Allows to build testsuite reports

testsuite/Cargo.toml

@@ -13,6 +13,7 @@ publish = false
 
 [dependencies]
 anyhow = "1"
+argh = "0.1"
 chrono = "0.4"
 oxigraph = { version = "0.2", path="../lib" }
 text-diff = "0.4"

testsuite/src/evaluator.rs

@@ -0,0 +1,41 @@
+use crate::manifest::Test;
+use crate::report::TestResult;
+use anyhow::{anyhow, Result};
+use chrono::Utc;
+use std::collections::HashMap;
+
+#[derive(Default)]
+pub struct TestEvaluator {
+    handlers: HashMap<String, Box<dyn Fn(&Test) -> Result<()>>>,
+}
+
+impl TestEvaluator {
+    pub fn register(
+        &mut self,
+        test_type: impl Into<String>,
+        handler: impl Fn(&Test) -> Result<()> + 'static,
+    ) {
+        self.handlers.insert(test_type.into(), Box::new(handler));
+    }
+
+    pub fn evaluate(
+        &self,
+        manifest: impl Iterator<Item = Result<Test>>,
+    ) -> Result<Vec<TestResult>> {
+        manifest
+            .map(|test| {
+                let test = test?;
+                let outcome = if let Some(handler) = self.handlers.get(test.kind.as_str()) {
+                    handler(&test)
+                } else {
+                    Err(anyhow!("The test type {} is not supported", test.kind))
+                };
+                Ok(TestResult {
+                    test: test.id,
+                    outcome,
+                    date: Utc::now(),
+                })
+            })
+            .collect()
+    }
+}

testsuite/src/lib.rs

@@ -10,6 +10,7 @@
     unused_qualifications
 )]
 
+pub mod evaluator;
 pub mod files;
 pub mod manifest;
 pub mod parser_evaluator;

testsuite/src/main.rs

@@ -0,0 +1,27 @@
+use anyhow::Result;
+use argh::FromArgs;
+use oxigraph_testsuite::evaluator::TestEvaluator;
+use oxigraph_testsuite::manifest::TestManifest;
+use oxigraph_testsuite::parser_evaluator::register_parser_tests;
+use oxigraph_testsuite::report::build_report;
+use oxigraph_testsuite::sparql_evaluator::register_sparql_tests;
+
+#[derive(FromArgs)]
+/// Oxigraph testsuite runner
+struct Args {
+    /// URI of the testsuite manifest to run
+    #[argh(positional)]
+    manifest: String,
+}
+
+fn main() -> Result<()> {
+    let args: Args = argh::from_env();
+
+    let mut evaluator = TestEvaluator::default();
+    register_parser_tests(&mut evaluator);
+    register_sparql_tests(&mut evaluator);
+    let manifest = TestManifest::new(vec![args.manifest]);
+    let results = evaluator.evaluate(manifest)?;
+    print!("{}", build_report(results));
+    Ok(())
+}

testsuite/src/parser_evaluator.rs

@@ -1,55 +1,95 @@
+use crate::evaluator::TestEvaluator;
 use crate::files::load_dataset;
 use crate::manifest::Test;
-use crate::report::{dataset_diff, TestResult};
+use crate::report::dataset_diff;
 use anyhow::{anyhow, Result};
-use chrono::Utc;
 
-pub fn evaluate_parser_tests(
+pub fn register_parser_tests(evaluator: &mut TestEvaluator) {
-    manifest: impl Iterator<Item = Result<Test>>,
+    evaluator.register(
-) -> Result<Vec<TestResult>> {
+        "http://www.w3.org/ns/rdftest#TestNTriplesPositiveSyntax",
-    manifest
+        evaluate_positive_syntax_test,
-        .map(|test| {
+    );
-            let test = test?;
+    evaluator.register(
-            let outcome = evaluate_parser_test(&test);
+        "http://www.w3.org/ns/rdftest#TestNQuadsPositiveSyntax",
-            Ok(TestResult {
+        evaluate_positive_syntax_test,
-                test: test.id,
+    );
-                outcome,
+    evaluator.register(
-                date: Utc::now(),
+        "http://www.w3.org/ns/rdftest#TestTurtlePositiveSyntax",
-            })
+        evaluate_positive_syntax_test,
-        })
+    );
-        .collect()
+    evaluator.register(
+        "http://www.w3.org/ns/rdftest#TestTrigPositiveSyntax",
+        evaluate_positive_syntax_test,
+    );
+    evaluator.register(
+        "http://www.w3.org/ns/rdftest#TestNTriplesNegativeSyntax",
+        evaluate_negative_syntax_test,
+    );
+    evaluator.register(
+        "http://www.w3.org/ns/rdftest#TestNQuadsNegativeSyntax",
+        evaluate_negative_syntax_test,
+    );
+    evaluator.register(
+        "http://www.w3.org/ns/rdftest#TestTurtleNegativeSyntax",
+        evaluate_negative_syntax_test,
+    );
+    evaluator.register(
+        "http://www.w3.org/ns/rdftest#TestTurtleNegativeEval",
+        evaluate_negative_syntax_test,
+    );
+    evaluator.register(
+        "http://www.w3.org/ns/rdftest#TestTrigNegativeSyntax",
+        evaluate_negative_syntax_test,
+    );
+    evaluator.register(
+        "http://www.w3.org/ns/rdftest#TestTrigNegativeEval",
+        evaluate_negative_syntax_test,
+    );
+    evaluator.register(
+        "http://www.w3.org/ns/rdftest#TestXMLNegativeSyntax",
+        evaluate_negative_syntax_test,
+    );
+    evaluator.register(
+        "http://www.w3.org/ns/rdftest#TestTurtleEval",
+        evaluate_eval_test,
+    );
+    evaluator.register(
+        "http://www.w3.org/ns/rdftest#TestTrigEval",
+        evaluate_eval_test,
+    );
+    evaluator.register(
+        "http://www.w3.org/ns/rdftest#TestXMLEval",
+        evaluate_eval_test,
+    );
 }
 
-fn evaluate_parser_test(test: &Test) -> Result<()> {
+fn evaluate_positive_syntax_test(test: &Test) -> Result<()> {
     let action = test
         .action
         .as_deref()
         .ok_or_else(|| anyhow!("No action found for test {}", test))?;
-    if test.kind == "http://www.w3.org/ns/rdftest#TestNTriplesPositiveSyntax"
-        || test.kind == "http://www.w3.org/ns/rdftest#TestNQuadsPositiveSyntax"
-        || test.kind == "http://www.w3.org/ns/rdftest#TestTurtlePositiveSyntax"
-        || test.kind == "http://www.w3.org/ns/rdftest#TestTrigPositiveSyntax"
-    {
     match load_dataset(action) {
         Ok(_) => Ok(()),
         Err(e) => Err(anyhow!(format!("Parse error: {}", e))),
     }
-    } else if test.kind == "http://www.w3.org/ns/rdftest#TestNTriplesNegativeSyntax"
+}
-        || test.kind == "http://www.w3.org/ns/rdftest#TestNQuadsNegativeSyntax"
+
-        || test.kind == "http://www.w3.org/ns/rdftest#TestTurtleNegativeSyntax"
+fn evaluate_negative_syntax_test(test: &Test) -> Result<()> {
-        || test.kind == "http://www.w3.org/ns/rdftest#TestTurtleNegativeEval"
+    let action = test
-        || test.kind == "http://www.w3.org/ns/rdftest#TestTrigNegativeSyntax"
+        .action
-        || test.kind == "http://www.w3.org/ns/rdftest#TestTrigNegativeEval"
+        .as_deref()
-        || test.kind == "http://www.w3.org/ns/rdftest#TestXMLNegativeSyntax"
+        .ok_or_else(|| anyhow!("No action found for test {}", test))?;
-    {
     match load_dataset(action) {
         Ok(_) => Err(anyhow!("File parsed with an error even if it should not",)),
         Err(_) => Ok(()),
     }
-    } else if test.kind == "http://www.w3.org/ns/rdftest#TestTurtleEval"
+}
-        || test.kind == "http://www.w3.org/ns/rdftest#TestTrigEval"
+
-        || test.kind == "http://www.w3.org/ns/rdftest#TestXMLEval"
+fn evaluate_eval_test(test: &Test) -> Result<()> {
-    {
+    let action = test
+        .action
+        .as_deref()
+        .ok_or_else(|| anyhow!("No action found for test {}", test))?;
     match load_dataset(action) {
         Ok(mut actual_graph) => {
             actual_graph.canonicalize();
@@ -74,7 +114,4 @@ fn evaluate_parser_test(test: &Test) -> Result<()> {
         }
         Err(e) => Err(anyhow!("Parse error on file {}: {}", action, e)),
     }
-    } else {
-        Err(anyhow!("Unsupported test type: {}", test.kind))
-    }
 }

testsuite/src/report.rs

@@ -1,6 +1,7 @@
 use anyhow::Result;
 use chrono::{DateTime, Utc};
 use oxigraph::model::{Dataset, NamedNode};
+use std::fmt::Write;
 use text_diff::{diff, Difference};
 
 #[derive(Debug)]
@@ -46,3 +47,104 @@ fn normalize_dataset_text(store: &Dataset) -> String {
     quads.sort();
     quads.join("\n")
 }
+
+#[allow(unused_must_use)]
+pub fn build_report(results: impl IntoIterator<Item = TestResult>) -> String {
+    let mut buffer = String::new();
+    writeln!(&mut buffer, "@prefix dc: <http://purl.org/dc/terms/> .");
+    writeln!(
+        &mut buffer,
+        "@prefix doap: <http://usefulinc.com/ns/doap#> ."
+    );
+    writeln!(&mut buffer, "@prefix earl: <http://www.w3.org/ns/earl#> .");
+    writeln!(&mut buffer, "@prefix foaf: <http://xmlns.com/foaf/0.1/> .");
+    writeln!(
+        &mut buffer,
+        "@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> ."
+    );
+    writeln!(
+        &mut buffer,
+        "@prefix xsd: <http://www.w3.org/2001/XMLSchema#> ."
+    );
+    writeln!(&mut buffer);
+    writeln!(&mut buffer, "<> foaf:primaryTopic <http://oxigraph.org/> ;");
+    writeln!(
+        &mut buffer,
+        "\tdc:issued \"{}\"^^xsd:dateTime ;",
+        Utc::now().to_rfc3339()
+    );
+    writeln!(
+        &mut buffer,
+        "\tfoaf:maker <https://thomas.pellissier-tanon.fr/#me> ."
+    );
+    writeln!(&mut buffer);
+    writeln!(
+        &mut buffer,
+        "<http://oxigraph.org/> a doap:Project, earl:TestSubject, earl:Software ;"
+    );
+    writeln!(&mut buffer, "\tdoap:name \"Oxigraph\" ;");
+    writeln!(&mut buffer, "\tdoap:release [");
+    writeln!(
+        &mut buffer,
+        "\t\tdoap:name \"Oxigraph {}\";",
+        env!("CARGO_PKG_VERSION")
+    );
+    writeln!(
+        &mut buffer,
+        "\t\tdoap:revision \"{}\" ;",
+        env!("CARGO_PKG_VERSION")
+    );
+    writeln!(&mut buffer, "\t] ;");
+    writeln!(
+        &mut buffer,
+        "\tdoap:developer <https://thomas.pellissier-tanon.fr/#me> ;"
+    );
+    writeln!(&mut buffer, "\tdoap:homepage <https://oxigraph.org/> ;");
+    writeln!(
+        &mut buffer,
+        "\tdoap:description \"Oxigraph is an embedded triple store.\"@en ;"
+    );
+    writeln!(&mut buffer, "\tdoap:programming-language \"Rust\" .");
+    writeln!(&mut buffer);
+    writeln!(
+        &mut buffer,
+        "<https://thomas.pellissier-tanon.fr/#me> a foaf:Person, earl:Assertor ;"
+    );
+    writeln!(&mut buffer, "\tfoaf:name \"Thomas Tanon\"; ");
+    writeln!(
+        &mut buffer,
+        "\tfoaf:homepage <https://thomas.pellissier-tanon.fr/> ."
+    );
+    writeln!(&mut buffer);
+    for result in results {
+        writeln!(&mut buffer);
+        writeln!(&mut buffer, "[");
+        writeln!(&mut buffer, "\ta earl:Assertion ;");
+        writeln!(
+            &mut buffer,
+            "\tearl:assertedBy <https://thomas.pellissier-tanon.fr/#me> ;"
+        );
+        writeln!(&mut buffer, "\tearl:subject <http://oxigraph.org/> ;");
+        writeln!(&mut buffer, "\tearl:test {} ;", result.test);
+        writeln!(&mut buffer, "\tearl:result [");
+        writeln!(&mut buffer, "\t\ta earl:TestResult ;");
+        writeln!(
+            &mut buffer,
+            "\t\tearl:outcome earl:{} ;",
+            if result.outcome.is_ok() {
+                "passed"
+            } else {
+                "failed"
+            }
+        );
+        writeln!(
+            &mut buffer,
+            "\t\tdc:date \"{}\"^^xsd:dateTime",
+            result.date.to_rfc3339()
+        );
+        writeln!(&mut buffer, "\t] ;");
+        writeln!(&mut buffer, "\tearl:mode earl:automatic");
+        writeln!(&mut buffer, "] .");
+    }
+    buffer
+}

testsuite/src/sparql_evaluator.rs

@@ -1,9 +1,9 @@
+use crate::evaluator::TestEvaluator;
 use crate::files::*;
 use crate::manifest::*;
-use crate::report::{dataset_diff, TestResult};
+use crate::report::dataset_diff;
 use crate::vocab::*;
 use anyhow::{anyhow, Result};
-use chrono::Utc;
 use oxigraph::model::vocab::*;
 use oxigraph::model::*;
 use oxigraph::sparql::*;
@@ -13,27 +13,42 @@ use std::str::FromStr;
 use std::sync::Arc;
 use std::{fmt, io};
 
-pub fn evaluate_sparql_tests(
+pub fn register_sparql_tests(evaluator: &mut TestEvaluator) {
-    manifest: impl Iterator<Item = Result<Test>>,
+    evaluator.register(
-) -> Result<Vec<TestResult>> {
+        "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#PositiveSyntaxTest",
-    manifest
+        evaluate_positive_syntax_test,
-        .map(|test| {
+    );
-            let test = test?;
+    evaluator.register(
-            let outcome = evaluate_sparql_test(&test);
+        "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#PositiveSyntaxTest11",
-            Ok(TestResult {
+        evaluate_positive_syntax_test,
-                test: test.id,
+    );
-                outcome,
+    evaluator.register(
-                date: Utc::now(),
+        "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#NegativeSyntaxTest",
-            })
+        evaluate_negative_syntax_test,
-        })
+    );
-        .collect()
+    evaluator.register(
+        "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#NegativeSyntaxTest11",
+        evaluate_negative_syntax_test,
+    );
+    evaluator.register(
+        "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#QueryEvaluationTest",
+        evaluate_evaluation_test,
+    );
+    evaluator.register(
+        "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#PositiveUpdateSyntaxTest11",
+        evaluate_positive_update_syntax_test,
+    );
+    evaluator.register(
+        "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#NegativeUpdateSyntaxTest11",
+        evaluate_negative_update_syntax_test,
+    );
+    evaluator.register(
+        "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#UpdateEvaluationTest",
+        evaluate_update_evaluation_test,
+    );
 }
 
-fn evaluate_sparql_test(test: &Test) -> Result<()> {
+fn evaluate_positive_syntax_test(test: &Test) -> Result<()> {
-    if test.kind == "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#PositiveSyntaxTest"
-        || test.kind
-            == "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#PositiveSyntaxTest11"
-    {
     let query_file = test
         .action
         .as_deref()
@@ -50,11 +65,9 @@ fn evaluate_sparql_test(test: &Test) -> Result<()> {
             )),
         },
     }
-    } else if test.kind
+}
-        == "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#NegativeSyntaxTest"
+
-        || test.kind
+fn evaluate_negative_syntax_test(test: &Test) -> Result<()> {
-            == "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#NegativeSyntaxTest11"
-    {
     let query_file = test
         .action
         .as_deref()
@@ -67,9 +80,9 @@ fn evaluate_sparql_test(test: &Test) -> Result<()> {
         )),
         Err(_) => Ok(()),
     }
-    } else if test.kind
+}
-        == "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#QueryEvaluationTest"
+
-    {
+fn evaluate_evaluation_test(test: &Test) -> Result<()> {
     let store = Store::new()?;
     if let Some(data) = &test.data {
         load_to_store(data, &store, GraphNameRef::DefaultGraph)?;
@@ -96,11 +109,7 @@ fn evaluate_sparql_test(test: &Test) -> Result<()> {
                     if let GraphName::NamedNode(graph_name) = graph_name {
                         load_to_store(graph_name.as_str(), &store, graph_name.as_ref())?;
                     } else {
-                            return Err(anyhow!(
+                        return Err(anyhow!("Invalid FROM in query {} for test {}", query, test));
-                                "Invalid FROM in query {} for test {}",
-                                query,
-                                test
-                            ));
                     }
                 }
                 for graph_name in query.dataset().available_named_graphs().unwrap_or(&[]) {
@@ -122,15 +131,12 @@ fn evaluate_sparql_test(test: &Test) -> Result<()> {
                     error
                 )),
                 Ok(actual_results) => {
-                        let expected_results = load_sparql_query_result(
+                    let expected_results = load_sparql_query_result(test.result.as_ref().unwrap())
-                            test.result.as_ref().unwrap(),
-                        )
                         .map_err(|e| {
                             anyhow!("Error constructing expected graph for {}: {}", test, e)
                         })?;
-                        let with_order = if let StaticQueryResults::Solutions { ordered, .. } =
+                    let with_order =
-                            &expected_results
+                        if let StaticQueryResults::Solutions { ordered, .. } = &expected_results {
-                        {
                             *ordered
                         } else {
                             false
@@ -153,9 +159,9 @@ fn evaluate_sparql_test(test: &Test) -> Result<()> {
             }
         }
     }
-    } else if test.kind
+}
-        == "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#PositiveUpdateSyntaxTest11"
+
-    {
+fn evaluate_positive_update_syntax_test(test: &Test) -> Result<()> {
     let update_file = test
         .action
         .as_deref()
@@ -172,9 +178,9 @@ fn evaluate_sparql_test(test: &Test) -> Result<()> {
             )),
         },
     }
-    } else if test.kind
+}
-        == "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#NegativeUpdateSyntaxTest11"
+
-    {
+fn evaluate_negative_update_syntax_test(test: &Test) -> Result<()> {
     let update_file = test
         .action
         .as_deref()
@@ -187,9 +193,9 @@ fn evaluate_sparql_test(test: &Test) -> Result<()> {
         )),
         Err(_) => Ok(()),
     }
-    } else if test.kind
+}
-        == "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#UpdateEvaluationTest"
+
-    {
+fn evaluate_update_evaluation_test(test: &Test) -> Result<()> {
     let store = Store::new()?;
     if let Some(data) = &test.data {
         load_to_store(data, &store, &GraphName::DefaultGraph)?;
@@ -242,9 +248,6 @@ fn evaluate_sparql_test(test: &Test) -> Result<()> {
             }
         },
     }
-    } else {
-        Err(anyhow!("Unsupported test type: {}", test.kind))
-    }
 }
 
 fn load_sparql_query_result(url: &str) -> Result<StaticQueryResults> {

testsuite/tests/oxigraph.rs

@@ -1,10 +1,13 @@
 use anyhow::Result;
+use oxigraph_testsuite::evaluator::TestEvaluator;
 use oxigraph_testsuite::manifest::TestManifest;
-use oxigraph_testsuite::sparql_evaluator::evaluate_sparql_tests;
+use oxigraph_testsuite::sparql_evaluator::register_sparql_tests;
 
 fn run_testsuite(manifest_urls: Vec<&str>) -> Result<()> {
+    let mut evaluator = TestEvaluator::default();
+    register_sparql_tests(&mut evaluator);
     let manifest = TestManifest::new(manifest_urls);
-    let results = evaluate_sparql_tests(manifest)?;
+    let results = evaluator.evaluate(manifest)?;
 
     let mut errors = Vec::default();
     for result in results {

testsuite/tests/parser.rs

@@ -1,10 +1,13 @@
 use anyhow::Result;
+use oxigraph_testsuite::evaluator::TestEvaluator;
 use oxigraph_testsuite::manifest::TestManifest;
-use oxigraph_testsuite::parser_evaluator::evaluate_parser_tests;
+use oxigraph_testsuite::parser_evaluator::register_parser_tests;
 
 fn run_testsuite(manifest_url: &str) -> Result<()> {
+    let mut evaluator = TestEvaluator::default();
+    register_parser_tests(&mut evaluator);
     let manifest = TestManifest::new(vec![manifest_url]);
-    let results = evaluate_parser_tests(manifest)?;
+    let results = evaluator.evaluate(manifest)?;
 
     let mut errors = Vec::default();
     for result in results {

testsuite/tests/sparql.rs

@@ -1,10 +1,13 @@
 use anyhow::Result;
+use oxigraph_testsuite::evaluator::TestEvaluator;
 use oxigraph_testsuite::manifest::TestManifest;
-use oxigraph_testsuite::sparql_evaluator::evaluate_sparql_tests;
+use oxigraph_testsuite::sparql_evaluator::register_sparql_tests;
 
 fn run_testsuite(manifest_url: &str, ignored_tests: Vec<&str>) -> Result<()> {
+    let mut evaluator = TestEvaluator::default();
+    register_sparql_tests(&mut evaluator);
     let manifest = TestManifest::new(vec![manifest_url]);
-    let results = evaluate_sparql_tests(manifest)?;
+    let results = evaluator.evaluate(manifest)?;
 
     let mut errors = Vec::default();
     for result in results {