Allows to build testsuite reports

testsuite/Cargo.toml

@@ -13,6 +13,7 @@ publish = false
 
 [dependencies]
 anyhow = "1"
+argh = "0.1"
 chrono = "0.4"
 oxigraph = { version = "0.2", path="../lib" }
 text-diff = "0.4"

testsuite/src/evaluator.rs

@@ -0,0 +1,41 @@
+use crate::manifest::Test;
+use crate::report::TestResult;
+use anyhow::{anyhow, Result};
+use chrono::Utc;
+use std::collections::HashMap;
+
+#[derive(Default)]
+pub struct TestEvaluator {
+    handlers: HashMap<String, Box<dyn Fn(&Test) -> Result<()>>>,
+}
+
+impl TestEvaluator {
+    pub fn register(
+        &mut self,
+        test_type: impl Into<String>,
+        handler: impl Fn(&Test) -> Result<()> + 'static,
+    ) {
+        self.handlers.insert(test_type.into(), Box::new(handler));
+    }
+
+    pub fn evaluate(
+        &self,
+        manifest: impl Iterator<Item = Result<Test>>,
+    ) -> Result<Vec<TestResult>> {
+        manifest
+            .map(|test| {
+                let test = test?;
+                let outcome = if let Some(handler) = self.handlers.get(test.kind.as_str()) {
+                    handler(&test)
+                } else {
+                    Err(anyhow!("The test type {} is not supported", test.kind))
+                };
+                Ok(TestResult {
+                    test: test.id,
+                    outcome,
+                    date: Utc::now(),
+                })
+            })
+            .collect()
+    }
+}

testsuite/src/lib.rs

@@ -10,6 +10,7 @@
     unused_qualifications
 )]
 
+pub mod evaluator;
 pub mod files;
 pub mod manifest;
 pub mod parser_evaluator;

testsuite/src/main.rs

@@ -0,0 +1,27 @@
+use anyhow::Result;
+use argh::FromArgs;
+use oxigraph_testsuite::evaluator::TestEvaluator;
+use oxigraph_testsuite::manifest::TestManifest;
+use oxigraph_testsuite::parser_evaluator::register_parser_tests;
+use oxigraph_testsuite::report::build_report;
+use oxigraph_testsuite::sparql_evaluator::register_sparql_tests;
+
+#[derive(FromArgs)]
+/// Oxigraph testsuite runner
+struct Args {
+    /// URI of the testsuite manifest to run
+    #[argh(positional)]
+    manifest: String,
+}
+
+fn main() -> Result<()> {
+    let args: Args = argh::from_env();
+
+    let mut evaluator = TestEvaluator::default();
+    register_parser_tests(&mut evaluator);
+    register_sparql_tests(&mut evaluator);
+    let manifest = TestManifest::new(vec![args.manifest]);
+    let results = evaluator.evaluate(manifest)?;
+    print!("{}", build_report(results));
+    Ok(())
+}

testsuite/src/parser_evaluator.rs

@@ -1,80 +1,117 @@
+use crate::evaluator::TestEvaluator;
 use crate::files::load_dataset;
 use crate::manifest::Test;
-use crate::report::{dataset_diff, TestResult};
+use crate::report::dataset_diff;
 use anyhow::{anyhow, Result};
-use chrono::Utc;
 
-pub fn evaluate_parser_tests(
+pub fn register_parser_tests(evaluator: &mut TestEvaluator) {
-    manifest: impl Iterator<Item = Result<Test>>,
+    evaluator.register(
-) -> Result<Vec<TestResult>> {
+        "http://www.w3.org/ns/rdftest#TestNTriplesPositiveSyntax",
-    manifest
+        evaluate_positive_syntax_test,
-        .map(|test| {
+    );
-            let test = test?;
+    evaluator.register(
-            let outcome = evaluate_parser_test(&test);
+        "http://www.w3.org/ns/rdftest#TestNQuadsPositiveSyntax",
-            Ok(TestResult {
+        evaluate_positive_syntax_test,
-                test: test.id,
+    );
-                outcome,
+    evaluator.register(
-                date: Utc::now(),
+        "http://www.w3.org/ns/rdftest#TestTurtlePositiveSyntax",
-            })
+        evaluate_positive_syntax_test,
-        })
+    );
-        .collect()
+    evaluator.register(
+        "http://www.w3.org/ns/rdftest#TestTrigPositiveSyntax",
+        evaluate_positive_syntax_test,
+    );
+    evaluator.register(
+        "http://www.w3.org/ns/rdftest#TestNTriplesNegativeSyntax",
+        evaluate_negative_syntax_test,
+    );
+    evaluator.register(
+        "http://www.w3.org/ns/rdftest#TestNQuadsNegativeSyntax",
+        evaluate_negative_syntax_test,
+    );
+    evaluator.register(
+        "http://www.w3.org/ns/rdftest#TestTurtleNegativeSyntax",
+        evaluate_negative_syntax_test,
+    );
+    evaluator.register(
+        "http://www.w3.org/ns/rdftest#TestTurtleNegativeEval",
+        evaluate_negative_syntax_test,
+    );
+    evaluator.register(
+        "http://www.w3.org/ns/rdftest#TestTrigNegativeSyntax",
+        evaluate_negative_syntax_test,
+    );
+    evaluator.register(
+        "http://www.w3.org/ns/rdftest#TestTrigNegativeEval",
+        evaluate_negative_syntax_test,
+    );
+    evaluator.register(
+        "http://www.w3.org/ns/rdftest#TestXMLNegativeSyntax",
+        evaluate_negative_syntax_test,
+    );
+    evaluator.register(
+        "http://www.w3.org/ns/rdftest#TestTurtleEval",
+        evaluate_eval_test,
+    );
+    evaluator.register(
+        "http://www.w3.org/ns/rdftest#TestTrigEval",
+        evaluate_eval_test,
+    );
+    evaluator.register(
+        "http://www.w3.org/ns/rdftest#TestXMLEval",
+        evaluate_eval_test,
+    );
 }
 
-fn evaluate_parser_test(test: &Test) -> Result<()> {
+fn evaluate_positive_syntax_test(test: &Test) -> Result<()> {
     let action = test
         .action
         .as_deref()
         .ok_or_else(|| anyhow!("No action found for test {}", test))?;
-    if test.kind == "http://www.w3.org/ns/rdftest#TestNTriplesPositiveSyntax"
+    match load_dataset(action) {
-        || test.kind == "http://www.w3.org/ns/rdftest#TestNQuadsPositiveSyntax"
+        Ok(_) => Ok(()),
-        || test.kind == "http://www.w3.org/ns/rdftest#TestTurtlePositiveSyntax"
+        Err(e) => Err(anyhow!(format!("Parse error: {}", e))),
-        || test.kind == "http://www.w3.org/ns/rdftest#TestTrigPositiveSyntax"
+    }
-    {
+}
-        match load_dataset(action) {
+
-            Ok(_) => Ok(()),
+fn evaluate_negative_syntax_test(test: &Test) -> Result<()> {
-            Err(e) => Err(anyhow!(format!("Parse error: {}", e))),
+    let action = test
-        }
+        .action
-    } else if test.kind == "http://www.w3.org/ns/rdftest#TestNTriplesNegativeSyntax"
+        .as_deref()
-        || test.kind == "http://www.w3.org/ns/rdftest#TestNQuadsNegativeSyntax"
+        .ok_or_else(|| anyhow!("No action found for test {}", test))?;
-        || test.kind == "http://www.w3.org/ns/rdftest#TestTurtleNegativeSyntax"
+    match load_dataset(action) {
-        || test.kind == "http://www.w3.org/ns/rdftest#TestTurtleNegativeEval"
+        Ok(_) => Err(anyhow!("File parsed with an error even if it should not",)),
-        || test.kind == "http://www.w3.org/ns/rdftest#TestTrigNegativeSyntax"
+        Err(_) => Ok(()),
-        || test.kind == "http://www.w3.org/ns/rdftest#TestTrigNegativeEval"
+    }
-        || test.kind == "http://www.w3.org/ns/rdftest#TestXMLNegativeSyntax"
+}
-    {
+
-        match load_dataset(action) {
+fn evaluate_eval_test(test: &Test) -> Result<()> {
-            Ok(_) => Err(anyhow!("File parsed with an error even if it should not",)),
+    let action = test
-            Err(_) => Ok(()),
+        .action
-        }
+        .as_deref()
-    } else if test.kind == "http://www.w3.org/ns/rdftest#TestTurtleEval"
+        .ok_or_else(|| anyhow!("No action found for test {}", test))?;
-        || test.kind == "http://www.w3.org/ns/rdftest#TestTrigEval"
+    match load_dataset(action) {
-        || test.kind == "http://www.w3.org/ns/rdftest#TestXMLEval"
+        Ok(mut actual_graph) => {
-    {
+            actual_graph.canonicalize();
-        match load_dataset(action) {
+            if let Some(result) = &test.result {
-            Ok(mut actual_graph) => {
+                match load_dataset(result) {
-                actual_graph.canonicalize();
+                    Ok(mut expected_graph) => {
-                if let Some(result) = &test.result {
+                        expected_graph.canonicalize();
-                    match load_dataset(result) {
+                        if expected_graph == actual_graph {
-                        Ok(mut expected_graph) => {
+                            Ok(())
-                            expected_graph.canonicalize();
+                        } else {
-                            if expected_graph == actual_graph {
+                            Err(anyhow!(
-                                Ok(())
+                                "The two files are not isomorphic. Diff:\n{}",
-                            } else {
+                                dataset_diff(&expected_graph, &actual_graph)
-                                Err(anyhow!(
+                            ))
-                                    "The two files are not isomorphic. Diff:\n{}",
-                                    dataset_diff(&expected_graph, &actual_graph)
-                                ))
-                            }
                         }
-                        Err(e) => Err(anyhow!("Parse error on file {}: {}", action, e)),
                     }
-                } else {
+                    Err(e) => Err(anyhow!("Parse error on file {}: {}", action, e)),
-                    Err(anyhow!("No tests result found"))
                 }
+            } else {
+                Err(anyhow!("No tests result found"))
             }
-            Err(e) => Err(anyhow!("Parse error on file {}: {}", action, e)),
         }
-    } else {
+        Err(e) => Err(anyhow!("Parse error on file {}: {}", action, e)),
-        Err(anyhow!("Unsupported test type: {}", test.kind))
     }
 }

testsuite/src/report.rs

@@ -1,6 +1,7 @@
 use anyhow::Result;
 use chrono::{DateTime, Utc};
 use oxigraph::model::{Dataset, NamedNode};
+use std::fmt::Write;
 use text_diff::{diff, Difference};
 
 #[derive(Debug)]
@@ -46,3 +47,104 @@ fn normalize_dataset_text(store: &Dataset) -> String {
     quads.sort();
     quads.join("\n")
 }
+
+#[allow(unused_must_use)]
+pub fn build_report(results: impl IntoIterator<Item = TestResult>) -> String {
+    let mut buffer = String::new();
+    writeln!(&mut buffer, "@prefix dc: <http://purl.org/dc/terms/> .");
+    writeln!(
+        &mut buffer,
+        "@prefix doap: <http://usefulinc.com/ns/doap#> ."
+    );
+    writeln!(&mut buffer, "@prefix earl: <http://www.w3.org/ns/earl#> .");
+    writeln!(&mut buffer, "@prefix foaf: <http://xmlns.com/foaf/0.1/> .");
+    writeln!(
+        &mut buffer,
+        "@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> ."
+    );
+    writeln!(
+        &mut buffer,
+        "@prefix xsd: <http://www.w3.org/2001/XMLSchema#> ."
+    );
+    writeln!(&mut buffer);
+    writeln!(&mut buffer, "<> foaf:primaryTopic <http://oxigraph.org/> ;");
+    writeln!(
+        &mut buffer,
+        "\tdc:issued \"{}\"^^xsd:dateTime ;",
+        Utc::now().to_rfc3339()
+    );
+    writeln!(
+        &mut buffer,
+        "\tfoaf:maker <https://thomas.pellissier-tanon.fr/#me> ."
+    );
+    writeln!(&mut buffer);
+    writeln!(
+        &mut buffer,
+        "<http://oxigraph.org/> a doap:Project, earl:TestSubject, earl:Software ;"
+    );
+    writeln!(&mut buffer, "\tdoap:name \"Oxigraph\" ;");
+    writeln!(&mut buffer, "\tdoap:release [");
+    writeln!(
+        &mut buffer,
+        "\t\tdoap:name \"Oxigraph {}\";",
+        env!("CARGO_PKG_VERSION")
+    );
+    writeln!(
+        &mut buffer,
+        "\t\tdoap:revision \"{}\" ;",
+        env!("CARGO_PKG_VERSION")
+    );
+    writeln!(&mut buffer, "\t] ;");
+    writeln!(
+        &mut buffer,
+        "\tdoap:developer <https://thomas.pellissier-tanon.fr/#me> ;"
+    );
+    writeln!(&mut buffer, "\tdoap:homepage <https://oxigraph.org/> ;");
+    writeln!(
+        &mut buffer,
+        "\tdoap:description \"Oxigraph is an embedded triple store.\"@en ;"
+    );
+    writeln!(&mut buffer, "\tdoap:programming-language \"Rust\" .");
+    writeln!(&mut buffer);
+    writeln!(
+        &mut buffer,
+        "<https://thomas.pellissier-tanon.fr/#me> a foaf:Person, earl:Assertor ;"
+    );
+    writeln!(&mut buffer, "\tfoaf:name \"Thomas Tanon\"; ");
+    writeln!(
+        &mut buffer,
+        "\tfoaf:homepage <https://thomas.pellissier-tanon.fr/> ."
+    );
+    writeln!(&mut buffer);
+    for result in results {
+        writeln!(&mut buffer);
+        writeln!(&mut buffer, "[");
+        writeln!(&mut buffer, "\ta earl:Assertion ;");
+        writeln!(
+            &mut buffer,
+            "\tearl:assertedBy <https://thomas.pellissier-tanon.fr/#me> ;"
+        );
+        writeln!(&mut buffer, "\tearl:subject <http://oxigraph.org/> ;");
+        writeln!(&mut buffer, "\tearl:test {} ;", result.test);
+        writeln!(&mut buffer, "\tearl:result [");
+        writeln!(&mut buffer, "\t\ta earl:TestResult ;");
+        writeln!(
+            &mut buffer,
+            "\t\tearl:outcome earl:{} ;",
+            if result.outcome.is_ok() {
+                "passed"
+            } else {
+                "failed"
+            }
+        );
+        writeln!(
+            &mut buffer,
+            "\t\tdc:date \"{}\"^^xsd:dateTime",
+            result.date.to_rfc3339()
+        );
+        writeln!(&mut buffer, "\t] ;");
+        writeln!(&mut buffer, "\tearl:mode earl:automatic");
+        writeln!(&mut buffer, "] .");
+    }
+    buffer
+}

testsuite/src/sparql_evaluator.rs

@@ -1,9 +1,9 @@
+use crate::evaluator::TestEvaluator;
 use crate::files::*;
 use crate::manifest::*;
-use crate::report::{dataset_diff, TestResult};
+use crate::report::dataset_diff;
 use crate::vocab::*;
 use anyhow::{anyhow, Result};
-use chrono::Utc;
 use oxigraph::model::vocab::*;
 use oxigraph::model::*;
 use oxigraph::sparql::*;
@@ -13,237 +13,240 @@ use std::str::FromStr;
 use std::sync::Arc;
 use std::{fmt, io};
 
-pub fn evaluate_sparql_tests(
+pub fn register_sparql_tests(evaluator: &mut TestEvaluator) {
-    manifest: impl Iterator<Item = Result<Test>>,
+    evaluator.register(
-) -> Result<Vec<TestResult>> {
+        "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#PositiveSyntaxTest",
-    manifest
+        evaluate_positive_syntax_test,
-        .map(|test| {
+    );
-            let test = test?;
+    evaluator.register(
-            let outcome = evaluate_sparql_test(&test);
+        "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#PositiveSyntaxTest11",
-            Ok(TestResult {
+        evaluate_positive_syntax_test,
-                test: test.id,
+    );
-                outcome,
+    evaluator.register(
-                date: Utc::now(),
+        "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#NegativeSyntaxTest",
-            })
+        evaluate_negative_syntax_test,
-        })
+    );
-        .collect()
+    evaluator.register(
+        "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#NegativeSyntaxTest11",
+        evaluate_negative_syntax_test,
+    );
+    evaluator.register(
+        "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#QueryEvaluationTest",
+        evaluate_evaluation_test,
+    );
+    evaluator.register(
+        "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#PositiveUpdateSyntaxTest11",
+        evaluate_positive_update_syntax_test,
+    );
+    evaluator.register(
+        "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#NegativeUpdateSyntaxTest11",
+        evaluate_negative_update_syntax_test,
+    );
+    evaluator.register(
+        "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#UpdateEvaluationTest",
+        evaluate_update_evaluation_test,
+    );
 }
 
-fn evaluate_sparql_test(test: &Test) -> Result<()> {
+fn evaluate_positive_syntax_test(test: &Test) -> Result<()> {
-    if test.kind == "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#PositiveSyntaxTest"
+    let query_file = test
-        || test.kind
+        .action
-            == "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#PositiveSyntaxTest11"
+        .as_deref()
-    {
+        .ok_or_else(|| anyhow!("No action found for test {}", test))?;
-        let query_file = test
+    match Query::parse(&read_file_to_string(&query_file)?, Some(&query_file)) {
-            .action
+        Err(error) => Err(anyhow!("Not able to parse {} with error: {}", test, error)),
-            .as_deref()
+        Ok(query) => match Query::parse(&query.to_string(), None) {
-            .ok_or_else(|| anyhow!("No action found for test {}", test))?;
+            Ok(_) => Ok(()),
-        match Query::parse(&read_file_to_string(&query_file)?, Some(&query_file)) {
-            Err(error) => Err(anyhow!("Not able to parse {} with error: {}", test, error)),
-            Ok(query) => match Query::parse(&query.to_string(), None) {
-                Ok(_) => Ok(()),
-                Err(error) => Err(anyhow!(
-                    "Failure to deserialize \"{}\" of {} with error: {}",
-                    query.to_string(),
-                    test,
-                    error
-                )),
-            },
-        }
-    } else if test.kind
-        == "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#NegativeSyntaxTest"
-        || test.kind
-            == "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#NegativeSyntaxTest11"
-    {
-        let query_file = test
-            .action
-            .as_deref()
-            .ok_or_else(|| anyhow!("No action found for test {}", test))?;
-        match Query::parse(&read_file_to_string(query_file)?, Some(query_file)) {
-            Ok(result) => Err(anyhow!(
-                "Oxigraph parses even if it should not {}. The output tree is: {}",
-                test,
-                result
-            )),
-            Err(_) => Ok(()),
-        }
-    } else if test.kind
-        == "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#QueryEvaluationTest"
-    {
-        let store = Store::new()?;
-        if let Some(data) = &test.data {
-            load_to_store(data, &store, GraphNameRef::DefaultGraph)?;
-        }
-        for (name, value) in &test.graph_data {
-            load_to_store(value, &store, name)?;
-        }
-        let query_file = test
-            .query
-            .as_deref()
-            .ok_or_else(|| anyhow!("No action found for test {}", test))?;
-        let options = QueryOptions::default()
-            .with_service_handler(StaticServiceHandler::new(&test.service_data)?);
-        match Query::parse(&read_file_to_string(query_file)?, Some(query_file)) {
             Err(error) => Err(anyhow!(
-                "Failure to parse query of {} with error: {}",
+                "Failure to deserialize \"{}\" of {} with error: {}",
+                query.to_string(),
                 test,
                 error
             )),
-            Ok(query) => {
+        },
-                // FROM and FROM NAMED support. We make sure the data is in the store
+    }
-                if !query.dataset().is_default_dataset() {
+}
-                    for graph_name in query.dataset().default_graph_graphs().unwrap_or(&[]) {
+
-                        if let GraphName::NamedNode(graph_name) = graph_name {
+fn evaluate_negative_syntax_test(test: &Test) -> Result<()> {
-                            load_to_store(graph_name.as_str(), &store, graph_name.as_ref())?;
+    let query_file = test
-                        } else {
+        .action
-                            return Err(anyhow!(
+        .as_deref()
-                                "Invalid FROM in query {} for test {}",
+        .ok_or_else(|| anyhow!("No action found for test {}", test))?;
-                                query,
+    match Query::parse(&read_file_to_string(query_file)?, Some(query_file)) {
-                                test
+        Ok(result) => Err(anyhow!(
-                            ));
+            "Oxigraph parses even if it should not {}. The output tree is: {}",
-                        }
+            test,
+            result
+        )),
+        Err(_) => Ok(()),
+    }
+}
+
+fn evaluate_evaluation_test(test: &Test) -> Result<()> {
+    let store = Store::new()?;
+    if let Some(data) = &test.data {
+        load_to_store(data, &store, GraphNameRef::DefaultGraph)?;
+    }
+    for (name, value) in &test.graph_data {
+        load_to_store(value, &store, name)?;
+    }
+    let query_file = test
+        .query
+        .as_deref()
+        .ok_or_else(|| anyhow!("No action found for test {}", test))?;
+    let options = QueryOptions::default()
+        .with_service_handler(StaticServiceHandler::new(&test.service_data)?);
+    match Query::parse(&read_file_to_string(query_file)?, Some(query_file)) {
+        Err(error) => Err(anyhow!(
+            "Failure to parse query of {} with error: {}",
+            test,
+            error
+        )),
+        Ok(query) => {
+            // FROM and FROM NAMED support. We make sure the data is in the store
+            if !query.dataset().is_default_dataset() {
+                for graph_name in query.dataset().default_graph_graphs().unwrap_or(&[]) {
+                    if let GraphName::NamedNode(graph_name) = graph_name {
+                        load_to_store(graph_name.as_str(), &store, graph_name.as_ref())?;
+                    } else {
+                        return Err(anyhow!("Invalid FROM in query {} for test {}", query, test));
                     }
-                    for graph_name in query.dataset().available_named_graphs().unwrap_or(&[]) {
+                }
-                        if let Subject::NamedNode(graph_name) = graph_name {
+                for graph_name in query.dataset().available_named_graphs().unwrap_or(&[]) {
-                            load_to_store(graph_name.as_str(), &store, graph_name.as_ref())?;
+                    if let Subject::NamedNode(graph_name) = graph_name {
-                        } else {
+                        load_to_store(graph_name.as_str(), &store, graph_name.as_ref())?;
-                            return Err(anyhow!(
+                    } else {
-                                "Invalid FROM NAMED in query {} for test {}",
+                        return Err(anyhow!(
-                                query,
+                            "Invalid FROM NAMED in query {} for test {}",
-                                test
+                            query,
-                            ));
+                            test
-                        }
+                        ));
                     }
                 }
-                match store.query_opt(query, options) {
+            }
-                    Err(error) => Err(anyhow!(
+            match store.query_opt(query, options) {
-                        "Failure to execute query of {} with error: {}",
+                Err(error) => Err(anyhow!(
-                        test,
+                    "Failure to execute query of {} with error: {}",
-                        error
+                    test,
-                    )),
+                    error
-                    Ok(actual_results) => {
+                )),
-                        let expected_results = load_sparql_query_result(
+                Ok(actual_results) => {
-                            test.result.as_ref().unwrap(),
+                    let expected_results = load_sparql_query_result(test.result.as_ref().unwrap())
-                        )
                         .map_err(|e| {
                             anyhow!("Error constructing expected graph for {}: {}", test, e)
                         })?;
-                        let with_order = if let StaticQueryResults::Solutions { ordered, .. } =
+                    let with_order =
-                            &expected_results
+                        if let StaticQueryResults::Solutions { ordered, .. } = &expected_results {
-                        {
                             *ordered
                         } else {
                             false
                         };
-                        let actual_results =
+                    let actual_results =
-                            StaticQueryResults::from_query_results(actual_results, with_order)?;
+                        StaticQueryResults::from_query_results(actual_results, with_order)?;
 
-                        if are_query_results_isomorphic(&expected_results, &actual_results) {
+                    if are_query_results_isomorphic(&expected_results, &actual_results) {
-                            Ok(())
+                        Ok(())
-                        } else {
+                    } else {
-                            Err(anyhow!("Failure on {}.\nExpected file:\n{}\nOutput file:\n{}\nParsed query:\n{}\nData:\n{}\n",
+                        Err(anyhow!("Failure on {}.\nExpected file:\n{}\nOutput file:\n{}\nParsed query:\n{}\nData:\n{}\n",
                                                test,
                                                expected_results,
                                                actual_results,
                                                Query::parse(&read_file_to_string(query_file)?, Some(query_file)).unwrap(),
                                                store
                             ))
-                        }
                     }
                 }
             }
         }
-    } else if test.kind
+    }
-        == "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#PositiveUpdateSyntaxTest11"
+}
-    {
+
-        let update_file = test
+fn evaluate_positive_update_syntax_test(test: &Test) -> Result<()> {
-            .action
+    let update_file = test
-            .as_deref()
+        .action
-            .ok_or_else(|| anyhow!("No action found for test {}", test))?;
+        .as_deref()
-        match Update::parse(&read_file_to_string(&update_file)?, Some(&update_file)) {
+        .ok_or_else(|| anyhow!("No action found for test {}", test))?;
-            Err(error) => Err(anyhow!("Not able to parse {} with error: {}", test, error)),
+    match Update::parse(&read_file_to_string(&update_file)?, Some(&update_file)) {
-            Ok(update) => match Update::parse(&update.to_string(), None) {
+        Err(error) => Err(anyhow!("Not able to parse {} with error: {}", test, error)),
-                Ok(_) => Ok(()),
+        Ok(update) => match Update::parse(&update.to_string(), None) {
-                Err(error) => Err(anyhow!(
+            Ok(_) => Ok(()),
-                    "Failure to deserialize \"{}\" of {} with error: {}",
+            Err(error) => Err(anyhow!(
-                    update.to_string(),
+                "Failure to deserialize \"{}\" of {} with error: {}",
-                    test,
+                update.to_string(),
-                    error
-                )),
-            },
-        }
-    } else if test.kind
-        == "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#NegativeUpdateSyntaxTest11"
-    {
-        let update_file = test
-            .action
-            .as_deref()
-            .ok_or_else(|| anyhow!("No action found for test {}", test))?;
-        match Query::parse(&read_file_to_string(update_file)?, Some(update_file)) {
-            Ok(result) => Err(anyhow!(
-                "Oxigraph parses even if it should not {}. The output tree is: {}",
                 test,
-                result
+                error
             )),
-            Err(_) => Ok(()),
+        },
-        }
+    }
-    } else if test.kind
+}
-        == "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#UpdateEvaluationTest"
-    {
-        let store = Store::new()?;
-        if let Some(data) = &test.data {
-            load_to_store(data, &store, &GraphName::DefaultGraph)?;
-        }
-        for (name, value) in &test.graph_data {
-            load_to_store(value, &store, name)?;
-        }
 
-        let result_store = Store::new()?;
+fn evaluate_negative_update_syntax_test(test: &Test) -> Result<()> {
-        if let Some(data) = &test.result {
+    let update_file = test
-            load_to_store(data, &result_store, &GraphName::DefaultGraph)?;
+        .action
-        }
+        .as_deref()
-        for (name, value) in &test.result_graph_data {
+        .ok_or_else(|| anyhow!("No action found for test {}", test))?;
-            load_to_store(value, &result_store, name)?;
+    match Query::parse(&read_file_to_string(update_file)?, Some(update_file)) {
-        }
+        Ok(result) => Err(anyhow!(
+            "Oxigraph parses even if it should not {}. The output tree is: {}",
+            test,
+            result
+        )),
+        Err(_) => Ok(()),
+    }
+}
 
-        let update_file = test
+fn evaluate_update_evaluation_test(test: &Test) -> Result<()> {
-            .update
+    let store = Store::new()?;
-            .as_deref()
+    if let Some(data) = &test.data {
-            .ok_or_else(|| anyhow!("No action found for test {}", test))?;
+        load_to_store(data, &store, &GraphName::DefaultGraph)?;
-        match Update::parse(&read_file_to_string(update_file)?, Some(update_file)) {
+    }
+    for (name, value) in &test.graph_data {
+        load_to_store(value, &store, name)?;
+    }
+
+    let result_store = Store::new()?;
+    if let Some(data) = &test.result {
+        load_to_store(data, &result_store, &GraphName::DefaultGraph)?;
+    }
+    for (name, value) in &test.result_graph_data {
+        load_to_store(value, &result_store, name)?;
+    }
+
+    let update_file = test
+        .update
+        .as_deref()
+        .ok_or_else(|| anyhow!("No action found for test {}", test))?;
+    match Update::parse(&read_file_to_string(update_file)?, Some(update_file)) {
+        Err(error) => Err(anyhow!(
+            "Failure to parse update of {} with error: {}",
+            test,
+            error
+        )),
+        Ok(update) => match store.update(update) {
             Err(error) => Err(anyhow!(
-                "Failure to parse update of {} with error: {}",
+                "Failure to execute update of {} with error: {}",
                 test,
                 error
             )),
-            Ok(update) => match store.update(update) {
+            Ok(()) => {
-                Err(error) => Err(anyhow!(
+                let mut store_dataset: Dataset = store.iter().collect::<Result<_, _>>()?;
-                    "Failure to execute update of {} with error: {}",
+                store_dataset.canonicalize();
-                    test,
+                let mut result_store_dataset: Dataset =
-                    error
+                    result_store.iter().collect::<Result<_, _>>()?;
-                )),
+                result_store_dataset.canonicalize();
-                Ok(()) => {
+                if store_dataset == result_store_dataset {
-                    let mut store_dataset: Dataset = store.iter().collect::<Result<_, _>>()?;
+                    Ok(())
-                    store_dataset.canonicalize();
+                } else {
-                    let mut result_store_dataset: Dataset =
+                    Err(anyhow!(
-                        result_store.iter().collect::<Result<_, _>>()?;
+                        "Failure on {}.\nDiff:\n{}\nParsed update:\n{}\n",
-                    result_store_dataset.canonicalize();
+                        test,
-                    if store_dataset == result_store_dataset {
+                        dataset_diff(&result_store_dataset, &store_dataset),
-                        Ok(())
+                        Update::parse(&read_file_to_string(update_file)?, Some(update_file))
-                    } else {
+                            .unwrap(),
-                        Err(anyhow!(
+                    ))
-                            "Failure on {}.\nDiff:\n{}\nParsed update:\n{}\n",
-                            test,
-                            dataset_diff(&result_store_dataset, &store_dataset),
-                            Update::parse(&read_file_to_string(update_file)?, Some(update_file))
-                                .unwrap(),
-                        ))
-                    }
                 }
-            },
+            }
-        }
+        },
-    } else {
-        Err(anyhow!("Unsupported test type: {}", test.kind))
     }
 }
 

testsuite/tests/oxigraph.rs

@@ -1,10 +1,13 @@
 use anyhow::Result;
+use oxigraph_testsuite::evaluator::TestEvaluator;
 use oxigraph_testsuite::manifest::TestManifest;
-use oxigraph_testsuite::sparql_evaluator::evaluate_sparql_tests;
+use oxigraph_testsuite::sparql_evaluator::register_sparql_tests;
 
 fn run_testsuite(manifest_urls: Vec<&str>) -> Result<()> {
+    let mut evaluator = TestEvaluator::default();
+    register_sparql_tests(&mut evaluator);
     let manifest = TestManifest::new(manifest_urls);
-    let results = evaluate_sparql_tests(manifest)?;
+    let results = evaluator.evaluate(manifest)?;
 
     let mut errors = Vec::default();
     for result in results {

testsuite/tests/parser.rs

@@ -1,10 +1,13 @@
 use anyhow::Result;
+use oxigraph_testsuite::evaluator::TestEvaluator;
 use oxigraph_testsuite::manifest::TestManifest;
-use oxigraph_testsuite::parser_evaluator::evaluate_parser_tests;
+use oxigraph_testsuite::parser_evaluator::register_parser_tests;
 
 fn run_testsuite(manifest_url: &str) -> Result<()> {
+    let mut evaluator = TestEvaluator::default();
+    register_parser_tests(&mut evaluator);
     let manifest = TestManifest::new(vec![manifest_url]);
-    let results = evaluate_parser_tests(manifest)?;
+    let results = evaluator.evaluate(manifest)?;
 
     let mut errors = Vec::default();
     for result in results {

testsuite/tests/sparql.rs

@@ -1,10 +1,13 @@
 use anyhow::Result;
+use oxigraph_testsuite::evaluator::TestEvaluator;
 use oxigraph_testsuite::manifest::TestManifest;
-use oxigraph_testsuite::sparql_evaluator::evaluate_sparql_tests;
+use oxigraph_testsuite::sparql_evaluator::register_sparql_tests;
 
 fn run_testsuite(manifest_url: &str, ignored_tests: Vec<&str>) -> Result<()> {
+    let mut evaluator = TestEvaluator::default();
+    register_sparql_tests(&mut evaluator);
     let manifest = TestManifest::new(vec![manifest_url]);
-    let results = evaluate_sparql_tests(manifest)?;
+    let results = evaluator.evaluate(manifest)?;
 
     let mut errors = Vec::default();
     for result in results {