From 8427eb23c527f7d26c1db09c8685994618568e27 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 2 May 2021 09:44:17 +0200 Subject: [PATCH] Allows to build testsuite reports --- testsuite/Cargo.toml | 1 + testsuite/src/evaluator.rs | 41 +++ testsuite/src/lib.rs | 1 + testsuite/src/main.rs | 27 ++ testsuite/src/parser_evaluator.rs | 163 +++++++----- testsuite/src/report.rs | 102 ++++++++ testsuite/src/sparql_evaluator.rs | 397 +++++++++++++++--------------- testsuite/tests/oxigraph.rs | 7 +- testsuite/tests/parser.rs | 7 +- testsuite/tests/sparql.rs | 7 +- 10 files changed, 487 insertions(+), 266 deletions(-) create mode 100644 testsuite/src/evaluator.rs create mode 100644 testsuite/src/main.rs diff --git a/testsuite/Cargo.toml b/testsuite/Cargo.toml index 7ea2a6e6..520d6e14 100644 --- a/testsuite/Cargo.toml +++ b/testsuite/Cargo.toml @@ -13,6 +13,7 @@ publish = false [dependencies] anyhow = "1" +argh = "0.1" chrono = "0.4" oxigraph = { version = "0.2", path="../lib" } text-diff = "0.4" diff --git a/testsuite/src/evaluator.rs b/testsuite/src/evaluator.rs new file mode 100644 index 00000000..7dbd8b80 --- /dev/null +++ b/testsuite/src/evaluator.rs @@ -0,0 +1,41 @@ +use crate::manifest::Test; +use crate::report::TestResult; +use anyhow::{anyhow, Result}; +use chrono::Utc; +use std::collections::HashMap; + +#[derive(Default)] +pub struct TestEvaluator { + handlers: HashMap Result<()>>>, +} + +impl TestEvaluator { + pub fn register( + &mut self, + test_type: impl Into, + handler: impl Fn(&Test) -> Result<()> + 'static, + ) { + self.handlers.insert(test_type.into(), Box::new(handler)); + } + + pub fn evaluate( + &self, + manifest: impl Iterator>, + ) -> Result> { + manifest + .map(|test| { + let test = test?; + let outcome = if let Some(handler) = self.handlers.get(test.kind.as_str()) { + handler(&test) + } else { + Err(anyhow!("The test type {} is not supported", test.kind)) + }; + Ok(TestResult { + test: test.id, + outcome, + date: Utc::now(), + }) + }) + .collect() + } +} diff --git a/testsuite/src/lib.rs b/testsuite/src/lib.rs index d501de75..ac160782 100644 --- a/testsuite/src/lib.rs +++ b/testsuite/src/lib.rs @@ -10,6 +10,7 @@ unused_qualifications )] +pub mod evaluator; pub mod files; pub mod manifest; pub mod parser_evaluator; diff --git a/testsuite/src/main.rs b/testsuite/src/main.rs new file mode 100644 index 00000000..6cd6b885 --- /dev/null +++ b/testsuite/src/main.rs @@ -0,0 +1,27 @@ +use anyhow::Result; +use argh::FromArgs; +use oxigraph_testsuite::evaluator::TestEvaluator; +use oxigraph_testsuite::manifest::TestManifest; +use oxigraph_testsuite::parser_evaluator::register_parser_tests; +use oxigraph_testsuite::report::build_report; +use oxigraph_testsuite::sparql_evaluator::register_sparql_tests; + +#[derive(FromArgs)] +/// Oxigraph testsuite runner +struct Args { + /// URI of the testsuite manifest to run + #[argh(positional)] + manifest: String, +} + +fn main() -> Result<()> { + let args: Args = argh::from_env(); + + let mut evaluator = TestEvaluator::default(); + register_parser_tests(&mut evaluator); + register_sparql_tests(&mut evaluator); + let manifest = TestManifest::new(vec![args.manifest]); + let results = evaluator.evaluate(manifest)?; + print!("{}", build_report(results)); + Ok(()) +} diff --git a/testsuite/src/parser_evaluator.rs b/testsuite/src/parser_evaluator.rs index 04804811..35216f6c 100644 --- a/testsuite/src/parser_evaluator.rs +++ b/testsuite/src/parser_evaluator.rs @@ -1,80 +1,117 @@ +use crate::evaluator::TestEvaluator; use crate::files::load_dataset; use crate::manifest::Test; -use crate::report::{dataset_diff, TestResult}; +use crate::report::dataset_diff; use anyhow::{anyhow, Result}; -use chrono::Utc; -pub fn evaluate_parser_tests( - manifest: impl Iterator>, -) -> Result> { - manifest - .map(|test| { - let test = test?; - let outcome = evaluate_parser_test(&test); - Ok(TestResult { - test: test.id, - outcome, - date: Utc::now(), - }) - }) - .collect() +pub fn register_parser_tests(evaluator: &mut TestEvaluator) { + evaluator.register( + "http://www.w3.org/ns/rdftest#TestNTriplesPositiveSyntax", + evaluate_positive_syntax_test, + ); + evaluator.register( + "http://www.w3.org/ns/rdftest#TestNQuadsPositiveSyntax", + evaluate_positive_syntax_test, + ); + evaluator.register( + "http://www.w3.org/ns/rdftest#TestTurtlePositiveSyntax", + evaluate_positive_syntax_test, + ); + evaluator.register( + "http://www.w3.org/ns/rdftest#TestTrigPositiveSyntax", + evaluate_positive_syntax_test, + ); + evaluator.register( + "http://www.w3.org/ns/rdftest#TestNTriplesNegativeSyntax", + evaluate_negative_syntax_test, + ); + evaluator.register( + "http://www.w3.org/ns/rdftest#TestNQuadsNegativeSyntax", + evaluate_negative_syntax_test, + ); + evaluator.register( + "http://www.w3.org/ns/rdftest#TestTurtleNegativeSyntax", + evaluate_negative_syntax_test, + ); + evaluator.register( + "http://www.w3.org/ns/rdftest#TestTurtleNegativeEval", + evaluate_negative_syntax_test, + ); + evaluator.register( + "http://www.w3.org/ns/rdftest#TestTrigNegativeSyntax", + evaluate_negative_syntax_test, + ); + evaluator.register( + "http://www.w3.org/ns/rdftest#TestTrigNegativeEval", + evaluate_negative_syntax_test, + ); + evaluator.register( + "http://www.w3.org/ns/rdftest#TestXMLNegativeSyntax", + evaluate_negative_syntax_test, + ); + evaluator.register( + "http://www.w3.org/ns/rdftest#TestTurtleEval", + evaluate_eval_test, + ); + evaluator.register( + "http://www.w3.org/ns/rdftest#TestTrigEval", + evaluate_eval_test, + ); + evaluator.register( + "http://www.w3.org/ns/rdftest#TestXMLEval", + evaluate_eval_test, + ); } -fn evaluate_parser_test(test: &Test) -> Result<()> { +fn evaluate_positive_syntax_test(test: &Test) -> Result<()> { let action = test .action .as_deref() .ok_or_else(|| anyhow!("No action found for test {}", test))?; - if test.kind == "http://www.w3.org/ns/rdftest#TestNTriplesPositiveSyntax" - || test.kind == "http://www.w3.org/ns/rdftest#TestNQuadsPositiveSyntax" - || test.kind == "http://www.w3.org/ns/rdftest#TestTurtlePositiveSyntax" - || test.kind == "http://www.w3.org/ns/rdftest#TestTrigPositiveSyntax" - { - match load_dataset(action) { - Ok(_) => Ok(()), - Err(e) => Err(anyhow!(format!("Parse error: {}", e))), - } - } else if test.kind == "http://www.w3.org/ns/rdftest#TestNTriplesNegativeSyntax" - || test.kind == "http://www.w3.org/ns/rdftest#TestNQuadsNegativeSyntax" - || test.kind == "http://www.w3.org/ns/rdftest#TestTurtleNegativeSyntax" - || test.kind == "http://www.w3.org/ns/rdftest#TestTurtleNegativeEval" - || test.kind == "http://www.w3.org/ns/rdftest#TestTrigNegativeSyntax" - || test.kind == "http://www.w3.org/ns/rdftest#TestTrigNegativeEval" - || test.kind == "http://www.w3.org/ns/rdftest#TestXMLNegativeSyntax" - { - match load_dataset(action) { - Ok(_) => Err(anyhow!("File parsed with an error even if it should not",)), - Err(_) => Ok(()), - } - } else if test.kind == "http://www.w3.org/ns/rdftest#TestTurtleEval" - || test.kind == "http://www.w3.org/ns/rdftest#TestTrigEval" - || test.kind == "http://www.w3.org/ns/rdftest#TestXMLEval" - { - match load_dataset(action) { - Ok(mut actual_graph) => { - actual_graph.canonicalize(); - if let Some(result) = &test.result { - match load_dataset(result) { - Ok(mut expected_graph) => { - expected_graph.canonicalize(); - if expected_graph == actual_graph { - Ok(()) - } else { - Err(anyhow!( - "The two files are not isomorphic. Diff:\n{}", - dataset_diff(&expected_graph, &actual_graph) - )) - } + match load_dataset(action) { + Ok(_) => Ok(()), + Err(e) => Err(anyhow!(format!("Parse error: {}", e))), + } +} + +fn evaluate_negative_syntax_test(test: &Test) -> Result<()> { + let action = test + .action + .as_deref() + .ok_or_else(|| anyhow!("No action found for test {}", test))?; + match load_dataset(action) { + Ok(_) => Err(anyhow!("File parsed with an error even if it should not",)), + Err(_) => Ok(()), + } +} + +fn evaluate_eval_test(test: &Test) -> Result<()> { + let action = test + .action + .as_deref() + .ok_or_else(|| anyhow!("No action found for test {}", test))?; + match load_dataset(action) { + Ok(mut actual_graph) => { + actual_graph.canonicalize(); + if let Some(result) = &test.result { + match load_dataset(result) { + Ok(mut expected_graph) => { + expected_graph.canonicalize(); + if expected_graph == actual_graph { + Ok(()) + } else { + Err(anyhow!( + "The two files are not isomorphic. Diff:\n{}", + dataset_diff(&expected_graph, &actual_graph) + )) } - Err(e) => Err(anyhow!("Parse error on file {}: {}", action, e)), } - } else { - Err(anyhow!("No tests result found")) + Err(e) => Err(anyhow!("Parse error on file {}: {}", action, e)), } + } else { + Err(anyhow!("No tests result found")) } - Err(e) => Err(anyhow!("Parse error on file {}: {}", action, e)), } - } else { - Err(anyhow!("Unsupported test type: {}", test.kind)) + Err(e) => Err(anyhow!("Parse error on file {}: {}", action, e)), } } diff --git a/testsuite/src/report.rs b/testsuite/src/report.rs index 68a4accb..e9fb2e13 100644 --- a/testsuite/src/report.rs +++ b/testsuite/src/report.rs @@ -1,6 +1,7 @@ use anyhow::Result; use chrono::{DateTime, Utc}; use oxigraph::model::{Dataset, NamedNode}; +use std::fmt::Write; use text_diff::{diff, Difference}; #[derive(Debug)] @@ -46,3 +47,104 @@ fn normalize_dataset_text(store: &Dataset) -> String { quads.sort(); quads.join("\n") } + +#[allow(unused_must_use)] +pub fn build_report(results: impl IntoIterator) -> String { + let mut buffer = String::new(); + writeln!(&mut buffer, "@prefix dc: ."); + writeln!( + &mut buffer, + "@prefix doap: ." + ); + writeln!(&mut buffer, "@prefix earl: ."); + writeln!(&mut buffer, "@prefix foaf: ."); + writeln!( + &mut buffer, + "@prefix rdf: ." + ); + writeln!( + &mut buffer, + "@prefix xsd: ." + ); + writeln!(&mut buffer); + writeln!(&mut buffer, "<> foaf:primaryTopic ;"); + writeln!( + &mut buffer, + "\tdc:issued \"{}\"^^xsd:dateTime ;", + Utc::now().to_rfc3339() + ); + writeln!( + &mut buffer, + "\tfoaf:maker ." + ); + writeln!(&mut buffer); + writeln!( + &mut buffer, + " a doap:Project, earl:TestSubject, earl:Software ;" + ); + writeln!(&mut buffer, "\tdoap:name \"Oxigraph\" ;"); + writeln!(&mut buffer, "\tdoap:release ["); + writeln!( + &mut buffer, + "\t\tdoap:name \"Oxigraph {}\";", + env!("CARGO_PKG_VERSION") + ); + writeln!( + &mut buffer, + "\t\tdoap:revision \"{}\" ;", + env!("CARGO_PKG_VERSION") + ); + writeln!(&mut buffer, "\t] ;"); + writeln!( + &mut buffer, + "\tdoap:developer ;" + ); + writeln!(&mut buffer, "\tdoap:homepage ;"); + writeln!( + &mut buffer, + "\tdoap:description \"Oxigraph is an embedded triple store.\"@en ;" + ); + writeln!(&mut buffer, "\tdoap:programming-language \"Rust\" ."); + writeln!(&mut buffer); + writeln!( + &mut buffer, + " a foaf:Person, earl:Assertor ;" + ); + writeln!(&mut buffer, "\tfoaf:name \"Thomas Tanon\"; "); + writeln!( + &mut buffer, + "\tfoaf:homepage ." + ); + writeln!(&mut buffer); + for result in results { + writeln!(&mut buffer); + writeln!(&mut buffer, "["); + writeln!(&mut buffer, "\ta earl:Assertion ;"); + writeln!( + &mut buffer, + "\tearl:assertedBy ;" + ); + writeln!(&mut buffer, "\tearl:subject ;"); + writeln!(&mut buffer, "\tearl:test {} ;", result.test); + writeln!(&mut buffer, "\tearl:result ["); + writeln!(&mut buffer, "\t\ta earl:TestResult ;"); + writeln!( + &mut buffer, + "\t\tearl:outcome earl:{} ;", + if result.outcome.is_ok() { + "passed" + } else { + "failed" + } + ); + writeln!( + &mut buffer, + "\t\tdc:date \"{}\"^^xsd:dateTime", + result.date.to_rfc3339() + ); + writeln!(&mut buffer, "\t] ;"); + writeln!(&mut buffer, "\tearl:mode earl:automatic"); + writeln!(&mut buffer, "] ."); + } + buffer +} diff --git a/testsuite/src/sparql_evaluator.rs b/testsuite/src/sparql_evaluator.rs index 5d726367..1aa5a66c 100644 --- a/testsuite/src/sparql_evaluator.rs +++ b/testsuite/src/sparql_evaluator.rs @@ -1,9 +1,9 @@ +use crate::evaluator::TestEvaluator; use crate::files::*; use crate::manifest::*; -use crate::report::{dataset_diff, TestResult}; +use crate::report::dataset_diff; use crate::vocab::*; use anyhow::{anyhow, Result}; -use chrono::Utc; use oxigraph::model::vocab::*; use oxigraph::model::*; use oxigraph::sparql::*; @@ -13,237 +13,240 @@ use std::str::FromStr; use std::sync::Arc; use std::{fmt, io}; -pub fn evaluate_sparql_tests( - manifest: impl Iterator>, -) -> Result> { - manifest - .map(|test| { - let test = test?; - let outcome = evaluate_sparql_test(&test); - Ok(TestResult { - test: test.id, - outcome, - date: Utc::now(), - }) - }) - .collect() +pub fn register_sparql_tests(evaluator: &mut TestEvaluator) { + evaluator.register( + "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#PositiveSyntaxTest", + evaluate_positive_syntax_test, + ); + evaluator.register( + "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#PositiveSyntaxTest11", + evaluate_positive_syntax_test, + ); + evaluator.register( + "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#NegativeSyntaxTest", + evaluate_negative_syntax_test, + ); + evaluator.register( + "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#NegativeSyntaxTest11", + evaluate_negative_syntax_test, + ); + evaluator.register( + "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#QueryEvaluationTest", + evaluate_evaluation_test, + ); + evaluator.register( + "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#PositiveUpdateSyntaxTest11", + evaluate_positive_update_syntax_test, + ); + evaluator.register( + "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#NegativeUpdateSyntaxTest11", + evaluate_negative_update_syntax_test, + ); + evaluator.register( + "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#UpdateEvaluationTest", + evaluate_update_evaluation_test, + ); } -fn evaluate_sparql_test(test: &Test) -> Result<()> { - if test.kind == "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#PositiveSyntaxTest" - || test.kind - == "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#PositiveSyntaxTest11" - { - let query_file = test - .action - .as_deref() - .ok_or_else(|| anyhow!("No action found for test {}", test))?; - match Query::parse(&read_file_to_string(&query_file)?, Some(&query_file)) { - Err(error) => Err(anyhow!("Not able to parse {} with error: {}", test, error)), - Ok(query) => match Query::parse(&query.to_string(), None) { - Ok(_) => Ok(()), - Err(error) => Err(anyhow!( - "Failure to deserialize \"{}\" of {} with error: {}", - query.to_string(), - test, - error - )), - }, - } - } else if test.kind - == "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#NegativeSyntaxTest" - || test.kind - == "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#NegativeSyntaxTest11" - { - let query_file = test - .action - .as_deref() - .ok_or_else(|| anyhow!("No action found for test {}", test))?; - match Query::parse(&read_file_to_string(query_file)?, Some(query_file)) { - Ok(result) => Err(anyhow!( - "Oxigraph parses even if it should not {}. The output tree is: {}", - test, - result - )), - Err(_) => Ok(()), - } - } else if test.kind - == "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#QueryEvaluationTest" - { - let store = Store::new()?; - if let Some(data) = &test.data { - load_to_store(data, &store, GraphNameRef::DefaultGraph)?; - } - for (name, value) in &test.graph_data { - load_to_store(value, &store, name)?; - } - let query_file = test - .query - .as_deref() - .ok_or_else(|| anyhow!("No action found for test {}", test))?; - let options = QueryOptions::default() - .with_service_handler(StaticServiceHandler::new(&test.service_data)?); - match Query::parse(&read_file_to_string(query_file)?, Some(query_file)) { +fn evaluate_positive_syntax_test(test: &Test) -> Result<()> { + let query_file = test + .action + .as_deref() + .ok_or_else(|| anyhow!("No action found for test {}", test))?; + match Query::parse(&read_file_to_string(&query_file)?, Some(&query_file)) { + Err(error) => Err(anyhow!("Not able to parse {} with error: {}", test, error)), + Ok(query) => match Query::parse(&query.to_string(), None) { + Ok(_) => Ok(()), Err(error) => Err(anyhow!( - "Failure to parse query of {} with error: {}", + "Failure to deserialize \"{}\" of {} with error: {}", + query.to_string(), test, error )), - Ok(query) => { - // FROM and FROM NAMED support. We make sure the data is in the store - if !query.dataset().is_default_dataset() { - for graph_name in query.dataset().default_graph_graphs().unwrap_or(&[]) { - if let GraphName::NamedNode(graph_name) = graph_name { - load_to_store(graph_name.as_str(), &store, graph_name.as_ref())?; - } else { - return Err(anyhow!( - "Invalid FROM in query {} for test {}", - query, - test - )); - } + }, + } +} + +fn evaluate_negative_syntax_test(test: &Test) -> Result<()> { + let query_file = test + .action + .as_deref() + .ok_or_else(|| anyhow!("No action found for test {}", test))?; + match Query::parse(&read_file_to_string(query_file)?, Some(query_file)) { + Ok(result) => Err(anyhow!( + "Oxigraph parses even if it should not {}. The output tree is: {}", + test, + result + )), + Err(_) => Ok(()), + } +} + +fn evaluate_evaluation_test(test: &Test) -> Result<()> { + let store = Store::new()?; + if let Some(data) = &test.data { + load_to_store(data, &store, GraphNameRef::DefaultGraph)?; + } + for (name, value) in &test.graph_data { + load_to_store(value, &store, name)?; + } + let query_file = test + .query + .as_deref() + .ok_or_else(|| anyhow!("No action found for test {}", test))?; + let options = QueryOptions::default() + .with_service_handler(StaticServiceHandler::new(&test.service_data)?); + match Query::parse(&read_file_to_string(query_file)?, Some(query_file)) { + Err(error) => Err(anyhow!( + "Failure to parse query of {} with error: {}", + test, + error + )), + Ok(query) => { + // FROM and FROM NAMED support. We make sure the data is in the store + if !query.dataset().is_default_dataset() { + for graph_name in query.dataset().default_graph_graphs().unwrap_or(&[]) { + if let GraphName::NamedNode(graph_name) = graph_name { + load_to_store(graph_name.as_str(), &store, graph_name.as_ref())?; + } else { + return Err(anyhow!("Invalid FROM in query {} for test {}", query, test)); } - for graph_name in query.dataset().available_named_graphs().unwrap_or(&[]) { - if let Subject::NamedNode(graph_name) = graph_name { - load_to_store(graph_name.as_str(), &store, graph_name.as_ref())?; - } else { - return Err(anyhow!( - "Invalid FROM NAMED in query {} for test {}", - query, - test - )); - } + } + for graph_name in query.dataset().available_named_graphs().unwrap_or(&[]) { + if let Subject::NamedNode(graph_name) = graph_name { + load_to_store(graph_name.as_str(), &store, graph_name.as_ref())?; + } else { + return Err(anyhow!( + "Invalid FROM NAMED in query {} for test {}", + query, + test + )); } } - match store.query_opt(query, options) { - Err(error) => Err(anyhow!( - "Failure to execute query of {} with error: {}", - test, - error - )), - Ok(actual_results) => { - let expected_results = load_sparql_query_result( - test.result.as_ref().unwrap(), - ) + } + match store.query_opt(query, options) { + Err(error) => Err(anyhow!( + "Failure to execute query of {} with error: {}", + test, + error + )), + Ok(actual_results) => { + let expected_results = load_sparql_query_result(test.result.as_ref().unwrap()) .map_err(|e| { anyhow!("Error constructing expected graph for {}: {}", test, e) })?; - let with_order = if let StaticQueryResults::Solutions { ordered, .. } = - &expected_results - { + let with_order = + if let StaticQueryResults::Solutions { ordered, .. } = &expected_results { *ordered } else { false }; - let actual_results = - StaticQueryResults::from_query_results(actual_results, with_order)?; + let actual_results = + StaticQueryResults::from_query_results(actual_results, with_order)?; - if are_query_results_isomorphic(&expected_results, &actual_results) { - Ok(()) - } else { - Err(anyhow!("Failure on {}.\nExpected file:\n{}\nOutput file:\n{}\nParsed query:\n{}\nData:\n{}\n", + if are_query_results_isomorphic(&expected_results, &actual_results) { + Ok(()) + } else { + Err(anyhow!("Failure on {}.\nExpected file:\n{}\nOutput file:\n{}\nParsed query:\n{}\nData:\n{}\n", test, expected_results, actual_results, Query::parse(&read_file_to_string(query_file)?, Some(query_file)).unwrap(), store )) - } } } } } - } else if test.kind - == "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#PositiveUpdateSyntaxTest11" - { - let update_file = test - .action - .as_deref() - .ok_or_else(|| anyhow!("No action found for test {}", test))?; - match Update::parse(&read_file_to_string(&update_file)?, Some(&update_file)) { - Err(error) => Err(anyhow!("Not able to parse {} with error: {}", test, error)), - Ok(update) => match Update::parse(&update.to_string(), None) { - Ok(_) => Ok(()), - Err(error) => Err(anyhow!( - "Failure to deserialize \"{}\" of {} with error: {}", - update.to_string(), - test, - error - )), - }, - } - } else if test.kind - == "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#NegativeUpdateSyntaxTest11" - { - let update_file = test - .action - .as_deref() - .ok_or_else(|| anyhow!("No action found for test {}", test))?; - match Query::parse(&read_file_to_string(update_file)?, Some(update_file)) { - Ok(result) => Err(anyhow!( - "Oxigraph parses even if it should not {}. The output tree is: {}", + } +} + +fn evaluate_positive_update_syntax_test(test: &Test) -> Result<()> { + let update_file = test + .action + .as_deref() + .ok_or_else(|| anyhow!("No action found for test {}", test))?; + match Update::parse(&read_file_to_string(&update_file)?, Some(&update_file)) { + Err(error) => Err(anyhow!("Not able to parse {} with error: {}", test, error)), + Ok(update) => match Update::parse(&update.to_string(), None) { + Ok(_) => Ok(()), + Err(error) => Err(anyhow!( + "Failure to deserialize \"{}\" of {} with error: {}", + update.to_string(), test, - result + error )), - Err(_) => Ok(()), - } - } else if test.kind - == "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#UpdateEvaluationTest" - { - let store = Store::new()?; - if let Some(data) = &test.data { - load_to_store(data, &store, &GraphName::DefaultGraph)?; - } - for (name, value) in &test.graph_data { - load_to_store(value, &store, name)?; - } + }, + } +} - let result_store = Store::new()?; - if let Some(data) = &test.result { - load_to_store(data, &result_store, &GraphName::DefaultGraph)?; - } - for (name, value) in &test.result_graph_data { - load_to_store(value, &result_store, name)?; - } +fn evaluate_negative_update_syntax_test(test: &Test) -> Result<()> { + let update_file = test + .action + .as_deref() + .ok_or_else(|| anyhow!("No action found for test {}", test))?; + match Query::parse(&read_file_to_string(update_file)?, Some(update_file)) { + Ok(result) => Err(anyhow!( + "Oxigraph parses even if it should not {}. The output tree is: {}", + test, + result + )), + Err(_) => Ok(()), + } +} - let update_file = test - .update - .as_deref() - .ok_or_else(|| anyhow!("No action found for test {}", test))?; - match Update::parse(&read_file_to_string(update_file)?, Some(update_file)) { +fn evaluate_update_evaluation_test(test: &Test) -> Result<()> { + let store = Store::new()?; + if let Some(data) = &test.data { + load_to_store(data, &store, &GraphName::DefaultGraph)?; + } + for (name, value) in &test.graph_data { + load_to_store(value, &store, name)?; + } + + let result_store = Store::new()?; + if let Some(data) = &test.result { + load_to_store(data, &result_store, &GraphName::DefaultGraph)?; + } + for (name, value) in &test.result_graph_data { + load_to_store(value, &result_store, name)?; + } + + let update_file = test + .update + .as_deref() + .ok_or_else(|| anyhow!("No action found for test {}", test))?; + match Update::parse(&read_file_to_string(update_file)?, Some(update_file)) { + Err(error) => Err(anyhow!( + "Failure to parse update of {} with error: {}", + test, + error + )), + Ok(update) => match store.update(update) { Err(error) => Err(anyhow!( - "Failure to parse update of {} with error: {}", + "Failure to execute update of {} with error: {}", test, error )), - Ok(update) => match store.update(update) { - Err(error) => Err(anyhow!( - "Failure to execute update of {} with error: {}", - test, - error - )), - Ok(()) => { - let mut store_dataset: Dataset = store.iter().collect::>()?; - store_dataset.canonicalize(); - let mut result_store_dataset: Dataset = - result_store.iter().collect::>()?; - result_store_dataset.canonicalize(); - if store_dataset == result_store_dataset { - Ok(()) - } else { - Err(anyhow!( - "Failure on {}.\nDiff:\n{}\nParsed update:\n{}\n", - test, - dataset_diff(&result_store_dataset, &store_dataset), - Update::parse(&read_file_to_string(update_file)?, Some(update_file)) - .unwrap(), - )) - } + Ok(()) => { + let mut store_dataset: Dataset = store.iter().collect::>()?; + store_dataset.canonicalize(); + let mut result_store_dataset: Dataset = + result_store.iter().collect::>()?; + result_store_dataset.canonicalize(); + if store_dataset == result_store_dataset { + Ok(()) + } else { + Err(anyhow!( + "Failure on {}.\nDiff:\n{}\nParsed update:\n{}\n", + test, + dataset_diff(&result_store_dataset, &store_dataset), + Update::parse(&read_file_to_string(update_file)?, Some(update_file)) + .unwrap(), + )) } - }, - } - } else { - Err(anyhow!("Unsupported test type: {}", test.kind)) + } + }, } } diff --git a/testsuite/tests/oxigraph.rs b/testsuite/tests/oxigraph.rs index 93a1ff2e..5336d9d2 100644 --- a/testsuite/tests/oxigraph.rs +++ b/testsuite/tests/oxigraph.rs @@ -1,10 +1,13 @@ use anyhow::Result; +use oxigraph_testsuite::evaluator::TestEvaluator; use oxigraph_testsuite::manifest::TestManifest; -use oxigraph_testsuite::sparql_evaluator::evaluate_sparql_tests; +use oxigraph_testsuite::sparql_evaluator::register_sparql_tests; fn run_testsuite(manifest_urls: Vec<&str>) -> Result<()> { + let mut evaluator = TestEvaluator::default(); + register_sparql_tests(&mut evaluator); let manifest = TestManifest::new(manifest_urls); - let results = evaluate_sparql_tests(manifest)?; + let results = evaluator.evaluate(manifest)?; let mut errors = Vec::default(); for result in results { diff --git a/testsuite/tests/parser.rs b/testsuite/tests/parser.rs index eeed641f..69a4364a 100644 --- a/testsuite/tests/parser.rs +++ b/testsuite/tests/parser.rs @@ -1,10 +1,13 @@ use anyhow::Result; +use oxigraph_testsuite::evaluator::TestEvaluator; use oxigraph_testsuite::manifest::TestManifest; -use oxigraph_testsuite::parser_evaluator::evaluate_parser_tests; +use oxigraph_testsuite::parser_evaluator::register_parser_tests; fn run_testsuite(manifest_url: &str) -> Result<()> { + let mut evaluator = TestEvaluator::default(); + register_parser_tests(&mut evaluator); let manifest = TestManifest::new(vec![manifest_url]); - let results = evaluate_parser_tests(manifest)?; + let results = evaluator.evaluate(manifest)?; let mut errors = Vec::default(); for result in results { diff --git a/testsuite/tests/sparql.rs b/testsuite/tests/sparql.rs index cc7ce8be..1c31a4a8 100644 --- a/testsuite/tests/sparql.rs +++ b/testsuite/tests/sparql.rs @@ -1,10 +1,13 @@ use anyhow::Result; +use oxigraph_testsuite::evaluator::TestEvaluator; use oxigraph_testsuite::manifest::TestManifest; -use oxigraph_testsuite::sparql_evaluator::evaluate_sparql_tests; +use oxigraph_testsuite::sparql_evaluator::register_sparql_tests; fn run_testsuite(manifest_url: &str, ignored_tests: Vec<&str>) -> Result<()> { + let mut evaluator = TestEvaluator::default(); + register_sparql_tests(&mut evaluator); let manifest = TestManifest::new(vec![manifest_url]); - let results = evaluate_sparql_tests(manifest)?; + let results = evaluator.evaluate(manifest)?; let mut errors = Vec::default(); for result in results {