NextGraph
/
oxigraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Activity

Allows to build testsuite reports

Browse Source
pull/171/head
Tpt 4 years ago
parent 7e9e3a0743
commit 8427eb23c5
10 changed files with 487 additions and 266 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 1
      testsuite/Cargo.toml
  2. 41
      testsuite/src/evaluator.rs
  3. 1
      testsuite/src/lib.rs
  4. 27
      testsuite/src/main.rs
  5. 111
      testsuite/src/parser_evaluator.rs
  6. 102
      testsuite/src/report.rs
  7. 107
      testsuite/src/sparql_evaluator.rs
  8. 7
      testsuite/tests/oxigraph.rs
  9. 7
      testsuite/tests/parser.rs
  10. 7
      testsuite/tests/sparql.rs

1
testsuite/Cargo.toml
Unescape View File

@ -13,6 +13,7 @@ publish = false
[dependencies]
anyhow = "1"
argh = "0.1"
chrono = "0.4"
oxigraph = { version = "0.2", path="../lib" }
text-diff = "0.4"

41
testsuite/src/evaluator.rs
Unescape View File

@ -0,0 +1,41 @@
use crate::manifest::Test;
use crate::report::TestResult;
use anyhow::{anyhow, Result};
use chrono::Utc;
use std::collections::HashMap;
#[derive(Default)]
pub struct TestEvaluator {
handlers: HashMap<String, Box<dyn Fn(&Test) -> Result<()>>>,
}
impl TestEvaluator {
pub fn register(
&mut self,
test_type: impl Into<String>,
handler: impl Fn(&Test) -> Result<()> + 'static,
) {
self.handlers.insert(test_type.into(), Box::new(handler));
}
pub fn evaluate(
&self,
manifest: impl Iterator<Item = Result<Test>>,
) -> Result<Vec<TestResult>> {
manifest
.map(|test| {
let test = test?;
let outcome = if let Some(handler) = self.handlers.get(test.kind.as_str()) {
handler(&test)
} else {
Err(anyhow!("The test type {} is not supported", test.kind))
};
Ok(TestResult {
test: test.id,
outcome,
date: Utc::now(),
})
})
.collect()
}
}

1
testsuite/src/lib.rs
Unescape View File

@ -10,6 +10,7 @@
unused_qualifications
)]
pub mod evaluator;
pub mod files;
pub mod manifest;
pub mod parser_evaluator;

27
testsuite/src/main.rs
Unescape View File

@ -0,0 +1,27 @@
use anyhow::Result;
use argh::FromArgs;
use oxigraph_testsuite::evaluator::TestEvaluator;
use oxigraph_testsuite::manifest::TestManifest;
use oxigraph_testsuite::parser_evaluator::register_parser_tests;
use oxigraph_testsuite::report::build_report;
use oxigraph_testsuite::sparql_evaluator::register_sparql_tests;
#[derive(FromArgs)]
/// Oxigraph testsuite runner
struct Args {
/// URI of the testsuite manifest to run
#[argh(positional)]
manifest: String,
}
fn main() -> Result<()> {
let args: Args = argh::from_env();
let mut evaluator = TestEvaluator::default();
register_parser_tests(&mut evaluator);
register_sparql_tests(&mut evaluator);
let manifest = TestManifest::new(vec![args.manifest]);
let results = evaluator.evaluate(manifest)?;
print!("{}", build_report(results));
Ok(())
}

111
testsuite/src/parser_evaluator.rs
Unescape View File

@ -1,55 +1,95 @@
use crate::evaluator::TestEvaluator;
use crate::files::load_dataset;
use crate::manifest::Test;
use crate::report::{dataset_diff, TestResult};
use crate::report::dataset_diff;
use anyhow::{anyhow, Result};
use chrono::Utc;
pub fn evaluate_parser_tests(
manifest: impl Iterator<Item = Result<Test>>,
) -> Result<Vec<TestResult>> {
manifest
.map(|test| {
let test = test?;
let outcome = evaluate_parser_test(&test);
Ok(TestResult {
test: test.id,
outcome,
date: Utc::now(),
})
})
.collect()
pub fn register_parser_tests(evaluator: &mut TestEvaluator) {
evaluator.register(
"http://www.w3.org/ns/rdftest#TestNTriplesPositiveSyntax",
evaluate_positive_syntax_test,
);
evaluator.register(
"http://www.w3.org/ns/rdftest#TestNQuadsPositiveSyntax",
evaluate_positive_syntax_test,
);
evaluator.register(
"http://www.w3.org/ns/rdftest#TestTurtlePositiveSyntax",
evaluate_positive_syntax_test,
);
evaluator.register(
"http://www.w3.org/ns/rdftest#TestTrigPositiveSyntax",
evaluate_positive_syntax_test,
);
evaluator.register(
"http://www.w3.org/ns/rdftest#TestNTriplesNegativeSyntax",
evaluate_negative_syntax_test,
);
evaluator.register(
"http://www.w3.org/ns/rdftest#TestNQuadsNegativeSyntax",
evaluate_negative_syntax_test,
);
evaluator.register(
"http://www.w3.org/ns/rdftest#TestTurtleNegativeSyntax",
evaluate_negative_syntax_test,
);
evaluator.register(
"http://www.w3.org/ns/rdftest#TestTurtleNegativeEval",
evaluate_negative_syntax_test,
);
evaluator.register(
"http://www.w3.org/ns/rdftest#TestTrigNegativeSyntax",
evaluate_negative_syntax_test,
);
evaluator.register(
"http://www.w3.org/ns/rdftest#TestTrigNegativeEval",
evaluate_negative_syntax_test,
);
evaluator.register(
"http://www.w3.org/ns/rdftest#TestXMLNegativeSyntax",
evaluate_negative_syntax_test,
);
evaluator.register(
"http://www.w3.org/ns/rdftest#TestTurtleEval",
evaluate_eval_test,
);
evaluator.register(
"http://www.w3.org/ns/rdftest#TestTrigEval",
evaluate_eval_test,
);
evaluator.register(
"http://www.w3.org/ns/rdftest#TestXMLEval",
evaluate_eval_test,
);
}
fn evaluate_parser_test(test: &Test) -> Result<()> {
fn evaluate_positive_syntax_test(test: &Test) -> Result<()> {
let action = test
.action
.as_deref()
.ok_or_else(|| anyhow!("No action found for test {}", test))?;
if test.kind == "http://www.w3.org/ns/rdftest#TestNTriplesPositiveSyntax"
|| test.kind == "http://www.w3.org/ns/rdftest#TestNQuadsPositiveSyntax"
|| test.kind == "http://www.w3.org/ns/rdftest#TestTurtlePositiveSyntax"
|| test.kind == "http://www.w3.org/ns/rdftest#TestTrigPositiveSyntax"
{
match load_dataset(action) {
Ok(_) => Ok(()),
Err(e) => Err(anyhow!(format!("Parse error: {}", e))),
}
} else if test.kind == "http://www.w3.org/ns/rdftest#TestNTriplesNegativeSyntax"
|| test.kind == "http://www.w3.org/ns/rdftest#TestNQuadsNegativeSyntax"
|| test.kind == "http://www.w3.org/ns/rdftest#TestTurtleNegativeSyntax"
|| test.kind == "http://www.w3.org/ns/rdftest#TestTurtleNegativeEval"
|| test.kind == "http://www.w3.org/ns/rdftest#TestTrigNegativeSyntax"
|| test.kind == "http://www.w3.org/ns/rdftest#TestTrigNegativeEval"
|| test.kind == "http://www.w3.org/ns/rdftest#TestXMLNegativeSyntax"
{
}
fn evaluate_negative_syntax_test(test: &Test) -> Result<()> {
let action = test
.action
.as_deref()
.ok_or_else(|| anyhow!("No action found for test {}", test))?;
match load_dataset(action) {
Ok(_) => Err(anyhow!("File parsed with an error even if it should not",)),
Err(_) => Ok(()),
}
} else if test.kind == "http://www.w3.org/ns/rdftest#TestTurtleEval"
|| test.kind == "http://www.w3.org/ns/rdftest#TestTrigEval"
|| test.kind == "http://www.w3.org/ns/rdftest#TestXMLEval"
{
}
fn evaluate_eval_test(test: &Test) -> Result<()> {
let action = test
.action
.as_deref()
.ok_or_else(|| anyhow!("No action found for test {}", test))?;
match load_dataset(action) {
Ok(mut actual_graph) => {
actual_graph.canonicalize();
@ -74,7 +114,4 @@ fn evaluate_parser_test(test: &Test) -> Result<()> {
}
Err(e) => Err(anyhow!("Parse error on file {}: {}", action, e)),
}
} else {
Err(anyhow!("Unsupported test type: {}", test.kind))
}
}

102
testsuite/src/report.rs
Unescape View File

@ -1,6 +1,7 @@
use anyhow::Result;
use chrono::{DateTime, Utc};
use oxigraph::model::{Dataset, NamedNode};
use std::fmt::Write;
use text_diff::{diff, Difference};
#[derive(Debug)]
@ -46,3 +47,104 @@ fn normalize_dataset_text(store: &Dataset) -> String {
quads.sort();
quads.join("\n")
}
#[allow(unused_must_use)]
pub fn build_report(results: impl IntoIterator<Item = TestResult>) -> String {
let mut buffer = String::new();
writeln!(&mut buffer, "@prefix dc: <http://purl.org/dc/terms/> .");
writeln!(
&mut buffer,
"@prefix doap: <http://usefulinc.com/ns/doap#> ."
);
writeln!(&mut buffer, "@prefix earl: <http://www.w3.org/ns/earl#> .");
writeln!(&mut buffer, "@prefix foaf: <http://xmlns.com/foaf/0.1/> .");
writeln!(
&mut buffer,
"@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> ."
);
writeln!(
&mut buffer,
"@prefix xsd: <http://www.w3.org/2001/XMLSchema#> ."
);
writeln!(&mut buffer);
writeln!(&mut buffer, "<> foaf:primaryTopic <http://oxigraph.org/> ;");
writeln!(
&mut buffer,
"\tdc:issued \"{}\"^^xsd:dateTime ;",
Utc::now().to_rfc3339()
);
writeln!(
&mut buffer,
"\tfoaf:maker <https://thomas.pellissier-tanon.fr/#me> ."
);
writeln!(&mut buffer);
writeln!(
&mut buffer,
"<http://oxigraph.org/> a doap:Project, earl:TestSubject, earl:Software ;"
);
writeln!(&mut buffer, "\tdoap:name \"Oxigraph\" ;");
writeln!(&mut buffer, "\tdoap:release [");
writeln!(
&mut buffer,
"\t\tdoap:name \"Oxigraph {}\";",
env!("CARGO_PKG_VERSION")
);
writeln!(
&mut buffer,
"\t\tdoap:revision \"{}\" ;",
env!("CARGO_PKG_VERSION")
);
writeln!(&mut buffer, "\t] ;");
writeln!(
&mut buffer,
"\tdoap:developer <https://thomas.pellissier-tanon.fr/#me> ;"
);
writeln!(&mut buffer, "\tdoap:homepage <https://oxigraph.org/> ;");
writeln!(
&mut buffer,
"\tdoap:description \"Oxigraph is an embedded triple store.\"@en ;"
);
writeln!(&mut buffer, "\tdoap:programming-language \"Rust\" .");
writeln!(&mut buffer);
writeln!(
&mut buffer,
"<https://thomas.pellissier-tanon.fr/#me> a foaf:Person, earl:Assertor ;"
);
writeln!(&mut buffer, "\tfoaf:name \"Thomas Tanon\"; ");
writeln!(
&mut buffer,
"\tfoaf:homepage <https://thomas.pellissier-tanon.fr/> ."
);
writeln!(&mut buffer);
for result in results {
writeln!(&mut buffer);
writeln!(&mut buffer, "[");
writeln!(&mut buffer, "\ta earl:Assertion ;");
writeln!(
&mut buffer,
"\tearl:assertedBy <https://thomas.pellissier-tanon.fr/#me> ;"
);
writeln!(&mut buffer, "\tearl:subject <http://oxigraph.org/> ;");
writeln!(&mut buffer, "\tearl:test {} ;", result.test);
writeln!(&mut buffer, "\tearl:result [");
writeln!(&mut buffer, "\t\ta earl:TestResult ;");
writeln!(
&mut buffer,
"\t\tearl:outcome earl:{} ;",
if result.outcome.is_ok() {
"passed"
} else {
"failed"
}
);
writeln!(
&mut buffer,
"\t\tdc:date \"{}\"^^xsd:dateTime",
result.date.to_rfc3339()
);
writeln!(&mut buffer, "\t] ;");
writeln!(&mut buffer, "\tearl:mode earl:automatic");
writeln!(&mut buffer, "] .");
}
buffer
}

107
testsuite/src/sparql_evaluator.rs
Unescape View File

@ -1,9 +1,9 @@
use crate::evaluator::TestEvaluator;
use crate::files::*;
use crate::manifest::*;
use crate::report::{dataset_diff, TestResult};
use crate::report::dataset_diff;
use crate::vocab::*;
use anyhow::{anyhow, Result};
use chrono::Utc;
use oxigraph::model::vocab::*;
use oxigraph::model::*;
use oxigraph::sparql::*;
@ -13,27 +13,42 @@ use std::str::FromStr;
use std::sync::Arc;
use std::{fmt, io};
pub fn evaluate_sparql_tests(
manifest: impl Iterator<Item = Result<Test>>,
) -> Result<Vec<TestResult>> {
manifest
.map(|test| {
let test = test?;
let outcome = evaluate_sparql_test(&test);
Ok(TestResult {
test: test.id,
outcome,
date: Utc::now(),
})
})
.collect()
pub fn register_sparql_tests(evaluator: &mut TestEvaluator) {
evaluator.register(
"http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#PositiveSyntaxTest",
evaluate_positive_syntax_test,
);
evaluator.register(
"http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#PositiveSyntaxTest11",
evaluate_positive_syntax_test,
);
evaluator.register(
"http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#NegativeSyntaxTest",
evaluate_negative_syntax_test,
);
evaluator.register(
"http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#NegativeSyntaxTest11",
evaluate_negative_syntax_test,
);
evaluator.register(
"http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#QueryEvaluationTest",
evaluate_evaluation_test,
);
evaluator.register(
"http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#PositiveUpdateSyntaxTest11",
evaluate_positive_update_syntax_test,
);
evaluator.register(
"http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#NegativeUpdateSyntaxTest11",
evaluate_negative_update_syntax_test,
);
evaluator.register(
"http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#UpdateEvaluationTest",
evaluate_update_evaluation_test,
);
}
fn evaluate_sparql_test(test: &Test) -> Result<()> {
if test.kind == "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#PositiveSyntaxTest"
|| test.kind
== "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#PositiveSyntaxTest11"
{
fn evaluate_positive_syntax_test(test: &Test) -> Result<()> {
let query_file = test
.action
.as_deref()
@ -50,11 +65,9 @@ fn evaluate_sparql_test(test: &Test) -> Result<()> {
)),
},
}
} else if test.kind
== "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#NegativeSyntaxTest"
|| test.kind
== "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#NegativeSyntaxTest11"
{
}
fn evaluate_negative_syntax_test(test: &Test) -> Result<()> {
let query_file = test
.action
.as_deref()
@ -67,9 +80,9 @@ fn evaluate_sparql_test(test: &Test) -> Result<()> {
)),
Err(_) => Ok(()),
}
} else if test.kind
== "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#QueryEvaluationTest"
{
}
fn evaluate_evaluation_test(test: &Test) -> Result<()> {
let store = Store::new()?;
if let Some(data) = &test.data {
load_to_store(data, &store, GraphNameRef::DefaultGraph)?;
@ -96,11 +109,7 @@ fn evaluate_sparql_test(test: &Test) -> Result<()> {
if let GraphName::NamedNode(graph_name) = graph_name {
load_to_store(graph_name.as_str(), &store, graph_name.as_ref())?;
} else {
return Err(anyhow!(
"Invalid FROM in query {} for test {}",
query,
test
));
return Err(anyhow!("Invalid FROM in query {} for test {}", query, test));
}
}
for graph_name in query.dataset().available_named_graphs().unwrap_or(&[]) {
@ -122,15 +131,12 @@ fn evaluate_sparql_test(test: &Test) -> Result<()> {
error
)),
Ok(actual_results) => {
let expected_results = load_sparql_query_result(
test.result.as_ref().unwrap(),
)
let expected_results = load_sparql_query_result(test.result.as_ref().unwrap())
.map_err(|e| {
anyhow!("Error constructing expected graph for {}: {}", test, e)
})?;
let with_order = if let StaticQueryResults::Solutions { ordered, .. } =
&expected_results
{
let with_order =
if let StaticQueryResults::Solutions { ordered, .. } = &expected_results {
*ordered
} else {
false
@ -153,9 +159,9 @@ fn evaluate_sparql_test(test: &Test) -> Result<()> {
}
}
}
} else if test.kind
== "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#PositiveUpdateSyntaxTest11"
{
}
fn evaluate_positive_update_syntax_test(test: &Test) -> Result<()> {
let update_file = test
.action
.as_deref()
@ -172,9 +178,9 @@ fn evaluate_sparql_test(test: &Test) -> Result<()> {
)),
},
}
} else if test.kind
== "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#NegativeUpdateSyntaxTest11"
{
}
fn evaluate_negative_update_syntax_test(test: &Test) -> Result<()> {
let update_file = test
.action
.as_deref()
@ -187,9 +193,9 @@ fn evaluate_sparql_test(test: &Test) -> Result<()> {
)),
Err(_) => Ok(()),
}
} else if test.kind
== "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#UpdateEvaluationTest"
{
}
fn evaluate_update_evaluation_test(test: &Test) -> Result<()> {
let store = Store::new()?;
if let Some(data) = &test.data {
load_to_store(data, &store, &GraphName::DefaultGraph)?;
@ -242,9 +248,6 @@ fn evaluate_sparql_test(test: &Test) -> Result<()> {
}
},
}
} else {
Err(anyhow!("Unsupported test type: {}", test.kind))
}
}
fn load_sparql_query_result(url: &str) -> Result<StaticQueryResults> {

7
testsuite/tests/oxigraph.rs
Unescape View File

@ -1,10 +1,13 @@
use anyhow::Result;
use oxigraph_testsuite::evaluator::TestEvaluator;
use oxigraph_testsuite::manifest::TestManifest;
use oxigraph_testsuite::sparql_evaluator::evaluate_sparql_tests;
use oxigraph_testsuite::sparql_evaluator::register_sparql_tests;
fn run_testsuite(manifest_urls: Vec<&str>) -> Result<()> {
let mut evaluator = TestEvaluator::default();
register_sparql_tests(&mut evaluator);
let manifest = TestManifest::new(manifest_urls);
let results = evaluate_sparql_tests(manifest)?;
let results = evaluator.evaluate(manifest)?;
let mut errors = Vec::default();
for result in results {

7
testsuite/tests/parser.rs
Unescape View File

@ -1,10 +1,13 @@
use anyhow::Result;
use oxigraph_testsuite::evaluator::TestEvaluator;
use oxigraph_testsuite::manifest::TestManifest;
use oxigraph_testsuite::parser_evaluator::evaluate_parser_tests;
use oxigraph_testsuite::parser_evaluator::register_parser_tests;
fn run_testsuite(manifest_url: &str) -> Result<()> {
let mut evaluator = TestEvaluator::default();
register_parser_tests(&mut evaluator);
let manifest = TestManifest::new(vec![manifest_url]);
let results = evaluate_parser_tests(manifest)?;
let results = evaluator.evaluate(manifest)?;
let mut errors = Vec::default();
for result in results {

7
testsuite/tests/sparql.rs
Unescape View File

@ -1,10 +1,13 @@
use anyhow::Result;
use oxigraph_testsuite::evaluator::TestEvaluator;
use oxigraph_testsuite::manifest::TestManifest;
use oxigraph_testsuite::sparql_evaluator::evaluate_sparql_tests;
use oxigraph_testsuite::sparql_evaluator::register_sparql_tests;
fn run_testsuite(manifest_url: &str, ignored_tests: Vec<&str>) -> Result<()> {
let mut evaluator = TestEvaluator::default();
register_sparql_tests(&mut evaluator);
let manifest = TestManifest::new(vec![manifest_url]);
let results = evaluate_sparql_tests(manifest)?;
let results = evaluator.evaluate(manifest)?;
let mut errors = Vec::default();
for result in results {

Write Preview
Loading…
Cancel
Save