From b24bcf52acd4316ee42aaff8c074c9c2d3acfaee Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Apr 2021 09:17:35 +0200 Subject: [PATCH 1/9] Update rocksdb requirement from 0.15 to 0.16 (#98) Updates the requirements on [rocksdb](https://github.com/rust-rocksdb/rust-rocksdb) to permit the latest version. - [Release notes](https://github.com/rust-rocksdb/rust-rocksdb/releases) - [Changelog](https://github.com/rust-rocksdb/rust-rocksdb/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-rocksdb/rust-rocksdb/compare/v0.15.0...v0.16.0) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- lib/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 3f71c7e1..1035d5cc 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -22,7 +22,7 @@ sophia = ["sophia_api"] http_client = ["httparse", "native-tls"] [dependencies] -rocksdb = { version = "0.15", optional = true } +rocksdb = { version = "0.16", optional = true } sled = { version = "0.34", optional = true } quick-xml = "0.22" rand = "0.8" From dbe0465cf1886699ae3a04a54890a3bd503fbe52 Mon Sep 17 00:00:00 2001 From: Tpt Date: Wed, 28 Apr 2021 18:20:24 +0200 Subject: [PATCH 2/9] Exposes Sled flush operation Required to materialize changes on some platforms (Android, Windows...) Bug #99 --- lib/src/store/sled.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/lib/src/store/sled.rs b/lib/src/store/sled.rs index 3c477280..e52b54a5 100644 --- a/lib/src/store/sled.rs +++ b/lib/src/store/sled.rs @@ -664,6 +664,28 @@ impl SledStore { (&mut this).clear() } + /// Flushes all buffers and ensures that all writes are saved on disk. + /// + /// Flushes are automatically done for most platform using background threads. + /// However, calling this method explicitly is still required for Windows and Android. + /// + /// An [async version](SledStore::flush_async) is also available. + pub fn flush(&self) -> Result<(), io::Error> { + self.default.flush()?; + Ok(()) + } + + /// Asynchronously flushes all buffers and ensures that all writes are saved on disk. + /// + /// Flushes are automatically done for most platform using background threads. + /// However, calling this method explicitly is still required for Windows and Android. + /// + /// A [sync version](SledStore::flush) is also available. + pub async fn flush_async(&self) -> Result<(), io::Error> { + self.default.flush_async().await?; + Ok(()) + } + fn contains_encoded(&self, quad: &EncodedQuad) -> Result { let mut buffer = Vec::with_capacity(4 * WRITTEN_TERM_MAX_SIZE); if quad.graph_name.is_default_graph() { From bdeb73868cbe40025d45d19e8cc598a932ee5003 Mon Sep 17 00:00:00 2001 From: Tpt Date: Fri, 9 Apr 2021 20:24:01 +0200 Subject: [PATCH 3/9] Server: Add union-default-graph parameter Bug #92 --- server/src/main.rs | 126 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 96 insertions(+), 30 deletions(-) diff --git a/server/src/main.rs b/server/src/main.rs index ef35ff51..a4346ad2 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -374,6 +374,7 @@ fn configure_and_evaluate_sparql_query( ) -> Result { let mut default_graph_uris = Vec::new(); let mut named_graph_uris = Vec::new(); + let mut use_default_graph_as_union = false; for (k, v) in form_urlencoded::parse(&encoded) { match k.as_ref() { "query" => { @@ -383,12 +384,20 @@ fn configure_and_evaluate_sparql_query( query = Some(v.into_owned()) } "default-graph-uri" => default_graph_uris.push(v.into_owned()), + "union-default-graph" => use_default_graph_as_union = true, "named-graph-uri" => named_graph_uris.push(v.into_owned()), _ => (), } } if let Some(query) = query { - evaluate_sparql_query(store, query, default_graph_uris, named_graph_uris, request) + evaluate_sparql_query( + store, + query, + use_default_graph_as_union, + default_graph_uris, + named_graph_uris, + request, + ) } else { bail_status!(400, "You should set the 'query' parameter") } @@ -397,28 +406,37 @@ fn configure_and_evaluate_sparql_query( fn evaluate_sparql_query( store: Store, query: String, + use_default_graph_as_union: bool, default_graph_uris: Vec, named_graph_uris: Vec, request: Request, ) -> Result { let mut query = Query::parse(&query, Some(base_url(&request)?.as_str())).map_err(bad_request)?; - let default_graph_uris = default_graph_uris - .into_iter() - .map(|e| Ok(NamedNode::new(e)?.into())) - .collect::>>() - .map_err(bad_request)?; - let named_graph_uris = named_graph_uris - .into_iter() - .map(|e| Ok(NamedNode::new(e)?.into())) - .collect::>>() - .map_err(bad_request)?; - - if !default_graph_uris.is_empty() || !named_graph_uris.is_empty() { - query.dataset_mut().set_default_graph(default_graph_uris); - query - .dataset_mut() - .set_available_named_graphs(named_graph_uris); + + if use_default_graph_as_union { + if !default_graph_uris.is_empty() || !named_graph_uris.is_empty() { + bail_status!( + 400, + "default-graph-uri or named-graph-uri and union-default-graph should not be set at the same time" + ); + } + query.dataset_mut().set_default_graph_as_union() + } else if !default_graph_uris.is_empty() || !named_graph_uris.is_empty() { + query.dataset_mut().set_default_graph( + default_graph_uris + .into_iter() + .map(|e| Ok(NamedNode::new(e)?.into())) + .collect::>() + .map_err(bad_request)?, + ); + query.dataset_mut().set_available_named_graphs( + named_graph_uris + .into_iter() + .map(|e| Ok(NamedNode::new(e)?.into())) + .collect::>() + .map_err(bad_request)?, + ); } let results = store.query(query)?; @@ -455,6 +473,7 @@ fn configure_and_evaluate_sparql_update( mut update: Option, request: Request, ) -> Result { + let mut use_default_graph_as_union = false; let mut default_graph_uris = Vec::new(); let mut named_graph_uris = Vec::new(); for (k, v) in form_urlencoded::parse(&encoded) { @@ -466,12 +485,20 @@ fn configure_and_evaluate_sparql_update( update = Some(v.into_owned()) } "using-graph-uri" => default_graph_uris.push(v.into_owned()), + "using-union-graph" => use_default_graph_as_union = true, "using-named-graph-uri" => named_graph_uris.push(v.into_owned()), _ => (), } } if let Some(update) = update { - evaluate_sparql_update(store, update, default_graph_uris, named_graph_uris, request) + evaluate_sparql_update( + store, + update, + use_default_graph_as_union, + default_graph_uris, + named_graph_uris, + request, + ) } else { bail_status!(400, "You should set the 'update' parameter") } @@ -480,27 +507,48 @@ fn configure_and_evaluate_sparql_update( fn evaluate_sparql_update( store: Store, update: String, + use_default_graph_as_union: bool, default_graph_uris: Vec, named_graph_uris: Vec, request: Request, ) -> Result { let mut update = Update::parse(&update, Some(base_url(&request)?.as_str())).map_err(bad_request)?; - let default_graph_uris = default_graph_uris - .into_iter() - .map(|e| Ok(NamedNode::new(e)?.into())) - .collect::>>() - .map_err(bad_request)?; - let named_graph_uris = named_graph_uris - .into_iter() - .map(|e| Ok(NamedNode::new(e)?.into())) - .collect::>>() - .map_err(bad_request)?; - if !default_graph_uris.is_empty() || !named_graph_uris.is_empty() { + + if use_default_graph_as_union { + if !default_graph_uris.is_empty() || !named_graph_uris.is_empty() { + bail_status!( + 400, + "using-graph-uri or using-named-graph-uri and using-union-graph should not be set at the same time" + ); + } for operation in &mut update.operations { if let GraphUpdateOperation::DeleteInsert { using, .. } = operation { if !using.is_default_dataset() { - bail_status!(400, + bail_status!( + 400, + "using-union-graph must not be used with a SPARQL UPDATE containing USING", + ); + } + using.set_default_graph_as_union(); + } + } + } else if !default_graph_uris.is_empty() || !named_graph_uris.is_empty() { + let default_graph_uris = default_graph_uris + .into_iter() + .map(|e| Ok(NamedNode::new(e)?.into())) + .collect::>>() + .map_err(bad_request)?; + let named_graph_uris = named_graph_uris + .into_iter() + .map(|e| Ok(NamedNode::new(e)?.into())) + .collect::>>() + .map_err(bad_request)?; + for operation in &mut update.operations { + if let GraphUpdateOperation::DeleteInsert { using, .. } = operation { + if !using.is_default_dataset() { + bail_status!( + 400, "using-graph-uri and using-named-graph-uri must not be used with a SPARQL UPDATE containing USING", ); } @@ -786,6 +834,24 @@ mod tests { ); } + #[test] + fn get_query_union_graph() { + ServerTest::new().test_status(Request::new( + Method::Get, + Url::parse("http://localhost/query?query=SELECT%20*%20WHERE%20{%20?s%20?p%20?o%20}&union-default-graph") + .unwrap(), + ), StatusCode::Ok); + } + + #[test] + fn get_query_union_graph_and_default_graph() { + ServerTest::new().test_status(Request::new( + Method::Get, + Url::parse("http://localhost/query?query=SELECT%20*%20WHERE%20{%20?s%20?p%20?o%20}&union-default-graph&default-graph-uri=http://example.com") + .unwrap(), + ), StatusCode::BadRequest); + } + #[test] fn get_without_query() { ServerTest::new().test_status( From 8671fb6060fd5e28ef6ec6674dac14ada4a79f0f Mon Sep 17 00:00:00 2001 From: Tpt Date: Wed, 28 Apr 2021 19:06:15 +0200 Subject: [PATCH 4/9] Fixes an out of bound panic in SPARQL evaluation --- lib/src/sparql/eval.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/sparql/eval.rs b/lib/src/sparql/eval.rs index ec27237b..139bef72 100644 --- a/lib/src/sparql/eval.rs +++ b/lib/src/sparql/eval.rs @@ -2682,7 +2682,7 @@ fn get_triple_template_value( TripleTemplateValue::Variable(v) => tuple.get(*v), TripleTemplateValue::BlankNode(id) => { if *id >= bnodes.len() { - bnodes.resize_with(*id, new_bnode) + bnodes.resize_with(*id + 1, new_bnode) } Some(bnodes[*id]) } From 3e05c6ef4049623e77500196b4136e25b6f7789d Mon Sep 17 00:00:00 2001 From: Tpt Date: Wed, 28 Apr 2021 18:54:57 +0200 Subject: [PATCH 5/9] Releases v0.2.4 --- CHANGELOG.md | 9 +++++++++ js/Cargo.toml | 2 +- lib/Cargo.toml | 2 +- python/Cargo.toml | 2 +- server/Cargo.toml | 2 +- testsuite/Cargo.toml | 2 +- wikibase/Cargo.toml | 2 +- 7 files changed, 15 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0921ee75..b3fc551e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +## [0.2.4] - 2021-04-28 + +### Changed +- The HTTP server allows to query the union of all graphs using the `union-default-graph` query parameter and to use the union graph for update `WHERE` clauses using the `using-union-graph` parameter. +- Exposes Sled flush operation (useful for platforms without auto-flush like Windows or Android). +- Fixes a possible out of bound panic in SPARQL query evaluation. +- Upgrades RocksDB to 6.17.3. + + ## [0.2.3] - 2021-04-11 ### Changed diff --git a/js/Cargo.toml b/js/Cargo.toml index 058419a6..1d964191 100644 --- a/js/Cargo.toml +++ b/js/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxigraph_js" -version = "0.2.3" +version = "0.2.4" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 1035d5cc..8ee256da 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxigraph" -version = "0.2.3" +version = "0.2.4" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" diff --git a/python/Cargo.toml b/python/Cargo.toml index 5eabfc99..40045c9e 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyoxigraph" -version = "0.2.3" +version = "0.2.4" authors = ["Tpt"] license = "MIT OR Apache-2.0" readme = "README.md" diff --git a/server/Cargo.toml b/server/Cargo.toml index da187173..5e827758 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxigraph_server" -version = "0.2.3" +version = "0.2.4" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" diff --git a/testsuite/Cargo.toml b/testsuite/Cargo.toml index 7ea2a6e6..706f95ea 100644 --- a/testsuite/Cargo.toml +++ b/testsuite/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxigraph_testsuite" -version = "0.2.3" +version = "0.2.4" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "../README.md" diff --git a/wikibase/Cargo.toml b/wikibase/Cargo.toml index ad863eb2..07bf1352 100644 --- a/wikibase/Cargo.toml +++ b/wikibase/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oxigraph_wikibase" -version = "0.2.3" +version = "0.2.4" authors = ["Tpt "] license = "MIT OR Apache-2.0" readme = "README.md" From c8bfea0b9c6b3490e4e2c3570b3fe7c42487d08c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 May 2021 07:51:17 +0200 Subject: [PATCH 6/9] Bump crazy-max/ghaction-docker-meta from v2 to v3 (#100) * Bump crazy-max/ghaction-docker-meta from v2 to v3 Bumps [crazy-max/ghaction-docker-meta](https://github.com/crazy-max/ghaction-docker-meta) from v2 to v3. - [Release notes](https://github.com/crazy-max/ghaction-docker-meta/releases) - [Commits](https://github.com/crazy-max/ghaction-docker-meta/compare/v2...2af9c6a52b5431eea749f0e923b7503b84813f77) Signed-off-by: dependabot[bot] * Update release.yml Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ce54e508..fabfdb58 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: crazy-max/ghaction-docker-meta@v2 + - uses: docker/metadata-action@v3 id: docker_meta with: images: oxigraph/oxigraph @@ -30,7 +30,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: crazy-max/ghaction-docker-meta@v2 + - uses: docker/metadata-action@v3 id: docker_meta with: images: oxigraph/oxigraph-wikibase From 1852f34c25393af51c8296c9072eb7a81a6167f6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 May 2021 06:04:51 +0000 Subject: [PATCH 7/9] Bump testsuite/rdf-tests from `634dadc` to `43b0eb9` Bumps [testsuite/rdf-tests](https://github.com/w3c/rdf-tests) from `634dadc` to `43b0eb9`. - [Release notes](https://github.com/w3c/rdf-tests/releases) - [Commits](https://github.com/w3c/rdf-tests/compare/634dadcedba0177eb47975d642ae66949774834a...43b0eb9078f144c0d1c36f2c751b81922030eb1a) Signed-off-by: dependabot[bot] --- testsuite/rdf-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testsuite/rdf-tests b/testsuite/rdf-tests index 634dadce..43b0eb90 160000 --- a/testsuite/rdf-tests +++ b/testsuite/rdf-tests @@ -1 +1 @@ -Subproject commit 634dadcedba0177eb47975d642ae66949774834a +Subproject commit 43b0eb9078f144c0d1c36f2c751b81922030eb1a From 7b98d58b1071db090143326ec9cc7fef832315dc Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 30 May 2021 09:25:34 +0200 Subject: [PATCH 8/9] Url: fixes depreciation warning --- server/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main.rs b/server/src/main.rs index a4346ad2..d6948761 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -349,7 +349,7 @@ fn base_url(request: &Request) -> Result { } url.set_query(None); url.set_fragment(None); - Ok(url.into_string()) + Ok(url.into()) } fn resolve_with_base(request: &Request, url: &str) -> Result { From cde2672cdd7b23f461fac64f4bd105cbb4a354c6 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 30 May 2021 09:29:58 +0200 Subject: [PATCH 9/9] Adds JSON deserializer Closes #47 --- lib/Cargo.toml | 1 + lib/src/sparql/json_results.rs | 375 ++++++++++++++++++++++++++------- lib/src/sparql/model.rs | 78 ++++++- lib/src/sparql/xml_results.rs | 6 +- testsuite/tests/sparql.rs | 5 +- 5 files changed, 377 insertions(+), 88 deletions(-) diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 8ee256da..48809c72 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -45,6 +45,7 @@ sophia_api = { version = "0.6.2", optional = true } http = "0.2" httparse = { version = "1", optional = true } native-tls = { version = "0.2", optional = true } +json-event-parser = "0.1" [target.'cfg(target_arch = "wasm32")'.dependencies] js-sys = "0.3" diff --git a/lib/src/sparql/json_results.rs b/lib/src/sparql/json_results.rs index 609aa888..0650f3e7 100644 --- a/lib/src/sparql/json_results.rs +++ b/lib/src/sparql/json_results.rs @@ -1,80 +1,89 @@ //! Implementation of [SPARQL Query Results JSON Format](https://www.w3.org/TR/sparql11-results-json/) -use crate::error::invalid_input_error; +use crate::error::{invalid_data_error, invalid_input_error}; use crate::model::*; use crate::sparql::error::EvaluationError; use crate::sparql::model::*; -use std::io::Write; +use json_event_parser::{JsonEvent, JsonReader, JsonWriter}; +use std::collections::BTreeMap; +use std::io; +use std::io::{BufRead, Write}; +use std::rc::Rc; -pub fn write_json_results( - results: QueryResults, - mut sink: impl Write, -) -> Result<(), EvaluationError> { +pub fn write_json_results(results: QueryResults, sink: impl Write) -> Result<(), EvaluationError> { + let mut writer = JsonWriter::from_writer(sink); match results { QueryResults::Boolean(value) => { - sink.write_all(b"{\"head\":{},\"boolean\":")?; - sink.write_all(if value { b"true" } else { b"false" })?; - sink.write_all(b"}")?; + writer.write_event(JsonEvent::StartObject)?; + writer.write_event(JsonEvent::ObjectKey("head"))?; + writer.write_event(JsonEvent::StartObject)?; + writer.write_event(JsonEvent::EndObject)?; + writer.write_event(JsonEvent::ObjectKey("boolean"))?; + writer.write_event(JsonEvent::Boolean(value))?; + writer.write_event(JsonEvent::EndObject)?; Ok(()) } QueryResults::Solutions(solutions) => { - sink.write_all(b"{\"head\":{\"vars\":[")?; - let mut start_vars = true; + writer.write_event(JsonEvent::StartObject)?; + writer.write_event(JsonEvent::ObjectKey("head"))?; + writer.write_event(JsonEvent::StartObject)?; + writer.write_event(JsonEvent::ObjectKey("vars"))?; + writer.write_event(JsonEvent::StartArray)?; for variable in solutions.variables() { - if start_vars { - start_vars = false; - } else { - sink.write_all(b",")?; - } - write_escaped_json_string(variable.as_str(), &mut sink)?; + writer.write_event(JsonEvent::String(variable.as_str()))?; } - sink.write_all(b"]},\"results\":{\"bindings\":[")?; - let mut start_bindings = true; + writer.write_event(JsonEvent::EndArray)?; + writer.write_event(JsonEvent::EndObject)?; + writer.write_event(JsonEvent::ObjectKey("results"))?; + writer.write_event(JsonEvent::StartObject)?; + writer.write_event(JsonEvent::ObjectKey("bindings"))?; + writer.write_event(JsonEvent::StartArray)?; for solution in solutions { - if start_bindings { - start_bindings = false; - } else { - sink.write_all(b",")?; - } - sink.write_all(b"{")?; + writer.write_event(JsonEvent::StartObject)?; let solution = solution?; - let mut start_binding = true; for (variable, value) in solution.iter() { - if start_binding { - start_binding = false; - } else { - sink.write_all(b",")?; - } - write_escaped_json_string(variable.as_str(), &mut sink)?; + writer.write_event(JsonEvent::ObjectKey(variable.as_str()))?; match value { Term::NamedNode(uri) => { - sink.write_all(b":{\"type\":\"uri\",\"value\":")?; - write_escaped_json_string(uri.as_str(), &mut sink)?; - sink.write_all(b"}")?; + writer.write_event(JsonEvent::StartObject)?; + writer.write_event(JsonEvent::ObjectKey("type"))?; + writer.write_event(JsonEvent::String("uri"))?; + writer.write_event(JsonEvent::ObjectKey("value"))?; + writer.write_event(JsonEvent::String(uri.as_str()))?; + writer.write_event(JsonEvent::EndObject)?; } Term::BlankNode(bnode) => { - sink.write_all(b":{\"type\":\"bnode\",\"value\":")?; - write_escaped_json_string(bnode.as_str(), &mut sink)?; - sink.write_all(b"}")?; + writer.write_event(JsonEvent::StartObject)?; + writer.write_event(JsonEvent::ObjectKey("type"))?; + writer.write_event(JsonEvent::String("bnode"))?; + writer.write_event(JsonEvent::ObjectKey("value"))?; + writer.write_event(JsonEvent::String(bnode.as_str()))?; + writer.write_event(JsonEvent::EndObject)?; } Term::Literal(literal) => { - sink.write_all(b":{\"type\":\"literal\",\"value\":")?; - write_escaped_json_string(literal.value(), &mut sink)?; + writer.write_event(JsonEvent::StartObject)?; + writer.write_event(JsonEvent::ObjectKey("type"))?; + writer.write_event(JsonEvent::String("literal"))?; + writer.write_event(JsonEvent::ObjectKey("value"))?; + writer.write_event(JsonEvent::String(literal.value()))?; if let Some(language) = literal.language() { - sink.write_all(b",\"xml:lang\":")?; - write_escaped_json_string(language, &mut sink)?; + writer.write_event(JsonEvent::ObjectKey("xml:lang"))?; + writer.write_event(JsonEvent::String(language))?; } else if !literal.is_plain() { - sink.write_all(b",\"datatype\":")?; - write_escaped_json_string(literal.datatype().as_str(), &mut sink)?; + writer.write_event(JsonEvent::ObjectKey("datatype"))?; + writer + .write_event(JsonEvent::String(literal.datatype().as_str()))?; } - sink.write_all(b"}")?; + writer.write_event(JsonEvent::EndObject)?; } } } - sink.write_all(b"}")?; + writer.write_event(JsonEvent::EndObject)?; } - sink.write_all(b"]}}")?; + writer.write_event(JsonEvent::EndArray)?; + writer.write_event(JsonEvent::EndObject)?; + writer.write_event(JsonEvent::EndObject)?; Ok(()) } QueryResults::Graph(_) => Err(invalid_input_error( @@ -84,37 +93,253 @@ pub fn write_json_results( } } -fn write_escaped_json_string(s: &str, mut sink: impl Write) -> Result<(), EvaluationError> { - sink.write_all(b"\"")?; - for c in s.chars() { - match c { - '\\' => sink.write_all(b"\\\\"), - '"' => sink.write_all(b"\\\""), - c => { - if c < char::from(32) { - match c { - '\u{08}' => sink.write_all(b"\\b"), - '\u{0C}' => sink.write_all(b"\\f"), - '\n' => sink.write_all(b"\\n"), - '\r' => sink.write_all(b"\\r"), - '\t' => sink.write_all(b"\\t"), - c => { - let mut c = c as u8; - let mut result = [b'\\', b'u', 0, 0, 0, 0]; - for i in (2..6).rev() { - let ch = c % 16; - result[i] = ch + if ch < 10 { b'0' } else { b'A' }; - c /= 16; - } - sink.write_all(&result) +pub fn read_json_results(source: impl BufRead + 'static) -> Result { + let mut reader = JsonReader::from_reader(source); + let mut buffer = Vec::default(); + let mut variables = None; + + if reader.read_event(&mut buffer)? != JsonEvent::StartObject { + return Err(invalid_data_error( + "SPARQL JSON results should be an object", + )); + } + + loop { + let event = reader.read_event(&mut buffer)?; + match event { + JsonEvent::ObjectKey(key) => match key { + "head" => variables = Some(read_head(&mut reader, &mut buffer)?), + "results" => { + if reader.read_event(&mut buffer)? != JsonEvent::StartObject { + return Err(invalid_data_error("'results' should be an object")); + } + if reader.read_event(&mut buffer)? != JsonEvent::ObjectKey("bindings") { + return Err(invalid_data_error( + "'results' should contain a 'bindings' key", + )); + } + if reader.read_event(&mut buffer)? != JsonEvent::StartArray { + return Err(invalid_data_error("'bindings' should be an object")); + } + return if let Some(variables) = variables { + let mut mapping = BTreeMap::default(); + for (i, var) in variables.iter().enumerate() { + mapping.insert(var.clone(), i); } + Ok(QueryResults::Solutions(QuerySolutionIter::new( + Rc::new( + variables + .into_iter() + .map(Variable::new) + .collect::, _>>() + .map_err(invalid_data_error)?, + ), + Box::new(ResultsIterator { + reader, + buffer, + mapping, + }), + ))) + } else { + Err(invalid_data_error( + "SPARQL tuple query results should contain a head key", + )) + }; + } + "boolean" => { + return if let JsonEvent::Boolean(v) = reader.read_event(&mut buffer)? { + Ok(QueryResults::Boolean(v)) + } else { + Err(invalid_data_error("Unexpected boolean value")) } - } else { - write!(sink, "{}", c) } + _ => { + return Err(invalid_data_error(format!( + "Expecting head or result key, found {}", + key + ))); + } + }, + JsonEvent::EndObject => { + return Err(invalid_data_error( + "SPARQL results should contain a bindings key or a boolean key", + )) } - }?; + JsonEvent::Eof => return Err(io::Error::from(io::ErrorKind::UnexpectedEof)), + _ => return Err(invalid_data_error("Invalid SPARQL results serialization")), + } + } +} + +fn read_head( + reader: &mut JsonReader, + buffer: &mut Vec, +) -> io::Result> { + if reader.read_event(buffer)? != JsonEvent::StartObject { + return Err(invalid_data_error("head should be an object")); + } + let mut variables = None; + loop { + match reader.read_event(buffer)? { + JsonEvent::ObjectKey(key) => match key { + "vars" => variables = Some(read_string_array(reader, buffer)?), + "link" => { + read_string_array(reader, buffer)?; + } + _ => { + return Err(invalid_data_error(format!( + "Unexpected key in head: '{}'", + key + ))) + } + }, + JsonEvent::EndObject => return Ok(variables.unwrap_or_else(Vec::new)), + _ => return Err(invalid_data_error("Invalid head serialization")), + } + } +} + +fn read_string_array( + reader: &mut JsonReader, + buffer: &mut Vec, +) -> io::Result> { + if reader.read_event(buffer)? != JsonEvent::StartArray { + return Err(invalid_data_error("Variable list should be an array")); + } + let mut elements = Vec::new(); + loop { + match reader.read_event(buffer)? { + JsonEvent::String(s) => { + elements.push(s.into()); + } + JsonEvent::EndArray => return Ok(elements), + _ => return Err(invalid_data_error("Variable names should be strings")), + } + } +} + +struct ResultsIterator { + reader: JsonReader, + buffer: Vec, + mapping: BTreeMap, +} + +impl Iterator for ResultsIterator { + type Item = Result>, EvaluationError>; + + fn next(&mut self) -> Option>, EvaluationError>> { + self.read_next().map_err(EvaluationError::from).transpose() + } +} + +impl ResultsIterator { + fn read_next(&mut self) -> io::Result>>> { + let mut new_bindings = vec![None; self.mapping.len()]; + loop { + match self.reader.read_event(&mut self.buffer)? { + JsonEvent::StartObject => (), + JsonEvent::EndObject => return Ok(Some(new_bindings)), + JsonEvent::EndArray | JsonEvent::Eof => return Ok(None), + JsonEvent::ObjectKey(key) => { + let k = *self.mapping.get(key).ok_or_else(|| { + invalid_data_error(format!( + "The variable {} has not been defined in the header", + key + )) + })?; + new_bindings[k] = Some(self.read_value()?) + } + _ => return Err(invalid_data_error("Invalid result serialization")), + } + } + } + fn read_value(&mut self) -> io::Result { + enum Type { + Uri, + BNode, + Literal, + } + enum State { + Type, + Value, + Lang, + Datatype, + } + let mut state = None; + let mut t = None; + let mut value = None; + let mut lang = None; + let mut datatype = None; + if self.reader.read_event(&mut self.buffer)? != JsonEvent::StartObject { + return Err(invalid_data_error( + "Term serializations should be an object", + )); + } + loop { + match self.reader.read_event(&mut self.buffer)? { + JsonEvent::ObjectKey(key) => match key { + "type" => state = Some(State::Type), + "value" => state = Some(State::Value), + "xml:lang" => state = Some(State::Lang), + "datatype" => state = Some(State::Datatype), + _ => { + return Err(invalid_data_error(format!( + "Unexpected key in term serialization: '{}'", + key + ))) + } + }, + JsonEvent::String(s) => match state { + None => (), // impossible + Some(State::Type) => match s { + "uri" => t = Some(Type::Uri), + "bnode" => t = Some(Type::BNode), + "literal" => t = Some(Type::Literal), + _ => { + return Err(invalid_data_error(format!( + "Unexpected term type: '{}'", + s + ))) + } + }, + Some(State::Value) => value = Some(s.to_owned()), + Some(State::Lang) => lang = Some(s.to_owned()), + Some(State::Datatype) => datatype = Some(s.to_owned()), + }, + JsonEvent::EndObject => { + let value = value.ok_or_else(|| { + invalid_data_error("Term serialization should have a value key") + })?; + return match t { + None => Err(invalid_data_error( + "Term serialization should have a type key", + )), + Some(Type::Uri) => Ok(NamedNode::new(value) + .map_err(|e| invalid_data_error(format!("Invalid uri value: {}", e)))? + .into()), + Some(Type::BNode) => Ok(BlankNode::new(value) + .map_err(|e| invalid_data_error(format!("Invalid bnode value: {}", e)))? + .into()), + Some(Type::Literal) => Ok(match datatype { + Some(datatype) => Literal::new_typed_literal( + value, + NamedNode::new(datatype).map_err(|e| { + invalid_data_error(format!("Invalid datatype value: {}", e)) + })?, + ), + None => match lang { + Some(lang) => Literal::new_language_tagged_literal(value, lang) + .map_err(|e| { + invalid_data_error(format!("Invalid xml:lang value: {}", e)) + })?, + None => Literal::new_simple_literal(value), + }, + } + .into()), + }; + } + _ => return Err(invalid_data_error("Invalid term serialization")), + } + } } - sink.write_all(b"\"")?; - Ok(()) } diff --git a/lib/src/sparql/model.rs b/lib/src/sparql/model.rs index f8ff1605..ec056a6b 100644 --- a/lib/src/sparql/model.rs +++ b/lib/src/sparql/model.rs @@ -4,7 +4,7 @@ use crate::io::GraphSerializer; use crate::model::*; use crate::sparql::csv_results::{read_tsv_results, write_csv_results, write_tsv_results}; use crate::sparql::error::EvaluationError; -use crate::sparql::json_results::write_json_results; +use crate::sparql::json_results::{read_json_results, write_json_results}; use crate::sparql::xml_results::{read_xml_results, write_xml_results}; use rand::random; use std::error::Error; @@ -30,11 +30,9 @@ impl QueryResults { ) -> Result { match format { QueryResultsFormat::Xml => read_xml_results(reader), - QueryResultsFormat::Json => Err(invalid_input_error( - "JSON SPARQL results format parsing has not been implemented yet", - )), //TODO: implement + QueryResultsFormat::Json => read_json_results(reader), QueryResultsFormat::Csv => Err(invalid_input_error( - "CSV and TSV SPARQL results format parsing is not implemented", + "CSV SPARQL results format parsing is not implemented", )), QueryResultsFormat::Tsv => read_tsv_results(reader), } @@ -520,3 +518,73 @@ impl fmt::Display for VariableNameParseError { } impl Error for VariableNameParseError {} + +#[test] +fn test_serialization_rountrip() -> Result<(), EvaluationError> { + use std::io::Cursor; + use std::str; + + for format in &[ + QueryResultsFormat::Xml, + QueryResultsFormat::Json, + QueryResultsFormat::Tsv, + ] { + let results = vec![ + QueryResults::Boolean(true), + QueryResults::Boolean(false), + QueryResults::Solutions(QuerySolutionIter::new( + Rc::new(vec![ + Variable::new_unchecked("foo"), + Variable::new_unchecked("bar"), + ]), + Box::new( + vec![ + Ok(vec![None, None]), + Ok(vec![ + Some(NamedNode::new_unchecked("http://example.com").into()), + None, + ]), + Ok(vec![ + None, + Some(NamedNode::new_unchecked("http://example.com").into()), + ]), + Ok(vec![ + Some(BlankNode::new_unchecked("foo").into()), + Some(BlankNode::new_unchecked("bar").into()), + ]), + Ok(vec![Some(Literal::new_simple_literal("foo").into()), None]), + Ok(vec![ + Some( + Literal::new_language_tagged_literal_unchecked("foo", "fr").into(), + ), + None, + ]), + Ok(vec![ + Some(Literal::from(1).into()), + Some(Literal::from(true).into()), + ]), + Ok(vec![ + Some(Literal::from(1.33).into()), + Some(Literal::from(false).into()), + ]), + ] + .into_iter(), + ), + )), + ]; + + for ex in results { + let mut buffer = Vec::new(); + ex.write(&mut buffer, *format)?; + let ex2 = QueryResults::read(Cursor::new(buffer.clone()), *format)?; + let mut buffer2 = Vec::new(); + ex2.write(&mut buffer2, *format)?; + assert_eq!( + str::from_utf8(&buffer).unwrap(), + str::from_utf8(&buffer2).unwrap() + ); + } + } + + Ok(()) +} diff --git a/lib/src/sparql/xml_results.rs b/lib/src/sparql/xml_results.rs index 88a55d5f..e0ca3617 100644 --- a/lib/src/sparql/xml_results.rs +++ b/lib/src/sparql/xml_results.rs @@ -318,8 +318,7 @@ impl ResultsIterator { } let mut state = State::Start; - let mut new_bindings = Vec::default(); - new_bindings.resize(self.mapping.len(), None); + let mut new_bindings = vec![None; self.mapping.len()]; let mut current_var = None; let mut term: Option = None; @@ -474,13 +473,12 @@ impl ResultsIterator { State::Result => return Ok(Some(new_bindings)), State::Binding => { if let Some(var) = ¤t_var { - new_bindings[self.mapping[var]] = term.clone() + new_bindings[self.mapping[var]] = term.take() } else { return Err( invalid_data_error("No name found for tag").into() ); } - term = None; state = State::Result; } State::Uri | State::BNode => state = State::Binding, diff --git a/testsuite/tests/sparql.rs b/testsuite/tests/sparql.rs index 2b10b63f..f73b4f94 100644 --- a/testsuite/tests/sparql.rs +++ b/testsuite/tests/sparql.rs @@ -85,15 +85,12 @@ fn sparql11_query_w3c_evaluation_testsuite() -> Result<()> { "http://www.w3.org/2009/sparql/docs/tests/data-sparql11/syntax-query/manifest#test_61a", "http://www.w3.org/2009/sparql/docs/tests/data-sparql11/syntax-query/manifest#test_62a", "http://www.w3.org/2009/sparql/docs/tests/data-sparql11/syntax-query/manifest#test_65", - // SPARQL 1.1 JSON query results deserialization is not implemented yet - "http://www.w3.org/2009/sparql/docs/tests/data-sparql11/aggregates/manifest#agg-empty-group-count-1", - "http://www.w3.org/2009/sparql/docs/tests/data-sparql11/aggregates/manifest#agg-empty-group-count-2", //BNODE() scope is currently wrong "http://www.w3.org/2009/sparql/docs/tests/data-sparql11/functions/manifest#bnode01", //Property path with unbound graph name are not supported yet "http://www.w3.org/2009/sparql/docs/tests/data-sparql11/property-path/manifest#pp35", //SERVICE name from a BGP - "http://www.w3.org/2009/sparql/docs/tests/data-sparql11/service/manifest#service5" + "http://www.w3.org/2009/sparql/docs/tests/data-sparql11/service/manifest#service5", ], ) }