diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fd7e1b05..fec4ff6d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: crazy-max/ghaction-docker-meta@v2 + - uses: docker/metadata-action@v3 id: docker_meta with: images: oxigraph/oxigraph @@ -30,7 +30,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: crazy-max/ghaction-docker-meta@v2 + - uses: docker/metadata-action@v3 id: docker_meta with: images: oxigraph/oxigraph-wikibase diff --git a/CHANGELOG.md b/CHANGELOG.md index 0921ee75..b3fc551e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +## [0.2.4] - 2021-04-28 + +### Changed +- The HTTP server allows to query the union of all graphs using the `union-default-graph` query parameter and to use the union graph for update `WHERE` clauses using the `using-union-graph` parameter. +- Exposes Sled flush operation (useful for platforms without auto-flush like Windows or Android). +- Fixes a possible out of bound panic in SPARQL query evaluation. +- Upgrades RocksDB to 6.17.3. + + ## [0.2.3] - 2021-04-11 ### Changed diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 47d7bd16..3f90e365 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -43,6 +43,7 @@ sophia_api = { version = "0.6.2", optional = true } http = "0.2" httparse = { version = "1", optional = true } native-tls = { version = "0.2", optional = true } +json-event-parser = "0.1" spargebra = { version = "0.1", path="../spargebra" } diff --git a/lib/src/sparql/json_results.rs b/lib/src/sparql/json_results.rs index f9fa4a79..6b3321f1 100644 --- a/lib/src/sparql/json_results.rs +++ b/lib/src/sparql/json_results.rs @@ -1,58 +1,56 @@ //! Implementation of [SPARQL Query Results JSON Format](https://www.w3.org/TR/sparql11-results-json/) -use crate::error::invalid_input_error; +use crate::error::{invalid_data_error, invalid_input_error}; use crate::model::*; use crate::sparql::error::EvaluationError; use crate::sparql::model::*; -use std::io::Write; +use json_event_parser::{JsonEvent, JsonReader, JsonWriter}; +use std::collections::BTreeMap; +use std::io; +use std::io::{BufRead, Write}; +use std::rc::Rc; -pub fn write_json_results( - results: QueryResults, - mut sink: impl Write, -) -> Result<(), EvaluationError> { +pub fn write_json_results(results: QueryResults, sink: impl Write) -> Result<(), EvaluationError> { + let mut writer = JsonWriter::from_writer(sink); match results { QueryResults::Boolean(value) => { - sink.write_all(b"{\"head\":{},\"boolean\":")?; - sink.write_all(if value { b"true" } else { b"false" })?; - sink.write_all(b"}")?; + writer.write_event(JsonEvent::StartObject)?; + writer.write_event(JsonEvent::ObjectKey("head"))?; + writer.write_event(JsonEvent::StartObject)?; + writer.write_event(JsonEvent::EndObject)?; + writer.write_event(JsonEvent::ObjectKey("boolean"))?; + writer.write_event(JsonEvent::Boolean(value))?; + writer.write_event(JsonEvent::EndObject)?; Ok(()) } QueryResults::Solutions(solutions) => { - sink.write_all(b"{\"head\":{\"vars\":[")?; - let mut start_vars = true; + writer.write_event(JsonEvent::StartObject)?; + writer.write_event(JsonEvent::ObjectKey("head"))?; + writer.write_event(JsonEvent::StartObject)?; + writer.write_event(JsonEvent::ObjectKey("vars"))?; + writer.write_event(JsonEvent::StartArray)?; for variable in solutions.variables() { - if start_vars { - start_vars = false; - } else { - sink.write_all(b",")?; - } - write_escaped_json_string(variable.as_str(), &mut sink)?; + writer.write_event(JsonEvent::String(variable.as_str()))?; } - sink.write_all(b"]},\"results\":{\"bindings\":[")?; - let mut start_bindings = true; + writer.write_event(JsonEvent::EndArray)?; + writer.write_event(JsonEvent::EndObject)?; + writer.write_event(JsonEvent::ObjectKey("results"))?; + writer.write_event(JsonEvent::StartObject)?; + writer.write_event(JsonEvent::ObjectKey("bindings"))?; + writer.write_event(JsonEvent::StartArray)?; for solution in solutions { - if start_bindings { - start_bindings = false; - } else { - sink.write_all(b",")?; - } - sink.write_all(b"{")?; + writer.write_event(JsonEvent::StartObject)?; let solution = solution?; - let mut start_binding = true; for (variable, value) in solution.iter() { - if start_binding { - start_binding = false; - } else { - sink.write_all(b",")?; - } - write_escaped_json_string(variable.as_str(), &mut sink)?; - sink.write_all(b":")?; - write_json_term(value.as_ref(), &mut sink)?; + writer.write_event(JsonEvent::ObjectKey(variable.as_str()))?; + write_json_term(value.as_ref(), &mut writer)?; } - sink.write_all(b"}")?; + writer.write_event(JsonEvent::EndObject)?; } - sink.write_all(b"]}}")?; + writer.write_event(JsonEvent::EndArray)?; + writer.write_event(JsonEvent::EndObject)?; + writer.write_event(JsonEvent::EndObject)?; Ok(()) } QueryResults::Graph(_) => Err(invalid_input_error( @@ -62,74 +60,398 @@ pub fn write_json_results( } } -fn write_json_term(term: TermRef<'_>, sink: &mut impl Write) -> Result<(), EvaluationError> { +fn write_json_term( + term: TermRef<'_>, + writer: &mut JsonWriter, +) -> Result<(), EvaluationError> { match term { TermRef::NamedNode(uri) => { - sink.write_all(b"{\"type\":\"uri\",\"value\":")?; - write_escaped_json_string(uri.as_str(), sink)?; - sink.write_all(b"}")?; + writer.write_event(JsonEvent::StartObject)?; + writer.write_event(JsonEvent::ObjectKey("type"))?; + writer.write_event(JsonEvent::String("uri"))?; + writer.write_event(JsonEvent::ObjectKey("value"))?; + writer.write_event(JsonEvent::String(uri.as_str()))?; + writer.write_event(JsonEvent::EndObject)?; } TermRef::BlankNode(bnode) => { - sink.write_all(b"{\"type\":\"bnode\",\"value\":")?; - write_escaped_json_string(bnode.as_str(), sink)?; - sink.write_all(b"}")?; + writer.write_event(JsonEvent::StartObject)?; + writer.write_event(JsonEvent::ObjectKey("type"))?; + writer.write_event(JsonEvent::String("bnode"))?; + writer.write_event(JsonEvent::ObjectKey("value"))?; + writer.write_event(JsonEvent::String(bnode.as_str()))?; + writer.write_event(JsonEvent::EndObject)?; } TermRef::Literal(literal) => { - sink.write_all(b"{\"type\":\"literal\",\"value\":")?; - write_escaped_json_string(literal.value(), sink)?; + writer.write_event(JsonEvent::StartObject)?; + writer.write_event(JsonEvent::ObjectKey("type"))?; + writer.write_event(JsonEvent::String("literal"))?; + writer.write_event(JsonEvent::ObjectKey("value"))?; + writer.write_event(JsonEvent::String(literal.value()))?; if let Some(language) = literal.language() { - sink.write_all(b",\"xml:lang\":")?; - write_escaped_json_string(language, sink)?; + writer.write_event(JsonEvent::ObjectKey("xml:lang"))?; + writer.write_event(JsonEvent::String(language))?; } else if !literal.is_plain() { - sink.write_all(b",\"datatype\":")?; - write_escaped_json_string(literal.datatype().as_str(), sink)?; + writer.write_event(JsonEvent::ObjectKey("datatype"))?; + writer.write_event(JsonEvent::String(literal.datatype().as_str()))?; } - sink.write_all(b"}")?; + writer.write_event(JsonEvent::EndObject)?; } TermRef::Triple(triple) => { - sink.write_all(b":{\"type\":\"triple\",\"value\":{\"subject\":")?; - write_json_term(triple.subject.as_ref().into(), sink)?; - sink.write_all(b":,\"predicate\":")?; - write_json_term(triple.predicate.as_ref().into(), sink)?; - sink.write_all(b":,\"object\":")?; - write_json_term(triple.object.as_ref(), sink)?; - sink.write_all(b"}}")?; + writer.write_event(JsonEvent::StartObject)?; + writer.write_event(JsonEvent::ObjectKey("type"))?; + writer.write_event(JsonEvent::String("triple"))?; + writer.write_event(JsonEvent::ObjectKey("value"))?; + writer.write_event(JsonEvent::StartObject)?; + writer.write_event(JsonEvent::ObjectKey("subject"))?; + write_json_term(triple.subject.as_ref().into(), writer)?; + writer.write_event(JsonEvent::ObjectKey("predicate"))?; + write_json_term(triple.predicate.as_ref().into(), writer)?; + writer.write_event(JsonEvent::ObjectKey("object"))?; + write_json_term(triple.object.as_ref(), writer)?; + writer.write_event(JsonEvent::EndObject)?; + writer.write_event(JsonEvent::EndObject)?; } } Ok(()) } -fn write_escaped_json_string(s: &str, sink: &mut impl Write) -> Result<(), EvaluationError> { - sink.write_all(b"\"")?; - for c in s.chars() { - match c { - '\\' => sink.write_all(b"\\\\"), - '"' => sink.write_all(b"\\\""), - c => { - if c < char::from(32) { - match c { - '\u{08}' => sink.write_all(b"\\b"), - '\u{0C}' => sink.write_all(b"\\f"), - '\n' => sink.write_all(b"\\n"), - '\r' => sink.write_all(b"\\r"), - '\t' => sink.write_all(b"\\t"), - c => { - let mut c = c as u8; - let mut result = [b'\\', b'u', 0, 0, 0, 0]; - for i in (2..6).rev() { - let ch = c % 16; - result[i] = ch + if ch < 10 { b'0' } else { b'A' }; - c /= 16; +pub fn read_json_results(source: impl BufRead + 'static) -> Result { + let mut reader = JsonReader::from_reader(source); + let mut buffer = Vec::default(); + let mut variables = None; + + if reader.read_event(&mut buffer)? != JsonEvent::StartObject { + return Err(invalid_data_error( + "SPARQL JSON results should be an object", + )); + } + + loop { + let event = reader.read_event(&mut buffer)?; + match event { + JsonEvent::ObjectKey(key) => match key { + "head" => variables = Some(read_head(&mut reader, &mut buffer)?), + "results" => { + if reader.read_event(&mut buffer)? != JsonEvent::StartObject { + return Err(invalid_data_error("'results' should be an object")); + } + if reader.read_event(&mut buffer)? != JsonEvent::ObjectKey("bindings") { + return Err(invalid_data_error( + "'results' should contain a 'bindings' key", + )); + } + if reader.read_event(&mut buffer)? != JsonEvent::StartArray { + return Err(invalid_data_error("'bindings' should be an object")); + } + return if let Some(variables) = variables { + let mut mapping = BTreeMap::default(); + for (i, var) in variables.iter().enumerate() { + mapping.insert(var.clone(), i); + } + Ok(QueryResults::Solutions(QuerySolutionIter::new( + Rc::new( + variables + .into_iter() + .map(Variable::new) + .collect::, _>>() + .map_err(invalid_data_error)?, + ), + Box::new(ResultsIterator { + reader, + buffer, + mapping, + }), + ))) + } else { + Err(invalid_data_error( + "SPARQL tuple query results should contain a head key", + )) + }; + } + "boolean" => { + return if let JsonEvent::Boolean(v) = reader.read_event(&mut buffer)? { + Ok(QueryResults::Boolean(v)) + } else { + Err(invalid_data_error("Unexpected boolean value")) + } + } + _ => { + return Err(invalid_data_error(format!( + "Expecting head or result key, found {}", + key + ))); + } + }, + JsonEvent::EndObject => { + return Err(invalid_data_error( + "SPARQL results should contain a bindings key or a boolean key", + )) + } + JsonEvent::Eof => return Err(io::Error::from(io::ErrorKind::UnexpectedEof)), + _ => return Err(invalid_data_error("Invalid SPARQL results serialization")), + } + } +} + +fn read_head( + reader: &mut JsonReader, + buffer: &mut Vec, +) -> io::Result> { + if reader.read_event(buffer)? != JsonEvent::StartObject { + return Err(invalid_data_error("head should be an object")); + } + let mut variables = None; + loop { + match reader.read_event(buffer)? { + JsonEvent::ObjectKey(key) => match key { + "vars" => variables = Some(read_string_array(reader, buffer)?), + "link" => { + read_string_array(reader, buffer)?; + } + _ => { + return Err(invalid_data_error(format!( + "Unexpected key in head: '{}'", + key + ))) + } + }, + JsonEvent::EndObject => return Ok(variables.unwrap_or_else(Vec::new)), + _ => return Err(invalid_data_error("Invalid head serialization")), + } + } +} + +fn read_string_array( + reader: &mut JsonReader, + buffer: &mut Vec, +) -> io::Result> { + if reader.read_event(buffer)? != JsonEvent::StartArray { + return Err(invalid_data_error("Variable list should be an array")); + } + let mut elements = Vec::new(); + loop { + match reader.read_event(buffer)? { + JsonEvent::String(s) => { + elements.push(s.into()); + } + JsonEvent::EndArray => return Ok(elements), + _ => return Err(invalid_data_error("Variable names should be strings")), + } + } +} + +struct ResultsIterator { + reader: JsonReader, + buffer: Vec, + mapping: BTreeMap, +} + +impl Iterator for ResultsIterator { + type Item = Result>, EvaluationError>; + + fn next(&mut self) -> Option>, EvaluationError>> { + self.read_next().map_err(EvaluationError::from).transpose() + } +} + +impl ResultsIterator { + fn read_next(&mut self) -> io::Result>>> { + let mut new_bindings = vec![None; self.mapping.len()]; + loop { + match self.reader.read_event(&mut self.buffer)? { + JsonEvent::StartObject => (), + JsonEvent::EndObject => return Ok(Some(new_bindings)), + JsonEvent::EndArray | JsonEvent::Eof => return Ok(None), + JsonEvent::ObjectKey(key) => { + let k = *self.mapping.get(key).ok_or_else(|| { + invalid_data_error(format!( + "The variable {} has not been defined in the header", + key + )) + })?; + new_bindings[k] = Some(self.read_value()?) + } + _ => return Err(invalid_data_error("Invalid result serialization")), + } + } + } + fn read_value(&mut self) -> io::Result { + enum Type { + Uri, + BNode, + Literal, + Triple, + } + #[derive(Eq, PartialEq)] + enum State { + Type, + Value, + Lang, + Datatype, + } + let mut state = None; + let mut t = None; + let mut value = None; + let mut lang = None; + let mut datatype = None; + let mut subject = None; + let mut predicate = None; + let mut object = None; + if self.reader.read_event(&mut self.buffer)? != JsonEvent::StartObject { + return Err(invalid_data_error( + "Term serializations should be an object", + )); + } + loop { + match self.reader.read_event(&mut self.buffer)? { + JsonEvent::ObjectKey(key) => match key { + "type" => state = Some(State::Type), + "value" => state = Some(State::Value), + "xml:lang" => state = Some(State::Lang), + "datatype" => state = Some(State::Datatype), + "subject" => subject = Some(self.read_value()?), + "predicate" => predicate = Some(self.read_value()?), + "object" => object = Some(self.read_value()?), + _ => { + return Err(invalid_data_error(format!( + "Unexpected key in term serialization: '{}'", + key + ))) + } + }, + JsonEvent::StartObject => { + if state != Some(State::Value) { + return Err(invalid_data_error( + "Unexpected nested object in term serialization", + )); + } + } + JsonEvent::String(s) => match state { + Some(State::Type) => { + match s { + "uri" => t = Some(Type::Uri), + "bnode" => t = Some(Type::BNode), + "literal" => t = Some(Type::Literal), + "triple" => t = Some(Type::Triple), + _ => { + return Err(invalid_data_error(format!( + "Unexpected term type: '{}'", + s + ))) } - sink.write_all(&result) + }; + state = None; + } + Some(State::Value) => { + value = Some(s.to_owned()); + state = None; + } + Some(State::Lang) => { + lang = Some(s.to_owned()); + state = None; + } + Some(State::Datatype) => { + datatype = Some(s.to_owned()); + state = None; + } + _ => (), // impossible + }, + JsonEvent::EndObject => { + if let Some(s) = state { + if s == State::Value { + state = None; //End of triple + } else { + return Err(invalid_data_error( + "Term description values should be string", + )); } + } else { + return match t { + None => Err(invalid_data_error( + "Term serialization should have a 'type' key", + )), + Some(Type::Uri) => Ok(NamedNode::new(value.ok_or_else(|| { + invalid_data_error("uri serialization should have a 'value' key") + })?) + .map_err(|e| invalid_data_error(format!("Invalid uri value: {}", e)))? + .into()), + Some(Type::BNode) => Ok(BlankNode::new(value.ok_or_else(|| { + invalid_data_error("bnode serialization should have a 'value' key") + })?) + .map_err(|e| invalid_data_error(format!("Invalid bnode value: {}", e)))? + .into()), + Some(Type::Literal) => { + let value = value.ok_or_else(|| { + invalid_data_error( + "uri serialization should have a 'value' key", + ) + })?; + Ok(match datatype { + Some(datatype) => Literal::new_typed_literal( + value, + NamedNode::new(datatype).map_err(|e| { + invalid_data_error(format!( + "Invalid datatype value: {}", + e + )) + })?, + ), + None => match lang { + Some(lang) => { + Literal::new_language_tagged_literal(value, lang) + .map_err(|e| { + invalid_data_error(format!( + "Invalid xml:lang value: {}", + e + )) + })? + } + None => Literal::new_simple_literal(value), + }, + } + .into()) + } + Some(Type::Triple) => Ok(Triple::new( + match subject.ok_or_else(|| { + invalid_data_error( + "triple serialization should have a 'subject' key", + ) + })? { + Term::NamedNode(subject) => subject.into(), + Term::BlankNode(subject) => subject.into(), + Term::Triple(subject) => Subject::Triple(subject), + Term::Literal(_) => { + return Err(invalid_data_error( + "The 'subject' value should not be a literal", + ) + .into()) + } + }, + match predicate.ok_or_else(|| { + invalid_data_error( + "triple serialization should have a 'predicate' key", + ) + })? { + Term::NamedNode(predicate) => predicate, + _ => { + return Err(invalid_data_error( + "The 'predicate' value should be a uri", + ) + .into()) + } + }, + object.ok_or_else(|| { + invalid_data_error( + "triple serialization should have a 'object' key", + ) + })?, + ) + .into()), + }; } - } else { - write!(sink, "{}", c) } + _ => return Err(invalid_data_error("Invalid term serialization")), } - }?; + } } - sink.write_all(b"\"")?; - Ok(()) } diff --git a/lib/src/sparql/model.rs b/lib/src/sparql/model.rs index 0dac7437..078ae7ba 100644 --- a/lib/src/sparql/model.rs +++ b/lib/src/sparql/model.rs @@ -4,7 +4,7 @@ use crate::io::GraphSerializer; use crate::model::*; use crate::sparql::csv_results::{read_tsv_results, write_csv_results, write_tsv_results}; use crate::sparql::error::EvaluationError; -use crate::sparql::json_results::write_json_results; +use crate::sparql::json_results::{read_json_results, write_json_results}; use crate::sparql::xml_results::{read_xml_results, write_xml_results}; use std::error::Error; use std::io::{BufRead, Write}; @@ -29,11 +29,9 @@ impl QueryResults { ) -> Result { match format { QueryResultsFormat::Xml => read_xml_results(reader), - QueryResultsFormat::Json => Err(invalid_input_error( - "JSON SPARQL results format parsing has not been implemented yet", - )), //TODO: implement + QueryResultsFormat::Json => read_json_results(reader), QueryResultsFormat::Csv => Err(invalid_input_error( - "CSV and TSV SPARQL results format parsing is not implemented", + "CSV SPARQL results format parsing is not implemented", )), QueryResultsFormat::Tsv => read_tsv_results(reader), } @@ -514,3 +512,84 @@ impl fmt::Display for VariableNameParseError { } impl Error for VariableNameParseError {} + +#[test] +fn test_serialization_rountrip() -> Result<(), EvaluationError> { + use std::io::Cursor; + use std::str; + + for format in &[ + QueryResultsFormat::Xml, + QueryResultsFormat::Json, + QueryResultsFormat::Tsv, + ] { + let results = vec![ + QueryResults::Boolean(true), + QueryResults::Boolean(false), + QueryResults::Solutions(QuerySolutionIter::new( + Rc::new(vec![ + Variable::new_unchecked("foo"), + Variable::new_unchecked("bar"), + ]), + Box::new( + vec![ + Ok(vec![None, None]), + Ok(vec![ + Some(NamedNode::new_unchecked("http://example.com").into()), + None, + ]), + Ok(vec![ + None, + Some(NamedNode::new_unchecked("http://example.com").into()), + ]), + Ok(vec![ + Some(BlankNode::new_unchecked("foo").into()), + Some(BlankNode::new_unchecked("bar").into()), + ]), + Ok(vec![Some(Literal::new_simple_literal("foo").into()), None]), + Ok(vec![ + Some( + Literal::new_language_tagged_literal_unchecked("foo", "fr").into(), + ), + None, + ]), + Ok(vec![ + Some(Literal::from(1).into()), + Some(Literal::from(true).into()), + ]), + Ok(vec![ + Some(Literal::from(1.33).into()), + Some(Literal::from(false).into()), + ]), + Ok(vec![ + Some( + Triple::new( + NamedNode::new_unchecked("http://example.com/s"), + NamedNode::new_unchecked("http://example.com/p"), + NamedNode::new_unchecked("http://example.com/o"), + ) + .into(), + ), + None, + ]), + ] + .into_iter(), + ), + )), + ]; + + for ex in results { + let mut buffer = Vec::new(); + ex.write(&mut buffer, *format)?; + let ex2 = QueryResults::read(Cursor::new(buffer.clone()), *format)?; + let mut buffer2 = Vec::new(); + ex2.write(&mut buffer2, *format)?; + assert_eq!( + str::from_utf8(&buffer).unwrap(), + str::from_utf8(&buffer2).unwrap() + ); + } + } + + Ok(()) +} diff --git a/lib/src/sparql/xml_results.rs b/lib/src/sparql/xml_results.rs index 044dac8f..d056e8fe 100644 --- a/lib/src/sparql/xml_results.rs +++ b/lib/src/sparql/xml_results.rs @@ -362,8 +362,7 @@ impl ResultsIterator { fn read_next(&mut self) -> Result>>, EvaluationError> { let mut state = State::Start; - let mut new_bindings = Vec::default(); - new_bindings.resize(self.mapping.len(), None); + let mut new_bindings = vec![None; self.mapping.len()]; let mut current_var = None; let mut term: Option = None; @@ -568,34 +567,37 @@ impl ResultsIterator { state = self.stack.pop().unwrap(); } State::Triple => { + println!("foo"); if let (Some(subject), Some(predicate), Some(object)) = (subject.take(), predicate.take(), object.take()) { - term = - Some( - Triple::new( - match subject { - Term::NamedNode(subject) => subject.into(), - Term::BlankNode(subject) => subject.into(), - Term::Triple(subject) => Subject::Triple(subject), - Term::Literal(_) => return Err(invalid_data_error( - "The value should not be a ", + term = Some( + Triple::new( + match subject { + Term::NamedNode(subject) => subject.into(), + Term::BlankNode(subject) => subject.into(), + Term::Triple(subject) => Subject::Triple(subject), + Term::Literal(_) => { + return Err(invalid_data_error( + "The value should not be a ", ) - .into()), - }, - match predicate { - Term::NamedNode(predicate) => predicate, - _ => { - return Err(invalid_data_error( - "The value should be an ", - ) - .into()) - } - }, - object, - ) - .into(), + .into()) + } + }, + match predicate { + Term::NamedNode(predicate) => predicate, + _ => { + return Err(invalid_data_error( + "The value should be an ", + ) + .into()) + } + }, + object, ) + .into(), + ); + state = self.stack.pop().unwrap(); } else { return Err( invalid_data_error("A should contain a , a and an ").into() diff --git a/lib/src/storage/mod.rs b/lib/src/storage/mod.rs index 09585fed..a223a300 100644 --- a/lib/src/storage/mod.rs +++ b/lib/src/storage/mod.rs @@ -603,6 +603,16 @@ impl Storage { Ok(()) } + pub fn flush(&self) -> Result<(), std::io::Error> { + self.default.flush()?; + Ok(()) + } + + pub async fn flush_async(&self) -> Result<(), std::io::Error> { + self.default.flush_async().await?; + Ok(()) + } + pub fn get_str(&self, key: &StrHash) -> Result, std::io::Error> { self.id2str .get(key.to_be_bytes())? diff --git a/lib/src/store.rs b/lib/src/store.rs index 4802b64c..46e0da73 100644 --- a/lib/src/store.rs +++ b/lib/src/store.rs @@ -567,6 +567,26 @@ impl Store { pub fn clear(&self) -> Result<(), io::Error> { self.storage.clear() } + + /// Flushes all buffers and ensures that all writes are saved on disk. + /// + /// Flushes are automatically done for most platform using background threads. + /// However, calling this method explicitly is still required for Windows and Android. + /// + /// An [async version](SledStore::flush_async) is also available. + pub fn flush(&self) -> Result<(), io::Error> { + self.storage.flush() + } + + /// Asynchronously flushes all buffers and ensures that all writes are saved on disk. + /// + /// Flushes are automatically done for most platform using background threads. + /// However, calling this method explicitly is still required for Windows and Android. + /// + /// A [sync version](SledStore::flush) is also available. + pub async fn flush_async(&self) -> Result<(), io::Error> { + self.storage.flush_async().await + } } impl fmt::Display for Store { diff --git a/server/src/main.rs b/server/src/main.rs index 1e03256e..3963fdfa 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -345,7 +345,7 @@ fn base_url(request: &Request) -> Result { } url.set_query(None); url.set_fragment(None); - Ok(url.into_string()) + Ok(url.into()) } fn resolve_with_base(request: &Request, url: &str) -> Result { @@ -370,6 +370,7 @@ fn configure_and_evaluate_sparql_query( ) -> Result { let mut default_graph_uris = Vec::new(); let mut named_graph_uris = Vec::new(); + let mut use_default_graph_as_union = false; for (k, v) in form_urlencoded::parse(&encoded) { match k.as_ref() { "query" => { @@ -379,12 +380,20 @@ fn configure_and_evaluate_sparql_query( query = Some(v.into_owned()) } "default-graph-uri" => default_graph_uris.push(v.into_owned()), + "union-default-graph" => use_default_graph_as_union = true, "named-graph-uri" => named_graph_uris.push(v.into_owned()), _ => (), } } if let Some(query) = query { - evaluate_sparql_query(store, query, default_graph_uris, named_graph_uris, request) + evaluate_sparql_query( + store, + query, + use_default_graph_as_union, + default_graph_uris, + named_graph_uris, + request, + ) } else { bail_status!(400, "You should set the 'query' parameter") } @@ -393,28 +402,37 @@ fn configure_and_evaluate_sparql_query( fn evaluate_sparql_query( store: Store, query: String, + use_default_graph_as_union: bool, default_graph_uris: Vec, named_graph_uris: Vec, request: Request, ) -> Result { let mut query = Query::parse(&query, Some(base_url(&request)?.as_str())).map_err(bad_request)?; - let default_graph_uris = default_graph_uris - .into_iter() - .map(|e| Ok(NamedNode::new(e)?.into())) - .collect::>>() - .map_err(bad_request)?; - let named_graph_uris = named_graph_uris - .into_iter() - .map(|e| Ok(NamedNode::new(e)?.into())) - .collect::>>() - .map_err(bad_request)?; - - if !default_graph_uris.is_empty() || !named_graph_uris.is_empty() { - query.dataset_mut().set_default_graph(default_graph_uris); - query - .dataset_mut() - .set_available_named_graphs(named_graph_uris); + + if use_default_graph_as_union { + if !default_graph_uris.is_empty() || !named_graph_uris.is_empty() { + bail_status!( + 400, + "default-graph-uri or named-graph-uri and union-default-graph should not be set at the same time" + ); + } + query.dataset_mut().set_default_graph_as_union() + } else if !default_graph_uris.is_empty() || !named_graph_uris.is_empty() { + query.dataset_mut().set_default_graph( + default_graph_uris + .into_iter() + .map(|e| Ok(NamedNode::new(e)?.into())) + .collect::>() + .map_err(bad_request)?, + ); + query.dataset_mut().set_available_named_graphs( + named_graph_uris + .into_iter() + .map(|e| Ok(NamedNode::new(e)?.into())) + .collect::>() + .map_err(bad_request)?, + ); } let results = store.query(query)?; @@ -451,6 +469,7 @@ fn configure_and_evaluate_sparql_update( mut update: Option, request: Request, ) -> Result { + let mut use_default_graph_as_union = false; let mut default_graph_uris = Vec::new(); let mut named_graph_uris = Vec::new(); for (k, v) in form_urlencoded::parse(&encoded) { @@ -462,12 +481,20 @@ fn configure_and_evaluate_sparql_update( update = Some(v.into_owned()) } "using-graph-uri" => default_graph_uris.push(v.into_owned()), + "using-union-graph" => use_default_graph_as_union = true, "using-named-graph-uri" => named_graph_uris.push(v.into_owned()), _ => (), } } if let Some(update) = update { - evaluate_sparql_update(store, update, default_graph_uris, named_graph_uris, request) + evaluate_sparql_update( + store, + update, + use_default_graph_as_union, + default_graph_uris, + named_graph_uris, + request, + ) } else { bail_status!(400, "You should set the 'update' parameter") } @@ -476,26 +503,45 @@ fn configure_and_evaluate_sparql_update( fn evaluate_sparql_update( store: Store, update: String, + use_default_graph_as_union: bool, default_graph_uris: Vec, named_graph_uris: Vec, request: Request, ) -> Result { let mut update = Update::parse(&update, Some(base_url(&request)?.as_str())).map_err(bad_request)?; - let default_graph_uris = default_graph_uris - .into_iter() - .map(|e| Ok(NamedNode::new(e)?.into())) - .collect::>>() - .map_err(bad_request)?; - let named_graph_uris = named_graph_uris - .into_iter() - .map(|e| Ok(NamedNode::new(e)?.into())) - .collect::>>() - .map_err(bad_request)?; - if !default_graph_uris.is_empty() || !named_graph_uris.is_empty() { + + if use_default_graph_as_union { + if !default_graph_uris.is_empty() || !named_graph_uris.is_empty() { + bail_status!( + 400, + "using-graph-uri or using-named-graph-uri and using-union-graph should not be set at the same time" + ); + } + for using in update.using_datasets_mut() { + if !using.is_default_dataset() { + bail_status!( + 400, + "using-union-graph must not be used with a SPARQL UPDATE containing USING", + ); + } + using.set_default_graph_as_union(); + } + } else if !default_graph_uris.is_empty() || !named_graph_uris.is_empty() { + let default_graph_uris = default_graph_uris + .into_iter() + .map(|e| Ok(NamedNode::new(e)?.into())) + .collect::>>() + .map_err(bad_request)?; + let named_graph_uris = named_graph_uris + .into_iter() + .map(|e| Ok(NamedNode::new(e)?.into())) + .collect::>>() + .map_err(bad_request)?; for using in update.using_datasets_mut() { if !using.is_default_dataset() { - bail_status!(400, + bail_status!( + 400, "using-graph-uri and using-named-graph-uri must not be used with a SPARQL UPDATE containing USING", ); } @@ -780,6 +826,24 @@ mod tests { ); } + #[test] + fn get_query_union_graph() { + ServerTest::new().test_status(Request::new( + Method::Get, + Url::parse("http://localhost/query?query=SELECT%20*%20WHERE%20{%20?s%20?p%20?o%20}&union-default-graph") + .unwrap(), + ), StatusCode::Ok); + } + + #[test] + fn get_query_union_graph_and_default_graph() { + ServerTest::new().test_status(Request::new( + Method::Get, + Url::parse("http://localhost/query?query=SELECT%20*%20WHERE%20{%20?s%20?p%20?o%20}&union-default-graph&default-graph-uri=http://example.com") + .unwrap(), + ), StatusCode::BadRequest); + } + #[test] fn get_without_query() { ServerTest::new().test_status( diff --git a/testsuite/rdf-tests b/testsuite/rdf-tests index 634dadce..43b0eb90 160000 --- a/testsuite/rdf-tests +++ b/testsuite/rdf-tests @@ -1 +1 @@ -Subproject commit 634dadcedba0177eb47975d642ae66949774834a +Subproject commit 43b0eb9078f144c0d1c36f2c751b81922030eb1a diff --git a/testsuite/tests/sparql.rs b/testsuite/tests/sparql.rs index 1c31a4a8..bb91dbb3 100644 --- a/testsuite/tests/sparql.rs +++ b/testsuite/tests/sparql.rs @@ -87,15 +87,12 @@ fn sparql11_query_w3c_evaluation_testsuite() -> Result<()> { "http://www.w3.org/2009/sparql/docs/tests/data-sparql11/syntax-query/manifest#test_61a", "http://www.w3.org/2009/sparql/docs/tests/data-sparql11/syntax-query/manifest#test_62a", "http://www.w3.org/2009/sparql/docs/tests/data-sparql11/syntax-query/manifest#test_65", - // SPARQL 1.1 JSON query results deserialization is not implemented yet - "http://www.w3.org/2009/sparql/docs/tests/data-sparql11/aggregates/manifest#agg-empty-group-count-1", - "http://www.w3.org/2009/sparql/docs/tests/data-sparql11/aggregates/manifest#agg-empty-group-count-2", //BNODE() scope is currently wrong "http://www.w3.org/2009/sparql/docs/tests/data-sparql11/functions/manifest#bnode01", //Property path with unbound graph name are not supported yet "http://www.w3.org/2009/sparql/docs/tests/data-sparql11/property-path/manifest#pp35", //SERVICE name from a BGP - "http://www.w3.org/2009/sparql/docs/tests/data-sparql11/service/manifest#service5" + "http://www.w3.org/2009/sparql/docs/tests/data-sparql11/service/manifest#service5", ], ) } @@ -119,6 +116,14 @@ fn sparql11_update_w3c_evaluation_testsuite() -> Result<()> { ) } +#[test] +fn sparql11_json_w3c_evaluation_testsuite() -> Result<()> { + run_testsuite( + "http://www.w3.org/2009/sparql/docs/tests/data-sparql11/json-res/manifest.ttl", + vec![], + ) +} + #[test] fn sparql11_tsv_w3c_evaluation_testsuite() -> Result<()> { run_testsuite(