From 6aa27d488596f55af2a37d278abed4c90c1f80ea Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 3 Jan 2021 16:01:39 +0100 Subject: [PATCH] Makes Oxigraph store "graph aware" --- CHANGELOG.md | 2 +- lib/src/sparql/dataset.rs | 18 +- lib/src/sparql/parser.rs | 4 +- lib/src/sparql/update.rs | 136 ++++++++++---- lib/src/store/binary_encoder.rs | 6 +- lib/src/store/memory.rs | 282 +++++++++++++++++++++++++---- lib/src/store/mod.rs | 25 +++ lib/src/store/rocksdb.rs | 308 ++++++++++++++++++++++++++------ lib/src/store/sled.rs | 256 +++++++++++++++++++++++--- lib/tests/rocksdb_store.rs | 21 ++- lib/tests/sled_store.rs | 21 ++- python/src/memory_store.rs | 74 ++++++++ python/src/sled_store.rs | 82 +++++++++ python/tests/test_store.py | 14 ++ 14 files changed, 1082 insertions(+), 167 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98dc7d71..cf07879e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,10 @@ - [SPARQL 1.1 Update](https://www.w3.org/TR/sparql11-update/) support for Rust, Python and JavaScript. All store-like classes now provide an `update` method. - [SPARQL 1.1 Query Results CSV and TSV Formats](https://www.w3.org/TR/sparql11-results-csv-tsv/) serializers and TSV format parser. - The SPARQL Query and Update algebra is now public. +- The stores are now "graph aware" i.e. it is possible to create and keep empty named graphs. - A simple built-in HTTP client. In the Rust library, is disabled by default behind the `http_client` feature. It powers SPARQL federation and SPARQL UPDATE `LOAD` operations. - `std::str::FromStr` implementations to `NamedNode`, `BlankNode`, `Literal`, `Term` and `Variable` allowing to easily parse Turtle/SPARQL serialization of these terms. - Optional Sled storage for `oxigraph_server`. -- `(Memory|RocksDB|Sled)Store::drop_graph` and `(Memory|RocksDB|Sled)Store::clear`. ### Removed - The `default_graph_uris` and `named_graph_uris` parameters from `pyoxigraph` `query` methods. diff --git a/lib/src/sparql/dataset.rs b/lib/src/sparql/dataset.rs index 4a4e5b08..c410e671 100644 --- a/lib/src/sparql/dataset.rs +++ b/lib/src/sparql/dataset.rs @@ -6,7 +6,7 @@ use crate::store::numeric_encoder::{ use crate::store::ReadableEncodedStore; use lasso::{Rodeo, Spur}; use std::cell::RefCell; -use std::iter::empty; +use std::iter::{empty, once, Once}; pub(crate) struct DatasetView { store: S, @@ -182,6 +182,7 @@ impl StrLookup for DatasetView { impl ReadableEncodedStore for DatasetView { type QuadsIter = Box>, EvaluationError>>>; + type GraphsIter = Once>, EvaluationError>>; fn encoded_quads_for_pattern( &self, @@ -199,6 +200,21 @@ impl ReadableEncodedStore for DatasetView { Box::new(empty()) } } + + fn encoded_named_graphs(&self) -> Self::GraphsIter { + once(Err(EvaluationError::msg( + "Graphs lookup is not implemented by DatasetView", + ))) + } + + fn contains_encoded_named_graph( + &self, + _: EncodedTerm, + ) -> Result { + Err(EvaluationError::msg( + "Graphs lookup is not implemented by DatasetView", + )) + } } fn map_iter<'a, I: StrId>( diff --git a/lib/src/sparql/parser.rs b/lib/src/sparql/parser.rs index a3f41886..d824f7f8 100644 --- a/lib/src/sparql/parser.rs +++ b/lib/src/sparql/parser.rs @@ -986,7 +986,7 @@ parser! { Vec::new() // identity case } else { let bgp = GraphPattern::BGP(vec![TriplePattern::new(Variable::new_unchecked("s"), Variable::new_unchecked("p"), Variable::new_unchecked("o"))]); - vec![GraphUpdateOperation::Drop { silent, graph: to.clone().map_or(GraphTarget::DefaultGraph, GraphTarget::NamedNode) }, copy_graph(from.clone(), to.map(NamedNodeOrVariable::NamedNode)), GraphUpdateOperation::Drop { silent, graph: from.map_or(GraphTarget::DefaultGraph, GraphTarget::NamedNode) }] + vec![GraphUpdateOperation::Drop { silent: true, graph: to.clone().map_or(GraphTarget::DefaultGraph, GraphTarget::NamedNode) }, copy_graph(from.clone(), to.map(NamedNodeOrVariable::NamedNode)), GraphUpdateOperation::Drop { silent, graph: from.map_or(GraphTarget::DefaultGraph, GraphTarget::NamedNode) }] } } @@ -997,7 +997,7 @@ parser! { Vec::new() // identity case } else { let bgp = GraphPattern::BGP(vec![TriplePattern::new(Variable::new_unchecked("s"), Variable::new_unchecked("p"), Variable::new_unchecked("o"))]); - vec![GraphUpdateOperation::Drop { silent, graph: to.clone().map_or(GraphTarget::DefaultGraph, GraphTarget::NamedNode) }, copy_graph(from, to.map(NamedNodeOrVariable::NamedNode))] + vec![GraphUpdateOperation::Drop { silent: true, graph: to.clone().map_or(GraphTarget::DefaultGraph, GraphTarget::NamedNode) }, copy_graph(from, to.map(NamedNodeOrVariable::NamedNode))] } } diff --git a/lib/src/sparql/update.rs b/lib/src/sparql/update.rs index 2b156664..b1cd3b61 100644 --- a/lib/src/sparql/update.rs +++ b/lib/src/sparql/update.rs @@ -81,9 +81,9 @@ where Ok(()) } } - GraphUpdateOperation::Clear { graph, .. } => self.eval_clear(graph), - GraphUpdateOperation::Create { .. } => Ok(()), - GraphUpdateOperation::Drop { graph, .. } => self.eval_clear(graph), + GraphUpdateOperation::Clear { graph, silent } => self.eval_clear(graph, *silent), + GraphUpdateOperation::Create { graph, silent } => self.eval_create(graph, *silent), + GraphUpdateOperation::Drop { graph, silent } => self.eval_drop(graph, *silent), } } @@ -227,55 +227,129 @@ where Ok(()) } - fn eval_clear(&mut self, graph: &GraphTarget) -> Result<(), EvaluationError> { + fn eval_create(&mut self, graph: &NamedNode, silent: bool) -> Result<(), EvaluationError> { + let encoded_graph_name = self + .write + .encode_named_node(graph.as_ref()) + .map_err(to_eval_error)?; + if self + .read + .contains_encoded_named_graph(encoded_graph_name) + .map_err(to_eval_error)? + { + if silent { + Ok(()) + } else { + Err(EvaluationError::msg(format!( + "The graph {} already exists", + graph + ))) + } + } else { + self.write + .insert_encoded_named_graph(encoded_graph_name) + .map_err(to_eval_error) + } + } + + fn eval_clear(&mut self, graph: &GraphTarget, silent: bool) -> Result<(), EvaluationError> { match graph { - GraphTarget::NamedNode(graph) => { - if let Some(graph) = self + GraphTarget::NamedNode(graph_name) => { + if let Some(graph_name) = self .read - .get_encoded_named_node(graph.into()) + .get_encoded_named_node(graph_name.as_ref()) .map_err(to_eval_error)? { - for quad in self + if self .read - .encoded_quads_for_pattern(None, None, None, Some(graph)) + .contains_encoded_named_graph(graph_name) + .map_err(to_eval_error)? { - self.write - .remove_encoded(&quad.map_err(to_eval_error)?) - .map_err(to_eval_error)?; + return self + .write + .clear_encoded_graph(graph_name) + .map_err(to_eval_error); } + } + if silent { + Ok(()) } else { - //we do not track created graph so it's fine + Err(EvaluationError::msg(format!( + "The graph {} does not exists", + graph + ))) + } + } + GraphTarget::DefaultGraph => self + .write + .clear_encoded_graph(EncodedTerm::DefaultGraph) + .map_err(to_eval_error), + GraphTarget::NamedGraphs => { + // TODO: optimize? + for graph in self.read.encoded_named_graphs() { + self.write + .clear_encoded_graph(graph.map_err(to_eval_error)?) + .map_err(to_eval_error)?; } + Ok(()) } - GraphTarget::DefaultGraph => { - for quad in self.read.encoded_quads_for_pattern( - None, - None, - None, - Some(EncodedTerm::DefaultGraph), - ) { + GraphTarget::AllGraphs => { + // TODO: optimize? + for graph in self.read.encoded_named_graphs() { self.write - .remove_encoded(&quad.map_err(to_eval_error)?) + .clear_encoded_graph(graph.map_err(to_eval_error)?) .map_err(to_eval_error)?; } + self.write + .clear_encoded_graph(EncodedTerm::DefaultGraph) + .map_err(to_eval_error) } - GraphTarget::NamedGraphs => { - for quad in self.read.encoded_quads_for_pattern(None, None, None, None) { - let quad = quad.map_err(to_eval_error)?; - if !quad.graph_name.is_default_graph() { - self.write.remove_encoded(&quad).map_err(to_eval_error)?; + } + } + + fn eval_drop(&mut self, graph: &GraphTarget, silent: bool) -> Result<(), EvaluationError> { + match graph { + GraphTarget::NamedNode(graph_name) => { + if let Some(graph_name) = self + .read + .get_encoded_named_node(graph_name.as_ref()) + .map_err(to_eval_error)? + { + if self + .read + .contains_encoded_named_graph(graph_name) + .map_err(to_eval_error)? + { + return self + .write + .remove_encoded_named_graph(graph_name) + .map_err(to_eval_error); } } + if silent { + Ok(()) + } else { + Err(EvaluationError::msg(format!( + "The graph {} does not exists", + graph + ))) + } } - GraphTarget::AllGraphs => { - for quad in self.read.encoded_quads_for_pattern(None, None, None, None) { + GraphTarget::DefaultGraph => self + .write + .clear_encoded_graph(EncodedTerm::DefaultGraph) + .map_err(to_eval_error), + GraphTarget::NamedGraphs => { + // TODO: optimize? + for graph in self.read.encoded_named_graphs() { self.write - .remove_encoded(&quad.map_err(to_eval_error)?) + .remove_encoded_named_graph(graph.map_err(to_eval_error)?) .map_err(to_eval_error)?; } + Ok(()) } - }; - Ok(()) + GraphTarget::AllGraphs => self.write.clear().map_err(to_eval_error), + } } fn encode_quad_for_insertion( diff --git a/lib/src/store/binary_encoder.rs b/lib/src/store/binary_encoder.rs index 7d6fea92..59172c51 100644 --- a/lib/src/store/binary_encoder.rs +++ b/lib/src/store/binary_encoder.rs @@ -11,7 +11,7 @@ use std::mem::size_of; type EncodedTerm = crate::store::numeric_encoder::EncodedTerm; type EncodedQuad = crate::store::numeric_encoder::EncodedQuad; -pub const LATEST_STORAGE_VERSION: u64 = 0; +pub const LATEST_STORAGE_VERSION: u64 = 1; pub const WRITTEN_TERM_MAX_SIZE: usize = size_of::() + 2 * size_of::(); // Encoded term type blocks @@ -111,6 +111,10 @@ impl QuadEncoding { } } +pub fn decode_term(buffer: &[u8]) -> Result { + Cursor::new(&buffer).read_term() +} + pub trait TermReader { fn read_term(&mut self) -> Result; diff --git a/lib/src/store/memory.rs b/lib/src/store/memory.rs index d7b5e522..f2fb53ac 100644 --- a/lib/src/store/memory.rs +++ b/lib/src/store/memory.rs @@ -455,7 +455,75 @@ impl MemoryStore { dump_dataset(self.iter().map(Ok), writer, format) } - /// Removes a graph from this store. + /// Returns all the store named graphs + /// + /// Usage example: + /// ``` + /// use oxigraph::MemoryStore; + /// use oxigraph::model::{NamedNode, Quad, NamedOrBlankNode}; + /// + /// let ex = NamedNode::new("http://example.com")?; + /// let store = MemoryStore::new(); + /// store.insert(Quad::new(ex.clone(), ex.clone(), ex.clone(), ex.clone())); + /// store.insert(Quad::new(ex.clone(), ex.clone(), ex.clone(), None)); + /// assert_eq!(vec![NamedOrBlankNode::from(ex)], store.named_graphs().collect::>()); + /// # Result::<_,Box>::Ok(()) + /// ``` + pub fn named_graphs(&self) -> MemoryGraphNameIter { + MemoryGraphNameIter { + iter: self.encoded_named_graphs(), + store: self.clone(), + } + } + + /// Checks if the store contains a given graph + /// + /// Usage example: + /// ``` + /// use oxigraph::MemoryStore; + /// use oxigraph::model::{NamedNode, Quad}; + /// + /// let ex = NamedNode::new("http://example.com")?; + /// let store = MemoryStore::new(); + /// store.insert(Quad::new(ex.clone(), ex.clone(), ex.clone(), ex.clone())); + /// assert!(store.contains_named_graph(&ex)); + /// # Result::<_,Box>::Ok(()) + /// ``` + pub fn contains_named_graph<'a>(&self, graph_name: impl Into>) -> bool { + if let Some(graph_name) = self + .get_encoded_named_or_blank_node(graph_name.into()) + .unwrap_infallible() + { + self.contains_encoded_named_graph(graph_name) + .unwrap_infallible() + } else { + false + } + } + + /// Inserts a graph into this store + /// + /// Usage example: + /// ``` + /// use oxigraph::MemoryStore; + /// use oxigraph::model::NamedNode; + /// + /// let ex = NamedNode::new("http://example.com")?; + /// let store = MemoryStore::new(); + /// store.insert_named_graph(ex.clone()); + /// assert_eq!(store.named_graphs().count(), 1); + /// # Result::<_,Box>::Ok(()) + /// ``` + pub fn insert_named_graph(&self, graph_name: impl Into) { + let mut this = self; + let graph_name = this + .encode_named_or_blank_node(graph_name.into().as_ref()) + .unwrap_infallible(); + this.insert_encoded_named_graph(graph_name) + .unwrap_infallible() + } + + /// Clears a graph from this store. /// /// Usage example: /// ``` @@ -465,22 +533,50 @@ impl MemoryStore { /// let ex = NamedNode::new("http://example.com")?; /// let quad = Quad::new(ex.clone(), ex.clone(), ex.clone(), ex.clone()); /// let store = MemoryStore::new(); - /// store.insert(quad.clone()); + /// store.insert(quad.clone()); /// assert_eq!(1, store.len()); /// - /// store.drop_graph(&ex); + /// store.clear_graph(&ex); /// assert_eq!(0, store.len()); + /// assert_eq!(1, store.named_graphs().count()); /// # Result::<_,Box>::Ok(()) /// ``` - pub fn drop_graph<'a>(&self, graph_name: impl Into>) { + pub fn clear_graph<'a>(&self, graph_name: impl Into>) { if let Some(graph_name) = self .get_encoded_graph_name(graph_name.into()) .unwrap_infallible() { - for quad in self.encoded_quads_for_pattern_inner(None, None, None, Some(graph_name)) { - let mut this = self; - this.remove_encoded(&quad).unwrap_infallible(); - } + let mut this = self; + this.clear_encoded_graph(graph_name).unwrap_infallible() + } + } + + /// Removes a graph from this store. + /// + /// Usage example: + /// ``` + /// use oxigraph::MemoryStore; + /// use oxigraph::model::{NamedNode, Quad}; + /// + /// let ex = NamedNode::new("http://example.com")?; + /// let quad = Quad::new(ex.clone(), ex.clone(), ex.clone(), ex.clone()); + /// let store = MemoryStore::new(); + /// store.insert(quad.clone()); + /// assert_eq!(1, store.len()); + /// + /// store.remove_named_graph(&ex); + /// assert_eq!(0, store.len()); + /// assert_eq!(0, store.named_graphs().count()); + /// # Result::<_,Box>::Ok(()) + /// ``` + pub fn remove_named_graph<'a>(&self, graph_name: impl Into>) { + if let Some(graph_name) = self + .get_encoded_named_or_blank_node(graph_name.into()) + .unwrap_infallible() + { + let mut this = self; + this.remove_encoded_named_graph(graph_name) + .unwrap_infallible() } } @@ -502,7 +598,7 @@ impl MemoryStore { /// # Result::<_,Box>::Ok(()) /// ``` pub fn clear(&self) { - *self.indexes_mut() = MemoryStoreIndexes::default(); + self.indexes_mut().clear().unwrap_infallible() } #[allow(clippy::expect_used)] @@ -891,6 +987,7 @@ impl<'a> StrContainer for &'a MemoryStore { impl<'a> ReadableEncodedStore for MemoryStore { type QuadsIter = EncodedQuadsIter; + type GraphsIter = EncodedGraphsIter; fn encoded_quads_for_pattern( &self, @@ -905,6 +1002,22 @@ impl<'a> ReadableEncodedStore for MemoryStore { .into_iter(), } } + + fn encoded_named_graphs(&self) -> Self::GraphsIter { + EncodedGraphsIter { + iter: self + .indexes() + .gosp + .keys() + .cloned() + .collect::>() + .into_iter(), + } + } + + fn contains_encoded_named_graph(&self, graph_name: EncodedTerm) -> Result { + Ok(self.indexes().gspo.contains_key(&graph_name)) + } } impl<'a> WritableEncodedStore for &'a MemoryStore { @@ -915,6 +1028,22 @@ impl<'a> WritableEncodedStore for &'a MemoryStore { fn remove_encoded(&mut self, quad: &EncodedQuad) -> Result<(), Infallible> { self.indexes_mut().remove_encoded(quad) } + + fn insert_encoded_named_graph(&mut self, graph_name: EncodedTerm) -> Result<(), Infallible> { + self.indexes_mut().insert_encoded_named_graph(graph_name) + } + + fn clear_encoded_graph(&mut self, graph_name: EncodedTerm) -> Result<(), Infallible> { + self.indexes_mut().clear_encoded_graph(graph_name) + } + + fn remove_encoded_named_graph(&mut self, graph_name: EncodedTerm) -> Result<(), Infallible> { + self.indexes_mut().remove_encoded_named_graph(graph_name) + } + + fn clear(&mut self) -> Result<(), Self::Error> { + self.indexes_mut().clear() + } } impl StrEncodingAware for MemoryStoreIndexes { @@ -1011,27 +1140,15 @@ impl WritableEncodedStore for MemoryStoreIndexes { &quad.predicate, ); } else { - remove_from_quad_map( - &mut self.gspo, - &quad.graph_name, - &quad.subject, - &quad.predicate, - &quad.object, - ); - remove_from_quad_map( - &mut self.gpos, - &quad.graph_name, - &quad.predicate, - &quad.object, - &quad.subject, - ); - remove_from_quad_map( - &mut self.gosp, - &quad.graph_name, - &quad.object, - &quad.subject, - &quad.predicate, - ); + if let Some(spo) = self.gspo.get_mut(&quad.graph_name) { + remove_from_triple_map(spo, &quad.subject, &quad.predicate, &quad.object); + } + if let Some(pos) = self.gpos.get_mut(&quad.graph_name) { + remove_from_triple_map(pos, &quad.predicate, &quad.object, &quad.subject); + } + if let Some(osp) = self.gosp.get_mut(&quad.graph_name) { + remove_from_triple_map(osp, &quad.object, &quad.subject, &quad.predicate); + } remove_from_quad_map( &mut self.spog, &quad.subject, @@ -1056,6 +1173,66 @@ impl WritableEncodedStore for MemoryStoreIndexes { } Ok(()) } + + fn insert_encoded_named_graph(&mut self, graph_name: EncodedTerm) -> Result<(), Infallible> { + self.gspo.entry(graph_name).or_default(); + self.gpos.entry(graph_name).or_default(); + self.gosp.entry(graph_name).or_default(); + Ok(()) + } + + fn clear_encoded_graph(&mut self, graph_name: EncodedTerm) -> Result<(), Infallible> { + if graph_name.is_default_graph() { + self.default_spo.clear(); + self.default_pos.clear(); + self.default_osp.clear(); + } else { + if let Some(spo) = self.gspo.get(&graph_name) { + for (s, po) in spo { + for (p, os) in po { + for o in os { + remove_from_quad_map(&mut self.spog, s, p, o, &graph_name); + remove_from_quad_map(&mut self.posg, p, o, s, &graph_name); + remove_from_quad_map(&mut self.ospg, o, s, p, &graph_name); + } + } + } + } + if let Some(spo) = self.gspo.get_mut(&graph_name) { + spo.clear(); + } + if let Some(pos) = self.gpos.get_mut(&graph_name) { + pos.clear(); + } + if let Some(osp) = self.gosp.get_mut(&graph_name) { + osp.clear(); + } + } + Ok(()) + } + + fn remove_encoded_named_graph(&mut self, graph_name: EncodedTerm) -> Result<(), Infallible> { + if let Some(spo) = self.gspo.get(&graph_name) { + for (s, po) in spo { + for (p, os) in po { + for o in os { + remove_from_quad_map(&mut self.spog, s, p, o, &graph_name); + remove_from_quad_map(&mut self.posg, p, o, s, &graph_name); + remove_from_quad_map(&mut self.ospg, o, s, p, &graph_name); + } + } + } + } + self.gspo.remove(&graph_name); + self.gpos.remove(&graph_name); + self.gosp.remove(&graph_name); + Ok(()) + } + + fn clear(&mut self) -> Result<(), Infallible> { + *self = MemoryStoreIndexes::default(); + Ok(()) + } } fn insert_into_triple_map(map: &mut TripleMap, e1: T, e2: T, e3: T) { @@ -1337,6 +1514,29 @@ impl Iterator for EncodedQuadsIter { } } +pub(crate) struct EncodedGraphsIter { + iter: IntoIter, +} + +impl Iterator for EncodedGraphsIter { + type Item = Result; + + fn next(&mut self) -> Option> { + self.iter.next().map(Ok) + } + + fn size_hint(&self) -> (usize, Option) { + self.iter.size_hint() + } + + fn fold(self, init: Acc, mut g: G) -> Acc + where + G: FnMut(Acc, Self::Item) -> Acc, + { + self.iter.fold(init, |acc, elt| g(acc, Ok(elt))) + } +} + /// An iterator returning the quads contained in a [`MemoryStore`]. pub struct MemoryQuadIter { iter: IntoIter, @@ -1355,6 +1555,28 @@ impl Iterator for MemoryQuadIter { } } +/// An iterator returning the graph names contained in a [`MemoryStore`]. +pub struct MemoryGraphNameIter { + iter: EncodedGraphsIter, + store: MemoryStore, +} + +impl Iterator for MemoryGraphNameIter { + type Item = NamedOrBlankNode; + + fn next(&mut self) -> Option { + Some( + self.store + .decode_named_or_blank_node(self.iter.next()?.unwrap_infallible()) + .unwrap(), + ) + } + + fn size_hint(&self) -> (usize, Option) { + self.iter.size_hint() + } +} + impl StrId for LargeSpur {} // Isomorphism implementation diff --git a/lib/src/store/mod.rs b/lib/src/store/mod.rs index caa49c3f..3957f476 100644 --- a/lib/src/store/mod.rs +++ b/lib/src/store/mod.rs @@ -34,6 +34,7 @@ use std::iter::Iterator; pub(crate) trait ReadableEncodedStore: StrLookup { type QuadsIter: Iterator, Self::Error>> + 'static; + type GraphsIter: Iterator, Self::Error>> + 'static; fn encoded_quads_for_pattern( &self, @@ -42,12 +43,36 @@ pub(crate) trait ReadableEncodedStore: StrLookup { object: Option>, graph_name: Option>, ) -> Self::QuadsIter; + + fn encoded_named_graphs(&self) -> Self::GraphsIter; + + fn contains_encoded_named_graph( + &self, + graph_name: EncodedTerm, + ) -> Result; } pub(crate) trait WritableEncodedStore: StrEncodingAware { fn insert_encoded(&mut self, quad: &EncodedQuad) -> Result<(), Self::Error>; fn remove_encoded(&mut self, quad: &EncodedQuad) -> Result<(), Self::Error>; + + fn insert_encoded_named_graph( + &mut self, + graph_name: EncodedTerm, + ) -> Result<(), Self::Error>; + + fn clear_encoded_graph( + &mut self, + graph_name: EncodedTerm, + ) -> Result<(), Self::Error>; + + fn remove_encoded_named_graph( + &mut self, + graph_name: EncodedTerm, + ) -> Result<(), Self::Error>; + + fn clear(&mut self) -> Result<(), Self::Error>; } pub(crate) fn load_graph( diff --git a/lib/src/store/rocksdb.rs b/lib/src/store/rocksdb.rs index c304a61a..bd556b51 100644 --- a/lib/src/store/rocksdb.rs +++ b/lib/src/store/rocksdb.rs @@ -77,9 +77,11 @@ const GOSP_CF: &str = "gosp"; const DSPO_CF: &str = "dspo"; const DPOS_CF: &str = "dpos"; const DOSP_CF: &str = "dosp"; +const GRAPHS_CF: &str = "graphs"; -const COLUMN_FAMILIES: [&str; 10] = [ +const COLUMN_FAMILIES: [&str; 11] = [ ID2STR_CF, SPOG_CF, POSG_CF, OSPG_CF, GSPO_CF, GPOS_CF, GOSP_CF, DSPO_CF, DPOS_CF, DOSP_CF, + GRAPHS_CF, ]; const MAX_TRANSACTION_SIZE: usize = 1024; @@ -96,15 +98,33 @@ impl RocksDbStore { db: Arc::new(DB::open_cf(&options, path, &COLUMN_FAMILIES).map_err(map_err)?), }; - let version = this.ensure_version()?; - if version != LATEST_STORAGE_VERSION { - return Err(invalid_data_error(format!( - "The RocksDB database is still using the encoding version {}, please upgrade it", - version - ))); + let mut version = this.ensure_version()?; + if version == 0 { + // We migrate to v1 + let mut transaction = this.auto_batch_writer(); + for quad in this.encoded_quads_for_pattern(None, None, None, None) { + let quad = quad?; + if !quad.graph_name.is_default_graph() { + transaction.insert_encoded_named_graph(quad.graph_name)?; + } + } + transaction.apply()?; + version = 1; + this.set_version(version)?; + this.flush()?; } - Ok(this) + match version { + _ if version < LATEST_STORAGE_VERSION => Err(invalid_data_error(format!( + "The RocksDB database is using the outdated encoding version {}. Automated migration is not supported, please dump the store dataset using a compatible Oxigraph version and load it again using the current version", + version + ))), + LATEST_STORAGE_VERSION => Ok(this), + _ => Err(invalid_data_error(format!( + "The RocksDB database is using the too recent version {}. Upgrade to the latest Oxigraph version to load this database", + version + ))) + } } fn ensure_version(&self) -> Result { @@ -114,14 +134,25 @@ impl RocksDbStore { buffer.copy_from_slice(&version); u64::from_be_bytes(buffer) } else { - self.db - .put("oxversion", &LATEST_STORAGE_VERSION.to_be_bytes()) - .map_err(map_err)?; + self.set_version(LATEST_STORAGE_VERSION)?; LATEST_STORAGE_VERSION }, ) } + fn set_version(&self, version: u64) -> Result<(), io::Error> { + self.db + .put("oxversion", &version.to_be_bytes()) + .map_err(map_err) + } + + fn flush(&self) -> Result<(), io::Error> { + let mut options = FlushOptions::new(); + options.set_wait(true); + self.db.flush_opt(&options).map_err(map_err)?; + Ok(()) + } + /// Executes a [SPARQL 1.1 query](https://www.w3.org/TR/sparql11-query/). /// /// See [`MemoryStore`](super::memory::MemoryStore::query()) for a usage example. @@ -352,15 +383,68 @@ impl RocksDbStore { dump_dataset(self.iter(), writer, syntax) } - /// Removes a graph from this store. + /// Returns all the store named graphs + /// + /// See [`MemoryStore`](super::memory::MemoryStore::named_graphs()) for a usage example. + pub fn named_graphs(&self) -> impl Iterator> { + let this = self.clone(); + self.encoded_named_graphs() + .map(move |g| Ok(this.decode_named_or_blank_node(g?)?)) + } + + /// Checks if the store contains a given graph + /// + /// See [`MemoryStore`](super::memory::MemoryStore::contains_named_graph()) for a usage example. + pub fn contains_named_graph<'a>( + &self, + graph_name: impl Into>, + ) -> Result { + if let Some(graph_name) = self.get_encoded_named_or_blank_node(graph_name.into())? { + self.contains_encoded_named_graph(graph_name) + } else { + Ok(false) + } + } + + /// Inserts a graph into this store + /// + /// See [`MemoryStore`](super::memory::MemoryStore::insert_named_graph()) for a usage example. + pub fn insert_named_graph<'a>( + &self, + graph_name: impl Into>, + ) -> Result<(), io::Error> { + let mut transaction = self.auto_batch_writer(); + let graph_name = transaction.encode_named_or_blank_node(graph_name.into())?; + transaction.insert_encoded_named_graph(graph_name)?; + transaction.apply() + } + + /// Clears a graph from this store. /// - /// See [`MemoryStore`](super::memory::MemoryStore::drop_graph()) for a usage example. - pub fn drop_graph<'a>(&self, graph_name: impl Into>) -> Result<(), io::Error> { + /// See [`MemoryStore`](super::memory::MemoryStore::clear_graph()) for a usage example. + pub fn clear_graph<'a>( + &self, + graph_name: impl Into>, + ) -> Result<(), io::Error> { if let Some(graph_name) = self.get_encoded_graph_name(graph_name.into())? { let mut transaction = self.auto_batch_writer(); - for quad in self.encoded_quads_for_pattern(None, None, None, Some(graph_name)) { - transaction.remove_encoded(&quad?)?; - } + transaction.clear_encoded_graph(graph_name)?; + transaction.apply() + } else { + Ok(()) + } + } + + /// Removes a graph from this store. + /// + /// See [`MemoryStore`](super::memory::MemoryStore::remove_named_graph()) for a usage example. + pub fn remove_named_graph<'a>( + &self, + graph_name: impl Into>, + ) -> Result<(), io::Error> { + if let Some(graph_name) = self.get_encoded_named_or_blank_node(graph_name.into())? { + let mut transaction = self.auto_batch_writer(); + transaction.remove_encoded_named_graph(graph_name)?; transaction.apply() } else { Ok(()) @@ -371,17 +455,9 @@ impl RocksDbStore { /// /// See [`MemoryStore`](super::memory::MemoryStore::clear()) for a usage example. pub fn clear(&self) -> Result<(), io::Error> { - self.clear_cf(self.id2str_cf())?; - self.clear_cf(self.spog_cf())?; - self.clear_cf(self.posg_cf())?; - self.clear_cf(self.ospg_cf())?; - self.clear_cf(self.gspo_cf())?; - self.clear_cf(self.gpos_cf())?; - self.clear_cf(self.gosp_cf())?; - self.clear_cf(self.dspo_cf())?; - self.clear_cf(self.dpos_cf())?; - self.clear_cf(self.dosp_cf())?; - Ok(()) + let mut transaction = self.auto_batch_writer(); + transaction.clear()?; + transaction.apply() } fn id2str_cf(&self) -> &ColumnFamily { @@ -424,6 +500,9 @@ impl RocksDbStore { get_cf(&self.db, DOSP_CF) } + fn graphs_cf(&self) -> &ColumnFamily { + get_cf(&self.db, GRAPHS_CF) + } fn auto_batch_writer(&self) -> AutoBatchWriter<'_> { AutoBatchWriter { store: self, @@ -657,48 +736,25 @@ impl RocksDbStore { self.inner_quads(self.dosp_cf(), prefix, QuadEncoding::DOSP) } - #[allow(unsafe_code)] fn inner_quads( &self, cf: &ColumnFamily, prefix: Vec, encoding: QuadEncoding, ) -> DecodingIndexIterator { - let mut iter = self.db.raw_iterator_cf(cf); - iter.seek(&prefix); + let mut iter = self.db_iter(cf); + iter.iter.seek(&prefix); DecodingIndexIterator { - iter: unsafe { StaticDBRowIterator::new(iter, self.db.clone()) }, // This is safe because the iterator belongs to DB + iter, prefix, encoding, } } - fn clear_cf(&self, cf: &ColumnFamily) -> Result<(), io::Error> { - self.db - .delete_range_cf( - cf, - [ - u8::MIN, - u8::MIN, - u8::MIN, - u8::MIN, - u8::MIN, - u8::MIN, - u8::MIN, - u8::MIN, - ], - [ - u8::MAX, - u8::MAX, - u8::MAX, - u8::MAX, - u8::MAX, - u8::MAX, - u8::MAX, - u8::MAX, - ], - ) - .map_err(map_err) + #[allow(unsafe_code)] + fn db_iter(&self, cf: &ColumnFamily) -> StaticDBRowIterator { + // Valid because it's the same database so db can't be dropped before iter + unsafe { StaticDBRowIterator::new(self.db.raw_iterator_cf(cf), self.db.clone()) } } } @@ -745,6 +801,7 @@ impl StrLookup for RocksDbStore { impl ReadableEncodedStore for RocksDbStore { type QuadsIter = DecodingIndexesIterator; + type GraphsIter = DecodingGraphIterator; fn encoded_quads_for_pattern( &self, @@ -809,6 +866,20 @@ impl ReadableEncodedStore for RocksDbStore { }, } } + + fn encoded_named_graphs(&self) -> DecodingGraphIterator { + let mut iter = self.db_iter(self.graphs_cf()); + iter.iter.seek_to_first(); + DecodingGraphIterator { iter } + } + + fn contains_encoded_named_graph(&self, graph_name: EncodedTerm) -> Result { + Ok(self + .db + .get_cf(self.graphs_cf(), &encode_term(graph_name)) + .map_err(map_err)? + .is_some()) + } } struct AutoBatchWriter<'a> { @@ -831,6 +902,32 @@ impl AutoBatchWriter<'_> { } Ok(()) } + + fn clear_cf(&mut self, cf: &ColumnFamily) { + self.batch.delete_range_cf( + cf, + [ + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + ], + [ + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + ], + ) + } } impl StrEncodingAware for AutoBatchWriter<'_> { @@ -885,6 +982,10 @@ impl WritableEncodedStore for AutoBatchWriter<'_> { write_gosp_quad(&mut self.buffer, quad); self.batch.put_cf(self.store.gosp_cf(), &self.buffer, &[]); self.buffer.clear(); + + write_term(&mut self.buffer, quad.graph_name); + self.batch.put_cf(self.store.graphs_cf(), &self.buffer, &[]); + self.buffer.clear(); } self.apply_if_big() @@ -931,6 +1032,49 @@ impl WritableEncodedStore for AutoBatchWriter<'_> { self.apply_if_big() } + + fn insert_encoded_named_graph(&mut self, graph_name: EncodedTerm) -> Result<(), io::Error> { + self.batch + .put_cf(self.store.graphs_cf(), &encode_term(graph_name), &[]); + self.apply_if_big() + } + + fn clear_encoded_graph(&mut self, graph_name: EncodedTerm) -> Result<(), io::Error> { + if graph_name.is_default_graph() { + self.clear_cf(self.store.dspo_cf()); + self.clear_cf(self.store.dpos_cf()); + self.clear_cf(self.store.dosp_cf()); + } else { + for quad in self.store.quads_for_graph(graph_name) { + self.remove_encoded(&quad?)?; + } + } + self.apply_if_big() + } + + fn remove_encoded_named_graph(&mut self, graph_name: EncodedTerm) -> Result<(), io::Error> { + for quad in self.store.quads_for_graph(graph_name) { + self.remove_encoded(&quad?)?; + } + self.batch + .delete_cf(self.store.graphs_cf(), &encode_term(graph_name)); + self.apply_if_big() + } + + fn clear(&mut self) -> Result<(), io::Error> { + self.clear_cf(self.store.spog_cf()); + self.clear_cf(self.store.posg_cf()); + self.clear_cf(self.store.ospg_cf()); + self.clear_cf(self.store.gspo_cf()); + self.clear_cf(self.store.gpos_cf()); + self.clear_cf(self.store.gosp_cf()); + self.clear_cf(self.store.dspo_cf()); + self.clear_cf(self.store.dpos_cf()); + self.clear_cf(self.store.dosp_cf()); + self.clear_cf(self.store.graphs_cf()); + self.clear_cf(self.store.id2str_cf()); + self.apply_if_big() + } } /// Allows inserting and deleting quads during an ACID transaction with the [`RocksDbStore`]. @@ -1093,6 +1237,10 @@ impl WritableEncodedStore for RocksDbTransaction<'_> { write_gosp_quad(&mut self.buffer, quad); self.batch.put_cf(self.store.gosp_cf(), &self.buffer, &[]); self.buffer.clear(); + + write_term(&mut self.buffer, quad.graph_name); + self.batch.put_cf(self.store.graphs_cf(), &self.buffer, &[]); + self.buffer.clear(); } Ok(()) @@ -1139,6 +1287,33 @@ impl WritableEncodedStore for RocksDbTransaction<'_> { Ok(()) } + + fn insert_encoded_named_graph(&mut self, graph_name: EncodedTerm) -> Result<(), io::Error> { + self.batch + .put_cf(self.store.graphs_cf(), &encode_term(graph_name), &[]); + Ok(()) + } + + fn clear_encoded_graph(&mut self, _: EncodedTerm) -> Result<(), io::Error> { + Err(io::Error::new( + io::ErrorKind::Other, + "CLEAR is not implemented in RocksDB transactions", + )) + } + + fn remove_encoded_named_graph(&mut self, _: EncodedTerm) -> Result<(), io::Error> { + Err(io::Error::new( + io::ErrorKind::Other, + "DROP is not implemented in RocksDB transactions", + )) + } + + fn clear(&mut self) -> Result<(), Self::Error> { + Err(io::Error::new( + io::ErrorKind::Other, + "CLEAR ALL is not implemented in RocksDB transactions", + )) + } } #[allow(clippy::expect_used)] @@ -1267,6 +1442,23 @@ impl Iterator for RocksDbQuadIter { } } +pub(crate) struct DecodingGraphIterator { + iter: StaticDBRowIterator, +} + +impl Iterator for DecodingGraphIterator { + type Item = Result; + + fn next(&mut self) -> Option> { + if let Some(key) = self.iter.key() { + let result = decode_term(key); + self.iter.next(); + Some(result) + } else { + None + } + } +} #[test] fn store() -> Result<(), io::Error> { use crate::model::*; diff --git a/lib/src/store/sled.rs b/lib/src/store/sled.rs index 890786e3..1c0115bc 100644 --- a/lib/src/store/sled.rs +++ b/lib/src/store/sled.rs @@ -75,6 +75,7 @@ pub struct SledStore { dspo: Tree, dpos: Tree, dosp: Tree, + graphs: Tree, } type EncodedTerm = crate::store::numeric_encoder::EncodedTerm; @@ -107,17 +108,35 @@ impl SledStore { dspo: db.open_tree("dspo")?, dpos: db.open_tree("dpos")?, dosp: db.open_tree("dosp")?, + graphs: db.open_tree("graphs")?, }; - let version = this.ensure_version()?; - if version != LATEST_STORAGE_VERSION { - return Err(invalid_data_error(format!( - "The Sled database is still using the encoding version {}, please upgrade it", - version - ))); + let mut version = this.ensure_version()?; + if version == 0 { + // We migrate to v1 + for quad in this.encoded_quads_for_pattern(None, None, None, None) { + let mut this_mut = &this; + let quad = quad?; + if !quad.graph_name.is_default_graph() { + this_mut.insert_encoded_named_graph(quad.graph_name)?; + } + } + version = 1; + this.set_version(version)?; + this.graphs.flush()?; } - Ok(this) + match version { + _ if version < LATEST_STORAGE_VERSION => Err(invalid_data_error(format!( + "The Sled database is using the outdated encoding version {}. Automated migration is not supported, please dump the store dataset using a compatible Oxigraph version and load it again using the current version", + version + ))), + LATEST_STORAGE_VERSION => Ok(this), + _ => Err(invalid_data_error(format!( + "The Sled database is using the too recent version {}. Upgrade to the latest Oxigraph version to load this database", + version + ))) + } } fn ensure_version(&self) -> Result { @@ -126,12 +145,16 @@ impl SledStore { buffer.copy_from_slice(&version); u64::from_be_bytes(buffer) } else { - self.default - .insert("oxversion", &LATEST_STORAGE_VERSION.to_be_bytes())?; + self.set_version(LATEST_STORAGE_VERSION)?; LATEST_STORAGE_VERSION }) } + fn set_version(&self, version: u64) -> Result<(), io::Error> { + self.default.insert("oxversion", &version.to_be_bytes())?; + Ok(()) + } + /// Executes a [SPARQL 1.1 query](https://www.w3.org/TR/sparql11-query/). /// /// See [`MemoryStore`](super::memory::MemoryStore::query()) for a usage example. @@ -268,9 +291,10 @@ impl SledStore { &self.dspo, &self.dpos, &self.dosp, + &self.graphs, ) .transaction( - move |(id2str, spog, posg, ospg, gspo, gpos, gosp, dspo, dpos, dosp)| { + move |(id2str, spog, posg, ospg, gspo, gpos, gosp, dspo, dpos, dosp, graphs)| { Ok(f(SledTransaction { id2str, spog, @@ -282,6 +306,7 @@ impl SledStore { dspo, dpos, dosp, + graphs, })?) }, )?) @@ -385,34 +410,78 @@ impl SledStore { dump_dataset(self.iter(), writer, format) } - /// Removes a graph from this store. + /// Returns all the store named graphs + /// + /// See [`MemoryStore`](super::memory::MemoryStore::named_graphs()) for a usage example. + pub fn named_graphs(&self) -> SledGraphNameIter { + SledGraphNameIter { + iter: self.encoded_named_graphs(), + store: self.clone(), + } + } + + /// Checks if the store contains a given graph /// - /// See [`MemoryStore`](super::memory::MemoryStore::drop_graph()) for a usage example. - pub fn drop_graph<'a>(&self, graph_name: impl Into>) -> Result<(), io::Error> { + /// See [`MemoryStore`](super::memory::MemoryStore::contains_named_graph()) for a usage example. + pub fn contains_named_graph<'a>( + &self, + graph_name: impl Into>, + ) -> Result { + if let Some(graph_name) = self.get_encoded_named_or_blank_node(graph_name.into())? { + self.contains_encoded_named_graph(graph_name) + } else { + Ok(false) + } + } + + /// Inserts a graph into this store + /// + /// See [`MemoryStore`](super::memory::MemoryStore::insert_named_graph()) for a usage example. + pub fn insert_named_graph<'a>( + &self, + graph_name: impl Into>, + ) -> Result<(), io::Error> { + let mut this = self; + let graph_name = this.encode_named_or_blank_node(graph_name.into())?; + this.insert_encoded_named_graph(graph_name) + } + + /// Clears a graph from this store. + /// + /// See [`MemoryStore`](super::memory::MemoryStore::clear_graph()) for a usage example. + pub fn clear_graph<'a>( + &self, + graph_name: impl Into>, + ) -> Result<(), io::Error> { if let Some(graph_name) = self.get_encoded_graph_name(graph_name.into())? { - for quad in self.encoded_quads_for_pattern(None, None, None, Some(graph_name)) { - let mut this = self; - this.remove_encoded(&quad?)?; - } + let mut this = self; + this.clear_encoded_graph(graph_name) + } else { + Ok(()) + } + } + + /// Removes a graph from this store. + /// + /// See [`MemoryStore`](super::memory::MemoryStore::remove_named_graph()) for a usage example. + pub fn remove_named_graph<'a>( + &self, + graph_name: impl Into>, + ) -> Result<(), io::Error> { + if let Some(graph_name) = self.get_encoded_named_or_blank_node(graph_name.into())? { + let mut this = self; + this.remove_encoded_named_graph(graph_name) + } else { + Ok(()) } - Ok(()) } /// Clears the store. /// /// See [`MemoryStore`](super::memory::MemoryStore::clear()) for a usage example. pub fn clear(&self) -> Result<(), io::Error> { - self.dspo.clear()?; - self.dpos.clear()?; - self.dosp.clear()?; - self.gspo.clear()?; - self.gpos.clear()?; - self.gosp.clear()?; - self.spog.clear()?; - self.posg.clear()?; - self.ospg.clear()?; - self.id2str.clear()?; - Ok(()) + let mut this = self; + (&mut this).clear() } fn contains_encoded(&self, quad: &EncodedQuad) -> Result { @@ -679,6 +748,7 @@ impl StrLookup for SledStore { impl ReadableEncodedStore for SledStore { type QuadsIter = DecodingQuadsIterator; + type GraphsIter = DecodingGraphIterator; fn encoded_quads_for_pattern( &self, @@ -742,6 +812,16 @@ impl ReadableEncodedStore for SledStore { }, } } + + fn encoded_named_graphs(&self) -> DecodingGraphIterator { + DecodingGraphIterator { + iter: self.graphs.iter(), + } + } + + fn contains_encoded_named_graph(&self, graph_name: EncodedTerm) -> Result { + Ok(self.graphs.contains_key(&encode_term(graph_name))?) + } } impl<'a> StrContainer for &'a SledStore { @@ -792,6 +872,10 @@ impl<'a> WritableEncodedStore for &'a SledStore { write_gosp_quad(&mut buffer, quad); self.gosp.insert(buffer.as_slice(), &[])?; buffer.clear(); + + write_term(&mut buffer, quad.graph_name); + self.graphs.insert(&buffer, &[])?; + buffer.clear(); } Ok(()) @@ -840,6 +924,47 @@ impl<'a> WritableEncodedStore for &'a SledStore { Ok(()) } + + fn insert_encoded_named_graph(&mut self, graph_name: EncodedTerm) -> Result<(), io::Error> { + self.graphs.insert(&encode_term(graph_name), &[])?; + Ok(()) + } + + fn clear_encoded_graph(&mut self, graph_name: EncodedTerm) -> Result<(), io::Error> { + if graph_name.is_default_graph() { + self.dspo.clear()?; + self.dpos.clear()?; + self.dosp.clear()?; + } else { + for quad in self.quads_for_graph(graph_name) { + self.remove_encoded(&quad?)?; + } + } + Ok(()) + } + + fn remove_encoded_named_graph(&mut self, graph_name: EncodedTerm) -> Result<(), io::Error> { + for quad in self.quads_for_graph(graph_name) { + self.remove_encoded(&quad?)?; + } + self.graphs.remove(&encode_term(graph_name))?; + Ok(()) + } + + fn clear(&mut self) -> Result<(), io::Error> { + self.dspo.clear()?; + self.dpos.clear()?; + self.dosp.clear()?; + self.gspo.clear()?; + self.gpos.clear()?; + self.gosp.clear()?; + self.spog.clear()?; + self.posg.clear()?; + self.ospg.clear()?; + self.graphs.clear()?; + self.id2str.clear()?; + Ok(()) + } } /// Allows inserting and deleting quads during an ACID transaction with the [`SledStore`]. @@ -854,6 +979,7 @@ pub struct SledTransaction<'a> { dspo: &'a TransactionalTree, dpos: &'a TransactionalTree, dosp: &'a TransactionalTree, + graphs: &'a TransactionalTree, } impl SledTransaction<'_> { @@ -1058,6 +1184,41 @@ impl<'a> WritableEncodedStore for &'a SledTransaction<'a> { Ok(()) } + + fn insert_encoded_named_graph( + &mut self, + graph_name: EncodedTerm, + ) -> Result<(), SledUnabortableTransactionError> { + self.graphs.insert(encode_term(graph_name), &[])?; + Ok(()) + } + + fn clear_encoded_graph( + &mut self, + _: EncodedTerm, + ) -> Result<(), SledUnabortableTransactionError> { + Err(SledUnabortableTransactionError::Storage(io::Error::new( + io::ErrorKind::Other, + "CLEAR is not implemented in Sled transactions", + ))) + } + + fn remove_encoded_named_graph( + &mut self, + _: EncodedTerm, + ) -> Result<(), SledUnabortableTransactionError> { + Err(SledUnabortableTransactionError::Storage(io::Error::new( + io::ErrorKind::Other, + "DROP is not implemented in Sled transactions", + ))) + } + + fn clear(&mut self) -> Result<(), SledUnabortableTransactionError> { + Err(SledUnabortableTransactionError::Storage(io::Error::new( + io::ErrorKind::Other, + "CLEAR ALL is not implemented in Sled transactions", + ))) + } } /// Error returned by a Sled transaction @@ -1263,6 +1424,21 @@ impl Iterator for DecodingQuadIterator { } } +pub(crate) struct DecodingGraphIterator { + iter: Iter, +} + +impl Iterator for DecodingGraphIterator { + type Item = Result; + + fn next(&mut self) -> Option> { + Some(match self.iter.next()? { + Ok((encoded, _)) => decode_term(&encoded), + Err(error) => Err(error.into()), + }) + } +} + /// An iterator returning the quads contained in a [`SledStore`]. pub struct SledQuadIter { inner: QuadIterInner, @@ -1292,6 +1468,28 @@ impl Iterator for SledQuadIter { } } +/// An iterator returning the graph names contained in a [`SledStore`]. +pub struct SledGraphNameIter { + iter: DecodingGraphIterator, + store: SledStore, +} + +impl Iterator for SledGraphNameIter { + type Item = Result; + + fn next(&mut self) -> Option> { + Some( + self.iter + .next()? + .and_then(|graph_name| Ok(self.store.decode_named_or_blank_node(graph_name)?)), + ) + } + + fn size_hint(&self) -> (usize, Option) { + self.iter.size_hint() + } +} + #[test] fn store() -> Result<(), io::Error> { use crate::model::*; diff --git a/lib/tests/rocksdb_store.rs b/lib/tests/rocksdb_store.rs index ec045161..664fa36e 100644 --- a/lib/tests/rocksdb_store.rs +++ b/lib/tests/rocksdb_store.rs @@ -1,12 +1,11 @@ -#![cfg(features = "rocksdb")] - use oxigraph::model::vocab::{rdf, xsd}; use oxigraph::model::*; use oxigraph::RocksDbStore; use std::io; use std::process::Command; -fn quads(graph_name: GraphNameRef<'static>) -> Vec> { +fn quads(graph_name: impl Into>) -> Vec> { + let graph_name = graph_name.into(); let paris = NamedNodeRef::new_unchecked("http://www.wikidata.org/entity/Q90"); let france = NamedNodeRef::new_unchecked("http://www.wikidata.org/entity/Q142"); let city = NamedNodeRef::new_unchecked("http://schema.org/City"); @@ -65,18 +64,26 @@ fn test_backward_compatibility() -> io::Result<()> { for q in quads(GraphNameRef::DefaultGraph) { assert!(store.contains(q)?); } - for q in quads( - NamedNodeRef::new_unchecked("http://www.wikidata.org/wiki/Special:EntityData/Q90") - .into(), - ) { + let graph_name = + NamedNodeRef::new_unchecked("http://www.wikidata.org/wiki/Special:EntityData/Q90"); + for q in quads(graph_name) { assert!(store.contains(q)?); } + assert!(store.contains_named_graph(graph_name)?); + assert_eq!( + vec![NamedOrBlankNode::from(graph_name)], + store.named_graphs().collect::>>()? + ); }; reset_dir("tests/rockdb_bc_data")?; Ok(()) } fn reset_dir(dir: &str) -> io::Result<()> { + assert!(Command::new("git") + .args(&["clean", "-fX", dir]) + .status()? + .success()); assert!(Command::new("git") .args(&["checkout", "HEAD", "--", dir]) .status()? diff --git a/lib/tests/sled_store.rs b/lib/tests/sled_store.rs index 591af749..6cf542a3 100644 --- a/lib/tests/sled_store.rs +++ b/lib/tests/sled_store.rs @@ -1,5 +1,3 @@ -#![cfg(features = "sled")] - use oxigraph::io::{DatasetFormat, GraphFormat}; use oxigraph::model::vocab::{rdf, xsd}; use oxigraph::model::*; @@ -24,7 +22,8 @@ wd:Q90 a schema:City ; "#; const NUMBER_OF_TRIPLES: usize = 8; -fn quads(graph_name: GraphNameRef<'static>) -> Vec> { +fn quads(graph_name: impl Into>) -> Vec> { + let graph_name = graph_name.into(); let paris = NamedNodeRef::new_unchecked("http://www.wikidata.org/entity/Q90"); let france = NamedNodeRef::new_unchecked("http://www.wikidata.org/entity/Q142"); let city = NamedNodeRef::new_unchecked("http://schema.org/City"); @@ -161,18 +160,26 @@ fn test_backward_compatibility() -> io::Result<()> { for q in quads(GraphNameRef::DefaultGraph) { assert!(store.contains(q)?); } - for q in quads( - NamedNodeRef::new_unchecked("http://www.wikidata.org/wiki/Special:EntityData/Q90") - .into(), - ) { + let graph_name = + NamedNodeRef::new_unchecked("http://www.wikidata.org/wiki/Special:EntityData/Q90"); + for q in quads(graph_name) { assert!(store.contains(q)?); } + assert!(store.contains_named_graph(graph_name)?); + assert_eq!( + vec![NamedOrBlankNode::from(graph_name)], + store.named_graphs().collect::>>()? + ); }; reset_dir("tests/sled_bc_data")?; Ok(()) } fn reset_dir(dir: &str) -> io::Result<()> { + assert!(Command::new("git") + .args(&["clean", "-fX", dir]) + .status()? + .success()); assert!(Command::new("git") .args(&["checkout", "HEAD", "--", dir]) .status()? diff --git a/python/src/memory_store.rs b/python/src/memory_store.rs index 9c5f93df..bcce09a3 100644 --- a/python/src/memory_store.rs +++ b/python/src/memory_store.rs @@ -3,6 +3,7 @@ use crate::model::*; use crate::sparql::*; use crate::store_utils::*; use oxigraph::io::{DatasetFormat, GraphFormat}; +use oxigraph::model::GraphNameRef; use oxigraph::store::memory::*; use pyo3::basic::CompareOp; use pyo3::exceptions::{PyNotImplementedError, PyValueError}; @@ -333,6 +334,63 @@ impl PyMemoryStore { ))) } } + + /// Returns an iterator over all the store named graphs + /// + /// >>> store = MemoryStore() + /// >>> store.add(Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'), NamedNode('http://example.com/g'))) + /// >>> list(store.named_graphs()) + /// [] + #[text_signature = "($self)"] + fn named_graphs(&self) -> GraphNameIter { + GraphNameIter { + inner: self.inner.named_graphs(), + } + } + + /// Adds a named graph to the store + /// + /// :param graph_name: the quad to add + /// :type graph_name: NamedNode or BlankNode + /// + /// >>> store = MemoryStore() + /// >>> store.add_graph(NamedNode('http://example.com/g')) + /// >>> list(store.named_graphs()) + /// [] + #[text_signature = "($self, graph_name)"] + fn add_graph(&self, graph_name: PyGraphName) { + match graph_name { + PyGraphName::DefaultGraph(_) => (), + PyGraphName::NamedNode(graph_name) => self.inner.insert_named_graph(graph_name), + PyGraphName::BlankNode(graph_name) => self.inner.insert_named_graph(graph_name), + } + } + + /// Removes a graph from the store + /// + /// The default graph will not be remove but just cleared. + /// + /// :param graph_name: the quad to add + /// :type graph_name: NamedNode or BlankNode or DefaultGraph + /// + /// >>> store = MemoryStore() + /// >>> quad = Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'), NamedNode('http://example.com/g')) + /// >>> store.remove_graph(NamedNode('http://example.com/g')) + /// >>> list(store) + /// [] + #[text_signature = "($self, graph_name)"] + fn remove_graph(&self, graph_name: &PyAny) -> PyResult<()> { + match PyGraphNameRef::try_from(graph_name)? { + PyGraphNameRef::DefaultGraph => self.inner.clear_graph(GraphNameRef::DefaultGraph), + PyGraphNameRef::NamedNode(graph_name) => self + .inner + .remove_named_graph(&PyNamedOrBlankNodeRef::NamedNode(graph_name)), + PyGraphNameRef::BlankNode(graph_name) => self + .inner + .remove_named_graph(&PyNamedOrBlankNodeRef::BlankNode(graph_name)), + } + Ok(()) + } } #[pyproto] @@ -392,3 +450,19 @@ impl PyIterProtocol for QuadIter { slf.inner.next().map(|q| q.into()) } } + +#[pyclass(unsendable, module = "oxigraph")] +pub struct GraphNameIter { + inner: MemoryGraphNameIter, +} + +#[pyproto] +impl PyIterProtocol for GraphNameIter { + fn __iter__(slf: PyRefMut) -> Py { + slf.into() + } + + fn __next__(mut slf: PyRefMut) -> Option { + slf.inner.next().map(|q| q.into()) + } +} diff --git a/python/src/sled_store.rs b/python/src/sled_store.rs index 93ce7aea..82621d67 100644 --- a/python/src/sled_store.rs +++ b/python/src/sled_store.rs @@ -3,6 +3,7 @@ use crate::model::*; use crate::sparql::*; use crate::store_utils::*; use oxigraph::io::{DatasetFormat, GraphFormat}; +use oxigraph::model::GraphNameRef; use oxigraph::store::sled::*; use pyo3::exceptions::PyValueError; use pyo3::prelude::{ @@ -351,6 +352,68 @@ impl PySledStore { ))) } } + + /// Returns an iterator over all the store named graphs + /// + /// >>> store = MemoryStore() + /// >>> store.add(Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'), NamedNode('http://example.com/g'))) + /// >>> list(store.named_graphs()) + /// [] + #[text_signature = "($self)"] + fn named_graphs(&self) -> GraphNameIter { + GraphNameIter { + inner: self.inner.named_graphs(), + } + } + + /// Adds a named graph to the store + /// + /// :param graph_name: the quad to add + /// :type graph_name: NamedNode or BlankNode + /// + /// >>> store = MemoryStore() + /// >>> store.add_graph(NamedNode('http://example.com/g')) + /// >>> list(store.named_graphs()) + /// [] + #[text_signature = "($self, graph_name)"] + fn add_graph(&self, graph_name: &PyAny) -> PyResult<()> { + match PyGraphNameRef::try_from(graph_name)? { + PyGraphNameRef::DefaultGraph => Ok(()), + PyGraphNameRef::NamedNode(graph_name) => self + .inner + .insert_named_graph(&PyNamedOrBlankNodeRef::NamedNode(graph_name)), + PyGraphNameRef::BlankNode(graph_name) => self + .inner + .insert_named_graph(&PyNamedOrBlankNodeRef::BlankNode(graph_name)), + } + .map_err(map_io_err) + } + + /// Removes a graph from the store + /// + /// The default graph will not be remove but just cleared. + /// + /// :param graph_name: the quad to add + /// :type graph_name: NamedNode or BlankNode or DefaultGraph + /// + /// >>> store = MemoryStore() + /// >>> quad = Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'), NamedNode('http://example.com/g')) + /// >>> store.remove_graph(NamedNode('http://example.com/g')) + /// >>> list(store) + /// [] + #[text_signature = "($self, graph_name)"] + fn remove_graph(&self, graph_name: &PyAny) -> PyResult<()> { + match PyGraphNameRef::try_from(graph_name)? { + PyGraphNameRef::DefaultGraph => self.inner.clear_graph(GraphNameRef::DefaultGraph), + PyGraphNameRef::NamedNode(graph_name) => self + .inner + .remove_named_graph(&PyNamedOrBlankNodeRef::NamedNode(graph_name)), + PyGraphNameRef::BlankNode(graph_name) => self + .inner + .remove_named_graph(&PyNamedOrBlankNodeRef::BlankNode(graph_name)), + } + .map_err(map_io_err) + } } #[pyproto] @@ -402,3 +465,22 @@ impl PyIterProtocol for QuadIter { .transpose() } } + +#[pyclass(unsendable, module = "oxigraph")] +pub struct GraphNameIter { + inner: SledGraphNameIter, +} + +#[pyproto] +impl PyIterProtocol for GraphNameIter { + fn __iter__(slf: PyRefMut) -> Py { + slf.into() + } + + fn __next__(mut slf: PyRefMut) -> PyResult> { + slf.inner + .next() + .map(|q| Ok(q.map_err(map_io_err)?.into())) + .transpose() + } +} diff --git a/python/tests/test_store.py b/python/tests/test_store.py index 95edfcab..a979c03c 100644 --- a/python/tests/test_store.py +++ b/python/tests/test_store.py @@ -244,6 +244,20 @@ class TestAbstractStore(unittest.TestCase, ABC): store.add(Quad(triple.object, triple.predicate, triple.subject)) self.assertEqual(len(store), 4) + def test_add_graph(self): + store = self.store() + store.add_graph(graph) + self.assertEqual(list(store.named_graphs()), [graph]) + + def test_remove_graph(self): + store = self.store() + store.add(Quad(foo, bar, baz, graph)) + store.add_graph(NamedNode("http://graph2")) + store.remove_graph(graph) + store.remove_graph(NamedNode("http://graph2")) + self.assertEqual(list(store.named_graphs()), []) + self.assertEqual(list(store), []) + class TestMemoryStore(TestAbstractStore): def store(self):