Makes Oxigraph store "graph aware"

pull/73/head
Tpt 4 years ago
parent 60f601d6d3
commit 6aa27d4885
  1. 2
      CHANGELOG.md
  2. 18
      lib/src/sparql/dataset.rs
  3. 4
      lib/src/sparql/parser.rs
  4. 136
      lib/src/sparql/update.rs
  5. 6
      lib/src/store/binary_encoder.rs
  6. 276
      lib/src/store/memory.rs
  7. 25
      lib/src/store/mod.rs
  8. 306
      lib/src/store/rocksdb.rs
  9. 250
      lib/src/store/sled.rs
  10. 21
      lib/tests/rocksdb_store.rs
  11. 21
      lib/tests/sled_store.rs
  12. 74
      python/src/memory_store.rs
  13. 82
      python/src/sled_store.rs
  14. 14
      python/tests/test_store.py

@ -4,10 +4,10 @@
- [SPARQL 1.1 Update](https://www.w3.org/TR/sparql11-update/) support for Rust, Python and JavaScript. All store-like classes now provide an `update` method.
- [SPARQL 1.1 Query Results CSV and TSV Formats](https://www.w3.org/TR/sparql11-results-csv-tsv/) serializers and TSV format parser.
- The SPARQL Query and Update algebra is now public.
- The stores are now "graph aware" i.e. it is possible to create and keep empty named graphs.
- A simple built-in HTTP client. In the Rust library, is disabled by default behind the `http_client` feature. It powers SPARQL federation and SPARQL UPDATE `LOAD` operations.
- `std::str::FromStr` implementations to `NamedNode`, `BlankNode`, `Literal`, `Term` and `Variable` allowing to easily parse Turtle/SPARQL serialization of these terms.
- Optional Sled storage for `oxigraph_server`.
- `(Memory|RocksDB|Sled)Store::drop_graph` and `(Memory|RocksDB|Sled)Store::clear`.
### Removed
- The `default_graph_uris` and `named_graph_uris` parameters from `pyoxigraph` `query` methods.

@ -6,7 +6,7 @@ use crate::store::numeric_encoder::{
use crate::store::ReadableEncodedStore;
use lasso::{Rodeo, Spur};
use std::cell::RefCell;
use std::iter::empty;
use std::iter::{empty, once, Once};
pub(crate) struct DatasetView<S: ReadableEncodedStore> {
store: S,
@ -182,6 +182,7 @@ impl<S: ReadableEncodedStore> StrLookup for DatasetView<S> {
impl<S: ReadableEncodedStore> ReadableEncodedStore for DatasetView<S> {
type QuadsIter =
Box<dyn Iterator<Item = Result<EncodedQuad<DatasetStrId<S::StrId>>, EvaluationError>>>;
type GraphsIter = Once<Result<EncodedTerm<DatasetStrId<S::StrId>>, EvaluationError>>;
fn encoded_quads_for_pattern(
&self,
@ -199,6 +200,21 @@ impl<S: ReadableEncodedStore> ReadableEncodedStore for DatasetView<S> {
Box::new(empty())
}
}
fn encoded_named_graphs(&self) -> Self::GraphsIter {
once(Err(EvaluationError::msg(
"Graphs lookup is not implemented by DatasetView",
)))
}
fn contains_encoded_named_graph(
&self,
_: EncodedTerm<Self::StrId>,
) -> Result<bool, EvaluationError> {
Err(EvaluationError::msg(
"Graphs lookup is not implemented by DatasetView",
))
}
}
fn map_iter<'a, I: StrId>(

@ -986,7 +986,7 @@ parser! {
Vec::new() // identity case
} else {
let bgp = GraphPattern::BGP(vec![TriplePattern::new(Variable::new_unchecked("s"), Variable::new_unchecked("p"), Variable::new_unchecked("o"))]);
vec![GraphUpdateOperation::Drop { silent, graph: to.clone().map_or(GraphTarget::DefaultGraph, GraphTarget::NamedNode) }, copy_graph(from.clone(), to.map(NamedNodeOrVariable::NamedNode)), GraphUpdateOperation::Drop { silent, graph: from.map_or(GraphTarget::DefaultGraph, GraphTarget::NamedNode) }]
vec![GraphUpdateOperation::Drop { silent: true, graph: to.clone().map_or(GraphTarget::DefaultGraph, GraphTarget::NamedNode) }, copy_graph(from.clone(), to.map(NamedNodeOrVariable::NamedNode)), GraphUpdateOperation::Drop { silent, graph: from.map_or(GraphTarget::DefaultGraph, GraphTarget::NamedNode) }]
}
}
@ -997,7 +997,7 @@ parser! {
Vec::new() // identity case
} else {
let bgp = GraphPattern::BGP(vec![TriplePattern::new(Variable::new_unchecked("s"), Variable::new_unchecked("p"), Variable::new_unchecked("o"))]);
vec![GraphUpdateOperation::Drop { silent, graph: to.clone().map_or(GraphTarget::DefaultGraph, GraphTarget::NamedNode) }, copy_graph(from, to.map(NamedNodeOrVariable::NamedNode))]
vec![GraphUpdateOperation::Drop { silent: true, graph: to.clone().map_or(GraphTarget::DefaultGraph, GraphTarget::NamedNode) }, copy_graph(from, to.map(NamedNodeOrVariable::NamedNode))]
}
}

@ -81,9 +81,9 @@ where
Ok(())
}
}
GraphUpdateOperation::Clear { graph, .. } => self.eval_clear(graph),
GraphUpdateOperation::Create { .. } => Ok(()),
GraphUpdateOperation::Drop { graph, .. } => self.eval_clear(graph),
GraphUpdateOperation::Clear { graph, silent } => self.eval_clear(graph, *silent),
GraphUpdateOperation::Create { graph, silent } => self.eval_create(graph, *silent),
GraphUpdateOperation::Drop { graph, silent } => self.eval_drop(graph, *silent),
}
}
@ -227,56 +227,130 @@ where
Ok(())
}
fn eval_clear(&mut self, graph: &GraphTarget) -> Result<(), EvaluationError> {
fn eval_create(&mut self, graph: &NamedNode, silent: bool) -> Result<(), EvaluationError> {
let encoded_graph_name = self
.write
.encode_named_node(graph.as_ref())
.map_err(to_eval_error)?;
if self
.read
.contains_encoded_named_graph(encoded_graph_name)
.map_err(to_eval_error)?
{
if silent {
Ok(())
} else {
Err(EvaluationError::msg(format!(
"The graph {} already exists",
graph
)))
}
} else {
self.write
.insert_encoded_named_graph(encoded_graph_name)
.map_err(to_eval_error)
}
}
fn eval_clear(&mut self, graph: &GraphTarget, silent: bool) -> Result<(), EvaluationError> {
match graph {
GraphTarget::NamedNode(graph) => {
if let Some(graph) = self
GraphTarget::NamedNode(graph_name) => {
if let Some(graph_name) = self
.read
.get_encoded_named_node(graph.into())
.get_encoded_named_node(graph_name.as_ref())
.map_err(to_eval_error)?
{
for quad in self
if self
.read
.encoded_quads_for_pattern(None, None, None, Some(graph))
.contains_encoded_named_graph(graph_name)
.map_err(to_eval_error)?
{
self.write
.remove_encoded(&quad.map_err(to_eval_error)?)
.map_err(to_eval_error)?;
return self
.write
.clear_encoded_graph(graph_name)
.map_err(to_eval_error);
}
}
if silent {
Ok(())
} else {
//we do not track created graph so it's fine
Err(EvaluationError::msg(format!(
"The graph {} does not exists",
graph
)))
}
}
GraphTarget::DefaultGraph => {
for quad in self.read.encoded_quads_for_pattern(
None,
None,
None,
Some(EncodedTerm::DefaultGraph),
) {
GraphTarget::DefaultGraph => self
.write
.clear_encoded_graph(EncodedTerm::DefaultGraph)
.map_err(to_eval_error),
GraphTarget::NamedGraphs => {
// TODO: optimize?
for graph in self.read.encoded_named_graphs() {
self.write
.remove_encoded(&quad.map_err(to_eval_error)?)
.clear_encoded_graph(graph.map_err(to_eval_error)?)
.map_err(to_eval_error)?;
}
Ok(())
}
GraphTarget::NamedGraphs => {
for quad in self.read.encoded_quads_for_pattern(None, None, None, None) {
let quad = quad.map_err(to_eval_error)?;
if !quad.graph_name.is_default_graph() {
self.write.remove_encoded(&quad).map_err(to_eval_error)?;
GraphTarget::AllGraphs => {
// TODO: optimize?
for graph in self.read.encoded_named_graphs() {
self.write
.clear_encoded_graph(graph.map_err(to_eval_error)?)
.map_err(to_eval_error)?;
}
self.write
.clear_encoded_graph(EncodedTerm::DefaultGraph)
.map_err(to_eval_error)
}
}
GraphTarget::AllGraphs => {
for quad in self.read.encoded_quads_for_pattern(None, None, None, None) {
}
fn eval_drop(&mut self, graph: &GraphTarget, silent: bool) -> Result<(), EvaluationError> {
match graph {
GraphTarget::NamedNode(graph_name) => {
if let Some(graph_name) = self
.read
.get_encoded_named_node(graph_name.as_ref())
.map_err(to_eval_error)?
{
if self
.read
.contains_encoded_named_graph(graph_name)
.map_err(to_eval_error)?
{
return self
.write
.remove_encoded_named_graph(graph_name)
.map_err(to_eval_error);
}
}
if silent {
Ok(())
} else {
Err(EvaluationError::msg(format!(
"The graph {} does not exists",
graph
)))
}
}
GraphTarget::DefaultGraph => self
.write
.clear_encoded_graph(EncodedTerm::DefaultGraph)
.map_err(to_eval_error),
GraphTarget::NamedGraphs => {
// TODO: optimize?
for graph in self.read.encoded_named_graphs() {
self.write
.remove_encoded(&quad.map_err(to_eval_error)?)
.remove_encoded_named_graph(graph.map_err(to_eval_error)?)
.map_err(to_eval_error)?;
}
}
};
Ok(())
}
GraphTarget::AllGraphs => self.write.clear().map_err(to_eval_error),
}
}
fn encode_quad_for_insertion(
&mut self,

@ -11,7 +11,7 @@ use std::mem::size_of;
type EncodedTerm = crate::store::numeric_encoder::EncodedTerm<StrHash>;
type EncodedQuad = crate::store::numeric_encoder::EncodedQuad<StrHash>;
pub const LATEST_STORAGE_VERSION: u64 = 0;
pub const LATEST_STORAGE_VERSION: u64 = 1;
pub const WRITTEN_TERM_MAX_SIZE: usize = size_of::<u8>() + 2 * size_of::<StrHash>();
// Encoded term type blocks
@ -111,6 +111,10 @@ impl QuadEncoding {
}
}
pub fn decode_term(buffer: &[u8]) -> Result<EncodedTerm, io::Error> {
Cursor::new(&buffer).read_term()
}
pub trait TermReader {
fn read_term(&mut self) -> Result<EncodedTerm, io::Error>;

@ -455,7 +455,75 @@ impl MemoryStore {
dump_dataset(self.iter().map(Ok), writer, format)
}
/// Removes a graph from this store.
/// Returns all the store named graphs
///
/// Usage example:
/// ```
/// use oxigraph::MemoryStore;
/// use oxigraph::model::{NamedNode, Quad, NamedOrBlankNode};
///
/// let ex = NamedNode::new("http://example.com")?;
/// let store = MemoryStore::new();
/// store.insert(Quad::new(ex.clone(), ex.clone(), ex.clone(), ex.clone()));
/// store.insert(Quad::new(ex.clone(), ex.clone(), ex.clone(), None));
/// assert_eq!(vec![NamedOrBlankNode::from(ex)], store.named_graphs().collect::<Vec<_>>());
/// # Result::<_,Box<dyn std::error::Error>>::Ok(())
/// ```
pub fn named_graphs(&self) -> MemoryGraphNameIter {
MemoryGraphNameIter {
iter: self.encoded_named_graphs(),
store: self.clone(),
}
}
/// Checks if the store contains a given graph
///
/// Usage example:
/// ```
/// use oxigraph::MemoryStore;
/// use oxigraph::model::{NamedNode, Quad};
///
/// let ex = NamedNode::new("http://example.com")?;
/// let store = MemoryStore::new();
/// store.insert(Quad::new(ex.clone(), ex.clone(), ex.clone(), ex.clone()));
/// assert!(store.contains_named_graph(&ex));
/// # Result::<_,Box<dyn std::error::Error>>::Ok(())
/// ```
pub fn contains_named_graph<'a>(&self, graph_name: impl Into<NamedOrBlankNodeRef<'a>>) -> bool {
if let Some(graph_name) = self
.get_encoded_named_or_blank_node(graph_name.into())
.unwrap_infallible()
{
self.contains_encoded_named_graph(graph_name)
.unwrap_infallible()
} else {
false
}
}
/// Inserts a graph into this store
///
/// Usage example:
/// ```
/// use oxigraph::MemoryStore;
/// use oxigraph::model::NamedNode;
///
/// let ex = NamedNode::new("http://example.com")?;
/// let store = MemoryStore::new();
/// store.insert_named_graph(ex.clone());
/// assert_eq!(store.named_graphs().count(), 1);
/// # Result::<_,Box<dyn std::error::Error>>::Ok(())
/// ```
pub fn insert_named_graph(&self, graph_name: impl Into<NamedOrBlankNode>) {
let mut this = self;
let graph_name = this
.encode_named_or_blank_node(graph_name.into().as_ref())
.unwrap_infallible();
this.insert_encoded_named_graph(graph_name)
.unwrap_infallible()
}
/// Clears a graph from this store.
///
/// Usage example:
/// ```
@ -468,20 +536,48 @@ impl MemoryStore {
/// store.insert(quad.clone());
/// assert_eq!(1, store.len());
///
/// store.drop_graph(&ex);
/// store.clear_graph(&ex);
/// assert_eq!(0, store.len());
/// assert_eq!(1, store.named_graphs().count());
/// # Result::<_,Box<dyn std::error::Error>>::Ok(())
/// ```
pub fn drop_graph<'a>(&self, graph_name: impl Into<GraphNameRef<'a>>) {
pub fn clear_graph<'a>(&self, graph_name: impl Into<GraphNameRef<'a>>) {
if let Some(graph_name) = self
.get_encoded_graph_name(graph_name.into())
.unwrap_infallible()
{
for quad in self.encoded_quads_for_pattern_inner(None, None, None, Some(graph_name)) {
let mut this = self;
this.remove_encoded(&quad).unwrap_infallible();
this.clear_encoded_graph(graph_name).unwrap_infallible()
}
}
/// Removes a graph from this store.
///
/// Usage example:
/// ```
/// use oxigraph::MemoryStore;
/// use oxigraph::model::{NamedNode, Quad};
///
/// let ex = NamedNode::new("http://example.com")?;
/// let quad = Quad::new(ex.clone(), ex.clone(), ex.clone(), ex.clone());
/// let store = MemoryStore::new();
/// store.insert(quad.clone());
/// assert_eq!(1, store.len());
///
/// store.remove_named_graph(&ex);
/// assert_eq!(0, store.len());
/// assert_eq!(0, store.named_graphs().count());
/// # Result::<_,Box<dyn std::error::Error>>::Ok(())
/// ```
pub fn remove_named_graph<'a>(&self, graph_name: impl Into<NamedOrBlankNodeRef<'a>>) {
if let Some(graph_name) = self
.get_encoded_named_or_blank_node(graph_name.into())
.unwrap_infallible()
{
let mut this = self;
this.remove_encoded_named_graph(graph_name)
.unwrap_infallible()
}
}
/// Clears the store.
@ -502,7 +598,7 @@ impl MemoryStore {
/// # Result::<_,Box<dyn std::error::Error>>::Ok(())
/// ```
pub fn clear(&self) {
*self.indexes_mut() = MemoryStoreIndexes::default();
self.indexes_mut().clear().unwrap_infallible()
}
#[allow(clippy::expect_used)]
@ -891,6 +987,7 @@ impl<'a> StrContainer for &'a MemoryStore {
impl<'a> ReadableEncodedStore for MemoryStore {
type QuadsIter = EncodedQuadsIter;
type GraphsIter = EncodedGraphsIter;
fn encoded_quads_for_pattern(
&self,
@ -905,6 +1002,22 @@ impl<'a> ReadableEncodedStore for MemoryStore {
.into_iter(),
}
}
fn encoded_named_graphs(&self) -> Self::GraphsIter {
EncodedGraphsIter {
iter: self
.indexes()
.gosp
.keys()
.cloned()
.collect::<Vec<_>>()
.into_iter(),
}
}
fn contains_encoded_named_graph(&self, graph_name: EncodedTerm) -> Result<bool, Infallible> {
Ok(self.indexes().gspo.contains_key(&graph_name))
}
}
impl<'a> WritableEncodedStore for &'a MemoryStore {
@ -915,6 +1028,22 @@ impl<'a> WritableEncodedStore for &'a MemoryStore {
fn remove_encoded(&mut self, quad: &EncodedQuad) -> Result<(), Infallible> {
self.indexes_mut().remove_encoded(quad)
}
fn insert_encoded_named_graph(&mut self, graph_name: EncodedTerm) -> Result<(), Infallible> {
self.indexes_mut().insert_encoded_named_graph(graph_name)
}
fn clear_encoded_graph(&mut self, graph_name: EncodedTerm) -> Result<(), Infallible> {
self.indexes_mut().clear_encoded_graph(graph_name)
}
fn remove_encoded_named_graph(&mut self, graph_name: EncodedTerm) -> Result<(), Infallible> {
self.indexes_mut().remove_encoded_named_graph(graph_name)
}
fn clear(&mut self) -> Result<(), Self::Error> {
self.indexes_mut().clear()
}
}
impl StrEncodingAware for MemoryStoreIndexes {
@ -1011,27 +1140,15 @@ impl WritableEncodedStore for MemoryStoreIndexes {
&quad.predicate,
);
} else {
remove_from_quad_map(
&mut self.gspo,
&quad.graph_name,
&quad.subject,
&quad.predicate,
&quad.object,
);
remove_from_quad_map(
&mut self.gpos,
&quad.graph_name,
&quad.predicate,
&quad.object,
&quad.subject,
);
remove_from_quad_map(
&mut self.gosp,
&quad.graph_name,
&quad.object,
&quad.subject,
&quad.predicate,
);
if let Some(spo) = self.gspo.get_mut(&quad.graph_name) {
remove_from_triple_map(spo, &quad.subject, &quad.predicate, &quad.object);
}
if let Some(pos) = self.gpos.get_mut(&quad.graph_name) {
remove_from_triple_map(pos, &quad.predicate, &quad.object, &quad.subject);
}
if let Some(osp) = self.gosp.get_mut(&quad.graph_name) {
remove_from_triple_map(osp, &quad.object, &quad.subject, &quad.predicate);
}
remove_from_quad_map(
&mut self.spog,
&quad.subject,
@ -1056,6 +1173,66 @@ impl WritableEncodedStore for MemoryStoreIndexes {
}
Ok(())
}
fn insert_encoded_named_graph(&mut self, graph_name: EncodedTerm) -> Result<(), Infallible> {
self.gspo.entry(graph_name).or_default();
self.gpos.entry(graph_name).or_default();
self.gosp.entry(graph_name).or_default();
Ok(())
}
fn clear_encoded_graph(&mut self, graph_name: EncodedTerm) -> Result<(), Infallible> {
if graph_name.is_default_graph() {
self.default_spo.clear();
self.default_pos.clear();
self.default_osp.clear();
} else {
if let Some(spo) = self.gspo.get(&graph_name) {
for (s, po) in spo {
for (p, os) in po {
for o in os {
remove_from_quad_map(&mut self.spog, s, p, o, &graph_name);
remove_from_quad_map(&mut self.posg, p, o, s, &graph_name);
remove_from_quad_map(&mut self.ospg, o, s, p, &graph_name);
}
}
}
}
if let Some(spo) = self.gspo.get_mut(&graph_name) {
spo.clear();
}
if let Some(pos) = self.gpos.get_mut(&graph_name) {
pos.clear();
}
if let Some(osp) = self.gosp.get_mut(&graph_name) {
osp.clear();
}
}
Ok(())
}
fn remove_encoded_named_graph(&mut self, graph_name: EncodedTerm) -> Result<(), Infallible> {
if let Some(spo) = self.gspo.get(&graph_name) {
for (s, po) in spo {
for (p, os) in po {
for o in os {
remove_from_quad_map(&mut self.spog, s, p, o, &graph_name);
remove_from_quad_map(&mut self.posg, p, o, s, &graph_name);
remove_from_quad_map(&mut self.ospg, o, s, p, &graph_name);
}
}
}
}
self.gspo.remove(&graph_name);
self.gpos.remove(&graph_name);
self.gosp.remove(&graph_name);
Ok(())
}
fn clear(&mut self) -> Result<(), Infallible> {
*self = MemoryStoreIndexes::default();
Ok(())
}
}
fn insert_into_triple_map<T: Eq + Hash>(map: &mut TripleMap<T>, e1: T, e2: T, e3: T) {
@ -1337,6 +1514,29 @@ impl Iterator for EncodedQuadsIter {
}
}
pub(crate) struct EncodedGraphsIter {
iter: IntoIter<EncodedTerm>,
}
impl Iterator for EncodedGraphsIter {
type Item = Result<EncodedTerm, Infallible>;
fn next(&mut self) -> Option<Result<EncodedTerm, Infallible>> {
self.iter.next().map(Ok)
}
fn size_hint(&self) -> (usize, Option<usize>) {
self.iter.size_hint()
}
fn fold<Acc, G>(self, init: Acc, mut g: G) -> Acc
where
G: FnMut(Acc, Self::Item) -> Acc,
{
self.iter.fold(init, |acc, elt| g(acc, Ok(elt)))
}
}
/// An iterator returning the quads contained in a [`MemoryStore`].
pub struct MemoryQuadIter {
iter: IntoIter<EncodedQuad>,
@ -1355,6 +1555,28 @@ impl Iterator for MemoryQuadIter {
}
}
/// An iterator returning the graph names contained in a [`MemoryStore`].
pub struct MemoryGraphNameIter {
iter: EncodedGraphsIter,
store: MemoryStore,
}
impl Iterator for MemoryGraphNameIter {
type Item = NamedOrBlankNode;
fn next(&mut self) -> Option<NamedOrBlankNode> {
Some(
self.store
.decode_named_or_blank_node(self.iter.next()?.unwrap_infallible())
.unwrap(),
)
}
fn size_hint(&self) -> (usize, Option<usize>) {
self.iter.size_hint()
}
}
impl StrId for LargeSpur {}
// Isomorphism implementation

@ -34,6 +34,7 @@ use std::iter::Iterator;
pub(crate) trait ReadableEncodedStore: StrLookup {
type QuadsIter: Iterator<Item = Result<EncodedQuad<Self::StrId>, Self::Error>> + 'static;
type GraphsIter: Iterator<Item = Result<EncodedTerm<Self::StrId>, Self::Error>> + 'static;
fn encoded_quads_for_pattern(
&self,
@ -42,12 +43,36 @@ pub(crate) trait ReadableEncodedStore: StrLookup {
object: Option<EncodedTerm<Self::StrId>>,
graph_name: Option<EncodedTerm<Self::StrId>>,
) -> Self::QuadsIter;
fn encoded_named_graphs(&self) -> Self::GraphsIter;
fn contains_encoded_named_graph(
&self,
graph_name: EncodedTerm<Self::StrId>,
) -> Result<bool, Self::Error>;
}
pub(crate) trait WritableEncodedStore: StrEncodingAware {
fn insert_encoded(&mut self, quad: &EncodedQuad<Self::StrId>) -> Result<(), Self::Error>;
fn remove_encoded(&mut self, quad: &EncodedQuad<Self::StrId>) -> Result<(), Self::Error>;
fn insert_encoded_named_graph(
&mut self,
graph_name: EncodedTerm<Self::StrId>,
) -> Result<(), Self::Error>;
fn clear_encoded_graph(
&mut self,
graph_name: EncodedTerm<Self::StrId>,
) -> Result<(), Self::Error>;
fn remove_encoded_named_graph(
&mut self,
graph_name: EncodedTerm<Self::StrId>,
) -> Result<(), Self::Error>;
fn clear(&mut self) -> Result<(), Self::Error>;
}
pub(crate) fn load_graph<S: WritableEncodedStore + StrContainer>(

@ -77,9 +77,11 @@ const GOSP_CF: &str = "gosp";
const DSPO_CF: &str = "dspo";
const DPOS_CF: &str = "dpos";
const DOSP_CF: &str = "dosp";
const GRAPHS_CF: &str = "graphs";
const COLUMN_FAMILIES: [&str; 10] = [
const COLUMN_FAMILIES: [&str; 11] = [
ID2STR_CF, SPOG_CF, POSG_CF, OSPG_CF, GSPO_CF, GPOS_CF, GOSP_CF, DSPO_CF, DPOS_CF, DOSP_CF,
GRAPHS_CF,
];
const MAX_TRANSACTION_SIZE: usize = 1024;
@ -96,15 +98,33 @@ impl RocksDbStore {
db: Arc::new(DB::open_cf(&options, path, &COLUMN_FAMILIES).map_err(map_err)?),
};
let version = this.ensure_version()?;
if version != LATEST_STORAGE_VERSION {
return Err(invalid_data_error(format!(
"The RocksDB database is still using the encoding version {}, please upgrade it",
version
)));
let mut version = this.ensure_version()?;
if version == 0 {
// We migrate to v1
let mut transaction = this.auto_batch_writer();
for quad in this.encoded_quads_for_pattern(None, None, None, None) {
let quad = quad?;
if !quad.graph_name.is_default_graph() {
transaction.insert_encoded_named_graph(quad.graph_name)?;
}
}
transaction.apply()?;
version = 1;
this.set_version(version)?;
this.flush()?;
}
Ok(this)
match version {
_ if version < LATEST_STORAGE_VERSION => Err(invalid_data_error(format!(
"The RocksDB database is using the outdated encoding version {}. Automated migration is not supported, please dump the store dataset using a compatible Oxigraph version and load it again using the current version",
version
))),
LATEST_STORAGE_VERSION => Ok(this),
_ => Err(invalid_data_error(format!(
"The RocksDB database is using the too recent version {}. Upgrade to the latest Oxigraph version to load this database",
version
)))
}
}
fn ensure_version(&self) -> Result<u64, io::Error> {
@ -114,14 +134,25 @@ impl RocksDbStore {
buffer.copy_from_slice(&version);
u64::from_be_bytes(buffer)
} else {
self.db
.put("oxversion", &LATEST_STORAGE_VERSION.to_be_bytes())
.map_err(map_err)?;
self.set_version(LATEST_STORAGE_VERSION)?;
LATEST_STORAGE_VERSION
},
)
}
fn set_version(&self, version: u64) -> Result<(), io::Error> {
self.db
.put("oxversion", &version.to_be_bytes())
.map_err(map_err)
}
fn flush(&self) -> Result<(), io::Error> {
let mut options = FlushOptions::new();
options.set_wait(true);
self.db.flush_opt(&options).map_err(map_err)?;
Ok(())
}
/// Executes a [SPARQL 1.1 query](https://www.w3.org/TR/sparql11-query/).
///
/// See [`MemoryStore`](super::memory::MemoryStore::query()) for a usage example.
@ -352,15 +383,68 @@ impl RocksDbStore {
dump_dataset(self.iter(), writer, syntax)
}
/// Removes a graph from this store.
/// Returns all the store named graphs
///
/// See [`MemoryStore`](super::memory::MemoryStore::named_graphs()) for a usage example.
pub fn named_graphs(&self) -> impl Iterator<Item = Result<NamedOrBlankNode, io::Error>> {
let this = self.clone();
self.encoded_named_graphs()
.map(move |g| Ok(this.decode_named_or_blank_node(g?)?))
}
/// Checks if the store contains a given graph
///
/// See [`MemoryStore`](super::memory::MemoryStore::drop_graph()) for a usage example.
pub fn drop_graph<'a>(&self, graph_name: impl Into<GraphNameRef<'a>>) -> Result<(), io::Error> {
/// See [`MemoryStore`](super::memory::MemoryStore::contains_named_graph()) for a usage example.
pub fn contains_named_graph<'a>(
&self,
graph_name: impl Into<NamedOrBlankNodeRef<'a>>,
) -> Result<bool, io::Error> {
if let Some(graph_name) = self.get_encoded_named_or_blank_node(graph_name.into())? {
self.contains_encoded_named_graph(graph_name)
} else {
Ok(false)
}
}
/// Inserts a graph into this store
///
/// See [`MemoryStore`](super::memory::MemoryStore::insert_named_graph()) for a usage example.
pub fn insert_named_graph<'a>(
&self,
graph_name: impl Into<NamedOrBlankNodeRef<'a>>,
) -> Result<(), io::Error> {
let mut transaction = self.auto_batch_writer();
let graph_name = transaction.encode_named_or_blank_node(graph_name.into())?;
transaction.insert_encoded_named_graph(graph_name)?;
transaction.apply()
}
/// Clears a graph from this store.
///
/// See [`MemoryStore`](super::memory::MemoryStore::clear_graph()) for a usage example.
pub fn clear_graph<'a>(
&self,
graph_name: impl Into<GraphNameRef<'a>>,
) -> Result<(), io::Error> {
if let Some(graph_name) = self.get_encoded_graph_name(graph_name.into())? {
let mut transaction = self.auto_batch_writer();
for quad in self.encoded_quads_for_pattern(None, None, None, Some(graph_name)) {
transaction.remove_encoded(&quad?)?;
transaction.clear_encoded_graph(graph_name)?;
transaction.apply()
} else {
Ok(())
}
}
/// Removes a graph from this store.
///
/// See [`MemoryStore`](super::memory::MemoryStore::remove_named_graph()) for a usage example.
pub fn remove_named_graph<'a>(
&self,
graph_name: impl Into<NamedOrBlankNodeRef<'a>>,
) -> Result<(), io::Error> {
if let Some(graph_name) = self.get_encoded_named_or_blank_node(graph_name.into())? {
let mut transaction = self.auto_batch_writer();
transaction.remove_encoded_named_graph(graph_name)?;
transaction.apply()
} else {
Ok(())
@ -371,17 +455,9 @@ impl RocksDbStore {
///
/// See [`MemoryStore`](super::memory::MemoryStore::clear()) for a usage example.
pub fn clear(&self) -> Result<(), io::Error> {
self.clear_cf(self.id2str_cf())?;
self.clear_cf(self.spog_cf())?;
self.clear_cf(self.posg_cf())?;
self.clear_cf(self.ospg_cf())?;
self.clear_cf(self.gspo_cf())?;
self.clear_cf(self.gpos_cf())?;
self.clear_cf(self.gosp_cf())?;
self.clear_cf(self.dspo_cf())?;
self.clear_cf(self.dpos_cf())?;
self.clear_cf(self.dosp_cf())?;
Ok(())
let mut transaction = self.auto_batch_writer();
transaction.clear()?;
transaction.apply()
}
fn id2str_cf(&self) -> &ColumnFamily {
@ -424,6 +500,9 @@ impl RocksDbStore {
get_cf(&self.db, DOSP_CF)
}
fn graphs_cf(&self) -> &ColumnFamily {
get_cf(&self.db, GRAPHS_CF)
}
fn auto_batch_writer(&self) -> AutoBatchWriter<'_> {
AutoBatchWriter {
store: self,
@ -657,48 +736,25 @@ impl RocksDbStore {
self.inner_quads(self.dosp_cf(), prefix, QuadEncoding::DOSP)
}
#[allow(unsafe_code)]
fn inner_quads(
&self,
cf: &ColumnFamily,
prefix: Vec<u8>,
encoding: QuadEncoding,
) -> DecodingIndexIterator {
let mut iter = self.db.raw_iterator_cf(cf);
iter.seek(&prefix);
let mut iter = self.db_iter(cf);
iter.iter.seek(&prefix);
DecodingIndexIterator {
iter: unsafe { StaticDBRowIterator::new(iter, self.db.clone()) }, // This is safe because the iterator belongs to DB
iter,
prefix,
encoding,
}
}
fn clear_cf(&self, cf: &ColumnFamily) -> Result<(), io::Error> {
self.db
.delete_range_cf(
cf,
[
u8::MIN,
u8::MIN,
u8::MIN,
u8::MIN,
u8::MIN,
u8::MIN,
u8::MIN,
u8::MIN,
],
[
u8::MAX,
u8::MAX,
u8::MAX,
u8::MAX,
u8::MAX,
u8::MAX,
u8::MAX,
u8::MAX,
],
)
.map_err(map_err)
#[allow(unsafe_code)]
fn db_iter(&self, cf: &ColumnFamily) -> StaticDBRowIterator {
// Valid because it's the same database so db can't be dropped before iter
unsafe { StaticDBRowIterator::new(self.db.raw_iterator_cf(cf), self.db.clone()) }
}
}
@ -745,6 +801,7 @@ impl StrLookup for RocksDbStore {
impl ReadableEncodedStore for RocksDbStore {
type QuadsIter = DecodingIndexesIterator;
type GraphsIter = DecodingGraphIterator;
fn encoded_quads_for_pattern(
&self,
@ -809,6 +866,20 @@ impl ReadableEncodedStore for RocksDbStore {
},
}
}
fn encoded_named_graphs(&self) -> DecodingGraphIterator {
let mut iter = self.db_iter(self.graphs_cf());
iter.iter.seek_to_first();
DecodingGraphIterator { iter }
}
fn contains_encoded_named_graph(&self, graph_name: EncodedTerm) -> Result<bool, io::Error> {
Ok(self
.db
.get_cf(self.graphs_cf(), &encode_term(graph_name))
.map_err(map_err)?
.is_some())
}
}
struct AutoBatchWriter<'a> {
@ -831,6 +902,32 @@ impl AutoBatchWriter<'_> {
}
Ok(())
}
fn clear_cf(&mut self, cf: &ColumnFamily) {
self.batch.delete_range_cf(
cf,
[
u8::MIN,
u8::MIN,
u8::MIN,
u8::MIN,
u8::MIN,
u8::MIN,
u8::MIN,
u8::MIN,
],
[
u8::MAX,
u8::MAX,
u8::MAX,
u8::MAX,
u8::MAX,
u8::MAX,
u8::MAX,
u8::MAX,
],
)
}
}
impl StrEncodingAware for AutoBatchWriter<'_> {
@ -885,6 +982,10 @@ impl WritableEncodedStore for AutoBatchWriter<'_> {
write_gosp_quad(&mut self.buffer, quad);
self.batch.put_cf(self.store.gosp_cf(), &self.buffer, &[]);
self.buffer.clear();
write_term(&mut self.buffer, quad.graph_name);
self.batch.put_cf(self.store.graphs_cf(), &self.buffer, &[]);
self.buffer.clear();
}
self.apply_if_big()
@ -931,6 +1032,49 @@ impl WritableEncodedStore for AutoBatchWriter<'_> {
self.apply_if_big()
}
fn insert_encoded_named_graph(&mut self, graph_name: EncodedTerm) -> Result<(), io::Error> {
self.batch
.put_cf(self.store.graphs_cf(), &encode_term(graph_name), &[]);
self.apply_if_big()
}
fn clear_encoded_graph(&mut self, graph_name: EncodedTerm) -> Result<(), io::Error> {
if graph_name.is_default_graph() {
self.clear_cf(self.store.dspo_cf());
self.clear_cf(self.store.dpos_cf());
self.clear_cf(self.store.dosp_cf());
} else {
for quad in self.store.quads_for_graph(graph_name) {
self.remove_encoded(&quad?)?;
}
}
self.apply_if_big()
}
fn remove_encoded_named_graph(&mut self, graph_name: EncodedTerm) -> Result<(), io::Error> {
for quad in self.store.quads_for_graph(graph_name) {
self.remove_encoded(&quad?)?;
}
self.batch
.delete_cf(self.store.graphs_cf(), &encode_term(graph_name));
self.apply_if_big()
}
fn clear(&mut self) -> Result<(), io::Error> {
self.clear_cf(self.store.spog_cf());
self.clear_cf(self.store.posg_cf());
self.clear_cf(self.store.ospg_cf());
self.clear_cf(self.store.gspo_cf());
self.clear_cf(self.store.gpos_cf());
self.clear_cf(self.store.gosp_cf());
self.clear_cf(self.store.dspo_cf());
self.clear_cf(self.store.dpos_cf());
self.clear_cf(self.store.dosp_cf());
self.clear_cf(self.store.graphs_cf());
self.clear_cf(self.store.id2str_cf());
self.apply_if_big()
}
}
/// Allows inserting and deleting quads during an ACID transaction with the [`RocksDbStore`].
@ -1093,6 +1237,10 @@ impl WritableEncodedStore for RocksDbTransaction<'_> {
write_gosp_quad(&mut self.buffer, quad);
self.batch.put_cf(self.store.gosp_cf(), &self.buffer, &[]);
self.buffer.clear();
write_term(&mut self.buffer, quad.graph_name);
self.batch.put_cf(self.store.graphs_cf(), &self.buffer, &[]);
self.buffer.clear();
}
Ok(())
@ -1139,6 +1287,33 @@ impl WritableEncodedStore for RocksDbTransaction<'_> {
Ok(())
}
fn insert_encoded_named_graph(&mut self, graph_name: EncodedTerm) -> Result<(), io::Error> {
self.batch
.put_cf(self.store.graphs_cf(), &encode_term(graph_name), &[]);
Ok(())
}
fn clear_encoded_graph(&mut self, _: EncodedTerm) -> Result<(), io::Error> {
Err(io::Error::new(
io::ErrorKind::Other,
"CLEAR is not implemented in RocksDB transactions",
))
}
fn remove_encoded_named_graph(&mut self, _: EncodedTerm) -> Result<(), io::Error> {
Err(io::Error::new(
io::ErrorKind::Other,
"DROP is not implemented in RocksDB transactions",
))
}
fn clear(&mut self) -> Result<(), Self::Error> {
Err(io::Error::new(
io::ErrorKind::Other,
"CLEAR ALL is not implemented in RocksDB transactions",
))
}
}
#[allow(clippy::expect_used)]
@ -1267,6 +1442,23 @@ impl Iterator for RocksDbQuadIter {
}
}
pub(crate) struct DecodingGraphIterator {
iter: StaticDBRowIterator,
}
impl Iterator for DecodingGraphIterator {
type Item = Result<EncodedTerm, io::Error>;
fn next(&mut self) -> Option<Result<EncodedTerm, io::Error>> {
if let Some(key) = self.iter.key() {
let result = decode_term(key);
self.iter.next();
Some(result)
} else {
None
}
}
}
#[test]
fn store() -> Result<(), io::Error> {
use crate::model::*;

@ -75,6 +75,7 @@ pub struct SledStore {
dspo: Tree,
dpos: Tree,
dosp: Tree,
graphs: Tree,
}
type EncodedTerm = crate::store::numeric_encoder::EncodedTerm<StrHash>;
@ -107,17 +108,35 @@ impl SledStore {
dspo: db.open_tree("dspo")?,
dpos: db.open_tree("dpos")?,
dosp: db.open_tree("dosp")?,
graphs: db.open_tree("graphs")?,
};
let version = this.ensure_version()?;
if version != LATEST_STORAGE_VERSION {
return Err(invalid_data_error(format!(
"The Sled database is still using the encoding version {}, please upgrade it",
version
)));
let mut version = this.ensure_version()?;
if version == 0 {
// We migrate to v1
for quad in this.encoded_quads_for_pattern(None, None, None, None) {
let mut this_mut = &this;
let quad = quad?;
if !quad.graph_name.is_default_graph() {
this_mut.insert_encoded_named_graph(quad.graph_name)?;
}
}
version = 1;
this.set_version(version)?;
this.graphs.flush()?;
}
Ok(this)
match version {
_ if version < LATEST_STORAGE_VERSION => Err(invalid_data_error(format!(
"The Sled database is using the outdated encoding version {}. Automated migration is not supported, please dump the store dataset using a compatible Oxigraph version and load it again using the current version",
version
))),
LATEST_STORAGE_VERSION => Ok(this),
_ => Err(invalid_data_error(format!(
"The Sled database is using the too recent version {}. Upgrade to the latest Oxigraph version to load this database",
version
)))
}
}
fn ensure_version(&self) -> Result<u64, io::Error> {
@ -126,12 +145,16 @@ impl SledStore {
buffer.copy_from_slice(&version);
u64::from_be_bytes(buffer)
} else {
self.default
.insert("oxversion", &LATEST_STORAGE_VERSION.to_be_bytes())?;
self.set_version(LATEST_STORAGE_VERSION)?;
LATEST_STORAGE_VERSION
})
}
fn set_version(&self, version: u64) -> Result<(), io::Error> {
self.default.insert("oxversion", &version.to_be_bytes())?;
Ok(())
}
/// Executes a [SPARQL 1.1 query](https://www.w3.org/TR/sparql11-query/).
///
/// See [`MemoryStore`](super::memory::MemoryStore::query()) for a usage example.
@ -268,9 +291,10 @@ impl SledStore {
&self.dspo,
&self.dpos,
&self.dosp,
&self.graphs,
)
.transaction(
move |(id2str, spog, posg, ospg, gspo, gpos, gosp, dspo, dpos, dosp)| {
move |(id2str, spog, posg, ospg, gspo, gpos, gosp, dspo, dpos, dosp, graphs)| {
Ok(f(SledTransaction {
id2str,
spog,
@ -282,6 +306,7 @@ impl SledStore {
dspo,
dpos,
dosp,
graphs,
})?)
},
)?)
@ -385,34 +410,78 @@ impl SledStore {
dump_dataset(self.iter(), writer, format)
}
/// Removes a graph from this store.
/// Returns all the store named graphs
///
/// See [`MemoryStore`](super::memory::MemoryStore::named_graphs()) for a usage example.
pub fn named_graphs(&self) -> SledGraphNameIter {
SledGraphNameIter {
iter: self.encoded_named_graphs(),
store: self.clone(),
}
}
/// Checks if the store contains a given graph
///
/// See [`MemoryStore`](super::memory::MemoryStore::drop_graph()) for a usage example.
pub fn drop_graph<'a>(&self, graph_name: impl Into<GraphNameRef<'a>>) -> Result<(), io::Error> {
/// See [`MemoryStore`](super::memory::MemoryStore::contains_named_graph()) for a usage example.
pub fn contains_named_graph<'a>(
&self,
graph_name: impl Into<NamedOrBlankNodeRef<'a>>,
) -> Result<bool, io::Error> {
if let Some(graph_name) = self.get_encoded_named_or_blank_node(graph_name.into())? {
self.contains_encoded_named_graph(graph_name)
} else {
Ok(false)
}
}
/// Inserts a graph into this store
///
/// See [`MemoryStore`](super::memory::MemoryStore::insert_named_graph()) for a usage example.
pub fn insert_named_graph<'a>(
&self,
graph_name: impl Into<NamedOrBlankNodeRef<'a>>,
) -> Result<(), io::Error> {
let mut this = self;
let graph_name = this.encode_named_or_blank_node(graph_name.into())?;
this.insert_encoded_named_graph(graph_name)
}
/// Clears a graph from this store.
///
/// See [`MemoryStore`](super::memory::MemoryStore::clear_graph()) for a usage example.
pub fn clear_graph<'a>(
&self,
graph_name: impl Into<GraphNameRef<'a>>,
) -> Result<(), io::Error> {
if let Some(graph_name) = self.get_encoded_graph_name(graph_name.into())? {
for quad in self.encoded_quads_for_pattern(None, None, None, Some(graph_name)) {
let mut this = self;
this.remove_encoded(&quad?)?;
this.clear_encoded_graph(graph_name)
} else {
Ok(())
}
}
/// Removes a graph from this store.
///
/// See [`MemoryStore`](super::memory::MemoryStore::remove_named_graph()) for a usage example.
pub fn remove_named_graph<'a>(
&self,
graph_name: impl Into<NamedOrBlankNodeRef<'a>>,
) -> Result<(), io::Error> {
if let Some(graph_name) = self.get_encoded_named_or_blank_node(graph_name.into())? {
let mut this = self;
this.remove_encoded_named_graph(graph_name)
} else {
Ok(())
}
}
/// Clears the store.
///
/// See [`MemoryStore`](super::memory::MemoryStore::clear()) for a usage example.
pub fn clear(&self) -> Result<(), io::Error> {
self.dspo.clear()?;
self.dpos.clear()?;
self.dosp.clear()?;
self.gspo.clear()?;
self.gpos.clear()?;
self.gosp.clear()?;
self.spog.clear()?;
self.posg.clear()?;
self.ospg.clear()?;
self.id2str.clear()?;
Ok(())
let mut this = self;
(&mut this).clear()
}
fn contains_encoded(&self, quad: &EncodedQuad) -> Result<bool, io::Error> {
@ -679,6 +748,7 @@ impl StrLookup for SledStore {
impl ReadableEncodedStore for SledStore {
type QuadsIter = DecodingQuadsIterator;
type GraphsIter = DecodingGraphIterator;
fn encoded_quads_for_pattern(
&self,
@ -742,6 +812,16 @@ impl ReadableEncodedStore for SledStore {
},
}
}
fn encoded_named_graphs(&self) -> DecodingGraphIterator {
DecodingGraphIterator {
iter: self.graphs.iter(),
}
}
fn contains_encoded_named_graph(&self, graph_name: EncodedTerm) -> Result<bool, io::Error> {
Ok(self.graphs.contains_key(&encode_term(graph_name))?)
}
}
impl<'a> StrContainer for &'a SledStore {
@ -792,6 +872,10 @@ impl<'a> WritableEncodedStore for &'a SledStore {
write_gosp_quad(&mut buffer, quad);
self.gosp.insert(buffer.as_slice(), &[])?;
buffer.clear();
write_term(&mut buffer, quad.graph_name);
self.graphs.insert(&buffer, &[])?;
buffer.clear();
}
Ok(())
@ -840,6 +924,47 @@ impl<'a> WritableEncodedStore for &'a SledStore {
Ok(())
}
fn insert_encoded_named_graph(&mut self, graph_name: EncodedTerm) -> Result<(), io::Error> {
self.graphs.insert(&encode_term(graph_name), &[])?;
Ok(())
}
fn clear_encoded_graph(&mut self, graph_name: EncodedTerm) -> Result<(), io::Error> {
if graph_name.is_default_graph() {
self.dspo.clear()?;
self.dpos.clear()?;
self.dosp.clear()?;
} else {
for quad in self.quads_for_graph(graph_name) {
self.remove_encoded(&quad?)?;
}
}
Ok(())
}
fn remove_encoded_named_graph(&mut self, graph_name: EncodedTerm) -> Result<(), io::Error> {
for quad in self.quads_for_graph(graph_name) {
self.remove_encoded(&quad?)?;
}
self.graphs.remove(&encode_term(graph_name))?;
Ok(())
}
fn clear(&mut self) -> Result<(), io::Error> {
self.dspo.clear()?;
self.dpos.clear()?;
self.dosp.clear()?;
self.gspo.clear()?;
self.gpos.clear()?;
self.gosp.clear()?;
self.spog.clear()?;
self.posg.clear()?;
self.ospg.clear()?;
self.graphs.clear()?;
self.id2str.clear()?;
Ok(())
}
}
/// Allows inserting and deleting quads during an ACID transaction with the [`SledStore`].
@ -854,6 +979,7 @@ pub struct SledTransaction<'a> {
dspo: &'a TransactionalTree,
dpos: &'a TransactionalTree,
dosp: &'a TransactionalTree,
graphs: &'a TransactionalTree,
}
impl SledTransaction<'_> {
@ -1058,6 +1184,41 @@ impl<'a> WritableEncodedStore for &'a SledTransaction<'a> {
Ok(())
}
fn insert_encoded_named_graph(
&mut self,
graph_name: EncodedTerm,
) -> Result<(), SledUnabortableTransactionError> {
self.graphs.insert(encode_term(graph_name), &[])?;
Ok(())
}
fn clear_encoded_graph(
&mut self,
_: EncodedTerm,
) -> Result<(), SledUnabortableTransactionError> {
Err(SledUnabortableTransactionError::Storage(io::Error::new(
io::ErrorKind::Other,
"CLEAR is not implemented in Sled transactions",
)))
}
fn remove_encoded_named_graph(
&mut self,
_: EncodedTerm,
) -> Result<(), SledUnabortableTransactionError> {
Err(SledUnabortableTransactionError::Storage(io::Error::new(
io::ErrorKind::Other,
"DROP is not implemented in Sled transactions",
)))
}
fn clear(&mut self) -> Result<(), SledUnabortableTransactionError> {
Err(SledUnabortableTransactionError::Storage(io::Error::new(
io::ErrorKind::Other,
"CLEAR ALL is not implemented in Sled transactions",
)))
}
}
/// Error returned by a Sled transaction
@ -1263,6 +1424,21 @@ impl Iterator for DecodingQuadIterator {
}
}
pub(crate) struct DecodingGraphIterator {
iter: Iter,
}
impl Iterator for DecodingGraphIterator {
type Item = Result<EncodedTerm, io::Error>;
fn next(&mut self) -> Option<Result<EncodedTerm, io::Error>> {
Some(match self.iter.next()? {
Ok((encoded, _)) => decode_term(&encoded),
Err(error) => Err(error.into()),
})
}
}
/// An iterator returning the quads contained in a [`SledStore`].
pub struct SledQuadIter {
inner: QuadIterInner,
@ -1292,6 +1468,28 @@ impl Iterator for SledQuadIter {
}
}
/// An iterator returning the graph names contained in a [`SledStore`].
pub struct SledGraphNameIter {
iter: DecodingGraphIterator,
store: SledStore,
}
impl Iterator for SledGraphNameIter {
type Item = Result<NamedOrBlankNode, io::Error>;
fn next(&mut self) -> Option<Result<NamedOrBlankNode, io::Error>> {
Some(
self.iter
.next()?
.and_then(|graph_name| Ok(self.store.decode_named_or_blank_node(graph_name)?)),
)
}
fn size_hint(&self) -> (usize, Option<usize>) {
self.iter.size_hint()
}
}
#[test]
fn store() -> Result<(), io::Error> {
use crate::model::*;

@ -1,12 +1,11 @@
#![cfg(features = "rocksdb")]
use oxigraph::model::vocab::{rdf, xsd};
use oxigraph::model::*;
use oxigraph::RocksDbStore;
use std::io;
use std::process::Command;
fn quads(graph_name: GraphNameRef<'static>) -> Vec<QuadRef<'static>> {
fn quads(graph_name: impl Into<GraphNameRef<'static>>) -> Vec<QuadRef<'static>> {
let graph_name = graph_name.into();
let paris = NamedNodeRef::new_unchecked("http://www.wikidata.org/entity/Q90");
let france = NamedNodeRef::new_unchecked("http://www.wikidata.org/entity/Q142");
let city = NamedNodeRef::new_unchecked("http://schema.org/City");
@ -65,18 +64,26 @@ fn test_backward_compatibility() -> io::Result<()> {
for q in quads(GraphNameRef::DefaultGraph) {
assert!(store.contains(q)?);
}
for q in quads(
NamedNodeRef::new_unchecked("http://www.wikidata.org/wiki/Special:EntityData/Q90")
.into(),
) {
let graph_name =
NamedNodeRef::new_unchecked("http://www.wikidata.org/wiki/Special:EntityData/Q90");
for q in quads(graph_name) {
assert!(store.contains(q)?);
}
assert!(store.contains_named_graph(graph_name)?);
assert_eq!(
vec![NamedOrBlankNode::from(graph_name)],
store.named_graphs().collect::<io::Result<Vec<_>>>()?
);
};
reset_dir("tests/rockdb_bc_data")?;
Ok(())
}
fn reset_dir(dir: &str) -> io::Result<()> {
assert!(Command::new("git")
.args(&["clean", "-fX", dir])
.status()?
.success());
assert!(Command::new("git")
.args(&["checkout", "HEAD", "--", dir])
.status()?

@ -1,5 +1,3 @@
#![cfg(features = "sled")]
use oxigraph::io::{DatasetFormat, GraphFormat};
use oxigraph::model::vocab::{rdf, xsd};
use oxigraph::model::*;
@ -24,7 +22,8 @@ wd:Q90 a schema:City ;
"#;
const NUMBER_OF_TRIPLES: usize = 8;
fn quads(graph_name: GraphNameRef<'static>) -> Vec<QuadRef<'static>> {
fn quads(graph_name: impl Into<GraphNameRef<'static>>) -> Vec<QuadRef<'static>> {
let graph_name = graph_name.into();
let paris = NamedNodeRef::new_unchecked("http://www.wikidata.org/entity/Q90");
let france = NamedNodeRef::new_unchecked("http://www.wikidata.org/entity/Q142");
let city = NamedNodeRef::new_unchecked("http://schema.org/City");
@ -161,18 +160,26 @@ fn test_backward_compatibility() -> io::Result<()> {
for q in quads(GraphNameRef::DefaultGraph) {
assert!(store.contains(q)?);
}
for q in quads(
NamedNodeRef::new_unchecked("http://www.wikidata.org/wiki/Special:EntityData/Q90")
.into(),
) {
let graph_name =
NamedNodeRef::new_unchecked("http://www.wikidata.org/wiki/Special:EntityData/Q90");
for q in quads(graph_name) {
assert!(store.contains(q)?);
}
assert!(store.contains_named_graph(graph_name)?);
assert_eq!(
vec![NamedOrBlankNode::from(graph_name)],
store.named_graphs().collect::<io::Result<Vec<_>>>()?
);
};
reset_dir("tests/sled_bc_data")?;
Ok(())
}
fn reset_dir(dir: &str) -> io::Result<()> {
assert!(Command::new("git")
.args(&["clean", "-fX", dir])
.status()?
.success());
assert!(Command::new("git")
.args(&["checkout", "HEAD", "--", dir])
.status()?

@ -3,6 +3,7 @@ use crate::model::*;
use crate::sparql::*;
use crate::store_utils::*;
use oxigraph::io::{DatasetFormat, GraphFormat};
use oxigraph::model::GraphNameRef;
use oxigraph::store::memory::*;
use pyo3::basic::CompareOp;
use pyo3::exceptions::{PyNotImplementedError, PyValueError};
@ -333,6 +334,63 @@ impl PyMemoryStore {
)))
}
}
/// Returns an iterator over all the store named graphs
///
/// >>> store = MemoryStore()
/// >>> store.add(Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'), NamedNode('http://example.com/g')))
/// >>> list(store.named_graphs())
/// [<NamedNode value=http://example.com/g>]
#[text_signature = "($self)"]
fn named_graphs(&self) -> GraphNameIter {
GraphNameIter {
inner: self.inner.named_graphs(),
}
}
/// Adds a named graph to the store
///
/// :param graph_name: the quad to add
/// :type graph_name: NamedNode or BlankNode
///
/// >>> store = MemoryStore()
/// >>> store.add_graph(NamedNode('http://example.com/g'))
/// >>> list(store.named_graphs())
/// [<NamedNode value=http://example.com/g>]
#[text_signature = "($self, graph_name)"]
fn add_graph(&self, graph_name: PyGraphName) {
match graph_name {
PyGraphName::DefaultGraph(_) => (),
PyGraphName::NamedNode(graph_name) => self.inner.insert_named_graph(graph_name),
PyGraphName::BlankNode(graph_name) => self.inner.insert_named_graph(graph_name),
}
}
/// Removes a graph from the store
///
/// The default graph will not be remove but just cleared.
///
/// :param graph_name: the quad to add
/// :type graph_name: NamedNode or BlankNode or DefaultGraph
///
/// >>> store = MemoryStore()
/// >>> quad = Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'), NamedNode('http://example.com/g'))
/// >>> store.remove_graph(NamedNode('http://example.com/g'))
/// >>> list(store)
/// []
#[text_signature = "($self, graph_name)"]
fn remove_graph(&self, graph_name: &PyAny) -> PyResult<()> {
match PyGraphNameRef::try_from(graph_name)? {
PyGraphNameRef::DefaultGraph => self.inner.clear_graph(GraphNameRef::DefaultGraph),
PyGraphNameRef::NamedNode(graph_name) => self
.inner
.remove_named_graph(&PyNamedOrBlankNodeRef::NamedNode(graph_name)),
PyGraphNameRef::BlankNode(graph_name) => self
.inner
.remove_named_graph(&PyNamedOrBlankNodeRef::BlankNode(graph_name)),
}
Ok(())
}
}
#[pyproto]
@ -392,3 +450,19 @@ impl PyIterProtocol for QuadIter {
slf.inner.next().map(|q| q.into())
}
}
#[pyclass(unsendable, module = "oxigraph")]
pub struct GraphNameIter {
inner: MemoryGraphNameIter,
}
#[pyproto]
impl PyIterProtocol for GraphNameIter {
fn __iter__(slf: PyRefMut<Self>) -> Py<Self> {
slf.into()
}
fn __next__(mut slf: PyRefMut<Self>) -> Option<PyNamedOrBlankNode> {
slf.inner.next().map(|q| q.into())
}
}

@ -3,6 +3,7 @@ use crate::model::*;
use crate::sparql::*;
use crate::store_utils::*;
use oxigraph::io::{DatasetFormat, GraphFormat};
use oxigraph::model::GraphNameRef;
use oxigraph::store::sled::*;
use pyo3::exceptions::PyValueError;
use pyo3::prelude::{
@ -351,6 +352,68 @@ impl PySledStore {
)))
}
}
/// Returns an iterator over all the store named graphs
///
/// >>> store = MemoryStore()
/// >>> store.add(Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'), NamedNode('http://example.com/g')))
/// >>> list(store.named_graphs())
/// [<NamedNode value=http://example.com/g>]
#[text_signature = "($self)"]
fn named_graphs(&self) -> GraphNameIter {
GraphNameIter {
inner: self.inner.named_graphs(),
}
}
/// Adds a named graph to the store
///
/// :param graph_name: the quad to add
/// :type graph_name: NamedNode or BlankNode
///
/// >>> store = MemoryStore()
/// >>> store.add_graph(NamedNode('http://example.com/g'))
/// >>> list(store.named_graphs())
/// [<NamedNode value=http://example.com/g>]
#[text_signature = "($self, graph_name)"]
fn add_graph(&self, graph_name: &PyAny) -> PyResult<()> {
match PyGraphNameRef::try_from(graph_name)? {
PyGraphNameRef::DefaultGraph => Ok(()),
PyGraphNameRef::NamedNode(graph_name) => self
.inner
.insert_named_graph(&PyNamedOrBlankNodeRef::NamedNode(graph_name)),
PyGraphNameRef::BlankNode(graph_name) => self
.inner
.insert_named_graph(&PyNamedOrBlankNodeRef::BlankNode(graph_name)),
}
.map_err(map_io_err)
}
/// Removes a graph from the store
///
/// The default graph will not be remove but just cleared.
///
/// :param graph_name: the quad to add
/// :type graph_name: NamedNode or BlankNode or DefaultGraph
///
/// >>> store = MemoryStore()
/// >>> quad = Quad(NamedNode('http://example.com'), NamedNode('http://example.com/p'), Literal('1'), NamedNode('http://example.com/g'))
/// >>> store.remove_graph(NamedNode('http://example.com/g'))
/// >>> list(store)
/// []
#[text_signature = "($self, graph_name)"]
fn remove_graph(&self, graph_name: &PyAny) -> PyResult<()> {
match PyGraphNameRef::try_from(graph_name)? {
PyGraphNameRef::DefaultGraph => self.inner.clear_graph(GraphNameRef::DefaultGraph),
PyGraphNameRef::NamedNode(graph_name) => self
.inner
.remove_named_graph(&PyNamedOrBlankNodeRef::NamedNode(graph_name)),
PyGraphNameRef::BlankNode(graph_name) => self
.inner
.remove_named_graph(&PyNamedOrBlankNodeRef::BlankNode(graph_name)),
}
.map_err(map_io_err)
}
}
#[pyproto]
@ -402,3 +465,22 @@ impl PyIterProtocol for QuadIter {
.transpose()
}
}
#[pyclass(unsendable, module = "oxigraph")]
pub struct GraphNameIter {
inner: SledGraphNameIter,
}
#[pyproto]
impl PyIterProtocol for GraphNameIter {
fn __iter__(slf: PyRefMut<Self>) -> Py<Self> {
slf.into()
}
fn __next__(mut slf: PyRefMut<Self>) -> PyResult<Option<PyNamedOrBlankNode>> {
slf.inner
.next()
.map(|q| Ok(q.map_err(map_io_err)?.into()))
.transpose()
}
}

@ -244,6 +244,20 @@ class TestAbstractStore(unittest.TestCase, ABC):
store.add(Quad(triple.object, triple.predicate, triple.subject))
self.assertEqual(len(store), 4)
def test_add_graph(self):
store = self.store()
store.add_graph(graph)
self.assertEqual(list(store.named_graphs()), [graph])
def test_remove_graph(self):
store = self.store()
store.add(Quad(foo, bar, baz, graph))
store.add_graph(NamedNode("http://graph2"))
store.remove_graph(graph)
store.remove_graph(NamedNode("http://graph2"))
self.assertEqual(list(store.named_graphs()), [])
self.assertEqual(list(store), [])
class TestMemoryStore(TestAbstractStore):
def store(self):

Loading…
Cancel
Save