// partial Copyright (c) 2022-2024 Niko Bonnieure, Par le Peuple, NextGraph.org developers // All rights reserved. // partial Copyright (c) 2018 Oxigraph developers // All work licensed under the Apache License, Version 2.0 // // or the MIT license , // at your option. All files in the project carrying such // notice or not, may not be copied, modified, or distributed except // according to those terms. //! API to access an on-disk [RDF dataset](https://www.w3.org/TR/rdf11-concepts/#dfn-rdf-dataset). //! //! The entry point of the module is the [`Store`] struct. //! //! Usage example: //! ``` //! use oxigraph::model::*; //! use oxigraph::sparql::QueryResults; //! use oxigraph::store::Store; //! //! let store = Store::new()?; //! //! // insertion //! let ex = NamedNode::new("http://example.com")?; //! let quad = Quad::new(ex.clone(), ex.clone(), ex.clone(), GraphName::DefaultGraph); //! store.insert(&quad)?; //! //! // quad filter //! let results: Result, _> = store.quads_for_pattern(None, None, None, None).collect(); //! assert_eq!(vec![quad], results?); //! //! // SPARQL query //! if let QueryResults::Solutions(mut solutions) = store.query("SELECT ?s WHERE { ?s ?p ?o }")? { //! assert_eq!(solutions.next().unwrap()?.get("s"), Some(&ex.into())); //! }; //! # Result::<_, Box>::Ok(()) //! ``` #[cfg(all(not(target_family = "wasm"), not(docsrs)))] use super::io::RdfParseError; use super::io::{RdfFormat, RdfParser, RdfSerializer}; use super::model::*; use super::sparql::{ evaluate_query, evaluate_update, EvaluationError, Query, QueryExplanation, QueryOptions, QueryResults, Update, UpdateOptions, }; use super::storage::numeric_encoder::{Decoder, EncodedQuad, EncodedTerm, StrHash}; #[cfg(all(not(target_family = "wasm"), not(docsrs)))] use super::storage::StorageBulkLoader; pub use super::storage::{CorruptionError, LoaderError, SerializerError, StorageError}; use super::storage::{DecodingGraphIterator, Storage, StorageReader, StorageWriter}; use std::collections::HashSet; use std::error::Error; use std::io::{Read, Write}; #[cfg(all(not(target_family = "wasm"), not(docsrs)))] use std::path::Path; use std::{fmt, str}; /// An on-disk [RDF dataset](https://www.w3.org/TR/rdf11-concepts/#dfn-rdf-dataset). /// Allows to query and update it using SPARQL. /// It is based on the [RocksDB](https://rocksdb.org/) key-value store. /// /// This store ensures the "repeatable read" isolation level: the store only exposes changes that have /// been "committed" (i.e. no partial writes) and the exposed state does not change for the complete duration /// of a read operation (e.g. a SPARQL query) or a read/write operation (e.g. a SPARQL update). /// /// Usage example: /// ``` /// use oxigraph::model::*; /// use oxigraph::sparql::QueryResults; /// use oxigraph::store::Store; /// # use std::fs::remove_dir_all; /// /// # { /// let store = Store::open("example.db")?; /// /// // insertion /// let ex = NamedNode::new("http://example.com")?; /// let quad = Quad::new(ex.clone(), ex.clone(), ex.clone(), GraphName::DefaultGraph); /// store.insert(&quad)?; /// /// // quad filter /// let results: Result, _> = store.quads_for_pattern(None, None, None, None).collect(); /// assert_eq!(vec![quad], results?); /// /// // SPARQL query /// if let QueryResults::Solutions(mut solutions) = store.query("SELECT ?s WHERE { ?s ?p ?o }")? { /// assert_eq!(solutions.next().unwrap()?.get("s"), Some(&ex.into())); /// }; /// # /// # }; /// # remove_dir_all("example.db")?; /// # Result::<_, Box>::Ok(()) /// ``` #[derive(Clone)] pub struct Store { storage: Storage, } impl Store { /// Creates a temporary [`Store`] that will be deleted after drop. pub fn new() -> Result { Ok(Self { storage: Storage::new()?, }) } /// Opens a read-write [`Store`] and creates it if it does not exist yet. /// /// Only one read-write [`Store`] can exist at the same time. /// If you want to have extra [`Store`] instance opened on a same data /// use [`Store::open_read_only`]. #[cfg(all(not(target_family = "wasm"), not(docsrs)))] pub fn open(path: impl AsRef) -> Result { Ok(Self { storage: Storage::open(path.as_ref(), None)?, }) } #[cfg(all(not(target_family = "wasm"), not(docsrs)))] pub fn open_with_key(path: impl AsRef, key: [u8; 32]) -> Result { Ok(Self { storage: Storage::open(path.as_ref(), Some(key))?, }) } // /// Opens a read-only clone of a running read-write [`Store`]. // /// // /// Changes done while this process is running will be replicated after a possible lag. // /// // /// It should only be used if a primary instance opened with [`Store::open`] is running at the same time. // /// `primary_path` must be the path of the primary instance. // /// This secondary instance will use temporary storage for the secondary instance cache. // /// If you prefer persistent storage use [`Store::open_persistent_secondary`]. // /// // /// If you want to simple read-only [`Store`] use [`Store::open_read_only`]. // #[cfg(all(not(target_family = "wasm"),not(docsrs)))] // pub fn open_secondary(primary_path: impl AsRef) -> Result { // Ok(Self { // storage: Storage::open_secondary(primary_path.as_ref())?, // }) // } /// Opens a read-only clone of a running read-write [`Store`] with persistence of the secondary instance cache. /// /// Changes done while this process is running will be replicated after a possible lag. /// /// It should only be used if a primary instance opened with [`Store::open`] is running at the same time. /// `primary_path` must be the path of the primary instance and `secondary_path` an other directory for the secondary instance cache. /// /// If you want to simple read-only [`Store`] use [`Store::open_read_only`]. // #[cfg(all(not(target_family = "wasm"),not(docsrs)))] // pub fn open_persistent_secondary( // primary_path: impl AsRef, // secondary_path: impl AsRef, // ) -> Result { // Ok(Self { // storage: Storage::open_persistent_secondary( // primary_path.as_ref(), // secondary_path.as_ref(), // )?, // }) // } /// Opens a read-only [`Store`] from disk. /// /// Opening as read-only while having an other process writing the database is undefined behavior. #[cfg(all(not(target_family = "wasm"), not(docsrs)))] pub fn open_read_only( path: impl AsRef, key: Option<[u8; 32]>, ) -> Result { Ok(Self { storage: Storage::open_read_only(path.as_ref(), key)?, }) } /// Executes a [SPARQL 1.1 query](https://www.w3.org/TR/sparql11-query/). /// /// Usage example: /// ``` /// use oxigraph::model::*; /// use oxigraph::sparql::QueryResults; /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// /// // insertions /// let ex = NamedNodeRef::new("http://example.com")?; /// store.insert(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?; /// /// // SPARQL query /// if let QueryResults::Solutions(mut solutions) = store.query("SELECT ?s WHERE { ?s ?p ?o }")? { /// assert_eq!( /// solutions.next().unwrap()?.get("s"), /// Some(&ex.into_owned().into()) /// ); /// } /// # Result::<_, Box>::Ok(()) /// ``` pub fn query( &self, query: impl TryInto>, default_graph: Option, ) -> Result { let mut opts = QueryOptions::default(); opts.set_default_graph(default_graph); self.query_opt(query, opts) } /// Executes a [SPARQL 1.1 query](https://www.w3.org/TR/sparql11-query/) with some options. /// /// Usage example with a custom function serializing terms to N-Triples: /// ``` /// use oxigraph::model::*; /// use oxigraph::sparql::{QueryOptions, QueryResults}; /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// if let QueryResults::Solutions(mut solutions) = store.query_opt( /// "SELECT ((1) AS ?nt) WHERE {}", /// QueryOptions::default().with_custom_function( /// NamedNode::new("http://www.w3.org/ns/formats/N-Triples")?, /// |args| args.get(0).map(|t| Literal::from(t.to_string()).into()), /// ), /// )? { /// assert_eq!( /// solutions.next().unwrap()?.get("nt"), /// Some(&Literal::from("\"1\"^^").into()) /// ); /// } /// # Result::<_, Box>::Ok(()) /// ``` pub fn query_opt( &self, query: impl TryInto>, options: QueryOptions, ) -> Result { let (results, _) = self.explain_query_opt(query, options, false)?; results } /// Executes a [SPARQL 1.1 query](https://www.w3.org/TR/sparql11-query/) with some options and /// returns a query explanation with some statistics (if enabled with the `with_stats` parameter). /// ///
If you want to compute statistics you need to exhaust the results iterator before having a look at them.
/// /// Usage example serialising the explanation with statistics in JSON: /// ``` /// use oxigraph::sparql::{QueryOptions, QueryResults}; /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// if let (Ok(QueryResults::Solutions(solutions)), explanation) = store.explain_query_opt( /// "SELECT ?s WHERE { VALUES ?s { 1 2 3 } }", /// QueryOptions::default(), /// true, /// )? { /// // We make sure to have read all the solutions /// for _ in solutions {} /// let mut buf = Vec::new(); /// explanation.write_in_json(&mut buf)?; /// } /// # Result::<_, Box>::Ok(()) /// ``` pub fn explain_query_opt( &self, query: impl TryInto>, options: QueryOptions, with_stats: bool, ) -> Result<(Result, QueryExplanation), EvaluationError> { evaluate_query(self.storage.snapshot(), query, options, with_stats) } /// Retrieves quads with a filter on each quad component /// /// Usage example: /// ``` /// use oxigraph::model::*; /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// /// // insertion /// let ex = NamedNode::new("http://example.com")?; /// let quad = Quad::new(ex.clone(), ex.clone(), ex.clone(), GraphName::DefaultGraph); /// store.insert(&quad)?; /// /// // quad filter by object /// let results = store /// .quads_for_pattern(None, None, Some((&ex).into()), None) /// .collect::, _>>()?; /// assert_eq!(vec![quad], results); /// # Result::<_, Box>::Ok(()) /// ``` pub fn quads_for_pattern( &self, subject: Option>, predicate: Option>, object: Option>, graph_name: Option>, ) -> QuadIter { let reader = self.storage.snapshot(); QuadIter { iter: reader.quads_for_pattern( subject.map(EncodedTerm::from).as_ref(), predicate.map(EncodedTerm::from).as_ref(), object.map(EncodedTerm::from).as_ref(), graph_name.map(|graph_name_ref| { if let GraphName::NamedNode(nn) = graph_name_ref.into_owned() { reader.parse_graph_name(nn.as_string(), None).unwrap() //TODO improve error mng (remove unwrap) } else { panic!("invalid graph name"); } }), ), reader, } } /// Returns all the quads contained in the store. /// /// Usage example: /// ``` /// use oxigraph::model::*; /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// /// // insertion /// let ex = NamedNode::new("http://example.com")?; /// let quad = Quad::new(ex.clone(), ex.clone(), ex.clone(), GraphName::DefaultGraph); /// store.insert(&quad)?; /// /// // quad filter by object /// let results = store.iter().collect::, _>>()?; /// assert_eq!(vec![quad], results); /// # Result::<_, Box>::Ok(()) /// ``` pub fn iter(&self) -> QuadIter { self.quads_for_pattern(None, None, None, None) } /// Checks if this store contains a given quad. /// /// Usage example: /// ``` /// use oxigraph::model::*; /// use oxigraph::store::Store; /// /// let ex = NamedNodeRef::new("http://example.com")?; /// let quad = QuadRef::new(ex, ex, ex, ex); /// /// let store = Store::new()?; /// assert!(!store.contains(quad)?); /// /// store.insert(quad)?; /// assert!(store.contains(quad)?); /// # Result::<_, Box>::Ok(()) /// ``` pub fn contains<'a>(&self, quad: impl Into>) -> Result { let quad = EncodedQuad::from(quad.into()); self.storage.snapshot().contains(&quad) } /// Returns the number of quads in the store. /// ///
This function executes a full scan.
/// /// Usage example: /// ``` /// use oxigraph::model::*; /// use oxigraph::store::Store; /// /// let ex = NamedNodeRef::new("http://example.com")?; /// let store = Store::new()?; /// store.insert(QuadRef::new(ex, ex, ex, ex))?; /// store.insert(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?; /// assert_eq!(2, store.len()?); /// # Result::<_, Box>::Ok(()) /// ``` pub fn len(&self) -> Result { self.storage.snapshot().len() } /// Returns if the store is empty. /// /// Usage example: /// ``` /// use oxigraph::model::*; /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// assert!(store.is_empty()?); /// /// let ex = NamedNodeRef::new("http://example.com")?; /// store.insert(QuadRef::new(ex, ex, ex, ex))?; /// assert!(!store.is_empty()?); /// # Result::<_, Box>::Ok(()) /// ``` pub fn is_empty(&self) -> Result { self.storage.snapshot().is_empty() } /// Executes a transaction. /// /// Transactions ensure the "repeatable read" isolation level: the store only exposes changes that have /// been "committed" (i.e. no partial writes) and the exposed state does not change for the complete duration /// of a read operation (e.g. a SPARQL query) or a read/write operation (e.g. a SPARQL update). /// /// Usage example: /// ``` /// use oxigraph::model::*; /// use oxigraph::store::{StorageError, Store}; /// /// let store = Store::new()?; /// let a = NamedNodeRef::new("http://example.com/a")?; /// let b = NamedNodeRef::new("http://example.com/b")?; /// /// // Copy all triples about ex:a to triples about ex:b /// store.transaction(|mut transaction| { /// for q in transaction.quads_for_pattern(Some(a.into()), None, None, None) { /// let q = q?; /// transaction.insert(QuadRef::new(b, &q.predicate, &q.object, &q.graph_name))?; /// } /// Result::<_, StorageError>::Ok(()) /// })?; /// # Result::<_, Box>::Ok(()) /// ``` fn transaction<'a, 'b: 'a, T, E: Error + 'static + From>( &'b self, f: impl Fn(Transaction<'a>) -> Result, ) -> Result { self.storage .ng_transaction(|writer| f(Transaction { writer })) } /// Executes a [SPARQL 1.1 update](https://www.w3.org/TR/sparql11-update/). /// /// Usage example: /// ``` /// use oxigraph::model::*; /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// /// // insertion /// store /// .update("INSERT DATA { }")?; /// /// // we inspect the store contents /// let ex = NamedNodeRef::new("http://example.com")?; /// assert!(store.contains(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?); /// # Result::<_, Box>::Ok(()) /// ``` pub fn ng_update( &self, update: impl TryInto>, default_graph: Option, ) -> Result<(HashSet, HashSet), EvaluationError> { let mut opts = UpdateOptions::default(); opts.set_default_graph(default_graph); self.ng_update_opt(update, opts) } /// Executes a [SPARQL 1.1 update](https://www.w3.org/TR/sparql11-update/) with some options. /// /// ``` /// use oxigraph::store::Store; /// use oxigraph::model::*; /// use oxigraph::sparql::QueryOptions; /// /// let store = Store::new()?; /// store.update_opt( /// "INSERT { ?s ?n } WHERE { ?s ?p ?o BIND((?s) AS ?nt) }", /// QueryOptions::default().with_custom_function( /// NamedNode::new("http://www.w3.org/ns/formats/N-Triples")?, /// |args| args.get(0).map(|t| Literal::from(t.to_string()).into()) /// ) /// )?; /// # Result::<_, Box>::Ok(()) /// ``` pub fn ng_update_opt( &self, update: impl TryInto>, options: impl Into, ) -> Result<(HashSet, HashSet), EvaluationError> { let update = update.try_into().map_err(Into::into)?; let options = options.into(); self.storage.transaction(|mut t| { evaluate_update(&mut t, &update, &options)?; Ok(t.get_update()) }) } /// INTERNAL FOR NG // pub fn ng_update_opt( // &self, // update: impl TryInto>, // options: impl Into, // ) -> Result<(), EvaluationError> { // let update = update.try_into().map_err(Into::into)?; // let options = options.into(); // self.storage // .ng_transaction(|mut t| evaluate_update(&mut t, &update, &options)) // } // pub fn ng_update( // &self, // update: impl TryInto>, // ) -> Result<(), EvaluationError> { // self.ng_update_opt(update, UpdateOptions::default()) // } #[doc(hidden)] pub fn ng_transaction<'a, 'b: 'a, T, E: Error + 'static + From>( &'b self, f: impl Fn(Transaction<'a>) -> Result, ) -> Result { self.storage .ng_transaction(|writer| f(Transaction { writer })) } /// Loads a RDF file under into the store. /// /// This function is atomic, quite slow and memory hungry. To get much better performances you might want to use the [`bulk_loader`](Store::bulk_loader). /// /// Usage example: /// ``` /// use oxigraph::store::Store; /// use oxigraph::io::RdfFormat; /// use oxigraph::model::*; /// use oxrdfio::RdfParser; /// /// let store = Store::new()?; /// /// // insert a dataset file (former load_dataset method) /// let file = b" ."; /// store.load_from_read(RdfFormat::NQuads, file.as_ref())?; /// /// // insert a graph file (former load_graph method) /// let file = b"<> <> <> ."; /// store.load_from_read( /// RdfParser::from_format(RdfFormat::Turtle) /// .with_base_iri("http://example.com")? /// .without_named_graphs() // No named graphs allowed in the input /// .with_default_graph(NamedNodeRef::new("http://example.com/g2")?), // we put the file default graph inside of a named graph /// file.as_ref() /// )?; /// /// // we inspect the store contents /// let ex = NamedNodeRef::new("http://example.com")?; /// assert!(store.contains(QuadRef::new(ex, ex, ex, NamedNodeRef::new("http://example.com/g")?))?); /// assert!(store.contains(QuadRef::new(ex, ex, ex, NamedNodeRef::new("http://example.com/g2")?))?); /// # Result::<_, Box>::Ok(()) /// ``` pub fn load_from_read( &self, parser: impl Into, read: impl Read, ) -> Result<(), LoaderError> { unimplemented!(); let quads = parser .into() .rename_blank_nodes() .parse_read(read) .collect::, _>>()?; self.storage.transaction(move |mut t| { for quad in &quads { t.insert(quad.as_ref())?; } Ok(()) }) } /// Loads a graph file (i.e. triples) into the store. /// /// This function is atomic, quite slow and memory hungry. To get much better performances you might want to use the [`bulk_loader`](Store::bulk_loader). /// /// Usage example: /// ``` /// use oxigraph::io::RdfFormat; /// use oxigraph::model::*; /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// /// // insertion /// let file = b" ."; /// store.load_graph( /// file.as_ref(), /// RdfFormat::NTriples, /// GraphName::DefaultGraph, /// None, /// )?; /// /// // we inspect the store contents /// let ex = NamedNodeRef::new("http://example.com")?; /// assert!(store.contains(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?); /// # Result::<_, Box>::Ok(()) /// ``` #[deprecated(note = "use Store.load_from_read instead", since = "0.4.0")] pub fn load_graph( &self, read: impl Read, format: impl Into, to_graph_name: impl Into, base_iri: Option<&str>, ) -> Result<(), LoaderError> { let mut parser = RdfParser::from_format(format.into()) .without_named_graphs() .with_default_graph(to_graph_name); if let Some(base_iri) = base_iri { parser = parser .with_base_iri(base_iri) .map_err(|e| LoaderError::InvalidBaseIri { iri: base_iri.into(), error: e, })?; } self.load_from_read(parser, read) } /// Loads a dataset file (i.e. quads) into the store. /// /// This function is atomic, quite slow and memory hungry. To get much better performances you might want to use the [`bulk_loader`](Store::bulk_loader). /// /// Usage example: /// ``` /// use oxigraph::io::RdfFormat; /// use oxigraph::model::*; /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// /// // insertion /// let file = /// b" ."; /// store.load_dataset(file.as_ref(), RdfFormat::NQuads, None)?; /// /// // we inspect the store contents /// let ex = NamedNodeRef::new("http://example.com")?; /// assert!(store.contains(QuadRef::new(ex, ex, ex, ex))?); /// # Result::<_, Box>::Ok(()) /// ``` #[deprecated(note = "use Store.load_from_read instead", since = "0.4.0")] pub fn load_dataset( &self, read: impl Read, format: impl Into, base_iri: Option<&str>, ) -> Result<(), LoaderError> { let mut parser = RdfParser::from_format(format.into()); if let Some(base_iri) = base_iri { parser = parser .with_base_iri(base_iri) .map_err(|e| LoaderError::InvalidBaseIri { iri: base_iri.into(), error: e, })?; } self.load_from_read(parser, read) } /// Adds a quad to this store. /// /// Returns `true` if the quad was not already in the store. /// /// Usage example: /// ``` /// use oxigraph::model::*; /// use oxigraph::store::Store; /// /// let ex = NamedNodeRef::new("http://example.com")?; /// let quad = QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph); /// /// let store = Store::new()?; /// assert!(store.insert(quad)?); /// assert!(!store.insert(quad)?); /// /// assert!(store.contains(quad)?); /// # Result::<_, Box>::Ok(()) /// ``` // pub fn insert<'a>(&self, quad: impl Into>) -> Result { // let quad = quad.into(); // self.transaction(|mut t| t.insert(quad)) // } // /// Adds atomically a set of quads to this store. // /// // ///
// /// // /// This operation uses a memory heavy transaction internally, use the [`bulk_loader`](Store::bulk_loader) if you plan to add ten of millions of triples.
// pub fn extend( // &self, // quads: impl IntoIterator>, // ) -> Result<(), StorageError> { // let quads = quads.into_iter().map(Into::into).collect::>(); // self.transaction(move |mut t| t.extend(&quads)) // } /// Removes a quad from this store. /// /// Returns `true` if the quad was in the store and has been removed. /// /// Usage example: /// ``` /// use oxigraph::model::*; /// use oxigraph::store::Store; /// /// let ex = NamedNodeRef::new("http://example.com")?; /// let quad = QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph); /// /// let store = Store::new()?; /// store.insert(quad)?; /// assert!(store.remove(quad)?); /// assert!(!store.remove(quad)?); /// /// assert!(!store.contains(quad)?); /// # Result::<_, Box>::Ok(()) /// ``` pub fn remove<'a>(&self, quad: impl Into>) -> Result { unimplemented!(); let quad = quad.into(); self.transaction(move |mut t| t.remove(quad)) } /// Dumps the store into a file. /// /// ``` /// use oxigraph::io::RdfFormat; /// use oxigraph::store::Store; /// /// let file = /// " .\n" /// .as_bytes(); /// /// let store = Store::new()?; /// store.load_from_read(RdfFormat::NQuads, file)?; /// /// let buffer = store.dump_to_write(RdfFormat::NQuads, Vec::new())?; /// assert_eq!(file, buffer.as_slice()); /// # std::io::Result::Ok(()) /// ``` pub fn dump_to_write( &self, serializer: impl Into, write: W, ) -> Result { let serializer = serializer.into(); if !serializer.format().supports_datasets() { return Err(SerializerError::DatasetFormatExpected(serializer.format())); } let mut writer = serializer.serialize_to_write(write); for quad in self { writer.write_quad(&quad?)?; } Ok(writer.finish()?) } /// Dumps a store graph into a file. /// /// Usage example: /// ``` /// use oxigraph::io::RdfFormat; /// use oxigraph::model::*; /// use oxigraph::store::Store; /// /// let file = " .\n".as_bytes(); /// /// let store = Store::new()?; /// store.load_graph(file, RdfFormat::NTriples, GraphName::DefaultGraph, None)?; /// /// let mut buffer = Vec::new(); /// store.dump_graph_to_write(GraphNameRef::DefaultGraph, RdfFormat::NTriples, &mut buffer)?; /// assert_eq!(file, buffer.as_slice()); /// # std::io::Result::Ok(()) /// ``` pub fn dump_graph_to_write<'a, W: Write>( &self, from_graph_name: impl Into>, serializer: impl Into, write: W, ) -> Result { let mut writer = serializer.into().serialize_to_write(write); for quad in self.quads_for_pattern(None, None, None, Some(from_graph_name.into())) { writer.write_triple(quad?.as_ref())?; } Ok(writer.finish()?) } /// Dumps a store graph into a file. /// /// Usage example: /// ``` /// use oxigraph::io::RdfFormat; /// use oxigraph::model::*; /// use oxigraph::store::Store; /// /// let file = " .\n".as_bytes(); /// /// let store = Store::new()?; /// store.load_graph(file, RdfFormat::NTriples, GraphName::DefaultGraph, None)?; /// /// let mut buffer = Vec::new(); /// store.dump_graph(&mut buffer, RdfFormat::NTriples, GraphNameRef::DefaultGraph)?; /// assert_eq!(file, buffer.as_slice()); /// # std::io::Result::Ok(()) /// ``` #[deprecated(note = "use Store.dump_graph_to_write instead", since = "0.4.0")] pub fn dump_graph<'a, W: Write>( &self, write: W, format: impl Into, from_graph_name: impl Into>, ) -> Result { self.dump_graph_to_write(from_graph_name, format.into(), write) } /// Dumps the store into a file. /// /// ``` /// use oxigraph::io::RdfFormat; /// use oxigraph::store::Store; /// /// let file = /// " .\n" /// .as_bytes(); /// /// let store = Store::new()?; /// store.load_from_read(RdfFormat::NQuads, file)?; /// /// let buffer = store.dump_dataset(Vec::new(), RdfFormat::NQuads)?; /// assert_eq!(file, buffer.as_slice()); /// # std::io::Result::Ok(()) /// ``` #[deprecated(note = "use Store.dump_to_write instead", since = "0.4.0")] pub fn dump_dataset( &self, write: W, format: impl Into, ) -> Result { self.dump_to_write(format.into(), write) } /// Returns all the store named graphs. /// /// Usage example: /// ``` /// use oxigraph::model::*; /// use oxigraph::store::Store; /// /// let ex = NamedNode::new("http://example.com")?; /// let store = Store::new()?; /// store.insert(QuadRef::new(&ex, &ex, &ex, &ex))?; /// store.insert(QuadRef::new(&ex, &ex, &ex, GraphNameRef::DefaultGraph))?; /// assert_eq!( /// vec![NamedOrBlankNode::from(ex)], /// store.named_graphs().collect::, _>>()? /// ); /// # Result::<_, Box>::Ok(()) /// ``` // pub fn named_graphs(&self) -> GraphNameIter { // let reader = self.storage.snapshot(); // GraphNameIter { // iter: reader.named_graphs(), // reader, // } // } /// Checks if the store contains a given graph /// /// Usage example: /// ``` /// use oxigraph::model::{NamedNode, QuadRef}; /// use oxigraph::store::Store; /// /// let ex = NamedNode::new("http://example.com")?; /// let store = Store::new()?; /// store.insert(QuadRef::new(&ex, &ex, &ex, &ex))?; /// assert!(store.contains_named_graph(&ex)?); /// # Result::<_, Box>::Ok(()) /// ``` // pub fn contains_named_graph<'a>( // &self, // graph_name: impl Into>, // ) -> Result { // let graph_name = EncodedTerm::from(graph_name.into()); // self.storage.snapshot().contains_named_graph(&graph_name) // } /// Inserts a graph into this store. /// /// Returns `true` if the graph was not already in the store. /// /// Usage example: /// ``` /// use oxigraph::model::NamedNodeRef; /// use oxigraph::store::Store; /// /// let ex = NamedNodeRef::new("http://example.com")?; /// let store = Store::new()?; /// store.insert_named_graph(ex)?; /// /// assert_eq!( /// store.named_graphs().collect::, _>>()?, /// vec![ex.into_owned().into()] /// ); /// # Result::<_, Box>::Ok(()) /// ``` // pub fn insert_named_graph<'a>( // &self, // graph_name: impl Into>, // ) -> Result { // let graph_name = graph_name.into(); // self.transaction(|mut t| t.insert_named_graph(graph_name)) // } /// Clears a graph from this store. /// /// Usage example: /// ``` /// use oxigraph::model::{NamedNodeRef, QuadRef}; /// use oxigraph::store::Store; /// /// let ex = NamedNodeRef::new("http://example.com")?; /// let quad = QuadRef::new(ex, ex, ex, ex); /// let store = Store::new()?; /// store.insert(quad)?; /// assert_eq!(1, store.len()?); /// /// store.clear_graph(ex)?; /// assert!(store.is_empty()?); /// assert_eq!(1, store.named_graphs().count()); /// # Result::<_, Box>::Ok(()) /// ``` pub fn clear_graph<'a>( &self, graph_name: impl Into>, ) -> Result<(), StorageError> { let graph_name = graph_name.into(); self.transaction(|mut t| t.clear_graph(graph_name)) } /// Removes a graph from this store. /// /// Returns `true` if the graph was in the store and has been removed. /// /// Usage example: /// ``` /// use oxigraph::model::{NamedNodeRef, QuadRef}; /// use oxigraph::store::Store; /// /// let ex = NamedNodeRef::new("http://example.com")?; /// let quad = QuadRef::new(ex, ex, ex, ex); /// let store = Store::new()?; /// store.insert(quad)?; /// assert_eq!(1, store.len()?); /// /// assert!(store.remove_named_graph(ex)?); /// assert!(store.is_empty()?); /// assert_eq!(0, store.named_graphs().count()); /// # Result::<_, Box>::Ok(()) /// ``` pub fn remove_named_graph<'a>( &self, graph_name: impl Into>, ) -> Result { let graph_name = graph_name.into(); self.transaction(|mut t| t.remove_named_graph(graph_name)) } /// Clears the store. /// /// Usage example: /// ``` /// use oxigraph::model::*; /// use oxigraph::store::Store; /// /// let ex = NamedNodeRef::new("http://example.com")?; /// let store = Store::new()?; /// store.insert(QuadRef::new(ex, ex, ex, ex))?; /// store.insert(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?; /// assert_eq!(2, store.len()?); /// /// store.clear()?; /// assert!(store.is_empty()?); /// # Result::<_, Box>::Ok(()) /// ``` pub fn clear(&self) -> Result<(), StorageError> { self.transaction(|mut t| t.clear()) } /// Flushes all buffers and ensures that all writes are saved on disk. /// /// Flushes are automatically done using background threads but might lag a little bit. #[cfg(all(not(target_family = "wasm"), not(docsrs)))] pub fn flush(&self) -> Result<(), StorageError> { self.storage.flush() } /// Optimizes the database for future workload. /// /// Useful to call after a batch upload or another similar operation. /// ///
Can take hours on huge databases.
#[cfg(all(not(target_family = "wasm"), not(docsrs)))] pub fn optimize(&self) -> Result<(), StorageError> { self.storage.compact() } /// Creates database backup into the `target_directory`. /// /// After its creation, the backup is usable using [`Store::open`] /// like a regular Oxigraph database and operates independently from the original database. /// ///
/// /// Backups are only possible for on-disk databases created using [`Store::open`].
/// Temporary in-memory databases created using [`Store::new`] are not compatible with RocksDB backup system. /// ///
An error is raised if the `target_directory` already exists.
/// /// If the target directory is in the same file system as the current database, /// the database content will not be fully copied /// but hard links will be used to point to the original database immutable snapshots. /// This allows cheap regular backups. /// /// If you want to move your data to another RDF storage system, you should have a look at the [`Store::dump_to_write`] function instead. #[cfg(all(not(target_family = "wasm"), not(docsrs)))] pub fn backup(&self, target_directory: impl AsRef) -> Result<(), StorageError> { self.storage.backup(target_directory.as_ref()) } /// Creates a bulk loader allowing to load at lot of data quickly into the store. /// /// Usage example: /// ``` /// use oxigraph::io::RdfFormat; /// use oxigraph::model::*; /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// /// // quads file insertion /// let file = /// b" ."; /// store /// .bulk_loader() /// .load_from_read(RdfFormat::NQuads, file.as_ref())?; /// /// // we inspect the store contents /// let ex = NamedNodeRef::new("http://example.com")?; /// assert!(store.contains(QuadRef::new(ex, ex, ex, ex))?); /// # Result::<_, Box>::Ok(()) /// ``` #[cfg(all(not(target_family = "wasm"), not(docsrs)))] pub fn bulk_loader(&self) -> BulkLoader { BulkLoader { storage: StorageBulkLoader::new(self.storage.clone()), on_parse_error: None, } } /// Validates that all the store invariants held in the data #[doc(hidden)] pub fn validate(&self) -> Result<(), StorageError> { self.storage.snapshot().validate() } } impl fmt::Display for Store { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { for t in self { writeln!(f, "{} .", t.map_err(|_| fmt::Error)?)?; } Ok(()) } } impl IntoIterator for &Store { type IntoIter = QuadIter; type Item = Result; #[inline] fn into_iter(self) -> Self::IntoIter { self.iter() } } /// An object to do operations during a transaction. /// /// See [`Store::transaction`] for a more detailed description. pub struct Transaction<'a> { writer: StorageWriter<'a>, } impl<'a> Transaction<'a> { /// Executes a [SPARQL 1.1 query](https://www.w3.org/TR/sparql11-query/). /// /// Usage example: /// ``` /// use oxigraph::model::*; /// use oxigraph::sparql::{EvaluationError, QueryResults}; /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// store.transaction(|mut transaction| { /// if let QueryResults::Solutions(solutions) = /// transaction.query("SELECT ?s WHERE { ?s ?p ?o }")? /// { /// for solution in solutions { /// if let Some(Term::NamedNode(s)) = solution?.get("s") { /// transaction.insert(QuadRef::new( /// s, /// vocab::rdf::TYPE, /// NamedNodeRef::new_unchecked("http://example.com"), /// GraphNameRef::DefaultGraph, /// ))?; /// } /// } /// } /// Result::<_, EvaluationError>::Ok(()) /// })?; /// # Result::<_, EvaluationError>::Ok(()) /// ``` pub fn query( &self, query: impl TryInto>, ) -> Result { self.query_opt(query, QueryOptions::default()) } /// Executes a [SPARQL 1.1 query](https://www.w3.org/TR/sparql11-query/) with some options. /// /// Usage example with a custom function serializing terms to N-Triples: /// ``` /// use oxigraph::model::*; /// use oxigraph::sparql::{EvaluationError, QueryOptions, QueryResults}; /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// store.transaction(|mut transaction| { /// if let QueryResults::Solutions(solutions) = transaction.query_opt( /// "SELECT ?s ((?s) AS ?nt) WHERE { ?s ?p ?o }", /// QueryOptions::default().with_custom_function( /// NamedNode::new_unchecked("http://www.w3.org/ns/formats/N-Triples"), /// |args| args.get(0).map(|t| Literal::from(t.to_string()).into()), /// ), /// )? { /// for solution in solutions { /// let solution = solution?; /// if let (Some(Term::NamedNode(s)), Some(nt)) = /// (solution.get("s"), solution.get("nt")) /// { /// transaction.insert(QuadRef::new( /// s, /// NamedNodeRef::new_unchecked("http://example.com/n-triples-representation"), /// nt, /// GraphNameRef::DefaultGraph, /// ))?; /// } /// } /// } /// Result::<_, EvaluationError>::Ok(()) /// })?; /// # Result::<_, EvaluationError>::Ok(()) /// ``` pub fn query_opt( &self, query: impl TryInto>, options: QueryOptions, ) -> Result { let (results, _) = evaluate_query(self.writer.reader(), query, options, false)?; results } /// Retrieves quads with a filter on each quad component. /// /// Usage example: /// ``` /// use oxigraph::model::*; /// use oxigraph::store::{StorageError, Store}; /// /// let store = Store::new()?; /// let a = NamedNodeRef::new("http://example.com/a")?; /// let b = NamedNodeRef::new("http://example.com/b")?; /// /// // Copy all triples about ex:a to triples about ex:b /// store.transaction(|mut transaction| { /// for q in transaction.quads_for_pattern(Some(a.into()), None, None, None) { /// let q = q?; /// transaction.insert(QuadRef::new(b, &q.predicate, &q.object, &q.graph_name))?; /// } /// Result::<_, StorageError>::Ok(()) /// })?; /// # Result::<_, Box>::Ok(()) /// ``` pub fn quads_for_pattern( &self, subject: Option>, predicate: Option>, object: Option>, graph_name: Option>, ) -> QuadIter { let reader = self.writer.reader(); QuadIter { iter: reader.quads_for_pattern( subject.map(EncodedTerm::from).as_ref(), predicate.map(EncodedTerm::from).as_ref(), object.map(EncodedTerm::from).as_ref(), graph_name.map(|graph_name_ref| { if let GraphName::NamedNode(nn) = graph_name_ref.into_owned() { reader.parse_graph_name(nn.as_string(), None).unwrap() //TODO improve error mng (remove unwrap) } else { panic!("invalid graph name"); } }), ), reader, } } /// Returns all the quads contained in the store. pub fn iter(&self) -> QuadIter { self.quads_for_pattern(None, None, None, None) } /// Checks if this store contains a given quad. pub fn contains<'b>(&self, quad: impl Into>) -> Result { let quad = EncodedQuad::from(quad.into()); self.writer.reader().contains(&quad) } /// Returns the number of quads in the store. /// ///
this function executes a full scan.
pub fn len(&self) -> Result { self.writer.reader().len() } /// Returns if the store is empty. pub fn is_empty(&self) -> Result { self.writer.reader().is_empty() } /// Executes a [SPARQL 1.1 update](https://www.w3.org/TR/sparql11-update/). /// /// Usage example: /// ``` /// use oxigraph::model::*; /// use oxigraph::sparql::EvaluationError; /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// store.transaction(|mut transaction| { /// // insertion /// transaction.update( /// "INSERT DATA { }", /// )?; /// /// // we inspect the store contents /// let ex = NamedNodeRef::new_unchecked("http://example.com"); /// assert!(transaction.contains(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?); /// Result::<_, EvaluationError>::Ok(()) /// })?; /// # Result::<_, EvaluationError>::Ok(()) /// ``` // pub fn update( // &mut self, // update: impl TryInto>, // ) -> Result<(), EvaluationError> { // self.update_opt(update, UpdateOptions::default()) // } // /// Executes a [SPARQL 1.1 update](https://www.w3.org/TR/sparql11-update/) with some options. // pub fn update_opt( // &mut self, // update: impl TryInto>, // options: impl Into, // ) -> Result<(), EvaluationError> { // evaluate_update( // &mut self.writer, // &update.try_into().map_err(Into::into)?, // &options.into(), // ) // } /// Loads a RDF file into the store. /// /// This function is atomic, quite slow and memory hungry. To get much better performances you might want to use the [`bulk_loader`](Store::bulk_loader). /// /// Usage example: /// ``` /// use oxigraph::store::Store; /// use oxigraph::io::RdfFormat; /// use oxigraph::model::*; /// use oxrdfio::RdfParser; /// /// let store = Store::new()?; /// /// // insert a dataset file (former load_dataset method) /// let file = b" ."; /// store.transaction(|mut t| t.load_from_read(RdfFormat::NQuads, file.as_ref()))?; /// /// // insert a graph file (former load_graph method) /// let file = b"<> <> <> ."; /// store.transaction(|mut t| /// t.load_from_read( /// RdfParser::from_format(RdfFormat::Turtle) /// .with_base_iri("http://example.com") /// .unwrap() /// .without_named_graphs() // No named graphs allowed in the input /// .with_default_graph(NamedNodeRef::new("http://example.com/g2").unwrap()), // we put the file default graph inside of a named graph /// file.as_ref() /// ) /// )?; /// /// // we inspect the store contents /// let ex = NamedNodeRef::new("http://example.com")?; /// assert!(store.contains(QuadRef::new(ex, ex, ex, NamedNodeRef::new("http://example.com/g")?))?); /// assert!(store.contains(QuadRef::new(ex, ex, ex, NamedNodeRef::new("http://example.com/g2")?))?); /// # Result::<_, Box>::Ok(()) /// ``` // pub fn load_from_read( // &mut self, // parser: impl Into, // read: impl Read, // ) -> Result<(), LoaderError> { // for quad in parser.into().rename_blank_nodes().parse_read(read) { // self.insert(quad?.as_ref())?; // } // Ok(()) // } /// Loads a graph file (i.e. triples) into the store. /// /// Usage example: /// ``` /// use oxigraph::io::RdfFormat; /// use oxigraph::model::*; /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// /// // insertion /// let file = b" ."; /// store.transaction(|mut transaction| { /// transaction.load_graph( /// file.as_ref(), /// RdfFormat::NTriples, /// GraphName::DefaultGraph, /// None, /// ) /// })?; /// /// // we inspect the store contents /// let ex = NamedNodeRef::new_unchecked("http://example.com"); /// assert!(store.contains(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?); /// # Result::<_,oxigraph::store::LoaderError>::Ok(()) /// ``` // #[deprecated(note = "use Transaction.load_from_read instead", since = "0.4.0")] // pub fn load_graph( // &mut self, // read: impl Read, // format: impl Into, // to_graph_name: impl Into, // base_iri: Option<&str>, // ) -> Result<(), LoaderError> { // let mut parser = RdfParser::from_format(format.into()) // .without_named_graphs() // .with_default_graph(to_graph_name); // if let Some(base_iri) = base_iri { // parser = parser // .with_base_iri(base_iri) // .map_err(|e| LoaderError::InvalidBaseIri { // iri: base_iri.into(), // error: e, // })?; // } // self.load_from_read(parser, read) // } /// Loads a dataset file (i.e. quads) into the store. /// /// Usage example: /// ``` /// use oxigraph::io::RdfFormat; /// use oxigraph::model::*; /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// /// // insertion /// let file = /// b" ."; /// store.transaction(|mut transaction| { /// transaction.load_dataset(file.as_ref(), RdfFormat::NQuads, None) /// })?; /// /// // we inspect the store contents /// let ex = NamedNodeRef::new_unchecked("http://example.com"); /// assert!(store.contains(QuadRef::new(ex, ex, ex, ex))?); /// # Result::<_,oxigraph::store::LoaderError>::Ok(()) /// ``` // #[deprecated(note = "use Transaction.load_from_read instead", since = "0.4.0")] // pub fn load_dataset( // &mut self, // read: impl Read, // format: impl Into, // base_iri: Option<&str>, // ) -> Result<(), LoaderError> { // let mut parser = RdfParser::from_format(format.into()); // if let Some(base_iri) = base_iri { // parser = parser // .with_base_iri(base_iri) // .map_err(|e| LoaderError::InvalidBaseIri { // iri: base_iri.into(), // error: e, // })?; // } // self.load_from_read(parser, read) // } pub fn ng_get_heads( &self, topic: &StrHash, overlay: &StrHash, ) -> Result, StorageError> { self.writer.reader().ng_get_heads(topic, overlay) } pub fn ng_get_reader(&self) -> StorageReader { self.writer.reader() } pub fn update_heads( &mut self, topic: &StrHash, overlay: &StrHash, commit: &StrHash, direct_causal_past: &HashSet, ) -> Result<(), StorageError> { self.writer .ng_update_heads(topic, overlay, commit, direct_causal_past) } pub fn update_past( &mut self, commit: &StrHash, direct_causal_past: &HashSet, skip_has_no_graph: bool, ) -> Result<(), StorageError> { self.writer .ng_update_past(commit, direct_causal_past, skip_has_no_graph) } pub fn update_branch_and_token( &mut self, overlay_encoded: &StrHash, branch_encoded: &StrHash, topic_encoded: &StrHash, token_encoded: &StrHash, ) -> Result<(), StorageError> { self.writer.update_branch_and_token( overlay_encoded, branch_encoded, topic_encoded, token_encoded, ) } pub fn doc_in_store( &mut self, graph_name: NamedNodeRef<'_>, overlay: &StrHash, remove: bool, ) -> Result<(), StorageError> { self.writer.doc_in_store(graph_name, overlay, remove) } pub fn named_commit_or_branch( &mut self, ov_graph_name: NamedNodeRef<'_>, name: &String, // if None: remove value: &Option>, ) -> Result<(), StorageError> { self.writer .named_commit_or_branch(ov_graph_name, name, value) } /// Adds a quad to this store. /// /// Returns `true` if the quad was not already in the store. /// /// Usage example: /// ``` /// use oxigraph::model::*; /// use oxigraph::store::Store; /// /// let ex = NamedNodeRef::new_unchecked("http://example.com"); /// let quad = QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph); /// /// let store = Store::new()?; /// store.transaction(|mut transaction| transaction.insert(quad))?; /// assert!(store.contains(quad)?); /// # Result::<_,oxigraph::store::StorageError>::Ok(()) /// ``` pub fn insert<'b>( &mut self, quad: impl Into>, value: u8, cv: bool, ) -> Result<(), StorageError> { self.writer.ng_insert(quad.into(), value, cv) } pub fn insert_encoded( &mut self, encoded: &EncodedQuad, value: u8, cv: bool, ) -> Result { self.writer.ng_insert_encoded(encoded, value, cv) } pub fn ng_remove(&mut self, quad: &EncodedQuad, commit: &StrHash) -> Result<(), StorageError> { self.writer.ng_remove(quad, commit) } // /// Adds a set of quads to this store. // pub fn extend<'b>( // &mut self, // quads: impl IntoIterator>>, // ) -> Result<(), StorageError> { // for quad in quads { // self.writer.insert(quad.into())?; // } // Ok(()) // } /// Removes a quad from this store. /// /// Returns `true` if the quad was in the store and has been removed. /// /// Usage example: /// ``` /// use oxigraph::model::*; /// use oxigraph::store::Store; /// /// let ex = NamedNodeRef::new_unchecked("http://example.com"); /// let quad = QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph); /// let store = Store::new()?; /// store.transaction(|mut transaction| { /// transaction.insert(quad)?; /// transaction.remove(quad) /// })?; /// assert!(!store.contains(quad)?); /// # Result::<_,oxigraph::store::StorageError>::Ok(()) /// ``` pub fn remove<'b>(&mut self, quad: impl Into>) -> Result { self.writer.remove(quad.into()) } /// Returns all the store named graphs. // pub fn named_graphs(&self) -> GraphNameIter { // let reader = self.writer.reader(); // GraphNameIter { // iter: reader.named_graphs(), // reader, // } // } /// Checks if the store contains a given graph. // pub fn contains_named_graph<'b>( // &self, // graph_name: impl Into>, // ) -> Result { // self.writer // .reader() // .contains_named_graph(&EncodedTerm::from(graph_name.into())) // } /// Inserts a graph into this store. /// /// Returns `true` if the graph was not already in the store. /// /// Usage example: /// ``` /// use oxigraph::model::NamedNodeRef; /// use oxigraph::store::Store; /// /// let ex = NamedNodeRef::new_unchecked("http://example.com"); /// let store = Store::new()?; /// store.transaction(|mut transaction| transaction.insert_named_graph(ex))?; /// assert_eq!( /// store.named_graphs().collect::, _>>()?, /// vec![ex.into_owned().into()] /// ); /// # Result::<_,oxigraph::store::StorageError>::Ok(()) /// ``` // pub fn insert_named_graph<'b>( // &mut self, // graph_name: impl Into>, // ) -> Result { // self.writer.insert_named_graph(graph_name.into()) // } /// Clears a graph from this store. /// /// Usage example: /// ``` /// use oxigraph::model::{NamedNodeRef, QuadRef}; /// use oxigraph::store::Store; /// /// let ex = NamedNodeRef::new_unchecked("http://example.com"); /// let quad = QuadRef::new(ex, ex, ex, ex); /// let store = Store::new()?; /// store.transaction(|mut transaction| { /// transaction.insert(quad)?; /// transaction.clear_graph(ex) /// })?; /// assert!(store.is_empty()?); /// assert_eq!(1, store.named_graphs().count()); /// # Result::<_,oxigraph::store::StorageError>::Ok(()) /// ``` pub fn clear_graph<'b>( &mut self, graph_name: impl Into>, ) -> Result<(), StorageError> { self.writer.clear_graph(graph_name.into()) } /// Removes a graph from this store. /// /// Returns `true` if the graph was in the store and has been removed. /// /// Usage example: /// ``` /// use oxigraph::model::{NamedNodeRef, QuadRef}; /// use oxigraph::store::Store; /// /// let ex = NamedNodeRef::new_unchecked("http://example.com"); /// let quad = QuadRef::new(ex, ex, ex, ex); /// let store = Store::new()?; /// store.transaction(|mut transaction| { /// transaction.insert(quad)?; /// transaction.remove_named_graph(ex) /// })?; /// assert!(store.is_empty()?); /// assert_eq!(0, store.named_graphs().count()); /// # Result::<_,oxigraph::store::StorageError>::Ok(()) /// ``` pub fn remove_named_graph<'b>( &mut self, graph_name: impl Into>, ) -> Result { self.writer.remove_named_graph(graph_name.into()) } /// Clears the store. /// /// Usage example: /// ``` /// use oxigraph::model::*; /// use oxigraph::store::Store; /// /// let ex = NamedNodeRef::new_unchecked("http://example.com"); /// let store = Store::new()?; /// store.transaction(|mut transaction| { /// transaction.insert(QuadRef::new(ex, ex, ex, ex))?; /// transaction.clear() /// })?; /// assert!(store.is_empty()?); /// # Result::<_,oxigraph::store::StorageError>::Ok(()) /// ``` pub fn clear(&mut self) -> Result<(), StorageError> { self.writer.clear() } } impl IntoIterator for &Transaction<'_> { type IntoIter = QuadIter; type Item = Result; #[inline] fn into_iter(self) -> Self::IntoIter { self.iter() } } /// An iterator returning the quads contained in a [`Store`]. pub struct QuadIter { iter: Box>>, reader: StorageReader, } impl Iterator for QuadIter { type Item = Result; fn next(&mut self) -> Option { Some(match self.iter.next()? { Ok(quad) => self.reader.decode_quad(&quad), Err(error) => Err(error), }) } } /// An iterator returning the graph names contained in a [`Store`]. pub struct GraphNameIter { iter: DecodingGraphIterator, reader: StorageReader, } impl Iterator for GraphNameIter { type Item = Result; fn next(&mut self) -> Option { Some( self.iter .next()? .and_then(|graph_name| self.reader.decode_named_or_blank_node(&graph_name)), ) } fn size_hint(&self) -> (usize, Option) { self.iter.size_hint() } } /// A bulk loader allowing to load at lot of data quickly into the store. /// ///
The operations provided here are not atomic. /// If the operation fails in the middle, only a part of the data may be written to the store. /// Results might get weird if you delete data during the loading process.
/// ///
/// /// It is optimized for speed.
/// Memory usage is configurable using [`with_max_memory_size_in_megabytes`](Self::with_max_memory_size_in_megabytes) /// and the number of used threads with [`with_num_threads`](Self::with_num_threads). /// By default the memory consumption target (excluding the system and RocksDB internal consumption) /// is around 2GB per thread and 2 threads. /// These targets are considered per loaded file. /// /// Usage example with loading a dataset: /// ``` /// use oxigraph::io::RdfFormat; /// use oxigraph::model::*; /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// /// // quads file insertion /// let file = /// b" ."; /// store /// .bulk_loader() /// .load_from_read(RdfFormat::NQuads, file.as_ref())?; /// /// // we inspect the store contents /// let ex = NamedNodeRef::new("http://example.com")?; /// assert!(store.contains(QuadRef::new(ex, ex, ex, ex))?); /// # Result::<_, Box>::Ok(()) /// ``` #[cfg(all(not(target_family = "wasm"), not(docsrs)))] #[must_use] pub struct BulkLoader { storage: StorageBulkLoader, on_parse_error: Option Result<(), RdfParseError>>>, } #[cfg(all(not(target_family = "wasm"), not(docsrs)))] impl BulkLoader { /// Sets the maximal number of threads to be used by the bulk loader per operation. /// /// This number must be at last 2 (one for parsing and one for loading). /// /// The default value is 2. pub fn with_num_threads(mut self, num_threads: usize) -> Self { self.storage = self.storage.with_num_threads(num_threads); self } #[doc(hidden)] #[deprecated(note = "Use with_num_threads", since = "0.4.0")] pub fn set_num_threads(self, num_threads: usize) -> Self { self.with_num_threads(num_threads) } /// Sets a rough idea of the maximal amount of memory to be used by this operation. /// /// This number must be at last a few megabytes per thread. /// /// Memory used by RocksDB and the system is not taken into account in this limit. /// Note that depending on the system behavior this amount might never be reached or be blown up /// (for example if the data contains very long IRIs or literals). /// /// By default, a target 2GB per used thread is used. pub fn with_max_memory_size_in_megabytes(mut self, max_memory_size: usize) -> Self { self.storage = self .storage .with_max_memory_size_in_megabytes(max_memory_size); self } #[doc(hidden)] #[deprecated(note = "Use with_max_memory_size_in_megabytes", since = "0.4.0")] pub fn set_max_memory_size_in_megabytes(self, max_memory_size: usize) -> Self { self.with_max_memory_size_in_megabytes(max_memory_size) } /// Adds a `callback` evaluated from time to time with the number of loaded triples. pub fn on_progress(mut self, callback: impl Fn(u64) + 'static) -> Self { self.storage = self.storage.on_progress(callback); self } /// Adds a `callback` catching all parse errors and choosing if the parsing should continue /// by returning `Ok` or fail by returning `Err`. /// /// By default the parsing fails. pub fn on_parse_error( mut self, callback: impl Fn(RdfParseError) -> Result<(), RdfParseError> + 'static, ) -> Self { self.on_parse_error = Some(Box::new(callback)); self } /// Loads a file using the bulk loader. /// /// This function is optimized for large dataset loading speed. For small files, [`Store::load_from_read`] might be more convenient. /// ///
This method is not atomic. /// If the parsing fails in the middle of the file, only a part of it may be written to the store. /// Results might get weird if you delete data during the loading process.
/// ///
/// /// This method is optimized for speed. See [the struct](Self) documentation for more details.
/// /// To get better speed on valid datasets, consider enabling [`RdfParser::unchecked`] option to skip some validations. /// /// Usage example: /// ``` /// use oxigraph::store::Store; /// use oxigraph::io::{RdfParser, RdfFormat}; /// use oxigraph::model::*; /// /// let store = Store::new()?; /// /// // insert a dataset file (former load_dataset method) /// let file = b" ."; /// store.bulk_loader().load_from_read( /// RdfParser::from_format(RdfFormat::NQuads).unchecked(), // we inject a custom parser with options /// file.as_ref() /// )?; /// /// // insert a graph file (former load_graph method) /// let file = b"<> <> <> ."; /// store.bulk_loader().load_from_read( /// RdfParser::from_format(RdfFormat::Turtle) /// .with_base_iri("http://example.com")? /// .without_named_graphs() // No named graphs allowed in the input /// .with_default_graph(NamedNodeRef::new("http://example.com/g2")?), // we put the file default graph inside of a named graph /// file.as_ref() /// )?; /// /// // we inspect the store contents /// let ex = NamedNodeRef::new("http://example.com")?; /// assert!(store.contains(QuadRef::new(ex, ex, ex, NamedNodeRef::new("http://example.com/g")?))?); /// assert!(store.contains(QuadRef::new(ex, ex, ex, NamedNodeRef::new("http://example.com/g2")?))?); /// # Result::<_, Box>::Ok(()) /// ``` pub fn load_from_read( &self, parser: impl Into, read: impl Read, ) -> Result<(), LoaderError> { self.load_ok_quads( parser .into() .rename_blank_nodes() .parse_read(read) .filter_map(|r| match r { Ok(q) => Some(Ok(q)), Err(e) => { if let Some(callback) = &self.on_parse_error { if let Err(e) = callback(e) { Some(Err(e)) } else { None } } else { Some(Err(e)) } } }), ) } /// Loads a dataset file using the bulk loader. /// /// This function is optimized for large dataset loading speed. For small files, [`Store::load_dataset`] might be more convenient. /// ///
This method is not atomic. /// If the parsing fails in the middle of the file, only a part of it may be written to the store. /// Results might get weird if you delete data during the loading process.
/// ///
/// /// This method is optimized for speed. See [the struct](Self) documentation for more details.
/// /// Usage example: /// ``` /// use oxigraph::io::RdfFormat; /// use oxigraph::model::*; /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// /// // insertion /// let file = /// b" ."; /// store /// .bulk_loader() /// .load_dataset(file.as_ref(), RdfFormat::NQuads, None)?; /// /// // we inspect the store contents /// let ex = NamedNodeRef::new("http://example.com")?; /// assert!(store.contains(QuadRef::new(ex, ex, ex, ex))?); /// # Result::<_, Box>::Ok(()) /// ``` #[deprecated(note = "use BulkLoader.load_from_read instead", since = "0.4.0")] pub fn load_dataset( &self, read: impl Read, format: impl Into, base_iri: Option<&str>, ) -> Result<(), LoaderError> { let mut parser = RdfParser::from_format(format.into()).rename_blank_nodes(); if let Some(base_iri) = base_iri { parser = parser .with_base_iri(base_iri) .map_err(|e| LoaderError::InvalidBaseIri { iri: base_iri.into(), error: e, })?; } self.load_ok_quads(parser.parse_read(read).filter_map(|r| match r { Ok(q) => Some(Ok(q)), Err(e) => { if let Some(callback) = &self.on_parse_error { if let Err(e) = callback(e) { Some(Err(e)) } else { None } } else { Some(Err(e)) } } })) } /// Loads a graph file using the bulk loader. /// /// This function is optimized for large graph loading speed. For small files, [`Store::load_graph`] might be more convenient. /// ///
This method is not atomic. /// If the parsing fails in the middle of the file, only a part of it may be written to the store. /// Results might get weird if you delete data during the loading process.
/// ///
/// /// This method is optimized for speed. See [the struct](Self) documentation for more details.
/// /// Usage example: /// ``` /// use oxigraph::io::RdfFormat; /// use oxigraph::model::*; /// use oxigraph::store::Store; /// /// let store = Store::new()?; /// /// // insertion /// let file = b" ."; /// store.bulk_loader().load_graph( /// file.as_ref(), /// RdfFormat::NTriples, /// GraphName::DefaultGraph, /// None, /// )?; /// /// // we inspect the store contents /// let ex = NamedNodeRef::new("http://example.com")?; /// assert!(store.contains(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?); /// # Result::<_, Box>::Ok(()) /// ``` #[deprecated(note = "use BulkLoader.load_from_read instead", since = "0.4.0")] pub fn load_graph( &self, read: impl Read, format: impl Into, to_graph_name: impl Into, base_iri: Option<&str>, ) -> Result<(), LoaderError> { let mut parser = RdfParser::from_format(format.into()) .without_named_graphs() .with_default_graph(to_graph_name) .rename_blank_nodes(); if let Some(base_iri) = base_iri { parser = parser .with_base_iri(base_iri) .map_err(|e| LoaderError::InvalidBaseIri { iri: base_iri.into(), error: e, })?; } self.load_ok_quads(parser.parse_read(read).filter_map(|r| match r { Ok(q) => Some(Ok(q)), Err(e) => { if let Some(callback) = &self.on_parse_error { if let Err(e) = callback(e) { Some(Err(e)) } else { None } } else { Some(Err(e)) } } })) } /// Adds a set of quads using the bulk loader. /// ///
This method is not atomic. /// If the process fails in the middle of the file, only a part of the data may be written to the store. /// Results might get weird if you delete data during the loading process.
/// ///
/// /// This method is optimized for speed. See [the struct](Self) documentation for more details.
pub fn load_quads( &self, quads: impl IntoIterator>, ) -> Result<(), StorageError> { self.load_ok_quads(quads.into_iter().map(Ok::<_, StorageError>)) } /// Adds a set of quads using the bulk loader while breaking in the middle of the process in case of error. /// ///
This method is not atomic. /// If the process fails in the middle of the file, only a part of the data may be written to the store. /// Results might get weird if you delete data during the loading process.
/// ///
/// /// This method is optimized for speed. See [the struct](Self) documentation for more details.
pub fn load_ok_quads + From>( &self, quads: impl IntoIterator, EI>>, ) -> Result<(), EO> { self.storage .load(quads.into_iter().map(|q| q.map(Into::into))) } } #[cfg(test)] #[allow(clippy::panic_in_result_fn)] mod tests { use super::*; #[test] fn store() -> Result<(), StorageError> { use super::super::model::*; let main_s = Subject::from(BlankNode::default()); let main_p = NamedNode::new("http://example.com").unwrap(); let main_o = Term::from(Literal::from(1)); let main_g = GraphName::from(BlankNode::default()); let default_quad = Quad::new( main_s.clone(), main_p.clone(), main_o.clone(), GraphName::DefaultGraph, ); let named_quad = Quad::new( main_s.clone(), main_p.clone(), main_o.clone(), main_g.clone(), ); let default_quads = vec![ Quad::new( main_s.clone(), main_p.clone(), Literal::from(0), GraphName::DefaultGraph, ), default_quad.clone(), Quad::new( main_s.clone(), main_p.clone(), Literal::from(200_000_000), GraphName::DefaultGraph, ), ]; let all_quads = vec![ Quad::new( main_s.clone(), main_p.clone(), Literal::from(0), GraphName::DefaultGraph, ), default_quad.clone(), Quad::new( main_s.clone(), main_p.clone(), Literal::from(200_000_000), GraphName::DefaultGraph, ), named_quad.clone(), ]; let store = Store::new()?; // for t in &default_quads { // assert!(store.insert(t)?); // } // assert!(!store.insert(&default_quad)?); // assert!(store.remove(&default_quad)?); // assert!(!store.remove(&default_quad)?); // assert!(store.insert(&named_quad)?); // assert!(!store.insert(&named_quad)?); // assert!(store.insert(&default_quad)?); // assert!(!store.insert(&default_quad)?); assert_eq!(store.len()?, 4); assert_eq!(store.iter().collect::, _>>()?, all_quads); assert_eq!( store .quads_for_pattern(Some(main_s.as_ref()), None, None, None) .collect::, _>>()?, all_quads ); assert_eq!( store .quads_for_pattern(Some(main_s.as_ref()), Some(main_p.as_ref()), None, None) .collect::, _>>()?, all_quads ); assert_eq!( store .quads_for_pattern( Some(main_s.as_ref()), Some(main_p.as_ref()), Some(main_o.as_ref()), None ) .collect::, _>>()?, vec![default_quad.clone(), named_quad.clone()] ); assert_eq!( store .quads_for_pattern( Some(main_s.as_ref()), Some(main_p.as_ref()), Some(main_o.as_ref()), Some(GraphNameRef::DefaultGraph) ) .collect::, _>>()?, vec![default_quad.clone()] ); assert_eq!( store .quads_for_pattern( Some(main_s.as_ref()), Some(main_p.as_ref()), Some(main_o.as_ref()), Some(main_g.as_ref()) ) .collect::, _>>()?, vec![named_quad.clone()] ); assert_eq!( store .quads_for_pattern( Some(main_s.as_ref()), Some(main_p.as_ref()), None, Some(GraphNameRef::DefaultGraph) ) .collect::, _>>()?, default_quads ); assert_eq!( store .quads_for_pattern(Some(main_s.as_ref()), None, Some(main_o.as_ref()), None) .collect::, _>>()?, vec![default_quad.clone(), named_quad.clone()] ); assert_eq!( store .quads_for_pattern( Some(main_s.as_ref()), None, Some(main_o.as_ref()), Some(GraphNameRef::DefaultGraph) ) .collect::, _>>()?, vec![default_quad.clone()] ); assert_eq!( store .quads_for_pattern( Some(main_s.as_ref()), None, Some(main_o.as_ref()), Some(main_g.as_ref()) ) .collect::, _>>()?, vec![named_quad.clone()] ); assert_eq!( store .quads_for_pattern( Some(main_s.as_ref()), None, None, Some(GraphNameRef::DefaultGraph) ) .collect::, _>>()?, default_quads ); assert_eq!( store .quads_for_pattern(None, Some(main_p.as_ref()), None, None) .collect::, _>>()?, all_quads ); assert_eq!( store .quads_for_pattern(None, Some(main_p.as_ref()), Some(main_o.as_ref()), None) .collect::, _>>()?, vec![default_quad.clone(), named_quad.clone()] ); assert_eq!( store .quads_for_pattern(None, None, Some(main_o.as_ref()), None) .collect::, _>>()?, vec![default_quad.clone(), named_quad.clone()] ); assert_eq!( store .quads_for_pattern(None, None, None, Some(GraphNameRef::DefaultGraph)) .collect::, _>>()?, default_quads ); assert_eq!( store .quads_for_pattern( None, Some(main_p.as_ref()), Some(main_o.as_ref()), Some(GraphNameRef::DefaultGraph) ) .collect::, _>>()?, vec![default_quad] ); assert_eq!( store .quads_for_pattern( None, Some(main_p.as_ref()), Some(main_o.as_ref()), Some(main_g.as_ref()) ) .collect::, _>>()?, vec![named_quad] ); Ok(()) } }