From 19d9ddb56e4a7375d8b77dda7fe2905272e87c0f Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 26 Jul 2020 22:56:23 +0200 Subject: [PATCH] Adds methods to dump store content into a graph --- lib/src/store/memory.rs | 70 +++++++++++++++++++++++++++++++++++++--- lib/src/store/mod.rs | 65 +++++++++++++++++++++++++++++++++++-- lib/src/store/rocksdb.rs | 34 +++++++++++++++++-- lib/src/store/sled.rs | 34 +++++++++++++++++-- 4 files changed, 192 insertions(+), 11 deletions(-) diff --git a/lib/src/store/memory.rs b/lib/src/store/memory.rs index 9e9a1008..eb4c9e98 100644 --- a/lib/src/store/memory.rs +++ b/lib/src/store/memory.rs @@ -4,14 +4,16 @@ use crate::error::UnwrapInfallible; use crate::model::*; use crate::sparql::{Query, QueryOptions, QueryResult, SimplePreparedQuery}; use crate::store::numeric_encoder::*; -use crate::store::{load_dataset, load_graph, ReadableEncodedStore, WritableEncodedStore}; +use crate::store::{ + dump_dataset, dump_graph, load_dataset, load_graph, ReadableEncodedStore, WritableEncodedStore, +}; use crate::{DatasetSyntax, Error, GraphSyntax}; use std::collections::hash_map::DefaultHasher; use std::collections::{HashMap, HashSet}; use std::convert::{Infallible, TryInto}; use std::fmt; use std::hash::{BuildHasherDefault, Hash, Hasher}; -use std::io::BufRead; +use std::io::{BufRead, Write}; use std::iter::FromIterator; use std::mem::size_of; use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard}; @@ -255,7 +257,7 @@ impl MemoryStore { /// /// // insertion /// let file = b" ."; - /// store.load_graph(file.as_ref(), GraphSyntax::NTriples, &GraphName::DefaultGraph, None); + /// store.load_graph(file.as_ref(), GraphSyntax::NTriples, &GraphName::DefaultGraph, None)?; /// /// // quad filter /// let results: Vec = store.quads_for_pattern(None, None, None, None).collect(); @@ -285,7 +287,7 @@ impl MemoryStore { /// /// // insertion /// let file = b" ."; - /// store.load_dataset(file.as_ref(), DatasetSyntax::NQuads, None); + /// store.load_dataset(file.as_ref(), DatasetSyntax::NQuads, None)?; /// /// // quad filter /// let results: Vec = store.quads_for_pattern(None, None, None, None).collect(); @@ -328,6 +330,66 @@ impl MemoryStore { iso_canonicalize(self) == iso_canonicalize(other) } + /// Dumps a store graph into a file. + /// + /// Usage example: + /// ``` + /// use oxigraph::model::*; + /// use oxigraph::{MemoryStore, Result, GraphSyntax}; + /// + /// let file = " .\n".as_bytes(); + /// + /// let store = MemoryStore::new(); + /// store.load_graph(file, GraphSyntax::NTriples, &GraphName::DefaultGraph, None)?; + /// + /// let mut buffer = Vec::new(); + /// store.dump_graph(&mut buffer, GraphSyntax::NTriples, &GraphName::DefaultGraph)?; + /// assert_eq!(file, buffer.as_slice()); + /// # Result::Ok(()) + /// ``` + pub fn dump_graph( + &self, + writer: &mut impl Write, + syntax: GraphSyntax, + from_graph_name: &GraphName, + ) -> crate::Result<()> { + dump_graph( + self.quads_for_pattern(None, None, None, Some(from_graph_name)) + .map(|q| Ok(q.into())), + writer, + syntax, + ) + } + + /// Dumps the store dataset into a file. + /// + /// Usage example: + /// ``` + /// use oxigraph::model::*; + /// use oxigraph::{MemoryStore, Result, DatasetSyntax}; + /// + /// let file = " .\n".as_bytes(); + /// + /// let store = MemoryStore::new(); + /// store.load_dataset(file, DatasetSyntax::NQuads, None)?; + /// + /// let mut buffer = Vec::new(); + /// store.dump_dataset(&mut buffer, DatasetSyntax::NQuads)?; + /// assert_eq!(file, buffer.as_slice()); + /// # Result::Ok(()) + /// ``` + pub fn dump_dataset( + &self, + writer: &mut impl Write, + syntax: DatasetSyntax, + ) -> crate::Result<()> { + dump_dataset( + self.quads_for_pattern(None, None, None, None).map(Ok), + writer, + syntax, + ) + } + #[allow(clippy::expect_used)] fn indexes(&self) -> RwLockReadGuard<'_, MemoryStoreIndexes> { self.indexes diff --git a/lib/src/store/mod.rs b/lib/src/store/mod.rs index 6b71fc1a..078c7246 100644 --- a/lib/src/store/mod.rs +++ b/lib/src/store/mod.rs @@ -19,11 +19,15 @@ pub use crate::store::sled::SledStore; use crate::model::*; use crate::store::numeric_encoder::*; use crate::{DatasetSyntax, Error, GraphSyntax, Result}; +use rio_api::formatter::{QuadsFormatter, TriplesFormatter}; use rio_api::parser::{QuadsParser, TriplesParser}; -use rio_turtle::{NQuadsParser, NTriplesParser, TriGParser, TurtleParser}; -use rio_xml::RdfXmlParser; +use rio_turtle::{ + NQuadsFormatter, NQuadsParser, NTriplesFormatter, NTriplesParser, TriGFormatter, TriGParser, + TurtleFormatter, TurtleParser, +}; +use rio_xml::{RdfXmlFormatter, RdfXmlParser}; use std::collections::HashMap; -use std::io::BufRead; +use std::io::{BufRead, Write}; use std::iter::Iterator; pub(crate) trait ReadableEncodedStore: StrLookup { @@ -93,6 +97,37 @@ where }) } +fn dump_graph( + triples: impl Iterator>, + writer: &mut impl Write, + syntax: GraphSyntax, +) -> Result<()> { + match syntax { + GraphSyntax::NTriples => { + let mut formatter = NTriplesFormatter::new(writer); + for triple in triples { + formatter.format(&(&triple?).into())?; + } + formatter.finish(); + } + GraphSyntax::Turtle => { + let mut formatter = TurtleFormatter::new(writer); + for triple in triples { + formatter.format(&(&triple?).into())?; + } + formatter.finish()?; + } + GraphSyntax::RdfXml => { + let mut formatter = RdfXmlFormatter::new(writer)?; + for triple in triples { + formatter.format(&(&triple?).into())?; + } + formatter.finish()?; + } + } + Ok(()) +} + fn load_dataset( store: &mut S, reader: impl BufRead, @@ -121,3 +156,27 @@ where store.insert_encoded(&quad).map_err(|e| e.into()) }) } + +fn dump_dataset( + quads: impl Iterator>, + writer: &mut impl Write, + syntax: DatasetSyntax, +) -> Result<()> { + match syntax { + DatasetSyntax::NQuads => { + let mut formatter = NQuadsFormatter::new(writer); + for quad in quads { + formatter.format(&(&quad?).into())?; + } + formatter.finish(); + } + DatasetSyntax::TriG => { + let mut formatter = TriGFormatter::new(writer); + for quad in quads { + formatter.format(&(&quad?).into())?; + } + formatter.finish()?; + } + } + Ok(()) +} diff --git a/lib/src/store/rocksdb.rs b/lib/src/store/rocksdb.rs index 393c5917..cd4e6de9 100644 --- a/lib/src/store/rocksdb.rs +++ b/lib/src/store/rocksdb.rs @@ -4,11 +4,13 @@ use crate::error::UnwrapInfallible; use crate::model::*; use crate::sparql::{Query, QueryOptions, QueryResult, SimplePreparedQuery}; use crate::store::numeric_encoder::*; -use crate::store::{load_dataset, load_graph, ReadableEncodedStore, WritableEncodedStore}; +use crate::store::{ + dump_dataset, dump_graph, load_dataset, load_graph, ReadableEncodedStore, WritableEncodedStore, +}; use crate::{DatasetSyntax, GraphSyntax, Result}; use rocksdb::*; use std::convert::{Infallible, TryInto}; -use std::io::{BufRead, Cursor}; +use std::io::{BufRead, Cursor, Write}; use std::mem::{take, transmute}; use std::path::Path; use std::sync::Arc; @@ -226,6 +228,34 @@ impl RocksDbStore { transaction.apply() } + /// Dumps a store graph into a file. + /// + /// See `MemoryStore` for a usage example. + pub fn dump_graph( + &self, + writer: &mut impl Write, + syntax: GraphSyntax, + from_graph_name: &GraphName, + ) -> Result<()> { + dump_graph( + self.quads_for_pattern(None, None, None, Some(from_graph_name)) + .map(|q| Ok(q?.into())), + writer, + syntax, + ) + } + + /// Dumps the store dataset into a file. + /// + /// See `MemoryStore` for a usage example. + pub fn dump_dataset(&self, writer: &mut impl Write, syntax: DatasetSyntax) -> Result<()> { + dump_dataset( + self.quads_for_pattern(None, None, None, None), + writer, + syntax, + ) + } + fn id2str_cf(&self) -> &ColumnFamily { get_cf(&self.db, ID2STR_CF) } diff --git a/lib/src/store/sled.rs b/lib/src/store/sled.rs index edb995ae..a369c233 100644 --- a/lib/src/store/sled.rs +++ b/lib/src/store/sled.rs @@ -4,11 +4,13 @@ use crate::error::UnwrapInfallible; use crate::model::*; use crate::sparql::{Query, QueryOptions, QueryResult, SimplePreparedQuery}; use crate::store::numeric_encoder::*; -use crate::store::{load_dataset, load_graph, ReadableEncodedStore, WritableEncodedStore}; +use crate::store::{ + dump_dataset, dump_graph, load_dataset, load_graph, ReadableEncodedStore, WritableEncodedStore, +}; use crate::{DatasetSyntax, Error, GraphSyntax, Result}; use sled::{Batch, Config, Iter, Tree}; use std::convert::{Infallible, TryInto}; -use std::io::{BufRead, Cursor}; +use std::io::{BufRead, Cursor, Write}; use std::path::Path; use std::{fmt, str}; @@ -211,6 +213,34 @@ impl SledStore { DirectWriter::new(self).remove_encoded(&quad) } + /// Dumps a store graph into a file. + /// + /// See `MemoryStore` for a usage example. + pub fn dump_graph( + &self, + writer: &mut impl Write, + syntax: GraphSyntax, + from_graph_name: &GraphName, + ) -> Result<()> { + dump_graph( + self.quads_for_pattern(None, None, None, Some(from_graph_name)) + .map(|q| Ok(q?.into())), + writer, + syntax, + ) + } + + /// Dumps the store dataset into a file. + /// + /// See `MemoryStore` for a usage example. + pub fn dump_dataset(&self, writer: &mut impl Write, syntax: DatasetSyntax) -> Result<()> { + dump_dataset( + self.quads_for_pattern(None, None, None, None), + writer, + syntax, + ) + } + fn contains_encoded(&self, quad: &EncodedQuad) -> Result { let mut buffer = Vec::with_capacity(4 * WRITTEN_TERM_MAX_SIZE); write_spog_quad(&mut buffer, quad);