From 35fe15f796c1230af3b7ab2bc9c8f086aae68839 Mon Sep 17 00:00:00 2001 From: Tpt Date: Wed, 14 Apr 2021 18:55:26 +0200 Subject: [PATCH] Drops RocksDBStore --- README.md | 4 +- bench/bsbm_oxigraph.sh | 6 +- lib/Cargo.toml | 3 +- lib/README.md | 7 +- lib/benches/store.rs | 34 +- lib/src/lib.rs | 7 - lib/src/store/mod.rs | 6 +- lib/src/store/rocksdb.rs | 1665 ---------------------- lib/src/store/sophia.rs | 63 +- lib/tests/rockdb_bc_data/000003.log | Bin 8399 -> 0 bytes lib/tests/rockdb_bc_data/CURRENT | 1 - lib/tests/rockdb_bc_data/IDENTITY | 1 - lib/tests/rockdb_bc_data/LOCK | 0 lib/tests/rockdb_bc_data/MANIFEST-000004 | Bin 559 -> 0 bytes lib/tests/rockdb_bc_data/OPTIONS-000026 | 964 ------------- lib/tests/rocksdb_store.rs | 92 -- server/Cargo.toml | 7 +- server/Dockerfile | 1 - server/README.md | 4 +- server/src/main.rs | 3 - wikibase/Cargo.toml | 2 +- wikibase/Dockerfile | 1 - wikibase/README.md | 2 +- wikibase/src/loader.rs | 39 +- wikibase/src/main.rs | 10 +- 25 files changed, 40 insertions(+), 2882 deletions(-) delete mode 100644 lib/src/store/rocksdb.rs delete mode 100644 lib/tests/rockdb_bc_data/000003.log delete mode 100644 lib/tests/rockdb_bc_data/CURRENT delete mode 100644 lib/tests/rockdb_bc_data/IDENTITY delete mode 100644 lib/tests/rockdb_bc_data/LOCK delete mode 100644 lib/tests/rockdb_bc_data/MANIFEST-000004 delete mode 100644 lib/tests/rockdb_bc_data/OPTIONS-000026 delete mode 100644 lib/tests/rocksdb_store.rs diff --git a/README.md b/README.md index d411cd93..42c53ca0 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Oxigraph Oxigraph is a graph database implementing the [SPARQL](https://www.w3.org/TR/sparql11-overview/) standard. -Its goal is to provide a compliant, safe, and fast graph database based on the [RocksDB](https://rocksdb.org/) and [Sled](https://sled.rs/) key-value stores. +Its goal is to provide a compliant, safe, and fast graph database based on the [Sled](https://sled.rs/) key-value store. It is written in Rust. It also provides a set of utility functions for reading, writing, and processing RDF files. @@ -23,7 +23,7 @@ It is split into multiple parts: * [`pyoxigraph` that exposes Oxigraph to the Python world](https://oxigraph.org/pyoxigraph/). Its source code is in the `python` directory. [![PyPI](https://img.shields.io/pypi/v/pyoxigraph)](https://pypi.org/project/pyoxigraph/) * [JavaScript bindings for Oxigraph](https://www.npmjs.com/package/oxigraph). WebAssembly is used to package Oxigraph into a NodeJS compatible NPM package. Its source code is in the `js` directory. [![npm](https://img.shields.io/npm/v/oxigraph)](https://www.npmjs.com/package/oxigraph) -* [Oxigraph server](https://crates.io/crates/oxigraph_server) that provides a standalone binary of a web server implementing the [SPARQL 1.1 Protocol](https://www.w3.org/TR/sparql11-protocol/) and the [SPARQL 1.1 Graph Store Protocol](https://www.w3.org/TR/sparql11-http-rdf-update/). It uses the [RocksDB](https://rocksdb.org/) key-value store. Its source code is in the `server` directory. +* [Oxigraph server](https://crates.io/crates/oxigraph_server) that provides a standalone binary of a web server implementing the [SPARQL 1.1 Protocol](https://www.w3.org/TR/sparql11-protocol/) and the [SPARQL 1.1 Graph Store Protocol](https://www.w3.org/TR/sparql11-http-rdf-update/). Its source code is in the `server` directory. [![Latest Version](https://img.shields.io/crates/v/oxigraph_server.svg)](https://crates.io/crates/oxigraph_server) [![Docker Image Version (latest semver)](https://img.shields.io/docker/v/oxigraph/oxigraph?sort=semver)](https://hub.docker.com/repository/docker/oxigraph/oxigraph) * [Oxigraph Wikibase](https://crates.io/crates/oxigraph_wikibase), a web server able to synchronize with a [Wikibase instance](https://wikiba.se/). Its source code is in the `wikibase` directory. diff --git a/bench/bsbm_oxigraph.sh b/bench/bsbm_oxigraph.sh index b8e26348..c7d9174f 100755 --- a/bench/bsbm_oxigraph.sh +++ b/bench/bsbm_oxigraph.sh @@ -8,9 +8,9 @@ cargo build --release --manifest-path="../../server/Cargo.toml" ./../../target/release/oxigraph_server --file oxigraph_data --bind 127.0.0.1:7878 & sleep 5 curl -f -X POST -H 'Content-Type:application/n-triples' --data-binary "@explore-${DATASET_SIZE}.nt" http://127.0.0.1:7878/store?default -./testdriver -mt ${PARALLELISM} -ucf usecases/explore/sparql.txt -o "../bsbm.explore.oxigraph.${DATASET_SIZE}.${PARALLELISM}.main-rocksdb.xml" http://127.0.0.1:7878/query -./testdriver -mt ${PARALLELISM} -ucf usecases/exploreAndUpdate/sparql.txt -o "../bsbm.exploreAndUpdate.oxigraph.${DATASET_SIZE}.${PARALLELISM}.main-rocksdb.xml" http://127.0.0.1:7878/query -u http://127.0.0.1:7878/update -udataset "explore-update-${DATASET_SIZE}.nt" -./testdriver -mt ${PARALLELISM} -ucf usecases/businessIntelligence/sparql.txt -o "../bsbm.businessIntelligence.${DATASET_SIZE}.${PARALLELISM}.main-rocksdb.xml" "http://127.0.0.1:7878/query" +./testdriver -mt ${PARALLELISM} -ucf usecases/explore/sparql.txt -o "../bsbm.explore.oxigraph.${DATASET_SIZE}.${PARALLELISM}.main.xml" http://127.0.0.1:7878/query +./testdriver -mt ${PARALLELISM} -ucf usecases/exploreAndUpdate/sparql.txt -o "../bsbm.exploreAndUpdate.oxigraph.${DATASET_SIZE}.${PARALLELISM}.main.xml" http://127.0.0.1:7878/query -u http://127.0.0.1:7878/update -udataset "explore-update-${DATASET_SIZE}.nt" +./testdriver -mt ${PARALLELISM} -ucf usecases/businessIntelligence/sparql.txt -o "../bsbm.businessIntelligence.${DATASET_SIZE}.${PARALLELISM}.main.xml" "http://127.0.0.1:7878/query" kill $! rm -r oxigraph_data rm "explore-${DATASET_SIZE}.nt" diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 3f71c7e1..1c21e881 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -22,7 +22,6 @@ sophia = ["sophia_api"] http_client = ["httparse", "native-tls"] [dependencies] -rocksdb = { version = "0.15", optional = true } sled = { version = "0.34", optional = true } quick-xml = "0.22" rand = "0.8" @@ -61,4 +60,4 @@ wasm-bindgen-test = "0.3" [[bench]] name = "store" harness = false -required-features = ["sled", "rocksdb"] +required-features = ["sled"] diff --git a/lib/README.md b/lib/README.md index afb27c9a..0cc2b9cd 100644 --- a/lib/README.md +++ b/lib/README.md @@ -14,13 +14,8 @@ It also provides a set of utility functions for reading, writing, and processing It currently provides three store implementations providing [SPARQL](https://www.w3.org/TR/sparql11-overview/) capability: * `MemoryStore`: a simple in memory implementation. -* `RocksDbStore`: a file system implementation based on the [RocksDB](https://rocksdb.org/) key-value store. - It requires the `"rocksdb"` feature to be activated. - The [clang](https://clang.llvm.org/) compiler needs to be installed to compile RocksDB. -* `SledStore`: another file system implementation based on the [Sled](https://sled.rs/) key-value store. +* `SledStore`: a file system implementation based on the [Sled](https://sled.rs/) key-value store. It requires the `"sled"` feature to be activated. - Sled is much faster to build than RockDB and does not require a C++ compiler. - However, Sled is still in development, less tested and data load seems much slower than RocksDB. Oxigraph is in heavy development and SPARQL query evaluation has not been optimized yet. diff --git a/lib/benches/store.rs b/lib/benches/store.rs index a8a82701..7453489e 100644 --- a/lib/benches/store.rs +++ b/lib/benches/store.rs @@ -1,16 +1,9 @@ use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; use oxigraph::model::{NamedNode, Quad}; -use oxigraph::{MemoryStore, RocksDbStore, SledStore}; +use oxigraph::{MemoryStore, SledStore}; use rand::random; -use std::env::temp_dir; -use std::fs::remove_dir_all; -criterion_group!( - store_load, - memory_load_bench, - sled_load_bench, - rocksdb_load_bench -); +criterion_group!(store_load, memory_load_bench, sled_load_bench); criterion_main!(store_load); @@ -52,29 +45,6 @@ fn sled_load_bench(c: &mut Criterion) { group.finish(); } -fn rocksdb_load_bench(c: &mut Criterion) { - let mut group = c.benchmark_group("rocksdb"); - group.nresamples(10); - group.sample_size(10); - let temp_dir = temp_dir(); - for size in [100, 1_000, 10_000].iter() { - group.throughput(Throughput::Elements(*size as u64)); - let quads = create_quads(*size); - group.bench_function(BenchmarkId::from_parameter(size), |b| { - b.iter(|| { - let mut dir = temp_dir.clone(); - dir.push(random::().to_string()); - let store = RocksDbStore::open(&dir).unwrap(); - for quad in &quads { - store.insert(quad).unwrap(); - } - remove_dir_all(&dir).unwrap(); - }); - }); - } - group.finish(); -} - fn create_quads(size: u64) -> Vec { (0..size) .map(|_| { diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 3e7c7ffd..6f7c2642 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -5,13 +5,8 @@ //! //! It currently provides three store implementations providing [SPARQL](https://www.w3.org/TR/sparql11-overview/) capability: //! * [`MemoryStore`](store::memory::MemoryStore): a simple in memory implementation. -//! * [`RocksDbStore`](store::rocksdb::RocksDbStore): a file system implementation based on the [RocksDB](https://rocksdb.org/) key-value store. -//! It requires the `"rocksdb"` feature to be activated. -//! The [clang](https://clang.llvm.org/) compiler needs to be installed to compile RocksDB. //! * [`SledStore`](store::sled::SledStore): another file system implementation based on the [Sled](https://sled.rs/) key-value store. //! It requires the `"sled"` feature to be activated. -//! Sled is much faster to build than RockDB and does not require a C++ compiler. -//! However, Sled is still in developpment, less tested and data load seems much slower than RocksDB. //! //! Oxigraph is in heavy development and SPARQL query evaluation has not been optimized yet. //! @@ -128,7 +123,5 @@ pub mod sparql; pub mod store; pub use crate::store::memory::MemoryStore; -#[cfg(feature = "rocksdb")] -pub use crate::store::rocksdb::RocksDbStore; #[cfg(feature = "sled")] pub use crate::store::sled::SledStore; diff --git a/lib/src/store/mod.rs b/lib/src/store/mod.rs index 3957f476..59d9b032 100644 --- a/lib/src/store/mod.rs +++ b/lib/src/store/mod.rs @@ -1,11 +1,9 @@ //! RDF [RDF dataset](https://www.w3.org/TR/rdf11-concepts/#dfn-rdf-dataset) storage implementations. -#[cfg(any(feature = "rocksdb", feature = "sled"))] +#[cfg(feature = "sled")] mod binary_encoder; pub mod memory; pub(crate) mod numeric_encoder; -#[cfg(feature = "rocksdb")] -pub mod rocksdb; #[cfg(feature = "sled")] pub mod sled; pub(crate) mod small_string; @@ -13,8 +11,6 @@ pub(crate) mod small_string; mod sophia; pub use crate::store::memory::MemoryStore; -#[cfg(feature = "rocksdb")] -pub use crate::store::rocksdb::RocksDbStore; #[cfg(feature = "sled")] pub use crate::store::sled::SledStore; diff --git a/lib/src/store/rocksdb.rs b/lib/src/store/rocksdb.rs deleted file mode 100644 index c95c5070..00000000 --- a/lib/src/store/rocksdb.rs +++ /dev/null @@ -1,1665 +0,0 @@ -//! Store based on the [RocksDB](https://rocksdb.org/) key-value database. - -use crate::error::invalid_data_error; -use crate::io::{DatasetFormat, GraphFormat}; -use crate::model::*; -use crate::sparql::{ - evaluate_query, evaluate_update, EvaluationError, Query, QueryOptions, QueryResults, Update, - UpdateOptions, -}; -use crate::store::binary_encoder::*; -use crate::store::numeric_encoder::{ - Decoder, ReadEncoder, StrContainer, StrEncodingAware, StrLookup, WriteEncoder, -}; -use crate::store::{ - dump_dataset, dump_graph, get_encoded_quad_pattern, load_dataset, load_graph, - ReadableEncodedStore, WritableEncodedStore, -}; -use rocksdb::*; -use std::collections::HashMap; -use std::convert::TryInto; -use std::io; -use std::io::{BufRead, Write}; -use std::iter::{once, Once}; -use std::mem::{take, transmute}; -use std::path::Path; -use std::sync::Arc; -use std::{fmt, str}; - -/// Store based on the [RocksDB](https://rocksdb.org/) key-value database. -/// It encodes a [RDF dataset](https://www.w3.org/TR/rdf11-concepts/#dfn-rdf-dataset) and allows to query it using SPARQL. -/// -/// To use it, the `"rocksdb"` feature needs to be activated. -/// -/// Usage example: -/// ``` -/// use oxigraph::RocksDbStore; -/// use oxigraph::model::*; -/// use oxigraph::sparql::QueryResults; -/// # use std::fs::remove_dir_all; -/// -/// # { -/// let store = RocksDbStore::open("example.db")?; -/// -/// // insertion -/// let ex = NamedNode::new("http://example.com")?; -/// let quad = Quad::new(ex.clone(), ex.clone(), ex.clone(), None); -/// store.insert(&quad)?; -/// -/// // quad filter -/// let results: Result,_> = store.quads_for_pattern(None, None, None, None).collect(); -/// assert_eq!(vec![quad], results?); -/// -/// // SPARQL query -/// if let QueryResults::Solutions(mut solutions) = store.query("SELECT ?s WHERE { ?s ?p ?o }")? { -/// assert_eq!(solutions.next().unwrap()?.get("s"), Some(&ex.into())); -/// } -/// # -/// # }; -/// # remove_dir_all("example.db")?; -/// # Result::<_,Box>::Ok(()) -/// ``` -#[derive(Clone)] -pub struct RocksDbStore { - db: Arc, -} - -type EncodedTerm = crate::store::numeric_encoder::EncodedTerm; -type EncodedQuad = crate::store::numeric_encoder::EncodedQuad; - -const ID2STR_CF: &str = "id2str"; -const SPOG_CF: &str = "spog"; -const POSG_CF: &str = "posg"; -const OSPG_CF: &str = "ospg"; -const GSPO_CF: &str = "gspo"; -const GPOS_CF: &str = "gpos"; -const GOSP_CF: &str = "gosp"; -const DSPO_CF: &str = "dspo"; -const DPOS_CF: &str = "dpos"; -const DOSP_CF: &str = "dosp"; -const GRAPHS_CF: &str = "graphs"; - -const COLUMN_FAMILIES: [&str; 11] = [ - ID2STR_CF, SPOG_CF, POSG_CF, OSPG_CF, GSPO_CF, GPOS_CF, GOSP_CF, DSPO_CF, DPOS_CF, DOSP_CF, - GRAPHS_CF, -]; - -const MAX_TRANSACTION_SIZE: usize = 1024; - -impl RocksDbStore { - /// Opens a [`RocksDbStore`]() - pub fn open(path: impl AsRef) -> Result { - let mut options = Options::default(); - options.create_if_missing(true); - options.create_missing_column_families(true); - options.set_compaction_style(DBCompactionStyle::Universal); - - let this = Self { - db: Arc::new(DB::open_cf(&options, path, &COLUMN_FAMILIES).map_err(map_err)?), - }; - - let mut version = this.ensure_version()?; - if version == 0 { - // We migrate to v1 - let mut transaction = this.auto_batch_writer(); - for quad in this.encoded_quads_for_pattern(None, None, None, None) { - let quad = quad?; - if !quad.graph_name.is_default_graph() { - transaction.insert_encoded_named_graph(quad.graph_name)?; - } - } - transaction.apply()?; - version = 1; - this.set_version(version)?; - this.flush()?; - } - - match version { - _ if version < LATEST_STORAGE_VERSION => Err(invalid_data_error(format!( - "The RocksDB database is using the outdated encoding version {}. Automated migration is not supported, please dump the store dataset using a compatible Oxigraph version and load it again using the current version", - version - ))), - LATEST_STORAGE_VERSION => Ok(this), - _ => Err(invalid_data_error(format!( - "The RocksDB database is using the too recent version {}. Upgrade to the latest Oxigraph version to load this database", - version - ))) - } - } - - fn ensure_version(&self) -> Result { - Ok( - if let Some(version) = self.db.get("oxversion").map_err(map_err)? { - let mut buffer = [0; 8]; - buffer.copy_from_slice(&version); - u64::from_be_bytes(buffer) - } else { - self.set_version(LATEST_STORAGE_VERSION)?; - LATEST_STORAGE_VERSION - }, - ) - } - - fn set_version(&self, version: u64) -> Result<(), io::Error> { - self.db - .put("oxversion", &version.to_be_bytes()) - .map_err(map_err) - } - - fn flush(&self) -> Result<(), io::Error> { - let mut options = FlushOptions::new(); - options.set_wait(true); - self.db.flush_opt(&options).map_err(map_err)?; - Ok(()) - } - - /// Executes a [SPARQL 1.1 query](https://www.w3.org/TR/sparql11-query/). - /// - /// See [`MemoryStore`](super::memory::MemoryStore::query()) for a usage example. - pub fn query( - &self, - query: impl TryInto>, - ) -> Result { - self.query_opt(query, QueryOptions::default()) - } - - /// Executes a [SPARQL 1.1 query](https://www.w3.org/TR/sparql11-query/) with some options. - pub fn query_opt( - &self, - query: impl TryInto>, - options: QueryOptions, - ) -> Result { - evaluate_query(self.clone(), query, options) - } - - /// Retrieves quads with a filter on each quad component - /// - /// See [`MemoryStore`](super::memory::MemoryStore::quads_for_pattern()) for a usage example. - pub fn quads_for_pattern( - &self, - subject: Option>, - predicate: Option>, - object: Option>, - graph_name: Option>, - ) -> RocksDbQuadIter { - RocksDbQuadIter { - inner: match get_encoded_quad_pattern(self, subject, predicate, object, graph_name) { - Ok(Some((subject, predicate, object, graph_name))) => QuadIterInner::Quads { - iter: self.encoded_quads_for_pattern(subject, predicate, object, graph_name), - store: self.clone(), - }, - Ok(None) => QuadIterInner::Empty, - Err(error) => QuadIterInner::Error(once(error)), - }, - } - } - - /// Returns all the quads contained in the store - pub fn iter(&self) -> RocksDbQuadIter { - self.quads_for_pattern(None, None, None, None) - } - - /// Checks if this store contains a given quad - pub fn contains<'a>(&self, quad: impl Into>) -> Result { - if let Some(quad) = self.get_encoded_quad(quad.into())? { - self.contains_encoded(&quad) - } else { - Ok(false) - } - } - - /// Returns the number of quads in the store - /// - /// Warning: this function executes a full scan - pub fn len(&self) -> usize { - let default = self - .db - .full_iterator_cf(self.dspo_cf(), IteratorMode::Start) - .count(); - let named = self - .db - .full_iterator_cf(self.gspo_cf(), IteratorMode::Start) - .count(); - default + named - } - - /// Returns if the store is empty - pub fn is_empty(&self) -> bool { - let default = self - .db - .full_iterator_cf(self.dspo_cf(), IteratorMode::Start) - .next() - .is_none(); - let named = self - .db - .full_iterator_cf(self.gspo_cf(), IteratorMode::Start) - .next() - .is_none(); - default && named - } - - /// Executes a [SPARQL 1.1 update](https://www.w3.org/TR/sparql11-update/). - /// - /// The store does not track the existence of empty named graphs. - /// This method has no ACID guarantees. - /// - /// See [`MemoryStore`](super::memory::MemoryStore::update()) for a usage example. - pub fn update( - &self, - update: impl TryInto>, - ) -> Result<(), EvaluationError> { - self.update_opt(update, UpdateOptions::default()) - } - - /// Executes a [SPARQL 1.1 update](https://www.w3.org/TR/sparql11-update/) with some options. - pub fn update_opt( - &self, - update: impl TryInto>, - options: UpdateOptions, - ) -> Result<(), EvaluationError> { - let mut writer = self.auto_batch_writer(); - evaluate_update( - self.clone(), - &mut writer, - update.try_into().map_err(|e| e.into())?, - options, - )?; - Ok(writer.apply()?) - } - - /// Executes an ACID transaction. - /// - /// The transaction is executed if the given closure returns `Ok`. - /// The transaction is rollbacked if the closure returns `Err`. - /// - /// The transaction data are stored in memory while the transaction is not committed or rollbacked. - /// - /// See [`MemoryStore`](super::memory::MemoryStore::transaction()) for a usage example. - pub fn transaction<'a, E: From>( - &'a self, - f: impl FnOnce(&mut RocksDbTransaction<'a>) -> Result<(), E>, - ) -> Result<(), E> { - let mut transaction = RocksDbTransaction { - store: self, - batch: WriteBatch::default(), - buffer: Vec::new(), - new_strings: HashMap::new(), - }; - f(&mut transaction)?; - Ok(transaction.apply()?) - } - - /// Loads a graph file (i.e. triples) into the store - /// - /// Warning: This functions saves the triples in batch. If the parsing fails in the middle of the file, - /// only a part of it may be written. Use a (memory greedy) [transaction](RocksDbStore::transaction()) if you do not want that. - /// - /// See [`MemoryStore`](super::memory::MemoryStore::load_graph()) for a usage example. - /// - /// Errors related to parameter validation like the base IRI use the [`InvalidInput`](std::io::ErrorKind::InvalidInput) error kind. - /// Errors related to a bad syntax in the loaded file use the [`InvalidData`](std::io::ErrorKind::InvalidData) or [`UnexpectedEof`](std::io::ErrorKind::UnexpectedEof) error kinds. - /// Errors related to data loading into the store use the other error kinds. - pub fn load_graph<'a>( - &self, - reader: impl BufRead, - format: GraphFormat, - to_graph_name: impl Into>, - base_iri: Option<&str>, - ) -> Result<(), io::Error> { - let mut transaction = self.auto_batch_writer(); - load_graph( - &mut transaction, - reader, - format, - to_graph_name.into(), - base_iri, - )?; - transaction.apply() - } - - /// Loads a dataset file (i.e. quads) into the store. - /// - /// Warning: This functions saves the quads in batch. If the parsing fails in the middle of the file, - /// only a part of it may be written. Use a (memory greedy) [transaction](RocksDbStore::transaction()) if you do not want that. - /// - /// See [`MemoryStore`](super::memory::MemoryStore::load_dataset()) for a usage example. - /// - /// Errors related to parameter validation like the base IRI use the [`InvalidInput`](std::io::ErrorKind::InvalidInput) error kind. - /// Errors related to a bad syntax in the loaded file use the [`InvalidData`](std::io::ErrorKind::InvalidData) or [`UnexpectedEof`](std::io::ErrorKind::UnexpectedEof) error kinds. - /// Errors related to data loading into the store use the other error kinds. - pub fn load_dataset( - &self, - reader: impl BufRead, - format: DatasetFormat, - base_iri: Option<&str>, - ) -> Result<(), io::Error> { - let mut transaction = self.auto_batch_writer(); - load_dataset(&mut transaction, reader, format, base_iri)?; - transaction.apply() - } - - /// Adds a quad to this store. - /// This operation is atomic and could not leave the store in a bad state. - pub fn insert<'a>(&self, quad: impl Into>) -> Result<(), io::Error> { - let mut transaction = self.auto_batch_writer(); - let quad = transaction.encode_quad(quad.into())?; - transaction.insert_encoded(&quad)?; - transaction.apply() - } - - /// Removes a quad from this store. - /// This operation is atomic and could not leave the store in a bad state. - pub fn remove<'a>(&self, quad: impl Into>) -> Result<(), io::Error> { - if let Some(quad) = self.get_encoded_quad(quad.into())? { - let mut transaction = self.auto_batch_writer(); - transaction.remove_encoded(&quad)?; - transaction.apply() - } else { - Ok(()) - } - } - - /// Dumps a store graph into a file. - /// - /// See [`MemoryStore`](super::memory::MemoryStore::dump_graph()) for a usage example. - pub fn dump_graph<'a>( - &self, - writer: impl Write, - format: GraphFormat, - from_graph_name: impl Into>, - ) -> Result<(), io::Error> { - dump_graph( - self.quads_for_pattern(None, None, None, Some(from_graph_name.into())) - .map(|q| Ok(q?.into())), - writer, - format, - ) - } - - /// Dumps the store into a file. - /// - /// See [`MemoryStore`](super::memory::MemoryStore::dump_dataset()) for a usage example. - pub fn dump_dataset(&self, writer: impl Write, syntax: DatasetFormat) -> Result<(), io::Error> { - dump_dataset(self.iter(), writer, syntax) - } - - /// Returns all the store named graphs - /// - /// See [`MemoryStore`](super::memory::MemoryStore::named_graphs()) for a usage example. - pub fn named_graphs(&self) -> impl Iterator> { - let this = self.clone(); - self.encoded_named_graphs() - .map(move |g| Ok(this.decode_named_or_blank_node(g?)?)) - } - - /// Checks if the store contains a given graph - /// - /// See [`MemoryStore`](super::memory::MemoryStore::contains_named_graph()) for a usage example. - pub fn contains_named_graph<'a>( - &self, - graph_name: impl Into>, - ) -> Result { - if let Some(graph_name) = self.get_encoded_named_or_blank_node(graph_name.into())? { - self.contains_encoded_named_graph(graph_name) - } else { - Ok(false) - } - } - - /// Inserts a graph into this store - /// - /// See [`MemoryStore`](super::memory::MemoryStore::insert_named_graph()) for a usage example. - pub fn insert_named_graph<'a>( - &self, - graph_name: impl Into>, - ) -> Result<(), io::Error> { - let mut transaction = self.auto_batch_writer(); - let graph_name = transaction.encode_named_or_blank_node(graph_name.into())?; - transaction.insert_encoded_named_graph(graph_name)?; - transaction.apply() - } - - /// Clears a graph from this store. - /// - /// See [`MemoryStore`](super::memory::MemoryStore::clear_graph()) for a usage example. - pub fn clear_graph<'a>( - &self, - graph_name: impl Into>, - ) -> Result<(), io::Error> { - if let Some(graph_name) = self.get_encoded_graph_name(graph_name.into())? { - let mut transaction = self.auto_batch_writer(); - transaction.clear_encoded_graph(graph_name)?; - transaction.apply() - } else { - Ok(()) - } - } - - /// Removes a graph from this store. - /// - /// See [`MemoryStore`](super::memory::MemoryStore::remove_named_graph()) for a usage example. - pub fn remove_named_graph<'a>( - &self, - graph_name: impl Into>, - ) -> Result<(), io::Error> { - if let Some(graph_name) = self.get_encoded_named_or_blank_node(graph_name.into())? { - let mut transaction = self.auto_batch_writer(); - transaction.remove_encoded_named_graph(graph_name)?; - transaction.apply() - } else { - Ok(()) - } - } - - /// Clears the store. - /// - /// See [`MemoryStore`](super::memory::MemoryStore::clear()) for a usage example. - pub fn clear(&self) -> Result<(), io::Error> { - let mut transaction = self.auto_batch_writer(); - transaction.clear()?; - transaction.apply() - } - - fn id2str_cf(&self) -> &ColumnFamily { - get_cf(&self.db, ID2STR_CF) - } - - fn spog_cf(&self) -> &ColumnFamily { - get_cf(&self.db, SPOG_CF) - } - - fn posg_cf(&self) -> &ColumnFamily { - get_cf(&self.db, POSG_CF) - } - - fn ospg_cf(&self) -> &ColumnFamily { - get_cf(&self.db, OSPG_CF) - } - - fn gspo_cf(&self) -> &ColumnFamily { - get_cf(&self.db, GSPO_CF) - } - - fn gpos_cf(&self) -> &ColumnFamily { - get_cf(&self.db, GPOS_CF) - } - - fn gosp_cf(&self) -> &ColumnFamily { - get_cf(&self.db, GOSP_CF) - } - - fn dspo_cf(&self) -> &ColumnFamily { - get_cf(&self.db, DSPO_CF) - } - - fn dpos_cf(&self) -> &ColumnFamily { - get_cf(&self.db, DPOS_CF) - } - - fn dosp_cf(&self) -> &ColumnFamily { - get_cf(&self.db, DOSP_CF) - } - - fn graphs_cf(&self) -> &ColumnFamily { - get_cf(&self.db, GRAPHS_CF) - } - fn auto_batch_writer(&self) -> AutoBatchWriter<'_> { - AutoBatchWriter { - store: self, - batch: WriteBatch::default(), - buffer: Vec::default(), - } - } - - fn contains_encoded(&self, quad: &EncodedQuad) -> Result { - let mut buffer = Vec::with_capacity(4 * WRITTEN_TERM_MAX_SIZE); - if quad.graph_name.is_default_graph() { - write_spo_quad(&mut buffer, quad); - Ok(self - .db - .get_pinned_cf(self.dspo_cf(), &buffer) - .map_err(map_err)? - .is_some()) - } else { - write_gspo_quad(&mut buffer, quad); - Ok(self - .db - .get_pinned_cf(self.gspo_cf(), &buffer) - .map_err(map_err)? - .is_some()) - } - } - - fn quads(&self) -> DecodingIndexesIterator { - DecodingIndexesIterator::pair( - self.dspo_quads(Vec::default()), - self.gspo_quads(Vec::default()), - ) - } - - fn quads_for_subject(&self, subject: EncodedTerm) -> DecodingIndexesIterator { - DecodingIndexesIterator::pair( - self.dspo_quads(encode_term(subject)), - self.spog_quads(encode_term(subject)), - ) - } - - fn quads_for_subject_predicate( - &self, - subject: EncodedTerm, - predicate: EncodedTerm, - ) -> DecodingIndexesIterator { - DecodingIndexesIterator::pair( - self.dspo_quads(encode_term_pair(subject, predicate)), - self.spog_quads(encode_term_pair(subject, predicate)), - ) - } - - fn quads_for_subject_predicate_object( - &self, - subject: EncodedTerm, - predicate: EncodedTerm, - object: EncodedTerm, - ) -> DecodingIndexesIterator { - DecodingIndexesIterator::pair( - self.dspo_quads(encode_term_triple(subject, predicate, object)), - self.spog_quads(encode_term_triple(subject, predicate, object)), - ) - } - - fn quads_for_subject_object( - &self, - subject: EncodedTerm, - object: EncodedTerm, - ) -> DecodingIndexesIterator { - DecodingIndexesIterator::pair( - self.dosp_quads(encode_term_pair(object, subject)), - self.ospg_quads(encode_term_pair(object, subject)), - ) - } - - fn quads_for_predicate(&self, predicate: EncodedTerm) -> DecodingIndexesIterator { - DecodingIndexesIterator::pair( - self.dpos_quads(encode_term(predicate)), - self.posg_quads(encode_term(predicate)), - ) - } - - fn quads_for_predicate_object( - &self, - predicate: EncodedTerm, - object: EncodedTerm, - ) -> DecodingIndexesIterator { - DecodingIndexesIterator::pair( - self.dpos_quads(encode_term_pair(predicate, object)), - self.posg_quads(encode_term_pair(predicate, object)), - ) - } - - fn quads_for_object(&self, object: EncodedTerm) -> DecodingIndexesIterator { - DecodingIndexesIterator::pair( - self.dosp_quads(encode_term(object)), - self.ospg_quads(encode_term(object)), - ) - } - - fn quads_for_graph(&self, graph_name: EncodedTerm) -> DecodingIndexesIterator { - DecodingIndexesIterator::new(if graph_name.is_default_graph() { - self.dspo_quads(Vec::default()) - } else { - self.gspo_quads(encode_term(graph_name)) - }) - } - - fn quads_for_subject_graph( - &self, - subject: EncodedTerm, - graph_name: EncodedTerm, - ) -> DecodingIndexesIterator { - DecodingIndexesIterator::new(if graph_name.is_default_graph() { - self.dspo_quads(encode_term(subject)) - } else { - self.gspo_quads(encode_term_pair(graph_name, subject)) - }) - } - - fn quads_for_subject_predicate_graph( - &self, - subject: EncodedTerm, - predicate: EncodedTerm, - graph_name: EncodedTerm, - ) -> DecodingIndexesIterator { - DecodingIndexesIterator::new(if graph_name.is_default_graph() { - self.dspo_quads(encode_term_pair(subject, predicate)) - } else { - self.gspo_quads(encode_term_triple(graph_name, subject, predicate)) - }) - } - - fn quads_for_subject_predicate_object_graph( - &self, - subject: EncodedTerm, - predicate: EncodedTerm, - object: EncodedTerm, - graph_name: EncodedTerm, - ) -> DecodingIndexesIterator { - DecodingIndexesIterator::new(if graph_name.is_default_graph() { - self.dspo_quads(encode_term_triple(subject, predicate, object)) - } else { - self.gspo_quads(encode_term_quad(graph_name, subject, predicate, object)) - }) - } - - fn quads_for_subject_object_graph( - &self, - subject: EncodedTerm, - object: EncodedTerm, - graph_name: EncodedTerm, - ) -> DecodingIndexesIterator { - DecodingIndexesIterator::new(if graph_name.is_default_graph() { - self.dosp_quads(encode_term_pair(object, subject)) - } else { - self.gosp_quads(encode_term_triple(graph_name, object, subject)) - }) - } - - fn quads_for_predicate_graph( - &self, - predicate: EncodedTerm, - graph_name: EncodedTerm, - ) -> DecodingIndexesIterator { - DecodingIndexesIterator::new(if graph_name.is_default_graph() { - self.dpos_quads(encode_term(predicate)) - } else { - self.gpos_quads(encode_term_pair(graph_name, predicate)) - }) - } - - fn quads_for_predicate_object_graph( - &self, - predicate: EncodedTerm, - object: EncodedTerm, - graph_name: EncodedTerm, - ) -> DecodingIndexesIterator { - DecodingIndexesIterator::new(if graph_name.is_default_graph() { - self.dpos_quads(encode_term_pair(predicate, object)) - } else { - self.gpos_quads(encode_term_triple(graph_name, predicate, object)) - }) - } - - fn quads_for_object_graph( - &self, - object: EncodedTerm, - graph_name: EncodedTerm, - ) -> DecodingIndexesIterator { - DecodingIndexesIterator::new(if graph_name.is_default_graph() { - self.dosp_quads(encode_term(object)) - } else { - self.gosp_quads(encode_term_pair(graph_name, object)) - }) - } - - fn spog_quads(&self, prefix: Vec) -> DecodingIndexIterator { - self.inner_quads(self.spog_cf(), prefix, QuadEncoding::Spog) - } - - fn posg_quads(&self, prefix: Vec) -> DecodingIndexIterator { - self.inner_quads(self.posg_cf(), prefix, QuadEncoding::Posg) - } - - fn ospg_quads(&self, prefix: Vec) -> DecodingIndexIterator { - self.inner_quads(self.ospg_cf(), prefix, QuadEncoding::Ospg) - } - - fn gspo_quads(&self, prefix: Vec) -> DecodingIndexIterator { - self.inner_quads(self.gspo_cf(), prefix, QuadEncoding::Gspo) - } - - fn gpos_quads(&self, prefix: Vec) -> DecodingIndexIterator { - self.inner_quads(self.gpos_cf(), prefix, QuadEncoding::Gpos) - } - - fn gosp_quads(&self, prefix: Vec) -> DecodingIndexIterator { - self.inner_quads(self.gosp_cf(), prefix, QuadEncoding::Gosp) - } - - fn dspo_quads(&self, prefix: Vec) -> DecodingIndexIterator { - self.inner_quads(self.dspo_cf(), prefix, QuadEncoding::Dspo) - } - - fn dpos_quads(&self, prefix: Vec) -> DecodingIndexIterator { - self.inner_quads(self.dpos_cf(), prefix, QuadEncoding::Dpos) - } - - fn dosp_quads(&self, prefix: Vec) -> DecodingIndexIterator { - self.inner_quads(self.dosp_cf(), prefix, QuadEncoding::Dosp) - } - - fn inner_quads( - &self, - cf: &ColumnFamily, - prefix: Vec, - encoding: QuadEncoding, - ) -> DecodingIndexIterator { - let mut iter = self.db_iter(cf); - iter.iter.seek(&prefix); - DecodingIndexIterator { - iter, - prefix, - encoding, - } - } - - #[allow(unsafe_code)] - fn db_iter(&self, cf: &ColumnFamily) -> StaticDbRowIterator { - // Valid because it's the same database so db can't be dropped before iter - unsafe { StaticDbRowIterator::new(self.db.raw_iterator_cf(cf), self.db.clone()) } - } -} - -impl fmt::Display for RocksDbStore { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - for t in self.iter() { - writeln!(f, "{}", t.map_err(|_| fmt::Error)?)?; - } - Ok(()) - } -} - -impl StrEncodingAware for RocksDbStore { - type Error = io::Error; - type StrId = StrHash; -} - -impl StrLookup for RocksDbStore { - fn get_str(&self, id: StrHash) -> Result, io::Error> { - self.db - .get_cf(self.id2str_cf(), &id.to_be_bytes()) - .map_err(map_err)? - .map(String::from_utf8) - .transpose() - .map_err(invalid_data_error) - } - - fn get_str_id(&self, value: &str) -> Result, io::Error> { - let id = StrHash::new(value); - Ok( - if self - .db - .get_cf(self.id2str_cf(), &id.to_be_bytes()) - .map_err(map_err)? - .is_some() - { - Some(id) - } else { - None - }, - ) - } -} - -impl ReadableEncodedStore for RocksDbStore { - type QuadsIter = DecodingIndexesIterator; - type GraphsIter = DecodingGraphIterator; - - fn encoded_quads_for_pattern( - &self, - subject: Option, - predicate: Option, - object: Option, - graph_name: Option, - ) -> DecodingIndexesIterator { - match subject { - Some(subject) => match predicate { - Some(predicate) => match object { - Some(object) => match graph_name { - Some(graph_name) => self.quads_for_subject_predicate_object_graph( - subject, predicate, object, graph_name, - ), - None => self.quads_for_subject_predicate_object(subject, predicate, object), - }, - None => match graph_name { - Some(graph_name) => { - self.quads_for_subject_predicate_graph(subject, predicate, graph_name) - } - None => self.quads_for_subject_predicate(subject, predicate), - }, - }, - None => match object { - Some(object) => match graph_name { - Some(graph_name) => { - self.quads_for_subject_object_graph(subject, object, graph_name) - } - None => self.quads_for_subject_object(subject, object), - }, - None => match graph_name { - Some(graph_name) => self.quads_for_subject_graph(subject, graph_name), - None => self.quads_for_subject(subject), - }, - }, - }, - None => match predicate { - Some(predicate) => match object { - Some(object) => match graph_name { - Some(graph_name) => { - self.quads_for_predicate_object_graph(predicate, object, graph_name) - } - - None => self.quads_for_predicate_object(predicate, object), - }, - None => match graph_name { - Some(graph_name) => self.quads_for_predicate_graph(predicate, graph_name), - None => self.quads_for_predicate(predicate), - }, - }, - None => match object { - Some(object) => match graph_name { - Some(graph_name) => self.quads_for_object_graph(object, graph_name), - None => self.quads_for_object(object), - }, - None => match graph_name { - Some(graph_name) => self.quads_for_graph(graph_name), - None => self.quads(), - }, - }, - }, - } - } - - fn encoded_named_graphs(&self) -> DecodingGraphIterator { - let mut iter = self.db_iter(self.graphs_cf()); - iter.iter.seek_to_first(); - DecodingGraphIterator { iter } - } - - fn contains_encoded_named_graph(&self, graph_name: EncodedTerm) -> Result { - Ok(self - .db - .get_cf(self.graphs_cf(), &encode_term(graph_name)) - .map_err(map_err)? - .is_some()) - } -} - -struct AutoBatchWriter<'a> { - store: &'a RocksDbStore, - batch: WriteBatch, - buffer: Vec, -} - -impl AutoBatchWriter<'_> { - fn apply(self) -> Result<(), io::Error> { - self.store.db.write(self.batch).map_err(map_err) - } - - fn apply_if_big(&mut self) -> Result<(), io::Error> { - if self.batch.len() > MAX_TRANSACTION_SIZE { - self.store - .db - .write(take(&mut self.batch)) - .map_err(map_err)?; - } - Ok(()) - } - - fn clear_cf(&mut self, cf: &ColumnFamily) { - self.batch.delete_range_cf( - cf, - [ - u8::MIN, - u8::MIN, - u8::MIN, - u8::MIN, - u8::MIN, - u8::MIN, - u8::MIN, - u8::MIN, - ], - [ - u8::MAX, - u8::MAX, - u8::MAX, - u8::MAX, - u8::MAX, - u8::MAX, - u8::MAX, - u8::MAX, - ], - ) - } -} - -impl StrEncodingAware for AutoBatchWriter<'_> { - type Error = io::Error; - type StrId = StrHash; -} - -impl StrContainer for AutoBatchWriter<'_> { - fn insert_str(&mut self, value: &str) -> Result { - let key = StrHash::new(value); - self.batch - .put_cf(self.store.id2str_cf(), &key.to_be_bytes(), value); - Ok(key) - } -} - -impl WritableEncodedStore for AutoBatchWriter<'_> { - fn insert_encoded(&mut self, quad: &EncodedQuad) -> Result<(), io::Error> { - if quad.graph_name.is_default_graph() { - write_spo_quad(&mut self.buffer, quad); - self.batch.put_cf(self.store.dspo_cf(), &self.buffer, &[]); - self.buffer.clear(); - - write_pos_quad(&mut self.buffer, quad); - self.batch.put_cf(self.store.dpos_cf(), &self.buffer, &[]); - self.buffer.clear(); - - write_osp_quad(&mut self.buffer, quad); - self.batch.put_cf(self.store.dosp_cf(), &self.buffer, &[]); - self.buffer.clear(); - } else { - write_spog_quad(&mut self.buffer, quad); - self.batch.put_cf(self.store.spog_cf(), &self.buffer, &[]); - self.buffer.clear(); - - write_posg_quad(&mut self.buffer, quad); - self.batch.put_cf(self.store.posg_cf(), &self.buffer, &[]); - self.buffer.clear(); - - write_ospg_quad(&mut self.buffer, quad); - self.batch.put_cf(self.store.ospg_cf(), &self.buffer, &[]); - self.buffer.clear(); - - write_gspo_quad(&mut self.buffer, quad); - self.batch.put_cf(self.store.gspo_cf(), &self.buffer, &[]); - self.buffer.clear(); - - write_gpos_quad(&mut self.buffer, quad); - self.batch.put_cf(self.store.gpos_cf(), &self.buffer, &[]); - self.buffer.clear(); - - write_gosp_quad(&mut self.buffer, quad); - self.batch.put_cf(self.store.gosp_cf(), &self.buffer, &[]); - self.buffer.clear(); - - write_term(&mut self.buffer, quad.graph_name); - self.batch.put_cf(self.store.graphs_cf(), &self.buffer, &[]); - self.buffer.clear(); - } - - self.apply_if_big() - } - - fn remove_encoded(&mut self, quad: &EncodedQuad) -> Result<(), io::Error> { - if quad.graph_name.is_default_graph() { - write_spo_quad(&mut self.buffer, quad); - self.batch.delete_cf(self.store.dspo_cf(), &self.buffer); - self.buffer.clear(); - - write_pos_quad(&mut self.buffer, quad); - self.batch.delete_cf(self.store.dpos_cf(), &self.buffer); - self.buffer.clear(); - - write_osp_quad(&mut self.buffer, quad); - self.batch.delete_cf(self.store.dosp_cf(), &self.buffer); - self.buffer.clear(); - } else { - write_spog_quad(&mut self.buffer, quad); - self.batch.delete_cf(self.store.spog_cf(), &self.buffer); - self.buffer.clear(); - - write_posg_quad(&mut self.buffer, quad); - self.batch.delete_cf(self.store.posg_cf(), &self.buffer); - self.buffer.clear(); - - write_ospg_quad(&mut self.buffer, quad); - self.batch.delete_cf(self.store.ospg_cf(), &self.buffer); - self.buffer.clear(); - - write_gspo_quad(&mut self.buffer, quad); - self.batch.delete_cf(self.store.gspo_cf(), &self.buffer); - self.buffer.clear(); - - write_gpos_quad(&mut self.buffer, quad); - self.batch.delete_cf(self.store.gpos_cf(), &self.buffer); - self.buffer.clear(); - - write_gosp_quad(&mut self.buffer, quad); - self.batch.delete_cf(self.store.gosp_cf(), &self.buffer); - self.buffer.clear(); - } - - self.apply_if_big() - } - - fn insert_encoded_named_graph(&mut self, graph_name: EncodedTerm) -> Result<(), io::Error> { - self.batch - .put_cf(self.store.graphs_cf(), &encode_term(graph_name), &[]); - self.apply_if_big() - } - - fn clear_encoded_graph(&mut self, graph_name: EncodedTerm) -> Result<(), io::Error> { - if graph_name.is_default_graph() { - self.clear_cf(self.store.dspo_cf()); - self.clear_cf(self.store.dpos_cf()); - self.clear_cf(self.store.dosp_cf()); - } else { - for quad in self.store.quads_for_graph(graph_name) { - self.remove_encoded(&quad?)?; - } - } - self.apply_if_big() - } - - fn remove_encoded_named_graph(&mut self, graph_name: EncodedTerm) -> Result<(), io::Error> { - for quad in self.store.quads_for_graph(graph_name) { - self.remove_encoded(&quad?)?; - } - self.batch - .delete_cf(self.store.graphs_cf(), &encode_term(graph_name)); - self.apply_if_big() - } - - fn clear(&mut self) -> Result<(), io::Error> { - self.clear_cf(self.store.spog_cf()); - self.clear_cf(self.store.posg_cf()); - self.clear_cf(self.store.ospg_cf()); - self.clear_cf(self.store.gspo_cf()); - self.clear_cf(self.store.gpos_cf()); - self.clear_cf(self.store.gosp_cf()); - self.clear_cf(self.store.dspo_cf()); - self.clear_cf(self.store.dpos_cf()); - self.clear_cf(self.store.dosp_cf()); - self.clear_cf(self.store.graphs_cf()); - self.clear_cf(self.store.id2str_cf()); - self.apply_if_big() - } -} - -/// Allows inserting and deleting quads during an ACID transaction with the [`RocksDbStore`]. -pub struct RocksDbTransaction<'a> { - store: &'a RocksDbStore, - batch: WriteBatch, - buffer: Vec, - new_strings: HashMap, -} - -impl RocksDbTransaction<'_> { - /// Loads a graph file (i.e. triples) into the store during the transaction. - /// - /// Warning: Because the load happens during a transaction, - /// the full file content is temporarily stored in main memory. - /// Do not use for big files. - /// - /// See [`MemoryTransaction`](super::memory::MemoryTransaction::load_graph()) for a usage example. - /// - /// If the file parsing fails in the middle of the file, the triples read before are still - /// considered by the transaction. Rollback the transaction by making the transaction closure - /// return an error if you don't want that. - /// - /// Errors related to parameter validation like the base IRI use the [`InvalidInput`](std::io::ErrorKind::InvalidInput) error kind. - /// Errors related to a bad syntax in the loaded file use the [`InvalidData`](std::io::ErrorKind::InvalidData) or [`UnexpectedEof`](std::io::ErrorKind::UnexpectedEof) error kinds. - pub fn load_graph<'a>( - &mut self, - reader: impl BufRead, - syntax: GraphFormat, - to_graph_name: impl Into>, - base_iri: Option<&str>, - ) -> Result<(), io::Error> { - load_graph(self, reader, syntax, to_graph_name.into(), base_iri)?; - Ok(()) - } - - /// Loads a dataset file (i.e. quads) into the store. into the store during the transaction. - /// - /// Warning: Because the load happens during a transaction, - /// the full file content is temporarily stored in main memory. - /// Do not use for big files. - /// - /// See [`MemoryTransaction`](super::memory::MemoryTransaction::load_dataset()) for a usage example. - /// - /// If the file parsing fails in the middle of the file, the quads read before are still - /// considered by the transaction. Rollback the transaction by making the transaction closure - /// return an error if you don't want that. - /// - /// Errors related to parameter validation like the base IRI use the [`InvalidInput`](std::io::ErrorKind::InvalidInput) error kind. - /// Errors related to a bad syntax in the loaded file use the [`InvalidData`](std::io::ErrorKind::InvalidData) or [`UnexpectedEof`](std::io::ErrorKind::UnexpectedEof) error kinds. - pub fn load_dataset( - &mut self, - reader: impl BufRead, - format: DatasetFormat, - base_iri: Option<&str>, - ) -> Result<(), io::Error> { - load_dataset(self, reader, format, base_iri)?; - Ok(()) - } - - /// Adds a quad to this store during the transaction. - pub fn insert<'a>(&mut self, quad: impl Into>) -> Result<(), io::Error> { - let quad = self.encode_quad(quad.into())?; - self.insert_encoded(&quad) - } - - /// Removes a quad from this store during the transaction. - pub fn remove<'a>(&mut self, quad: impl Into>) -> Result<(), io::Error> { - // Works because all strings could be encoded - if let Some(quad) = self.get_encoded_quad(quad.into()).unwrap() { - self.remove_encoded(&quad) - } else { - Ok(()) - } - } - - fn apply(self) -> Result<(), io::Error> { - self.store.db.write(self.batch).map_err(map_err) - } -} - -impl StrEncodingAware for RocksDbTransaction<'_> { - type Error = io::Error; - type StrId = StrHash; -} - -impl StrLookup for RocksDbTransaction<'_> { - fn get_str(&self, id: StrHash) -> Result, io::Error> { - if let Some(str) = self.new_strings.get(&id) { - Ok(Some(str.clone())) - } else { - self.store.get_str(id) - } - } - - fn get_str_id(&self, value: &str) -> Result, io::Error> { - let id = StrHash::new(value); - Ok( - if self.new_strings.contains_key(&id) - || self - .store - .db - .get_cf(self.store.id2str_cf(), &id.to_be_bytes()) - .map_err(map_err)? - .is_some() - { - Some(id) - } else { - None - }, - ) - } -} - -impl StrContainer for RocksDbTransaction<'_> { - fn insert_str(&mut self, value: &str) -> Result { - let key = StrHash::new(value); - self.batch - .put_cf(self.store.id2str_cf(), &key.to_be_bytes(), value); - self.new_strings.insert(key, value.to_owned()); - Ok(key) - } -} - -impl WritableEncodedStore for RocksDbTransaction<'_> { - fn insert_encoded(&mut self, quad: &EncodedQuad) -> Result<(), io::Error> { - if quad.graph_name.is_default_graph() { - write_spo_quad(&mut self.buffer, quad); - self.batch.put_cf(self.store.dspo_cf(), &self.buffer, &[]); - self.buffer.clear(); - - write_pos_quad(&mut self.buffer, quad); - self.batch.put_cf(self.store.dpos_cf(), &self.buffer, &[]); - self.buffer.clear(); - - write_osp_quad(&mut self.buffer, quad); - self.batch.put_cf(self.store.dosp_cf(), &self.buffer, &[]); - self.buffer.clear(); - } else { - write_spog_quad(&mut self.buffer, quad); - self.batch.put_cf(self.store.spog_cf(), &self.buffer, &[]); - self.buffer.clear(); - - write_posg_quad(&mut self.buffer, quad); - self.batch.put_cf(self.store.posg_cf(), &self.buffer, &[]); - self.buffer.clear(); - - write_ospg_quad(&mut self.buffer, quad); - self.batch.put_cf(self.store.ospg_cf(), &self.buffer, &[]); - self.buffer.clear(); - - write_gspo_quad(&mut self.buffer, quad); - self.batch.put_cf(self.store.gspo_cf(), &self.buffer, &[]); - self.buffer.clear(); - - write_gpos_quad(&mut self.buffer, quad); - self.batch.put_cf(self.store.gpos_cf(), &self.buffer, &[]); - self.buffer.clear(); - - write_gosp_quad(&mut self.buffer, quad); - self.batch.put_cf(self.store.gosp_cf(), &self.buffer, &[]); - self.buffer.clear(); - - write_term(&mut self.buffer, quad.graph_name); - self.batch.put_cf(self.store.graphs_cf(), &self.buffer, &[]); - self.buffer.clear(); - } - - Ok(()) - } - - fn remove_encoded(&mut self, quad: &EncodedQuad) -> Result<(), io::Error> { - if quad.graph_name.is_default_graph() { - write_spo_quad(&mut self.buffer, quad); - self.batch.delete_cf(self.store.dspo_cf(), &self.buffer); - self.buffer.clear(); - - write_pos_quad(&mut self.buffer, quad); - self.batch.delete_cf(self.store.dpos_cf(), &self.buffer); - self.buffer.clear(); - - write_osp_quad(&mut self.buffer, quad); - self.batch.delete_cf(self.store.dosp_cf(), &self.buffer); - self.buffer.clear(); - } else { - write_spog_quad(&mut self.buffer, quad); - self.batch.delete_cf(self.store.spog_cf(), &self.buffer); - self.buffer.clear(); - - write_posg_quad(&mut self.buffer, quad); - self.batch.delete_cf(self.store.posg_cf(), &self.buffer); - self.buffer.clear(); - - write_ospg_quad(&mut self.buffer, quad); - self.batch.delete_cf(self.store.ospg_cf(), &self.buffer); - self.buffer.clear(); - - write_gspo_quad(&mut self.buffer, quad); - self.batch.delete_cf(self.store.gspo_cf(), &self.buffer); - self.buffer.clear(); - - write_gpos_quad(&mut self.buffer, quad); - self.batch.delete_cf(self.store.gpos_cf(), &self.buffer); - self.buffer.clear(); - - write_gosp_quad(&mut self.buffer, quad); - self.batch.delete_cf(self.store.gosp_cf(), &self.buffer); - self.buffer.clear(); - } - - Ok(()) - } - - fn insert_encoded_named_graph(&mut self, graph_name: EncodedTerm) -> Result<(), io::Error> { - self.batch - .put_cf(self.store.graphs_cf(), &encode_term(graph_name), &[]); - Ok(()) - } - - fn clear_encoded_graph(&mut self, _: EncodedTerm) -> Result<(), io::Error> { - Err(io::Error::new( - io::ErrorKind::Other, - "CLEAR is not implemented in RocksDB transactions", - )) - } - - fn remove_encoded_named_graph(&mut self, _: EncodedTerm) -> Result<(), io::Error> { - Err(io::Error::new( - io::ErrorKind::Other, - "DROP is not implemented in RocksDB transactions", - )) - } - - fn clear(&mut self) -> Result<(), Self::Error> { - Err(io::Error::new( - io::ErrorKind::Other, - "CLEAR ALL is not implemented in RocksDB transactions", - )) - } -} - -#[allow(clippy::expect_used)] -fn get_cf<'a>(db: &'a DB, name: &str) -> &'a ColumnFamily { - db.cf_handle(name) - .expect("A column family that should exist in RocksDB does not exist") -} - -struct StaticDbRowIterator { - iter: DBRawIterator<'static>, - _db: Arc, // needed to ensure that DB still lives while iter is used -} - -impl StaticDbRowIterator { - /// Creates a static iterator from a non static one by keeping a ARC reference to the database - /// Caller must ensure that the iterator belongs to the same database - /// - /// This unsafe method is required to get static iterators and ease the usage of the library - /// and make streaming Python bindings possible - #[allow(unsafe_code)] - unsafe fn new(iter: DBRawIterator<'_>, db: Arc) -> Self { - Self { - iter: transmute(iter), - _db: db, - } - } - - fn key(&self) -> Option<&[u8]> { - self.iter.key() - } - - fn next(&mut self) { - self.iter.next() - } -} - -pub(crate) struct DecodingIndexesIterator { - first: DecodingIndexIterator, - second: Option, -} - -impl DecodingIndexesIterator { - fn new(first: DecodingIndexIterator) -> Self { - Self { - first, - second: None, - } - } - - fn pair(first: DecodingIndexIterator, second: DecodingIndexIterator) -> Self { - Self { - first, - second: Some(second), - } - } -} - -impl Iterator for DecodingIndexesIterator { - type Item = Result; - - fn next(&mut self) -> Option> { - if let Some(result) = self.first.next() { - Some(result) - } else if let Some(second) = self.second.as_mut() { - second.next() - } else { - None - } - } -} - -struct DecodingIndexIterator { - iter: StaticDbRowIterator, - prefix: Vec, - encoding: QuadEncoding, -} - -impl Iterator for DecodingIndexIterator { - type Item = Result; - - fn next(&mut self) -> Option> { - if let Some(key) = self.iter.key() { - if key.starts_with(&self.prefix) { - let result = self.encoding.decode(key); - self.iter.next(); - Some(result) - } else { - None - } - } else { - None - } - } -} - -fn map_err(e: Error) -> io::Error { - io::Error::new(io::ErrorKind::Other, e) -} - -/// An iterator returning the quads contained in a [`RocksDbStore`]. -pub struct RocksDbQuadIter { - inner: QuadIterInner, -} - -enum QuadIterInner { - Quads { - iter: DecodingIndexesIterator, - store: RocksDbStore, - }, - Error(Once), - Empty, -} - -impl Iterator for RocksDbQuadIter { - type Item = Result; - - fn next(&mut self) -> Option> { - match &mut self.inner { - QuadIterInner::Quads { iter, store } => Some(match iter.next()? { - Ok(quad) => store.decode_quad(&quad).map_err(|e| e.into()), - Err(error) => Err(error), - }), - QuadIterInner::Error(iter) => iter.next().map(Err), - QuadIterInner::Empty => None, - } - } -} - -pub(crate) struct DecodingGraphIterator { - iter: StaticDbRowIterator, -} - -impl Iterator for DecodingGraphIterator { - type Item = Result; - - fn next(&mut self) -> Option> { - if let Some(key) = self.iter.key() { - let result = decode_term(key); - self.iter.next(); - Some(result) - } else { - None - } - } -} -#[test] -fn store() -> Result<(), io::Error> { - use crate::model::*; - use rand::random; - use std::env::temp_dir; - use std::fs::remove_dir_all; - - let main_s = NamedOrBlankNode::from(BlankNode::default()); - let main_p = NamedNode::new("http://example.com").unwrap(); - let main_o = Term::from(Literal::from(1)); - let main_g = GraphName::from(BlankNode::default()); - - let default_quad = Quad::new(main_s.clone(), main_p.clone(), main_o.clone(), None); - let named_quad = Quad::new( - main_s.clone(), - main_p.clone(), - main_o.clone(), - main_g.clone(), - ); - let default_quads = vec![ - Quad::new(main_s.clone(), main_p.clone(), Literal::from(0), None), - default_quad.clone(), - Quad::new( - main_s.clone(), - main_p.clone(), - Literal::from(200000000), - None, - ), - ]; - let all_quads = vec![ - Quad::new(main_s.clone(), main_p.clone(), Literal::from(0), None), - default_quad.clone(), - Quad::new( - main_s.clone(), - main_p.clone(), - Literal::from(200000000), - None, - ), - named_quad.clone(), - ]; - - let mut repo_path = temp_dir(); - repo_path.push(random::().to_string()); - - { - let store = RocksDbStore::open(&repo_path)?; - for t in &default_quads { - store.insert(t)?; - } - - store.transaction(|t| { - t.remove(&default_quad)?; - t.insert(&named_quad)?; - t.insert(&default_quad) - })?; - - assert_eq!(store.len(), 4); - assert_eq!(store.iter().collect::, _>>()?, all_quads); - assert_eq!( - store - .quads_for_pattern(Some(main_s.as_ref()), None, None, None) - .collect::, _>>()?, - all_quads - ); - assert_eq!( - store - .quads_for_pattern(Some(main_s.as_ref()), Some(main_p.as_ref()), None, None) - .collect::, _>>()?, - all_quads - ); - assert_eq!( - store - .quads_for_pattern( - Some(main_s.as_ref()), - Some(main_p.as_ref()), - Some(main_o.as_ref()), - None - ) - .collect::, _>>()?, - vec![default_quad.clone(), named_quad.clone()] - ); - assert_eq!( - store - .quads_for_pattern( - Some(main_s.as_ref()), - Some(main_p.as_ref()), - Some(main_o.as_ref()), - Some(GraphNameRef::DefaultGraph) - ) - .collect::, _>>()?, - vec![default_quad.clone()] - ); - assert_eq!( - store - .quads_for_pattern( - Some(main_s.as_ref()), - Some(main_p.as_ref()), - Some(main_o.as_ref()), - Some(main_g.as_ref()) - ) - .collect::, _>>()?, - vec![named_quad.clone()] - ); - assert_eq!( - store - .quads_for_pattern( - Some(main_s.as_ref()), - Some(main_p.as_ref()), - None, - Some(GraphNameRef::DefaultGraph) - ) - .collect::, _>>()?, - default_quads - ); - assert_eq!( - store - .quads_for_pattern(Some(main_s.as_ref()), None, Some(main_o.as_ref()), None) - .collect::, _>>()?, - vec![default_quad.clone(), named_quad.clone()] - ); - assert_eq!( - store - .quads_for_pattern( - Some(main_s.as_ref()), - None, - Some(main_o.as_ref()), - Some(GraphNameRef::DefaultGraph) - ) - .collect::, _>>()?, - vec![default_quad.clone()] - ); - assert_eq!( - store - .quads_for_pattern( - Some(main_s.as_ref()), - None, - Some(main_o.as_ref()), - Some(main_g.as_ref()) - ) - .collect::, _>>()?, - vec![named_quad.clone()] - ); - assert_eq!( - store - .quads_for_pattern( - Some(main_s.as_ref()), - None, - None, - Some(GraphNameRef::DefaultGraph) - ) - .collect::, _>>()?, - default_quads - ); - assert_eq!( - store - .quads_for_pattern(None, Some(main_p.as_ref()), None, None) - .collect::, _>>()?, - all_quads - ); - assert_eq!( - store - .quads_for_pattern(None, Some(main_p.as_ref()), Some(main_o.as_ref()), None) - .collect::, _>>()?, - vec![default_quad.clone(), named_quad.clone()] - ); - assert_eq!( - store - .quads_for_pattern(None, None, Some(main_o.as_ref()), None) - .collect::, _>>()?, - vec![default_quad.clone(), named_quad.clone()] - ); - assert_eq!( - store - .quads_for_pattern(None, None, None, Some(GraphNameRef::DefaultGraph)) - .collect::, _>>()?, - default_quads - ); - assert_eq!( - store - .quads_for_pattern( - None, - Some(main_p.as_ref()), - Some(main_o.as_ref()), - Some(GraphNameRef::DefaultGraph) - ) - .collect::, _>>()?, - vec![default_quad] - ); - assert_eq!( - store - .quads_for_pattern( - None, - Some(main_p.as_ref()), - Some(main_o.as_ref()), - Some(main_g.as_ref()) - ) - .collect::, _>>()?, - vec![named_quad] - ); - } - - remove_dir_all(&repo_path)?; - Ok(()) -} diff --git a/lib/src/store/sophia.rs b/lib/src/store/sophia.rs index 2eb0ed04..03613cff 100644 --- a/lib/src/store/sophia.rs +++ b/lib/src/store/sophia.rs @@ -560,65 +560,6 @@ mod sled { sophia_api::test_dataset_impl!(test, SledStore, false, false); } -#[cfg(feature = "rocksdb")] -mod rocksdb { - use super::*; - impl_dataset!(RocksDbStore, std::io::Error, io_quad_map, io_err_map); - - impl MutableDataset for RocksDbStore { - type MutationError = std::io::Error; - fn insert( - &mut self, - s: &TS, - p: &TP, - o: &TO, - g: Option<&TG>, - ) -> MDResult - where - TS: TTerm + ?Sized, - TP: TTerm + ?Sized, - TO: TTerm + ?Sized, - TG: TTerm + ?Sized, - { - let mut buf_s = String::new(); - let mut buf_p = String::new(); - let mut buf_o = String::new(); - let mut buf_g = String::new(); - let quadref = - match convert_quadref(s, p, o, g, &mut buf_s, &mut buf_p, &mut buf_o, &mut buf_g) { - Some(quad) => quad, - None => return Ok(false), - }; - RocksDbStore::insert(self, quadref).map(|_| true) - } - - fn remove( - &mut self, - s: &TS, - p: &TP, - o: &TO, - g: Option<&TG>, - ) -> MDResult - where - TS: TTerm + ?Sized, - TP: TTerm + ?Sized, - TO: TTerm + ?Sized, - TG: TTerm + ?Sized, - { - let mut buf_s = String::new(); - let mut buf_p = String::new(); - let mut buf_o = String::new(); - let mut buf_g = String::new(); - let quadref = - match convert_quadref(s, p, o, g, &mut buf_s, &mut buf_p, &mut buf_o, &mut buf_g) { - Some(quad) => quad, - None => return Ok(false), - }; - RocksDbStore::remove(self, quadref).map(|_| true) - } - } -} - // helper functions #[allow(clippy::unnecessary_wraps)] fn infallible_quad_map<'a>(q: Quad) -> Result, Infallible> { @@ -630,7 +571,7 @@ fn infallible_err_map(_: EvaluationError) -> Infallible { panic!("Unexpected error") } -#[cfg(any(feature = "rocksdb", feature = "sled"))] +#[cfg(feature = "sled")] fn io_quad_map<'a>( res: Result, ) -> Result, std::io::Error> { @@ -640,7 +581,7 @@ fn io_quad_map<'a>( }) } -#[cfg(any(feature = "rocksdb", feature = "sled"))] +#[cfg(feature = "sled")] fn io_err_map(err: EvaluationError) -> std::io::Error { match err { EvaluationError::Io(err) => err, diff --git a/lib/tests/rockdb_bc_data/000003.log b/lib/tests/rockdb_bc_data/000003.log deleted file mode 100644 index fac33cee6c6d85fed4876cab01c04c5d486d3f1e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8399 zcmds5e@xV67=Pd6zQUnyG)iMOyaA&c!W|rfM^L#t7n(4QGq9}u3J09rFYgCGMul4@ znfix?uB}{dA!M-Crj}W@HY{puxfSK6E~Q1LTMbMJs8;WN`GV|uUV87iH`69k-^Z^)RfcYXmHfX0XfIxt25Z$0Y{+4u+eN}RweDh zpSE`IyK?Pi=kupuPaQMNv@$8{bIoS6!I)>r%ggrF)Mooz+yS{c+wD&ew0P|dqUKop zk@ZETy>BI5>i_8EK=Kk5!e3o)cTKvqFo#ltiNY8ubWb4|k#r{UDg;cVL=(b7s}L}8 zChtjJP3aWXQZ&?_`u&@SkL!+(C2Z&|{eQD&O zp|ck^zxF{x{NDSzOE#t8WUlrE-2q<<%ly)=i~`rs@8yoa@>|CKEz6%tpIJP3B_}sO zk6Q+yR|=7pG9X;VW7veS&?*Xn$%7N*)IWr#sEkHc2EO*XKOGpzEni$PdbRKLj`Djj zqh60U=#&Gj#!CrVI6R1;SHeo&wUe=uMQ8mL0WrO0P1~Yh5fCx&E*#uRX{zFmkf@UQ zmag>gM?OE+QDCa*@3@#hxD2!456HfNRStyiltWz~|8eNt$pyO}Sa;%P`^1}>GnF-Y zE#(=FxrVB;^%crBlU}#kF8jE(6uQ0}f@hnIMs<&&hgM1Nh^-Kzq4XB|LRJ(yCcg8_ z+BzxCP(zGoPsR|uO2fPEGo2asbz>KLhkMEoAH4a=p1bfd1bt2hUv*>es{PNdYPztn zqdKMYsi&2Nezl$OhAy9+TAzU}+>-tNj`5-whLSDysXxSzSKcc;DcRk!>9Mukqk`e3 zKy(@?D0EDE?mx0`DWzHJQAJfZI;7IUCk8ied-3}-FCAUfdX`%^ zSk5>to*H(qahfZLkW{#m<%}9zNJUCj6|O@2W^9=d7FtCiFsT}!XztR`sa#8pM#!*n zoX5VECXFl|zT87xy01ph#d*3{Y%|T&_{@7Q)Mpziy!L8`>?~TN^x3SS$x5%Sv0{qg zE^x~VT@xwJirQ9e2CDG?ve^j3^_2?Nn9yJKStPOXY7M&pLt%aKs9F0{q}u^a}%@h=slml14;)kKax zbUS;5toC91*_R`#$nM7B4>fUgE1CR@>Pm<6hIyLI#7cOa%O37PJaoZ=V!dtTFyRQ* zCXNQdz(|l*!@TSvjxZGdWgjsNN+s@PFN_Akpd%vhUeBt`jHBDtL|Dk4|A*PwJSGdA zS&7G9++>Ly2B{q5YHi|Z5DbiDuE_#%grV@0C1Mz)a&EE+qd_ouI=RBUrp}?EK{Z*T z8fn1c9=Ge5o=(O}&zvq727(8ZSZA!u6`LWUVH~X$dqZZ>6HlLlorU29BhyJnmg0i^be*&1`v_~{ z1ZiOgYAMJsP7h)ajU%ju8>EE=s3pI+AbqFox9xZ~(QW03G)0=H7dRwTObWaDub|9d?R+ RYCT~s;vg+tAT2>v)BLt diff --git a/lib/tests/rockdb_bc_data/OPTIONS-000026 b/lib/tests/rockdb_bc_data/OPTIONS-000026 deleted file mode 100644 index 518dd627..00000000 --- a/lib/tests/rockdb_bc_data/OPTIONS-000026 +++ /dev/null @@ -1,964 +0,0 @@ -# This is a RocksDB option file. -# -# For detailed file format spec, please refer to the example file -# in examples/rocksdb_option_file_example.ini -# - -[Version] - rocksdb_version=6.7.3 - options_file_version=1.1 - -[DBOptions] - write_dbid_to_manifest=false - avoid_unnecessary_blocking_io=false - two_write_queues=false - allow_ingest_behind=false - writable_file_max_buffer_size=1048576 - avoid_flush_during_shutdown=false - avoid_flush_during_recovery=false - info_log_level=INFO_LEVEL - access_hint_on_compaction_start=NORMAL - allow_concurrent_memtable_write=true - enable_pipelined_write=false - stats_dump_period_sec=600 - stats_persist_period_sec=600 - strict_bytes_per_sync=false - WAL_ttl_seconds=0 - WAL_size_limit_MB=0 - max_subcompactions=1 - dump_malloc_stats=false - db_log_dir= - wal_recovery_mode=kPointInTimeRecovery - log_file_time_to_roll=0 - enable_write_thread_adaptive_yield=true - recycle_log_file_num=0 - table_cache_numshardbits=6 - atomic_flush=false - preserve_deletes=false - stats_history_buffer_size=1048576 - max_open_files=-1 - max_file_opening_threads=16 - delete_obsolete_files_period_micros=21600000000 - max_background_flushes=-1 - write_thread_slow_yield_usec=3 - base_background_compactions=-1 - manual_wal_flush=false - wal_dir=tests/rockdb_bc_data - max_background_compactions=-1 - bytes_per_sync=0 - max_background_jobs=2 - use_fsync=false - unordered_write=false - fail_if_options_file_error=false - random_access_max_buffer_size=1048576 - compaction_readahead_size=0 - wal_bytes_per_sync=0 - new_table_reader_for_compaction_inputs=false - skip_stats_update_on_db_open=false - persist_stats_to_disk=false - skip_log_error_on_recovery=false - log_readahead_size=0 - is_fd_close_on_exec=true - use_adaptive_mutex=false - error_if_exists=false - write_thread_max_yield_usec=100 - enable_thread_tracking=false - db_write_buffer_size=0 - create_missing_column_families=true - paranoid_checks=true - create_if_missing=true - max_manifest_file_size=1073741824 - allow_2pc=false - max_total_wal_size=0 - use_direct_io_for_flush_and_compaction=false - manifest_preallocation_size=4194304 - use_direct_reads=false - delayed_write_rate=16777216 - allow_fallocate=true - max_write_batch_group_size_bytes=1048576 - keep_log_file_num=1000 - allow_mmap_reads=false - max_log_file_size=0 - allow_mmap_writes=false - advise_random_on_open=true - - -[CFOptions "default"] - sample_for_compression=0 - compaction_pri=kMinOverlappingRatio - merge_operator=nullptr - compaction_filter_factory=nullptr - memtable_factory=SkipListFactory - memtable_insert_with_hint_prefix_extractor=nullptr - comparator=leveldb.BytewiseComparator - target_file_size_base=67108864 - max_sequential_skip_in_iterations=8 - compaction_style=kCompactionStyleLevel - max_bytes_for_level_base=268435456 - bloom_locality=0 - write_buffer_size=67108864 - compression_per_level= - memtable_huge_page_size=0 - max_successive_merges=0 - arena_block_size=8388608 - memtable_whole_key_filtering=false - target_file_size_multiplier=1 - max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 - num_levels=7 - min_write_buffer_number_to_merge=1 - max_write_buffer_number_to_maintain=0 - max_write_buffer_number=2 - compression=kSnappyCompression - level0_stop_writes_trigger=36 - level0_slowdown_writes_trigger=20 - compaction_filter=nullptr - level0_file_num_compaction_trigger=4 - max_compaction_bytes=1677721600 - compaction_options_universal={stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;allow_trivial_move=false;max_merge_width=4294967295;max_size_amplification_percent=200;min_merge_width=2;size_ratio=1;} - memtable_prefix_bloom_size_ratio=0.000000 - max_write_buffer_size_to_maintain=0 - hard_pending_compaction_bytes_limit=274877906944 - ttl=2592000 - table_factory=BlockBasedTable - soft_pending_compaction_bytes_limit=68719476736 - prefix_extractor=nullptr - bottommost_compression=kDisableCompressionOption - force_consistency_checks=false - paranoid_file_checks=false - compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;} - max_bytes_for_level_multiplier=10.000000 - optimize_filters_for_hits=false - level_compaction_dynamic_level_bytes=false - inplace_update_num_locks=10000 - inplace_update_support=false - periodic_compaction_seconds=0 - disable_auto_compactions=false - report_bg_io_stats=false - -[TableOptions/BlockBasedTable "default"] - pin_top_level_index_and_filter=true - enable_index_compression=true - read_amp_bytes_per_bit=8589934592 - format_version=2 - block_align=false - metadata_block_size=4096 - block_size_deviation=10 - partition_filters=false - block_size=4096 - index_block_restart_interval=1 - no_block_cache=false - checksum=kCRC32c - whole_key_filtering=true - index_shortening=kShortenSeparators - data_block_index_type=kDataBlockBinarySearch - index_type=kBinarySearch - verify_compression=false - filter_policy=nullptr - data_block_hash_table_util_ratio=0.750000 - pin_l0_filter_and_index_blocks_in_cache=false - block_restart_interval=16 - cache_index_and_filter_blocks_with_high_priority=true - cache_index_and_filter_blocks=false - hash_index_allow_collision=true - flush_block_policy_factory=FlushBlockBySizePolicyFactory - - -[CFOptions "id2str"] - sample_for_compression=0 - compaction_pri=kMinOverlappingRatio - merge_operator=nullptr - compaction_filter_factory=nullptr - memtable_factory=SkipListFactory - memtable_insert_with_hint_prefix_extractor=nullptr - comparator=leveldb.BytewiseComparator - target_file_size_base=67108864 - max_sequential_skip_in_iterations=8 - compaction_style=kCompactionStyleLevel - max_bytes_for_level_base=268435456 - bloom_locality=0 - write_buffer_size=67108864 - compression_per_level= - memtable_huge_page_size=0 - max_successive_merges=0 - arena_block_size=8388608 - memtable_whole_key_filtering=false - target_file_size_multiplier=1 - max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 - num_levels=7 - min_write_buffer_number_to_merge=1 - max_write_buffer_number_to_maintain=0 - max_write_buffer_number=2 - compression=kSnappyCompression - level0_stop_writes_trigger=36 - level0_slowdown_writes_trigger=20 - compaction_filter=nullptr - level0_file_num_compaction_trigger=4 - max_compaction_bytes=1677721600 - compaction_options_universal={stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;allow_trivial_move=false;max_merge_width=4294967295;max_size_amplification_percent=200;min_merge_width=2;size_ratio=1;} - memtable_prefix_bloom_size_ratio=0.000000 - max_write_buffer_size_to_maintain=0 - hard_pending_compaction_bytes_limit=274877906944 - ttl=2592000 - table_factory=BlockBasedTable - soft_pending_compaction_bytes_limit=68719476736 - prefix_extractor=nullptr - bottommost_compression=kDisableCompressionOption - force_consistency_checks=false - paranoid_file_checks=false - compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;} - max_bytes_for_level_multiplier=10.000000 - optimize_filters_for_hits=false - level_compaction_dynamic_level_bytes=false - inplace_update_num_locks=10000 - inplace_update_support=false - periodic_compaction_seconds=0 - disable_auto_compactions=false - report_bg_io_stats=false - -[TableOptions/BlockBasedTable "id2str"] - pin_top_level_index_and_filter=true - enable_index_compression=true - read_amp_bytes_per_bit=8589934592 - format_version=2 - block_align=false - metadata_block_size=4096 - block_size_deviation=10 - partition_filters=false - block_size=4096 - index_block_restart_interval=1 - no_block_cache=false - checksum=kCRC32c - whole_key_filtering=true - index_shortening=kShortenSeparators - data_block_index_type=kDataBlockBinarySearch - index_type=kBinarySearch - verify_compression=false - filter_policy=nullptr - data_block_hash_table_util_ratio=0.750000 - pin_l0_filter_and_index_blocks_in_cache=false - block_restart_interval=16 - cache_index_and_filter_blocks_with_high_priority=true - cache_index_and_filter_blocks=false - hash_index_allow_collision=true - flush_block_policy_factory=FlushBlockBySizePolicyFactory - - -[CFOptions "spog"] - sample_for_compression=0 - compaction_pri=kMinOverlappingRatio - merge_operator=nullptr - compaction_filter_factory=nullptr - memtable_factory=SkipListFactory - memtable_insert_with_hint_prefix_extractor=nullptr - comparator=leveldb.BytewiseComparator - target_file_size_base=67108864 - max_sequential_skip_in_iterations=8 - compaction_style=kCompactionStyleLevel - max_bytes_for_level_base=268435456 - bloom_locality=0 - write_buffer_size=67108864 - compression_per_level= - memtable_huge_page_size=0 - max_successive_merges=0 - arena_block_size=8388608 - memtable_whole_key_filtering=false - target_file_size_multiplier=1 - max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 - num_levels=7 - min_write_buffer_number_to_merge=1 - max_write_buffer_number_to_maintain=0 - max_write_buffer_number=2 - compression=kSnappyCompression - level0_stop_writes_trigger=36 - level0_slowdown_writes_trigger=20 - compaction_filter=nullptr - level0_file_num_compaction_trigger=4 - max_compaction_bytes=1677721600 - compaction_options_universal={stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;allow_trivial_move=false;max_merge_width=4294967295;max_size_amplification_percent=200;min_merge_width=2;size_ratio=1;} - memtable_prefix_bloom_size_ratio=0.000000 - max_write_buffer_size_to_maintain=0 - hard_pending_compaction_bytes_limit=274877906944 - ttl=2592000 - table_factory=BlockBasedTable - soft_pending_compaction_bytes_limit=68719476736 - prefix_extractor=nullptr - bottommost_compression=kDisableCompressionOption - force_consistency_checks=false - paranoid_file_checks=false - compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;} - max_bytes_for_level_multiplier=10.000000 - optimize_filters_for_hits=false - level_compaction_dynamic_level_bytes=false - inplace_update_num_locks=10000 - inplace_update_support=false - periodic_compaction_seconds=0 - disable_auto_compactions=false - report_bg_io_stats=false - -[TableOptions/BlockBasedTable "spog"] - pin_top_level_index_and_filter=true - enable_index_compression=true - read_amp_bytes_per_bit=8589934592 - format_version=2 - block_align=false - metadata_block_size=4096 - block_size_deviation=10 - partition_filters=false - block_size=4096 - index_block_restart_interval=1 - no_block_cache=false - checksum=kCRC32c - whole_key_filtering=true - index_shortening=kShortenSeparators - data_block_index_type=kDataBlockBinarySearch - index_type=kBinarySearch - verify_compression=false - filter_policy=nullptr - data_block_hash_table_util_ratio=0.750000 - pin_l0_filter_and_index_blocks_in_cache=false - block_restart_interval=16 - cache_index_and_filter_blocks_with_high_priority=true - cache_index_and_filter_blocks=false - hash_index_allow_collision=true - flush_block_policy_factory=FlushBlockBySizePolicyFactory - - -[CFOptions "posg"] - sample_for_compression=0 - compaction_pri=kMinOverlappingRatio - merge_operator=nullptr - compaction_filter_factory=nullptr - memtable_factory=SkipListFactory - memtable_insert_with_hint_prefix_extractor=nullptr - comparator=leveldb.BytewiseComparator - target_file_size_base=67108864 - max_sequential_skip_in_iterations=8 - compaction_style=kCompactionStyleLevel - max_bytes_for_level_base=268435456 - bloom_locality=0 - write_buffer_size=67108864 - compression_per_level= - memtable_huge_page_size=0 - max_successive_merges=0 - arena_block_size=8388608 - memtable_whole_key_filtering=false - target_file_size_multiplier=1 - max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 - num_levels=7 - min_write_buffer_number_to_merge=1 - max_write_buffer_number_to_maintain=0 - max_write_buffer_number=2 - compression=kSnappyCompression - level0_stop_writes_trigger=36 - level0_slowdown_writes_trigger=20 - compaction_filter=nullptr - level0_file_num_compaction_trigger=4 - max_compaction_bytes=1677721600 - compaction_options_universal={stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;allow_trivial_move=false;max_merge_width=4294967295;max_size_amplification_percent=200;min_merge_width=2;size_ratio=1;} - memtable_prefix_bloom_size_ratio=0.000000 - max_write_buffer_size_to_maintain=0 - hard_pending_compaction_bytes_limit=274877906944 - ttl=2592000 - table_factory=BlockBasedTable - soft_pending_compaction_bytes_limit=68719476736 - prefix_extractor=nullptr - bottommost_compression=kDisableCompressionOption - force_consistency_checks=false - paranoid_file_checks=false - compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;} - max_bytes_for_level_multiplier=10.000000 - optimize_filters_for_hits=false - level_compaction_dynamic_level_bytes=false - inplace_update_num_locks=10000 - inplace_update_support=false - periodic_compaction_seconds=0 - disable_auto_compactions=false - report_bg_io_stats=false - -[TableOptions/BlockBasedTable "posg"] - pin_top_level_index_and_filter=true - enable_index_compression=true - read_amp_bytes_per_bit=8589934592 - format_version=2 - block_align=false - metadata_block_size=4096 - block_size_deviation=10 - partition_filters=false - block_size=4096 - index_block_restart_interval=1 - no_block_cache=false - checksum=kCRC32c - whole_key_filtering=true - index_shortening=kShortenSeparators - data_block_index_type=kDataBlockBinarySearch - index_type=kBinarySearch - verify_compression=false - filter_policy=nullptr - data_block_hash_table_util_ratio=0.750000 - pin_l0_filter_and_index_blocks_in_cache=false - block_restart_interval=16 - cache_index_and_filter_blocks_with_high_priority=true - cache_index_and_filter_blocks=false - hash_index_allow_collision=true - flush_block_policy_factory=FlushBlockBySizePolicyFactory - - -[CFOptions "ospg"] - sample_for_compression=0 - compaction_pri=kMinOverlappingRatio - merge_operator=nullptr - compaction_filter_factory=nullptr - memtable_factory=SkipListFactory - memtable_insert_with_hint_prefix_extractor=nullptr - comparator=leveldb.BytewiseComparator - target_file_size_base=67108864 - max_sequential_skip_in_iterations=8 - compaction_style=kCompactionStyleLevel - max_bytes_for_level_base=268435456 - bloom_locality=0 - write_buffer_size=67108864 - compression_per_level= - memtable_huge_page_size=0 - max_successive_merges=0 - arena_block_size=8388608 - memtable_whole_key_filtering=false - target_file_size_multiplier=1 - max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 - num_levels=7 - min_write_buffer_number_to_merge=1 - max_write_buffer_number_to_maintain=0 - max_write_buffer_number=2 - compression=kSnappyCompression - level0_stop_writes_trigger=36 - level0_slowdown_writes_trigger=20 - compaction_filter=nullptr - level0_file_num_compaction_trigger=4 - max_compaction_bytes=1677721600 - compaction_options_universal={stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;allow_trivial_move=false;max_merge_width=4294967295;max_size_amplification_percent=200;min_merge_width=2;size_ratio=1;} - memtable_prefix_bloom_size_ratio=0.000000 - max_write_buffer_size_to_maintain=0 - hard_pending_compaction_bytes_limit=274877906944 - ttl=2592000 - table_factory=BlockBasedTable - soft_pending_compaction_bytes_limit=68719476736 - prefix_extractor=nullptr - bottommost_compression=kDisableCompressionOption - force_consistency_checks=false - paranoid_file_checks=false - compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;} - max_bytes_for_level_multiplier=10.000000 - optimize_filters_for_hits=false - level_compaction_dynamic_level_bytes=false - inplace_update_num_locks=10000 - inplace_update_support=false - periodic_compaction_seconds=0 - disable_auto_compactions=false - report_bg_io_stats=false - -[TableOptions/BlockBasedTable "ospg"] - pin_top_level_index_and_filter=true - enable_index_compression=true - read_amp_bytes_per_bit=8589934592 - format_version=2 - block_align=false - metadata_block_size=4096 - block_size_deviation=10 - partition_filters=false - block_size=4096 - index_block_restart_interval=1 - no_block_cache=false - checksum=kCRC32c - whole_key_filtering=true - index_shortening=kShortenSeparators - data_block_index_type=kDataBlockBinarySearch - index_type=kBinarySearch - verify_compression=false - filter_policy=nullptr - data_block_hash_table_util_ratio=0.750000 - pin_l0_filter_and_index_blocks_in_cache=false - block_restart_interval=16 - cache_index_and_filter_blocks_with_high_priority=true - cache_index_and_filter_blocks=false - hash_index_allow_collision=true - flush_block_policy_factory=FlushBlockBySizePolicyFactory - - -[CFOptions "gspo"] - sample_for_compression=0 - compaction_pri=kMinOverlappingRatio - merge_operator=nullptr - compaction_filter_factory=nullptr - memtable_factory=SkipListFactory - memtable_insert_with_hint_prefix_extractor=nullptr - comparator=leveldb.BytewiseComparator - target_file_size_base=67108864 - max_sequential_skip_in_iterations=8 - compaction_style=kCompactionStyleLevel - max_bytes_for_level_base=268435456 - bloom_locality=0 - write_buffer_size=67108864 - compression_per_level= - memtable_huge_page_size=0 - max_successive_merges=0 - arena_block_size=8388608 - memtable_whole_key_filtering=false - target_file_size_multiplier=1 - max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 - num_levels=7 - min_write_buffer_number_to_merge=1 - max_write_buffer_number_to_maintain=0 - max_write_buffer_number=2 - compression=kSnappyCompression - level0_stop_writes_trigger=36 - level0_slowdown_writes_trigger=20 - compaction_filter=nullptr - level0_file_num_compaction_trigger=4 - max_compaction_bytes=1677721600 - compaction_options_universal={stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;allow_trivial_move=false;max_merge_width=4294967295;max_size_amplification_percent=200;min_merge_width=2;size_ratio=1;} - memtable_prefix_bloom_size_ratio=0.000000 - max_write_buffer_size_to_maintain=0 - hard_pending_compaction_bytes_limit=274877906944 - ttl=2592000 - table_factory=BlockBasedTable - soft_pending_compaction_bytes_limit=68719476736 - prefix_extractor=nullptr - bottommost_compression=kDisableCompressionOption - force_consistency_checks=false - paranoid_file_checks=false - compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;} - max_bytes_for_level_multiplier=10.000000 - optimize_filters_for_hits=false - level_compaction_dynamic_level_bytes=false - inplace_update_num_locks=10000 - inplace_update_support=false - periodic_compaction_seconds=0 - disable_auto_compactions=false - report_bg_io_stats=false - -[TableOptions/BlockBasedTable "gspo"] - pin_top_level_index_and_filter=true - enable_index_compression=true - read_amp_bytes_per_bit=8589934592 - format_version=2 - block_align=false - metadata_block_size=4096 - block_size_deviation=10 - partition_filters=false - block_size=4096 - index_block_restart_interval=1 - no_block_cache=false - checksum=kCRC32c - whole_key_filtering=true - index_shortening=kShortenSeparators - data_block_index_type=kDataBlockBinarySearch - index_type=kBinarySearch - verify_compression=false - filter_policy=nullptr - data_block_hash_table_util_ratio=0.750000 - pin_l0_filter_and_index_blocks_in_cache=false - block_restart_interval=16 - cache_index_and_filter_blocks_with_high_priority=true - cache_index_and_filter_blocks=false - hash_index_allow_collision=true - flush_block_policy_factory=FlushBlockBySizePolicyFactory - - -[CFOptions "gpos"] - sample_for_compression=0 - compaction_pri=kMinOverlappingRatio - merge_operator=nullptr - compaction_filter_factory=nullptr - memtable_factory=SkipListFactory - memtable_insert_with_hint_prefix_extractor=nullptr - comparator=leveldb.BytewiseComparator - target_file_size_base=67108864 - max_sequential_skip_in_iterations=8 - compaction_style=kCompactionStyleLevel - max_bytes_for_level_base=268435456 - bloom_locality=0 - write_buffer_size=67108864 - compression_per_level= - memtable_huge_page_size=0 - max_successive_merges=0 - arena_block_size=8388608 - memtable_whole_key_filtering=false - target_file_size_multiplier=1 - max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 - num_levels=7 - min_write_buffer_number_to_merge=1 - max_write_buffer_number_to_maintain=0 - max_write_buffer_number=2 - compression=kSnappyCompression - level0_stop_writes_trigger=36 - level0_slowdown_writes_trigger=20 - compaction_filter=nullptr - level0_file_num_compaction_trigger=4 - max_compaction_bytes=1677721600 - compaction_options_universal={stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;allow_trivial_move=false;max_merge_width=4294967295;max_size_amplification_percent=200;min_merge_width=2;size_ratio=1;} - memtable_prefix_bloom_size_ratio=0.000000 - max_write_buffer_size_to_maintain=0 - hard_pending_compaction_bytes_limit=274877906944 - ttl=2592000 - table_factory=BlockBasedTable - soft_pending_compaction_bytes_limit=68719476736 - prefix_extractor=nullptr - bottommost_compression=kDisableCompressionOption - force_consistency_checks=false - paranoid_file_checks=false - compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;} - max_bytes_for_level_multiplier=10.000000 - optimize_filters_for_hits=false - level_compaction_dynamic_level_bytes=false - inplace_update_num_locks=10000 - inplace_update_support=false - periodic_compaction_seconds=0 - disable_auto_compactions=false - report_bg_io_stats=false - -[TableOptions/BlockBasedTable "gpos"] - pin_top_level_index_and_filter=true - enable_index_compression=true - read_amp_bytes_per_bit=8589934592 - format_version=2 - block_align=false - metadata_block_size=4096 - block_size_deviation=10 - partition_filters=false - block_size=4096 - index_block_restart_interval=1 - no_block_cache=false - checksum=kCRC32c - whole_key_filtering=true - index_shortening=kShortenSeparators - data_block_index_type=kDataBlockBinarySearch - index_type=kBinarySearch - verify_compression=false - filter_policy=nullptr - data_block_hash_table_util_ratio=0.750000 - pin_l0_filter_and_index_blocks_in_cache=false - block_restart_interval=16 - cache_index_and_filter_blocks_with_high_priority=true - cache_index_and_filter_blocks=false - hash_index_allow_collision=true - flush_block_policy_factory=FlushBlockBySizePolicyFactory - - -[CFOptions "gosp"] - sample_for_compression=0 - compaction_pri=kMinOverlappingRatio - merge_operator=nullptr - compaction_filter_factory=nullptr - memtable_factory=SkipListFactory - memtable_insert_with_hint_prefix_extractor=nullptr - comparator=leveldb.BytewiseComparator - target_file_size_base=67108864 - max_sequential_skip_in_iterations=8 - compaction_style=kCompactionStyleLevel - max_bytes_for_level_base=268435456 - bloom_locality=0 - write_buffer_size=67108864 - compression_per_level= - memtable_huge_page_size=0 - max_successive_merges=0 - arena_block_size=8388608 - memtable_whole_key_filtering=false - target_file_size_multiplier=1 - max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 - num_levels=7 - min_write_buffer_number_to_merge=1 - max_write_buffer_number_to_maintain=0 - max_write_buffer_number=2 - compression=kSnappyCompression - level0_stop_writes_trigger=36 - level0_slowdown_writes_trigger=20 - compaction_filter=nullptr - level0_file_num_compaction_trigger=4 - max_compaction_bytes=1677721600 - compaction_options_universal={stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;allow_trivial_move=false;max_merge_width=4294967295;max_size_amplification_percent=200;min_merge_width=2;size_ratio=1;} - memtable_prefix_bloom_size_ratio=0.000000 - max_write_buffer_size_to_maintain=0 - hard_pending_compaction_bytes_limit=274877906944 - ttl=2592000 - table_factory=BlockBasedTable - soft_pending_compaction_bytes_limit=68719476736 - prefix_extractor=nullptr - bottommost_compression=kDisableCompressionOption - force_consistency_checks=false - paranoid_file_checks=false - compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;} - max_bytes_for_level_multiplier=10.000000 - optimize_filters_for_hits=false - level_compaction_dynamic_level_bytes=false - inplace_update_num_locks=10000 - inplace_update_support=false - periodic_compaction_seconds=0 - disable_auto_compactions=false - report_bg_io_stats=false - -[TableOptions/BlockBasedTable "gosp"] - pin_top_level_index_and_filter=true - enable_index_compression=true - read_amp_bytes_per_bit=8589934592 - format_version=2 - block_align=false - metadata_block_size=4096 - block_size_deviation=10 - partition_filters=false - block_size=4096 - index_block_restart_interval=1 - no_block_cache=false - checksum=kCRC32c - whole_key_filtering=true - index_shortening=kShortenSeparators - data_block_index_type=kDataBlockBinarySearch - index_type=kBinarySearch - verify_compression=false - filter_policy=nullptr - data_block_hash_table_util_ratio=0.750000 - pin_l0_filter_and_index_blocks_in_cache=false - block_restart_interval=16 - cache_index_and_filter_blocks_with_high_priority=true - cache_index_and_filter_blocks=false - hash_index_allow_collision=true - flush_block_policy_factory=FlushBlockBySizePolicyFactory - - -[CFOptions "dspo"] - sample_for_compression=0 - compaction_pri=kMinOverlappingRatio - merge_operator=nullptr - compaction_filter_factory=nullptr - memtable_factory=SkipListFactory - memtable_insert_with_hint_prefix_extractor=nullptr - comparator=leveldb.BytewiseComparator - target_file_size_base=67108864 - max_sequential_skip_in_iterations=8 - compaction_style=kCompactionStyleLevel - max_bytes_for_level_base=268435456 - bloom_locality=0 - write_buffer_size=67108864 - compression_per_level= - memtable_huge_page_size=0 - max_successive_merges=0 - arena_block_size=8388608 - memtable_whole_key_filtering=false - target_file_size_multiplier=1 - max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 - num_levels=7 - min_write_buffer_number_to_merge=1 - max_write_buffer_number_to_maintain=0 - max_write_buffer_number=2 - compression=kSnappyCompression - level0_stop_writes_trigger=36 - level0_slowdown_writes_trigger=20 - compaction_filter=nullptr - level0_file_num_compaction_trigger=4 - max_compaction_bytes=1677721600 - compaction_options_universal={stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;allow_trivial_move=false;max_merge_width=4294967295;max_size_amplification_percent=200;min_merge_width=2;size_ratio=1;} - memtable_prefix_bloom_size_ratio=0.000000 - max_write_buffer_size_to_maintain=0 - hard_pending_compaction_bytes_limit=274877906944 - ttl=2592000 - table_factory=BlockBasedTable - soft_pending_compaction_bytes_limit=68719476736 - prefix_extractor=nullptr - bottommost_compression=kDisableCompressionOption - force_consistency_checks=false - paranoid_file_checks=false - compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;} - max_bytes_for_level_multiplier=10.000000 - optimize_filters_for_hits=false - level_compaction_dynamic_level_bytes=false - inplace_update_num_locks=10000 - inplace_update_support=false - periodic_compaction_seconds=0 - disable_auto_compactions=false - report_bg_io_stats=false - -[TableOptions/BlockBasedTable "dspo"] - pin_top_level_index_and_filter=true - enable_index_compression=true - read_amp_bytes_per_bit=8589934592 - format_version=2 - block_align=false - metadata_block_size=4096 - block_size_deviation=10 - partition_filters=false - block_size=4096 - index_block_restart_interval=1 - no_block_cache=false - checksum=kCRC32c - whole_key_filtering=true - index_shortening=kShortenSeparators - data_block_index_type=kDataBlockBinarySearch - index_type=kBinarySearch - verify_compression=false - filter_policy=nullptr - data_block_hash_table_util_ratio=0.750000 - pin_l0_filter_and_index_blocks_in_cache=false - block_restart_interval=16 - cache_index_and_filter_blocks_with_high_priority=true - cache_index_and_filter_blocks=false - hash_index_allow_collision=true - flush_block_policy_factory=FlushBlockBySizePolicyFactory - - -[CFOptions "dpos"] - sample_for_compression=0 - compaction_pri=kMinOverlappingRatio - merge_operator=nullptr - compaction_filter_factory=nullptr - memtable_factory=SkipListFactory - memtable_insert_with_hint_prefix_extractor=nullptr - comparator=leveldb.BytewiseComparator - target_file_size_base=67108864 - max_sequential_skip_in_iterations=8 - compaction_style=kCompactionStyleLevel - max_bytes_for_level_base=268435456 - bloom_locality=0 - write_buffer_size=67108864 - compression_per_level= - memtable_huge_page_size=0 - max_successive_merges=0 - arena_block_size=8388608 - memtable_whole_key_filtering=false - target_file_size_multiplier=1 - max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 - num_levels=7 - min_write_buffer_number_to_merge=1 - max_write_buffer_number_to_maintain=0 - max_write_buffer_number=2 - compression=kSnappyCompression - level0_stop_writes_trigger=36 - level0_slowdown_writes_trigger=20 - compaction_filter=nullptr - level0_file_num_compaction_trigger=4 - max_compaction_bytes=1677721600 - compaction_options_universal={stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;allow_trivial_move=false;max_merge_width=4294967295;max_size_amplification_percent=200;min_merge_width=2;size_ratio=1;} - memtable_prefix_bloom_size_ratio=0.000000 - max_write_buffer_size_to_maintain=0 - hard_pending_compaction_bytes_limit=274877906944 - ttl=2592000 - table_factory=BlockBasedTable - soft_pending_compaction_bytes_limit=68719476736 - prefix_extractor=nullptr - bottommost_compression=kDisableCompressionOption - force_consistency_checks=false - paranoid_file_checks=false - compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;} - max_bytes_for_level_multiplier=10.000000 - optimize_filters_for_hits=false - level_compaction_dynamic_level_bytes=false - inplace_update_num_locks=10000 - inplace_update_support=false - periodic_compaction_seconds=0 - disable_auto_compactions=false - report_bg_io_stats=false - -[TableOptions/BlockBasedTable "dpos"] - pin_top_level_index_and_filter=true - enable_index_compression=true - read_amp_bytes_per_bit=8589934592 - format_version=2 - block_align=false - metadata_block_size=4096 - block_size_deviation=10 - partition_filters=false - block_size=4096 - index_block_restart_interval=1 - no_block_cache=false - checksum=kCRC32c - whole_key_filtering=true - index_shortening=kShortenSeparators - data_block_index_type=kDataBlockBinarySearch - index_type=kBinarySearch - verify_compression=false - filter_policy=nullptr - data_block_hash_table_util_ratio=0.750000 - pin_l0_filter_and_index_blocks_in_cache=false - block_restart_interval=16 - cache_index_and_filter_blocks_with_high_priority=true - cache_index_and_filter_blocks=false - hash_index_allow_collision=true - flush_block_policy_factory=FlushBlockBySizePolicyFactory - - -[CFOptions "dosp"] - sample_for_compression=0 - compaction_pri=kMinOverlappingRatio - merge_operator=nullptr - compaction_filter_factory=nullptr - memtable_factory=SkipListFactory - memtable_insert_with_hint_prefix_extractor=nullptr - comparator=leveldb.BytewiseComparator - target_file_size_base=67108864 - max_sequential_skip_in_iterations=8 - compaction_style=kCompactionStyleLevel - max_bytes_for_level_base=268435456 - bloom_locality=0 - write_buffer_size=67108864 - compression_per_level= - memtable_huge_page_size=0 - max_successive_merges=0 - arena_block_size=8388608 - memtable_whole_key_filtering=false - target_file_size_multiplier=1 - max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 - num_levels=7 - min_write_buffer_number_to_merge=1 - max_write_buffer_number_to_maintain=0 - max_write_buffer_number=2 - compression=kSnappyCompression - level0_stop_writes_trigger=36 - level0_slowdown_writes_trigger=20 - compaction_filter=nullptr - level0_file_num_compaction_trigger=4 - max_compaction_bytes=1677721600 - compaction_options_universal={stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;allow_trivial_move=false;max_merge_width=4294967295;max_size_amplification_percent=200;min_merge_width=2;size_ratio=1;} - memtable_prefix_bloom_size_ratio=0.000000 - max_write_buffer_size_to_maintain=0 - hard_pending_compaction_bytes_limit=274877906944 - ttl=2592000 - table_factory=BlockBasedTable - soft_pending_compaction_bytes_limit=68719476736 - prefix_extractor=nullptr - bottommost_compression=kDisableCompressionOption - force_consistency_checks=false - paranoid_file_checks=false - compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;} - max_bytes_for_level_multiplier=10.000000 - optimize_filters_for_hits=false - level_compaction_dynamic_level_bytes=false - inplace_update_num_locks=10000 - inplace_update_support=false - periodic_compaction_seconds=0 - disable_auto_compactions=false - report_bg_io_stats=false - -[TableOptions/BlockBasedTable "dosp"] - pin_top_level_index_and_filter=true - enable_index_compression=true - read_amp_bytes_per_bit=8589934592 - format_version=2 - block_align=false - metadata_block_size=4096 - block_size_deviation=10 - partition_filters=false - block_size=4096 - index_block_restart_interval=1 - no_block_cache=false - checksum=kCRC32c - whole_key_filtering=true - index_shortening=kShortenSeparators - data_block_index_type=kDataBlockBinarySearch - index_type=kBinarySearch - verify_compression=false - filter_policy=nullptr - data_block_hash_table_util_ratio=0.750000 - pin_l0_filter_and_index_blocks_in_cache=false - block_restart_interval=16 - cache_index_and_filter_blocks_with_high_priority=true - cache_index_and_filter_blocks=false - hash_index_allow_collision=true - flush_block_policy_factory=FlushBlockBySizePolicyFactory - diff --git a/lib/tests/rocksdb_store.rs b/lib/tests/rocksdb_store.rs deleted file mode 100644 index 664fa36e..00000000 --- a/lib/tests/rocksdb_store.rs +++ /dev/null @@ -1,92 +0,0 @@ -use oxigraph::model::vocab::{rdf, xsd}; -use oxigraph::model::*; -use oxigraph::RocksDbStore; -use std::io; -use std::process::Command; - -fn quads(graph_name: impl Into>) -> Vec> { - let graph_name = graph_name.into(); - let paris = NamedNodeRef::new_unchecked("http://www.wikidata.org/entity/Q90"); - let france = NamedNodeRef::new_unchecked("http://www.wikidata.org/entity/Q142"); - let city = NamedNodeRef::new_unchecked("http://schema.org/City"); - let name = NamedNodeRef::new_unchecked("http://schema.org/name"); - let country = NamedNodeRef::new_unchecked("http://schema.org/country"); - let population = NamedNodeRef::new_unchecked("http://schema.org/population"); - let start_date = NamedNodeRef::new_unchecked("http://schema.org/startDate"); - let url = NamedNodeRef::new_unchecked("http://schema.org/url"); - let postal_code = NamedNodeRef::new_unchecked("http://schema.org/postalCode"); - vec![ - QuadRef::new(paris, rdf::TYPE, city, graph_name), - QuadRef::new( - paris, - name, - LiteralRef::new_language_tagged_literal_unchecked("Paris", "fr"), - graph_name, - ), - QuadRef::new( - paris, - name, - LiteralRef::new_language_tagged_literal_unchecked("la ville lumière", "fr"), - graph_name, - ), - QuadRef::new(paris, country, france, graph_name), - QuadRef::new( - paris, - population, - LiteralRef::new_typed_literal("2000000", xsd::INTEGER), - graph_name, - ), - QuadRef::new( - paris, - start_date, - LiteralRef::new_typed_literal("-300", xsd::G_YEAR), - graph_name, - ), - QuadRef::new( - paris, - url, - LiteralRef::new_typed_literal("https://www.paris.fr/", xsd::ANY_URI), - graph_name, - ), - QuadRef::new( - paris, - postal_code, - LiteralRef::new_simple_literal("75001"), - graph_name, - ), - ] -} - -#[test] -fn test_backward_compatibility() -> io::Result<()> { - { - let store = RocksDbStore::open("tests/rockdb_bc_data")?; - for q in quads(GraphNameRef::DefaultGraph) { - assert!(store.contains(q)?); - } - let graph_name = - NamedNodeRef::new_unchecked("http://www.wikidata.org/wiki/Special:EntityData/Q90"); - for q in quads(graph_name) { - assert!(store.contains(q)?); - } - assert!(store.contains_named_graph(graph_name)?); - assert_eq!( - vec![NamedOrBlankNode::from(graph_name)], - store.named_graphs().collect::>>()? - ); - }; - reset_dir("tests/rockdb_bc_data")?; - Ok(()) -} - -fn reset_dir(dir: &str) -> io::Result<()> { - assert!(Command::new("git") - .args(&["clean", "-fX", dir]) - .status()? - .success()); - assert!(Command::new("git") - .args(&["checkout", "HEAD", "--", dir]) - .status()? - .success()); - Ok(()) -} diff --git a/server/Cargo.toml b/server/Cargo.toml index da187173..ca0b23b1 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -11,17 +11,12 @@ SPARQL server based on Oxigraph """ edition = "2018" -[features] -sled = ["oxigraph/sled"] -rocksdb = ["oxigraph/rocksdb"] -default = ["rocksdb"] - [dependencies] argh = "0.1" async-std = { version = "1", features = ["attributes"] } async-h1 = "2" http-types = "2" -oxigraph = { version = "0.2", path="../lib", features = ["http_client"] } +oxigraph = { version = "0.2", path="../lib", features = ["sled", "http_client"] } rand = "0.8" url = "2" oxiri = "0.1" diff --git a/server/Dockerfile b/server/Dockerfile index fbf531fd..4a635278 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -1,6 +1,5 @@ FROM rust:1-buster as builder -RUN apt-get update && apt-get install clang -y COPY . /oxigraph RUN cd /oxigraph/server && cargo build --release diff --git a/server/README.md b/server/README.md index 27698352..22ebfc88 100644 --- a/server/README.md +++ b/server/README.md @@ -11,7 +11,7 @@ Oxigraph Server Oxigraph Server is a standalone HTTP server providing a graph database implementing the [SPARQL](https://www.w3.org/TR/sparql11-overview/) standard. -Its goal is to provide a compliant, safe, and fast graph database based on the [RocksDB](https://rocksdb.org/) key-value stores. +Its goal is to provide a compliant, safe, and fast graph database based on the [Sled](https://sled.rs/) key-value stores. It is written in Rust. It also provides a set of utility functions for reading, writing, and processing RDF files. @@ -34,7 +34,7 @@ A preliminary benchmark [is provided](../bench/README.md). ## Installation -You need to have [a recent stable version of Rust and Cargo installed](https://www.rust-lang.org/tools/install). You also need [clang](https://clang.llvm.org/) to build RocksDB. +You need to have [a recent stable version of Rust and Cargo installed](https://www.rust-lang.org/tools/install). To download, build and install the latest released version run `cargo install oxigraph_server`. There is no need to clone the git repository. diff --git a/server/src/main.rs b/server/src/main.rs index ef35ff51..68cff0ba 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -24,9 +24,6 @@ use oxigraph::io::{DatasetFormat, GraphFormat}; use oxigraph::model::{GraphName, GraphNameRef, NamedNode, NamedOrBlankNode}; use oxigraph::sparql::algebra::GraphUpdateOperation; use oxigraph::sparql::{Query, QueryResults, QueryResultsFormat, Update}; -#[cfg(feature = "rocksdb")] -use oxigraph::RocksDbStore as Store; -#[cfg(all(feature = "sled", not(feature = "rocksdb")))] use oxigraph::SledStore as Store; use oxiri::Iri; use rand::random; diff --git a/wikibase/Cargo.toml b/wikibase/Cargo.toml index ad863eb2..d361d909 100644 --- a/wikibase/Cargo.toml +++ b/wikibase/Cargo.toml @@ -17,6 +17,6 @@ async-h1 = "2" chrono = "0.4" http-client = { version = "6", features = ["h1_client"] } http-types = "2" -oxigraph = { version = "0.2", path ="../lib", features = ["rocksdb", "http_client"] } +oxigraph = { version = "0.2", path ="../lib", features = ["sled", "http_client"] } serde_json = "1" url = "2" \ No newline at end of file diff --git a/wikibase/Dockerfile b/wikibase/Dockerfile index 4d11d2d6..bf415b40 100644 --- a/wikibase/Dockerfile +++ b/wikibase/Dockerfile @@ -1,6 +1,5 @@ FROM rust:1-buster as builder -RUN apt-get update && apt-get install clang -y COPY . /oxigraph RUN cd /oxigraph/wikibase && cargo build --release diff --git a/wikibase/README.md b/wikibase/README.md index c3b16236..a4040c0a 100644 --- a/wikibase/README.md +++ b/wikibase/README.md @@ -16,7 +16,7 @@ Oxigraph and Oxigraph Wikibase are in heavy development and not been optimized y ## Installation -You need to have [a recent stable version of Rust and Cargo installed](https://www.rust-lang.org/tools/install). You also need [clang](https://clang.llvm.org/) to build RocksDB. +You need to have [a recent stable version of Rust and Cargo installed](https://www.rust-lang.org/tools/install). To download, build and install the latest released version run `cargo install oxigraph_wikibase`. There is no need to clone the git repository. diff --git a/wikibase/src/loader.rs b/wikibase/src/loader.rs index f5170114..e70676c2 100644 --- a/wikibase/src/loader.rs +++ b/wikibase/src/loader.rs @@ -7,7 +7,7 @@ use http_client::HttpClient; use http_types::{headers, Method, Request, Result}; use oxigraph::io::GraphFormat; use oxigraph::model::NamedNodeRef; -use oxigraph::RocksDbStore; +use oxigraph::SledStore; use serde_json::Value; use std::collections::{HashMap, HashSet}; use std::io::{BufReader, Cursor, Read}; @@ -16,7 +16,7 @@ use std::time::Duration; use url::{form_urlencoded, Url}; pub struct WikibaseLoader { - store: RocksDbStore, + store: SledStore, client: H1Client, api_url: Url, entity_data_url: Url, @@ -28,7 +28,7 @@ pub struct WikibaseLoader { impl WikibaseLoader { pub fn new( - store: RocksDbStore, + store: SledStore, api_url: &str, pages_base_url: &str, namespaces: &[u32], @@ -230,23 +230,20 @@ impl WikibaseLoader { } fn load_entity_data(&self, uri: &str, data: impl Read) -> Result<()> { - let graph_name = NamedNodeRef::new(uri)?.into(); - self.store.transaction(|transaction| { - let to_remove = self - .store - .quads_for_pattern(None, None, None, Some(graph_name)) - .collect::, _>>()?; - for q in to_remove { - transaction.remove(&q)?; - } - - transaction.load_graph( - BufReader::new(data), - GraphFormat::NTriples, - NamedNodeRef::new(uri)?, - None, - )?; - Ok(()) - }) + let graph_name = NamedNodeRef::new(uri)?; + //TODO: proper transaction + for q in self + .store + .quads_for_pattern(None, None, None, Some(graph_name.into())) + { + self.store.remove(&q?)?; + } + self.store.load_graph( + BufReader::new(data), + GraphFormat::NTriples, + graph_name, + None, + )?; + Ok(()) } } diff --git a/wikibase/src/main.rs b/wikibase/src/main.rs index 021fd589..cf419f7a 100644 --- a/wikibase/src/main.rs +++ b/wikibase/src/main.rs @@ -23,7 +23,7 @@ use http_types::{ use oxigraph::io::GraphFormat; use oxigraph::model::{GraphName, NamedNode, NamedOrBlankNode}; use oxigraph::sparql::{Query, QueryResults, QueryResultsFormat}; -use oxigraph::RocksDbStore; +use oxigraph::SledStore; use std::str::FromStr; use std::time::Duration; use url::form_urlencoded; @@ -65,7 +65,7 @@ struct Args { pub async fn main() -> Result<()> { let args: Args = argh::from_env(); - let store = RocksDbStore::open(args.file)?; + let store = SledStore::open(args.file)?; let mediawiki_api = args.mediawiki_api.clone(); let mediawiki_base_url = args.mediawiki_base_url.clone(); let namespaces = args @@ -106,7 +106,7 @@ pub async fn main() -> Result<()> { .await } -async fn handle_request(request: Request, store: RocksDbStore) -> Result { +async fn handle_request(request: Request, store: SledStore) -> Result { Ok(match (request.url().path(), request.method()) { ("/query", Method::Get) => { configure_and_evaluate_sparql_query(store, url_query(&request), None, request)? @@ -159,7 +159,7 @@ fn url_query(request: &Request) -> Vec { } fn configure_and_evaluate_sparql_query( - store: RocksDbStore, + store: SledStore, encoded: Vec, mut query: Option, request: Request, @@ -187,7 +187,7 @@ fn configure_and_evaluate_sparql_query( } fn evaluate_sparql_query( - store: RocksDbStore, + store: SledStore, query: String, default_graph_uris: Vec, named_graph_uris: Vec,