Drops RocksDBStore

pull/171/head
Tpt 4 years ago
parent 8da6165226
commit 35fe15f796
  1. 4
      README.md
  2. 6
      bench/bsbm_oxigraph.sh
  3. 3
      lib/Cargo.toml
  4. 7
      lib/README.md
  5. 34
      lib/benches/store.rs
  6. 7
      lib/src/lib.rs
  7. 6
      lib/src/store/mod.rs
  8. 1665
      lib/src/store/rocksdb.rs
  9. 63
      lib/src/store/sophia.rs
  10. BIN
      lib/tests/rockdb_bc_data/000003.log
  11. 1
      lib/tests/rockdb_bc_data/CURRENT
  12. 1
      lib/tests/rockdb_bc_data/IDENTITY
  13. 0
      lib/tests/rockdb_bc_data/LOCK
  14. BIN
      lib/tests/rockdb_bc_data/MANIFEST-000004
  15. 964
      lib/tests/rockdb_bc_data/OPTIONS-000026
  16. 92
      lib/tests/rocksdb_store.rs
  17. 7
      server/Cargo.toml
  18. 1
      server/Dockerfile
  19. 4
      server/README.md
  20. 3
      server/src/main.rs
  21. 2
      wikibase/Cargo.toml
  22. 1
      wikibase/Dockerfile
  23. 2
      wikibase/README.md
  24. 25
      wikibase/src/loader.rs
  25. 10
      wikibase/src/main.rs

@ -10,7 +10,7 @@ Oxigraph
Oxigraph is a graph database implementing the [SPARQL](https://www.w3.org/TR/sparql11-overview/) standard. Oxigraph is a graph database implementing the [SPARQL](https://www.w3.org/TR/sparql11-overview/) standard.
Its goal is to provide a compliant, safe, and fast graph database based on the [RocksDB](https://rocksdb.org/) and [Sled](https://sled.rs/) key-value stores. Its goal is to provide a compliant, safe, and fast graph database based on the [Sled](https://sled.rs/) key-value store.
It is written in Rust. It is written in Rust.
It also provides a set of utility functions for reading, writing, and processing RDF files. It also provides a set of utility functions for reading, writing, and processing RDF files.
@ -23,7 +23,7 @@ It is split into multiple parts:
* [`pyoxigraph` that exposes Oxigraph to the Python world](https://oxigraph.org/pyoxigraph/). Its source code is in the `python` directory. [![PyPI](https://img.shields.io/pypi/v/pyoxigraph)](https://pypi.org/project/pyoxigraph/) * [`pyoxigraph` that exposes Oxigraph to the Python world](https://oxigraph.org/pyoxigraph/). Its source code is in the `python` directory. [![PyPI](https://img.shields.io/pypi/v/pyoxigraph)](https://pypi.org/project/pyoxigraph/)
* [JavaScript bindings for Oxigraph](https://www.npmjs.com/package/oxigraph). WebAssembly is used to package Oxigraph into a NodeJS compatible NPM package. Its source code is in the `js` directory. * [JavaScript bindings for Oxigraph](https://www.npmjs.com/package/oxigraph). WebAssembly is used to package Oxigraph into a NodeJS compatible NPM package. Its source code is in the `js` directory.
[![npm](https://img.shields.io/npm/v/oxigraph)](https://www.npmjs.com/package/oxigraph) [![npm](https://img.shields.io/npm/v/oxigraph)](https://www.npmjs.com/package/oxigraph)
* [Oxigraph server](https://crates.io/crates/oxigraph_server) that provides a standalone binary of a web server implementing the [SPARQL 1.1 Protocol](https://www.w3.org/TR/sparql11-protocol/) and the [SPARQL 1.1 Graph Store Protocol](https://www.w3.org/TR/sparql11-http-rdf-update/). It uses the [RocksDB](https://rocksdb.org/) key-value store. Its source code is in the `server` directory. * [Oxigraph server](https://crates.io/crates/oxigraph_server) that provides a standalone binary of a web server implementing the [SPARQL 1.1 Protocol](https://www.w3.org/TR/sparql11-protocol/) and the [SPARQL 1.1 Graph Store Protocol](https://www.w3.org/TR/sparql11-http-rdf-update/). Its source code is in the `server` directory.
[![Latest Version](https://img.shields.io/crates/v/oxigraph_server.svg)](https://crates.io/crates/oxigraph_server) [![Latest Version](https://img.shields.io/crates/v/oxigraph_server.svg)](https://crates.io/crates/oxigraph_server)
[![Docker Image Version (latest semver)](https://img.shields.io/docker/v/oxigraph/oxigraph?sort=semver)](https://hub.docker.com/repository/docker/oxigraph/oxigraph) [![Docker Image Version (latest semver)](https://img.shields.io/docker/v/oxigraph/oxigraph?sort=semver)](https://hub.docker.com/repository/docker/oxigraph/oxigraph)
* [Oxigraph Wikibase](https://crates.io/crates/oxigraph_wikibase), a web server able to synchronize with a [Wikibase instance](https://wikiba.se/). Its source code is in the `wikibase` directory. * [Oxigraph Wikibase](https://crates.io/crates/oxigraph_wikibase), a web server able to synchronize with a [Wikibase instance](https://wikiba.se/). Its source code is in the `wikibase` directory.

@ -8,9 +8,9 @@ cargo build --release --manifest-path="../../server/Cargo.toml"
./../../target/release/oxigraph_server --file oxigraph_data --bind 127.0.0.1:7878 & ./../../target/release/oxigraph_server --file oxigraph_data --bind 127.0.0.1:7878 &
sleep 5 sleep 5
curl -f -X POST -H 'Content-Type:application/n-triples' --data-binary "@explore-${DATASET_SIZE}.nt" http://127.0.0.1:7878/store?default curl -f -X POST -H 'Content-Type:application/n-triples' --data-binary "@explore-${DATASET_SIZE}.nt" http://127.0.0.1:7878/store?default
./testdriver -mt ${PARALLELISM} -ucf usecases/explore/sparql.txt -o "../bsbm.explore.oxigraph.${DATASET_SIZE}.${PARALLELISM}.main-rocksdb.xml" http://127.0.0.1:7878/query ./testdriver -mt ${PARALLELISM} -ucf usecases/explore/sparql.txt -o "../bsbm.explore.oxigraph.${DATASET_SIZE}.${PARALLELISM}.main.xml" http://127.0.0.1:7878/query
./testdriver -mt ${PARALLELISM} -ucf usecases/exploreAndUpdate/sparql.txt -o "../bsbm.exploreAndUpdate.oxigraph.${DATASET_SIZE}.${PARALLELISM}.main-rocksdb.xml" http://127.0.0.1:7878/query -u http://127.0.0.1:7878/update -udataset "explore-update-${DATASET_SIZE}.nt" ./testdriver -mt ${PARALLELISM} -ucf usecases/exploreAndUpdate/sparql.txt -o "../bsbm.exploreAndUpdate.oxigraph.${DATASET_SIZE}.${PARALLELISM}.main.xml" http://127.0.0.1:7878/query -u http://127.0.0.1:7878/update -udataset "explore-update-${DATASET_SIZE}.nt"
./testdriver -mt ${PARALLELISM} -ucf usecases/businessIntelligence/sparql.txt -o "../bsbm.businessIntelligence.${DATASET_SIZE}.${PARALLELISM}.main-rocksdb.xml" "http://127.0.0.1:7878/query" ./testdriver -mt ${PARALLELISM} -ucf usecases/businessIntelligence/sparql.txt -o "../bsbm.businessIntelligence.${DATASET_SIZE}.${PARALLELISM}.main.xml" "http://127.0.0.1:7878/query"
kill $! kill $!
rm -r oxigraph_data rm -r oxigraph_data
rm "explore-${DATASET_SIZE}.nt" rm "explore-${DATASET_SIZE}.nt"

@ -22,7 +22,6 @@ sophia = ["sophia_api"]
http_client = ["httparse", "native-tls"] http_client = ["httparse", "native-tls"]
[dependencies] [dependencies]
rocksdb = { version = "0.15", optional = true }
sled = { version = "0.34", optional = true } sled = { version = "0.34", optional = true }
quick-xml = "0.22" quick-xml = "0.22"
rand = "0.8" rand = "0.8"
@ -61,4 +60,4 @@ wasm-bindgen-test = "0.3"
[[bench]] [[bench]]
name = "store" name = "store"
harness = false harness = false
required-features = ["sled", "rocksdb"] required-features = ["sled"]

@ -14,13 +14,8 @@ It also provides a set of utility functions for reading, writing, and processing
It currently provides three store implementations providing [SPARQL](https://www.w3.org/TR/sparql11-overview/) capability: It currently provides three store implementations providing [SPARQL](https://www.w3.org/TR/sparql11-overview/) capability:
* `MemoryStore`: a simple in memory implementation. * `MemoryStore`: a simple in memory implementation.
* `RocksDbStore`: a file system implementation based on the [RocksDB](https://rocksdb.org/) key-value store. * `SledStore`: a file system implementation based on the [Sled](https://sled.rs/) key-value store.
It requires the `"rocksdb"` feature to be activated.
The [clang](https://clang.llvm.org/) compiler needs to be installed to compile RocksDB.
* `SledStore`: another file system implementation based on the [Sled](https://sled.rs/) key-value store.
It requires the `"sled"` feature to be activated. It requires the `"sled"` feature to be activated.
Sled is much faster to build than RockDB and does not require a C++ compiler.
However, Sled is still in development, less tested and data load seems much slower than RocksDB.
Oxigraph is in heavy development and SPARQL query evaluation has not been optimized yet. Oxigraph is in heavy development and SPARQL query evaluation has not been optimized yet.

@ -1,16 +1,9 @@
use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
use oxigraph::model::{NamedNode, Quad}; use oxigraph::model::{NamedNode, Quad};
use oxigraph::{MemoryStore, RocksDbStore, SledStore}; use oxigraph::{MemoryStore, SledStore};
use rand::random; use rand::random;
use std::env::temp_dir;
use std::fs::remove_dir_all;
criterion_group!( criterion_group!(store_load, memory_load_bench, sled_load_bench);
store_load,
memory_load_bench,
sled_load_bench,
rocksdb_load_bench
);
criterion_main!(store_load); criterion_main!(store_load);
@ -52,29 +45,6 @@ fn sled_load_bench(c: &mut Criterion) {
group.finish(); group.finish();
} }
fn rocksdb_load_bench(c: &mut Criterion) {
let mut group = c.benchmark_group("rocksdb");
group.nresamples(10);
group.sample_size(10);
let temp_dir = temp_dir();
for size in [100, 1_000, 10_000].iter() {
group.throughput(Throughput::Elements(*size as u64));
let quads = create_quads(*size);
group.bench_function(BenchmarkId::from_parameter(size), |b| {
b.iter(|| {
let mut dir = temp_dir.clone();
dir.push(random::<u64>().to_string());
let store = RocksDbStore::open(&dir).unwrap();
for quad in &quads {
store.insert(quad).unwrap();
}
remove_dir_all(&dir).unwrap();
});
});
}
group.finish();
}
fn create_quads(size: u64) -> Vec<Quad> { fn create_quads(size: u64) -> Vec<Quad> {
(0..size) (0..size)
.map(|_| { .map(|_| {

@ -5,13 +5,8 @@
//! //!
//! It currently provides three store implementations providing [SPARQL](https://www.w3.org/TR/sparql11-overview/) capability: //! It currently provides three store implementations providing [SPARQL](https://www.w3.org/TR/sparql11-overview/) capability:
//! * [`MemoryStore`](store::memory::MemoryStore): a simple in memory implementation. //! * [`MemoryStore`](store::memory::MemoryStore): a simple in memory implementation.
//! * [`RocksDbStore`](store::rocksdb::RocksDbStore): a file system implementation based on the [RocksDB](https://rocksdb.org/) key-value store.
//! It requires the `"rocksdb"` feature to be activated.
//! The [clang](https://clang.llvm.org/) compiler needs to be installed to compile RocksDB.
//! * [`SledStore`](store::sled::SledStore): another file system implementation based on the [Sled](https://sled.rs/) key-value store. //! * [`SledStore`](store::sled::SledStore): another file system implementation based on the [Sled](https://sled.rs/) key-value store.
//! It requires the `"sled"` feature to be activated. //! It requires the `"sled"` feature to be activated.
//! Sled is much faster to build than RockDB and does not require a C++ compiler.
//! However, Sled is still in developpment, less tested and data load seems much slower than RocksDB.
//! //!
//! Oxigraph is in heavy development and SPARQL query evaluation has not been optimized yet. //! Oxigraph is in heavy development and SPARQL query evaluation has not been optimized yet.
//! //!
@ -128,7 +123,5 @@ pub mod sparql;
pub mod store; pub mod store;
pub use crate::store::memory::MemoryStore; pub use crate::store::memory::MemoryStore;
#[cfg(feature = "rocksdb")]
pub use crate::store::rocksdb::RocksDbStore;
#[cfg(feature = "sled")] #[cfg(feature = "sled")]
pub use crate::store::sled::SledStore; pub use crate::store::sled::SledStore;

@ -1,11 +1,9 @@
//! RDF [RDF dataset](https://www.w3.org/TR/rdf11-concepts/#dfn-rdf-dataset) storage implementations. //! RDF [RDF dataset](https://www.w3.org/TR/rdf11-concepts/#dfn-rdf-dataset) storage implementations.
#[cfg(any(feature = "rocksdb", feature = "sled"))] #[cfg(feature = "sled")]
mod binary_encoder; mod binary_encoder;
pub mod memory; pub mod memory;
pub(crate) mod numeric_encoder; pub(crate) mod numeric_encoder;
#[cfg(feature = "rocksdb")]
pub mod rocksdb;
#[cfg(feature = "sled")] #[cfg(feature = "sled")]
pub mod sled; pub mod sled;
pub(crate) mod small_string; pub(crate) mod small_string;
@ -13,8 +11,6 @@ pub(crate) mod small_string;
mod sophia; mod sophia;
pub use crate::store::memory::MemoryStore; pub use crate::store::memory::MemoryStore;
#[cfg(feature = "rocksdb")]
pub use crate::store::rocksdb::RocksDbStore;
#[cfg(feature = "sled")] #[cfg(feature = "sled")]
pub use crate::store::sled::SledStore; pub use crate::store::sled::SledStore;

File diff suppressed because it is too large Load Diff

@ -560,65 +560,6 @@ mod sled {
sophia_api::test_dataset_impl!(test, SledStore, false, false); sophia_api::test_dataset_impl!(test, SledStore, false, false);
} }
#[cfg(feature = "rocksdb")]
mod rocksdb {
use super::*;
impl_dataset!(RocksDbStore, std::io::Error, io_quad_map, io_err_map);
impl MutableDataset for RocksDbStore {
type MutationError = std::io::Error;
fn insert<TS, TP, TO, TG>(
&mut self,
s: &TS,
p: &TP,
o: &TO,
g: Option<&TG>,
) -> MDResult<Self, bool>
where
TS: TTerm + ?Sized,
TP: TTerm + ?Sized,
TO: TTerm + ?Sized,
TG: TTerm + ?Sized,
{
let mut buf_s = String::new();
let mut buf_p = String::new();
let mut buf_o = String::new();
let mut buf_g = String::new();
let quadref =
match convert_quadref(s, p, o, g, &mut buf_s, &mut buf_p, &mut buf_o, &mut buf_g) {
Some(quad) => quad,
None => return Ok(false),
};
RocksDbStore::insert(self, quadref).map(|_| true)
}
fn remove<TS, TP, TO, TG>(
&mut self,
s: &TS,
p: &TP,
o: &TO,
g: Option<&TG>,
) -> MDResult<Self, bool>
where
TS: TTerm + ?Sized,
TP: TTerm + ?Sized,
TO: TTerm + ?Sized,
TG: TTerm + ?Sized,
{
let mut buf_s = String::new();
let mut buf_p = String::new();
let mut buf_o = String::new();
let mut buf_g = String::new();
let quadref =
match convert_quadref(s, p, o, g, &mut buf_s, &mut buf_p, &mut buf_o, &mut buf_g) {
Some(quad) => quad,
None => return Ok(false),
};
RocksDbStore::remove(self, quadref).map(|_| true)
}
}
}
// helper functions // helper functions
#[allow(clippy::unnecessary_wraps)] #[allow(clippy::unnecessary_wraps)]
fn infallible_quad_map<'a>(q: Quad) -> Result<StreamedSophiaQuad<'a>, Infallible> { fn infallible_quad_map<'a>(q: Quad) -> Result<StreamedSophiaQuad<'a>, Infallible> {
@ -630,7 +571,7 @@ fn infallible_err_map(_: EvaluationError) -> Infallible {
panic!("Unexpected error") panic!("Unexpected error")
} }
#[cfg(any(feature = "rocksdb", feature = "sled"))] #[cfg(feature = "sled")]
fn io_quad_map<'a>( fn io_quad_map<'a>(
res: Result<Quad, std::io::Error>, res: Result<Quad, std::io::Error>,
) -> Result<StreamedSophiaQuad<'a>, std::io::Error> { ) -> Result<StreamedSophiaQuad<'a>, std::io::Error> {
@ -640,7 +581,7 @@ fn io_quad_map<'a>(
}) })
} }
#[cfg(any(feature = "rocksdb", feature = "sled"))] #[cfg(feature = "sled")]
fn io_err_map(err: EvaluationError) -> std::io::Error { fn io_err_map(err: EvaluationError) -> std::io::Error {
match err { match err {
EvaluationError::Io(err) => err, EvaluationError::Io(err) => err,

@ -1 +0,0 @@
f08a4c5b-0479-408c-80d3-7d4b10d7c7aa

@ -1,964 +0,0 @@
# This is a RocksDB option file.
#
# For detailed file format spec, please refer to the example file
# in examples/rocksdb_option_file_example.ini
#
[Version]
rocksdb_version=6.7.3
options_file_version=1.1
[DBOptions]
write_dbid_to_manifest=false
avoid_unnecessary_blocking_io=false
two_write_queues=false
allow_ingest_behind=false
writable_file_max_buffer_size=1048576
avoid_flush_during_shutdown=false
avoid_flush_during_recovery=false
info_log_level=INFO_LEVEL
access_hint_on_compaction_start=NORMAL
allow_concurrent_memtable_write=true
enable_pipelined_write=false
stats_dump_period_sec=600
stats_persist_period_sec=600
strict_bytes_per_sync=false
WAL_ttl_seconds=0
WAL_size_limit_MB=0
max_subcompactions=1
dump_malloc_stats=false
db_log_dir=
wal_recovery_mode=kPointInTimeRecovery
log_file_time_to_roll=0
enable_write_thread_adaptive_yield=true
recycle_log_file_num=0
table_cache_numshardbits=6
atomic_flush=false
preserve_deletes=false
stats_history_buffer_size=1048576
max_open_files=-1
max_file_opening_threads=16
delete_obsolete_files_period_micros=21600000000
max_background_flushes=-1
write_thread_slow_yield_usec=3
base_background_compactions=-1
manual_wal_flush=false
wal_dir=tests/rockdb_bc_data
max_background_compactions=-1
bytes_per_sync=0
max_background_jobs=2
use_fsync=false
unordered_write=false
fail_if_options_file_error=false
random_access_max_buffer_size=1048576
compaction_readahead_size=0
wal_bytes_per_sync=0
new_table_reader_for_compaction_inputs=false
skip_stats_update_on_db_open=false
persist_stats_to_disk=false
skip_log_error_on_recovery=false
log_readahead_size=0
is_fd_close_on_exec=true
use_adaptive_mutex=false
error_if_exists=false
write_thread_max_yield_usec=100
enable_thread_tracking=false
db_write_buffer_size=0
create_missing_column_families=true
paranoid_checks=true
create_if_missing=true
max_manifest_file_size=1073741824
allow_2pc=false
max_total_wal_size=0
use_direct_io_for_flush_and_compaction=false
manifest_preallocation_size=4194304
use_direct_reads=false
delayed_write_rate=16777216
allow_fallocate=true
max_write_batch_group_size_bytes=1048576
keep_log_file_num=1000
allow_mmap_reads=false
max_log_file_size=0
allow_mmap_writes=false
advise_random_on_open=true
[CFOptions "default"]
sample_for_compression=0
compaction_pri=kMinOverlappingRatio
merge_operator=nullptr
compaction_filter_factory=nullptr
memtable_factory=SkipListFactory
memtable_insert_with_hint_prefix_extractor=nullptr
comparator=leveldb.BytewiseComparator
target_file_size_base=67108864
max_sequential_skip_in_iterations=8
compaction_style=kCompactionStyleLevel
max_bytes_for_level_base=268435456
bloom_locality=0
write_buffer_size=67108864
compression_per_level=
memtable_huge_page_size=0
max_successive_merges=0
arena_block_size=8388608
memtable_whole_key_filtering=false
target_file_size_multiplier=1
max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
num_levels=7
min_write_buffer_number_to_merge=1
max_write_buffer_number_to_maintain=0
max_write_buffer_number=2
compression=kSnappyCompression
level0_stop_writes_trigger=36
level0_slowdown_writes_trigger=20
compaction_filter=nullptr
level0_file_num_compaction_trigger=4
max_compaction_bytes=1677721600
compaction_options_universal={stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;allow_trivial_move=false;max_merge_width=4294967295;max_size_amplification_percent=200;min_merge_width=2;size_ratio=1;}
memtable_prefix_bloom_size_ratio=0.000000
max_write_buffer_size_to_maintain=0
hard_pending_compaction_bytes_limit=274877906944
ttl=2592000
table_factory=BlockBasedTable
soft_pending_compaction_bytes_limit=68719476736
prefix_extractor=nullptr
bottommost_compression=kDisableCompressionOption
force_consistency_checks=false
paranoid_file_checks=false
compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;}
max_bytes_for_level_multiplier=10.000000
optimize_filters_for_hits=false
level_compaction_dynamic_level_bytes=false
inplace_update_num_locks=10000
inplace_update_support=false
periodic_compaction_seconds=0
disable_auto_compactions=false
report_bg_io_stats=false
[TableOptions/BlockBasedTable "default"]
pin_top_level_index_and_filter=true
enable_index_compression=true
read_amp_bytes_per_bit=8589934592
format_version=2
block_align=false
metadata_block_size=4096
block_size_deviation=10
partition_filters=false
block_size=4096
index_block_restart_interval=1
no_block_cache=false
checksum=kCRC32c
whole_key_filtering=true
index_shortening=kShortenSeparators
data_block_index_type=kDataBlockBinarySearch
index_type=kBinarySearch
verify_compression=false
filter_policy=nullptr
data_block_hash_table_util_ratio=0.750000
pin_l0_filter_and_index_blocks_in_cache=false
block_restart_interval=16
cache_index_and_filter_blocks_with_high_priority=true
cache_index_and_filter_blocks=false
hash_index_allow_collision=true
flush_block_policy_factory=FlushBlockBySizePolicyFactory
[CFOptions "id2str"]
sample_for_compression=0
compaction_pri=kMinOverlappingRatio
merge_operator=nullptr
compaction_filter_factory=nullptr
memtable_factory=SkipListFactory
memtable_insert_with_hint_prefix_extractor=nullptr
comparator=leveldb.BytewiseComparator
target_file_size_base=67108864
max_sequential_skip_in_iterations=8
compaction_style=kCompactionStyleLevel
max_bytes_for_level_base=268435456
bloom_locality=0
write_buffer_size=67108864
compression_per_level=
memtable_huge_page_size=0
max_successive_merges=0
arena_block_size=8388608
memtable_whole_key_filtering=false
target_file_size_multiplier=1
max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
num_levels=7
min_write_buffer_number_to_merge=1
max_write_buffer_number_to_maintain=0
max_write_buffer_number=2
compression=kSnappyCompression
level0_stop_writes_trigger=36
level0_slowdown_writes_trigger=20
compaction_filter=nullptr
level0_file_num_compaction_trigger=4
max_compaction_bytes=1677721600
compaction_options_universal={stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;allow_trivial_move=false;max_merge_width=4294967295;max_size_amplification_percent=200;min_merge_width=2;size_ratio=1;}
memtable_prefix_bloom_size_ratio=0.000000
max_write_buffer_size_to_maintain=0
hard_pending_compaction_bytes_limit=274877906944
ttl=2592000
table_factory=BlockBasedTable
soft_pending_compaction_bytes_limit=68719476736
prefix_extractor=nullptr
bottommost_compression=kDisableCompressionOption
force_consistency_checks=false
paranoid_file_checks=false
compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;}
max_bytes_for_level_multiplier=10.000000
optimize_filters_for_hits=false
level_compaction_dynamic_level_bytes=false
inplace_update_num_locks=10000
inplace_update_support=false
periodic_compaction_seconds=0
disable_auto_compactions=false
report_bg_io_stats=false
[TableOptions/BlockBasedTable "id2str"]
pin_top_level_index_and_filter=true
enable_index_compression=true
read_amp_bytes_per_bit=8589934592
format_version=2
block_align=false
metadata_block_size=4096
block_size_deviation=10
partition_filters=false
block_size=4096
index_block_restart_interval=1
no_block_cache=false
checksum=kCRC32c
whole_key_filtering=true
index_shortening=kShortenSeparators
data_block_index_type=kDataBlockBinarySearch
index_type=kBinarySearch
verify_compression=false
filter_policy=nullptr
data_block_hash_table_util_ratio=0.750000
pin_l0_filter_and_index_blocks_in_cache=false
block_restart_interval=16
cache_index_and_filter_blocks_with_high_priority=true
cache_index_and_filter_blocks=false
hash_index_allow_collision=true
flush_block_policy_factory=FlushBlockBySizePolicyFactory
[CFOptions "spog"]
sample_for_compression=0
compaction_pri=kMinOverlappingRatio
merge_operator=nullptr
compaction_filter_factory=nullptr
memtable_factory=SkipListFactory
memtable_insert_with_hint_prefix_extractor=nullptr
comparator=leveldb.BytewiseComparator
target_file_size_base=67108864
max_sequential_skip_in_iterations=8
compaction_style=kCompactionStyleLevel
max_bytes_for_level_base=268435456
bloom_locality=0
write_buffer_size=67108864
compression_per_level=
memtable_huge_page_size=0
max_successive_merges=0
arena_block_size=8388608
memtable_whole_key_filtering=false
target_file_size_multiplier=1
max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
num_levels=7
min_write_buffer_number_to_merge=1
max_write_buffer_number_to_maintain=0
max_write_buffer_number=2
compression=kSnappyCompression
level0_stop_writes_trigger=36
level0_slowdown_writes_trigger=20
compaction_filter=nullptr
level0_file_num_compaction_trigger=4
max_compaction_bytes=1677721600
compaction_options_universal={stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;allow_trivial_move=false;max_merge_width=4294967295;max_size_amplification_percent=200;min_merge_width=2;size_ratio=1;}
memtable_prefix_bloom_size_ratio=0.000000
max_write_buffer_size_to_maintain=0
hard_pending_compaction_bytes_limit=274877906944
ttl=2592000
table_factory=BlockBasedTable
soft_pending_compaction_bytes_limit=68719476736
prefix_extractor=nullptr
bottommost_compression=kDisableCompressionOption
force_consistency_checks=false
paranoid_file_checks=false
compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;}
max_bytes_for_level_multiplier=10.000000
optimize_filters_for_hits=false
level_compaction_dynamic_level_bytes=false
inplace_update_num_locks=10000
inplace_update_support=false
periodic_compaction_seconds=0
disable_auto_compactions=false
report_bg_io_stats=false
[TableOptions/BlockBasedTable "spog"]
pin_top_level_index_and_filter=true
enable_index_compression=true
read_amp_bytes_per_bit=8589934592
format_version=2
block_align=false
metadata_block_size=4096
block_size_deviation=10
partition_filters=false
block_size=4096
index_block_restart_interval=1
no_block_cache=false
checksum=kCRC32c
whole_key_filtering=true
index_shortening=kShortenSeparators
data_block_index_type=kDataBlockBinarySearch
index_type=kBinarySearch
verify_compression=false
filter_policy=nullptr
data_block_hash_table_util_ratio=0.750000
pin_l0_filter_and_index_blocks_in_cache=false
block_restart_interval=16
cache_index_and_filter_blocks_with_high_priority=true
cache_index_and_filter_blocks=false
hash_index_allow_collision=true
flush_block_policy_factory=FlushBlockBySizePolicyFactory
[CFOptions "posg"]
sample_for_compression=0
compaction_pri=kMinOverlappingRatio
merge_operator=nullptr
compaction_filter_factory=nullptr
memtable_factory=SkipListFactory
memtable_insert_with_hint_prefix_extractor=nullptr
comparator=leveldb.BytewiseComparator
target_file_size_base=67108864
max_sequential_skip_in_iterations=8
compaction_style=kCompactionStyleLevel
max_bytes_for_level_base=268435456
bloom_locality=0
write_buffer_size=67108864
compression_per_level=
memtable_huge_page_size=0
max_successive_merges=0
arena_block_size=8388608
memtable_whole_key_filtering=false
target_file_size_multiplier=1
max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
num_levels=7
min_write_buffer_number_to_merge=1
max_write_buffer_number_to_maintain=0
max_write_buffer_number=2
compression=kSnappyCompression
level0_stop_writes_trigger=36
level0_slowdown_writes_trigger=20
compaction_filter=nullptr
level0_file_num_compaction_trigger=4
max_compaction_bytes=1677721600
compaction_options_universal={stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;allow_trivial_move=false;max_merge_width=4294967295;max_size_amplification_percent=200;min_merge_width=2;size_ratio=1;}
memtable_prefix_bloom_size_ratio=0.000000
max_write_buffer_size_to_maintain=0
hard_pending_compaction_bytes_limit=274877906944
ttl=2592000
table_factory=BlockBasedTable
soft_pending_compaction_bytes_limit=68719476736
prefix_extractor=nullptr
bottommost_compression=kDisableCompressionOption
force_consistency_checks=false
paranoid_file_checks=false
compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;}
max_bytes_for_level_multiplier=10.000000
optimize_filters_for_hits=false
level_compaction_dynamic_level_bytes=false
inplace_update_num_locks=10000
inplace_update_support=false
periodic_compaction_seconds=0
disable_auto_compactions=false
report_bg_io_stats=false
[TableOptions/BlockBasedTable "posg"]
pin_top_level_index_and_filter=true
enable_index_compression=true
read_amp_bytes_per_bit=8589934592
format_version=2
block_align=false
metadata_block_size=4096
block_size_deviation=10
partition_filters=false
block_size=4096
index_block_restart_interval=1
no_block_cache=false
checksum=kCRC32c
whole_key_filtering=true
index_shortening=kShortenSeparators
data_block_index_type=kDataBlockBinarySearch
index_type=kBinarySearch
verify_compression=false
filter_policy=nullptr
data_block_hash_table_util_ratio=0.750000
pin_l0_filter_and_index_blocks_in_cache=false
block_restart_interval=16
cache_index_and_filter_blocks_with_high_priority=true
cache_index_and_filter_blocks=false
hash_index_allow_collision=true
flush_block_policy_factory=FlushBlockBySizePolicyFactory
[CFOptions "ospg"]
sample_for_compression=0
compaction_pri=kMinOverlappingRatio
merge_operator=nullptr
compaction_filter_factory=nullptr
memtable_factory=SkipListFactory
memtable_insert_with_hint_prefix_extractor=nullptr
comparator=leveldb.BytewiseComparator
target_file_size_base=67108864
max_sequential_skip_in_iterations=8
compaction_style=kCompactionStyleLevel
max_bytes_for_level_base=268435456
bloom_locality=0
write_buffer_size=67108864
compression_per_level=
memtable_huge_page_size=0
max_successive_merges=0
arena_block_size=8388608
memtable_whole_key_filtering=false
target_file_size_multiplier=1
max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
num_levels=7
min_write_buffer_number_to_merge=1
max_write_buffer_number_to_maintain=0
max_write_buffer_number=2
compression=kSnappyCompression
level0_stop_writes_trigger=36
level0_slowdown_writes_trigger=20
compaction_filter=nullptr
level0_file_num_compaction_trigger=4
max_compaction_bytes=1677721600
compaction_options_universal={stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;allow_trivial_move=false;max_merge_width=4294967295;max_size_amplification_percent=200;min_merge_width=2;size_ratio=1;}
memtable_prefix_bloom_size_ratio=0.000000
max_write_buffer_size_to_maintain=0
hard_pending_compaction_bytes_limit=274877906944
ttl=2592000
table_factory=BlockBasedTable
soft_pending_compaction_bytes_limit=68719476736
prefix_extractor=nullptr
bottommost_compression=kDisableCompressionOption
force_consistency_checks=false
paranoid_file_checks=false
compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;}
max_bytes_for_level_multiplier=10.000000
optimize_filters_for_hits=false
level_compaction_dynamic_level_bytes=false
inplace_update_num_locks=10000
inplace_update_support=false
periodic_compaction_seconds=0
disable_auto_compactions=false
report_bg_io_stats=false
[TableOptions/BlockBasedTable "ospg"]
pin_top_level_index_and_filter=true
enable_index_compression=true
read_amp_bytes_per_bit=8589934592
format_version=2
block_align=false
metadata_block_size=4096
block_size_deviation=10
partition_filters=false
block_size=4096
index_block_restart_interval=1
no_block_cache=false
checksum=kCRC32c
whole_key_filtering=true
index_shortening=kShortenSeparators
data_block_index_type=kDataBlockBinarySearch
index_type=kBinarySearch
verify_compression=false
filter_policy=nullptr
data_block_hash_table_util_ratio=0.750000
pin_l0_filter_and_index_blocks_in_cache=false
block_restart_interval=16
cache_index_and_filter_blocks_with_high_priority=true
cache_index_and_filter_blocks=false
hash_index_allow_collision=true
flush_block_policy_factory=FlushBlockBySizePolicyFactory
[CFOptions "gspo"]
sample_for_compression=0
compaction_pri=kMinOverlappingRatio
merge_operator=nullptr
compaction_filter_factory=nullptr
memtable_factory=SkipListFactory
memtable_insert_with_hint_prefix_extractor=nullptr
comparator=leveldb.BytewiseComparator
target_file_size_base=67108864
max_sequential_skip_in_iterations=8
compaction_style=kCompactionStyleLevel
max_bytes_for_level_base=268435456
bloom_locality=0
write_buffer_size=67108864
compression_per_level=
memtable_huge_page_size=0
max_successive_merges=0
arena_block_size=8388608
memtable_whole_key_filtering=false
target_file_size_multiplier=1
max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
num_levels=7
min_write_buffer_number_to_merge=1
max_write_buffer_number_to_maintain=0
max_write_buffer_number=2
compression=kSnappyCompression
level0_stop_writes_trigger=36
level0_slowdown_writes_trigger=20
compaction_filter=nullptr
level0_file_num_compaction_trigger=4
max_compaction_bytes=1677721600
compaction_options_universal={stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;allow_trivial_move=false;max_merge_width=4294967295;max_size_amplification_percent=200;min_merge_width=2;size_ratio=1;}
memtable_prefix_bloom_size_ratio=0.000000
max_write_buffer_size_to_maintain=0
hard_pending_compaction_bytes_limit=274877906944
ttl=2592000
table_factory=BlockBasedTable
soft_pending_compaction_bytes_limit=68719476736
prefix_extractor=nullptr
bottommost_compression=kDisableCompressionOption
force_consistency_checks=false
paranoid_file_checks=false
compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;}
max_bytes_for_level_multiplier=10.000000
optimize_filters_for_hits=false
level_compaction_dynamic_level_bytes=false
inplace_update_num_locks=10000
inplace_update_support=false
periodic_compaction_seconds=0
disable_auto_compactions=false
report_bg_io_stats=false
[TableOptions/BlockBasedTable "gspo"]
pin_top_level_index_and_filter=true
enable_index_compression=true
read_amp_bytes_per_bit=8589934592
format_version=2
block_align=false
metadata_block_size=4096
block_size_deviation=10
partition_filters=false
block_size=4096
index_block_restart_interval=1
no_block_cache=false
checksum=kCRC32c
whole_key_filtering=true
index_shortening=kShortenSeparators
data_block_index_type=kDataBlockBinarySearch
index_type=kBinarySearch
verify_compression=false
filter_policy=nullptr
data_block_hash_table_util_ratio=0.750000
pin_l0_filter_and_index_blocks_in_cache=false
block_restart_interval=16
cache_index_and_filter_blocks_with_high_priority=true
cache_index_and_filter_blocks=false
hash_index_allow_collision=true
flush_block_policy_factory=FlushBlockBySizePolicyFactory
[CFOptions "gpos"]
sample_for_compression=0
compaction_pri=kMinOverlappingRatio
merge_operator=nullptr
compaction_filter_factory=nullptr
memtable_factory=SkipListFactory
memtable_insert_with_hint_prefix_extractor=nullptr
comparator=leveldb.BytewiseComparator
target_file_size_base=67108864
max_sequential_skip_in_iterations=8
compaction_style=kCompactionStyleLevel
max_bytes_for_level_base=268435456
bloom_locality=0
write_buffer_size=67108864
compression_per_level=
memtable_huge_page_size=0
max_successive_merges=0
arena_block_size=8388608
memtable_whole_key_filtering=false
target_file_size_multiplier=1
max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
num_levels=7
min_write_buffer_number_to_merge=1
max_write_buffer_number_to_maintain=0
max_write_buffer_number=2
compression=kSnappyCompression
level0_stop_writes_trigger=36
level0_slowdown_writes_trigger=20
compaction_filter=nullptr
level0_file_num_compaction_trigger=4
max_compaction_bytes=1677721600
compaction_options_universal={stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;allow_trivial_move=false;max_merge_width=4294967295;max_size_amplification_percent=200;min_merge_width=2;size_ratio=1;}
memtable_prefix_bloom_size_ratio=0.000000
max_write_buffer_size_to_maintain=0
hard_pending_compaction_bytes_limit=274877906944
ttl=2592000
table_factory=BlockBasedTable
soft_pending_compaction_bytes_limit=68719476736
prefix_extractor=nullptr
bottommost_compression=kDisableCompressionOption
force_consistency_checks=false
paranoid_file_checks=false
compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;}
max_bytes_for_level_multiplier=10.000000
optimize_filters_for_hits=false
level_compaction_dynamic_level_bytes=false
inplace_update_num_locks=10000
inplace_update_support=false
periodic_compaction_seconds=0
disable_auto_compactions=false
report_bg_io_stats=false
[TableOptions/BlockBasedTable "gpos"]
pin_top_level_index_and_filter=true
enable_index_compression=true
read_amp_bytes_per_bit=8589934592
format_version=2
block_align=false
metadata_block_size=4096
block_size_deviation=10
partition_filters=false
block_size=4096
index_block_restart_interval=1
no_block_cache=false
checksum=kCRC32c
whole_key_filtering=true
index_shortening=kShortenSeparators
data_block_index_type=kDataBlockBinarySearch
index_type=kBinarySearch
verify_compression=false
filter_policy=nullptr
data_block_hash_table_util_ratio=0.750000
pin_l0_filter_and_index_blocks_in_cache=false
block_restart_interval=16
cache_index_and_filter_blocks_with_high_priority=true
cache_index_and_filter_blocks=false
hash_index_allow_collision=true
flush_block_policy_factory=FlushBlockBySizePolicyFactory
[CFOptions "gosp"]
sample_for_compression=0
compaction_pri=kMinOverlappingRatio
merge_operator=nullptr
compaction_filter_factory=nullptr
memtable_factory=SkipListFactory
memtable_insert_with_hint_prefix_extractor=nullptr
comparator=leveldb.BytewiseComparator
target_file_size_base=67108864
max_sequential_skip_in_iterations=8
compaction_style=kCompactionStyleLevel
max_bytes_for_level_base=268435456
bloom_locality=0
write_buffer_size=67108864
compression_per_level=
memtable_huge_page_size=0
max_successive_merges=0
arena_block_size=8388608
memtable_whole_key_filtering=false
target_file_size_multiplier=1
max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
num_levels=7
min_write_buffer_number_to_merge=1
max_write_buffer_number_to_maintain=0
max_write_buffer_number=2
compression=kSnappyCompression
level0_stop_writes_trigger=36
level0_slowdown_writes_trigger=20
compaction_filter=nullptr
level0_file_num_compaction_trigger=4
max_compaction_bytes=1677721600
compaction_options_universal={stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;allow_trivial_move=false;max_merge_width=4294967295;max_size_amplification_percent=200;min_merge_width=2;size_ratio=1;}
memtable_prefix_bloom_size_ratio=0.000000
max_write_buffer_size_to_maintain=0
hard_pending_compaction_bytes_limit=274877906944
ttl=2592000
table_factory=BlockBasedTable
soft_pending_compaction_bytes_limit=68719476736
prefix_extractor=nullptr
bottommost_compression=kDisableCompressionOption
force_consistency_checks=false
paranoid_file_checks=false
compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;}
max_bytes_for_level_multiplier=10.000000
optimize_filters_for_hits=false
level_compaction_dynamic_level_bytes=false
inplace_update_num_locks=10000
inplace_update_support=false
periodic_compaction_seconds=0
disable_auto_compactions=false
report_bg_io_stats=false
[TableOptions/BlockBasedTable "gosp"]
pin_top_level_index_and_filter=true
enable_index_compression=true
read_amp_bytes_per_bit=8589934592
format_version=2
block_align=false
metadata_block_size=4096
block_size_deviation=10
partition_filters=false
block_size=4096
index_block_restart_interval=1
no_block_cache=false
checksum=kCRC32c
whole_key_filtering=true
index_shortening=kShortenSeparators
data_block_index_type=kDataBlockBinarySearch
index_type=kBinarySearch
verify_compression=false
filter_policy=nullptr
data_block_hash_table_util_ratio=0.750000
pin_l0_filter_and_index_blocks_in_cache=false
block_restart_interval=16
cache_index_and_filter_blocks_with_high_priority=true
cache_index_and_filter_blocks=false
hash_index_allow_collision=true
flush_block_policy_factory=FlushBlockBySizePolicyFactory
[CFOptions "dspo"]
sample_for_compression=0
compaction_pri=kMinOverlappingRatio
merge_operator=nullptr
compaction_filter_factory=nullptr
memtable_factory=SkipListFactory
memtable_insert_with_hint_prefix_extractor=nullptr
comparator=leveldb.BytewiseComparator
target_file_size_base=67108864
max_sequential_skip_in_iterations=8
compaction_style=kCompactionStyleLevel
max_bytes_for_level_base=268435456
bloom_locality=0
write_buffer_size=67108864
compression_per_level=
memtable_huge_page_size=0
max_successive_merges=0
arena_block_size=8388608
memtable_whole_key_filtering=false
target_file_size_multiplier=1
max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
num_levels=7
min_write_buffer_number_to_merge=1
max_write_buffer_number_to_maintain=0
max_write_buffer_number=2
compression=kSnappyCompression
level0_stop_writes_trigger=36
level0_slowdown_writes_trigger=20
compaction_filter=nullptr
level0_file_num_compaction_trigger=4
max_compaction_bytes=1677721600
compaction_options_universal={stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;allow_trivial_move=false;max_merge_width=4294967295;max_size_amplification_percent=200;min_merge_width=2;size_ratio=1;}
memtable_prefix_bloom_size_ratio=0.000000
max_write_buffer_size_to_maintain=0
hard_pending_compaction_bytes_limit=274877906944
ttl=2592000
table_factory=BlockBasedTable
soft_pending_compaction_bytes_limit=68719476736
prefix_extractor=nullptr
bottommost_compression=kDisableCompressionOption
force_consistency_checks=false
paranoid_file_checks=false
compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;}
max_bytes_for_level_multiplier=10.000000
optimize_filters_for_hits=false
level_compaction_dynamic_level_bytes=false
inplace_update_num_locks=10000
inplace_update_support=false
periodic_compaction_seconds=0
disable_auto_compactions=false
report_bg_io_stats=false
[TableOptions/BlockBasedTable "dspo"]
pin_top_level_index_and_filter=true
enable_index_compression=true
read_amp_bytes_per_bit=8589934592
format_version=2
block_align=false
metadata_block_size=4096
block_size_deviation=10
partition_filters=false
block_size=4096
index_block_restart_interval=1
no_block_cache=false
checksum=kCRC32c
whole_key_filtering=true
index_shortening=kShortenSeparators
data_block_index_type=kDataBlockBinarySearch
index_type=kBinarySearch
verify_compression=false
filter_policy=nullptr
data_block_hash_table_util_ratio=0.750000
pin_l0_filter_and_index_blocks_in_cache=false
block_restart_interval=16
cache_index_and_filter_blocks_with_high_priority=true
cache_index_and_filter_blocks=false
hash_index_allow_collision=true
flush_block_policy_factory=FlushBlockBySizePolicyFactory
[CFOptions "dpos"]
sample_for_compression=0
compaction_pri=kMinOverlappingRatio
merge_operator=nullptr
compaction_filter_factory=nullptr
memtable_factory=SkipListFactory
memtable_insert_with_hint_prefix_extractor=nullptr
comparator=leveldb.BytewiseComparator
target_file_size_base=67108864
max_sequential_skip_in_iterations=8
compaction_style=kCompactionStyleLevel
max_bytes_for_level_base=268435456
bloom_locality=0
write_buffer_size=67108864
compression_per_level=
memtable_huge_page_size=0
max_successive_merges=0
arena_block_size=8388608
memtable_whole_key_filtering=false
target_file_size_multiplier=1
max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
num_levels=7
min_write_buffer_number_to_merge=1
max_write_buffer_number_to_maintain=0
max_write_buffer_number=2
compression=kSnappyCompression
level0_stop_writes_trigger=36
level0_slowdown_writes_trigger=20
compaction_filter=nullptr
level0_file_num_compaction_trigger=4
max_compaction_bytes=1677721600
compaction_options_universal={stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;allow_trivial_move=false;max_merge_width=4294967295;max_size_amplification_percent=200;min_merge_width=2;size_ratio=1;}
memtable_prefix_bloom_size_ratio=0.000000
max_write_buffer_size_to_maintain=0
hard_pending_compaction_bytes_limit=274877906944
ttl=2592000
table_factory=BlockBasedTable
soft_pending_compaction_bytes_limit=68719476736
prefix_extractor=nullptr
bottommost_compression=kDisableCompressionOption
force_consistency_checks=false
paranoid_file_checks=false
compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;}
max_bytes_for_level_multiplier=10.000000
optimize_filters_for_hits=false
level_compaction_dynamic_level_bytes=false
inplace_update_num_locks=10000
inplace_update_support=false
periodic_compaction_seconds=0
disable_auto_compactions=false
report_bg_io_stats=false
[TableOptions/BlockBasedTable "dpos"]
pin_top_level_index_and_filter=true
enable_index_compression=true
read_amp_bytes_per_bit=8589934592
format_version=2
block_align=false
metadata_block_size=4096
block_size_deviation=10
partition_filters=false
block_size=4096
index_block_restart_interval=1
no_block_cache=false
checksum=kCRC32c
whole_key_filtering=true
index_shortening=kShortenSeparators
data_block_index_type=kDataBlockBinarySearch
index_type=kBinarySearch
verify_compression=false
filter_policy=nullptr
data_block_hash_table_util_ratio=0.750000
pin_l0_filter_and_index_blocks_in_cache=false
block_restart_interval=16
cache_index_and_filter_blocks_with_high_priority=true
cache_index_and_filter_blocks=false
hash_index_allow_collision=true
flush_block_policy_factory=FlushBlockBySizePolicyFactory
[CFOptions "dosp"]
sample_for_compression=0
compaction_pri=kMinOverlappingRatio
merge_operator=nullptr
compaction_filter_factory=nullptr
memtable_factory=SkipListFactory
memtable_insert_with_hint_prefix_extractor=nullptr
comparator=leveldb.BytewiseComparator
target_file_size_base=67108864
max_sequential_skip_in_iterations=8
compaction_style=kCompactionStyleLevel
max_bytes_for_level_base=268435456
bloom_locality=0
write_buffer_size=67108864
compression_per_level=
memtable_huge_page_size=0
max_successive_merges=0
arena_block_size=8388608
memtable_whole_key_filtering=false
target_file_size_multiplier=1
max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
num_levels=7
min_write_buffer_number_to_merge=1
max_write_buffer_number_to_maintain=0
max_write_buffer_number=2
compression=kSnappyCompression
level0_stop_writes_trigger=36
level0_slowdown_writes_trigger=20
compaction_filter=nullptr
level0_file_num_compaction_trigger=4
max_compaction_bytes=1677721600
compaction_options_universal={stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;allow_trivial_move=false;max_merge_width=4294967295;max_size_amplification_percent=200;min_merge_width=2;size_ratio=1;}
memtable_prefix_bloom_size_ratio=0.000000
max_write_buffer_size_to_maintain=0
hard_pending_compaction_bytes_limit=274877906944
ttl=2592000
table_factory=BlockBasedTable
soft_pending_compaction_bytes_limit=68719476736
prefix_extractor=nullptr
bottommost_compression=kDisableCompressionOption
force_consistency_checks=false
paranoid_file_checks=false
compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;}
max_bytes_for_level_multiplier=10.000000
optimize_filters_for_hits=false
level_compaction_dynamic_level_bytes=false
inplace_update_num_locks=10000
inplace_update_support=false
periodic_compaction_seconds=0
disable_auto_compactions=false
report_bg_io_stats=false
[TableOptions/BlockBasedTable "dosp"]
pin_top_level_index_and_filter=true
enable_index_compression=true
read_amp_bytes_per_bit=8589934592
format_version=2
block_align=false
metadata_block_size=4096
block_size_deviation=10
partition_filters=false
block_size=4096
index_block_restart_interval=1
no_block_cache=false
checksum=kCRC32c
whole_key_filtering=true
index_shortening=kShortenSeparators
data_block_index_type=kDataBlockBinarySearch
index_type=kBinarySearch
verify_compression=false
filter_policy=nullptr
data_block_hash_table_util_ratio=0.750000
pin_l0_filter_and_index_blocks_in_cache=false
block_restart_interval=16
cache_index_and_filter_blocks_with_high_priority=true
cache_index_and_filter_blocks=false
hash_index_allow_collision=true
flush_block_policy_factory=FlushBlockBySizePolicyFactory

@ -1,92 +0,0 @@
use oxigraph::model::vocab::{rdf, xsd};
use oxigraph::model::*;
use oxigraph::RocksDbStore;
use std::io;
use std::process::Command;
fn quads(graph_name: impl Into<GraphNameRef<'static>>) -> Vec<QuadRef<'static>> {
let graph_name = graph_name.into();
let paris = NamedNodeRef::new_unchecked("http://www.wikidata.org/entity/Q90");
let france = NamedNodeRef::new_unchecked("http://www.wikidata.org/entity/Q142");
let city = NamedNodeRef::new_unchecked("http://schema.org/City");
let name = NamedNodeRef::new_unchecked("http://schema.org/name");
let country = NamedNodeRef::new_unchecked("http://schema.org/country");
let population = NamedNodeRef::new_unchecked("http://schema.org/population");
let start_date = NamedNodeRef::new_unchecked("http://schema.org/startDate");
let url = NamedNodeRef::new_unchecked("http://schema.org/url");
let postal_code = NamedNodeRef::new_unchecked("http://schema.org/postalCode");
vec![
QuadRef::new(paris, rdf::TYPE, city, graph_name),
QuadRef::new(
paris,
name,
LiteralRef::new_language_tagged_literal_unchecked("Paris", "fr"),
graph_name,
),
QuadRef::new(
paris,
name,
LiteralRef::new_language_tagged_literal_unchecked("la ville lumière", "fr"),
graph_name,
),
QuadRef::new(paris, country, france, graph_name),
QuadRef::new(
paris,
population,
LiteralRef::new_typed_literal("2000000", xsd::INTEGER),
graph_name,
),
QuadRef::new(
paris,
start_date,
LiteralRef::new_typed_literal("-300", xsd::G_YEAR),
graph_name,
),
QuadRef::new(
paris,
url,
LiteralRef::new_typed_literal("https://www.paris.fr/", xsd::ANY_URI),
graph_name,
),
QuadRef::new(
paris,
postal_code,
LiteralRef::new_simple_literal("75001"),
graph_name,
),
]
}
#[test]
fn test_backward_compatibility() -> io::Result<()> {
{
let store = RocksDbStore::open("tests/rockdb_bc_data")?;
for q in quads(GraphNameRef::DefaultGraph) {
assert!(store.contains(q)?);
}
let graph_name =
NamedNodeRef::new_unchecked("http://www.wikidata.org/wiki/Special:EntityData/Q90");
for q in quads(graph_name) {
assert!(store.contains(q)?);
}
assert!(store.contains_named_graph(graph_name)?);
assert_eq!(
vec![NamedOrBlankNode::from(graph_name)],
store.named_graphs().collect::<io::Result<Vec<_>>>()?
);
};
reset_dir("tests/rockdb_bc_data")?;
Ok(())
}
fn reset_dir(dir: &str) -> io::Result<()> {
assert!(Command::new("git")
.args(&["clean", "-fX", dir])
.status()?
.success());
assert!(Command::new("git")
.args(&["checkout", "HEAD", "--", dir])
.status()?
.success());
Ok(())
}

@ -11,17 +11,12 @@ SPARQL server based on Oxigraph
""" """
edition = "2018" edition = "2018"
[features]
sled = ["oxigraph/sled"]
rocksdb = ["oxigraph/rocksdb"]
default = ["rocksdb"]
[dependencies] [dependencies]
argh = "0.1" argh = "0.1"
async-std = { version = "1", features = ["attributes"] } async-std = { version = "1", features = ["attributes"] }
async-h1 = "2" async-h1 = "2"
http-types = "2" http-types = "2"
oxigraph = { version = "0.2", path="../lib", features = ["http_client"] } oxigraph = { version = "0.2", path="../lib", features = ["sled", "http_client"] }
rand = "0.8" rand = "0.8"
url = "2" url = "2"
oxiri = "0.1" oxiri = "0.1"

@ -1,6 +1,5 @@
FROM rust:1-buster as builder FROM rust:1-buster as builder
RUN apt-get update && apt-get install clang -y
COPY . /oxigraph COPY . /oxigraph
RUN cd /oxigraph/server && cargo build --release RUN cd /oxigraph/server && cargo build --release

@ -11,7 +11,7 @@ Oxigraph Server
Oxigraph Server is a standalone HTTP server providing a graph database implementing the [SPARQL](https://www.w3.org/TR/sparql11-overview/) standard. Oxigraph Server is a standalone HTTP server providing a graph database implementing the [SPARQL](https://www.w3.org/TR/sparql11-overview/) standard.
Its goal is to provide a compliant, safe, and fast graph database based on the [RocksDB](https://rocksdb.org/) key-value stores. Its goal is to provide a compliant, safe, and fast graph database based on the [Sled](https://sled.rs/) key-value stores.
It is written in Rust. It is written in Rust.
It also provides a set of utility functions for reading, writing, and processing RDF files. It also provides a set of utility functions for reading, writing, and processing RDF files.
@ -34,7 +34,7 @@ A preliminary benchmark [is provided](../bench/README.md).
## Installation ## Installation
You need to have [a recent stable version of Rust and Cargo installed](https://www.rust-lang.org/tools/install). You also need [clang](https://clang.llvm.org/) to build RocksDB. You need to have [a recent stable version of Rust and Cargo installed](https://www.rust-lang.org/tools/install).
To download, build and install the latest released version run `cargo install oxigraph_server`. To download, build and install the latest released version run `cargo install oxigraph_server`.
There is no need to clone the git repository. There is no need to clone the git repository.

@ -24,9 +24,6 @@ use oxigraph::io::{DatasetFormat, GraphFormat};
use oxigraph::model::{GraphName, GraphNameRef, NamedNode, NamedOrBlankNode}; use oxigraph::model::{GraphName, GraphNameRef, NamedNode, NamedOrBlankNode};
use oxigraph::sparql::algebra::GraphUpdateOperation; use oxigraph::sparql::algebra::GraphUpdateOperation;
use oxigraph::sparql::{Query, QueryResults, QueryResultsFormat, Update}; use oxigraph::sparql::{Query, QueryResults, QueryResultsFormat, Update};
#[cfg(feature = "rocksdb")]
use oxigraph::RocksDbStore as Store;
#[cfg(all(feature = "sled", not(feature = "rocksdb")))]
use oxigraph::SledStore as Store; use oxigraph::SledStore as Store;
use oxiri::Iri; use oxiri::Iri;
use rand::random; use rand::random;

@ -17,6 +17,6 @@ async-h1 = "2"
chrono = "0.4" chrono = "0.4"
http-client = { version = "6", features = ["h1_client"] } http-client = { version = "6", features = ["h1_client"] }
http-types = "2" http-types = "2"
oxigraph = { version = "0.2", path ="../lib", features = ["rocksdb", "http_client"] } oxigraph = { version = "0.2", path ="../lib", features = ["sled", "http_client"] }
serde_json = "1" serde_json = "1"
url = "2" url = "2"

@ -1,6 +1,5 @@
FROM rust:1-buster as builder FROM rust:1-buster as builder
RUN apt-get update && apt-get install clang -y
COPY . /oxigraph COPY . /oxigraph
RUN cd /oxigraph/wikibase && cargo build --release RUN cd /oxigraph/wikibase && cargo build --release

@ -16,7 +16,7 @@ Oxigraph and Oxigraph Wikibase are in heavy development and not been optimized y
## Installation ## Installation
You need to have [a recent stable version of Rust and Cargo installed](https://www.rust-lang.org/tools/install). You also need [clang](https://clang.llvm.org/) to build RocksDB. You need to have [a recent stable version of Rust and Cargo installed](https://www.rust-lang.org/tools/install).
To download, build and install the latest released version run `cargo install oxigraph_wikibase`. To download, build and install the latest released version run `cargo install oxigraph_wikibase`.
There is no need to clone the git repository. There is no need to clone the git repository.

@ -7,7 +7,7 @@ use http_client::HttpClient;
use http_types::{headers, Method, Request, Result}; use http_types::{headers, Method, Request, Result};
use oxigraph::io::GraphFormat; use oxigraph::io::GraphFormat;
use oxigraph::model::NamedNodeRef; use oxigraph::model::NamedNodeRef;
use oxigraph::RocksDbStore; use oxigraph::SledStore;
use serde_json::Value; use serde_json::Value;
use std::collections::{HashMap, HashSet}; use std::collections::{HashMap, HashSet};
use std::io::{BufReader, Cursor, Read}; use std::io::{BufReader, Cursor, Read};
@ -16,7 +16,7 @@ use std::time::Duration;
use url::{form_urlencoded, Url}; use url::{form_urlencoded, Url};
pub struct WikibaseLoader { pub struct WikibaseLoader {
store: RocksDbStore, store: SledStore,
client: H1Client, client: H1Client,
api_url: Url, api_url: Url,
entity_data_url: Url, entity_data_url: Url,
@ -28,7 +28,7 @@ pub struct WikibaseLoader {
impl WikibaseLoader { impl WikibaseLoader {
pub fn new( pub fn new(
store: RocksDbStore, store: SledStore,
api_url: &str, api_url: &str,
pages_base_url: &str, pages_base_url: &str,
namespaces: &[u32], namespaces: &[u32],
@ -230,23 +230,20 @@ impl WikibaseLoader {
} }
fn load_entity_data(&self, uri: &str, data: impl Read) -> Result<()> { fn load_entity_data(&self, uri: &str, data: impl Read) -> Result<()> {
let graph_name = NamedNodeRef::new(uri)?.into(); let graph_name = NamedNodeRef::new(uri)?;
self.store.transaction(|transaction| { //TODO: proper transaction
let to_remove = self for q in self
.store .store
.quads_for_pattern(None, None, None, Some(graph_name)) .quads_for_pattern(None, None, None, Some(graph_name.into()))
.collect::<std::result::Result<Vec<_>, _>>()?; {
for q in to_remove { self.store.remove(&q?)?;
transaction.remove(&q)?;
} }
self.store.load_graph(
transaction.load_graph(
BufReader::new(data), BufReader::new(data),
GraphFormat::NTriples, GraphFormat::NTriples,
NamedNodeRef::new(uri)?, graph_name,
None, None,
)?; )?;
Ok(()) Ok(())
})
} }
} }

@ -23,7 +23,7 @@ use http_types::{
use oxigraph::io::GraphFormat; use oxigraph::io::GraphFormat;
use oxigraph::model::{GraphName, NamedNode, NamedOrBlankNode}; use oxigraph::model::{GraphName, NamedNode, NamedOrBlankNode};
use oxigraph::sparql::{Query, QueryResults, QueryResultsFormat}; use oxigraph::sparql::{Query, QueryResults, QueryResultsFormat};
use oxigraph::RocksDbStore; use oxigraph::SledStore;
use std::str::FromStr; use std::str::FromStr;
use std::time::Duration; use std::time::Duration;
use url::form_urlencoded; use url::form_urlencoded;
@ -65,7 +65,7 @@ struct Args {
pub async fn main() -> Result<()> { pub async fn main() -> Result<()> {
let args: Args = argh::from_env(); let args: Args = argh::from_env();
let store = RocksDbStore::open(args.file)?; let store = SledStore::open(args.file)?;
let mediawiki_api = args.mediawiki_api.clone(); let mediawiki_api = args.mediawiki_api.clone();
let mediawiki_base_url = args.mediawiki_base_url.clone(); let mediawiki_base_url = args.mediawiki_base_url.clone();
let namespaces = args let namespaces = args
@ -106,7 +106,7 @@ pub async fn main() -> Result<()> {
.await .await
} }
async fn handle_request(request: Request, store: RocksDbStore) -> Result<Response> { async fn handle_request(request: Request, store: SledStore) -> Result<Response> {
Ok(match (request.url().path(), request.method()) { Ok(match (request.url().path(), request.method()) {
("/query", Method::Get) => { ("/query", Method::Get) => {
configure_and_evaluate_sparql_query(store, url_query(&request), None, request)? configure_and_evaluate_sparql_query(store, url_query(&request), None, request)?
@ -159,7 +159,7 @@ fn url_query(request: &Request) -> Vec<u8> {
} }
fn configure_and_evaluate_sparql_query( fn configure_and_evaluate_sparql_query(
store: RocksDbStore, store: SledStore,
encoded: Vec<u8>, encoded: Vec<u8>,
mut query: Option<String>, mut query: Option<String>,
request: Request, request: Request,
@ -187,7 +187,7 @@ fn configure_and_evaluate_sparql_query(
} }
fn evaluate_sparql_query( fn evaluate_sparql_query(
store: RocksDbStore, store: SledStore,
query: String, query: String,
default_graph_uris: Vec<String>, default_graph_uris: Vec<String>,
named_graph_uris: Vec<String>, named_graph_uris: Vec<String>,

Loading…
Cancel
Save