Adds small backward compatibility tests for Sled and RocksDB

pull/46/head
Tpt 4 years ago
parent 1c2bf8f995
commit 10e4c1518d
  1. 4
      .gitignore
  2. 3
      js/src/store.rs
  3. 4
      lib/src/store/memory.rs
  4. 9
      lib/src/store/sled.rs
  5. BIN
      lib/tests/rockdb_bc_data/000003.log
  6. 1
      lib/tests/rockdb_bc_data/CURRENT
  7. 1
      lib/tests/rockdb_bc_data/IDENTITY
  8. 0
      lib/tests/rockdb_bc_data/LOCK
  9. BIN
      lib/tests/rockdb_bc_data/MANIFEST-000004
  10. 964
      lib/tests/rockdb_bc_data/OPTIONS-000026
  11. 83
      lib/tests/rocksdb_store.rs
  12. 4
      lib/tests/sled_bc_data/conf
  13. BIN
      lib/tests/sled_bc_data/db
  14. 179
      lib/tests/sled_store.rs

4
.gitignore vendored

@ -3,4 +3,6 @@ target/
Cargo.lock Cargo.lock
.idea .idea
*.iml *.iml
js/node_modules js/node_modules
lib/tests/rockdb_bc_data
lib/tests/sled_bc_data

@ -38,7 +38,8 @@ impl JsMemoryStore {
} }
pub fn add(&self, quad: &JsValue) -> Result<(), JsValue> { pub fn add(&self, quad: &JsValue) -> Result<(), JsValue> {
self.store.insert(self.from_js.to_quad(quad)?.try_into()?); self.store
.insert(Quad::try_from(self.from_js.to_quad(quad)?)?);
Ok(()) Ok(())
} }

@ -354,9 +354,9 @@ impl MemoryStore {
/// Adds a quad to this store. /// Adds a quad to this store.
#[allow(clippy::needless_pass_by_value)] #[allow(clippy::needless_pass_by_value)]
pub fn insert(&self, quad: Quad) { pub fn insert(&self, quad: impl Into<Quad>) {
let mut this = self; let mut this = self;
let quad = this.encode_quad(quad.as_ref()).unwrap_infallible(); let quad = this.encode_quad(quad.into().as_ref()).unwrap_infallible();
this.insert_encoded(&quad).unwrap_infallible(); this.insert_encoded(&quad).unwrap_infallible();
} }

@ -1040,6 +1040,15 @@ impl<T> From<TransactionError<T>> for SledTransactionError<T> {
} }
} }
impl<T: Into<io::Error>> From<SledTransactionError<T>> for io::Error {
fn from(e: SledTransactionError<T>) -> Self {
match e {
SledTransactionError::Abort(e) => e.into(),
SledTransactionError::Storage(e) => e,
}
}
}
/// An error returned from the transaction methods. /// An error returned from the transaction methods.
/// Should be returned as it is /// Should be returned as it is
#[derive(Debug)] #[derive(Debug)]

@ -0,0 +1 @@
f08a4c5b-0479-408c-80d3-7d4b10d7c7aa

@ -0,0 +1,964 @@
# This is a RocksDB option file.
#
# For detailed file format spec, please refer to the example file
# in examples/rocksdb_option_file_example.ini
#
[Version]
rocksdb_version=6.7.3
options_file_version=1.1
[DBOptions]
write_dbid_to_manifest=false
avoid_unnecessary_blocking_io=false
two_write_queues=false
allow_ingest_behind=false
writable_file_max_buffer_size=1048576
avoid_flush_during_shutdown=false
avoid_flush_during_recovery=false
info_log_level=INFO_LEVEL
access_hint_on_compaction_start=NORMAL
allow_concurrent_memtable_write=true
enable_pipelined_write=false
stats_dump_period_sec=600
stats_persist_period_sec=600
strict_bytes_per_sync=false
WAL_ttl_seconds=0
WAL_size_limit_MB=0
max_subcompactions=1
dump_malloc_stats=false
db_log_dir=
wal_recovery_mode=kPointInTimeRecovery
log_file_time_to_roll=0
enable_write_thread_adaptive_yield=true
recycle_log_file_num=0
table_cache_numshardbits=6
atomic_flush=false
preserve_deletes=false
stats_history_buffer_size=1048576
max_open_files=-1
max_file_opening_threads=16
delete_obsolete_files_period_micros=21600000000
max_background_flushes=-1
write_thread_slow_yield_usec=3
base_background_compactions=-1
manual_wal_flush=false
wal_dir=tests/rockdb_bc_data
max_background_compactions=-1
bytes_per_sync=0
max_background_jobs=2
use_fsync=false
unordered_write=false
fail_if_options_file_error=false
random_access_max_buffer_size=1048576
compaction_readahead_size=0
wal_bytes_per_sync=0
new_table_reader_for_compaction_inputs=false
skip_stats_update_on_db_open=false
persist_stats_to_disk=false
skip_log_error_on_recovery=false
log_readahead_size=0
is_fd_close_on_exec=true
use_adaptive_mutex=false
error_if_exists=false
write_thread_max_yield_usec=100
enable_thread_tracking=false
db_write_buffer_size=0
create_missing_column_families=true
paranoid_checks=true
create_if_missing=true
max_manifest_file_size=1073741824
allow_2pc=false
max_total_wal_size=0
use_direct_io_for_flush_and_compaction=false
manifest_preallocation_size=4194304
use_direct_reads=false
delayed_write_rate=16777216
allow_fallocate=true
max_write_batch_group_size_bytes=1048576
keep_log_file_num=1000
allow_mmap_reads=false
max_log_file_size=0
allow_mmap_writes=false
advise_random_on_open=true
[CFOptions "default"]
sample_for_compression=0
compaction_pri=kMinOverlappingRatio
merge_operator=nullptr
compaction_filter_factory=nullptr
memtable_factory=SkipListFactory
memtable_insert_with_hint_prefix_extractor=nullptr
comparator=leveldb.BytewiseComparator
target_file_size_base=67108864
max_sequential_skip_in_iterations=8
compaction_style=kCompactionStyleLevel
max_bytes_for_level_base=268435456
bloom_locality=0
write_buffer_size=67108864
compression_per_level=
memtable_huge_page_size=0
max_successive_merges=0
arena_block_size=8388608
memtable_whole_key_filtering=false
target_file_size_multiplier=1
max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
num_levels=7
min_write_buffer_number_to_merge=1
max_write_buffer_number_to_maintain=0
max_write_buffer_number=2
compression=kSnappyCompression
level0_stop_writes_trigger=36
level0_slowdown_writes_trigger=20
compaction_filter=nullptr
level0_file_num_compaction_trigger=4
max_compaction_bytes=1677721600
compaction_options_universal={stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;allow_trivial_move=false;max_merge_width=4294967295;max_size_amplification_percent=200;min_merge_width=2;size_ratio=1;}
memtable_prefix_bloom_size_ratio=0.000000
max_write_buffer_size_to_maintain=0
hard_pending_compaction_bytes_limit=274877906944
ttl=2592000
table_factory=BlockBasedTable
soft_pending_compaction_bytes_limit=68719476736
prefix_extractor=nullptr
bottommost_compression=kDisableCompressionOption
force_consistency_checks=false
paranoid_file_checks=false
compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;}
max_bytes_for_level_multiplier=10.000000
optimize_filters_for_hits=false
level_compaction_dynamic_level_bytes=false
inplace_update_num_locks=10000
inplace_update_support=false
periodic_compaction_seconds=0
disable_auto_compactions=false
report_bg_io_stats=false
[TableOptions/BlockBasedTable "default"]
pin_top_level_index_and_filter=true
enable_index_compression=true
read_amp_bytes_per_bit=8589934592
format_version=2
block_align=false
metadata_block_size=4096
block_size_deviation=10
partition_filters=false
block_size=4096
index_block_restart_interval=1
no_block_cache=false
checksum=kCRC32c
whole_key_filtering=true
index_shortening=kShortenSeparators
data_block_index_type=kDataBlockBinarySearch
index_type=kBinarySearch
verify_compression=false
filter_policy=nullptr
data_block_hash_table_util_ratio=0.750000
pin_l0_filter_and_index_blocks_in_cache=false
block_restart_interval=16
cache_index_and_filter_blocks_with_high_priority=true
cache_index_and_filter_blocks=false
hash_index_allow_collision=true
flush_block_policy_factory=FlushBlockBySizePolicyFactory
[CFOptions "id2str"]
sample_for_compression=0
compaction_pri=kMinOverlappingRatio
merge_operator=nullptr
compaction_filter_factory=nullptr
memtable_factory=SkipListFactory
memtable_insert_with_hint_prefix_extractor=nullptr
comparator=leveldb.BytewiseComparator
target_file_size_base=67108864
max_sequential_skip_in_iterations=8
compaction_style=kCompactionStyleLevel
max_bytes_for_level_base=268435456
bloom_locality=0
write_buffer_size=67108864
compression_per_level=
memtable_huge_page_size=0
max_successive_merges=0
arena_block_size=8388608
memtable_whole_key_filtering=false
target_file_size_multiplier=1
max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
num_levels=7
min_write_buffer_number_to_merge=1
max_write_buffer_number_to_maintain=0
max_write_buffer_number=2
compression=kSnappyCompression
level0_stop_writes_trigger=36
level0_slowdown_writes_trigger=20
compaction_filter=nullptr
level0_file_num_compaction_trigger=4
max_compaction_bytes=1677721600
compaction_options_universal={stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;allow_trivial_move=false;max_merge_width=4294967295;max_size_amplification_percent=200;min_merge_width=2;size_ratio=1;}
memtable_prefix_bloom_size_ratio=0.000000
max_write_buffer_size_to_maintain=0
hard_pending_compaction_bytes_limit=274877906944
ttl=2592000
table_factory=BlockBasedTable
soft_pending_compaction_bytes_limit=68719476736
prefix_extractor=nullptr
bottommost_compression=kDisableCompressionOption
force_consistency_checks=false
paranoid_file_checks=false
compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;}
max_bytes_for_level_multiplier=10.000000
optimize_filters_for_hits=false
level_compaction_dynamic_level_bytes=false
inplace_update_num_locks=10000
inplace_update_support=false
periodic_compaction_seconds=0
disable_auto_compactions=false
report_bg_io_stats=false
[TableOptions/BlockBasedTable "id2str"]
pin_top_level_index_and_filter=true
enable_index_compression=true
read_amp_bytes_per_bit=8589934592
format_version=2
block_align=false
metadata_block_size=4096
block_size_deviation=10
partition_filters=false
block_size=4096
index_block_restart_interval=1
no_block_cache=false
checksum=kCRC32c
whole_key_filtering=true
index_shortening=kShortenSeparators
data_block_index_type=kDataBlockBinarySearch
index_type=kBinarySearch
verify_compression=false
filter_policy=nullptr
data_block_hash_table_util_ratio=0.750000
pin_l0_filter_and_index_blocks_in_cache=false
block_restart_interval=16
cache_index_and_filter_blocks_with_high_priority=true
cache_index_and_filter_blocks=false
hash_index_allow_collision=true
flush_block_policy_factory=FlushBlockBySizePolicyFactory
[CFOptions "spog"]
sample_for_compression=0
compaction_pri=kMinOverlappingRatio
merge_operator=nullptr
compaction_filter_factory=nullptr
memtable_factory=SkipListFactory
memtable_insert_with_hint_prefix_extractor=nullptr
comparator=leveldb.BytewiseComparator
target_file_size_base=67108864
max_sequential_skip_in_iterations=8
compaction_style=kCompactionStyleLevel
max_bytes_for_level_base=268435456
bloom_locality=0
write_buffer_size=67108864
compression_per_level=
memtable_huge_page_size=0
max_successive_merges=0
arena_block_size=8388608
memtable_whole_key_filtering=false
target_file_size_multiplier=1
max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
num_levels=7
min_write_buffer_number_to_merge=1
max_write_buffer_number_to_maintain=0
max_write_buffer_number=2
compression=kSnappyCompression
level0_stop_writes_trigger=36
level0_slowdown_writes_trigger=20
compaction_filter=nullptr
level0_file_num_compaction_trigger=4
max_compaction_bytes=1677721600
compaction_options_universal={stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;allow_trivial_move=false;max_merge_width=4294967295;max_size_amplification_percent=200;min_merge_width=2;size_ratio=1;}
memtable_prefix_bloom_size_ratio=0.000000
max_write_buffer_size_to_maintain=0
hard_pending_compaction_bytes_limit=274877906944
ttl=2592000
table_factory=BlockBasedTable
soft_pending_compaction_bytes_limit=68719476736
prefix_extractor=nullptr
bottommost_compression=kDisableCompressionOption
force_consistency_checks=false
paranoid_file_checks=false
compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;}
max_bytes_for_level_multiplier=10.000000
optimize_filters_for_hits=false
level_compaction_dynamic_level_bytes=false
inplace_update_num_locks=10000
inplace_update_support=false
periodic_compaction_seconds=0
disable_auto_compactions=false
report_bg_io_stats=false
[TableOptions/BlockBasedTable "spog"]
pin_top_level_index_and_filter=true
enable_index_compression=true
read_amp_bytes_per_bit=8589934592
format_version=2
block_align=false
metadata_block_size=4096
block_size_deviation=10
partition_filters=false
block_size=4096
index_block_restart_interval=1
no_block_cache=false
checksum=kCRC32c
whole_key_filtering=true
index_shortening=kShortenSeparators
data_block_index_type=kDataBlockBinarySearch
index_type=kBinarySearch
verify_compression=false
filter_policy=nullptr
data_block_hash_table_util_ratio=0.750000
pin_l0_filter_and_index_blocks_in_cache=false
block_restart_interval=16
cache_index_and_filter_blocks_with_high_priority=true
cache_index_and_filter_blocks=false
hash_index_allow_collision=true
flush_block_policy_factory=FlushBlockBySizePolicyFactory
[CFOptions "posg"]
sample_for_compression=0
compaction_pri=kMinOverlappingRatio
merge_operator=nullptr
compaction_filter_factory=nullptr
memtable_factory=SkipListFactory
memtable_insert_with_hint_prefix_extractor=nullptr
comparator=leveldb.BytewiseComparator
target_file_size_base=67108864
max_sequential_skip_in_iterations=8
compaction_style=kCompactionStyleLevel
max_bytes_for_level_base=268435456
bloom_locality=0
write_buffer_size=67108864
compression_per_level=
memtable_huge_page_size=0
max_successive_merges=0
arena_block_size=8388608
memtable_whole_key_filtering=false
target_file_size_multiplier=1
max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
num_levels=7
min_write_buffer_number_to_merge=1
max_write_buffer_number_to_maintain=0
max_write_buffer_number=2
compression=kSnappyCompression
level0_stop_writes_trigger=36
level0_slowdown_writes_trigger=20
compaction_filter=nullptr
level0_file_num_compaction_trigger=4
max_compaction_bytes=1677721600
compaction_options_universal={stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;allow_trivial_move=false;max_merge_width=4294967295;max_size_amplification_percent=200;min_merge_width=2;size_ratio=1;}
memtable_prefix_bloom_size_ratio=0.000000
max_write_buffer_size_to_maintain=0
hard_pending_compaction_bytes_limit=274877906944
ttl=2592000
table_factory=BlockBasedTable
soft_pending_compaction_bytes_limit=68719476736
prefix_extractor=nullptr
bottommost_compression=kDisableCompressionOption
force_consistency_checks=false
paranoid_file_checks=false
compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;}
max_bytes_for_level_multiplier=10.000000
optimize_filters_for_hits=false
level_compaction_dynamic_level_bytes=false
inplace_update_num_locks=10000
inplace_update_support=false
periodic_compaction_seconds=0
disable_auto_compactions=false
report_bg_io_stats=false
[TableOptions/BlockBasedTable "posg"]
pin_top_level_index_and_filter=true
enable_index_compression=true
read_amp_bytes_per_bit=8589934592
format_version=2
block_align=false
metadata_block_size=4096
block_size_deviation=10
partition_filters=false
block_size=4096
index_block_restart_interval=1
no_block_cache=false
checksum=kCRC32c
whole_key_filtering=true
index_shortening=kShortenSeparators
data_block_index_type=kDataBlockBinarySearch
index_type=kBinarySearch
verify_compression=false
filter_policy=nullptr
data_block_hash_table_util_ratio=0.750000
pin_l0_filter_and_index_blocks_in_cache=false
block_restart_interval=16
cache_index_and_filter_blocks_with_high_priority=true
cache_index_and_filter_blocks=false
hash_index_allow_collision=true
flush_block_policy_factory=FlushBlockBySizePolicyFactory
[CFOptions "ospg"]
sample_for_compression=0
compaction_pri=kMinOverlappingRatio
merge_operator=nullptr
compaction_filter_factory=nullptr
memtable_factory=SkipListFactory
memtable_insert_with_hint_prefix_extractor=nullptr
comparator=leveldb.BytewiseComparator
target_file_size_base=67108864
max_sequential_skip_in_iterations=8
compaction_style=kCompactionStyleLevel
max_bytes_for_level_base=268435456
bloom_locality=0
write_buffer_size=67108864
compression_per_level=
memtable_huge_page_size=0
max_successive_merges=0
arena_block_size=8388608
memtable_whole_key_filtering=false
target_file_size_multiplier=1
max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
num_levels=7
min_write_buffer_number_to_merge=1
max_write_buffer_number_to_maintain=0
max_write_buffer_number=2
compression=kSnappyCompression
level0_stop_writes_trigger=36
level0_slowdown_writes_trigger=20
compaction_filter=nullptr
level0_file_num_compaction_trigger=4
max_compaction_bytes=1677721600
compaction_options_universal={stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;allow_trivial_move=false;max_merge_width=4294967295;max_size_amplification_percent=200;min_merge_width=2;size_ratio=1;}
memtable_prefix_bloom_size_ratio=0.000000
max_write_buffer_size_to_maintain=0
hard_pending_compaction_bytes_limit=274877906944
ttl=2592000
table_factory=BlockBasedTable
soft_pending_compaction_bytes_limit=68719476736
prefix_extractor=nullptr
bottommost_compression=kDisableCompressionOption
force_consistency_checks=false
paranoid_file_checks=false
compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;}
max_bytes_for_level_multiplier=10.000000
optimize_filters_for_hits=false
level_compaction_dynamic_level_bytes=false
inplace_update_num_locks=10000
inplace_update_support=false
periodic_compaction_seconds=0
disable_auto_compactions=false
report_bg_io_stats=false
[TableOptions/BlockBasedTable "ospg"]
pin_top_level_index_and_filter=true
enable_index_compression=true
read_amp_bytes_per_bit=8589934592
format_version=2
block_align=false
metadata_block_size=4096
block_size_deviation=10
partition_filters=false
block_size=4096
index_block_restart_interval=1
no_block_cache=false
checksum=kCRC32c
whole_key_filtering=true
index_shortening=kShortenSeparators
data_block_index_type=kDataBlockBinarySearch
index_type=kBinarySearch
verify_compression=false
filter_policy=nullptr
data_block_hash_table_util_ratio=0.750000
pin_l0_filter_and_index_blocks_in_cache=false
block_restart_interval=16
cache_index_and_filter_blocks_with_high_priority=true
cache_index_and_filter_blocks=false
hash_index_allow_collision=true
flush_block_policy_factory=FlushBlockBySizePolicyFactory
[CFOptions "gspo"]
sample_for_compression=0
compaction_pri=kMinOverlappingRatio
merge_operator=nullptr
compaction_filter_factory=nullptr
memtable_factory=SkipListFactory
memtable_insert_with_hint_prefix_extractor=nullptr
comparator=leveldb.BytewiseComparator
target_file_size_base=67108864
max_sequential_skip_in_iterations=8
compaction_style=kCompactionStyleLevel
max_bytes_for_level_base=268435456
bloom_locality=0
write_buffer_size=67108864
compression_per_level=
memtable_huge_page_size=0
max_successive_merges=0
arena_block_size=8388608
memtable_whole_key_filtering=false
target_file_size_multiplier=1
max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
num_levels=7
min_write_buffer_number_to_merge=1
max_write_buffer_number_to_maintain=0
max_write_buffer_number=2
compression=kSnappyCompression
level0_stop_writes_trigger=36
level0_slowdown_writes_trigger=20
compaction_filter=nullptr
level0_file_num_compaction_trigger=4
max_compaction_bytes=1677721600
compaction_options_universal={stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;allow_trivial_move=false;max_merge_width=4294967295;max_size_amplification_percent=200;min_merge_width=2;size_ratio=1;}
memtable_prefix_bloom_size_ratio=0.000000
max_write_buffer_size_to_maintain=0
hard_pending_compaction_bytes_limit=274877906944
ttl=2592000
table_factory=BlockBasedTable
soft_pending_compaction_bytes_limit=68719476736
prefix_extractor=nullptr
bottommost_compression=kDisableCompressionOption
force_consistency_checks=false
paranoid_file_checks=false
compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;}
max_bytes_for_level_multiplier=10.000000
optimize_filters_for_hits=false
level_compaction_dynamic_level_bytes=false
inplace_update_num_locks=10000
inplace_update_support=false
periodic_compaction_seconds=0
disable_auto_compactions=false
report_bg_io_stats=false
[TableOptions/BlockBasedTable "gspo"]
pin_top_level_index_and_filter=true
enable_index_compression=true
read_amp_bytes_per_bit=8589934592
format_version=2
block_align=false
metadata_block_size=4096
block_size_deviation=10
partition_filters=false
block_size=4096
index_block_restart_interval=1
no_block_cache=false
checksum=kCRC32c
whole_key_filtering=true
index_shortening=kShortenSeparators
data_block_index_type=kDataBlockBinarySearch
index_type=kBinarySearch
verify_compression=false
filter_policy=nullptr
data_block_hash_table_util_ratio=0.750000
pin_l0_filter_and_index_blocks_in_cache=false
block_restart_interval=16
cache_index_and_filter_blocks_with_high_priority=true
cache_index_and_filter_blocks=false
hash_index_allow_collision=true
flush_block_policy_factory=FlushBlockBySizePolicyFactory
[CFOptions "gpos"]
sample_for_compression=0
compaction_pri=kMinOverlappingRatio
merge_operator=nullptr
compaction_filter_factory=nullptr
memtable_factory=SkipListFactory
memtable_insert_with_hint_prefix_extractor=nullptr
comparator=leveldb.BytewiseComparator
target_file_size_base=67108864
max_sequential_skip_in_iterations=8
compaction_style=kCompactionStyleLevel
max_bytes_for_level_base=268435456
bloom_locality=0
write_buffer_size=67108864
compression_per_level=
memtable_huge_page_size=0
max_successive_merges=0
arena_block_size=8388608
memtable_whole_key_filtering=false
target_file_size_multiplier=1
max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
num_levels=7
min_write_buffer_number_to_merge=1
max_write_buffer_number_to_maintain=0
max_write_buffer_number=2
compression=kSnappyCompression
level0_stop_writes_trigger=36
level0_slowdown_writes_trigger=20
compaction_filter=nullptr
level0_file_num_compaction_trigger=4
max_compaction_bytes=1677721600
compaction_options_universal={stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;allow_trivial_move=false;max_merge_width=4294967295;max_size_amplification_percent=200;min_merge_width=2;size_ratio=1;}
memtable_prefix_bloom_size_ratio=0.000000
max_write_buffer_size_to_maintain=0
hard_pending_compaction_bytes_limit=274877906944
ttl=2592000
table_factory=BlockBasedTable
soft_pending_compaction_bytes_limit=68719476736
prefix_extractor=nullptr
bottommost_compression=kDisableCompressionOption
force_consistency_checks=false
paranoid_file_checks=false
compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;}
max_bytes_for_level_multiplier=10.000000
optimize_filters_for_hits=false
level_compaction_dynamic_level_bytes=false
inplace_update_num_locks=10000
inplace_update_support=false
periodic_compaction_seconds=0
disable_auto_compactions=false
report_bg_io_stats=false
[TableOptions/BlockBasedTable "gpos"]
pin_top_level_index_and_filter=true
enable_index_compression=true
read_amp_bytes_per_bit=8589934592
format_version=2
block_align=false
metadata_block_size=4096
block_size_deviation=10
partition_filters=false
block_size=4096
index_block_restart_interval=1
no_block_cache=false
checksum=kCRC32c
whole_key_filtering=true
index_shortening=kShortenSeparators
data_block_index_type=kDataBlockBinarySearch
index_type=kBinarySearch
verify_compression=false
filter_policy=nullptr
data_block_hash_table_util_ratio=0.750000
pin_l0_filter_and_index_blocks_in_cache=false
block_restart_interval=16
cache_index_and_filter_blocks_with_high_priority=true
cache_index_and_filter_blocks=false
hash_index_allow_collision=true
flush_block_policy_factory=FlushBlockBySizePolicyFactory
[CFOptions "gosp"]
sample_for_compression=0
compaction_pri=kMinOverlappingRatio
merge_operator=nullptr
compaction_filter_factory=nullptr
memtable_factory=SkipListFactory
memtable_insert_with_hint_prefix_extractor=nullptr
comparator=leveldb.BytewiseComparator
target_file_size_base=67108864
max_sequential_skip_in_iterations=8
compaction_style=kCompactionStyleLevel
max_bytes_for_level_base=268435456
bloom_locality=0
write_buffer_size=67108864
compression_per_level=
memtable_huge_page_size=0
max_successive_merges=0
arena_block_size=8388608
memtable_whole_key_filtering=false
target_file_size_multiplier=1
max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
num_levels=7
min_write_buffer_number_to_merge=1
max_write_buffer_number_to_maintain=0
max_write_buffer_number=2
compression=kSnappyCompression
level0_stop_writes_trigger=36
level0_slowdown_writes_trigger=20
compaction_filter=nullptr
level0_file_num_compaction_trigger=4
max_compaction_bytes=1677721600
compaction_options_universal={stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;allow_trivial_move=false;max_merge_width=4294967295;max_size_amplification_percent=200;min_merge_width=2;size_ratio=1;}
memtable_prefix_bloom_size_ratio=0.000000
max_write_buffer_size_to_maintain=0
hard_pending_compaction_bytes_limit=274877906944
ttl=2592000
table_factory=BlockBasedTable
soft_pending_compaction_bytes_limit=68719476736
prefix_extractor=nullptr
bottommost_compression=kDisableCompressionOption
force_consistency_checks=false
paranoid_file_checks=false
compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;}
max_bytes_for_level_multiplier=10.000000
optimize_filters_for_hits=false
level_compaction_dynamic_level_bytes=false
inplace_update_num_locks=10000
inplace_update_support=false
periodic_compaction_seconds=0
disable_auto_compactions=false
report_bg_io_stats=false
[TableOptions/BlockBasedTable "gosp"]
pin_top_level_index_and_filter=true
enable_index_compression=true
read_amp_bytes_per_bit=8589934592
format_version=2
block_align=false
metadata_block_size=4096
block_size_deviation=10
partition_filters=false
block_size=4096
index_block_restart_interval=1
no_block_cache=false
checksum=kCRC32c
whole_key_filtering=true
index_shortening=kShortenSeparators
data_block_index_type=kDataBlockBinarySearch
index_type=kBinarySearch
verify_compression=false
filter_policy=nullptr
data_block_hash_table_util_ratio=0.750000
pin_l0_filter_and_index_blocks_in_cache=false
block_restart_interval=16
cache_index_and_filter_blocks_with_high_priority=true
cache_index_and_filter_blocks=false
hash_index_allow_collision=true
flush_block_policy_factory=FlushBlockBySizePolicyFactory
[CFOptions "dspo"]
sample_for_compression=0
compaction_pri=kMinOverlappingRatio
merge_operator=nullptr
compaction_filter_factory=nullptr
memtable_factory=SkipListFactory
memtable_insert_with_hint_prefix_extractor=nullptr
comparator=leveldb.BytewiseComparator
target_file_size_base=67108864
max_sequential_skip_in_iterations=8
compaction_style=kCompactionStyleLevel
max_bytes_for_level_base=268435456
bloom_locality=0
write_buffer_size=67108864
compression_per_level=
memtable_huge_page_size=0
max_successive_merges=0
arena_block_size=8388608
memtable_whole_key_filtering=false
target_file_size_multiplier=1
max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
num_levels=7
min_write_buffer_number_to_merge=1
max_write_buffer_number_to_maintain=0
max_write_buffer_number=2
compression=kSnappyCompression
level0_stop_writes_trigger=36
level0_slowdown_writes_trigger=20
compaction_filter=nullptr
level0_file_num_compaction_trigger=4
max_compaction_bytes=1677721600
compaction_options_universal={stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;allow_trivial_move=false;max_merge_width=4294967295;max_size_amplification_percent=200;min_merge_width=2;size_ratio=1;}
memtable_prefix_bloom_size_ratio=0.000000
max_write_buffer_size_to_maintain=0
hard_pending_compaction_bytes_limit=274877906944
ttl=2592000
table_factory=BlockBasedTable
soft_pending_compaction_bytes_limit=68719476736
prefix_extractor=nullptr
bottommost_compression=kDisableCompressionOption
force_consistency_checks=false
paranoid_file_checks=false
compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;}
max_bytes_for_level_multiplier=10.000000
optimize_filters_for_hits=false
level_compaction_dynamic_level_bytes=false
inplace_update_num_locks=10000
inplace_update_support=false
periodic_compaction_seconds=0
disable_auto_compactions=false
report_bg_io_stats=false
[TableOptions/BlockBasedTable "dspo"]
pin_top_level_index_and_filter=true
enable_index_compression=true
read_amp_bytes_per_bit=8589934592
format_version=2
block_align=false
metadata_block_size=4096
block_size_deviation=10
partition_filters=false
block_size=4096
index_block_restart_interval=1
no_block_cache=false
checksum=kCRC32c
whole_key_filtering=true
index_shortening=kShortenSeparators
data_block_index_type=kDataBlockBinarySearch
index_type=kBinarySearch
verify_compression=false
filter_policy=nullptr
data_block_hash_table_util_ratio=0.750000
pin_l0_filter_and_index_blocks_in_cache=false
block_restart_interval=16
cache_index_and_filter_blocks_with_high_priority=true
cache_index_and_filter_blocks=false
hash_index_allow_collision=true
flush_block_policy_factory=FlushBlockBySizePolicyFactory
[CFOptions "dpos"]
sample_for_compression=0
compaction_pri=kMinOverlappingRatio
merge_operator=nullptr
compaction_filter_factory=nullptr
memtable_factory=SkipListFactory
memtable_insert_with_hint_prefix_extractor=nullptr
comparator=leveldb.BytewiseComparator
target_file_size_base=67108864
max_sequential_skip_in_iterations=8
compaction_style=kCompactionStyleLevel
max_bytes_for_level_base=268435456
bloom_locality=0
write_buffer_size=67108864
compression_per_level=
memtable_huge_page_size=0
max_successive_merges=0
arena_block_size=8388608
memtable_whole_key_filtering=false
target_file_size_multiplier=1
max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
num_levels=7
min_write_buffer_number_to_merge=1
max_write_buffer_number_to_maintain=0
max_write_buffer_number=2
compression=kSnappyCompression
level0_stop_writes_trigger=36
level0_slowdown_writes_trigger=20
compaction_filter=nullptr
level0_file_num_compaction_trigger=4
max_compaction_bytes=1677721600
compaction_options_universal={stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;allow_trivial_move=false;max_merge_width=4294967295;max_size_amplification_percent=200;min_merge_width=2;size_ratio=1;}
memtable_prefix_bloom_size_ratio=0.000000
max_write_buffer_size_to_maintain=0
hard_pending_compaction_bytes_limit=274877906944
ttl=2592000
table_factory=BlockBasedTable
soft_pending_compaction_bytes_limit=68719476736
prefix_extractor=nullptr
bottommost_compression=kDisableCompressionOption
force_consistency_checks=false
paranoid_file_checks=false
compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;}
max_bytes_for_level_multiplier=10.000000
optimize_filters_for_hits=false
level_compaction_dynamic_level_bytes=false
inplace_update_num_locks=10000
inplace_update_support=false
periodic_compaction_seconds=0
disable_auto_compactions=false
report_bg_io_stats=false
[TableOptions/BlockBasedTable "dpos"]
pin_top_level_index_and_filter=true
enable_index_compression=true
read_amp_bytes_per_bit=8589934592
format_version=2
block_align=false
metadata_block_size=4096
block_size_deviation=10
partition_filters=false
block_size=4096
index_block_restart_interval=1
no_block_cache=false
checksum=kCRC32c
whole_key_filtering=true
index_shortening=kShortenSeparators
data_block_index_type=kDataBlockBinarySearch
index_type=kBinarySearch
verify_compression=false
filter_policy=nullptr
data_block_hash_table_util_ratio=0.750000
pin_l0_filter_and_index_blocks_in_cache=false
block_restart_interval=16
cache_index_and_filter_blocks_with_high_priority=true
cache_index_and_filter_blocks=false
hash_index_allow_collision=true
flush_block_policy_factory=FlushBlockBySizePolicyFactory
[CFOptions "dosp"]
sample_for_compression=0
compaction_pri=kMinOverlappingRatio
merge_operator=nullptr
compaction_filter_factory=nullptr
memtable_factory=SkipListFactory
memtable_insert_with_hint_prefix_extractor=nullptr
comparator=leveldb.BytewiseComparator
target_file_size_base=67108864
max_sequential_skip_in_iterations=8
compaction_style=kCompactionStyleLevel
max_bytes_for_level_base=268435456
bloom_locality=0
write_buffer_size=67108864
compression_per_level=
memtable_huge_page_size=0
max_successive_merges=0
arena_block_size=8388608
memtable_whole_key_filtering=false
target_file_size_multiplier=1
max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
num_levels=7
min_write_buffer_number_to_merge=1
max_write_buffer_number_to_maintain=0
max_write_buffer_number=2
compression=kSnappyCompression
level0_stop_writes_trigger=36
level0_slowdown_writes_trigger=20
compaction_filter=nullptr
level0_file_num_compaction_trigger=4
max_compaction_bytes=1677721600
compaction_options_universal={stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;allow_trivial_move=false;max_merge_width=4294967295;max_size_amplification_percent=200;min_merge_width=2;size_ratio=1;}
memtable_prefix_bloom_size_ratio=0.000000
max_write_buffer_size_to_maintain=0
hard_pending_compaction_bytes_limit=274877906944
ttl=2592000
table_factory=BlockBasedTable
soft_pending_compaction_bytes_limit=68719476736
prefix_extractor=nullptr
bottommost_compression=kDisableCompressionOption
force_consistency_checks=false
paranoid_file_checks=false
compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;}
max_bytes_for_level_multiplier=10.000000
optimize_filters_for_hits=false
level_compaction_dynamic_level_bytes=false
inplace_update_num_locks=10000
inplace_update_support=false
periodic_compaction_seconds=0
disable_auto_compactions=false
report_bg_io_stats=false
[TableOptions/BlockBasedTable "dosp"]
pin_top_level_index_and_filter=true
enable_index_compression=true
read_amp_bytes_per_bit=8589934592
format_version=2
block_align=false
metadata_block_size=4096
block_size_deviation=10
partition_filters=false
block_size=4096
index_block_restart_interval=1
no_block_cache=false
checksum=kCRC32c
whole_key_filtering=true
index_shortening=kShortenSeparators
data_block_index_type=kDataBlockBinarySearch
index_type=kBinarySearch
verify_compression=false
filter_policy=nullptr
data_block_hash_table_util_ratio=0.750000
pin_l0_filter_and_index_blocks_in_cache=false
block_restart_interval=16
cache_index_and_filter_blocks_with_high_priority=true
cache_index_and_filter_blocks=false
hash_index_allow_collision=true
flush_block_policy_factory=FlushBlockBySizePolicyFactory

@ -0,0 +1,83 @@
use oxigraph::model::vocab::{rdf, xsd};
use oxigraph::model::*;
use oxigraph::RocksDbStore;
use std::io;
use std::process::Command;
fn quads(graph_name: GraphNameRef<'static>) -> Vec<QuadRef<'static>> {
let paris = NamedNodeRef::new_unchecked("http://www.wikidata.org/entity/Q90");
let france = NamedNodeRef::new_unchecked("http://www.wikidata.org/entity/Q142");
let city = NamedNodeRef::new_unchecked("http://schema.org/City");
let name = NamedNodeRef::new_unchecked("http://schema.org/name");
let country = NamedNodeRef::new_unchecked("http://schema.org/country");
let population = NamedNodeRef::new_unchecked("http://schema.org/population");
let start_date = NamedNodeRef::new_unchecked("http://schema.org/startDate");
let url = NamedNodeRef::new_unchecked("http://schema.org/url");
let postal_code = NamedNodeRef::new_unchecked("http://schema.org/postalCode");
vec![
QuadRef::new(paris, rdf::TYPE, city, graph_name),
QuadRef::new(
paris,
name,
LiteralRef::new_language_tagged_literal_unchecked("Paris", "fr"),
graph_name,
),
QuadRef::new(
paris,
name,
LiteralRef::new_language_tagged_literal_unchecked("la ville lumière", "fr"),
graph_name,
),
QuadRef::new(paris, country, france, graph_name),
QuadRef::new(
paris,
population,
LiteralRef::new_typed_literal("2000000", xsd::INTEGER),
graph_name,
),
QuadRef::new(
paris,
start_date,
LiteralRef::new_typed_literal("-300", xsd::G_YEAR),
graph_name,
),
QuadRef::new(
paris,
url,
LiteralRef::new_typed_literal("https://www.paris.fr/", xsd::ANY_URI),
graph_name,
),
QuadRef::new(
paris,
postal_code,
LiteralRef::new_simple_literal("75001"),
graph_name,
),
]
}
#[test]
fn test_backward_compatibility() -> io::Result<()> {
{
let store = RocksDbStore::open("tests/rockdb_bc_data")?;
for q in quads(GraphNameRef::DefaultGraph) {
assert!(store.contains(q)?);
}
for q in quads(
NamedNodeRef::new_unchecked("http://www.wikidata.org/wiki/Special:EntityData/Q90")
.into(),
) {
assert!(store.contains(q)?);
}
};
reset_dir("tests/rockdb_bc_data")?;
Ok(())
}
fn reset_dir(dir: &str) -> io::Result<()> {
assert!(Command::new("git")
.args(&["checkout", "HEAD", "--", dir])
.status()?
.success());
Ok(())
}

@ -0,0 +1,4 @@
segment_size: 524288
use_compression: false
version: 0.34
v

Binary file not shown.

@ -0,0 +1,179 @@
use oxigraph::io::{DatasetFormat, GraphFormat};
use oxigraph::model::vocab::{rdf, xsd};
use oxigraph::model::*;
use oxigraph::store::sled::SledConflictableTransactionError;
use oxigraph::SledStore;
use std::io;
use std::io::Cursor;
use std::process::Command;
const DATA: &str = r#"
@prefix schema: <http://schema.org/> .
@prefix wd: <http://www.wikidata.org/entity/> .
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
wd:Q90 a schema:City ;
schema:name "Paris"@fr , "la ville lumière"@fr ;
schema:country wd:Q142 ;
schema:population 2000000 ;
schema:startDate "-300"^^xsd:gYear ;
schema:url "https://www.paris.fr/"^^xsd:anyURI ;
schema:postalCode "75001" .
"#;
const NUMBER_OF_TRIPLES: usize = 8;
fn quads(graph_name: GraphNameRef<'static>) -> Vec<QuadRef<'static>> {
let paris = NamedNodeRef::new_unchecked("http://www.wikidata.org/entity/Q90");
let france = NamedNodeRef::new_unchecked("http://www.wikidata.org/entity/Q142");
let city = NamedNodeRef::new_unchecked("http://schema.org/City");
let name = NamedNodeRef::new_unchecked("http://schema.org/name");
let country = NamedNodeRef::new_unchecked("http://schema.org/country");
let population = NamedNodeRef::new_unchecked("http://schema.org/population");
let start_date = NamedNodeRef::new_unchecked("http://schema.org/startDate");
let url = NamedNodeRef::new_unchecked("http://schema.org/url");
let postal_code = NamedNodeRef::new_unchecked("http://schema.org/postalCode");
vec![
QuadRef::new(paris, rdf::TYPE, city, graph_name),
QuadRef::new(
paris,
name,
LiteralRef::new_language_tagged_literal_unchecked("Paris", "fr"),
graph_name,
),
QuadRef::new(
paris,
name,
LiteralRef::new_language_tagged_literal_unchecked("la ville lumière", "fr"),
graph_name,
),
QuadRef::new(paris, country, france, graph_name),
QuadRef::new(
paris,
population,
LiteralRef::new_typed_literal("2000000", xsd::INTEGER),
graph_name,
),
QuadRef::new(
paris,
start_date,
LiteralRef::new_typed_literal("-300", xsd::G_YEAR),
graph_name,
),
QuadRef::new(
paris,
url,
LiteralRef::new_typed_literal("https://www.paris.fr/", xsd::ANY_URI),
graph_name,
),
QuadRef::new(
paris,
postal_code,
LiteralRef::new_simple_literal("75001"),
graph_name,
),
]
}
#[test]
fn test_load_graph() -> io::Result<()> {
let store = SledStore::new()?;
store.load_graph(Cursor::new(DATA), GraphFormat::Turtle, None, None)?;
for q in quads(GraphNameRef::DefaultGraph) {
assert!(store.contains(q)?);
}
Ok(())
}
#[test]
fn test_load_dataset() -> io::Result<()> {
let store = SledStore::new()?;
store.load_dataset(Cursor::new(DATA), DatasetFormat::TriG, None)?;
for q in quads(GraphNameRef::DefaultGraph) {
assert!(store.contains(q)?);
}
Ok(())
}
#[test]
fn test_dump_graph() -> io::Result<()> {
let store = SledStore::new()?;
for q in quads(GraphNameRef::DefaultGraph) {
store.insert(q)?;
}
let mut buffer = Vec::new();
store.dump_graph(&mut buffer, GraphFormat::NTriples, None)?;
assert_eq!(
buffer.into_iter().filter(|c| *c == b'\n').count(),
NUMBER_OF_TRIPLES
);
Ok(())
}
#[test]
fn test_dump_dataset() -> io::Result<()> {
let store = SledStore::new()?;
for q in quads(GraphNameRef::DefaultGraph) {
store.insert(q)?;
}
let mut buffer = Vec::new();
store.dump_dataset(&mut buffer, DatasetFormat::NQuads)?;
assert_eq!(
buffer.into_iter().filter(|c| *c == b'\n').count(),
NUMBER_OF_TRIPLES
);
Ok(())
}
#[test]
fn test_transaction_load_graph() -> io::Result<()> {
let store = SledStore::new()?;
store.transaction(|t| {
t.load_graph(Cursor::new(DATA), GraphFormat::Turtle, None, None)?;
Ok(()) as Result<_, SledConflictableTransactionError<io::Error>>
})?;
for q in quads(GraphNameRef::DefaultGraph) {
assert!(store.contains(q)?);
}
Ok(())
}
#[test]
fn test_transaction_load_dataset() -> io::Result<()> {
let store = SledStore::new()?;
store.transaction(|t| {
t.load_dataset(Cursor::new(DATA), DatasetFormat::TriG, None)?;
Ok(()) as Result<_, SledConflictableTransactionError<io::Error>>
})?;
for q in quads(GraphNameRef::DefaultGraph) {
assert!(store.contains(q)?);
}
Ok(())
}
#[test]
fn test_backward_compatibility() -> io::Result<()> {
{
let store = SledStore::open("tests/sled_bc_data")?;
for q in quads(GraphNameRef::DefaultGraph) {
assert!(store.contains(q)?);
}
for q in quads(
NamedNodeRef::new_unchecked("http://www.wikidata.org/wiki/Special:EntityData/Q90")
.into(),
) {
assert!(store.contains(q)?);
}
};
reset_dir("tests/sled_bc_data")?;
Ok(())
}
fn reset_dir(dir: &str) -> io::Result<()> {
assert!(Command::new("git")
.args(&["checkout", "HEAD", "--", dir])
.status()?
.success());
Ok(())
}
Loading…
Cancel
Save