From 85bc98462e699d8fbc846ad142c89dc071964161 Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Wed, 20 Apr 2016 18:18:21 +0800 Subject: [PATCH 01/17] options: add compression option --- src/ffi.rs | 2 +- src/lib.rs | 2 +- src/rocksdb_options.rs | 8 +++++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/ffi.rs b/src/ffi.rs index f13472b..3182d0c 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -178,7 +178,7 @@ extern "C" { pub fn rocksdb_options_set_compaction_style(options: DBOptions, cs: DBCompactionStyle); pub fn rocksdb_options_set_compression(options: DBOptions, - compression_style_no: c_int); + compression_style_no: DBCompressionType); pub fn rocksdb_options_set_max_background_compactions( options: DBOptions, max_bg_compactions: c_int); pub fn rocksdb_options_set_max_background_flushes(options: DBOptions, diff --git a/src/lib.rs b/src/lib.rs index 8b774f4..dbadb71 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,7 +13,7 @@ // limitations under the License. // pub use ffi as rocksdb_ffi; -pub use ffi::{DBCompactionStyle, DBComparator, new_bloom_filter}; +pub use ffi::{DBCompactionStyle, DBComparator, DBCompressionType, new_bloom_filter}; pub use rocksdb::{DB, DBIterator, DBVector, Direction, IteratorMode, Writable, WriteBatch}; pub use rocksdb_options::{BlockBasedOptions, Options, WriteOptions}; diff --git a/src/rocksdb_options.rs b/src/rocksdb_options.rs index 8a20dbc..adbec5f 100644 --- a/src/rocksdb_options.rs +++ b/src/rocksdb_options.rs @@ -17,7 +17,7 @@ use self::libc::c_int; use std::ffi::CString; use std::mem; -use rocksdb_ffi; +use rocksdb_ffi::{self, DBCompressionType}; use merge_operator::{self, MergeFn, MergeOperatorCallback, full_merge_callback, partial_merge_callback}; use comparator::{self, ComparatorCallback, compare_callback}; @@ -102,6 +102,12 @@ impl Options { } } + pub fn compression(&mut self, t: DBCompressionType) { + unsafe { + rocksdb_ffi::rocksdb_options_set_compression(self.inner, t); + } + } + pub fn add_merge_operator(&mut self, name: &str, merge_fn: MergeFn) { From e2c087738872d5ece0d910e356a70c5b1ed801b7 Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Sun, 24 Apr 2016 18:32:30 +0800 Subject: [PATCH 02/17] option: add max_bytes_for_level_base option --- src/ffi.rs | 2 ++ src/rocksdb_options.rs | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/src/ffi.rs b/src/ffi.rs index 3182d0c..99ff01f 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -167,6 +167,8 @@ extern "C" { bytes: u64); pub fn rocksdb_options_set_target_file_size_multiplier(options: DBOptions, mul: c_int); + pub fn rocksdb_options_set_max_bytes_for_level_base(options: DBOptions, bytes: u64); + pub fn rocksdb_options_set_max_bytes_for_level_multiplier(options: DBOptions, mul: c_int); pub fn rocksdb_options_set_max_log_file_size(options: DBOptions, bytes: usize); pub fn rocksdb_options_set_max_manifest_file_size(options: DBOptions, diff --git a/src/rocksdb_options.rs b/src/rocksdb_options.rs index adbec5f..1a1dc06 100644 --- a/src/rocksdb_options.rs +++ b/src/rocksdb_options.rs @@ -214,6 +214,18 @@ impl Options { } } + pub fn set_max_bytes_for_level_base(&mut self, size: u64) { + unsafe { + rocksdb_ffi::rocksdb_options_set_max_bytes_for_level_base(self.inner, size); + } + } + + pub fn set_max_bytes_for_level_multiplier(&mut self, mul: i32) { + unsafe { + rocksdb_ffi::rocksdb_options_set_max_bytes_for_level_multiplier(self.inner, mul); + } + } + pub fn set_target_file_size_base(&mut self, size: u64) { unsafe { rocksdb_ffi::rocksdb_options_set_target_file_size_base(self.inner, From f92f15f111915b742c8502b31a45d1de6b0cc4ab Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Mon, 25 Apr 2016 03:18:39 +0800 Subject: [PATCH 03/17] add lru cache conf --- src/rocksdb_options.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/rocksdb_options.rs b/src/rocksdb_options.rs index 1a1dc06..f45a4e3 100644 --- a/src/rocksdb_options.rs +++ b/src/rocksdb_options.rs @@ -65,6 +65,15 @@ impl BlockBasedOptions { size); } } + + pub fn set_lru_cache(&mut self, size: size_t) { + let cache = rocksdb_ffi::new_cache(size); + unsafe { + // because cache is wrapped in shared_ptr, so we don't need to call + // rocksdb_cache_destroy explicitly. + rocksdb_ffi::rocksdb_block_based_options_set_block_cache(self.inner, cache); + } + } } impl Default for BlockBasedOptions { From f781adc84b81fe2e848420043e41af298008ffa7 Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Mon, 25 Apr 2016 03:36:19 +0800 Subject: [PATCH 04/17] add allow os buffer conf --- src/ffi.rs | 1 + src/rocksdb_options.rs | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/src/ffi.rs b/src/ffi.rs index 99ff01f..5045d2f 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -147,6 +147,7 @@ extern "C" { pub fn rocksdb_options_set_bytes_per_sync(options: DBOptions, bytes: u64); pub fn rocksdb_options_set_disable_data_sync(options: DBOptions, v: c_int); + pub fn rocksdb_options_set_allow_os_buffer(options: DBOptions, is_allow: bool); pub fn rocksdb_options_optimize_for_point_lookup(options: DBOptions, block_cache_size_mb: u64); pub fn rocksdb_options_set_table_cache_numshardbits(options: DBOptions, diff --git a/src/rocksdb_options.rs b/src/rocksdb_options.rs index f45a4e3..6b16265 100644 --- a/src/rocksdb_options.rs +++ b/src/rocksdb_options.rs @@ -195,6 +195,13 @@ impl Options { } } + pub fn allow_os_buffer(&mut self, is_allow: bool) { + unsafe { + rocksdb_ffi::rocksdb_options_set_allow_os_buffer(self.inner, + is_allow); + } + } + pub fn set_table_cache_num_shard_bits(&mut self, nbits: c_int) { unsafe { rocksdb_ffi::rocksdb_options_set_table_cache_numshardbits(self.inner, From 0dfdc06ddfa49fd5509257b72f23fd0cc58b1efe Mon Sep 17 00:00:00 2001 From: ngaut Date: Wed, 11 May 2016 22:59:51 +0800 Subject: [PATCH 05/17] *: add an option to disable write-ahead-log. --- src/rocksdb.rs | 6 ++++++ src/rocksdb_options.rs | 11 +++++++++++ 2 files changed, 17 insertions(+) diff --git a/src/rocksdb.rs b/src/rocksdb.rs index 93fcff7..62fd506 100644 --- a/src/rocksdb.rs +++ b/src/rocksdb.rs @@ -434,6 +434,12 @@ impl DB { self.write_opt(batch, &WriteOptions::default()) } + pub fn write_withou_wal(&self, batch: WriteBatch) -> Result<(), String> { + let mut wo = WriteOptions::new(); + wo.disable_wal(true); + self.write_opt(batch, &wo) + } + pub fn get_opt(&self, key: &[u8], readopts: &ReadOptions) diff --git a/src/rocksdb_options.rs b/src/rocksdb_options.rs index 6b16265..20f8a9c 100644 --- a/src/rocksdb_options.rs +++ b/src/rocksdb_options.rs @@ -334,11 +334,22 @@ impl WriteOptions { pub fn new() -> WriteOptions { WriteOptions::default() } + pub fn set_sync(&mut self, sync: bool) { unsafe { rocksdb_ffi::rocksdb_writeoptions_set_sync(self.inner, sync); } } + + pub fn disable_wal(&mut self, disable: bool) { + unsafe { + if disable { + rocksdb_ffi::rocksdb_writeoptions_disable_WAL(self.inner, 1); + } else { + rocksdb_ffi::rocksdb_writeoptions_disable_WAL(self.inner, 0); + } + } + } } impl Default for WriteOptions { From a7830930483ab2aa2ea2c32f92c4da93fb4fb02d Mon Sep 17 00:00:00 2001 From: jinpeng zhang Date: Wed, 8 Jun 2016 17:01:32 +0800 Subject: [PATCH 06/17] add report_gb_io_stats option --- src/ffi.rs | 1 + src/rocksdb_options.rs | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/src/ffi.rs b/src/ffi.rs index 5045d2f..9b160ae 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -189,6 +189,7 @@ extern "C" { pub fn rocksdb_options_set_filter_deletes(options: DBOptions, v: bool); pub fn rocksdb_options_set_disable_auto_compactions(options: DBOptions, v: c_int); + pub fn rocksdb_options_set_report_bg_io_stats(options: DBOptions, v: c_int); pub fn rocksdb_filterpolicy_create_bloom(bits_per_key: c_int) -> DBFilterPolicy; pub fn rocksdb_open(options: DBOptions, diff --git a/src/rocksdb_options.rs b/src/rocksdb_options.rs index 20f8a9c..4e3dfb3 100644 --- a/src/rocksdb_options.rs +++ b/src/rocksdb_options.rs @@ -315,6 +315,16 @@ impl Options { rocksdb_ffi::rocksdb_options_set_block_based_table_factory(self.inner, factory.inner); } } + + pub fn set_report_bg_io_stats(&mut self, enable: bool) { + unsafe { + if enable { + rocksdb_ffi::rocksdb_options_set_report_bg_io_stats(self.inner, 1); + } else { + rocksdb_ffi::rocksdb_options_set_report_bg_io_stats(self.inner, 0); + } + } + } } impl Default for Options { From b9bcc0900ef06c0c587136288efd5cdac932dcd2 Mon Sep 17 00:00:00 2001 From: zhangjinpeng1987 Date: Sun, 12 Jun 2016 11:41:04 +0800 Subject: [PATCH 07/17] add compression_per_level options --- src/ffi.rs | 3 +++ src/rocksdb_options.rs | 10 +++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/ffi.rs b/src/ffi.rs index 9b160ae..1437c6c 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -182,6 +182,9 @@ extern "C" { cs: DBCompactionStyle); pub fn rocksdb_options_set_compression(options: DBOptions, compression_style_no: DBCompressionType); + pub fn rocksdb_options_set_compression_per_level(options: DBOptions, + level_values: *const DBCompressionType, + num_levels: size_t); pub fn rocksdb_options_set_max_background_compactions( options: DBOptions, max_bg_compactions: c_int); pub fn rocksdb_options_set_max_background_flushes(options: DBOptions, diff --git a/src/rocksdb_options.rs b/src/rocksdb_options.rs index 4e3dfb3..92dd2a1 100644 --- a/src/rocksdb_options.rs +++ b/src/rocksdb_options.rs @@ -13,7 +13,7 @@ // limitations under the License. // extern crate libc; -use self::libc::c_int; +use self::libc::{c_int,size_t}; use std::ffi::CString; use std::mem; @@ -117,6 +117,14 @@ impl Options { } } + pub fn compression_per_level(&mut self, level_types: &[DBCompressionType]) { + unsafe { + rocksdb_ffi::rocksdb_options_set_compression_per_level(self.inner, + level_types.as_ptr(), + level_types.len() as size_t) + } + } + pub fn add_merge_operator(&mut self, name: &str, merge_fn: MergeFn) { From ab9028c7785c0170b61a9b7e2fd1497256cee4ac Mon Sep 17 00:00:00 2001 From: zhangjinpeng1987 Date: Mon, 13 Jun 2016 18:06:47 +0800 Subject: [PATCH 08/17] DBComressionType copyable --- src/ffi.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ffi.rs b/src/ffi.rs index 1437c6c..367a211 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -70,6 +70,7 @@ pub fn new_cache(capacity: size_t) -> DBCache { unsafe { rocksdb_cache_create_lru(capacity) } } +#[derive(Copy, Clone)] #[repr(C)] pub enum DBCompressionType { None = 0, From c9e104a4e5e0c95425c585b5c67bdabd11502851 Mon Sep 17 00:00:00 2001 From: zhangjinpeng1987 Date: Mon, 11 Jul 2016 16:22:21 +0800 Subject: [PATCH 09/17] add filter policy interface && add set_cache_index_and_filter_blocks --- src/ffi.rs | 6 +++++- src/rocksdb_options.rs | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/ffi.rs b/src/ffi.rs index 367a211..3ffceff 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -13,7 +13,7 @@ // limitations under the License. // extern crate libc; -use self::libc::{c_char, c_int, c_void, size_t}; +use self::libc::{c_char, c_uchar, c_int, c_void, size_t}; use std::ffi::CStr; use std::str::from_utf8; @@ -122,6 +122,8 @@ extern "C" { pub fn rocksdb_block_based_options_set_block_restart_interval( block_options: DBBlockBasedTableOptions, block_restart_interval: c_int); + pub fn rocksdb_block_based_options_set_cache_index_and_filter_blocks( + block_options: DBBlockBasedTableOptions, v: c_uchar); pub fn rocksdb_block_based_options_set_filter_policy( block_options: DBBlockBasedTableOptions, filter_policy: DBFilterPolicy); @@ -194,6 +196,8 @@ extern "C" { pub fn rocksdb_options_set_disable_auto_compactions(options: DBOptions, v: c_int); pub fn rocksdb_options_set_report_bg_io_stats(options: DBOptions, v: c_int); + pub fn rocksdb_filterpolicy_create_bloom_full(bits_per_key: c_int) + -> DBFilterPolicy; pub fn rocksdb_filterpolicy_create_bloom(bits_per_key: c_int) -> DBFilterPolicy; pub fn rocksdb_open(options: DBOptions, diff --git a/src/rocksdb_options.rs b/src/rocksdb_options.rs index 92dd2a1..29a4931 100644 --- a/src/rocksdb_options.rs +++ b/src/rocksdb_options.rs @@ -74,6 +74,26 @@ impl BlockBasedOptions { rocksdb_ffi::rocksdb_block_based_options_set_block_cache(self.inner, cache); } } + + pub fn set_bloom_filter(&mut self, bits_per_key: c_int, block_based: bool) { + unsafe { + let bloom = if block_based { + rocksdb_ffi::rocksdb_filterpolicy_create_bloom(bits_per_key) + } else { + rocksdb_ffi::rocksdb_filterpolicy_create_bloom_full(bits_per_key) + }; + + rocksdb_ffi::rocksdb_block_based_options_set_filter_policy(self.inner, + bloom); + } + } + + pub fn set_cache_index_and_filter_blocks(&mut self, v: bool) { + unsafe { + rocksdb_ffi::rocksdb_block_based_options_set_cache_index_and_filter_blocks(self.inner, + v as u8); + } + } } impl Default for BlockBasedOptions { From 5454956385fd118b664dc6d840fc656194af60a2 Mon Sep 17 00:00:00 2001 From: zhangjinpeng1987 Date: Tue, 2 Aug 2016 18:34:48 +0800 Subject: [PATCH 10/17] add set_level0_compaction_trigger api --- src/rocksdb_options.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/rocksdb_options.rs b/src/rocksdb_options.rs index 29a4931..1fea826 100644 --- a/src/rocksdb_options.rs +++ b/src/rocksdb_options.rs @@ -284,6 +284,13 @@ impl Options { } } + pub fn set_level_zero_compaction_trigger(&mut self, n: c_int) { + unsafe { + rocksdb_ffi::rocksdb_options_set_level0_file_num_compaction_trigger( + self.inner, n); + } + } + pub fn set_level_zero_slowdown_writes_trigger(&mut self, n: c_int) { unsafe { rocksdb_ffi::rocksdb_options_set_level0_slowdown_writes_trigger( From c884ee38b7de4ab2756773c08e82a388538b2ee8 Mon Sep 17 00:00:00 2001 From: zhangjinpeng1987 Date: Tue, 2 Aug 2016 18:36:03 +0800 Subject: [PATCH 11/17] add set_level_zero_file_num_compaction_trigger api --- src/rocksdb_options.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rocksdb_options.rs b/src/rocksdb_options.rs index 1fea826..3eab0a2 100644 --- a/src/rocksdb_options.rs +++ b/src/rocksdb_options.rs @@ -284,7 +284,7 @@ impl Options { } } - pub fn set_level_zero_compaction_trigger(&mut self, n: c_int) { + pub fn set_level_zero_file_num_compaction_trigger(&mut self, n: c_int) { unsafe { rocksdb_ffi::rocksdb_options_set_level0_file_num_compaction_trigger( self.inner, n); From ad09a5fbe17e96d661b91b060c123ee7618228f6 Mon Sep 17 00:00:00 2001 From: Dylan Wen Date: Sun, 18 Sep 2016 11:43:37 +0800 Subject: [PATCH 12/17] add set_max_manifest_file_size for Options --- src/rocksdb_options.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/rocksdb_options.rs b/src/rocksdb_options.rs index 3eab0a2..78a8dcc 100644 --- a/src/rocksdb_options.rs +++ b/src/rocksdb_options.rs @@ -270,6 +270,12 @@ impl Options { } } + pub fn set_max_manifest_file_size(&mut self, size: usize) { + unsafe { + rocksdb_ffi::rocksdb_options_set_max_manifest_file_size(self.inner, size); + } + } + pub fn set_target_file_size_base(&mut self, size: u64) { unsafe { rocksdb_ffi::rocksdb_options_set_target_file_size_base(self.inner, @@ -406,3 +412,15 @@ impl Default for WriteOptions { WriteOptions { inner: write_opts } } } + +#[cfg(test)] +mod tests { + use super::Options; + + #[test] + fn test_set_max_manifest_file_size() { + let mut opts = Options::default(); + let size = 20 * 1024 * 1024; + opts.set_max_manifest_file_size(size) + } +} From f542987ba715a0849e44827ebf76f39e7cd629e7 Mon Sep 17 00:00:00 2001 From: zhangjinpeng1987 Date: Wed, 14 Sep 2016 13:22:49 +0800 Subject: [PATCH 13/17] wal recovery mode --- src/ffi.rs | 10 ++++++++++ src/lib.rs | 2 +- src/rocksdb_options.rs | 8 +++++++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/ffi.rs b/src/ffi.rs index 3ffceff..23f641d 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -94,6 +94,15 @@ pub enum DBUniversalCompactionStyle { rocksdb_total_size_compaction_stop_style = 1, } +#[derive(Copy, Clone, PartialEq)] +#[repr(C)] +pub enum DBRecoveryMode { + TolerateCorruptedTailRecords = 0, + AbsoluteConsistency = 1, + PointInTime = 2, + SkipAnyCorruptedRecords = 3, +} + pub fn error_message(ptr: *const i8) -> String { let c_str = unsafe { CStr::from_ptr(ptr as *const _) }; let s = from_utf8(c_str.to_bytes()).unwrap().to_owned(); @@ -196,6 +205,7 @@ extern "C" { pub fn rocksdb_options_set_disable_auto_compactions(options: DBOptions, v: c_int); pub fn rocksdb_options_set_report_bg_io_stats(options: DBOptions, v: c_int); + pub fn rocksdb_options_set_wal_recovery_mode(options: DBOptions, mode: DBRecoveryMode); pub fn rocksdb_filterpolicy_create_bloom_full(bits_per_key: c_int) -> DBFilterPolicy; pub fn rocksdb_filterpolicy_create_bloom(bits_per_key: c_int) diff --git a/src/lib.rs b/src/lib.rs index dbadb71..3b9d1ad 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,7 +13,7 @@ // limitations under the License. // pub use ffi as rocksdb_ffi; -pub use ffi::{DBCompactionStyle, DBComparator, DBCompressionType, new_bloom_filter}; +pub use ffi::{DBCompactionStyle, DBComparator, DBCompressionType, DBRecoveryMode, new_bloom_filter}; pub use rocksdb::{DB, DBIterator, DBVector, Direction, IteratorMode, Writable, WriteBatch}; pub use rocksdb_options::{BlockBasedOptions, Options, WriteOptions}; diff --git a/src/rocksdb_options.rs b/src/rocksdb_options.rs index 78a8dcc..c6112cb 100644 --- a/src/rocksdb_options.rs +++ b/src/rocksdb_options.rs @@ -17,7 +17,7 @@ use self::libc::{c_int,size_t}; use std::ffi::CString; use std::mem; -use rocksdb_ffi::{self, DBCompressionType}; +use rocksdb_ffi::{self, DBCompressionType, DBRecoveryMode}; use merge_operator::{self, MergeFn, MergeOperatorCallback, full_merge_callback, partial_merge_callback}; use comparator::{self, ComparatorCallback, compare_callback}; @@ -366,6 +366,12 @@ impl Options { } } } + + pub fn set_wal_recovery_mode(&mut self, mode: DBRecoveryMode) { + unsafe { + rocksdb_ffi::rocksdb_options_set_wal_recovery_mode(self.inner, mode); + } + } } impl Default for Options { From fa038809e15a39e396ddaf361952202f0646e742 Mon Sep 17 00:00:00 2001 From: zhangjinpeng1987 Date: Wed, 21 Sep 2016 14:08:02 +0800 Subject: [PATCH 14/17] enable statistics --- src/ffi.rs | 2 ++ src/rocksdb_options.rs | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/src/ffi.rs b/src/ffi.rs index 23f641d..19634fc 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -206,6 +206,8 @@ extern "C" { v: c_int); pub fn rocksdb_options_set_report_bg_io_stats(options: DBOptions, v: c_int); pub fn rocksdb_options_set_wal_recovery_mode(options: DBOptions, mode: DBRecoveryMode); + pub fn rocksdb_options_enable_statistics(options: DBOptions); + pub fn rocksdb_options_set_stats_dump_period_sec(options: DBOptions, v: usize); pub fn rocksdb_filterpolicy_create_bloom_full(bits_per_key: c_int) -> DBFilterPolicy; pub fn rocksdb_filterpolicy_create_bloom(bits_per_key: c_int) diff --git a/src/rocksdb_options.rs b/src/rocksdb_options.rs index c6112cb..9cdfc87 100644 --- a/src/rocksdb_options.rs +++ b/src/rocksdb_options.rs @@ -372,6 +372,19 @@ impl Options { rocksdb_ffi::rocksdb_options_set_wal_recovery_mode(self.inner, mode); } } + + pub fn enable_statistics(&mut self) { + unsafe { + rocksdb_ffi::rocksdb_options_enable_statistics(self.inner); + } + } + + pub fn set_stats_dump_period_sec(&mut self, period: usize) { + unsafe { + rocksdb_ffi::rocksdb_options_set_stats_dump_period_sec(self.inner, + period); + } + } } impl Default for Options { @@ -429,4 +442,11 @@ mod tests { let size = 20 * 1024 * 1024; opts.set_max_manifest_file_size(size) } + + #[test] + fn test_enable_statistics() { + let mut opts = Options::default(); + opts.enable_statistics(); + opts.set_stats_dump_period_sec(60); + } } From 42c5bb92dcb25c4224770e7ea3ca1ff6b9f800fb Mon Sep 17 00:00:00 2001 From: zhangjinpeng1987 Date: Fri, 30 Sep 2016 16:57:45 +0800 Subject: [PATCH 15/17] set num levels --- src/ffi.rs | 1 + src/rocksdb_options.rs | 6 ++++++ test/test.rs | 1 + test/test_rocksdb_options.rs | 12 ++++++++++++ 4 files changed, 20 insertions(+) create mode 100644 test/test_rocksdb_options.rs diff --git a/src/ffi.rs b/src/ffi.rs index 19634fc..15c7ac1 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -208,6 +208,7 @@ extern "C" { pub fn rocksdb_options_set_wal_recovery_mode(options: DBOptions, mode: DBRecoveryMode); pub fn rocksdb_options_enable_statistics(options: DBOptions); pub fn rocksdb_options_set_stats_dump_period_sec(options: DBOptions, v: usize); + pub fn rocksdb_options_set_num_levels(options: DBOptions, v: c_int); pub fn rocksdb_filterpolicy_create_bloom_full(bits_per_key: c_int) -> DBFilterPolicy; pub fn rocksdb_filterpolicy_create_bloom(bits_per_key: c_int) diff --git a/src/rocksdb_options.rs b/src/rocksdb_options.rs index 9cdfc87..568d5d1 100644 --- a/src/rocksdb_options.rs +++ b/src/rocksdb_options.rs @@ -385,6 +385,12 @@ impl Options { period); } } + + pub fn set_num_levels(&mut self, n: c_int) { + unsafe { + rocksdb_ffi::rocksdb_options_set_num_levels(self.inner, n); + } + } } impl Default for Options { diff --git a/test/test.rs b/test/test.rs index 20a1b69..06e09ad 100644 --- a/test/test.rs +++ b/test/test.rs @@ -3,3 +3,4 @@ extern crate rocksdb; mod test_iterator; mod test_multithreaded; mod test_column_family; +mod test_rocksdb_options; diff --git a/test/test_rocksdb_options.rs b/test/test_rocksdb_options.rs new file mode 100644 index 0000000..cf1accf --- /dev/null +++ b/test/test_rocksdb_options.rs @@ -0,0 +1,12 @@ +use rocksdb::{DB, Options}; + + +#[test] +fn test_set_num_levels() { + let path = "_rust_rocksdb_test_set_num_levels"; + let mut opts = Options::default(); + opts.create_if_missing(true); + opts.set_num_levels(2); + let db = DB::open(&opts, path).unwrap(); + drop(db); +} From edb0859029dce8abf3d53b4801d6925aa3798230 Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 20 Oct 2016 09:25:36 +0800 Subject: [PATCH 16/17] support get statistics --- src/ffi.rs | 1 + src/rocksdb_options.rs | 24 ++++++++++++++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/ffi.rs b/src/ffi.rs index 15c7ac1..a9f7e4e 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -207,6 +207,7 @@ extern "C" { pub fn rocksdb_options_set_report_bg_io_stats(options: DBOptions, v: c_int); pub fn rocksdb_options_set_wal_recovery_mode(options: DBOptions, mode: DBRecoveryMode); pub fn rocksdb_options_enable_statistics(options: DBOptions); + pub fn rocksdb_options_statistics_get_string(options: DBOptions) -> *const c_char; pub fn rocksdb_options_set_stats_dump_period_sec(options: DBOptions, v: usize); pub fn rocksdb_options_set_num_levels(options: DBOptions, v: c_int); pub fn rocksdb_filterpolicy_create_bloom_full(bits_per_key: c_int) diff --git a/src/rocksdb_options.rs b/src/rocksdb_options.rs index 568d5d1..5f0c67e 100644 --- a/src/rocksdb_options.rs +++ b/src/rocksdb_options.rs @@ -13,8 +13,8 @@ // limitations under the License. // extern crate libc; -use self::libc::{c_int,size_t}; -use std::ffi::CString; +use self::libc::{c_int, size_t, c_void}; +use std::ffi::{CStr, CString}; use std::mem; use rocksdb_ffi::{self, DBCompressionType, DBRecoveryMode}; @@ -379,6 +379,22 @@ impl Options { } } + pub fn get_statistics(&self) -> Option { + unsafe { + let value = rocksdb_ffi::rocksdb_options_statistics_get_string(self.inner); + + + if value.is_null() { + return None; + } + + // Must valid UTF-8 format. + let s = CStr::from_ptr(value).to_str().unwrap().to_owned(); + libc::free(value as *mut c_void); + Some(s) + } + } + pub fn set_stats_dump_period_sec(&mut self, period: usize) { unsafe { rocksdb_ffi::rocksdb_options_set_stats_dump_period_sec(self.inner, @@ -454,5 +470,9 @@ mod tests { let mut opts = Options::default(); opts.enable_statistics(); opts.set_stats_dump_period_sec(60); + assert!(opts.get_statistics().is_some()); + + let opts = Options::default(); + assert!(opts.get_statistics().is_none()); } } From c0e0331d0cb1f652c541d35cebd66de00df26992 Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Wed, 26 Oct 2016 13:53:30 +0100 Subject: [PATCH 17/17] Add docstrings from RocksDB comments All the docstrings were taken from https://github.com/facebook/rocksdb/blob/869ae5d7868d7d48dd55b7be02a23581101ed69a/include/rocksdb/options.h --- src/rocksdb_options.rs | 217 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 216 insertions(+), 1 deletion(-) diff --git a/src/rocksdb_options.rs b/src/rocksdb_options.rs index 5f0c67e..1120a7b 100644 --- a/src/rocksdb_options.rs +++ b/src/rocksdb_options.rs @@ -109,6 +109,11 @@ impl Default for BlockBasedOptions { } impl Options { + /// By default, RocksDB uses only one background thread for flush and + /// compaction. Calling this function will set it up such that total of + /// `total_threads` is used. Good value for `total_threads` is the number of + /// cores. You almost definitely want to call this function if your system is + /// bottlenecked by RocksDB. pub fn increase_parallelism(&mut self, parallelism: i32) { unsafe { rocksdb_ffi::rocksdb_options_increase_parallelism(self.inner, @@ -124,6 +129,9 @@ impl Options { } } + /// If true, the database will be created if it is missing. + /// + /// Default: false pub fn create_if_missing(&mut self, create_if_missing: bool) { unsafe { rocksdb_ffi::rocksdb_options_set_create_if_missing( @@ -131,12 +139,24 @@ impl Options { } } + /// Sets the compression algorithm that will be used for the bottommost level that + /// contain files. If level-compaction is used, this option will only affect + /// levels after base level. + /// + /// Default: DBCompressionType::None pub fn compression(&mut self, t: DBCompressionType) { unsafe { rocksdb_ffi::rocksdb_options_set_compression(self.inner, t); } } + /// Different levels can have different compression policies. There + /// are cases where most lower levels would like to use quick compression + /// algorithms while the higher levels (which have more data) use + /// compression algorithms that have better compression but could + /// be slower. This array, if non-empty, should have an entry for + /// each level of the database; these override the value specified in + /// the previous field 'compression'. pub fn compression_per_level(&mut self, level_types: &[DBCompressionType]) { unsafe { rocksdb_ffi::rocksdb_options_set_compression_per_level(self.inner, @@ -165,6 +185,12 @@ impl Options { } } + /// Sets the comparator used to define the order of keys in the table. + /// Default: a comparator that uses lexicographic byte-wise ordering + /// + /// The client must ensure that the comparator supplied here has the + //// same name and orders keys *exactly* the same as the comparator + /// provided to previous open calls on the same DB. pub fn add_comparator(&mut self, name: &str, compare_fn: fn(&[u8], &[u8]) -> i32) { @@ -183,7 +209,6 @@ impl Options { } } - pub fn set_block_cache_size_mb(&mut self, cache_size: u64) { unsafe { rocksdb_ffi::rocksdb_options_optimize_for_point_lookup(self.inner, @@ -191,12 +216,24 @@ impl Options { } } + /// Sets the number of open files that can be used by the DB. You may need to + /// increase this if your database has a large working set. Value -1 means + /// files opened are always kept open. You can estimate number of files based + /// on target_file_size_base and target_file_size_multiplier for level-based + /// compaction. For universal-style compaction, you can usually set it to -1. + /// + /// Default: -1 pub fn set_max_open_files(&mut self, nfiles: c_int) { unsafe { rocksdb_ffi::rocksdb_options_set_max_open_files(self.inner, nfiles); } } + /// If true, then every store to stable storage will issue a fsync. + /// If false, then every store to stable storage will issue a fdatasync. + /// This parameter should be set to true while storing data to + /// filesystem like ext3 that can lose files after a reboot. + /// Default: false pub fn set_use_fsync(&mut self, useit: bool) { unsafe { if useit { @@ -207,6 +244,19 @@ impl Options { } } + /// Allows OS to incrementally sync files to disk while they are being + /// written, asynchronously, in the background. This operation can be used + /// to smooth out write I/Os over time. Users shouldn't rely on it for + /// persistency guarantee. + /// Issue one request for every bytes_per_sync written. 0 turns it off. + /// + /// Default: 0 + /// + /// You may consider using rate_limiter to regulate write rate to device. + /// When rate limiter is enabled, it automatically enables bytes_per_sync + /// to 1MB. + /// + /// This option applies to table files pub fn set_bytes_per_sync(&mut self, nbytes: u64) { unsafe { rocksdb_ffi::rocksdb_options_set_bytes_per_sync(self.inner, nbytes); @@ -223,6 +273,24 @@ impl Options { } } + /// Hints to the OS that it should not buffer disk I/O. Enabling this + /// parameter may improve performance but increases pressure on the + /// system cache. + /// + /// The exact behavior of this parameter is platform dependent. + /// + /// On POSIX systems, after RocksDB reads data from disk it will + /// mark the pages as "unneeded". The operating system may - or may not + /// - evict these pages from memory, reducing pressure on the system + /// cache. If the disk block is requested again this can result in + /// additional disk I/O. + /// + /// On WINDOWS system, files will be opened in "unbuffered I/O" mode + /// which means that data read from the disk will not be cached or + /// bufferized. The hardware buffer of the devices may however still + /// be used. Memory mapped files are not impacted by this parameter. + /// + /// Default: true pub fn allow_os_buffer(&mut self, is_allow: bool) { unsafe { rocksdb_ffi::rocksdb_options_set_allow_os_buffer(self.inner, @@ -230,6 +298,7 @@ impl Options { } } + /// Sets the number of shards used for table cache. pub fn set_table_cache_num_shard_bits(&mut self, nbits: c_int) { unsafe { rocksdb_ffi::rocksdb_options_set_table_cache_numshardbits(self.inner, @@ -237,6 +306,15 @@ impl Options { } } + /// Sets the minimum number of write buffers that will be merged together + /// before writing to storage. If set to 1, then + /// all write buffers are flushed to L0 as individual files and this increases + /// read amplification because a get request has to check in all of these + /// files. Also, an in-memory merge may result in writing lesser + /// data to storage if there are duplicate records in each of these + /// individual write buffers. + /// + /// Default: 1 pub fn set_min_write_buffer_number(&mut self, nbuf: c_int) { unsafe { rocksdb_ffi::rocksdb_options_set_min_write_buffer_number_to_merge( @@ -244,6 +322,30 @@ impl Options { } } + /// Sets the total maximum number of write buffers to maintain in memory including + /// copies of buffers that have already been flushed. Unlike + /// max_write_buffer_number, this parameter does not affect flushing. + /// This controls the minimum amount of write history that will be available + /// in memory for conflict checking when Transactions are used. + /// + /// When using an OptimisticTransactionDB: + /// If this value is too low, some transactions may fail at commit time due + /// to not being able to determine whether there were any write conflicts. + /// + /// When using a TransactionDB: + /// If Transaction::SetSnapshot is used, TransactionDB will read either + /// in-memory write buffers or SST files to do write-conflict checking. + /// Increasing this value can reduce the number of reads to SST files + /// done for conflict detection. + /// + /// Setting this value to 0 will cause write buffers to be freed immediately + /// after they are flushed. + /// If this value is set to -1, 'max_write_buffer_number' will be used. + /// + /// Default: + /// If using a TransactionDB/OptimisticTransactionDB, the default value will + /// be set to the value of 'max_write_buffer_number' if it is not explicitly + /// set by the user. Otherwise, the default is 0. pub fn set_max_write_buffer_number(&mut self, nbuf: c_int) { unsafe { rocksdb_ffi::rocksdb_options_set_max_write_buffer_number(self.inner, @@ -251,6 +353,22 @@ impl Options { } } + /// Sets the amount of data to build up in memory (backed by an unsorted log + /// on disk) before converting to a sorted on-disk file. + /// + /// Larger values increase performance, especially during bulk loads. + /// Up to max_write_buffer_number write buffers may be held in memory + /// at the same time, + /// so you may wish to adjust this parameter to control memory usage. + /// Also, a larger write buffer will result in a longer recovery time + /// the next time the database is opened. + /// + /// Note that write_buffer_size is enforced per column family. + /// See db_write_buffer_size for sharing memory across column families. + /// + /// Default: 64MB + /// + /// Dynamically changeable through SetOptions() API pub fn set_write_buffer_size(&mut self, size: usize) { unsafe { rocksdb_ffi::rocksdb_options_set_write_buffer_size(self.inner, @@ -258,24 +376,52 @@ impl Options { } } + /// Control maximum total data size for a level. + /// max_bytes_for_level_base is the max total for level-1. + /// Maximum number of bytes for level L can be calculated as + /// (max_bytes_for_level_base) * (max_bytes_for_level_multiplier ^ (L-1)) + /// For example, if max_bytes_for_level_base is 200MB, and if + /// max_bytes_for_level_multiplier is 10, total data size for level-1 + /// will be 200MB, total file size for level-2 will be 2GB, + /// and total file size for level-3 will be 20GB. + /// + /// Default: 256MB. + /// + /// Dynamically changeable through SetOptions() API pub fn set_max_bytes_for_level_base(&mut self, size: u64) { unsafe { rocksdb_ffi::rocksdb_options_set_max_bytes_for_level_base(self.inner, size); } } + /// Default: 10 pub fn set_max_bytes_for_level_multiplier(&mut self, mul: i32) { unsafe { rocksdb_ffi::rocksdb_options_set_max_bytes_for_level_multiplier(self.inner, mul); } } + /// The manifest file is rolled over on reaching this limit. + /// The older manifest file be deleted. + /// The default value is MAX_INT so that roll-over does not take place. pub fn set_max_manifest_file_size(&mut self, size: usize) { unsafe { rocksdb_ffi::rocksdb_options_set_max_manifest_file_size(self.inner, size); } } + /// Sets the target file size for compaction. + /// target_file_size_base is per-file size for level-1. + /// Target file size for level L can be calculated by + /// target_file_size_base * (target_file_size_multiplier ^ (L-1)) + /// For example, if target_file_size_base is 2MB and + /// target_file_size_multiplier is 10, then each file on level-1 will + /// be 2MB, and each file on level 2 will be 20MB, + /// and each file on level-3 will be 200MB. + /// + /// Default: 64MB. + /// + /// Dynamically changeable through SetOptions() API pub fn set_target_file_size_base(&mut self, size: u64) { unsafe { rocksdb_ffi::rocksdb_options_set_target_file_size_base(self.inner, @@ -283,6 +429,15 @@ impl Options { } } + /// Sets the minimum number of write buffers that will be merged together + /// before writing to storage. If set to 1, then + /// all write buffers are flushed to L0 as individual files and this increases + /// read amplification because a get request has to check in all of these + /// files. Also, an in-memory merge may result in writing lesser + /// data to storage if there are duplicate records in each of these + /// individual write buffers. + /// + /// Default: 1 pub fn set_min_write_buffer_number_to_merge(&mut self, to_merge: c_int) { unsafe { rocksdb_ffi::rocksdb_options_set_min_write_buffer_number_to_merge( @@ -290,6 +445,12 @@ impl Options { } } + /// Sets the number of files to trigger level-0 compaction. A value <0 means that + /// level-0 compaction will not be triggered by number of files at all. + /// + /// Default: 4 + /// + /// Dynamically changeable through SetOptions() API pub fn set_level_zero_file_num_compaction_trigger(&mut self, n: c_int) { unsafe { rocksdb_ffi::rocksdb_options_set_level0_file_num_compaction_trigger( @@ -297,6 +458,11 @@ impl Options { } } + /// Sets the soft limit on number of level-0 files. We start slowing down writes at this + /// point. A value <0 means that no writing slow down will be triggered by + /// number of files in level-0. + /// + /// Dynamically changeable through SetOptions() API pub fn set_level_zero_slowdown_writes_trigger(&mut self, n: c_int) { unsafe { rocksdb_ffi::rocksdb_options_set_level0_slowdown_writes_trigger( @@ -304,6 +470,9 @@ impl Options { } } + /// Sets the maximum number of level-0 files. We stop writes at this point. + /// + /// Dynamically changeable through SetOptions() API pub fn set_level_zero_stop_writes_trigger(&mut self, n: c_int) { unsafe { rocksdb_ffi::rocksdb_options_set_level0_stop_writes_trigger( @@ -311,6 +480,9 @@ impl Options { } } + /// Sets the compaction style. + /// + /// Default: DBCompactionStyle::Level pub fn set_compaction_style(&mut self, style: rocksdb_ffi::DBCompactionStyle) { unsafe { @@ -319,6 +491,19 @@ impl Options { } } + + /// Sets the maximum number of concurrent background compaction jobs, submitted to + /// the default LOW priority thread pool. + /// We first try to schedule compactions based on + /// `base_background_compactions`. If the compaction cannot catch up , we + /// will increase number of compaction threads up to + /// `max_background_compactions`. + /// + /// If you're increasing this, also consider increasing number of threads in + /// LOW priority thread pool. For more information, see + /// Env::SetBackgroundThreads + /// + /// Default: 1 pub fn set_max_background_compactions(&mut self, n: c_int) { unsafe { rocksdb_ffi::rocksdb_options_set_max_background_compactions( @@ -326,6 +511,22 @@ impl Options { } } + /// Sets the maximum number of concurrent background memtable flush jobs, submitted to + /// the HIGH priority thread pool. + /// + /// By default, all background jobs (major compaction and memtable flush) go + /// to the LOW priority pool. If this option is set to a positive number, + /// memtable flush jobs will be submitted to the HIGH priority pool. + /// It is important when the same Env is shared by multiple db instances. + /// Without a separate pool, long running major compaction jobs could + /// potentially block memtable flush jobs of other db instances, leading to + /// unnecessary Put stalls. + /// + /// If you're increasing this, also consider increasing number of threads in + /// HIGH priority thread pool. For more information, see + /// Env::SetBackgroundThreads + /// + /// Default: 1 pub fn set_max_background_flushes(&mut self, n: c_int) { unsafe { rocksdb_ffi::rocksdb_options_set_max_background_flushes(self.inner, @@ -339,6 +540,10 @@ impl Options { } } + /// Disables automatic compactions. Manual compactions can still + /// be issued on this column family + /// + /// Dynamically changeable through SetOptions() API pub fn set_disable_auto_compactions(&mut self, disable: bool) { let c_bool = if disable { 1 @@ -357,6 +562,9 @@ impl Options { } } + /// Measure IO stats in compactions and flushes, if true. + /// + /// Default: false pub fn set_report_bg_io_stats(&mut self, enable: bool) { unsafe { if enable { @@ -367,6 +575,9 @@ impl Options { } } + /// Recovery mode to control the consistency while replaying WAL + /// + /// Default: DBRecoveryMode::PointInTime pub fn set_wal_recovery_mode(&mut self, mode: DBRecoveryMode) { unsafe { rocksdb_ffi::rocksdb_options_set_wal_recovery_mode(self.inner, mode); @@ -395,6 +606,9 @@ impl Options { } } + /// If not zero, dump rocksdb.stats to LOG every stats_dump_period_sec + /// + /// Default: 600 (10 min) pub fn set_stats_dump_period_sec(&mut self, period: usize) { unsafe { rocksdb_ffi::rocksdb_options_set_stats_dump_period_sec(self.inner, @@ -402,6 +616,7 @@ impl Options { } } + /// Sets the number of levels for this database pub fn set_num_levels(&mut self, n: c_int) { unsafe { rocksdb_ffi::rocksdb_options_set_num_levels(self.inner, n);