From 955336556df4103d3bd6f3694d42c5dd3cd1714f Mon Sep 17 00:00:00 2001 From: Lucjan Suski Date: Mon, 29 Apr 2019 11:51:58 +0200 Subject: [PATCH 1/8] Allow to set max_dict_bytes and zstd_max_training_bytes --- src/db_options.rs | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/db_options.rs b/src/db_options.rs index 890760e..3d159f5 100644 --- a/src/db_options.rs +++ b/src/db_options.rs @@ -269,6 +269,41 @@ impl Options { ) } } + + /// Maximum size of dictionaries used to prime the compression library. + /// Enabling dictionary can improve compression ratios when there are + /// repetitions across data blocks. + /// + /// The dictionary is created by sampling the SST file data. If + /// `zstd_max_train_bytes` is nonzero, the samples are passed through zstd's + /// dictionary generator. Otherwise, the random samples are used directly as + /// the dictionary. + /// + /// When compression dictionary is disabled, we compress and write each block + /// before buffering data for the next one. When compression dictionary is + /// enabled, we buffer all SST file data in-memory so we can sample it, as data + /// can only be compressed and written after the dictionary has been finalized. + /// So users of this feature may see increased memory usage. + /// + /// Default: `0` + pub fn set_max_dict_bytes(&mut self, max_dict_bytes: u32) { + unsafe { + ffi::rocksdb_options_set_max_dict_bytes(self.inner, max_dict_bytes as u32); + } + } + + /// Maximum size of training data passed to zstd's dictionary trainer. Using + /// zstd's dictionary trainer can achieve even better compression ratio + /// improvements than using `max_dict_bytes` alone. + /// + /// The training data will be used to generate a dictionary of max_dict_bytes. + /// + /// Default: `0` + pub fn set_zstd_max_train_bytes(&mut self, max_dict_bytes: u32) { + unsafe { + ffi::rocksdb_options_zstd_max_train_bytes(self.inner, max_dict_bytes as u32); + } + } /// If non-zero, we perform bigger reads when doing compaction. If you're /// running RocksDB on spinning disks, you should set this to at least 2MB. From e44633dde7d56fa1d6b33e3413b366bea8947650 Mon Sep 17 00:00:00 2001 From: Lucjan Suski Date: Mon, 29 Apr 2019 12:22:30 +0200 Subject: [PATCH 2/8] Enable set_compression_options --- src/db_options.rs | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/src/db_options.rs b/src/db_options.rs index 3d159f5..6cf3949 100644 --- a/src/db_options.rs +++ b/src/db_options.rs @@ -286,24 +286,14 @@ impl Options { /// So users of this feature may see increased memory usage. /// /// Default: `0` - pub fn set_max_dict_bytes(&mut self, max_dict_bytes: u32) { + pub fn set_compression_options(&mut self, w_bits: c_int, level: c_int, strategy: c_int, max_dict_bytes: c_int + + enabled: bool) { unsafe { - ffi::rocksdb_options_set_max_dict_bytes(self.inner, max_dict_bytes as u32); + ffi::rocksdb_options_set_compression_options(self.inner, w_bits, level, strategy, max_dict_bytes, enabled); } } - /// Maximum size of training data passed to zstd's dictionary trainer. Using - /// zstd's dictionary trainer can achieve even better compression ratio - /// improvements than using `max_dict_bytes` alone. - /// - /// The training data will be used to generate a dictionary of max_dict_bytes. - /// - /// Default: `0` - pub fn set_zstd_max_train_bytes(&mut self, max_dict_bytes: u32) { - unsafe { - ffi::rocksdb_options_zstd_max_train_bytes(self.inner, max_dict_bytes as u32); - } - } /// If non-zero, we perform bigger reads when doing compaction. If you're /// running RocksDB on spinning disks, you should set this to at least 2MB. From 845a332fd321346a07e630b7e735e056e1dcd582 Mon Sep 17 00:00:00 2001 From: Lucjan Suski Date: Mon, 29 Apr 2019 12:31:00 +0200 Subject: [PATCH 3/8] There is no enabled param on compression options --- src/db_options.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/db_options.rs b/src/db_options.rs index 6cf3949..c056e3c 100644 --- a/src/db_options.rs +++ b/src/db_options.rs @@ -286,11 +286,9 @@ impl Options { /// So users of this feature may see increased memory usage. /// /// Default: `0` - pub fn set_compression_options(&mut self, w_bits: c_int, level: c_int, strategy: c_int, max_dict_bytes: c_int - - enabled: bool) { + pub fn set_compression_options(&mut self, w_bits: c_int, level: c_int, strategy: c_int, max_dict_bytes: c_int) { unsafe { - ffi::rocksdb_options_set_compression_options(self.inner, w_bits, level, strategy, max_dict_bytes, enabled); + ffi::rocksdb_options_set_compression_options(self.inner, w_bits, level, strategy, max_dict_bytes); } } From 9c06fab60fb50b73c37d90a4ad900729f401b401 Mon Sep 17 00:00:00 2001 From: Lucjan Suski Date: Tue, 29 Oct 2019 15:20:35 +0100 Subject: [PATCH 4/8] Allow setting optimize_filter_for_hits --- src/db_options.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/db_options.rs b/src/db_options.rs index c056e3c..29d0131 100644 --- a/src/db_options.rs +++ b/src/db_options.rs @@ -414,6 +414,12 @@ impl Options { } } + pub fn set_optimize_filters_for_hits(&mut self, optimize_for_hits: bool) { + unsafe { + ffi::rocksdb_options_set_optimize_filters_for_hits(self.inner, optimize_for_hits as c_uchar); + } + } + /// Sets the number of open files that can be used by the DB. You may need to /// increase this if your database has a large working set. Value `-1` means /// files opened are always kept open. You can estimate number of files based From f4d1e81e04ec6151431ccd4b86b181c6e85ed937 Mon Sep 17 00:00:00 2001 From: Lucjan Suski Date: Tue, 29 Oct 2019 16:33:40 +0100 Subject: [PATCH 5/8] Add example, fix type --- src/db_options.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/db_options.rs b/src/db_options.rs index 29d0131..0ed1ca9 100644 --- a/src/db_options.rs +++ b/src/db_options.rs @@ -414,12 +414,24 @@ impl Options { } } + /// Sets the optimize_filters_for_hits flag + /// + /// Default: `false` + /// + /// # Example + /// + /// ``` + /// use rocksdb::Options; + /// + /// let mut opts = Options::default(); + /// opts.set_optimize_filters_for_hits(true); + /// ``` pub fn set_optimize_filters_for_hits(&mut self, optimize_for_hits: bool) { unsafe { - ffi::rocksdb_options_set_optimize_filters_for_hits(self.inner, optimize_for_hits as c_uchar); + ffi::rocksdb_options_set_optimize_filters_for_hits(self.inner, optimize_for_hits as c_int); } } - + /// Sets the number of open files that can be used by the DB. You may need to /// increase this if your database has a large working set. Value `-1` means /// files opened are always kept open. You can estimate number of files based From 421149b4da13b7b41b6e09b7c5424866327ef59c Mon Sep 17 00:00:00 2001 From: Lucjan Suski Date: Tue, 29 Oct 2019 17:21:36 +0100 Subject: [PATCH 6/8] Formatting --- src/db_options.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/db_options.rs b/src/db_options.rs index 0ed1ca9..fbd8d7b 100644 --- a/src/db_options.rs +++ b/src/db_options.rs @@ -428,7 +428,10 @@ impl Options { /// ``` pub fn set_optimize_filters_for_hits(&mut self, optimize_for_hits: bool) { unsafe { - ffi::rocksdb_options_set_optimize_filters_for_hits(self.inner, optimize_for_hits as c_int); + ffi::rocksdb_options_set_optimize_filters_for_hits( + self.inner, + optimize_for_hits as c_int, + ); } } From f64c74cc2e758cf79dcf70bffa8acdf04f6c4f05 Mon Sep 17 00:00:00 2001 From: Lucjan Suski Date: Wed, 30 Oct 2019 08:47:09 +0100 Subject: [PATCH 7/8] Formatting --- src/db_options.rs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/db_options.rs b/src/db_options.rs index fbd8d7b..25aedc4 100644 --- a/src/db_options.rs +++ b/src/db_options.rs @@ -269,7 +269,7 @@ impl Options { ) } } - + /// Maximum size of dictionaries used to prime the compression library. /// Enabling dictionary can improve compression ratios when there are /// repetitions across data blocks. @@ -286,12 +286,23 @@ impl Options { /// So users of this feature may see increased memory usage. /// /// Default: `0` - pub fn set_compression_options(&mut self, w_bits: c_int, level: c_int, strategy: c_int, max_dict_bytes: c_int) { + pub fn set_compression_options( + &mut self, + w_bits: c_int, + level: c_int, + strategy: c_int, + max_dict_bytes: c_int, + ) { unsafe { - ffi::rocksdb_options_set_compression_options(self.inner, w_bits, level, strategy, max_dict_bytes); + ffi::rocksdb_options_set_compression_options( + self.inner, + w_bits, + level, + strategy, + max_dict_bytes, + ); } } - /// If non-zero, we perform bigger reads when doing compaction. If you're /// running RocksDB on spinning disks, you should set this to at least 2MB. From bb65693abe137ec7ab9e1668ca72a82f86eccf52 Mon Sep 17 00:00:00 2001 From: Lucjan Suski Date: Wed, 30 Oct 2019 08:49:38 +0100 Subject: [PATCH 8/8] Add example --- src/db_options.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/db_options.rs b/src/db_options.rs index 25aedc4..b5339c2 100644 --- a/src/db_options.rs +++ b/src/db_options.rs @@ -286,6 +286,15 @@ impl Options { /// So users of this feature may see increased memory usage. /// /// Default: `0` + /// + /// # Example + /// + /// ``` + /// use rocksdb::Options; + /// + /// let mut opts = Options::default(); + /// opts.set_compression_options(4, 5, 6, 7); + /// ``` pub fn set_compression_options( &mut self, w_bits: c_int,