Upgrade to RocksDB 6.28.2 (#600)

librocksdb-sys/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "librocksdb-sys"
-version = "6.20.3"
+version = "6.28.2"
 edition = "2018"
 authors = ["Karl Hobley <karlhobley10@gmail.com>", "Arkadiy Paronyan <arkadiy@ethcore.io>"]
 license = "MIT/Apache-2.0/BSD-3-Clause"

librocksdb-sys/build_version.cc

@@ -7,9 +7,9 @@
 
 // The build script may replace these values with real values based
 // on whether or not GIT is available and the platform settings
-static const std::string rocksdb_build_git_sha  = "rocksdb_build_git_sha:51b540921dd7495c9cf2265eb58942dad1f2ef72";
+static const std::string rocksdb_build_git_sha  = "rocksdb_build_git_sha:3122cb435875d720fc3d23a48eb7c0fa89d869aa";
-static const std::string rocksdb_build_git_tag = "rocksdb_build_git_tag:v6.22.1";
+static const std::string rocksdb_build_git_tag = "rocksdb_build_git_tag:6.28.2";
-static const std::string rocksdb_build_date = "rocksdb_build_date:2021-06-25 14:15:04";
+static const std::string rocksdb_build_date = "rocksdb_build_date:2022-02-02 06:19:00";
 
 namespace ROCKSDB_NAMESPACE {
 static void AddProperty(std::unordered_map<std::string, std::string> *props, const std::string& name) {

librocksdb-sys/rocksdb

@@ -1 +1 @@
-Subproject commit 51b540921dd7495c9cf2265eb58942dad1f2ef72
+Subproject commit df4d3cf6fd52907f9a9a9bb62f124891787610eb

librocksdb-sys/rocksdb_lib_sources.txt

@@ -1,5 +1,7 @@
 cache/cache.cc
 cache/cache_entry_roles.cc
+cache/cache_key.cc
+cache/cache_reservation_manager.cc
 cache/clock_cache.cc
 cache/lru_cache.cc
 cache/sharded_cache.cc
@@ -11,9 +13,11 @@ db/blob/blob_file_cache.cc
 db/blob/blob_file_garbage.cc
 db/blob/blob_file_meta.cc
 db/blob/blob_file_reader.cc
+db/blob/blob_garbage_meter.cc
 db/blob/blob_log_format.cc
 db/blob/blob_log_sequential_reader.cc
 db/blob/blob_log_writer.cc
+db/blob/prefetch_buffer_collection.cc
 db/builder.cc
 db/c.cc
 db/column_family.cc
@@ -90,6 +94,7 @@ env/fs_remap.cc
 env/file_system_tracer.cc
 env/io_posix.cc
 env/mock_env.cc
+env/unique_id_gen.cc
 file/delete_scheduler.cc
 file/file_prefetch_buffer.cc
 file/file_util.cc
@@ -108,6 +113,7 @@ memory/arena.cc
 memory/concurrent_arena.cc
 memory/jemalloc_nodump_allocator.cc
 memory/memkind_kmem_allocator.cc
+memory/memory_allocator.cc
 memtable/alloc_tracker.cc
 memtable/hash_linklist_rep.cc
 memtable/hash_skiplist_rep.cc
@@ -185,10 +191,14 @@ table/sst_file_writer.cc
 table/table_factory.cc
 table/table_properties.cc
 table/two_level_iterator.cc
+table/unique_id.cc
 test_util/sync_point.cc
 test_util/sync_point_impl.cc
 test_util/transaction_test_util.cc
 tools/dump/db_dump_tool.cc
+trace_replay/trace_record_handler.cc
+trace_replay/trace_record_result.cc
+trace_replay/trace_record.cc
 trace_replay/trace_replay.cc
 trace_replay/block_cache_tracer.cc
 trace_replay/io_tracer.cc
@@ -199,12 +209,14 @@ util/comparator.cc
 util/compression_context_cache.cc
 util/concurrent_task_limiter_impl.cc
 util/crc32c.cc
+util/crc32c_arm64.cc
 util/dynamic_bloom.cc
 util/hash.cc
 util/murmurhash.cc
 util/random.cc
 util/rate_limiter.cc
 util/ribbon_config.cc
+util/regex.cc
 util/slice.cc
 util/file_checksum_helper.cc
 util/status.cc
@@ -218,10 +230,13 @@ utilities/blob_db/blob_db.cc
 utilities/blob_db/blob_db_impl.cc
 utilities/blob_db/blob_db_impl_filesnapshot.cc
 utilities/blob_db/blob_file.cc
+utilities/cache_dump_load.cc
+utilities/cache_dump_load_impl.cc
 utilities/cassandra/cassandra_compaction_filter.cc
 utilities/cassandra/format.cc
 utilities/cassandra/merge_operator.cc
 utilities/checkpoint/checkpoint_impl.cc
+utilities/compaction_filters.cc
 utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc
 utilities/convenience/info_log_finder.cc
 utilities/debug.cc
@@ -229,8 +244,10 @@ utilities/env_mirror.cc
 utilities/env_timed.cc
 utilities/fault_injection_env.cc
 utilities/fault_injection_fs.cc
+utilities/fault_injection_secondary_cache.cc
 utilities/leveldb_options/leveldb_options.cc
 utilities/memory/memory_util.cc
+utilities/merge_operators.cc
 utilities/merge_operators/max.cc
 utilities/merge_operators/put.cc
 utilities/merge_operators/sortlist.cc
@@ -250,6 +267,7 @@ utilities/simulator_cache/cache_simulator.cc
 utilities/simulator_cache/sim_cache.cc
 utilities/table_properties_collectors/compact_on_deletion_collector.cc
 utilities/trace/file_trace_reader_writer.cc
+utilities/trace/replayer_impl.cc
 utilities/transactions/lock/lock_manager.cc
 utilities/transactions/lock/point/point_lock_tracker.cc
 utilities/transactions/lock/point/point_lock_manager.cc
@@ -266,5 +284,6 @@ utilities/transactions/write_prepared_txn_db.cc
 utilities/transactions/write_unprepared_txn.cc
 utilities/transactions/write_unprepared_txn_db.cc
 utilities/ttl/db_ttl_impl.cc
+utilities/wal_filter.cc
 utilities/write_batch_with_index/write_batch_with_index.cc
 utilities/write_batch_with_index/write_batch_with_index_internal.cc

librocksdb-sys/tests/ffi.rs

@@ -802,7 +802,7 @@ fn ffi() {
                     Some(FilterName),
                 )
             } else {
-                rocksdb_filterpolicy_create_bloom(10)
+                rocksdb_filterpolicy_create_bloom(10.0)
             };
 
             rocksdb_block_based_options_set_filter_policy(table_options, policy);

src/db.rs

@@ -1734,6 +1734,8 @@ impl<T: ThreadMode> DBWithThreadMode<T> {
                 let mut key_size: usize = 0;
 
                 for i in 0..n {
+                    let column_family_name =
+                        from_cstr(ffi::rocksdb_livefiles_column_family_name(files, i));
                     let name = from_cstr(ffi::rocksdb_livefiles_name(files, i));
                     let size = ffi::rocksdb_livefiles_size(files, i);
                     let level = ffi::rocksdb_livefiles_level(files, i) as i32;
@@ -1747,6 +1749,7 @@ impl<T: ThreadMode> DBWithThreadMode<T> {
                     let largest_key = raw_data(largest_key, key_size);
 
                     livefiles.push(LiveFile {
+                        column_family_name,
                         name,
                         size,
                         level,
@@ -1910,6 +1913,8 @@ impl<T: ThreadMode> fmt::Debug for DBWithThreadMode<T> {
 /// The metadata that describes a SST file
 #[derive(Debug, Clone)]
 pub struct LiveFile {
+    /// Name of the column family the file belongs to
+    pub column_family_name: String,
     /// Name of the file
     pub name: String,
     /// Size of the file

src/db_options.rs

@@ -16,7 +16,7 @@ use std::ffi::{CStr, CString};
 use std::path::Path;
 use std::sync::Arc;
 
-use libc::{self, c_char, c_int, c_uchar, c_uint, c_void, size_t};
+use libc::{self, c_char, c_double, c_int, c_uchar, c_uint, c_void, size_t};
 
 use crate::{
     compaction_filter::{self, CompactionFilterCallback, CompactionFilterFn},
@@ -562,7 +562,7 @@ impl BlockBasedOptions {
     }
 
     /// Sets the filter policy to reduce disk reads
-    pub fn set_bloom_filter(&mut self, bits_per_key: c_int, block_based: bool) {
+    pub fn set_bloom_filter(&mut self, bits_per_key: c_double, block_based: bool) {
         unsafe {
             let bloom = if block_based {
                 ffi::rocksdb_filterpolicy_create_bloom(bits_per_key)
@@ -1661,17 +1661,6 @@ impl Options {
         }
     }
 
-    /// Enable/disable skipping of log corruption error on recovery (If client is ok with
-    /// losing most recent changes)
-    ///
-    /// Default: false
-    #[deprecated(since = "0.15.0", note = "This option is no longer used")]
-    pub fn set_skip_log_error_on_recovery(&mut self, enabled: bool) {
-        unsafe {
-            ffi::rocksdb_options_set_skip_log_error_on_recovery(self.inner, enabled as c_uchar);
-        }
-    }
-
     /// Hints to the OS that it should not buffer disk I/O. Enabling this
     /// parameter may improve performance but increases pressure on the
     /// system cache.
@@ -2721,18 +2710,6 @@ impl Options {
         }
     }
 
-    /// Enable/disable purging of duplicate/deleted keys when a memtable is flushed to storage.
-    ///
-    /// Default: true
-    pub fn set_purge_redundant_kvs_while_flush(&mut self, enabled: bool) {
-        unsafe {
-            ffi::rocksdb_options_set_purge_redundant_kvs_while_flush(
-                self.inner,
-                enabled as c_uchar,
-            );
-        }
-    }
-
     /// If true, then DB::Open() will not update the statistics used to optimize
     /// compaction decision by loading table properties from many files.
     /// Turning off this feature will improve DBOpen time especially in disk environment.
@@ -2938,32 +2915,6 @@ impl Options {
         }
     }
 
-    /// Sets the soft rate limit.
-    ///
-    /// Puts are delayed 0-1 ms when any level has a compaction score that exceeds
-    /// soft_rate_limit. This is ignored when == 0.0.
-    /// CONSTRAINT: soft_rate_limit <= hard_rate_limit. If this constraint does not
-    /// hold, RocksDB will set soft_rate_limit = hard_rate_limit
-    ///
-    /// Default: 0.0 (disabled)
-    pub fn set_soft_rate_limit(&mut self, limit: f64) {
-        unsafe {
-            ffi::rocksdb_options_set_soft_rate_limit(self.inner, limit);
-        }
-    }
-
-    /// Sets the hard rate limit.
-    ///
-    /// Puts are delayed 1ms at a time when any level has a compaction score that
-    /// exceeds hard_rate_limit. This is ignored when <= 1.0.
-    ///
-    /// Default: 0.0 (disabled)
-    pub fn set_hard_rate_limit(&mut self, limit: f64) {
-        unsafe {
-            ffi::rocksdb_options_set_hard_rate_limit(self.inner, limit);
-        }
-    }
-
     /// Sets the threshold at which all writes will be slowed down to at least delayed_write_rate if estimated
     /// bytes needed to be compaction exceed this threshold.
     ///
@@ -2984,16 +2935,6 @@ impl Options {
         }
     }
 
-    /// Sets the max time a put will be stalled when hard_rate_limit is enforced.
-    /// If 0, then there is no limit.
-    ///
-    /// Default: 1000
-    pub fn set_rate_limit_delay_max_milliseconds(&mut self, millis: c_uint) {
-        unsafe {
-            ffi::rocksdb_options_set_rate_limit_delay_max_milliseconds(self.inner, millis);
-        }
-    }
-
     /// Sets the size of one block in arena memory allocation.
     ///
     /// If <= 0, a proper value is automatically calculated (usually 1/10 of

tests/test_db.rs

@@ -660,6 +660,19 @@ fn fifo_compaction_test() {
             let expect = format!("block_cache_hit_count = {}", block_cache_hit_count);
             assert!(ctx.report(true).contains(&expect));
         }
+
+        // check live files (sst files meta)
+        let livefiles = db.live_files().unwrap();
+        assert_eq!(livefiles.len(), 1);
+        livefiles.iter().for_each(|f| {
+            assert_eq!(f.level, 1);
+            assert_eq!(f.column_family_name, "cf1");
+            assert!(!f.name.is_empty());
+            assert_eq!(f.start_key.as_ref().unwrap().as_slice(), "k1".as_bytes());
+            assert_eq!(f.end_key.as_ref().unwrap().as_slice(), "k5".as_bytes());
+            assert_eq!(f.num_entries, 5);
+            assert_eq!(f.num_deletions, 0);
+        });
     }
 }
 
@@ -798,6 +811,7 @@ fn get_with_cache_and_bulkload_test() {
         assert_eq!(livefiles.len(), 1);
         livefiles.iter().for_each(|f| {
             assert_eq!(f.level, 2);
+            assert_eq!(f.column_family_name, "default");
             assert!(!f.name.is_empty());
             assert_eq!(
                 f.start_key.as_ref().unwrap().as_slice(),