From 871c461580ea81cd92afd6819fd5b5ee2ea59641 Mon Sep 17 00:00:00 2001 From: Artem Krylysov Date: Thu, 3 Feb 2022 09:46:58 -0500 Subject: [PATCH] Upgrade to RocksDB 6.28.2 (#600) --- librocksdb-sys/Cargo.toml | 2 +- librocksdb-sys/build_version.cc | 6 +-- librocksdb-sys/rocksdb | 2 +- librocksdb-sys/rocksdb_lib_sources.txt | 19 ++++++++ librocksdb-sys/tests/ffi.rs | 2 +- src/db.rs | 5 ++ src/db_options.rs | 63 +------------------------- tests/test_db.rs | 14 ++++++ 8 files changed, 46 insertions(+), 67 deletions(-) diff --git a/librocksdb-sys/Cargo.toml b/librocksdb-sys/Cargo.toml index 064736c..cbf1ae6 100644 --- a/librocksdb-sys/Cargo.toml +++ b/librocksdb-sys/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "librocksdb-sys" -version = "6.20.3" +version = "6.28.2" edition = "2018" authors = ["Karl Hobley ", "Arkadiy Paronyan "] license = "MIT/Apache-2.0/BSD-3-Clause" diff --git a/librocksdb-sys/build_version.cc b/librocksdb-sys/build_version.cc index 01f721e..2a4ef78 100644 --- a/librocksdb-sys/build_version.cc +++ b/librocksdb-sys/build_version.cc @@ -7,9 +7,9 @@ // The build script may replace these values with real values based // on whether or not GIT is available and the platform settings -static const std::string rocksdb_build_git_sha = "rocksdb_build_git_sha:51b540921dd7495c9cf2265eb58942dad1f2ef72"; -static const std::string rocksdb_build_git_tag = "rocksdb_build_git_tag:v6.22.1"; -static const std::string rocksdb_build_date = "rocksdb_build_date:2021-06-25 14:15:04"; +static const std::string rocksdb_build_git_sha = "rocksdb_build_git_sha:3122cb435875d720fc3d23a48eb7c0fa89d869aa"; +static const std::string rocksdb_build_git_tag = "rocksdb_build_git_tag:6.28.2"; +static const std::string rocksdb_build_date = "rocksdb_build_date:2022-02-02 06:19:00"; namespace ROCKSDB_NAMESPACE { static void AddProperty(std::unordered_map *props, const std::string& name) { diff --git a/librocksdb-sys/rocksdb b/librocksdb-sys/rocksdb index 51b5409..df4d3cf 160000 --- a/librocksdb-sys/rocksdb +++ b/librocksdb-sys/rocksdb @@ -1 +1 @@ -Subproject commit 51b540921dd7495c9cf2265eb58942dad1f2ef72 +Subproject commit df4d3cf6fd52907f9a9a9bb62f124891787610eb diff --git a/librocksdb-sys/rocksdb_lib_sources.txt b/librocksdb-sys/rocksdb_lib_sources.txt index d326c86..1397d61 100644 --- a/librocksdb-sys/rocksdb_lib_sources.txt +++ b/librocksdb-sys/rocksdb_lib_sources.txt @@ -1,5 +1,7 @@ cache/cache.cc cache/cache_entry_roles.cc +cache/cache_key.cc +cache/cache_reservation_manager.cc cache/clock_cache.cc cache/lru_cache.cc cache/sharded_cache.cc @@ -11,9 +13,11 @@ db/blob/blob_file_cache.cc db/blob/blob_file_garbage.cc db/blob/blob_file_meta.cc db/blob/blob_file_reader.cc +db/blob/blob_garbage_meter.cc db/blob/blob_log_format.cc db/blob/blob_log_sequential_reader.cc db/blob/blob_log_writer.cc +db/blob/prefetch_buffer_collection.cc db/builder.cc db/c.cc db/column_family.cc @@ -90,6 +94,7 @@ env/fs_remap.cc env/file_system_tracer.cc env/io_posix.cc env/mock_env.cc +env/unique_id_gen.cc file/delete_scheduler.cc file/file_prefetch_buffer.cc file/file_util.cc @@ -108,6 +113,7 @@ memory/arena.cc memory/concurrent_arena.cc memory/jemalloc_nodump_allocator.cc memory/memkind_kmem_allocator.cc +memory/memory_allocator.cc memtable/alloc_tracker.cc memtable/hash_linklist_rep.cc memtable/hash_skiplist_rep.cc @@ -185,10 +191,14 @@ table/sst_file_writer.cc table/table_factory.cc table/table_properties.cc table/two_level_iterator.cc +table/unique_id.cc test_util/sync_point.cc test_util/sync_point_impl.cc test_util/transaction_test_util.cc tools/dump/db_dump_tool.cc +trace_replay/trace_record_handler.cc +trace_replay/trace_record_result.cc +trace_replay/trace_record.cc trace_replay/trace_replay.cc trace_replay/block_cache_tracer.cc trace_replay/io_tracer.cc @@ -199,12 +209,14 @@ util/comparator.cc util/compression_context_cache.cc util/concurrent_task_limiter_impl.cc util/crc32c.cc +util/crc32c_arm64.cc util/dynamic_bloom.cc util/hash.cc util/murmurhash.cc util/random.cc util/rate_limiter.cc util/ribbon_config.cc +util/regex.cc util/slice.cc util/file_checksum_helper.cc util/status.cc @@ -218,10 +230,13 @@ utilities/blob_db/blob_db.cc utilities/blob_db/blob_db_impl.cc utilities/blob_db/blob_db_impl_filesnapshot.cc utilities/blob_db/blob_file.cc +utilities/cache_dump_load.cc +utilities/cache_dump_load_impl.cc utilities/cassandra/cassandra_compaction_filter.cc utilities/cassandra/format.cc utilities/cassandra/merge_operator.cc utilities/checkpoint/checkpoint_impl.cc +utilities/compaction_filters.cc utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc utilities/convenience/info_log_finder.cc utilities/debug.cc @@ -229,8 +244,10 @@ utilities/env_mirror.cc utilities/env_timed.cc utilities/fault_injection_env.cc utilities/fault_injection_fs.cc +utilities/fault_injection_secondary_cache.cc utilities/leveldb_options/leveldb_options.cc utilities/memory/memory_util.cc +utilities/merge_operators.cc utilities/merge_operators/max.cc utilities/merge_operators/put.cc utilities/merge_operators/sortlist.cc @@ -250,6 +267,7 @@ utilities/simulator_cache/cache_simulator.cc utilities/simulator_cache/sim_cache.cc utilities/table_properties_collectors/compact_on_deletion_collector.cc utilities/trace/file_trace_reader_writer.cc +utilities/trace/replayer_impl.cc utilities/transactions/lock/lock_manager.cc utilities/transactions/lock/point/point_lock_tracker.cc utilities/transactions/lock/point/point_lock_manager.cc @@ -266,5 +284,6 @@ utilities/transactions/write_prepared_txn_db.cc utilities/transactions/write_unprepared_txn.cc utilities/transactions/write_unprepared_txn_db.cc utilities/ttl/db_ttl_impl.cc +utilities/wal_filter.cc utilities/write_batch_with_index/write_batch_with_index.cc utilities/write_batch_with_index/write_batch_with_index_internal.cc diff --git a/librocksdb-sys/tests/ffi.rs b/librocksdb-sys/tests/ffi.rs index 4a15d16..eed96c8 100644 --- a/librocksdb-sys/tests/ffi.rs +++ b/librocksdb-sys/tests/ffi.rs @@ -802,7 +802,7 @@ fn ffi() { Some(FilterName), ) } else { - rocksdb_filterpolicy_create_bloom(10) + rocksdb_filterpolicy_create_bloom(10.0) }; rocksdb_block_based_options_set_filter_policy(table_options, policy); diff --git a/src/db.rs b/src/db.rs index e8e0338..4fdf809 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1734,6 +1734,8 @@ impl DBWithThreadMode { let mut key_size: usize = 0; for i in 0..n { + let column_family_name = + from_cstr(ffi::rocksdb_livefiles_column_family_name(files, i)); let name = from_cstr(ffi::rocksdb_livefiles_name(files, i)); let size = ffi::rocksdb_livefiles_size(files, i); let level = ffi::rocksdb_livefiles_level(files, i) as i32; @@ -1747,6 +1749,7 @@ impl DBWithThreadMode { let largest_key = raw_data(largest_key, key_size); livefiles.push(LiveFile { + column_family_name, name, size, level, @@ -1910,6 +1913,8 @@ impl fmt::Debug for DBWithThreadMode { /// The metadata that describes a SST file #[derive(Debug, Clone)] pub struct LiveFile { + /// Name of the column family the file belongs to + pub column_family_name: String, /// Name of the file pub name: String, /// Size of the file diff --git a/src/db_options.rs b/src/db_options.rs index 421aa83..bf763f5 100644 --- a/src/db_options.rs +++ b/src/db_options.rs @@ -16,7 +16,7 @@ use std::ffi::{CStr, CString}; use std::path::Path; use std::sync::Arc; -use libc::{self, c_char, c_int, c_uchar, c_uint, c_void, size_t}; +use libc::{self, c_char, c_double, c_int, c_uchar, c_uint, c_void, size_t}; use crate::{ compaction_filter::{self, CompactionFilterCallback, CompactionFilterFn}, @@ -562,7 +562,7 @@ impl BlockBasedOptions { } /// Sets the filter policy to reduce disk reads - pub fn set_bloom_filter(&mut self, bits_per_key: c_int, block_based: bool) { + pub fn set_bloom_filter(&mut self, bits_per_key: c_double, block_based: bool) { unsafe { let bloom = if block_based { ffi::rocksdb_filterpolicy_create_bloom(bits_per_key) @@ -1661,17 +1661,6 @@ impl Options { } } - /// Enable/disable skipping of log corruption error on recovery (If client is ok with - /// losing most recent changes) - /// - /// Default: false - #[deprecated(since = "0.15.0", note = "This option is no longer used")] - pub fn set_skip_log_error_on_recovery(&mut self, enabled: bool) { - unsafe { - ffi::rocksdb_options_set_skip_log_error_on_recovery(self.inner, enabled as c_uchar); - } - } - /// Hints to the OS that it should not buffer disk I/O. Enabling this /// parameter may improve performance but increases pressure on the /// system cache. @@ -2721,18 +2710,6 @@ impl Options { } } - /// Enable/disable purging of duplicate/deleted keys when a memtable is flushed to storage. - /// - /// Default: true - pub fn set_purge_redundant_kvs_while_flush(&mut self, enabled: bool) { - unsafe { - ffi::rocksdb_options_set_purge_redundant_kvs_while_flush( - self.inner, - enabled as c_uchar, - ); - } - } - /// If true, then DB::Open() will not update the statistics used to optimize /// compaction decision by loading table properties from many files. /// Turning off this feature will improve DBOpen time especially in disk environment. @@ -2938,32 +2915,6 @@ impl Options { } } - /// Sets the soft rate limit. - /// - /// Puts are delayed 0-1 ms when any level has a compaction score that exceeds - /// soft_rate_limit. This is ignored when == 0.0. - /// CONSTRAINT: soft_rate_limit <= hard_rate_limit. If this constraint does not - /// hold, RocksDB will set soft_rate_limit = hard_rate_limit - /// - /// Default: 0.0 (disabled) - pub fn set_soft_rate_limit(&mut self, limit: f64) { - unsafe { - ffi::rocksdb_options_set_soft_rate_limit(self.inner, limit); - } - } - - /// Sets the hard rate limit. - /// - /// Puts are delayed 1ms at a time when any level has a compaction score that - /// exceeds hard_rate_limit. This is ignored when <= 1.0. - /// - /// Default: 0.0 (disabled) - pub fn set_hard_rate_limit(&mut self, limit: f64) { - unsafe { - ffi::rocksdb_options_set_hard_rate_limit(self.inner, limit); - } - } - /// Sets the threshold at which all writes will be slowed down to at least delayed_write_rate if estimated /// bytes needed to be compaction exceed this threshold. /// @@ -2984,16 +2935,6 @@ impl Options { } } - /// Sets the max time a put will be stalled when hard_rate_limit is enforced. - /// If 0, then there is no limit. - /// - /// Default: 1000 - pub fn set_rate_limit_delay_max_milliseconds(&mut self, millis: c_uint) { - unsafe { - ffi::rocksdb_options_set_rate_limit_delay_max_milliseconds(self.inner, millis); - } - } - /// Sets the size of one block in arena memory allocation. /// /// If <= 0, a proper value is automatically calculated (usually 1/10 of diff --git a/tests/test_db.rs b/tests/test_db.rs index a362d46..b7b48ab 100644 --- a/tests/test_db.rs +++ b/tests/test_db.rs @@ -660,6 +660,19 @@ fn fifo_compaction_test() { let expect = format!("block_cache_hit_count = {}", block_cache_hit_count); assert!(ctx.report(true).contains(&expect)); } + + // check live files (sst files meta) + let livefiles = db.live_files().unwrap(); + assert_eq!(livefiles.len(), 1); + livefiles.iter().for_each(|f| { + assert_eq!(f.level, 1); + assert_eq!(f.column_family_name, "cf1"); + assert!(!f.name.is_empty()); + assert_eq!(f.start_key.as_ref().unwrap().as_slice(), "k1".as_bytes()); + assert_eq!(f.end_key.as_ref().unwrap().as_slice(), "k5".as_bytes()); + assert_eq!(f.num_entries, 5); + assert_eq!(f.num_deletions, 0); + }); } } @@ -798,6 +811,7 @@ fn get_with_cache_and_bulkload_test() { assert_eq!(livefiles.len(), 1); livefiles.iter().for_each(|f| { assert_eq!(f.level, 2); + assert_eq!(f.column_family_name, "default"); assert!(!f.name.is_empty()); assert_eq!( f.start_key.as_ref().unwrap().as_slice(),