From 530dc462d51b7ac4f54bb1d9a99a8a217700f1e3 Mon Sep 17 00:00:00 2001 From: Martin Ek Date: Mon, 9 Apr 2018 18:59:17 +0200 Subject: [PATCH 1/5] Add DBOptions.set_wal_dir --- src/db_options.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/db_options.rs b/src/db_options.rs index 716b6f7..12fe213 100644 --- a/src/db_options.rs +++ b/src/db_options.rs @@ -14,6 +14,7 @@ use std::ffi::{CStr, CString}; use std::mem; +use std::path::Path; use libc::{self, c_int, c_uchar, c_uint, c_void, size_t, uint64_t}; @@ -980,6 +981,26 @@ impl Options { ffi::rocksdb_options_set_num_levels(self.inner, n); } } + + /// Specifies the absolute path of the directory the + /// write-ahead log (WAL) should be written to. + /// + /// Default: same directory as the database + /// + /// # Example + /// + /// ``` + /// use rocksdb::Options; + /// + /// let mut opts = Options::default(); + /// opts.set_wal_dir("/path/to/dir"); + /// ``` + pub fn set_wal_dir>(&mut self, path: P) { + let p = CString::new(path.as_ref().to_string_lossy().as_bytes()).unwrap(); + unsafe { + ffi::rocksdb_options_set_wal_dir(self.inner, p.as_ptr()); + } + } } impl Default for Options { From 45edfd2e900d196e0d0bcd4e251e69cc8ee4e943 Mon Sep 17 00:00:00 2001 From: Roman Zeyde Date: Mon, 9 Apr 2018 21:37:58 +0300 Subject: [PATCH 2/5] Expose compaction_readahead_size option --- src/db_options.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/db_options.rs b/src/db_options.rs index 716b6f7..912f5c6 100644 --- a/src/db_options.rs +++ b/src/db_options.rs @@ -221,6 +221,20 @@ impl Options { } } + /// If non-zero, we perform bigger reads when doing compaction. If you're + /// running RocksDB on spinning disks, you should set this to at least 2MB. + /// That way RocksDB's compaction is doing sequential instead of random reads. + /// + /// When non-zero, we also force new_table_reader_for_compaction_inputs to + /// true. + /// + /// Default: `0` + pub fn set_compaction_readahead_size(&mut self, compaction_readahead_size: usize) { + unsafe { + ffi::rocksdb_options_compaction_readahead_size(self.inner, compaction_readahead_size as usize); + } + } + pub fn set_merge_operator(&mut self, name: &str, full_merge_fn: MergeFn, partial_merge_fn: Option) { From e08ad288c48ab6fccd45bfb00da162daa69281b1 Mon Sep 17 00:00:00 2001 From: Martin Ek Date: Thu, 12 Apr 2018 16:20:27 +0200 Subject: [PATCH 3/5] Add disable_cache method --- src/db_options.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/db_options.rs b/src/db_options.rs index 716b6f7..7adc6ad 100644 --- a/src/db_options.rs +++ b/src/db_options.rs @@ -72,6 +72,12 @@ impl BlockBasedOptions { } } + pub fn disable_cache(&mut self) { + unsafe { + ffi::rocksdb_block_based_options_set_no_block_cache(self.inner, true as c_uchar); + } + } + pub fn set_bloom_filter(&mut self, bits_per_key: c_int, block_based: bool) { unsafe { let bloom = if block_based { From f1c3b48b3f1ffc0f3b27cfc4d0c62e5894e76fad Mon Sep 17 00:00:00 2001 From: Martin Ek Date: Sat, 14 Jul 2018 19:45:06 +0700 Subject: [PATCH 4/5] Update bindgen to 0.37 --- librocksdb-sys/Cargo.toml | 2 +- librocksdb-sys/build.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/librocksdb-sys/Cargo.toml b/librocksdb-sys/Cargo.toml index d28b889..af83aa6 100644 --- a/librocksdb-sys/Cargo.toml +++ b/librocksdb-sys/Cargo.toml @@ -24,4 +24,4 @@ const-cstr = "0.2" [build-dependencies] cc = { version = "^1.0", features = ["parallel"] } make-cmd = "0.1" -bindgen = "0.29" +bindgen = "0.37" diff --git a/librocksdb-sys/build.rs b/librocksdb-sys/build.rs index 1f19287..7b6ada9 100644 --- a/librocksdb-sys/build.rs +++ b/librocksdb-sys/build.rs @@ -32,7 +32,7 @@ fn fail_on_empty_directory(name: &str) { fn bindgen_rocksdb() { let bindings = bindgen::Builder::default() .header("rocksdb/include/rocksdb/c.h") - .hide_type("max_align_t") // https://github.com/rust-lang-nursery/rust-bindgen/issues/550 + .blacklist_type("max_align_t") // https://github.com/rust-lang-nursery/rust-bindgen/issues/550 .ctypes_prefix("libc") .generate() .expect("unable to generate rocksdb bindings"); From 7c6d756347f9bba688cdb5f0d4bd02a189d0ee62 Mon Sep 17 00:00:00 2001 From: Martin Ek Date: Sat, 14 Jul 2018 20:01:14 +0700 Subject: [PATCH 5/5] Upgrade RocksDB to 5.14.2 --- Cargo.toml | 2 +- librocksdb-sys/Cargo.toml | 2 +- librocksdb-sys/rocksdb | 2 +- librocksdb-sys/rocksdb_lib_sources.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 8ebecae..4e4f4e5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,4 +19,4 @@ valgrind = [] [dependencies] libc = "0.2" -librocksdb-sys = { path = "librocksdb-sys", version = "5.11.3" } +librocksdb-sys = { path = "librocksdb-sys", version = "5.14.2" } diff --git a/librocksdb-sys/Cargo.toml b/librocksdb-sys/Cargo.toml index d28b889..2286bc7 100644 --- a/librocksdb-sys/Cargo.toml +++ b/librocksdb-sys/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "librocksdb-sys" -version = "5.11.3" +version = "5.14.2" authors = ["Karl Hobley ", "Arkadiy Paronyan "] license = "MIT/Apache-2.0/BSD-3-Clause" description = "Native bindings to librocksdb" diff --git a/librocksdb-sys/rocksdb b/librocksdb-sys/rocksdb index dbd8fa0..5089e12 160000 --- a/librocksdb-sys/rocksdb +++ b/librocksdb-sys/rocksdb @@ -1 +1 @@ -Subproject commit dbd8fa09b823826dd2a30bc119dad7a6fa9a4c6d +Subproject commit 5089e121166c46956a8a21c8ef967f1896c239de diff --git a/librocksdb-sys/rocksdb_lib_sources.txt b/librocksdb-sys/rocksdb_lib_sources.txt index 886b530..fc927e2 100644 --- a/librocksdb-sys/rocksdb_lib_sources.txt +++ b/librocksdb-sys/rocksdb_lib_sources.txt @@ -1 +1 @@ -cache/clock_cache.cc cache/lru_cache.cc cache/sharded_cache.cc db/builder.cc db/c.cc db/column_family.cc db/compacted_db_impl.cc db/compaction.cc db/compaction_iterator.cc db/compaction_job.cc db/compaction_picker.cc db/compaction_picker_universal.cc db/convenience.cc db/db_filesnapshot.cc db/db_impl.cc db/db_impl_compaction_flush.cc db/db_impl_debug.cc db/db_impl_experimental.cc db/db_impl_files.cc db/db_impl_open.cc db/db_impl_readonly.cc db/db_impl_write.cc db/db_info_dumper.cc db/db_iter.cc db/dbformat.cc db/event_helpers.cc db/experimental.cc db/external_sst_file_ingestion_job.cc db/file_indexer.cc db/flush_job.cc db/flush_scheduler.cc db/forward_iterator.cc db/internal_stats.cc db/log_reader.cc db/log_writer.cc db/malloc_stats.cc db/managed_iterator.cc db/memtable.cc db/memtable_list.cc db/merge_helper.cc db/merge_operator.cc db/range_del_aggregator.cc db/repair.cc db/snapshot_impl.cc db/table_cache.cc db/table_properties_collector.cc db/transaction_log_impl.cc db/version_builder.cc db/version_edit.cc db/version_set.cc db/wal_manager.cc db/write_batch.cc db/write_batch_base.cc db/write_controller.cc db/write_thread.cc env/env.cc env/env_chroot.cc env/env_encryption.cc env/env_hdfs.cc env/env_posix.cc env/io_posix.cc env/mock_env.cc memtable/alloc_tracker.cc memtable/hash_cuckoo_rep.cc memtable/hash_linklist_rep.cc memtable/hash_skiplist_rep.cc memtable/skiplistrep.cc memtable/vectorrep.cc memtable/write_buffer_manager.cc monitoring/histogram.cc monitoring/histogram_windowing.cc monitoring/instrumented_mutex.cc monitoring/iostats_context.cc monitoring/perf_context.cc monitoring/perf_level.cc monitoring/statistics.cc monitoring/thread_status_impl.cc monitoring/thread_status_updater.cc monitoring/thread_status_updater_debug.cc monitoring/thread_status_util.cc monitoring/thread_status_util_debug.cc options/cf_options.cc options/db_options.cc options/options.cc options/options_helper.cc options/options_parser.cc options/options_sanity_check.cc port/port_posix.cc port/stack_trace.cc table/adaptive_table_factory.cc table/block.cc table/block_based_filter_block.cc table/block_based_table_builder.cc table/block_based_table_factory.cc table/block_based_table_reader.cc table/block_builder.cc table/block_fetcher.cc table/block_prefix_index.cc table/bloom_block.cc table/cuckoo_table_builder.cc table/cuckoo_table_factory.cc table/cuckoo_table_reader.cc table/flush_block_policy.cc table/format.cc table/full_filter_block.cc table/get_context.cc table/index_builder.cc table/iterator.cc table/merging_iterator.cc table/meta_blocks.cc table/partitioned_filter_block.cc table/persistent_cache_helper.cc table/plain_table_builder.cc table/plain_table_factory.cc table/plain_table_index.cc table/plain_table_key_coding.cc table/plain_table_reader.cc table/sst_file_writer.cc table/table_properties.cc table/two_level_iterator.cc tools/dump/db_dump_tool.cc util/arena.cc util/auto_roll_logger.cc util/bloom.cc util/build_version.cc util/coding.cc util/compaction_job_stats_impl.cc util/comparator.cc util/concurrent_arena.cc util/crc32c.cc util/delete_scheduler.cc util/dynamic_bloom.cc util/event_logger.cc util/file_reader_writer.cc util/file_util.cc util/filename.cc util/filter_policy.cc util/hash.cc util/log_buffer.cc util/murmurhash.cc util/random.cc util/rate_limiter.cc util/slice.cc util/sst_file_manager_impl.cc util/status.cc util/status_message.cc util/string_util.cc util/sync_point.cc util/thread_local.cc util/threadpool_imp.cc util/transaction_test_util.cc util/xxhash.cc utilities/backupable/backupable_db.cc utilities/blob_db/blob_db.cc utilities/blob_db/blob_db_impl.cc utilities/blob_db/blob_file.cc utilities/blob_db/blob_log_format.cc utilities/blob_db/blob_log_reader.cc utilities/blob_db/blob_log_writer.cc utilities/blob_db/ttl_extractor.cc utilities/cassandra/cassandra_compaction_filter.cc utilities/cassandra/format.cc utilities/cassandra/merge_operator.cc utilities/checkpoint/checkpoint_impl.cc utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc utilities/convenience/info_log_finder.cc utilities/date_tiered/date_tiered_db_impl.cc utilities/debug.cc utilities/document/document_db.cc utilities/document/json_document.cc utilities/document/json_document_builder.cc utilities/env_mirror.cc utilities/env_timed.cc utilities/geodb/geodb_impl.cc utilities/leveldb_options/leveldb_options.cc utilities/lua/rocks_lua_compaction_filter.cc utilities/memory/memory_util.cc utilities/merge_operators/max.cc utilities/merge_operators/put.cc utilities/merge_operators/string_append/stringappend.cc utilities/merge_operators/string_append/stringappend2.cc utilities/merge_operators/uint64add.cc utilities/merge_operators/bytesxor.cc utilities/option_change_migration/option_change_migration.cc utilities/options/options_util.cc utilities/persistent_cache/block_cache_tier.cc utilities/persistent_cache/block_cache_tier_file.cc utilities/persistent_cache/block_cache_tier_metadata.cc utilities/persistent_cache/persistent_cache_tier.cc utilities/persistent_cache/volatile_tier_impl.cc utilities/redis/redis_lists.cc utilities/simulator_cache/sim_cache.cc utilities/spatialdb/spatial_db.cc utilities/table_properties_collectors/compact_on_deletion_collector.cc utilities/transactions/optimistic_transaction.cc utilities/transactions/optimistic_transaction_db_impl.cc utilities/transactions/pessimistic_transaction.cc utilities/transactions/pessimistic_transaction_db.cc utilities/transactions/snapshot_checker.cc utilities/transactions/transaction_base.cc utilities/transactions/transaction_db_mutex_impl.cc utilities/transactions/transaction_lock_mgr.cc utilities/transactions/transaction_util.cc utilities/transactions/write_prepared_txn.cc utilities/transactions/write_prepared_txn_db.cc utilities/ttl/db_ttl_impl.cc utilities/write_batch_with_index/write_batch_with_index.cc utilities/write_batch_with_index/write_batch_with_index_internal.cc \ No newline at end of file +cache/clock_cache.cc cache/lru_cache.cc cache/sharded_cache.cc db/builder.cc db/c.cc db/column_family.cc db/compacted_db_impl.cc db/compaction.cc db/compaction_iterator.cc db/compaction_job.cc db/compaction_picker.cc db/compaction_picker_universal.cc db/convenience.cc db/db_filesnapshot.cc db/db_impl.cc db/db_impl_compaction_flush.cc db/db_impl_debug.cc db/db_impl_experimental.cc db/db_impl_files.cc db/db_impl_open.cc db/db_impl_readonly.cc db/db_impl_write.cc db/db_info_dumper.cc db/db_iter.cc db/dbformat.cc db/event_helpers.cc db/experimental.cc db/external_sst_file_ingestion_job.cc db/file_indexer.cc db/flush_job.cc db/flush_scheduler.cc db/forward_iterator.cc db/internal_stats.cc db/logs_with_prep_tracker.cc db/log_reader.cc db/log_writer.cc db/malloc_stats.cc db/managed_iterator.cc db/memtable.cc db/memtable_list.cc db/merge_helper.cc db/merge_operator.cc db/range_del_aggregator.cc db/repair.cc db/snapshot_impl.cc db/table_cache.cc db/table_properties_collector.cc db/transaction_log_impl.cc db/version_builder.cc db/version_edit.cc db/version_set.cc db/wal_manager.cc db/write_batch.cc db/write_batch_base.cc db/write_controller.cc db/write_thread.cc env/env.cc env/env_chroot.cc env/env_encryption.cc env/env_hdfs.cc env/env_posix.cc env/io_posix.cc env/mock_env.cc memtable/alloc_tracker.cc memtable/hash_cuckoo_rep.cc memtable/hash_linklist_rep.cc memtable/hash_skiplist_rep.cc memtable/skiplistrep.cc memtable/vectorrep.cc memtable/write_buffer_manager.cc monitoring/histogram.cc monitoring/histogram_windowing.cc monitoring/instrumented_mutex.cc monitoring/iostats_context.cc monitoring/perf_context.cc monitoring/perf_level.cc monitoring/statistics.cc monitoring/thread_status_impl.cc monitoring/thread_status_updater.cc monitoring/thread_status_updater_debug.cc monitoring/thread_status_util.cc monitoring/thread_status_util_debug.cc options/cf_options.cc options/db_options.cc options/options.cc options/options_helper.cc options/options_parser.cc options/options_sanity_check.cc port/port_posix.cc port/stack_trace.cc table/adaptive_table_factory.cc table/block.cc table/block_based_filter_block.cc table/block_based_table_builder.cc table/block_based_table_factory.cc table/block_based_table_reader.cc table/block_builder.cc table/block_fetcher.cc table/block_prefix_index.cc table/bloom_block.cc table/cuckoo_table_builder.cc table/cuckoo_table_factory.cc table/cuckoo_table_reader.cc table/flush_block_policy.cc table/format.cc table/full_filter_block.cc table/get_context.cc table/index_builder.cc table/iterator.cc table/merging_iterator.cc table/meta_blocks.cc table/partitioned_filter_block.cc table/persistent_cache_helper.cc table/plain_table_builder.cc table/plain_table_factory.cc table/plain_table_index.cc table/plain_table_key_coding.cc table/plain_table_reader.cc table/sst_file_writer.cc table/table_properties.cc table/two_level_iterator.cc tools/dump/db_dump_tool.cc util/arena.cc util/auto_roll_logger.cc util/bloom.cc util/build_version.cc util/coding.cc util/compaction_job_stats_impl.cc util/comparator.cc util/concurrent_arena.cc util/crc32c.cc util/delete_scheduler.cc util/dynamic_bloom.cc util/event_logger.cc util/file_reader_writer.cc util/file_util.cc util/filename.cc util/filter_policy.cc util/hash.cc util/log_buffer.cc util/murmurhash.cc util/random.cc util/rate_limiter.cc util/slice.cc util/sst_file_manager_impl.cc util/status.cc util/status_message.cc util/string_util.cc util/sync_point.cc util/sync_point_impl.cc util/thread_local.cc util/threadpool_imp.cc util/transaction_test_util.cc util/xxhash.cc utilities/backupable/backupable_db.cc utilities/blob_db/blob_compaction_filter.cc utilities/blob_db/blob_db.cc utilities/blob_db/blob_db_impl.cc utilities/blob_db/blob_file.cc utilities/blob_db/blob_log_format.cc utilities/blob_db/blob_log_reader.cc utilities/blob_db/blob_log_writer.cc utilities/blob_db/ttl_extractor.cc utilities/cassandra/cassandra_compaction_filter.cc utilities/cassandra/format.cc utilities/cassandra/merge_operator.cc utilities/checkpoint/checkpoint_impl.cc utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc utilities/convenience/info_log_finder.cc utilities/date_tiered/date_tiered_db_impl.cc utilities/debug.cc utilities/document/document_db.cc utilities/document/json_document.cc utilities/document/json_document_builder.cc utilities/env_mirror.cc utilities/env_timed.cc utilities/geodb/geodb_impl.cc utilities/leveldb_options/leveldb_options.cc utilities/lua/rocks_lua_compaction_filter.cc utilities/memory/memory_util.cc utilities/merge_operators/max.cc utilities/merge_operators/put.cc utilities/merge_operators/string_append/stringappend.cc utilities/merge_operators/string_append/stringappend2.cc utilities/merge_operators/uint64add.cc utilities/merge_operators/bytesxor.cc utilities/option_change_migration/option_change_migration.cc utilities/options/options_util.cc utilities/persistent_cache/block_cache_tier.cc utilities/persistent_cache/block_cache_tier_file.cc utilities/persistent_cache/block_cache_tier_metadata.cc utilities/persistent_cache/persistent_cache_tier.cc utilities/persistent_cache/volatile_tier_impl.cc utilities/redis/redis_lists.cc utilities/simulator_cache/sim_cache.cc utilities/spatialdb/spatial_db.cc utilities/table_properties_collectors/compact_on_deletion_collector.cc utilities/transactions/optimistic_transaction.cc utilities/transactions/optimistic_transaction_db_impl.cc utilities/transactions/pessimistic_transaction.cc utilities/transactions/pessimistic_transaction_db.cc utilities/transactions/snapshot_checker.cc utilities/transactions/transaction_base.cc utilities/transactions/transaction_db_mutex_impl.cc utilities/transactions/transaction_lock_mgr.cc utilities/transactions/transaction_util.cc utilities/transactions/write_prepared_txn.cc utilities/transactions/write_prepared_txn_db.cc utilities/ttl/db_ttl_impl.cc utilities/write_batch_with_index/write_batch_with_index.cc utilities/write_batch_with_index/write_batch_with_index_internal.cc \ No newline at end of file