From 87f802479dc3dd78cb63b69463293264622bb3d2 Mon Sep 17 00:00:00 2001 From: Richard Dodd Date: Tue, 13 Jun 2017 11:14:43 +0100 Subject: [PATCH 1/9] Make rocksdb work with gcc 7 (v5.4.6) --- librocksdb-sys/rocksdb | 2 +- librocksdb-sys/rocksdb_lib_sources.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/librocksdb-sys/rocksdb b/librocksdb-sys/rocksdb index f201a44..6e05979 160000 --- a/librocksdb-sys/rocksdb +++ b/librocksdb-sys/rocksdb @@ -1 +1 @@ -Subproject commit f201a44b4102308b840b15d9b89122af787476f1 +Subproject commit 6e0597951e51e8dd3b28104d31e52962e00c101f diff --git a/librocksdb-sys/rocksdb_lib_sources.txt b/librocksdb-sys/rocksdb_lib_sources.txt index 209deb5..97e9791 100644 --- a/librocksdb-sys/rocksdb_lib_sources.txt +++ b/librocksdb-sys/rocksdb_lib_sources.txt @@ -1 +1 @@ -db/auto_roll_logger.cc db/builder.cc db/c.cc db/column_family.cc db/compacted_db_impl.cc db/compaction.cc db/compaction_iterator.cc db/compaction_job.cc db/compaction_picker.cc db/convenience.cc db/range_del_aggregator.cc db/db_filesnapshot.cc db/dbformat.cc db/db_impl.cc db/db_impl_debug.cc db/db_impl_readonly.cc db/db_impl_experimental.cc db/db_info_dumper.cc db/db_iter.cc db/external_sst_file_ingestion_job.cc db/experimental.cc db/event_helpers.cc db/file_indexer.cc db/filename.cc db/flush_job.cc db/flush_scheduler.cc db/forward_iterator.cc db/internal_stats.cc db/log_reader.cc db/log_writer.cc db/managed_iterator.cc db/memtable_allocator.cc db/memtable.cc db/memtable_list.cc db/merge_helper.cc db/merge_operator.cc db/repair.cc db/snapshot_impl.cc db/table_cache.cc db/table_properties_collector.cc db/transaction_log_impl.cc db/version_builder.cc db/version_edit.cc db/version_set.cc db/wal_manager.cc db/write_batch.cc db/write_batch_base.cc db/write_controller.cc db/write_thread.cc db/xfunc_test_points.cc memtable/hash_cuckoo_rep.cc memtable/hash_linklist_rep.cc memtable/hash_skiplist_rep.cc memtable/skiplistrep.cc memtable/vectorrep.cc port/stack_trace.cc port/port_posix.cc table/adaptive_table_factory.cc table/block_based_filter_block.cc table/block_based_table_builder.cc table/block_based_table_factory.cc table/block_based_table_reader.cc table/block_builder.cc table/block.cc table/block_prefix_index.cc table/bloom_block.cc table/cuckoo_table_builder.cc table/cuckoo_table_factory.cc table/cuckoo_table_reader.cc table/flush_block_policy.cc table/format.cc table/full_filter_block.cc table/get_context.cc table/iterator.cc table/merger.cc table/meta_blocks.cc table/sst_file_writer.cc table/plain_table_builder.cc table/plain_table_factory.cc table/plain_table_index.cc table/plain_table_key_coding.cc table/plain_table_reader.cc table/persistent_cache_helper.cc table/table_properties.cc table/two_level_iterator.cc tools/dump/db_dump_tool.cc util/arena.cc util/bloom.cc util/build_version.cc util/cf_options.cc util/clock_cache.cc util/coding.cc util/comparator.cc util/compaction_job_stats_impl.cc util/concurrent_arena.cc util/crc32c.cc util/db_options.cc util/delete_scheduler.cc util/dynamic_bloom.cc util/env.cc util/env_chroot.cc util/env_hdfs.cc util/env_posix.cc util/event_logger.cc util/file_util.cc util/file_reader_writer.cc util/filter_policy.cc util/hash.cc util/histogram.cc util/histogram_windowing.cc util/instrumented_mutex.cc util/iostats_context.cc util/io_posix.cc util/log_buffer.cc util/logging.cc util/lru_cache.cc util/memenv.cc util/murmurhash.cc util/options.cc util/options_helper.cc util/options_parser.cc util/options_sanity_check.cc util/perf_context.cc util/perf_level.cc util/random.cc util/rate_limiter.cc util/sharded_cache.cc util/slice.cc util/sst_file_manager_impl.cc util/statistics.cc util/status.cc util/status_message.cc util/string_util.cc util/sync_point.cc util/thread_local.cc util/thread_status_impl.cc util/thread_status_updater.cc util/thread_status_updater_debug.cc util/thread_status_util.cc util/thread_status_util_debug.cc util/threadpool_imp.cc util/transaction_test_util.cc util/xfunc.cc util/xxhash.cc utilities/backupable/backupable_db.cc utilities/blob_db/blob_db.cc utilities/convenience/info_log_finder.cc utilities/checkpoint/checkpoint.cc utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc utilities/document/document_db.cc utilities/document/json_document_builder.cc utilities/document/json_document.cc utilities/env_mirror.cc utilities/env_registry.cc utilities/flashcache/flashcache.cc utilities/geodb/geodb_impl.cc utilities/leveldb_options/leveldb_options.cc utilities/memory/memory_util.cc utilities/merge_operators/put.cc utilities/merge_operators/max.cc utilities/merge_operators/string_append/stringappend2.cc utilities/merge_operators/string_append/stringappend.cc utilities/merge_operators/uint64add.cc utilities/option_change_migration/option_change_migration.cc utilities/options/options_util.cc utilities/persistent_cache/persistent_cache_tier.cc utilities/persistent_cache/volatile_tier_impl.cc utilities/persistent_cache/block_cache_tier_file.cc utilities/persistent_cache/block_cache_tier_metadata.cc utilities/persistent_cache/block_cache_tier.cc utilities/redis/redis_lists.cc utilities/simulator_cache/sim_cache.cc utilities/spatialdb/spatial_db.cc utilities/table_properties_collectors/compact_on_deletion_collector.cc utilities/transactions/optimistic_transaction_impl.cc utilities/transactions/optimistic_transaction_db_impl.cc utilities/transactions/transaction_base.cc utilities/transactions/transaction_db_impl.cc utilities/transactions/transaction_db_mutex_impl.cc utilities/transactions/transaction_lock_mgr.cc utilities/transactions/transaction_impl.cc utilities/transactions/transaction_util.cc utilities/ttl/db_ttl_impl.cc utilities/date_tiered/date_tiered_db_impl.cc utilities/write_batch_with_index/write_batch_with_index.cc utilities/write_batch_with_index/write_batch_with_index_internal.cc \ No newline at end of file +cache/clock_cache.cc cache/lru_cache.cc cache/sharded_cache.cc db/builder.cc db/c.cc db/column_family.cc db/compacted_db_impl.cc db/compaction.cc db/compaction_iterator.cc db/compaction_job.cc db/compaction_picker.cc db/compaction_picker_universal.cc db/convenience.cc db/db_filesnapshot.cc db/db_impl.cc db/db_impl_write.cc db/db_impl_compaction_flush.cc db/db_impl_files.cc db/db_impl_open.cc db/db_impl_debug.cc db/db_impl_experimental.cc db/db_impl_readonly.cc db/db_info_dumper.cc db/db_iter.cc db/dbformat.cc db/event_helpers.cc db/experimental.cc db/external_sst_file_ingestion_job.cc db/file_indexer.cc db/flush_job.cc db/flush_scheduler.cc db/forward_iterator.cc db/internal_stats.cc db/log_reader.cc db/log_writer.cc db/managed_iterator.cc db/memtable.cc db/memtable_list.cc db/merge_helper.cc db/merge_operator.cc db/range_del_aggregator.cc db/repair.cc db/snapshot_impl.cc db/table_cache.cc db/table_properties_collector.cc db/transaction_log_impl.cc db/version_builder.cc db/version_edit.cc db/version_set.cc db/wal_manager.cc db/write_batch.cc db/write_batch_base.cc db/write_controller.cc db/write_thread.cc env/env.cc env/env_chroot.cc env/env_hdfs.cc env/env_posix.cc env/io_posix.cc env/memenv.cc memtable/hash_cuckoo_rep.cc memtable/hash_linklist_rep.cc memtable/hash_skiplist_rep.cc memtable/memtable_allocator.cc memtable/skiplistrep.cc memtable/vectorrep.cc monitoring/histogram.cc monitoring/histogram_windowing.cc monitoring/instrumented_mutex.cc monitoring/iostats_context.cc monitoring/perf_context.cc monitoring/perf_level.cc monitoring/statistics.cc monitoring/thread_status_impl.cc monitoring/thread_status_updater.cc monitoring/thread_status_updater_debug.cc monitoring/thread_status_util.cc monitoring/thread_status_util_debug.cc options/cf_options.cc options/db_options.cc options/options.cc options/options_helper.cc options/options_parser.cc options/options_sanity_check.cc port/port_posix.cc port/stack_trace.cc table/adaptive_table_factory.cc table/block.cc table/block_based_filter_block.cc table/block_based_table_builder.cc table/block_based_table_factory.cc table/block_based_table_reader.cc table/block_builder.cc table/block_prefix_index.cc table/bloom_block.cc table/cuckoo_table_builder.cc table/cuckoo_table_factory.cc table/cuckoo_table_reader.cc table/flush_block_policy.cc table/format.cc table/full_filter_block.cc table/get_context.cc table/index_builder.cc table/iterator.cc table/merging_iterator.cc table/meta_blocks.cc table/partitioned_filter_block.cc table/persistent_cache_helper.cc table/plain_table_builder.cc table/plain_table_factory.cc table/plain_table_index.cc table/plain_table_key_coding.cc table/plain_table_reader.cc table/sst_file_writer.cc table/table_properties.cc table/two_level_iterator.cc tools/dump/db_dump_tool.cc util/arena.cc util/auto_roll_logger.cc util/bloom.cc util/build_version.cc util/coding.cc util/compaction_job_stats_impl.cc util/comparator.cc util/concurrent_arena.cc util/crc32c.cc util/delete_scheduler.cc util/dynamic_bloom.cc util/event_logger.cc util/file_reader_writer.cc util/file_util.cc util/filename.cc util/filter_policy.cc util/hash.cc util/log_buffer.cc util/murmurhash.cc util/random.cc util/rate_limiter.cc util/slice.cc util/sst_file_manager_impl.cc util/status.cc util/status_message.cc util/string_util.cc util/sync_point.cc util/thread_local.cc util/threadpool_imp.cc util/transaction_test_util.cc util/xxhash.cc utilities/backupable/backupable_db.cc utilities/blob_db/blob_db.cc utilities/blob_db/blob_db_impl.cc utilities/blob_db/blob_db_options_impl.cc utilities/blob_db/blob_file.cc utilities/blob_db/blob_log_reader.cc utilities/blob_db/blob_log_writer.cc utilities/blob_db/blob_log_format.cc utilities/checkpoint/checkpoint.cc utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc utilities/convenience/info_log_finder.cc utilities/date_tiered/date_tiered_db_impl.cc utilities/document/document_db.cc utilities/document/json_document.cc utilities/document/json_document_builder.cc utilities/env_mirror.cc utilities/env_timed.cc utilities/geodb/geodb_impl.cc utilities/leveldb_options/leveldb_options.cc utilities/lua/rocks_lua_compaction_filter.cc utilities/memory/memory_util.cc utilities/merge_operators/max.cc utilities/merge_operators/put.cc utilities/merge_operators/string_append/stringappend.cc utilities/merge_operators/string_append/stringappend2.cc utilities/merge_operators/uint64add.cc utilities/option_change_migration/option_change_migration.cc utilities/options/options_util.cc utilities/persistent_cache/block_cache_tier.cc utilities/persistent_cache/block_cache_tier_file.cc utilities/persistent_cache/block_cache_tier_metadata.cc utilities/persistent_cache/persistent_cache_tier.cc utilities/persistent_cache/volatile_tier_impl.cc utilities/redis/redis_lists.cc utilities/simulator_cache/sim_cache.cc utilities/spatialdb/spatial_db.cc utilities/table_properties_collectors/compact_on_deletion_collector.cc utilities/transactions/optimistic_transaction_db_impl.cc utilities/transactions/optimistic_transaction_impl.cc utilities/transactions/transaction_base.cc utilities/transactions/transaction_db_impl.cc utilities/transactions/transaction_db_mutex_impl.cc utilities/transactions/transaction_impl.cc utilities/transactions/transaction_lock_mgr.cc utilities/transactions/transaction_util.cc utilities/ttl/db_ttl_impl.cc utilities/write_batch_with_index/write_batch_with_index.cc utilities/write_batch_with_index/write_batch_with_index_internal.cc \ No newline at end of file From 6b407a3486c654ab507f9ca3616a9f24a6bd0242 Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Tue, 25 Jul 2017 10:10:31 +0100 Subject: [PATCH 2/9] Implement support for DirectIO As per: https://github.com/facebook/rocksdb/commit/972f96b3fbae1a4675043bdf4279c9072ad69645 --- librocksdb-sys/src/lib.rs | 3 ++- src/db_options.rs | 54 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/librocksdb-sys/src/lib.rs b/librocksdb-sys/src/lib.rs index fb1ce6c..6e86a38 100644 --- a/librocksdb-sys/src/lib.rs +++ b/librocksdb-sys/src/lib.rs @@ -633,7 +633,8 @@ extern "C" { pub fn rocksdb_options_set_purge_redundant_kvs_while_flush(opt: *mut rocksdb_options_t, v: c_uchar); - pub fn rocksdb_options_set_allow_os_buffer(opt: *mut rocksdb_options_t, v: c_uchar); + pub fn rocksdb_options_set_use_direct_reads(opt: *mut rocksdb_options_t, v: c_uchar); + pub fn rocksdb_options_set_use_direct_writes(opt: *mut rocksdb_options_t, v: c_uchar); pub fn rocksdb_options_set_allow_mmap_reads(opt: *mut rocksdb_options_t, v: c_uchar); diff --git a/src/db_options.rs b/src/db_options.rs index 5c5ede9..8952d97 100644 --- a/src/db_options.rs +++ b/src/db_options.rs @@ -351,6 +351,54 @@ impl Options { unsafe { ffi::rocksdb_options_set_disable_data_sync(self.inner, disable as c_int) } } + /// Enable direct I/O mode for reading + /// they may or may not improve performance depending on the use case + /// + /// Files will be opened in "direct I/O" mode + /// which means that data read from the disk will not be cached or + /// buffered. The hardware buffer of the devices may however still + /// be used. Memory mapped files are not impacted by these parameters. + /// + /// Default: false + /// + /// # Example + /// + /// ``` + /// use rocksdb::Options; + /// + /// let mut opts = Options::default(); + /// opts.set_use_direct_reads(true); + /// ``` + pub fn set_use_direct_reads(&mut self, enabled: bool) { + unsafe { + ffi::rocksdb_options_set_use_direct_reads(self.inner, enabled as c_uchar); + } + } + + /// Enable direct I/O mode for writing + /// they may or may not improve performance depending on the use case + /// + /// Files will be opened in "direct I/O" mode + /// which means that data written to the disk will not be cached or + /// buffered. The hardware buffer of the devices may however still + /// be used. Memory mapped files are not impacted by these parameters. + /// + /// Default: false + /// + /// # Example + /// + /// ``` + /// use rocksdb::Options; + /// + /// let mut opts = Options::default(); + /// opts.set_use_direct_writes(true); + /// ``` + pub fn set_use_direct_writes(&mut self, enabled: bool) { + unsafe { + ffi::rocksdb_options_set_use_direct_writes(self.inner, enabled as c_uchar); + } + } + /// Hints to the OS that it should not buffer disk I/O. Enabling this /// parameter may improve performance but increases pressure on the /// system cache. @@ -378,10 +426,10 @@ impl Options { /// let mut opts = Options::default(); /// opts.set_allow_os_buffer(false); /// ``` + #[deprecated(since="0.7.0", note="replaced with set_use_direct_reads/set_use_direct_writes methods")] pub fn set_allow_os_buffer(&mut self, is_allow: bool) { - unsafe { - ffi::rocksdb_options_set_allow_os_buffer(self.inner, is_allow as c_uchar); - } + self.set_use_direct_reads(!is_allow); + self.set_use_direct_writes(!is_allow); } /// Sets the number of shards used for table cache. From 1c3333636d6f468fb76c72cb8d883a092d8d78c8 Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Tue, 25 Jul 2017 10:39:02 +0100 Subject: [PATCH 3/9] Implement RawIterator.seek_for_prev --- librocksdb-sys/src/lib.rs | 2 ++ src/db.rs | 12 ++++++++---- tests/test_raw_iterator.rs | 30 +++++++++++++++++++++++++++++- 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/librocksdb-sys/src/lib.rs b/librocksdb-sys/src/lib.rs index 6e86a38..7cd922a 100644 --- a/librocksdb-sys/src/lib.rs +++ b/librocksdb-sys/src/lib.rs @@ -294,6 +294,8 @@ extern "C" { pub fn rocksdb_iter_seek(iterator: *mut rocksdb_iterator_t, k: *const c_char, klen: size_t); + pub fn rocksdb_iter_seek_for_prev(iterator: *mut rocksdb_iterator_t, k: *const c_char, klen: size_t); + pub fn rocksdb_iter_next(iterator: *mut rocksdb_iterator_t); pub fn rocksdb_iter_prev(iterator: *mut rocksdb_iterator_t); diff --git a/src/db.rs b/src/db.rs index 37a370f..3067928 100644 --- a/src/db.rs +++ b/src/db.rs @@ -135,6 +135,14 @@ pub struct Snapshot<'a> { /// iter.next(); /// } /// +/// // Reverse iteration from key +/// // Note, use seek_for_prev when reversing because if this key doesn't exist, +/// // this will make the iterator start from the previous key rather than the next. +/// iter.seek_for_prev(b"my key"); +/// while iter.valid() { +/// println!("Saw {:?} {:?}", iter.key(), iter.value()); +/// iter.prev(); +/// } /// ``` pub struct DBRawIterator { inner: *mut ffi::rocksdb_iterator_t, @@ -307,9 +315,6 @@ impl DBRawIterator { unsafe { ffi::rocksdb_iter_seek(self.inner, key.as_ptr() as *const c_char, key.len() as size_t); } } -/* - SeekForPrev was added in RocksDB 4.13 but not implemented in the C API until RocksDB 5.0 - /// Seeks to the specified key, or the first key that lexicographically precedes it. /// /// Like ``.seek()`` this method will attempt to seek to the specified key. @@ -336,7 +341,6 @@ impl DBRawIterator { pub fn seek_for_prev(&mut self, key: &[u8]) { unsafe { ffi::rocksdb_iter_seek_for_prev(self.inner, key.as_ptr() as *const c_char, key.len() as size_t); } } -*/ /// Seeks to the next key. /// diff --git a/tests/test_raw_iterator.rs b/tests/test_raw_iterator.rs index 7d99859..1591d90 100644 --- a/tests/test_raw_iterator.rs +++ b/tests/test_raw_iterator.rs @@ -98,7 +98,6 @@ pub fn test_seek() { let db = setup_test_db("seek"); db.put(b"k1", b"v1").unwrap(); db.put(b"k2", b"v2").unwrap(); - db.put(b"k3", b"v3").unwrap(); db.put(b"k4", b"v4").unwrap(); let mut iter = db.raw_iterator(); @@ -107,6 +106,13 @@ pub fn test_seek() { assert_eq!(iter.valid(), true); assert_eq!(iter.key(), Some(b"k2".to_vec())); assert_eq!(iter.value(), Some(b"v2".to_vec())); + + // Check it gets the next key when the key doesn't exist + iter.seek(b"k3"); + + assert_eq!(iter.valid(), true); + assert_eq!(iter.key(), Some(b"k4".to_vec())); + assert_eq!(iter.value(), Some(b"v4".to_vec())); } @@ -124,3 +130,25 @@ pub fn test_seek_to_nonexistant() { assert_eq!(iter.key(), Some(b"k3".to_vec())); assert_eq!(iter.value(), Some(b"v3".to_vec())); } + +#[test] +pub fn test_seek_for_prev() { + let db = setup_test_db("seek_for_prev"); + db.put(b"k1", b"v1").unwrap(); + db.put(b"k2", b"v2").unwrap(); + db.put(b"k4", b"v4").unwrap(); + + let mut iter = db.raw_iterator(); + iter.seek(b"k2"); + + assert_eq!(iter.valid(), true); + assert_eq!(iter.key(), Some(b"k2".to_vec())); + assert_eq!(iter.value(), Some(b"v2".to_vec())); + + // Check it gets the previous key when the key doesn't exist + iter.seek_for_prev(b"k3"); + + assert_eq!(iter.valid(), true); + assert_eq!(iter.key(), Some(b"k2".to_vec())); + assert_eq!(iter.value(), Some(b"v2".to_vec())); +} From ccb40173fe1c619ae07a2e1e66d18a637b5cc528 Mon Sep 17 00:00:00 2001 From: Richard Dodd Date: Tue, 25 Jul 2017 12:07:27 +0100 Subject: [PATCH 4/9] Update rocksdb Remove option that no longer exists in rocksdb --- librocksdb-sys/rocksdb | 2 +- librocksdb-sys/rocksdb_lib_sources.txt | 2 +- src/db_options.rs | 33 -------------------------- 3 files changed, 2 insertions(+), 35 deletions(-) diff --git a/librocksdb-sys/rocksdb b/librocksdb-sys/rocksdb index 6e05979..2e98ac0 160000 --- a/librocksdb-sys/rocksdb +++ b/librocksdb-sys/rocksdb @@ -1 +1 @@ -Subproject commit 6e0597951e51e8dd3b28104d31e52962e00c101f +Subproject commit 2e98ac018f8e1d886c75ecc2268e8a52f685bf39 diff --git a/librocksdb-sys/rocksdb_lib_sources.txt b/librocksdb-sys/rocksdb_lib_sources.txt index 97e9791..4890333 100644 --- a/librocksdb-sys/rocksdb_lib_sources.txt +++ b/librocksdb-sys/rocksdb_lib_sources.txt @@ -1 +1 @@ -cache/clock_cache.cc cache/lru_cache.cc cache/sharded_cache.cc db/builder.cc db/c.cc db/column_family.cc db/compacted_db_impl.cc db/compaction.cc db/compaction_iterator.cc db/compaction_job.cc db/compaction_picker.cc db/compaction_picker_universal.cc db/convenience.cc db/db_filesnapshot.cc db/db_impl.cc db/db_impl_write.cc db/db_impl_compaction_flush.cc db/db_impl_files.cc db/db_impl_open.cc db/db_impl_debug.cc db/db_impl_experimental.cc db/db_impl_readonly.cc db/db_info_dumper.cc db/db_iter.cc db/dbformat.cc db/event_helpers.cc db/experimental.cc db/external_sst_file_ingestion_job.cc db/file_indexer.cc db/flush_job.cc db/flush_scheduler.cc db/forward_iterator.cc db/internal_stats.cc db/log_reader.cc db/log_writer.cc db/managed_iterator.cc db/memtable.cc db/memtable_list.cc db/merge_helper.cc db/merge_operator.cc db/range_del_aggregator.cc db/repair.cc db/snapshot_impl.cc db/table_cache.cc db/table_properties_collector.cc db/transaction_log_impl.cc db/version_builder.cc db/version_edit.cc db/version_set.cc db/wal_manager.cc db/write_batch.cc db/write_batch_base.cc db/write_controller.cc db/write_thread.cc env/env.cc env/env_chroot.cc env/env_hdfs.cc env/env_posix.cc env/io_posix.cc env/memenv.cc memtable/hash_cuckoo_rep.cc memtable/hash_linklist_rep.cc memtable/hash_skiplist_rep.cc memtable/memtable_allocator.cc memtable/skiplistrep.cc memtable/vectorrep.cc monitoring/histogram.cc monitoring/histogram_windowing.cc monitoring/instrumented_mutex.cc monitoring/iostats_context.cc monitoring/perf_context.cc monitoring/perf_level.cc monitoring/statistics.cc monitoring/thread_status_impl.cc monitoring/thread_status_updater.cc monitoring/thread_status_updater_debug.cc monitoring/thread_status_util.cc monitoring/thread_status_util_debug.cc options/cf_options.cc options/db_options.cc options/options.cc options/options_helper.cc options/options_parser.cc options/options_sanity_check.cc port/port_posix.cc port/stack_trace.cc table/adaptive_table_factory.cc table/block.cc table/block_based_filter_block.cc table/block_based_table_builder.cc table/block_based_table_factory.cc table/block_based_table_reader.cc table/block_builder.cc table/block_prefix_index.cc table/bloom_block.cc table/cuckoo_table_builder.cc table/cuckoo_table_factory.cc table/cuckoo_table_reader.cc table/flush_block_policy.cc table/format.cc table/full_filter_block.cc table/get_context.cc table/index_builder.cc table/iterator.cc table/merging_iterator.cc table/meta_blocks.cc table/partitioned_filter_block.cc table/persistent_cache_helper.cc table/plain_table_builder.cc table/plain_table_factory.cc table/plain_table_index.cc table/plain_table_key_coding.cc table/plain_table_reader.cc table/sst_file_writer.cc table/table_properties.cc table/two_level_iterator.cc tools/dump/db_dump_tool.cc util/arena.cc util/auto_roll_logger.cc util/bloom.cc util/build_version.cc util/coding.cc util/compaction_job_stats_impl.cc util/comparator.cc util/concurrent_arena.cc util/crc32c.cc util/delete_scheduler.cc util/dynamic_bloom.cc util/event_logger.cc util/file_reader_writer.cc util/file_util.cc util/filename.cc util/filter_policy.cc util/hash.cc util/log_buffer.cc util/murmurhash.cc util/random.cc util/rate_limiter.cc util/slice.cc util/sst_file_manager_impl.cc util/status.cc util/status_message.cc util/string_util.cc util/sync_point.cc util/thread_local.cc util/threadpool_imp.cc util/transaction_test_util.cc util/xxhash.cc utilities/backupable/backupable_db.cc utilities/blob_db/blob_db.cc utilities/blob_db/blob_db_impl.cc utilities/blob_db/blob_db_options_impl.cc utilities/blob_db/blob_file.cc utilities/blob_db/blob_log_reader.cc utilities/blob_db/blob_log_writer.cc utilities/blob_db/blob_log_format.cc utilities/checkpoint/checkpoint.cc utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc utilities/convenience/info_log_finder.cc utilities/date_tiered/date_tiered_db_impl.cc utilities/document/document_db.cc utilities/document/json_document.cc utilities/document/json_document_builder.cc utilities/env_mirror.cc utilities/env_timed.cc utilities/geodb/geodb_impl.cc utilities/leveldb_options/leveldb_options.cc utilities/lua/rocks_lua_compaction_filter.cc utilities/memory/memory_util.cc utilities/merge_operators/max.cc utilities/merge_operators/put.cc utilities/merge_operators/string_append/stringappend.cc utilities/merge_operators/string_append/stringappend2.cc utilities/merge_operators/uint64add.cc utilities/option_change_migration/option_change_migration.cc utilities/options/options_util.cc utilities/persistent_cache/block_cache_tier.cc utilities/persistent_cache/block_cache_tier_file.cc utilities/persistent_cache/block_cache_tier_metadata.cc utilities/persistent_cache/persistent_cache_tier.cc utilities/persistent_cache/volatile_tier_impl.cc utilities/redis/redis_lists.cc utilities/simulator_cache/sim_cache.cc utilities/spatialdb/spatial_db.cc utilities/table_properties_collectors/compact_on_deletion_collector.cc utilities/transactions/optimistic_transaction_db_impl.cc utilities/transactions/optimistic_transaction_impl.cc utilities/transactions/transaction_base.cc utilities/transactions/transaction_db_impl.cc utilities/transactions/transaction_db_mutex_impl.cc utilities/transactions/transaction_impl.cc utilities/transactions/transaction_lock_mgr.cc utilities/transactions/transaction_util.cc utilities/ttl/db_ttl_impl.cc utilities/write_batch_with_index/write_batch_with_index.cc utilities/write_batch_with_index/write_batch_with_index_internal.cc \ No newline at end of file +cache/clock_cache.cc cache/lru_cache.cc cache/sharded_cache.cc db/builder.cc db/c.cc db/column_family.cc db/compacted_db_impl.cc db/compaction.cc db/compaction_iterator.cc db/compaction_job.cc db/compaction_picker.cc db/compaction_picker_universal.cc db/convenience.cc db/db_filesnapshot.cc db/db_impl.cc db/db_impl_write.cc db/db_impl_compaction_flush.cc db/db_impl_files.cc db/db_impl_open.cc db/db_impl_debug.cc db/db_impl_experimental.cc db/db_impl_readonly.cc db/db_info_dumper.cc db/db_iter.cc db/dbformat.cc db/event_helpers.cc db/experimental.cc db/external_sst_file_ingestion_job.cc db/file_indexer.cc db/flush_job.cc db/flush_scheduler.cc db/forward_iterator.cc db/internal_stats.cc db/log_reader.cc db/log_writer.cc db/managed_iterator.cc db/memtable.cc db/memtable_list.cc db/merge_helper.cc db/merge_operator.cc db/range_del_aggregator.cc db/repair.cc db/snapshot_impl.cc db/table_cache.cc db/table_properties_collector.cc db/transaction_log_impl.cc db/version_builder.cc db/version_edit.cc db/version_set.cc db/wal_manager.cc db/write_batch.cc db/write_batch_base.cc db/write_controller.cc db/write_thread.cc env/env.cc env/env_chroot.cc env/env_hdfs.cc env/env_posix.cc env/io_posix.cc env/memenv.cc memtable/hash_cuckoo_rep.cc memtable/hash_linklist_rep.cc memtable/hash_skiplist_rep.cc memtable/memtable_allocator.cc memtable/skiplistrep.cc memtable/vectorrep.cc monitoring/histogram.cc monitoring/histogram_windowing.cc monitoring/instrumented_mutex.cc monitoring/iostats_context.cc monitoring/perf_context.cc monitoring/perf_level.cc monitoring/statistics.cc monitoring/thread_status_impl.cc monitoring/thread_status_updater.cc monitoring/thread_status_updater_debug.cc monitoring/thread_status_util.cc monitoring/thread_status_util_debug.cc options/cf_options.cc options/db_options.cc options/options.cc options/options_helper.cc options/options_parser.cc options/options_sanity_check.cc port/port_posix.cc port/stack_trace.cc table/adaptive_table_factory.cc table/block.cc table/block_based_filter_block.cc table/block_based_table_builder.cc table/block_based_table_factory.cc table/block_based_table_reader.cc table/block_builder.cc table/block_prefix_index.cc table/bloom_block.cc table/cuckoo_table_builder.cc table/cuckoo_table_factory.cc table/cuckoo_table_reader.cc table/flush_block_policy.cc table/format.cc table/full_filter_block.cc table/get_context.cc table/index_builder.cc table/iterator.cc table/merging_iterator.cc table/meta_blocks.cc table/partitioned_filter_block.cc table/persistent_cache_helper.cc table/plain_table_builder.cc table/plain_table_factory.cc table/plain_table_index.cc table/plain_table_key_coding.cc table/plain_table_reader.cc table/sst_file_writer.cc table/table_properties.cc table/two_level_iterator.cc tools/dump/db_dump_tool.cc util/arena.cc util/auto_roll_logger.cc util/bloom.cc util/build_version.cc util/coding.cc util/compaction_job_stats_impl.cc util/comparator.cc util/concurrent_arena.cc util/crc32c.cc util/delete_scheduler.cc util/dynamic_bloom.cc util/event_logger.cc util/file_reader_writer.cc util/file_util.cc util/filename.cc util/filter_policy.cc util/hash.cc util/log_buffer.cc util/murmurhash.cc util/random.cc util/rate_limiter.cc util/slice.cc util/sst_file_manager_impl.cc util/status.cc util/status_message.cc util/string_util.cc util/sync_point.cc util/thread_local.cc util/threadpool_imp.cc util/transaction_test_util.cc util/xxhash.cc utilities/backupable/backupable_db.cc utilities/blob_db/blob_db.cc utilities/blob_db/blob_db_impl.cc utilities/blob_db/blob_db_options_impl.cc utilities/blob_db/blob_file.cc utilities/blob_db/blob_log_reader.cc utilities/blob_db/blob_log_writer.cc utilities/blob_db/blob_log_format.cc utilities/checkpoint/checkpoint_impl.cc utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc utilities/convenience/info_log_finder.cc utilities/date_tiered/date_tiered_db_impl.cc utilities/debug.cc utilities/document/document_db.cc utilities/document/json_document.cc utilities/document/json_document_builder.cc utilities/env_mirror.cc utilities/env_timed.cc utilities/geodb/geodb_impl.cc utilities/leveldb_options/leveldb_options.cc utilities/lua/rocks_lua_compaction_filter.cc utilities/memory/memory_util.cc utilities/merge_operators/max.cc utilities/merge_operators/put.cc utilities/merge_operators/string_append/stringappend.cc utilities/merge_operators/string_append/stringappend2.cc utilities/merge_operators/uint64add.cc utilities/option_change_migration/option_change_migration.cc utilities/options/options_util.cc utilities/persistent_cache/block_cache_tier.cc utilities/persistent_cache/block_cache_tier_file.cc utilities/persistent_cache/block_cache_tier_metadata.cc utilities/persistent_cache/persistent_cache_tier.cc utilities/persistent_cache/volatile_tier_impl.cc utilities/redis/redis_lists.cc utilities/simulator_cache/sim_cache.cc utilities/spatialdb/spatial_db.cc utilities/table_properties_collectors/compact_on_deletion_collector.cc utilities/transactions/optimistic_transaction_db_impl.cc utilities/transactions/optimistic_transaction_impl.cc utilities/transactions/transaction_base.cc utilities/transactions/transaction_db_impl.cc utilities/transactions/transaction_db_mutex_impl.cc utilities/transactions/transaction_impl.cc utilities/transactions/transaction_lock_mgr.cc utilities/transactions/transaction_util.cc utilities/ttl/db_ttl_impl.cc utilities/write_batch_with_index/write_batch_with_index.cc utilities/write_batch_with_index/write_batch_with_index_internal.cc \ No newline at end of file diff --git a/src/db_options.rs b/src/db_options.rs index 8952d97..515af19 100644 --- a/src/db_options.rs +++ b/src/db_options.rs @@ -399,39 +399,6 @@ impl Options { } } - /// Hints to the OS that it should not buffer disk I/O. Enabling this - /// parameter may improve performance but increases pressure on the - /// system cache. - /// - /// The exact behavior of this parameter is platform dependent. - /// - /// On POSIX systems, after RocksDB reads data from disk it will - /// mark the pages as "unneeded". The operating system may - or may not - /// - evict these pages from memory, reducing pressure on the system - /// cache. If the disk block is requested again this can result in - /// additional disk I/O. - /// - /// On WINDOWS systems, files will be opened in "unbuffered I/O" mode - /// which means that data read from the disk will not be cached or - /// bufferized. The hardware buffer of the devices may however still - /// be used. Memory mapped files are not impacted by this parameter. - /// - /// Default: true - /// - /// # Example - /// - /// ``` - /// use rocksdb::Options; - /// - /// let mut opts = Options::default(); - /// opts.set_allow_os_buffer(false); - /// ``` - #[deprecated(since="0.7.0", note="replaced with set_use_direct_reads/set_use_direct_writes methods")] - pub fn set_allow_os_buffer(&mut self, is_allow: bool) { - self.set_use_direct_reads(!is_allow); - self.set_use_direct_writes(!is_allow); - } - /// Sets the number of shards used for table cache. /// /// Default: `6` From 2a7ab0f805883a08a54aa229ac15cd446d5fe10a Mon Sep 17 00:00:00 2001 From: Richard Dodd Date: Tue, 25 Jul 2017 12:36:31 +0100 Subject: [PATCH 5/9] Include change from 5.4.5 Change "use_direct_writes" to "use_direct_io_for_flush_and_compaction" --- librocksdb-sys/src/lib.rs | 3 +- src/db_options.rs | 64 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/librocksdb-sys/src/lib.rs b/librocksdb-sys/src/lib.rs index 7cd922a..f2320ac 100644 --- a/librocksdb-sys/src/lib.rs +++ b/librocksdb-sys/src/lib.rs @@ -636,7 +636,8 @@ extern "C" { v: c_uchar); pub fn rocksdb_options_set_use_direct_reads(opt: *mut rocksdb_options_t, v: c_uchar); - pub fn rocksdb_options_set_use_direct_writes(opt: *mut rocksdb_options_t, v: c_uchar); + pub fn rocksdb_options_set_use_direct_io_for_flush_and_compaction(opt: *mut rocksdb_options_t, + v: c_uchar); pub fn rocksdb_options_set_allow_mmap_reads(opt: *mut rocksdb_options_t, v: c_uchar); diff --git a/src/db_options.rs b/src/db_options.rs index 515af19..11a0b1a 100644 --- a/src/db_options.rs +++ b/src/db_options.rs @@ -388,17 +388,79 @@ impl Options { /// # Example /// /// ``` + /// #[allow(deprecated)] /// use rocksdb::Options; /// /// let mut opts = Options::default(); /// opts.set_use_direct_writes(true); /// ``` + #[deprecated(note = "replaced by `Options::set_use_direct_io_for_flush_and_compaction`")] pub fn set_use_direct_writes(&mut self, enabled: bool) { unsafe { - ffi::rocksdb_options_set_use_direct_writes(self.inner, enabled as c_uchar); + ffi::rocksdb_options_set_use_direct_io_for_flush_and_compaction(self.inner, + enabled as c_uchar); } } + /// Enable direct I/O mode for writing + /// they may or may not improve performance depending on the use case + /// + /// Files will be opened in "direct I/O" mode + /// which means that data written to the disk will not be cached or + /// buffered. The hardware buffer of the devices may however still + /// be used. Memory mapped files are not impacted by these parameters. + /// + /// Default: false + /// + /// # Example + /// + /// ``` + /// use rocksdb::Options; + /// + /// let mut opts = Options::default(); + /// opts.set_use_direct_io_for_flush_and_compaction(true); + /// ``` + pub fn set_use_direct_io_for_flush_and_compaction(&mut self, enabled: bool) { + unsafe { + ffi::rocksdb_options_set_use_direct_io_for_flush_and_compaction(self.inner, + enabled as c_uchar); + } + } + + /// Hints to the OS that it should not buffer disk I/O. Enabling this + /// parameter may improve performance but increases pressure on the + /// system cache. + /// + /// The exact behavior of this parameter is platform dependent. + /// + /// On POSIX systems, after RocksDB reads data from disk it will + /// mark the pages as "unneeded". The operating system may - or may not + /// - evict these pages from memory, reducing pressure on the system + /// cache. If the disk block is requested again this can result in + /// additional disk I/O. + /// + /// On WINDOWS systems, files will be opened in "unbuffered I/O" mode + /// which means that data read from the disk will not be cached or + /// bufferized. The hardware buffer of the devices may however still + /// be used. Memory mapped files are not impacted by this parameter. + /// + /// Default: true + /// + /// # Example + /// + /// ``` + /// #[allow(deprecated)] + /// use rocksdb::Options; + /// + /// let mut opts = Options::default(); + /// opts.set_allow_os_buffer(false); + /// ``` + #[deprecated(since="0.7.0", note="replaced with set_use_direct_reads/set_use_direct_io_for_flush_and_compaction methods")] + pub fn set_allow_os_buffer(&mut self, is_allow: bool) { + self.set_use_direct_reads(!is_allow); + self.set_use_direct_io_for_flush_and_compaction(!is_allow); + } + /// Sets the number of shards used for table cache. /// /// Default: `6` From 90f5e0103eb311218d593ec560c251bf7d919923 Mon Sep 17 00:00:00 2001 From: Richard Dodd Date: Tue, 25 Jul 2017 12:44:50 +0100 Subject: [PATCH 6/9] Remove unnecessary function --- src/db_options.rs | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/src/db_options.rs b/src/db_options.rs index 11a0b1a..2b180c4 100644 --- a/src/db_options.rs +++ b/src/db_options.rs @@ -375,33 +375,6 @@ impl Options { } } - /// Enable direct I/O mode for writing - /// they may or may not improve performance depending on the use case - /// - /// Files will be opened in "direct I/O" mode - /// which means that data written to the disk will not be cached or - /// buffered. The hardware buffer of the devices may however still - /// be used. Memory mapped files are not impacted by these parameters. - /// - /// Default: false - /// - /// # Example - /// - /// ``` - /// #[allow(deprecated)] - /// use rocksdb::Options; - /// - /// let mut opts = Options::default(); - /// opts.set_use_direct_writes(true); - /// ``` - #[deprecated(note = "replaced by `Options::set_use_direct_io_for_flush_and_compaction`")] - pub fn set_use_direct_writes(&mut self, enabled: bool) { - unsafe { - ffi::rocksdb_options_set_use_direct_io_for_flush_and_compaction(self.inner, - enabled as c_uchar); - } - } - /// Enable direct I/O mode for writing /// they may or may not improve performance depending on the use case /// From 620091d31b7886d4c8760509242b64ba6a228ab6 Mon Sep 17 00:00:00 2001 From: Richard Dodd Date: Tue, 25 Jul 2017 12:46:54 +0100 Subject: [PATCH 7/9] Make docs match new function --- src/db_options.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/db_options.rs b/src/db_options.rs index 2b180c4..cab0cac 100644 --- a/src/db_options.rs +++ b/src/db_options.rs @@ -375,13 +375,13 @@ impl Options { } } - /// Enable direct I/O mode for writing - /// they may or may not improve performance depending on the use case + /// Enable direct I/O mode for flush and compaction /// /// Files will be opened in "direct I/O" mode /// which means that data written to the disk will not be cached or /// buffered. The hardware buffer of the devices may however still /// be used. Memory mapped files are not impacted by these parameters. + /// they may or may not improve performance depending on the use case /// /// Default: false /// From 514e8904d7a08df2d896cc9fb752db2f67f16bde Mon Sep 17 00:00:00 2001 From: Richard Dodd Date: Tue, 25 Jul 2017 15:24:26 +0100 Subject: [PATCH 8/9] Add to options - set_allow_concurrent_memtable_write And fix test --- librocksdb-sys/src/lib.rs | 3 +++ librocksdb-sys/tests/ffi.rs | 1 + src/db_options.rs | 22 ++++++++++++++++++++++ 3 files changed, 26 insertions(+) diff --git a/librocksdb-sys/src/lib.rs b/librocksdb-sys/src/lib.rs index f2320ac..a5d9268 100644 --- a/librocksdb-sys/src/lib.rs +++ b/librocksdb-sys/src/lib.rs @@ -659,6 +659,9 @@ extern "C" { pub fn rocksdb_options_set_bytes_per_sync(opt: *mut rocksdb_options_t, v: uint64_t); + pub fn rocksdb_options_set_allow_concurrent_memtable_write(opt: *mut rocksdb_options_t, + v: c_uchar); + pub fn rocksdb_options_set_verify_checksums_in_compaction(opt: *mut rocksdb_options_t, v: c_uchar); diff --git a/librocksdb-sys/tests/ffi.rs b/librocksdb-sys/tests/ffi.rs index 15e8d9b..d230322 100644 --- a/librocksdb-sys/tests/ffi.rs +++ b/librocksdb-sys/tests/ffi.rs @@ -1030,6 +1030,7 @@ fn ffi() { rocksdb_slicetransform_create_fixed_prefix(3)); rocksdb_options_set_hash_skip_list_rep(options, 5000, 4, 4); rocksdb_options_set_plain_table_factory(options, 4, 10, 0.75, 16); + rocksdb_options_set_allow_concurrent_memtable_write(options, 0); db = rocksdb_open(options, dbname, &mut err); CheckNoError!(err); diff --git a/src/db_options.rs b/src/db_options.rs index cab0cac..eea8e97 100644 --- a/src/db_options.rs +++ b/src/db_options.rs @@ -347,6 +347,28 @@ impl Options { } } + /// If true, allow multi-writers to update mem tables in parallel. + /// Only some memtable_factory-s support concurrent writes; currently it + /// is implemented only for SkipListFactory. Concurrent memtable writes + /// are not compatible with inplace_update_support or filter_deletes. + /// It is strongly recommended to set enable_write_thread_adaptive_yield + /// if you are going to use this feature. + /// + /// Default: true + /// + /// # Example + /// + /// ``` + /// use rocksdb::Options; + /// + /// let mut opts = Options::default(); + /// opts.set_allow_concurrent_memtable_write(false); + /// ``` + pub fn set_allow_concurrent_memtable_write(&mut self, allow: bool) { + unsafe { ffi::rocksdb_options_set_allow_concurrent_memtable_write(self.inner, + allow as c_uchar) } + } + pub fn set_disable_data_sync(&mut self, disable: bool) { unsafe { ffi::rocksdb_options_set_disable_data_sync(self.inner, disable as c_int) } } From b8ecb424605937217ca408570c52f99cdc09ba34 Mon Sep 17 00:00:00 2001 From: Richard Dodd Date: Wed, 26 Jul 2017 09:29:59 +0100 Subject: [PATCH 9/9] Nit --- librocksdb-sys/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/librocksdb-sys/src/lib.rs b/librocksdb-sys/src/lib.rs index a5d9268..8e9b631 100644 --- a/librocksdb-sys/src/lib.rs +++ b/librocksdb-sys/src/lib.rs @@ -636,6 +636,7 @@ extern "C" { v: c_uchar); pub fn rocksdb_options_set_use_direct_reads(opt: *mut rocksdb_options_t, v: c_uchar); + pub fn rocksdb_options_set_use_direct_io_for_flush_and_compaction(opt: *mut rocksdb_options_t, v: c_uchar);