diff --git a/librocksdb-sys/rocksdb b/librocksdb-sys/rocksdb index f201a44..2e98ac0 160000 --- a/librocksdb-sys/rocksdb +++ b/librocksdb-sys/rocksdb @@ -1 +1 @@ -Subproject commit f201a44b4102308b840b15d9b89122af787476f1 +Subproject commit 2e98ac018f8e1d886c75ecc2268e8a52f685bf39 diff --git a/librocksdb-sys/rocksdb_lib_sources.txt b/librocksdb-sys/rocksdb_lib_sources.txt index 209deb5..4890333 100644 --- a/librocksdb-sys/rocksdb_lib_sources.txt +++ b/librocksdb-sys/rocksdb_lib_sources.txt @@ -1 +1 @@ -db/auto_roll_logger.cc db/builder.cc db/c.cc db/column_family.cc db/compacted_db_impl.cc db/compaction.cc db/compaction_iterator.cc db/compaction_job.cc db/compaction_picker.cc db/convenience.cc db/range_del_aggregator.cc db/db_filesnapshot.cc db/dbformat.cc db/db_impl.cc db/db_impl_debug.cc db/db_impl_readonly.cc db/db_impl_experimental.cc db/db_info_dumper.cc db/db_iter.cc db/external_sst_file_ingestion_job.cc db/experimental.cc db/event_helpers.cc db/file_indexer.cc db/filename.cc db/flush_job.cc db/flush_scheduler.cc db/forward_iterator.cc db/internal_stats.cc db/log_reader.cc db/log_writer.cc db/managed_iterator.cc db/memtable_allocator.cc db/memtable.cc db/memtable_list.cc db/merge_helper.cc db/merge_operator.cc db/repair.cc db/snapshot_impl.cc db/table_cache.cc db/table_properties_collector.cc db/transaction_log_impl.cc db/version_builder.cc db/version_edit.cc db/version_set.cc db/wal_manager.cc db/write_batch.cc db/write_batch_base.cc db/write_controller.cc db/write_thread.cc db/xfunc_test_points.cc memtable/hash_cuckoo_rep.cc memtable/hash_linklist_rep.cc memtable/hash_skiplist_rep.cc memtable/skiplistrep.cc memtable/vectorrep.cc port/stack_trace.cc port/port_posix.cc table/adaptive_table_factory.cc table/block_based_filter_block.cc table/block_based_table_builder.cc table/block_based_table_factory.cc table/block_based_table_reader.cc table/block_builder.cc table/block.cc table/block_prefix_index.cc table/bloom_block.cc table/cuckoo_table_builder.cc table/cuckoo_table_factory.cc table/cuckoo_table_reader.cc table/flush_block_policy.cc table/format.cc table/full_filter_block.cc table/get_context.cc table/iterator.cc table/merger.cc table/meta_blocks.cc table/sst_file_writer.cc table/plain_table_builder.cc table/plain_table_factory.cc table/plain_table_index.cc table/plain_table_key_coding.cc table/plain_table_reader.cc table/persistent_cache_helper.cc table/table_properties.cc table/two_level_iterator.cc tools/dump/db_dump_tool.cc util/arena.cc util/bloom.cc util/build_version.cc util/cf_options.cc util/clock_cache.cc util/coding.cc util/comparator.cc util/compaction_job_stats_impl.cc util/concurrent_arena.cc util/crc32c.cc util/db_options.cc util/delete_scheduler.cc util/dynamic_bloom.cc util/env.cc util/env_chroot.cc util/env_hdfs.cc util/env_posix.cc util/event_logger.cc util/file_util.cc util/file_reader_writer.cc util/filter_policy.cc util/hash.cc util/histogram.cc util/histogram_windowing.cc util/instrumented_mutex.cc util/iostats_context.cc util/io_posix.cc util/log_buffer.cc util/logging.cc util/lru_cache.cc util/memenv.cc util/murmurhash.cc util/options.cc util/options_helper.cc util/options_parser.cc util/options_sanity_check.cc util/perf_context.cc util/perf_level.cc util/random.cc util/rate_limiter.cc util/sharded_cache.cc util/slice.cc util/sst_file_manager_impl.cc util/statistics.cc util/status.cc util/status_message.cc util/string_util.cc util/sync_point.cc util/thread_local.cc util/thread_status_impl.cc util/thread_status_updater.cc util/thread_status_updater_debug.cc util/thread_status_util.cc util/thread_status_util_debug.cc util/threadpool_imp.cc util/transaction_test_util.cc util/xfunc.cc util/xxhash.cc utilities/backupable/backupable_db.cc utilities/blob_db/blob_db.cc utilities/convenience/info_log_finder.cc utilities/checkpoint/checkpoint.cc utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc utilities/document/document_db.cc utilities/document/json_document_builder.cc utilities/document/json_document.cc utilities/env_mirror.cc utilities/env_registry.cc utilities/flashcache/flashcache.cc utilities/geodb/geodb_impl.cc utilities/leveldb_options/leveldb_options.cc utilities/memory/memory_util.cc utilities/merge_operators/put.cc utilities/merge_operators/max.cc utilities/merge_operators/string_append/stringappend2.cc utilities/merge_operators/string_append/stringappend.cc utilities/merge_operators/uint64add.cc utilities/option_change_migration/option_change_migration.cc utilities/options/options_util.cc utilities/persistent_cache/persistent_cache_tier.cc utilities/persistent_cache/volatile_tier_impl.cc utilities/persistent_cache/block_cache_tier_file.cc utilities/persistent_cache/block_cache_tier_metadata.cc utilities/persistent_cache/block_cache_tier.cc utilities/redis/redis_lists.cc utilities/simulator_cache/sim_cache.cc utilities/spatialdb/spatial_db.cc utilities/table_properties_collectors/compact_on_deletion_collector.cc utilities/transactions/optimistic_transaction_impl.cc utilities/transactions/optimistic_transaction_db_impl.cc utilities/transactions/transaction_base.cc utilities/transactions/transaction_db_impl.cc utilities/transactions/transaction_db_mutex_impl.cc utilities/transactions/transaction_lock_mgr.cc utilities/transactions/transaction_impl.cc utilities/transactions/transaction_util.cc utilities/ttl/db_ttl_impl.cc utilities/date_tiered/date_tiered_db_impl.cc utilities/write_batch_with_index/write_batch_with_index.cc utilities/write_batch_with_index/write_batch_with_index_internal.cc \ No newline at end of file +cache/clock_cache.cc cache/lru_cache.cc cache/sharded_cache.cc db/builder.cc db/c.cc db/column_family.cc db/compacted_db_impl.cc db/compaction.cc db/compaction_iterator.cc db/compaction_job.cc db/compaction_picker.cc db/compaction_picker_universal.cc db/convenience.cc db/db_filesnapshot.cc db/db_impl.cc db/db_impl_write.cc db/db_impl_compaction_flush.cc db/db_impl_files.cc db/db_impl_open.cc db/db_impl_debug.cc db/db_impl_experimental.cc db/db_impl_readonly.cc db/db_info_dumper.cc db/db_iter.cc db/dbformat.cc db/event_helpers.cc db/experimental.cc db/external_sst_file_ingestion_job.cc db/file_indexer.cc db/flush_job.cc db/flush_scheduler.cc db/forward_iterator.cc db/internal_stats.cc db/log_reader.cc db/log_writer.cc db/managed_iterator.cc db/memtable.cc db/memtable_list.cc db/merge_helper.cc db/merge_operator.cc db/range_del_aggregator.cc db/repair.cc db/snapshot_impl.cc db/table_cache.cc db/table_properties_collector.cc db/transaction_log_impl.cc db/version_builder.cc db/version_edit.cc db/version_set.cc db/wal_manager.cc db/write_batch.cc db/write_batch_base.cc db/write_controller.cc db/write_thread.cc env/env.cc env/env_chroot.cc env/env_hdfs.cc env/env_posix.cc env/io_posix.cc env/memenv.cc memtable/hash_cuckoo_rep.cc memtable/hash_linklist_rep.cc memtable/hash_skiplist_rep.cc memtable/memtable_allocator.cc memtable/skiplistrep.cc memtable/vectorrep.cc monitoring/histogram.cc monitoring/histogram_windowing.cc monitoring/instrumented_mutex.cc monitoring/iostats_context.cc monitoring/perf_context.cc monitoring/perf_level.cc monitoring/statistics.cc monitoring/thread_status_impl.cc monitoring/thread_status_updater.cc monitoring/thread_status_updater_debug.cc monitoring/thread_status_util.cc monitoring/thread_status_util_debug.cc options/cf_options.cc options/db_options.cc options/options.cc options/options_helper.cc options/options_parser.cc options/options_sanity_check.cc port/port_posix.cc port/stack_trace.cc table/adaptive_table_factory.cc table/block.cc table/block_based_filter_block.cc table/block_based_table_builder.cc table/block_based_table_factory.cc table/block_based_table_reader.cc table/block_builder.cc table/block_prefix_index.cc table/bloom_block.cc table/cuckoo_table_builder.cc table/cuckoo_table_factory.cc table/cuckoo_table_reader.cc table/flush_block_policy.cc table/format.cc table/full_filter_block.cc table/get_context.cc table/index_builder.cc table/iterator.cc table/merging_iterator.cc table/meta_blocks.cc table/partitioned_filter_block.cc table/persistent_cache_helper.cc table/plain_table_builder.cc table/plain_table_factory.cc table/plain_table_index.cc table/plain_table_key_coding.cc table/plain_table_reader.cc table/sst_file_writer.cc table/table_properties.cc table/two_level_iterator.cc tools/dump/db_dump_tool.cc util/arena.cc util/auto_roll_logger.cc util/bloom.cc util/build_version.cc util/coding.cc util/compaction_job_stats_impl.cc util/comparator.cc util/concurrent_arena.cc util/crc32c.cc util/delete_scheduler.cc util/dynamic_bloom.cc util/event_logger.cc util/file_reader_writer.cc util/file_util.cc util/filename.cc util/filter_policy.cc util/hash.cc util/log_buffer.cc util/murmurhash.cc util/random.cc util/rate_limiter.cc util/slice.cc util/sst_file_manager_impl.cc util/status.cc util/status_message.cc util/string_util.cc util/sync_point.cc util/thread_local.cc util/threadpool_imp.cc util/transaction_test_util.cc util/xxhash.cc utilities/backupable/backupable_db.cc utilities/blob_db/blob_db.cc utilities/blob_db/blob_db_impl.cc utilities/blob_db/blob_db_options_impl.cc utilities/blob_db/blob_file.cc utilities/blob_db/blob_log_reader.cc utilities/blob_db/blob_log_writer.cc utilities/blob_db/blob_log_format.cc utilities/checkpoint/checkpoint_impl.cc utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc utilities/convenience/info_log_finder.cc utilities/date_tiered/date_tiered_db_impl.cc utilities/debug.cc utilities/document/document_db.cc utilities/document/json_document.cc utilities/document/json_document_builder.cc utilities/env_mirror.cc utilities/env_timed.cc utilities/geodb/geodb_impl.cc utilities/leveldb_options/leveldb_options.cc utilities/lua/rocks_lua_compaction_filter.cc utilities/memory/memory_util.cc utilities/merge_operators/max.cc utilities/merge_operators/put.cc utilities/merge_operators/string_append/stringappend.cc utilities/merge_operators/string_append/stringappend2.cc utilities/merge_operators/uint64add.cc utilities/option_change_migration/option_change_migration.cc utilities/options/options_util.cc utilities/persistent_cache/block_cache_tier.cc utilities/persistent_cache/block_cache_tier_file.cc utilities/persistent_cache/block_cache_tier_metadata.cc utilities/persistent_cache/persistent_cache_tier.cc utilities/persistent_cache/volatile_tier_impl.cc utilities/redis/redis_lists.cc utilities/simulator_cache/sim_cache.cc utilities/spatialdb/spatial_db.cc utilities/table_properties_collectors/compact_on_deletion_collector.cc utilities/transactions/optimistic_transaction_db_impl.cc utilities/transactions/optimistic_transaction_impl.cc utilities/transactions/transaction_base.cc utilities/transactions/transaction_db_impl.cc utilities/transactions/transaction_db_mutex_impl.cc utilities/transactions/transaction_impl.cc utilities/transactions/transaction_lock_mgr.cc utilities/transactions/transaction_util.cc utilities/ttl/db_ttl_impl.cc utilities/write_batch_with_index/write_batch_with_index.cc utilities/write_batch_with_index/write_batch_with_index_internal.cc \ No newline at end of file diff --git a/librocksdb-sys/src/lib.rs b/librocksdb-sys/src/lib.rs index fb1ce6c..8e9b631 100644 --- a/librocksdb-sys/src/lib.rs +++ b/librocksdb-sys/src/lib.rs @@ -294,6 +294,8 @@ extern "C" { pub fn rocksdb_iter_seek(iterator: *mut rocksdb_iterator_t, k: *const c_char, klen: size_t); + pub fn rocksdb_iter_seek_for_prev(iterator: *mut rocksdb_iterator_t, k: *const c_char, klen: size_t); + pub fn rocksdb_iter_next(iterator: *mut rocksdb_iterator_t); pub fn rocksdb_iter_prev(iterator: *mut rocksdb_iterator_t); @@ -633,7 +635,10 @@ extern "C" { pub fn rocksdb_options_set_purge_redundant_kvs_while_flush(opt: *mut rocksdb_options_t, v: c_uchar); - pub fn rocksdb_options_set_allow_os_buffer(opt: *mut rocksdb_options_t, v: c_uchar); + pub fn rocksdb_options_set_use_direct_reads(opt: *mut rocksdb_options_t, v: c_uchar); + + pub fn rocksdb_options_set_use_direct_io_for_flush_and_compaction(opt: *mut rocksdb_options_t, + v: c_uchar); pub fn rocksdb_options_set_allow_mmap_reads(opt: *mut rocksdb_options_t, v: c_uchar); @@ -655,6 +660,9 @@ extern "C" { pub fn rocksdb_options_set_bytes_per_sync(opt: *mut rocksdb_options_t, v: uint64_t); + pub fn rocksdb_options_set_allow_concurrent_memtable_write(opt: *mut rocksdb_options_t, + v: c_uchar); + pub fn rocksdb_options_set_verify_checksums_in_compaction(opt: *mut rocksdb_options_t, v: c_uchar); diff --git a/librocksdb-sys/tests/ffi.rs b/librocksdb-sys/tests/ffi.rs index 15e8d9b..d230322 100644 --- a/librocksdb-sys/tests/ffi.rs +++ b/librocksdb-sys/tests/ffi.rs @@ -1030,6 +1030,7 @@ fn ffi() { rocksdb_slicetransform_create_fixed_prefix(3)); rocksdb_options_set_hash_skip_list_rep(options, 5000, 4, 4); rocksdb_options_set_plain_table_factory(options, 4, 10, 0.75, 16); + rocksdb_options_set_allow_concurrent_memtable_write(options, 0); db = rocksdb_open(options, dbname, &mut err); CheckNoError!(err); diff --git a/src/db.rs b/src/db.rs index 37a370f..3067928 100644 --- a/src/db.rs +++ b/src/db.rs @@ -135,6 +135,14 @@ pub struct Snapshot<'a> { /// iter.next(); /// } /// +/// // Reverse iteration from key +/// // Note, use seek_for_prev when reversing because if this key doesn't exist, +/// // this will make the iterator start from the previous key rather than the next. +/// iter.seek_for_prev(b"my key"); +/// while iter.valid() { +/// println!("Saw {:?} {:?}", iter.key(), iter.value()); +/// iter.prev(); +/// } /// ``` pub struct DBRawIterator { inner: *mut ffi::rocksdb_iterator_t, @@ -307,9 +315,6 @@ impl DBRawIterator { unsafe { ffi::rocksdb_iter_seek(self.inner, key.as_ptr() as *const c_char, key.len() as size_t); } } -/* - SeekForPrev was added in RocksDB 4.13 but not implemented in the C API until RocksDB 5.0 - /// Seeks to the specified key, or the first key that lexicographically precedes it. /// /// Like ``.seek()`` this method will attempt to seek to the specified key. @@ -336,7 +341,6 @@ impl DBRawIterator { pub fn seek_for_prev(&mut self, key: &[u8]) { unsafe { ffi::rocksdb_iter_seek_for_prev(self.inner, key.as_ptr() as *const c_char, key.len() as size_t); } } -*/ /// Seeks to the next key. /// diff --git a/src/db_options.rs b/src/db_options.rs index 5c5ede9..eea8e97 100644 --- a/src/db_options.rs +++ b/src/db_options.rs @@ -347,10 +347,81 @@ impl Options { } } + /// If true, allow multi-writers to update mem tables in parallel. + /// Only some memtable_factory-s support concurrent writes; currently it + /// is implemented only for SkipListFactory. Concurrent memtable writes + /// are not compatible with inplace_update_support or filter_deletes. + /// It is strongly recommended to set enable_write_thread_adaptive_yield + /// if you are going to use this feature. + /// + /// Default: true + /// + /// # Example + /// + /// ``` + /// use rocksdb::Options; + /// + /// let mut opts = Options::default(); + /// opts.set_allow_concurrent_memtable_write(false); + /// ``` + pub fn set_allow_concurrent_memtable_write(&mut self, allow: bool) { + unsafe { ffi::rocksdb_options_set_allow_concurrent_memtable_write(self.inner, + allow as c_uchar) } + } + pub fn set_disable_data_sync(&mut self, disable: bool) { unsafe { ffi::rocksdb_options_set_disable_data_sync(self.inner, disable as c_int) } } + /// Enable direct I/O mode for reading + /// they may or may not improve performance depending on the use case + /// + /// Files will be opened in "direct I/O" mode + /// which means that data read from the disk will not be cached or + /// buffered. The hardware buffer of the devices may however still + /// be used. Memory mapped files are not impacted by these parameters. + /// + /// Default: false + /// + /// # Example + /// + /// ``` + /// use rocksdb::Options; + /// + /// let mut opts = Options::default(); + /// opts.set_use_direct_reads(true); + /// ``` + pub fn set_use_direct_reads(&mut self, enabled: bool) { + unsafe { + ffi::rocksdb_options_set_use_direct_reads(self.inner, enabled as c_uchar); + } + } + + /// Enable direct I/O mode for flush and compaction + /// + /// Files will be opened in "direct I/O" mode + /// which means that data written to the disk will not be cached or + /// buffered. The hardware buffer of the devices may however still + /// be used. Memory mapped files are not impacted by these parameters. + /// they may or may not improve performance depending on the use case + /// + /// Default: false + /// + /// # Example + /// + /// ``` + /// use rocksdb::Options; + /// + /// let mut opts = Options::default(); + /// opts.set_use_direct_io_for_flush_and_compaction(true); + /// ``` + pub fn set_use_direct_io_for_flush_and_compaction(&mut self, enabled: bool) { + unsafe { + ffi::rocksdb_options_set_use_direct_io_for_flush_and_compaction(self.inner, + enabled as c_uchar); + } + } + /// Hints to the OS that it should not buffer disk I/O. Enabling this /// parameter may improve performance but increases pressure on the /// system cache. @@ -373,15 +444,16 @@ impl Options { /// # Example /// /// ``` + /// #[allow(deprecated)] /// use rocksdb::Options; /// /// let mut opts = Options::default(); /// opts.set_allow_os_buffer(false); /// ``` + #[deprecated(since="0.7.0", note="replaced with set_use_direct_reads/set_use_direct_io_for_flush_and_compaction methods")] pub fn set_allow_os_buffer(&mut self, is_allow: bool) { - unsafe { - ffi::rocksdb_options_set_allow_os_buffer(self.inner, is_allow as c_uchar); - } + self.set_use_direct_reads(!is_allow); + self.set_use_direct_io_for_flush_and_compaction(!is_allow); } /// Sets the number of shards used for table cache. diff --git a/tests/test_raw_iterator.rs b/tests/test_raw_iterator.rs index 7d99859..1591d90 100644 --- a/tests/test_raw_iterator.rs +++ b/tests/test_raw_iterator.rs @@ -98,7 +98,6 @@ pub fn test_seek() { let db = setup_test_db("seek"); db.put(b"k1", b"v1").unwrap(); db.put(b"k2", b"v2").unwrap(); - db.put(b"k3", b"v3").unwrap(); db.put(b"k4", b"v4").unwrap(); let mut iter = db.raw_iterator(); @@ -107,6 +106,13 @@ pub fn test_seek() { assert_eq!(iter.valid(), true); assert_eq!(iter.key(), Some(b"k2".to_vec())); assert_eq!(iter.value(), Some(b"v2".to_vec())); + + // Check it gets the next key when the key doesn't exist + iter.seek(b"k3"); + + assert_eq!(iter.valid(), true); + assert_eq!(iter.key(), Some(b"k4".to_vec())); + assert_eq!(iter.value(), Some(b"v4".to_vec())); } @@ -124,3 +130,25 @@ pub fn test_seek_to_nonexistant() { assert_eq!(iter.key(), Some(b"k3".to_vec())); assert_eq!(iter.value(), Some(b"v3".to_vec())); } + +#[test] +pub fn test_seek_for_prev() { + let db = setup_test_db("seek_for_prev"); + db.put(b"k1", b"v1").unwrap(); + db.put(b"k2", b"v2").unwrap(); + db.put(b"k4", b"v4").unwrap(); + + let mut iter = db.raw_iterator(); + iter.seek(b"k2"); + + assert_eq!(iter.valid(), true); + assert_eq!(iter.key(), Some(b"k2".to_vec())); + assert_eq!(iter.value(), Some(b"v2".to_vec())); + + // Check it gets the previous key when the key doesn't exist + iter.seek_for_prev(b"k3"); + + assert_eq!(iter.valid(), true); + assert_eq!(iter.key(), Some(b"k2".to_vec())); + assert_eq!(iter.value(), Some(b"v2".to_vec())); +}