Merge branch 'master' into set_zstd_max_train_bytes

master
Oleksandr Anyshchenko 4 years ago committed by GitHub
commit 02e663fc7a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 1
      CHANGELOG.md
  2. 4
      Cargo.toml
  3. 4
      librocksdb-sys/Cargo.toml
  4. 4
      librocksdb-sys/build.rs
  5. 66
      librocksdb-sys/build_version.cc
  6. 15
      librocksdb-sys/build_version.h
  7. 2
      librocksdb-sys/rocksdb
  8. 52
      librocksdb-sys/rocksdb_lib_sources.txt
  9. 49
      src/db.rs
  10. 100
      src/db_options.rs
  11. 6
      src/lib.rs
  12. 60
      tests/test_db.rs

@ -2,6 +2,7 @@
## [Unreleased] ## [Unreleased]
* Bump `librocksdb-sys` up to 6.19.3 (olegnn)
* Add `DB::key_may_exist_cf_opt` method (stanislav-tkach) * Add `DB::key_may_exist_cf_opt` method (stanislav-tkach)
* Add `Options::set_zstd_max_train_bytes` method (stanislav-tkach) * Add `Options::set_zstd_max_train_bytes` method (stanislav-tkach)

@ -1,7 +1,7 @@
[package] [package]
name = "rocksdb" name = "rocksdb"
description = "Rust wrapper for Facebook's RocksDB embeddable database" description = "Rust wrapper for Facebook's RocksDB embeddable database"
version = "0.16.0" version = "0.17.0"
edition = "2018" edition = "2018"
authors = ["Tyler Neely <t@jujit.su>", "David Greenberg <dsg123456789@gmail.com>"] authors = ["Tyler Neely <t@jujit.su>", "David Greenberg <dsg123456789@gmail.com>"]
repository = "https://github.com/rust-rocksdb/rust-rocksdb" repository = "https://github.com/rust-rocksdb/rust-rocksdb"
@ -28,7 +28,7 @@ multi-threaded-cf = []
[dependencies] [dependencies]
libc = "0.2" libc = "0.2"
librocksdb-sys = { path = "librocksdb-sys", version = "6.17.3" } librocksdb-sys = { path = "librocksdb-sys", version = "6.19.3" }
[dev-dependencies] [dev-dependencies]
trybuild = "1.0" trybuild = "1.0"

@ -1,6 +1,6 @@
[package] [package]
name = "librocksdb-sys" name = "librocksdb-sys"
version = "6.17.3" version = "6.19.3"
edition = "2018" edition = "2018"
authors = ["Karl Hobley <karlhobley10@gmail.com>", "Arkadiy Paronyan <arkadiy@ethcore.io>"] authors = ["Karl Hobley <karlhobley10@gmail.com>", "Arkadiy Paronyan <arkadiy@ethcore.io>"]
license = "MIT/Apache-2.0/BSD-3-Clause" license = "MIT/Apache-2.0/BSD-3-Clause"
@ -29,5 +29,5 @@ uuid = { version = "0.8", features = ["v4"] }
[build-dependencies] [build-dependencies]
cc = { version = "1.0", features = ["parallel"] } cc = { version = "1.0", features = ["parallel"] }
bindgen = { version = "0.57.0", default-features = false, features = ["runtime"] } bindgen = { version = "0.58.1", default-features = false, features = ["runtime"] }
glob = "0.3" glob = "0.3"

@ -37,7 +37,7 @@ fn bindgen_rocksdb() {
let bindings = bindgen::Builder::default() let bindings = bindgen::Builder::default()
.header(rocksdb_include_dir() + "/rocksdb/c.h") .header(rocksdb_include_dir() + "/rocksdb/c.h")
.derive_debug(false) .derive_debug(false)
.blacklist_type("max_align_t") // https://github.com/rust-lang-nursery/rust-bindgen/issues/550 .blocklist_type("max_align_t") // https://github.com/rust-lang-nursery/rust-bindgen/issues/550
.ctypes_prefix("libc") .ctypes_prefix("libc")
.size_t_is_usize(true) .size_t_is_usize(true)
.generate() .generate()
@ -96,7 +96,7 @@ fn build_rocksdb() {
lib_sources = lib_sources lib_sources = lib_sources
.iter() .iter()
.cloned() .cloned()
.filter(|file| *file != "util/build_version.cc") .filter(|&file| file != "util/build_version.cc")
.collect::<Vec<&'static str>>(); .collect::<Vec<&'static str>>();
if target.contains("x86_64") { if target.contains("x86_64") {

@ -1,4 +1,62 @@
#include "build_version.h" // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
const char* rocksdb_build_git_sha = "rocksdb_build_git_sha:@09c7e96eac1ab983f97ce9e0406730b8014b3398@";
const char* rocksdb_build_git_date = "rocksdb_build_git_date:@2021/02/18 09:36:23@"; #include <memory>
const char* rocksdb_build_compile_date = __DATE__;
#include "rocksdb/version.h"
#include "util/string_util.h"
// The build script may replace these values with real values based
// on whether or not GIT is available and the platform settings
static const std::string rocksdb_build_git_sha = "rocksdb_build_git_sha:645c445978bb22c8c7791f90a0d9931928a58045";
static const std::string rocksdb_build_git_tag = "rocksdb_build_git_tag:master";
static const std::string rocksdb_build_date = "rocksdb_build_date:2021-04-19 19:57:42";
namespace ROCKSDB_NAMESPACE {
static void AddProperty(std::unordered_map<std::string, std::string> *props, const std::string& name) {
size_t colon = name.find(":");
if (colon != std::string::npos && colon > 0 && colon < name.length() - 1) {
// If we found a "@:", then this property was a build-time substitution that failed. Skip it
size_t at = name.find("@", colon);
if (at != colon + 1) {
// Everything before the colon is the name, after is the value
(*props)[name.substr(0, colon)] = name.substr(colon + 1);
}
}
}
static std::unordered_map<std::string, std::string>* LoadPropertiesSet() {
auto * properties = new std::unordered_map<std::string, std::string>();
AddProperty(properties, rocksdb_build_git_sha);
AddProperty(properties, rocksdb_build_git_tag);
AddProperty(properties, rocksdb_build_date);
return properties;
}
const std::unordered_map<std::string, std::string>& GetRocksBuildProperties() {
static std::unique_ptr<std::unordered_map<std::string, std::string>> props(LoadPropertiesSet());
return *props;
}
std::string GetRocksVersionAsString(bool with_patch) {
std::string version = ToString(ROCKSDB_MAJOR) + "." + ToString(ROCKSDB_MINOR);
if (with_patch) {
return version + "." + ToString(ROCKSDB_PATCH);
} else {
return version;
}
}
std::string GetRocksBuildInfoAsString(const std::string& program, bool verbose) {
std::string info = program + " (RocksDB) " + GetRocksVersionAsString(true);
if (verbose) {
for (const auto& it : GetRocksBuildProperties()) {
info.append("\n ");
info.append(it.first);
info.append(": ");
info.append(it.second);
}
}
return info;
}
} // namespace ROCKSDB_NAMESPACE

@ -1,15 +0,0 @@
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
//
#pragma once
#if !defined(IOS_CROSS_COMPILE)
// if we compile with Xcode, we don't run build_detect_vesion, so we don't
// generate these variables
// this variable tells us about the git revision
extern const char* rocksdb_build_git_sha;
// Date on which the code was compiled:
extern const char* rocksdb_build_compile_date;
#endif

@ -1 +1 @@
Subproject commit 09c7e96eac1ab983f97ce9e0406730b8014b3398 Subproject commit 645c445978bb22c8c7791f90a0d9931928a58045

@ -76,20 +76,22 @@ db/write_batch.cc
db/write_batch_base.cc db/write_batch_base.cc
db/write_controller.cc db/write_controller.cc
db/write_thread.cc db/write_thread.cc
env/composite_env.cc
env/env.cc env/env.cc
env/env_chroot.cc env/env_chroot.cc
env/env_encryption.cc env/env_encryption.cc
env/env_hdfs.cc env/env_hdfs.cc
env/env_posix.cc env/env_posix.cc
env/file_system.cc env/file_system.cc
env/file_system_tracer.cc
env/fs_posix.cc env/fs_posix.cc
env/file_system_tracer.cc
env/io_posix.cc env/io_posix.cc
env/mock_env.cc env/mock_env.cc
file/delete_scheduler.cc file/delete_scheduler.cc
file/file_prefetch_buffer.cc file/file_prefetch_buffer.cc
file/file_util.cc file/file_util.cc
file/filename.cc file/filename.cc
file/line_file_reader.cc
file/random_access_file_reader.cc file/random_access_file_reader.cc
file/read_write_util.cc file/read_write_util.cc
file/readahead_raf.cc file/readahead_raf.cc
@ -131,6 +133,12 @@ options/options.cc
options/options_helper.cc options/options_helper.cc
options/options_parser.cc options/options_parser.cc
port/port_posix.cc port/port_posix.cc
port/win/env_default.cc
port/win/env_win.cc
port/win/io_win.cc
port/win/port_win.cc
port/win/win_logger.cc
port/win/win_thread.cc
port/stack_trace.cc port/stack_trace.cc
table/adaptive/adaptive_table_factory.cc table/adaptive/adaptive_table_factory.cc
table/block_based/binary_search_index_reader.cc table/block_based/binary_search_index_reader.cc
@ -182,15 +190,8 @@ table/table_properties.cc
table/two_level_iterator.cc table/two_level_iterator.cc
test_util/sync_point.cc test_util/sync_point.cc
test_util/sync_point_impl.cc test_util/sync_point_impl.cc
test_util/testutil.cc
test_util/transaction_test_util.cc test_util/transaction_test_util.cc
tools/block_cache_analyzer/block_cache_trace_analyzer.cc
tools/dump/db_dump_tool.cc tools/dump/db_dump_tool.cc
tools/io_tracer_parser_tool.cc
tools/ldb_cmd.cc
tools/ldb_tool.cc
tools/sst_dump_tool.cc
tools/trace_analyzer_tool.cc
trace_replay/trace_replay.cc trace_replay/trace_replay.cc
trace_replay/block_cache_tracer.cc trace_replay/block_cache_tracer.cc
trace_replay/io_tracer.cc trace_replay/io_tracer.cc
@ -206,6 +207,7 @@ util/hash.cc
util/murmurhash.cc util/murmurhash.cc
util/random.cc util/random.cc
util/rate_limiter.cc util/rate_limiter.cc
util/ribbon_config.cc
util/slice.cc util/slice.cc
util/file_checksum_helper.cc util/file_checksum_helper.cc
util/status.cc util/status.cc
@ -218,7 +220,6 @@ utilities/blob_db/blob_compaction_filter.cc
utilities/blob_db/blob_db.cc utilities/blob_db/blob_db.cc
utilities/blob_db/blob_db_impl.cc utilities/blob_db/blob_db_impl.cc
utilities/blob_db/blob_db_impl_filesnapshot.cc utilities/blob_db/blob_db_impl_filesnapshot.cc
utilities/blob_db/blob_dump_tool.cc
utilities/blob_db/blob_file.cc utilities/blob_db/blob_file.cc
utilities/cassandra/cassandra_compaction_filter.cc utilities/cassandra/cassandra_compaction_filter.cc
utilities/cassandra/format.cc utilities/cassandra/format.cc
@ -233,13 +234,13 @@ utilities/fault_injection_env.cc
utilities/fault_injection_fs.cc utilities/fault_injection_fs.cc
utilities/leveldb_options/leveldb_options.cc utilities/leveldb_options/leveldb_options.cc
utilities/memory/memory_util.cc utilities/memory/memory_util.cc
utilities/merge_operators/bytesxor.cc
utilities/merge_operators/max.cc utilities/merge_operators/max.cc
utilities/merge_operators/put.cc utilities/merge_operators/put.cc
utilities/merge_operators/sortlist.cc utilities/merge_operators/sortlist.cc
utilities/merge_operators/string_append/stringappend.cc utilities/merge_operators/string_append/stringappend.cc
utilities/merge_operators/string_append/stringappend2.cc utilities/merge_operators/string_append/stringappend2.cc
utilities/merge_operators/uint64add.cc utilities/merge_operators/uint64add.cc
utilities/merge_operators/bytesxor.cc
utilities/object_registry.cc utilities/object_registry.cc
utilities/option_change_migration/option_change_migration.cc utilities/option_change_migration/option_change_migration.cc
utilities/options/options_util.cc utilities/options/options_util.cc
@ -255,22 +256,8 @@ utilities/trace/file_trace_reader_writer.cc
utilities/transactions/lock/lock_manager.cc utilities/transactions/lock/lock_manager.cc
utilities/transactions/lock/point/point_lock_tracker.cc utilities/transactions/lock/point/point_lock_tracker.cc
utilities/transactions/lock/point/point_lock_manager.cc utilities/transactions/lock/point/point_lock_manager.cc
utilities/transactions/lock/range/range_tree/lib/locktree/concurrent_tree.cc
utilities/transactions/lock/range/range_tree/lib/locktree/keyrange.cc
utilities/transactions/lock/range/range_tree/lib/locktree/lock_request.cc
utilities/transactions/lock/range/range_tree/lib/locktree/locktree.cc
utilities/transactions/lock/range/range_tree/lib/locktree/manager.cc
utilities/transactions/lock/range/range_tree/lib/locktree/range_buffer.cc
utilities/transactions/lock/range/range_tree/lib/locktree/treenode.cc
utilities/transactions/lock/range/range_tree/lib/locktree/txnid_set.cc
utilities/transactions/lock/range/range_tree/lib/locktree/wfg.cc
utilities/transactions/lock/range/range_tree/lib/standalone_port.cc
utilities/transactions/lock/range/range_tree/lib/util/dbt.cc
utilities/transactions/lock/range/range_tree/lib/util/memarena.cc
utilities/transactions/lock/range/range_tree/range_tree_lock_manager.cc
utilities/transactions/lock/range/range_tree/range_tree_lock_tracker.cc
utilities/transactions/optimistic_transaction_db_impl.cc
utilities/transactions/optimistic_transaction.cc utilities/transactions/optimistic_transaction.cc
utilities/transactions/optimistic_transaction_db_impl.cc
utilities/transactions/pessimistic_transaction.cc utilities/transactions/pessimistic_transaction.cc
utilities/transactions/pessimistic_transaction_db.cc utilities/transactions/pessimistic_transaction_db.cc
utilities/transactions/snapshot_checker.cc utilities/transactions/snapshot_checker.cc
@ -284,3 +271,18 @@ utilities/transactions/write_unprepared_txn_db.cc
utilities/ttl/db_ttl_impl.cc utilities/ttl/db_ttl_impl.cc
utilities/write_batch_with_index/write_batch_with_index.cc utilities/write_batch_with_index/write_batch_with_index.cc
utilities/write_batch_with_index/write_batch_with_index_internal.cc utilities/write_batch_with_index/write_batch_with_index_internal.cc
util/crc32c_arm64.cc
utilities/transactions/lock/range/range_tree/lib/locktree/concurrent_tree.cc
utilities/transactions/lock/range/range_tree/lib/locktree/keyrange.cc
utilities/transactions/lock/range/range_tree/lib/locktree/lock_request.cc
utilities/transactions/lock/range/range_tree/lib/locktree/locktree.cc
utilities/transactions/lock/range/range_tree/lib/locktree/manager.cc
utilities/transactions/lock/range/range_tree/lib/locktree/range_buffer.cc
utilities/transactions/lock/range/range_tree/lib/locktree/treenode.cc
utilities/transactions/lock/range/range_tree/lib/locktree/txnid_set.cc
utilities/transactions/lock/range/range_tree/lib/locktree/wfg.cc
utilities/transactions/lock/range/range_tree/lib/standalone_port.cc
utilities/transactions/lock/range/range_tree/lib/util/dbt.cc
utilities/transactions/lock/range/range_tree/lib/util/memarena.cc
utilities/transactions/lock/range/range_tree/range_tree_lock_manager.cc
utilities/transactions/lock/range/range_tree/range_tree_lock_tracker.cc

@ -747,7 +747,7 @@ impl<T: ThreadMode> DBWithThreadMode<T> {
} }
/// Return the values associated with the given keys. /// Return the values associated with the given keys.
pub fn multi_get<K, I>(&self, keys: I) -> Result<Vec<Vec<u8>>, Error> pub fn multi_get<K, I>(&self, keys: I) -> Vec<Result<Option<Vec<u8>>, Error>>
where where
K: AsRef<[u8]>, K: AsRef<[u8]>,
I: IntoIterator<Item = K>, I: IntoIterator<Item = K>,
@ -760,7 +760,7 @@ impl<T: ThreadMode> DBWithThreadMode<T> {
&self, &self,
keys: I, keys: I,
readopts: &ReadOptions, readopts: &ReadOptions,
) -> Result<Vec<Vec<u8>>, Error> ) -> Vec<Result<Option<Vec<u8>>, Error>>
where where
K: AsRef<[u8]>, K: AsRef<[u8]>,
I: IntoIterator<Item = K>, I: IntoIterator<Item = K>,
@ -773,8 +773,9 @@ impl<T: ThreadMode> DBWithThreadMode<T> {
let mut values = vec![ptr::null_mut(); keys.len()]; let mut values = vec![ptr::null_mut(); keys.len()];
let mut values_sizes = vec![0_usize; keys.len()]; let mut values_sizes = vec![0_usize; keys.len()];
let mut errors = vec![ptr::null_mut(); keys.len()];
unsafe { unsafe {
ffi_try!(ffi::rocksdb_multi_get( ffi::rocksdb_multi_get(
self.inner, self.inner,
readopts.inner, readopts.inner,
ptr_keys.len(), ptr_keys.len(),
@ -782,14 +783,15 @@ impl<T: ThreadMode> DBWithThreadMode<T> {
keys_sizes.as_ptr(), keys_sizes.as_ptr(),
values.as_mut_ptr(), values.as_mut_ptr(),
values_sizes.as_mut_ptr(), values_sizes.as_mut_ptr(),
)); errors.as_mut_ptr(),
);
} }
Ok(convert_values(values, values_sizes)) convert_values(values, values_sizes, errors)
} }
/// Return the values associated with the given keys and column families. /// Return the values associated with the given keys and column families.
pub fn multi_get_cf<K, I, W>(&self, keys: I) -> Result<Vec<Vec<u8>>, Error> pub fn multi_get_cf<K, I, W>(&self, keys: I) -> Vec<Result<Option<Vec<u8>>, Error>>
where where
K: AsRef<[u8]>, K: AsRef<[u8]>,
I: IntoIterator<Item = (W, K)>, I: IntoIterator<Item = (W, K)>,
@ -803,7 +805,7 @@ impl<T: ThreadMode> DBWithThreadMode<T> {
&self, &self,
keys: I, keys: I,
readopts: &ReadOptions, readopts: &ReadOptions,
) -> Result<Vec<Vec<u8>>, Error> ) -> Vec<Result<Option<Vec<u8>>, Error>>
where where
K: AsRef<[u8]>, K: AsRef<[u8]>,
I: IntoIterator<Item = (W, K)>, I: IntoIterator<Item = (W, K)>,
@ -828,8 +830,9 @@ impl<T: ThreadMode> DBWithThreadMode<T> {
let mut values = vec![ptr::null_mut(); boxed_keys.len()]; let mut values = vec![ptr::null_mut(); boxed_keys.len()];
let mut values_sizes = vec![0_usize; boxed_keys.len()]; let mut values_sizes = vec![0_usize; boxed_keys.len()];
let mut errors = vec![ptr::null_mut(); boxed_keys.len()];
unsafe { unsafe {
ffi_try!(ffi::rocksdb_multi_get_cf( ffi::rocksdb_multi_get_cf(
self.inner, self.inner,
readopts.inner, readopts.inner,
ptr_cfs.as_ptr(), ptr_cfs.as_ptr(),
@ -838,20 +841,21 @@ impl<T: ThreadMode> DBWithThreadMode<T> {
keys_sizes.as_ptr(), keys_sizes.as_ptr(),
values.as_mut_ptr(), values.as_mut_ptr(),
values_sizes.as_mut_ptr(), values_sizes.as_mut_ptr(),
)); errors.as_mut_ptr(),
);
} }
Ok(convert_values(values, values_sizes)) convert_values(values, values_sizes, errors)
} }
/// Returns `false` if the given key definitely doesn't exist in the database, otherwise returns /// Returns `false` if the given key definitely doesn't exist in the database, otherwise returns
/// `false`. This function uses default `ReadOptions`. /// `true`. This function uses default `ReadOptions`.
pub fn key_may_exist<K: AsRef<[u8]>>(&self, key: K) -> bool { pub fn key_may_exist<K: AsRef<[u8]>>(&self, key: K) -> bool {
self.key_may_exist_opt(key, &ReadOptions::default()) self.key_may_exist_opt(key, &ReadOptions::default())
} }
/// Returns `false` if the given key definitely doesn't exist in the database, otherwise returns /// Returns `false` if the given key definitely doesn't exist in the database, otherwise returns
/// `false`. /// `true`.
pub fn key_may_exist_opt<K: AsRef<[u8]>>(&self, key: K, readopts: &ReadOptions) -> bool { pub fn key_may_exist_opt<K: AsRef<[u8]>>(&self, key: K, readopts: &ReadOptions) -> bool {
let key = key.as_ref(); let key = key.as_ref();
unsafe { unsafe {
@ -870,13 +874,13 @@ impl<T: ThreadMode> DBWithThreadMode<T> {
} }
/// Returns `false` if the given key definitely doesn't exist in the specified column family, /// Returns `false` if the given key definitely doesn't exist in the specified column family,
/// otherwise returns `false`. This function uses default `ReadOptions`. /// otherwise returns `true`. This function uses default `ReadOptions`.
pub fn key_may_exist_cf<K: AsRef<[u8]>>(&self, cf: impl AsColumnFamilyRef, key: K) -> bool { pub fn key_may_exist_cf<K: AsRef<[u8]>>(&self, cf: impl AsColumnFamilyRef, key: K) -> bool {
self.key_may_exist_cf_opt(cf, key, &ReadOptions::default()) self.key_may_exist_cf_opt(cf, key, &ReadOptions::default())
} }
/// Returns `false` if the given key definitely doesn't exist in the specified column family, /// Returns `false` if the given key definitely doesn't exist in the specified column family,
/// otherwise returns `false`. /// otherwise returns `true`.
pub fn key_may_exist_cf_opt<K: AsRef<[u8]>>( pub fn key_may_exist_cf_opt<K: AsRef<[u8]>>(
&self, &self,
cf: impl AsColumnFamilyRef, cf: impl AsColumnFamilyRef,
@ -1835,16 +1839,25 @@ fn convert_options(opts: &[(&str, &str)]) -> Result<Vec<(CString, CString)>, Err
.collect() .collect()
} }
fn convert_values(values: Vec<*mut c_char>, values_sizes: Vec<usize>) -> Vec<Vec<u8>> { fn convert_values(
values: Vec<*mut c_char>,
values_sizes: Vec<usize>,
errors: Vec<*mut c_char>,
) -> Vec<Result<Option<Vec<u8>>, Error>> {
values values
.into_iter() .into_iter()
.zip(values_sizes.into_iter()) .zip(values_sizes.into_iter())
.map(|(v, s)| { .zip(errors.into_iter())
let value = unsafe { slice::from_raw_parts(v as *const u8, s) }.into(); .map(|((v, s), e)| {
if e.is_null() {
let value = unsafe { crate::ffi_util::raw_data(v, s) };
unsafe { unsafe {
ffi::rocksdb_free(v as *mut c_void); ffi::rocksdb_free(v as *mut c_void);
} }
value Ok(value)
} else {
Err(Error::new(crate::ffi_util::error_message(e)))
}
}) })
.collect() .collect()
} }

@ -342,6 +342,11 @@ pub struct ReadOptions {
iterate_lower_bound: Option<Vec<u8>>, iterate_lower_bound: Option<Vec<u8>>,
} }
/// Configuration of cuckoo-based storage.
pub struct CuckooTableOptions {
pub(crate) inner: *mut ffi::rocksdb_cuckoo_table_options_t,
}
/// For configuring external files ingestion. /// For configuring external files ingestion.
/// ///
/// # Examples /// # Examples
@ -376,6 +381,7 @@ pub struct IngestExternalFileOptions {
unsafe impl Send for Options {} unsafe impl Send for Options {}
unsafe impl Send for WriteOptions {} unsafe impl Send for WriteOptions {}
unsafe impl Send for BlockBasedOptions {} unsafe impl Send for BlockBasedOptions {}
unsafe impl Send for CuckooTableOptions {}
unsafe impl Send for ReadOptions {} unsafe impl Send for ReadOptions {}
unsafe impl Send for IngestExternalFileOptions {} unsafe impl Send for IngestExternalFileOptions {}
@ -384,6 +390,7 @@ unsafe impl Send for IngestExternalFileOptions {}
unsafe impl Sync for Options {} unsafe impl Sync for Options {}
unsafe impl Sync for WriteOptions {} unsafe impl Sync for WriteOptions {}
unsafe impl Sync for BlockBasedOptions {} unsafe impl Sync for BlockBasedOptions {}
unsafe impl Sync for CuckooTableOptions {}
unsafe impl Sync for ReadOptions {} unsafe impl Sync for ReadOptions {}
unsafe impl Sync for IngestExternalFileOptions {} unsafe impl Sync for IngestExternalFileOptions {}
@ -416,6 +423,14 @@ impl Drop for BlockBasedOptions {
} }
} }
impl Drop for CuckooTableOptions {
fn drop(&mut self) {
unsafe {
ffi::rocksdb_cuckoo_options_destroy(self.inner);
}
}
}
impl Drop for FlushOptions { impl Drop for FlushOptions {
fn drop(&mut self) { fn drop(&mut self) {
unsafe { unsafe {
@ -695,6 +710,65 @@ impl Default for BlockBasedOptions {
} }
} }
impl CuckooTableOptions {
/// Determines the utilization of hash tables. Smaller values
/// result in larger hash tables with fewer collisions.
/// Default: 0.9
pub fn set_hash_ratio(&mut self, ratio: f64) {
unsafe { ffi::rocksdb_cuckoo_options_set_hash_ratio(self.inner, ratio) }
}
/// A property used by builder to determine the depth to go to
/// to search for a path to displace elements in case of
/// collision. See Builder.MakeSpaceForKey method. Higher
/// values result in more efficient hash tables with fewer
/// lookups but take more time to build.
/// Default: 100
pub fn set_max_search_depth(&mut self, depth: u32) {
unsafe { ffi::rocksdb_cuckoo_options_set_max_search_depth(self.inner, depth) }
}
/// In case of collision while inserting, the builder
/// attempts to insert in the next cuckoo_block_size
/// locations before skipping over to the next Cuckoo hash
/// function. This makes lookups more cache friendly in case
/// of collisions.
/// Default: 5
pub fn set_cuckoo_block_size(&mut self, size: u32) {
unsafe { ffi::rocksdb_cuckoo_options_set_cuckoo_block_size(self.inner, size) }
}
/// If this option is enabled, user key is treated as uint64_t and its value
/// is used as hash value directly. This option changes builder's behavior.
/// Reader ignore this option and behave according to what specified in
/// table property.
/// Default: false
pub fn set_identity_as_first_hash(&mut self, flag: bool) {
let v = flag as u8;
unsafe { ffi::rocksdb_cuckoo_options_set_identity_as_first_hash(self.inner, v) }
}
/// If this option is set to true, module is used during hash calculation.
/// This often yields better space efficiency at the cost of performance.
/// If this option is set to false, # of entries in table is constrained to
/// be power of two, and bit and is used to calculate hash, which is faster in general.
/// Default: true
pub fn set_use_module_hash(&mut self, flag: bool) {
let v = flag as u8;
unsafe { ffi::rocksdb_cuckoo_options_set_use_module_hash(self.inner, v) }
}
}
impl Default for CuckooTableOptions {
fn default() -> CuckooTableOptions {
let opts = unsafe { ffi::rocksdb_cuckoo_options_create() };
if opts.is_null() {
panic!("Could not create RocksDB cuckoo options");
}
CuckooTableOptions { inner: opts }
}
}
impl Options { impl Options {
/// By default, RocksDB uses only one background thread for flush and /// By default, RocksDB uses only one background thread for flush and
/// compaction. Calling this function will set it up such that total of /// compaction. Calling this function will set it up such that total of
@ -2176,6 +2250,32 @@ impl Options {
self.outlive.block_based = Some(factory.outlive.clone()); self.outlive.block_based = Some(factory.outlive.clone());
} }
/// Sets the table factory to a CuckooTableFactory (the default table
/// factory is a block-based table factory that provides a default
/// implementation of TableBuilder and TableReader with default
/// BlockBasedTableOptions).
/// See official [wiki](https://github.com/facebook/rocksdb/wiki/CuckooTable-Format) for more information on this table format.
/// # Examples
///
/// ```
/// use rocksdb::{Options, CuckooTableOptions};
///
/// let mut opts = Options::default();
/// let mut factory_opts = CuckooTableOptions::default();
/// factory_opts.set_hash_ratio(0.8);
/// factory_opts.set_max_search_depth(20);
/// factory_opts.set_cuckoo_block_size(10);
/// factory_opts.set_identity_as_first_hash(true);
/// factory_opts.set_use_module_hash(false);
///
/// opts.set_cuckoo_table_factory(&factory_opts);
/// ```
pub fn set_cuckoo_table_factory(&mut self, factory: &CuckooTableOptions) {
unsafe {
ffi::rocksdb_options_set_cuckoo_table_factory(self.inner, factory.inner);
}
}
// This is a factory that provides TableFactory objects. // This is a factory that provides TableFactory objects.
// Default: a block-based table factory that provides a default // Default: a block-based table factory that provides a default
// implementation of TableBuilder and TableReader with default // implementation of TableBuilder and TableReader with default

@ -109,9 +109,9 @@ pub use crate::{
}, },
db_options::{ db_options::{
BlockBasedIndexType, BlockBasedOptions, BottommostLevelCompaction, Cache, CompactOptions, BlockBasedIndexType, BlockBasedOptions, BottommostLevelCompaction, Cache, CompactOptions,
DBCompactionStyle, DBCompressionType, DBPath, DBRecoveryMode, DataBlockIndexType, Env, CuckooTableOptions, DBCompactionStyle, DBCompressionType, DBPath, DBRecoveryMode,
FifoCompactOptions, FlushOptions, IngestExternalFileOptions, MemtableFactory, Options, DataBlockIndexType, Env, FifoCompactOptions, FlushOptions, IngestExternalFileOptions,
PlainTableFactoryOptions, ReadOptions, UniversalCompactOptions, MemtableFactory, Options, PlainTableFactoryOptions, ReadOptions, UniversalCompactOptions,
UniversalCompactionStopStyle, WriteOptions, UniversalCompactionStopStyle, WriteOptions,
}, },
db_pinnable_slice::DBPinnableSlice, db_pinnable_slice::DBPinnableSlice,

@ -20,10 +20,10 @@ use pretty_assertions::assert_eq;
use rocksdb::{ use rocksdb::{
perf::get_memory_usage_stats, BlockBasedOptions, BottommostLevelCompaction, Cache, perf::get_memory_usage_stats, BlockBasedOptions, BottommostLevelCompaction, Cache,
CompactOptions, DBCompactionStyle, DBWithThreadMode, Env, Error, FifoCompactOptions, CompactOptions, CuckooTableOptions, DBCompactionStyle, DBWithThreadMode, Env, Error,
IteratorMode, MultiThreaded, Options, PerfContext, PerfMetric, ReadOptions, SingleThreaded, FifoCompactOptions, IteratorMode, MultiThreaded, Options, PerfContext, PerfMetric, ReadOptions,
SliceTransform, Snapshot, UniversalCompactOptions, UniversalCompactionStopStyle, WriteBatch, SingleThreaded, SliceTransform, Snapshot, UniversalCompactOptions,
DB, UniversalCompactionStopStyle, WriteBatch, DB,
}; };
use util::DBPath; use util::DBPath;
@ -928,13 +928,17 @@ fn multi_get() {
db.put(b"k1", b"v1").unwrap(); db.put(b"k1", b"v1").unwrap();
db.put(b"k2", b"v2").unwrap(); db.put(b"k2", b"v2").unwrap();
let _ = db.multi_get(&[b"k0"; 40]);
let values = db let values = db
.multi_get(&[b"k0", b"k1", b"k2"]) .multi_get(&[b"k0", b"k1", b"k2"])
.expect("multi_get failed"); .into_iter()
.map(Result::unwrap)
.collect::<Vec<_>>();
assert_eq!(3, values.len()); assert_eq!(3, values.len());
assert!(values[0].is_empty()); assert_eq!(values[0], None);
assert_eq!(values[1], b"v1"); assert_eq!(values[1], Some(b"v1".to_vec()));
assert_eq!(values[2], b"v2"); assert_eq!(values[2], Some(b"v2".to_vec()));
} }
} }
@ -958,11 +962,13 @@ fn multi_get_cf() {
let values = db let values = db
.multi_get_cf(vec![(cf0, b"k0"), (cf1, b"k1"), (cf2, b"k2")]) .multi_get_cf(vec![(cf0, b"k0"), (cf1, b"k1"), (cf2, b"k2")])
.expect("multi_get failed"); .into_iter()
.map(Result::unwrap)
.collect::<Vec<_>>();
assert_eq!(3, values.len()); assert_eq!(3, values.len());
assert!(values[0].is_empty()); assert_eq!(values[0], None);
assert_eq!(values[1], b"v1"); assert_eq!(values[1], Some(b"v1".to_vec()));
assert_eq!(values[2], b"v2"); assert_eq!(values[2], Some(b"v2".to_vec()));
} }
} }
@ -1004,3 +1010,33 @@ fn test_snapshot_outlive_db() {
let t = trybuild::TestCases::new(); let t = trybuild::TestCases::new();
t.compile_fail("tests/fail/snapshot_outlive_db.rs"); t.compile_fail("tests/fail/snapshot_outlive_db.rs");
} }
#[test]
fn cuckoo() {
let path = DBPath::new("_rust_rocksdb_cuckoo");
{
let mut opts = Options::default();
let mut factory_opts = CuckooTableOptions::default();
factory_opts.set_hash_ratio(0.8);
factory_opts.set_max_search_depth(20);
factory_opts.set_cuckoo_block_size(10);
factory_opts.set_identity_as_first_hash(true);
factory_opts.set_use_module_hash(false);
opts.set_cuckoo_table_factory(&factory_opts);
opts.create_if_missing(true);
let db = DB::open(&opts, &path).unwrap();
db.put(b"k1", b"v1").unwrap();
db.put(b"k2", b"v2").unwrap();
let r: Result<Option<Vec<u8>>, Error> = db.get(b"k1");
assert_eq!(r.unwrap().unwrap(), b"v1");
let r: Result<Option<Vec<u8>>, Error> = db.get(b"k2");
assert_eq!(r.unwrap().unwrap(), b"v2");
assert!(db.delete(b"k1").is_ok());
assert!(db.get(b"k1").unwrap().is_none());
}
}

Loading…
Cancel
Save