Merge branch 'master' into drop_after_tests

master
Tyler Neely 6 years ago committed by GitHub
commit 2b8dc16fc8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 2
      Cargo.toml
  2. 4
      librocksdb-sys/Cargo.toml
  3. 3
      librocksdb-sys/build.rs
  4. 2
      librocksdb-sys/rocksdb
  5. 2
      librocksdb-sys/rocksdb_lib_sources.txt
  6. 19
      src/db.rs
  7. 99
      src/db_options.rs
  8. 8
      src/lib.rs
  9. 54
      tests/test_iterator.rs

@ -19,4 +19,4 @@ valgrind = []
[dependencies]
libc = "0.2"
librocksdb-sys = { path = "librocksdb-sys", version = "5.11.3" }
librocksdb-sys = { path = "librocksdb-sys", version = "5.14.2" }

@ -1,6 +1,6 @@
[package]
name = "librocksdb-sys"
version = "5.11.3"
version = "5.14.2"
authors = ["Karl Hobley <karlhobley10@gmail.com>", "Arkadiy Paronyan <arkadiy@ethcore.io>"]
license = "MIT/Apache-2.0/BSD-3-Clause"
description = "Native bindings to librocksdb"
@ -24,4 +24,4 @@ const-cstr = "0.2"
[build-dependencies]
cc = { version = "^1.0", features = ["parallel"] }
make-cmd = "0.1"
bindgen = "0.29"
bindgen = "0.37"

@ -32,7 +32,7 @@ fn fail_on_empty_directory(name: &str) {
fn bindgen_rocksdb() {
let bindings = bindgen::Builder::default()
.header("rocksdb/include/rocksdb/c.h")
.hide_type("max_align_t") // https://github.com/rust-lang-nursery/rust-bindgen/issues/550
.blacklist_type("max_align_t") // https://github.com/rust-lang-nursery/rust-bindgen/issues/550
.ctypes_prefix("libc")
.generate()
.expect("unable to generate rocksdb bindings");
@ -105,6 +105,7 @@ fn build_rocksdb() {
lib_sources.push("port/win/env_default.cc");
lib_sources.push("port/win/win_logger.cc");
lib_sources.push("port/win/io_win.cc");
lib_sources.push("port/win/win_thread.cc");
}
if cfg!(target_env = "msvc") {

@ -1 +1 @@
Subproject commit dbd8fa09b823826dd2a30bc119dad7a6fa9a4c6d
Subproject commit 5089e121166c46956a8a21c8ef967f1896c239de

File diff suppressed because one or more lines are too long

@ -888,6 +888,15 @@ impl DB {
DBIterator::new(self, &opts, mode)
}
/// Opens an interator with `set_total_order_seek` enabled.
/// This must be used to iterate across prefixes when `set_memtable_factory` has been called
/// with a Hash-based implementation.
pub fn full_iterator(&self, mode: IteratorMode) -> DBIterator {
let mut opts = ReadOptions::default();
opts.set_total_order_seek(true);
DBIterator::new(self, &opts, mode)
}
pub fn prefix_iterator<'a>(&self, prefix: &'a [u8]) -> DBIterator {
let mut opts = ReadOptions::default();
opts.set_prefix_same_as_start(true);
@ -903,6 +912,16 @@ impl DB {
DBIterator::new_cf(self, cf_handle, &opts, mode)
}
pub fn full_iterator_cf(
&self,
cf_handle: ColumnFamily,
mode: IteratorMode,
) -> Result<DBIterator, Error> {
let mut opts = ReadOptions::default();
opts.set_total_order_seek(true);
DBIterator::new_cf(self, cf_handle, &opts, mode)
}
pub fn prefix_iterator_cf<'a>(
&self,
cf_handle: ColumnFamily,

@ -14,12 +14,13 @@
use std::ffi::{CStr, CString};
use std::mem;
use std::path::Path;
use libc::{self, c_int, c_uchar, c_uint, c_void, size_t, uint64_t};
use ffi;
use {BlockBasedOptions, DBCompactionStyle, DBCompressionType, DBRecoveryMode,
Options, WriteOptions};
use {BlockBasedOptions, DBCompactionStyle, DBCompressionType, DBRecoveryMode, MemtableFactory,
Options, WriteOptions};
use compaction_filter::{self, CompactionFilterCallback, CompactionFilterFn, filter_callback};
use comparator::{self, ComparatorCallback, CompareFn};
use merge_operator::{self, MergeFn, MergeOperatorCallback, full_merge_callback,
@ -72,6 +73,12 @@ impl BlockBasedOptions {
}
}
pub fn disable_cache(&mut self) {
unsafe {
ffi::rocksdb_block_based_options_set_no_block_cache(self.inner, true as c_uchar);
}
}
pub fn set_bloom_filter(&mut self, bits_per_key: c_int, block_based: bool) {
unsafe {
let bloom = if block_based {
@ -221,6 +228,20 @@ impl Options {
}
}
/// If non-zero, we perform bigger reads when doing compaction. If you're
/// running RocksDB on spinning disks, you should set this to at least 2MB.
/// That way RocksDB's compaction is doing sequential instead of random reads.
///
/// When non-zero, we also force new_table_reader_for_compaction_inputs to
/// true.
///
/// Default: `0`
pub fn set_compaction_readahead_size(&mut self, compaction_readahead_size: usize) {
unsafe {
ffi::rocksdb_options_compaction_readahead_size(self.inner, compaction_readahead_size as usize);
}
}
pub fn set_merge_operator(&mut self, name: &str,
full_merge_fn: MergeFn,
partial_merge_fn: Option<MergeFn>) {
@ -883,6 +904,47 @@ impl Options {
unsafe { ffi::rocksdb_options_set_disable_auto_compactions(self.inner, disable as c_int) }
}
/// Defines the underlying memtable implementation.
/// See https://github.com/facebook/rocksdb/wiki/MemTable for more information.
/// Defaults to using a skiplist.
///
/// # Example
///
/// ```
/// use rocksdb::{Options, MemtableFactory};
/// let mut opts = Options::default();
/// let factory = MemtableFactory::HashSkipList {
/// bucket_count: 1_000_000,
/// height: 4,
/// branching_factor: 4,
/// };
///
/// opts.set_allow_concurrent_memtable_write(false);
/// opts.set_memtable_factory(factory);
/// ```
pub fn set_memtable_factory(&mut self, factory: MemtableFactory) {
match factory {
MemtableFactory::Vector => unsafe {
ffi::rocksdb_options_set_memtable_vector_rep(self.inner);
},
MemtableFactory::HashSkipList {
bucket_count,
height,
branching_factor,
} => unsafe {
ffi::rocksdb_options_set_hash_skip_list_rep(
self.inner,
bucket_count,
height,
branching_factor,
);
},
MemtableFactory::HashLinkList { bucket_count } => unsafe {
ffi::rocksdb_options_set_hash_link_list_rep(self.inner, bucket_count);
},
};
}
pub fn set_block_based_table_factory(&mut self, factory: &BlockBasedOptions) {
unsafe {
ffi::rocksdb_options_set_block_based_table_factory(self.inner, factory.inner);
@ -980,6 +1042,26 @@ impl Options {
ffi::rocksdb_options_set_num_levels(self.inner, n);
}
}
/// Specifies the absolute path of the directory the
/// write-ahead log (WAL) should be written to.
///
/// Default: same directory as the database
///
/// # Example
///
/// ```
/// use rocksdb::Options;
///
/// let mut opts = Options::default();
/// opts.set_wal_dir("/path/to/dir");
/// ```
pub fn set_wal_dir<P: AsRef<Path>>(&mut self, path: P) {
let p = CString::new(path.as_ref().to_string_lossy().as_bytes()).unwrap();
unsafe {
ffi::rocksdb_options_set_wal_dir(self.inner, p.as_ptr());
}
}
}
impl Default for Options {
@ -1024,6 +1106,7 @@ impl Default for WriteOptions {
#[cfg(test)]
mod tests {
use MemtableFactory;
use Options;
#[test]
@ -1036,4 +1119,16 @@ mod tests {
let opts = Options::default();
assert!(opts.get_statistics().is_none());
}
#[test]
fn test_set_memtable_factory() {
let mut opts = Options::default();
opts.set_memtable_factory(MemtableFactory::Vector);
opts.set_memtable_factory(MemtableFactory::HashLinkList { bucket_count: 100 });
opts.set_memtable_factory(MemtableFactory::HashSkipList {
bucket_count: 100,
height: 4,
branching_factor: 4,
});
}
}

@ -136,6 +136,14 @@ pub struct BlockBasedOptions {
inner: *mut ffi::rocksdb_block_based_table_options_t,
}
/// Defines the underlying memtable implementation.
/// See https://github.com/facebook/rocksdb/wiki/MemTable for more information.
pub enum MemtableFactory {
Vector,
HashSkipList { bucket_count: usize, height: i32, branching_factor: i32 },
HashLinkList { bucket_count: usize }
}
/// Database-wide options around performance and behavior.
///
/// Please read [the official tuning guide](https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide), and most importantly, measure performance under realistic workloads with realistic hardware.

@ -15,7 +15,7 @@
extern crate rocksdb;
mod util;
use rocksdb::{DB, Direction, IteratorMode, Options};
use rocksdb::{DB, Direction, IteratorMode, MemtableFactory, Options};
use util::DBName;
fn cba(input: &Box<[u8]>) -> Box<[u8]> {
@ -171,7 +171,7 @@ pub fn test_prefix_iterator() {
let prefix_extractor = rocksdb::SliceTransform::create_fixed_prefix(3);
let mut opts = Options::default();
opts.create_if_missing(true);
opts.create_if_missing(true);
opts.set_prefix_extractor(prefix_extractor);
let db = DB::open(&opts, &n.name).unwrap();
@ -193,4 +193,54 @@ pub fn test_prefix_iterator() {
assert_eq!(b_iterator.collect::<Vec<_>>(), expected)
}
}
let opts = Options::default();
assert!(DB::destroy(&opts, path).is_ok());
}
#[test]
pub fn test_full_iterator() {
let path = "_rust_rocksdb_fulliteratortest";
{
let a1: Box<[u8]> = key(b"aaa1");
let a2: Box<[u8]> = key(b"aaa2");
let b1: Box<[u8]> = key(b"bbb1");
let b2: Box<[u8]> = key(b"bbb2");
let prefix_extractor = rocksdb::SliceTransform::create_fixed_prefix(3);
let factory = MemtableFactory::HashSkipList {
bucket_count: 1_000_000,
height: 4,
branching_factor: 4,
};
let mut opts = Options::default();
opts.create_if_missing(true);
opts.set_prefix_extractor(prefix_extractor);
opts.set_allow_concurrent_memtable_write(false);
opts.set_memtable_factory(factory);
let db = DB::open(&opts, path).unwrap();
assert!(db.put(&*a1, &*a1).is_ok());
assert!(db.put(&*a2, &*a2).is_ok());
assert!(db.put(&*b1, &*b1).is_ok());
assert!(db.put(&*b2, &*b2).is_ok());
// A normal iterator won't work here since we're using a HashSkipList for our memtable
// implementation (which buckets keys based on their prefix):
let bad_iterator = db.iterator(IteratorMode::Start);
assert_eq!(bad_iterator.collect::<Vec<_>>(), vec![]);
let expected = vec![
(cba(&a1), cba(&a1)),
(cba(&a2), cba(&a2)),
(cba(&b1), cba(&b1)),
(cba(&b2), cba(&b2)),
];
let a_iterator = db.full_iterator(IteratorMode::Start);
assert_eq!(a_iterator.collect::<Vec<_>>(), expected)
}
let opts = Options::default();
assert!(DB::destroy(&opts, path).is_ok());
}

Loading…
Cancel
Save