diff --git a/Cargo.toml b/Cargo.toml index e722c47..3137ad5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "rocksdb" description = "A Rust wrapper for Facebook's RocksDB embeddable database." -version = "0.0.6" +version = "0.0.7" authors = ["Tyler Neely ", "David Greenberg "] license = "Apache-2.0" exclude = [ diff --git a/README.md b/README.md index 444c034..922b426 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ rust-rocksdb ============ [![Build Status](https://travis-ci.org/spacejam/rust-rocksdb.svg?branch=master)](https://travis-ci.org/spacejam/rust-rocksdb) -This library has been tested against RocksDB 3.8.1 on linux and OSX. The 0.0.6 crate should work with the Rust nightly release as of 7/12/15. +This library has been tested against RocksDB 3.8.1 on linux and OSX. The 0.0.7 crate should work with the Rust nightly release as of 7/16/15. ### status - [x] basic open/put/get/delete/close @@ -12,7 +12,7 @@ This library has been tested against RocksDB 3.8.1 on linux and OSX. The 0.0.6 - [x] LRU cache - [x] destroy/repair - [x] iterator - - [ ] comparator + - [x] comparator - [x] snapshot - [ ] column family operations - [ ] slicetransform @@ -31,12 +31,12 @@ sudo make install ###### Cargo.toml ```rust [dependencies] -rocksdb = "~0.0.6" +rocksdb = "~0.0.7" ``` ###### Code ```rust extern crate rocksdb; -use rocksdb::RocksDB; +use rocksdb::{RocksDB, Writable}; fn main() { let mut db = RocksDB::open_default("/path/for/rocksdb/storage").unwrap(); @@ -107,7 +107,7 @@ fn main() { ###### Rustic Merge Operator ```rust extern crate rocksdb; -use rocksdb::{RocksDBOptions, RocksDB, MergeOperands}; +use rocksdb::{Options, RocksDB, MergeOperands, Writable}; fn concat_merge(new_key: &[u8], existing_val: Option<&[u8]>, operands: &mut MergeOperands) -> Vec { @@ -121,10 +121,10 @@ fn concat_merge(new_key: &[u8], existing_val: Option<&[u8]>, fn main() { let path = "/path/to/rocksdb"; - let opts = RocksDBOptions::new(); + let mut opts = Options::new(); opts.create_if_missing(true); opts.add_merge_operator("test operator", concat_merge); - let mut db = RocksDB::open(opts, path).unwrap(); + let mut db = RocksDB::open(&opts, path).unwrap(); let p = db.put(b"k1", b"a"); db.merge(b"k1", b"b"); db.merge(b"k1", b"c"); @@ -138,14 +138,13 @@ fn main() { ###### Apply Some Tunings Please read [the official tuning guide](https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide), and most importantly, measure performance under realistic workloads with realistic hardware. ```rust -use rocksdb::{RocksDBOptions, RocksDB, new_bloom_filter}; +use rocksdb::{Options, RocksDB}; use rocksdb::RocksDBCompactionStyle::RocksDBUniversalCompaction; -fn tuned_for_somebody_elses_disk() -> RocksDB { +fn badly_tuned_for_somebody_elses_disk() -> RocksDB { let path = "_rust_rocksdb_optimizetest"; - let opts = RocksDBOptions::new(); + let mut opts = Options::new(); opts.create_if_missing(true); - opts.set_block_size(524288); opts.set_max_open_files(10000); opts.set_use_fsync(false); opts.set_bytes_per_sync(8388608); @@ -164,10 +163,7 @@ fn tuned_for_somebody_elses_disk() -> RocksDB { opts.set_filter_deletes(false); opts.set_disable_auto_compactions(true); - let filter = new_bloom_filter(10); - opts.set_filter(filter); - - RocksDB::open(opts, path).unwrap() + RocksDB::open(&opts, path).unwrap() } ``` diff --git a/src/comparator.rs b/src/comparator.rs index 9a466c2..6cf8d0d 100644 --- a/src/comparator.rs +++ b/src/comparator.rs @@ -25,7 +25,7 @@ use rocksdb::RocksDB; pub struct ComparatorCallback { pub name: CString, - pub f: fn (&[u8], &[u8]) -> i32, + pub f: fn(&[u8], &[u8]) -> i32, } pub extern "C" fn destructor_callback(raw_cb: *mut c_void) { @@ -42,10 +42,12 @@ pub extern "C" fn name_callback(raw_cb: *mut c_void) -> *const c_char { } } -pub extern "C" fn compare_callback( - raw_cb: *mut c_void, - a_raw: *const c_char, a_len: size_t, - b_raw: *const c_char, b_len: size_t) -> c_int { +pub extern "C" fn compare_callback(raw_cb: *mut c_void, + a_raw: *const c_char, + a_len: size_t, + b_raw: *const c_char, + b_len: size_t) + -> c_int { unsafe { let cb: &mut ComparatorCallback = &mut *(raw_cb as *mut ComparatorCallback); diff --git a/src/ffi.rs b/src/ffi.rs index fa79285..10c8cc9 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -71,25 +71,25 @@ pub fn new_cache(capacity: size_t) -> RocksDBCache { #[repr(C)] pub enum RocksDBCompressionType { - RocksDBNoCompression = 0, + RocksDBNoCompression = 0, RocksDBSnappyCompression = 1, - RocksDBZlibCompression = 2, - RocksDBBz2Compression = 3, - RocksDBLz4Compression = 4, - RocksDBLz4hcCompression = 5 + RocksDBZlibCompression = 2, + RocksDBBz2Compression = 3, + RocksDBLz4Compression = 4, + RocksDBLz4hcCompression = 5, } #[repr(C)] pub enum RocksDBCompactionStyle { - RocksDBLevelCompaction = 0, + RocksDBLevelCompaction = 0, RocksDBUniversalCompaction = 1, - RocksDBFifoCompaction = 2 + RocksDBFifoCompaction = 2, } #[repr(C)] pub enum RocksDBUniversalCompactionStyle { rocksdb_similar_size_compaction_stop_style = 0, - rocksdb_total_size_compaction_stop_style = 1 + rocksdb_total_size_compaction_stop_style = 1, } //TODO audit the use of boolean arguments, b/c I think they need to be u8 instead... diff --git a/src/lib.rs b/src/lib.rs index 73c7d81..621add8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -21,26 +21,10 @@ #![feature(raw)] pub use ffi as rocksdb_ffi; -pub use ffi::{ - new_bloom_filter, - RocksDBCompactionStyle, - RocksDBComparator, -}; -pub use rocksdb::{ - RocksDB, - RocksDBResult, - RocksDBVector, - WriteBatch, - Writable, - Direction, -}; -pub use rocksdb_options::{ - Options, - BlockBasedOptions, -}; -pub use merge_operator::{ - MergeOperands, -}; +pub use ffi::{new_bloom_filter, RocksDBCompactionStyle, RocksDBComparator}; +pub use rocksdb::{RocksDB, RocksDBResult, RocksDBVector, WriteBatch, Writable, Direction}; +pub use rocksdb_options::{Options, BlockBasedOptions}; +pub use merge_operator::MergeOperands; pub mod rocksdb; pub mod ffi; pub mod rocksdb_options; diff --git a/src/merge_operator.rs b/src/merge_operator.rs index aa40e32..38766e6 100644 --- a/src/merge_operator.rs +++ b/src/merge_operator.rs @@ -20,12 +20,12 @@ use std::mem; use std::ptr; use std::slice; -use rocksdb_options::{Options}; +use rocksdb_options::Options; use rocksdb::{RocksDB, RocksDBResult, RocksDBVector, Writable}; pub struct MergeOperatorCallback { pub name: CString, - pub merge_fn: fn (&[u8], Option<&[u8]>, &mut MergeOperands) -> Vec, + pub merge_fn: fn(&[u8], Option<&[u8]>, &mut MergeOperands) -> Vec, } pub extern "C" fn destructor_callback(raw_cb: *mut c_void) { @@ -43,12 +43,17 @@ pub extern "C" fn name_callback(raw_cb: *mut c_void) -> *const c_char { } } -pub extern "C" fn full_merge_callback( - raw_cb: *mut c_void, raw_key: *const c_char, key_len: size_t, - existing_value: *const c_char, existing_value_len: size_t, - operands_list: *const *const c_char, operands_list_len: *const size_t, - num_operands: c_int, - success: *mut u8, new_value_length: *mut size_t) -> *const c_char { +pub extern "C" fn full_merge_callback(raw_cb: *mut c_void, + raw_key: *const c_char, + key_len: size_t, + existing_value: *const c_char, + existing_value_len: size_t, + operands_list: *const *const c_char, + operands_list_len: *const size_t, + num_operands: c_int, + success: *mut u8, + new_value_length: *mut size_t) + -> *const c_char { unsafe { let cb: &mut MergeOperatorCallback = &mut *(raw_cb as *mut MergeOperatorCallback); @@ -72,11 +77,15 @@ pub extern "C" fn full_merge_callback( } } -pub extern "C" fn partial_merge_callback( - raw_cb: *mut c_void, raw_key: *const c_char, key_len: size_t, - operands_list: *const *const c_char, operands_list_len: *const size_t, - num_operands: c_int, - success: *mut u8, new_value_length: *mut size_t) -> *const c_char { +pub extern "C" fn partial_merge_callback(raw_cb: *mut c_void, + raw_key: *const c_char, + key_len: size_t, + operands_list: *const *const c_char, + operands_list_len: *const size_t, + num_operands: c_int, + success: *mut u8, + new_value_length: *mut size_t) + -> *const c_char { unsafe { let cb: &mut MergeOperatorCallback = &mut *(raw_cb as *mut MergeOperatorCallback); @@ -107,7 +116,8 @@ pub struct MergeOperands { impl MergeOperands { fn new(operands_list: *const *const c_char, operands_list_len: *const size_t, - num_operands: c_int) -> MergeOperands { + num_operands: c_int) + -> MergeOperands { assert!(num_operands >= 0); MergeOperands { operands_list: operands_list, @@ -148,8 +158,10 @@ impl<'a> Iterator for &'a mut MergeOperands { } } -fn test_provided_merge(new_key: &[u8], existing_val: Option<&[u8]>, - mut operands: &mut MergeOperands) -> Vec { +fn test_provided_merge(new_key: &[u8], + existing_val: Option<&[u8]>, + mut operands: &mut MergeOperands) + -> Vec { let nops = operands.size_hint().0; let mut result: Vec = Vec::with_capacity(nops); match existing_val { diff --git a/src/rocksdb.rs b/src/rocksdb.rs index a922b85..dba957a 100644 --- a/src/rocksdb.rs +++ b/src/rocksdb.rs @@ -46,14 +46,16 @@ pub struct Snapshot<'a> { } pub struct DBIterator { -//TODO: should have a reference to DB to enforce scope, but it's trickier than I thought to add + // TODO: should have a reference to DB to enforce scope, but it's trickier than I + // thought to add inner: rocksdb_ffi::RocksDBIterator, direction: Direction, - just_seeked: bool + just_seeked: bool, } pub enum Direction { - forward, reverse + forward, + reverse, } pub struct SubDBIterator<'a> { @@ -92,7 +94,7 @@ impl <'a> Iterator for SubDBIterator<'a> { } impl DBIterator { -//TODO alias db & opts to different lifetimes, and DBIterator to the db's lifetime +//TODO alias db & opts to different lifetimes, and DBIterator to the db's lifetime fn new(db: &RocksDB, readopts: &ReadOptions) -> DBIterator { unsafe { let iterator = rocksdb_ffi::rocksdb_create_iterator(db.inner, readopts.inner); @@ -106,7 +108,7 @@ impl DBIterator { unsafe { rocksdb_ffi::rocksdb_iter_seek_to_first(self.inner); }; - SubDBIterator{ iter: self, direction: Direction::forward, } + SubDBIterator { iter: self, direction: Direction::forward } } pub fn from_end(&mut self) -> SubDBIterator { @@ -114,7 +116,7 @@ impl DBIterator { unsafe { rocksdb_ffi::rocksdb_iter_seek_to_last(self.inner); }; - SubDBIterator{ iter: self, direction: Direction::reverse, } + SubDBIterator { iter: self, direction: Direction::reverse } } pub fn from(&mut self, key: &[u8], dir: Direction) -> SubDBIterator { @@ -122,7 +124,7 @@ impl DBIterator { unsafe { rocksdb_ffi::rocksdb_iter_seek(self.inner, key.as_ptr(), key.len() as size_t); } - SubDBIterator{ iter: self, direction: dir, } + SubDBIterator { iter: self, direction: dir } } } @@ -137,7 +139,7 @@ impl Drop for DBIterator { impl <'a> Snapshot<'a> { pub fn new(db: &RocksDB) -> Snapshot { let snapshot = unsafe { rocksdb_ffi::rocksdb_create_snapshot(db.inner) }; - Snapshot{db: db, inner: snapshot} + Snapshot { db: db, inner: snapshot } } pub fn iterator(&self) -> DBIterator { @@ -180,8 +182,9 @@ impl RocksDB { pub fn open(opts: &Options, path: &str) -> Result { let cpath = match CString::new(path.as_bytes()) { - Ok(c) => c, - Err(_) => return Err("Failed to convert path to CString when opening rocksdb".to_string()), + Ok(c) => c, + Err(_) => + return Err("Failed to convert path to CString when opening rocksdb".to_string()), }; let cpath_ptr = cpath.as_ptr(); @@ -208,7 +211,7 @@ impl RocksDB { if db_ptr.is_null() { return Err("Could not initialize database.".to_string()); } - Ok(RocksDB{inner: db}) + Ok(RocksDB { inner: db }) } pub fn destroy(opts: &Options, path: &str) -> Result<(), String> { @@ -358,7 +361,7 @@ impl WriteBatch { WriteBatch { inner: unsafe { rocksdb_ffi::rocksdb_writebatch_create() - } + }, } } } @@ -419,8 +422,9 @@ impl ReadOptions { ReadOptions{inner: rocksdb_ffi::rocksdb_readoptions_create()} } } -//TODO add snapshot setting here -//TODO add snapshot wrapper structs with proper destructors; that struct needs an "iterator" impl too. +// TODO add snapshot setting here +// TODO add snapshot wrapper structs with proper destructors; +// that struct needs an "iterator" impl too. fn fill_cache(&mut self, v: bool) { unsafe { rocksdb_ffi::rocksdb_readoptions_set_fill_cache(self.inner, v); @@ -505,7 +509,7 @@ impl RocksDBResult { } } - pub fn on_absent()>(self, f: F) -> RocksDBResult { + pub fn on_absent ()>(self, f: F) -> RocksDBResult { match self { RocksDBResult::Some(x) => RocksDBResult::Some(x), RocksDBResult::None => { diff --git a/src/rocksdb_options.rs b/src/rocksdb_options.rs index e5c4b53..08f290c 100644 --- a/src/rocksdb_options.rs +++ b/src/rocksdb_options.rs @@ -19,8 +19,8 @@ use std::ffi::CString; use std::mem; use rocksdb_ffi; -use merge_operator::{self, MergeOperatorCallback, MergeOperands, full_merge_callback, - partial_merge_callback}; +use merge_operator::{self, MergeOperatorCallback, MergeOperands, + full_merge_callback, partial_merge_callback}; use comparator::{self, ComparatorCallback, compare_callback}; pub struct BlockBasedOptions { @@ -54,7 +54,7 @@ impl BlockBasedOptions { if opt_ptr.is_null() { panic!("Could not create rocksdb block based options".to_string()); } - BlockBasedOptions{ inner: block_opts, } + BlockBasedOptions { inner: block_opts } } pub fn set_block_size(&mut self, size: u64) { @@ -107,8 +107,7 @@ impl Options { } } - pub fn optimize_level_style_compaction(&mut self, - memtable_memory_budget: i32) { + pub fn optimize_level_style_compaction(&mut self, memtable_memory_budget: i32) { unsafe { rocksdb_ffi::rocksdb_options_optimize_level_style_compaction( self.inner, memtable_memory_budget); @@ -122,8 +121,9 @@ impl Options { } } - pub fn add_merge_operator<'a>(&mut self, name: &str, - merge_fn: fn (&[u8], Option<&[u8]>, &mut MergeOperands) -> Vec) { + pub fn add_merge_operator<'a>(&mut self, + name: &str, + merge_fn: fn(&[u8], Option<&[u8]>, &mut MergeOperands) -> Vec) { let cb = Box::new(MergeOperatorCallback { name: CString::new(name.as_bytes()).unwrap(), merge_fn: merge_fn, @@ -258,8 +258,7 @@ impl Options { } } - pub fn set_compaction_style(&mut self, style: - rocksdb_ffi::RocksDBCompactionStyle) { + pub fn set_compaction_style(&mut self, style: rocksdb_ffi::RocksDBCompactionStyle) { unsafe { rocksdb_ffi::rocksdb_options_set_compaction_style( self.inner, style); @@ -306,5 +305,3 @@ impl Options { } } } - - diff --git a/test/test_iterator.rs b/test/test_iterator.rs index 0985bfd..f5cb0c0 100644 --- a/test/test_iterator.rs +++ b/test/test_iterator.rs @@ -1,6 +1,9 @@ use rocksdb::{Options, RocksDB, Writable, Direction}; use std; +fn cba(input: &Box<[u8]>) -> Box<[u8]> { + input.iter().cloned().collect::>().into_boxed_slice() +} #[test] pub fn test_iterator() { @@ -22,7 +25,7 @@ pub fn test_iterator() { let p = db.put(&*k3, &*v3); assert!(p.is_ok()); let mut view1 = db.iterator(); - let expected = vec![(k1, v1), (k2, v2), (k3, v3)]; + let expected = vec![(cba(&k1), cba(&v1)), (cba(&k2), cba(&v2)), (cba(&k3), cba(&v3))]; { let mut iterator1 = view1.from_start(); assert_eq!(iterator1.collect::>(), expected); @@ -82,26 +85,27 @@ pub fn test_iterator() { } let mut view2 = db.iterator(); - let p = db.put(k4, v4); + let p = db.put(&*k4, &*v4); assert!(p.is_ok()); let mut view3 = db.iterator(); - let expected2 = vec![(k1, v1), (k2, v2), (k3, v3), (k4, v4)]; + let expected2 = vec![(cba(&k1), cba(&v1)), (cba(&k2), cba(&v2)), (cba(&k3), cba(&v3)), (cba(&k4), cba(&v4))]; { let mut iterator1 = view1.from_start(); - assert_eq!(iterator1.collect::>(), expected2); + assert_eq!(iterator1.collect::>(), expected); } -//TODO continue implementing tests! - println!("See the output of the third iter"); - for (k,v) in view3.from_start() { - //println!("Hello {}: {}", std::str::from_utf8(k).unwrap(), std::str::from_utf8(v).unwrap()); + { + let mut iterator1 = view3.from_start(); + assert_eq!(iterator1.collect::>(), expected2); } - println!("now the 3rd iter from k2 fwd"); - for (k,v) in view3.from(b"k2", Direction::forward) { - //println!("Hello {}: {}", std::str::from_utf8(k).unwrap(), std::str::from_utf8(v).unwrap()); + { + let mut iterator1 = view3.from(b"k2", Direction::forward); + let expected = vec![(cba(&k2), cba(&v2)), (cba(&k3), cba(&v3)), (cba(&k4), cba(&v4))]; + assert_eq!(iterator1.collect::>(), expected); } - println!("now the 3rd iter from k2 and back"); - for (k,v) in view3.from(b"k2", Direction::reverse) { - //println!("Hello {}: {}", std::str::from_utf8(k).unwrap(), std::str::from_utf8(v).unwrap()); + { + let mut iterator1 = view3.from(b"k2", Direction::reverse); + let expected = vec![(cba(&k2), cba(&v2)), (cba(&k1), cba(&v1))]; + assert_eq!(iterator1.collect::>(), expected); } } let opts = Options::new();