From 6e3d781c15d4076120f2339081d37ef49b2e535d Mon Sep 17 00:00:00 2001 From: Oleksandr Anyshchenko Date: Thu, 22 Jul 2021 17:30:38 +0300 Subject: [PATCH] Release 0.17.0 (#539) --- .github/workflows/rust.yml | 2 +- CHANGELOG.md | 12 +++- Cargo.toml | 4 +- librocksdb-sys/Cargo.toml | 2 +- src/column_family.rs | 2 +- src/compaction_filter.rs | 10 ++-- src/compaction_filter_factory.rs | 8 +-- src/db.rs | 4 +- src/db_iterator.rs | 2 +- tests/test_db.rs | 13 +++-- tests/test_merge_operator.rs | 99 +++++++++++++------------------- 11 files changed, 75 insertions(+), 83 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index a21299f..07311fb 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -41,7 +41,7 @@ jobs: uses: actions-rs/clippy-check@v1 with: token: ${{ secrets.GITHUB_TOKEN }} - args: -- -D warnings + args: --all-targets -- -D warnings audit: name: Security audit diff --git a/CHANGELOG.md b/CHANGELOG.md index 14cf0d0..2c3b56c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,12 +2,20 @@ ## [Unreleased] +## 0.17.0 (2021-07-22) + +* Fix `multi_get` method (mikhailOK) * Bump `librocksdb-sys` up to 6.19.3 (olegnn) -* Make SSE inclusion conditional for target features. - RocksDB is not compiled with SSE4 instructions anymore unless the corresponding features are enabled in rustc (mbargull) +* Add support for the cuckoo table format (rbost) +* RocksDB is not compiled with SSE4 instructions anymore unless the corresponding features are enabled in rustc (mbargull) * Bump `librocksdb-sys` up to 6.20.3 (olegnn, akrylysov) * Add `DB::key_may_exist_cf_opt` method (stanislav-tkach) * Add `Options::set_zstd_max_train_bytes` method (stanislav-tkach) +* Mark Cache and Env as Send and Sync (akrylysov) +* Allow cloning the Cache and Env (duarten) +* Make SSE inclusion conditional for target features (mbargull) +* Use Self where possible (adamnemecek) +* Don't leak dropped column families (ryoqun) ## 0.16.0 (2021-04-18) diff --git a/Cargo.toml b/Cargo.toml index 721d6a1..7a5b880 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,9 +28,11 @@ multi-threaded-cf = [] [dependencies] libc = "0.2" -librocksdb-sys = { path = "librocksdb-sys", version = "6.19.3" } +librocksdb-sys = { path = "librocksdb-sys", version = "6.20.3" } [dev-dependencies] trybuild = "1.0" tempfile = "3.1" pretty_assertions = "0.7" +bincode = "1.3" +serde = { version = "1", features = [ "derive" ] } \ No newline at end of file diff --git a/librocksdb-sys/Cargo.toml b/librocksdb-sys/Cargo.toml index 672c060..06f2012 100644 --- a/librocksdb-sys/Cargo.toml +++ b/librocksdb-sys/Cargo.toml @@ -29,5 +29,5 @@ uuid = { version = "0.8", features = ["v4"] } [build-dependencies] cc = { version = "1.0", features = ["parallel"] } -bindgen = { version = "0.58.1", default-features = false, features = ["runtime"] } +bindgen = { version = "0.59", default-features = false, features = ["runtime"] } glob = "0.3" diff --git a/src/column_family.rs b/src/column_family.rs index 3184c65..1cde254 100644 --- a/src/column_family.rs +++ b/src/column_family.rs @@ -130,7 +130,7 @@ impl<'a> AsColumnFamilyRef for &'a ColumnFamily { } // Only implement for Arc-ed BoundColumnFamily as this tightly coupled and -// implmetation detail, considering use of std::mem::transmute. BoundColumnFamily +// implementation detail, considering use of std::mem::transmute. BoundColumnFamily // isn't expected to be used as naked. // Also, ColumnFamilyRef might not be Arc> depending crate // feature flags so, we can't use the type alias here. diff --git a/src/compaction_filter.rs b/src/compaction_filter.rs index 3e82730..2680408 100644 --- a/src/compaction_filter.rs +++ b/src/compaction_filter.rs @@ -49,7 +49,7 @@ pub trait CompactionFilter { /// /// Note that RocksDB snapshots (i.e. call GetSnapshot() API on a /// DB* object) will not guarantee to preserve the state of the DB with - /// CompactionFilter. Data seen from a snapshot might disppear after a + /// CompactionFilter. Data seen from a snapshot might disappear after a /// compaction finishes. If you use snapshots, think twice about whether you /// want to use compaction filter and whether you are using it in a safe way. /// @@ -158,15 +158,15 @@ fn test_filter(level: u32, key: &[u8], value: &[u8]) -> Decision { fn compaction_filter_test() { use crate::{Options, DB}; - let path = "_rust_rocksdb_filtertest"; + let path = "_rust_rocksdb_filter_test"; let mut opts = Options::default(); opts.create_if_missing(true); opts.set_compaction_filter("test", test_filter); { let db = DB::open(&opts, path).unwrap(); - let _ = db.put(b"k1", b"a"); - let _ = db.put(b"_k", b"b"); - let _ = db.put(b"%k", b"c"); + let _r = db.put(b"k1", b"a"); + let _r = db.put(b"_k", b"b"); + let _r = db.put(b"%k", b"c"); db.compact_range(None::<&[u8]>, None::<&[u8]>); assert_eq!(&*db.get(b"k1").unwrap().unwrap(), b"a"); assert!(db.get(b"_k").unwrap().is_none()); diff --git a/src/compaction_filter_factory.rs b/src/compaction_filter_factory.rs index 1e3159c..5b144d0 100644 --- a/src/compaction_filter_factory.rs +++ b/src/compaction_filter_factory.rs @@ -122,15 +122,15 @@ mod tests { #[test] fn compaction_filter_factory_test() { - let path = "_rust_rocksdb_filterfactorytest"; + let path = "_rust_rocksdb_filter_factory_test"; let mut opts = Options::default(); opts.create_if_missing(true); opts.set_compaction_filter_factory(TestFactory(CString::new("TestFactory").unwrap())); { let db = DB::open(&opts, path).unwrap(); - let _ = db.put(b"k1", b"a"); - let _ = db.put(b"_k", b"b"); - let _ = db.put(b"%k", b"c"); + let _r = db.put(b"k1", b"a"); + let _r = db.put(b"_rk", b"b"); + let _r = db.put(b"%k", b"c"); db.compact_range(None::<&[u8]>, None::<&[u8]>); assert_eq!(db.get(b"%k1").unwrap(), None); } diff --git a/src/db.rs b/src/db.rs index 1890d8f..789129f 100644 --- a/src/db.rs +++ b/src/db.rs @@ -184,7 +184,7 @@ impl DBAccess for DBWithThreadMode { /// Even with [`SingleThreaded`], almost all of RocksDB operations is /// multi-threaded unless the underlying RocksDB instance is /// specifically configured otherwise. `SingleThreaded` only forces -/// serialization of column family alternations by requring `&mut self` of DB +/// serialization of column family alternations by requiring `&mut self` of DB /// instance due to its wrapper implementation details. /// /// # Multi-threaded mode @@ -1779,7 +1779,7 @@ impl DBWithThreadMode { } /// Returns the underlying column family handle - pub fn cf_handle<'a>(&'a self, name: &str) -> Option<&'a ColumnFamily> { + pub fn cf_handle(&self, name: &str) -> Option<&ColumnFamily> { self.cfs.cfs.get(name) } } diff --git a/src/db_iterator.rs b/src/db_iterator.rs index 5027aeb..bc8e22e 100644 --- a/src/db_iterator.rs +++ b/src/db_iterator.rs @@ -471,7 +471,7 @@ impl<'a, D: DBAccess> Iterator for DBIteratorWithThreadMode<'a, D> { } if self.raw.valid() { - // .key() and .value() only ever return None if valid == false, which we've just cheked + // .key() and .value() only ever return None if valid == false, which we've just checked Some(( Box::from(self.raw.key().unwrap()), Box::from(self.raw.value().unwrap()), diff --git a/tests/test_db.rs b/tests/test_db.rs index 0504475..8f229ec 100644 --- a/tests/test_db.rs +++ b/tests/test_db.rs @@ -72,9 +72,9 @@ fn errors_do_stuff() { match DB::destroy(&opts, &path) { Err(s) => { let message = s.to_string(); - assert!(message.find("IO error:").is_some()); - assert!(message.find("_rust_rocksdb_error").is_some()); - assert!(message.find("/LOCK:").is_some()); + assert!(message.contains("IO error:")); + assert!(message.contains("_rust_rocksdb_error")); + assert!(message.contains("/LOCK:")); } Ok(_) => panic!("should fail"), } @@ -680,9 +680,10 @@ fn env_and_dbpaths_test() { } { - let mut paths = Vec::new(); - paths.push(rocksdb::DBPath::new(&path1, 20 << 20).unwrap()); - paths.push(rocksdb::DBPath::new(&path2, 30 << 20).unwrap()); + let paths = vec![ + rocksdb::DBPath::new(&path1, 20 << 20).unwrap(), + rocksdb::DBPath::new(&path2, 30 << 20).unwrap(), + ]; opts.set_db_paths(&paths); } diff --git a/tests/test_merge_operator.rs b/tests/test_merge_operator.rs index 021eb2a..7ba2fbd 100644 --- a/tests/test_merge_operator.rs +++ b/tests/test_merge_operator.rs @@ -15,9 +15,8 @@ mod util; use pretty_assertions::assert_eq; - -use rocksdb::merge_operator::MergeFn; -use rocksdb::{DBCompactionStyle, MergeOperands, Options, DB}; +use rocksdb::{merge_operator::MergeFn, DBCompactionStyle, MergeOperands, Options, DB}; +use serde::{Deserialize, Serialize}; use util::DBPath; fn test_provided_merge( @@ -77,26 +76,7 @@ fn merge_test() { assert!(db.get(b"k1").unwrap().is_none()); } -unsafe fn to_slice(p: &T) -> &[u8] { - ::std::slice::from_raw_parts((p as *const T) as *const u8, ::std::mem::size_of::()) -} - -fn from_slice(s: &[u8]) -> Option<&T> { - if std::mem::size_of::() == s.len() { - unsafe { Some(&*(s.as_ptr() as *const T)) } - } else { - println!( - "slice {:?} is len {}, but T is size {}", - s, - s.len(), - std::mem::size_of::() - ); - None - } -} - -#[repr(packed)] -#[derive(Copy, Clone, Debug, Default)] +#[derive(Serialize, Deserialize, Copy, Clone, Debug, Default)] struct ValueCounts { num_a: u32, num_b: u32, @@ -104,6 +84,16 @@ struct ValueCounts { num_d: u32, } +impl ValueCounts { + fn from_slice(slice: &[u8]) -> Option { + bincode::deserialize::(slice).ok() + } + + fn as_bytes(&self) -> Option> { + bincode::serialize(self).ok() + } +} + fn test_counting_partial_merge( _new_key: &[u8], _existing_val: Option<&[u8]>, @@ -124,11 +114,10 @@ fn test_counting_full_merge( existing_val: Option<&[u8]>, operands: &mut MergeOperands, ) -> Option> { - let mut counts = if let Some(v) = existing_val { - *from_slice::(v).unwrap_or(&ValueCounts::default()) - } else { - ValueCounts::default() - }; + let mut counts = existing_val + .map(|v| ValueCounts::from_slice(v)) + .flatten() + .unwrap_or_default(); for op in operands { for e in op { @@ -141,15 +130,13 @@ fn test_counting_full_merge( } } } - let slc = unsafe { to_slice(&counts) }; - Some(slc.to_vec()) + + counts.as_bytes() } #[test] -#[allow(clippy::too_many_lines)] fn counting_merge_test() { - use std::sync::Arc; - use std::thread; + use std::{sync::Arc, thread}; let db_path = DBPath::new("_rust_rocksdb_partial_merge_test"); let mut opts = Options::default(); @@ -234,35 +221,29 @@ fn counting_merge_test() { } }); let m = db.merge(b"k1", b"b"); + assert!(m.is_ok()); h3.join().unwrap(); h1.join().unwrap(); - match db.get(b"k2") { - Ok(Some(value)) => match from_slice::(&*value) { - Some(v) => unsafe { - assert_eq!(v.num_a, 1000); - assert_eq!(v.num_b, 500); - assert_eq!(v.num_c, 2000); - assert_eq!(v.num_d, 500); - }, - None => panic!("Failed to get ValueCounts from db"), - }, - Err(e) => panic!("error reading value {:?}", e), - _ => panic!("value not present"), - } - match db.get(b"k1") { - Ok(Some(value)) => match from_slice::(&*value) { - Some(v) => unsafe { - assert_eq!(v.num_a, 3); - assert_eq!(v.num_b, 2); - assert_eq!(v.num_c, 0); - assert_eq!(v.num_d, 1); - }, - None => panic!("Failed to get ValueCounts from db"), - }, + + let value_getter = |key| match db.get(key) { + Ok(Some(value)) => ValueCounts::from_slice(&value) + .map_or_else(|| panic!("unable to create ValueCounts from bytes"), |v| v), + Ok(None) => panic!("value not present"), Err(e) => panic!("error reading value {:?}", e), - _ => panic!("value not present"), - } + }; + + let counts = value_getter(b"k2"); + assert_eq!(counts.num_a, 1000); + assert_eq!(counts.num_b, 500); + assert_eq!(counts.num_c, 2000); + assert_eq!(counts.num_d, 500); + + let counts = value_getter(b"k1"); + assert_eq!(counts.num_a, 3); + assert_eq!(counts.num_b, 2); + assert_eq!(counts.num_c, 0); + assert_eq!(counts.num_d, 1); } #[test] @@ -311,7 +292,7 @@ fn make_merge_max_with_limit(limit: u64) -> impl MergeFn + Clone { #[test] fn test_merge_state() { use {Options, DB}; - let path = "_rust_rocksdb_mergetest_state"; + let path = "_rust_rocksdb_merge_test_state"; let mut opts = Options::default(); opts.create_if_missing(true); opts.set_merge_operator_associative("max-limit-12", make_merge_max_with_limit(12));