From 5aeb49137d9821c6672a96fc23ba6c4e252e9ef4 Mon Sep 17 00:00:00 2001 From: Jihyun Yu Date: Sun, 21 Apr 2019 15:06:50 +0900 Subject: [PATCH 01/20] enable sse4.2/pclmul for accelerated crc32c --- librocksdb-sys/build.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/librocksdb-sys/build.rs b/librocksdb-sys/build.rs index 503fecf..d8549f3 100644 --- a/librocksdb-sys/build.rs +++ b/librocksdb-sys/build.rs @@ -90,6 +90,18 @@ fn build_rocksdb() { .filter(|file| *file != "util/build_version.cc") .collect::>(); + if cfg!(target_arch = "x86_64") { + // This is needed to enable hardware CRC32C. Technically, SSE 4.2 is + // only available since Intel Nehalem (about 2010) and AMD Bulldozer + // (about 2011). + config.define("HAVE_PCLMUL", Some("1")); + config.define("HAVE_SSE42", Some("1")); + config.flag("-msse2"); + config.flag("-msse4.1"); + config.flag("-msse4.2"); + config.flag("-mpclmul"); + } + if cfg!(target_os = "macos") { config.define("OS_MACOSX", Some("1")); config.define("ROCKSDB_PLATFORM_POSIX", Some("1")); From 6cf5be4a61c361da5680ca3448683052d7e82dae Mon Sep 17 00:00:00 2001 From: Renar Narubin Date: Tue, 23 Apr 2019 14:08:31 -0700 Subject: [PATCH 02/20] Add raw_iterator_cf_opt to the DB API The underlying functionality already existed, but it wasn't exposed to the top-level API. --- src/db.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/db.rs b/src/db.rs index 4c1e233..d82a8f2 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1167,6 +1167,14 @@ impl DB { DBRawIterator::new_cf(self, cf_handle, &opts) } + pub fn raw_iterator_cf_opt( + &self, + cf_handle: ColumnFamily, + readopts: &ReadOptions, + ) -> Result { + DBRawIterator::new_cf(self, cf_handle, readopts) + } + pub fn snapshot(&self) -> Snapshot { Snapshot::new(self) } From 10b1ef7f8540d065a9e603413a56017307079095 Mon Sep 17 00:00:00 2001 From: pavel-mukhanov Date: Tue, 25 Jun 2019 12:24:15 +0300 Subject: [PATCH 03/20] Implement Send and Sync for Snapshot. --- src/db.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/db.rs b/src/db.rs index ca8430c..c799b01 100644 --- a/src/db.rs +++ b/src/db.rs @@ -104,6 +104,9 @@ pub struct Snapshot<'a> { inner: *const ffi::rocksdb_snapshot_t, } +unsafe impl<'a> Send for Snapshot<'a> {} +unsafe impl<'a> Sync for Snapshot<'a> {} + /// An iterator over a database or column family, with specifiable /// ranges and direction. /// From e08fe8f6e066587b1d47f35d4787f9626385788e Mon Sep 17 00:00:00 2001 From: pavel-mukhanov Date: Wed, 26 Jun 2019 16:04:01 +0300 Subject: [PATCH 04/20] Add test for sync snapshot. --- tests/test_db.rs | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/test_db.rs b/tests/test_db.rs index 47dac85..f0ec54c 100644 --- a/tests/test_db.rs +++ b/tests/test_db.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![feature(thread_spawn_unchecked)] + extern crate libc; extern crate rocksdb; @@ -21,6 +23,7 @@ use libc::size_t; use rocksdb::{DBVector, Error, IteratorMode, Options, WriteBatch, DB}; use util::DBPath; +use std::thread; #[test] fn test_db_vector() { @@ -163,6 +166,36 @@ fn snapshot_test() { } } +#[test] +fn sync_snapshot_test() { + let path = DBPath::new("_rust_rocksdb_sync_snapshottest"); + { + let db = DB::open_default(&path).unwrap(); + + assert!(db.put(b"k1", b"v1").is_ok()); + assert!(db.put(b"k2", b"v2").is_ok()); + + let snapshot = db.snapshot(); + + // Unsafe here is safe, because `handler.join()` is called at the end of the + // method to ensure that snapshot will not outlive database. + let handler_1 = unsafe { + thread::Builder::new().spawn_unchecked(|| { + assert_eq!(snapshot.get(b"k1").unwrap().unwrap().to_utf8().unwrap(), "v1"); + }).unwrap() + }; + + let handler_2 = unsafe { + thread::Builder::new().spawn_unchecked(|| { + assert_eq!(snapshot.get(b"k2").unwrap().unwrap().to_utf8().unwrap(), "v2"); + }).unwrap() + }; + + handler_1.join().unwrap(); + handler_2.join().unwrap(); + } +} + #[test] fn set_option_test() { let path = DBPath::new("_rust_rocksdb_set_optionstest"); From c8d9f6e56f17d546355960e152e38337089fc0d2 Mon Sep 17 00:00:00 2001 From: pavel-mukhanov Date: Mon, 1 Jul 2019 13:51:12 +0300 Subject: [PATCH 05/20] Fix fmt. --- tests/test_db.rs | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/tests/test_db.rs b/tests/test_db.rs index f0ec54c..151513b 100644 --- a/tests/test_db.rs +++ b/tests/test_db.rs @@ -22,8 +22,8 @@ mod util; use libc::size_t; use rocksdb::{DBVector, Error, IteratorMode, Options, WriteBatch, DB}; -use util::DBPath; use std::thread; +use util::DBPath; #[test] fn test_db_vector() { @@ -180,15 +180,25 @@ fn sync_snapshot_test() { // Unsafe here is safe, because `handler.join()` is called at the end of the // method to ensure that snapshot will not outlive database. let handler_1 = unsafe { - thread::Builder::new().spawn_unchecked(|| { - assert_eq!(snapshot.get(b"k1").unwrap().unwrap().to_utf8().unwrap(), "v1"); - }).unwrap() + thread::Builder::new() + .spawn_unchecked(|| { + assert_eq!( + snapshot.get(b"k1").unwrap().unwrap().to_utf8().unwrap(), + "v1" + ); + }) + .unwrap() }; let handler_2 = unsafe { - thread::Builder::new().spawn_unchecked(|| { - assert_eq!(snapshot.get(b"k2").unwrap().unwrap().to_utf8().unwrap(), "v2"); - }).unwrap() + thread::Builder::new() + .spawn_unchecked(|| { + assert_eq!( + snapshot.get(b"k2").unwrap().unwrap().to_utf8().unwrap(), + "v2" + ); + }) + .unwrap() }; handler_1.join().unwrap(); From 7449fda48e7b82d1ec57a418073a3ba7aa558f37 Mon Sep 17 00:00:00 2001 From: pavel-mukhanov Date: Mon, 1 Jul 2019 14:29:54 +0300 Subject: [PATCH 06/20] Rewrite sync_snapshot_test without unchecked threads. --- tests/test_db.rs | 71 ++++++++++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 38 deletions(-) diff --git a/tests/test_db.rs b/tests/test_db.rs index 151513b..d8403b3 100644 --- a/tests/test_db.rs +++ b/tests/test_db.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![feature(thread_spawn_unchecked)] - extern crate libc; extern crate rocksdb; @@ -21,8 +19,8 @@ mod util; use libc::size_t; -use rocksdb::{DBVector, Error, IteratorMode, Options, WriteBatch, DB}; -use std::thread; +use rocksdb::{DBVector, Error, IteratorMode, Options, Snapshot, WriteBatch, DB}; +use std::{mem, thread}; use util::DBPath; #[test] @@ -166,44 +164,41 @@ fn snapshot_test() { } } +struct SnapshotWrapper { + snapshot: Snapshot<'static>, +} + +impl SnapshotWrapper { + fn new(db: &DB) -> Self { + Self { + snapshot: unsafe { mem::transmute(db.snapshot()) }, + } + } + + fn check(&self, key: K, value: &str) -> bool + where + K: AsRef<[u8]>, + { + self.snapshot.get(key).unwrap().unwrap().to_utf8().unwrap() == value + } +} + #[test] fn sync_snapshot_test() { let path = DBPath::new("_rust_rocksdb_sync_snapshottest"); - { - let db = DB::open_default(&path).unwrap(); + let db = DB::open_default(&path).unwrap(); - assert!(db.put(b"k1", b"v1").is_ok()); - assert!(db.put(b"k2", b"v2").is_ok()); - - let snapshot = db.snapshot(); - - // Unsafe here is safe, because `handler.join()` is called at the end of the - // method to ensure that snapshot will not outlive database. - let handler_1 = unsafe { - thread::Builder::new() - .spawn_unchecked(|| { - assert_eq!( - snapshot.get(b"k1").unwrap().unwrap().to_utf8().unwrap(), - "v1" - ); - }) - .unwrap() - }; - - let handler_2 = unsafe { - thread::Builder::new() - .spawn_unchecked(|| { - assert_eq!( - snapshot.get(b"k2").unwrap().unwrap().to_utf8().unwrap(), - "v2" - ); - }) - .unwrap() - }; - - handler_1.join().unwrap(); - handler_2.join().unwrap(); - } + assert!(db.put(b"k1", b"v1").is_ok()); + assert!(db.put(b"k2", b"v2").is_ok()); + + let wrapper = SnapshotWrapper::new(&db); + let handler_1 = thread::spawn(move || wrapper.check("k1", "v1")); + + let wrapper = SnapshotWrapper::new(&db); + let handler_2 = thread::spawn(move || wrapper.check("k2", "v2")); + + assert!(handler_1.join().unwrap()); + assert!(handler_2.join().unwrap()); } #[test] From bc63f2b057e7015e8e34444d8c86c68aa58d11d4 Mon Sep 17 00:00:00 2001 From: pavel-mukhanov Date: Mon, 1 Jul 2019 14:34:13 +0300 Subject: [PATCH 07/20] Clone snapshot in test instead of create new. --- tests/test_db.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/test_db.rs b/tests/test_db.rs index d8403b3..4240d96 100644 --- a/tests/test_db.rs +++ b/tests/test_db.rs @@ -20,6 +20,7 @@ mod util; use libc::size_t; use rocksdb::{DBVector, Error, IteratorMode, Options, Snapshot, WriteBatch, DB}; +use std::sync::Arc; use std::{mem, thread}; use util::DBPath; @@ -164,14 +165,15 @@ fn snapshot_test() { } } +#[derive(Clone)] struct SnapshotWrapper { - snapshot: Snapshot<'static>, + snapshot: Arc>, } impl SnapshotWrapper { fn new(db: &DB) -> Self { Self { - snapshot: unsafe { mem::transmute(db.snapshot()) }, + snapshot: Arc::new(unsafe { mem::transmute(db.snapshot()) }), } } @@ -192,10 +194,11 @@ fn sync_snapshot_test() { assert!(db.put(b"k2", b"v2").is_ok()); let wrapper = SnapshotWrapper::new(&db); - let handler_1 = thread::spawn(move || wrapper.check("k1", "v1")); + let wrapper_1 = wrapper.clone(); + let handler_1 = thread::spawn(move || wrapper_1.check("k1", "v1")); - let wrapper = SnapshotWrapper::new(&db); - let handler_2 = thread::spawn(move || wrapper.check("k2", "v2")); + let wrapper_2 = wrapper.clone(); + let handler_2 = thread::spawn(move || wrapper_2.check("k2", "v2")); assert!(handler_1.join().unwrap()); assert!(handler_2.join().unwrap()); From fbc8c4ccb382e97e53efd41342e49fa091739ac1 Mon Sep 17 00:00:00 2001 From: wjhuang2016 Date: Mon, 1 Jul 2019 20:16:55 +0800 Subject: [PATCH 08/20] set_max_write_buffer_number --- src/db_options.rs | 35 ++++++++++------------------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/src/db_options.rs b/src/db_options.rs index 0a9b4f7..43c7092 100644 --- a/src/db_options.rs +++ b/src/db_options.rs @@ -603,30 +603,15 @@ impl Options { } } - /// Sets the total maximum number of write buffers to maintain in memory including - /// copies of buffers that have already been flushed. Unlike - /// max_write_buffer_number, this parameter does not affect flushing. - /// This controls the minimum amount of write history that will be available - /// in memory for conflict checking when Transactions are used. - /// - /// When using an OptimisticTransactionDB: - /// If this value is too low, some transactions may fail at commit time due - /// to not being able to determine whether there were any write conflicts. - /// - /// When using a TransactionDB: - /// If Transaction::SetSnapshot is used, TransactionDB will read either - /// in-memory write buffers or SST files to do write-conflict checking. - /// Increasing this value can reduce the number of reads to SST files - /// done for conflict detection. - /// - /// Setting this value to `0` will cause write buffers to be freed immediately - /// after they are flushed. - /// If this value is set to `-1`, 'max_write_buffer_number' will be used. - /// - /// Default: - /// If using a TransactionDB/OptimisticTransactionDB, the default value will - /// be set to the value of 'max_write_buffer_number' if it is not explicitly - /// set by the user. Otherwise, the default is 0. + /// Sets the maximum number of write buffers that are built up in memory. + /// The default and the minimum number is 2, so that when 1 write buffer + /// is being flushed to storage, new writes can continue to the other + /// write buffer. + /// If max_write_buffer_number > 3, writing will be slowed down to + /// options.delayed_write_rate if we are writing to the last write buffer + /// allowed. + /// + /// Default: `2` /// /// # Example /// @@ -634,7 +619,7 @@ impl Options { /// use rocksdb::Options; /// /// let mut opts = Options::default(); - /// opts.set_min_write_buffer_number(4); + /// opts.set_max_write_buffer_number(4); /// ``` pub fn set_max_write_buffer_number(&mut self, nbuf: c_int) { unsafe { From 892dea0fb7b755dd78d27ac863a823db6bf1d2be Mon Sep 17 00:00:00 2001 From: pavel-mukhanov Date: Tue, 9 Jul 2019 12:49:03 +0300 Subject: [PATCH 09/20] Add changelog and comment. --- CHANGELOG.md | 6 ++++++ src/db.rs | 2 ++ 2 files changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 117b034..be588ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 0.12.3 + +### Changes + +* Added `Sync` and `Send` implementations to `Snapshot` (pavel-mukhanov) + ## 0.12.2 (2019-05-03) ### Changes diff --git a/src/db.rs b/src/db.rs index c799b01..c59a871 100644 --- a/src/db.rs +++ b/src/db.rs @@ -104,6 +104,8 @@ pub struct Snapshot<'a> { inner: *const ffi::rocksdb_snapshot_t, } +/// `Send` and `Sync` implementations for `Snapshot` are safe, because `Snapshot` is +/// immutable and can be safely shared between threads. unsafe impl<'a> Send for Snapshot<'a> {} unsafe impl<'a> Sync for Snapshot<'a> {} From b5cca05e0193adb39085117b745a3f082c59979a Mon Sep 17 00:00:00 2001 From: pavel-mukhanov Date: Tue, 9 Jul 2019 13:14:59 +0300 Subject: [PATCH 10/20] Tiny changelog fix. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index be588ba..91928dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 0.12.3 +## Unreleased ### Changes From d69d1bcd2e9e1a9497bf9334a06da538a4b4cd91 Mon Sep 17 00:00:00 2001 From: Oleksandr Anyshchenko Date: Fri, 19 Jul 2019 11:55:37 +0300 Subject: [PATCH 11/20] Release 0.12.3 --- CHANGELOG.md | 5 ++++- Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 91928dd..49a02da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,11 @@ ## Unreleased -### Changes +### 0.12.3 (2019-07-19) +* Enabled sse4.2/pclmul for accelerated crc32c (yjh0502) +* Added `set_db_write_buffer_size` to the Options API (rnarubin) +* Bumped RocksDB to 6.1.2 (lispy) * Added `Sync` and `Send` implementations to `Snapshot` (pavel-mukhanov) ## 0.12.2 (2019-05-03) diff --git a/Cargo.toml b/Cargo.toml index a2db300..4b0f81f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rocksdb" description = "Rust wrapper for Facebook's RocksDB embeddable database" -version = "0.12.2" +version = "0.12.3" authors = ["Tyler Neely ", "David Greenberg "] license = "Apache-2.0" keywords = ["database", "embedded", "LSM-tree", "persistence"] From 1159f3198780fe3dc548eccec7ed1d102a553aae Mon Sep 17 00:00:00 2001 From: Renar Narubin Date: Fri, 19 Jul 2019 12:34:53 -0700 Subject: [PATCH 12/20] Document the raw_iterator methods --- src/db.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/db.rs b/src/db.rs index d82a8f2..21b8c40 100644 --- a/src/db.rs +++ b/src/db.rs @@ -588,21 +588,25 @@ impl<'a> Snapshot<'a> { DBIterator::new_cf(self.db, cf_handle, &readopts, mode) } + /// Opens a raw iterator over the data in this snapshot, using the default read options. pub fn raw_iterator(&self) -> DBRawIterator { let readopts = ReadOptions::default(); self.raw_iterator_opt(readopts) } + /// Opens a raw iterator over the data in this snapshot under the given column family, using the default read options. pub fn raw_iterator_cf(&self, cf_handle: ColumnFamily) -> Result { let readopts = ReadOptions::default(); self.raw_iterator_cf_opt(cf_handle, readopts) } + /// Opens a raw iterator over the data in this snapshot, using the given read options. pub fn raw_iterator_opt(&self, mut readopts: ReadOptions) -> DBRawIterator { readopts.set_snapshot(self); DBRawIterator::new(self.db, &readopts) } + /// Opens a raw iterator over the data in this snapshot under the given column family, using the given read options. pub fn raw_iterator_cf_opt( &self, cf_handle: ColumnFamily, @@ -1157,16 +1161,24 @@ impl DB { ) } + /// Opens a raw iterator over the database, using the default read options pub fn raw_iterator(&self) -> DBRawIterator { let opts = ReadOptions::default(); DBRawIterator::new(self, &opts) } + /// Opens a raw iterator over the given column family, using the default read options pub fn raw_iterator_cf(&self, cf_handle: ColumnFamily) -> Result { let opts = ReadOptions::default(); DBRawIterator::new_cf(self, cf_handle, &opts) } + /// Opens a raw iterator over the database, using the given read options + pub fn raw_iterator_opt(&self, readopts: &ReadOptions) -> DBRawIterator { + DBRawIterator::new(self, readopts) + } + + /// Opens a raw iterator over the given column family, using the given read options pub fn raw_iterator_cf_opt( &self, cf_handle: ColumnFamily, From b117d44dfca35617b6299f615a5d61bd6bc2c672 Mon Sep 17 00:00:00 2001 From: Oleksandr Anyshchenko Date: Mon, 22 Jul 2019 09:35:12 +0300 Subject: [PATCH 13/20] Add an entry in CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 49a02da..de46e50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ * Added `set_db_write_buffer_size` to the Options API (rnarubin) * Bumped RocksDB to 6.1.2 (lispy) * Added `Sync` and `Send` implementations to `Snapshot` (pavel-mukhanov) +* Added `raw_iterator_cf_opt` to the DB API (rnarubin) ## 0.12.2 (2019-05-03) From d7cfbdcb6d2a338bdf8f6b5ce01deab24c4a314f Mon Sep 17 00:00:00 2001 From: John-John Tedro Date: Mon, 22 Jul 2019 23:06:18 +0200 Subject: [PATCH 14/20] Build RocksDB with support for unicode filenames on Windows --- librocksdb-sys/build.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/librocksdb-sys/build.rs b/librocksdb-sys/build.rs index d8549f3..2170d1b 100644 --- a/librocksdb-sys/build.rs +++ b/librocksdb-sys/build.rs @@ -123,6 +123,7 @@ fn build_rocksdb() { link("rpcrt4", false); link("shlwapi", false); config.define("OS_WIN", Some("1")); + config.define("ROCKSDB_WINDOWS_UTF8_FILENAMES", Some("1")); // Remove POSIX-specific sources lib_sources = lib_sources From 54aff620fa0b3cdb2453afe7e9ba2ec335f46564 Mon Sep 17 00:00:00 2001 From: Andrew Dirksen Date: Mon, 26 Aug 2019 14:20:14 -0700 Subject: [PATCH 15/20] enable cross compilation by fix #322 --- librocksdb-sys/build.rs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/librocksdb-sys/build.rs b/librocksdb-sys/build.rs index 2170d1b..273573c 100644 --- a/librocksdb-sys/build.rs +++ b/librocksdb-sys/build.rs @@ -45,6 +45,8 @@ fn bindgen_rocksdb() { } fn build_rocksdb() { + let target = env::var("TARGET").unwrap(); + let mut config = cc::Build::new(); config.include("rocksdb/include/"); config.include("rocksdb/"); @@ -90,7 +92,7 @@ fn build_rocksdb() { .filter(|file| *file != "util/build_version.cc") .collect::>(); - if cfg!(target_arch = "x86_64") { + if target.contains("x86_64") { // This is needed to enable hardware CRC32C. Technically, SSE 4.2 is // only available since Intel Nehalem (about 2010) and AMD Bulldozer // (about 2011). @@ -102,24 +104,24 @@ fn build_rocksdb() { config.flag("-mpclmul"); } - if cfg!(target_os = "macos") { + if target.contains("darwin") { config.define("OS_MACOSX", Some("1")); config.define("ROCKSDB_PLATFORM_POSIX", Some("1")); config.define("ROCKSDB_LIB_IO_POSIX", Some("1")); } - if cfg!(target_os = "linux") { + if target.contains("linux") { config.define("OS_LINUX", Some("1")); config.define("ROCKSDB_PLATFORM_POSIX", Some("1")); config.define("ROCKSDB_LIB_IO_POSIX", Some("1")); // COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp" } - if cfg!(target_os = "freebsd") { + if target.contains("freebsd") { config.define("OS_FREEBSD", Some("1")); config.define("ROCKSDB_PLATFORM_POSIX", Some("1")); config.define("ROCKSDB_LIB_IO_POSIX", Some("1")); } - if cfg!(target_os = "windows") { + if target.contains("windows") { link("rpcrt4", false); link("shlwapi", false); config.define("OS_WIN", Some("1")); @@ -144,7 +146,7 @@ fn build_rocksdb() { lib_sources.push("port/win/win_thread.cc"); } - if cfg!(target_env = "msvc") { + if target.contains("msvc") { config.flag("-EHsc"); } else { config.flag("-std=c++11"); @@ -165,13 +167,15 @@ fn build_rocksdb() { } fn build_snappy() { + let target = env::var("TARGET").unwrap(); + let mut config = cc::Build::new(); config.include("snappy/"); config.include("."); config.define("NDEBUG", Some("1")); - if cfg!(target_env = "msvc") { + if target.contains("msvc") { config.flag("-EHsc"); } else { config.flag("-std=c++11"); From de431e3e5ec5b01b00b39fa00c73c7bed7fe7fcd Mon Sep 17 00:00:00 2001 From: Andrew Dirksen Date: Tue, 27 Aug 2019 09:01:07 -0700 Subject: [PATCH 16/20] fmt --- librocksdb-sys/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/librocksdb-sys/build.rs b/librocksdb-sys/build.rs index 273573c..1b1567a 100644 --- a/librocksdb-sys/build.rs +++ b/librocksdb-sys/build.rs @@ -168,7 +168,7 @@ fn build_rocksdb() { fn build_snappy() { let target = env::var("TARGET").unwrap(); - + let mut config = cc::Build::new(); config.include("snappy/"); config.include("."); From 9e75179e771fa08a74503eb34593df8ecb496d70 Mon Sep 17 00:00:00 2001 From: rnarubin Date: Sat, 7 Sep 2019 05:59:55 -0700 Subject: [PATCH 17/20] Implement Send and Sync for useful types (#318) The Send and Sync traits aren't auto-implemented on the many types which wrap a raw pointer to some RocksDB struct. Most of these types are nevertheless safe to use in Send and Sync context, so they should be marked as such --- src/db.rs | 12 ++++++++++-- src/db_options.rs | 10 ++++++++++ src/lib.rs | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 2 deletions(-) diff --git a/src/db.rs b/src/db.rs index 96a3f9a..9af7e72 100644 --- a/src/db.rs +++ b/src/db.rs @@ -199,8 +199,6 @@ pub struct DBIterator<'a> { just_seeked: bool, } -unsafe impl<'a> Send for DBIterator<'a> {} - pub enum Direction { Forward, Reverse, @@ -1854,6 +1852,16 @@ impl Default for ReadOptions { } } +// Safety note: auto-implementing Send on most db-related types is prevented by the inner FFI +// pointer. In most cases, however, this pointer is Send-safe because it is never aliased and +// rocksdb internally does not rely on thread-local information for its user-exposed types. +unsafe impl<'a> Send for DBRawIterator<'a> {} +unsafe impl Send for ReadOptions {} + +// Sync is similarly safe for many types because they do not expose interior mutability, and their +// use within the rocksdb library is generally behind a const reference +unsafe impl Sync for ReadOptions {} + /// Vector of bytes stored in the database. /// /// This is a `C` allocated byte array and a length value. diff --git a/src/db_options.rs b/src/db_options.rs index 43c7092..0baca2e 100644 --- a/src/db_options.rs +++ b/src/db_options.rs @@ -34,7 +34,17 @@ pub fn new_cache(capacity: size_t) -> *mut ffi::rocksdb_cache_t { unsafe { ffi::rocksdb_cache_create_lru(capacity) } } +// Safety note: auto-implementing Send on most db-related types is prevented by the inner FFI +// pointer. In most cases, however, this pointer is Send-safe because it is never aliased and +// rocksdb internally does not rely on thread-local information for its user-exposed types. unsafe impl Send for Options {} +unsafe impl Send for WriteOptions {} +unsafe impl Send for BlockBasedOptions {} +// Sync is similarly safe for many types because they do not expose interior mutability, and their +// use within the rocksdb library is generally behind a const reference +unsafe impl Sync for Options {} +unsafe impl Sync for WriteOptions {} +unsafe impl Sync for BlockBasedOptions {} impl Drop for Options { fn drop(&mut self) { diff --git a/src/lib.rs b/src/lib.rs index e6fbd99..67e6bb9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -290,3 +290,49 @@ pub struct ColumnFamily<'a> { } unsafe impl<'a> Send for ColumnFamily<'a> {} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn is_send() { + // test (at compile time) that certain types implement the auto-trait Send, either directly for + // pointer-wrapping types or transitively for types with all Send fields + + fn is_send() { + // dummy function just used for its parameterized type bound + } + + is_send::(); + is_send::>(); + is_send::>(); + is_send::(); + is_send::(); + is_send::(); + is_send::(); + is_send::(); + is_send::(); + is_send::(); + is_send::(); + } + + #[test] + fn is_sync() { + // test (at compile time) that certain types implement the auto-trait Sync + + fn is_sync() { + // dummy function just used for its parameterized type bound + } + + is_sync::(); + is_sync::(); + is_sync::(); + is_sync::(); + is_sync::(); + is_sync::(); + is_sync::(); + is_sync::(); + } + +} From c5e8c9f5ef4bf8b85877b4dbb8c7c83f51e04c1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Sat, 7 Sep 2019 15:04:08 +0200 Subject: [PATCH 18/20] fix: Make the set_iterate_upper_bound ReadOptions method unsafe (#309) --- src/db.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/db.rs b/src/db.rs index 9af7e72..2fd59c5 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1804,8 +1804,15 @@ impl ReadOptions { } } - /// Set the upper bound for an iterator, and the upper bound itself is not included on the iteration result. - pub fn set_iterate_upper_bound>(&mut self, key: K) { + /// Set the upper bound for an iterator. + /// The upper bound itself is not included on the iteration result. + /// + /// # Safety + /// + /// This function will store a clone of key and will give a raw pointer of it to the + /// underlying C++ API, therefore, when given to any other [`DB`] method you must ensure + /// that this [`ReadOptions`] value does not leave the scope too early (e.g. `DB::iterator_cf_opt`). + pub unsafe fn set_iterate_upper_bound>(&mut self, key: K) { let key = key.as_ref(); unsafe { ffi::rocksdb_readoptions_set_iterate_upper_bound( From 932a70e1f170e6bb7c76dc6e241920e021600229 Mon Sep 17 00:00:00 2001 From: Ilya Bogdanov Date: Sat, 7 Sep 2019 16:26:40 +0300 Subject: [PATCH 19/20] Add `DB::latest_sequence_number` method. (#326) --- CHANGELOG.md | 1 + src/db.rs | 5 +++++ tests/test_db.rs | 11 +++++++++++ 3 files changed, 17 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index de46e50..85e2c0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ * Bumped RocksDB to 6.1.2 (lispy) * Added `Sync` and `Send` implementations to `Snapshot` (pavel-mukhanov) * Added `raw_iterator_cf_opt` to the DB API (rnarubin) +* Added `DB::latest_sequence_number` method (vitvakatu) ## 0.12.2 (2019-05-03) diff --git a/src/db.rs b/src/db.rs index 2fd59c5..6e5959e 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1560,6 +1560,11 @@ impl DB { Err(e) => Err(e), } } + + /// The sequence number of the most recent transaction. + pub fn latest_sequence_number(&self) -> u64 { + unsafe { ffi::rocksdb_get_latest_sequence_number(self.inner) } + } } impl WriteBatch { diff --git a/tests/test_db.rs b/tests/test_db.rs index 4240d96..a2da749 100644 --- a/tests/test_db.rs +++ b/tests/test_db.rs @@ -240,3 +240,14 @@ fn set_option_test() { db.set_options(&multiple_options).unwrap(); } } + +#[test] +fn test_sequence_number() { + let path = DBPath::new("_rust_rocksdb_test_sequence_number"); + { + let db = DB::open_default(&path).unwrap(); + assert_eq!(db.latest_sequence_number(), 0); + db.put(b"key", b"value"); + assert_eq!(db.latest_sequence_number(), 1); + } +} From f435fa7e75b195cf14e7a44c964fa5c0cb53ef21 Mon Sep 17 00:00:00 2001 From: rnarubin Date: Sun, 8 Sep 2019 01:49:31 -0700 Subject: [PATCH 20/20] Expose the `status` method on iterators (#302) --- src/db.rs | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/db.rs b/src/db.rs index 6e5959e..1cd361f 100644 --- a/src/db.rs +++ b/src/db.rs @@ -235,11 +235,28 @@ impl<'a> DBRawIterator<'a> { } } - /// Returns true if the iterator is valid. + /// Returns `true` if the iterator is valid. An iterator is invalidated when + /// it reaches the end of its defined range, or when it encounters an error. + /// + /// To check whether the iterator encountered an error after `valid` has + /// returned `false`, use the [`status`](DBRawIterator::status) method. `status` will never + /// return an error when `valid` is `true`. pub fn valid(&self) -> bool { unsafe { ffi::rocksdb_iter_valid(self.inner) != 0 } } + /// Returns an error `Result` if the iterator has encountered an error + /// during operation. When an error is encountered, the iterator is + /// invalidated and [`valid`](DBRawIterator::valid) will return `false` when called. + /// + /// Performing a seek will discard the current status. + pub fn status(&self) -> Result<(), Error> { + unsafe { + ffi_try!(ffi::rocksdb_iter_get_error(self.inner,)); + } + Ok(()) + } + /// Seeks to the first key in the database. /// /// # Examples @@ -515,9 +532,15 @@ impl<'a> DBIterator<'a> { self.just_seeked = true; } + /// See [`valid`](DBRawIterator::valid) pub fn valid(&self) -> bool { self.raw.valid() } + + /// See [`status`](DBRawIterator::status) + pub fn status(&self) -> Result<(), Error> { + self.raw.status() + } } impl<'a> Iterator for DBIterator<'a> {