Merge remote-tracking branch 'prime/master' into ergonomics-and-safety

CHANGELOG.md

@@ -1,11 +1,11 @@
 # Changelog
 
-## 0.11.0 (2018-01-10)
+## 0.11.0 (2019-01-10)
 
 ### Announcements
 
 * This is the first release under the new [Maintainership](MAINTAINERSHIP.md) model.
-  Three contributors have been selected to help maintain this library -- Oleksandr Anyshchenko ([@aleksus](https://github.com/aleksuss)), Jordan Terrell ([@iSynaptic](https://github.com/iSynaptic)), and Ilya Bogdanov ([@vitvakatu](https://github.com/vitvakatu)). Many thanks to Tyler Neely ([@spacejam](https://github.com/spacejam)) for your support while taking on this new role.
+  Three contributors have been selected to help maintain this library -- Oleksandr Anyshchenko ([@aleksuss](https://github.com/aleksuss)), Jordan Terrell ([@iSynaptic](https://github.com/iSynaptic)), and Ilya Bogdanov ([@vitvakatu](https://github.com/vitvakatu)). Many thanks to Tyler Neely ([@spacejam](https://github.com/spacejam)) for your support while taking on this new role.
 
 * A [gitter.im chat room](https://gitter.im/rust-rocksdb/Lobby) has been created. Although it's not guaranteed to be "staffed", it may help to collaborate on changes to `rust-rocksdb`.
 

librocksdb-sys/build.rs

@@ -231,6 +231,7 @@ fn build_zlib() {
         }
     }
 
+    compiler.flag_if_supported("-Wno-implicit-function-declaration");
     compiler.opt_level(3);
     compiler.compile("libz.a");
 }
@@ -251,6 +252,7 @@ fn build_bzip2() {
         .define("_FILE_OFFSET_BITS", Some("64"))
         .define("BZ_NO_STDIO", None);
 
+    compiler.extra_warnings(false);
     compiler.opt_level(3);
     compiler.compile("libbz2.a");
 }

src/compaction_filter.rs

@@ -101,3 +101,36 @@ where
         }
     }
 }
+
+#[cfg(test)]
+#[allow(unused_variables)]
+fn test_filter(level: u32, key: &[u8], value: &[u8]) -> Decision {
+    use self::Decision::*;
+    match key.first() {
+        Some(&b'_') => Remove,
+        Some(&b'%') => Change(b"secret"),
+        _ => Keep,
+    }
+}
+
+#[test]
+fn compaction_filter_test() {
+    use {Options, DB};
+
+    let path = "_rust_rocksdb_filtertest";
+    let mut opts = Options::default();
+    opts.create_if_missing(true);
+    opts.set_compaction_filter("test", test_filter);
+    {
+        let db = DB::open(&opts, path).unwrap();
+        let _ = db.put(b"k1", b"a");
+        let _ = db.put(b"_k", b"b");
+        let _ = db.put(b"%k", b"c");
+        db.compact_range(None::<&[u8]>, None::<&[u8]>);
+        assert_eq!(&*db.get(b"k1").unwrap().unwrap(), b"a");
+        assert!(db.get(b"_k").unwrap().is_none());
+        assert_eq!(&*db.get(b"%k").unwrap().unwrap(), b"secret");
+    }
+    let result = DB::destroy(&opts, path);
+    assert!(result.is_ok());
+}

src/db.rs

@@ -64,16 +64,18 @@ pub enum DBRecoveryMode {
 /// Making an atomic commit of several writes:
 ///
 /// ```
-/// use rocksdb::{DB, WriteBatch};
+/// use rocksdb::{DB, Options, WriteBatch};
 ///
-/// let db = DB::open_default("path/for/rocksdb/storage1").unwrap();
+/// let path = "_path_for_rocksdb_storage1";
 /// {
+///     let db = DB::open_default(path).unwrap();
 ///     let mut batch = WriteBatch::default();
 ///     batch.put(b"my key", b"my value");
 ///     batch.put(b"key2", b"value2");
 ///     batch.put(b"key3", b"value3");
 ///     db.write(batch); // Atomically commits the batch
 /// }
+/// let _ = DB::destroy(&Options::default(), path);
 /// ```
 pub struct WriteBatch {
     inner: *mut ffi::rocksdb_writebatch_t,
@@ -86,11 +88,15 @@ pub struct ReadOptions {
 /// A consistent view of the database at the point of creation.
 ///
 /// ```
-/// use rocksdb::{DB, IteratorMode};
+/// use rocksdb::{DB, IteratorMode, Options};
 ///
-/// let db = DB::open_default("path/for/rocksdb/storage3").unwrap();
+/// let path = "_path_for_rocksdb_storage3";
+/// {
+///     let db = DB::open_default(path).unwrap();
 ///     let snapshot = db.snapshot(); // Creates a longer-term snapshot of the DB, but closed when goes out of scope
 ///     let mut iter = snapshot.iterator(IteratorMode::Start); // Make as many iterators as you'd like from one snapshot
+/// }
+/// let _ = DB::destroy(&Options::default(), path);
 /// ```
 ///
 pub struct Snapshot<'a> {
@@ -107,9 +113,11 @@ pub struct Snapshot<'a> {
 /// widely recognised Rust idioms.
 ///
 /// ```
-/// use rocksdb::DB;
+/// use rocksdb::{DB, Options};
 ///
-/// let mut db = DB::open_default("path/for/rocksdb/storage4").unwrap();
+/// let path = "_path_for_rocksdb_storage4";
+/// {
+///     let db = DB::open_default(path).unwrap();
 ///     let mut iter = db.raw_iterator();
 ///
 ///     // Forwards iteration
@@ -141,6 +149,8 @@ pub struct Snapshot<'a> {
 ///         println!("Saw {:?} {:?}", iter.key(), iter.value());
 ///         iter.prev();
 ///     }
+/// }
+/// let _ = DB::destroy(&Options::default(), path);
 /// ```
 pub struct DBRawIterator<'a> {
     inner: *mut ffi::rocksdb_iterator_t,
@@ -151,9 +161,11 @@ pub struct DBRawIterator<'a> {
 /// ranges and direction.
 ///
 /// ```
-/// use rocksdb::{DB, Direction, IteratorMode};
+/// use rocksdb::{DB, Direction, IteratorMode, Options};
 ///
-/// let mut db = DB::open_default("path/for/rocksdb/storage2").unwrap();
+/// let path = "_path_for_rocksdb_storage2";
+/// {
+///     let db = DB::open_default(path).unwrap();
 ///     let mut iter = db.iterator(IteratorMode::Start); // Always iterates forward
 ///     for (key, value) in iter {
 ///         println!("Saw {:?} {:?}", key, value);
@@ -173,6 +185,8 @@ pub struct DBRawIterator<'a> {
 ///     for (key, value) in iter {
 ///         println!("Saw {:?} {:?}", key, value);
 ///     }
+/// }
+/// let _ = DB::destroy(&Options::default(), path);
 /// ```
 pub struct DBIterator<'a> {
     raw: DBRawIterator<'a>,
@@ -228,23 +242,22 @@ impl<'a> DBRawIterator<'a> {
     /// # Examples
     ///
     /// ```rust
-    /// use rocksdb::DB;
+    /// use rocksdb::{DB, Options};
     ///
-    /// let mut db = DB::open_default("path/for/rocksdb/storage5").unwrap();
+    /// let path = "_path_for_rocksdb_storage5";
+    /// {
+    ///     let db = DB::open_default(path).unwrap();
     ///     let mut iter = db.raw_iterator();
     ///
     ///     // Iterate all keys from the start in lexicographic order
-    ///
     ///     iter.seek_to_first();
     ///
     ///     while iter.valid() {
     ///         println!("{:?} {:?}", iter.key(), iter.value());
-    ///
     ///         iter.next();
     ///     }
     ///
     ///     // Read just the first key
-    ///
     ///     iter.seek_to_first();
     ///
     ///     if iter.valid() {
@@ -252,6 +265,8 @@ impl<'a> DBRawIterator<'a> {
     ///     } else {
     ///         // There are no keys in the database
     ///     }
+    /// }
+    /// let _ = DB::destroy(&Options::default(), path);
     /// ```
     pub fn seek_to_first(&mut self) {
         unsafe {
@@ -264,23 +279,22 @@ impl<'a> DBRawIterator<'a> {
     /// # Examples
     ///
     /// ```rust
-    /// use rocksdb::DB;
+    /// use rocksdb::{DB, Options};
     ///
-    /// let mut db = DB::open_default("path/for/rocksdb/storage6").unwrap();
+    /// let path = "_path_for_rocksdb_storage6";
+    /// {
+    ///     let db = DB::open_default(path).unwrap();
     ///     let mut iter = db.raw_iterator();
     ///
     ///     // Iterate all keys from the end in reverse lexicographic order
-    ///
     ///     iter.seek_to_last();
     ///
     ///     while iter.valid() {
     ///         println!("{:?} {:?}", iter.key(), iter.value());
-    ///
     ///         iter.prev();
     ///     }
     ///
     ///     // Read just the last key
-    ///
     ///     iter.seek_to_last();
     ///
     ///     if iter.valid() {
@@ -288,6 +302,8 @@ impl<'a> DBRawIterator<'a> {
     ///     } else {
     ///         // There are no keys in the database
     ///     }
+    /// }
+    /// let _ = DB::destroy(&Options::default(), path);
     /// ```
     pub fn seek_to_last(&mut self) {
         unsafe {
@@ -303,13 +319,14 @@ impl<'a> DBRawIterator<'a> {
     /// # Examples
     ///
     /// ```rust
-    /// use rocksdb::DB;
+    /// use rocksdb::{DB, Options};
     ///
-    /// let mut db = DB::open_default("path/for/rocksdb/storage7").unwrap();
+    /// let path = "_path_for_rocksdb_storage7";
+    /// {
+    ///     let db = DB::open_default(path).unwrap();
     ///     let mut iter = db.raw_iterator();
     ///
     ///     // Read the first key that starts with 'a'
-    ///
     ///     iter.seek(b"a");
     ///
     ///     if iter.valid() {
@@ -317,6 +334,8 @@ impl<'a> DBRawIterator<'a> {
     ///     } else {
     ///         // There are no keys in the database
     ///     }
+    /// }
+    /// let _ = DB::destroy(&Options::default(), path);
     /// ```
     pub fn seek<K: AsRef<[u8]>>(&mut self, key: K) {
         let key = key.as_ref();
@@ -339,13 +358,14 @@ impl<'a> DBRawIterator<'a> {
     /// # Examples
     ///
     /// ```rust
-    /// use rocksdb::DB;
+    /// use rocksdb::{DB, Options};
     ///
-    /// let mut db = DB::open_default("path/for/rocksdb/storage8").unwrap();
+    /// let path = "_path_for_rocksdb_storage8";
+    /// {
+    ///     let db = DB::open_default(path).unwrap();
     ///     let mut iter = db.raw_iterator();
     ///
     ///     // Read the last key that starts with 'a'
-    ///
     ///     iter.seek_for_prev(b"b");
     ///
     ///     if iter.valid() {
@@ -353,6 +373,9 @@ impl<'a> DBRawIterator<'a> {
     ///     } else {
     ///         // There are no keys in the database
     ///     }
+    /// }
+    /// let _ = DB::destroy(&Options::default(), path);
+    /// ```
     pub fn seek_for_prev<K: AsRef<[u8]>>(&mut self, key: K) {
         let key = key.as_ref();
 
@@ -673,9 +696,7 @@ impl DB {
 
         if let Err(e) = fs::create_dir_all(&path) {
             return Err(Error::new(format!(
-                "Failed to create RocksDB\
+                "Failed to create RocksDB directory: `{:?}`.", e
-                 directory: `{:?}`.",
-                e
             )));
         }
 
@@ -752,17 +773,7 @@ impl DB {
     }
 
     pub fn list_cf<P: AsRef<Path>>(opts: &Options, path: P) -> Result<Vec<String>, Error> {
-        let cpath = match CString::new(path.as_ref().to_string_lossy().as_bytes()) {
+        let cpath = to_cpath(path)?;
-            Ok(c) => c,
-            Err(_) => {
-                return Err(Error::new(
-                    "Failed to convert path to CString \
-                     when opening DB."
-                        .to_owned(),
-                ))
-            }
-        };
-
         let mut length = 0;
 
         unsafe {
@@ -782,7 +793,7 @@ impl DB {
     }
 
     pub fn destroy<P: AsRef<Path>>(opts: &Options, path: P) -> Result<(), Error> {
-        let cpath = CString::new(path.as_ref().to_string_lossy().as_bytes()).unwrap();
+        let cpath = to_cpath(path)?;
         unsafe {
             ffi_try!(ffi::rocksdb_destroy_db(opts.inner, cpath.as_ptr(),));
         }
@@ -790,7 +801,7 @@ impl DB {
     }
 
     pub fn repair<P: AsRef<Path>>(opts: Options, path: P) -> Result<(), Error> {
-        let cpath = CString::new(path.as_ref().to_string_lossy().as_bytes()).unwrap();
+        let cpath = to_cpath(path)?;
         unsafe {
             ffi_try!(ffi::rocksdb_repair_db(opts.inner, cpath.as_ptr(),));
         }
@@ -1379,6 +1390,14 @@ impl WriteBatch {
             Ok(())
         }
     }
+
+    /// Clear all updates buffered in this batch.
+    pub fn clear(&mut self) -> Result<(), Error> {
+        unsafe {
+            ffi::rocksdb_writebatch_clear(self.inner);
+        }
+        Ok(())
+    }
 }
 
 impl Default for WriteBatch {
@@ -1523,3 +1542,187 @@ impl DBVector {
         str::from_utf8(self.deref()).ok()
     }
 }
+
+fn to_cpath<P: AsRef<Path>>(path: P) -> Result<CString, Error> {
+    match CString::new(path.as_ref().to_string_lossy().as_bytes()) {
+        Ok(c) => Ok(c),
+        Err(_) => Err(Error::new(
+            "Failed to convert path to CString when opening DB.".to_owned(),
+        )),
+    }
+}
+
+#[test]
+fn test_db_vector() {
+    use std::mem;
+    let len: size_t = 4;
+    let data: *mut u8 = unsafe { mem::transmute(libc::calloc(len, mem::size_of::<u8>())) };
+    let v = unsafe { DBVector::from_c(data, len) };
+    let ctrl = [0u8, 0, 0, 0];
+    assert_eq!(&*v, &ctrl[..]);
+}
+
+#[test]
+fn external() {
+    let path = "_rust_rocksdb_externaltest";
+    {
+        let db = DB::open_default(path).unwrap();
+        let p = db.put(b"k1", b"v1111");
+        assert!(p.is_ok());
+        let r: Result<Option<DBVector>, Error> = db.get(b"k1");
+        assert!(r.unwrap().unwrap().to_utf8().unwrap() == "v1111");
+        assert!(db.delete(b"k1").is_ok());
+        assert!(db.get(b"k1").unwrap().is_none());
+    }
+    let opts = Options::default();
+    let result = DB::destroy(&opts, path);
+    assert!(result.is_ok());
+}
+
+#[test]
+fn errors_do_stuff() {
+    let path = "_rust_rocksdb_error";
+    {
+        let _db = DB::open_default(path).unwrap();
+        let opts = Options::default();
+        // The DB will still be open when we try to destroy it and the lock should fail.
+        match DB::destroy(&opts, path) {
+            Err(s) => {
+                let message = s.to_string();
+                assert!(message.find("IO error:").is_some());
+                assert!(message.find("_rust_rocksdb_error/LOCK:").is_some());
+            }
+            Ok(_) => panic!("should fail"),
+        }
+    }
+    let opts = Options::default();
+    let result = DB::destroy(&opts, path);
+    assert!(result.is_ok());
+}
+
+#[test]
+fn writebatch_works() {
+    let path = "_rust_rocksdb_writebacktest";
+    {
+        let db = DB::open_default(path).unwrap();
+        {
+            // test put
+            let mut batch = WriteBatch::default();
+            assert!(db.get(b"k1").unwrap().is_none());
+            assert_eq!(batch.len(), 0);
+            assert!(batch.is_empty());
+            let _ = batch.put(b"k1", b"v1111");
+            assert_eq!(batch.len(), 1);
+            assert!(!batch.is_empty());
+            assert!(db.get(b"k1").unwrap().is_none());
+            let p = db.write(batch);
+            assert!(p.is_ok());
+            let r: Result<Option<DBVector>, Error> = db.get(b"k1");
+            assert!(r.unwrap().unwrap().to_utf8().unwrap() == "v1111");
+        }
+        {
+            // test delete
+            let mut batch = WriteBatch::default();
+            let _ = batch.delete(b"k1");
+            assert_eq!(batch.len(), 1);
+            assert!(!batch.is_empty());
+            let p = db.write(batch);
+            assert!(p.is_ok());
+            assert!(db.get(b"k1").unwrap().is_none());
+        }
+        {
+            // test size_in_bytes
+            let mut batch = WriteBatch::default();
+            let before = batch.size_in_bytes();
+            let _ = batch.put(b"k1", b"v1234567890");
+            let after = batch.size_in_bytes();
+            assert!(before + 10 <= after);
+        }
+    }
+    let opts = Options::default();
+    assert!(DB::destroy(&opts, path).is_ok());
+}
+
+#[test]
+fn iterator_test() {
+    let path = "_rust_rocksdb_iteratortest";
+    {
+        let db = DB::open_default(path).unwrap();
+        let p = db.put(b"k1", b"v1111");
+        assert!(p.is_ok());
+        let p = db.put(b"k2", b"v2222");
+        assert!(p.is_ok());
+        let p = db.put(b"k3", b"v3333");
+        assert!(p.is_ok());
+        let iter = db.iterator(IteratorMode::Start);
+        for (k, v) in iter {
+            println!(
+                "Hello {}: {}",
+                str::from_utf8(&*k).unwrap(),
+                str::from_utf8(&*v).unwrap()
+            );
+        }
+    }
+    let opts = Options::default();
+    assert!(DB::destroy(&opts, path).is_ok());
+}
+
+#[test]
+fn snapshot_test() {
+    let path = "_rust_rocksdb_snapshottest";
+    {
+        let db = DB::open_default(path).unwrap();
+        let p = db.put(b"k1", b"v1111");
+        assert!(p.is_ok());
+
+        let snap = db.snapshot();
+        let r: Result<Option<DBVector>, Error> = snap.get(b"k1");
+        assert!(r.unwrap().unwrap().to_utf8().unwrap() == "v1111");
+
+        let p = db.put(b"k2", b"v2222");
+        assert!(p.is_ok());
+
+        assert!(db.get(b"k2").unwrap().is_some());
+        assert!(snap.get(b"k2").unwrap().is_none());
+    }
+    let opts = Options::default();
+    assert!(DB::destroy(&opts, path).is_ok());
+}
+
+#[test]
+fn set_option_test() {
+    let path = "_rust_rocksdb_set_optionstest";
+    {
+        let db = DB::open_default(path).unwrap();
+        // set an option to valid values
+        assert!(db
+            .set_options(&[("disable_auto_compactions", "true")])
+            .is_ok());
+        assert!(db
+            .set_options(&[("disable_auto_compactions", "false")])
+            .is_ok());
+        // invalid names/values should result in an error
+        assert!(db
+            .set_options(&[("disable_auto_compactions", "INVALID_VALUE")])
+            .is_err());
+        assert!(db
+            .set_options(&[("INVALID_NAME", "INVALID_VALUE")])
+            .is_err());
+        // option names/values must not contain NULLs
+        assert!(db
+            .set_options(&[("disable_auto_compactions", "true\0")])
+            .is_err());
+        assert!(db
+            .set_options(&[("disable_auto_compactions\0", "true")])
+            .is_err());
+        // empty options are not allowed
+        assert!(db.set_options(&[]).is_err());
+        // multiple options can be set in a single API call
+        let multiple_options = [
+            ("paranoid_file_checks", "true"),
+            ("report_bg_io_stats", "true"),
+        ];
+        db.set_options(&multiple_options).unwrap();
+    }
+    assert!(DB::destroy(&Options::default(), path).is_ok());
+}

src/lib.rs

@@ -18,9 +18,11 @@
 //! # Examples
 //!
 //! ```
-//!  use rocksdb::DB;
+//! use rocksdb::{DB, Options};
 //! // NB: db is automatically closed at end of lifetime
-//!  let db = DB::open_default("path/for/rocksdb/storage").unwrap();
+//! let path = "_path_for_rocksdb_storage";
+//! {
+//!    let db = DB::open_default(path).unwrap();
 //!    db.put(b"my key", b"my value");
 //!    match db.get(b"my key") {
 //!        Ok(Some(value)) => println!("retrieved value {}", value.to_utf8().unwrap()),
@@ -28,12 +30,16 @@
 //!        Err(e) => println!("operational problem encountered: {}", e),
 //!    }
 //!    db.delete(b"my key").unwrap();
+//! }
+//! let _ = DB::destroy(&Options::default(), path);
 //! ```
 //!
 //! Opening a database and a single column family with custom options:
 //!
 //! ```
 //! use rocksdb::{DB, ColumnFamilyDescriptor, Options};
+//!
+//! let path = "_path_for_rocksdb_storage_with_cfs";
 //! let mut cf_opts = Options::default();
 //! cf_opts.set_max_write_buffer_number(16);
 //! let cf = ColumnFamilyDescriptor::new("cf1", cf_opts);
@@ -41,8 +47,10 @@
 //! let mut db_opts = Options::default();
 //! db_opts.create_missing_column_families(true);
 //! db_opts.create_if_missing(true);
-//!
+//! {
-//! let db = DB::open_cf_descriptors(&db_opts, "path/for/rocksdb/storage_with_cfs", vec![cf]).unwrap();
+//!     let db = DB::open_cf_descriptors(&db_opts, path, vec![cf]).unwrap();
+//! }
+//! let _ = DB::destroy(&db_opts, path);
 //! ```
 //!
 
@@ -212,10 +220,11 @@ pub struct Options {
 /// Making an unsafe write of a batch:
 ///
 /// ```
-/// use rocksdb::{DB, WriteBatch, WriteOptions};
+/// use rocksdb::{DB, Options, WriteBatch, WriteOptions};
-///
-/// let db = DB::open_default("path/for/rocksdb/storageY").unwrap();
 ///
+/// let path = "_path_for_rocksdb_storageY";
+/// {
+///     let db = DB::open_default(path).unwrap();
 ///     let mut batch = WriteBatch::default();
 ///     batch.put(b"my key", b"my value");
 ///     batch.put(b"key2", b"value2");
@@ -226,6 +235,8 @@ pub struct Options {
 ///     write_options.disable_wal(true);
 ///
 ///     db.write_opt(batch, &write_options);
+/// }
+/// let _ = DB::destroy(&Options::default(), path);
 /// ```
 pub struct WriteOptions {
     inner: *mut ffi::rocksdb_writeoptions_t,

src/merge_operator.rs

@@ -38,10 +38,11 @@
 //! }
 //!
 //! fn main() {
-//!    let path = "path/to/rocksdb";
+//!    let path = "_rust_path_to_rocksdb";
 //!    let mut opts = Options::default();
 //!    opts.create_if_missing(true);
 //!    opts.set_merge_operator("test operator", concat_merge, None);
+//!    {
 //!        let db = DB::open(&opts, path).unwrap();
 //!         let p = db.put(b"k1", b"a");
 //!         db.merge(b"k1", b"b");
@@ -51,6 +52,8 @@
 //!         let r = db.get(b"k1");
 //!         assert!(r.unwrap().unwrap().to_utf8().unwrap() == "abcdefg");
 //!    }
+//!    let _ = DB::destroy(&opts, path);
+//! }
 //! ```
 
 use libc::{self, c_char, c_int, c_void, size_t};

tests/test_write_batch.rs

@@ -0,0 +1,27 @@
+// Copyright 2019 Tyler Neely
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+extern crate rocksdb;
+
+use rocksdb::WriteBatch;
+
+#[test]
+fn test_write_batch_clear() {
+    let mut batch = WriteBatch::default();
+    let _ = batch.put(b"1", b"2");
+    assert_eq!(batch.len(), 1);
+    let _ = batch.clear();
+    assert_eq!(batch.len(), 0);
+    assert!(batch.is_empty());
+}