Merge pull request #52 from mykmelez/separate-benches-from-tests

Separate benches from tests

.appveyor.yml

@@ -1,11 +1,21 @@
 environment:
   matrix:
     - TARGET: x86_64-pc-windows-msvc
+      TOOLCHAIN: stable
     - TARGET: i686-pc-windows-msvc
+      TOOLCHAIN: stable
+    - TARGET: x86_64-pc-windows-msvc
+      TOOLCHAIN: beta
+    - TARGET: i686-pc-windows-msvc
+      TOOLCHAIN: beta
+    - TARGET: x86_64-pc-windows-msvc
+      TOOLCHAIN: nightly
+    - TARGET: i686-pc-windows-msvc
+      TOOLCHAIN: nightly
 
 install:
   - curl -sSf -o rustup-init.exe https://win.rustup.rs/
-  - rustup-init.exe -y --default-host %TARGET% --default-toolchain nightly
+  - rustup-init.exe -y --default-host %TARGET% --default-toolchain %TOOLCHAIN%
   - set PATH=%PATH%;C:\Users\appveyor\.cargo\bin
   - rustc -Vv
   - cargo -V
@@ -15,9 +25,11 @@ build_script:
 
 test_script:
   - SET RUST_BACKTRACE=1
-  - cargo test --target %TARGET% --all -v
+  - cargo test --target %TARGET% --all --verbose
-  - cargo test --release --target %TARGET% --all -v
+  - cargo test --release --target %TARGET% --all --verbose
-
+  - if [%TOOLCHAIN%]==[nightly] (
+      cargo bench --target %TARGET% --all --verbose
+    )
 cache:
   - C:\Users\appveyor\.cargo\registry
   - target

.travis.yml

@@ -11,11 +11,14 @@ os:
 rust:
 - 1.30.0
 - stable
+- beta
 - nightly
 
 script:
   - cargo build --verbose
+  - export RUST_BACKTRACE=1
+  - cargo test --all --verbose
+  - cargo test --release --all --verbose
   - if [[ $TRAVIS_RUST_VERSION = nightly* ]]; then
-      env RUST_BACKTRACE=1 cargo test --all -v;
+      cargo bench --all --verbose;
-      env RUST_BACKTRACE=1 cargo test --all -v --release;
     fi

azure-pipelines-template.yml

@@ -32,18 +32,27 @@ jobs:
     displayName: Query rust and cargo versions
   - script: cargo build
     displayName: Build
+  - script: |
+        cargo test --all --verbose
+        cargo test --release --all --verbose
+    displayName: Test
   # Linux and macOS w/nightly toolchain.
   # Ideally we'd only run the script for the nightly toolchain, but I can't
   # figure out how to determine that within the Azure Pipelines conditional.
   - ${{ if ne(parameters.name, 'Windows') }}:
     - script: |
-        if [ "$RUSTUP_TOOLCHAIN" = 'nightly' ]
+        export RUST_BACKTRACE=1
-          then cargo test
+        if [ "$RUSTUP_TOOLCHAIN" = "nightly" ]
+          then cargo bench --all --verbose;
         fi
-      displayName: Test
+      displayName: Bench
   # Windows w/nightly toolchain.
   # Ideally we'd only run the script for the nightly toolchain, but I can't
   # figure out how to determine that within the Azure Pipelines conditional.
   - ${{ if eq(parameters.name, 'Windows') }}:
-    - script: if "%RUSTUP_TOOLCHAIN%" == "nightly" cargo test
+    - script: |
-      displayName: Test
+        SET RUST_BACKTRACE=1
+        if "%RUSTUP_TOOLCHAIN%" == "nightly" (
+          cargo bench --all --verbose
+        )
+      displayName: Bench

benches/cursor.rs

@@ -0,0 +1,106 @@
+#![feature(test)]
+
+extern crate lmdb;
+extern crate lmdb_rkv_sys as ffi;
+extern crate test;
+
+mod utils;
+
+use ffi::*;
+use lmdb::{Cursor, Result, RoCursor, Transaction};
+use std::ptr;
+use test::{Bencher, black_box};
+use utils::*;
+
+/// Benchmark of iterator sequential read performance.
+#[bench]
+fn bench_get_seq_iter(b: &mut Bencher) {
+    let n = 100;
+    let (_dir, env) = setup_bench_db(n);
+    let db = env.open_db(None).unwrap();
+    let txn = env.begin_ro_txn().unwrap();
+
+    b.iter(|| {
+        let mut cursor = txn.open_ro_cursor(db).unwrap();
+        let mut i = 0;
+        let mut count = 0u32;
+
+        for (key, data) in cursor.iter().map(Result::unwrap) {
+            i = i + key.len() + data.len();
+            count = count + 1;
+        }
+        for (key, data) in cursor.iter().filter_map(Result::ok) {
+            i = i + key.len() + data.len();
+            count = count + 1;
+        }
+
+        fn iterate<'a>(cursor: &mut RoCursor) -> Result<()> {
+            let mut i = 0;
+            let mut count = 0u32;
+            for result in cursor.iter() {
+                let (key, data) = result?;
+                i = i + key.len() + data.len();
+                count = count + 1;
+            }
+            Ok(())
+        }
+        iterate(&mut cursor).unwrap();
+
+        black_box(i);
+        assert_eq!(count, n);
+    });
+}
+
+/// Benchmark of cursor sequential read performance.
+#[bench]
+fn bench_get_seq_cursor(b: &mut Bencher) {
+    let n = 100;
+    let (_dir, env) = setup_bench_db(n);
+    let db = env.open_db(None).unwrap();
+    let txn = env.begin_ro_txn().unwrap();
+
+    b.iter(|| {
+        let cursor = txn.open_ro_cursor(db).unwrap();
+        let mut i = 0;
+        let mut count = 0u32;
+
+        while let Ok((key_opt, val)) = cursor.get(None, None, MDB_NEXT) {
+            i += key_opt.map(|key| key.len()).unwrap_or(0) + val.len();
+            count += 1;
+        }
+
+        black_box(i);
+        assert_eq!(count, n);
+    });
+}
+
+/// Benchmark of raw LMDB sequential read performance (control).
+#[bench]
+fn bench_get_seq_raw(b: &mut Bencher) {
+    let n = 100;
+    let (_dir, env) = setup_bench_db(n);
+    let db = env.open_db(None).unwrap();
+
+    let dbi: MDB_dbi = db.dbi();
+    let _txn = env.begin_ro_txn().unwrap();
+    let txn = _txn.txn();
+
+    let mut key = MDB_val { mv_size: 0, mv_data: ptr::null_mut() };
+    let mut data = MDB_val { mv_size: 0, mv_data: ptr::null_mut() };
+    let mut cursor: *mut MDB_cursor = ptr::null_mut();
+
+    b.iter(|| unsafe {
+        mdb_cursor_open(txn, dbi, &mut cursor);
+        let mut i = 0;
+        let mut count = 0u32;
+
+        while mdb_cursor_get(cursor, &mut key, &mut data, MDB_NEXT) == 0 {
+            i += key.mv_size + data.mv_size;
+            count += 1;
+        };
+
+        black_box(i);
+        assert_eq!(count, n);
+        mdb_cursor_close(cursor);
+    });
+}

benches/transaction.rs

@@ -0,0 +1,118 @@
+#![feature(test)]
+
+extern crate libc;
+extern crate lmdb;
+extern crate lmdb_rkv_sys as ffi;
+extern crate rand;
+extern crate test;
+
+mod utils;
+
+use ffi::*;
+use libc::size_t;
+use lmdb::{Transaction, WriteFlags};
+use rand::{Rng, XorShiftRng};
+use std::ptr;
+use test::{Bencher, black_box};
+use utils::*;
+
+#[bench]
+fn bench_get_rand(b: &mut Bencher) {
+    let n = 100u32;
+    let (_dir, env) = setup_bench_db(n);
+    let db = env.open_db(None).unwrap();
+    let txn = env.begin_ro_txn().unwrap();
+
+    let mut keys: Vec<String> = (0..n).map(|n| get_key(n)).collect();
+    XorShiftRng::new_unseeded().shuffle(&mut keys[..]);
+
+    b.iter(|| {
+        let mut i = 0usize;
+        for key in &keys {
+            i = i + txn.get(db, key).unwrap().len();
+        }
+        black_box(i);
+    });
+}
+
+#[bench]
+fn bench_get_rand_raw(b: &mut Bencher) {
+    let n = 100u32;
+    let (_dir, env) = setup_bench_db(n);
+    let db = env.open_db(None).unwrap();
+    let _txn = env.begin_ro_txn().unwrap();
+
+    let mut keys: Vec<String> = (0..n).map(|n| get_key(n)).collect();
+    XorShiftRng::new_unseeded().shuffle(&mut keys[..]);
+
+    let dbi = db.dbi();
+    let txn = _txn.txn();
+
+    let mut key_val: MDB_val = MDB_val { mv_size: 0, mv_data: ptr::null_mut() };
+    let mut data_val: MDB_val = MDB_val { mv_size: 0, mv_data: ptr::null_mut() };
+
+    b.iter(|| unsafe {
+        let mut i: size_t = 0;
+        for key in &keys {
+            key_val.mv_size = key.len() as size_t;
+            key_val.mv_data = key.as_bytes().as_ptr() as *mut _;
+
+            mdb_get(txn, dbi, &mut key_val, &mut data_val);
+
+            i = i + key_val.mv_size;
+        }
+        black_box(i);
+    });
+}
+
+#[bench]
+fn bench_put_rand(b: &mut Bencher) {
+    let n = 100u32;
+    let (_dir, env) = setup_bench_db(0);
+    let db = env.open_db(None).unwrap();
+
+    let mut items: Vec<(String, String)> = (0..n).map(|n| (get_key(n), get_data(n))).collect();
+    XorShiftRng::new_unseeded().shuffle(&mut items[..]);
+
+    b.iter(|| {
+        let mut txn = env.begin_rw_txn().unwrap();
+        for &(ref key, ref data) in items.iter() {
+            txn.put(db, key, data, WriteFlags::empty()).unwrap();
+        }
+        txn.abort();
+    });
+}
+
+#[bench]
+fn bench_put_rand_raw(b: &mut Bencher) {
+    let n = 100u32;
+    let (_dir, _env) = setup_bench_db(0);
+    let db = _env.open_db(None).unwrap();
+
+    let mut items: Vec<(String, String)> = (0..n).map(|n| (get_key(n), get_data(n))).collect();
+    XorShiftRng::new_unseeded().shuffle(&mut items[..]);
+
+    let dbi = db.dbi();
+    let env = _env.env();
+
+    let mut key_val: MDB_val = MDB_val { mv_size: 0, mv_data: ptr::null_mut() };
+    let mut data_val: MDB_val = MDB_val { mv_size: 0, mv_data: ptr::null_mut() };
+
+    b.iter(|| unsafe {
+        let mut txn: *mut MDB_txn = ptr::null_mut();
+        mdb_txn_begin(env, ptr::null_mut(), 0, &mut txn);
+
+        let mut i: ::libc::c_int = 0;
+        for &(ref key, ref data) in items.iter() {
+
+            key_val.mv_size = key.len() as size_t;
+            key_val.mv_data = key.as_bytes().as_ptr() as *mut _;
+            data_val.mv_size = data.len() as size_t;
+            data_val.mv_data = data.as_bytes().as_ptr() as *mut _;
+
+            i += mdb_put(txn, dbi, &mut key_val, &mut data_val, 0);
+        }
+        assert_eq!(0, i);
+        mdb_txn_abort(txn);
+    });
+}

benches/utils.rs

@@ -0,0 +1,28 @@
+extern crate lmdb;
+extern crate tempdir;
+
+use lmdb::{Environment, Transaction, WriteFlags};
+use self::tempdir::TempDir;
+
+pub fn get_key(n: u32) -> String {
+    format!("key{}", n)
+}
+
+pub fn get_data(n: u32) -> String {
+    format!("data{}", n)
+}
+
+pub fn setup_bench_db<'a>(num_rows: u32) -> (TempDir, Environment) {
+    let dir = TempDir::new("test").unwrap();
+    let env = Environment::new().open(dir.path()).unwrap();
+
+    {
+        let db = env.open_db(None).unwrap();
+        let mut txn = env.begin_rw_txn().unwrap();
+        for i in 0..num_rows {
+            txn.put(db, &get_key(i), &get_data(i), WriteFlags::empty()).unwrap();
+        }
+        txn.commit().unwrap();
+    }
+    (dir, env)
+}

src/cursor.rs

@@ -350,17 +350,12 @@ impl <'txn> Iterator for IterDup<'txn> {
 
 #[cfg(test)]
 mod test {
-
-    use std::ptr;
-    use test::{Bencher, black_box};
-
     use tempdir::TempDir;
 
     use environment::*;
     use ffi::*;
     use flags::*;
     use super::*;
-    use test_utils::*;
 
     #[test]
     fn test_get() {
@@ -613,97 +608,4 @@ mod test {
         assert_eq!((Some(&b"key2"[..]), &b"val2"[..]),
                    cursor.get(None, None, MDB_LAST).unwrap());
     }
-
-    /// Benchmark of iterator sequential read performance.
-    #[bench]
-    fn bench_get_seq_iter(b: &mut Bencher) {
-        let n = 100;
-        let (_dir, env) = setup_bench_db(n);
-        let db = env.open_db(None).unwrap();
-        let txn = env.begin_ro_txn().unwrap();
-
-        b.iter(|| {
-            let mut cursor = txn.open_ro_cursor(db).unwrap();
-            let mut i = 0;
-            let mut count = 0u32;
-
-            for (key, data) in cursor.iter().map(Result::unwrap) {
-                i = i + key.len() + data.len();
-                count = count + 1;
-            }
-            for (key, data) in cursor.iter().filter_map(Result::ok) {
-                i = i + key.len() + data.len();
-                count = count + 1;
-            }
-
-            fn iterate<'a>(cursor: &mut RoCursor) -> Result<()> {
-                let mut i = 0;
-                let mut count = 0u32;
-                for result in cursor.iter() {
-                    let (key, data) = result?;
-                    i = i + key.len() + data.len();
-                    count = count + 1;
-                }
-                Ok(())
-            }
-            iterate(&mut cursor).unwrap();
-
-            black_box(i);
-            assert_eq!(count, n);
-        });
-    }
-
-    /// Benchmark of cursor sequential read performance.
-    #[bench]
-    fn bench_get_seq_cursor(b: &mut Bencher) {
-        let n = 100;
-        let (_dir, env) = setup_bench_db(n);
-        let db = env.open_db(None).unwrap();
-        let txn = env.begin_ro_txn().unwrap();
-
-        b.iter(|| {
-            let cursor = txn.open_ro_cursor(db).unwrap();
-            let mut i = 0;
-            let mut count = 0u32;
-
-            while let Ok((key_opt, val)) = cursor.get(None, None, MDB_NEXT) {
-                i += key_opt.map(|key| key.len()).unwrap_or(0) + val.len();
-                count += 1;
-            }
-
-            black_box(i);
-            assert_eq!(count, n);
-        });
-    }
-
-    /// Benchmark of raw LMDB sequential read performance (control).
-    #[bench]
-    fn bench_get_seq_raw(b: &mut Bencher) {
-        let n = 100;
-        let (_dir, env) = setup_bench_db(n);
-        let db = env.open_db(None).unwrap();
-
-        let dbi: MDB_dbi = db.dbi();
-        let _txn = env.begin_ro_txn().unwrap();
-        let txn = _txn.txn();
-
-        let mut key = MDB_val { mv_size: 0, mv_data: ptr::null_mut() };
-        let mut data = MDB_val { mv_size: 0, mv_data: ptr::null_mut() };
-        let mut cursor: *mut MDB_cursor = ptr::null_mut();
-
-        b.iter(|| unsafe {
-            mdb_cursor_open(txn, dbi, &mut cursor);
-            let mut i = 0;
-            let mut count = 0u32;
-
-            while mdb_cursor_get(cursor, &mut key, &mut data, MDB_NEXT) == 0 {
-                i += key.mv_size + data.mv_size;
-                count += 1;
-            };
-
-            black_box(i);
-            assert_eq!(count, n);
-            mdb_cursor_close(cursor);
-        });
-    }
 }

src/lib.rs

@@ -1,7 +1,6 @@
 //! Idiomatic and safe APIs for interacting with the
 //! [Lightning Memory-mapped Database (LMDB)](https://symas.com/lmdb).
 
-#![cfg_attr(test, feature(test))]
 #![deny(missing_docs)]
 #![doc(html_root_url = "https://docs.rs/lmdb-rkv/0.11.4")]
 
@@ -9,9 +8,7 @@ extern crate libc;
 extern crate lmdb_rkv_sys as ffi;
 extern crate byteorder;
 
-#[cfg(test)] extern crate rand;
 #[cfg(test)] extern crate tempdir;
-#[cfg(test)] extern crate test;
 #[macro_use] extern crate bitflags;
 
 pub use cursor::{
@@ -73,29 +70,6 @@ mod test_utils {
 
     use super::*;
 
-    pub fn get_key(n: u32) -> String {
-        format!("key{}", n)
-    }
-
-    pub fn get_data(n: u32) -> String {
-        format!("data{}", n)
-    }
-
-    pub fn setup_bench_db<'a>(num_rows: u32) -> (TempDir, Environment) {
-        let dir = TempDir::new("test").unwrap();
-        let env = Environment::new().open(dir.path()).unwrap();
-
-        {
-            let db = env.open_db(None).unwrap();
-            let mut txn = env.begin_rw_txn().unwrap();
-            for i in 0..num_rows {
-                txn.put(db, &get_key(i), &get_data(i), WriteFlags::empty()).unwrap();
-            }
-            txn.commit().unwrap();
-        }
-        (dir, env)
-    }
-
     /// Regression test for https://github.com/danburkert/lmdb-rs/issues/21.
     /// This test reliably segfaults when run against lmbdb compiled with opt level -O3 and newer
     /// GCC compilers.

src/transaction.rs

@@ -395,22 +395,16 @@ impl <'env> Transaction for RwTransaction<'env> {
 #[cfg(test)]
 mod test {
 
-    use libc::size_t;
-    use rand::{Rng, XorShiftRng};
     use std::io::Write;
-    use std::ptr;
     use std::sync::{Arc, Barrier};
     use std::thread::{self, JoinHandle};
-    use test::{Bencher, black_box};
 
     use tempdir::TempDir;
 
     use environment::*;
     use error::*;
-    use ffi::*;
     use flags::*;
     use super::*;
-    use test_utils::*;
     use cursor::Cursor;
 
     #[test]
@@ -752,105 +746,4 @@ mod test {
             assert_eq!(stat.entries(), 8);
         }
     }
-
-    #[bench]
-    fn bench_get_rand(b: &mut Bencher) {
-        let n = 100u32;
-        let (_dir, env) = setup_bench_db(n);
-        let db = env.open_db(None).unwrap();
-        let txn = env.begin_ro_txn().unwrap();
-
-        let mut keys: Vec<String> = (0..n).map(|n| get_key(n)).collect();
-        XorShiftRng::new_unseeded().shuffle(&mut keys[..]);
-
-        b.iter(|| {
-            let mut i = 0usize;
-            for key in &keys {
-                i = i + txn.get(db, key).unwrap().len();
-            }
-            black_box(i);
-        });
-    }
-
-    #[bench]
-    fn bench_get_rand_raw(b: &mut Bencher) {
-        let n = 100u32;
-        let (_dir, env) = setup_bench_db(n);
-        let db = env.open_db(None).unwrap();
-        let _txn = env.begin_ro_txn().unwrap();
-
-        let mut keys: Vec<String> = (0..n).map(|n| get_key(n)).collect();
-        XorShiftRng::new_unseeded().shuffle(&mut keys[..]);
-
-        let dbi = db.dbi();
-        let txn = _txn.txn();
-
-        let mut key_val: MDB_val = MDB_val { mv_size: 0, mv_data: ptr::null_mut() };
-        let mut data_val: MDB_val = MDB_val { mv_size: 0, mv_data: ptr::null_mut() };
-
-        b.iter(|| unsafe {
-            let mut i: size_t = 0;
-            for key in &keys {
-                key_val.mv_size = key.len() as size_t;
-                key_val.mv_data = key.as_bytes().as_ptr() as *mut _;
-
-                mdb_get(txn, dbi, &mut key_val, &mut data_val);
-
-                i = i + key_val.mv_size;
-            }
-            black_box(i);
-        });
-    }
-
-    #[bench]
-    fn bench_put_rand(b: &mut Bencher) {
-        let n = 100u32;
-        let (_dir, env) = setup_bench_db(0);
-        let db = env.open_db(None).unwrap();
-
-        let mut items: Vec<(String, String)> = (0..n).map(|n| (get_key(n), get_data(n))).collect();
-        XorShiftRng::new_unseeded().shuffle(&mut items[..]);
-
-        b.iter(|| {
-            let mut txn = env.begin_rw_txn().unwrap();
-            for &(ref key, ref data) in items.iter() {
-                txn.put(db, key, data, WriteFlags::empty()).unwrap();
-            }
-            txn.abort();
-        });
-    }
-
-    #[bench]
-    fn bench_put_rand_raw(b: &mut Bencher) {
-        let n = 100u32;
-        let (_dir, _env) = setup_bench_db(0);
-        let db = _env.open_db(None).unwrap();
-
-        let mut items: Vec<(String, String)> = (0..n).map(|n| (get_key(n), get_data(n))).collect();
-        XorShiftRng::new_unseeded().shuffle(&mut items[..]);
-
-        let dbi = db.dbi();
-        let env = _env.env();
-
-        let mut key_val: MDB_val = MDB_val { mv_size: 0, mv_data: ptr::null_mut() };
-        let mut data_val: MDB_val = MDB_val { mv_size: 0, mv_data: ptr::null_mut() };
-
-        b.iter(|| unsafe {
-            let mut txn: *mut MDB_txn = ptr::null_mut();
-            mdb_txn_begin(env, ptr::null_mut(), 0, &mut txn);
-
-            let mut i: ::libc::c_int = 0;
-            for &(ref key, ref data) in items.iter() {
-
-                key_val.mv_size = key.len() as size_t;
-                key_val.mv_data = key.as_bytes().as_ptr() as *mut _;
-                data_val.mv_size = data.len() as size_t;
-                data_val.mv_data = data.as_bytes().as_ptr() as *mut _;
-
-                i += mdb_put(txn, dbi, &mut key_val, &mut data_val, 0);
-            }
-            assert_eq!(0, i);
-            mdb_txn_abort(txn);
-        });
-    }
 }