diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 0000000..cad9135 --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,24 @@ +environment: + matrix: + - TARGET: x86_64-pc-windows-msvc + - TARGET: i686-pc-windows-msvc + - TARGET: x86_64-pc-windows-gnu + MSYS_BITS: 64 + - TARGET: i686-pc-windows-gnu + MSYS_BITS: 32 + +install: + - ps: Start-FileDownload "https://static.rust-lang.org/dist/rust-nightly-${env:TARGET}.exe" + - rust-nightly-%TARGET%.exe /VERYSILENT /NORESTART /DIR="C:\Program Files (x86)\Rust" + - set PATH=%PATH%;C:\Program Files (x86)\Rust\bin + - if defined MSYS_BITS set PATH=%PATH%;C:\msys64\mingw%MSYS_BITS%\bin + - rustc -V + - cargo -V + +build_script: + - git submodule -q update --init + - cargo build --target %TARGET% --all -v + +test_script: + - SET RUST_BACKTRACE=1 + - cargo test --target %TARGET% --all -v diff --git a/.rustfmt.toml b/.rustfmt.toml new file mode 100644 index 0000000..c7ad93b --- /dev/null +++ b/.rustfmt.toml @@ -0,0 +1 @@ +disable_all_formatting = true diff --git a/.travis.yml b/.travis.yml index 64d9a5c..d7bf1e7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,30 +1,21 @@ language: rust +dist: trusty sudo: false -rust: -- 1.2.0 -- nightly +cache: cargo os: - linux - osx -before_script: - - pip install 'travis-cargo<0.2' --user - - export PATH=$HOME/.local/bin:$PATH # Linux - - export PATH=$HOME/Library/Python/2.7/bin:$PATH # OS X +rust: +- 1.20.0 +- stable +- nightly script: - - | - travis-cargo build && - travis-cargo --only nightly test && - travis-cargo bench && - travis-cargo --only nightly doc - -after_success: - - travis-cargo --only nightly doc-upload - -env: - global: - - TRAVIS_CARGO_NIGHTLY_FEATURE="" - - secure: "BCsYNBS56hCKCMBZFLYDxKk3+UbFkAt+JRpPG9aiQyTVGkVqXNm4jPhiNSc4/JMuxoBVne+EzUrKVgdv3D7jTOx0sPsWEiE/oFK1K+6xusvsA87ZdDBC0hTRMiEHaufgtaGFJNwWW2tLzwlcdzvGUPVGYogao2MAxl1xlpppvso=" + - cargo build --verbose + - if [[ $TRAVIS_RUST_VERSION = nightly* ]]; then + env RUST_BACKTRACE=1 cargo test --all -v; + env RUST_BACKTRACE=1 cargo test --all -v --release; + fi diff --git a/Cargo.toml b/Cargo.toml index 8c32d77..82ed4ed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,24 +1,33 @@ [package] name = "lmdb" -version = "0.5.0" +# NB: When modifying, also modify html_root_url in lib.rs +version = "0.8.0" authors = ["Dan Burkert "] license = "Apache-2.0" description = "Idiomatic and safe LMDB wrapper." repository = "https://github.com/danburkert/lmdb-rs.git" readme = "README.md" +documentation = "https://docs.rs/lmdb" keywords = ["LMDB", "database", "storage-engine", "bindings", "library"] -documentation = "http://danburkert.github.io/lmdb-rs/lmdb/index.html" +categories = ["database"] -[dependencies.lmdb-sys] -path = "lmdb-sys" -version = "0.5.0" +[badges] +travis-ci = { repository = "danburkert/lmdb-rs" } +appveyor = { repository = "danburkert/lmdb-rs" } + +[workspace] +members = [ + "lmdb-sys", +] [dependencies] -bitflags = "0.4" +bitflags = "1" libc = "0.2" +lmdb-sys = { version = "0.8.0", path = "lmdb-sys" } [dev-dependencies] -rand = "0.3" +rand = "0.4" tempdir = "0.3" +byteorder = "1.0" diff --git a/README.md b/README.md index 43f91a1..59d4198 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,7 @@ [![Build Status](https://travis-ci.org/danburkert/lmdb-rs.svg?branch=master)](https://travis-ci.org/danburkert/lmdb-rs) - -[Documentation](http://danburkert.github.io/lmdb-rs/lmdb/index.html) - -[Cargo](https://crates.io/crates/lmdb) +[![Windows Build Status](https://ci.appveyor.com/api/projects/status/0bw21yfqsrsv3soh/branch/master?svg=true)](https://ci.appveyor.com/project/danburkert/lmdb-rs/branch/master) +[![Documentation](https://docs.rs/lmdb/badge.svg)](https://docs.rs/lmdb/) +[![Crate](https://img.shields.io/crates/v/lmdb.svg)](https://crates.io/crates/lmdb) # lmdb-rs @@ -17,10 +16,10 @@ cd lmdb-rs cargo build ``` -## TODO +## Features * [x] lmdb-sys. * [x] Cursors. * [x] Zero-copy put API. -* [ ] Nested transactions. -* [ ] Database statistics. +* [x] Nested transactions. +* [x] Database statistics. diff --git a/lmdb-sys/Cargo.toml b/lmdb-sys/Cargo.toml index cf6d02d..5c8f433 100644 --- a/lmdb-sys/Cargo.toml +++ b/lmdb-sys/Cargo.toml @@ -1,13 +1,17 @@ [package] name = "lmdb-sys" -version = "0.5.0" +# NB: When modifying, also modify html_root_url in lib.rs +version = "0.8.0" authors = ["Dan Burkert "] license = "Apache-2.0" + description = "Rust bindings for liblmdb." repository = "https://github.com/danburkert/lmdb-rs.git" readme = "../README.md" -documentation = "http://danburkert.github.io/lmdb-rs/lmdb_sys/index.html" +documentation = "https://docs.rs/lmdb-sys" +keywords = ["LMDB", "database", "storage-engine", "bindings", "library"] +categories = ["database", "external-ffi-bindings"] build = "build.rs" @@ -16,4 +20,4 @@ libc = "0.2" [build-dependencies] pkg-config = "0.3" -gcc = "0.3" +cc = "1" diff --git a/lmdb-sys/build.rs b/lmdb-sys/build.rs index 36332d5..8ae769a 100644 --- a/lmdb-sys/build.rs +++ b/lmdb-sys/build.rs @@ -1,25 +1,21 @@ extern crate pkg_config; -extern crate gcc; +extern crate cc; use std::env; use std::path::PathBuf; fn main() { - let mut lmdb: PathBuf = PathBuf::from(&env::var("CARGO_MANIFEST_DIR").unwrap()); lmdb.push("lmdb"); lmdb.push("libraries"); lmdb.push("liblmdb"); - let mut mdb: PathBuf = lmdb.clone(); - let mut midl: PathBuf = lmdb.clone(); - - mdb.push("mdb.c"); - midl.push("midl.c"); - if !pkg_config::find_library("liblmdb").is_ok() { - gcc::compile_library("liblmdb.a", - &[(*mdb).to_str().unwrap(), - (*midl).to_str().unwrap()]); + cc::Build::new() + .file(lmdb.join("mdb.c")) + .file(lmdb.join("midl.c")) + // https://github.com/LMDB/lmdb/blob/LMDB_0.9.21/libraries/liblmdb/Makefile#L25 + .opt_level(2) + .compile("liblmdb.a") } } diff --git a/lmdb-sys/lmdb b/lmdb-sys/lmdb index ad8488c..60d5002 160000 --- a/lmdb-sys/lmdb +++ b/lmdb-sys/lmdb @@ -1 +1 @@ -Subproject commit ad8488cfac644d7a289e428ab3c403c859d844cb +Subproject commit 60d500206a108b2c64ca7e36b0113b2cd3711b98 diff --git a/lmdb-sys/src/ffi.rs b/lmdb-sys/src/ffi.rs index 44d71cf..a998ea2 100644 --- a/lmdb-sys/src/ffi.rs +++ b/lmdb-sys/src/ffi.rs @@ -62,7 +62,7 @@ extern "C" { pub fn mdb_version(major: *mut ::libc::c_int, minor: *mut ::libc::c_int, patch: *mut ::libc::c_int) -> *mut ::libc::c_char; pub fn mdb_strerror(err: ::libc::c_int) -> *mut ::libc::c_char; pub fn mdb_env_create(env: *mut *mut MDB_env) -> ::libc::c_int; - pub fn mdb_env_open(env: *mut MDB_env, path: *const ::libc::c_char, flags: ::libc::c_uint, mode: ::libc::mode_t) -> ::libc::c_int; + pub fn mdb_env_open(env: *mut MDB_env, path: *const ::libc::c_char, flags: ::libc::c_uint, mode: super::mode_t) -> ::libc::c_int; pub fn mdb_env_copy(env: *mut MDB_env, path: *const ::libc::c_char) -> ::libc::c_int; pub fn mdb_env_copyfd(env: *mut MDB_env, fd: ::libc::c_int) -> ::libc::c_int; pub fn mdb_env_copy2(env: *mut MDB_env, path: *const ::libc::c_char, flags: ::libc::c_uint) -> ::libc::c_int; diff --git a/lmdb-sys/src/lib.rs b/lmdb-sys/src/lib.rs index 217fd92..601ef69 100644 --- a/lmdb-sys/src/lib.rs +++ b/lmdb-sys/src/lib.rs @@ -1,7 +1,16 @@ #![allow(non_camel_case_types)] +#![deny(warnings)] +#![doc(html_root_url = "https://docs.rs/lmdb-sys/0.8.0")] extern crate libc; +#[cfg(unix)] +#[allow(non_camel_case_types)] +pub type mode_t = ::libc::mode_t; +#[cfg(windows)] +#[allow(non_camel_case_types)] +pub type mode_t = ::libc::c_int; + pub use constants::*; pub use ffi::*; diff --git a/src/cursor.rs b/src/cursor.rs index 0c4daf5..61058bd 100644 --- a/src/cursor.rs +++ b/src/cursor.rs @@ -1,5 +1,5 @@ use libc::{c_void, size_t, c_uint}; -use std::{ptr, slice}; +use std::{fmt, ptr, result, slice}; use std::marker::PhantomData; use database::Database; @@ -28,7 +28,7 @@ pub trait Cursor<'txn> { let mut key_val = slice_to_val(key); let mut data_val = slice_to_val(data); let key_ptr = key_val.mv_data; - try!(lmdb_result(ffi::mdb_cursor_get(self.cursor(), &mut key_val, &mut data_val, op))); + lmdb_result(ffi::mdb_cursor_get(self.cursor(), &mut key_val, &mut data_val, op))?; let key_out = if key_ptr != key_val.mv_data { Some(val_to_slice(key_val)) } else { None }; let data_out = val_to_slice(data_val); Ok((key_out, data_out)) @@ -91,7 +91,7 @@ pub trait Cursor<'txn> { /// key. fn iter_dup_of(&mut self, key: &K) -> Result> where K: AsRef<[u8]> { - try!(self.get(Some(key.as_ref()), None, ffi::MDB_SET)); + self.get(Some(key.as_ref()), None, ffi::MDB_SET)?; Ok(Iter::new(self.cursor(), ffi::MDB_GET_CURRENT, ffi::MDB_NEXT_DUP)) } } @@ -108,6 +108,12 @@ impl <'txn> Cursor<'txn> for RoCursor<'txn> { } } +impl <'txn> fmt::Debug for RoCursor<'txn> { + fn fmt(&self, f: &mut fmt::Formatter) -> result::Result<(), fmt::Error> { + f.debug_struct("RoCursor").finish() + } +} + impl <'txn> Drop for RoCursor<'txn> { fn drop(&mut self) { unsafe { ffi::mdb_cursor_close(self.cursor) } @@ -118,10 +124,9 @@ impl <'txn> RoCursor<'txn> { /// Creates a new read-only cursor in the given database and transaction. /// Prefer using `Transaction::open_cursor`. - #[doc(hidden)] - pub fn new(txn: &'txn T, db: Database) -> Result> where T: Transaction { + pub(crate) fn new(txn: &'txn T, db: Database) -> Result> where T: Transaction { let mut cursor: *mut ffi::MDB_cursor = ptr::null_mut(); - unsafe { try!(lmdb_result(ffi::mdb_cursor_open(txn.txn(), db.dbi(), &mut cursor))); } + unsafe { lmdb_result(ffi::mdb_cursor_open(txn.txn(), db.dbi(), &mut cursor))?; } Ok(RoCursor { cursor: cursor, _marker: PhantomData, @@ -129,7 +134,7 @@ impl <'txn> RoCursor<'txn> { } } -/// A read-only cursor for navigating items within a database. +/// A read-write cursor for navigating items within a database. pub struct RwCursor<'txn> { cursor: *mut ffi::MDB_cursor, _marker: PhantomData &'txn ()>, @@ -141,6 +146,12 @@ impl <'txn> Cursor<'txn> for RwCursor<'txn> { } } +impl <'txn> fmt::Debug for RwCursor<'txn> { + fn fmt(&self, f: &mut fmt::Formatter) -> result::Result<(), fmt::Error> { + f.debug_struct("RwCursor").finish() + } +} + impl <'txn> Drop for RwCursor<'txn> { fn drop(&mut self) { unsafe { ffi::mdb_cursor_close(self.cursor) } @@ -151,10 +162,9 @@ impl <'txn> RwCursor<'txn> { /// Creates a new read-only cursor in the given database and transaction. /// Prefer using `RwTransaction::open_rw_cursor`. - #[doc(hidden)] - pub fn new(txn: &'txn T, db: Database) -> Result> where T: Transaction { + pub(crate) fn new(txn: &'txn T, db: Database) -> Result> where T: Transaction { let mut cursor: *mut ffi::MDB_cursor = ptr::null_mut(); - unsafe { try!(lmdb_result(ffi::mdb_cursor_open(txn.txn(), db.dbi(), &mut cursor))); } + unsafe { lmdb_result(ffi::mdb_cursor_open(txn.txn(), db.dbi(), &mut cursor))?; } Ok(RwCursor { cursor: cursor, _marker: PhantomData }) } @@ -202,6 +212,7 @@ unsafe fn val_to_slice<'a>(val: ffi::MDB_val) -> &'a [u8] { slice::from_raw_parts(val.mv_data as *const u8, val.mv_size as usize) } +/// An iterator over the values in an LMDB database. pub struct Iter<'txn> { cursor: *mut ffi::MDB_cursor, op: c_uint, @@ -217,6 +228,12 @@ impl <'txn> Iter<'txn> { } } +impl <'txn> fmt::Debug for Iter<'txn> { + fn fmt(&self, f: &mut fmt::Formatter) -> result::Result<(), fmt::Error> { + f.debug_struct("Iter").finish() + } +} + impl <'txn> Iterator for Iter<'txn> { type Item = (&'txn [u8], &'txn [u8]); @@ -243,6 +260,10 @@ impl <'txn> Iterator for Iter<'txn> { } } +/// An iterator over the keys and duplicate values in an LMDB database. +/// +/// The yielded items of the iterator are themselves iterators over the duplicate values for a +/// specific key. pub struct IterDup<'txn> { cursor: *mut ffi::MDB_cursor, op: c_uint, @@ -257,6 +278,12 @@ impl <'txn> IterDup<'txn> { } } +impl <'txn> fmt::Debug for IterDup<'txn> { + fn fmt(&self, f: &mut fmt::Formatter) -> result::Result<(), fmt::Error> { + f.debug_struct("IterDup").finish() + } +} + impl <'txn> Iterator for IterDup<'txn> { type Item = Iter<'txn>; @@ -290,7 +317,6 @@ mod test { use flags::*; use super::*; use test_utils::*; - use transaction::*; #[test] fn test_get() { @@ -326,7 +352,7 @@ mod test { fn test_get_dup() { let dir = TempDir::new("test").unwrap(); let env = Environment::new().open(dir.path()).unwrap(); - let db = env.create_db(None, DUP_SORT).unwrap(); + let db = env.create_db(None, DatabaseFlags::DUP_SORT).unwrap(); let mut txn = env.begin_rw_txn().unwrap(); txn.put(db, b"key1", b"val1", WriteFlags::empty()).unwrap(); @@ -372,7 +398,7 @@ mod test { fn test_get_dupfixed() { let dir = TempDir::new("test").unwrap(); let env = Environment::new().open(dir.path()).unwrap(); - let db = env.create_db(None, DUP_SORT | DUP_FIXED).unwrap(); + let db = env.create_db(None, DatabaseFlags::DUP_SORT | DatabaseFlags::DUP_FIXED).unwrap(); let mut txn = env.begin_rw_txn().unwrap(); txn.put(db, b"key1", b"val1", WriteFlags::empty()).unwrap(); @@ -428,7 +454,7 @@ mod test { fn test_iter_dup() { let dir = TempDir::new("test").unwrap(); let env = Environment::new().open(dir.path()).unwrap(); - let db = env.create_db(None, DUP_SORT).unwrap(); + let db = env.create_db(None, DatabaseFlags::DUP_SORT).unwrap(); let items: Vec<(&[u8], &[u8])> = vec!((b"a", b"1"), (b"a", b"2"), diff --git a/src/database.rs b/src/database.rs index 8a92ad1..61656dc 100644 --- a/src/database.rs +++ b/src/database.rs @@ -20,15 +20,14 @@ impl Database { /// /// Prefer using `Environment::open_db`, `Environment::create_db`, `TransactionExt::open_db`, /// or `RwTransaction::create_db`. - #[doc(hidden)] - pub unsafe fn new(txn: *mut ffi::MDB_txn, - name: Option<&str>, - flags: c_uint) - -> Result { + pub(crate) unsafe fn new(txn: *mut ffi::MDB_txn, + name: Option<&str>, + flags: c_uint) + -> Result { let c_name = name.map(|n| CString::new(n).unwrap()); let name_ptr = if let Some(ref c_name) = c_name { c_name.as_ptr() } else { ptr::null() }; let mut dbi: ffi::MDB_dbi = 0; - try!(lmdb_result(ffi::mdb_dbi_open(txn, name_ptr, flags, &mut dbi))); + lmdb_result(ffi::mdb_dbi_open(txn, name_ptr, flags, &mut dbi))?; Ok(Database { dbi: dbi }) } diff --git a/src/environment.rs b/src/environment.rs index 8d48654..db31f7a 100644 --- a/src/environment.rs +++ b/src/environment.rs @@ -1,8 +1,11 @@ -use libc::{c_uint, size_t, mode_t}; +use libc::{c_uint, size_t}; +use std::{fmt, ptr, result, mem}; use std::ffi::CString; +#[cfg(unix)] use std::os::unix::ffi::OsStrExt; +#[cfg(windows)] +use std::ffi::OsStr; use std::path::Path; -use std::ptr; use std::sync::Mutex; use ffi; @@ -12,6 +15,18 @@ use database::Database; use transaction::{RoTransaction, RwTransaction, Transaction}; use flags::{DatabaseFlags, EnvironmentFlags}; +#[cfg(windows)] +/// Adding a 'missing' trait from windows OsStrExt +trait OsStrExtLmdb { + fn as_bytes(&self) -> &[u8]; +} +#[cfg(windows)] +impl OsStrExtLmdb for OsStr { + fn as_bytes(&self) -> &[u8] { + &self.to_str().unwrap().as_bytes() + } +} + /// An LMDB environment. /// /// An environment supports multiple databases, all residing in the same shared-memory map. @@ -56,9 +71,9 @@ impl Environment { /// The database name may not contain the null character. pub fn open_db<'env>(&'env self, name: Option<&str>) -> Result { let mutex = self.dbi_open_mutex.lock(); - let txn = try!(self.begin_ro_txn()); - let db = unsafe { try!(txn.open_db(name)) }; - try!(txn.commit()); + let txn = self.begin_ro_txn()?; + let db = unsafe { txn.open_db(name)? }; + txn.commit()?; drop(mutex); Ok(db) } @@ -82,18 +97,21 @@ impl Environment { flags: DatabaseFlags) -> Result { let mutex = self.dbi_open_mutex.lock(); - let txn = try!(self.begin_rw_txn()); - let db = unsafe { try!(txn.create_db(name, flags)) }; - try!(txn.commit()); + let txn = self.begin_rw_txn()?; + let db = unsafe { txn.create_db(name, flags)? }; + txn.commit()?; drop(mutex); Ok(db) } + /// Retrieves the set of flags which the database is opened with. + /// + /// The database must belong to to this environment. pub fn get_db_flags<'env>(&'env self, db: Database) -> Result { - let txn = try!(self.begin_ro_txn()); + let txn = self.begin_ro_txn()?; let mut flags: c_uint = 0; unsafe { - try!(lmdb_result(ffi::mdb_dbi_flags(txn.txn(), db.dbi(), &mut flags))); + lmdb_result(ffi::mdb_dbi_flags(txn.txn(), db.dbi(), &mut flags))?; } Ok(DatabaseFlags::from_bits(flags).unwrap()) } @@ -136,11 +154,69 @@ impl Environment { pub unsafe fn close_db(&mut self, db: Database) { ffi::mdb_dbi_close(self.env, db.dbi()); } + + /// Retrieves statistics about this environment. + pub fn stat(&self) -> Result { + unsafe { + let mut stat = Stat(mem::zeroed()); + lmdb_try!(ffi::mdb_env_stat(self.env(), &mut stat.0)); + Ok(stat) + } + } +} + +/// Environment statistics. +/// +/// Contains information about the size and layout of an LMDB environment. +pub struct Stat(ffi::MDB_stat); + +impl Stat { + /// Size of a database page. This is the same for all databases in the environment. + #[inline] + pub fn page_size(&self) -> u32 { + self.0.ms_psize + } + + /// Depth (height) of the B-tree. + #[inline] + pub fn depth(&self) -> u32 { + self.0.ms_depth + } + + /// Number of internal (non-leaf) pages. + #[inline] + pub fn branch_pages(&self) -> usize { + self.0.ms_branch_pages + } + + /// Number of leaf pages. + #[inline] + pub fn leaf_pages(&self) -> usize { + self.0.ms_leaf_pages + } + + /// Number of overflow pages. + #[inline] + pub fn overflow_pages(&self) -> usize { + self.0.ms_overflow_pages + } + + /// Number of data items. + #[inline] + pub fn entries(&self) -> usize { + self.0.ms_entries + } } unsafe impl Send for Environment {} unsafe impl Sync for Environment {} +impl fmt::Debug for Environment { + fn fmt(&self, f: &mut fmt::Formatter) -> result::Result<(), fmt::Error> { + f.debug_struct("Environment").finish() + } +} + impl Drop for Environment { fn drop(&mut self) { unsafe { ffi::mdb_env_close(self.env) } @@ -176,7 +252,7 @@ impl EnvironmentBuilder { /// On Windows, the permissions will be ignored. /// /// The path may not contain the null character. - pub fn open_with_permissions(&self, path: &Path, mode: mode_t) -> Result { + pub fn open_with_permissions(&self, path: &Path, mode: ffi::mode_t) -> Result { let mut env: *mut ffi::MDB_env = ptr::null_mut(); unsafe { lmdb_try!(ffi::mdb_env_create(&mut env)); @@ -192,16 +268,18 @@ impl EnvironmentBuilder { lmdb_try_with_cleanup!(ffi::mdb_env_set_mapsize(env, map_size), ffi::mdb_env_close(env)) } - lmdb_try_with_cleanup!(ffi::mdb_env_open(env, - CString::new(path.as_os_str().as_bytes()).unwrap().as_ptr(), - self.flags.bits(), - mode), + let path = match CString::new(path.as_os_str().as_bytes()) { + Ok(path) => path, + Err(..) => return Err(::Error::Invalid), + }; + lmdb_try_with_cleanup!(ffi::mdb_env_open(env, path.as_ptr(), self.flags.bits(), mode), ffi::mdb_env_close(env)); } Ok(Environment { env: env, dbi_open_mutex: Mutex::new(()) }) } + /// Sets the provided options in the environment. pub fn set_flags(&mut self, flags: EnvironmentFlags) -> &mut EnvironmentBuilder { self.flags = flags; self @@ -252,7 +330,10 @@ impl EnvironmentBuilder { #[cfg(test)] mod test { + extern crate byteorder; + use tempdir::TempDir; + use self::byteorder::{ByteOrder, LittleEndian}; use flags::*; @@ -263,7 +344,7 @@ mod test { let dir = TempDir::new("test").unwrap(); // opening non-existent env with read-only should fail - assert!(Environment::new().set_flags(READ_ONLY) + assert!(Environment::new().set_flags(EnvironmentFlags::READ_ONLY) .open(dir.path()) .is_err()); @@ -271,7 +352,7 @@ mod test { assert!(Environment::new().open(dir.path()).is_ok()); // opening env with read-only should succeed - assert!(Environment::new().set_flags(READ_ONLY) + assert!(Environment::new().set_flags(EnvironmentFlags::READ_ONLY) .open(dir.path()) .is_ok()); } @@ -288,7 +369,7 @@ mod test { } { // read-only environment - let env = Environment::new().set_flags(READ_ONLY) + let env = Environment::new().set_flags(EnvironmentFlags::READ_ONLY) .open(dir.path()) .unwrap(); @@ -338,10 +419,45 @@ mod test { let env = Environment::new().open(dir.path()).unwrap(); assert!(env.sync(true).is_ok()); } { - let env = Environment::new().set_flags(READ_ONLY) + let env = Environment::new().set_flags(EnvironmentFlags::READ_ONLY) .open(dir.path()) .unwrap(); assert!(env.sync(true).is_err()); } } + + #[test] + fn test_stat() { + let dir = TempDir::new("test").unwrap(); + let env = Environment::new().open(dir.path()).unwrap(); + + // Stats should be empty initially. + let stat = env.stat().unwrap(); + assert_eq!(stat.page_size(), 4096); + assert_eq!(stat.depth(), 0); + assert_eq!(stat.branch_pages(), 0); + assert_eq!(stat.leaf_pages(), 0); + assert_eq!(stat.overflow_pages(), 0); + assert_eq!(stat.entries(), 0); + + let db = env.open_db(None).unwrap(); + + // Write a few small values. + for i in 0..64 { + let mut value = [0u8; 8]; + LittleEndian::write_u64(&mut value, i); + let mut tx = env.begin_rw_txn().expect("begin_rw_txn"); + tx.put(db, &value, &value, WriteFlags::default()).expect("tx.put"); + tx.commit().expect("tx.commit") + } + + // Stats should now reflect inserted values. + let stat = env.stat().unwrap(); + assert_eq!(stat.page_size(), 4096); + assert_eq!(stat.depth(), 1); + assert_eq!(stat.branch_pages(), 0); + assert_eq!(stat.leaf_pages(), 1); + assert_eq!(stat.overflow_pages(), 0); + assert_eq!(stat.entries(), 64); + } } diff --git a/src/error.rs b/src/error.rs index 5f73391..9225e85 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,10 +1,12 @@ use libc::c_int; use std::error::Error as StdError; use std::ffi::CStr; +use std::os::raw::c_char; use std::{fmt, result, str}; use ffi; +/// An LMDB error kind. #[derive(Debug, Eq, PartialEq, Copy, Clone)] pub enum Error { /// key/data pair already exists. @@ -52,6 +54,8 @@ pub enum Error { } impl Error { + + /// Converts a raw error code to an `Error`. pub fn from_err_code(err_code: c_int) -> Error { match err_code { ffi::MDB_KEYEXIST => Error::KeyExist, @@ -78,6 +82,7 @@ impl Error { } } + /// Converts an `Error` to the raw error code. pub fn to_err_code(&self) -> c_int { match *self { Error::KeyExist => ffi::MDB_KEYEXIST, @@ -115,12 +120,13 @@ impl StdError for Error { fn description(&self) -> &str { unsafe { // This is safe since the error messages returned from mdb_strerror are static. - let err: *const i8 = ffi::mdb_strerror(self.to_err_code()) as *const i8; + let err: *const c_char = ffi::mdb_strerror(self.to_err_code()) as *const c_char; str::from_utf8_unchecked(CStr::from_ptr(err).to_bytes()) } } } +/// An LMDB result. pub type Result = result::Result; pub fn lmdb_result(err_code: c_int) -> Result<()> { @@ -145,5 +151,4 @@ mod test { assert_eq!("MDB_NOTFOUND: No matching key/data pair found", Error::NotFound.description()); } - } diff --git a/src/flags.rs b/src/flags.rs index 01f277e..c467405 100644 --- a/src/flags.rs +++ b/src/flags.rs @@ -3,8 +3,9 @@ use libc::c_uint; use ffi::*; bitflags! { - #[doc="Environment Options"] - flags EnvironmentFlags: c_uint { + #[doc="Environment options."] + #[derive(Default)] + pub struct EnvironmentFlags: c_uint { #[doc="Use a fixed address for the mmap region. This flag must be specified"] #[doc="when creating the environment, and is stored persistently in the environment."] @@ -13,25 +14,25 @@ bitflags! { #[doc="across multiple invocations. This option may not always work, depending on"] #[doc="how the operating system has allocated memory to shared libraries and other uses."] #[doc="The feature is highly experimental."] - const FIXED_MAP = MDB_FIXEDMAP, + const FIXED_MAP = MDB_FIXEDMAP; #[doc="By default, LMDB creates its environment in a directory whose pathname is given in"] #[doc="`path`, and creates its data and lock files under that directory. With this option,"] #[doc="`path` is used as-is for the database main data file. The database lock file is the"] #[doc="`path` with `-lock` appended."] - const NO_SUB_DIR = MDB_NOSUBDIR, + const NO_SUB_DIR = MDB_NOSUBDIR; #[doc="Use a writeable memory map unless `READ_ONLY` is set. This is faster and uses"] #[doc="fewer mallocs, but loses protection from application bugs like wild pointer writes"] #[doc="and other bad updates into the database. Incompatible with nested transactions."] #[doc="Processes with and without `WRITE_MAP` on the same environment do not cooperate"] #[doc="well."] - const WRITE_MAP = MDB_WRITEMAP, + const WRITE_MAP = MDB_WRITEMAP; #[doc="Open the environment in read-only mode. No write operations will be allowed."] #[doc="When opening an environment, LMDB will still modify the lock file - except on"] #[doc="read-only filesystems, where LMDB does not use locks."] - const READ_ONLY = MDB_RDONLY, + const READ_ONLY = MDB_RDONLY; #[doc="Flush system buffers to disk only once per transaction, omit the metadata flush."] #[doc="Defer that until the system flushes files to disk, or next non-`READ_ONLY` commit"] @@ -39,7 +40,7 @@ bitflags! { #[doc="system crash may undo the last committed transaction. I.e. it preserves the ACI"] #[doc="(atomicity, consistency, isolation) but not D (durability) database property."] #[doc="\n\nThis flag may be changed at any time using `Environment::set_flags`."] - const NO_META_SYNC = MDB_NOMETASYNC, + const NO_META_SYNC = MDB_NOMETASYNC; #[doc="Don't flush system buffers to disk when committing a transaction. This optimization"] #[doc="means a system crash can corrupt the database or lose the last transactions if"] @@ -52,36 +53,36 @@ bitflags! { #[doc="system with no hint for when to write transactions to disk, unless"] #[doc="`Environment::sync` is called. (`MAP_ASYNC | WRITE_MAP`) may be preferable."] #[doc="\n\nThis flag may be changed at any time using `Environment::set_flags`."] - const NO_SYNC = MDB_NOSYNC, + const NO_SYNC = MDB_NOSYNC; #[doc="When using `WRITE_MAP`, use asynchronous flushes to disk. As with `NO_SYNC`, a"] #[doc="system crash can then corrupt the database or lose the last transactions. Calling"] #[doc="`Environment::sync` ensures on-disk database integrity until next commit."] #[doc="\n\nThis flag may be changed at any time using `Environment::set_flags`."] - const MAP_ASYNC = MDB_MAPASYNC, + const MAP_ASYNC = MDB_MAPASYNC; #[doc="Don't use thread-local storage. Tie reader locktable slots to transaction objects"] - #[doc="instead of to threads. I.e. `RoTransaction::reset` keeps the slot reseved for the"] + #[doc="instead of to threads. I.e. `RoTransaction::reset` keeps the slot reserved for the"] #[doc="transaction object. A thread may use parallel read-only transactions. A read-only"] #[doc="transaction may span threads if the user synchronizes its use. Applications that"] #[doc="multiplex many the user synchronizes its use. Applications that multiplex many user"] #[doc="threads over individual OS threads need this option. Such an application must also"] #[doc="serialize the write transactions in an OS thread, since LMDB's write locking is"] #[doc="unaware of the user threads."] - const NO_TLS = MDB_NOTLS, + const NO_TLS = MDB_NOTLS; #[doc="Do not do any locking. If concurrent access is anticipated, the caller must manage"] #[doc="all concurrency themself. For proper operation the caller must enforce"] #[doc="single-writer semantics, and must ensure that no readers are using old"] #[doc="transactions while a writer is active. The simplest approach is to use an exclusive"] #[doc="lock so that no readers may be active at all when a writer begins."] - const NO_LOCK = MDB_NOLOCK, + const NO_LOCK = MDB_NOLOCK; #[doc="Turn off readahead. Most operating systems perform readahead on read requests by"] #[doc="default. This option turns it off if the OS supports it. Turning it off may help"] #[doc="random read performance when the DB is larger than RAM and system RAM is full."] #[doc="The option is not implemented on Windows."] - const NO_READAHEAD = MDB_NORDAHEAD, + const NO_READAHEAD = MDB_NORDAHEAD; #[doc="Do not initialize malloc'd memory before writing to unused spaces in the data file."] #[doc="By default, memory for pages written to the data file is obtained using malloc."] @@ -98,73 +99,75 @@ bitflags! { #[doc="with reserve; the caller is expected to overwrite all of the memory that was"] #[doc="reserved in that case."] #[doc="\n\nThis flag may be changed at any time using `Environment::set_flags`."] - const NO_MEM_INIT = MDB_NOMEMINIT, + const NO_MEM_INIT = MDB_NOMEMINIT; } } bitflags! { - #[doc="Database Options"] - flags DatabaseFlags: c_uint { + #[doc="Database options."] + #[derive(Default)] + pub struct DatabaseFlags: c_uint { #[doc="Keys are strings to be compared in reverse order, from the end of the strings"] #[doc="to the beginning. By default, Keys are treated as strings and compared from"] #[doc="beginning to end."] - const REVERSE_KEY = MDB_REVERSEKEY, + const REVERSE_KEY = MDB_REVERSEKEY; #[doc="Duplicate keys may be used in the database. (Or, from another perspective,"] #[doc="keys may have multiple data items, stored in sorted order.) By default"] #[doc="keys must be unique and may have only a single data item."] - const DUP_SORT = MDB_DUPSORT, + const DUP_SORT = MDB_DUPSORT; #[doc="Keys are binary integers in native byte order. Setting this option requires all"] #[doc="keys to be the same size, typically 32 or 64 bits."] - const INTEGER_KEY = MDB_INTEGERKEY, + const INTEGER_KEY = MDB_INTEGERKEY; #[doc="This flag may only be used in combination with `DUP_SORT`. This option tells"] #[doc="the library that the data items for this database are all the same size, which"] #[doc="allows further optimizations in storage and retrieval. When all data items are"] #[doc="the same size, the `GET_MULTIPLE` and `NEXT_MULTIPLE` cursor operations may be"] #[doc="used to retrieve multiple items at once."] - const DUP_FIXED = MDB_DUPFIXED, + const DUP_FIXED = MDB_DUPFIXED; #[doc="This option specifies that duplicate data items are also integers, and"] #[doc="should be sorted as such."] - const INTEGER_DUP = MDB_INTEGERDUP, + const INTEGER_DUP = MDB_INTEGERDUP; #[doc="This option specifies that duplicate data items should be compared as strings"] #[doc="in reverse order."] - const REVERSE_DUP = MDB_REVERSEDUP, + const REVERSE_DUP = MDB_REVERSEDUP; } } bitflags! { - #[doc="Write Options"] - flags WriteFlags: c_uint { + #[doc="Write options."] + #[derive(Default)] + pub struct WriteFlags: c_uint { #[doc="Insert the new item only if the key does not already appear in the database."] #[doc="The function will return `LmdbError::KeyExist` if the key already appears in the"] #[doc="database, even if the database supports duplicates (`DUP_SORT`)."] - const NO_OVERWRITE = MDB_NOOVERWRITE, + const NO_OVERWRITE = MDB_NOOVERWRITE; #[doc="Insert the new item only if it does not already appear in the database."] #[doc="This flag may only be specified if the database was opened with `DUP_SORT`."] #[doc="The function will return `LmdbError::KeyExist` if the item already appears in the"] #[doc="database."] - const NO_DUP_DATA = MDB_NODUPDATA, + const NO_DUP_DATA = MDB_NODUPDATA; #[doc="For `Cursor::put`. Replace the item at the current cursor position. The key"] #[doc="parameter must match the current position. If using sorted duplicates (`DUP_SORT`)"] #[doc="the data item must still sort into the same position. This is intended to be used"] #[doc="when the new data is the same size as the old. Otherwise it will simply perform a"] #[doc="delete of the old record followed by an insert."] - const CURRENT = MDB_CURRENT, + const CURRENT = MDB_CURRENT; #[doc="Append the given item to the end of the database. No key comparisons are performed."] #[doc="This option allows fast bulk loading when keys are already known to be in the"] #[doc="correct order. Loading unsorted keys with this flag will cause data corruption."] - const APPEND = MDB_APPEND, + const APPEND = MDB_APPEND; #[doc="Same as `APPEND`, but for sorted dup data."] - const APPEND_DUP = MDB_APPENDDUP, + const APPEND_DUP = MDB_APPENDDUP; } } diff --git a/src/lib.rs b/src/lib.rs index 6e89c0d..bee6bfe 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,7 +1,9 @@ //! Idiomatic and safe APIs for interacting with the -//! [Symas Lightning Memory-Mapped Database (LMDB)](http://symas.com/mdb/). +//! [Lightning Memory-mapped Database (LMDB)](https://symas.com/lmdb). #![cfg_attr(test, feature(test))] +#![deny(missing_docs)] +#![doc(html_root_url = "https://docs.rs/lmdb/0.8.0")] extern crate libc; extern crate lmdb_sys as ffi; @@ -14,10 +16,12 @@ extern crate lmdb_sys as ffi; pub use cursor::{ Cursor, RoCursor, - RwCursor + RwCursor, + Iter, + IterDup, }; pub use database::Database; -pub use environment::{Environment, EnvironmentBuilder}; +pub use environment::{Environment, Stat, EnvironmentBuilder}; pub use error::{Error, Result}; pub use flags::*; pub use transaction::{ @@ -58,6 +62,9 @@ mod transaction; #[cfg(test)] mod test_utils { + extern crate byteorder; + + use self::byteorder::{ByteOrder, LittleEndian}; use tempdir::TempDir; use super::*; @@ -84,4 +91,33 @@ mod test_utils { } (dir, env) } + + /// Regression test for https://github.com/danburkert/lmdb-rs/issues/21. + /// This test reliably segfaults when run against lmbdb compiled with opt level -O3 and newer + /// GCC compilers. + #[test] + fn issue_21_regression() { + const HEIGHT_KEY: [u8; 1] = [0]; + + let dir = TempDir::new("test").unwrap(); + + let env = { + let mut builder = Environment::new(); + builder.set_max_dbs(2); + builder.set_map_size(1_000_000); + builder.open(dir.path()).expect("open lmdb env") + }; + let index = env.create_db(None, DatabaseFlags::DUP_SORT).expect("open index db"); + + for height in 0..1000 { + let mut value = [0u8; 8]; + LittleEndian::write_u64(&mut value, height); + let mut tx = env.begin_rw_txn().expect("begin_rw_txn"); + tx.put(index, + &HEIGHT_KEY, + &value, + WriteFlags::empty()).expect("tx.put"); + tx.commit().expect("tx.commit") + } + } } diff --git a/src/transaction.rs b/src/transaction.rs index ab481a4..590fd9f 100644 --- a/src/transaction.rs +++ b/src/transaction.rs @@ -1,5 +1,5 @@ use libc::{c_uint, c_void, size_t}; -use std::{mem, ptr, slice}; +use std::{fmt, mem, ptr, result, slice}; use std::marker::PhantomData ; use ffi; @@ -98,7 +98,7 @@ pub trait Transaction : Sized { fn db_flags(&self, db: Database) -> Result { let mut flags: c_uint = 0; unsafe { - try!(lmdb_result(ffi::mdb_dbi_flags(self.txn(), db.dbi(), &mut flags))); + lmdb_result(ffi::mdb_dbi_flags(self.txn(), db.dbi(), &mut flags))?; } Ok(DatabaseFlags::from_bits_truncate(flags)) } @@ -110,6 +110,12 @@ pub struct RoTransaction<'env> { _marker: PhantomData<&'env ()>, } +impl <'env> fmt::Debug for RoTransaction<'env> { + fn fmt(&self, f: &mut fmt::Formatter) -> result::Result<(), fmt::Error> { + f.debug_struct("RoTransaction").finish() + } +} + impl <'env> Drop for RoTransaction<'env> { fn drop(&mut self) { unsafe { ffi::mdb_txn_abort(self.txn) } @@ -120,14 +126,10 @@ impl <'env> RoTransaction<'env> { /// Creates a new read-only transaction in the given environment. Prefer /// using `Environment::begin_ro_txn`. - #[doc(hidden)] - pub fn new(env: &'env Environment) -> Result> { + pub(crate) fn new(env: &'env Environment) -> Result> { let mut txn: *mut ffi::MDB_txn = ptr::null_mut(); unsafe { - try!(lmdb_result(ffi::mdb_txn_begin(env.env(), - ptr::null_mut(), - ffi::MDB_RDONLY, - &mut txn))); + lmdb_result(ffi::mdb_txn_begin(env.env(), ptr::null_mut(), ffi::MDB_RDONLY, &mut txn))?; Ok(RoTransaction { txn: txn, _marker: PhantomData }) } } @@ -166,6 +168,12 @@ pub struct InactiveTransaction<'env> { _marker: PhantomData<&'env ()>, } +impl <'env> fmt::Debug for InactiveTransaction<'env> { + fn fmt(&self, f: &mut fmt::Formatter) -> result::Result<(), fmt::Error> { + f.debug_struct("InactiveTransaction").finish() + } +} + impl <'env> Drop for InactiveTransaction<'env> { fn drop(&mut self) { unsafe { ffi::mdb_txn_abort(self.txn) } @@ -183,7 +191,7 @@ impl <'env> InactiveTransaction<'env> { let txn = self.txn; unsafe { mem::forget(self); - try!(lmdb_result(ffi::mdb_txn_renew(txn))) + lmdb_result(ffi::mdb_txn_renew(txn))? }; Ok(RoTransaction { txn: txn, _marker: PhantomData }) } @@ -195,6 +203,12 @@ pub struct RwTransaction<'env> { _marker: PhantomData<&'env ()>, } +impl <'env> fmt::Debug for RwTransaction<'env> { + fn fmt(&self, f: &mut fmt::Formatter) -> result::Result<(), fmt::Error> { + f.debug_struct("RwTransaction").finish() + } +} + impl <'env> Drop for RwTransaction<'env> { fn drop(&mut self) { unsafe { ffi::mdb_txn_abort(self.txn) } @@ -205,14 +219,13 @@ impl <'env> RwTransaction<'env> { /// Creates a new read-write transaction in the given environment. Prefer /// using `Environment::begin_ro_txn`. - #[doc(hidden)] - pub fn new(env: &'env Environment) -> Result> { + pub(crate) fn new(env: &'env Environment) -> Result> { let mut txn: *mut ffi::MDB_txn = ptr::null_mut(); unsafe { - try!(lmdb_result(ffi::mdb_txn_begin(env.env(), - ptr::null_mut(), - EnvironmentFlags::empty().bits(), - &mut txn))); + lmdb_result(ffi::mdb_txn_begin(env.env(), + ptr::null_mut(), + EnvironmentFlags::empty().bits(), + &mut txn))?; Ok(RwTransaction { txn: txn, _marker: PhantomData }) } } @@ -229,7 +242,7 @@ impl <'env> RwTransaction<'env> { /// /// ## Safety /// - /// * This function (as well as `Environment::open_db`, + /// This function (as well as `Environment::open_db`, /// `Environment::create_db`, and `Database::open`) **must not** be called /// from multiple concurrent transactions in the same environment. A /// transaction which uses this function must finish (either commit or @@ -287,11 +300,11 @@ impl <'env> RwTransaction<'env> { let mut data_val: ffi::MDB_val = ffi::MDB_val { mv_size: len, mv_data: ptr::null_mut::() }; unsafe { - try!(lmdb_result(ffi::mdb_put(self.txn(), - database.dbi(), - &mut key_val, - &mut data_val, - flags.bits() | ffi::MDB_RESERVE))); + lmdb_result(ffi::mdb_put(self.txn(), + database.dbi(), + &mut key_val, + &mut data_val, + flags.bits() | ffi::MDB_RESERVE))?; Ok(slice::from_raw_parts_mut(data_val.mv_data as *mut u8, data_val.mv_size as usize)) }