support architecture-independent database migration (#142)

Cargo.toml

@@ -10,6 +10,7 @@ readme = "README.md"
 keywords = ["lmdb", "database", "storage"]
 categories = ["database"]
 edition = "2018"
+exclude = ["/tests/envs/*"]
 
 [features]
 default = []
@@ -18,6 +19,8 @@ backtrace = ["failure/backtrace", "failure/std"]
 [dependencies]
 arrayref = "0.3"
 bincode = "1.0"
+bitflags = "1"
+byteorder = "1"
 lazy_static = "1.0.2"
 lmdb-rkv = "0.11.4"
 ordered-float = "1.0"

README.md

@@ -40,6 +40,8 @@ The project includes unit and doc tests embedded in the `src/` files, integratio
 ./run-all-examples.sh
 ```
 
+Note: the test fixtures in the `tests/envs/` subdirectory aren't included in the package published to crates.io, so you must clone this repository in order to run the tests that depend on those fixtures or use the `rand` and `dump` executables to recreate them.
+
 ## Contribute
 
 Of the various open source archetypes described in [A Framework for Purposeful Open Source](https://medium.com/mozilla-open-innovation/whats-your-open-source-strategy-here-are-10-answers-383221b3f9d3), the rkv project most closely resembles the Specialty Library, and we welcome contributions. Please report problems or ask questions using this repo's GitHub [issue tracker](https://github.com/mozilla/rkv/issues) and submit [pull requests](https://github.com/mozilla/rkv/pulls) for code and documentation changes.

src/bin/dump.rs

@@ -0,0 +1,55 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+extern crate rkv;
+
+use rkv::{
+    error::MigrateError,
+    migrate::Migrator,
+};
+use std::{
+    env::args,
+    io,
+    path::Path,
+};
+
+fn main() -> Result<(), MigrateError> {
+    let mut cli_args = args();
+    let mut db_name = None;
+    let mut env_path = None;
+
+    // The first arg is the name of the program, which we can ignore.
+    cli_args.next();
+
+    while let Some(arg) = cli_args.next() {
+        if &arg[0..1] == "-" {
+            match &arg[1..] {
+                "s" => {
+                    db_name = match cli_args.next() {
+                        None => return Err("-s must be followed by database name".into()),
+                        Some(str) => Some(str),
+                    };
+                },
+                str => return Err(format!("arg -{} not recognized", str).into()),
+            }
+        } else {
+            if env_path.is_some() {
+                return Err("must provide only one path to the LMDB environment".into());
+            }
+            env_path = Some(arg);
+        }
+    }
+
+    let env_path = env_path.ok_or("must provide a path to the LMDB environment")?;
+    let mut migrator: Migrator = Migrator::new(Path::new(&env_path))?;
+    migrator.dump(db_name.as_ref().map(String::as_str), io::stdout()).unwrap();
+
+    Ok(())
+}

src/bin/rand.rs

@@ -0,0 +1,114 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+//! A command-line utility to create an LMDB environment containing random data.
+//! It requires one flag, `-s path/to/environment`, which specifies the location
+//! where the tool should create the environment.  Optionally, you may specify
+//! the number of key/value pairs to create via the `-n <number>` flag
+//! (for which the default value is 50).
+
+extern crate rkv;
+
+use rkv::{
+    Rkv,
+    SingleStore,
+    StoreOptions,
+    Value,
+};
+use std::{
+    env::args,
+    fs::{
+        create_dir_all,
+        File,
+    },
+    io::Read,
+    path::Path,
+};
+
+fn main() {
+    let mut args = args();
+    let mut database = None;
+    let mut path = None;
+    let mut num_pairs = 50;
+
+    // The first arg is the name of the program, which we can ignore.
+    args.next();
+
+    while let Some(arg) = args.next() {
+        if &arg[0..1] == "-" {
+            match &arg[1..] {
+                "s" => {
+                    database = match args.next() {
+                        None => panic!("-s must be followed by database arg"),
+                        Some(str) => Some(str),
+                    };
+                },
+                "n" => {
+                    num_pairs = match args.next() {
+                        None => panic!("-s must be followed by number of pairs"),
+                        Some(str) => str.parse().expect("number"),
+                    };
+                },
+                str => panic!("arg -{} not recognized", str),
+            }
+        } else {
+            if path.is_some() {
+                panic!("must provide only one path to the LMDB environment");
+            }
+            path = Some(arg);
+        }
+    }
+
+    if path.is_none() {
+        panic!("must provide a path to the LMDB environment");
+    }
+    let path = path.unwrap();
+
+    create_dir_all(&path).expect("dir created");
+
+    let mut builder = Rkv::environment_builder();
+    builder.set_max_dbs(2);
+    // Allocate enough map to accommodate the largest random collection.
+    // We currently do this by allocating twice the maximum possible size
+    // of the pairs (assuming maximum key and value sizes).
+    builder.set_map_size((511 + 65535) * num_pairs * 2);
+    let rkv = Rkv::from_env(Path::new(&path), builder).expect("Rkv");
+    let store: SingleStore =
+        rkv.open_single(database.as_ref().map(|x| x.as_str()), StoreOptions::create()).expect("opened");
+    let mut writer = rkv.write().expect("writer");
+
+    // Generate random values for the number of keys and key/value lengths.
+    // On Linux, "Just use /dev/urandom!" <https://www.2uo.de/myths-about-urandom/>.
+    // On macOS it doesn't matter (/dev/random and /dev/urandom are identical).
+    let mut random = File::open("/dev/urandom").unwrap();
+    let mut nums = [0u8; 4];
+    random.read_exact(&mut nums).unwrap();
+
+    // Generate 0–255 pairs.
+    for _ in 0..num_pairs {
+        // Generate key and value lengths.  The key must be 1–511 bytes long.
+        // The value length can be 0 and is essentially unbounded; we generate
+        // value lengths of 0–0xffff (65535).
+        // NB: the modulus method for generating a random number within a range
+        // introduces distribution skew, but we don't need it to be perfect.
+        let key_len = ((u16::from(nums[0]) + (u16::from(nums[1]) << 8)) % 511 + 1) as usize;
+        let value_len = (u16::from(nums[2]) + (u16::from(nums[3]) << 8)) as usize;
+
+        let mut key: Vec<u8> = vec![0; key_len];
+        random.read_exact(&mut key[0..key_len]).unwrap();
+
+        let mut value: Vec<u8> = vec![0; value_len];
+        random.read_exact(&mut value[0..value_len]).unwrap();
+
+        store.put(&mut writer, key, &Value::Blob(&value)).expect("wrote");
+    }
+
+    writer.commit().expect("committed");
+}

src/error.rs

@@ -96,3 +96,95 @@ impl From<::std::io::Error> for StoreError {
         StoreError::IoError(e)
     }
 }
+
+#[derive(Debug, Fail)]
+pub enum MigrateError {
+    #[fail(display = "database not found: {:?}", _0)]
+    DatabaseNotFound(String),
+
+    #[fail(display = "{}", _0)]
+    FromString(String),
+
+    #[fail(display = "couldn't determine bit depth")]
+    IndeterminateBitDepth,
+
+    #[fail(display = "I/O error: {:?}", _0)]
+    IoError(::std::io::Error),
+
+    #[fail(display = "invalid DatabaseFlags bits")]
+    InvalidDatabaseBits,
+
+    #[fail(display = "invalid data version")]
+    InvalidDataVersion,
+
+    #[fail(display = "invalid magic number")]
+    InvalidMagicNum,
+
+    #[fail(display = "invalid NodeFlags bits")]
+    InvalidNodeBits,
+
+    #[fail(display = "invalid PageFlags bits")]
+    InvalidPageBits,
+
+    #[fail(display = "invalid page number")]
+    InvalidPageNum,
+
+    #[fail(display = "lmdb error: {}", _0)]
+    LmdbError(lmdb::Error),
+
+    #[fail(display = "string conversion error")]
+    StringConversionError,
+
+    #[fail(display = "TryFromInt error: {:?}", _0)]
+    TryFromIntError(::std::num::TryFromIntError),
+
+    #[fail(display = "unexpected Page variant")]
+    UnexpectedPageVariant,
+
+    #[fail(display = "unexpected PageHeader variant")]
+    UnexpectedPageHeaderVariant,
+
+    #[fail(display = "unsupported PageHeader variant")]
+    UnsupportedPageHeaderVariant,
+
+    #[fail(display = "UTF8 error: {:?}", _0)]
+    Utf8Error(::std::str::Utf8Error),
+}
+
+impl From<::std::io::Error> for MigrateError {
+    fn from(e: ::std::io::Error) -> MigrateError {
+        MigrateError::IoError(e)
+    }
+}
+
+impl From<::std::str::Utf8Error> for MigrateError {
+    fn from(e: ::std::str::Utf8Error) -> MigrateError {
+        MigrateError::Utf8Error(e)
+    }
+}
+
+impl From<::std::num::TryFromIntError> for MigrateError {
+    fn from(e: ::std::num::TryFromIntError) -> MigrateError {
+        MigrateError::TryFromIntError(e)
+    }
+}
+
+impl From<&str> for MigrateError {
+    fn from(e: &str) -> MigrateError {
+        MigrateError::FromString(e.to_string())
+    }
+}
+
+impl From<String> for MigrateError {
+    fn from(e: String) -> MigrateError {
+        MigrateError::FromString(e)
+    }
+}
+
+impl From<lmdb::Error> for MigrateError {
+    fn from(e: lmdb::Error) -> MigrateError {
+        match e {
+            e => MigrateError::LmdbError(e),
+        }
+    }
+}

src/lib.rs

@@ -209,6 +209,7 @@ pub use lmdb::{
 mod env;
 pub mod error;
 mod manager;
+pub mod migrate;
 mod readwrite;
 pub mod store;
 pub mod value;

src/manager.rs

@@ -38,6 +38,8 @@ use crate::error::StoreError;
 use crate::Rkv;
 
 lazy_static! {
+    /// A process is only permitted to have one open handle to each Rkv environment.
+    /// This manager exists to enforce that constraint: don't open environments directly.
     static ref MANAGER: RwLock<Manager> = RwLock::new(Manager::new());
 }
 

tests/envs/ref_dump.txt

@@ -0,0 +1,8 @@
+VERSION=3
+format=bytevalue
+type=btree
+mapsize=6604600
+maxreaders=126
+db_pagesize=4096
+HEADER=END
+DATA=END