mirror of https://github.com/nextgraph-org/rkv.git
Merge pull request #200 from mozilla/migrator
Implement a simple migrator between multiple backendswithout.crypto
Victor Porof
4 years ago
committed by
GitHub
commit
a6a616ad3d
@ -0,0 +1,107 @@ |
||||
// Copyright 2018-2019 Mozilla
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
|
||||
// this file except in compliance with the License. You may obtain a copy of the
|
||||
// License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software distributed
|
||||
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
|
||||
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations under the License.
|
||||
|
||||
use std::{ |
||||
io, |
||||
num, |
||||
str, |
||||
}; |
||||
|
||||
use failure::Fail; |
||||
|
||||
#[derive(Debug, Fail)] |
||||
pub enum MigrateError { |
||||
#[fail(display = "database not found: {:?}", _0)] |
||||
DatabaseNotFound(String), |
||||
|
||||
#[fail(display = "{}", _0)] |
||||
FromString(String), |
||||
|
||||
#[fail(display = "couldn't determine bit depth")] |
||||
IndeterminateBitDepth, |
||||
|
||||
#[fail(display = "I/O error: {:?}", _0)] |
||||
IoError(io::Error), |
||||
|
||||
#[fail(display = "invalid DatabaseFlags bits")] |
||||
InvalidDatabaseBits, |
||||
|
||||
#[fail(display = "invalid data version")] |
||||
InvalidDataVersion, |
||||
|
||||
#[fail(display = "invalid magic number")] |
||||
InvalidMagicNum, |
||||
|
||||
#[fail(display = "invalid NodeFlags bits")] |
||||
InvalidNodeBits, |
||||
|
||||
#[fail(display = "invalid PageFlags bits")] |
||||
InvalidPageBits, |
||||
|
||||
#[fail(display = "invalid page number")] |
||||
InvalidPageNum, |
||||
|
||||
#[fail(display = "lmdb backend error: {}", _0)] |
||||
LmdbError(lmdb::Error), |
||||
|
||||
#[fail(display = "string conversion error")] |
||||
StringConversionError, |
||||
|
||||
#[fail(display = "TryFromInt error: {:?}", _0)] |
||||
TryFromIntError(num::TryFromIntError), |
||||
|
||||
#[fail(display = "unexpected Page variant")] |
||||
UnexpectedPageVariant, |
||||
|
||||
#[fail(display = "unexpected PageHeader variant")] |
||||
UnexpectedPageHeaderVariant, |
||||
|
||||
#[fail(display = "unsupported PageHeader variant")] |
||||
UnsupportedPageHeaderVariant, |
||||
|
||||
#[fail(display = "UTF8 error: {:?}", _0)] |
||||
Utf8Error(str::Utf8Error), |
||||
} |
||||
|
||||
impl From<io::Error> for MigrateError { |
||||
fn from(e: io::Error) -> MigrateError { |
||||
MigrateError::IoError(e) |
||||
} |
||||
} |
||||
|
||||
impl From<str::Utf8Error> for MigrateError { |
||||
fn from(e: str::Utf8Error) -> MigrateError { |
||||
MigrateError::Utf8Error(e) |
||||
} |
||||
} |
||||
|
||||
impl From<num::TryFromIntError> for MigrateError { |
||||
fn from(e: num::TryFromIntError) -> MigrateError { |
||||
MigrateError::TryFromIntError(e) |
||||
} |
||||
} |
||||
|
||||
impl From<&str> for MigrateError { |
||||
fn from(e: &str) -> MigrateError { |
||||
MigrateError::FromString(e.to_string()) |
||||
} |
||||
} |
||||
|
||||
impl From<String> for MigrateError { |
||||
fn from(e: String) -> MigrateError { |
||||
MigrateError::FromString(e) |
||||
} |
||||
} |
||||
|
||||
impl From<lmdb::Error> for MigrateError { |
||||
fn from(e: lmdb::Error) -> MigrateError { |
||||
MigrateError::LmdbError(e) |
||||
} |
||||
} |
||||
@ -0,0 +1,91 @@ |
||||
// Copyright 2018-2019 Mozilla
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
|
||||
// this file except in compliance with the License. You may obtain a copy of the
|
||||
// License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software distributed
|
||||
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
|
||||
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations under the License.
|
||||
|
||||
//! A simple utility for migrating data from one RVK environment to another. Notably, this
|
||||
//! tool can migrate data from an enviroment created with a different backend than the
|
||||
//! current RKV consumer (e.g from Lmdb to SafeMode).
|
||||
//!
|
||||
//! The utility doesn't support migrating between 32-bit and 64-bit LMDB environments yet,
|
||||
//! see `arch_migrator` if this is needed. However, this utility is ultimately intended to
|
||||
//! handle all possible migrations.
|
||||
//!
|
||||
//! The destination environment should be empty of data, otherwise an error is returned.
|
||||
//!
|
||||
//! The tool currently has these limitations:
|
||||
//!
|
||||
//! 1. It doesn't support migration from environments created with
|
||||
//! `EnvironmentFlags::NO_SUB_DIR`. To migrate such an environment, create a temporary
|
||||
//! directory, copy the environment's data files in the temporary directory, then
|
||||
//! migrate the temporary directory as the source environment.
|
||||
//! 2. It doesn't support migration from databases created with DatabaseFlags::DUP_SORT`
|
||||
//! (with or without `DatabaseFlags::DUP_FIXED`) nor with `DatabaseFlags::INTEGER_KEY`.
|
||||
//! This effectively means that migration is limited to `SingleStore`s.
|
||||
//! 3. It doesn't allow for existing data in the destination environment, which means that
|
||||
//! it cannot overwrite nor append data.
|
||||
|
||||
use crate::{ |
||||
backend::{ |
||||
LmdbEnvironment, |
||||
SafeModeEnvironment, |
||||
}, |
||||
error::MigrateError, |
||||
Rkv, |
||||
StoreOptions, |
||||
}; |
||||
|
||||
pub use crate::backend::{ |
||||
LmdbArchMigrateError, |
||||
LmdbArchMigrateResult, |
||||
LmdbArchMigrator, |
||||
}; |
||||
|
||||
// FIXME: should parametrize this instead.
|
||||
macro_rules! fn_migrator { |
||||
($name:tt, $src:ty, $dst:ty) => { |
||||
/// Migrate all data in all of databases from the source environment to the destination
|
||||
/// environment. This includes all key/value pairs in the main database that aren't
|
||||
/// metadata about subdatabases and all key/value pairs in all subdatabases.
|
||||
///
|
||||
/// Other backend-specific metadata such as map size or maximum databases left intact on
|
||||
/// the given environments.
|
||||
///
|
||||
/// The destination environment should be empty of data, otherwise an error is returned.
|
||||
pub fn $name(src_env: &Rkv<$src>, dst_env: &Rkv<$dst>) -> Result<(), MigrateError> { |
||||
let src_dbs = src_env.get_dbs().unwrap(); |
||||
if src_dbs.is_empty() { |
||||
return Err(MigrateError::SourceEmpty); |
||||
} |
||||
let dst_dbs = dst_env.get_dbs().unwrap(); |
||||
if !dst_dbs.is_empty() { |
||||
return Err(MigrateError::DestinationNotEmpty); |
||||
} |
||||
for name in src_dbs { |
||||
let src_store = src_env.open_single(name.as_deref(), StoreOptions::default())?; |
||||
let dst_store = dst_env.open_single(name.as_deref(), StoreOptions::create())?; |
||||
let reader = src_env.read()?; |
||||
let mut writer = dst_env.write()?; |
||||
let mut iter = src_store.iter_start(&reader)?; |
||||
while let Some(Ok((key, value))) = iter.next() { |
||||
dst_store.put(&mut writer, key, &value).expect("wrote"); |
||||
} |
||||
writer.commit()?; |
||||
} |
||||
Ok(()) |
||||
} |
||||
}; |
||||
} |
||||
|
||||
pub struct Migrator; |
||||
|
||||
impl Migrator { |
||||
fn_migrator!(migrate_lmdb_to_safe_mode, LmdbEnvironment, SafeModeEnvironment); |
||||
|
||||
fn_migrator!(migrate_safe_mode_to_lmdb, SafeModeEnvironment, LmdbEnvironment); |
||||
} |
||||
@ -0,0 +1,138 @@ |
||||
// Copyright 2018-2019 Mozilla
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
|
||||
// this file except in compliance with the License. You may obtain a copy of the
|
||||
// License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software distributed
|
||||
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
|
||||
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations under the License.
|
||||
|
||||
use std::fs; |
||||
|
||||
use tempfile::Builder; |
||||
|
||||
use rkv::{ |
||||
backend::{ |
||||
BackendEnvironmentBuilder, |
||||
Lmdb, |
||||
SafeMode, |
||||
}, |
||||
migrator::Migrator, |
||||
Rkv, |
||||
StoreOptions, |
||||
Value, |
||||
}; |
||||
|
||||
macro_rules! populate_store { |
||||
($env:expr) => { |
||||
let store = $env.open_single("store", StoreOptions::create()).expect("opened"); |
||||
let mut writer = $env.write().expect("writer"); |
||||
store.put(&mut writer, "foo", &Value::I64(1234)).expect("wrote"); |
||||
store.put(&mut writer, "bar", &Value::Bool(true)).expect("wrote"); |
||||
store.put(&mut writer, "baz", &Value::Str("héllo, yöu")).expect("wrote"); |
||||
writer.commit().expect("committed"); |
||||
}; |
||||
} |
||||
|
||||
#[test] |
||||
#[should_panic(expected = "new succeeded: EnvironmentDoesNotExistError")] |
||||
fn test_migrator_lmdb_to_safe_0() { |
||||
let mut builder = Lmdb::new(); |
||||
builder.set_check_if_env_exists(true); |
||||
|
||||
let root = Builder::new().prefix("test_migrate_lmdb_to_safe").tempdir().expect("tempdir"); |
||||
let _ = Rkv::from_builder::<Lmdb>(root.path(), builder).expect("new succeeded"); |
||||
} |
||||
|
||||
#[test] |
||||
#[should_panic(expected = "migrated: SourceEmpty")] |
||||
fn test_migrator_lmdb_to_safe_1() { |
||||
let root = Builder::new().prefix("test_migrate_lmdb_to_safe").tempdir().expect("tempdir"); |
||||
fs::create_dir_all(root.path()).expect("dir created"); |
||||
|
||||
let src_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); |
||||
let dst_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); |
||||
Migrator::migrate_lmdb_to_safe_mode(&src_env, &dst_env).expect("migrated"); |
||||
} |
||||
|
||||
#[test] |
||||
#[should_panic(expected = "migrated: DestinationNotEmpty")] |
||||
fn test_migrator_lmdb_to_safe_2() { |
||||
let root = Builder::new().prefix("test_migrate_lmdb_to_safe").tempdir().expect("tempdir"); |
||||
fs::create_dir_all(root.path()).expect("dir created"); |
||||
|
||||
let src_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); |
||||
populate_store!(&src_env); |
||||
let dst_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); |
||||
populate_store!(&dst_env); |
||||
Migrator::migrate_lmdb_to_safe_mode(&src_env, &dst_env).expect("migrated"); |
||||
} |
||||
|
||||
#[test] |
||||
fn test_migrator_lmdb_to_safe_3() { |
||||
let root = Builder::new().prefix("test_migrate_lmdb_to_safe").tempdir().expect("tempdir"); |
||||
fs::create_dir_all(root.path()).expect("dir created"); |
||||
|
||||
let src_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); |
||||
populate_store!(&src_env); |
||||
let dst_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); |
||||
Migrator::migrate_lmdb_to_safe_mode(&src_env, &dst_env).expect("migrated"); |
||||
|
||||
let store = dst_env.open_single("store", StoreOptions::default()).expect("opened"); |
||||
let reader = dst_env.read().expect("reader"); |
||||
assert_eq!(store.get(&reader, "foo").expect("read"), Some(Value::I64(1234))); |
||||
assert_eq!(store.get(&reader, "bar").expect("read"), Some(Value::Bool(true))); |
||||
assert_eq!(store.get(&reader, "baz").expect("read"), Some(Value::Str("héllo, yöu"))); |
||||
} |
||||
|
||||
#[test] |
||||
#[should_panic(expected = "new succeeded: EnvironmentDoesNotExistError")] |
||||
fn test_migrator_safe_to_lmdb_0() { |
||||
let mut builder = SafeMode::new(); |
||||
builder.set_check_if_env_exists(true); |
||||
|
||||
let root = Builder::new().prefix("test_migrate_safe_to_lmdb").tempdir().expect("tempdir"); |
||||
let _ = Rkv::from_builder::<SafeMode>(root.path(), builder).expect("new succeeded"); |
||||
} |
||||
|
||||
#[test] |
||||
#[should_panic(expected = "migrated: SourceEmpty")] |
||||
fn test_migrator_safe_to_lmdb_1() { |
||||
let root = Builder::new().prefix("test_migrate_safe_to_lmdb").tempdir().expect("tempdir"); |
||||
fs::create_dir_all(root.path()).expect("dir created"); |
||||
|
||||
let src_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); |
||||
let dst_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); |
||||
Migrator::migrate_safe_mode_to_lmdb(&src_env, &dst_env).expect("migrated"); |
||||
} |
||||
|
||||
#[test] |
||||
#[should_panic(expected = "migrated: DestinationNotEmpty")] |
||||
fn test_migrator_safe_to_lmdb_2() { |
||||
let root = Builder::new().prefix("test_migrate_safe_to_lmdb").tempdir().expect("tempdir"); |
||||
fs::create_dir_all(root.path()).expect("dir created"); |
||||
|
||||
let src_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); |
||||
populate_store!(&src_env); |
||||
let dst_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); |
||||
populate_store!(&dst_env); |
||||
Migrator::migrate_safe_mode_to_lmdb(&src_env, &dst_env).expect("migrated"); |
||||
} |
||||
|
||||
#[test] |
||||
fn test_migrator_safe_to_lmdb_3() { |
||||
let root = Builder::new().prefix("test_migrate_safe_to_lmdb").tempdir().expect("tempdir"); |
||||
fs::create_dir_all(root.path()).expect("dir created"); |
||||
|
||||
let src_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); |
||||
populate_store!(&src_env); |
||||
let dst_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); |
||||
Migrator::migrate_safe_mode_to_lmdb(&src_env, &dst_env).expect("migrated"); |
||||
|
||||
let store = dst_env.open_single("store", StoreOptions::default()).expect("opened"); |
||||
let reader = dst_env.read().expect("reader"); |
||||
assert_eq!(store.get(&reader, "foo").expect("read"), Some(Value::I64(1234))); |
||||
assert_eq!(store.get(&reader, "bar").expect("read"), Some(Value::Bool(true))); |
||||
assert_eq!(store.get(&reader, "baz").expect("read"), Some(Value::Str("héllo, yöu"))); |
||||
} |
||||
Loading…
Reference in new issue