NextGraph
/
rkv
mirror of https://github.com/nextgraph-org/rkv.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Hide all of LMDB behind a feature

Browse Source
without.crypto
Jan-Erik Rediger 2 years ago committed by Kagami Sascha Rosylight
parent c03fcfbf3d
commit dd42887903
17 changed files with 103 additions and 61 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 33
      Cargo.toml
  2. 12
      examples/iterator.rs
  3. 19
      examples/simple-store.rs
  4. 2
      src/backend.rs
  5. 4
      src/backend/impl_lmdb/arch_migrator.rs
  6. 2
      src/env.rs
  7. 1
      src/error.rs
  8. 8
      src/lib.rs
  9. 10
      src/manager.rs
  10. 14
      src/store/integer.rs
  11. 14
      src/store/integermulti.rs
  12. 2
      tests/env-lmdb.rs
  13. 2
      tests/env-safe.rs
  14. 4
      tests/integer-store.rs
  15. 11
      tests/manager.rs
  16. 4
      tests/multi-integer-store.rs
  17. 22
      tests/test_txn.rs

33
Cargo.toml
Unescape View File

@ -18,14 +18,37 @@ readme = "README.md"
repository = "https://github.com/mozilla/rkv"
version = "0.17.1"
[[bin]]
name = "rand"
path = "src/bin/rand.rs"
required-features = ["lmdb"]
[[bin]]
name = "dump"
path = "src/bin/dump.rs"
required-features = ["lmdb"]
[[test]]
name = "env-all"
required-features = ["lmdb"]
[[test]]
name = "env-lmdb"
required-features = ["lmdb"]
[[test]]
name = "env-migration"
required-features = ["lmdb"]
[features]
lmdb = ["lmdb-rkv"]
db-dup-sort = []
db-int-key = []
default = ["db-dup-sort", "db-int-key"]
default = ["db-dup-sort", "db-int-key", "lmdb"]
no-canonicalize-path = []
with-asan = ["lmdb-rkv/with-asan"]
with-fuzzer = ["lmdb-rkv/with-fuzzer"]
with-fuzzer-no-link = ["lmdb-rkv/with-fuzzer-no-link"]
with-asan = ["lmdb", "lmdb-rkv/with-asan"]
with-fuzzer = ["lmdb", "lmdb-rkv/with-fuzzer"]
with-fuzzer-no-link = ["lmdb", "lmdb-rkv/with-fuzzer-no-link"]
[dependencies]
arrayref = "0.3"
@ -34,7 +57,7 @@ bitflags = "~1.2"
byteorder = "1"
id-arena = "2.2"
lazy_static = "1.1"
lmdb-rkv = "0.14"
lmdb-rkv = { version = "0.14", optional = true }
log = "0.4.4"
ordered-float = "3.0.0"
paste = "1.0.6"

12
examples/iterator.rs
Unescape View File

@ -16,9 +16,9 @@ use tempfile::Builder;
use rkv::{
backend::{
Lmdb,
LmdbDatabase,
LmdbEnvironment,
SafeMode,
SafeModeDatabase,
SafeModeEnvironment,
},
Manager,
Rkv,
@ -33,8 +33,8 @@ fn main() {
fs::create_dir_all(root.path()).unwrap();
let p = root.path();
let mut manager = Manager::<LmdbEnvironment>::singleton().write().unwrap();
let created_arc = manager.get_or_create(p, Rkv::new::<Lmdb>).unwrap();
let mut manager = Manager::<SafeModeEnvironment>::singleton().write().unwrap();
let created_arc = manager.get_or_create(p, Rkv::new::<SafeMode>).unwrap();
let k = created_arc.read().unwrap();
let store = k.open_single("store", StoreOptions::create()).unwrap();
@ -67,7 +67,7 @@ fn main() {
}
}
fn populate_store(k: &Rkv<LmdbEnvironment>, store: SingleStore<LmdbDatabase>) -> Result<(), StoreError> {
fn populate_store(k: &Rkv<SafeModeEnvironment>, store: SingleStore<SafeModeDatabase>) -> Result<(), StoreError> {
let mut writer = k.write()?;
for (country, city) in vec![
("Canada", Value::Str("Ottawa")),

19
examples/simple-store.rs
Unescape View File

@ -13,11 +13,10 @@ use tempfile::Builder;
use rkv::{
backend::{
BackendStat,
Lmdb,
LmdbDatabase,
LmdbEnvironment,
LmdbRwTransaction,
SafeMode,
SafeModeDatabase,
SafeModeEnvironment,
SafeModeRwTransaction,
},
Manager,
Rkv,
@ -25,8 +24,8 @@ use rkv::{
Value,
};
type MultiStore = rkv::MultiStore<LmdbDatabase>;
type Writer<'w> = rkv::Writer<LmdbRwTransaction<'w>>;
type MultiStore = rkv::MultiStore<SafeModeDatabase>;
type Writer<'w> = rkv::Writer<SafeModeRwTransaction<'w>>;
fn getput<'w, 's>(store: MultiStore, writer: &'w mut Writer, ids: &'s mut Vec<String>) {
let keys = vec!["str1", "str2", "str3"];
@ -62,8 +61,8 @@ fn main() {
let p = root.path();
// The manager enforces that each process opens the same lmdb environment at most once
let mut manager = Manager::<LmdbEnvironment>::singleton().write().unwrap();
let created_arc = manager.get_or_create(p, Rkv::new::<Lmdb>).unwrap();
let mut manager = Manager::<SafeModeEnvironment>::singleton().write().unwrap();
let created_arc = manager.get_or_create(p, Rkv::new::<SafeMode>).unwrap();
let k = created_arc.read().unwrap();
// Creates a store called "store"
@ -189,6 +188,4 @@ fn main() {
println!("Get from store value: {:?}", store.get(&reader, "foo").unwrap());
println!("Get from another store value: {:?}", another_store.get(&reader, "foo").unwrap());
}
println!("Environment statistics: btree depth = {}", k.stat().unwrap().depth());
}

2
src/backend.rs
Unescape View File

@ -9,6 +9,7 @@
// specific language governing permissions and limitations under the License.
mod common;
#[cfg(feature = "lmdb")]
mod impl_lmdb;
mod impl_safe;
mod traits;
@ -16,6 +17,7 @@ mod traits;
pub use common::*;
pub use traits::*;
#[cfg(feature = "lmdb")]
pub use impl_lmdb::{
ArchMigrateError as LmdbArchMigrateError,
ArchMigrateResult as LmdbArchMigrateResult,

4
src/backend/impl_lmdb/arch_migrator.rs
Unescape View File

@ -801,10 +801,10 @@ mod tests {
loop {
match ref_file.read(ref_buf) {
Err(err) => panic!(err),
Err(err) => panic!("{}", err),
Ok(ref_len) => {
match new_file.read(new_buf) {
Err(err) => panic!(err),
Err(err) => panic!("{}", err),
Ok(new_len) => {
assert_eq!(ref_len, new_len);
if ref_len == 0 {

2
src/env.rs
Unescape View File

@ -192,6 +192,7 @@ where
if opts.create {
self.env.create_db(name.into(), opts.flags).map_err(|e| {
match e.into() {
#[cfg(feature = "lmdb")]
StoreError::LmdbError(lmdb::Error::BadRslot) => StoreError::open_during_transaction(),
StoreError::SafeModeError(SafeModeError::DbsIllegalOpen) => StoreError::open_during_transaction(),
e => e,
@ -200,6 +201,7 @@ where
} else {
self.env.open_db(name.into()).map_err(|e| {
match e.into() {
#[cfg(feature = "lmdb")]
StoreError::LmdbError(lmdb::Error::BadRslot) => StoreError::open_during_transaction(),
StoreError::SafeModeError(SafeModeError::DbsIllegalOpen) => StoreError::open_during_transaction(),
e => e,

1
src/error.rs
Unescape View File

@ -83,6 +83,7 @@ pub enum StoreError {
#[error("data error: {0:?}")]
DataError(#[from] DataError),
#[cfg(feature = "lmdb")]
#[error("lmdb backend error: {0}")]
LmdbError(lmdb::Error),

8
src/lib.rs
Unescape View File

@ -45,7 +45,7 @@
//! ## Basic Usage
//! ```
//! use rkv::{Manager, Rkv, SingleStore, Value, StoreOptions};
//! use rkv::backend::{Lmdb, LmdbEnvironment};
//! use rkv::backend::{SafeMode, SafeModeEnvironment};
//! use std::fs;
//! use tempfile::Builder;
//!
@ -64,8 +64,8 @@
//! // The `Manager` enforces that each process opens the same environment at most once by
//! // caching a handle to each environment that it opens. Use it to retrieve the handle
//! // to an opened environment—or create one if it hasn't already been opened:
//! let mut manager = Manager::<LmdbEnvironment>::singleton().write().unwrap();
//! let created_arc = manager.get_or_create(path, Rkv::new::<Lmdb>).unwrap();
//! let mut manager = Manager::<SafeModeEnvironment>::singleton().write().unwrap();
//! let created_arc = manager.get_or_create(path, Rkv::new::<SafeMode>).unwrap();
//! let env = created_arc.read().unwrap();
//!
//! // Then you can use the environment handle to get a handle to a datastore:
@ -208,6 +208,7 @@ mod manager;
mod readwrite;
pub mod backend;
#[cfg(feature = "lmdb")]
pub mod migrator;
pub mod store;
pub mod value;
@ -224,6 +225,7 @@ pub use error::{
StoreError,
};
pub use manager::Manager;
#[cfg(feature = "lmdb")]
pub use migrator::Migrator;
pub use readwrite::{
Readable,

10
src/manager.rs
Unescape View File

@ -27,11 +27,12 @@ use std::{
use lazy_static::lazy_static;
#[cfg(feature = "lmdb")]
use crate::backend::LmdbEnvironment;
use crate::{
backend::{
BackendEnvironment,
BackendEnvironmentBuilder,
LmdbEnvironment,
SafeModeEnvironment,
},
error::{
@ -47,8 +48,12 @@ type Result<T> = result::Result<T, StoreError>;
type CloseResult<T> = result::Result<T, CloseError>;
type SharedRkv<E> = Arc<RwLock<Rkv<E>>>;
#[cfg(feature = "lmdb")]
lazy_static! {
static ref MANAGER_LMDB: RwLock<Manager<LmdbEnvironment>> = RwLock::new(Manager::new());
}
lazy_static! {
static ref MANAGER_SAFE_MODE: RwLock<Manager<SafeModeEnvironment>> = RwLock::new(Manager::new());
}
@ -174,6 +179,7 @@ where
}
}
#[cfg(feature = "lmdb")]
impl Manager<LmdbEnvironment> {
pub fn singleton() -> &'static RwLock<Manager<LmdbEnvironment>> {
&*MANAGER_LMDB
@ -195,9 +201,11 @@ mod tests {
use tempfile::Builder;
#[cfg(feature = "lmdb")]
use backend::Lmdb;
/// Test that one can mutate managed Rkv instances in surprising ways.
#[cfg(feature = "lmdb")]
#[test]
fn test_mutate_managed_rkv() {
let mut manager = Manager::<LmdbEnvironment>::new();

14
src/store/integer.rs
Unescape View File

@ -93,7 +93,7 @@ mod tests {
let root = Builder::new().prefix("test_integer_keys").tempdir().expect("tempdir");
fs::create_dir_all(root.path()).expect("dir created");
let k = Rkv::new::<backend::Lmdb>(root.path()).expect("new succeeded");
let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
let s = k.open_integer("s", StoreOptions::create()).expect("open");
macro_rules! test_integer_keys {
@ -118,7 +118,7 @@ mod tests {
let root = Builder::new().prefix("test_integer_clear").tempdir().expect("tempdir");
fs::create_dir_all(root.path()).expect("dir created");
let k = Rkv::new::<backend::Lmdb>(root.path()).expect("new succeeded");
let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
let s = k.open_integer("s", StoreOptions::create()).expect("open");
{
@ -149,7 +149,7 @@ mod tests {
let root = Builder::new().prefix("test_integer_dup").tempdir().expect("tempdir");
fs::create_dir_all(root.path()).expect("dir created");
let k = Rkv::new::<backend::Lmdb>(root.path()).expect("new succeeded");
let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
let s = k.open_integer("s", StoreOptions::create()).expect("open");
{
@ -180,7 +180,7 @@ mod tests {
let root = Builder::new().prefix("test_integer_del").tempdir().expect("tempdir");
fs::create_dir_all(root.path()).expect("dir created");
let k = Rkv::new::<backend::Lmdb>(root.path()).expect("new succeeded");
let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
let s = k.open_integer("s", StoreOptions::create()).expect("open");
{
@ -228,7 +228,7 @@ mod tests {
fs::create_dir_all(root.path()).expect("dir created");
{
let k = Rkv::new::<backend::Lmdb>(root.path()).expect("new succeeded");
let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
let s = k.open_integer("s", StoreOptions::create()).expect("open");
let mut writer = k.write().expect("writer");
@ -242,7 +242,7 @@ mod tests {
}
{
let k = Rkv::new::<backend::Lmdb>(root.path()).expect("new succeeded");
let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
let s = k.open_integer("s", StoreOptions::create()).expect("open");
let reader = k.read().expect("reader");
@ -257,7 +257,7 @@ mod tests {
let root = Builder::new().prefix("test_integer_intertwine_read_write").tempdir().expect("tempdir");
fs::create_dir_all(root.path()).expect("dir created");
let k = Rkv::new::<backend::Lmdb>(root.path()).expect("new succeeded");
let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
let s = k.open_integer("s", StoreOptions::create()).expect("open");
{

14
src/store/integermulti.rs
Unescape View File

@ -122,7 +122,7 @@ mod tests {
let root = Builder::new().prefix("test_integer_keys").tempdir().expect("tempdir");
fs::create_dir_all(root.path()).expect("dir created");
let k = Rkv::new::<backend::Lmdb>(root.path()).expect("new succeeded");
let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
let s = k.open_multi_integer("s", StoreOptions::create()).expect("open");
macro_rules! test_integer_keys {
@ -147,7 +147,7 @@ mod tests {
let root = Builder::new().prefix("test_multi_integer_clear").tempdir().expect("tempdir");
fs::create_dir_all(root.path()).expect("dir created");
let k = Rkv::new::<backend::Lmdb>(root.path()).expect("new succeeded");
let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
let s = k.open_multi_integer("s", StoreOptions::create()).expect("open");
{
@ -178,7 +178,7 @@ mod tests {
let root = Builder::new().prefix("test_multi_integer_dup").tempdir().expect("tempdir");
fs::create_dir_all(root.path()).expect("dir created");
let k = Rkv::new::<backend::Lmdb>(root.path()).expect("new succeeded");
let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
let s = k.open_multi_integer("s", StoreOptions::create()).expect("open");
{
@ -209,7 +209,7 @@ mod tests {
let root = Builder::new().prefix("test_multi_integer_dup").tempdir().expect("tempdir");
fs::create_dir_all(root.path()).expect("dir created");
let k = Rkv::new::<backend::Lmdb>(root.path()).expect("new succeeded");
let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
let s = k.open_multi_integer("s", StoreOptions::create()).expect("open");
{
@ -230,7 +230,7 @@ mod tests {
let root = Builder::new().prefix("test_multi_integer_dup").tempdir().expect("tempdir");
fs::create_dir_all(root.path()).expect("dir created");
let k = Rkv::new::<backend::Lmdb>(root.path()).expect("new succeeded");
let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
let s = k.open_multi_integer("s", StoreOptions::create()).expect("open");
{
@ -296,7 +296,7 @@ mod tests {
fs::create_dir_all(root.path()).expect("dir created");
{
let k = Rkv::new::<backend::Lmdb>(root.path()).expect("new succeeded");
let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
let s = k.open_multi_integer("s", StoreOptions::create()).expect("open");
let mut writer = k.write().expect("writer");
@ -313,7 +313,7 @@ mod tests {
}
{
let k = Rkv::new::<backend::Lmdb>(root.path()).expect("new succeeded");
let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
let s = k.open_multi_integer("s", StoreOptions::create()).expect("open");
let reader = k.read().expect("reader");

2
tests/env-lmdb.rs
Unescape View File

@ -1222,7 +1222,7 @@ fn test_store_multiple_thread() {
Ok(Some(Value::U64(value))) => value,
Ok(Some(_)) => panic!("value type unexpected"),
Ok(None) => panic!("value not found"),
Err(err) => panic!(err),
Err(err) => panic!("{}", err),
};
assert_eq!(value, i);
value

2
tests/env-safe.rs
Unescape View File

@ -993,7 +993,7 @@ fn test_store_multiple_thread_safe() {
Ok(Some(Value::U64(value))) => value,
Ok(Some(_)) => panic!("value type unexpected"),
Ok(None) => panic!("value not found"),
Err(err) => panic!(err),
Err(err) => panic!("{}", err),
};
assert_eq!(value, i);
value

4
tests/integer-store.rs
Unescape View File

@ -16,7 +16,7 @@ use serde_derive::Serialize;
use tempfile::Builder;
use rkv::{
backend::Lmdb,
backend::SafeMode,
PrimitiveInt,
Rkv,
StoreOptions,
@ -28,7 +28,7 @@ fn test_integer_keys() {
let root = Builder::new().prefix("test_integer_keys").tempdir().expect("tempdir");
fs::create_dir_all(root.path()).expect("dir created");
let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
let s = k.open_integer("s", StoreOptions::create()).expect("open");
macro_rules! test_integer_keys {

11
tests/manager.rs
Unescape View File

@ -15,11 +15,14 @@ use std::{
use tempfile::Builder;
#[cfg(feature = "lmdb")]
use rkv::backend::{
Lmdb,
LmdbEnvironment,
};
use rkv::{
backend::{
BackendEnvironmentBuilder,
Lmdb,
LmdbEnvironment,
SafeMode,
SafeModeEnvironment,
},
@ -30,6 +33,7 @@ use rkv::{
};
/// Test that a manager can be created with simple type inference.
#[cfg(feature = "lmdb")]
#[test]
#[allow(clippy::let_underscore_lock)]
fn test_simple() {
@ -48,6 +52,7 @@ fn test_simple_safe() {
}
/// Test that a shared Rkv instance can be created with simple type inference.
#[cfg(feature = "lmdb")]
#[test]
fn test_simple_2() {
type Manager = rkv::Manager<LmdbEnvironment>;
@ -72,6 +77,7 @@ fn test_simple_safe_2() {
}
/// Test that the manager will return the same Rkv instance each time for each path.
#[cfg(feature = "lmdb")]
#[test]
fn test_same() {
type Manager = rkv::Manager<LmdbEnvironment>;
@ -104,6 +110,7 @@ fn test_same_safe() {
}
/// Test that the manager will return the same Rkv instance each time for each path.
#[cfg(feature = "lmdb")]
#[test]
fn test_same_with_capacity() {
type Manager = rkv::Manager<LmdbEnvironment>;

4
tests/multi-integer-store.rs
Unescape View File

@ -16,7 +16,7 @@ use serde_derive::Serialize;
use tempfile::Builder;
use rkv::{
backend::Lmdb,
backend::SafeMode,
PrimitiveInt,
Rkv,
StoreOptions,
@ -28,7 +28,7 @@ fn test_multi_integer_keys() {
let root = Builder::new().prefix("test_integer_keys").tempdir().expect("tempdir");
fs::create_dir_all(root.path()).expect("dir created");
let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
let s = k.open_multi_integer("s", StoreOptions::create()).expect("open");
macro_rules! test_integer_keys {

22
tests/test_txn.rs
Unescape View File

@ -15,10 +15,10 @@ use tempfile::Builder;
use rkv::{
backend::{
Lmdb,
LmdbDatabase,
LmdbRoCursor,
LmdbRwTransaction,
SafeMode,
SafeModeDatabase,
SafeModeRoCursor,
SafeModeRwTransaction,
},
Readable,
Rkv,
@ -41,14 +41,14 @@ use rkv::{
/// Note that the reader functions take `Readable` because they might run within a Read
/// Transaction or a Write Transaction. The test demonstrates fetching values via both.
type SingleStore = rkv::SingleStore<LmdbDatabase>;
type MultiStore = rkv::MultiStore<LmdbDatabase>;
type SingleStore = rkv::SingleStore<SafeModeDatabase>;
type MultiStore = rkv::MultiStore<SafeModeDatabase>;
#[test]
fn read_many() {
let root = Builder::new().prefix("test_txns").tempdir().expect("tempdir");
fs::create_dir_all(root.path()).expect("dir created");
let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
let samplestore = k.open_single("s", StoreOptions::create()).expect("open");
let datestore = k.open_multi("m", StoreOptions::create()).expect("open");
let valuestore = k.open_multi("m", StoreOptions::create()).expect("open");
@ -92,7 +92,7 @@ fn read_many() {
fn get_ids_by_field<'t, T>(txn: &'t T, store: MultiStore, field: &'t str) -> Vec<u64>
where
T: Readable<'t, Database = LmdbDatabase, RoCursor = LmdbRoCursor<'t>>,
T: Readable<'t, Database = SafeModeDatabase, RoCursor = SafeModeRoCursor<'t>>,
{
store
.get(txn, field)
@ -108,7 +108,7 @@ where
fn get_samples<'t, T>(txn: &'t T, samplestore: SingleStore, ids: &[u64]) -> Vec<String>
where
T: Readable<'t, Database = LmdbDatabase, RoCursor = LmdbRoCursor<'t>>,
T: Readable<'t, Database = SafeModeDatabase, RoCursor = SafeModeRoCursor<'t>>,
{
ids.iter()
.map(|id| {
@ -122,11 +122,11 @@ where
.collect::<Vec<String>>()
}
fn put_sample(txn: &mut Writer<LmdbRwTransaction>, samplestore: SingleStore, id: u64, value: &str) {
fn put_sample(txn: &mut Writer<SafeModeRwTransaction>, samplestore: SingleStore, id: u64, value: &str) {
let idbytes = id.to_be_bytes();
samplestore.put(txn, &idbytes, &Value::Str(value)).expect("put id");
}
fn put_id_field(txn: &mut Writer<LmdbRwTransaction>, store: MultiStore, field: &str, id: u64) {
fn put_id_field(txn: &mut Writer<SafeModeRwTransaction>, store: MultiStore, field: &str, id: u64) {
store.put(txn, field, &Value::U64(id)).expect("put id");
}

Write Preview
Loading…
Cancel
Save