NextGraph
/
rkv
mirror of https://github.com/nextgraph-org/rkv.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactor into store-centric API (#101)

Browse Source
without.crypto
Rick Richardson 5 years ago committed by Myk Melez
parent a2f760faaa
commit a81dfbeb10
13 changed files with 954 additions and 584 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 2
      Cargo.toml
  2. 18
      examples/iterator.rs
  3. 125
      examples/simple-store.rs
  4. 476
      src/env.rs
  5. 109
      src/lib.rs
  6. 174
      src/readwrite.rs
  7. 21
      src/store.rs
  8. 91
      src/store/integer.rs
  9. 117
      src/store/integermulti.rs
  10. 161
      src/store/multi.rs
  11. 116
      src/store/single.rs
  12. 38
      tests/integer-store.rs
  13. 90
      tests/multi-integer-store.rs

2
Cargo.toml
Unescape View File

@ -18,8 +18,8 @@ backtrace = ["failure/backtrace", "failure/std"]
[dependencies]
arrayref = "0.3"
bincode = "1.0"
lmdb-rkv = "0.11"
lazy_static = "1.0.2"
lmdb-rkv = "0.9"
ordered-float = "1.0"
uuid = "0.7"
serde = "1.0"

18
examples/iterator.rs
Unescape View File

@ -10,8 +10,10 @@
use rkv::{
Manager,
Rkv,
Store,
SingleStore,
StoreError,
StoreOptions,
Transaction,
Value,
};
use tempfile::Builder;
@ -26,7 +28,7 @@ fn main() {
let created_arc = Manager::singleton().write().unwrap().get_or_create(p, Rkv::new).unwrap();
let k = created_arc.read().unwrap();
let store = k.open_or_create("store").unwrap();
let store = k.open_single("store", StoreOptions::create()).unwrap();
populate_store(&k, store).unwrap();
@ -35,7 +37,7 @@ fn main() {
println!("Iterating from the beginning...");
// Reader::iter_start() iterates from the first item in the store, and
// returns the (key, value) tuples in order.
let mut iter = reader.iter_start(store).unwrap();
let mut iter = store.iter_start(&reader).unwrap();
while let Some((country, city)) = iter.next() {
println!("{}, {:?}", str::from_utf8(country).unwrap(), city);
}
@ -44,20 +46,20 @@ fn main() {
println!("Iterating from the given key...");
// Reader::iter_from() iterates from the first key equal to or greater
// than the given key.
let mut iter = reader.iter_from(store, "Japan").unwrap();
let mut iter = store.iter_from(&reader, "Japan").unwrap();
while let Some((country, city)) = iter.next() {
println!("{}, {:?}", str::from_utf8(country).unwrap(), city);
}
println!("");
println!("Iterating from the given prefix...");
let mut iter = reader.iter_from(store, "Un").unwrap();
let mut iter = store.iter_from(&reader, "Un").unwrap();
while let Some((country, city)) = iter.next() {
println!("{}, {:?}", str::from_utf8(country).unwrap(), city);
}
}
fn populate_store(k: &Rkv, store: Store) -> Result<(), StoreError> {
fn populate_store(k: &Rkv, mut store: SingleStore) -> Result<(), StoreError> {
let mut writer = k.write()?;
for (country, city) in vec![
("Canada", Value::Str("Ottawa")),
@ -68,7 +70,7 @@ fn populate_store(k: &Rkv, store: Store) -> Result<(), StoreError> {
("United Kingdom", Value::Str("London")),
("Japan", Value::Str("Tokyo")),
] {
writer.put(store, country, &city)?;
store.put(&mut writer, country, &city)?;
}
writer.commit()
writer.commit().map_err(|e| e.into())
}

125
examples/simple-store.rs
Unescape View File

@ -9,13 +9,45 @@
use rkv::{
Manager,
MultiStore,
Rkv,
RwTransaction,
StoreOptions,
Transaction,
Value,
};
use tempfile::Builder;
use std::fs;
fn getput<'env, 's>(mut store: MultiStore, writer: &'env mut RwTransaction, ids: &'s mut Vec<String>) {
let keys = vec!["str1", "str2", "str3"];
// we convert the writer into a cursor so that we can safely read
for k in keys.iter() {
// this is a multi-valued database, so get returns an iterator
let iter = store.get(writer, k).unwrap();
for (_key, val) in iter {
if let Value::Str(s) = val.unwrap().unwrap() {
ids.push(s.to_owned());
} else {
panic!("didn't get a string back!");
}
}
}
for i in 0..ids.len() {
let _r = store.put(writer, &ids[i], &Value::Blob(b"weeeeeee")).unwrap();
}
}
fn delete<'env, 's>(mut store: MultiStore, writer: &'env mut RwTransaction) {
let keys = vec!["str1", "str2", "str3"];
let vals = vec!["string uno", "string quatro", "string siete"];
// we convert the writer into a cursor so that we can safely read
for i in 0..keys.len() {
store.delete(writer, &keys[i], &Value::Str(vals[i])).unwrap();
}
}
fn main() {
let root = Builder::new().prefix("simple-db").tempdir().unwrap();
fs::create_dir_all(root.path()).unwrap();
@ -26,61 +58,82 @@ fn main() {
let k = created_arc.read().unwrap();
// Creates a store called "store"
let store = k.open_or_create("store").unwrap();
let mut store = k.open_single("store", StoreOptions::create()).unwrap();
let mut multistore = k.open_multi("multistore", StoreOptions::create()).unwrap();
println!("Inserting data...");
{
// Use a writer to mutate the store
let mut writer = k.write().unwrap();
writer.put(store, "int", &Value::I64(1234)).unwrap();
writer.put(store, "uint", &Value::U64(1234_u64)).unwrap();
writer.put(store, "float", &Value::F64(1234.0.into())).unwrap();
writer.put(store, "instant", &Value::Instant(1528318073700)).unwrap();
writer.put(store, "boolean", &Value::Bool(true)).unwrap();
writer.put(store, "string", &Value::Str("héllo, yöu")).unwrap();
writer.put(store, "json", &Value::Json(r#"{"foo":"bar", "number": 1}"#)).unwrap();
writer.put(store, "blob", &Value::Blob(b"blob")).unwrap();
store.put(&mut writer, "int", &Value::I64(1234)).unwrap();
store.put(&mut writer, "uint", &Value::U64(1234_u64)).unwrap();
store.put(&mut writer, "float", &Value::F64(1234.0.into())).unwrap();
store.put(&mut writer, "instant", &Value::Instant(1528318073700)).unwrap();
store.put(&mut writer, "boolean", &Value::Bool(true)).unwrap();
store.put(&mut writer, "string", &Value::Str("héllo, yöu")).unwrap();
store.put(&mut writer, "json", &Value::Json(r#"{"foo":"bar", "number": 1}"#)).unwrap();
store.put(&mut writer, "blob", &Value::Blob(b"blob")).unwrap();
writer.commit().unwrap();
}
println!("Testing getput");
{
let mut ids = Vec::new();
let mut writer = k.write().unwrap();
multistore.put(&mut writer, "str1", &Value::Str("string uno")).unwrap();
multistore.put(&mut writer, "str1", &Value::Str("string dos")).unwrap();
multistore.put(&mut writer, "str1", &Value::Str("string tres")).unwrap();
multistore.put(&mut writer, "str2", &Value::Str("string quatro")).unwrap();
multistore.put(&mut writer, "str2", &Value::Str("string cinco")).unwrap();
multistore.put(&mut writer, "str2", &Value::Str("string seis")).unwrap();
multistore.put(&mut writer, "str3", &Value::Str("string siete")).unwrap();
multistore.put(&mut writer, "str3", &Value::Str("string ocho")).unwrap();
multistore.put(&mut writer, "str3", &Value::Str("string nueve")).unwrap();
getput(multistore, &mut writer, &mut ids);
writer.commit().unwrap();
let mut writer = k.write().unwrap();
delete(multistore, &mut writer);
writer.commit().unwrap();
}
println!("Looking up keys...");
{
// Use a reader to query the store
let reader = k.read().unwrap();
println!("Get int {:?}", reader.get(store, "int").unwrap());
println!("Get uint {:?}", reader.get(store, "uint").unwrap());
println!("Get float {:?}", reader.get(store, "float").unwrap());
println!("Get instant {:?}", reader.get(store, "instant").unwrap());
println!("Get boolean {:?}", reader.get(store, "boolean").unwrap());
println!("Get string {:?}", reader.get(store, "string").unwrap());
println!("Get json {:?}", reader.get(store, "json").unwrap());
println!("Get blob {:?}", reader.get(store, "blob").unwrap());
println!("Get non-existent {:?}", reader.get(store, "non-existent").unwrap());
println!("Get int {:?}", store.get(&reader, "int").unwrap());
println!("Get uint {:?}", store.get(&reader, "uint").unwrap());
println!("Get float {:?}", store.get(&reader, "float").unwrap());
println!("Get instant {:?}", store.get(&reader, "instant").unwrap());
println!("Get boolean {:?}", store.get(&reader, "boolean").unwrap());
println!("Get string {:?}", store.get(&reader, "string").unwrap());
println!("Get json {:?}", store.get(&reader, "json").unwrap());
println!("Get blob {:?}", store.get(&reader, "blob").unwrap());
println!("Get non-existent {:?}", store.get(&reader, "non-existent").unwrap());
}
println!("Looking up keys via Writer.get()...");
{
let mut writer = k.write().unwrap();
writer.put(store, "foo", &Value::Str("bar")).unwrap();
writer.put(store, "bar", &Value::Str("baz")).unwrap();
writer.delete(store, "foo").unwrap();
println!("It should be None! ({:?})", writer.get(store, "foo").unwrap());
println!("Get bar ({:?})", writer.get(store, "bar").unwrap());
store.put(&mut writer, "foo", &Value::Str("bar")).unwrap();
store.put(&mut writer, "bar", &Value::Str("baz")).unwrap();
store.delete(&mut writer, "foo").unwrap();
println!("It should be None! ({:?})", store.get(&writer, "foo").unwrap());
println!("Get bar ({:?})", store.get(&writer, "bar").unwrap());
writer.commit().unwrap();
let reader = k.read().expect("reader");
println!("It should be None! ({:?})", reader.get(store, "foo").unwrap());
println!("Get bar {:?}", reader.get(store, "bar").unwrap());
println!("It should be None! ({:?})", store.get(&reader, "foo").unwrap());
println!("Get bar {:?}", store.get(&reader, "bar").unwrap());
}
println!("Aborting transaction...");
{
// Aborting a write transaction rollbacks the change(s)
let mut writer = k.write().unwrap();
writer.put(store, "foo", &Value::Str("bar")).unwrap();
store.put(&mut writer, "foo", &Value::Str("bar")).unwrap();
writer.abort();
let reader = k.read().expect("reader");
println!("It should be None! ({:?})", reader.get(store, "foo").unwrap());
println!("It should be None! ({:?})", store.get(&reader, "foo").unwrap());
// Explicitly aborting a transaction is not required unless an early
// abort is desired, since both read and write transactions will
// implicitly be aborted once they go out of scope.
@ -90,27 +143,27 @@ fn main() {
{
// Deleting a key/value also requires a write transaction
let mut writer = k.write().unwrap();
writer.put(store, "foo", &Value::Str("bar")).unwrap();
writer.delete(store, "foo").unwrap();
println!("It should be None! ({:?})", writer.get(store, "foo").unwrap());
store.put(&mut writer, "foo", &Value::Str("bar")).unwrap();
store.delete(&mut writer, "foo").unwrap();
println!("It should be None! ({:?})", store.get(&writer, "foo").unwrap());
writer.commit().unwrap();
// Committing a transaction consumes the writer, preventing you
// from reusing it by failing and reporting a compile-time error.
// This line would report error[E0382]: use of moved value: `writer`.
// writer.put(store, "baz", &Value::Str("buz")).unwrap();
// store.put(&mut writer, "baz", &Value::Str("buz")).unwrap();
}
println!("Write and read on multiple stores...");
{
let another_store = k.open_or_create("another_store").unwrap();
let mut another_store = k.open_single("another_store", StoreOptions::create()).unwrap();
let mut writer = k.write().unwrap();
writer.put(store, "foo", &Value::Str("bar")).unwrap();
writer.put(another_store, "foo", &Value::Str("baz")).unwrap();
store.put(&mut writer, "foo", &Value::Str("bar")).unwrap();
another_store.put(&mut writer, "foo", &Value::Str("baz")).unwrap();
writer.commit().unwrap();
let reader = k.read().unwrap();
println!("Get from store value: {:?}", reader.get(store, "foo").unwrap());
println!("Get from another store value: {:?}", reader.get(another_store, "foo").unwrap());
println!("Get from store value: {:?}", store.get(&reader, "foo").unwrap());
println!("Get from another store value: {:?}", another_store.get(&reader, "foo").unwrap());
}
}

476
src/env.rs
Unescape View File

@ -18,26 +18,25 @@ use std::path::{
use lmdb;
use lmdb::{
Database,
DatabaseFlags,
Environment,
EnvironmentBuilder,
RoTransaction,
RwTransaction,
};
use crate::error::StoreError;
use crate::integer::{
IntegerReader,
use crate::store::integer::{
IntegerStore,
IntegerWriter,
Key,
PrimitiveInt,
};
use crate::readwrite::{
Reader,
Store,
Writer,
};
use crate::store::integermulti::MultiIntegerStore;
use crate::store::multi::MultiStore;
use crate::store::single::SingleStore;
use crate::store::Options as StoreOptions;
pub static DEFAULT_MAX_DBS: c_uint = 5;
@ -90,90 +89,84 @@ impl Rkv {
/// Store creation methods.
impl Rkv {
pub fn open_or_create_default(&self) -> Result<Store, StoreError> {
self.open_or_create(None)
}
pub fn open_or_create<'s, T>(&self, name: T) -> Result<Store, StoreError>
/// Create or Open an existing database in (&[u8] -> Single Value) mode.
/// Note: that create=true cannot be called concurrently with other operations
/// so if you are sure that the database exists, call this with create=false.
pub fn open_single<'s, T>(&self, name: T, opts: StoreOptions) -> Result<SingleStore, StoreError>
where
T: Into<Option<&'s str>>,
{
let flags = DatabaseFlags::empty();
self.open_or_create_with_flags(name, flags)
self.open(name, opts).map(SingleStore::new)
}
pub fn open_or_create_integer<'s, T>(&self, name: T) -> Result<IntegerStore, StoreError>
/// Create or Open an existing database in (Integer -> Single Value) mode.
/// Note: that create=true cannot be called concurrently with other operations
/// so if you are sure that the database exists, call this with create=false.
pub fn open_integer<'s, T, K: PrimitiveInt>(
&self,
name: T,
mut opts: StoreOptions,
) -> Result<IntegerStore<K>, StoreError>
where
T: Into<Option<&'s str>>,
{
let mut flags = DatabaseFlags::empty();
flags.toggle(DatabaseFlags::INTEGER_KEY);
let db = self.env.create_db(name.into(), flags).map_err(|e| match e {
lmdb::Error::BadRslot => StoreError::open_during_transaction(),
_ => e.into(),
})?;
Ok(IntegerStore::new(db))
opts.flags.set(DatabaseFlags::INTEGER_KEY, true);
self.open(name, opts).map(IntegerStore::new)
}
pub fn open_or_create_with_flags<'s, T>(&self, name: T, flags: DatabaseFlags) -> Result<Store, StoreError>
/// Create or Open an existing database in (&[u8] -> Multiple Values) mode.
/// Note: that create=true cannot be called concurrently with other operations
/// so if you are sure that the database exists, call this with create=false.
pub fn open_multi<'s, T>(&self, name: T, mut opts: StoreOptions) -> Result<MultiStore, StoreError>
where
T: Into<Option<&'s str>>,
{
let db = self.env.create_db(name.into(), flags).map_err(|e| match e {
lmdb::Error::BadRslot => StoreError::open_during_transaction(),
_ => e.into(),
})?;
Ok(Store::new(db))
}
/// Open an existing database, unlike other `open_or_create_*` functions, it
/// opens the given database by using a read transaction, which means other
/// in-flight write transaction will not block this call. This is preferred
/// to be used in the read_only scenarios.
pub fn open<'s, T>(&self, name: T) -> Result<Store, StoreError>
opts.flags.set(DatabaseFlags::DUP_SORT, true);
self.open(name, opts).map(MultiStore::new)
}
/// Create or Open an existing database in (Integer -> Multiple Values) mode.
/// Note: that create=true cannot be called concurrently with other operations
/// so if you are sure that the database exists, call this with create=false.
pub fn open_multi_integer<'s, T, K: PrimitiveInt>(
&self,
name: T,
mut opts: StoreOptions,
) -> Result<MultiIntegerStore<K>, StoreError>
where
T: Into<Option<&'s str>>,
{
let db = self.env.open_db(name.into()).map_err(|e| match e {
lmdb::Error::BadRslot => StoreError::open_during_transaction(),
_ => e.into(),
})?;
Ok(Store::new(db))
opts.flags.set(DatabaseFlags::INTEGER_KEY, true);
opts.flags.set(DatabaseFlags::DUP_SORT, true);
self.open(name, opts).map(MultiIntegerStore::new)
}
}
/// Read and write accessors.
impl Rkv {
pub fn read<K>(&self) -> Result<Reader<K>, StoreError>
fn open<'s, T>(&self, name: T, opts: StoreOptions) -> Result<Database, StoreError>
where
K: AsRef<[u8]>,
{
let txn = self.env.begin_ro_txn()?;
Ok(Reader::new(txn))
}
pub fn write<K>(&self) -> Result<Writer<K>, StoreError>
where
K: AsRef<[u8]>,
T: Into<Option<&'s str>>,
{
let txn = self.env.begin_rw_txn()?;
Ok(Writer::new(txn))
if opts.create {
self.env.create_db(name.into(), opts.flags).map_err(|e| match e {
lmdb::Error::BadRslot => StoreError::open_during_transaction(),
_ => e.into(),
})
} else {
self.env.open_db(name.into()).map_err(|e| match e {
lmdb::Error::BadRslot => StoreError::open_during_transaction(),
_ => e.into(),
})
}
}
}
pub fn read_int<K>(&self) -> Result<IntegerReader<K>, StoreError>
where
K: PrimitiveInt,
{
let reader = self.read::<Key<K>>()?;
Ok(IntegerReader::new(reader))
/// Read and write accessors.
impl Rkv {
pub fn read(&self) -> Result<RoTransaction, StoreError> {
self.env.begin_ro_txn().map_err(|e| e.into())
}
pub fn write_int<K>(&self) -> Result<IntegerWriter<K>, StoreError>
where
K: PrimitiveInt,
{
let write = self.write::<Key<K>>()?;
Ok(IntegerWriter::new(write))
pub fn write(&self) -> Result<RwTransaction, StoreError> {
self.env.begin_rw_txn().map_err(|e| e.into())
}
}
@ -216,12 +209,12 @@ mod tests {
}
fn check_rkv(k: &Rkv) {
let _ = k.open_or_create_default().expect("created default");
let _ = k.open_single("default", StoreOptions::create()).expect("created default");
let yyy = k.open_or_create("yyy").expect("opened");
let yyy = k.open_single("yyy", StoreOptions::create()).expect("opened");
let reader = k.read().expect("reader");
let result = reader.get(yyy, "foo");
let result = yyy.get(&reader, "foo");
assert_eq!(None, result.expect("success but no value"));
}
@ -245,7 +238,7 @@ mod tests {
assert!(root.path().is_dir());
let mut builder = Rkv::environment_builder();
builder.set_max_dbs(1);
builder.set_max_dbs(2);
let k = Rkv::from_env(root.path(), builder).expect("rkv");
check_rkv(&k);
@ -268,7 +261,7 @@ mod tests {
// (plus the default database, which doesn't count against the limit).
// This should really return an error rather than panicking, per
// <https://github.com/mozilla/lmdb-rs/issues/6>.
let _zzz = k.open_or_create("zzz").expect("opened");
let _zzz = k.open_single("zzz", StoreOptions::create()).expect("opened");
}
fn get_larger_than_default_map_size_value() -> usize {
@ -293,13 +286,13 @@ mod tests {
assert!(root.path().is_dir());
let k = Rkv::new(root.path()).expect("new succeeded");
let sk: Store = k.open_or_create_default().expect("opened");
let mut sk: SingleStore = k.open_single("test", StoreOptions::create()).expect("opened");
// Writing a large enough value should cause LMDB to fail on MapFull.
// We write a string that is larger than the default map size.
let val = "x".repeat(get_larger_than_default_map_size_value());
let mut writer = k.write().expect("writer");
writer.put(sk, "foo", &Value::Str(&val)).expect("wrote");
sk.put(&mut writer, "foo", &Value::Str(&val)).expect("wrote");
}
#[test]
@ -313,16 +306,17 @@ mod tests {
// Set the map size to the size of the value we'll store in it + 100KiB,
// which ensures that there's enough space for the value and metadata.
builder.set_map_size(get_larger_than_default_map_size_value() + 100 * 1024 /* 100KiB */);
builder.set_max_dbs(2);
let k = Rkv::from_env(root.path(), builder).unwrap();
let sk: Store = k.open_or_create_default().expect("opened");
let mut sk: SingleStore = k.open_single("test", StoreOptions::create()).expect("opened");
let val = "x".repeat(get_larger_than_default_map_size_value());
let mut writer = k.write().expect("writer");
writer.put(sk, "foo", &Value::Str(&val)).expect("wrote");
sk.put(&mut writer, "foo", &Value::Str(&val)).expect("wrote");
writer.commit().expect("committed");
let reader = k.read().unwrap();
assert_eq!(reader.get(sk, "foo").expect("read"), Some(Value::Str(&val)));
assert_eq!(sk.get(&reader, "foo").expect("read"), Some(Value::Str(&val)));
}
#[test]
@ -331,43 +325,43 @@ mod tests {
fs::create_dir_all(root.path()).expect("dir created");
let k = Rkv::new(root.path()).expect("new succeeded");
let sk: Store = k.open_or_create("sk").expect("opened");
let mut sk: SingleStore = k.open_single("sk", StoreOptions::create()).expect("opened");
{
let mut writer = k.write().expect("writer");
writer.put(sk, "foo", &Value::I64(1234)).expect("wrote");
writer.put(sk, "noo", &Value::F64(1234.0.into())).expect("wrote");
writer.put(sk, "bar", &Value::Bool(true)).expect("wrote");
writer.put(sk, "baz", &Value::Str("héllo, yöu")).expect("wrote");
assert_eq!(writer.get(sk, "foo").expect("read"), Some(Value::I64(1234)));
assert_eq!(writer.get(sk, "noo").expect("read"), Some(Value::F64(1234.0.into())));
assert_eq!(writer.get(sk, "bar").expect("read"), Some(Value::Bool(true)));
assert_eq!(writer.get(sk, "baz").expect("read"), Some(Value::Str("héllo, yöu")));
sk.put(&mut writer, "foo", &Value::I64(1234)).expect("wrote");
sk.put(&mut writer, "noo", &Value::F64(1234.0.into())).expect("wrote");
sk.put(&mut writer, "bar", &Value::Bool(true)).expect("wrote");
sk.put(&mut writer, "baz", &Value::Str("héllo, yöu")).expect("wrote");
assert_eq!(sk.get(&writer, "foo").expect("read"), Some(Value::I64(1234)));
assert_eq!(sk.get(&writer, "noo").expect("read"), Some(Value::F64(1234.0.into())));
assert_eq!(sk.get(&writer, "bar").expect("read"), Some(Value::Bool(true)));
assert_eq!(sk.get(&writer, "baz").expect("read"), Some(Value::Str("héllo, yöu")));
// Isolation. Reads won't return values.
let r = &k.read().unwrap();
assert_eq!(r.get(sk, "foo").expect("read"), None);
assert_eq!(r.get(sk, "bar").expect("read"), None);
assert_eq!(r.get(sk, "baz").expect("read"), None);
assert_eq!(sk.get(r, "foo").expect("read"), None);
assert_eq!(sk.get(r, "bar").expect("read"), None);
assert_eq!(sk.get(r, "baz").expect("read"), None);
}
// Dropped: tx rollback. Reads will still return nothing.
{
let r = &k.read().unwrap();
assert_eq!(r.get(sk, "foo").expect("read"), None);
assert_eq!(r.get(sk, "bar").expect("read"), None);
assert_eq!(r.get(sk, "baz").expect("read"), None);
assert_eq!(sk.get(r, "foo").expect("read"), None);
assert_eq!(sk.get(r, "bar").expect("read"), None);
assert_eq!(sk.get(r, "baz").expect("read"), None);
}
{
let mut writer = k.write().expect("writer");
writer.put(sk, "foo", &Value::I64(1234)).expect("wrote");
writer.put(sk, "bar", &Value::Bool(true)).expect("wrote");
writer.put(sk, "baz", &Value::Str("héllo, yöu")).expect("wrote");
assert_eq!(writer.get(sk, "foo").expect("read"), Some(Value::I64(1234)));
assert_eq!(writer.get(sk, "bar").expect("read"), Some(Value::Bool(true)));
assert_eq!(writer.get(sk, "baz").expect("read"), Some(Value::Str("héllo, yöu")));
sk.put(&mut writer, "foo", &Value::I64(1234)).expect("wrote");
sk.put(&mut writer, "bar", &Value::Bool(true)).expect("wrote");
sk.put(&mut writer, "baz", &Value::Str("héllo, yöu")).expect("wrote");
assert_eq!(sk.get(&writer, "foo").expect("read"), Some(Value::I64(1234)));
assert_eq!(sk.get(&writer, "bar").expect("read"), Some(Value::Bool(true)));
assert_eq!(sk.get(&writer, "baz").expect("read"), Some(Value::Str("héllo, yöu")));
writer.commit().expect("committed");
}
@ -375,44 +369,44 @@ mod tests {
// Committed. Reads will succeed.
{
let r = k.read().unwrap();
assert_eq!(r.get(sk, "foo").expect("read"), Some(Value::I64(1234)));
assert_eq!(r.get(sk, "bar").expect("read"), Some(Value::Bool(true)));
assert_eq!(r.get(sk, "baz").expect("read"), Some(Value::Str("héllo, yöu")));
assert_eq!(sk.get(&r, "foo").expect("read"), Some(Value::I64(1234)));
assert_eq!(sk.get(&r, "bar").expect("read"), Some(Value::Bool(true)));
assert_eq!(sk.get(&r, "baz").expect("read"), Some(Value::Str("héllo, yöu")));
}
{
let mut writer = k.write().expect("writer");
writer.delete(sk, "foo").expect("deleted");
writer.delete(sk, "bar").expect("deleted");
writer.delete(sk, "baz").expect("deleted");
assert_eq!(writer.get(sk, "foo").expect("read"), None);
assert_eq!(writer.get(sk, "bar").expect("read"), None);
assert_eq!(writer.get(sk, "baz").expect("read"), None);
sk.delete(&mut writer, "foo").expect("deleted");
sk.delete(&mut writer, "bar").expect("deleted");
sk.delete(&mut writer, "baz").expect("deleted");
assert_eq!(sk.get(&writer, "foo").expect("read"), None);
assert_eq!(sk.get(&writer, "bar").expect("read"), None);
assert_eq!(sk.get(&writer, "baz").expect("read"), None);
// Isolation. Reads still return values.
let r = k.read().unwrap();
assert_eq!(r.get(sk, "foo").expect("read"), Some(Value::I64(1234)));
assert_eq!(r.get(sk, "bar").expect("read"), Some(Value::Bool(true)));
assert_eq!(r.get(sk, "baz").expect("read"), Some(Value::Str("héllo, yöu")));
assert_eq!(sk.get(&r, "foo").expect("read"), Some(Value::I64(1234)));
assert_eq!(sk.get(&r, "bar").expect("read"), Some(Value::Bool(true)));
assert_eq!(sk.get(&r, "baz").expect("read"), Some(Value::Str("héllo, yöu")));
}
// Dropped: tx rollback. Reads will still return values.
{
let r = k.read().unwrap();
assert_eq!(r.get(sk, "foo").expect("read"), Some(Value::I64(1234)));
assert_eq!(r.get(sk, "bar").expect("read"), Some(Value::Bool(true)));
assert_eq!(r.get(sk, "baz").expect("read"), Some(Value::Str("héllo, yöu")));
assert_eq!(sk.get(&r, "foo").expect("read"), Some(Value::I64(1234)));
assert_eq!(sk.get(&r, "bar").expect("read"), Some(Value::Bool(true)));
assert_eq!(sk.get(&r, "baz").expect("read"), Some(Value::Str("héllo, yöu")));
}
{
let mut writer = k.write().expect("writer");
writer.delete(sk, "foo").expect("deleted");
writer.delete(sk, "bar").expect("deleted");
writer.delete(sk, "baz").expect("deleted");
assert_eq!(writer.get(sk, "foo").expect("read"), None);
assert_eq!(writer.get(sk, "bar").expect("read"), None);
assert_eq!(writer.get(sk, "baz").expect("read"), None);
sk.delete(&mut writer, "foo").expect("deleted");
sk.delete(&mut writer, "bar").expect("deleted");
sk.delete(&mut writer, "baz").expect("deleted");
assert_eq!(sk.get(&writer, "foo").expect("read"), None);
assert_eq!(sk.get(&writer, "bar").expect("read"), None);
assert_eq!(sk.get(&writer, "baz").expect("read"), None);
writer.commit().expect("committed");
}
@ -420,10 +414,40 @@ mod tests {
// Committed. Reads will succeed but return None to indicate a missing value.
{
let r = k.read().unwrap();
assert_eq!(r.get(sk, "foo").expect("read"), None);
assert_eq!(r.get(sk, "bar").expect("read"), None);
assert_eq!(r.get(sk, "baz").expect("read"), None);
assert_eq!(sk.get(&r, "foo").expect("read"), None);
assert_eq!(sk.get(&r, "bar").expect("read"), None);
assert_eq!(sk.get(&r, "baz").expect("read"), None);
}
}
#[test]
fn test_multi_put_get_del() {
let root = Builder::new().prefix("test_multi_put_get_del").tempdir().expect("tempdir");
fs::create_dir_all(root.path()).expect("dir created");
let k = Rkv::new(root.path()).expect("new succeeded");
let mut multistore = k.open_multi("multistore", StoreOptions::create()).unwrap();
let mut writer = k.write().unwrap();
multistore.put(&mut writer, "str1", &Value::Str("str1 foo")).unwrap();
multistore.put(&mut writer, "str1", &Value::Str("str1 bar")).unwrap();
multistore.put(&mut writer, "str2", &Value::Str("str2 foo")).unwrap();
multistore.put(&mut writer, "str2", &Value::Str("str2 bar")).unwrap();
multistore.put(&mut writer, "str3", &Value::Str("str3 foo")).unwrap();
multistore.put(&mut writer, "str3", &Value::Str("str3 bar")).unwrap();
writer.commit().unwrap();
let mut writer = k.write().unwrap();
{
let mut iter = multistore.get(&mut writer, "str1").unwrap();
let (id, val) = iter.next().unwrap();
assert_eq!((id, val.unwrap().unwrap()), (&b"str1"[..], Value::Str("str1 bar")));
let (id, val) = iter.next().unwrap();
assert_eq!((id, val.unwrap().unwrap()), (&b"str1"[..], Value::Str("str1 foo")));
}
writer.commit().unwrap();
let mut writer = k.write().unwrap();
multistore.delete(&mut writer, "str1", &Value::Str("str1 foo")).unwrap();
multistore.delete(&mut writer, "str2", &Value::Str("str2 bar")).unwrap();
multistore.delete(&mut writer, "str3", &Value::Str("str3 bar")).unwrap();
writer.commit().unwrap();
}
#[test]
@ -432,17 +456,17 @@ mod tests {
fs::create_dir_all(root.path()).expect("dir created");
let k = Rkv::new(root.path()).expect("new succeeded");
// First create the store, and start a write transaction on it.
let sk = k.open_or_create("sk").expect("opened");
let mut sk = k.open_single("sk", StoreOptions::create()).expect("opened");
let mut writer = k.write().expect("writer");
writer.put(sk, "foo", &Value::Str("bar")).expect("write");
sk.put(&mut writer, "foo", &Value::Str("bar")).expect("write");
// Open the same store for read, note that the write transaction is still in progress,
// it should not block the reader though.
let sk_readonly = k.open("sk").expect("opened");
let sk_readonly = k.open_single("sk", StoreOptions::default()).expect("opened");
writer.commit().expect("commit");
// Now the write transaction is committed, any followed reads should see its change.
let reader = k.read().expect("reader");
assert_eq!(reader.get(sk_readonly, "foo").expect("read"), Some(Value::Str("bar")));
assert_eq!(sk_readonly.get(&reader, "foo").expect("read"), Some(Value::Str("bar")));
}
#[test]
@ -451,7 +475,7 @@ mod tests {
let root = Builder::new().prefix("test_open_a_missing_store").tempdir().expect("tempdir");
fs::create_dir_all(root.path()).expect("dir created");
let k = Rkv::new(root.path()).expect("new succeeded");
let _sk = k.open("sk").expect("open a missing store");
let _sk = k.open("sk", StoreOptions::default()).expect("open a missing store");
}
#[test]
@ -460,11 +484,11 @@ mod tests {
fs::create_dir_all(root.path()).expect("dir created");
let k = Rkv::new(root.path()).expect("new succeeded");
// First create the store
let _sk = k.open_or_create("sk").expect("opened");
let _sk = k.open_single("sk", StoreOptions::create()).expect("opened");
// Open a reader on this store
let _reader = k.read::<&str>().expect("reader");
let _reader = k.read().expect("reader");
// Open the same store for read while the reader is in progress will panic
let store: Result<Store, StoreError> = k.open("sk");
let store: Result<SingleStore, StoreError> = k.open_single("sk", StoreOptions::default());
match store {
Err(StoreError::OpenAttemptedDuringTransaction(_thread_id)) => assert!(true),
_ => panic!("should panic"),
@ -476,26 +500,26 @@ mod tests {
let root = Builder::new().prefix("test_read_before_write_num").tempdir().expect("tempdir");
fs::create_dir_all(root.path()).expect("dir created");
let k = Rkv::new(root.path()).expect("new succeeded");
let sk: Store = k.open_or_create("sk").expect("opened");
let mut sk: SingleStore = k.open_single("sk", StoreOptions::create()).expect("opened");
// Test reading a number, modifying it, and then writing it back.
// We have to be done with the Value::I64 before calling Writer::put,
// as the Value::I64 borrows an immutable reference to the Writer.
// So we extract and copy its primitive value.
fn get_existing_foo(writer: &Writer<&str>, store: Store) -> Option<i64> {
match writer.get(store, "foo").expect("read") {
fn get_existing_foo(txn: &RwTransaction, store: &SingleStore) -> Option<i64> {
match store.get(txn, "foo").expect("read") {
Some(Value::I64(val)) => Some(val),
_ => None,
}
}
let mut writer = k.write().expect("writer");
let mut existing = get_existing_foo(&writer, sk).unwrap_or(99);
let mut existing = get_existing_foo(&writer, &sk).unwrap_or(99);
existing += 1;
writer.put(sk, "foo", &Value::I64(existing)).expect("success");
sk.put(&mut writer, "foo", &Value::I64(existing)).expect("success");
let updated = get_existing_foo(&writer, sk).unwrap_or(99);
let updated = get_existing_foo(&writer, &sk).unwrap_or(99);
assert_eq!(updated, 100);
writer.commit().expect("commit");
}
@ -505,7 +529,7 @@ mod tests {
let root = Builder::new().prefix("test_read_before_write_str").tempdir().expect("tempdir");
fs::create_dir_all(root.path()).expect("dir created");
let k = Rkv::new(root.path()).expect("new succeeded");
let sk: Store = k.open_or_create("sk").expect("opened");
let mut sk: SingleStore = k.open_single("sk", StoreOptions::create()).expect("opened");
// Test reading a string, modifying it, and then writing it back.
// We have to be done with the Value::Str before calling Writer::put,
@ -513,13 +537,13 @@ mod tests {
// reference to the Writer. So we copy it to a String.
let mut writer = k.write().expect("writer");
let mut existing = match writer.get(sk, "foo").expect("read") {
let mut existing = match sk.get(&writer, "foo").expect("read") {
Some(Value::Str(val)) => val,
_ => "",
}
.to_string();
existing.push('…');
writer.put(sk, "foo", &Value::Str(&existing)).expect("write");
sk.put(&mut writer, "foo", &Value::Str(&existing)).expect("write");
writer.commit().expect("commit");
}
@ -529,8 +553,8 @@ mod tests {
fs::create_dir_all(root.path()).expect("dir created");
let k = Rkv::new(root.path()).expect("new succeeded");
let _first = k.read::<&str>().expect("reader");
let second = k.read::<&str>();
let _first = k.read().expect("reader");
let second = k.read();
match second {
Err(StoreError::ReadTransactionAlreadyExists(t)) => {
@ -547,41 +571,41 @@ mod tests {
let root = Builder::new().prefix("test_isolation").tempdir().expect("tempdir");
fs::create_dir_all(root.path()).expect("dir created");
let k = Rkv::new(root.path()).expect("new succeeded");
let s: Store = k.open_or_create("s").expect("opened");
let mut s: SingleStore = k.open_single("s", StoreOptions::create()).expect("opened");
// Add one field.
{
let mut writer = k.write().expect("writer");
writer.put(s, "foo", &Value::I64(1234)).expect("wrote");
s.put(&mut writer, "foo", &Value::I64(1234)).expect("wrote");
writer.commit().expect("committed");
}
{
let reader = k.read().unwrap();
assert_eq!(reader.get(s, "foo").expect("read"), Some(Value::I64(1234)));
assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(1234)));
}
// Establish a long-lived reader that outlasts a writer.
let reader = k.read().expect("reader");
assert_eq!(reader.get(s, "foo").expect("read"), Some(Value::I64(1234)));
assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(1234)));
// Start a write transaction.
let mut writer = k.write().expect("writer");
writer.put(s, "foo", &Value::I64(999)).expect("wrote");
s.put(&mut writer, "foo", &Value::I64(999)).expect("wrote");
// The reader and writer are isolated.
assert_eq!(reader.get(s, "foo").expect("read"), Some(Value::I64(1234)));
assert_eq!(writer.get(s, "foo").expect("read"), Some(Value::I64(999)));
assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(1234)));
assert_eq!(s.get(&writer, "foo").expect("read"), Some(Value::I64(999)));
// If we commit the writer, we still have isolation.
writer.commit().expect("committed");
assert_eq!(reader.get(s, "foo").expect("read"), Some(Value::I64(1234)));
assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(1234)));
// A new reader sees the committed value. Note that LMDB doesn't allow two
// read transactions to exist in the same thread, so we abort the previous one.
reader.abort();
let reader = k.read().expect("reader");
assert_eq!(reader.get(s, "foo").expect("read"), Some(Value::I64(999)));
assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(999)));
}
#[test]
@ -589,12 +613,12 @@ mod tests {
let root = Builder::new().prefix("test_round_trip_blob").tempdir().expect("tempdir");
fs::create_dir_all(root.path()).expect("dir created");
let k = Rkv::new(root.path()).expect("new succeeded");
let sk: Store = k.open_or_create("sk").expect("opened");
let mut sk: SingleStore = k.open_single("sk", StoreOptions::create()).expect("opened");
let mut writer = k.write().expect("writer");
assert_eq!(writer.get(sk, "foo").expect("read"), None);
writer.put(sk, "foo", &Value::Blob(&[1, 2, 3, 4])).expect("wrote");
assert_eq!(writer.get(sk, "foo").expect("read"), Some(Value::Blob(&[1, 2, 3, 4])));
assert_eq!(sk.get(&writer, "foo").expect("read"), None);
sk.put(&mut writer, "foo", &Value::Blob(&[1, 2, 3, 4])).expect("wrote");
assert_eq!(sk.get(&writer, "foo").expect("read"), Some(Value::Blob(&[1, 2, 3, 4])));
fn u16_to_u8(src: &[u16]) -> Vec<u8> {
let mut dst = vec![0; 2 * src.len()];
@ -612,56 +636,42 @@ mod tests {
// their [u16] backing storage to [u8]. Test that converting, writing,
// reading, and converting back works as expected.
let u16_array = [1000, 10000, 54321, 65535];
assert_eq!(writer.get(sk, "bar").expect("read"), None);
writer.put(sk, "bar", &Value::Blob(&u16_to_u8(&u16_array))).expect("wrote");
let u8_array = match writer.get(sk, "bar").expect("read") {
assert_eq!(sk.get(&writer, "bar").expect("read"), None);
sk.put(&mut writer, "bar", &Value::Blob(&u16_to_u8(&u16_array))).expect("wrote");
let u8_array = match sk.get(&writer, "bar").expect("read") {
Some(Value::Blob(val)) => val,
_ => &[],
};
assert_eq!(u8_to_u16(u8_array), u16_array);
}
#[test]
#[should_panic(expected = "not yet implemented")]
fn test_delete_value() {
let root = Builder::new().prefix("test_delete_value").tempdir().expect("tempdir");
fs::create_dir_all(root.path()).expect("dir created");
let k = Rkv::new(root.path()).expect("new succeeded");
let sk: Store = k.open_or_create_with_flags("sk", DatabaseFlags::DUP_SORT).expect("opened");
let mut writer = k.write().expect("writer");
writer.put(sk, "foo", &Value::I64(1234)).expect("wrote");
writer.put(sk, "foo", &Value::I64(1235)).expect("wrote");
writer.delete_value(sk, "foo", &Value::I64(1234)).expect("deleted");
}
#[test]
fn test_iter() {
let root = Builder::new().prefix("test_iter").tempdir().expect("tempdir");
fs::create_dir_all(root.path()).expect("dir created");
let k = Rkv::new(root.path()).expect("new succeeded");
let sk: Store = k.open_or_create("sk").expect("opened");
let mut sk: SingleStore = k.open_single("sk", StoreOptions::create()).expect("opened");
// An iterator over an empty store returns no values.
{
let reader = k.read::<&str>().unwrap();
let mut iter = reader.iter_start(sk).unwrap();
let reader = k.read().unwrap();
let mut iter = sk.iter_start(&reader).unwrap();
assert!(iter.next().is_none());
}
let mut writer = k.write().expect("writer");
writer.put(sk, "foo", &Value::I64(1234)).expect("wrote");
writer.put(sk, "noo", &Value::F64(1234.0.into())).expect("wrote");
writer.put(sk, "bar", &Value::Bool(true)).expect("wrote");
writer.put(sk, "baz", &Value::Str("héllo, yöu")).expect("wrote");
writer.put(sk, "héllò, töűrîst", &Value::Str("Emil.RuleZ!")).expect("wrote");
writer.put(sk, "你好，遊客", &Value::Str("米克規則")).expect("wrote");
sk.put(&mut writer, "foo", &Value::I64(1234)).expect("wrote");
sk.put(&mut writer, "noo", &Value::F64(1234.0.into())).expect("wrote");
sk.put(&mut writer, "bar", &Value::Bool(true)).expect("wrote");
sk.put(&mut writer, "baz", &Value::Str("héllo, yöu")).expect("wrote");
sk.put(&mut writer, "héllò, töűrîst", &Value::Str("Emil.RuleZ!")).expect("wrote");
sk.put(&mut writer, "你好，遊客", &Value::Str("米克規則")).expect("wrote");
writer.commit().expect("committed");
let reader = k.read().unwrap();
// Reader.iter() returns (key, value) tuples ordered by key.
let mut iter = reader.iter_start(sk).unwrap();
let mut iter = sk.iter_start(&reader).unwrap();
let (key, val) = iter.next().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "bar");
assert_eq!(val.expect("value"), Some(Value::Bool(true)));
@ -688,7 +698,7 @@ mod tests {
// Reader.iter_from() begins iteration at the first key equal to
// or greater than the given key.
let mut iter = reader.iter_from(sk, "moo").unwrap();
let mut iter = sk.iter_from(&reader, "moo").unwrap();
let (key, val) = iter.next().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "noo");
assert_eq!(val.expect("value"), Some(Value::F64(1234.0.into())));
@ -699,7 +709,7 @@ mod tests {
// Reader.iter_from() works as expected when the given key is a prefix
// of a key in the store.
let mut iter = reader.iter_from(sk, "no").unwrap();
let mut iter = sk.iter_from(&reader, "no").unwrap();
let (key, val) = iter.next().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "noo");
assert_eq!(val.expect("value"), Some(Value::F64(1234.0.into())));
@ -714,17 +724,17 @@ mod tests {
let root = Builder::new().prefix("test_iter_from_key_greater_than_existing").tempdir().expect("tempdir");
fs::create_dir_all(root.path()).expect("dir created");
let k = Rkv::new(root.path()).expect("new succeeded");
let sk: Store = k.open_or_create("sk").expect("opened");
let mut sk: SingleStore = k.open_single("sk", StoreOptions::create()).expect("opened");
let mut writer = k.write().expect("writer");
writer.put(sk, "foo", &Value::I64(1234)).expect("wrote");
writer.put(sk, "noo", &Value::F64(1234.0.into())).expect("wrote");
writer.put(sk, "bar", &Value::Bool(true)).expect("wrote");
writer.put(sk, "baz", &Value::Str("héllo, yöu")).expect("wrote");
sk.put(&mut writer, "foo", &Value::I64(1234)).expect("wrote");
sk.put(&mut writer, "noo", &Value::F64(1234.0.into())).expect("wrote");
sk.put(&mut writer, "bar", &Value::Bool(true)).expect("wrote");
sk.put(&mut writer, "baz", &Value::Str("héllo, yöu")).expect("wrote");
writer.commit().expect("committed");
let reader = k.read().unwrap();
let mut iter = reader.iter_from(sk, "nuu").unwrap();
let mut iter = sk.iter_from(&reader, "nuu").unwrap();
assert!(iter.next().is_none());
}
@ -734,38 +744,38 @@ mod tests {
fs::create_dir_all(root.path()).expect("dir created");
let k = Rkv::new(root.path()).expect("new succeeded");
let s1: Store = k.open_or_create("store_1").expect("opened");
let s2: Store = k.open_or_create("store_2").expect("opened");
let s3: Store = k.open_or_create("store_3").expect("opened");
let mut s1: SingleStore = k.open_single("store_1", StoreOptions::create()).expect("opened");
let mut s2: SingleStore = k.open_single("store_2", StoreOptions::create()).expect("opened");
let mut s3: SingleStore = k.open_single("store_3", StoreOptions::create()).expect("opened");
let mut writer = k.write().expect("writer");
writer.put(s1, "foo", &Value::Str("bar")).expect("wrote");
writer.put(s2, "foo", &Value::I64(123)).expect("wrote");
writer.put(s3, "foo", &Value::Bool(true)).expect("wrote");
s1.put(&mut writer, "foo", &Value::Str("bar")).expect("wrote");
s2.put(&mut writer, "foo", &Value::I64(123)).expect("wrote");
s3.put(&mut writer, "foo", &Value::Bool(true)).expect("wrote");
assert_eq!(writer.get(s1, "foo").expect("read"), Some(Value::Str("bar")));
assert_eq!(writer.get(s2, "foo").expect("read"), Some(Value::I64(123)));
assert_eq!(writer.get(s3, "foo").expect("read"), Some(Value::Bool(true)));
assert_eq!(s1.get(&mut writer, "foo").expect("read"), Some(Value::Str("bar")));
assert_eq!(s2.get(&mut writer, "foo").expect("read"), Some(Value::I64(123)));
assert_eq!(s3.get(&mut writer, "foo").expect("read"), Some(Value::Bool(true)));
writer.commit().expect("committed");
let reader = k.read().expect("unbound_reader");
assert_eq!(reader.get(s1, "foo").expect("read"), Some(Value::Str("bar")));
assert_eq!(reader.get(s2, "foo").expect("read"), Some(Value::I64(123)));
assert_eq!(reader.get(s3, "foo").expect("read"), Some(Value::Bool(true)));
assert_eq!(s1.get(&reader, "foo").expect("read"), Some(Value::Str("bar")));
assert_eq!(s2.get(&reader, "foo").expect("read"), Some(Value::I64(123)));
assert_eq!(s3.get(&reader, "foo").expect("read"), Some(Value::Bool(true)));
reader.abort();
// test delete across multiple stores
let mut writer = k.write().expect("writer");
writer.delete(s1, "foo").expect("deleted");
writer.delete(s2, "foo").expect("deleted");
writer.delete(s3, "foo").expect("deleted");
s1.delete(&mut writer, "foo").expect("deleted");
s2.delete(&mut writer, "foo").expect("deleted");
s3.delete(&mut writer, "foo").expect("deleted");
writer.commit().expect("committed");
let reader = k.read().expect("reader");
assert_eq!(reader.get(s1, "key").expect("value"), None);
assert_eq!(reader.get(s2, "key").expect("value"), None);
assert_eq!(reader.get(s3, "key").expect("value"), None);
assert_eq!(s1.get(&reader, "key").expect("value"), None);
assert_eq!(s2.get(&reader, "key").expect("value"), None);
assert_eq!(s3.get(&reader, "key").expect("value"), None);
}
#[test]
@ -773,30 +783,30 @@ mod tests {
let root = Builder::new().prefix("test_multiple_store_iter").tempdir().expect("tempdir");
fs::create_dir_all(root.path()).expect("dir created");
let k = Rkv::new(root.path()).expect("new succeeded");
let s1: Store = k.open_or_create("store_1").expect("opened");
let s2: Store = k.open_or_create("store_2").expect("opened");
let mut s1: SingleStore = k.open_single("store_1", StoreOptions::create()).expect("opened");
let mut s2: SingleStore = k.open_single("store_2", StoreOptions::create()).expect("opened");
let mut writer = k.write().expect("writer");
// Write to "s1"
writer.put(s1, "foo", &Value::I64(1234)).expect("wrote");
writer.put(s1, "noo", &Value::F64(1234.0.into())).expect("wrote");
writer.put(s1, "bar", &Value::Bool(true)).expect("wrote");
writer.put(s1, "baz", &Value::Str("héllo, yöu")).expect("wrote");
writer.put(s1, "héllò, töűrîst", &Value::Str("Emil.RuleZ!")).expect("wrote");
writer.put(s1, "你好，遊客", &Value::Str("米克規則")).expect("wrote");
// Writer to "s2"
writer.put(s2, "foo", &Value::I64(1234)).expect("wrote");
writer.put(s2, "noo", &Value::F64(1234.0.into())).expect("wrote");
writer.put(s2, "bar", &Value::Bool(true)).expect("wrote");
writer.put(s2, "baz", &Value::Str("héllo, yöu")).expect("wrote");
writer.put(s2, "héllò, töűrîst", &Value::Str("Emil.RuleZ!")).expect("wrote");
writer.put(s2, "你好，遊客", &Value::Str("米克規則")).expect("wrote");
s1.put(&mut writer, "foo", &Value::I64(1234)).expect("wrote");
s1.put(&mut writer, "noo", &Value::F64(1234.0.into())).expect("wrote");
s1.put(&mut writer, "bar", &Value::Bool(true)).expect("wrote");
s1.put(&mut writer, "baz", &Value::Str("héllo, yöu")).expect("wrote");
s1.put(&mut writer, "héllò, töűrîst", &Value::Str("Emil.RuleZ!")).expect("wrote");
s1.put(&mut writer, "你好，遊客", &Value::Str("米克規則")).expect("wrote");
// &mut writer to "s2"
s2.put(&mut writer, "foo", &Value::I64(1234)).expect("wrote");
s2.put(&mut writer, "noo", &Value::F64(1234.0.into())).expect("wrote");
s2.put(&mut writer, "bar", &Value::Bool(true)).expect("wrote");
s2.put(&mut writer, "baz", &Value::Str("héllo, yöu")).expect("wrote");
s2.put(&mut writer, "héllò, töűrîst", &Value::Str("Emil.RuleZ!")).expect("wrote");
s2.put(&mut writer, "你好，遊客", &Value::Str("米克規則")).expect("wrote");
writer.commit().expect("committed");
let reader = k.read().unwrap();
// Iterate through the whole store in "s1"
let mut iter = reader.iter_start(s1).unwrap();
let mut iter = s1.iter_start(&reader).unwrap();
let (key, val) = iter.next().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "bar");
assert_eq!(val.expect("value"), Some(Value::Bool(true)));
@ -818,7 +828,7 @@ mod tests {
assert!(iter.next().is_none());
// Iterate through the whole store in "s2"
let mut iter = reader.iter_start(s2).unwrap();
let mut iter = s2.iter_start(&reader).unwrap();
let (key, val) = iter.next().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "bar");
assert_eq!(val.expect("value"), Some(Value::Bool(true)));
@ -840,7 +850,7 @@ mod tests {
assert!(iter.next().is_none());
// Iterate from a given key in "s1"
let mut iter = reader.iter_from(s1, "moo").unwrap();
let mut iter = s1.iter_from(&reader, "moo").unwrap();
let (key, val) = iter.next().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "noo");
assert_eq!(val.expect("value"), Some(Value::F64(1234.0.into())));
@ -850,7 +860,7 @@ mod tests {
assert!(iter.next().is_none());
// Iterate from a given key in "s2"
let mut iter = reader.iter_from(s2, "moo").unwrap();
let mut iter = s2.iter_from(&reader, "moo").unwrap();
let (key, val) = iter.next().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "noo");
assert_eq!(val.expect("value"), Some(Value::F64(1234.0.into())));
@ -860,7 +870,7 @@ mod tests {
assert!(iter.next().is_none());
// Iterate from a given prefix in "s1"
let mut iter = reader.iter_from(s1, "no").unwrap();
let mut iter = s1.iter_from(&reader, "no").unwrap();
let (key, val) = iter.next().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "noo");
assert_eq!(val.expect("value"), Some(Value::F64(1234.0.into())));
@ -870,7 +880,7 @@ mod tests {
assert!(iter.next().is_none());
// Iterate from a given prefix in "s2"
let mut iter = reader.iter_from(s2, "no").unwrap();
let mut iter = s2.iter_from(&reader, "no").unwrap();
let (key, val) = iter.next().unwrap();
assert_eq!(str::from_utf8(key).expect("key"), "noo");
assert_eq!(val.expect("value"), Some(Value::F64(1234.0.into())));
@ -885,7 +895,7 @@ mod tests {
let root = Builder::new().prefix("test_multiple_thread").tempdir().expect("tempdir");
fs::create_dir_all(root.path()).expect("dir created");
let rkv_arc = Arc::new(RwLock::new(Rkv::new(root.path()).expect("new succeeded")));
let store = rkv_arc.read().unwrap().open_or_create_default().expect("opened");
let mut store = rkv_arc.read().unwrap().open_single("test", StoreOptions::create()).expect("opened");
let num_threads = 10;
let mut write_handles = Vec::with_capacity(num_threads as usize);
@ -902,7 +912,7 @@ mod tests {
write_handles.push(thread::spawn(move || {
let rkv = rkv_arc.write().expect("rkv");
let mut writer = rkv.write().expect("writer");
writer.put(store, i.to_string(), &Value::U64(i)).expect("written");
store.put(&mut writer, i.to_string(), &Value::U64(i)).expect("written");
writer.commit().unwrap();
}));
}
@ -917,7 +927,7 @@ mod tests {
read_handles.push(thread::spawn(move || {
let rkv = rkv_arc.read().expect("rkv");
let reader = rkv.read().expect("reader");
let value = match reader.get(store, i.to_string()) {
let value = match store.get(&reader, i.to_string()) {
Ok(Some(Value::U64(value))) => value,
Ok(Some(_)) => panic!("value type unexpected"),
Ok(None) => panic!("value not found"),

109
src/lib.rs
Unescape View File

@ -15,7 +15,8 @@
//! - Avoid LMDB's sharp edges (e.g., obscure error codes for common situations).
//! - Report errors via [failure](https://docs.rs/failure/).
//! - Correctly restrict access to one handle per process via a [Manager](struct.Manager.html).
//! - Use Rust's type system to make single-typed key stores (including LMDB's own integer-keyed stores) safe and ergonomic.
//! - Use Rust's type system to make single-typed key stores (including LMDB's own integer-keyed stores)
//! safe and ergonomic.
//! - Encode and decode values via [bincode](https://docs.rs/bincode/)/[serde](https://docs.rs/serde/)
//! and type tags, achieving platform-independent storage and input/output flexibility.
//!
@ -25,7 +26,8 @@
//! - [Rkv](struct.Rkv.html): an LMDB environment, which contains a set of key/value databases
//! - [Store](struct.Store.html): an LMDB database, which contains a set of key/value pairs
//!
//! Keys can be anything that implements `AsRef<[u8]>` or integers (when accessing an [IntegerStore](struct.IntegerStore.html)).
//! Keys can be anything that implements `AsRef<[u8]>` or integers
//! (when accessing an [IntegerStore](struct.IntegerStore.html)).
//! Values can be any of the types defined by the [Value](value/enum.Value.html) enum, including:
//!
//! - booleans (`Value::Bool`)
@ -38,7 +40,7 @@
//!
//! ## Basic Usage
//! ```
//! use rkv::{Manager, Rkv, Store, Value};
//! use rkv::{Manager, Rkv, SingleStore, Value, Transaction, StoreOptions};
//! use std::fs;
//! use tempfile::Builder;
//!
@ -64,7 +66,7 @@
//!
//! // Call `Rkv.open_or_create_default()` to get a handle to the default
//! // (unnamed) store for the environment.
//! let store: Store = env.open_or_create_default().unwrap();
//! let mut store: SingleStore = env.open_single("mydb", StoreOptions::create()).unwrap();
//!
//! {
//! // Use a write transaction to mutate the store by calling
@ -76,14 +78,14 @@
//! // Writer takes a `Store` as the first argument.
//! // Keys are `AsRef<[u8]>`, while values are `Value` enum instances.
//! // Use the `Blob` variant to store arbitrary collections of bytes.
//! writer.put(store, "int", &Value::I64(1234)).unwrap();
//! writer.put(store, "uint", &Value::U64(1234_u64)).unwrap();
//! writer.put(store, "float", &Value::F64(1234.0.into())).unwrap();
//! writer.put(store, "instant", &Value::Instant(1528318073700)).unwrap();
//! writer.put(store, "boolean", &Value::Bool(true)).unwrap();
//! writer.put(store, "string", &Value::Str("héllo, yöu")).unwrap();
//! writer.put(store, "json", &Value::Json(r#"{"foo":"bar", "number": 1}"#)).unwrap();
//! writer.put(store, "blob", &Value::Blob(b"blob")).unwrap();
//! store.put(&mut writer, "int", &Value::I64(1234)).unwrap();
//! store.put(&mut writer, "uint", &Value::U64(1234_u64)).unwrap();
//! store.put(&mut writer, "float", &Value::F64(1234.0.into())).unwrap();
//! store.put(&mut writer, "instant", &Value::Instant(1528318073700)).unwrap();
//! store.put(&mut writer, "boolean", &Value::Bool(true)).unwrap();
//! store.put(&mut writer, "string", &Value::Str("héllo, yöu")).unwrap();
//! store.put(&mut writer, "json", &Value::Json(r#"{"foo":"bar", "number": 1}"#)).unwrap();
//! store.put(&mut writer, "blob", &Value::Blob(b"blob")).unwrap();
//!
//! // You must commit a write transaction before the writer goes out
//! // of scope, or the transaction will abort and the data won't persist.
@ -98,17 +100,17 @@
//!
//! // To retrieve data, call `Reader.get()`, passing it the target store
//! // and the key for the value to retrieve.
//! println!("Get int {:?}", reader.get(store, "int").unwrap());
//! println!("Get uint {:?}", reader.get(store, "uint").unwrap());
//! println!("Get float {:?}", reader.get(store, "float").unwrap());
//! println!("Get instant {:?}", reader.get(store, "instant").unwrap());
//! println!("Get boolean {:?}", reader.get(store, "boolean").unwrap());
//! println!("Get string {:?}", reader.get(store, "string").unwrap());
//! println!("Get json {:?}", reader.get(store, "json").unwrap());
//! println!("Get blob {:?}", reader.get(store, "blob").unwrap());
//! println!("Get int {:?}", store.get(&reader, "int").unwrap());
//! println!("Get uint {:?}", store.get(&reader, "uint").unwrap());
//! println!("Get float {:?}", store.get(&reader, "float").unwrap());
//! println!("Get instant {:?}", store.get(&reader, "instant").unwrap());
//! println!("Get boolean {:?}", store.get(&reader, "boolean").unwrap());
//! println!("Get string {:?}", store.get(&reader, "string").unwrap());
//! println!("Get json {:?}", store.get(&reader, "json").unwrap());
//! println!("Get blob {:?}", store.get(&reader, "blob").unwrap());
//!
//! // Retrieving a non-existent value returns `Ok(None)`.
//! println!("Get non-existent value {:?}", reader.get(store, "non-existent"));
//! println!("Get non-existent value {:?}", store.get(&reader, "non-existent"));
//!
//! // A read transaction will automatically close once the reader
//! // goes out of scope, so isn't necessary to close it explicitly,
@ -118,11 +120,11 @@
//! {
//! // Aborting a write transaction rolls back the change(s).
//! let mut writer = env.write().unwrap();
//! writer.put(store, "foo", &Value::Str("bar")).unwrap();
//! store.put(&mut writer, "foo", &Value::Str("bar")).unwrap();
//! writer.abort();
//!
//! let reader = env.read().expect("reader");
//! println!("It should be None! ({:?})", reader.get(store, "foo").unwrap());
//! println!("It should be None! ({:?})", store.get(&reader, "foo").unwrap());
//! }
//!
//! {
@ -131,83 +133,84 @@
//! // implicitly be aborted once they go out of scope.
//! {
//! let mut writer = env.write().unwrap();
//! writer.put(store, "foo", &Value::Str("bar")).unwrap();
//! store.put(&mut writer, "foo", &Value::Str("bar")).unwrap();
//! }
//! let reader = env.read().expect("reader");
//! println!("It should be None! ({:?})", reader.get(store, "foo").unwrap());
//! println!("It should be None! ({:?})", store.get(&reader, "foo").unwrap());
//! }
//!
//! {
//! // Deleting a key/value pair also requires a write transaction.
//! let mut writer = env.write().unwrap();
//! writer.put(store, "foo", &Value::Str("bar")).unwrap();
//! writer.put(store, "bar", &Value::Str("baz")).unwrap();
//! writer.delete(store, "foo").unwrap();
//! store.put(&mut writer, "foo", &Value::Str("bar")).unwrap();
//! store.put(&mut writer, "bar", &Value::Str("baz")).unwrap();
//! store.delete(&mut writer, "foo").unwrap();
//!
//! // A write transaction also supports reading, the version of the
//! // store that it reads includes changes it has made regardless of
//! // the commit state of that transaction.
//! // In the code above, "foo" and "bar" were put into the store,
//! // then "foo" was deleted so only "bar" will return a result.
//! println!("It should be None! ({:?})", writer.get(store, "foo").unwrap());
//! println!("Get bar ({:?})", writer.get(store, "bar").unwrap());
//! println!("It should be None! ({:?})", store.get(&writer, "foo").unwrap());
//! println!("Get bar ({:?})", store.get(&writer, "bar").unwrap());
//! writer.commit().unwrap();
//! let reader = env.read().expect("reader");
//! println!("It should be None! ({:?})", reader.get(store, "foo").unwrap());
//! println!("Get bar {:?}", reader.get(store, "bar").unwrap());
//! println!("It should be None! ({:?})", store.get(&reader, "foo").unwrap());
//! println!("Get bar {:?}", store.get(&reader, "bar").unwrap());
//!
//! // Committing a transaction consumes the writer, preventing you
//! // from reusing it by failing at compile time with an error.
//! // This line would report error[E0382]: use of moved value: `writer`.
//! // writer.put(store, "baz", &Value::Str("buz")).unwrap();
//! // store.put(&mut writer, "baz", &Value::Str("buz")).unwrap();
//! }
//! ```
#![allow(dead_code)]
use bincode;
use lmdb;
use ordered_float;
use serde; // So we can specify trait bounds. Everything else is bincode.
use url;
use uuid;
pub use lmdb::{
DatabaseFlags,
EnvironmentBuilder,
EnvironmentFlags,
Error as LmdbError,
RoTransaction,
RwTransaction,
Transaction,
WriteFlags,
};
mod env;
pub mod error;
mod integer;
mod manager;
mod readwrite;
pub mod store;
pub mod value;
pub use self::env::Rkv;
pub use self::error::{
DataError,
StoreError,
pub use lmdb::{
Cursor,
Database,
Iter as LmdbIter,
RoCursor,
};
pub use self::integer::{
IntegerReader,
pub use self::store::integer::{
IntegerStore,
IntegerWriter,
PrimitiveInt,
};
pub use self::store::integermulti::MultiIntegerStore;
pub use self::store::multi::MultiStore;
pub use self::store::single::SingleStore;
pub use self::store::Options as StoreOptions;
pub use self::manager::Manager;
pub use self::env::Rkv;
pub use self::readwrite::{
Reader,
Store,
Writer,
pub use self::error::{
DataError,
StoreError,
};
pub use self::manager::Manager;
pub use self::value::{
OwnedValue,
Value,

174
src/readwrite.rs
Unescape View File

@ -1,174 +0,0 @@
// Copyright 2018-2019 Mozilla
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
// this file except in compliance with the License. You may obtain a copy of the
// License at http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.
use lmdb;
use std::marker::PhantomData;
use lmdb::{
Cursor,
Database,
Iter as LmdbIter,
RoCursor,
RoTransaction,
RwTransaction,
Transaction,
};
use lmdb::WriteFlags;
use crate::error::StoreError;
use crate::value::Value;
fn read_transform(val: Result<&[u8], lmdb::Error>) -> Result<Option<Value>, StoreError> {
match val {
Ok(bytes) => Value::from_tagged_slice(bytes).map(Some).map_err(StoreError::DataError),
Err(lmdb::Error::NotFound) => Ok(None),
Err(e) => Err(StoreError::LmdbError(e)),
}
}
pub struct Writer<'env, K>
where
K: AsRef<[u8]>,
{
tx: RwTransaction<'env>,
phantom: PhantomData<K>,
}
pub struct Reader<'env, K>
where
K: AsRef<[u8]>,
{
tx: RoTransaction<'env>,
phantom: PhantomData<K>,
}
pub struct Iter<'env> {
iter: LmdbIter<'env>,
cursor: RoCursor<'env>,
}
impl<'env, K> Writer<'env, K>
where
K: AsRef<[u8]>,
{
pub(crate) fn new(txn: RwTransaction) -> Writer<K> {
Writer {
tx: txn,
phantom: PhantomData,
}
}
pub fn get(&self, store: Store, k: K) -> Result<Option<Value>, StoreError> {
let bytes = self.tx.get(store.0, &k);
read_transform(bytes)
}
// TODO: flags
pub fn put(&mut self, store: Store, k: K, v: &Value) -> Result<(), StoreError> {
// TODO: don't allocate twice.
let bytes = v.to_bytes()?;
self.tx.put(store.0, &k, &bytes, WriteFlags::empty()).map_err(StoreError::LmdbError)
}
pub fn delete(&mut self, store: Store, k: K) -> Result<(), StoreError> {
self.tx.del(store.0, &k, None).map_err(StoreError::LmdbError)
}
pub fn delete_value(&mut self, _store: Store, _k: K, _v: &Value) -> Result<(), StoreError> {
// Even better would be to make this a method only on a dupsort store —
// it would need a little bit of reorganizing of types and traits,
// but when I see "If the database does not support sorted duplicate
// data items (MDB_DUPSORT) the data parameter is ignored" in the docs,
// I see a footgun that we can avoid by using the type system.
unimplemented!();
}
pub fn commit(self) -> Result<(), StoreError> {
self.tx.commit().map_err(StoreError::LmdbError)
}
pub fn abort(self) {
self.tx.abort();
}
}
impl<'env, K> Reader<'env, K>
where
K: AsRef<[u8]>,
{
pub(crate) fn new(txn: RoTransaction) -> Reader<K> {
Reader {
tx: txn,
phantom: PhantomData,
}
}
pub fn get(&self, store: Store, k: K) -> Result<Option<Value>, StoreError> {
let bytes = self.tx.get(store.0, &k);
read_transform(bytes)
}
pub fn abort(self) {
self.tx.abort();
}
pub fn iter_start(&self, store: Store) -> Result<Iter, StoreError> {
let mut cursor = self.tx.open_ro_cursor(store.0).map_err(StoreError::LmdbError)?;
// We call Cursor.iter() instead of Cursor.iter_start() because
// the latter panics at "called `Result::unwrap()` on an `Err` value:
// NotFound" when there are no items in the store, whereas the former
// returns an iterator that yields no items.
//
// And since we create the Cursor and don't change its position, we can
// be sure that a call to Cursor.iter() will start at the beginning.
//
let iter = cursor.iter();
Ok(Iter {
iter,
cursor,
})
}
pub fn iter_from(&self, store: Store, k: K) -> Result<Iter, StoreError> {
let mut cursor = self.tx.open_ro_cursor(store.0).map_err(StoreError::LmdbError)?;
let iter = cursor.iter_from(k);
Ok(Iter {
iter,
cursor,
})
}
}
impl<'env> Iterator for Iter<'env> {
type Item = (&'env [u8], Result<Option<Value<'env>>, StoreError>);
fn next(&mut self) -> Option<Self::Item> {
match self.iter.next() {
None => None,
Some((key, bytes)) => Some((key, read_transform(Ok(bytes)))),
}
}
}
/// New type around an `lmdb::Database`. At this time, the underlying LMDB
/// handle (within lmdb-rs::Database) is a C integer, so Copy is automatic.
#[derive(Copy, Clone)]
pub struct Store(Database);
impl Store {
pub fn new(db: Database) -> Store {
Store(db)
}
}

21
src/store.rs
Unescape View File

@ -0,0 +1,21 @@
pub mod integer;
pub mod integermulti;
pub mod multi;
pub mod single;
use lmdb::DatabaseFlags;
#[derive(Default, Debug, Copy, Clone)]
pub struct Options {
pub create: bool,
pub flags: DatabaseFlags,
}
impl Options {
pub fn create() -> Options {
Options {
create: true,
flags: DatabaseFlags::empty(),
}
}
}

91
src/integer.rs → src/store/integer.rs
Unescape View File

@ -14,7 +14,11 @@ use bincode::serialize;
use serde::Serialize;
use lmdb::Database;
use lmdb::{
Database,
RwTransaction,
Transaction,
};
use crate::error::{
DataError,
@ -23,11 +27,7 @@ use crate::error::{
use crate::value::Value;
use crate::readwrite::{
Reader,
Store,
Writer,
};
use crate::store::single::SingleStore;
pub trait EncodableKey {
fn to_bytes(&self) -> Result<Vec<u8>, DataError>;
@ -65,7 +65,7 @@ impl<K> Key<K>
where
K: EncodableKey,
{
fn new(k: K) -> Result<Key<K>, DataError> {
pub(crate) fn new(k: K) -> Result<Key<K>, DataError> {
Ok(Key {
bytes: k.to_bytes()?,
phantom: PhantomData,
@ -73,72 +73,35 @@ where
}
}
pub struct IntegerReader<'env, K>
where
K: PrimitiveInt,
{
inner: Reader<'env, Key<K>>,
}
impl<'env, K> IntegerReader<'env, K>
where
K: PrimitiveInt,
{
pub(crate) fn new(reader: Reader<Key<K>>) -> IntegerReader<K> {
IntegerReader {
inner: reader,
}
}
pub fn get(&self, store: IntegerStore, k: K) -> Result<Option<Value>, StoreError> {
self.inner.get(store.0, Key::new(k)?)
}
pub fn abort(self) {
self.inner.abort();
}
}
pub struct IntegerWriter<'env, K>
pub struct IntegerStore<K>
where
K: PrimitiveInt,
{
inner: Writer<'env, Key<K>>,
inner: SingleStore,
phantom: PhantomData<K>,
}
impl<'env, K> IntegerWriter<'env, K>
impl<K> IntegerStore<K>
where
K: PrimitiveInt,
{
pub(crate) fn new(writer: Writer<Key<K>>) -> IntegerWriter<K> {
IntegerWriter {
inner: writer,
pub(crate) fn new(db: Database) -> IntegerStore<K> {
IntegerStore {
inner: SingleStore::new(db),
phantom: PhantomData,
}
}
pub fn get(&self, store: IntegerStore, k: K) -> Result<Option<Value>, StoreError> {
self.inner.get(store.0, Key::new(k)?)
pub fn get<'env, T: Transaction>(&self, txn: &'env T, k: K) -> Result<Option<Value<'env>>, StoreError> {
self.inner.get(txn, Key::new(k)?)
}
pub fn put(&mut self, store: IntegerStore, k: K, v: &Value) -> Result<(), StoreError> {
self.inner.put(store.0, Key::new(k)?, v)
pub fn put(&mut self, txn: &mut RwTransaction, k: K, v: &Value) -> Result<(), StoreError> {
self.inner.put(txn, Key::new(k)?, v)
}
pub fn abort(self) {
self.inner.abort();
}
pub fn commit(self) -> Result<(), StoreError> {
self.inner.commit()
}
}
#[derive(Copy, Clone)]
pub struct IntegerStore(Store);
impl IntegerStore {
pub fn new(db: Database) -> IntegerStore {
IntegerStore(Store::new(db))
pub fn delete(&mut self, txn: &mut RwTransaction, k: K) -> Result<(), StoreError> {
self.inner.delete(txn, Key::new(k)?)
}
}
@ -155,18 +118,18 @@ mod tests {
let root = Builder::new().prefix("test_integer_keys").tempdir().expect("tempdir");
fs::create_dir_all(root.path()).expect("dir created");
let k = Rkv::new(root.path()).expect("new succeeded");
let s = k.open_or_create_integer("s").expect("open");
let mut s = k.open_integer("s", StoreOptions::create()).expect("open");
macro_rules! test_integer_keys {
($type:ty, $key:expr) => {{
let mut writer = k.write_int::<$type>().expect("writer");
let mut writer = k.write().expect("writer");
writer.put(s, $key, &Value::Str("hello!")).expect("write");
assert_eq!(writer.get(s, $key).expect("read"), Some(Value::Str("hello!")));
s.put(&mut writer, $key, &Value::Str("hello!")).expect("write");
assert_eq!(s.get(&writer, $key).expect("read"), Some(Value::Str("hello!")));
writer.commit().expect("committed");
let reader = k.read_int::<$type>().expect("reader");
assert_eq!(reader.get(s, $key).expect("read"), Some(Value::Str("hello!")));
let reader = k.read().expect("reader");
assert_eq!(s.get(&reader, $key).expect("read"), Some(Value::Str("hello!")));
}};
}

117
src/store/integermulti.rs
Unescape View File

@ -0,0 +1,117 @@
// Copyright 2018 Mozilla
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
// this file except in compliance with the License. You may obtain a copy of the
// License at http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.
use lmdb::{
Database,
RwTransaction,
Transaction,
WriteFlags,
};
use std::marker::PhantomData;
use crate::error::StoreError;
use crate::value::Value;
use crate::store::multi::{
Iter,
MultiStore,
};
use crate::store::integer::{
Key,
PrimitiveInt,
};
pub struct MultiIntegerStore<K>
where
K: PrimitiveInt,
{
inner: MultiStore,
phantom: PhantomData<K>,
}
impl<K> MultiIntegerStore<K>
where
K: PrimitiveInt,
{
pub(crate) fn new(db: Database) -> MultiIntegerStore<K> {
MultiIntegerStore {
inner: MultiStore::new(db),
phantom: PhantomData,
}
}
pub fn get<'env, T: Transaction>(&self, txn: &'env T, k: K) -> Result<Iter<'env>, StoreError> {
self.inner.get(txn, Key::new(k)?)
}
pub fn get_first<'env, T: Transaction>(&self, txn: &'env T, k: K) -> Result<Option<Value<'env>>, StoreError> {
self.inner.get_first(txn, Key::new(k)?)
}
pub fn put(&mut self, txn: &mut RwTransaction, k: K, v: &Value) -> Result<(), StoreError> {
self.inner.put(txn, Key::new(k)?, v)
}
pub fn put_with_flags(
&mut self,
txn: &mut RwTransaction,
k: K,
v: &Value,
flags: WriteFlags,
) -> Result<(), StoreError> {
self.inner.put_with_flags(txn, Key::new(k)?, v, flags)
}
pub fn delete_all(&mut self, txn: &mut RwTransaction, k: K) -> Result<(), StoreError> {
self.inner.delete_all(txn, Key::new(k)?)
}
pub fn delete(&mut self, txn: &mut RwTransaction, k: K, v: &Value) -> Result<(), StoreError> {
self.inner.delete(txn, Key::new(k)?, v)
}
}
#[cfg(test)]
mod tests {
extern crate tempfile;
use self::tempfile::Builder;
use std::fs;
use super::*;
use crate::*;
#[test]
fn test_integer_keys() {
let root = Builder::new().prefix("test_integer_keys").tempdir().expect("tempdir");
fs::create_dir_all(root.path()).expect("dir created");
let k = Rkv::new(root.path()).expect("new succeeded");
let mut s = k.open_multi_integer("s", StoreOptions::create()).expect("open");
macro_rules! test_integer_keys {
($type:ty, $key:expr) => {{
let mut writer = k.write().expect("writer");
s.put(&mut writer, $key, &Value::Str("hello!")).expect("write");
assert_eq!(s.get_first(&writer, $key).expect("read"), Some(Value::Str("hello!")));
writer.commit().expect("committed");
let reader = k.read().expect("reader");
assert_eq!(s.get_first(&reader, $key).expect("read"), Some(Value::Str("hello!")));
}};
}
test_integer_keys!(u32, std::u32::MIN);
test_integer_keys!(u32, std::u32::MAX);
}
}

161
src/store/multi.rs
Unescape View File

@ -0,0 +1,161 @@
// Copyright 2018 Mozilla
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
// this file except in compliance with the License. You may obtain a copy of the
// License at http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.
use lmdb;
use lmdb::{
Cursor,
Database,
Iter as LmdbIter,
// IterDup as LmdbIterDup,
RoCursor,
RwTransaction,
Transaction,
WriteFlags,
};
use crate::error::StoreError;
use crate::value::{
OwnedValue,
Value,
};
fn read_transform(val: Result<&[u8], lmdb::Error>) -> Result<Option<Value>, StoreError> {
match val {
Ok(bytes) => Value::from_tagged_slice(bytes).map(Some).map_err(StoreError::DataError),
Err(lmdb::Error::NotFound) => Ok(None),
Err(e) => Err(StoreError::LmdbError(e)),
}
}
fn read_transform_owned(val: Result<&[u8], lmdb::Error>) -> Result<Option<OwnedValue>, StoreError> {
match val {
Ok(bytes) => Value::from_tagged_slice(bytes).map(|v| Some(OwnedValue::from(&v))).map_err(StoreError::DataError),
Err(lmdb::Error::NotFound) => Ok(None),
Err(e) => Err(StoreError::LmdbError(e)),
}
}
#[derive(Copy, Clone)]
pub struct MultiStore {
db: Database,
}
pub struct Iter<'env> {
iter: LmdbIter<'env>,
cursor: RoCursor<'env>,
}
impl MultiStore {
pub(crate) fn new(db: Database) -> MultiStore {
MultiStore {
db,
}
}
/// Provides a cursor to all of the values for the duplicate entries that match this key
pub fn get<'env, T: Transaction, K: AsRef<[u8]>>(&self, txn: &'env T, k: K) -> Result<Iter<'env>, StoreError> {
let mut cursor = txn.open_ro_cursor(self.db).map_err(StoreError::LmdbError)?;
let iter = cursor.iter_dup_of(k);
Ok(Iter {
iter,
cursor,
})
}
/// Provides a cursor to all of the values for the duplicate entries that match this key
pub fn get_first<'env, T: Transaction, K: AsRef<[u8]>>(
&self,
txn: &'env T,
k: K,
) -> Result<Option<Value<'env>>, StoreError> {
let result = txn.get(self.db, &k);
read_transform(result)
}
/// Insert a value at the specified key.
/// This put will allow duplicate entries. If you wish to have duplicate entries
/// rejected, use the `put_with_flags` function and specify NO_DUP_DATA
pub fn put<K: AsRef<[u8]>>(&mut self, txn: &mut RwTransaction, k: K, v: &Value) -> Result<(), StoreError> {
let bytes = v.to_bytes()?;
txn.put(self.db, &k, &bytes, WriteFlags::empty()).map_err(StoreError::LmdbError)
}
pub fn put_with_flags<K: AsRef<[u8]>>(
&mut self,
txn: &mut RwTransaction,
k: K,
v: &Value,
flags: WriteFlags,
) -> Result<(), StoreError> {
let bytes = v.to_bytes()?;
txn.put(self.db, &k, &bytes, flags).map_err(StoreError::LmdbError)
}
pub fn delete_all<K: AsRef<[u8]>>(&mut self, txn: &mut RwTransaction, k: K) -> Result<(), StoreError> {
txn.del(self.db, &k, None).map_err(StoreError::LmdbError)
}
pub fn delete<K: AsRef<[u8]>>(&mut self, txn: &mut RwTransaction, k: K, v: &Value) -> Result<(), StoreError> {
txn.del(self.db, &k, Some(&v.to_bytes()?)).map_err(StoreError::LmdbError)
}
/* TODO - Figure out how to solve the need to have the cursor stick around when
* we are producing iterators from MultiIter
/// Provides an iterator starting at the lexographically smallest value in the store
pub fn iter_start(&self, store: MultiStore) -> Result<MultiIter, StoreError> {
let mut cursor = self.tx.open_ro_cursor(store.0).map_err(StoreError::LmdbError)?;
// We call Cursor.iter() instead of Cursor.iter_start() because
// the latter panics at "called `Result::unwrap()` on an `Err` value:
// NotFound" when there are no items in the store, whereas the former
// returns an iterator that yields no items.
//
// And since we create the Cursor and don't change its position, we can
// be sure that a call to Cursor.iter() will start at the beginning.
//
let iter = cursor.iter_dup();
Ok(MultiIter {
iter,
cursor,
})
}
*/
}
/*
impl<'env> Iterator for MultiIter<'env> {
type Item = Iter<'env>;
fn next(&mut self) -> Option<Self::Item> {
match self.iter.next() {
None => None,
Some(iter) => Some(Iter {
iter,
cursor,
}),
}
}
}
*/
impl<'env> Iterator for Iter<'env> {
type Item = (&'env [u8], Result<Option<Value<'env>>, StoreError>);
fn next(&mut self) -> Option<Self::Item> {
match self.iter.next() {
None => None,
Some(Ok((key, bytes))) => Some((key, read_transform(Ok(bytes)))),
Some(Err(_)) => None,
}
}
}

116
src/store/single.rs
Unescape View File

@ -0,0 +1,116 @@
// Copyright 2018 Mozilla
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
// this file except in compliance with the License. You may obtain a copy of the
// License at http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.
use lmdb;
use lmdb::{
Cursor,
Database,
Iter as LmdbIter,
RoCursor,
RwTransaction,
Transaction,
};
use lmdb::WriteFlags;
use crate::error::StoreError;
use crate::value::Value;
fn read_transform(val: Result<&[u8], lmdb::Error>) -> Result<Option<Value>, StoreError> {
match val {
Ok(bytes) => Value::from_tagged_slice(bytes).map(Some).map_err(StoreError::DataError),
Err(lmdb::Error::NotFound) => Ok(None),
Err(e) => Err(StoreError::LmdbError(e)),
}
}
#[derive(Copy, Clone)]
pub struct SingleStore {
db: Database,
}
pub struct Iter<'env> {
iter: LmdbIter<'env>,
cursor: RoCursor<'env>,
}
impl SingleStore {
pub(crate) fn new(db: Database) -> SingleStore {
SingleStore {
db,
}
}
pub fn get<'env, T: Transaction, K: AsRef<[u8]>>(
&self,
txn: &'env T,
k: K,
) -> Result<Option<Value<'env>>, StoreError> {
let bytes = txn.get(self.db, &k);
read_transform(bytes)
}
// TODO: flags
pub fn put<K: AsRef<[u8]>>(&mut self, txn: &mut RwTransaction, k: K, v: &Value) -> Result<(), StoreError> {
// TODO: don't allocate twice.
let bytes = v.to_bytes()?;
txn.put(self.db, &k, &bytes, WriteFlags::empty()).map_err(StoreError::LmdbError)
}
pub fn delete<K: AsRef<[u8]>>(&mut self, txn: &mut RwTransaction, k: K) -> Result<(), StoreError> {
txn.del(self.db, &k, None).map_err(StoreError::LmdbError)
}
pub fn iter_start<'env, T: Transaction>(&self, txn: &'env T) -> Result<Iter<'env>, StoreError> {
let mut cursor = txn.open_ro_cursor(self.db).map_err(StoreError::LmdbError)?;
// We call Cursor.iter() instead of Cursor.iter_start() because
// the latter panics at "called `Result::unwrap()` on an `Err` value:
// NotFound" when there are no items in the store, whereas the former
// returns an iterator that yields no items.
//
// And since we create the Cursor and don't change its position, we can
// be sure that a call to Cursor.iter() will start at the beginning.
//
let iter = cursor.iter();
Ok(Iter {
iter,
cursor,
})
}
pub fn iter_from<'env, T: Transaction, K: AsRef<[u8]>>(
&self,
txn: &'env T,
k: K,
) -> Result<Iter<'env>, StoreError> {
let mut cursor = txn.open_ro_cursor(self.db).map_err(StoreError::LmdbError)?;
let iter = cursor.iter_from(k);
Ok(Iter {
iter,
cursor,
})
}
}
impl<'env> Iterator for Iter<'env> {
type Item = (&'env [u8], Result<Option<Value<'env>>, StoreError>);
fn next(&mut self) -> Option<Self::Item> {
match self.iter.next() {
None => None,
Some(Ok((key, bytes))) => Some((key, read_transform(Ok(bytes)))),
Some(Err(_)) => None,
}
}
}

38
tests/integer-store.rs
Unescape View File

@ -11,6 +11,8 @@
use rkv::{
PrimitiveInt,
Rkv,
StoreOptions,
Transaction,
Value,
};
use serde_derive::Serialize;
@ -22,25 +24,25 @@ fn test_integer_keys() {
let root = Builder::new().prefix("test_integer_keys").tempdir().expect("tempdir");
fs::create_dir_all(root.path()).expect("dir created");
let k = Rkv::new(root.path()).expect("new succeeded");
let s = k.open_or_create_integer("s").expect("open");
let mut s = k.open_integer("s", StoreOptions::create()).expect("open");
macro_rules! test_integer_keys {
($type:ty, $key:expr) => {{
let mut writer = k.write_int::<$type>().expect("writer");
($store:expr, $key:expr) => {{
let mut writer = k.write().expect("writer");
writer.put(s, $key, &Value::Str("hello!")).expect("write");
assert_eq!(writer.get(s, $key).expect("read"), Some(Value::Str("hello!")));
$store.put(&mut writer, $key, &Value::Str("hello!")).expect("write");
assert_eq!($store.get(&writer, $key).expect("read"), Some(Value::Str("hello!")));
writer.commit().expect("committed");
let reader = k.read_int::<$type>().expect("reader");
assert_eq!(reader.get(s, $key).expect("read"), Some(Value::Str("hello!")));
let reader = k.read().expect("reader");
assert_eq!($store.get(&reader, $key).expect("read"), Some(Value::Str("hello!")));
}};
}
// The integer module provides only the u32 integer key variant
// of IntegerStore, so we can use it without further ado.
test_integer_keys!(u32, std::u32::MIN);
test_integer_keys!(u32, std::u32::MAX);
test_integer_keys!(s, std::u32::MIN);
test_integer_keys!(s, std::u32::MAX);
// If you want to use another integer key variant, you need to implement
// a newtype, implement PrimitiveInt, and implement or derive Serialize
@ -50,21 +52,27 @@ fn test_integer_keys() {
// different integer key types, which may result in unexpected behavior.
// Make sure you know what you're doing!
let mut t = k.open_integer("s", StoreOptions::create()).expect("open");
#[derive(Serialize)]
struct I32(i32);
impl PrimitiveInt for I32 {}
test_integer_keys!(I32, I32(std::i32::MIN));
test_integer_keys!(I32, I32(std::i32::MAX));
test_integer_keys!(t, I32(std::i32::MIN));
test_integer_keys!(t, I32(std::i32::MAX));
let mut u = k.open_integer("s", StoreOptions::create()).expect("open");
#[derive(Serialize)]
struct U16(u16);
impl PrimitiveInt for U16 {}
test_integer_keys!(U16, U16(std::u16::MIN));
test_integer_keys!(U16, U16(std::u16::MAX));
test_integer_keys!(u, U16(std::u16::MIN));
test_integer_keys!(u, U16(std::u16::MAX));
let mut v = k.open_integer("s", StoreOptions::create()).expect("open");
#[derive(Serialize)]
struct U64(u64);
impl PrimitiveInt for U64 {}
test_integer_keys!(U64, U64(std::u64::MIN));
test_integer_keys!(U64, U64(std::u64::MAX));
test_integer_keys!(v, U64(std::u64::MIN));
test_integer_keys!(v, U64(std::u64::MAX));
}

90
tests/multi-integer-store.rs
Unescape View File

@ -0,0 +1,90 @@
// Copyright 2018 Mozilla
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
// this file except in compliance with the License. You may obtain a copy of the
// License at http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.
use rkv::{
PrimitiveInt,
Rkv,
StoreOptions,
Transaction,
Value,
};
use serde_derive::Serialize;
use std::fs;
use tempfile::Builder;
#[test]
fn test_multi_integer_keys() {
let root = Builder::new().prefix("test_integer_keys").tempdir().expect("tempdir");
fs::create_dir_all(root.path()).expect("dir created");
let k = Rkv::new(root.path()).expect("new succeeded");
let mut s = k.open_multi_integer("s", StoreOptions::create()).expect("open");
macro_rules! test_integer_keys {
($store:expr, $key:expr) => {{
let mut writer = k.write().expect("writer");
$store.put(&mut writer, $key, &Value::Str("hello1")).expect("write");
$store.put(&mut writer, $key, &Value::Str("hello2")).expect("write");
$store.put(&mut writer, $key, &Value::Str("hello3")).expect("write");
let vals = $store
.get(&writer, $key)
.expect("read")
.map(|(_, v)| v.expect("multi read"))
.collect::<Option<Vec<Value>>>();
assert_eq!(vals, Some(vec![Value::Str("hello1"), Value::Str("hello2"), Value::Str("hello3")]));
writer.commit().expect("committed");
let reader = k.read().expect("reader");
let vals = $store
.get(&reader, $key)
.expect("read")
.map(|(_, v)| v.expect("multi read"))
.collect::<Option<Vec<Value>>>();
assert_eq!(vals, Some(vec![Value::Str("hello1"), Value::Str("hello2"), Value::Str("hello3")]));
}};
}
// The integer module provides only the u32 integer key variant
// of IntegerStore, so we can use it without further ado.
test_integer_keys!(s, std::u32::MIN);
test_integer_keys!(s, std::u32::MAX);
// If you want to use another integer key variant, you need to implement
// a newtype, implement PrimitiveInt, and implement or derive Serialize
// for it. Here we do so for the i32 type.
// DANGER! Doing this enables you to open a store with multiple,
// different integer key types, which may result in unexpected behavior.
// Make sure you know what you're doing!
let mut t = k.open_multi_integer("s", StoreOptions::create()).expect("open");
#[derive(Serialize)]
struct I32(i32);
impl PrimitiveInt for I32 {}
test_integer_keys!(t, I32(std::i32::MIN));
test_integer_keys!(t, I32(std::i32::MAX));
let mut u = k.open_multi_integer("s", StoreOptions::create()).expect("open");
#[derive(Serialize)]
struct U16(u16);
impl PrimitiveInt for U16 {}
test_integer_keys!(u, U16(std::u16::MIN));
test_integer_keys!(u, U16(std::u16::MAX));
let mut v = k.open_multi_integer("s", StoreOptions::create()).expect("open");
#[derive(Serialize)]
struct U64(u64);
impl PrimitiveInt for U64 {}
test_integer_keys!(v, U64(std::u64::MIN));
test_integer_keys!(v, U64(std::u64::MAX));
}
Write Preview
Loading…
Cancel
Save