parent
92eabc251e
commit
54722e1f76
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@ -0,0 +1 @@ |
|||||||
|
data |
||||||
Binary file not shown.
@ -0,0 +1,28 @@ |
|||||||
|
<!-- |
||||||
|
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers |
||||||
|
// All rights reserved. |
||||||
|
// Licensed under the Apache License, Version 2.0 |
||||||
|
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0> |
||||||
|
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>, |
||||||
|
// at your option. All files in the project carrying such |
||||||
|
// notice may not be copied, modified, or distributed except |
||||||
|
// according to those terms. |
||||||
|
--> |
||||||
|
|
||||||
|
<script type="ts"> |
||||||
|
import { Button } from "flowbite-svelte"; |
||||||
|
import { link } from "svelte-spa-router"; |
||||||
|
import Install from "../../../../ng-app/src/lib/Install.svelte"; |
||||||
|
import { push } from "svelte-spa-router"; |
||||||
|
import { onMount, onDestroy } from "svelte"; |
||||||
|
|
||||||
|
let display_has_wallets_warning = false; |
||||||
|
let unsubscribe; |
||||||
|
onMount(() => {}); |
||||||
|
|
||||||
|
onDestroy(() => { |
||||||
|
unsubscribe(); |
||||||
|
}); |
||||||
|
</script> |
||||||
|
|
||||||
|
<Install {display_has_wallets_warning} /> |
||||||
@ -0,0 +1,20 @@ |
|||||||
|
[package] |
||||||
|
name = "stores-rocksdb" |
||||||
|
version = "0.1.0" |
||||||
|
edition = "2021" |
||||||
|
license = "MIT/Apache-2.0" |
||||||
|
authors = ["Niko PLP <niko@nextgraph.org>"] |
||||||
|
description = "P2P stores based on LMDB for NextGraph" |
||||||
|
repository = "https://git.nextgraph.org/NextGraph/nextgraph-rs" |
||||||
|
|
||||||
|
[dependencies] |
||||||
|
p2p-repo = { path = "../p2p-repo" } |
||||||
|
serde = { version = "1.0.142", features = ["derive"] } |
||||||
|
serde_bare = "0.5.0" |
||||||
|
tempfile = "3" |
||||||
|
hex = "0.4.3" |
||||||
|
|
||||||
|
[target.'cfg(not(target_arch = "wasm32"))'.dependencies.rocksdb] |
||||||
|
git = "https://git.nextgraph.org/NextGraph/rust-rocksdb.git" |
||||||
|
branch = "master" |
||||||
|
features = [ ] |
||||||
@ -0,0 +1,424 @@ |
|||||||
|
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers
|
||||||
|
// All rights reserved.
|
||||||
|
// Licensed under the Apache License, Version 2.0
|
||||||
|
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
|
||||||
|
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
|
||||||
|
// at your option. All files in the project carrying such
|
||||||
|
// notice may not be copied, modified, or distributed except
|
||||||
|
// according to those terms.
|
||||||
|
|
||||||
|
use p2p_repo::kcv_store::*; |
||||||
|
use p2p_repo::store::*; |
||||||
|
use p2p_repo::types::*; |
||||||
|
use p2p_repo::utils::*; |
||||||
|
|
||||||
|
use p2p_repo::log::*; |
||||||
|
|
||||||
|
use std::path::Path; |
||||||
|
use std::path::PathBuf; |
||||||
|
use std::sync::RwLockReadGuard; |
||||||
|
use std::sync::{Arc, RwLock}; |
||||||
|
|
||||||
|
use serde::{Deserialize, Serialize}; |
||||||
|
use serde_bare::error::Error; |
||||||
|
|
||||||
|
use rocksdb::{ |
||||||
|
ColumnFamilyDescriptor, Direction, Env, ErrorKind, IteratorMode, Options, SingleThreaded, |
||||||
|
TransactionDB, TransactionDBOptions, DB, |
||||||
|
}; |
||||||
|
|
||||||
|
pub struct RocksdbTransaction<'a> { |
||||||
|
store: &'a RocksdbKCVStore, |
||||||
|
tx: Option<rocksdb::Transaction<'a, TransactionDB>>, |
||||||
|
} |
||||||
|
|
||||||
|
impl<'a> RocksdbTransaction<'a> { |
||||||
|
fn commit(&mut self) { |
||||||
|
self.tx.take().unwrap().commit().unwrap(); |
||||||
|
} |
||||||
|
fn tx(&self) -> &rocksdb::Transaction<'a, TransactionDB> { |
||||||
|
self.tx.as_ref().unwrap() |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
impl<'a> ReadTransaction for RocksdbTransaction<'a> { |
||||||
|
fn get_all_keys_and_values( |
||||||
|
&self, |
||||||
|
prefix: u8, |
||||||
|
key_size: usize, |
||||||
|
key_prefix: Vec<u8>, |
||||||
|
suffix: Option<u8>, |
||||||
|
) -> Result<Vec<(Vec<u8>, Vec<u8>)>, StorageError> { |
||||||
|
self.store |
||||||
|
.get_all_keys_and_values(prefix, key_size, key_prefix, suffix) |
||||||
|
} |
||||||
|
|
||||||
|
/// Load a single value property from the store.
|
||||||
|
fn get(&self, prefix: u8, key: &Vec<u8>, suffix: Option<u8>) -> Result<Vec<u8>, StorageError> { |
||||||
|
let property = RocksdbKCVStore::compute_property(prefix, key, suffix); |
||||||
|
let mut res = self |
||||||
|
.tx() |
||||||
|
.get_for_update(property, true) |
||||||
|
.map_err(|e| StorageError::BackendError)?; |
||||||
|
match res { |
||||||
|
Some(val) => Ok(val), |
||||||
|
None => Err(StorageError::NotFound), |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/// Load all the values of a property from the store.
|
||||||
|
fn get_all( |
||||||
|
&self, |
||||||
|
prefix: u8, |
||||||
|
key: &Vec<u8>, |
||||||
|
suffix: Option<u8>, |
||||||
|
) -> Result<Vec<Vec<u8>>, StorageError> { |
||||||
|
unimplemented!(); |
||||||
|
} |
||||||
|
|
||||||
|
/// Check if a specific value exists for a property from the store.
|
||||||
|
fn has_property_value( |
||||||
|
&self, |
||||||
|
prefix: u8, |
||||||
|
key: &Vec<u8>, |
||||||
|
suffix: Option<u8>, |
||||||
|
value: &Vec<u8>, |
||||||
|
) -> Result<(), StorageError> { |
||||||
|
let property = RocksdbKCVStore::compute_property(prefix, key, suffix); |
||||||
|
let exists = self |
||||||
|
.tx() |
||||||
|
.get_for_update(property, true) |
||||||
|
.map_err(|e| StorageError::BackendError)?; |
||||||
|
match exists { |
||||||
|
Some(stored_value) => { |
||||||
|
if stored_value.eq(value) { |
||||||
|
Ok(()) |
||||||
|
} else { |
||||||
|
Err(StorageError::DifferentValue) |
||||||
|
} |
||||||
|
} |
||||||
|
None => Err(StorageError::NotFound), |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
impl<'a> WriteTransaction for RocksdbTransaction<'a> { |
||||||
|
/// Save a property value to the store.
|
||||||
|
fn put( |
||||||
|
&mut self, |
||||||
|
prefix: u8, |
||||||
|
key: &Vec<u8>, |
||||||
|
suffix: Option<u8>, |
||||||
|
value: &Vec<u8>, |
||||||
|
) -> Result<(), StorageError> { |
||||||
|
let property = RocksdbKCVStore::compute_property(prefix, key, suffix); |
||||||
|
self.tx() |
||||||
|
.put(property, value) |
||||||
|
.map_err(|e| StorageError::BackendError)?; |
||||||
|
|
||||||
|
Ok(()) |
||||||
|
} |
||||||
|
|
||||||
|
/// Replace the property of a key (single value) to the store.
|
||||||
|
fn replace( |
||||||
|
&mut self, |
||||||
|
prefix: u8, |
||||||
|
key: &Vec<u8>, |
||||||
|
suffix: Option<u8>, |
||||||
|
value: &Vec<u8>, |
||||||
|
) -> Result<(), StorageError> { |
||||||
|
let property = RocksdbKCVStore::compute_property(prefix, key, suffix); |
||||||
|
|
||||||
|
self.tx() |
||||||
|
.put(property, value) |
||||||
|
.map_err(|e| StorageError::BackendError)?; |
||||||
|
|
||||||
|
Ok(()) |
||||||
|
} |
||||||
|
|
||||||
|
/// Delete a property from the store.
|
||||||
|
fn del(&mut self, prefix: u8, key: &Vec<u8>, suffix: Option<u8>) -> Result<(), StorageError> { |
||||||
|
let property = RocksdbKCVStore::compute_property(prefix, key, suffix); |
||||||
|
let res = self.tx().delete(property); |
||||||
|
if res.is_err() { |
||||||
|
if let ErrorKind::NotFound = res.unwrap_err().kind() { |
||||||
|
return Ok(()); |
||||||
|
} |
||||||
|
return Err(StorageError::BackendError); |
||||||
|
} |
||||||
|
Ok(()) |
||||||
|
} |
||||||
|
|
||||||
|
/// Delete a specific value for a property from the store.
|
||||||
|
fn del_property_value( |
||||||
|
&mut self, |
||||||
|
prefix: u8, |
||||||
|
key: &Vec<u8>, |
||||||
|
suffix: Option<u8>, |
||||||
|
value: &Vec<u8>, |
||||||
|
) -> Result<(), StorageError> { |
||||||
|
let property = RocksdbKCVStore::compute_property(prefix, key, suffix); |
||||||
|
let exists = self |
||||||
|
.tx() |
||||||
|
.get_for_update(property.clone(), true) |
||||||
|
.map_err(|e| StorageError::BackendError)?; |
||||||
|
match exists { |
||||||
|
Some(val) => { |
||||||
|
if val.eq(value) { |
||||||
|
self.tx() |
||||||
|
.delete(property) |
||||||
|
.map_err(|e| StorageError::BackendError)?; |
||||||
|
} |
||||||
|
} |
||||||
|
None => return Err(StorageError::DifferentValue), |
||||||
|
} |
||||||
|
Ok(()) |
||||||
|
} |
||||||
|
|
||||||
|
/// Delete all properties of a key from the store.
|
||||||
|
fn del_all( |
||||||
|
&mut self, |
||||||
|
prefix: u8, |
||||||
|
key: &Vec<u8>, |
||||||
|
all_suffixes: &[u8], |
||||||
|
) -> Result<(), StorageError> { |
||||||
|
for suffix in all_suffixes { |
||||||
|
self.del(prefix, key, Some(*suffix))?; |
||||||
|
} |
||||||
|
if all_suffixes.is_empty() { |
||||||
|
self.del(prefix, key, None)?; |
||||||
|
} |
||||||
|
Ok(()) |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
pub struct RocksdbKCVStore { |
||||||
|
/// the main store where all the properties of keys are stored
|
||||||
|
main_db: TransactionDB, |
||||||
|
/// path for the storage backend data
|
||||||
|
path: String, |
||||||
|
} |
||||||
|
|
||||||
|
fn compare<T: Ord>(a: &[T], b: &[T]) -> std::cmp::Ordering { |
||||||
|
let mut iter_b = b.iter(); |
||||||
|
for v in a { |
||||||
|
match iter_b.next() { |
||||||
|
Some(w) => match v.cmp(w) { |
||||||
|
std::cmp::Ordering::Equal => continue, |
||||||
|
ord => return ord, |
||||||
|
}, |
||||||
|
None => break, |
||||||
|
} |
||||||
|
} |
||||||
|
return a.len().cmp(&b.len()); |
||||||
|
} |
||||||
|
|
||||||
|
impl ReadTransaction for RocksdbKCVStore { |
||||||
|
fn get_all_keys_and_values( |
||||||
|
&self, |
||||||
|
prefix: u8, |
||||||
|
key_size: usize, |
||||||
|
key_prefix: Vec<u8>, |
||||||
|
suffix: Option<u8>, |
||||||
|
) -> Result<Vec<(Vec<u8>, Vec<u8>)>, StorageError> { |
||||||
|
if key_prefix.len() > key_size { |
||||||
|
return Err(StorageError::InvalidValue); |
||||||
|
} |
||||||
|
|
||||||
|
let mut vec_key_start = key_prefix.clone(); |
||||||
|
let mut trailing_zeros = vec![0u8; key_size - key_prefix.len()]; |
||||||
|
vec_key_start.append(&mut trailing_zeros); |
||||||
|
|
||||||
|
let mut vec_key_end = key_prefix.clone(); |
||||||
|
let mut trailing_max = vec![255u8; key_size - key_prefix.len()]; |
||||||
|
vec_key_end.append(&mut trailing_max); |
||||||
|
|
||||||
|
let property_start = Self::compute_property(prefix, &vec_key_start, suffix); |
||||||
|
let property_end = |
||||||
|
Self::compute_property(prefix, &vec_key_end, Some(suffix.unwrap_or(255u8))); |
||||||
|
|
||||||
|
let mut iter = self |
||||||
|
.main_db |
||||||
|
.iterator(IteratorMode::From(&property_start, Direction::Forward)); |
||||||
|
let mut vector: Vec<(Vec<u8>, Vec<u8>)> = vec![]; |
||||||
|
while let res = iter.next() { |
||||||
|
match res { |
||||||
|
Some(Ok(val)) => { |
||||||
|
match compare(&val.0, property_end.as_slice()) { |
||||||
|
std::cmp::Ordering::Less | std::cmp::Ordering::Equal => { |
||||||
|
if suffix.is_some() { |
||||||
|
if val.0.len() < (key_size + 2) |
||||||
|
|| val.0[1 + key_size] != suffix.unwrap() |
||||||
|
{ |
||||||
|
continue; |
||||||
|
} |
||||||
|
// } else if val.0.len() > (key_size + 1) {
|
||||||
|
// continue;
|
||||||
|
} |
||||||
|
vector.push((val.0.to_vec(), val.1.to_vec())); |
||||||
|
} |
||||||
|
_ => {} //,
|
||||||
|
} |
||||||
|
} |
||||||
|
Some(Err(_e)) => return Err(StorageError::BackendError), |
||||||
|
None => { |
||||||
|
break; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
Ok(vector) |
||||||
|
} |
||||||
|
|
||||||
|
/// Load a single value property from the store.
|
||||||
|
fn get(&self, prefix: u8, key: &Vec<u8>, suffix: Option<u8>) -> Result<Vec<u8>, StorageError> { |
||||||
|
let property = Self::compute_property(prefix, key, suffix); |
||||||
|
let mut res = self |
||||||
|
.main_db |
||||||
|
.get(property) |
||||||
|
.map_err(|e| StorageError::BackendError)?; |
||||||
|
match res { |
||||||
|
Some(val) => Ok(val), |
||||||
|
None => Err(StorageError::NotFound), |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/// Load all the values of a property from the store.
|
||||||
|
fn get_all( |
||||||
|
&self, |
||||||
|
prefix: u8, |
||||||
|
key: &Vec<u8>, |
||||||
|
suffix: Option<u8>, |
||||||
|
) -> Result<Vec<Vec<u8>>, StorageError> { |
||||||
|
unimplemented!(); |
||||||
|
} |
||||||
|
|
||||||
|
/// Check if a specific value exists for a property from the store.
|
||||||
|
fn has_property_value( |
||||||
|
&self, |
||||||
|
prefix: u8, |
||||||
|
key: &Vec<u8>, |
||||||
|
suffix: Option<u8>, |
||||||
|
value: &Vec<u8>, |
||||||
|
) -> Result<(), StorageError> { |
||||||
|
let property = Self::compute_property(prefix, key, suffix); |
||||||
|
let exists = self |
||||||
|
.main_db |
||||||
|
.get(property) |
||||||
|
.map_err(|e| StorageError::BackendError)?; |
||||||
|
match exists { |
||||||
|
Some(stored_value) => { |
||||||
|
if stored_value.eq(value) { |
||||||
|
Ok(()) |
||||||
|
} else { |
||||||
|
Err(StorageError::DifferentValue) |
||||||
|
} |
||||||
|
} |
||||||
|
None => Err(StorageError::NotFound), |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
impl KCVStore for RocksdbKCVStore { |
||||||
|
fn write_transaction( |
||||||
|
&self, |
||||||
|
method: &mut dyn FnMut(&mut dyn WriteTransaction) -> Result<(), StorageError>, |
||||||
|
) -> Result<(), StorageError> { |
||||||
|
let tx = self.main_db.transaction(); |
||||||
|
|
||||||
|
let mut transaction = RocksdbTransaction { |
||||||
|
store: self, |
||||||
|
tx: Some(tx), |
||||||
|
}; |
||||||
|
let res = method(&mut transaction); |
||||||
|
if res.is_ok() { |
||||||
|
transaction.commit(); |
||||||
|
//lock.sync(true);
|
||||||
|
} |
||||||
|
res |
||||||
|
} |
||||||
|
|
||||||
|
/// Save a property value to the store.
|
||||||
|
fn put( |
||||||
|
&self, |
||||||
|
prefix: u8, |
||||||
|
key: &Vec<u8>, |
||||||
|
suffix: Option<u8>, |
||||||
|
value: Vec<u8>, |
||||||
|
) -> Result<(), StorageError> { |
||||||
|
self.write_transaction(&mut |tx| tx.put(prefix, key, suffix, &value)) |
||||||
|
} |
||||||
|
|
||||||
|
/// Replace the property of a key (single value) to the store.
|
||||||
|
fn replace( |
||||||
|
&self, |
||||||
|
prefix: u8, |
||||||
|
key: &Vec<u8>, |
||||||
|
suffix: Option<u8>, |
||||||
|
value: Vec<u8>, |
||||||
|
) -> Result<(), StorageError> { |
||||||
|
self.write_transaction(&mut |tx| tx.replace(prefix, key, suffix, &value)) |
||||||
|
} |
||||||
|
|
||||||
|
/// Delete a property from the store.
|
||||||
|
fn del(&self, prefix: u8, key: &Vec<u8>, suffix: Option<u8>) -> Result<(), StorageError> { |
||||||
|
self.write_transaction(&mut |tx| tx.del(prefix, key, suffix)) |
||||||
|
} |
||||||
|
|
||||||
|
/// Delete a specific value for a property from the store.
|
||||||
|
fn del_property_value( |
||||||
|
&self, |
||||||
|
prefix: u8, |
||||||
|
key: &Vec<u8>, |
||||||
|
suffix: Option<u8>, |
||||||
|
value: Vec<u8>, |
||||||
|
) -> Result<(), StorageError> { |
||||||
|
self.write_transaction(&mut |tx| tx.del_property_value(prefix, key, suffix, &value)) |
||||||
|
} |
||||||
|
|
||||||
|
/// Delete all properties of a key from the store.
|
||||||
|
fn del_all(&self, prefix: u8, key: &Vec<u8>, all_suffixes: &[u8]) -> Result<(), StorageError> { |
||||||
|
for suffix in all_suffixes { |
||||||
|
self.del(prefix, key, Some(*suffix))?; |
||||||
|
} |
||||||
|
if all_suffixes.is_empty() { |
||||||
|
self.del(prefix, key, None)?; |
||||||
|
} |
||||||
|
Ok(()) |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
impl RocksdbKCVStore { |
||||||
|
pub fn path(&self) -> PathBuf { |
||||||
|
PathBuf::from(&self.path) |
||||||
|
} |
||||||
|
|
||||||
|
fn compute_property(prefix: u8, key: &Vec<u8>, suffix: Option<u8>) -> Vec<u8> { |
||||||
|
let mut new: Vec<u8> = Vec::with_capacity(key.len() + 2); |
||||||
|
new.push(prefix); |
||||||
|
new.extend(key); |
||||||
|
if suffix.is_some() { |
||||||
|
new.push(suffix.unwrap()) |
||||||
|
} |
||||||
|
new |
||||||
|
} |
||||||
|
|
||||||
|
/// Opens the store and returns a KCVStore object that should be kept and used to manipulate the properties
|
||||||
|
/// The key is the encryption key for the data at rest.
|
||||||
|
pub fn open<'a>(path: &Path, key: [u8; 32]) -> Result<RocksdbKCVStore, StorageError> { |
||||||
|
let mut opts = Options::default(); |
||||||
|
opts.create_if_missing(true); |
||||||
|
opts.create_missing_column_families(true); |
||||||
|
let env = Env::enc_env(key).unwrap(); |
||||||
|
opts.set_env(&env); |
||||||
|
let tx_options = TransactionDBOptions::new(); |
||||||
|
let db: TransactionDB = |
||||||
|
TransactionDB::open_cf(&opts, &tx_options, &path, vec!["cf0", "cf1"]).unwrap(); |
||||||
|
|
||||||
|
log_info!("created db with Rocksdb Version: {}", Env::version()); |
||||||
|
|
||||||
|
Ok(RocksdbKCVStore { |
||||||
|
main_db: db, |
||||||
|
path: path.to_str().unwrap().to_string(), |
||||||
|
}) |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,5 @@ |
|||||||
|
// #[cfg(not(target_arch = "wasm32"))]
|
||||||
|
// pub mod repo_store;
|
||||||
|
|
||||||
|
#[cfg(not(target_arch = "wasm32"))] |
||||||
|
pub mod kcv_store; |
||||||
@ -0,0 +1,992 @@ |
|||||||
|
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers
|
||||||
|
// All rights reserved.
|
||||||
|
// Licensed under the Apache License, Version 2.0
|
||||||
|
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
|
||||||
|
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
|
||||||
|
// at your option. All files in the project carrying such
|
||||||
|
// notice may not be copied, modified, or distributed except
|
||||||
|
// according to those terms.
|
||||||
|
|
||||||
|
use p2p_repo::store::*; |
||||||
|
use p2p_repo::types::*; |
||||||
|
use p2p_repo::utils::*; |
||||||
|
|
||||||
|
use p2p_repo::log::*; |
||||||
|
use std::path::Path; |
||||||
|
use std::sync::{Arc, RwLock}; |
||||||
|
|
||||||
|
use serde::{Deserialize, Serialize}; |
||||||
|
use serde_bare::error::Error; |
||||||
|
|
||||||
|
#[derive(Debug)] |
||||||
|
pub struct LmdbRepoStore { |
||||||
|
/// the main store where all the repo blocks are stored
|
||||||
|
main_store: SingleStore<LmdbDatabase>, |
||||||
|
/// store for the pin boolean, recently_used timestamp, and synced boolean
|
||||||
|
meta_store: SingleStore<LmdbDatabase>, |
||||||
|
/// store for the expiry timestamp
|
||||||
|
expiry_store: MultiIntegerStore<LmdbDatabase, u32>, |
||||||
|
/// store for the LRU list
|
||||||
|
recently_used_store: MultiIntegerStore<LmdbDatabase, u32>, |
||||||
|
/// the opened environment so we can create new transactions
|
||||||
|
environment: Arc<RwLock<Rkv<LmdbEnvironment>>>, |
||||||
|
} |
||||||
|
|
||||||
|
// TODO: versioning V0
|
||||||
|
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] |
||||||
|
struct BlockMeta { |
||||||
|
pub pin: bool, |
||||||
|
pub last_used: Timestamp, |
||||||
|
pub synced: bool, |
||||||
|
} |
||||||
|
|
||||||
|
impl RepoStore for LmdbRepoStore { |
||||||
|
/// Retrieves a block from the storage backend.
|
||||||
|
fn get(&self, block_id: &BlockId) -> Result<Block, StorageError> { |
||||||
|
let lock = self.environment.read().unwrap(); |
||||||
|
let reader = lock.read().unwrap(); |
||||||
|
let block_id_ser = serde_bare::to_vec(&block_id).unwrap(); |
||||||
|
let block_ser_res = self.main_store.get(&reader, block_id_ser.clone()); |
||||||
|
match block_ser_res { |
||||||
|
Err(e) => Err(StorageError::BackendError), |
||||||
|
Ok(None) => Err(StorageError::NotFound), |
||||||
|
Ok(Some(block_ser)) => { |
||||||
|
// updating recently_used
|
||||||
|
// first getting the meta for this BlockId
|
||||||
|
let meta_ser = self.meta_store.get(&reader, block_id_ser.clone()).unwrap(); |
||||||
|
match meta_ser { |
||||||
|
Some(meta_value) => { |
||||||
|
let mut meta = |
||||||
|
serde_bare::from_slice::<BlockMeta>(&meta_value.to_bytes().unwrap()) |
||||||
|
.unwrap(); |
||||||
|
if meta.synced { |
||||||
|
let mut writer = lock.write().unwrap(); |
||||||
|
let now = now_timestamp(); |
||||||
|
if !meta.pin { |
||||||
|
// we remove the previous timestamp (last_used) from recently_used_store
|
||||||
|
self.remove_from_lru(&mut writer, &block_id_ser, &meta.last_used) |
||||||
|
.unwrap(); |
||||||
|
// we add an entry to recently_used_store with now
|
||||||
|
self.add_to_lru(&mut writer, &block_id_ser, &now).unwrap(); |
||||||
|
} |
||||||
|
// we save the new meta (with last_used:now)
|
||||||
|
meta.last_used = now; |
||||||
|
let new_meta_ser = serde_bare::to_vec(&meta).unwrap(); |
||||||
|
self.meta_store |
||||||
|
.put( |
||||||
|
&mut writer, |
||||||
|
block_id_ser, |
||||||
|
&Value::Blob(new_meta_ser.as_slice()), |
||||||
|
) |
||||||
|
.unwrap(); |
||||||
|
// commit
|
||||||
|
writer.commit().unwrap(); |
||||||
|
} |
||||||
|
} |
||||||
|
_ => {} // there is no meta. we do nothing since we start to record LRU only once synced == true.
|
||||||
|
} |
||||||
|
|
||||||
|
match serde_bare::from_slice::<Block>(&block_ser.to_bytes().unwrap()) { |
||||||
|
Err(_e) => Err(StorageError::InvalidValue), |
||||||
|
Ok(o) => { |
||||||
|
if o.id() != *block_id { |
||||||
|
log_debug!( |
||||||
|
"Invalid ObjectId.\nExp: {:?}\nGot: {:?}\nContent: {:?}", |
||||||
|
block_id, |
||||||
|
o.id(), |
||||||
|
o |
||||||
|
); |
||||||
|
panic!("CORRUPTION OF DATA !"); |
||||||
|
} |
||||||
|
Ok(o) |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/// Adds a block in the storage backend.
|
||||||
|
/// The block is persisted to disk.
|
||||||
|
/// Returns the BlockId of the Block.
|
||||||
|
fn put(&self, block: &Block) -> Result<BlockId, StorageError> { |
||||||
|
let block_ser = serde_bare::to_vec(&block).unwrap(); |
||||||
|
|
||||||
|
let block_id = block.id(); |
||||||
|
let block_id_ser = serde_bare::to_vec(&block_id).unwrap(); |
||||||
|
|
||||||
|
let lock = self.environment.read().unwrap(); |
||||||
|
let mut writer = lock.write().unwrap(); |
||||||
|
|
||||||
|
// TODO: check if the block is already in store? if yes, don't put it again.
|
||||||
|
// I didnt do it yet because it is extra cost. surely a get on the store is lighter than a put
|
||||||
|
// but doing a get in additing to a put for every call, is probably even costlier. better to deal with that at the higher level
|
||||||
|
|
||||||
|
self.main_store |
||||||
|
.put( |
||||||
|
&mut writer, |
||||||
|
&block_id_ser, |
||||||
|
&Value::Blob(block_ser.as_slice()), |
||||||
|
) |
||||||
|
.unwrap(); |
||||||
|
|
||||||
|
// if it has an expiry, adding the BlockId to the expiry_store
|
||||||
|
match block.expiry() { |
||||||
|
Some(expiry) => { |
||||||
|
self.expiry_store |
||||||
|
.put(&mut writer, expiry, &Value::Blob(block_id_ser.as_slice())) |
||||||
|
.unwrap(); |
||||||
|
} |
||||||
|
_ => {} |
||||||
|
} |
||||||
|
writer.commit().unwrap(); |
||||||
|
|
||||||
|
Ok(block_id) |
||||||
|
} |
||||||
|
|
||||||
|
/// Removes the block from the storage backend.
|
||||||
|
/// The removed block is returned, so it can be inspected.
|
||||||
|
/// Also returned is the approximate size of of free space that was reclaimed.
|
||||||
|
fn del(&self, block_id: &BlockId) -> Result<(Block, usize), StorageError> { |
||||||
|
let lock = self.environment.read().unwrap(); |
||||||
|
let mut writer = lock.write().unwrap(); |
||||||
|
let block_id_ser = serde_bare::to_vec(&block_id).unwrap(); |
||||||
|
// retrieving the block itself (we need the expiry)
|
||||||
|
let block_ser = self |
||||||
|
.main_store |
||||||
|
.get(&writer, block_id_ser.clone()) |
||||||
|
.unwrap() |
||||||
|
.ok_or(StorageError::NotFound)?; |
||||||
|
let slice = block_ser.to_bytes().unwrap(); |
||||||
|
let block = serde_bare::from_slice::<Block>(&slice).unwrap(); //FIXME propagate error?
|
||||||
|
let meta_res = self.meta_store.get(&writer, block_id_ser.clone()).unwrap(); |
||||||
|
if meta_res.is_some() { |
||||||
|
let meta = serde_bare::from_slice::<BlockMeta>(&meta_res.unwrap().to_bytes().unwrap()) |
||||||
|
.unwrap(); |
||||||
|
if meta.last_used != 0 { |
||||||
|
self.remove_from_lru(&mut writer, &block_id_ser.clone(), &meta.last_used) |
||||||
|
.unwrap(); |
||||||
|
} |
||||||
|
// removing the meta
|
||||||
|
self.meta_store |
||||||
|
.delete(&mut writer, block_id_ser.clone()) |
||||||
|
.unwrap(); |
||||||
|
} |
||||||
|
// delete block from main_store
|
||||||
|
self.main_store |
||||||
|
.delete(&mut writer, block_id_ser.clone()) |
||||||
|
.unwrap(); |
||||||
|
// remove BlockId from expiry_store, if any expiry
|
||||||
|
match block.expiry() { |
||||||
|
Some(expiry) => { |
||||||
|
self.expiry_store |
||||||
|
.delete( |
||||||
|
&mut writer, |
||||||
|
expiry, |
||||||
|
&Value::Blob(block_id_ser.clone().as_slice()), |
||||||
|
) |
||||||
|
.unwrap(); |
||||||
|
} |
||||||
|
_ => {} |
||||||
|
} |
||||||
|
|
||||||
|
writer.commit().unwrap(); |
||||||
|
Ok((block, slice.len())) |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
impl LmdbRepoStore { |
||||||
|
/// Opens the store and returns a RepoStore object that should be kept and used to call put/get/delete/pin
|
||||||
|
/// The key is the encryption key for the data at rest.
|
||||||
|
pub fn open<'a>(path: &Path, key: [u8; 32]) -> Result<LmdbRepoStore, StorageError> { |
||||||
|
let mut manager = Manager::<LmdbEnvironment>::singleton().write().unwrap(); |
||||||
|
let shared_rkv = manager |
||||||
|
.get_or_create(path, |path| { |
||||||
|
//Rkv::new::<Lmdb>(path) // use this instead to disable encryption
|
||||||
|
Rkv::with_encryption_key_and_mapsize::<Lmdb>(path, key, 1 * 1024 * 1024 * 1024) |
||||||
|
}) |
||||||
|
.map_err(|e| { |
||||||
|
log_debug!("open LMDB failed: {}", e); |
||||||
|
StorageError::BackendError |
||||||
|
})?; |
||||||
|
let env = shared_rkv.read().unwrap(); |
||||||
|
|
||||||
|
log_debug!( |
||||||
|
"created env with LMDB Version: {} key: {}", |
||||||
|
env.version(), |
||||||
|
hex::encode(&key) |
||||||
|
); |
||||||
|
|
||||||
|
let main_store = env.open_single("main", StoreOptions::create()).unwrap(); |
||||||
|
let meta_store = env.open_single("meta", StoreOptions::create()).unwrap(); |
||||||
|
let mut opts = StoreOptions::<LmdbDatabaseFlags>::create(); |
||||||
|
opts.flags.set(DatabaseFlags::DUP_FIXED, true); |
||||||
|
let expiry_store = env.open_multi_integer("expiry", opts).unwrap(); |
||||||
|
let recently_used_store = env.open_multi_integer("recently_used", opts).unwrap(); |
||||||
|
|
||||||
|
Ok(LmdbRepoStore { |
||||||
|
environment: shared_rkv.clone(), |
||||||
|
main_store, |
||||||
|
meta_store, |
||||||
|
expiry_store, |
||||||
|
recently_used_store, |
||||||
|
}) |
||||||
|
} |
||||||
|
|
||||||
|
//FIXME: use BlockId, not ObjectId. this is a block level operation
|
||||||
|
/// Pins the object
|
||||||
|
pub fn pin(&self, object_id: &ObjectId) -> Result<(), StorageError> { |
||||||
|
self.set_pin(object_id, true) |
||||||
|
} |
||||||
|
|
||||||
|
//FIXME: use BlockId, not ObjectId. this is a block level operation
|
||||||
|
/// Unpins the object
|
||||||
|
pub fn unpin(&self, object_id: &ObjectId) -> Result<(), StorageError> { |
||||||
|
self.set_pin(object_id, false) |
||||||
|
} |
||||||
|
|
||||||
|
//FIXME: use BlockId, not ObjectId. this is a block level operation
|
||||||
|
/// Sets the pin for that Object. if add is true, will add the pin. if false, will remove the pin.
|
||||||
|
/// A pin on an object prevents it from being removed when the store is making some disk space by using the LRU.
|
||||||
|
/// A pin does not override the expiry. If expiry is set and is reached, the obejct will be deleted, no matter what.
|
||||||
|
pub fn set_pin(&self, object_id: &ObjectId, add: bool) -> Result<(), StorageError> { |
||||||
|
let lock = self.environment.read().unwrap(); |
||||||
|
let mut writer = lock.write().unwrap(); |
||||||
|
let obj_id_ser = serde_bare::to_vec(&object_id).unwrap(); |
||||||
|
let meta_ser = self.meta_store.get(&writer, &obj_id_ser).unwrap(); |
||||||
|
let mut meta; |
||||||
|
|
||||||
|
// if adding a pin, if there is a meta (if already pinned, return) and is synced, remove the last_used timestamp from recently_used_store
|
||||||
|
// if no meta, create it with pin:true, synced: false
|
||||||
|
// if removing a pin (if pin already removed, return), if synced, add an entry to recently_used_store with the last_used timestamp (as found in meta, dont use now)
|
||||||
|
|
||||||
|
match meta_ser { |
||||||
|
Some(meta_value) => { |
||||||
|
meta = |
||||||
|
serde_bare::from_slice::<BlockMeta>(&meta_value.to_bytes().unwrap()).unwrap(); |
||||||
|
|
||||||
|
if add == meta.pin { |
||||||
|
// pinning while already pinned, or unpinning while already unpinned. NOP
|
||||||
|
return Ok(()); |
||||||
|
}; |
||||||
|
|
||||||
|
meta.pin = add; |
||||||
|
|
||||||
|
if meta.synced { |
||||||
|
if add { |
||||||
|
// we remove the previous timestamp (last_used) from recently_used_store
|
||||||
|
self.remove_from_lru(&mut writer, &obj_id_ser, &meta.last_used) |
||||||
|
.unwrap(); |
||||||
|
} else { |
||||||
|
// we add an entry to recently_used_store with last_used
|
||||||
|
self.add_to_lru(&mut writer, &obj_id_ser, &meta.last_used) |
||||||
|
.unwrap(); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
None => { |
||||||
|
if add { |
||||||
|
meta = BlockMeta { |
||||||
|
pin: true, |
||||||
|
synced: false, |
||||||
|
last_used: 0, |
||||||
|
} |
||||||
|
} else { |
||||||
|
// there is no meta, and user wants to unpin, so let's leave everything as it is.
|
||||||
|
return Ok(()); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
let new_meta_ser = serde_bare::to_vec(&meta).unwrap(); |
||||||
|
self.meta_store |
||||||
|
.put( |
||||||
|
&mut writer, |
||||||
|
obj_id_ser, |
||||||
|
&Value::Blob(new_meta_ser.as_slice()), |
||||||
|
) |
||||||
|
.unwrap(); |
||||||
|
// commit
|
||||||
|
writer.commit().unwrap(); |
||||||
|
|
||||||
|
Ok(()) |
||||||
|
} |
||||||
|
|
||||||
|
//FIXME: use BlockId, not ObjectId. this is a block level operation
|
||||||
|
/// the broker calls this method when the block has been retrieved/synced by enough peers and it
|
||||||
|
/// can now be included in the LRU for potential garbage collection.
|
||||||
|
/// If this method has not been called on a block, it will be kept in the store and will not enter LRU.
|
||||||
|
pub fn has_been_synced(&self, block_id: &BlockId, when: Option<u32>) -> Result<(), Error> { |
||||||
|
let lock = self.environment.read().unwrap(); |
||||||
|
let mut writer = lock.write().unwrap(); |
||||||
|
let block_id_ser = serde_bare::to_vec(&block_id).unwrap(); |
||||||
|
let meta_ser = self.meta_store.get(&writer, block_id_ser.clone()).unwrap(); |
||||||
|
let mut meta; |
||||||
|
let now = match when { |
||||||
|
None => now_timestamp(), |
||||||
|
Some(w) => w, |
||||||
|
}; |
||||||
|
// get the meta. if no meta, it is ok, we will create it after (with pin:false and synced:true)
|
||||||
|
// if already synced, return
|
||||||
|
// update the meta with last_used:now and synced:true
|
||||||
|
// if pinned, save and return
|
||||||
|
// otherwise add an entry to recently_used_store with now
|
||||||
|
|
||||||
|
match meta_ser { |
||||||
|
Some(meta_value) => { |
||||||
|
meta = |
||||||
|
serde_bare::from_slice::<BlockMeta>(&meta_value.to_bytes().unwrap()).unwrap(); |
||||||
|
|
||||||
|
if meta.synced { |
||||||
|
// already synced. NOP
|
||||||
|
return Ok(()); |
||||||
|
}; |
||||||
|
|
||||||
|
meta.synced = true; |
||||||
|
meta.last_used = now; |
||||||
|
|
||||||
|
if !meta.pin { |
||||||
|
// we add an entry to recently_used_store with now
|
||||||
|
log_debug!("adding to LRU"); |
||||||
|
self.add_to_lru(&mut writer, &block_id_ser, &now).unwrap(); |
||||||
|
} |
||||||
|
} |
||||||
|
None => { |
||||||
|
meta = BlockMeta { |
||||||
|
pin: false, |
||||||
|
synced: true, |
||||||
|
last_used: now, |
||||||
|
}; |
||||||
|
log_debug!("adding to LRU also"); |
||||||
|
self.add_to_lru(&mut writer, &block_id_ser, &now).unwrap(); |
||||||
|
} |
||||||
|
} |
||||||
|
let new_meta_ser = serde_bare::to_vec(&meta).unwrap(); |
||||||
|
self.meta_store |
||||||
|
.put( |
||||||
|
&mut writer, |
||||||
|
block_id_ser, |
||||||
|
&Value::Blob(new_meta_ser.as_slice()), |
||||||
|
) |
||||||
|
.unwrap(); |
||||||
|
// commit
|
||||||
|
writer.commit().unwrap(); |
||||||
|
|
||||||
|
Ok(()) |
||||||
|
} |
||||||
|
|
||||||
|
/// Removes all the blocks that have expired.
|
||||||
|
/// The broker should call this method periodically.
|
||||||
|
pub fn remove_expired(&self) -> Result<(), Error> { |
||||||
|
let mut block_ids: Vec<BlockId> = vec![]; |
||||||
|
|
||||||
|
{ |
||||||
|
let lock = self.environment.read().unwrap(); |
||||||
|
let reader = lock.read().unwrap(); |
||||||
|
|
||||||
|
let mut iter = self |
||||||
|
.expiry_store |
||||||
|
.iter_prev_dup_from(&reader, now_timestamp()) |
||||||
|
.unwrap(); |
||||||
|
|
||||||
|
while let Some(Ok(mut sub_iter)) = iter.next() { |
||||||
|
while let Some(Ok(k)) = sub_iter.next() { |
||||||
|
//log_debug!("removing {:?} {:?}", k.0, k.1);
|
||||||
|
let block_id = serde_bare::from_slice::<ObjectId>(k.1).unwrap(); |
||||||
|
block_ids.push(block_id); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
for block_id in block_ids { |
||||||
|
self.del(&block_id).unwrap(); |
||||||
|
} |
||||||
|
Ok(()) |
||||||
|
} |
||||||
|
|
||||||
|
/// Removes some blocks that haven't been used for a while, reclaiming some space on disk.
|
||||||
|
/// The oldest are removed first, until the total amount of data removed is at least equal to size,
|
||||||
|
/// or the LRU list became empty. The approximate size of the storage space that was reclaimed is returned.
|
||||||
|
pub fn remove_least_used(&self, size: usize) -> usize { |
||||||
|
let mut block_ids: Vec<BlockId> = vec![]; |
||||||
|
let mut total: usize = 0; |
||||||
|
|
||||||
|
{ |
||||||
|
let lock = self.environment.read().unwrap(); |
||||||
|
let reader = lock.read().unwrap(); |
||||||
|
|
||||||
|
let mut iter = self.recently_used_store.iter_start(&reader).unwrap(); |
||||||
|
|
||||||
|
while let Some(Ok(entry)) = iter.next() { |
||||||
|
let block_id = |
||||||
|
serde_bare::from_slice::<ObjectId>(entry.1.to_bytes().unwrap().as_slice()) |
||||||
|
.unwrap(); |
||||||
|
block_ids.push(block_id); |
||||||
|
} |
||||||
|
} |
||||||
|
for block_id in block_ids { |
||||||
|
let (block, block_size) = self.del(&block_id).unwrap(); |
||||||
|
log_debug!("removed {:?}", block_id); |
||||||
|
total += block_size; |
||||||
|
if total >= size { |
||||||
|
break; |
||||||
|
} |
||||||
|
} |
||||||
|
total |
||||||
|
} |
||||||
|
|
||||||
|
fn remove_from_lru( |
||||||
|
&self, |
||||||
|
writer: &mut Writer<LmdbRwTransaction>, |
||||||
|
block_id_ser: &Vec<u8>, |
||||||
|
time: &Timestamp, |
||||||
|
) -> Result<(), StoreError> { |
||||||
|
self.recently_used_store |
||||||
|
.delete(writer, *time, &Value::Blob(block_id_ser.as_slice())) |
||||||
|
} |
||||||
|
|
||||||
|
fn add_to_lru( |
||||||
|
&self, |
||||||
|
writer: &mut Writer<LmdbRwTransaction>, |
||||||
|
block_id_ser: &Vec<u8>, |
||||||
|
time: &Timestamp, |
||||||
|
) -> Result<(), StoreError> { |
||||||
|
let mut flag = LmdbWriteFlags::empty(); |
||||||
|
flag.set(WriteFlags::APPEND_DUP, true); |
||||||
|
self.recently_used_store.put_with_flags( |
||||||
|
writer, |
||||||
|
*time, |
||||||
|
&Value::Blob(block_id_ser.as_slice()), |
||||||
|
flag, |
||||||
|
) |
||||||
|
} |
||||||
|
|
||||||
|
fn list_all(&self) { |
||||||
|
let lock = self.environment.read().unwrap(); |
||||||
|
let reader = lock.read().unwrap(); |
||||||
|
log_debug!("MAIN"); |
||||||
|
let mut iter = self.main_store.iter_start(&reader).unwrap(); |
||||||
|
while let Some(Ok(entry)) = iter.next() { |
||||||
|
log_debug!("{:?} {:?}", entry.0, entry.1) |
||||||
|
} |
||||||
|
log_debug!("META"); |
||||||
|
let mut iter2 = self.meta_store.iter_start(&reader).unwrap(); |
||||||
|
while let Some(Ok(entry)) = iter2.next() { |
||||||
|
log_debug!("{:?} {:?}", entry.0, entry.1) |
||||||
|
} |
||||||
|
log_debug!("EXPIRY"); |
||||||
|
let mut iter3 = self.expiry_store.iter_start(&reader).unwrap(); |
||||||
|
while let Some(Ok(entry)) = iter3.next() { |
||||||
|
log_debug!("{:?} {:?}", entry.0, entry.1) |
||||||
|
} |
||||||
|
log_debug!("LRU"); |
||||||
|
let mut iter4 = self.recently_used_store.iter_start(&reader).unwrap(); |
||||||
|
while let Some(Ok(entry)) = iter4.next() { |
||||||
|
log_debug!("{:?} {:?}", entry.0, entry.1) |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
#[cfg(test)] |
||||||
|
mod test { |
||||||
|
|
||||||
|
use crate::repo_store::LmdbRepoStore; |
||||||
|
use p2p_repo::log::*; |
||||||
|
use p2p_repo::store::*; |
||||||
|
use p2p_repo::types::*; |
||||||
|
use p2p_repo::utils::*; |
||||||
|
use rkv::backend::{BackendInfo, BackendStat, Lmdb, LmdbEnvironment}; |
||||||
|
use rkv::{Manager, Rkv, StoreOptions, Value}; |
||||||
|
#[allow(unused_imports)] |
||||||
|
use std::time::Duration; |
||||||
|
#[allow(unused_imports)] |
||||||
|
use std::{fs, thread}; |
||||||
|
use tempfile::Builder; |
||||||
|
|
||||||
|
#[test] |
||||||
|
pub fn test_remove_least_used() { |
||||||
|
let path_str = "test-env"; |
||||||
|
let root = Builder::new().prefix(path_str).tempdir().unwrap(); |
||||||
|
let key: [u8; 32] = [0; 32]; |
||||||
|
fs::create_dir_all(root.path()).unwrap(); |
||||||
|
log_debug!("{}", root.path().to_str().unwrap()); |
||||||
|
let mut store = LmdbRepoStore::open(root.path(), key).unwrap(); |
||||||
|
let mut now = now_timestamp(); |
||||||
|
now -= 200; |
||||||
|
// TODO: fix the LMDB bug that is triggered with x max set to 86 !!!
|
||||||
|
for x in 1..85 { |
||||||
|
let block = Block::new( |
||||||
|
Vec::new(), |
||||||
|
ObjectDeps::ObjectIdList(Vec::new()), |
||||||
|
None, |
||||||
|
vec![x; 10], |
||||||
|
None, |
||||||
|
); |
||||||
|
let block_id = store.put(&block).unwrap(); |
||||||
|
log_debug!("#{} -> objId {:?}", x, block_id); |
||||||
|
store |
||||||
|
.has_been_synced(&block_id, Some(now + x as u32)) |
||||||
|
.unwrap(); |
||||||
|
} |
||||||
|
|
||||||
|
let ret = store.remove_least_used(200); |
||||||
|
log_debug!("removed {}", ret); |
||||||
|
assert_eq!(ret, 208) |
||||||
|
|
||||||
|
//store.list_all();
|
||||||
|
} |
||||||
|
|
||||||
|
#[test] |
||||||
|
pub fn test_set_pin() { |
||||||
|
let path_str = "test-env"; |
||||||
|
let root = Builder::new().prefix(path_str).tempdir().unwrap(); |
||||||
|
let key: [u8; 32] = [0; 32]; |
||||||
|
fs::create_dir_all(root.path()).unwrap(); |
||||||
|
log_debug!("{}", root.path().to_str().unwrap()); |
||||||
|
let mut store = LmdbRepoStore::open(root.path(), key).unwrap(); |
||||||
|
let mut now = now_timestamp(); |
||||||
|
now -= 200; |
||||||
|
// TODO: fix the LMDB bug that is triggered with x max set to 86 !!!
|
||||||
|
for x in 1..100 { |
||||||
|
let block = Block::new( |
||||||
|
Vec::new(), |
||||||
|
ObjectDeps::ObjectIdList(Vec::new()), |
||||||
|
None, |
||||||
|
vec![x; 10], |
||||||
|
None, |
||||||
|
); |
||||||
|
let obj_id = store.put(&block).unwrap(); |
||||||
|
log_debug!("#{} -> objId {:?}", x, obj_id); |
||||||
|
store.set_pin(&obj_id, true).unwrap(); |
||||||
|
store |
||||||
|
.has_been_synced(&obj_id, Some(now + x as u32)) |
||||||
|
.unwrap(); |
||||||
|
} |
||||||
|
|
||||||
|
let ret = store.remove_least_used(200); |
||||||
|
log_debug!("removed {}", ret); |
||||||
|
assert_eq!(ret, 0); |
||||||
|
|
||||||
|
store.list_all(); |
||||||
|
} |
||||||
|
|
||||||
|
#[test] |
||||||
|
pub fn test_get_valid_value_size() { |
||||||
|
assert_eq!(store_valid_value_size(0), 4072); |
||||||
|
assert_eq!(store_valid_value_size(2), 4072); |
||||||
|
assert_eq!(store_valid_value_size(4072), 4072); |
||||||
|
assert_eq!(store_valid_value_size(4072 + 1), 4072 + 4096); |
||||||
|
assert_eq!(store_valid_value_size(4072 + 4096), 4072 + 4096); |
||||||
|
assert_eq!(store_valid_value_size(4072 + 4096 + 1), 4072 + 4096 + 4096); |
||||||
|
assert_eq!( |
||||||
|
store_valid_value_size(4072 + 4096 + 4096), |
||||||
|
4072 + 4096 + 4096 |
||||||
|
); |
||||||
|
assert_eq!( |
||||||
|
store_valid_value_size(4072 + 4096 + 4096 + 1), |
||||||
|
4072 + 4096 + 4096 + 4096 |
||||||
|
); |
||||||
|
assert_eq!(store_valid_value_size(4072 + 4096 * 511), 4072 + 4096 * 511); |
||||||
|
assert_eq!( |
||||||
|
store_valid_value_size(4072 + 4096 * 511 + 1), |
||||||
|
4072 + 4096 * 511 |
||||||
|
); |
||||||
|
} |
||||||
|
|
||||||
|
#[test] |
||||||
|
pub fn test_remove_expired() { |
||||||
|
let path_str = "test-env"; |
||||||
|
let root = Builder::new().prefix(path_str).tempdir().unwrap(); |
||||||
|
let key: [u8; 32] = [0; 32]; |
||||||
|
fs::create_dir_all(root.path()).unwrap(); |
||||||
|
log_debug!("{}", root.path().to_str().unwrap()); |
||||||
|
let mut store = LmdbRepoStore::open(root.path(), key).unwrap(); |
||||||
|
|
||||||
|
let now = now_timestamp(); |
||||||
|
let list = [ |
||||||
|
now - 10, |
||||||
|
now - 6, |
||||||
|
now - 6, |
||||||
|
now - 3, |
||||||
|
now - 2, |
||||||
|
now - 1, //#5 should be removed, and above
|
||||||
|
now + 3, |
||||||
|
now + 4, |
||||||
|
now + 4, |
||||||
|
now + 5, |
||||||
|
now + 10, |
||||||
|
]; |
||||||
|
let mut block_ids: Vec<ObjectId> = Vec::with_capacity(11); |
||||||
|
log_debug!("now {}", now); |
||||||
|
|
||||||
|
let mut i = 0u8; |
||||||
|
for expiry in list { |
||||||
|
//let i: u8 = (expiry + 10 - now).try_into().unwrap();
|
||||||
|
let block = Block::new( |
||||||
|
Vec::new(), |
||||||
|
ObjectDeps::ObjectIdList(Vec::new()), |
||||||
|
Some(expiry), |
||||||
|
[i].to_vec(), |
||||||
|
None, |
||||||
|
); |
||||||
|
let block_id = store.put(&block).unwrap(); |
||||||
|
log_debug!("#{} -> objId {:?}", i, block_id); |
||||||
|
block_ids.push(block_id); |
||||||
|
i += 1; |
||||||
|
} |
||||||
|
|
||||||
|
store.remove_expired().unwrap(); |
||||||
|
|
||||||
|
assert!(store.get(block_ids.get(0).unwrap()).is_err()); |
||||||
|
assert!(store.get(block_ids.get(1).unwrap()).is_err()); |
||||||
|
assert!(store.get(block_ids.get(2).unwrap()).is_err()); |
||||||
|
assert!(store.get(block_ids.get(5).unwrap()).is_err()); |
||||||
|
assert!(store.get(block_ids.get(6).unwrap()).is_ok()); |
||||||
|
assert!(store.get(block_ids.get(7).unwrap()).is_ok()); |
||||||
|
|
||||||
|
//store.list_all();
|
||||||
|
} |
||||||
|
|
||||||
|
#[test] |
||||||
|
pub fn test_remove_all_expired() { |
||||||
|
let path_str = "test-env"; |
||||||
|
let root = Builder::new().prefix(path_str).tempdir().unwrap(); |
||||||
|
let key: [u8; 32] = [0; 32]; |
||||||
|
fs::create_dir_all(root.path()).unwrap(); |
||||||
|
log_debug!("{}", root.path().to_str().unwrap()); |
||||||
|
let mut store = LmdbRepoStore::open(root.path(), key).unwrap(); |
||||||
|
|
||||||
|
let now = now_timestamp(); |
||||||
|
let list = [ |
||||||
|
now - 10, |
||||||
|
now - 6, |
||||||
|
now - 6, |
||||||
|
now - 3, |
||||||
|
now - 2, |
||||||
|
now - 2, //#5 should be removed, and above
|
||||||
|
]; |
||||||
|
let mut block_ids: Vec<ObjectId> = Vec::with_capacity(6); |
||||||
|
log_debug!("now {}", now); |
||||||
|
|
||||||
|
let mut i = 0u8; |
||||||
|
for expiry in list { |
||||||
|
//let i: u8 = (expiry + 10 - now).try_into().unwrap();
|
||||||
|
let block = Block::new( |
||||||
|
Vec::new(), |
||||||
|
ObjectDeps::ObjectIdList(Vec::new()), |
||||||
|
Some(expiry), |
||||||
|
[i].to_vec(), |
||||||
|
None, |
||||||
|
); |
||||||
|
let block_id = store.put(&block).unwrap(); |
||||||
|
log_debug!("#{} -> objId {:?}", i, block_id); |
||||||
|
block_ids.push(block_id); |
||||||
|
i += 1; |
||||||
|
} |
||||||
|
|
||||||
|
store.remove_expired().unwrap(); |
||||||
|
|
||||||
|
assert!(store.get(block_ids.get(0).unwrap()).is_err()); |
||||||
|
assert!(store.get(block_ids.get(1).unwrap()).is_err()); |
||||||
|
assert!(store.get(block_ids.get(2).unwrap()).is_err()); |
||||||
|
assert!(store.get(block_ids.get(3).unwrap()).is_err()); |
||||||
|
assert!(store.get(block_ids.get(4).unwrap()).is_err()); |
||||||
|
assert!(store.get(block_ids.get(5).unwrap()).is_err()); |
||||||
|
} |
||||||
|
|
||||||
|
#[test] |
||||||
|
pub fn test_remove_empty_expired() { |
||||||
|
let path_str = "test-env"; |
||||||
|
let root = Builder::new().prefix(path_str).tempdir().unwrap(); |
||||||
|
let key: [u8; 32] = [0; 32]; |
||||||
|
fs::create_dir_all(root.path()).unwrap(); |
||||||
|
log_debug!("{}", root.path().to_str().unwrap()); |
||||||
|
let store = LmdbRepoStore::open(root.path(), key).unwrap(); |
||||||
|
store.remove_expired().unwrap(); |
||||||
|
} |
||||||
|
|
||||||
|
#[test] |
||||||
|
pub fn test_store_block() { |
||||||
|
let path_str = "test-env"; |
||||||
|
let root = Builder::new().prefix(path_str).tempdir().unwrap(); |
||||||
|
|
||||||
|
let key: [u8; 32] = [0; 32]; |
||||||
|
fs::create_dir_all(root.path()).unwrap(); |
||||||
|
|
||||||
|
log_debug!("{}", root.path().to_str().unwrap()); |
||||||
|
|
||||||
|
let mut store = LmdbRepoStore::open(root.path(), key).unwrap(); |
||||||
|
|
||||||
|
let block = Block::new( |
||||||
|
Vec::new(), |
||||||
|
ObjectDeps::ObjectIdList(Vec::new()), |
||||||
|
None, |
||||||
|
b"abc".to_vec(), |
||||||
|
None, |
||||||
|
); |
||||||
|
|
||||||
|
let block_id = store.put(&block).unwrap(); |
||||||
|
assert_eq!(block_id, block.id()); |
||||||
|
|
||||||
|
log_debug!("ObjectId: {:?}", block_id); |
||||||
|
assert_eq!( |
||||||
|
block_id, |
||||||
|
Digest::Blake3Digest32([ |
||||||
|
155, 83, 186, 17, 95, 10, 80, 31, 111, 24, 250, 64, 8, 145, 71, 193, 103, 246, 202, |
||||||
|
28, 202, 144, 63, 65, 85, 229, 136, 85, 202, 34, 13, 85 |
||||||
|
]) |
||||||
|
); |
||||||
|
|
||||||
|
let block_res = store.get(&block_id).unwrap(); |
||||||
|
|
||||||
|
log_debug!("Block: {:?}", block_res); |
||||||
|
assert_eq!(block_res.id(), block.id()); |
||||||
|
} |
||||||
|
|
||||||
|
#[test] |
||||||
|
pub fn test_lmdb() { |
||||||
|
let path_str = "test-env"; |
||||||
|
let root = Builder::new().prefix(path_str).tempdir().unwrap(); |
||||||
|
|
||||||
|
// we set an encryption key with all zeros... for test purpose only ;)
|
||||||
|
let key: [u8; 32] = [0; 32]; |
||||||
|
{ |
||||||
|
fs::create_dir_all(root.path()).unwrap(); |
||||||
|
|
||||||
|
log_debug!("{}", root.path().to_str().unwrap()); |
||||||
|
|
||||||
|
let mut manager = Manager::<LmdbEnvironment>::singleton().write().unwrap(); |
||||||
|
let shared_rkv = manager |
||||||
|
.get_or_create(root.path(), |path| { |
||||||
|
// Rkv::new::<Lmdb>(path) // use this instead to disable encryption
|
||||||
|
Rkv::with_encryption_key_and_mapsize::<Lmdb>(path, key, 1 * 1024 * 1024 * 1024) |
||||||
|
}) |
||||||
|
.unwrap(); |
||||||
|
let env = shared_rkv.read().unwrap(); |
||||||
|
|
||||||
|
log_debug!("LMDB Version: {}", env.version()); |
||||||
|
|
||||||
|
let store = env.open_single("testdb", StoreOptions::create()).unwrap(); |
||||||
|
|
||||||
|
{ |
||||||
|
// Use a write transaction to mutate the store via a `Writer`. There can be only
|
||||||
|
// one writer for a given environment, so opening a second one will block until
|
||||||
|
// the first completes.
|
||||||
|
let mut writer = env.write().unwrap(); |
||||||
|
|
||||||
|
// Keys are `AsRef<[u8]>`, while values are `Value` enum instances. Use the `Blob`
|
||||||
|
// variant to store arbitrary collections of bytes. Putting data returns a
|
||||||
|
// `Result<(), StoreError>`, where StoreError is an enum identifying the reason
|
||||||
|
// for a failure.
|
||||||
|
// store.put(&mut writer, "int", &Value::I64(1234)).unwrap();
|
||||||
|
// store
|
||||||
|
// .put(&mut writer, "uint", &Value::U64(1234_u64))
|
||||||
|
// .unwrap();
|
||||||
|
// store
|
||||||
|
// .put(&mut writer, "float", &Value::F64(1234.0.into()))
|
||||||
|
// .unwrap();
|
||||||
|
// store
|
||||||
|
// .put(&mut writer, "instant", &Value::Instant(1528318073700))
|
||||||
|
// .unwrap();
|
||||||
|
// store
|
||||||
|
// .put(&mut writer, "boolean", &Value::Bool(true))
|
||||||
|
// .unwrap();
|
||||||
|
// store
|
||||||
|
// .put(&mut writer, "string", &Value::Str("Héllo, wörld!"))
|
||||||
|
// .unwrap();
|
||||||
|
// store
|
||||||
|
// .put(
|
||||||
|
// &mut writer,
|
||||||
|
// "json",
|
||||||
|
// &Value::Json(r#"{"foo":"bar", "number": 1}"#),
|
||||||
|
// )
|
||||||
|
// .unwrap();
|
||||||
|
const EXTRA: usize = 2095; // + 4096 * 524280 + 0;
|
||||||
|
let key: [u8; 33] = [0; 33]; |
||||||
|
let key2: [u8; 33] = [2; 33]; |
||||||
|
let key3: [u8; 33] = [3; 33]; |
||||||
|
let key4: [u8; 33] = [4; 33]; |
||||||
|
//let value: [u8; 1977 + EXTRA] = [1; 1977 + EXTRA];
|
||||||
|
let value = vec![1; 1977 + EXTRA]; |
||||||
|
let value2: [u8; 1977 + 1] = [1; 1977 + 1]; |
||||||
|
let value4: [u8; 953 + 0] = [1; 953 + 0]; |
||||||
|
store.put(&mut writer, key, &Value::Blob(&value2)).unwrap(); |
||||||
|
store.put(&mut writer, key2, &Value::Blob(&value2)).unwrap(); |
||||||
|
// store.put(&mut writer, key3, &Value::Blob(&value)).unwrap();
|
||||||
|
// store.put(&mut writer, key4, &Value::Blob(&value4)).unwrap();
|
||||||
|
|
||||||
|
// You must commit a write transaction before the writer goes out of scope, or the
|
||||||
|
// transaction will abort and the data won't persist.
|
||||||
|
writer.commit().unwrap(); |
||||||
|
let reader = env.read().expect("reader"); |
||||||
|
let stat = store.stat(&reader).unwrap(); |
||||||
|
|
||||||
|
log_debug!("LMDB stat page_size : {}", stat.page_size()); |
||||||
|
log_debug!("LMDB stat depth : {}", stat.depth()); |
||||||
|
log_debug!("LMDB stat branch_pages : {}", stat.branch_pages()); |
||||||
|
log_debug!("LMDB stat leaf_pages : {}", stat.leaf_pages()); |
||||||
|
log_debug!("LMDB stat overflow_pages : {}", stat.overflow_pages()); |
||||||
|
log_debug!("LMDB stat entries : {}", stat.entries()); |
||||||
|
} |
||||||
|
|
||||||
|
// {
|
||||||
|
// // Use a read transaction to query the store via a `Reader`. There can be multiple
|
||||||
|
// // concurrent readers for a store, and readers never block on a writer nor other
|
||||||
|
// // readers.
|
||||||
|
// let reader = env.read().expect("reader");
|
||||||
|
|
||||||
|
// // Keys are `AsRef<u8>`, and the return value is `Result<Option<Value>, StoreError>`.
|
||||||
|
// // log_debug!("Get int {:?}", store.get(&reader, "int").unwrap());
|
||||||
|
// // log_debug!("Get uint {:?}", store.get(&reader, "uint").unwrap());
|
||||||
|
// // log_debug!("Get float {:?}", store.get(&reader, "float").unwrap());
|
||||||
|
// // log_debug!("Get instant {:?}", store.get(&reader, "instant").unwrap());
|
||||||
|
// // log_debug!("Get boolean {:?}", store.get(&reader, "boolean").unwrap());
|
||||||
|
// // log_debug!("Get string {:?}", store.get(&reader, "string").unwrap());
|
||||||
|
// // log_debug!("Get json {:?}", store.get(&reader, "json").unwrap());
|
||||||
|
// log_debug!("Get blob {:?}", store.get(&reader, "blob").unwrap());
|
||||||
|
|
||||||
|
// // Retrieving a non-existent value returns `Ok(None)`.
|
||||||
|
// log_debug!(
|
||||||
|
// "Get non-existent value {:?}",
|
||||||
|
// store.get(&reader, "non-existent").unwrap()
|
||||||
|
// );
|
||||||
|
|
||||||
|
// // A read transaction will automatically close once the reader goes out of scope,
|
||||||
|
// // so isn't necessary to close it explicitly, although you can do so by calling
|
||||||
|
// // `Reader.abort()`.
|
||||||
|
// }
|
||||||
|
|
||||||
|
// {
|
||||||
|
// // Aborting a write transaction rolls back the change(s).
|
||||||
|
// let mut writer = env.write().unwrap();
|
||||||
|
// store.put(&mut writer, "foo", &Value::Blob(b"bar")).unwrap();
|
||||||
|
// writer.abort();
|
||||||
|
// let reader = env.read().expect("reader");
|
||||||
|
// log_debug!(
|
||||||
|
// "It should be None! ({:?})",
|
||||||
|
// store.get(&reader, "foo").unwrap()
|
||||||
|
// );
|
||||||
|
// }
|
||||||
|
|
||||||
|
// {
|
||||||
|
// // Explicitly aborting a transaction is not required unless an early abort is
|
||||||
|
// // desired, since both read and write transactions will implicitly be aborted once
|
||||||
|
// // they go out of scope.
|
||||||
|
// {
|
||||||
|
// let mut writer = env.write().unwrap();
|
||||||
|
// store.put(&mut writer, "foo", &Value::Blob(b"bar")).unwrap();
|
||||||
|
// }
|
||||||
|
// let reader = env.read().expect("reader");
|
||||||
|
// log_debug!(
|
||||||
|
// "It should be None! ({:?})",
|
||||||
|
// store.get(&reader, "foo").unwrap()
|
||||||
|
// );
|
||||||
|
// }
|
||||||
|
|
||||||
|
// {
|
||||||
|
// // Deleting a key/value pair also requires a write transaction.
|
||||||
|
// let mut writer = env.write().unwrap();
|
||||||
|
// store.put(&mut writer, "foo", &Value::Blob(b"bar")).unwrap();
|
||||||
|
// store.put(&mut writer, "bar", &Value::Blob(b"baz")).unwrap();
|
||||||
|
// store.delete(&mut writer, "foo").unwrap();
|
||||||
|
|
||||||
|
// // A write transaction also supports reading, and the version of the store that it
|
||||||
|
// // reads includes the changes it has made regardless of the commit state of that
|
||||||
|
// // transaction.
|
||||||
|
// // In the code above, "foo" and "bar" were put into the store, then "foo" was
|
||||||
|
// // deleted so only "bar" will return a result when the database is queried via the
|
||||||
|
// // writer.
|
||||||
|
// log_debug!(
|
||||||
|
// "It should be None! ({:?})",
|
||||||
|
// store.get(&writer, "foo").unwrap()
|
||||||
|
// );
|
||||||
|
// log_debug!("Get bar ({:?})", store.get(&writer, "bar").unwrap());
|
||||||
|
|
||||||
|
// // But a reader won't see that change until the write transaction is committed.
|
||||||
|
// {
|
||||||
|
// let reader = env.read().expect("reader");
|
||||||
|
// log_debug!("Get foo {:?}", store.get(&reader, "foo").unwrap());
|
||||||
|
// log_debug!("Get bar {:?}", store.get(&reader, "bar").unwrap());
|
||||||
|
// }
|
||||||
|
// writer.commit().unwrap();
|
||||||
|
// {
|
||||||
|
// let reader = env.read().expect("reader");
|
||||||
|
// log_debug!(
|
||||||
|
// "It should be None! ({:?})",
|
||||||
|
// store.get(&reader, "foo").unwrap()
|
||||||
|
// );
|
||||||
|
// log_debug!("Get bar {:?}", store.get(&reader, "bar").unwrap());
|
||||||
|
// }
|
||||||
|
|
||||||
|
// // Committing a transaction consumes the writer, preventing you from reusing it by
|
||||||
|
// // failing at compile time with an error. This line would report "error[E0382]:
|
||||||
|
// // borrow of moved value: `writer`".
|
||||||
|
// // store.put(&mut writer, "baz", &Value::Str("buz")).unwrap();
|
||||||
|
// }
|
||||||
|
|
||||||
|
// {
|
||||||
|
// // Clearing all the entries in the store with a write transaction.
|
||||||
|
// {
|
||||||
|
// let mut writer = env.write().unwrap();
|
||||||
|
// store.put(&mut writer, "foo", &Value::Blob(b"bar")).unwrap();
|
||||||
|
// store.put(&mut writer, "bar", &Value::Blob(b"baz")).unwrap();
|
||||||
|
// writer.commit().unwrap();
|
||||||
|
// }
|
||||||
|
|
||||||
|
// // {
|
||||||
|
// // let mut writer = env.write().unwrap();
|
||||||
|
// // store.clear(&mut writer).unwrap();
|
||||||
|
// // writer.commit().unwrap();
|
||||||
|
// // }
|
||||||
|
|
||||||
|
// // {
|
||||||
|
// // let reader = env.read().expect("reader");
|
||||||
|
// // log_debug!(
|
||||||
|
// // "It should be None! ({:?})",
|
||||||
|
// // store.get(&reader, "foo").unwrap()
|
||||||
|
// // );
|
||||||
|
// // log_debug!(
|
||||||
|
// // "It should be None! ({:?})",
|
||||||
|
// // store.get(&reader, "bar").unwrap()
|
||||||
|
// // );
|
||||||
|
// // }
|
||||||
|
// }
|
||||||
|
|
||||||
|
let stat = env.stat().unwrap(); |
||||||
|
let info = env.info().unwrap(); |
||||||
|
log_debug!("LMDB info map_size : {}", info.map_size()); |
||||||
|
log_debug!("LMDB info last_pgno : {}", info.last_pgno()); |
||||||
|
log_debug!("LMDB info last_txnid : {}", info.last_txnid()); |
||||||
|
log_debug!("LMDB info max_readers : {}", info.max_readers()); |
||||||
|
log_debug!("LMDB info num_readers : {}", info.num_readers()); |
||||||
|
log_debug!("LMDB stat page_size : {}", stat.page_size()); |
||||||
|
log_debug!("LMDB stat depth : {}", stat.depth()); |
||||||
|
log_debug!("LMDB stat branch_pages : {}", stat.branch_pages()); |
||||||
|
log_debug!("LMDB stat leaf_pages : {}", stat.leaf_pages()); |
||||||
|
log_debug!("LMDB stat overflow_pages : {}", stat.overflow_pages()); |
||||||
|
log_debug!("LMDB stat entries : {}", stat.entries()); |
||||||
|
} |
||||||
|
// We reopen the env and data to see if it was well saved to disk.
|
||||||
|
{ |
||||||
|
let mut manager = Manager::<LmdbEnvironment>::singleton().write().unwrap(); |
||||||
|
let shared_rkv = manager |
||||||
|
.get_or_create(root.path(), |path| { |
||||||
|
//Rkv::new::<Lmdb>(path) // use this instead to disable encryption
|
||||||
|
Rkv::with_encryption_key_and_mapsize::<Lmdb>(path, key, 1 * 1024 * 1024 * 1024) |
||||||
|
}) |
||||||
|
.unwrap(); |
||||||
|
let env = shared_rkv.read().unwrap(); |
||||||
|
|
||||||
|
log_debug!("LMDB Version: {}", env.version()); |
||||||
|
|
||||||
|
let mut store = env.open_single("testdb", StoreOptions::default()).unwrap(); //StoreOptions::create()
|
||||||
|
|
||||||
|
{ |
||||||
|
let reader = env.read().expect("reader"); |
||||||
|
log_debug!( |
||||||
|
"It should be baz! ({:?})", |
||||||
|
store.get(&reader, "bar").unwrap() |
||||||
|
); |
||||||
|
} |
||||||
|
} |
||||||
|
// Here the database and environment is closed, but the files are still present in the temp directory.
|
||||||
|
// uncomment this if you need time to copy them somewhere for analysis, before the temp folder get destroyed
|
||||||
|
//thread::sleep(Duration::from_millis(20000));
|
||||||
|
} |
||||||
|
} |
||||||
Loading…
Reference in new issue