NextGraph
/
nextgraph-rs
1
3
Fork 0
Code Issues 1 Pull Requests Packages Projects 2 Releases 3 Activity

fix bloomfilter, due to hash function issue on wasm32

Browse Source
pull/37/head
Niko PLP 4 months ago
parent b6db6b0d76
commit 263df00e93
10 changed files with 66 additions and 49 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 52
      Cargo.lock
  2. 2
      ng-oxigraph/src/oxigraph/storage/backend/fallback.rs
  3. 2
      ng-oxigraph/src/oxigraph/storage/backend/oxi_rocksdb.rs
  4. 2
      ng-oxigraph/src/oxigraph/storage/mod.rs
  5. 4
      ng-oxigraph/src/oxigraph/store.rs
  6. 2
      ng-repo/Cargo.toml
  7. 14
      ng-repo/src/branch.rs
  8. 23
      ng-repo/src/types.rs
  9. 2
      ng-verifier/Cargo.toml
  10. 12
      ng-verifier/src/verifier.rs

52
Cargo.lock generated
Unescape View File

@ -562,15 +562,6 @@ dependencies = [
"syn 2.0.58",
]
[[package]]
name = "bit-vec"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb"
dependencies = [
"serde",
]
[[package]]
name = "bit_field"
version = "0.10.2"
@ -670,17 +661,6 @@ dependencies = [
"log",
]
[[package]]
name = "bloomfilter"
version = "1.0.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b64d54e47a7f4fd723f082e8f11429f3df6ba8adaeca355a76556f9f0602bbcf"
dependencies = [
"bit-vec",
"getrandom 0.2.10",
"siphasher 1.0.1",
]
[[package]]
name = "brotli"
version = "3.3.4"
@ -1743,6 +1723,12 @@ dependencies = [
"getrandom 0.2.10",
]
[[package]]
name = "fastrange-rs"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e90a1392cd6ec5ebe42ccaf251f2b7ba6be654c377f05c913f3898bfb2172512"
[[package]]
name = "fdeflate"
version = "0.3.0"
@ -3463,7 +3449,6 @@ version = "0.1.0-preview.1"
dependencies = [
"base64-url",
"blake3",
"bloomfilter",
"chacha20",
"crypto_box",
"current_platform",
@ -3478,6 +3463,7 @@ dependencies = [
"once_cell",
"os_info",
"rand 0.7.3",
"sbbf-rs-safe",
"serde",
"serde_bare",
"serde_bytes",
@ -3566,7 +3552,6 @@ dependencies = [
"async-std",
"async-trait",
"automerge",
"bloomfilter",
"either",
"futures",
"getrandom 0.2.10",
@ -3575,6 +3560,7 @@ dependencies = [
"ng-repo",
"ng-storage-rocksdb",
"rand 0.7.3",
"sbbf-rs-safe",
"serde",
"serde_bare",
"serde_bytes",
@ -4904,6 +4890,25 @@ dependencies = [
"winapi-util",
]
[[package]]
name = "sbbf-rs"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5525db49c7719816ac719ea8ffd0d0b4586db1a3f5d3e7751593230dacc642fd"
dependencies = [
"cpufeatures",
"fastrange-rs",
]
[[package]]
name = "sbbf-rs-safe"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9902ffeb2cff3f8c072c60c7d526ac9560fc9a66fe1dfc3c240eba5e2151ba3c"
dependencies = [
"sbbf-rs",
]
[[package]]
name = "schannel"
version = "0.1.22"
@ -5242,9 +5247,6 @@ name = "siphasher"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d"
dependencies = [
"serde",
]
[[package]]
name = "sized-chunks"

2
ng-oxigraph/src/oxigraph/storage/backend/fallback.rs
Unescape View File

@ -81,7 +81,7 @@ impl Db {
pub fn ng_transaction<'a, 'b: 'a, T, E: Error + 'static + From<StorageError>>(
&'b self,
f: impl Fn(Transaction<'a>) -> Result<T, E>,
mut f: impl FnMut(Transaction<'a>) -> Result<T, E>,
) -> Result<T, E> {
let mut t = Transaction::new(Rc::new(RefCell::new(self.db.write().unwrap())));
let res = f(t.clone());

2
ng-oxigraph/src/oxigraph/storage/backend/oxi_rocksdb.rs
Unescape View File

@ -714,7 +714,7 @@ impl Db {
pub fn ng_transaction<'a, 'b: 'a, T, E: Error + 'static + From<StorageError>>(
&'b self,
f: impl Fn(Transaction<'a>) -> Result<T, E>,
mut f: impl FnMut(Transaction<'a>) -> Result<T, E>,
) -> Result<T, E> {
if let DbKind::ReadWrite(db) = &self.inner {
loop {

2
ng-oxigraph/src/oxigraph/storage/mod.rs
Unescape View File

@ -378,7 +378,7 @@ impl Storage {
pub fn ng_transaction<'a, 'b: 'a, T, E: Error + 'static + From<StorageError>>(
&'b self,
f: impl Fn(StorageWriter<'a>) -> Result<T, E>,
mut f: impl FnMut(StorageWriter<'a>) -> Result<T, E>,
) -> Result<T, E> {
self.db.ng_transaction(|transaction| {
f(StorageWriter {

4
ng-oxigraph/src/oxigraph/store.rs
Unescape View File

@ -439,7 +439,7 @@ impl Store {
/// ```
fn transaction<'a, 'b: 'a, T, E: Error + 'static + From<StorageError>>(
&'b self,
f: impl Fn(Transaction<'a>) -> Result<T, E>,
mut f: impl FnMut(Transaction<'a>) -> Result<T, E>,
) -> Result<T, E> {
self.storage
.ng_transaction(|writer| f(Transaction { writer }))
@ -524,7 +524,7 @@ impl Store {
#[doc(hidden)]
pub fn ng_transaction<'a, 'b: 'a, T, E: Error + 'static + From<StorageError>>(
&'b self,
f: impl Fn(Transaction<'a>) -> Result<T, E>,
mut f: impl FnMut(Transaction<'a>) -> Result<T, E>,
) -> Result<T, E> {
self.storage
.ng_transaction(|writer| f(Transaction { writer }))

2
ng-repo/Cargo.toml
Unescape View File

@ -34,7 +34,7 @@ rand = { version = "0.7", features = ["getrandom"] }
blake3 = "1.3.1"
chacha20 = "0.9.0"
ed25519-dalek = "1.0.1"
bloomfilter = { version = "1.0.13", features = ["random","serde"] }
sbbf-rs-safe = "0.3.2"
curve25519-dalek = "3.2.0"
threshold_crypto = "0.4.0"
crypto_box = { version = "0.8.2", features = ["seal"] }

14
ng-repo/src/branch.rs
Unescape View File

@ -13,7 +13,7 @@ use std::collections::HashMap;
use std::collections::HashSet;
use std::fmt;
use bloomfilter::Bloom;
use sbbf_rs_safe::Filter;
use zeroize::Zeroize;
use crate::errors::*;
@ -187,7 +187,7 @@ impl Branch {
missing: &mut Option<&mut HashSet<ObjectId>>,
future: Option<ObjectId>,
theirs_found: &mut Option<&mut HashSet<ObjectId>>,
theirs_filter: &Option<Bloom<ObjectId>>,
theirs_filter: &Option<Filter>,
) -> Result<(), ObjectParseError> {
let id = cobj.id();
@ -195,7 +195,8 @@ impl Branch {
// load deps, stop at the root(including it in visited) or if this is a commit object from known_heads
let found_in_filter = if let Some(filter) = theirs_filter {
filter.check(&id)
let hash = id.get_hash();
filter.contains_hash(hash)
} else {
false
};
@ -276,7 +277,7 @@ impl Branch {
// we silently discard any load error on the known_heads as the responder might not know them (yet).
}
//log_debug!("their causal past \n{}", Dag(&theirs));
// log_debug!("their causal past \n{}", Dag(&theirs));
let mut visited = HashMap::new();
@ -284,8 +285,7 @@ impl Branch {
let filter = if let Some(filter) = known_commits.as_ref() {
Some(
serde_bare::from_slice(filter.filter())
.map_err(|_| ObjectParseError::FilterDeserializationError)?,
filter.filter(), //.map_err(|_| ObjectParseError::FilterDeserializationError)?,
)
} else {
None
@ -309,7 +309,7 @@ impl Branch {
// we silently discard any load error on the target_heads as they can be wrong if the requester is confused about what the responder has locally.
}
//log_debug!("what we have here \n{}", Dag(&visited));
// log_debug!("what we have here \n{}", Dag(&visited));
// now ordering to respect causal partial order.
let mut next_generations = HashSet::new();

23
ng-repo/src/types.rs
Unescape View File

@ -13,9 +13,11 @@
use core::fmt;
use std::cmp::Ordering;
use std::hash::Hash;
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
use once_cell::sync::OnceCell;
use sbbf_rs_safe::Filter;
use serde::{Deserialize, Serialize};
use threshold_crypto::serde_impl::SerdeSecret;
use zeroize::{Zeroize, ZeroizeOnDrop};
@ -70,6 +72,16 @@ impl Digest {
Self::Blake3Digest32(o) => o,
}
}
/// returns a hash that is consistent across platforms (32/64 bits. important for WASM32 compatibility with the rest)
/// see https://www.reddit.com/r/rust/comments/fwpki6/a_debugging_mystery_hashing_slices_in_wasm_works/
pub fn get_hash(&self) -> u64 {
let mut hasher = DefaultHasher::new();
let ser = serde_bare::to_vec(&self).unwrap();
for e in ser {
e.hash(&mut hasher);
}
hasher.finish()
}
}
impl fmt::Display for Digest {
@ -373,11 +385,16 @@ pub enum BloomFilter {
}
impl BloomFilter {
pub fn filter(&self) -> &Vec<u8> {
pub fn filter(&self) -> Filter {
match self {
Self::V0(v0) => &v0.f,
Self::V0(v0) => Filter::from_bytes(&v0.f).unwrap(),
}
}
pub fn from_filter(filter: &Filter) -> Self {
BloomFilter::V0(BloomFilterV0 {
f: filter.as_bytes().to_vec(),
})
}
}
//

2
ng-verifier/Cargo.toml
Unescape View File

@ -31,7 +31,7 @@ async-trait = "0.1.64"
async-std = { version = "1.12.0", features = [ "attributes", "unstable" ] }
automerge = "0.5.9"
yrs = "0.18.2"
bloomfilter = { version = "1.0.13", features = ["random","serde"] }
sbbf-rs-safe = "0.3.2"
ng-repo = { path = "../ng-repo", version = "0.1.0-preview.1" }
ng-net = { path = "../ng-net", version = "0.1.0-preview.1" }
ng-oxigraph = { path = "../ng-oxigraph", version = "0.4.0-alpha.7-ngpreview6" }

12
ng-verifier/src/verifier.rs
Unescape View File

@ -23,9 +23,9 @@ use std::{collections::HashMap, sync::Arc};
use async_std::stream::StreamExt;
use async_std::sync::{Mutex, RwLockReadGuard};
use bloomfilter::Bloom;
use futures::channel::mpsc;
use futures::SinkExt;
use sbbf_rs_safe::Filter;
use serde::{Deserialize, Serialize};
use web_time::SystemTime;
@ -1585,6 +1585,7 @@ impl Verifier {
let mut theirs_found = HashSet::new();
let mut visited = HashMap::new();
for our in ours_set.iter() {
//log_info!("OUR HEADS {}", our);
if let Ok(cobj) = Object::load(*our, None, &repo.store) {
let _ = Branch::load_causal_past(
&cobj,
@ -1611,14 +1612,11 @@ impl Verifier {
// prepare bloom filter
let expected_elements =
remote_commits_nbr + max(visited.len() as u64, branch_info.commits_nbr);
let mut filter =
Bloom::<ObjectId>::new_for_fp_rate(expected_elements as usize, 0.01);
let mut filter = Filter::new(27, expected_elements as usize);
for commit_id in visited.keys() {
filter.set(commit_id);
filter.insert_hash(commit_id.get_hash());
}
Some(BloomFilter::V0(BloomFilterV0 {
f: serde_bare::to_vec(&filter)?,
}))
Some(BloomFilter::from_filter(&filter))
}
};

Write Preview
Loading…
Cancel
Save