diff --git a/Cargo.lock b/Cargo.lock index 493e5bb..3055355 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -562,15 +562,6 @@ dependencies = [ "syn 2.0.58", ] -[[package]] -name = "bit-vec" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" -dependencies = [ - "serde", -] - [[package]] name = "bit_field" version = "0.10.2" @@ -670,17 +661,6 @@ dependencies = [ "log", ] -[[package]] -name = "bloomfilter" -version = "1.0.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b64d54e47a7f4fd723f082e8f11429f3df6ba8adaeca355a76556f9f0602bbcf" -dependencies = [ - "bit-vec", - "getrandom 0.2.10", - "siphasher 1.0.1", -] - [[package]] name = "brotli" version = "3.3.4" @@ -1743,6 +1723,12 @@ dependencies = [ "getrandom 0.2.10", ] +[[package]] +name = "fastrange-rs" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e90a1392cd6ec5ebe42ccaf251f2b7ba6be654c377f05c913f3898bfb2172512" + [[package]] name = "fdeflate" version = "0.3.0" @@ -3463,7 +3449,6 @@ version = "0.1.0-preview.1" dependencies = [ "base64-url", "blake3", - "bloomfilter", "chacha20", "crypto_box", "current_platform", @@ -3478,6 +3463,7 @@ dependencies = [ "once_cell", "os_info", "rand 0.7.3", + "sbbf-rs-safe", "serde", "serde_bare", "serde_bytes", @@ -3566,7 +3552,6 @@ dependencies = [ "async-std", "async-trait", "automerge", - "bloomfilter", "either", "futures", "getrandom 0.2.10", @@ -3575,6 +3560,7 @@ dependencies = [ "ng-repo", "ng-storage-rocksdb", "rand 0.7.3", + "sbbf-rs-safe", "serde", "serde_bare", "serde_bytes", @@ -4904,6 +4890,25 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "sbbf-rs" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5525db49c7719816ac719ea8ffd0d0b4586db1a3f5d3e7751593230dacc642fd" +dependencies = [ + "cpufeatures", + "fastrange-rs", +] + +[[package]] +name = "sbbf-rs-safe" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9902ffeb2cff3f8c072c60c7d526ac9560fc9a66fe1dfc3c240eba5e2151ba3c" +dependencies = [ + "sbbf-rs", +] + [[package]] name = "schannel" version = "0.1.22" @@ -5242,9 +5247,6 @@ name = "siphasher" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" -dependencies = [ - "serde", -] [[package]] name = "sized-chunks" diff --git a/ng-oxigraph/src/oxigraph/storage/backend/fallback.rs b/ng-oxigraph/src/oxigraph/storage/backend/fallback.rs index 9f76951..6c95384 100644 --- a/ng-oxigraph/src/oxigraph/storage/backend/fallback.rs +++ b/ng-oxigraph/src/oxigraph/storage/backend/fallback.rs @@ -81,7 +81,7 @@ impl Db { pub fn ng_transaction<'a, 'b: 'a, T, E: Error + 'static + From>( &'b self, - f: impl Fn(Transaction<'a>) -> Result, + mut f: impl FnMut(Transaction<'a>) -> Result, ) -> Result { let mut t = Transaction::new(Rc::new(RefCell::new(self.db.write().unwrap()))); let res = f(t.clone()); diff --git a/ng-oxigraph/src/oxigraph/storage/backend/oxi_rocksdb.rs b/ng-oxigraph/src/oxigraph/storage/backend/oxi_rocksdb.rs index 0f4c593..0463294 100644 --- a/ng-oxigraph/src/oxigraph/storage/backend/oxi_rocksdb.rs +++ b/ng-oxigraph/src/oxigraph/storage/backend/oxi_rocksdb.rs @@ -714,7 +714,7 @@ impl Db { pub fn ng_transaction<'a, 'b: 'a, T, E: Error + 'static + From>( &'b self, - f: impl Fn(Transaction<'a>) -> Result, + mut f: impl FnMut(Transaction<'a>) -> Result, ) -> Result { if let DbKind::ReadWrite(db) = &self.inner { loop { diff --git a/ng-oxigraph/src/oxigraph/storage/mod.rs b/ng-oxigraph/src/oxigraph/storage/mod.rs index 4c9ac56..f478f81 100644 --- a/ng-oxigraph/src/oxigraph/storage/mod.rs +++ b/ng-oxigraph/src/oxigraph/storage/mod.rs @@ -378,7 +378,7 @@ impl Storage { pub fn ng_transaction<'a, 'b: 'a, T, E: Error + 'static + From>( &'b self, - f: impl Fn(StorageWriter<'a>) -> Result, + mut f: impl FnMut(StorageWriter<'a>) -> Result, ) -> Result { self.db.ng_transaction(|transaction| { f(StorageWriter { diff --git a/ng-oxigraph/src/oxigraph/store.rs b/ng-oxigraph/src/oxigraph/store.rs index cf4d0f3..a68a7f4 100644 --- a/ng-oxigraph/src/oxigraph/store.rs +++ b/ng-oxigraph/src/oxigraph/store.rs @@ -439,7 +439,7 @@ impl Store { /// ``` fn transaction<'a, 'b: 'a, T, E: Error + 'static + From>( &'b self, - f: impl Fn(Transaction<'a>) -> Result, + mut f: impl FnMut(Transaction<'a>) -> Result, ) -> Result { self.storage .ng_transaction(|writer| f(Transaction { writer })) @@ -524,7 +524,7 @@ impl Store { #[doc(hidden)] pub fn ng_transaction<'a, 'b: 'a, T, E: Error + 'static + From>( &'b self, - f: impl Fn(Transaction<'a>) -> Result, + mut f: impl FnMut(Transaction<'a>) -> Result, ) -> Result { self.storage .ng_transaction(|writer| f(Transaction { writer })) diff --git a/ng-repo/Cargo.toml b/ng-repo/Cargo.toml index 66f0da7..bfcb0c8 100644 --- a/ng-repo/Cargo.toml +++ b/ng-repo/Cargo.toml @@ -34,7 +34,7 @@ rand = { version = "0.7", features = ["getrandom"] } blake3 = "1.3.1" chacha20 = "0.9.0" ed25519-dalek = "1.0.1" -bloomfilter = { version = "1.0.13", features = ["random","serde"] } +sbbf-rs-safe = "0.3.2" curve25519-dalek = "3.2.0" threshold_crypto = "0.4.0" crypto_box = { version = "0.8.2", features = ["seal"] } diff --git a/ng-repo/src/branch.rs b/ng-repo/src/branch.rs index 9862ed1..e1a4171 100644 --- a/ng-repo/src/branch.rs +++ b/ng-repo/src/branch.rs @@ -13,7 +13,7 @@ use std::collections::HashMap; use std::collections::HashSet; use std::fmt; -use bloomfilter::Bloom; +use sbbf_rs_safe::Filter; use zeroize::Zeroize; use crate::errors::*; @@ -187,7 +187,7 @@ impl Branch { missing: &mut Option<&mut HashSet>, future: Option, theirs_found: &mut Option<&mut HashSet>, - theirs_filter: &Option>, + theirs_filter: &Option, ) -> Result<(), ObjectParseError> { let id = cobj.id(); @@ -195,7 +195,8 @@ impl Branch { // load deps, stop at the root(including it in visited) or if this is a commit object from known_heads let found_in_filter = if let Some(filter) = theirs_filter { - filter.check(&id) + let hash = id.get_hash(); + filter.contains_hash(hash) } else { false }; @@ -276,7 +277,7 @@ impl Branch { // we silently discard any load error on the known_heads as the responder might not know them (yet). } - //log_debug!("their causal past \n{}", Dag(&theirs)); + // log_debug!("their causal past \n{}", Dag(&theirs)); let mut visited = HashMap::new(); @@ -284,8 +285,7 @@ impl Branch { let filter = if let Some(filter) = known_commits.as_ref() { Some( - serde_bare::from_slice(filter.filter()) - .map_err(|_| ObjectParseError::FilterDeserializationError)?, + filter.filter(), //.map_err(|_| ObjectParseError::FilterDeserializationError)?, ) } else { None @@ -309,7 +309,7 @@ impl Branch { // we silently discard any load error on the target_heads as they can be wrong if the requester is confused about what the responder has locally. } - //log_debug!("what we have here \n{}", Dag(&visited)); + // log_debug!("what we have here \n{}", Dag(&visited)); // now ordering to respect causal partial order. let mut next_generations = HashSet::new(); diff --git a/ng-repo/src/types.rs b/ng-repo/src/types.rs index 5310c6f..e359674 100644 --- a/ng-repo/src/types.rs +++ b/ng-repo/src/types.rs @@ -13,9 +13,11 @@ use core::fmt; use std::cmp::Ordering; -use std::hash::Hash; +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; use once_cell::sync::OnceCell; +use sbbf_rs_safe::Filter; use serde::{Deserialize, Serialize}; use threshold_crypto::serde_impl::SerdeSecret; use zeroize::{Zeroize, ZeroizeOnDrop}; @@ -70,6 +72,16 @@ impl Digest { Self::Blake3Digest32(o) => o, } } + /// returns a hash that is consistent across platforms (32/64 bits. important for WASM32 compatibility with the rest) + /// see https://www.reddit.com/r/rust/comments/fwpki6/a_debugging_mystery_hashing_slices_in_wasm_works/ + pub fn get_hash(&self) -> u64 { + let mut hasher = DefaultHasher::new(); + let ser = serde_bare::to_vec(&self).unwrap(); + for e in ser { + e.hash(&mut hasher); + } + hasher.finish() + } } impl fmt::Display for Digest { @@ -373,11 +385,16 @@ pub enum BloomFilter { } impl BloomFilter { - pub fn filter(&self) -> &Vec { + pub fn filter(&self) -> Filter { match self { - Self::V0(v0) => &v0.f, + Self::V0(v0) => Filter::from_bytes(&v0.f).unwrap(), } } + pub fn from_filter(filter: &Filter) -> Self { + BloomFilter::V0(BloomFilterV0 { + f: filter.as_bytes().to_vec(), + }) + } } // diff --git a/ng-verifier/Cargo.toml b/ng-verifier/Cargo.toml index 30d8888..4a1edcb 100644 --- a/ng-verifier/Cargo.toml +++ b/ng-verifier/Cargo.toml @@ -31,7 +31,7 @@ async-trait = "0.1.64" async-std = { version = "1.12.0", features = [ "attributes", "unstable" ] } automerge = "0.5.9" yrs = "0.18.2" -bloomfilter = { version = "1.0.13", features = ["random","serde"] } +sbbf-rs-safe = "0.3.2" ng-repo = { path = "../ng-repo", version = "0.1.0-preview.1" } ng-net = { path = "../ng-net", version = "0.1.0-preview.1" } ng-oxigraph = { path = "../ng-oxigraph", version = "0.4.0-alpha.7-ngpreview6" } diff --git a/ng-verifier/src/verifier.rs b/ng-verifier/src/verifier.rs index ff1cbc4..70e0ad6 100644 --- a/ng-verifier/src/verifier.rs +++ b/ng-verifier/src/verifier.rs @@ -23,9 +23,9 @@ use std::{collections::HashMap, sync::Arc}; use async_std::stream::StreamExt; use async_std::sync::{Mutex, RwLockReadGuard}; -use bloomfilter::Bloom; use futures::channel::mpsc; use futures::SinkExt; +use sbbf_rs_safe::Filter; use serde::{Deserialize, Serialize}; use web_time::SystemTime; @@ -1585,6 +1585,7 @@ impl Verifier { let mut theirs_found = HashSet::new(); let mut visited = HashMap::new(); for our in ours_set.iter() { + //log_info!("OUR HEADS {}", our); if let Ok(cobj) = Object::load(*our, None, &repo.store) { let _ = Branch::load_causal_past( &cobj, @@ -1611,14 +1612,11 @@ impl Verifier { // prepare bloom filter let expected_elements = remote_commits_nbr + max(visited.len() as u64, branch_info.commits_nbr); - let mut filter = - Bloom::::new_for_fp_rate(expected_elements as usize, 0.01); + let mut filter = Filter::new(27, expected_elements as usize); for commit_id in visited.keys() { - filter.set(commit_id); + filter.insert_hash(commit_id.get_hash()); } - Some(BloomFilter::V0(BloomFilterV0 { - f: serde_bare::to_vec(&filter)?, - })) + Some(BloomFilter::from_filter(&filter)) } };