From 9370a9216eddc345c58b70a8637cc6ec30eef92f Mon Sep 17 00:00:00 2001
From: Niko PLP <niko@nextgraph.org>
Date: Wed, 10 Apr 2024 19:49:56 +0300
Subject: [PATCH] ng-verifier, integration of oxigraph, yrs, automerge
---
Cargo.lock | 423 +++++++-
Cargo.toml | 6 +-
README.md | 8 +-
nextgraph/Cargo.toml | 1 -
nextgraph/README.md | 2 +-
nextgraph/examples/in_memory.rs | 6 +-
nextgraph/examples/persistent.rs | 5 +-
nextgraph/src/lib.rs | 2 +-
nextgraph/src/local_broker.rs | 397 +++++--
ng-app/README.md | 4 +-
ng-app/prepare-app-file.cjs | 2 +
ng-app/src-tauri/src/lib.rs | 21 +-
ng-app/src/api.ts | 13 +-
ng-broker/Cargo.toml | 2 +-
ng-broker/README.md | 2 +-
.../account.rs | 10 +-
.../config.rs | 4 +-
.../invitation.rs | 8 +-
.../{broker_store => broker_storage}/mod.rs | 2 -
.../overlay.rs | 4 +-
.../{broker_store => broker_storage}/peer.rs | 4 +-
.../{broker_store => broker_storage}/topic.rs | 4 +-
.../wallet.rs | 6 +-
ng-broker/src/broker_store/repostoreinfo.rs | 100 --
ng-broker/src/lib.rs | 2 +-
ng-broker/src/server.rs | 973 ------------------
ng-broker/src/server_storage.rs | 12 +-
ng-client-ws/README.md | 2 +-
ng-net/Cargo.toml | 2 +-
ng-net/README.md | 2 +-
ng-net/src/broker.rs | 9 +-
ng-net/src/broker_connection.rs | 2 +-
ng-net/src/errors.rs | 3 +-
ng-net/src/types.rs | 29 +-
ng-repo/README.md | 2 +-
ng-repo/src/{store.rs => block_storage.rs} | 36 +-
ng-repo/src/branch.rs | 25 +-
ng-repo/src/commit.rs | 42 +-
ng-repo/src/errors.rs | 25 +-
ng-repo/src/event.rs | 6 +-
ng-repo/src/file.rs | 60 +-
ng-repo/src/{kcv_store.rs => kcv_storage.rs} | 2 +-
ng-repo/src/lib.rs | 4 +-
ng-repo/src/object.rs | 19 +-
ng-repo/src/repo.rs | 19 +-
ng-repo/src/types.rs | 43 +-
ng-repo/src/utils.rs | 6 +-
ng-sdk-js/js/browser.js | 15 +
ng-sdk-js/js/node.js | 10 +
ng-sdk-js/src/lib.rs | 43 +-
.../Cargo.toml | 2 +-
.../src/block_storage.rs | 26 +-
.../src/kcv_storage.rs | 0
.../src/lib.rs | 0
.../Cargo.toml | 2 +-
.../README.md | 4 +-
.../src/block_storage.rs | 28 +-
.../src/kcv_storage.rs | 4 +-
.../src/lib.rs | 4 +-
ng-verifier/Cargo.toml | 6 +
ng-verifier/README.md | 2 +-
ng-verifier/src/lib.rs | 6 +
ng-verifier/src/rocksdb_user_storage.rs | 39 +
ng-verifier/src/types.rs | 268 +++++
ng-verifier/src/user_storage.rs | 46 +
ng-wallet/README.md | 2 +-
ng-wallet/src/types.rs | 19 +-
ngaccount/Cargo.toml | 2 +-
ngcli/Cargo.toml | 2 +-
ngcli/README.md | 2 +-
ngcli/src/main.rs | 4 +-
ngcli/src/old.rs | 14 +-
ngd/README.md | 2 +-
ngone/Cargo.toml | 2 +-
ngone/src/main.rs | 4 +-
ngone/src/store/dynpeer.rs | 4 +-
ngone/src/store/wallet_record.rs | 4 +-
77 files changed, 1479 insertions(+), 1448 deletions(-)
rename ng-broker/src/{broker_store => broker_storage}/account.rs (97%)
rename ng-broker/src/{broker_store => broker_storage}/config.rs (97%)
rename ng-broker/src/{broker_store => broker_storage}/invitation.rs (97%)
rename ng-broker/src/{broker_store => broker_storage}/mod.rs (83%)
rename ng-broker/src/{broker_store => broker_storage}/overlay.rs (98%)
rename ng-broker/src/{broker_store => broker_storage}/peer.rs (98%)
rename ng-broker/src/{broker_store => broker_storage}/topic.rs (98%)
rename ng-broker/src/{broker_store => broker_storage}/wallet.rs (97%)
delete mode 100644 ng-broker/src/broker_store/repostoreinfo.rs
delete mode 100644 ng-broker/src/server.rs
rename ng-repo/src/{store.rs => block_storage.rs} (85%)
rename ng-repo/src/{kcv_store.rs => kcv_storage.rs} (99%)
rename {ng-stores-lmdb => ng-storage-lmdb}/Cargo.toml (96%)
rename ng-stores-lmdb/src/repo_store.rs => ng-storage-lmdb/src/block_storage.rs (97%)
rename ng-stores-lmdb/src/kcv_store.rs => ng-storage-lmdb/src/kcv_storage.rs (100%)
rename {ng-stores-lmdb => ng-storage-lmdb}/src/lib.rs (100%)
rename {ng-stores-rocksdb => ng-storage-rocksdb}/Cargo.toml (95%)
rename {ng-stores-rocksdb => ng-storage-rocksdb}/README.md (97%)
rename ng-stores-rocksdb/src/repo_store.rs => ng-storage-rocksdb/src/block_storage.rs (97%)
rename ng-stores-rocksdb/src/kcv_store.rs => ng-storage-rocksdb/src/kcv_storage.rs (99%)
rename {ng-stores-rocksdb => ng-storage-rocksdb}/src/lib.rs (62%)
create mode 100644 ng-verifier/src/rocksdb_user_storage.rs
create mode 100644 ng-verifier/src/user_storage.rs
diff --git a/Cargo.lock b/Cargo.lock
index d623a89..e41c72f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -181,6 +181,12 @@ version = "1.0.71"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8"
+[[package]]
+name = "arc-swap"
+version = "1.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457"
+
[[package]]
name = "argon2"
version = "0.5.0"
@@ -270,7 +276,7 @@ dependencies = [
"async-lock",
"async-task",
"concurrent-queue",
- "fastrand",
+ "fastrand 1.9.0",
"futures-lite",
"slab",
]
@@ -384,7 +390,7 @@ checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.18",
+ "syn 2.0.58",
]
[[package]]
@@ -429,7 +435,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.18",
+ "syn 2.0.58",
]
[[package]]
@@ -496,12 +502,40 @@ version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1181e1e0d1fce796a03db1ae795d67167da795f9cf4a39c37589e85ef57f26d3"
+[[package]]
+name = "atomic_refcell"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41e67cd8309bbd06cd603a9e693a784ac2e5d1e955f11286e355089fcab3047c"
+
[[package]]
name = "autocfg"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
+[[package]]
+name = "automerge"
+version = "0.5.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93b5e6ed2097a1e55cce3128d64c909cdb42c800d4880411c7382f3dfa2c808d"
+dependencies = [
+ "flate2",
+ "fxhash",
+ "hex",
+ "im",
+ "itertools 0.12.1",
+ "leb128",
+ "serde",
+ "sha2 0.10.7",
+ "smol_str",
+ "thiserror",
+ "tinyvec",
+ "tracing",
+ "unicode-segmentation",
+ "uuid",
+]
+
[[package]]
name = "backtrace"
version = "0.3.69"
@@ -562,7 +596,7 @@ dependencies = [
"regex",
"rustc-hash",
"shlex",
- "syn 2.0.18",
+ "syn 2.0.58",
]
[[package]]
@@ -577,6 +611,15 @@ version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+[[package]]
+name = "bitmaps"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2"
+dependencies = [
+ "typenum",
+]
+
[[package]]
name = "blake2"
version = "0.10.6"
@@ -644,7 +687,7 @@ dependencies = [
"async-lock",
"async-task",
"atomic-waker",
- "fastrand",
+ "fastrand 1.9.0",
"futures-lite",
"log",
]
@@ -901,7 +944,7 @@ dependencies = [
"heck",
"proc-macro2",
"quote",
- "syn 2.0.18",
+ "syn 2.0.58",
]
[[package]]
@@ -1187,7 +1230,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331"
dependencies = [
"quote",
- "syn 2.0.18",
+ "syn 2.0.58",
]
[[package]]
@@ -1274,7 +1317,7 @@ dependencies = [
"proc-macro2",
"quote",
"strsim",
- "syn 2.0.18",
+ "syn 2.0.58",
]
[[package]]
@@ -1285,7 +1328,7 @@ checksum = "29a358ff9f12ec09c3e61fef9b5a9902623a695a46a917b07f269bff1445611a"
dependencies = [
"darling_core",
"quote",
- "syn 2.0.18",
+ "syn 2.0.58",
]
[[package]]
@@ -1433,9 +1476,9 @@ checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b"
[[package]]
name = "duration-str"
-version = "0.5.1"
+version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9f037c488d179e21c87ef5fa9c331e8e62f5dddfa84618b41bb197da03edff1"
+checksum = "a8bb6a301a95ba86fa0ebaf71d49ae4838c51f8b84cb88ed140dfb66452bb3c4"
dependencies = [
"chrono",
"nom",
@@ -1520,7 +1563,7 @@ checksum = "5e9a1f9f7d83e59740248a6e14ecf93929ade55027844dfcea78beafccc15745"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.18",
+ "syn 2.0.58",
]
[[package]]
@@ -1628,6 +1671,15 @@ dependencies = [
"instant",
]
+[[package]]
+name = "fastrand"
+version = "2.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "658bd65b1cf4c852a3cc96f18a8ce7b5640f6b703f905c7d74532294c2a63984"
+dependencies = [
+ "getrandom 0.2.10",
+]
+
[[package]]
name = "fdeflate"
version = "0.3.0"
@@ -1734,7 +1786,7 @@ checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.18",
+ "syn 2.0.58",
]
[[package]]
@@ -1822,7 +1874,7 @@ version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce"
dependencies = [
- "fastrand",
+ "fastrand 1.9.0",
"futures-core",
"futures-io",
"memchr",
@@ -1839,7 +1891,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.18",
+ "syn 2.0.58",
]
[[package]]
@@ -2493,6 +2545,20 @@ dependencies = [
"unicode-normalization",
]
+[[package]]
+name = "im"
+version = "15.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d0acd33ff0285af998aaf9b57342af478078f53492322fafc47450e09397e0e9"
+dependencies = [
+ "bitmaps",
+ "rand_core 0.6.4",
+ "rand_xoshiro",
+ "sized-chunks",
+ "typenum",
+ "version_check",
+]
+
[[package]]
name = "image"
version = "0.24.6"
@@ -2588,6 +2654,15 @@ dependencies = [
"either",
]
+[[package]]
+name = "itertools"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
+dependencies = [
+ "either",
+]
+
[[package]]
name = "itoa"
version = "0.4.8"
@@ -2672,6 +2747,12 @@ dependencies = [
"wasm-bindgen",
]
+[[package]]
+name = "json-event-parser"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f850fafca79ebacd70eab9d80cb75a33aeda38bde8f3dd784c1837cdf0bde631"
+
[[package]]
name = "json-patch"
version = "1.0.0"
@@ -2729,6 +2810,12 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
+[[package]]
+name = "leb128"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67"
+
[[package]]
name = "lebe"
version = "0.5.2"
@@ -2784,7 +2871,7 @@ checksum = "7fc7aa29613bd6a620df431842069224d8bc9011086b1db4c0e0cd47fa03ec9a"
[[package]]
name = "librocksdb-sys"
version = "0.11.0+8.3.2"
-source = "git+https://git.nextgraph.org/NextGraph/rust-rocksdb.git?branch=master#846abdaaa238ad0005d2bcd16a4f6339c96b3a9f"
+source = "git+https://git.nextgraph.org/NextGraph/rust-rocksdb.git?branch=master#13b3c2022202abff8cfe921ee926d6ca567e66e8"
dependencies = [
"bindgen",
"bzip2-sys",
@@ -2900,6 +2987,16 @@ version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5"
+[[package]]
+name = "md-5"
+version = "0.10.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf"
+dependencies = [
+ "cfg-if",
+ "digest 0.10.7",
+]
+
[[package]]
name = "memalloc"
version = "0.1.0"
@@ -3130,7 +3227,6 @@ dependencies = [
"ng-client-ws",
"ng-net",
"ng-repo",
- "ng-stores-rocksdb",
"ng-verifier",
"ng-wallet",
"once_cell",
@@ -3177,7 +3273,7 @@ dependencies = [
"ng-client-ws",
"ng-net",
"ng-repo",
- "ng-stores-rocksdb",
+ "ng-storage-rocksdb",
"once_cell",
"rust-embed",
"serde",
@@ -3302,7 +3398,7 @@ dependencies = [
]
[[package]]
-name = "ng-stores-rocksdb"
+name = "ng-storage-rocksdb"
version = "0.1.0"
dependencies = [
"hex",
@@ -3317,13 +3413,17 @@ dependencies = [
name = "ng-verifier"
version = "0.1.0"
dependencies = [
+ "automerge",
"blake3",
"chacha20",
"ng-net",
"ng-repo",
+ "ng-storage-rocksdb",
+ "oxigraph",
"serde",
"serde_bare",
"serde_bytes",
+ "yrs",
]
[[package]]
@@ -3437,7 +3537,7 @@ dependencies = [
"log",
"ng-net",
"ng-repo",
- "ng-stores-rocksdb",
+ "ng-storage-rocksdb",
"ng-wallet",
"rust-embed",
"serde",
@@ -3658,7 +3758,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.18",
+ "syn 2.0.58",
]
[[package]]
@@ -3716,6 +3816,104 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
+[[package]]
+name = "oxigraph"
+version = "0.4.0-alpha.7-dev"
+source = "git+https://git.nextgraph.org/NextGraph/oxigraph.git?branch=main#b3ae51da8274693b75622349b4bba46efcaa8c3f"
+dependencies = [
+ "digest 0.10.7",
+ "getrandom 0.2.10",
+ "hex",
+ "json-event-parser",
+ "libc",
+ "md-5",
+ "oxilangtag",
+ "oxiri",
+ "oxrdf",
+ "oxrdfio",
+ "oxsdatatypes",
+ "rand 0.8.5",
+ "regex",
+ "rocksdb",
+ "sha1",
+ "sha2 0.10.7",
+ "siphasher",
+ "sparesults",
+ "spargebra",
+ "sparopt",
+ "thiserror",
+]
+
+[[package]]
+name = "oxilangtag"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23f3f87617a86af77fa3691e6350483e7154c2ead9f1261b75130e21ca0f8acb"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "oxiri"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d05417ee46e2eb40dd9d590b4d67fc2408208b3a48a6b7f71d2bc1d7ce12a3e0"
+
+[[package]]
+name = "oxrdf"
+version = "0.2.0-alpha.4"
+source = "git+https://git.nextgraph.org/NextGraph/oxigraph.git?branch=main#b3ae51da8274693b75622349b4bba46efcaa8c3f"
+dependencies = [
+ "oxilangtag",
+ "oxiri",
+ "oxsdatatypes",
+ "rand 0.8.5",
+ "thiserror",
+]
+
+[[package]]
+name = "oxrdfio"
+version = "0.1.0-alpha.5"
+source = "git+https://git.nextgraph.org/NextGraph/oxigraph.git?branch=main#b3ae51da8274693b75622349b4bba46efcaa8c3f"
+dependencies = [
+ "oxrdf",
+ "oxrdfxml",
+ "oxttl",
+ "thiserror",
+]
+
+[[package]]
+name = "oxrdfxml"
+version = "0.1.0-alpha.5"
+source = "git+https://git.nextgraph.org/NextGraph/oxigraph.git?branch=main#b3ae51da8274693b75622349b4bba46efcaa8c3f"
+dependencies = [
+ "oxilangtag",
+ "oxiri",
+ "oxrdf",
+ "quick-xml 0.31.0",
+ "thiserror",
+]
+
+[[package]]
+name = "oxsdatatypes"
+version = "0.2.0-alpha.1"
+source = "git+https://git.nextgraph.org/NextGraph/oxigraph.git?branch=main#b3ae51da8274693b75622349b4bba46efcaa8c3f"
+dependencies = [
+ "thiserror",
+]
+
+[[package]]
+name = "oxttl"
+version = "0.1.0-alpha.5"
+source = "git+https://git.nextgraph.org/NextGraph/oxigraph.git?branch=main#b3ae51da8274693b75622349b4bba46efcaa8c3f"
+dependencies = [
+ "memchr",
+ "oxilangtag",
+ "oxiri",
+ "oxrdf",
+ "thiserror",
+]
+
[[package]]
name = "packed_simd_2"
version = "0.3.8"
@@ -3816,6 +4014,33 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
+[[package]]
+name = "peg"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "400bcab7d219c38abf8bd7cc2054eb9bbbd4312d66f6a5557d572a203f646f61"
+dependencies = [
+ "peg-macros",
+ "peg-runtime",
+]
+
+[[package]]
+name = "peg-macros"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "46e61cce859b76d19090f62da50a9fe92bab7c2a5f09e183763559a2ac392c90"
+dependencies = [
+ "peg-runtime",
+ "proc-macro2",
+ "quote",
+]
+
+[[package]]
+name = "peg-runtime"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "36bae92c60fa2398ce4678b98b2c4b5a7c61099961ca1fa305aec04a9ad28922"
+
[[package]]
name = "percent-encoding"
version = "2.3.0"
@@ -3957,7 +4182,7 @@ checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.18",
+ "syn 2.0.58",
]
[[package]]
@@ -3993,7 +4218,7 @@ dependencies = [
"base64 0.21.2",
"indexmap",
"line-wrap",
- "quick-xml",
+ "quick-xml 0.28.2",
"serde",
"time 0.3.23",
]
@@ -4070,7 +4295,7 @@ checksum = "09963355b9f467184c04017ced4a2ba2d75cbcb4e7462690d388233253d4b1a9"
dependencies = [
"anstyle",
"difflib",
- "itertools",
+ "itertools 0.10.5",
"predicates-core",
]
@@ -4097,7 +4322,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9825a04601d60621feed79c4e6b56d65db77cdca55cef43b46b0de1096d1c282"
dependencies = [
"proc-macro2",
- "syn 2.0.18",
+ "syn 2.0.58",
]
[[package]]
@@ -4142,9 +4367,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068"
[[package]]
name = "proc-macro2"
-version = "1.0.60"
+version = "1.0.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406"
+checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e"
dependencies = [
"unicode-ident",
]
@@ -4182,11 +4407,20 @@ dependencies = [
"memchr",
]
+[[package]]
+name = "quick-xml"
+version = "0.31.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33"
+dependencies = [
+ "memchr",
+]
+
[[package]]
name = "quote"
-version = "1.0.28"
+version = "1.0.36"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488"
+checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
dependencies = [
"proc-macro2",
]
@@ -4281,6 +4515,15 @@ dependencies = [
"rand_core 0.5.1",
]
+[[package]]
+name = "rand_xoshiro"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa"
+dependencies = [
+ "rand_core 0.6.4",
+]
+
[[package]]
name = "raw-window-handle"
version = "0.5.2"
@@ -4412,7 +4655,7 @@ dependencies = [
[[package]]
name = "rocksdb"
version = "0.21.0"
-source = "git+https://git.nextgraph.org/NextGraph/rust-rocksdb.git?branch=master#846abdaaa238ad0005d2bcd16a4f6339c96b3a9f"
+source = "git+https://git.nextgraph.org/NextGraph/rust-rocksdb.git?branch=master#13b3c2022202abff8cfe921ee926d6ca567e66e8"
dependencies = [
"libc",
"librocksdb-sys",
@@ -4438,7 +4681,7 @@ dependencies = [
"proc-macro2",
"quote",
"rust-embed-utils",
- "syn 2.0.18",
+ "syn 2.0.58",
"walkdir",
]
@@ -4680,7 +4923,7 @@ checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.18",
+ "syn 2.0.58",
]
[[package]]
@@ -4702,7 +4945,7 @@ checksum = "bcec881020c684085e55a25f7fd888954d56609ef363479dc5a1305eb0d40cab"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.18",
+ "syn 2.0.58",
]
[[package]]
@@ -4751,7 +4994,7 @@ dependencies = [
"darling",
"proc-macro2",
"quote",
- "syn 2.0.18",
+ "syn 2.0.58",
]
[[package]]
@@ -4873,6 +5116,16 @@ version = "0.3.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de"
+[[package]]
+name = "sized-chunks"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "16d69225bde7a69b235da73377861095455d298f2b970996eec25ddbb42b3d1e"
+dependencies = [
+ "bitmaps",
+ "typenum",
+]
+
[[package]]
name = "slab"
version = "0.4.8"
@@ -4888,11 +5141,29 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64c963ee59ddedb5ab95dc2cd97c48b4a292572a52c5636fbbabdb9985bfe4c3"
+[[package]]
+name = "smallstr"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "63b1aefdf380735ff8ded0b15f31aab05daf1f70216c01c02a12926badd1df9d"
+dependencies = [
+ "smallvec",
+]
+
[[package]]
name = "smallvec"
-version = "1.10.0"
+version = "1.13.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
+
+[[package]]
+name = "smol_str"
+version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
+checksum = "e6845563ada680337a52d43bb0b29f396f2d911616f6573012645b9e3d048a49"
+dependencies = [
+ "serde",
+]
[[package]]
name = "socket2"
@@ -4932,6 +5203,41 @@ dependencies = [
"system-deps",
]
+[[package]]
+name = "sparesults"
+version = "0.2.0-alpha.4"
+source = "git+https://git.nextgraph.org/NextGraph/oxigraph.git?branch=main#b3ae51da8274693b75622349b4bba46efcaa8c3f"
+dependencies = [
+ "json-event-parser",
+ "memchr",
+ "oxrdf",
+ "quick-xml 0.31.0",
+ "thiserror",
+]
+
+[[package]]
+name = "spargebra"
+version = "0.3.0-alpha.4"
+source = "git+https://git.nextgraph.org/NextGraph/oxigraph.git?branch=main#b3ae51da8274693b75622349b4bba46efcaa8c3f"
+dependencies = [
+ "oxilangtag",
+ "oxiri",
+ "oxrdf",
+ "peg",
+ "rand 0.8.5",
+ "thiserror",
+]
+
+[[package]]
+name = "sparopt"
+version = "0.1.0-alpha.5-dev"
+source = "git+https://git.nextgraph.org/NextGraph/oxigraph.git?branch=main#b3ae51da8274693b75622349b4bba46efcaa8c3f"
+dependencies = [
+ "oxrdf",
+ "rand 0.8.5",
+ "spargebra",
+]
+
[[package]]
name = "spin"
version = "0.9.8"
@@ -5024,9 +5330,9 @@ dependencies = [
[[package]]
name = "syn"
-version = "2.0.18"
+version = "2.0.58"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e"
+checksum = "44cfb93f38070beee36b3fef7d4f5a16f27751d94b187b666a5cc5e9b0d30687"
dependencies = [
"proc-macro2",
"quote",
@@ -5354,7 +5660,7 @@ checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6"
dependencies = [
"autocfg",
"cfg-if",
- "fastrand",
+ "fastrand 1.9.0",
"redox_syscall 0.3.5",
"rustix",
"windows-sys 0.48.0",
@@ -5394,22 +5700,22 @@ checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c"
[[package]]
name = "thiserror"
-version = "1.0.40"
+version = "1.0.58"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac"
+checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
-version = "1.0.40"
+version = "1.0.58"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f"
+checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.18",
+ "syn 2.0.58",
]
[[package]]
@@ -5542,7 +5848,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.18",
+ "syn 2.0.58",
]
[[package]]
@@ -5653,7 +5959,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.18",
+ "syn 2.0.58",
]
[[package]]
@@ -5877,6 +6183,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fa2982af2eec27de306107c027578ff7f423d65f7250e40ce0fea8f45248b81"
dependencies = [
"getrandom 0.2.10",
+ "serde",
]
[[package]]
@@ -6044,7 +6351,7 @@ dependencies = [
"once_cell",
"proc-macro2",
"quote",
- "syn 2.0.18",
+ "syn 2.0.58",
"wasm-bindgen-shared",
]
@@ -6078,7 +6385,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.18",
+ "syn 2.0.58",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
@@ -6212,7 +6519,7 @@ checksum = "ac1345798ecd8122468840bcdf1b95e5dc6d2206c5e4b0eafa078d061f59c9bc"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.18",
+ "syn 2.0.58",
]
[[package]]
@@ -6685,6 +6992,22 @@ version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "735a71d46c4d68d71d4b24d03fdc2b98e38cea81730595801db779c04fe80d70"
+[[package]]
+name = "yrs"
+version = "0.18.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4058d69bbbc97181d53d9d093a4b892001b84601f2fc4e27f48c8862bc8b369"
+dependencies = [
+ "arc-swap",
+ "atomic_refcell",
+ "fastrand 2.0.2",
+ "serde",
+ "serde_json",
+ "smallstr",
+ "smallvec",
+ "thiserror",
+]
+
[[package]]
name = "zbus"
version = "3.14.1"
@@ -6768,7 +7091,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.18",
+ "syn 2.0.58",
]
[[package]]
diff --git a/Cargo.toml b/Cargo.toml
index d57fb4c..38cfaf0 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -9,7 +9,7 @@ members = [
"ng-client-ws",
"ng-verifier",
"ng-wallet",
- "ng-stores-rocksdb",
+ "ng-storage-rocksdb",
"ngone",
"ngaccount",
"ng-sdk-js",
@@ -20,7 +20,7 @@ default-members = [ "nextgraph", "ngcli", "ngd" ]
[workspace.package]
version = "0.1.0"
edition = "2021"
-rust-version = "1.64.0"
+rust-version = "1.74.0"
license = "MIT/Apache-2.0"
authors = ["Niko PLP <niko@nextgraph.org>"]
repository = "https://git.nextgraph.org/NextGraph/nextgraph-rs"
@@ -29,7 +29,7 @@ keywords = [
"crdt","dapp","decentralized","e2ee","local-first","p2p","semantic-web","eventual-consistency","json-ld","markdown",
"ocap","z-cap","offline-first","p2p-network","collaboration","privacy-protection","rdf","rich-text-editor","self-hosted",
"sparql","byzantine-fault-tolerance",
-"web3"
+"web3", "graph-database", "database","triplestore"
]
documentation = "https://docs.nextgraph.org/"
diff --git a/README.md b/README.md
index b875354..8f4e3f5 100644
--- a/README.md
+++ b/README.md
@@ -42,7 +42,7 @@ Read our [getting started guide](https://docs.nextgraph.org/en/getting-started/)
## For contributors
-- [Install Rust](https://www.rust-lang.org/tools/install) minimum required MSRV 1.64.0
+- [Install Rust](https://www.rust-lang.org/tools/install) minimum required MSRV 1.74.0
- [Install Nodejs](https://nodejs.org/en/download/)
- [Install LLVM](https://rust-lang.github.io/rust-bindgen/requirements.html)
@@ -71,7 +71,7 @@ The crates are organized as follow :
- ng-wallet : keeps the secret keys of all identities of the user in a safe wallet
- ng-broker : Core and Server Broker library
- ng-client-ws : Websocket client library
-- ng-stores-rocksdb : RocksDB backed stores. see also dependency [repo here](https://git.nextgraph.org/NextGraph/rust-rocksdb)
+- ng-storage-rocksdb : RocksDB backed stores. see also dependency [repo here](https://git.nextgraph.org/NextGraph/rust-rocksdb)
- [ng-sdk-js](ng-sdk-js/README.md) : contains the JS SDK, with example apps: web app, react app, or node service.
- [ng-app](ng-app/README.md) : all the native apps, based on Tauri, and the web app.
- ngone : server for nextgraph.one. helps user bootstrap into the right app. Not useful to you. Published here for transparency
@@ -170,7 +170,7 @@ For building the apps, see this [documentation](ng-app/README.md).
#### OpenBSD
On OpenBSD, a conflict between the installed LibreSSL library and the reqwest crate, needs a bit of attention.
-Before compiling the daemon for OpenBSD, please comment out lines 32-33 of `ng-net/Cargo.toml`. This will be solved soon in a more appropriate way.
+Before compiling the daemon for OpenBSD, please comment out lines 41-42 of `ng-net/Cargo.toml`. This will be solved soon by using `resolver = "2"`.
```
#[target.'cfg(target_arch = "wasm32")'.dependencies]
@@ -219,7 +219,7 @@ Licensed under either of
NextGraph received funding through the [NGI Assure Fund](https://nlnet.nl/project/NextGraph/index.html), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's [Next Generation Internet](https://ngi.eu/) programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 957073.
-[rustc-image]: https://img.shields.io/badge/rustc-1.64+-blue.svg
+[rustc-image]: https://img.shields.io/badge/rustc-1.74+-blue.svg
[license-image]: https://img.shields.io/badge/license-Apache2.0-blue.svg
[license-link]: https://git.nextgraph.org/NextGraph/nextgraph-rs/raw/branch/master/LICENSE-APACHE2
[license-image2]: https://img.shields.io/badge/license-MIT-blue.svg
diff --git a/nextgraph/Cargo.toml b/nextgraph/Cargo.toml
index 8eecf0b..238888a 100644
--- a/nextgraph/Cargo.toml
+++ b/nextgraph/Cargo.toml
@@ -21,7 +21,6 @@ ng-net = { path = "../ng-net", version = "0.1.0" }
ng-wallet = { path = "../ng-wallet", version = "0.1.0" }
ng-client-ws = { path = "../ng-client-ws", version = "0.1.0" }
ng-verifier = { path = "../ng-verifier", version = "0.1.0" }
-ng-stores-rocksdb = { path = "../ng-stores-rocksdb", version = "0.1.0" }
async-once-cell = "0.5.3"
once_cell = "1.17.1"
serde = { version = "1.0", features = ["derive"] }
diff --git a/nextgraph/README.md b/nextgraph/README.md
index c357d02..70089e4 100644
--- a/nextgraph/README.md
+++ b/nextgraph/README.md
@@ -72,7 +72,7 @@ additional terms or conditions.
NextGraph received funding through the [NGI Assure Fund](https://nlnet.nl/project/NextGraph/index.html), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's [Next Generation Internet](https://ngi.eu/) programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 957073.
-[rustc-image]: https://img.shields.io/badge/rustc-1.64+-blue.svg
+[rustc-image]: https://img.shields.io/badge/rustc-1.74+-blue.svg
[license-image]: https://img.shields.io/badge/license-Apache2.0-blue.svg
[license-link]: https://git.nextgraph.org/NextGraph/nextgraph-rs/raw/branch/master/LICENSE-APACHE2
[license-image2]: https://img.shields.io/badge/license-MIT-blue.svg
diff --git a/nextgraph/examples/in_memory.rs b/nextgraph/examples/in_memory.rs
index a79635c..bb61d32 100644
--- a/nextgraph/examples/in_memory.rs
+++ b/nextgraph/examples/in_memory.rs
@@ -111,7 +111,11 @@ async fn main() -> std::io::Result<()> {
// anyway, now that the wallet is opened, let's start a session.
// we pass the user_id and the wallet_name
- let _session = session_start(SessionConfig::new(&user_id, &wallet_result.wallet_name)).await?;
+ let _session = session_start(SessionConfig::new_in_memory(
+ &user_id,
+ &wallet_result.wallet_name,
+ ))
+ .await?;
// if the user has internet access, they can now decide to connect to its Server Broker, in order to sync data
let status = user_connect(&user_id).await?;
diff --git a/nextgraph/examples/persistent.rs b/nextgraph/examples/persistent.rs
index 7bd55c6..1115bc6 100644
--- a/nextgraph/examples/persistent.rs
+++ b/nextgraph/examples/persistent.rs
@@ -15,10 +15,11 @@ use std::fs::create_dir_all;
async fn main() -> std::io::Result<()> {
// get the current working directory
let mut current_path = current_dir()?;
- current_path.push("ng-example");
+ current_path.push(".ng");
+ current_path.push("example");
create_dir_all(current_path.clone())?;
- // initialize the local_broker with config to save to disk in a folder called `ng` in the current directory
+ // initialize the local_broker with config to save to disk in a folder called `.ng/example` in the current directory
init_local_broker(Box::new(move || {
LocalBrokerConfig::BasePath(current_path.clone())
}))
diff --git a/nextgraph/src/lib.rs b/nextgraph/src/lib.rs
index 3f844f4..1a900c5 100644
--- a/nextgraph/src/lib.rs
+++ b/nextgraph/src/lib.rs
@@ -17,7 +17,7 @@
//! - the blocks of the repos,
//! - the connection(s) to your Server Broker
//! - the events that you send to the Overlay, if there is no connectivity (Outbox)
-//! - A reference to the verifier (optional)
+//! - A reference to the verifier
//!
//! In addition, the API for creating and managing your wallet is provided here.
//!
diff --git a/nextgraph/src/local_broker.rs b/nextgraph/src/local_broker.rs
index c3cb3c9..b6cd9e2 100644
--- a/nextgraph/src/local_broker.rs
+++ b/nextgraph/src/local_broker.rs
@@ -12,7 +12,9 @@ use async_std::sync::{Arc, RwLock};
use core::fmt;
use ng_net::connection::{ClientConfig, IConnect, StartConfig};
use ng_net::types::{ClientInfo, ClientType};
+use ng_net::utils::{Receiver, Sender};
use ng_repo::os_info::get_os_info;
+use ng_verifier::types::*;
use ng_wallet::emojis::encode_pazzle;
use once_cell::sync::Lazy;
use serde_bare::to_vec;
@@ -26,6 +28,7 @@ use ng_net::broker::*;
use ng_repo::errors::NgError;
use ng_repo::log::*;
use ng_repo::types::*;
+use ng_repo::utils::derive_key;
use ng_wallet::{create_wallet_v0, types::*};
#[cfg(not(target_arch = "wasm32"))]
@@ -35,19 +38,105 @@ use ng_client_ws::remote_ws_wasm::ConnectionWebSocket;
type JsStorageReadFn = dyn Fn(String) -> Result<String, NgError> + 'static + Sync + Send;
type JsStorageWriteFn = dyn Fn(String, String) -> Result<(), NgError> + 'static + Sync + Send;
+type JsStorageDelFn = dyn Fn(String) -> Result<(), NgError> + 'static + Sync + Send;
type JsCallback = dyn Fn() + 'static + Sync + Send;
#[doc(hidden)]
pub struct JsStorageConfig {
pub local_read: Box<JsStorageReadFn>,
pub local_write: Box<JsStorageWriteFn>,
- pub session_read: Box<JsStorageReadFn>,
- pub session_write: Box<JsStorageWriteFn>,
+ pub session_read: Arc<Box<JsStorageReadFn>>,
+ pub session_write: Arc<Box<JsStorageWriteFn>>,
+ pub session_del: Arc<Box<JsStorageDelFn>>,
+ pub is_browser: bool,
+}
+
+impl JsStorageConfig {
+ fn get_js_storage_config(&self) -> JsSaveSessionConfig {
+ let session_read2 = Arc::clone(&self.session_read);
+ let session_write2 = Arc::clone(&self.session_write);
+ let session_read3 = Arc::clone(&self.session_read);
+ let session_write3 = Arc::clone(&self.session_write);
+ let session_read4 = Arc::clone(&self.session_read);
+ let session_del = Arc::clone(&self.session_del);
+ JsSaveSessionConfig {
+ last_seq_function: Box::new(move |peer_id: PubKey, qty: u16| -> Result<u64, NgError> {
+ let res = (session_read2)(format!("ng_peer_last_seq@{}", peer_id));
+ let val = match res {
+ Ok(old_str) => {
+ let decoded = base64_url::decode(&old_str)
+ .map_err(|_| NgError::SerializationError)?;
+ match serde_bare::from_slice(&decoded)? {
+ SessionPeerLastSeq::V0(old_val) => old_val,
+ _ => unimplemented!(),
+ }
+ }
+ Err(_) => 0,
+ };
+ let new_val = val + qty as u64;
+ let spls = SessionPeerLastSeq::V0(new_val);
+ let ser = serde_bare::to_vec(&spls)?;
+ //saving the new val
+ let encoded = base64_url::encode(&ser);
+ let r = (session_write2)(format!("ng_peer_last_seq@{}", peer_id), encoded);
+ if r.is_ok() {
+ return Err(NgError::SerializationError);
+ }
+ Ok(val)
+ }),
+ outbox_write_function: Box::new(
+ move |peer_id: PubKey, seq: u64, event: Vec<u8>| -> Result<(), NgError> {
+ let seq_str = format!("{}", seq);
+ let res = (session_read3)(format!("ng_outboxes@{}@start", peer_id));
+ let start = match res {
+ Err(_) => {
+ (session_write3)(format!("ng_outboxes@{}@start", peer_id), seq_str)?;
+ seq
+ }
+ Ok(start_str) => start_str
+ .parse::<u64>()
+ .map_err(|_| NgError::InvalidFileFormat)?,
+ };
+ let idx = seq - start;
+ let idx_str = format!("{:05}", idx);
+ let encoded = base64_url::encode(&event);
+ (session_write3)(format!("ng_outboxes@{}@{idx_str}", peer_id), encoded)
+ },
+ ),
+ outbox_read_function: Box::new(
+ move |peer_id: PubKey| -> Result<Vec<Vec<u8>>, NgError> {
+ let res = (session_read4)(format!("ng_outboxes@{}@start", peer_id));
+ let mut start = match res {
+ Err(_) => return Err(NgError::NotFound),
+ Ok(start_str) => start_str
+ .parse::<u64>()
+ .map_err(|_| NgError::InvalidFileFormat)?,
+ };
+ let mut result = vec![];
+ loop {
+ let idx_str = format!("{:05}", start);
+ let str = format!("ng_outboxes@{}@{idx_str}", peer_id);
+ let res = (session_read4)(str.clone());
+ let res = match res {
+ Err(_) => break,
+ Ok(res) => res,
+ };
+ (session_del)(str)?;
+ let decoded =
+ base64_url::decode(&res).map_err(|_| NgError::SerializationError)?;
+ result.push(decoded);
+ start += 1;
+ }
+ Ok(result)
+ },
+ ),
+ }
+ }
}
impl fmt::Debug for JsStorageConfig {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- write!(f, "JsStorageConfig")
+ write!(f, "JsStorageConfig. is_browser {}", self.is_browser)
}
}
@@ -86,25 +175,14 @@ impl LocalBrokerConfig {
}
}
-//type LastSeqFn = fn(PubKey, u16) -> Result<u64, NgError>;
-pub type LastSeqFn = dyn Fn(PubKey, u16) -> Result<u64, NgError> + 'static + Sync + Send;
-
-// peer_id: PubKey, seq_num:u64, event_ser: vec<u8>,
-pub type OutboxWriteFn =
- dyn Fn(PubKey, u64, Vec<u8>) -> Result<(), NgError> + 'static + Sync + Send;
-
-// peer_id: PubKey,
-pub type OutboxReadFn = dyn Fn(PubKey) -> Result<Vec<Vec<u8>>, NgError> + 'static + Sync + Send;
-
+#[derive(Debug)]
/// used to initiate a session at a local broker V0
pub struct SessionConfigV0 {
pub user_id: UserId,
pub wallet_name: String,
- // pub last_seq_function: Box<LastSeqFn>,
- // pub outbox_write_function: Box<OutboxWriteFn>,
- // pub outbox_read_function: Box<OutboxReadFn>,
+ pub verifier_type: VerifierType,
}
-
+#[derive(Debug)]
/// used to initiate a session at a local broker
pub enum SessionConfig {
V0(SessionConfigV0),
@@ -115,7 +193,7 @@ struct Session {
config: SessionConfig,
peer_key: PrivKey,
last_wallet_nonce: u64,
- //verifier,
+ verifier: Verifier,
}
impl SessionConfig {
@@ -129,34 +207,95 @@ impl SessionConfig {
Self::V0(v0) => v0.wallet_name.clone(),
}
}
- /// Creates a new SessionConfig with a UserId and a wallet name
+ pub fn verifier_type(&self) -> &VerifierType {
+ match self {
+ Self::V0(v0) => &v0.verifier_type,
+ }
+ }
+ /// Creates a new in_memory SessionConfig with a UserId and a wallet name
+ ///
/// that should be passed to [session_start]
- pub fn new(user_id: &UserId, wallet_name: &String) -> Self {
+ pub fn new_in_memory(user_id: &UserId, wallet_name: &String) -> Self {
SessionConfig::V0(SessionConfigV0 {
user_id: user_id.clone(),
wallet_name: wallet_name.clone(),
+ verifier_type: VerifierType::Memory,
})
}
-}
-impl fmt::Debug for SessionConfigV0 {
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- write!(
- f,
- "SessionConfigV0 user={} wallet={}",
- self.user_id, self.wallet_name
- )
+ /// Creates a new SessionConfig backed by RocksDb, with a UserId and a wallet name
+ ///
+ /// that should be passed to [session_start]
+ pub fn new_rocksdb(user_id: &UserId, wallet_name: &String) -> Self {
+ SessionConfig::V0(SessionConfigV0 {
+ user_id: user_id.clone(),
+ wallet_name: wallet_name.clone(),
+ verifier_type: VerifierType::RocksDb,
+ })
}
-}
-impl fmt::Debug for SessionConfig {
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- match self {
- SessionConfig::V0(v0) => v0.fmt(f),
+ /// Creates a new remote SessionConfig, with a UserId, a wallet name and optional remote peer_id
+ ///
+ /// that should be passed to [session_start]
+ pub fn new_remote(
+ user_id: &UserId,
+ wallet_name: &String,
+ remote_verifier_peer_id: Option<PubKey>,
+ ) -> Self {
+ SessionConfig::V0(SessionConfigV0 {
+ user_id: user_id.clone(),
+ wallet_name: wallet_name.clone(),
+ verifier_type: VerifierType::Remote(remote_verifier_peer_id),
+ })
+ }
+
+ fn valid_verifier_config_for_local_broker_config(
+ &mut self,
+ local_broker_config: &LocalBrokerConfig,
+ ) -> Result<(), NgError> {
+ if match self {
+ Self::V0(v0) => match local_broker_config {
+ LocalBrokerConfig::InMemory => {
+ v0.verifier_type = VerifierType::Memory;
+ true
+ }
+ LocalBrokerConfig::JsStorage(js_config) => match v0.verifier_type {
+ VerifierType::Memory | VerifierType::Remote(_) => true,
+ VerifierType::RocksDb => false,
+ VerifierType::WebRocksDb => js_config.is_browser,
+ },
+ LocalBrokerConfig::BasePath(_) => match v0.verifier_type {
+ VerifierType::RocksDb | VerifierType::Remote(_) => true,
+ //VerifierType::Memory => true,
+ _ => false,
+ },
+ },
+ } {
+ Ok(())
+ } else {
+ Err(NgError::InvalidArgument)
}
}
}
+// impl fmt::Debug for SessionConfigV0 {
+// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+// write!(
+// f,
+// "SessionConfigV0 user={} wallet={}",
+// self.user_id, self.wallet_name
+// )
+// }
+// }
+
+// impl fmt::Debug for SessionConfig {
+// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+// match self {
+// SessionConfig::V0(v0) => v0.fmt(f),
+// }
+// }
+// }
+
#[derive(Debug)]
struct LocalBroker {
pub config: LocalBrokerConfig,
@@ -167,19 +306,78 @@ struct LocalBroker {
pub sessions: HashMap<UserId, SessionPeerStorageV0>,
- pub opened_sessions: HashMap<UserId, Session>,
+ pub opened_sessions: HashMap<UserId, u8>,
+
+ pub opened_sessions_list: Vec<Option<Session>>,
}
impl ILocalBroker for LocalBroker {}
impl LocalBroker {
- fn get_wallet_for_session(&self, config: &SessionConfig) -> Result<&SensitiveWallet, NgError> {
- match config {
+ fn storage_path_for_user(&self, user_id: &UserId) -> Option<PathBuf> {
+ match &self.config {
+ LocalBrokerConfig::InMemory | LocalBrokerConfig::JsStorage(_) => None,
+ LocalBrokerConfig::BasePath(base) => {
+ let mut path = base.clone();
+ path.push(user_id.to_hash_string());
+ Some(path)
+ }
+ }
+ }
+
+ fn verifier_config_type_from_session_config(
+ &self,
+ config: &SessionConfig,
+ ) -> VerifierConfigType {
+ match (config.verifier_type(), &self.config) {
+ (VerifierType::Memory, LocalBrokerConfig::InMemory) => VerifierConfigType::Memory,
+ (VerifierType::RocksDb, LocalBrokerConfig::BasePath(base)) => {
+ let mut path = base.clone();
+ path.push(config.user_id().to_hash_string());
+ VerifierConfigType::RocksDb(path)
+ }
+ (VerifierType::Remote(to), _) => VerifierConfigType::Remote(*to),
+ (VerifierType::WebRocksDb, _) => VerifierConfigType::WebRocksDb,
+ (VerifierType::Memory, LocalBrokerConfig::JsStorage(js)) => {
+ VerifierConfigType::JsSaveSession(js.get_js_storage_config())
+ }
+ (_, _) => panic!("invalid combination in verifier_config_type_from_session_config"),
+ }
+ }
+
+ fn get_wallet_and_session(
+ &mut self,
+ user_id: &UserId,
+ ) -> Result<(&SensitiveWallet, &mut Session), NgError> {
+ let session_idx = self
+ .opened_sessions
+ .get(user_id)
+ .ok_or(NgError::SessionNotFound)?;
+ let session = self.opened_sessions_list[*session_idx as usize]
+ .as_mut()
+ .ok_or(NgError::SessionNotFound)?;
+ let wallet = match &session.config {
SessionConfig::V0(v0) => self
.opened_wallets
.get(&v0.wallet_name)
.ok_or(NgError::WalletNotFound),
+ }?;
+
+ Ok((wallet, session))
+ }
+ async fn disconnect_session(&mut self, user_id: &PubKey) -> Result<(), NgError> {
+ match self.opened_sessions.get(user_id) {
+ Some(session) => {
+ // TODO: change the logic here once it will be possible to have several users connected at the same time
+ Broker::close_all_connections().await;
+ let session = self.opened_sessions_list[*session as usize]
+ .as_mut()
+ .ok_or(NgError::SessionNotFound)?;
+ session.verifier.connected_server_id = None;
+ }
+ None => {}
}
+ Ok(())
}
}
@@ -224,6 +422,7 @@ async fn init_(config: LocalBrokerConfig) -> Result<Arc<RwLock<LocalBroker>>, Ng
opened_wallets: HashMap::new(),
sessions: HashMap::new(),
opened_sessions: HashMap::new(),
+ opened_sessions_list: vec![],
};
//log_debug!("{:?}", &local_broker);
@@ -492,24 +691,36 @@ pub async fn wallet_was_opened(mut wallet: SensitiveWallet) -> Result<ClientV0,
/// Starts a session with the LocalBroker. The type of verifier is selected at this moment.
///
/// The session is valid even if there is no internet. The local data will be used in this case.
-/// The returned value is not really useful. Might be removed
-//TODO: remove return value?
-pub async fn session_start(config: SessionConfig) -> Result<SessionPeerStorageV0, NgError> {
+/// Return value is the index of the session, will be used in all the doc_* API calls.
+pub async fn session_start(mut config: SessionConfig) -> Result<SessionInfo, NgError> {
let mut broker = match LOCAL_BROKER.get() {
None | Some(Err(_)) => return Err(NgError::LocalBrokerNotInitialized),
Some(Ok(broker)) => broker.write().await,
};
- let wallet_name = config.wallet_name();
+ config.valid_verifier_config_for_local_broker_config(&broker.config)?;
+
+ let wallet_name: String = config.wallet_name();
let wallet_id: PubKey = (*wallet_name).try_into()?;
let user_id = config.user_id();
match broker.opened_wallets.get(&wallet_name) {
None => return Err(NgError::WalletNotFound),
Some(wallet) => {
- if !wallet.has_user(&user_id) {
- return Err(NgError::NotFound);
- }
+ let credentials = match wallet.individual_site(&user_id) {
+ Some(creds) => creds.clone(),
+ None => return Err(NgError::NotFound),
+ };
+
+ let client_storage_master_key = serde_bare::to_vec(
+ &wallet
+ .client()
+ .as_ref()
+ .unwrap()
+ .sensitive_client_storage
+ .storage_master_key,
+ )
+ .unwrap();
let session = match broker.sessions.get(&user_id) {
Some(session) => session,
@@ -612,16 +823,35 @@ pub async fn session_start(config: SessionConfig) -> Result<SessionPeerStorageV0
}
};
let session = session.clone();
- broker.opened_sessions.insert(
- user_id,
- Session {
- config,
- peer_key: session.peer_key.clone(),
- last_wallet_nonce: session.last_wallet_nonce,
- },
+
+ // derive user_master_key from client's storage_master_key
+ let user_id_ser = serde_bare::to_vec(&user_id).unwrap();
+ let mut key_material = [user_id_ser, client_storage_master_key].concat(); //
+ let mut key: [u8; 32] = derive_key(
+ "NextGraph user_master_key BLAKE3 key",
+ key_material.as_slice(),
);
- // FIXME: is this return value useful ?
- Ok(session)
+ key_material.zeroize();
+ let verifier = Verifier::new(VerifierConfig {
+ config_type: broker.verifier_config_type_from_session_config(&config),
+ user_master_key: key,
+ peer_priv_key: session.peer_key.clone(),
+ user_priv_key: credentials.0,
+ private_store_read_cap: credentials.1,
+ })?;
+ key.zeroize();
+ broker.opened_sessions_list.push(Some(Session {
+ config,
+ peer_key: session.peer_key.clone(),
+ last_wallet_nonce: session.last_wallet_nonce,
+ verifier,
+ }));
+ let idx = broker.opened_sessions_list.len() - 1;
+ broker.opened_sessions.insert(user_id, idx as u8);
+ Ok(SessionInfo {
+ session_id: idx as u8,
+ user: user_id,
+ })
}
}
}
@@ -688,16 +918,14 @@ pub async fn user_connect_with_device_info(
user_id: &UserId,
location: Option<String>,
) -> Result<Vec<(String, String, String, Option<String>, f64)>, NgError> {
- let local_broker = match LOCAL_BROKER.get() {
+ //FIXME: release this write lock much sooner than at the end of the loop of all tries to connect to some servers ?
+ // or maybe it is good to block as we dont want concurrent connection attemps potentially to the same server
+ let mut local_broker = match LOCAL_BROKER.get() {
None | Some(Err(_)) => return Err(NgError::LocalBrokerNotInitialized),
- Some(Ok(broker)) => broker.read().await,
+ Some(Ok(broker)) => broker.write().await,
};
- let session = local_broker
- .opened_sessions
- .get(user_id)
- .ok_or(NgError::SessionNotFound)?;
- let wallet = local_broker.get_wallet_for_session(&session.config)?;
+ let (wallet, session) = local_broker.get_wallet_and_session(user_id)?;
let mut result: Vec<(String, String, String, Option<String>, f64)> = Vec::new();
let arc_cnx: Arc<Box<dyn IConnect>> = Arc::new(Box::new(ConnectionWebSocket {}));
@@ -791,6 +1019,7 @@ pub async fn user_connect_with_device_info(
));
}
if tried.is_some() && tried.as_ref().unwrap().3.is_none() {
+ session.verifier.connected_server_id = Some(server_key);
// successful. we can stop here
break;
} else {
@@ -825,9 +1054,13 @@ pub async fn session_stop(user_id: &UserId) -> Result<(), NgError> {
Some(Ok(broker)) => broker.write().await,
};
- if broker.opened_sessions.remove(user_id).is_some() {
- // TODO: change the logic here once it will be possible to have several users connected at the same time
- Broker::close_all_connections().await;
+ match broker.opened_sessions.remove(user_id) {
+ Some(id) => {
+ broker.opened_sessions_list[id as usize].take();
+ // TODO: change the logic here once it will be possible to have several users connected at the same time
+ Broker::close_all_connections().await;
+ }
+ None => {}
}
Ok(())
@@ -835,17 +1068,12 @@ pub async fn session_stop(user_id: &UserId) -> Result<(), NgError> {
/// Disconnects the user from the Server Broker(s), but keep all the local data opened and ready.
pub async fn user_disconnect(user_id: &UserId) -> Result<(), NgError> {
- let broker = match LOCAL_BROKER.get() {
+ let mut broker = match LOCAL_BROKER.get() {
None | Some(Err(_)) => return Err(NgError::LocalBrokerNotInitialized),
- Some(Ok(broker)) => broker.read().await,
+ Some(Ok(broker)) => broker.write().await,
};
- if broker.opened_sessions.get(user_id).is_some() {
- // TODO: change the logic here once it will be possible to have several users connected at the same time
- Broker::close_all_connections().await;
- }
-
- Ok(())
+ broker.disconnect_session(user_id).await
}
/// Closes a wallet, which means that the pazzle will have to be entered again if the user wants to use it
@@ -859,7 +1087,12 @@ pub async fn wallet_close(wallet_name: &String) -> Result<(), NgError> {
Some(mut wallet) => {
for user in wallet.sites() {
let key: PubKey = (user.as_str()).try_into().unwrap();
- broker.opened_sessions.remove(&key);
+ match broker.opened_sessions.remove(&key) {
+ Some(id) => {
+ broker.opened_sessions_list[id as usize].take();
+ }
+ None => {}
+ }
}
wallet.zeroize();
}
@@ -884,6 +1117,26 @@ pub async fn wallet_remove(wallet_name: String) -> Result<(), NgError> {
Ok(())
}
+/// fetches a document's content, or performs a mutation on the document.
+pub async fn doc_fetch(
+ session_id: u8,
+ nuri: String,
+ payload: Option<AppRequestPayload>,
+) -> Result<(Receiver<AppResponse>, CancelFn), NgError> {
+ let broker = match LOCAL_BROKER.get() {
+ None | Some(Err(_)) => return Err(NgError::LocalBrokerNotInitialized),
+ Some(Ok(broker)) => broker.read().await,
+ };
+ if session_id as usize >= broker.opened_sessions_list.len() {
+ return Err(NgError::InvalidArgument);
+ }
+ let session = broker.opened_sessions_list[session_id as usize]
+ .as_ref()
+ .ok_or(NgError::SessionNotFound)?;
+
+ session.verifier.doc_fetch(nuri, payload)
+}
+
#[cfg(test)]
mod test {
use super::*;
@@ -1003,7 +1256,7 @@ mod test {
.await
.expect("wallet_import");
- let _session = session_start(SessionConfig::new(&user_id, &wallet_name))
+ let _session = session_start(SessionConfig::new_in_memory(&user_id, &wallet_name))
.await
.expect("");
diff --git a/ng-app/README.md b/ng-app/README.md
index 049955d..b93b6e6 100644
--- a/ng-app/README.md
+++ b/ng-app/README.md
@@ -1,8 +1,8 @@
# NextGraph apps (Linux, MacOS, Windows, Android, iOS, web)
-NextGraph native apps use the Tauri framework.
+All the apps are based on Svelte and share the same code.
-All the native apps are using an embedded WebView that renders a Svelte app.
+The native apps are using the Tauri framework with an embedded WebView to render the Svelte app.
## Install
diff --git a/ng-app/prepare-app-file.cjs b/ng-app/prepare-app-file.cjs
index e02a481..96bf400 100644
--- a/ng-app/prepare-app-file.cjs
+++ b/ng-app/prepare-app-file.cjs
@@ -25,6 +25,8 @@ s.on('end', function() {
var buf = Buffer.concat(bufs);
gzip(buf).then((compressed) => {fs.writeFileSync(gzip_file, compressed);});
+ fs.rm(filename,()=>{});
+
})
diff --git a/ng-app/src-tauri/src/lib.rs b/ng-app/src-tauri/src/lib.rs
index bf89526..4677289 100644
--- a/ng-app/src-tauri/src/lib.rs
+++ b/ng-app/src-tauri/src/lib.rs
@@ -233,11 +233,21 @@ async fn session_start(
wallet_name: String,
user: PubKey,
app: tauri::AppHandle,
-) -> Result<SessionPeerStorageV0, String> {
- let config = SessionConfig::V0(SessionConfigV0 {
- user_id: user,
- wallet_name,
- });
+) -> Result<SessionInfo, String> {
+ let config = SessionConfig::new_rocksdb(&user, &wallet_name);
+ nextgraph::local_broker::session_start(config)
+ .await
+ .map_err(|e: NgError| e.to_string())
+}
+
+#[tauri::command(rename_all = "snake_case")]
+async fn session_start_remote(
+ wallet_name: String,
+ user: PubKey,
+ peer_id: Option<PubKey>,
+ app: tauri::AppHandle,
+) -> Result<SessionInfo, String> {
+ let config = SessionConfig::new_remote(&user, &wallet_name, peer_id);
nextgraph::local_broker::session_start(config)
.await
.map_err(|e: NgError| e.to_string())
@@ -497,6 +507,7 @@ impl AppBuilder {
wallet_close,
encode_create_account,
session_start,
+ session_start_remote,
session_stop,
get_wallets,
open_window,
diff --git a/ng-app/src/api.ts b/ng-app/src/api.ts
index 01dd2e3..b5eef4d 100644
--- a/ng-app/src/api.ts
+++ b/ng-app/src/api.ts
@@ -25,6 +25,7 @@ const mapping = {
"wallet_close": ["wallet_name"],
"encode_create_account": ["payload"],
"session_start": ["wallet_name","user"],
+ "session_start_remote": ["wallet_name","user","peer_id"],
"session_stop": ["user_id"],
"get_wallets": [],
"open_window": ["url","label","title"],
@@ -50,12 +51,12 @@ const handler = {
} else if (path[0] === "get_wallets") {
let wallets = await Reflect.apply(sdk[path], caller, args);
return Object.fromEntries(wallets || []);
- } else if (path[0] === "session_start") {
- let res = await Reflect.apply(sdk[path], caller, args);
- return res;
- } else if (path[0] === "wallet_create") {
- let res = await Reflect.apply(sdk[path], caller, args);
- return res;
+ // } else if (path[0] === "session_start") {
+ // let res = await Reflect.apply(sdk[path], caller, args);
+ // return res;
+ // } else if (path[0] === "wallet_create") {
+ // let res = await Reflect.apply(sdk[path], caller, args);
+ // return res;
} else {
return Reflect.apply(sdk[path], caller, args)
}
diff --git a/ng-broker/Cargo.toml b/ng-broker/Cargo.toml
index a9b70cd..d523116 100644
--- a/ng-broker/Cargo.toml
+++ b/ng-broker/Cargo.toml
@@ -19,7 +19,7 @@ maintenance = { status = "actively-developed" }
ng-repo = { path = "../ng-repo", version = "0.1.0" }
ng-net = { path = "../ng-net", version = "0.1.0" }
ng-client-ws = { path = "../ng-client-ws", version = "0.1.0" }
-ng-stores-rocksdb = { path = "../ng-stores-rocksdb", version = "0.1.0" }
+ng-storage-rocksdb = { path = "../ng-storage-rocksdb", version = "0.1.0" }
chacha20 = "0.9.0"
serde = { version = "1.0", features = ["derive"] }
serde_bare = "0.5.0"
diff --git a/ng-broker/README.md b/ng-broker/README.md
index efa9d43..94dd13a 100644
--- a/ng-broker/README.md
+++ b/ng-broker/README.md
@@ -48,7 +48,7 @@ additional terms or conditions.
NextGraph received funding through the [NGI Assure Fund](https://nlnet.nl/project/NextGraph/index.html), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's [Next Generation Internet](https://ngi.eu/) programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 957073.
-[rustc-image]: https://img.shields.io/badge/rustc-1.64+-blue.svg
+[rustc-image]: https://img.shields.io/badge/rustc71.74+-blue.svg
[license-image]: https://img.shields.io/badge/license-Apache2.0-blue.svg
[license-link]: https://git.nextgraph.org/NextGraph/nextgraph-rs/raw/branch/master/LICENSE-APACHE2
[license-image2]: https://img.shields.io/badge/license-MIT-blue.svg
diff --git a/ng-broker/src/broker_store/account.rs b/ng-broker/src/broker_storage/account.rs
similarity index 97%
rename from ng-broker/src/broker_store/account.rs
rename to ng-broker/src/broker_storage/account.rs
index 25aaef3..04e949b 100644
--- a/ng-broker/src/broker_store/account.rs
+++ b/ng-broker/src/broker_storage/account.rs
@@ -15,9 +15,9 @@ use std::hash::Hasher;
use std::time::SystemTime;
use ng_net::types::*;
-use ng_repo::kcv_store::KCVStore;
+use ng_repo::errors::StorageError;
+use ng_repo::kcv_storage::KCVStore;
use ng_repo::log::*;
-use ng_repo::store::*;
use ng_repo::types::UserId;
use serde_bare::{from_slice, to_vec};
@@ -225,14 +225,14 @@ impl<'a> Account<'a> {
#[cfg(test)]
mod test {
- use ng_repo::store::*;
+ use ng_repo::errors::StorageError;
use ng_repo::types::*;
use ng_repo::utils::*;
- use ng_stores_rocksdb::kcv_store::RocksdbKCVStore;
+ use ng_storage_rocksdb::kcv_storage::RocksdbKCVStore;
use std::fs;
use tempfile::Builder;
- use crate::broker_store::account::Account;
+ use crate::broker_storage::account::Account;
#[test]
pub fn test_account() {
diff --git a/ng-broker/src/broker_store/config.rs b/ng-broker/src/broker_storage/config.rs
similarity index 97%
rename from ng-broker/src/broker_store/config.rs
rename to ng-broker/src/broker_storage/config.rs
index a068ba0..6090de4 100644
--- a/ng-broker/src/broker_store/config.rs
+++ b/ng-broker/src/broker_storage/config.rs
@@ -10,8 +10,8 @@
//! Broker Config, persisted to store
use ng_net::types::*;
-use ng_repo::kcv_store::KCVStore;
-use ng_repo::store::*;
+use ng_repo::errors::StorageError;
+use ng_repo::kcv_storage::KCVStore;
use ng_repo::types::*;
use serde::{Deserialize, Serialize};
use serde_bare::{from_slice, to_vec};
diff --git a/ng-broker/src/broker_store/invitation.rs b/ng-broker/src/broker_storage/invitation.rs
similarity index 97%
rename from ng-broker/src/broker_store/invitation.rs
rename to ng-broker/src/broker_storage/invitation.rs
index 79429de..401b687 100644
--- a/ng-broker/src/broker_store/invitation.rs
+++ b/ng-broker/src/broker_storage/invitation.rs
@@ -16,8 +16,8 @@ use std::time::SystemTime;
use ng_net::errors::ProtocolError;
use ng_net::types::*;
-use ng_repo::kcv_store::KCVStore;
-use ng_repo::store::*;
+use ng_repo::errors::StorageError;
+use ng_repo::kcv_storage::KCVStore;
use ng_repo::types::SymKey;
use ng_repo::types::Timestamp;
use ng_repo::utils::now_timestamp;
@@ -178,13 +178,13 @@ impl<'a> Invitation<'a> {
#[cfg(test)]
mod test {
- use ng_repo::store::*;
+ use ng_repo::errors::StorageError;
use ng_repo::types::*;
use ng_repo::utils::*;
use std::fs;
use tempfile::Builder;
- use crate::broker_store::account::Account;
+ use crate::broker_storage::account::Account;
#[test]
pub fn test_invitation() {}
diff --git a/ng-broker/src/broker_store/mod.rs b/ng-broker/src/broker_storage/mod.rs
similarity index 83%
rename from ng-broker/src/broker_store/mod.rs
rename to ng-broker/src/broker_storage/mod.rs
index f429099..0d2ad4c 100644
--- a/ng-broker/src/broker_store/mod.rs
+++ b/ng-broker/src/broker_storage/mod.rs
@@ -6,8 +6,6 @@ pub mod overlay;
pub mod peer;
-pub mod repostoreinfo;
-
pub mod topic;
pub mod invitation;
diff --git a/ng-broker/src/broker_store/overlay.rs b/ng-broker/src/broker_storage/overlay.rs
similarity index 98%
rename from ng-broker/src/broker_store/overlay.rs
rename to ng-broker/src/broker_storage/overlay.rs
index d61a86b..a322220 100644
--- a/ng-broker/src/broker_store/overlay.rs
+++ b/ng-broker/src/broker_storage/overlay.rs
@@ -10,8 +10,8 @@
//! Overlay
use ng_net::types::*;
-use ng_repo::kcv_store::KCVStore;
-use ng_repo::store::*;
+use ng_repo::errors::StorageError;
+use ng_repo::kcv_storage::KCVStore;
use ng_repo::types::*;
use ng_repo::utils::now_timestamp;
use serde::{Deserialize, Serialize};
diff --git a/ng-broker/src/broker_store/peer.rs b/ng-broker/src/broker_storage/peer.rs
similarity index 98%
rename from ng-broker/src/broker_store/peer.rs
rename to ng-broker/src/broker_storage/peer.rs
index 393dd83..611aabe 100644
--- a/ng-broker/src/broker_store/peer.rs
+++ b/ng-broker/src/broker_storage/peer.rs
@@ -10,8 +10,8 @@
//! Peer
use ng_net::types::*;
-use ng_repo::kcv_store::KCVStore;
-use ng_repo::store::*;
+use ng_repo::errors::StorageError;
+use ng_repo::kcv_storage::KCVStore;
use ng_repo::types::*;
use serde::{Deserialize, Serialize};
use serde_bare::{from_slice, to_vec};
diff --git a/ng-broker/src/broker_store/topic.rs b/ng-broker/src/broker_storage/topic.rs
similarity index 98%
rename from ng-broker/src/broker_store/topic.rs
rename to ng-broker/src/broker_storage/topic.rs
index a64f9e6..ba61a51 100644
--- a/ng-broker/src/broker_store/topic.rs
+++ b/ng-broker/src/broker_storage/topic.rs
@@ -10,8 +10,8 @@
//! Topic
use ng_net::types::*;
-use ng_repo::kcv_store::KCVStore;
-use ng_repo::store::*;
+use ng_repo::errors::StorageError;
+use ng_repo::kcv_storage::KCVStore;
use ng_repo::types::*;
use serde::{Deserialize, Serialize};
use serde_bare::{from_slice, to_vec};
diff --git a/ng-broker/src/broker_store/wallet.rs b/ng-broker/src/broker_storage/wallet.rs
similarity index 97%
rename from ng-broker/src/broker_store/wallet.rs
rename to ng-broker/src/broker_storage/wallet.rs
index 1b91e1c..e41082a 100644
--- a/ng-broker/src/broker_store/wallet.rs
+++ b/ng-broker/src/broker_storage/wallet.rs
@@ -10,10 +10,10 @@
//! Broker Wallet, persists to store all the SymKeys needed to open other storages
use ng_net::types::*;
-use ng_repo::kcv_store::KCVStore;
-use ng_repo::kcv_store::WriteTransaction;
+use ng_repo::errors::StorageError;
+use ng_repo::kcv_storage::KCVStore;
+use ng_repo::kcv_storage::WriteTransaction;
use ng_repo::log::*;
-use ng_repo::store::*;
use ng_repo::types::*;
use serde::{Deserialize, Serialize};
use serde_bare::{from_slice, to_vec};
diff --git a/ng-broker/src/broker_store/repostoreinfo.rs b/ng-broker/src/broker_store/repostoreinfo.rs
deleted file mode 100644
index 0ef9cbf..0000000
--- a/ng-broker/src/broker_store/repostoreinfo.rs
+++ /dev/null
@@ -1,100 +0,0 @@
-// Copyright (c) 2022-2024 Niko Bonnieure, Par le Peuple, NextGraph.org developers
-// All rights reserved.
-// Licensed under the Apache License, Version 2.0
-// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
-// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
-// at your option. All files in the project carrying such
-// notice may not be copied, modified, or distributed except
-// according to those terms.
-
-//! RepoStore information about each RepoStore
-//! It contains the symKeys to open the RepoStores
-//! A repoStore is identified by its repo pubkey if in local mode
-//! In core mode, it is identified by the overlayid.
-
-use ng_net::types::*;
-use ng_repo::kcv_store::KCVStore;
-use ng_repo::store::*;
-use ng_repo::types::*;
-use serde::{Deserialize, Serialize};
-use serde_bare::{from_slice, to_vec};
-
-// #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]
-// pub enum RepoStoreId {
-// Overlay(OverlayId),
-// Repo(PubKey),
-// }
-
-// impl From<RepoStoreId> for String {
-// fn from(id: RepoStoreId) -> Self {
-// hex::encode(to_vec(&id).unwrap())
-// }
-// }
-
-pub struct RepoStoreInfo<'a> {
- /// RepoStore ID
- id: RepoHash,
- store: &'a dyn KCVStore,
-}
-
-impl<'a> RepoStoreInfo<'a> {
- const PREFIX: u8 = b"r"[0];
-
- // propertie's suffixes
- const KEY: u8 = b"k"[0];
-
- const ALL_PROPERTIES: [u8; 1] = [Self::KEY];
-
- const SUFFIX_FOR_EXIST_CHECK: u8 = Self::KEY;
-
- pub fn open(id: &RepoHash, store: &'a dyn KCVStore) -> Result<RepoStoreInfo<'a>, StorageError> {
- let opening = RepoStoreInfo {
- id: id.clone(),
- store,
- };
- if !opening.exists() {
- return Err(StorageError::NotFound);
- }
- Ok(opening)
- }
- pub fn create(
- id: &RepoHash,
- key: &SymKey,
- store: &'a dyn KCVStore,
- ) -> Result<RepoStoreInfo<'a>, StorageError> {
- let acc = RepoStoreInfo {
- id: id.clone(),
- store,
- };
- if acc.exists() {
- return Err(StorageError::BackendError);
- }
- store.put(Self::PREFIX, &to_vec(&id)?, Some(Self::KEY), to_vec(key)?)?;
- Ok(acc)
- }
- pub fn exists(&self) -> bool {
- self.store
- .get(
- Self::PREFIX,
- &to_vec(&self.id).unwrap(),
- Some(Self::SUFFIX_FOR_EXIST_CHECK),
- )
- .is_ok()
- }
- pub fn id(&self) -> &RepoHash {
- &self.id
- }
- pub fn key(&self) -> Result<SymKey, StorageError> {
- match self
- .store
- .get(Self::PREFIX, &to_vec(&self.id)?, Some(Self::KEY))
- {
- Ok(k) => Ok(from_slice::<SymKey>(&k)?),
- Err(e) => Err(e),
- }
- }
- pub fn del(&self) -> Result<(), StorageError> {
- self.store
- .del_all(Self::PREFIX, &to_vec(&self.id)?, &Self::ALL_PROPERTIES)
- }
-}
diff --git a/ng-broker/src/lib.rs b/ng-broker/src/lib.rs
index e8c77cd..527bfc4 100644
--- a/ng-broker/src/lib.rs
+++ b/ng-broker/src/lib.rs
@@ -1,4 +1,4 @@
-pub mod broker_store;
+pub mod broker_storage;
pub mod server_ws;
diff --git a/ng-broker/src/server.rs b/ng-broker/src/server.rs
deleted file mode 100644
index bc9d391..0000000
--- a/ng-broker/src/server.rs
+++ /dev/null
@@ -1,973 +0,0 @@
-// Copyright (c) 2022-2024 Niko Bonnieure, Par le Peuple, NextGraph.org developers
-// All rights reserved.
-// Licensed under the Apache License, Version 2.0
-// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
-// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
-// at your option. All files in the project carrying such
-// notice may not be copied, modified, or distributed except
-// according to those terms.
-
-//! A Broker server
-
-use std::collections::HashMap;
-use std::collections::HashSet;
-use std::net::SocketAddr;
-use std::pin::Pin;
-use std::sync::Arc;
-use std::sync::RwLock;
-
-use crate::broker_store::account::Account;
-use crate::broker_store::config::Config;
-use crate::broker_store::config::ConfigMode;
-use crate::broker_store::overlay::Overlay;
-use crate::broker_store::peer::Peer;
-use crate::broker_store::repostoreinfo::RepoStoreInfo;
-use async_std::task;
-use futures::future::BoxFuture;
-use futures::future::OptionFuture;
-use futures::FutureExt;
-use futures::Stream;
-use ng_net::actors::*;
-use ng_net::errors::*;
-use ng_net::types::*;
-use ng_repo::log::*;
-use ng_repo::object::Object;
-use ng_repo::store::RepoStore;
-use ng_repo::store::StorageError;
-use ng_repo::types::*;
-use ng_repo::utils::*;
-
-#[derive(Debug, Eq, PartialEq, Clone)]
-pub enum BrokerError {
- CannotStart,
- MismatchedMode,
- OverlayNotFound,
-}
-
-impl From<BrokerError> for ProtocolError {
- fn from(e: BrokerError) -> Self {
- match e {
- BrokerError::CannotStart => ProtocolError::OverlayNotFound,
- BrokerError::OverlayNotFound => ProtocolError::OverlayNotFound,
- _ => ProtocolError::BrokerError,
- }
- }
-}
-
-impl From<ng_repo::store::StorageError> for BrokerError {
- fn from(e: ng_repo::store::StorageError) -> Self {
- match e {
- ng_repo::store::StorageError::InvalidValue => BrokerError::MismatchedMode,
- _ => BrokerError::CannotStart,
- }
- }
-}
-
-#[derive(Debug)]
-enum ProtocolType {
- Start,
- Auth,
- Broker,
- Ext,
- Core,
-}
-
-pub struct ProtocolHandler {
- addr: Option<SocketAddr>,
- broker: Arc<BrokerServer>,
- protocol: ProtocolType,
- auth_protocol: Option<AuthProtocolHandler>,
- broker_protocol: Option<Arc<BrokerProtocolHandler>>,
- ext_protocol: Option<ExtProtocolHandler>,
- r: Option<async_channel::Receiver<Vec<u8>>>,
- s: async_channel::Sender<Vec<u8>>,
-}
-
-impl ProtocolHandler {
- pub fn register(&mut self, addr: SocketAddr) {
- self.addr = Some(addr);
- }
-
- pub fn deregister(&mut self) {
- match &self.protocol {
- ProtocolType::Start => (),
- ProtocolType::Auth => (),
- ProtocolType::Broker => {
- let _ = self
- .broker_protocol
- .as_ref()
- .unwrap()
- .deregister(self.addr.unwrap());
- }
- ProtocolType::Ext => (),
- ProtocolType::Core => (),
- }
- self.addr = None;
- }
-
- pub fn async_frames_receiver(&mut self) -> async_channel::Receiver<Vec<u8>> {
- self.r.take().unwrap()
- }
-
- /// Handle incoming message
- pub async fn handle_incoming(
- &mut self,
- frame: Vec<u8>,
- ) -> (
- Result<Vec<u8>, ProtocolError>,
- OptionFuture<BoxFuture<'static, u16>>,
- ) {
- //log_debug!("SERVER PROTOCOL {:?}", &self.protocol);
- match &self.protocol {
- ProtocolType::Start => {
- let message = serde_bare::from_slice::<StartProtocol>(&frame);
- match message {
- Ok(StartProtocol::Client(b)) => {
- self.protocol = ProtocolType::Auth;
- self.auth_protocol = Some(AuthProtocolHandler::new());
- return (
- self.auth_protocol.as_mut().unwrap().handle_init(b),
- OptionFuture::from(None),
- );
- }
- Ok(StartProtocol::Ext(ext)) => {
- self.protocol = ProtocolType::Ext;
- self.ext_protocol = Some(ExtProtocolHandler {});
- let reply = self.ext_protocol.as_ref().unwrap().handle_incoming(ext);
- return (
- Ok(serde_bare::to_vec(&reply).unwrap()),
- OptionFuture::from(None),
- );
- }
- Err(e) => {
- return (
- Err(ProtocolError::SerializationError),
- OptionFuture::from(None),
- )
- }
- }
- }
- ProtocolType::Auth => {
- let res = self.auth_protocol.as_mut().unwrap().handle_incoming(frame);
- match res.1.await {
- None => {
- // we switch to Broker protocol
- let bp = Arc::new(BrokerProtocolHandler {
- user: self.auth_protocol.as_ref().unwrap().get_user().unwrap(),
- broker: Arc::clone(&self.broker),
- async_frames_sender: self.s.clone(),
- });
- let registration = Arc::clone(&bp).register(self.addr.unwrap());
- match registration {
- Ok(_) => {
- self.protocol = ProtocolType::Broker;
- self.broker_protocol = Some(Arc::clone(&bp));
- self.auth_protocol = None;
- return (res.0, OptionFuture::from(None));
- }
- Err(e) => {
- let val = e.clone() as u16;
- let reply = AuthResult::V0(AuthResultV0 {
- result: val,
- metadata: vec![],
- });
- return (
- Ok(serde_bare::to_vec(&reply).unwrap()),
- OptionFuture::from(Some(async move { val }.boxed())),
- );
- }
- }
- }
- Some(e) => (res.0, OptionFuture::from(Some(async move { e }.boxed()))),
- }
- }
- ProtocolType::Broker => {
- let message = serde_bare::from_slice::<BrokerMessage>(&frame);
- match (message) {
- Ok(message) => {
- let reply = self
- .broker_protocol
- .as_ref()
- .unwrap()
- .handle_incoming(message)
- .await;
- (Ok(serde_bare::to_vec(&reply.0).unwrap()), reply.1)
- }
- Err(e_) => (
- Err(ProtocolError::SerializationError),
- OptionFuture::from(None),
- ),
- }
- }
- ProtocolType::Ext => {
- // Ext protocol is not accepting 2 extrequest in the same connection.
- // closing the connection
- (Err(ProtocolError::InvalidState), OptionFuture::from(None))
- }
- ProtocolType::Core => {
- unimplemented!()
- }
- }
- }
-}
-
-pub struct ExtProtocolHandler {}
-
-impl ExtProtocolHandler {
- pub fn handle_incoming(&self, msg: ExtHello) -> ExtResponse {
- unimplemented!()
- }
-}
-
-pub struct BrokerProtocolHandler {
- broker: Arc<BrokerServer>,
- user: PubKey,
- async_frames_sender: async_channel::Sender<Vec<u8>>,
-}
-use std::{thread, time};
-
-impl BrokerProtocolHandler {
- fn prepare_reply_broker_message(
- res: Result<(), ProtocolError>,
- id: i64,
- padding_size: usize,
- ) -> BrokerMessage {
- let result = match res {
- Ok(_) => 0,
- Err(e) => e.into(),
- };
- let msg = BrokerMessage::V0(BrokerMessageV0 {
- padding: vec![0; padding_size],
- content: BrokerMessageContentV0::BrokerResponse(BrokerResponse::V0(BrokerResponseV0 {
- id,
- result,
- content: BrokerResponseContentV0::EmptyResponse(()),
- })),
- });
- msg
- }
-
- fn prepare_reply_broker_overlay_message(
- res: Result<(), ProtocolError>,
- id: i64,
- overlay: OverlayId,
- block: Option<Block>,
- padding_size: usize,
- ) -> BrokerMessage {
- let result = match res {
- Ok(_) => 0,
- Err(e) => e.into(),
- };
- let content = match block {
- Some(b) => BrokerOverlayResponseContentV0::Block(b),
- None => BrokerOverlayResponseContentV0::EmptyResponse(()),
- };
- let msg = BrokerMessage::V0(BrokerMessageV0 {
- padding: vec![0; padding_size],
- content: BrokerMessageContentV0::BrokerOverlayMessage(BrokerOverlayMessage::V0(
- BrokerOverlayMessageV0 {
- overlay,
- content: BrokerOverlayMessageContentV0::BrokerOverlayResponse(
- BrokerOverlayResponse::V0(BrokerOverlayResponseV0 {
- id,
- result,
- content,
- }),
- ),
- },
- )),
- });
- msg
- }
-
- fn prepare_reply_broker_overlay_message_stream(
- res: Result<Block, ProtocolError>,
- id: i64,
- overlay: OverlayId,
- padding_size: usize,
- ) -> BrokerMessage {
- let result: u16 = match &res {
- Ok(r) => ProtocolError::PartialContent.into(),
- Err(e) => (*e).clone().into(),
- };
- let content = match res {
- Ok(r) => BrokerOverlayResponseContentV0::Block(r),
- Err(_) => BrokerOverlayResponseContentV0::EmptyResponse(()),
- };
- let msg = BrokerMessage::V0(BrokerMessageV0 {
- padding: vec![0; padding_size],
- content: BrokerMessageContentV0::BrokerOverlayMessage(BrokerOverlayMessage::V0(
- BrokerOverlayMessageV0 {
- overlay,
- content: BrokerOverlayMessageContentV0::BrokerOverlayResponse(
- BrokerOverlayResponse::V0(BrokerOverlayResponseV0 {
- id,
- result,
- content,
- }),
- ),
- },
- )),
- });
- msg
- }
-
- async fn send_block_stream_response_to_client(
- &self,
- res: Result<async_channel::Receiver<Block>, ProtocolError>,
- id: i64,
- overlay: OverlayId,
- padding_size: usize,
- ) -> (BrokerMessage, OptionFuture<BoxFuture<'static, u16>>) {
- // return an error or the first block, and setup a spawner for the remaining blocks to be sent.
- let one_reply: (
- Result<Block, ProtocolError>,
- OptionFuture<BoxFuture<'static, u16>>,
- ) = match res {
- Err(e) => (Err(e), OptionFuture::from(None)),
- Ok(stream) => {
- let one = stream
- .recv_blocking()
- .map_err(|e| ProtocolError::EndOfStream);
-
- if one.is_ok() {
- let sender = self.async_frames_sender.clone();
- let a = OptionFuture::from(Some(
- async move {
- while let Ok(next) = stream.recv().await {
- let msg = Self::prepare_reply_broker_overlay_message_stream(
- Ok(next),
- id,
- overlay,
- padding_size,
- );
- let res = sender.send(serde_bare::to_vec(&msg).unwrap()).await;
- if res.is_err() {
- break;
- }
- }
- // sending end of stream
- let msg = Self::prepare_reply_broker_overlay_message_stream(
- Err(ProtocolError::EndOfStream),
- id,
- overlay,
- padding_size,
- );
- let _ = sender.send(serde_bare::to_vec(&msg).unwrap()).await;
- 0
- }
- .boxed(),
- ));
- (one, a)
- } else {
- (one, OptionFuture::from(None))
- }
- }
- };
- return (
- Self::prepare_reply_broker_overlay_message_stream(
- one_reply.0,
- id,
- overlay,
- padding_size,
- ),
- one_reply.1,
- );
- }
-
- pub fn register(self: Arc<Self>, addr: SocketAddr) -> Result<(), ProtocolError> {
- //FIXME: peer_id must be real one
-
- self.broker
- .add_client_peer(PubKey::Ed25519PubKey([0; 32]), Arc::clone(&self))
- }
-
- pub fn deregister(&self, addr: SocketAddr) -> Result<(), ProtocolError> {
- self.broker
- .remove_client_peer(PubKey::Ed25519PubKey([0; 32]));
- Ok(())
- }
-
- pub async fn handle_incoming(
- &self,
- msg: BrokerMessage,
- ) -> (BrokerMessage, OptionFuture<BoxFuture<'static, u16>>) {
- let padding_size = 20; // TODO randomize, if config of server contains padding_max
-
- let id = msg.id();
- let content = msg.content();
- match content {
- BrokerMessageContentV0::BrokerRequest(req) => (
- Self::prepare_reply_broker_message(
- match req.content_v0() {
- BrokerRequestContentV0::AddUser(cmd) => {
- self.broker.add_user(self.user, cmd.user(), cmd.sig())
- }
- BrokerRequestContentV0::DelUser(cmd) => {
- self.broker.del_user(self.user, cmd.user(), cmd.sig())
- }
- BrokerRequestContentV0::AddClient(cmd) => {
- self.broker.add_client(self.user, cmd.client(), cmd.sig())
- }
- BrokerRequestContentV0::DelClient(cmd) => {
- self.broker.del_client(self.user, cmd.client(), cmd.sig())
- }
- },
- id,
- padding_size,
- ),
- OptionFuture::from(None),
- ),
- BrokerMessageContentV0::BrokerResponse(res) => (
- Self::prepare_reply_broker_message(
- Err(ProtocolError::InvalidState),
- id,
- padding_size,
- ),
- OptionFuture::from(None),
- ),
- BrokerMessageContentV0::BrokerOverlayMessage(omsg) => {
- let overlay = omsg.overlay_id();
- let block = None;
- let mut res = Err(ProtocolError::InvalidState);
-
- if omsg.is_request() {
- match omsg.overlay_request().content_v0() {
- BrokerOverlayRequestContentV0::OverlayConnect(_) => {
- res = self.broker.connect_overlay(self.user, overlay)
- }
- BrokerOverlayRequestContentV0::OverlayJoin(j) => {
- res = self.broker.join_overlay(
- self.user,
- overlay,
- j.repo_pubkey(),
- j.secret(),
- j.peers(),
- )
- }
- BrokerOverlayRequestContentV0::ObjectDel(op) => {
- res = self.broker.del_object(self.user, overlay, op.id())
- }
- BrokerOverlayRequestContentV0::ObjectPin(op) => {
- res = self.broker.pin_object(self.user, overlay, op.id())
- }
- BrokerOverlayRequestContentV0::ObjectUnpin(op) => {
- res = self.broker.unpin_object(self.user, overlay, op.id())
- }
- BrokerOverlayRequestContentV0::BlockPut(b) => {
- res = self.broker.put_block(self.user, overlay, b.block())
- }
- BrokerOverlayRequestContentV0::BranchSyncReq(b) => {
- let res = self.broker.sync_branch(
- self.user,
- &overlay,
- b.heads(),
- b.known_heads(),
- b.known_commits(),
- );
- return self
- .send_block_stream_response_to_client(
- res,
- id,
- overlay,
- padding_size,
- )
- .await;
- }
- BrokerOverlayRequestContentV0::BlockGet(b) => {
- let res = self.broker.get_block(
- self.user,
- overlay,
- b.id(),
- b.include_children(),
- b.topic(),
- );
- return self
- .send_block_stream_response_to_client(
- res,
- id,
- overlay,
- padding_size,
- )
- .await;
- }
- _ => {}
- }
- }
-
- (
- Self::prepare_reply_broker_overlay_message(
- res,
- id,
- overlay,
- block,
- padding_size,
- ),
- OptionFuture::from(None),
- )
- }
- }
- }
-}
-
-pub enum PeerConnection {
- CORE(IP),
- CLIENT(Arc<BrokerProtocolHandler>),
- NONE,
-}
-
-pub struct BrokerPeerInfo {
- lastPeerAdvert: Option<PeerAdvert>,
- connected: PeerConnection,
-}
-
-const REPO_STORES_SUBDIR: &str = "repos";
-
-pub struct BrokerServer {
- store: LmdbKCVStore,
- mode: ConfigMode,
- repo_stores: Arc<RwLock<HashMap<RepoHash, LmdbRepoStore>>>,
- // only used in ConfigMode::Local
- // try to change it to this version below in order to avoid double hashmap lookup in local mode. but hard to do...
- //overlayid_to_repostore: HashMap<RepoStoreId, &'a LmdbRepoStore>,
- //overlayid_to_repostore: Arc<RwLock<HashMap<OverlayId, RepoStoreId>>>,
- peers: RwLock<HashMap<DirectPeerId, BrokerPeerInfo>>,
- //local_connections:
-}
-
-impl BrokerServer {
- pub fn add_client_peer(
- &self,
- peer_id: DirectPeerId,
- bph: Arc<BrokerProtocolHandler>,
- ) -> Result<(), ProtocolError> {
- let mut writer = self.peers.write().expect("write peers hashmap");
- let bpi = BrokerPeerInfo {
- lastPeerAdvert: None, //TODO: load from store
- connected: PeerConnection::CLIENT(bph),
- };
- if !writer.get(&peer_id).is_none() {
- return Err(ProtocolError::PeerAlreadyConnected);
- }
- writer.insert(peer_id.clone(), bpi);
- Ok(())
- }
-
- pub fn remove_client_peer(&self, peer_id: DirectPeerId) {
- let mut writer = self.peers.write().expect("write peers hashmap");
- writer.remove(&peer_id);
- }
-
- pub fn new(store: LmdbKCVStore, mode: ConfigMode) -> Result<BrokerServer, BrokerError> {
- let mut configmode: ConfigMode;
- {
- let config = Config::get_or_create(&mode, &store)?;
- configmode = config.mode()?;
- }
- Ok(BrokerServer {
- store,
- mode: configmode,
- repo_stores: Arc::new(RwLock::new(HashMap::new())),
- //overlayid_to_repostore: Arc::new(RwLock::new(HashMap::new())),
- peers: RwLock::new(HashMap::new()),
- })
- }
-
- fn open_or_create_repostore<F, R>(&self, repo_hash: RepoHash, f: F) -> Result<R, ProtocolError>
- where
- F: FnOnce(&LmdbRepoStore) -> Result<R, ProtocolError>,
- {
- // first let's find it in the KCVStore.repostoreinfo table in order to get the encryption key
- let info = RepoStoreInfo::open(&repo_hash, &self.store)
- .map_err(|e| BrokerError::OverlayNotFound)?;
- let key = info.key()?;
- let mut path = self.store.path();
- path.push(REPO_STORES_SUBDIR);
- path.push::<String>(repo_hash.clone().into());
- std::fs::create_dir_all(path.clone()).map_err(|_e| ProtocolError::IoError)?;
- println!("path for repo store: {}", path.to_str().unwrap());
- let repo = LmdbRepoStore::open(&path, *key.slice());
- let mut writer = self.repo_stores.write().expect("write repo_store hashmap");
- writer.insert(repo_hash.clone(), repo);
- f(writer.get(&repo_hash).unwrap())
- }
-
- fn get_repostore_from_overlay_id<F, R>(
- &self,
- overlay_id: &OverlayId,
- f: F,
- ) -> Result<R, ProtocolError>
- where
- F: FnOnce(&LmdbRepoStore) -> Result<R, ProtocolError>,
- {
- //FIXME: the whole purpose of get_repostore_from_overlay_id is gone. review all of it
- let repostore_id = *overlay_id;
- {
- let reader = self.repo_stores.read().expect("read repo_store hashmap");
- let rep = reader.get(&repostore_id);
- if let Some(repo) = rep {
- return f(repo);
- }
- }
- // we need to open/create it
- // TODO: last_access
- return self.open_or_create_repostore(repostore_id, |repo| f(repo));
-
- // } else {
- // // it is ConfigMode::Local
- // {
- // let reader = self
- // .overlayid_to_repostore
- // .read()
- // .expect("read overlayid_to_repostore hashmap");
- // match reader.get(&overlay_id) {
- // Some(repostoreid) => {
- // let reader = self.repo_stores.read().expect("read repo_store hashmap");
- // match reader.get(repostoreid) {
- // Some(repo) => return f(repo),
- // None => return Err(ProtocolError::BrokerError),
- // }
- // }
- // None => {}
- // };
- // }
-
- // // we need to open/create it
- // // first let's find it in the KCVStore.overlay table to retrieve its repo_pubkey
- // log_debug!("searching for overlayId {}", overlay_id);
- // let overlay = Overlay::open(overlay_id, &self.store)?;
- // log_debug!("found overlayId {}", overlay_id);
- // let repo_id = overlay.repo()?;
- // let repostore_id = RepoStoreId::Repo(repo_id);
- // let mut writer = self
- // .overlayid_to_repostore
- // .write()
- // .expect("write overlayid_to_repostore hashmap");
- // writer.insert(*overlay_id, repostore_id.clone());
- // // now opening/creating the RepoStore
- // // TODO: last_access
- // return self.open_or_create_repostore(repostore_id, |repo| f(repo));
- // }
- }
-
- pub fn local_connection(&mut self, user: PubKey) -> BrokerConnectionLocal {
- BrokerConnectionLocal::new(self, user)
- }
-
- pub fn protocol_handler(self: Arc<Self>) -> ProtocolHandler {
- let (s, r) = async_channel::unbounded::<Vec<u8>>();
- return ProtocolHandler {
- addr: None,
- broker: Arc::clone(&self),
- protocol: ProtocolType::Start,
- auth_protocol: None,
- broker_protocol: None,
- ext_protocol: None,
- r: Some(r),
- s,
- };
- }
-
- pub fn add_user(
- &self,
- admin_user: PubKey,
- user_id: PubKey,
- sig: Sig,
- ) -> Result<(), ProtocolError> {
- log_debug!("ADDING USER {}", user_id);
- // TODO add is_admin boolean
- // TODO check that admin_user is indeed an admin
-
- // verify signature
- let op_content = AddUserContentV0 { user: user_id };
- let _ = verify(&serde_bare::to_vec(&op_content).unwrap(), sig, admin_user)?;
-
- // check user_id is not already present
- let account = Account::open(&user_id, &self.store);
- if account.is_ok() {
- Err(ProtocolError::UserAlreadyExists)
- }
- // if not, add to store
- else {
- let _ = Account::create(&user_id, false, &self.store)?;
- Ok(())
- }
- }
-
- pub fn del_user(
- &self,
- admin_user: PubKey,
- user_id: PubKey,
- sig: Sig,
- ) -> Result<(), ProtocolError> {
- // TODO implement del_user
- Ok(())
- }
- pub fn add_client(
- &self,
- user: PubKey,
- client_id: PubKey,
- sig: Sig,
- ) -> Result<(), ProtocolError> {
- // TODO implement add_client
- Ok(())
- }
-
- pub fn del_client(
- &self,
- user: PubKey,
- client_id: PubKey,
- sig: Sig,
- ) -> Result<(), ProtocolError> {
- // TODO implement del_client
- Ok(())
- }
-
- pub fn connect_overlay(&self, user: PubKey, overlay: OverlayId) -> Result<(), ProtocolError> {
- // TODO check that the broker has already joined this overlay. if not, send OverlayNotJoined
- Err(ProtocolError::OverlayNotJoined)
- }
-
- pub fn del_object(
- &self,
- user: PubKey,
- overlay: Digest,
- id: ObjectId,
- ) -> Result<(), ProtocolError> {
- self.get_repostore_from_overlay_id(&overlay, |store| {
- // TODO, only admin users can delete on a store on this broker
- let obj = Object::load(id, None, store);
- if obj.is_err() {
- return Err(ProtocolError::NotFound);
- }
- let o = obj.ok().unwrap();
- let mut deduplicated: HashSet<ObjectId> = HashSet::new();
- for block in o.blocks() {
- let id = block.id();
- if deduplicated.get(&id).is_none() {
- store.del(&id)?;
- deduplicated.insert(id);
- }
- }
- Ok(())
- })
- }
-
- pub fn pin_object(
- &self,
- user: PubKey,
- overlay: OverlayId,
- id: ObjectId,
- ) -> Result<(), ProtocolError> {
- self.get_repostore_from_overlay_id(&overlay, |store| {
- // TODO, store the user who pins, and manage reference counting on how many users pin/unpin
- let obj = Object::load(id, None, store);
- if obj.is_err() {
- return Err(ProtocolError::NotFound);
- }
- let o = obj.ok().unwrap();
- let mut deduplicated: HashSet<ObjectId> = HashSet::new();
- for block in o.blocks() {
- let id = block.id();
- if deduplicated.get(&id).is_none() {
- store.pin(&id)?;
- deduplicated.insert(id);
- }
- }
- Ok(())
- })
- }
-
- pub fn unpin_object(
- &self,
- user: PubKey,
- overlay: OverlayId,
- id: ObjectId,
- ) -> Result<(), ProtocolError> {
- self.get_repostore_from_overlay_id(&overlay, |store| {
- // TODO, store the user who pins, and manage reference counting on how many users pin/unpin
- let obj = Object::load(id, None, store);
- if obj.is_err() {
- return Err(ProtocolError::NotFound);
- }
- let o = obj.ok().unwrap();
- let mut deduplicated: HashSet<ObjectId> = HashSet::new();
- for block in o.blocks() {
- let id = block.id();
- if deduplicated.get(&id).is_none() {
- store.unpin(&id)?;
- deduplicated.insert(id);
- }
- }
- Ok(())
- })
- }
-
- pub fn copy_object(
- &self,
- user: PubKey,
- overlay: OverlayId,
- id: ObjectId,
- expiry: Option<Timestamp>,
- ) -> Result<ObjectId, ProtocolError> {
- // self.get_repostore_from_overlay_id(&overlay, |store| {
- // //let obj = Object::from_store(id, None, store);
- // //Ok(Object::copy(id, expiry, store)?)
- // });
- todo!();
- }
-
- pub fn put_block(
- &self,
- user: PubKey,
- overlay: OverlayId,
- block: &Block,
- ) -> Result<(), ProtocolError> {
- self.get_repostore_from_overlay_id(&overlay, |store| {
- let _ = store.put(block)?;
- Ok(())
- })
- }
-
- pub fn get_block(
- &self,
- user: PubKey,
- overlay: OverlayId,
- id: BlockId,
- include_children: bool,
- topic: Option<PubKey>,
- ) -> Result<async_channel::Receiver<Block>, ProtocolError> {
- self.get_repostore_from_overlay_id(&overlay, |store| {
- let (s, r) = async_channel::unbounded::<Block>();
- if !include_children {
- let block = store.get(&id)?;
- s.send_blocking(block)
- .map_err(|_e| ProtocolError::IoError)?;
- Ok(r)
- } else {
- let obj = Object::load(id, None, store);
- // TODO return partial blocks when some are missing ?
- if obj.is_err() {
- //&& obj.err().unwrap().len() == 1 && obj.err().unwrap()[0] == id {
- return Err(ProtocolError::NotFound);
- }
- // TODO use a task to send non blocking (streaming)
- let o = obj.ok().unwrap();
- //log_debug!("{} BLOCKS ", o.blocks().len());
- let mut deduplicated: HashSet<BlockId> = HashSet::new();
- for block in o.blocks() {
- let id = block.id();
- if deduplicated.get(&id).is_none() {
- s.send_blocking(block.clone())
- .map_err(|_e| ProtocolError::IoError)?;
- deduplicated.insert(id);
- }
- }
- Ok(r)
- }
- })
- }
-
- pub fn sync_branch(
- &self,
- user: PubKey,
- overlay: &OverlayId,
- heads: &Vec<ObjectId>,
- known_heads: &Vec<ObjectId>,
- known_commits: &BloomFilter,
- ) -> Result<async_channel::Receiver<Block>, ProtocolError> {
- //log_debug!("heads {:?}", heads);
- //log_debug!("known_heads {:?}", known_heads);
- //log_debug!("known_commits {:?}", known_commits);
-
- self.get_repostore_from_overlay_id(&overlay, |store| {
- let (s, r) = async_channel::unbounded::<Block>();
-
- let res = Branch::sync_req(heads, known_heads, known_commits, store)
- .map_err(|e| ProtocolError::ObjectParseError)?;
-
- // todo, use a task to send non blocking (streaming)
- log_debug!("SYNCING {} COMMITS", res.len());
-
- let mut deduplicated: HashSet<BlockId> = HashSet::new();
-
- for objectid in res {
- let object = Object::load(objectid, None, store)?;
-
- for block in object.blocks() {
- let id = block.id();
- if deduplicated.get(&id).is_none() {
- s.send_blocking(block.clone())
- .map_err(|_e| ProtocolError::IoError)?;
- deduplicated.insert(id);
- }
- }
- }
- Ok(r)
- })
- }
-
- // fn compute_repostore_id(&self, overlay: OverlayId, repo_id: Option<PubKey>) -> RepoStoreId {
- // match self.mode {
- // ConfigMode::Core => RepoStoreId::Overlay(overlay),
- // ConfigMode::Local => RepoStoreId::Repo(repo_id.unwrap()),
- // }
- // }
-
- pub fn join_overlay(
- &self,
- user: PubKey,
- overlay_id: OverlayId,
- repo_id: Option<PubKey>,
- secret: SymKey,
- peers: &Vec<PeerAdvert>,
- ) -> Result<(), ProtocolError> {
- // check if this overlay already exists
- //log_debug!("SEARCHING OVERLAY");
- let overlay_res = Overlay::open(&overlay_id, &self.store);
- let overlay = match overlay_res {
- Err(StorageError::NotFound) => {
- // we have to add it
- if self.mode == ConfigMode::Local && repo_id.is_none() {
- return Err(ProtocolError::RepoIdRequired);
- }
- let over = Overlay::create(
- &overlay_id,
- &secret,
- if self.mode == ConfigMode::Local {
- repo_id
- } else {
- None
- },
- &self.store,
- )?;
- // we need to add an encryption key for the repostore.
- let mut random_buf = [0u8; 32];
- getrandom::getrandom(&mut random_buf).unwrap();
- let key = SymKey::ChaCha20Key(random_buf);
-
- let _ = RepoStoreInfo::create(&overlay_id, &key, &self.store)?; // TODO in case of error, delete the previously created Overlay
- //log_debug!("KEY ADDED");
- over
- }
- Err(e) => return Err(e.into()),
- Ok(overlay) => overlay,
- };
- //log_debug!("OVERLAY FOUND");
- // add the peers to the overlay
- for advert in peers {
- Peer::update_or_create(advert, &self.store)?;
- overlay.add_peer(&advert.peer())?;
- }
- //log_debug!("PEERS ADDED");
-
- // now adding the overlay_id to the account
- let account = Account::open(&user, &self.store)?; // TODO in case of error, delete the previously created Overlay
- account.add_overlay(&overlay_id)?;
- //log_debug!("USER <-> OVERLAY");
-
- //TODO: connect to peers
-
- Ok(())
- }
-}
diff --git a/ng-broker/src/server_storage.rs b/ng-broker/src/server_storage.rs
index bcb0cb0..73caac1 100644
--- a/ng-broker/src/server_storage.rs
+++ b/ng-broker/src/server_storage.rs
@@ -15,18 +15,18 @@ use std::io::Write;
use std::path::{Path, PathBuf};
use std::sync::Mutex;
-use crate::broker_store::account::Account;
-use crate::broker_store::invitation::Invitation;
-use crate::broker_store::wallet::Wallet;
+use crate::broker_storage::account::Account;
+use crate::broker_storage::invitation::Invitation;
+use crate::broker_storage::wallet::Wallet;
use crate::types::*;
use ng_net::errors::{ProtocolError, ServerError};
use ng_net::server_storage::*;
use ng_net::types::{BootstrapContentV0, InvitationCode, InvitationV0};
-use ng_repo::kcv_store::KCVStore;
+use ng_repo::errors::StorageError;
+use ng_repo::kcv_storage::KCVStore;
use ng_repo::log::*;
-use ng_repo::store::StorageError;
use ng_repo::types::{PeerId, PubKey, SymKey};
-use ng_stores_rocksdb::kcv_store::RocksdbKCVStore;
+use ng_storage_rocksdb::kcv_storage::RocksdbKCVStore;
pub struct RocksdbServerStorage {
wallet_storage: RocksdbKCVStore,
diff --git a/ng-client-ws/README.md b/ng-client-ws/README.md
index 0eb5805..192a0e8 100644
--- a/ng-client-ws/README.md
+++ b/ng-client-ws/README.md
@@ -48,7 +48,7 @@ additional terms or conditions.
NextGraph received funding through the [NGI Assure Fund](https://nlnet.nl/project/NextGraph/index.html), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's [Next Generation Internet](https://ngi.eu/) programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 957073.
-[rustc-image]: https://img.shields.io/badge/rustc-1.64+-blue.svg
+[rustc-image]: https://img.shields.io/badge/rustc-1.74+-blue.svg
[license-image]: https://img.shields.io/badge/license-Apache2.0-blue.svg
[license-link]: https://git.nextgraph.org/NextGraph/nextgraph-rs/raw/branch/master/LICENSE-APACHE2
[license-image2]: https://img.shields.io/badge/license-MIT-blue.svg
diff --git a/ng-net/Cargo.toml b/ng-net/Cargo.toml
index e115220..c239d8b 100644
--- a/ng-net/Cargo.toml
+++ b/ng-net/Cargo.toml
@@ -48,4 +48,4 @@ features = ["js"]
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
getrandom = "0.2.7"
default-net = { git = "https://git.nextgraph.org/NextGraph/default-net.git" }
-# ng-stores-rocksdb = { path = "../ng-stores-rocksdb", version = "0.1.0" }
+# ng-storage-rocksdb = { path = "../ng-storage-rocksdb", version = "0.1.0" }
diff --git a/ng-net/README.md b/ng-net/README.md
index 4fabb6a..adf3a9f 100644
--- a/ng-net/README.md
+++ b/ng-net/README.md
@@ -48,7 +48,7 @@ additional terms or conditions.
NextGraph received funding through the [NGI Assure Fund](https://nlnet.nl/project/NextGraph/index.html), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's [Next Generation Internet](https://ngi.eu/) programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 957073.
-[rustc-image]: https://img.shields.io/badge/rustc-1.64+-blue.svg
+[rustc-image]: https://img.shields.io/badge/rustc-1.74+-blue.svg
[license-image]: https://img.shields.io/badge/license-Apache2.0-blue.svg
[license-link]: https://git.nextgraph.org/NextGraph/nextgraph-rs/raw/branch/master/LICENSE-APACHE2
[license-image2]: https://img.shields.io/badge/license-MIT-blue.svg
diff --git a/ng-net/src/broker.rs b/ng-net/src/broker.rs
index 32a38f9..e26229c 100644
--- a/ng-net/src/broker.rs
+++ b/ng-net/src/broker.rs
@@ -22,11 +22,11 @@ use async_std::sync::{Arc, RwLock};
use either::Either;
use futures::channel::mpsc;
use futures::SinkExt;
+use ng_repo::block_storage::HashMapBlockStorage;
use ng_repo::errors::NgError;
use ng_repo::errors::ObjectParseError;
use ng_repo::log::*;
use ng_repo::object::Object;
-use ng_repo::store::HashMapRepoStore;
use ng_repo::types::*;
use ng_repo::utils::generate_keypair;
use once_cell::sync::Lazy;
@@ -88,9 +88,6 @@ pub struct Broker<'a> {
tauri_streams: HashMap<String, Sender<Commit>>,
disconnections_sender: Sender<String>,
disconnections_receiver: Option<Receiver<String>>,
- //last_seq_function: Option<Box<LastSeqFn>>,
- //base_path: Option<PathBuf>,
- //in_memory: bool,
local_broker: Option<Box<dyn ILocalBroker + Send + Sync + 'a>>,
}
@@ -346,7 +343,7 @@ impl<'a> Broker<'a> {
let blockstream = self
.get_block_from_store_with_block_id(nuri, obj_ref.id, true)
.await?;
- let store = Box::new(HashMapRepoStore::from_block_stream(blockstream).await);
+ let store = Box::new(HashMapBlockStorage::from_block_stream(blockstream).await);
Object::load(obj_ref.id, Some(obj_ref.key), &store)
.map_err(|e| match e {
@@ -451,7 +448,7 @@ impl<'a> Broker<'a> {
// #[cfg(not(target_arch = "wasm32"))]
// pub fn test_storage(&self, path: PathBuf) {
- // use ng_stores_rocksdb::kcv_store::RocksdbKCVStore;
+ // use ng_storage_rocksdb::kcv_store::RocksdbKCVStore;
// let key: [u8; 32] = [0; 32];
// let test_storage = RocksdbKCVStore::open(&path, key);
diff --git a/ng-net/src/broker_connection.rs b/ng-net/src/broker_connection.rs
index 5970b14..883a3fb 100644
--- a/ng-net/src/broker_connection.rs
+++ b/ng-net/src/broker_connection.rs
@@ -266,7 +266,7 @@ where
topic: Option<PubKey>,
) -> Result<Object, ProtocolError> {
let mut blockstream = self.get_block(id, true, topic).await?;
- let mut store = HashMapRepoStore::new();
+ let mut store = HashMapBlockStorage::new();
while let Some(block) = blockstream.next().await {
store.put(&block).unwrap();
}
diff --git a/ng-net/src/errors.rs b/ng-net/src/errors.rs
index 3bcefe2..9659dbe 100644
--- a/ng-net/src/errors.rs
+++ b/ng-net/src/errors.rs
@@ -8,8 +8,7 @@
// according to those terms.
use core::fmt;
-use ng_repo::errors::ObjectParseError;
-use ng_repo::store::StorageError;
+use ng_repo::errors::{ObjectParseError, StorageError};
use num_enum::IntoPrimitive;
use num_enum::TryFromPrimitive;
use std::convert::From;
diff --git a/ng-net/src/types.rs b/ng-net/src/types.rs
index e7d75ee..ae0209b 100644
--- a/ng-net/src/types.rs
+++ b/ng-net/src/types.rs
@@ -154,6 +154,11 @@ pub struct BrokerServerV0 {
pub peer_id: PubKey,
}
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]
+pub enum BrokerServer {
+ V0(BrokerServerV0),
+}
+
impl BrokerServerV0 {
pub fn new_localhost(peer_id: PubKey) -> Self {
BrokerServerV0 {
@@ -2516,6 +2521,7 @@ impl AdminResponse {
///
/// When client will disconnect, the subscriptions and publisherAdvert of the topics will be removed,
/// except if a PinRepo occurred before or after the OpenRepo
+/// replied with a RepoOpened
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct OpenRepoV0 {
/// Repo Hash
@@ -2536,6 +2542,7 @@ pub struct OpenRepoV0 {
pub allowed_peers: Vec<DirectPeerId>,
/// Maximum number of peers to connect to for this overlay (only valid for an inner (RW/WO) overlay)
+ /// 0 means automatic/unlimited
pub max_peer_count: u16,
/// list of topics that should be subscribed to
@@ -2636,7 +2643,7 @@ pub struct RefreshPinRepoV0 {
/// The userId of banned user is revealed to the local broker where it was attached, which is a breach of privacy deemed acceptable
/// as only a broker that already knew the userid will enforce it, and
/// that broker might be interested to know that the offending user was banned from a repo, as only malicious users are banned.
- /// The broker might also discard this information, and just proceeed with the flush without much ado.
+ /// The broker might also discard this information, and just proceed with the flush without much ado.
/// Of course, if the broker is controlled by the malicious user, it might not proceed with the ban/flush. But who cares. That broker will keep old data forever, but it is a malicious broker anyway.
pub flush_topics: Vec<(TopicId, Sig)>,
}
@@ -2731,6 +2738,8 @@ impl RepoPinStatus {
}
/// Request subscription to a `Topic` of an already opened or pinned Repo
+///
+/// replied with a list of TopicSubRes containing the current heads that should be used to do a TopicSync
#[derive(Clone, Copy, Debug, Serialize, Deserialize)]
pub struct TopicSubV0 {
/// Topic to subscribe
@@ -3005,11 +3014,29 @@ impl BlocksFound {
}
}
+/// Topic subscription response V0
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct TopicSubResV0 {
+ /// Topic subscribed
+ pub topic: PubKey,
+ pub known_heads: Vec<ObjectId>,
+}
+
+/// Topic subscription response
+///
+/// it is a stream of blocks and or events.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum TopicSubRes {
+ V0(TopicSubResV0),
+}
+
/// Content of `ClientResponseV0`
#[derive(Clone, Debug, Serialize, Deserialize)]
pub enum ClientResponseContentV0 {
EmptyResponse,
Block(Block),
+ RepoOpened(Vec<TopicSubRes>),
+ TopicSubRes(TopicSubRes),
TopicSyncRes(TopicSyncRes),
BlocksFound(BlocksFound),
RepoPinStatus(RepoPinStatus),
diff --git a/ng-repo/README.md b/ng-repo/README.md
index 38b8a74..5940bb6 100644
--- a/ng-repo/README.md
+++ b/ng-repo/README.md
@@ -48,7 +48,7 @@ additional terms or conditions.
NextGraph received funding through the [NGI Assure Fund](https://nlnet.nl/project/NextGraph/index.html), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's [Next Generation Internet](https://ngi.eu/) programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 957073.
-[rustc-image]: https://img.shields.io/badge/rustc-1.64+-blue.svg
+[rustc-image]: https://img.shields.io/badge/rustc-1.74+-blue.svg
[license-image]: https://img.shields.io/badge/license-Apache2.0-blue.svg
[license-link]: https://git.nextgraph.org/NextGraph/nextgraph-rs/raw/branch/master/LICENSE-APACHE2
[license-image2]: https://img.shields.io/badge/license-MIT-blue.svg
diff --git a/ng-repo/src/store.rs b/ng-repo/src/block_storage.rs
similarity index 85%
rename from ng-repo/src/store.rs
rename to ng-repo/src/block_storage.rs
index 94d0770..51d7d63 100644
--- a/ng-repo/src/store.rs
+++ b/ng-repo/src/block_storage.rs
@@ -11,6 +11,7 @@
use futures::StreamExt;
+use crate::errors::*;
use crate::types::*;
use crate::utils::Receiver;
use std::sync::RwLock;
@@ -20,7 +21,7 @@ use std::{
mem::size_of_val,
};
-pub trait RepoStore: Send + Sync {
+pub trait BlockStorage: Send + Sync {
/// Load a block from the storage.
fn get(&self, id: &BlockId) -> Result<Block, StorageError>;
@@ -34,29 +35,6 @@ pub trait RepoStore: Send + Sync {
fn len(&self) -> Result<usize, StorageError>;
}
-#[derive(Debug, PartialEq, Eq, Clone)]
-pub enum StorageError {
- NotFound,
- InvalidValue,
- DifferentValue,
- BackendError,
- SerializationError,
- AlreadyExists,
- DataCorruption,
-}
-
-impl core::fmt::Display for StorageError {
- fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
- write!(f, "{:?}", self)
- }
-}
-
-impl From<serde_bare::error::Error> for StorageError {
- fn from(_e: serde_bare::error::Error) -> Self {
- StorageError::SerializationError
- }
-}
-
/* LMDB values:
const MIN_SIZE: usize = 4072;
@@ -102,13 +80,13 @@ pub const fn store_max_value_size() -> usize {
}
/// Store with a HashMap backend
-pub struct HashMapRepoStore {
+pub struct HashMapBlockStorage {
blocks: RwLock<HashMap<BlockId, Block>>,
}
-impl HashMapRepoStore {
- pub fn new() -> HashMapRepoStore {
- HashMapRepoStore {
+impl HashMapBlockStorage {
+ pub fn new() -> HashMapBlockStorage {
+ HashMapBlockStorage {
blocks: RwLock::new(HashMap::new()),
}
}
@@ -135,7 +113,7 @@ impl HashMapRepoStore {
}
}
-impl RepoStore for HashMapRepoStore {
+impl BlockStorage for HashMapBlockStorage {
fn get(&self, id: &BlockId) -> Result<Block, StorageError> {
match self.blocks.read().unwrap().get(id) {
Some(block) => {
diff --git a/ng-repo/src/branch.rs b/ng-repo/src/branch.rs
index cd6f0b2..079f4d4 100644
--- a/ng-repo/src/branch.rs
+++ b/ng-repo/src/branch.rs
@@ -14,9 +14,9 @@ use zeroize::{Zeroize, ZeroizeOnDrop};
// use fastbloom_rs::{BloomFilter as Filter, Membership};
+use crate::block_storage::*;
use crate::errors::*;
use crate::object::*;
-use crate::store::*;
use crate::types::*;
use crate::utils::encrypt_in_place;
@@ -33,6 +33,7 @@ impl BranchV0 {
let topic = topic_priv.to_pub();
BranchV0 {
id,
+ content_type: BranchContentType::None,
repo,
root_branch_readcap_id,
topic,
@@ -96,18 +97,18 @@ impl Branch {
target_heads: &[ObjectId],
known_heads: &[ObjectId],
//their_filter: &BloomFilter,
- store: &Box<impl RepoStore + ?Sized>,
+ store: &Box<impl BlockStorage + ?Sized>,
) -> Result<Vec<ObjectId>, ObjectParseError> {
//log_debug!(">> sync_req");
//log_debug!(" target_heads: {:?}", target_heads);
//log_debug!(" known_heads: {:?}", known_heads);
- /// Load causal past of a Commit `cobj` in a `Branch` from the `RepoStore`,
+ /// Load causal past of a Commit `cobj` in a `Branch` from the `BlockStorage`,
/// and collect in `visited` the ObjectIds encountered on the way, stopping at any commit already belonging to `theirs` or the root of DAG.
/// optionally collecting the missing objects/blocks that couldn't be found locally on the way
fn load_causal_past(
cobj: &Object,
- store: &Box<impl RepoStore + ?Sized>,
+ store: &Box<impl BlockStorage + ?Sized>,
theirs: &HashSet<ObjectId>,
visited: &mut HashSet<ObjectId>,
missing: &mut Option<&mut HashSet<ObjectId>>,
@@ -179,16 +180,16 @@ mod test {
//use fastbloom_rs::{BloomFilter as Filter, FilterBuilder, Membership};
struct Test<'a> {
- storage: Box<dyn RepoStore + Send + Sync + 'a>,
+ storage: Box<dyn BlockStorage + Send + Sync + 'a>,
}
impl<'a> Test<'a> {
- fn storage(s: impl RepoStore + 'a) -> Self {
+ fn storage(s: impl BlockStorage + 'a) -> Self {
Test {
storage: Box::new(s),
}
}
- fn s(&self) -> &Box<dyn RepoStore + Send + Sync + 'a> {
+ fn s(&self) -> &Box<dyn BlockStorage + Send + Sync + 'a> {
&self.storage
}
}
@@ -207,7 +208,7 @@ mod test {
header: Option<CommitHeader>,
store_pubkey: &StoreRepo,
store_secret: &ReadCapSecret,
- store: &Box<impl RepoStore + ?Sized>,
+ store: &Box<impl BlockStorage + ?Sized>,
) -> ObjectRef {
let max_object_size = 4000;
let mut obj = Object::new(
@@ -234,7 +235,7 @@ mod test {
body_ref: ObjectRef,
store_pubkey: &StoreRepo,
store_secret: &ReadCapSecret,
- store: &Box<impl RepoStore + ?Sized>,
+ store: &Box<impl BlockStorage + ?Sized>,
) -> ObjectRef {
let header = CommitHeader::new_with_deps_and_acks(
deps.iter().map(|r| r.id).collect(),
@@ -280,7 +281,7 @@ mod test {
branch: BranchV0,
store_pubkey: &StoreRepo,
store_secret: &ReadCapSecret,
- store: &Box<impl RepoStore + ?Sized>,
+ store: &Box<impl BlockStorage + ?Sized>,
) -> ObjectRef {
let body: CommitBodyV0 = CommitBodyV0::Branch(Branch::V0(branch));
//log_debug!("body: {:?}", body);
@@ -297,7 +298,7 @@ mod test {
header: Option<CommitHeader>,
store_pubkey: &StoreRepo,
store_secret: &ReadCapSecret,
- store: &Box<impl RepoStore + ?Sized>,
+ store: &Box<impl BlockStorage + ?Sized>,
) -> ObjectRef {
let content = [7u8; 777].to_vec();
let body = CommitBodyV0::AsyncTransaction(Transaction::V0(content));
@@ -311,7 +312,7 @@ mod test {
)
}
- let hashmap_storage = HashMapRepoStore::new();
+ let hashmap_storage = HashMapBlockStorage::new();
let t = Test::storage(hashmap_storage);
// repo
diff --git a/ng-repo/src/commit.rs b/ng-repo/src/commit.rs
index 5a094c1..dc2ff5e 100644
--- a/ng-repo/src/commit.rs
+++ b/ng-repo/src/commit.rs
@@ -15,11 +15,11 @@ use once_cell::sync::OnceCell;
use crate::errors::NgError;
+use crate::block_storage::*;
use crate::errors::*;
use crate::log::*;
use crate::object::*;
use crate::repo::Repo;
-use crate::store::*;
use crate::types::*;
use crate::utils::*;
use std::collections::HashSet;
@@ -129,7 +129,7 @@ impl CommitV0 {
block_size: usize,
store_pubkey: &StoreRepo,
store_secret: &ReadCapSecret,
- store: &Box<impl RepoStore + ?Sized>,
+ store: &Box<impl BlockStorage + ?Sized>,
) -> Result<ObjectRef, StorageError> {
if self.id.is_some() && self.key.is_some() {
return Ok(ObjectRef::from_id_key(
@@ -224,7 +224,7 @@ impl Commit {
body: CommitBody,
store_pubkey: &StoreRepo,
store_secret: &ReadCapSecret,
- storage: &Box<impl RepoStore + ?Sized>,
+ storage: &Box<impl BlockStorage + ?Sized>,
) -> Result<Commit, NgError> {
Self::new_with_body_and_save(
author_privkey,
@@ -263,7 +263,7 @@ impl Commit {
block_size: usize,
store_pubkey: &StoreRepo,
store_secret: &ReadCapSecret,
- storage: &Box<impl RepoStore + ?Sized>,
+ storage: &Box<impl BlockStorage + ?Sized>,
) -> Result<Commit, NgError> {
let (body_ref, mut saved_body) =
body.clone()
@@ -307,7 +307,7 @@ impl Commit {
block_size: usize,
store_pubkey: &StoreRepo,
store_secret: &ReadCapSecret,
- store: &Box<impl RepoStore + ?Sized>,
+ store: &Box<impl BlockStorage + ?Sized>,
) -> Result<ObjectRef, StorageError> {
match self {
Commit::V0(v0) => v0.save(block_size, store_pubkey, store_secret, store),
@@ -323,7 +323,7 @@ impl Commit {
/// Load commit from store
pub fn load(
commit_ref: ObjectRef,
- store: &Box<impl RepoStore + ?Sized>,
+ store: &Box<impl BlockStorage + ?Sized>,
with_body: bool,
) -> Result<Commit, CommitLoadError> {
let (id, key) = (commit_ref.id, commit_ref.key);
@@ -356,7 +356,7 @@ impl Commit {
/// Load commit body from store
pub fn load_body(
&self,
- store: &Box<impl RepoStore + ?Sized>,
+ store: &Box<impl BlockStorage + ?Sized>,
) -> Result<&CommitBody, CommitLoadError> {
if self.body().is_some() {
return Ok(self.body().unwrap());
@@ -457,7 +457,7 @@ impl Commit {
pub fn owners_signature_required(
&self,
- store: &Box<impl RepoStore + ?Sized>,
+ store: &Box<impl BlockStorage + ?Sized>,
) -> Result<bool, CommitLoadError> {
match self.load_body(store)? {
CommitBody::V0(CommitBodyV0::UpdateRootBranch(new_root)) => {
@@ -635,15 +635,15 @@ impl Commit {
/// or a list of missing blocks
pub fn verify_full_object_refs_of_branch_at_commit(
&self,
- store: &Box<impl RepoStore + ?Sized>,
+ store: &Box<impl BlockStorage + ?Sized>,
) -> Result<Vec<ObjectId>, CommitLoadError> {
//log_debug!(">> verify_full_object_refs_of_branch_at_commit: #{}", self.seq());
- /// Load `Commit`s of a `Branch` from the `RepoStore` starting from the given `Commit`,
+ /// Load `Commit`s of a `Branch` from the `BlockStorage` starting from the given `Commit`,
/// and collect missing `ObjectId`s
fn load_direct_object_refs(
commit: &Commit,
- store: &Box<impl RepoStore + ?Sized>,
+ store: &Box<impl BlockStorage + ?Sized>,
visited: &mut HashSet<ObjectId>,
missing: &mut HashSet<ObjectId>,
) -> Result<(), CommitLoadError> {
@@ -772,7 +772,7 @@ impl CommitBody {
block_size: usize,
store_pubkey: &StoreRepo,
store_secret: &ReadCapSecret,
- store: &Box<impl RepoStore + ?Sized>,
+ store: &Box<impl BlockStorage + ?Sized>,
) -> Result<(ObjectRef, Vec<BlockId>), StorageError> {
let obj = Object::new(
ObjectContent::V0(ObjectContentV0::CommitBody(self)),
@@ -1458,16 +1458,16 @@ mod test {
use crate::log::*;
struct Test<'a> {
- storage: Box<dyn RepoStore + Send + Sync + 'a>,
+ storage: Box<dyn BlockStorage + Send + Sync + 'a>,
}
impl<'a> Test<'a> {
- fn storage(s: impl RepoStore + 'a) -> Self {
+ fn storage(s: impl BlockStorage + 'a) -> Self {
Test {
storage: Box::new(s),
}
}
- fn s(&self) -> &Box<dyn RepoStore + Send + Sync + 'a> {
+ fn s(&self) -> &Box<dyn BlockStorage + Send + Sync + 'a> {
&self.storage
}
}
@@ -1511,7 +1511,7 @@ mod test {
let max_object_size = 0;
let (store_repo, store_secret) = StoreRepo::dummy_public_v0();
- let hashmap_storage = HashMapRepoStore::new();
+ let hashmap_storage = HashMapBlockStorage::new();
let storage = Box::new(hashmap_storage);
let commit_ref = commit
@@ -1579,7 +1579,7 @@ mod test {
&store_secret,
);
- let hashmap_storage = HashMapRepoStore::new();
+ let hashmap_storage = HashMapBlockStorage::new();
let storage = Box::new(hashmap_storage);
_ = obj.save(&storage).expect("save object");
@@ -1612,7 +1612,7 @@ mod test {
let max_object_size = 0;
let (store_repo, store_secret) = StoreRepo::dummy_public_v0();
- let hashmap_storage = HashMapRepoStore::new();
+ let hashmap_storage = HashMapBlockStorage::new();
let storage = Box::new(hashmap_storage);
let commit = Commit::new_with_body_and_save(
@@ -1676,7 +1676,7 @@ mod test {
.unwrap();
log_debug!("{}", commit);
- let hashmap_storage = HashMapRepoStore::new();
+ let hashmap_storage = HashMapBlockStorage::new();
let t = Test::storage(hashmap_storage);
let repo =
@@ -1735,7 +1735,7 @@ mod test {
let max_object_size = 0;
let (store_repo, store_secret) = StoreRepo::dummy_public_v0();
- let hashmap_storage = HashMapRepoStore::new();
+ let hashmap_storage = HashMapBlockStorage::new();
let t = Test::storage(hashmap_storage);
let commit = Commit::new_with_body_and_save(
@@ -1808,7 +1808,7 @@ mod test {
log_debug!("{}", commit);
- let hashmap_storage = HashMapRepoStore::new();
+ let hashmap_storage = HashMapBlockStorage::new();
let t = Test::storage(hashmap_storage);
let repo = Repo::new_with_member(
diff --git a/ng-repo/src/errors.rs b/ng-repo/src/errors.rs
index 6475f3a..9ddc986 100644
--- a/ng-repo/src/errors.rs
+++ b/ng-repo/src/errors.rs
@@ -10,7 +10,7 @@
//! Errors
use crate::commit::{CommitLoadError, CommitVerifyError};
-use crate::store::StorageError;
+
use crate::types::BlockId;
use core::fmt;
use std::error::Error;
@@ -149,3 +149,26 @@ pub enum ObjectParseError {
/// Error deserializing content of the object
ObjectDeserializeError,
}
+
+#[derive(Debug, PartialEq, Eq, Clone)]
+pub enum StorageError {
+ NotFound,
+ InvalidValue,
+ DifferentValue,
+ BackendError,
+ SerializationError,
+ AlreadyExists,
+ DataCorruption,
+}
+
+impl core::fmt::Display for StorageError {
+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+ write!(f, "{:?}", self)
+ }
+}
+
+impl From<serde_bare::error::Error> for StorageError {
+ fn from(_e: serde_bare::error::Error) -> Self {
+ StorageError::SerializationError
+ }
+}
diff --git a/ng-repo/src/event.rs b/ng-repo/src/event.rs
index e567fd9..243a83f 100644
--- a/ng-repo/src/event.rs
+++ b/ng-repo/src/event.rs
@@ -8,9 +8,9 @@
//! Event, a message sent in the PUB/SUB
+use crate::block_storage::*;
use crate::errors::*;
use crate::object::*;
-use crate::store::*;
use crate::types::*;
use crate::utils::*;
use core::fmt;
@@ -61,7 +61,7 @@ impl Event {
topic_id: TopicId,
branch_read_cap_secret: ReadCapSecret,
topic_priv_key: &BranchWriteCapSecret,
- storage: &'a Box<dyn RepoStore + Send + Sync + 'a>,
+ storage: &'a Box<dyn BlockStorage + Send + Sync + 'a>,
) -> Result<Event, NgError> {
Ok(Event::V0(EventV0::new(
publisher,
@@ -85,7 +85,7 @@ impl EventV0 {
topic_id: TopicId,
branch_read_cap_secret: ReadCapSecret,
topic_priv_key: &BranchWriteCapSecret,
- storage: &'a Box<dyn RepoStore + Send + Sync + 'a>,
+ storage: &'a Box<dyn BlockStorage + Send + Sync + 'a>,
) -> Result<EventV0, NgError> {
let mut blocks = vec![];
for bid in commit.blocks().iter() {
diff --git a/ng-repo/src/file.rs b/ng-repo/src/file.rs
index 8ac50fd..e0d0e3b 100644
--- a/ng-repo/src/file.rs
+++ b/ng-repo/src/file.rs
@@ -17,10 +17,10 @@ use chacha20::cipher::{KeyIvInit, StreamCipher};
use chacha20::ChaCha20;
use zeroize::Zeroize;
+use crate::block_storage::*;
use crate::errors::*;
use crate::log::*;
use crate::object::*;
-use crate::store::*;
use crate::types::*;
/// File errors
@@ -83,7 +83,7 @@ impl<'a> File<'a> {
pub fn open(
id: ObjectId,
key: SymKey,
- storage: &'a Box<dyn RepoStore + Send + Sync + 'a>,
+ storage: &'a Box<dyn BlockStorage + Send + Sync + 'a>,
) -> Result<File<'a>, FileError> {
let root_block = storage.get(&id)?;
@@ -133,8 +133,8 @@ impl ReadFile for SmallFileV0 {
/// A RandomAccessFile in memory. This is not used to serialize data
pub struct RandomAccessFile<'a> {
- //storage: Arc<&'a dyn RepoStore>,
- storage: &'a Box<dyn RepoStore + Send + Sync + 'a>,
+ //storage: Arc<&'a dyn BlockStorage>,
+ storage: &'a Box<dyn BlockStorage + Send + Sync + 'a>,
/// accurate once saved or opened
meta: RandomAccessFileMeta,
@@ -263,7 +263,7 @@ impl<'a> RandomAccessFile<'a> {
conv_key: &[u8; blake3::OUT_LEN],
children: Vec<ObjectId>,
already_existing: &mut HashMap<BlockKey, BlockId>,
- storage: &Box<dyn RepoStore + Send + Sync + 'a>,
+ storage: &Box<dyn BlockStorage + Send + Sync + 'a>,
) -> Result<(BlockId, BlockKey), StorageError> {
let key_hash = blake3::keyed_hash(conv_key, &content);
@@ -294,7 +294,7 @@ impl<'a> RandomAccessFile<'a> {
conv_key: &[u8; blake3::OUT_LEN],
children: Vec<(BlockId, BlockKey)>,
already_existing: &mut HashMap<BlockKey, BlockId>,
- storage: &Box<dyn RepoStore + Send + Sync + 'a>,
+ storage: &Box<dyn BlockStorage + Send + Sync + 'a>,
) -> Result<(BlockId, BlockKey), StorageError> {
let mut ids: Vec<BlockId> = Vec::with_capacity(children.len());
let mut keys: Vec<BlockKey> = Vec::with_capacity(children.len());
@@ -314,7 +314,7 @@ impl<'a> RandomAccessFile<'a> {
leaves: &[(BlockId, BlockKey)],
conv_key: &ChaCha20Key,
arity: u16,
- storage: &'a Box<dyn RepoStore + Send + Sync + 'a>,
+ storage: &'a Box<dyn BlockStorage + Send + Sync + 'a>,
) -> Result<(BlockId, BlockKey), StorageError> {
let mut parents: Vec<(BlockId, BlockKey)> = vec![];
let mut chunks = leaves.chunks(arity as usize);
@@ -347,7 +347,7 @@ impl<'a> RandomAccessFile<'a> {
blocks: &[(BlockId, BlockKey)],
meta: &mut RandomAccessFileMeta,
conv_key: &ChaCha20Key,
- storage: &'a Box<dyn RepoStore + Send + Sync + 'a>,
+ storage: &'a Box<dyn BlockStorage + Send + Sync + 'a>,
) -> Result<((BlockId, BlockKey), (BlockId, BlockKey)), FileError> {
let leaf_blocks_nbr = blocks.len();
let arity = meta.arity();
@@ -406,7 +406,7 @@ impl<'a> RandomAccessFile<'a> {
metadata: Vec<u8>,
store: &StoreRepo,
store_secret: &ReadCapSecret,
- storage: &'a Box<dyn RepoStore + Send + Sync + 'a>,
+ storage: &'a Box<dyn BlockStorage + Send + Sync + 'a>,
) -> Result<RandomAccessFile<'a>, FileError> {
//let max_block_size = store_max_value_size();
let valid_block_size = store_valid_value_size(block_size) - BLOCK_EXTRA;
@@ -477,7 +477,7 @@ impl<'a> RandomAccessFile<'a> {
metadata: Vec<u8>,
store: &StoreRepo,
store_secret: &ReadCapSecret,
- storage: &'a Box<dyn RepoStore + Send + Sync + 'a>,
+ storage: &'a Box<dyn BlockStorage + Send + Sync + 'a>,
) -> Self {
let valid_block_size = store_valid_value_size(block_size) - BLOCK_EXTRA;
@@ -617,7 +617,7 @@ impl<'a> RandomAccessFile<'a> {
pub fn open(
id: ObjectId,
key: SymKey,
- storage: &'a Box<dyn RepoStore + Send + Sync + 'a>,
+ storage: &'a Box<dyn BlockStorage + Send + Sync + 'a>,
) -> Result<RandomAccessFile<'a>, FileError> {
// load root block
let root_block = storage.get(&id)?;
@@ -735,16 +735,16 @@ mod test {
use std::io::Read;
struct Test<'a> {
- storage: Box<dyn RepoStore + Send + Sync + 'a>,
+ storage: Box<dyn BlockStorage + Send + Sync + 'a>,
}
impl<'a> Test<'a> {
- fn storage(s: impl RepoStore + 'a) -> Self {
+ fn storage(s: impl BlockStorage + 'a) -> Self {
Test {
storage: Box::new(s),
}
}
- fn s(&self) -> &Box<dyn RepoStore + Send + Sync + 'a> {
+ fn s(&self) -> &Box<dyn BlockStorage + Send + Sync + 'a> {
&self.storage
}
}
@@ -755,10 +755,10 @@ mod test {
let block_size = store_max_value_size();
//store_valid_value_size(0)
- let hashmap_storage = HashMapRepoStore::new();
+ let hashmap_storage = HashMapBlockStorage::new();
let t = Test::storage(hashmap_storage);
- //let storage: Arc<&dyn RepoStore> = Arc::new(&hashmap_storage);
+ //let storage: Arc<&dyn BlockStorage> = Arc::new(&hashmap_storage);
////// 1 MB of data!
let data_size = block_size - BLOCK_EXTRA;
@@ -834,7 +834,7 @@ mod test {
const MAX_ARITY_LEAVES: usize = 15887;
const MAX_DATA_PAYLOAD_SIZE: usize = 1048564;
- let hashmap_storage = HashMapRepoStore::new();
+ let hashmap_storage = HashMapBlockStorage::new();
let t = Test::storage(hashmap_storage);
////// 16 GB of data!
@@ -873,7 +873,7 @@ mod test {
const MAX_ARITY_LEAVES: usize = 15887;
const MAX_DATA_PAYLOAD_SIZE: usize = 1048564;
- let hashmap_storage = HashMapRepoStore::new();
+ let hashmap_storage = HashMapBlockStorage::new();
let t = Test::storage(hashmap_storage);
////// 16 GB of data!
@@ -911,7 +911,7 @@ mod test {
pub fn test_depth_3() {
const MAX_ARITY_LEAVES: usize = 61;
const MAX_DATA_PAYLOAD_SIZE: usize = 4084;
- let hashmap_storage = HashMapRepoStore::new();
+ let hashmap_storage = HashMapBlockStorage::new();
let t = Test::storage(hashmap_storage);
////// 900 MB of data!
@@ -981,7 +981,7 @@ mod test {
* MAX_ARITY_LEAVES
* MAX_DATA_PAYLOAD_SIZE;
- let hashmap_storage = HashMapRepoStore::new();
+ let hashmap_storage = HashMapBlockStorage::new();
let t = Test::storage(hashmap_storage);
let (store_repo, store_secret) = StoreRepo::dummy_public_v0();
@@ -1022,7 +1022,7 @@ mod test {
.read_to_end(&mut img_buffer)
.expect("read of test.jpg");
- let hashmap_storage = HashMapRepoStore::new();
+ let hashmap_storage = HashMapBlockStorage::new();
let t = Test::storage(hashmap_storage);
let (store_repo, store_secret) = StoreRepo::dummy_public_v0();
@@ -1097,7 +1097,7 @@ mod test {
.read_to_end(&mut img_buffer)
.expect("read of test.jpg");
- let hashmap_storage = HashMapRepoStore::new();
+ let hashmap_storage = HashMapBlockStorage::new();
let t = Test::storage(hashmap_storage);
let (store_repo, store_secret) = StoreRepo::dummy_public_v0();
@@ -1174,7 +1174,7 @@ mod test {
.read_to_end(&mut img_buffer)
.expect("read of test.jpg");
- let hashmap_storage = HashMapRepoStore::new();
+ let hashmap_storage = HashMapBlockStorage::new();
let t = Test::storage(hashmap_storage);
let (store_repo, store_secret) = StoreRepo::dummy_public_v0();
@@ -1259,7 +1259,7 @@ mod test {
let first_block_content = img_buffer[0..4084].to_vec();
- let hashmap_storage = HashMapRepoStore::new();
+ let hashmap_storage = HashMapBlockStorage::new();
let t = Test::storage(hashmap_storage);
let (store_repo, store_secret) = StoreRepo::dummy_public_v0();
@@ -1342,7 +1342,7 @@ mod test {
let chunk_nbr = data_size / 5000000;
let last_chunk = data_size % 5000000;
- let hashmap_storage = HashMapRepoStore::new();
+ let hashmap_storage = HashMapBlockStorage::new();
let t = Test::storage(hashmap_storage);
let (store_repo, store_secret) = StoreRepo::dummy_public_v0();
@@ -1396,7 +1396,7 @@ mod test {
.read_to_end(&mut img_buffer)
.expect("read of test.jpg");
- let hashmap_storage = HashMapRepoStore::new();
+ let hashmap_storage = HashMapBlockStorage::new();
let t = Test::storage(hashmap_storage);
let (store_repo, store_secret) = StoreRepo::dummy_public_v0();
@@ -1464,7 +1464,7 @@ mod test {
log_debug!("{}", obj);
- let hashmap_storage = HashMapRepoStore::new();
+ let hashmap_storage = HashMapBlockStorage::new();
let t = Test::storage(hashmap_storage);
let _ = obj.save_in_test(t.s()).expect("save");
@@ -1490,7 +1490,7 @@ mod test {
let max_object_size = store_max_value_size();
let (store_repo, store_secret) = StoreRepo::dummy_public_v0();
- let hashmap_storage = HashMapRepoStore::new();
+ let hashmap_storage = HashMapBlockStorage::new();
let t = Test::storage(hashmap_storage);
log_debug!("creating empty file");
@@ -1533,7 +1533,7 @@ mod test {
let f = std::fs::File::open("[enter path of a big file here]").expect("open of a big file");
let mut reader = BufReader::new(f);
- let hashmap_storage = HashMapRepoStore::new();
+ let hashmap_storage = HashMapBlockStorage::new();
let t = Test::storage(hashmap_storage);
let (store_repo, store_secret) = StoreRepo::dummy_public_v0();
@@ -1587,7 +1587,7 @@ mod test {
let f = std::fs::File::open("[enter path of a big file here]").expect("open of a big file");
let mut reader = BufReader::new(f);
- let hashmap_storage = HashMapRepoStore::new();
+ let hashmap_storage = HashMapBlockStorage::new();
let t = Test::storage(hashmap_storage);
let (store_repo, store_secret) = StoreRepo::dummy_public_v0();
diff --git a/ng-repo/src/kcv_store.rs b/ng-repo/src/kcv_storage.rs
similarity index 99%
rename from ng-repo/src/kcv_store.rs
rename to ng-repo/src/kcv_storage.rs
index e3a705d..9ce4a69 100644
--- a/ng-repo/src/kcv_store.rs
+++ b/ng-repo/src/kcv_storage.rs
@@ -8,7 +8,7 @@
//! KeyColumnValue Store abstraction
-use crate::store::StorageError;
+use crate::errors::StorageError;
// TODO:remove mut on self for trait WriteTransaction methods
diff --git a/ng-repo/src/lib.rs b/ng-repo/src/lib.rs
index 74648e5..2982d87 100644
--- a/ng-repo/src/lib.rs
+++ b/ng-repo/src/lib.rs
@@ -8,7 +8,7 @@
pub mod types;
-pub mod store;
+pub mod block_storage;
pub mod block;
@@ -30,7 +30,7 @@ pub mod utils;
pub mod errors;
-pub mod kcv_store;
+pub mod kcv_storage;
pub mod os_info;
diff --git a/ng-repo/src/object.rs b/ng-repo/src/object.rs
index deac4ff..3fec9de 100644
--- a/ng-repo/src/object.rs
+++ b/ng-repo/src/object.rs
@@ -18,9 +18,9 @@ use chacha20::cipher::{KeyIvInit, StreamCipher};
use chacha20::ChaCha20;
use zeroize::Zeroize;
+use crate::block_storage::*;
use crate::errors::*;
use crate::log::*;
-use crate::store::*;
use crate::types::*;
pub const BLOCK_EXTRA: usize = 12; // 8 is the smallest extra + BLOCK_MAX_DATA_EXTRA
@@ -418,17 +418,17 @@ impl Object {
}
}
- /// Load an Object from RepoStore
+ /// Load an Object from BlockStorage
///
/// Returns Ok(Object) or an Err(ObjectParseError::MissingBlocks(Vec<ObjectId>)) of missing BlockIds
pub fn load(
id: ObjectId,
key: Option<SymKey>,
- store: &Box<impl RepoStore + ?Sized>,
+ store: &Box<impl BlockStorage + ?Sized>,
) -> Result<Object, ObjectParseError> {
fn load_tree(
parents: Vec<BlockId>,
- store: &Box<impl RepoStore + ?Sized>,
+ store: &Box<impl BlockStorage + ?Sized>,
blocks: &mut Vec<BlockId>,
missing: &mut Vec<BlockId>,
block_contents: &mut HashMap<BlockId, Block>,
@@ -517,7 +517,10 @@ impl Object {
}
/// Save blocks of the object and the blocks of the header object in the store
- pub fn save(&self, store: &Box<impl RepoStore + ?Sized>) -> Result<Vec<BlockId>, StorageError> {
+ pub fn save(
+ &self,
+ store: &Box<impl BlockStorage + ?Sized>,
+ ) -> Result<Vec<BlockId>, StorageError> {
let mut deduplicated: HashSet<ObjectId> = HashSet::new();
//.chain(self.header_blocks.iter())
for block_id in self.blocks.iter() {
@@ -543,7 +546,7 @@ impl Object {
#[cfg(test)]
pub fn save_in_test(
&mut self,
- store: &Box<impl RepoStore + ?Sized>,
+ store: &Box<impl BlockStorage + ?Sized>,
) -> Result<Vec<BlockId>, StorageError> {
assert!(self.already_saved == false);
self.already_saved = true;
@@ -963,7 +966,7 @@ mod test {
use std::io::Read;
use std::io::Write;
- // Those constants are calculated with RepoStore::get_max_value_size
+ // Those constants are calculated with BlockStorage::get_max_value_size
/// Maximum arity of branch containing max number of leaves
// const MAX_ARITY_LEAVES: usize = 15887;
// /// Maximum arity of root branch
@@ -1064,7 +1067,7 @@ mod test {
}
Err(e) => panic!("Object parse error: {:?}", e),
}
- let store = Box::new(HashMapRepoStore::new());
+ let store = Box::new(HashMapBlockStorage::new());
obj.save_in_test(&store).expect("Object save error");
diff --git a/ng-repo/src/repo.rs b/ng-repo/src/repo.rs
index 8ea7c9d..e86053b 100644
--- a/ng-repo/src/repo.rs
+++ b/ng-repo/src/repo.rs
@@ -9,11 +9,11 @@
//! Repository
+use crate::block_storage::*;
use crate::errors::*;
use crate::event::*;
use crate::log::*;
use crate::object::Object;
-use crate::store::*;
use crate::types::*;
use crate::utils::generate_keypair;
use crate::utils::sign;
@@ -84,7 +84,7 @@ pub struct Repo<'a> {
pub members: HashMap<Digest, UserInfo>,
- storage: &'a Box<dyn RepoStore + Send + Sync + 'a>,
+ storage: &'a Box<dyn BlockStorage + Send + Sync + 'a>,
}
impl<'a> fmt::Display for Repo<'a> {
@@ -112,7 +112,7 @@ impl<'a> Repo<'a> {
peer_last_seq_num: &mut u64,
store_repo: &StoreRepo,
store_secret: &ReadCapSecret,
- storage: &'a Box<dyn RepoStore + Send + Sync + 'a>,
+ storage: &'a Box<dyn BlockStorage + Send + Sync + 'a>,
) -> Result<(Self, Vec<Event>), NgError> {
let mut events = Vec::with_capacity(6);
@@ -223,6 +223,7 @@ impl<'a> Repo<'a> {
let main_branch_commit_body = CommitBody::V0(CommitBodyV0::Branch(Branch::V0(BranchV0 {
id: main_branch_pub_key,
+ content_type: BranchContentType::None,
repo: repository_commit_ref.clone(),
root_branch_readcap_id: root_branch_commit.id().unwrap(),
topic: main_branch_topic_pub_key,
@@ -467,7 +468,7 @@ impl<'a> Repo<'a> {
member: &UserId,
perms: &[PermissionV0],
overlay: OverlayId,
- storage: &'a Box<dyn RepoStore + Send + Sync + 'a>,
+ storage: &'a Box<dyn BlockStorage + Send + Sync + 'a>,
) -> Self {
let mut members = HashMap::new();
let permissions = HashMap::from_iter(
@@ -510,7 +511,7 @@ impl<'a> Repo<'a> {
}
}
- pub fn get_storage(&self) -> &Box<dyn RepoStore + Send + Sync + 'a> {
+ pub fn get_storage(&self) -> &Box<dyn BlockStorage + Send + Sync + 'a> {
self.storage
}
}
@@ -522,16 +523,16 @@ mod test {
use crate::repo::*;
struct Test<'a> {
- storage: Box<dyn RepoStore + Send + Sync + 'a>,
+ storage: Box<dyn BlockStorage + Send + Sync + 'a>,
}
impl<'a> Test<'a> {
- fn storage(s: impl RepoStore + 'a) -> Self {
+ fn storage(s: impl BlockStorage + 'a) -> Self {
Test {
storage: Box::new(s),
}
}
- fn s(&self) -> &Box<dyn RepoStore + Send + Sync + 'a> {
+ fn s(&self) -> &Box<dyn BlockStorage + Send + Sync + 'a> {
&self.storage
}
}
@@ -546,7 +547,7 @@ mod test {
let mut peer_last_seq_num = 10;
let (store_repo, store_secret) = StoreRepo::dummy_public_v0();
- let hashmap_storage = HashMapRepoStore::new();
+ let hashmap_storage = HashMapBlockStorage::new();
let t = Test::storage(hashmap_storage);
let (repo, events) = Repo::new_default(
diff --git a/ng-repo/src/types.rs b/ng-repo/src/types.rs
index 7c9bec0..bb719c9 100644
--- a/ng-repo/src/types.rs
+++ b/ng-repo/src/types.rs
@@ -165,6 +165,11 @@ impl PubKey {
pub fn nil() -> Self {
PubKey::Ed25519PubKey([0u8; 32])
}
+
+ pub fn to_hash_string(&self) -> String {
+ let hash = blake3::hash(self.slice());
+ base64_url::encode(&hash.as_bytes())
+ }
}
impl fmt::Display for PubKey {
@@ -597,6 +602,22 @@ impl StoreOverlay {
StoreOverlay::Own(_) => unimplemented!(),
}
}
+
+ pub fn overlay_id_for_write_purpose(
+ &self,
+ store_overlay_branch_readcap_secret: ReadCapSecret,
+ ) -> OverlayId {
+ match self {
+ StoreOverlay::V0(StoreOverlayV0::PublicStore(id))
+ | StoreOverlay::V0(StoreOverlayV0::ProtectedStore(id))
+ | StoreOverlay::V0(StoreOverlayV0::PrivateStore(id))
+ | StoreOverlay::V0(StoreOverlayV0::Group(id)) => {
+ OverlayId::inner(id, store_overlay_branch_readcap_secret)
+ }
+ StoreOverlay::V0(StoreOverlayV0::Dialog(d)) => unimplemented!(),
+ StoreOverlay::Own(_) => unimplemented!(),
+ }
+ }
}
impl From<&StoreRepo> for StoreOverlay {
@@ -1115,6 +1136,21 @@ pub enum Quorum {
V0(QuorumV0),
}
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
+pub enum BranchContentType {
+ GraphOnly,
+ YMap,
+ YXml,
+ YText,
+ Automerge,
+ //Rdfs,
+ //Owl,
+ //Shacl,
+ //Shex,
+ None, // this is used by Store, Overlay and User BranchTypes
+ //Chat,
+}
+
/// Branch definition
///
/// First commit in a branch, signed by branch key
@@ -1128,6 +1164,8 @@ pub struct BranchV0 {
/// Branch public key ID
pub id: PubKey,
+ pub content_type: BranchContentType,
+
/// Reference to the repository commit
pub repo: ObjectRef,
@@ -1176,12 +1214,12 @@ pub enum Branch {
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
pub enum BranchType {
- Main,
+ Main, // Main is also transactional
Chat,
Store,
Overlay,
User,
- Transactional,
+ Transactional, // this could have been called OtherTransaction, but for the sake of simplicity, we use Transaction for any branch that is not the Main one.
}
impl fmt::Display for BranchType {
@@ -1594,6 +1632,7 @@ pub enum WalletUpdate {
///
/// DEPS to the previous ones.
/// this is used to speedup joining the overlay of such stores, for new devices on new brokers
+/// so they don't have to read the whole pub/sub of the StoreRepo in order to get the last ReadCap
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
pub struct StoreUpdateV0 {
// id of the store.
diff --git a/ng-repo/src/utils.rs b/ng-repo/src/utils.rs
index 0075523..ac441e6 100644
--- a/ng-repo/src/utils.rs
+++ b/ng-repo/src/utils.rs
@@ -22,6 +22,10 @@ use time::OffsetDateTime;
use web_time::{Duration, SystemTime, UNIX_EPOCH};
use zeroize::Zeroize;
+pub fn derive_key(context: &str, key_material: &[u8]) -> [u8; 32] {
+ blake3::derive_key(context, key_material)
+}
+
pub fn ed_keypair_from_priv_bytes(secret_key: [u8; 32]) -> (PrivKey, PubKey) {
let sk = SecretKey::from_bytes(&secret_key).unwrap();
let pk: PublicKey = (&sk).into();
@@ -200,4 +204,4 @@ pub fn display_timestamp(ts: &Timestamp) -> String {
.unwrap()
}
-pub type Receiver<T> = mpsc::UnboundedReceiver<T>;
+pub(crate) type Receiver<T> = mpsc::UnboundedReceiver<T>;
diff --git a/ng-sdk-js/js/browser.js b/ng-sdk-js/js/browser.js
index 098ccc8..b53572d 100644
--- a/ng-sdk-js/js/browser.js
+++ b/ng-sdk-js/js/browser.js
@@ -31,6 +31,10 @@ export function session_save(key,value) {
}
}
+export function is_browser() {
+ return true;
+}
+
function convert_error(e) {
if (
e == "The operation is insecure." ||
@@ -56,6 +60,17 @@ export function session_get(key) {
}
+export function session_remove(key) {
+
+ try {
+ return sessionStorage.removeItem(key);
+
+ } catch(e) {
+ console.error(e);
+ }
+
+}
+
export function local_save(key,value) {
try {
localStorage.setItem(key, value);
diff --git a/ng-sdk-js/js/node.js b/ng-sdk-js/js/node.js
index 625e9ea..4771cdb 100644
--- a/ng-sdk-js/js/node.js
+++ b/ng-sdk-js/js/node.js
@@ -140,6 +140,12 @@ module.exports.client_details = function () {
});
};
+
+module.exports.is_browser = function() {
+ return false;
+}
+
+
module.exports.session_save = function(key,value) {
}
@@ -148,6 +154,10 @@ module.exports.session_get = function(key) {
}
+module.exports.session_remove = function(key) {
+
+}
+
module.exports.local_save = function(key,value) {
}
diff --git a/ng-sdk-js/src/lib.rs b/ng-sdk-js/src/lib.rs
index 4a58582..e9afeaf 100644
--- a/ng-sdk-js/src/lib.rs
+++ b/ng-sdk-js/src/lib.rs
@@ -236,10 +236,28 @@ pub async fn session_start(wallet_name: String, user_js: JsValue) -> Result<JsVa
let user_id = serde_wasm_bindgen::from_value::<PubKey>(user_js)
.map_err(|_| "Deserialization error of user_id")?;
- let config = SessionConfig::V0(SessionConfigV0 {
- user_id,
- wallet_name,
- });
+ let config = SessionConfig::new_in_memory(&user_id, &wallet_name);
+ let res = nextgraph::local_broker::session_start(config)
+ .await
+ .map_err(|e: NgError| e.to_string())?;
+
+ Ok(serde_wasm_bindgen::to_value(&res).unwrap())
+}
+
+#[cfg(target_arch = "wasm32")]
+#[wasm_bindgen]
+pub async fn session_start_remote(
+ wallet_name: String,
+ user_js: JsValue,
+ peer_id_js: JsValue,
+) -> Result<JsValue, String> {
+ let user_id = serde_wasm_bindgen::from_value::<PubKey>(user_js)
+ .map_err(|_| "Deserialization error of user_id")?;
+
+ let peer_id = serde_wasm_bindgen::from_value::<Option<PubKey>>(peer_id_js)
+ .map_err(|_| "Deserialization error of peer_id")?;
+
+ let config = SessionConfig::new_remote(&user_id, &wallet_name, peer_id);
let res = nextgraph::local_broker::session_start(config)
.await
.map_err(|e: NgError| e.to_string())?;
@@ -274,8 +292,10 @@ pub async fn add_in_memory_wallet(lws_js: JsValue) -> Result<(), String> {
extern "C" {
fn session_save(key: String, value: String) -> Option<String>;
fn session_get(key: String) -> Option<String>;
+ fn session_remove(key: String);
fn local_save(key: String, value: String) -> Option<String>;
fn local_get(key: String) -> Option<String>;
+ fn is_browser() -> bool;
}
#[cfg(wasmpack_target = "nodejs")]
@@ -283,8 +303,10 @@ extern "C" {
extern "C" {
fn session_save(key: String, value: String) -> Option<String>;
fn session_get(key: String) -> Option<String>;
+ fn session_remove(key: String);
fn local_save(key: String, value: String) -> Option<String>;
fn local_get(key: String) -> Option<String>;
+ fn is_browser() -> bool;
}
#[cfg(target_arch = "wasm32")]
@@ -313,14 +335,22 @@ fn session_write(key: String, value: String) -> Result<(), NgError> {
}
}
+#[cfg(target_arch = "wasm32")]
+fn session_del(key: String) -> Result<(), NgError> {
+ session_remove(key);
+ Ok(())
+}
+
#[cfg(target_arch = "wasm32")]
static INIT_LOCAL_BROKER: Lazy<Box<ConfigInitFn>> = Lazy::new(|| {
Box::new(|| {
LocalBrokerConfig::JsStorage(JsStorageConfig {
local_read: Box::new(local_read),
local_write: Box::new(local_write),
- session_read: Box::new(session_read),
- session_write: Box::new(session_write),
+ session_read: Arc::new(Box::new(session_read)),
+ session_write: Arc::new(Box::new(session_write)),
+ session_del: Arc::new(Box::new(session_del)),
+ is_browser: is_browser(),
})
})
});
@@ -354,6 +384,7 @@ pub async fn wallet_get_file(wallet_name: String) -> Result<JsValue, JsValue> {
#[cfg(target_arch = "wasm32")]
#[wasm_bindgen]
pub async fn wallet_read_file(js_file: JsValue) -> Result<JsValue, String> {
+ init_local_broker_with_lazy(&INIT_LOCAL_BROKER).await;
let mut file = serde_wasm_bindgen::from_value::<serde_bytes::ByteBuf>(js_file)
.map_err(|_| "Deserialization error of file".to_string())?;
diff --git a/ng-stores-lmdb/Cargo.toml b/ng-storage-lmdb/Cargo.toml
similarity index 96%
rename from ng-stores-lmdb/Cargo.toml
rename to ng-storage-lmdb/Cargo.toml
index 7d8604f..aff09f5 100644
--- a/ng-stores-lmdb/Cargo.toml
+++ b/ng-storage-lmdb/Cargo.toml
@@ -1,5 +1,5 @@
[package]
-name = "ng-stores-lmdb"
+name = "ng-storage-lmdb"
# version = "0.1.0"
description = "Stores based on LMDB for NextGraph"
version.workspace = true
diff --git a/ng-stores-lmdb/src/repo_store.rs b/ng-storage-lmdb/src/block_storage.rs
similarity index 97%
rename from ng-stores-lmdb/src/repo_store.rs
rename to ng-storage-lmdb/src/block_storage.rs
index fdf2112..ab94721 100644
--- a/ng-stores-lmdb/src/repo_store.rs
+++ b/ng-storage-lmdb/src/block_storage.rs
@@ -28,7 +28,7 @@ use serde::{Deserialize, Serialize};
use serde_bare::error::Error;
#[derive(Debug)]
-pub struct LmdbRepoStore {
+pub struct LmdbBlockStorage {
/// the main store where all the repo blocks are stored
main_store: SingleStore<LmdbDatabase>,
/// store for the pin boolean, recently_used timestamp, and synced boolean
@@ -49,7 +49,7 @@ struct BlockMeta {
pub synced: bool,
}
-impl RepoStore for LmdbRepoStore {
+impl BlockStorage for LmdbBlockStorage {
/// Retrieves a block from the storage backend.
fn get(&self, block_id: &BlockId) -> Result<Block, StorageError> {
let lock = self.environment.read().unwrap();
@@ -203,10 +203,10 @@ impl RepoStore for LmdbRepoStore {
}
}
-impl LmdbRepoStore {
- /// Opens the store and returns a RepoStore object that should be kept and used to call put/get/delete/pin
+impl LmdbBlockStorage {
+ /// Opens the store and returns a BlockStorage object that should be kept and used to call put/get/delete/pin
/// The key is the encryption key for the data at rest.
- pub fn open<'a>(path: &Path, key: [u8; 32]) -> Result<LmdbRepoStore, StorageError> {
+ pub fn open<'a>(path: &Path, key: [u8; 32]) -> Result<LmdbBlockStorage, StorageError> {
let mut manager = Manager::<LmdbEnvironment>::singleton().write().unwrap();
let shared_rkv = manager
.get_or_create(path, |path| {
@@ -232,7 +232,7 @@ impl LmdbRepoStore {
let expiry_store = env.open_multi_integer("expiry", opts).unwrap();
let recently_used_store = env.open_multi_integer("recently_used", opts).unwrap();
- Ok(LmdbRepoStore {
+ Ok(LmdbBlockStorage {
environment: shared_rkv.clone(),
main_store,
meta_store,
@@ -495,7 +495,7 @@ impl LmdbRepoStore {
#[cfg(test)]
mod test {
- use crate::repo_store::LmdbRepoStore;
+ use crate::repo_store::LmdbBlockStorage;
use ng_repo::log::*;
use ng_repo::store::*;
use ng_repo::types::*;
@@ -515,7 +515,7 @@ mod test {
let key: [u8; 32] = [0; 32];
fs::create_dir_all(root.path()).unwrap();
log_debug!("{}", root.path().to_str().unwrap());
- let mut store = LmdbRepoStore::open(root.path(), key).unwrap();
+ let mut store = LmdbBlockStorage::open(root.path(), key).unwrap();
let mut now = now_timestamp();
now -= 200;
// TODO: fix the LMDB bug that is triggered with x max set to 86 !!!
@@ -548,7 +548,7 @@ mod test {
let key: [u8; 32] = [0; 32];
fs::create_dir_all(root.path()).unwrap();
log_debug!("{}", root.path().to_str().unwrap());
- let mut store = LmdbRepoStore::open(root.path(), key).unwrap();
+ let mut store = LmdbBlockStorage::open(root.path(), key).unwrap();
let mut now = now_timestamp();
now -= 200;
// TODO: fix the LMDB bug that is triggered with x max set to 86 !!!
@@ -605,7 +605,7 @@ mod test {
let key: [u8; 32] = [0; 32];
fs::create_dir_all(root.path()).unwrap();
log_debug!("{}", root.path().to_str().unwrap());
- let mut store = LmdbRepoStore::open(root.path(), key).unwrap();
+ let mut store = LmdbBlockStorage::open(root.path(), key).unwrap();
let now = now_timestamp();
let list = [
@@ -659,7 +659,7 @@ mod test {
let key: [u8; 32] = [0; 32];
fs::create_dir_all(root.path()).unwrap();
log_debug!("{}", root.path().to_str().unwrap());
- let mut store = LmdbRepoStore::open(root.path(), key).unwrap();
+ let mut store = LmdbBlockStorage::open(root.path(), key).unwrap();
let now = now_timestamp();
let list = [
@@ -706,7 +706,7 @@ mod test {
let key: [u8; 32] = [0; 32];
fs::create_dir_all(root.path()).unwrap();
log_debug!("{}", root.path().to_str().unwrap());
- let store = LmdbRepoStore::open(root.path(), key).unwrap();
+ let store = LmdbBlockStorage::open(root.path(), key).unwrap();
store.remove_expired().unwrap();
}
@@ -720,7 +720,7 @@ mod test {
log_debug!("{}", root.path().to_str().unwrap());
- let mut store = LmdbRepoStore::open(root.path(), key).unwrap();
+ let mut store = LmdbBlockStorage::open(root.path(), key).unwrap();
let block = Block::new(
Vec::new(),
diff --git a/ng-stores-lmdb/src/kcv_store.rs b/ng-storage-lmdb/src/kcv_storage.rs
similarity index 100%
rename from ng-stores-lmdb/src/kcv_store.rs
rename to ng-storage-lmdb/src/kcv_storage.rs
diff --git a/ng-stores-lmdb/src/lib.rs b/ng-storage-lmdb/src/lib.rs
similarity index 100%
rename from ng-stores-lmdb/src/lib.rs
rename to ng-storage-lmdb/src/lib.rs
diff --git a/ng-stores-rocksdb/Cargo.toml b/ng-storage-rocksdb/Cargo.toml
similarity index 95%
rename from ng-stores-rocksdb/Cargo.toml
rename to ng-storage-rocksdb/Cargo.toml
index 0c87739..9a35d54 100644
--- a/ng-stores-rocksdb/Cargo.toml
+++ b/ng-storage-rocksdb/Cargo.toml
@@ -1,5 +1,5 @@
[package]
-name = "ng-stores-rocksdb"
+name = "ng-storage-rocksdb"
# version = "0.1.0"
description = "Stores based on RocksDB for NextGraph"
version.workspace = true
diff --git a/ng-stores-rocksdb/README.md b/ng-storage-rocksdb/README.md
similarity index 97%
rename from ng-stores-rocksdb/README.md
rename to ng-storage-rocksdb/README.md
index f39fd3c..49b3696 100644
--- a/ng-stores-rocksdb/README.md
+++ b/ng-storage-rocksdb/README.md
@@ -1,4 +1,4 @@
-# ng-stores-rocksdb
+# ng-storage-rocksdb
![MSRV][rustc-image]
[![Apache 2.0 Licensed][license-image]][license-link]
@@ -48,7 +48,7 @@ additional terms or conditions.
NextGraph received funding through the [NGI Assure Fund](https://nlnet.nl/project/NextGraph/index.html), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's [Next Generation Internet](https://ngi.eu/) programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 957073.
-[rustc-image]: https://img.shields.io/badge/rustc-1.64+-blue.svg
+[rustc-image]: https://img.shields.io/badge/rustc-1.74+-blue.svg
[license-image]: https://img.shields.io/badge/license-Apache2.0-blue.svg
[license-link]: https://git.nextgraph.org/NextGraph/nextgraph-rs/raw/branch/master/LICENSE-APACHE2
[license-image2]: https://img.shields.io/badge/license-MIT-blue.svg
diff --git a/ng-stores-rocksdb/src/repo_store.rs b/ng-storage-rocksdb/src/block_storage.rs
similarity index 97%
rename from ng-stores-rocksdb/src/repo_store.rs
rename to ng-storage-rocksdb/src/block_storage.rs
index 9f94cad..4a1599b 100644
--- a/ng-stores-rocksdb/src/repo_store.rs
+++ b/ng-storage-rocksdb/src/block_storage.rs
@@ -7,7 +7,7 @@
// notice may not be copied, modified, or distributed except
// according to those terms.
-use ng_repo::store::*;
+use ng_repo::errors::StorageError;
use ng_repo::types::*;
use ng_repo::utils::*;
@@ -19,7 +19,7 @@ use serde::{Deserialize, Serialize};
use serde_bare::error::Error;
/*
#[derive(Debug)]
-pub struct LmdbRepoStore {
+pub struct LmdbBlockStorage {
/// the main store where all the repo blocks are stored
main_store: SingleStore<LmdbDatabase>,
/// store for the pin boolean, recently_used timestamp, and synced boolean
@@ -40,7 +40,7 @@ struct BlockMeta {
pub synced: bool,
}
-impl RepoStore for LmdbRepoStore {
+impl BlockStorage for LmdbBlockStorage {
/// Retrieves a block from the storage backend.
fn get(&self, block_id: &BlockId) -> Result<Block, StorageError> {
let lock = self.environment.read().unwrap();
@@ -194,10 +194,10 @@ impl RepoStore for LmdbRepoStore {
}
}
-impl LmdbRepoStore {
- /// Opens the store and returns a RepoStore object that should be kept and used to call put/get/delete/pin
+impl LmdbBlockStorage {
+ /// Opens the store and returns a BlockStorage object that should be kept and used to call put/get/delete/pin
/// The key is the encryption key for the data at rest.
- pub fn open<'a>(path: &Path, key: [u8; 32]) -> Result<LmdbRepoStore, StorageError> {
+ pub fn open<'a>(path: &Path, key: [u8; 32]) -> Result<LmdbBlockStorage, StorageError> {
let mut manager = Manager::<LmdbEnvironment>::singleton().write().unwrap();
let shared_rkv = manager
.get_or_create(path, |path| {
@@ -223,7 +223,7 @@ impl LmdbRepoStore {
let expiry_store = env.open_multi_integer("expiry", opts).unwrap();
let recently_used_store = env.open_multi_integer("recently_used", opts).unwrap();
- Ok(LmdbRepoStore {
+ Ok(LmdbBlockStorage {
environment: shared_rkv.clone(),
main_store,
meta_store,
@@ -487,8 +487,8 @@ impl LmdbRepoStore {
#[cfg(test)]
mod test {
+ use ng_repo::block_storage::*;
use ng_repo::log::*;
- use ng_repo::store::*;
use ng_repo::types::*;
use ng_repo::utils::*;
#[allow(unused_imports)]
@@ -504,7 +504,7 @@ mod test {
let key: [u8; 32] = [0; 32];
fs::create_dir_all(root.path()).unwrap();
log_debug!("{}", root.path().to_str().unwrap());
- let mut store = LmdbRepoStore::open(root.path(), key).unwrap();
+ let mut store = LmdbBlockStorage::open(root.path(), key).unwrap();
let mut now = now_timestamp();
now -= 200;
// TODO: fix the LMDB bug that is triggered with x max set to 86 !!!
@@ -537,7 +537,7 @@ mod test {
let key: [u8; 32] = [0; 32];
fs::create_dir_all(root.path()).unwrap();
log_debug!("{}", root.path().to_str().unwrap());
- let mut store = LmdbRepoStore::open(root.path(), key).unwrap();
+ let mut store = LmdbBlockStorage::open(root.path(), key).unwrap();
let mut now = now_timestamp();
now -= 200;
// TODO: fix the LMDB bug that is triggered with x max set to 86 !!!
@@ -595,7 +595,7 @@ mod test {
let key: [u8; 32] = [0; 32];
fs::create_dir_all(root.path()).unwrap();
log_debug!("{}", root.path().to_str().unwrap());
- let mut store = LmdbRepoStore::open(root.path(), key).unwrap();
+ let mut store = LmdbBlockStorage::open(root.path(), key).unwrap();
let now = now_timestamp();
let list = [
@@ -649,7 +649,7 @@ mod test {
let key: [u8; 32] = [0; 32];
fs::create_dir_all(root.path()).unwrap();
log_debug!("{}", root.path().to_str().unwrap());
- let mut store = LmdbRepoStore::open(root.path(), key).unwrap();
+ let mut store = LmdbBlockStorage::open(root.path(), key).unwrap();
let now = now_timestamp();
let list = [
@@ -696,7 +696,7 @@ mod test {
let key: [u8; 32] = [0; 32];
fs::create_dir_all(root.path()).unwrap();
log_debug!("{}", root.path().to_str().unwrap());
- let store = LmdbRepoStore::open(root.path(), key).unwrap();
+ let store = LmdbBlockStorage::open(root.path(), key).unwrap();
store.remove_expired().unwrap();
}
@@ -710,7 +710,7 @@ mod test {
log_debug!("{}", root.path().to_str().unwrap());
- let mut store = LmdbRepoStore::open(root.path(), key).unwrap();
+ let mut store = LmdbBlockStorage::open(root.path(), key).unwrap();
let block = Block::new(
Vec::new(),
diff --git a/ng-stores-rocksdb/src/kcv_store.rs b/ng-storage-rocksdb/src/kcv_storage.rs
similarity index 99%
rename from ng-stores-rocksdb/src/kcv_store.rs
rename to ng-storage-rocksdb/src/kcv_storage.rs
index 4aaa170..f569f6f 100644
--- a/ng-stores-rocksdb/src/kcv_store.rs
+++ b/ng-storage-rocksdb/src/kcv_storage.rs
@@ -7,9 +7,9 @@
// notice may not be copied, modified, or distributed except
// according to those terms.
-use ng_repo::kcv_store::*;
-use ng_repo::store::*;
+use ng_repo::kcv_storage::*;
+use ng_repo::errors::*;
use ng_repo::log::*;
use std::path::Path;
diff --git a/ng-stores-rocksdb/src/lib.rs b/ng-storage-rocksdb/src/lib.rs
similarity index 62%
rename from ng-stores-rocksdb/src/lib.rs
rename to ng-storage-rocksdb/src/lib.rs
index 7a62c33..cf018ea 100644
--- a/ng-stores-rocksdb/src/lib.rs
+++ b/ng-storage-rocksdb/src/lib.rs
@@ -1,5 +1,5 @@
#[cfg(not(target_arch = "wasm32"))]
-pub mod repo_store;
+pub mod block_storage;
#[cfg(not(target_arch = "wasm32"))]
-pub mod kcv_store;
+pub mod kcv_storage;
diff --git a/ng-verifier/Cargo.toml b/ng-verifier/Cargo.toml
index 150c45b..8421b83 100644
--- a/ng-verifier/Cargo.toml
+++ b/ng-verifier/Cargo.toml
@@ -23,3 +23,9 @@ chacha20 = "0.9.0"
serde = { version = "1.0", features = ["derive"] }
serde_bare = "0.5.0"
serde_bytes = "0.11.7"
+oxigraph = { git = "https://git.nextgraph.org/NextGraph/oxigraph.git", branch="main" }
+automerge = "0.5.9"
+yrs = "0.18.2"
+
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+ng-storage-rocksdb = { path = "../ng-storage-rocksdb", version = "0.1.0" }
\ No newline at end of file
diff --git a/ng-verifier/README.md b/ng-verifier/README.md
index 712c6f3..2133d01 100644
--- a/ng-verifier/README.md
+++ b/ng-verifier/README.md
@@ -48,7 +48,7 @@ additional terms or conditions.
NextGraph received funding through the [NGI Assure Fund](https://nlnet.nl/project/NextGraph/index.html), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's [Next Generation Internet](https://ngi.eu/) programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 957073.
-[rustc-image]: https://img.shields.io/badge/rustc-1.64+-blue.svg
+[rustc-image]: https://img.shields.io/badge/rustc-1.74+-blue.svg
[license-image]: https://img.shields.io/badge/license-Apache2.0-blue.svg
[license-link]: https://git.nextgraph.org/NextGraph/nextgraph-rs/raw/branch/master/LICENSE-APACHE2
[license-image2]: https://img.shields.io/badge/license-MIT-blue.svg
diff --git a/ng-verifier/src/lib.rs b/ng-verifier/src/lib.rs
index e69de29..f8ff6a9 100644
--- a/ng-verifier/src/lib.rs
+++ b/ng-verifier/src/lib.rs
@@ -0,0 +1,6 @@
+pub mod types;
+
+pub mod user_storage;
+
+#[cfg(not(target_family = "wasm"))]
+pub mod rocksdb_user_storage;
diff --git a/ng-verifier/src/rocksdb_user_storage.rs b/ng-verifier/src/rocksdb_user_storage.rs
new file mode 100644
index 0000000..d171005
--- /dev/null
+++ b/ng-verifier/src/rocksdb_user_storage.rs
@@ -0,0 +1,39 @@
+// Copyright (c) 2022-2024 Niko Bonnieure, Par le Peuple, NextGraph.org developers
+// All rights reserved.
+// Licensed under the Apache License, Version 2.0
+// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
+// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
+// at your option. All files in the project carrying such
+// notice may not be copied, modified, or distributed except
+// according to those terms.
+
+//! RocksDb Backend for UserStorage trait
+
+use crate::types::*;
+use crate::user_storage::*;
+use ng_repo::{errors::StorageError, types::*};
+use ng_storage_rocksdb::kcv_storage::RocksdbKCVStore;
+use std::path::PathBuf;
+use std::{
+ cmp::{max, min},
+ collections::HashMap,
+ mem::size_of_val,
+};
+
+pub(crate) struct RocksDbUserStorage {
+ user_storage: RocksdbKCVStore,
+}
+
+impl RocksDbUserStorage {
+ pub fn open(path: &PathBuf, master_key: [u8; 32]) -> Result<Self, StorageError> {
+ Ok(RocksDbUserStorage {
+ user_storage: RocksdbKCVStore::open(path, master_key)?,
+ })
+ }
+}
+
+impl UserStorage for RocksDbUserStorage {
+ fn repo_id_to_store_overlay(&self, id: &RepoId) -> Result<StoreOverlay, StorageError> {
+ unimplemented!();
+ }
+}
diff --git a/ng-verifier/src/types.rs b/ng-verifier/src/types.rs
index e69de29..43dadb4 100644
--- a/ng-verifier/src/types.rs
+++ b/ng-verifier/src/types.rs
@@ -0,0 +1,268 @@
+// Copyright (c) 2022-2024 Niko Bonnieure, Par le Peuple, NextGraph.org developers
+// All rights reserved.
+// Licensed under the Apache License, Version 2.0
+// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
+// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
+// at your option. All files in the project carrying such
+// notice may not be copied, modified, or distributed except
+// according to those terms.
+
+//! Types for Verifier
+
+use core::fmt;
+//use oxigraph::io::{RdfFormat, RdfParser, RdfSerializer};
+use oxigraph::store::Store;
+//use oxigraph::model::GroundQuad;
+#[cfg(not(target_family = "wasm"))]
+use crate::rocksdb_user_storage::RocksDbUserStorage;
+use crate::user_storage::{InMemoryUserStorage, UserStorage};
+use std::path::PathBuf;
+
+use ng_net::{
+ types::*,
+ utils::{Receiver, Sender},
+};
+use ng_repo::{
+ errors::{NgError, StorageError},
+ types::*,
+};
+use serde::{Deserialize, Serialize};
+//use yrs::{StateVector, Update};
+
+#[derive(Debug, Clone)]
+pub enum VerifierType {
+ /// nothing will be saved on disk during the session
+ Memory,
+ /// will save all user data locally, with RocksDb backend
+ RocksDb,
+ /// the verifier will be remote. a Noise connection will be opened
+ /// optional peerId to connect to. If None, will try any that has the flag `can_verify`
+ Remote(Option<PubKey>),
+ /// IndexedDb based rocksdb compiled to WASM... not ready yet. obviously. only works in the browser
+ WebRocksDb,
+}
+
+impl VerifierType {
+ pub fn is_memory(&self) -> bool {
+ match self {
+ Self::Memory => true,
+ _ => false,
+ }
+ }
+}
+
+//type LastSeqFn = fn(PubKey, u16) -> Result<u64, NgError>;
+pub type LastSeqFn = dyn Fn(PubKey, u16) -> Result<u64, NgError> + 'static + Sync + Send;
+
+// peer_id: PubKey, seq_num:u64, event_ser: vec<u8>,
+pub type OutboxWriteFn =
+ dyn Fn(PubKey, u64, Vec<u8>) -> Result<(), NgError> + 'static + Sync + Send;
+
+// peer_id: PubKey,
+pub type OutboxReadFn = dyn Fn(PubKey) -> Result<Vec<Vec<u8>>, NgError> + 'static + Sync + Send;
+
+pub struct JsSaveSessionConfig {
+ pub last_seq_function: Box<LastSeqFn>,
+ pub outbox_write_function: Box<OutboxWriteFn>,
+ pub outbox_read_function: Box<OutboxReadFn>,
+}
+
+impl fmt::Debug for JsSaveSessionConfig {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ write!(f, "JsSaveSessionConfig")
+ }
+}
+
+#[derive(Debug)]
+pub enum VerifierConfigType {
+ /// nothing will be saved on disk during the session
+ Memory,
+ JsSaveSession(JsSaveSessionConfig),
+ /// will save all user data locally, with RocksDb backend
+ RocksDb(PathBuf),
+ /// the verifier will be remote. a Noise connection will be opened
+ /// optional peerId to connect to. If None, will try any that has the flag `can_verify`
+ /// // TODO: Pass the AppConfig
+ Remote(Option<PubKey>),
+ /// IndexedDb based rocksdb compiled to WASM... not ready yet. obviously. only works in the browser
+ WebRocksDb,
+}
+
+#[derive(Debug)]
+pub struct VerifierConfig {
+ pub config_type: VerifierConfigType,
+ /// not used for Memory
+ pub user_master_key: [u8; 32],
+ /// not used for Memory
+ pub peer_priv_key: PrivKey,
+ pub user_priv_key: PrivKey,
+ pub private_store_read_cap: ObjectRef,
+}
+
+pub type CancelFn = Box<dyn FnOnce()>;
+
+pub struct Verifier {
+ pub config: VerifierConfig,
+ pub connected_server_id: Option<PubKey>,
+ graph_dataset: Option<Store>,
+ user_storage: Option<Box<dyn UserStorage>>,
+}
+
+impl fmt::Debug for Verifier {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ writeln!(f, "Verifier\nconfig: {:?}", self.config)?;
+ writeln!(f, "connected_server_id: {:?}", self.connected_server_id)
+ }
+}
+
+impl Verifier {
+ pub fn new(config: VerifierConfig) -> Result<Self, StorageError> {
+ let (graph, user) = match &config.config_type {
+ VerifierConfigType::Memory | VerifierConfigType::JsSaveSession(_) => (
+ Some(Store::new().unwrap()),
+ Some(Box::new(InMemoryUserStorage::new()) as Box<dyn UserStorage>),
+ ),
+ #[cfg(not(target_family = "wasm"))]
+ VerifierConfigType::RocksDb(path) => (
+ // FIXME BIG TIME: we are reusing the same encryption key here.
+ // this is very temporary, until we remove the code in oxi_rocksdb of oxigraph,
+ // and have oxigraph use directly the UserStorage
+ Some(Store::open_with_key(path, config.user_master_key).unwrap()),
+ Some(
+ Box::new(RocksDbUserStorage::open(path, config.user_master_key)?)
+ as Box<dyn UserStorage>,
+ ),
+ ),
+ VerifierConfigType::Remote(_) => (None, None),
+ _ => unimplemented!(), // can be WebRocksDb or RocksDb on wasm platforms
+ };
+ Ok(Verifier {
+ config,
+ connected_server_id: None,
+ graph_dataset: graph,
+ user_storage: user,
+ })
+ }
+
+ pub fn doc_fetch(
+ &self,
+ nuri: String,
+ payload: Option<AppRequestPayload>,
+ ) -> Result<(Receiver<AppResponse>, CancelFn), NgError> {
+ unimplemented!();
+ }
+}
+
+//
+// APP PROTOCOL (between APP and VERIFIER)
+//
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct AppRequestV0 {}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum AppRequest {
+ V0(AppRequestV0),
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct GraphUpdate {
+ sparql_update: String,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum DiscreteUpdate {
+ /// A yrs::Update
+ YMap(Vec<u8>),
+ YXml(Vec<u8>),
+ YText(Vec<u8>),
+ /// An automerge::Patch
+ Automerge(Vec<u8>),
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct AppUpdate {
+ heads: Vec<ObjectId>,
+ graph: Option<GraphUpdate>,
+ discrete: Option<DiscreteUpdate>,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct AppCreate {
+ store: StoreRepo,
+ content_type: BranchContentType,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct AppDelete {
+ /// Nuri of doc to delete
+ nuri: String,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum AppRequestPayloadV0 {
+ Create(AppCreate),
+ Update(AppUpdate),
+ Delete(AppDelete),
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum AppRequestPayload {
+ V0(AppRequestPayloadV0),
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum DiscretePatch {
+ /// A yrs::Update
+ YMap(Vec<u8>),
+ YXml(Vec<u8>),
+ YText(Vec<u8>),
+ /// An automerge::Patch
+ Automerge(Vec<u8>),
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct GraphPatch {
+ /// oxigraph::model::GroundQuad serialized in turtle with oxrdfio
+ pub adds: Vec<String>,
+ pub removes: Vec<String>,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum DiscreteState {
+ /// A yrs::StateVector
+ YMap(Vec<u8>),
+ YXml(Vec<u8>),
+ YText(Vec<u8>),
+ // the output of Automerge::save()
+ Automerge(Vec<u8>),
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct GraphState {
+ pub tuples: Vec<String>,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct AppState {
+ heads: Vec<ObjectId>,
+ graph: Option<GraphState>, // there is always a graph present in the branch. but it might not have been asked in the request
+ discrete: Option<DiscreteState>,
+}
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct AppPatch {
+ heads: Vec<ObjectId>,
+ graph: Option<GraphPatch>,
+ discrete: Option<DiscretePatch>,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum AppResponseV0 {
+ State(AppState),
+ Patch(AppPatch),
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum AppResponse {
+ V0(AppResponseV0),
+}
diff --git a/ng-verifier/src/user_storage.rs b/ng-verifier/src/user_storage.rs
new file mode 100644
index 0000000..a1a2a08
--- /dev/null
+++ b/ng-verifier/src/user_storage.rs
@@ -0,0 +1,46 @@
+// Copyright (c) 2022-2024 Niko Bonnieure, Par le Peuple, NextGraph.org developers
+// All rights reserved.
+// Licensed under the Apache License, Version 2.0
+// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
+// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
+// at your option. All files in the project carrying such
+// notice may not be copied, modified, or distributed except
+// according to those terms.
+
+//! Storage of user application data (RDF, content of rich-text document, etc)
+
+use ng_repo::{errors::StorageError, types::*};
+
+use crate::types::*;
+use std::{
+ cmp::{max, min},
+ collections::HashMap,
+ mem::size_of_val,
+};
+
+pub trait UserStorage: Send + Sync {
+ /// Gets the StoreRepo for a given RepoId
+ fn repo_id_to_store_overlay(&self, id: &RepoId) -> Result<StoreOverlay, StorageError>;
+}
+
+pub(crate) struct InMemoryUserStorage {
+ repo_id_to_store_overlay: HashMap<RepoId, StoreOverlay>,
+}
+
+impl InMemoryUserStorage {
+ pub fn new() -> Self {
+ InMemoryUserStorage {
+ repo_id_to_store_overlay: HashMap::new(),
+ }
+ }
+}
+
+impl UserStorage for InMemoryUserStorage {
+ fn repo_id_to_store_overlay(&self, id: &RepoId) -> Result<StoreOverlay, StorageError> {
+ Ok(self
+ .repo_id_to_store_overlay
+ .get(&id)
+ .ok_or(StorageError::NotFound)?
+ .to_owned())
+ }
+}
diff --git a/ng-wallet/README.md b/ng-wallet/README.md
index 69485e9..e34735b 100644
--- a/ng-wallet/README.md
+++ b/ng-wallet/README.md
@@ -48,7 +48,7 @@ additional terms or conditions.
NextGraph received funding through the [NGI Assure Fund](https://nlnet.nl/project/NextGraph/index.html), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's [Next Generation Internet](https://ngi.eu/) programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 957073.
-[rustc-image]: https://img.shields.io/badge/rustc-1.64+-blue.svg
+[rustc-image]: https://img.shields.io/badge/rustc-1.74+-blue.svg
[license-image]: https://img.shields.io/badge/license-Apache2.0-blue.svg
[license-link]: https://git.nextgraph.org/NextGraph/nextgraph-rs/raw/branch/master/LICENSE-APACHE2
[license-image2]: https://img.shields.io/badge/license-MIT-blue.svg
diff --git a/ng-wallet/src/types.rs b/ng-wallet/src/types.rs
index 37f2399..5c65bc4 100644
--- a/ng-wallet/src/types.rs
+++ b/ng-wallet/src/types.rs
@@ -140,6 +140,12 @@ impl SessionWalletStorageV0 {
// }
}
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct SessionInfo {
+ pub session_id: u8,
+ pub user: UserId,
+}
+
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct SessionPeerStorageV0 {
pub user: UserId,
@@ -173,7 +179,7 @@ pub struct LocalClientStorageV0 {
impl LocalClientStorageV0 {
fn crypt(text: &mut Vec<u8>, client: ClientId, wallet_privkey: PrivKey) {
let client_ser = serde_bare::to_vec(&client).unwrap();
- let mut wallet_privkey_ser = serde_bare::to_vec(&wallet_privkey).unwrap();
+ let wallet_privkey_ser = serde_bare::to_vec(&wallet_privkey).unwrap();
let mut key_material = [client_ser, wallet_privkey_ser].concat();
let mut key: [u8; 32] = blake3::derive_key(
@@ -462,6 +468,17 @@ impl SensitiveWallet {
Self::V0(v0) => v0.client = Some(client),
}
}
+ pub fn individual_site(&self, user_id: &UserId) -> Option<&(PrivKey, ReadCap)> {
+ match self {
+ Self::V0(v0) => match v0.sites.get(&user_id.to_string()) {
+ Some(site) => match &site.site_type {
+ SiteType::Individual(creds) => Some(creds),
+ _ => None,
+ },
+ None => None,
+ },
+ }
+ }
pub fn has_user(&self, user_id: &UserId) -> bool {
match self {
Self::V0(v0) => v0.sites.get(&user_id.to_string()).is_some(),
diff --git a/ngaccount/Cargo.toml b/ngaccount/Cargo.toml
index 14b7e2a..7c0648d 100644
--- a/ngaccount/Cargo.toml
+++ b/ngaccount/Cargo.toml
@@ -32,4 +32,4 @@ base64-url = "2.0.0"
serde_json = "1.0.96"
bytes = "1.0"
anyhow = "1.0.71"
-duration-str = "0.5.1"
\ No newline at end of file
+duration-str = "0.7.1"
\ No newline at end of file
diff --git a/ngcli/Cargo.toml b/ngcli/Cargo.toml
index 45e7980..962afcc 100644
--- a/ngcli/Cargo.toml
+++ b/ngcli/Cargo.toml
@@ -37,4 +37,4 @@ blake3 = "1.3.1"
serde = { version = "1.0", features = ["derive"] }
serde_bare = "0.5.0"
serde_bytes = "0.11.7"
-duration-str = "0.5.1"
\ No newline at end of file
+duration-str = "0.7.1"
\ No newline at end of file
diff --git a/ngcli/README.md b/ngcli/README.md
index 5d41a28..f06c43e 100644
--- a/ngcli/README.md
+++ b/ngcli/README.md
@@ -50,7 +50,7 @@ additional terms or conditions.
NextGraph received funding through the [NGI Assure Fund](https://nlnet.nl/project/NextGraph/index.html), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's [Next Generation Internet](https://ngi.eu/) programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 957073.
-[rustc-image]: https://img.shields.io/badge/rustc-1.64+-blue.svg
+[rustc-image]: https://img.shields.io/badge/rustc-1.74+-blue.svg
[license-image]: https://img.shields.io/badge/license-Apache2.0-blue.svg
[license-link]: https://git.nextgraph.org/NextGraph/nextgraph-rs/raw/branch/master/LICENSE-APACHE2
[license-image2]: https://img.shields.io/badge/license-MIT-blue.svg
diff --git a/ngcli/src/main.rs b/ngcli/src/main.rs
index d27bbce..a836b31 100644
--- a/ngcli/src/main.rs
+++ b/ngcli/src/main.rs
@@ -12,7 +12,9 @@ use ed25519_dalek::*;
use duration_str::parse;
use futures::{future, pin_mut, stream, SinkExt, StreamExt};
use ng_net::actors::*;
-use ng_repo::store::{store_max_value_size, store_valid_value_size, HashMapRepoStore, RepoStore};
+use ng_repo::block_storage::{
+ store_max_value_size, store_valid_value_size, BlockStorage, HashMapBlockStorage,
+};
use rand::rngs::OsRng;
use serde::{Deserialize, Serialize};
use serde_json::{from_str, to_string_pretty};
diff --git a/ngcli/src/old.rs b/ngcli/src/old.rs
index 1c7f541..df79027 100644
--- a/ngcli/src/old.rs
+++ b/ngcli/src/old.rs
@@ -10,7 +10,7 @@ async fn test_sync(cnx: &mut impl BrokerConnection, user_pub_key: PubKey, userpr
expiry: Option<Timestamp>,
repo_pubkey: PubKey,
repo_secret: SymKey,
- store: &mut impl RepoStore,
+ store: &mut impl BlockStorage,
) -> ObjectRef {
let max_object_size = 4000;
let obj = Object::new(
@@ -38,7 +38,7 @@ async fn test_sync(cnx: &mut impl BrokerConnection, user_pub_key: PubKey, userpr
body_ref: ObjectRef,
repo_pubkey: PubKey,
repo_secret: SymKey,
- store: &mut impl RepoStore,
+ store: &mut impl BlockStorage,
) -> ObjectRef {
let mut obj_deps: Vec<ObjectId> = vec![];
obj_deps.extend(deps.iter().map(|r| r.id));
@@ -80,7 +80,7 @@ async fn test_sync(cnx: &mut impl BrokerConnection, user_pub_key: PubKey, userpr
branch: Branch,
repo_pubkey: PubKey,
repo_secret: SymKey,
- store: &mut impl RepoStore,
+ store: &mut impl BlockStorage,
) -> ObjectRef {
let deps = vec![];
let expiry = None;
@@ -100,7 +100,7 @@ async fn test_sync(cnx: &mut impl BrokerConnection, user_pub_key: PubKey, userpr
deps: Vec<ObjectId>,
repo_pubkey: PubKey,
repo_secret: SymKey,
- store: &mut impl RepoStore,
+ store: &mut impl BlockStorage,
) -> ObjectRef {
let expiry = None;
let content = [7u8; 777].to_vec();
@@ -120,7 +120,7 @@ async fn test_sync(cnx: &mut impl BrokerConnection, user_pub_key: PubKey, userpr
deps: Vec<ObjectId>,
repo_pubkey: PubKey,
repo_secret: SymKey,
- store: &mut impl RepoStore,
+ store: &mut impl BlockStorage,
) -> ObjectRef {
let expiry = None;
let body = CommitBody::Ack(Ack::V0());
@@ -135,7 +135,7 @@ async fn test_sync(cnx: &mut impl BrokerConnection, user_pub_key: PubKey, userpr
)
}
- let mut store = HashMapRepoStore::new();
+ let mut store = HashMapBlockStorage::new();
let mut rng = OsRng {};
// repo
@@ -346,7 +346,7 @@ async fn test_sync(cnx: &mut impl BrokerConnection, user_pub_key: PubKey, userpr
// Now emptying the local store of the client, and adding only 1 commit into it (br)
// we also have received an commit (t5) but we don't know what to do with it...
- let mut store = HashMapRepoStore::new();
+ let mut store = HashMapBlockStorage::new();
let br = add_commit(
branch_body,
diff --git a/ngd/README.md b/ngd/README.md
index de3e264..34197df 100644
--- a/ngd/README.md
+++ b/ngd/README.md
@@ -108,7 +108,7 @@ additional terms or conditions.
NextGraph received funding through the [NGI Assure Fund](https://nlnet.nl/project/NextGraph/index.html), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's [Next Generation Internet](https://ngi.eu/) programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 957073.
-[rustc-image]: https://img.shields.io/badge/rustc-1.64+-blue.svg
+[rustc-image]: https://img.shields.io/badge/rustc-1.74+-blue.svg
[license-image]: https://img.shields.io/badge/license-Apache2.0-blue.svg
[license-link]: https://git.nextgraph.org/NextGraph/nextgraph-rs/raw/branch/master/LICENSE-APACHE2
[license-image2]: https://img.shields.io/badge/license-MIT-blue.svg
diff --git a/ngone/Cargo.toml b/ngone/Cargo.toml
index 9794dd3..e92dccd 100644
--- a/ngone/Cargo.toml
+++ b/ngone/Cargo.toml
@@ -20,7 +20,7 @@ warp-embed = "0.4"
rust-embed = "6"
log = "0.4"
env_logger = "0.10"
-ng-stores-rocksdb = { path = "../ng-stores-rocksdb" }
+ng-storage-rocksdb = { path = "../ng-storage-rocksdb" }
ng-repo = { path = "../ng-repo", features = ["server_log_output"] }
ng-net = { path = "../ng-net" }
ng-wallet = { path = "../ng-wallet" }
diff --git a/ngone/src/main.rs b/ngone/src/main.rs
index 7a617ad..64d5d73 100644
--- a/ngone/src/main.rs
+++ b/ngone/src/main.rs
@@ -12,7 +12,7 @@ extern crate slice_as_array;
mod store;
mod types;
-use ng_repo::store::StorageError;
+use ng_repo::errors::StorageError;
use warp::reply::Response;
use warp::{Filter, Reply};
@@ -28,7 +28,7 @@ use ng_net::types::{APP_NG_ONE_URL, NG_ONE_URL};
use ng_repo::log::*;
use ng_repo::types::*;
use ng_repo::utils::{generate_keypair, sign, verify};
-use ng_stores_rocksdb::kcv_store::RocksdbKCVStore;
+use ng_storage_rocksdb::kcv_storage::RocksdbKCVStore;
use ng_wallet::types::*;
#[derive(RustEmbed)]
diff --git a/ngone/src/store/dynpeer.rs b/ngone/src/store/dynpeer.rs
index 074489e..0490d85 100644
--- a/ngone/src/store/dynpeer.rs
+++ b/ngone/src/store/dynpeer.rs
@@ -10,8 +10,8 @@
//! ng-one bootstrap
use ng_net::types::*;
-use ng_repo::kcv_store::KCVStore;
-use ng_repo::store::*;
+use ng_repo::errors::StorageError;
+use ng_repo::kcv_storage::KCVStore;
use ng_repo::types::PubKey;
use serde::{Deserialize, Serialize};
diff --git a/ngone/src/store/wallet_record.rs b/ngone/src/store/wallet_record.rs
index 01aa3f3..fc5aee3 100644
--- a/ngone/src/store/wallet_record.rs
+++ b/ngone/src/store/wallet_record.rs
@@ -9,8 +9,8 @@
//! ng-wallet
-use ng_repo::kcv_store::KCVStore;
-use ng_repo::store::*;
+use ng_repo::errors::StorageError;
+use ng_repo::kcv_storage::KCVStore;
use ng_repo::types::*;
use ng_wallet::types::*;
use serde::{Deserialize, Serialize};