From 9370a9216eddc345c58b70a8637cc6ec30eef92f Mon Sep 17 00:00:00 2001 From: Niko PLP Date: Wed, 10 Apr 2024 19:49:56 +0300 Subject: [PATCH] ng-verifier, integration of oxigraph, yrs, automerge --- Cargo.lock | 423 +++++++- Cargo.toml | 6 +- README.md | 8 +- nextgraph/Cargo.toml | 1 - nextgraph/README.md | 2 +- nextgraph/examples/in_memory.rs | 6 +- nextgraph/examples/persistent.rs | 5 +- nextgraph/src/lib.rs | 2 +- nextgraph/src/local_broker.rs | 397 +++++-- ng-app/README.md | 4 +- ng-app/prepare-app-file.cjs | 2 + ng-app/src-tauri/src/lib.rs | 21 +- ng-app/src/api.ts | 13 +- ng-broker/Cargo.toml | 2 +- ng-broker/README.md | 2 +- .../account.rs | 10 +- .../config.rs | 4 +- .../invitation.rs | 8 +- .../{broker_store => broker_storage}/mod.rs | 2 - .../overlay.rs | 4 +- .../{broker_store => broker_storage}/peer.rs | 4 +- .../{broker_store => broker_storage}/topic.rs | 4 +- .../wallet.rs | 6 +- ng-broker/src/broker_store/repostoreinfo.rs | 100 -- ng-broker/src/lib.rs | 2 +- ng-broker/src/server.rs | 973 ------------------ ng-broker/src/server_storage.rs | 12 +- ng-client-ws/README.md | 2 +- ng-net/Cargo.toml | 2 +- ng-net/README.md | 2 +- ng-net/src/broker.rs | 9 +- ng-net/src/broker_connection.rs | 2 +- ng-net/src/errors.rs | 3 +- ng-net/src/types.rs | 29 +- ng-repo/README.md | 2 +- ng-repo/src/{store.rs => block_storage.rs} | 36 +- ng-repo/src/branch.rs | 25 +- ng-repo/src/commit.rs | 42 +- ng-repo/src/errors.rs | 25 +- ng-repo/src/event.rs | 6 +- ng-repo/src/file.rs | 60 +- ng-repo/src/{kcv_store.rs => kcv_storage.rs} | 2 +- ng-repo/src/lib.rs | 4 +- ng-repo/src/object.rs | 19 +- ng-repo/src/repo.rs | 19 +- ng-repo/src/types.rs | 43 +- ng-repo/src/utils.rs | 6 +- ng-sdk-js/js/browser.js | 15 + ng-sdk-js/js/node.js | 10 + ng-sdk-js/src/lib.rs | 43 +- .../Cargo.toml | 2 +- .../src/block_storage.rs | 26 +- .../src/kcv_storage.rs | 0 .../src/lib.rs | 0 .../Cargo.toml | 2 +- .../README.md | 4 +- .../src/block_storage.rs | 28 +- .../src/kcv_storage.rs | 4 +- .../src/lib.rs | 4 +- ng-verifier/Cargo.toml | 6 + ng-verifier/README.md | 2 +- ng-verifier/src/lib.rs | 6 + ng-verifier/src/rocksdb_user_storage.rs | 39 + ng-verifier/src/types.rs | 268 +++++ ng-verifier/src/user_storage.rs | 46 + ng-wallet/README.md | 2 +- ng-wallet/src/types.rs | 19 +- ngaccount/Cargo.toml | 2 +- ngcli/Cargo.toml | 2 +- ngcli/README.md | 2 +- ngcli/src/main.rs | 4 +- ngcli/src/old.rs | 14 +- ngd/README.md | 2 +- ngone/Cargo.toml | 2 +- ngone/src/main.rs | 4 +- ngone/src/store/dynpeer.rs | 4 +- ngone/src/store/wallet_record.rs | 4 +- 77 files changed, 1479 insertions(+), 1448 deletions(-) rename ng-broker/src/{broker_store => broker_storage}/account.rs (97%) rename ng-broker/src/{broker_store => broker_storage}/config.rs (97%) rename ng-broker/src/{broker_store => broker_storage}/invitation.rs (97%) rename ng-broker/src/{broker_store => broker_storage}/mod.rs (83%) rename ng-broker/src/{broker_store => broker_storage}/overlay.rs (98%) rename ng-broker/src/{broker_store => broker_storage}/peer.rs (98%) rename ng-broker/src/{broker_store => broker_storage}/topic.rs (98%) rename ng-broker/src/{broker_store => broker_storage}/wallet.rs (97%) delete mode 100644 ng-broker/src/broker_store/repostoreinfo.rs delete mode 100644 ng-broker/src/server.rs rename ng-repo/src/{store.rs => block_storage.rs} (85%) rename ng-repo/src/{kcv_store.rs => kcv_storage.rs} (99%) rename {ng-stores-lmdb => ng-storage-lmdb}/Cargo.toml (96%) rename ng-stores-lmdb/src/repo_store.rs => ng-storage-lmdb/src/block_storage.rs (97%) rename ng-stores-lmdb/src/kcv_store.rs => ng-storage-lmdb/src/kcv_storage.rs (100%) rename {ng-stores-lmdb => ng-storage-lmdb}/src/lib.rs (100%) rename {ng-stores-rocksdb => ng-storage-rocksdb}/Cargo.toml (95%) rename {ng-stores-rocksdb => ng-storage-rocksdb}/README.md (97%) rename ng-stores-rocksdb/src/repo_store.rs => ng-storage-rocksdb/src/block_storage.rs (97%) rename ng-stores-rocksdb/src/kcv_store.rs => ng-storage-rocksdb/src/kcv_storage.rs (99%) rename {ng-stores-rocksdb => ng-storage-rocksdb}/src/lib.rs (62%) create mode 100644 ng-verifier/src/rocksdb_user_storage.rs create mode 100644 ng-verifier/src/user_storage.rs diff --git a/Cargo.lock b/Cargo.lock index d623a89..e41c72f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -181,6 +181,12 @@ version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" +[[package]] +name = "arc-swap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" + [[package]] name = "argon2" version = "0.5.0" @@ -270,7 +276,7 @@ dependencies = [ "async-lock", "async-task", "concurrent-queue", - "fastrand", + "fastrand 1.9.0", "futures-lite", "slab", ] @@ -384,7 +390,7 @@ checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.58", ] [[package]] @@ -429,7 +435,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.58", ] [[package]] @@ -496,12 +502,40 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1181e1e0d1fce796a03db1ae795d67167da795f9cf4a39c37589e85ef57f26d3" +[[package]] +name = "atomic_refcell" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41e67cd8309bbd06cd603a9e693a784ac2e5d1e955f11286e355089fcab3047c" + [[package]] name = "autocfg" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "automerge" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93b5e6ed2097a1e55cce3128d64c909cdb42c800d4880411c7382f3dfa2c808d" +dependencies = [ + "flate2", + "fxhash", + "hex", + "im", + "itertools 0.12.1", + "leb128", + "serde", + "sha2 0.10.7", + "smol_str", + "thiserror", + "tinyvec", + "tracing", + "unicode-segmentation", + "uuid", +] + [[package]] name = "backtrace" version = "0.3.69" @@ -562,7 +596,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.18", + "syn 2.0.58", ] [[package]] @@ -577,6 +611,15 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitmaps" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2" +dependencies = [ + "typenum", +] + [[package]] name = "blake2" version = "0.10.6" @@ -644,7 +687,7 @@ dependencies = [ "async-lock", "async-task", "atomic-waker", - "fastrand", + "fastrand 1.9.0", "futures-lite", "log", ] @@ -901,7 +944,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.58", ] [[package]] @@ -1187,7 +1230,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" dependencies = [ "quote", - "syn 2.0.18", + "syn 2.0.58", ] [[package]] @@ -1274,7 +1317,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.18", + "syn 2.0.58", ] [[package]] @@ -1285,7 +1328,7 @@ checksum = "29a358ff9f12ec09c3e61fef9b5a9902623a695a46a917b07f269bff1445611a" dependencies = [ "darling_core", "quote", - "syn 2.0.18", + "syn 2.0.58", ] [[package]] @@ -1433,9 +1476,9 @@ checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b" [[package]] name = "duration-str" -version = "0.5.1" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9f037c488d179e21c87ef5fa9c331e8e62f5dddfa84618b41bb197da03edff1" +checksum = "a8bb6a301a95ba86fa0ebaf71d49ae4838c51f8b84cb88ed140dfb66452bb3c4" dependencies = [ "chrono", "nom", @@ -1520,7 +1563,7 @@ checksum = "5e9a1f9f7d83e59740248a6e14ecf93929ade55027844dfcea78beafccc15745" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.58", ] [[package]] @@ -1628,6 +1671,15 @@ dependencies = [ "instant", ] +[[package]] +name = "fastrand" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "658bd65b1cf4c852a3cc96f18a8ce7b5640f6b703f905c7d74532294c2a63984" +dependencies = [ + "getrandom 0.2.10", +] + [[package]] name = "fdeflate" version = "0.3.0" @@ -1734,7 +1786,7 @@ checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.58", ] [[package]] @@ -1822,7 +1874,7 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" dependencies = [ - "fastrand", + "fastrand 1.9.0", "futures-core", "futures-io", "memchr", @@ -1839,7 +1891,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.58", ] [[package]] @@ -2493,6 +2545,20 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "im" +version = "15.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0acd33ff0285af998aaf9b57342af478078f53492322fafc47450e09397e0e9" +dependencies = [ + "bitmaps", + "rand_core 0.6.4", + "rand_xoshiro", + "sized-chunks", + "typenum", + "version_check", +] + [[package]] name = "image" version = "0.24.6" @@ -2588,6 +2654,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "0.4.8" @@ -2672,6 +2747,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json-event-parser" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f850fafca79ebacd70eab9d80cb75a33aeda38bde8f3dd784c1837cdf0bde631" + [[package]] name = "json-patch" version = "1.0.0" @@ -2729,6 +2810,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" +[[package]] +name = "leb128" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" + [[package]] name = "lebe" version = "0.5.2" @@ -2784,7 +2871,7 @@ checksum = "7fc7aa29613bd6a620df431842069224d8bc9011086b1db4c0e0cd47fa03ec9a" [[package]] name = "librocksdb-sys" version = "0.11.0+8.3.2" -source = "git+https://git.nextgraph.org/NextGraph/rust-rocksdb.git?branch=master#846abdaaa238ad0005d2bcd16a4f6339c96b3a9f" +source = "git+https://git.nextgraph.org/NextGraph/rust-rocksdb.git?branch=master#13b3c2022202abff8cfe921ee926d6ca567e66e8" dependencies = [ "bindgen", "bzip2-sys", @@ -2900,6 +2987,16 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest 0.10.7", +] + [[package]] name = "memalloc" version = "0.1.0" @@ -3130,7 +3227,6 @@ dependencies = [ "ng-client-ws", "ng-net", "ng-repo", - "ng-stores-rocksdb", "ng-verifier", "ng-wallet", "once_cell", @@ -3177,7 +3273,7 @@ dependencies = [ "ng-client-ws", "ng-net", "ng-repo", - "ng-stores-rocksdb", + "ng-storage-rocksdb", "once_cell", "rust-embed", "serde", @@ -3302,7 +3398,7 @@ dependencies = [ ] [[package]] -name = "ng-stores-rocksdb" +name = "ng-storage-rocksdb" version = "0.1.0" dependencies = [ "hex", @@ -3317,13 +3413,17 @@ dependencies = [ name = "ng-verifier" version = "0.1.0" dependencies = [ + "automerge", "blake3", "chacha20", "ng-net", "ng-repo", + "ng-storage-rocksdb", + "oxigraph", "serde", "serde_bare", "serde_bytes", + "yrs", ] [[package]] @@ -3437,7 +3537,7 @@ dependencies = [ "log", "ng-net", "ng-repo", - "ng-stores-rocksdb", + "ng-storage-rocksdb", "ng-wallet", "rust-embed", "serde", @@ -3658,7 +3758,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.58", ] [[package]] @@ -3716,6 +3816,104 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "oxigraph" +version = "0.4.0-alpha.7-dev" +source = "git+https://git.nextgraph.org/NextGraph/oxigraph.git?branch=main#b3ae51da8274693b75622349b4bba46efcaa8c3f" +dependencies = [ + "digest 0.10.7", + "getrandom 0.2.10", + "hex", + "json-event-parser", + "libc", + "md-5", + "oxilangtag", + "oxiri", + "oxrdf", + "oxrdfio", + "oxsdatatypes", + "rand 0.8.5", + "regex", + "rocksdb", + "sha1", + "sha2 0.10.7", + "siphasher", + "sparesults", + "spargebra", + "sparopt", + "thiserror", +] + +[[package]] +name = "oxilangtag" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f3f87617a86af77fa3691e6350483e7154c2ead9f1261b75130e21ca0f8acb" +dependencies = [ + "serde", +] + +[[package]] +name = "oxiri" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05417ee46e2eb40dd9d590b4d67fc2408208b3a48a6b7f71d2bc1d7ce12a3e0" + +[[package]] +name = "oxrdf" +version = "0.2.0-alpha.4" +source = "git+https://git.nextgraph.org/NextGraph/oxigraph.git?branch=main#b3ae51da8274693b75622349b4bba46efcaa8c3f" +dependencies = [ + "oxilangtag", + "oxiri", + "oxsdatatypes", + "rand 0.8.5", + "thiserror", +] + +[[package]] +name = "oxrdfio" +version = "0.1.0-alpha.5" +source = "git+https://git.nextgraph.org/NextGraph/oxigraph.git?branch=main#b3ae51da8274693b75622349b4bba46efcaa8c3f" +dependencies = [ + "oxrdf", + "oxrdfxml", + "oxttl", + "thiserror", +] + +[[package]] +name = "oxrdfxml" +version = "0.1.0-alpha.5" +source = "git+https://git.nextgraph.org/NextGraph/oxigraph.git?branch=main#b3ae51da8274693b75622349b4bba46efcaa8c3f" +dependencies = [ + "oxilangtag", + "oxiri", + "oxrdf", + "quick-xml 0.31.0", + "thiserror", +] + +[[package]] +name = "oxsdatatypes" +version = "0.2.0-alpha.1" +source = "git+https://git.nextgraph.org/NextGraph/oxigraph.git?branch=main#b3ae51da8274693b75622349b4bba46efcaa8c3f" +dependencies = [ + "thiserror", +] + +[[package]] +name = "oxttl" +version = "0.1.0-alpha.5" +source = "git+https://git.nextgraph.org/NextGraph/oxigraph.git?branch=main#b3ae51da8274693b75622349b4bba46efcaa8c3f" +dependencies = [ + "memchr", + "oxilangtag", + "oxiri", + "oxrdf", + "thiserror", +] + [[package]] name = "packed_simd_2" version = "0.3.8" @@ -3816,6 +4014,33 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" +[[package]] +name = "peg" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "400bcab7d219c38abf8bd7cc2054eb9bbbd4312d66f6a5557d572a203f646f61" +dependencies = [ + "peg-macros", + "peg-runtime", +] + +[[package]] +name = "peg-macros" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46e61cce859b76d19090f62da50a9fe92bab7c2a5f09e183763559a2ac392c90" +dependencies = [ + "peg-runtime", + "proc-macro2", + "quote", +] + +[[package]] +name = "peg-runtime" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36bae92c60fa2398ce4678b98b2c4b5a7c61099961ca1fa305aec04a9ad28922" + [[package]] name = "percent-encoding" version = "2.3.0" @@ -3957,7 +4182,7 @@ checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.58", ] [[package]] @@ -3993,7 +4218,7 @@ dependencies = [ "base64 0.21.2", "indexmap", "line-wrap", - "quick-xml", + "quick-xml 0.28.2", "serde", "time 0.3.23", ] @@ -4070,7 +4295,7 @@ checksum = "09963355b9f467184c04017ced4a2ba2d75cbcb4e7462690d388233253d4b1a9" dependencies = [ "anstyle", "difflib", - "itertools", + "itertools 0.10.5", "predicates-core", ] @@ -4097,7 +4322,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9825a04601d60621feed79c4e6b56d65db77cdca55cef43b46b0de1096d1c282" dependencies = [ "proc-macro2", - "syn 2.0.18", + "syn 2.0.58", ] [[package]] @@ -4142,9 +4367,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.60" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" +checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" dependencies = [ "unicode-ident", ] @@ -4182,11 +4407,20 @@ dependencies = [ "memchr", ] +[[package]] +name = "quick-xml" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" +dependencies = [ + "memchr", +] + [[package]] name = "quote" -version = "1.0.28" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] @@ -4281,6 +4515,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_xoshiro" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" +dependencies = [ + "rand_core 0.6.4", +] + [[package]] name = "raw-window-handle" version = "0.5.2" @@ -4412,7 +4655,7 @@ dependencies = [ [[package]] name = "rocksdb" version = "0.21.0" -source = "git+https://git.nextgraph.org/NextGraph/rust-rocksdb.git?branch=master#846abdaaa238ad0005d2bcd16a4f6339c96b3a9f" +source = "git+https://git.nextgraph.org/NextGraph/rust-rocksdb.git?branch=master#13b3c2022202abff8cfe921ee926d6ca567e66e8" dependencies = [ "libc", "librocksdb-sys", @@ -4438,7 +4681,7 @@ dependencies = [ "proc-macro2", "quote", "rust-embed-utils", - "syn 2.0.18", + "syn 2.0.58", "walkdir", ] @@ -4680,7 +4923,7 @@ checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.58", ] [[package]] @@ -4702,7 +4945,7 @@ checksum = "bcec881020c684085e55a25f7fd888954d56609ef363479dc5a1305eb0d40cab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.58", ] [[package]] @@ -4751,7 +4994,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.58", ] [[package]] @@ -4873,6 +5116,16 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" +[[package]] +name = "sized-chunks" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16d69225bde7a69b235da73377861095455d298f2b970996eec25ddbb42b3d1e" +dependencies = [ + "bitmaps", + "typenum", +] + [[package]] name = "slab" version = "0.4.8" @@ -4888,11 +5141,29 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64c963ee59ddedb5ab95dc2cd97c48b4a292572a52c5636fbbabdb9985bfe4c3" +[[package]] +name = "smallstr" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b1aefdf380735ff8ded0b15f31aab05daf1f70216c01c02a12926badd1df9d" +dependencies = [ + "smallvec", +] + [[package]] name = "smallvec" -version = "1.10.0" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "smol_str" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +checksum = "e6845563ada680337a52d43bb0b29f396f2d911616f6573012645b9e3d048a49" +dependencies = [ + "serde", +] [[package]] name = "socket2" @@ -4932,6 +5203,41 @@ dependencies = [ "system-deps", ] +[[package]] +name = "sparesults" +version = "0.2.0-alpha.4" +source = "git+https://git.nextgraph.org/NextGraph/oxigraph.git?branch=main#b3ae51da8274693b75622349b4bba46efcaa8c3f" +dependencies = [ + "json-event-parser", + "memchr", + "oxrdf", + "quick-xml 0.31.0", + "thiserror", +] + +[[package]] +name = "spargebra" +version = "0.3.0-alpha.4" +source = "git+https://git.nextgraph.org/NextGraph/oxigraph.git?branch=main#b3ae51da8274693b75622349b4bba46efcaa8c3f" +dependencies = [ + "oxilangtag", + "oxiri", + "oxrdf", + "peg", + "rand 0.8.5", + "thiserror", +] + +[[package]] +name = "sparopt" +version = "0.1.0-alpha.5-dev" +source = "git+https://git.nextgraph.org/NextGraph/oxigraph.git?branch=main#b3ae51da8274693b75622349b4bba46efcaa8c3f" +dependencies = [ + "oxrdf", + "rand 0.8.5", + "spargebra", +] + [[package]] name = "spin" version = "0.9.8" @@ -5024,9 +5330,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.18" +version = "2.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +checksum = "44cfb93f38070beee36b3fef7d4f5a16f27751d94b187b666a5cc5e9b0d30687" dependencies = [ "proc-macro2", "quote", @@ -5354,7 +5660,7 @@ checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" dependencies = [ "autocfg", "cfg-if", - "fastrand", + "fastrand 1.9.0", "redox_syscall 0.3.5", "rustix", "windows-sys 0.48.0", @@ -5394,22 +5700,22 @@ checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c" [[package]] name = "thiserror" -version = "1.0.40" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.40" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.58", ] [[package]] @@ -5542,7 +5848,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.58", ] [[package]] @@ -5653,7 +5959,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.58", ] [[package]] @@ -5877,6 +6183,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fa2982af2eec27de306107c027578ff7f423d65f7250e40ce0fea8f45248b81" dependencies = [ "getrandom 0.2.10", + "serde", ] [[package]] @@ -6044,7 +6351,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.58", "wasm-bindgen-shared", ] @@ -6078,7 +6385,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.58", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -6212,7 +6519,7 @@ checksum = "ac1345798ecd8122468840bcdf1b95e5dc6d2206c5e4b0eafa078d061f59c9bc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.58", ] [[package]] @@ -6685,6 +6992,22 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "735a71d46c4d68d71d4b24d03fdc2b98e38cea81730595801db779c04fe80d70" +[[package]] +name = "yrs" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4058d69bbbc97181d53d9d093a4b892001b84601f2fc4e27f48c8862bc8b369" +dependencies = [ + "arc-swap", + "atomic_refcell", + "fastrand 2.0.2", + "serde", + "serde_json", + "smallstr", + "smallvec", + "thiserror", +] + [[package]] name = "zbus" version = "3.14.1" @@ -6768,7 +7091,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.58", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index d57fb4c..38cfaf0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ members = [ "ng-client-ws", "ng-verifier", "ng-wallet", - "ng-stores-rocksdb", + "ng-storage-rocksdb", "ngone", "ngaccount", "ng-sdk-js", @@ -20,7 +20,7 @@ default-members = [ "nextgraph", "ngcli", "ngd" ] [workspace.package] version = "0.1.0" edition = "2021" -rust-version = "1.64.0" +rust-version = "1.74.0" license = "MIT/Apache-2.0" authors = ["Niko PLP "] repository = "https://git.nextgraph.org/NextGraph/nextgraph-rs" @@ -29,7 +29,7 @@ keywords = [ "crdt","dapp","decentralized","e2ee","local-first","p2p","semantic-web","eventual-consistency","json-ld","markdown", "ocap","z-cap","offline-first","p2p-network","collaboration","privacy-protection","rdf","rich-text-editor","self-hosted", "sparql","byzantine-fault-tolerance", -"web3" +"web3", "graph-database", "database","triplestore" ] documentation = "https://docs.nextgraph.org/" diff --git a/README.md b/README.md index b875354..8f4e3f5 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ Read our [getting started guide](https://docs.nextgraph.org/en/getting-started/) ## For contributors -- [Install Rust](https://www.rust-lang.org/tools/install) minimum required MSRV 1.64.0 +- [Install Rust](https://www.rust-lang.org/tools/install) minimum required MSRV 1.74.0 - [Install Nodejs](https://nodejs.org/en/download/) - [Install LLVM](https://rust-lang.github.io/rust-bindgen/requirements.html) @@ -71,7 +71,7 @@ The crates are organized as follow : - ng-wallet : keeps the secret keys of all identities of the user in a safe wallet - ng-broker : Core and Server Broker library - ng-client-ws : Websocket client library -- ng-stores-rocksdb : RocksDB backed stores. see also dependency [repo here](https://git.nextgraph.org/NextGraph/rust-rocksdb) +- ng-storage-rocksdb : RocksDB backed stores. see also dependency [repo here](https://git.nextgraph.org/NextGraph/rust-rocksdb) - [ng-sdk-js](ng-sdk-js/README.md) : contains the JS SDK, with example apps: web app, react app, or node service. - [ng-app](ng-app/README.md) : all the native apps, based on Tauri, and the web app. - ngone : server for nextgraph.one. helps user bootstrap into the right app. Not useful to you. Published here for transparency @@ -170,7 +170,7 @@ For building the apps, see this [documentation](ng-app/README.md). #### OpenBSD On OpenBSD, a conflict between the installed LibreSSL library and the reqwest crate, needs a bit of attention. -Before compiling the daemon for OpenBSD, please comment out lines 32-33 of `ng-net/Cargo.toml`. This will be solved soon in a more appropriate way. +Before compiling the daemon for OpenBSD, please comment out lines 41-42 of `ng-net/Cargo.toml`. This will be solved soon by using `resolver = "2"`. ``` #[target.'cfg(target_arch = "wasm32")'.dependencies] @@ -219,7 +219,7 @@ Licensed under either of NextGraph received funding through the [NGI Assure Fund](https://nlnet.nl/project/NextGraph/index.html), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's [Next Generation Internet](https://ngi.eu/) programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 957073. -[rustc-image]: https://img.shields.io/badge/rustc-1.64+-blue.svg +[rustc-image]: https://img.shields.io/badge/rustc-1.74+-blue.svg [license-image]: https://img.shields.io/badge/license-Apache2.0-blue.svg [license-link]: https://git.nextgraph.org/NextGraph/nextgraph-rs/raw/branch/master/LICENSE-APACHE2 [license-image2]: https://img.shields.io/badge/license-MIT-blue.svg diff --git a/nextgraph/Cargo.toml b/nextgraph/Cargo.toml index 8eecf0b..238888a 100644 --- a/nextgraph/Cargo.toml +++ b/nextgraph/Cargo.toml @@ -21,7 +21,6 @@ ng-net = { path = "../ng-net", version = "0.1.0" } ng-wallet = { path = "../ng-wallet", version = "0.1.0" } ng-client-ws = { path = "../ng-client-ws", version = "0.1.0" } ng-verifier = { path = "../ng-verifier", version = "0.1.0" } -ng-stores-rocksdb = { path = "../ng-stores-rocksdb", version = "0.1.0" } async-once-cell = "0.5.3" once_cell = "1.17.1" serde = { version = "1.0", features = ["derive"] } diff --git a/nextgraph/README.md b/nextgraph/README.md index c357d02..70089e4 100644 --- a/nextgraph/README.md +++ b/nextgraph/README.md @@ -72,7 +72,7 @@ additional terms or conditions. NextGraph received funding through the [NGI Assure Fund](https://nlnet.nl/project/NextGraph/index.html), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's [Next Generation Internet](https://ngi.eu/) programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 957073. -[rustc-image]: https://img.shields.io/badge/rustc-1.64+-blue.svg +[rustc-image]: https://img.shields.io/badge/rustc-1.74+-blue.svg [license-image]: https://img.shields.io/badge/license-Apache2.0-blue.svg [license-link]: https://git.nextgraph.org/NextGraph/nextgraph-rs/raw/branch/master/LICENSE-APACHE2 [license-image2]: https://img.shields.io/badge/license-MIT-blue.svg diff --git a/nextgraph/examples/in_memory.rs b/nextgraph/examples/in_memory.rs index a79635c..bb61d32 100644 --- a/nextgraph/examples/in_memory.rs +++ b/nextgraph/examples/in_memory.rs @@ -111,7 +111,11 @@ async fn main() -> std::io::Result<()> { // anyway, now that the wallet is opened, let's start a session. // we pass the user_id and the wallet_name - let _session = session_start(SessionConfig::new(&user_id, &wallet_result.wallet_name)).await?; + let _session = session_start(SessionConfig::new_in_memory( + &user_id, + &wallet_result.wallet_name, + )) + .await?; // if the user has internet access, they can now decide to connect to its Server Broker, in order to sync data let status = user_connect(&user_id).await?; diff --git a/nextgraph/examples/persistent.rs b/nextgraph/examples/persistent.rs index 7bd55c6..1115bc6 100644 --- a/nextgraph/examples/persistent.rs +++ b/nextgraph/examples/persistent.rs @@ -15,10 +15,11 @@ use std::fs::create_dir_all; async fn main() -> std::io::Result<()> { // get the current working directory let mut current_path = current_dir()?; - current_path.push("ng-example"); + current_path.push(".ng"); + current_path.push("example"); create_dir_all(current_path.clone())?; - // initialize the local_broker with config to save to disk in a folder called `ng` in the current directory + // initialize the local_broker with config to save to disk in a folder called `.ng/example` in the current directory init_local_broker(Box::new(move || { LocalBrokerConfig::BasePath(current_path.clone()) })) diff --git a/nextgraph/src/lib.rs b/nextgraph/src/lib.rs index 3f844f4..1a900c5 100644 --- a/nextgraph/src/lib.rs +++ b/nextgraph/src/lib.rs @@ -17,7 +17,7 @@ //! - the blocks of the repos, //! - the connection(s) to your Server Broker //! - the events that you send to the Overlay, if there is no connectivity (Outbox) -//! - A reference to the verifier (optional) +//! - A reference to the verifier //! //! In addition, the API for creating and managing your wallet is provided here. //! diff --git a/nextgraph/src/local_broker.rs b/nextgraph/src/local_broker.rs index c3cb3c9..b6cd9e2 100644 --- a/nextgraph/src/local_broker.rs +++ b/nextgraph/src/local_broker.rs @@ -12,7 +12,9 @@ use async_std::sync::{Arc, RwLock}; use core::fmt; use ng_net::connection::{ClientConfig, IConnect, StartConfig}; use ng_net::types::{ClientInfo, ClientType}; +use ng_net::utils::{Receiver, Sender}; use ng_repo::os_info::get_os_info; +use ng_verifier::types::*; use ng_wallet::emojis::encode_pazzle; use once_cell::sync::Lazy; use serde_bare::to_vec; @@ -26,6 +28,7 @@ use ng_net::broker::*; use ng_repo::errors::NgError; use ng_repo::log::*; use ng_repo::types::*; +use ng_repo::utils::derive_key; use ng_wallet::{create_wallet_v0, types::*}; #[cfg(not(target_arch = "wasm32"))] @@ -35,19 +38,105 @@ use ng_client_ws::remote_ws_wasm::ConnectionWebSocket; type JsStorageReadFn = dyn Fn(String) -> Result + 'static + Sync + Send; type JsStorageWriteFn = dyn Fn(String, String) -> Result<(), NgError> + 'static + Sync + Send; +type JsStorageDelFn = dyn Fn(String) -> Result<(), NgError> + 'static + Sync + Send; type JsCallback = dyn Fn() + 'static + Sync + Send; #[doc(hidden)] pub struct JsStorageConfig { pub local_read: Box, pub local_write: Box, - pub session_read: Box, - pub session_write: Box, + pub session_read: Arc>, + pub session_write: Arc>, + pub session_del: Arc>, + pub is_browser: bool, +} + +impl JsStorageConfig { + fn get_js_storage_config(&self) -> JsSaveSessionConfig { + let session_read2 = Arc::clone(&self.session_read); + let session_write2 = Arc::clone(&self.session_write); + let session_read3 = Arc::clone(&self.session_read); + let session_write3 = Arc::clone(&self.session_write); + let session_read4 = Arc::clone(&self.session_read); + let session_del = Arc::clone(&self.session_del); + JsSaveSessionConfig { + last_seq_function: Box::new(move |peer_id: PubKey, qty: u16| -> Result { + let res = (session_read2)(format!("ng_peer_last_seq@{}", peer_id)); + let val = match res { + Ok(old_str) => { + let decoded = base64_url::decode(&old_str) + .map_err(|_| NgError::SerializationError)?; + match serde_bare::from_slice(&decoded)? { + SessionPeerLastSeq::V0(old_val) => old_val, + _ => unimplemented!(), + } + } + Err(_) => 0, + }; + let new_val = val + qty as u64; + let spls = SessionPeerLastSeq::V0(new_val); + let ser = serde_bare::to_vec(&spls)?; + //saving the new val + let encoded = base64_url::encode(&ser); + let r = (session_write2)(format!("ng_peer_last_seq@{}", peer_id), encoded); + if r.is_ok() { + return Err(NgError::SerializationError); + } + Ok(val) + }), + outbox_write_function: Box::new( + move |peer_id: PubKey, seq: u64, event: Vec| -> Result<(), NgError> { + let seq_str = format!("{}", seq); + let res = (session_read3)(format!("ng_outboxes@{}@start", peer_id)); + let start = match res { + Err(_) => { + (session_write3)(format!("ng_outboxes@{}@start", peer_id), seq_str)?; + seq + } + Ok(start_str) => start_str + .parse::() + .map_err(|_| NgError::InvalidFileFormat)?, + }; + let idx = seq - start; + let idx_str = format!("{:05}", idx); + let encoded = base64_url::encode(&event); + (session_write3)(format!("ng_outboxes@{}@{idx_str}", peer_id), encoded) + }, + ), + outbox_read_function: Box::new( + move |peer_id: PubKey| -> Result>, NgError> { + let res = (session_read4)(format!("ng_outboxes@{}@start", peer_id)); + let mut start = match res { + Err(_) => return Err(NgError::NotFound), + Ok(start_str) => start_str + .parse::() + .map_err(|_| NgError::InvalidFileFormat)?, + }; + let mut result = vec![]; + loop { + let idx_str = format!("{:05}", start); + let str = format!("ng_outboxes@{}@{idx_str}", peer_id); + let res = (session_read4)(str.clone()); + let res = match res { + Err(_) => break, + Ok(res) => res, + }; + (session_del)(str)?; + let decoded = + base64_url::decode(&res).map_err(|_| NgError::SerializationError)?; + result.push(decoded); + start += 1; + } + Ok(result) + }, + ), + } + } } impl fmt::Debug for JsStorageConfig { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "JsStorageConfig") + write!(f, "JsStorageConfig. is_browser {}", self.is_browser) } } @@ -86,25 +175,14 @@ impl LocalBrokerConfig { } } -//type LastSeqFn = fn(PubKey, u16) -> Result; -pub type LastSeqFn = dyn Fn(PubKey, u16) -> Result + 'static + Sync + Send; - -// peer_id: PubKey, seq_num:u64, event_ser: vec, -pub type OutboxWriteFn = - dyn Fn(PubKey, u64, Vec) -> Result<(), NgError> + 'static + Sync + Send; - -// peer_id: PubKey, -pub type OutboxReadFn = dyn Fn(PubKey) -> Result>, NgError> + 'static + Sync + Send; - +#[derive(Debug)] /// used to initiate a session at a local broker V0 pub struct SessionConfigV0 { pub user_id: UserId, pub wallet_name: String, - // pub last_seq_function: Box, - // pub outbox_write_function: Box, - // pub outbox_read_function: Box, + pub verifier_type: VerifierType, } - +#[derive(Debug)] /// used to initiate a session at a local broker pub enum SessionConfig { V0(SessionConfigV0), @@ -115,7 +193,7 @@ struct Session { config: SessionConfig, peer_key: PrivKey, last_wallet_nonce: u64, - //verifier, + verifier: Verifier, } impl SessionConfig { @@ -129,34 +207,95 @@ impl SessionConfig { Self::V0(v0) => v0.wallet_name.clone(), } } - /// Creates a new SessionConfig with a UserId and a wallet name + pub fn verifier_type(&self) -> &VerifierType { + match self { + Self::V0(v0) => &v0.verifier_type, + } + } + /// Creates a new in_memory SessionConfig with a UserId and a wallet name + /// /// that should be passed to [session_start] - pub fn new(user_id: &UserId, wallet_name: &String) -> Self { + pub fn new_in_memory(user_id: &UserId, wallet_name: &String) -> Self { SessionConfig::V0(SessionConfigV0 { user_id: user_id.clone(), wallet_name: wallet_name.clone(), + verifier_type: VerifierType::Memory, }) } -} -impl fmt::Debug for SessionConfigV0 { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - "SessionConfigV0 user={} wallet={}", - self.user_id, self.wallet_name - ) + /// Creates a new SessionConfig backed by RocksDb, with a UserId and a wallet name + /// + /// that should be passed to [session_start] + pub fn new_rocksdb(user_id: &UserId, wallet_name: &String) -> Self { + SessionConfig::V0(SessionConfigV0 { + user_id: user_id.clone(), + wallet_name: wallet_name.clone(), + verifier_type: VerifierType::RocksDb, + }) } -} -impl fmt::Debug for SessionConfig { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - SessionConfig::V0(v0) => v0.fmt(f), + /// Creates a new remote SessionConfig, with a UserId, a wallet name and optional remote peer_id + /// + /// that should be passed to [session_start] + pub fn new_remote( + user_id: &UserId, + wallet_name: &String, + remote_verifier_peer_id: Option, + ) -> Self { + SessionConfig::V0(SessionConfigV0 { + user_id: user_id.clone(), + wallet_name: wallet_name.clone(), + verifier_type: VerifierType::Remote(remote_verifier_peer_id), + }) + } + + fn valid_verifier_config_for_local_broker_config( + &mut self, + local_broker_config: &LocalBrokerConfig, + ) -> Result<(), NgError> { + if match self { + Self::V0(v0) => match local_broker_config { + LocalBrokerConfig::InMemory => { + v0.verifier_type = VerifierType::Memory; + true + } + LocalBrokerConfig::JsStorage(js_config) => match v0.verifier_type { + VerifierType::Memory | VerifierType::Remote(_) => true, + VerifierType::RocksDb => false, + VerifierType::WebRocksDb => js_config.is_browser, + }, + LocalBrokerConfig::BasePath(_) => match v0.verifier_type { + VerifierType::RocksDb | VerifierType::Remote(_) => true, + //VerifierType::Memory => true, + _ => false, + }, + }, + } { + Ok(()) + } else { + Err(NgError::InvalidArgument) } } } +// impl fmt::Debug for SessionConfigV0 { +// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { +// write!( +// f, +// "SessionConfigV0 user={} wallet={}", +// self.user_id, self.wallet_name +// ) +// } +// } + +// impl fmt::Debug for SessionConfig { +// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { +// match self { +// SessionConfig::V0(v0) => v0.fmt(f), +// } +// } +// } + #[derive(Debug)] struct LocalBroker { pub config: LocalBrokerConfig, @@ -167,19 +306,78 @@ struct LocalBroker { pub sessions: HashMap, - pub opened_sessions: HashMap, + pub opened_sessions: HashMap, + + pub opened_sessions_list: Vec>, } impl ILocalBroker for LocalBroker {} impl LocalBroker { - fn get_wallet_for_session(&self, config: &SessionConfig) -> Result<&SensitiveWallet, NgError> { - match config { + fn storage_path_for_user(&self, user_id: &UserId) -> Option { + match &self.config { + LocalBrokerConfig::InMemory | LocalBrokerConfig::JsStorage(_) => None, + LocalBrokerConfig::BasePath(base) => { + let mut path = base.clone(); + path.push(user_id.to_hash_string()); + Some(path) + } + } + } + + fn verifier_config_type_from_session_config( + &self, + config: &SessionConfig, + ) -> VerifierConfigType { + match (config.verifier_type(), &self.config) { + (VerifierType::Memory, LocalBrokerConfig::InMemory) => VerifierConfigType::Memory, + (VerifierType::RocksDb, LocalBrokerConfig::BasePath(base)) => { + let mut path = base.clone(); + path.push(config.user_id().to_hash_string()); + VerifierConfigType::RocksDb(path) + } + (VerifierType::Remote(to), _) => VerifierConfigType::Remote(*to), + (VerifierType::WebRocksDb, _) => VerifierConfigType::WebRocksDb, + (VerifierType::Memory, LocalBrokerConfig::JsStorage(js)) => { + VerifierConfigType::JsSaveSession(js.get_js_storage_config()) + } + (_, _) => panic!("invalid combination in verifier_config_type_from_session_config"), + } + } + + fn get_wallet_and_session( + &mut self, + user_id: &UserId, + ) -> Result<(&SensitiveWallet, &mut Session), NgError> { + let session_idx = self + .opened_sessions + .get(user_id) + .ok_or(NgError::SessionNotFound)?; + let session = self.opened_sessions_list[*session_idx as usize] + .as_mut() + .ok_or(NgError::SessionNotFound)?; + let wallet = match &session.config { SessionConfig::V0(v0) => self .opened_wallets .get(&v0.wallet_name) .ok_or(NgError::WalletNotFound), + }?; + + Ok((wallet, session)) + } + async fn disconnect_session(&mut self, user_id: &PubKey) -> Result<(), NgError> { + match self.opened_sessions.get(user_id) { + Some(session) => { + // TODO: change the logic here once it will be possible to have several users connected at the same time + Broker::close_all_connections().await; + let session = self.opened_sessions_list[*session as usize] + .as_mut() + .ok_or(NgError::SessionNotFound)?; + session.verifier.connected_server_id = None; + } + None => {} } + Ok(()) } } @@ -224,6 +422,7 @@ async fn init_(config: LocalBrokerConfig) -> Result>, Ng opened_wallets: HashMap::new(), sessions: HashMap::new(), opened_sessions: HashMap::new(), + opened_sessions_list: vec![], }; //log_debug!("{:?}", &local_broker); @@ -492,24 +691,36 @@ pub async fn wallet_was_opened(mut wallet: SensitiveWallet) -> Result Result { +/// Return value is the index of the session, will be used in all the doc_* API calls. +pub async fn session_start(mut config: SessionConfig) -> Result { let mut broker = match LOCAL_BROKER.get() { None | Some(Err(_)) => return Err(NgError::LocalBrokerNotInitialized), Some(Ok(broker)) => broker.write().await, }; - let wallet_name = config.wallet_name(); + config.valid_verifier_config_for_local_broker_config(&broker.config)?; + + let wallet_name: String = config.wallet_name(); let wallet_id: PubKey = (*wallet_name).try_into()?; let user_id = config.user_id(); match broker.opened_wallets.get(&wallet_name) { None => return Err(NgError::WalletNotFound), Some(wallet) => { - if !wallet.has_user(&user_id) { - return Err(NgError::NotFound); - } + let credentials = match wallet.individual_site(&user_id) { + Some(creds) => creds.clone(), + None => return Err(NgError::NotFound), + }; + + let client_storage_master_key = serde_bare::to_vec( + &wallet + .client() + .as_ref() + .unwrap() + .sensitive_client_storage + .storage_master_key, + ) + .unwrap(); let session = match broker.sessions.get(&user_id) { Some(session) => session, @@ -612,16 +823,35 @@ pub async fn session_start(config: SessionConfig) -> Result, ) -> Result, f64)>, NgError> { - let local_broker = match LOCAL_BROKER.get() { + //FIXME: release this write lock much sooner than at the end of the loop of all tries to connect to some servers ? + // or maybe it is good to block as we dont want concurrent connection attemps potentially to the same server + let mut local_broker = match LOCAL_BROKER.get() { None | Some(Err(_)) => return Err(NgError::LocalBrokerNotInitialized), - Some(Ok(broker)) => broker.read().await, + Some(Ok(broker)) => broker.write().await, }; - let session = local_broker - .opened_sessions - .get(user_id) - .ok_or(NgError::SessionNotFound)?; - let wallet = local_broker.get_wallet_for_session(&session.config)?; + let (wallet, session) = local_broker.get_wallet_and_session(user_id)?; let mut result: Vec<(String, String, String, Option, f64)> = Vec::new(); let arc_cnx: Arc> = Arc::new(Box::new(ConnectionWebSocket {})); @@ -791,6 +1019,7 @@ pub async fn user_connect_with_device_info( )); } if tried.is_some() && tried.as_ref().unwrap().3.is_none() { + session.verifier.connected_server_id = Some(server_key); // successful. we can stop here break; } else { @@ -825,9 +1054,13 @@ pub async fn session_stop(user_id: &UserId) -> Result<(), NgError> { Some(Ok(broker)) => broker.write().await, }; - if broker.opened_sessions.remove(user_id).is_some() { - // TODO: change the logic here once it will be possible to have several users connected at the same time - Broker::close_all_connections().await; + match broker.opened_sessions.remove(user_id) { + Some(id) => { + broker.opened_sessions_list[id as usize].take(); + // TODO: change the logic here once it will be possible to have several users connected at the same time + Broker::close_all_connections().await; + } + None => {} } Ok(()) @@ -835,17 +1068,12 @@ pub async fn session_stop(user_id: &UserId) -> Result<(), NgError> { /// Disconnects the user from the Server Broker(s), but keep all the local data opened and ready. pub async fn user_disconnect(user_id: &UserId) -> Result<(), NgError> { - let broker = match LOCAL_BROKER.get() { + let mut broker = match LOCAL_BROKER.get() { None | Some(Err(_)) => return Err(NgError::LocalBrokerNotInitialized), - Some(Ok(broker)) => broker.read().await, + Some(Ok(broker)) => broker.write().await, }; - if broker.opened_sessions.get(user_id).is_some() { - // TODO: change the logic here once it will be possible to have several users connected at the same time - Broker::close_all_connections().await; - } - - Ok(()) + broker.disconnect_session(user_id).await } /// Closes a wallet, which means that the pazzle will have to be entered again if the user wants to use it @@ -859,7 +1087,12 @@ pub async fn wallet_close(wallet_name: &String) -> Result<(), NgError> { Some(mut wallet) => { for user in wallet.sites() { let key: PubKey = (user.as_str()).try_into().unwrap(); - broker.opened_sessions.remove(&key); + match broker.opened_sessions.remove(&key) { + Some(id) => { + broker.opened_sessions_list[id as usize].take(); + } + None => {} + } } wallet.zeroize(); } @@ -884,6 +1117,26 @@ pub async fn wallet_remove(wallet_name: String) -> Result<(), NgError> { Ok(()) } +/// fetches a document's content, or performs a mutation on the document. +pub async fn doc_fetch( + session_id: u8, + nuri: String, + payload: Option, +) -> Result<(Receiver, CancelFn), NgError> { + let broker = match LOCAL_BROKER.get() { + None | Some(Err(_)) => return Err(NgError::LocalBrokerNotInitialized), + Some(Ok(broker)) => broker.read().await, + }; + if session_id as usize >= broker.opened_sessions_list.len() { + return Err(NgError::InvalidArgument); + } + let session = broker.opened_sessions_list[session_id as usize] + .as_ref() + .ok_or(NgError::SessionNotFound)?; + + session.verifier.doc_fetch(nuri, payload) +} + #[cfg(test)] mod test { use super::*; @@ -1003,7 +1256,7 @@ mod test { .await .expect("wallet_import"); - let _session = session_start(SessionConfig::new(&user_id, &wallet_name)) + let _session = session_start(SessionConfig::new_in_memory(&user_id, &wallet_name)) .await .expect(""); diff --git a/ng-app/README.md b/ng-app/README.md index 049955d..b93b6e6 100644 --- a/ng-app/README.md +++ b/ng-app/README.md @@ -1,8 +1,8 @@ # NextGraph apps (Linux, MacOS, Windows, Android, iOS, web) -NextGraph native apps use the Tauri framework. +All the apps are based on Svelte and share the same code. -All the native apps are using an embedded WebView that renders a Svelte app. +The native apps are using the Tauri framework with an embedded WebView to render the Svelte app. ## Install diff --git a/ng-app/prepare-app-file.cjs b/ng-app/prepare-app-file.cjs index e02a481..96bf400 100644 --- a/ng-app/prepare-app-file.cjs +++ b/ng-app/prepare-app-file.cjs @@ -25,6 +25,8 @@ s.on('end', function() { var buf = Buffer.concat(bufs); gzip(buf).then((compressed) => {fs.writeFileSync(gzip_file, compressed);}); + fs.rm(filename,()=>{}); + }) diff --git a/ng-app/src-tauri/src/lib.rs b/ng-app/src-tauri/src/lib.rs index bf89526..4677289 100644 --- a/ng-app/src-tauri/src/lib.rs +++ b/ng-app/src-tauri/src/lib.rs @@ -233,11 +233,21 @@ async fn session_start( wallet_name: String, user: PubKey, app: tauri::AppHandle, -) -> Result { - let config = SessionConfig::V0(SessionConfigV0 { - user_id: user, - wallet_name, - }); +) -> Result { + let config = SessionConfig::new_rocksdb(&user, &wallet_name); + nextgraph::local_broker::session_start(config) + .await + .map_err(|e: NgError| e.to_string()) +} + +#[tauri::command(rename_all = "snake_case")] +async fn session_start_remote( + wallet_name: String, + user: PubKey, + peer_id: Option, + app: tauri::AppHandle, +) -> Result { + let config = SessionConfig::new_remote(&user, &wallet_name, peer_id); nextgraph::local_broker::session_start(config) .await .map_err(|e: NgError| e.to_string()) @@ -497,6 +507,7 @@ impl AppBuilder { wallet_close, encode_create_account, session_start, + session_start_remote, session_stop, get_wallets, open_window, diff --git a/ng-app/src/api.ts b/ng-app/src/api.ts index 01dd2e3..b5eef4d 100644 --- a/ng-app/src/api.ts +++ b/ng-app/src/api.ts @@ -25,6 +25,7 @@ const mapping = { "wallet_close": ["wallet_name"], "encode_create_account": ["payload"], "session_start": ["wallet_name","user"], + "session_start_remote": ["wallet_name","user","peer_id"], "session_stop": ["user_id"], "get_wallets": [], "open_window": ["url","label","title"], @@ -50,12 +51,12 @@ const handler = { } else if (path[0] === "get_wallets") { let wallets = await Reflect.apply(sdk[path], caller, args); return Object.fromEntries(wallets || []); - } else if (path[0] === "session_start") { - let res = await Reflect.apply(sdk[path], caller, args); - return res; - } else if (path[0] === "wallet_create") { - let res = await Reflect.apply(sdk[path], caller, args); - return res; + // } else if (path[0] === "session_start") { + // let res = await Reflect.apply(sdk[path], caller, args); + // return res; + // } else if (path[0] === "wallet_create") { + // let res = await Reflect.apply(sdk[path], caller, args); + // return res; } else { return Reflect.apply(sdk[path], caller, args) } diff --git a/ng-broker/Cargo.toml b/ng-broker/Cargo.toml index a9b70cd..d523116 100644 --- a/ng-broker/Cargo.toml +++ b/ng-broker/Cargo.toml @@ -19,7 +19,7 @@ maintenance = { status = "actively-developed" } ng-repo = { path = "../ng-repo", version = "0.1.0" } ng-net = { path = "../ng-net", version = "0.1.0" } ng-client-ws = { path = "../ng-client-ws", version = "0.1.0" } -ng-stores-rocksdb = { path = "../ng-stores-rocksdb", version = "0.1.0" } +ng-storage-rocksdb = { path = "../ng-storage-rocksdb", version = "0.1.0" } chacha20 = "0.9.0" serde = { version = "1.0", features = ["derive"] } serde_bare = "0.5.0" diff --git a/ng-broker/README.md b/ng-broker/README.md index efa9d43..94dd13a 100644 --- a/ng-broker/README.md +++ b/ng-broker/README.md @@ -48,7 +48,7 @@ additional terms or conditions. NextGraph received funding through the [NGI Assure Fund](https://nlnet.nl/project/NextGraph/index.html), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's [Next Generation Internet](https://ngi.eu/) programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 957073. -[rustc-image]: https://img.shields.io/badge/rustc-1.64+-blue.svg +[rustc-image]: https://img.shields.io/badge/rustc71.74+-blue.svg [license-image]: https://img.shields.io/badge/license-Apache2.0-blue.svg [license-link]: https://git.nextgraph.org/NextGraph/nextgraph-rs/raw/branch/master/LICENSE-APACHE2 [license-image2]: https://img.shields.io/badge/license-MIT-blue.svg diff --git a/ng-broker/src/broker_store/account.rs b/ng-broker/src/broker_storage/account.rs similarity index 97% rename from ng-broker/src/broker_store/account.rs rename to ng-broker/src/broker_storage/account.rs index 25aaef3..04e949b 100644 --- a/ng-broker/src/broker_store/account.rs +++ b/ng-broker/src/broker_storage/account.rs @@ -15,9 +15,9 @@ use std::hash::Hasher; use std::time::SystemTime; use ng_net::types::*; -use ng_repo::kcv_store::KCVStore; +use ng_repo::errors::StorageError; +use ng_repo::kcv_storage::KCVStore; use ng_repo::log::*; -use ng_repo::store::*; use ng_repo::types::UserId; use serde_bare::{from_slice, to_vec}; @@ -225,14 +225,14 @@ impl<'a> Account<'a> { #[cfg(test)] mod test { - use ng_repo::store::*; + use ng_repo::errors::StorageError; use ng_repo::types::*; use ng_repo::utils::*; - use ng_stores_rocksdb::kcv_store::RocksdbKCVStore; + use ng_storage_rocksdb::kcv_storage::RocksdbKCVStore; use std::fs; use tempfile::Builder; - use crate::broker_store::account::Account; + use crate::broker_storage::account::Account; #[test] pub fn test_account() { diff --git a/ng-broker/src/broker_store/config.rs b/ng-broker/src/broker_storage/config.rs similarity index 97% rename from ng-broker/src/broker_store/config.rs rename to ng-broker/src/broker_storage/config.rs index a068ba0..6090de4 100644 --- a/ng-broker/src/broker_store/config.rs +++ b/ng-broker/src/broker_storage/config.rs @@ -10,8 +10,8 @@ //! Broker Config, persisted to store use ng_net::types::*; -use ng_repo::kcv_store::KCVStore; -use ng_repo::store::*; +use ng_repo::errors::StorageError; +use ng_repo::kcv_storage::KCVStore; use ng_repo::types::*; use serde::{Deserialize, Serialize}; use serde_bare::{from_slice, to_vec}; diff --git a/ng-broker/src/broker_store/invitation.rs b/ng-broker/src/broker_storage/invitation.rs similarity index 97% rename from ng-broker/src/broker_store/invitation.rs rename to ng-broker/src/broker_storage/invitation.rs index 79429de..401b687 100644 --- a/ng-broker/src/broker_store/invitation.rs +++ b/ng-broker/src/broker_storage/invitation.rs @@ -16,8 +16,8 @@ use std::time::SystemTime; use ng_net::errors::ProtocolError; use ng_net::types::*; -use ng_repo::kcv_store::KCVStore; -use ng_repo::store::*; +use ng_repo::errors::StorageError; +use ng_repo::kcv_storage::KCVStore; use ng_repo::types::SymKey; use ng_repo::types::Timestamp; use ng_repo::utils::now_timestamp; @@ -178,13 +178,13 @@ impl<'a> Invitation<'a> { #[cfg(test)] mod test { - use ng_repo::store::*; + use ng_repo::errors::StorageError; use ng_repo::types::*; use ng_repo::utils::*; use std::fs; use tempfile::Builder; - use crate::broker_store::account::Account; + use crate::broker_storage::account::Account; #[test] pub fn test_invitation() {} diff --git a/ng-broker/src/broker_store/mod.rs b/ng-broker/src/broker_storage/mod.rs similarity index 83% rename from ng-broker/src/broker_store/mod.rs rename to ng-broker/src/broker_storage/mod.rs index f429099..0d2ad4c 100644 --- a/ng-broker/src/broker_store/mod.rs +++ b/ng-broker/src/broker_storage/mod.rs @@ -6,8 +6,6 @@ pub mod overlay; pub mod peer; -pub mod repostoreinfo; - pub mod topic; pub mod invitation; diff --git a/ng-broker/src/broker_store/overlay.rs b/ng-broker/src/broker_storage/overlay.rs similarity index 98% rename from ng-broker/src/broker_store/overlay.rs rename to ng-broker/src/broker_storage/overlay.rs index d61a86b..a322220 100644 --- a/ng-broker/src/broker_store/overlay.rs +++ b/ng-broker/src/broker_storage/overlay.rs @@ -10,8 +10,8 @@ //! Overlay use ng_net::types::*; -use ng_repo::kcv_store::KCVStore; -use ng_repo::store::*; +use ng_repo::errors::StorageError; +use ng_repo::kcv_storage::KCVStore; use ng_repo::types::*; use ng_repo::utils::now_timestamp; use serde::{Deserialize, Serialize}; diff --git a/ng-broker/src/broker_store/peer.rs b/ng-broker/src/broker_storage/peer.rs similarity index 98% rename from ng-broker/src/broker_store/peer.rs rename to ng-broker/src/broker_storage/peer.rs index 393dd83..611aabe 100644 --- a/ng-broker/src/broker_store/peer.rs +++ b/ng-broker/src/broker_storage/peer.rs @@ -10,8 +10,8 @@ //! Peer use ng_net::types::*; -use ng_repo::kcv_store::KCVStore; -use ng_repo::store::*; +use ng_repo::errors::StorageError; +use ng_repo::kcv_storage::KCVStore; use ng_repo::types::*; use serde::{Deserialize, Serialize}; use serde_bare::{from_slice, to_vec}; diff --git a/ng-broker/src/broker_store/topic.rs b/ng-broker/src/broker_storage/topic.rs similarity index 98% rename from ng-broker/src/broker_store/topic.rs rename to ng-broker/src/broker_storage/topic.rs index a64f9e6..ba61a51 100644 --- a/ng-broker/src/broker_store/topic.rs +++ b/ng-broker/src/broker_storage/topic.rs @@ -10,8 +10,8 @@ //! Topic use ng_net::types::*; -use ng_repo::kcv_store::KCVStore; -use ng_repo::store::*; +use ng_repo::errors::StorageError; +use ng_repo::kcv_storage::KCVStore; use ng_repo::types::*; use serde::{Deserialize, Serialize}; use serde_bare::{from_slice, to_vec}; diff --git a/ng-broker/src/broker_store/wallet.rs b/ng-broker/src/broker_storage/wallet.rs similarity index 97% rename from ng-broker/src/broker_store/wallet.rs rename to ng-broker/src/broker_storage/wallet.rs index 1b91e1c..e41082a 100644 --- a/ng-broker/src/broker_store/wallet.rs +++ b/ng-broker/src/broker_storage/wallet.rs @@ -10,10 +10,10 @@ //! Broker Wallet, persists to store all the SymKeys needed to open other storages use ng_net::types::*; -use ng_repo::kcv_store::KCVStore; -use ng_repo::kcv_store::WriteTransaction; +use ng_repo::errors::StorageError; +use ng_repo::kcv_storage::KCVStore; +use ng_repo::kcv_storage::WriteTransaction; use ng_repo::log::*; -use ng_repo::store::*; use ng_repo::types::*; use serde::{Deserialize, Serialize}; use serde_bare::{from_slice, to_vec}; diff --git a/ng-broker/src/broker_store/repostoreinfo.rs b/ng-broker/src/broker_store/repostoreinfo.rs deleted file mode 100644 index 0ef9cbf..0000000 --- a/ng-broker/src/broker_store/repostoreinfo.rs +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright (c) 2022-2024 Niko Bonnieure, Par le Peuple, NextGraph.org developers -// All rights reserved. -// Licensed under the Apache License, Version 2.0 -// -// or the MIT license , -// at your option. All files in the project carrying such -// notice may not be copied, modified, or distributed except -// according to those terms. - -//! RepoStore information about each RepoStore -//! It contains the symKeys to open the RepoStores -//! A repoStore is identified by its repo pubkey if in local mode -//! In core mode, it is identified by the overlayid. - -use ng_net::types::*; -use ng_repo::kcv_store::KCVStore; -use ng_repo::store::*; -use ng_repo::types::*; -use serde::{Deserialize, Serialize}; -use serde_bare::{from_slice, to_vec}; - -// #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] -// pub enum RepoStoreId { -// Overlay(OverlayId), -// Repo(PubKey), -// } - -// impl From for String { -// fn from(id: RepoStoreId) -> Self { -// hex::encode(to_vec(&id).unwrap()) -// } -// } - -pub struct RepoStoreInfo<'a> { - /// RepoStore ID - id: RepoHash, - store: &'a dyn KCVStore, -} - -impl<'a> RepoStoreInfo<'a> { - const PREFIX: u8 = b"r"[0]; - - // propertie's suffixes - const KEY: u8 = b"k"[0]; - - const ALL_PROPERTIES: [u8; 1] = [Self::KEY]; - - const SUFFIX_FOR_EXIST_CHECK: u8 = Self::KEY; - - pub fn open(id: &RepoHash, store: &'a dyn KCVStore) -> Result, StorageError> { - let opening = RepoStoreInfo { - id: id.clone(), - store, - }; - if !opening.exists() { - return Err(StorageError::NotFound); - } - Ok(opening) - } - pub fn create( - id: &RepoHash, - key: &SymKey, - store: &'a dyn KCVStore, - ) -> Result, StorageError> { - let acc = RepoStoreInfo { - id: id.clone(), - store, - }; - if acc.exists() { - return Err(StorageError::BackendError); - } - store.put(Self::PREFIX, &to_vec(&id)?, Some(Self::KEY), to_vec(key)?)?; - Ok(acc) - } - pub fn exists(&self) -> bool { - self.store - .get( - Self::PREFIX, - &to_vec(&self.id).unwrap(), - Some(Self::SUFFIX_FOR_EXIST_CHECK), - ) - .is_ok() - } - pub fn id(&self) -> &RepoHash { - &self.id - } - pub fn key(&self) -> Result { - match self - .store - .get(Self::PREFIX, &to_vec(&self.id)?, Some(Self::KEY)) - { - Ok(k) => Ok(from_slice::(&k)?), - Err(e) => Err(e), - } - } - pub fn del(&self) -> Result<(), StorageError> { - self.store - .del_all(Self::PREFIX, &to_vec(&self.id)?, &Self::ALL_PROPERTIES) - } -} diff --git a/ng-broker/src/lib.rs b/ng-broker/src/lib.rs index e8c77cd..527bfc4 100644 --- a/ng-broker/src/lib.rs +++ b/ng-broker/src/lib.rs @@ -1,4 +1,4 @@ -pub mod broker_store; +pub mod broker_storage; pub mod server_ws; diff --git a/ng-broker/src/server.rs b/ng-broker/src/server.rs deleted file mode 100644 index bc9d391..0000000 --- a/ng-broker/src/server.rs +++ /dev/null @@ -1,973 +0,0 @@ -// Copyright (c) 2022-2024 Niko Bonnieure, Par le Peuple, NextGraph.org developers -// All rights reserved. -// Licensed under the Apache License, Version 2.0 -// -// or the MIT license , -// at your option. All files in the project carrying such -// notice may not be copied, modified, or distributed except -// according to those terms. - -//! A Broker server - -use std::collections::HashMap; -use std::collections::HashSet; -use std::net::SocketAddr; -use std::pin::Pin; -use std::sync::Arc; -use std::sync::RwLock; - -use crate::broker_store::account::Account; -use crate::broker_store::config::Config; -use crate::broker_store::config::ConfigMode; -use crate::broker_store::overlay::Overlay; -use crate::broker_store::peer::Peer; -use crate::broker_store::repostoreinfo::RepoStoreInfo; -use async_std::task; -use futures::future::BoxFuture; -use futures::future::OptionFuture; -use futures::FutureExt; -use futures::Stream; -use ng_net::actors::*; -use ng_net::errors::*; -use ng_net::types::*; -use ng_repo::log::*; -use ng_repo::object::Object; -use ng_repo::store::RepoStore; -use ng_repo::store::StorageError; -use ng_repo::types::*; -use ng_repo::utils::*; - -#[derive(Debug, Eq, PartialEq, Clone)] -pub enum BrokerError { - CannotStart, - MismatchedMode, - OverlayNotFound, -} - -impl From for ProtocolError { - fn from(e: BrokerError) -> Self { - match e { - BrokerError::CannotStart => ProtocolError::OverlayNotFound, - BrokerError::OverlayNotFound => ProtocolError::OverlayNotFound, - _ => ProtocolError::BrokerError, - } - } -} - -impl From for BrokerError { - fn from(e: ng_repo::store::StorageError) -> Self { - match e { - ng_repo::store::StorageError::InvalidValue => BrokerError::MismatchedMode, - _ => BrokerError::CannotStart, - } - } -} - -#[derive(Debug)] -enum ProtocolType { - Start, - Auth, - Broker, - Ext, - Core, -} - -pub struct ProtocolHandler { - addr: Option, - broker: Arc, - protocol: ProtocolType, - auth_protocol: Option, - broker_protocol: Option>, - ext_protocol: Option, - r: Option>>, - s: async_channel::Sender>, -} - -impl ProtocolHandler { - pub fn register(&mut self, addr: SocketAddr) { - self.addr = Some(addr); - } - - pub fn deregister(&mut self) { - match &self.protocol { - ProtocolType::Start => (), - ProtocolType::Auth => (), - ProtocolType::Broker => { - let _ = self - .broker_protocol - .as_ref() - .unwrap() - .deregister(self.addr.unwrap()); - } - ProtocolType::Ext => (), - ProtocolType::Core => (), - } - self.addr = None; - } - - pub fn async_frames_receiver(&mut self) -> async_channel::Receiver> { - self.r.take().unwrap() - } - - /// Handle incoming message - pub async fn handle_incoming( - &mut self, - frame: Vec, - ) -> ( - Result, ProtocolError>, - OptionFuture>, - ) { - //log_debug!("SERVER PROTOCOL {:?}", &self.protocol); - match &self.protocol { - ProtocolType::Start => { - let message = serde_bare::from_slice::(&frame); - match message { - Ok(StartProtocol::Client(b)) => { - self.protocol = ProtocolType::Auth; - self.auth_protocol = Some(AuthProtocolHandler::new()); - return ( - self.auth_protocol.as_mut().unwrap().handle_init(b), - OptionFuture::from(None), - ); - } - Ok(StartProtocol::Ext(ext)) => { - self.protocol = ProtocolType::Ext; - self.ext_protocol = Some(ExtProtocolHandler {}); - let reply = self.ext_protocol.as_ref().unwrap().handle_incoming(ext); - return ( - Ok(serde_bare::to_vec(&reply).unwrap()), - OptionFuture::from(None), - ); - } - Err(e) => { - return ( - Err(ProtocolError::SerializationError), - OptionFuture::from(None), - ) - } - } - } - ProtocolType::Auth => { - let res = self.auth_protocol.as_mut().unwrap().handle_incoming(frame); - match res.1.await { - None => { - // we switch to Broker protocol - let bp = Arc::new(BrokerProtocolHandler { - user: self.auth_protocol.as_ref().unwrap().get_user().unwrap(), - broker: Arc::clone(&self.broker), - async_frames_sender: self.s.clone(), - }); - let registration = Arc::clone(&bp).register(self.addr.unwrap()); - match registration { - Ok(_) => { - self.protocol = ProtocolType::Broker; - self.broker_protocol = Some(Arc::clone(&bp)); - self.auth_protocol = None; - return (res.0, OptionFuture::from(None)); - } - Err(e) => { - let val = e.clone() as u16; - let reply = AuthResult::V0(AuthResultV0 { - result: val, - metadata: vec![], - }); - return ( - Ok(serde_bare::to_vec(&reply).unwrap()), - OptionFuture::from(Some(async move { val }.boxed())), - ); - } - } - } - Some(e) => (res.0, OptionFuture::from(Some(async move { e }.boxed()))), - } - } - ProtocolType::Broker => { - let message = serde_bare::from_slice::(&frame); - match (message) { - Ok(message) => { - let reply = self - .broker_protocol - .as_ref() - .unwrap() - .handle_incoming(message) - .await; - (Ok(serde_bare::to_vec(&reply.0).unwrap()), reply.1) - } - Err(e_) => ( - Err(ProtocolError::SerializationError), - OptionFuture::from(None), - ), - } - } - ProtocolType::Ext => { - // Ext protocol is not accepting 2 extrequest in the same connection. - // closing the connection - (Err(ProtocolError::InvalidState), OptionFuture::from(None)) - } - ProtocolType::Core => { - unimplemented!() - } - } - } -} - -pub struct ExtProtocolHandler {} - -impl ExtProtocolHandler { - pub fn handle_incoming(&self, msg: ExtHello) -> ExtResponse { - unimplemented!() - } -} - -pub struct BrokerProtocolHandler { - broker: Arc, - user: PubKey, - async_frames_sender: async_channel::Sender>, -} -use std::{thread, time}; - -impl BrokerProtocolHandler { - fn prepare_reply_broker_message( - res: Result<(), ProtocolError>, - id: i64, - padding_size: usize, - ) -> BrokerMessage { - let result = match res { - Ok(_) => 0, - Err(e) => e.into(), - }; - let msg = BrokerMessage::V0(BrokerMessageV0 { - padding: vec![0; padding_size], - content: BrokerMessageContentV0::BrokerResponse(BrokerResponse::V0(BrokerResponseV0 { - id, - result, - content: BrokerResponseContentV0::EmptyResponse(()), - })), - }); - msg - } - - fn prepare_reply_broker_overlay_message( - res: Result<(), ProtocolError>, - id: i64, - overlay: OverlayId, - block: Option, - padding_size: usize, - ) -> BrokerMessage { - let result = match res { - Ok(_) => 0, - Err(e) => e.into(), - }; - let content = match block { - Some(b) => BrokerOverlayResponseContentV0::Block(b), - None => BrokerOverlayResponseContentV0::EmptyResponse(()), - }; - let msg = BrokerMessage::V0(BrokerMessageV0 { - padding: vec![0; padding_size], - content: BrokerMessageContentV0::BrokerOverlayMessage(BrokerOverlayMessage::V0( - BrokerOverlayMessageV0 { - overlay, - content: BrokerOverlayMessageContentV0::BrokerOverlayResponse( - BrokerOverlayResponse::V0(BrokerOverlayResponseV0 { - id, - result, - content, - }), - ), - }, - )), - }); - msg - } - - fn prepare_reply_broker_overlay_message_stream( - res: Result, - id: i64, - overlay: OverlayId, - padding_size: usize, - ) -> BrokerMessage { - let result: u16 = match &res { - Ok(r) => ProtocolError::PartialContent.into(), - Err(e) => (*e).clone().into(), - }; - let content = match res { - Ok(r) => BrokerOverlayResponseContentV0::Block(r), - Err(_) => BrokerOverlayResponseContentV0::EmptyResponse(()), - }; - let msg = BrokerMessage::V0(BrokerMessageV0 { - padding: vec![0; padding_size], - content: BrokerMessageContentV0::BrokerOverlayMessage(BrokerOverlayMessage::V0( - BrokerOverlayMessageV0 { - overlay, - content: BrokerOverlayMessageContentV0::BrokerOverlayResponse( - BrokerOverlayResponse::V0(BrokerOverlayResponseV0 { - id, - result, - content, - }), - ), - }, - )), - }); - msg - } - - async fn send_block_stream_response_to_client( - &self, - res: Result, ProtocolError>, - id: i64, - overlay: OverlayId, - padding_size: usize, - ) -> (BrokerMessage, OptionFuture>) { - // return an error or the first block, and setup a spawner for the remaining blocks to be sent. - let one_reply: ( - Result, - OptionFuture>, - ) = match res { - Err(e) => (Err(e), OptionFuture::from(None)), - Ok(stream) => { - let one = stream - .recv_blocking() - .map_err(|e| ProtocolError::EndOfStream); - - if one.is_ok() { - let sender = self.async_frames_sender.clone(); - let a = OptionFuture::from(Some( - async move { - while let Ok(next) = stream.recv().await { - let msg = Self::prepare_reply_broker_overlay_message_stream( - Ok(next), - id, - overlay, - padding_size, - ); - let res = sender.send(serde_bare::to_vec(&msg).unwrap()).await; - if res.is_err() { - break; - } - } - // sending end of stream - let msg = Self::prepare_reply_broker_overlay_message_stream( - Err(ProtocolError::EndOfStream), - id, - overlay, - padding_size, - ); - let _ = sender.send(serde_bare::to_vec(&msg).unwrap()).await; - 0 - } - .boxed(), - )); - (one, a) - } else { - (one, OptionFuture::from(None)) - } - } - }; - return ( - Self::prepare_reply_broker_overlay_message_stream( - one_reply.0, - id, - overlay, - padding_size, - ), - one_reply.1, - ); - } - - pub fn register(self: Arc, addr: SocketAddr) -> Result<(), ProtocolError> { - //FIXME: peer_id must be real one - - self.broker - .add_client_peer(PubKey::Ed25519PubKey([0; 32]), Arc::clone(&self)) - } - - pub fn deregister(&self, addr: SocketAddr) -> Result<(), ProtocolError> { - self.broker - .remove_client_peer(PubKey::Ed25519PubKey([0; 32])); - Ok(()) - } - - pub async fn handle_incoming( - &self, - msg: BrokerMessage, - ) -> (BrokerMessage, OptionFuture>) { - let padding_size = 20; // TODO randomize, if config of server contains padding_max - - let id = msg.id(); - let content = msg.content(); - match content { - BrokerMessageContentV0::BrokerRequest(req) => ( - Self::prepare_reply_broker_message( - match req.content_v0() { - BrokerRequestContentV0::AddUser(cmd) => { - self.broker.add_user(self.user, cmd.user(), cmd.sig()) - } - BrokerRequestContentV0::DelUser(cmd) => { - self.broker.del_user(self.user, cmd.user(), cmd.sig()) - } - BrokerRequestContentV0::AddClient(cmd) => { - self.broker.add_client(self.user, cmd.client(), cmd.sig()) - } - BrokerRequestContentV0::DelClient(cmd) => { - self.broker.del_client(self.user, cmd.client(), cmd.sig()) - } - }, - id, - padding_size, - ), - OptionFuture::from(None), - ), - BrokerMessageContentV0::BrokerResponse(res) => ( - Self::prepare_reply_broker_message( - Err(ProtocolError::InvalidState), - id, - padding_size, - ), - OptionFuture::from(None), - ), - BrokerMessageContentV0::BrokerOverlayMessage(omsg) => { - let overlay = omsg.overlay_id(); - let block = None; - let mut res = Err(ProtocolError::InvalidState); - - if omsg.is_request() { - match omsg.overlay_request().content_v0() { - BrokerOverlayRequestContentV0::OverlayConnect(_) => { - res = self.broker.connect_overlay(self.user, overlay) - } - BrokerOverlayRequestContentV0::OverlayJoin(j) => { - res = self.broker.join_overlay( - self.user, - overlay, - j.repo_pubkey(), - j.secret(), - j.peers(), - ) - } - BrokerOverlayRequestContentV0::ObjectDel(op) => { - res = self.broker.del_object(self.user, overlay, op.id()) - } - BrokerOverlayRequestContentV0::ObjectPin(op) => { - res = self.broker.pin_object(self.user, overlay, op.id()) - } - BrokerOverlayRequestContentV0::ObjectUnpin(op) => { - res = self.broker.unpin_object(self.user, overlay, op.id()) - } - BrokerOverlayRequestContentV0::BlockPut(b) => { - res = self.broker.put_block(self.user, overlay, b.block()) - } - BrokerOverlayRequestContentV0::BranchSyncReq(b) => { - let res = self.broker.sync_branch( - self.user, - &overlay, - b.heads(), - b.known_heads(), - b.known_commits(), - ); - return self - .send_block_stream_response_to_client( - res, - id, - overlay, - padding_size, - ) - .await; - } - BrokerOverlayRequestContentV0::BlockGet(b) => { - let res = self.broker.get_block( - self.user, - overlay, - b.id(), - b.include_children(), - b.topic(), - ); - return self - .send_block_stream_response_to_client( - res, - id, - overlay, - padding_size, - ) - .await; - } - _ => {} - } - } - - ( - Self::prepare_reply_broker_overlay_message( - res, - id, - overlay, - block, - padding_size, - ), - OptionFuture::from(None), - ) - } - } - } -} - -pub enum PeerConnection { - CORE(IP), - CLIENT(Arc), - NONE, -} - -pub struct BrokerPeerInfo { - lastPeerAdvert: Option, - connected: PeerConnection, -} - -const REPO_STORES_SUBDIR: &str = "repos"; - -pub struct BrokerServer { - store: LmdbKCVStore, - mode: ConfigMode, - repo_stores: Arc>>, - // only used in ConfigMode::Local - // try to change it to this version below in order to avoid double hashmap lookup in local mode. but hard to do... - //overlayid_to_repostore: HashMap, - //overlayid_to_repostore: Arc>>, - peers: RwLock>, - //local_connections: -} - -impl BrokerServer { - pub fn add_client_peer( - &self, - peer_id: DirectPeerId, - bph: Arc, - ) -> Result<(), ProtocolError> { - let mut writer = self.peers.write().expect("write peers hashmap"); - let bpi = BrokerPeerInfo { - lastPeerAdvert: None, //TODO: load from store - connected: PeerConnection::CLIENT(bph), - }; - if !writer.get(&peer_id).is_none() { - return Err(ProtocolError::PeerAlreadyConnected); - } - writer.insert(peer_id.clone(), bpi); - Ok(()) - } - - pub fn remove_client_peer(&self, peer_id: DirectPeerId) { - let mut writer = self.peers.write().expect("write peers hashmap"); - writer.remove(&peer_id); - } - - pub fn new(store: LmdbKCVStore, mode: ConfigMode) -> Result { - let mut configmode: ConfigMode; - { - let config = Config::get_or_create(&mode, &store)?; - configmode = config.mode()?; - } - Ok(BrokerServer { - store, - mode: configmode, - repo_stores: Arc::new(RwLock::new(HashMap::new())), - //overlayid_to_repostore: Arc::new(RwLock::new(HashMap::new())), - peers: RwLock::new(HashMap::new()), - }) - } - - fn open_or_create_repostore(&self, repo_hash: RepoHash, f: F) -> Result - where - F: FnOnce(&LmdbRepoStore) -> Result, - { - // first let's find it in the KCVStore.repostoreinfo table in order to get the encryption key - let info = RepoStoreInfo::open(&repo_hash, &self.store) - .map_err(|e| BrokerError::OverlayNotFound)?; - let key = info.key()?; - let mut path = self.store.path(); - path.push(REPO_STORES_SUBDIR); - path.push::(repo_hash.clone().into()); - std::fs::create_dir_all(path.clone()).map_err(|_e| ProtocolError::IoError)?; - println!("path for repo store: {}", path.to_str().unwrap()); - let repo = LmdbRepoStore::open(&path, *key.slice()); - let mut writer = self.repo_stores.write().expect("write repo_store hashmap"); - writer.insert(repo_hash.clone(), repo); - f(writer.get(&repo_hash).unwrap()) - } - - fn get_repostore_from_overlay_id( - &self, - overlay_id: &OverlayId, - f: F, - ) -> Result - where - F: FnOnce(&LmdbRepoStore) -> Result, - { - //FIXME: the whole purpose of get_repostore_from_overlay_id is gone. review all of it - let repostore_id = *overlay_id; - { - let reader = self.repo_stores.read().expect("read repo_store hashmap"); - let rep = reader.get(&repostore_id); - if let Some(repo) = rep { - return f(repo); - } - } - // we need to open/create it - // TODO: last_access - return self.open_or_create_repostore(repostore_id, |repo| f(repo)); - - // } else { - // // it is ConfigMode::Local - // { - // let reader = self - // .overlayid_to_repostore - // .read() - // .expect("read overlayid_to_repostore hashmap"); - // match reader.get(&overlay_id) { - // Some(repostoreid) => { - // let reader = self.repo_stores.read().expect("read repo_store hashmap"); - // match reader.get(repostoreid) { - // Some(repo) => return f(repo), - // None => return Err(ProtocolError::BrokerError), - // } - // } - // None => {} - // }; - // } - - // // we need to open/create it - // // first let's find it in the KCVStore.overlay table to retrieve its repo_pubkey - // log_debug!("searching for overlayId {}", overlay_id); - // let overlay = Overlay::open(overlay_id, &self.store)?; - // log_debug!("found overlayId {}", overlay_id); - // let repo_id = overlay.repo()?; - // let repostore_id = RepoStoreId::Repo(repo_id); - // let mut writer = self - // .overlayid_to_repostore - // .write() - // .expect("write overlayid_to_repostore hashmap"); - // writer.insert(*overlay_id, repostore_id.clone()); - // // now opening/creating the RepoStore - // // TODO: last_access - // return self.open_or_create_repostore(repostore_id, |repo| f(repo)); - // } - } - - pub fn local_connection(&mut self, user: PubKey) -> BrokerConnectionLocal { - BrokerConnectionLocal::new(self, user) - } - - pub fn protocol_handler(self: Arc) -> ProtocolHandler { - let (s, r) = async_channel::unbounded::>(); - return ProtocolHandler { - addr: None, - broker: Arc::clone(&self), - protocol: ProtocolType::Start, - auth_protocol: None, - broker_protocol: None, - ext_protocol: None, - r: Some(r), - s, - }; - } - - pub fn add_user( - &self, - admin_user: PubKey, - user_id: PubKey, - sig: Sig, - ) -> Result<(), ProtocolError> { - log_debug!("ADDING USER {}", user_id); - // TODO add is_admin boolean - // TODO check that admin_user is indeed an admin - - // verify signature - let op_content = AddUserContentV0 { user: user_id }; - let _ = verify(&serde_bare::to_vec(&op_content).unwrap(), sig, admin_user)?; - - // check user_id is not already present - let account = Account::open(&user_id, &self.store); - if account.is_ok() { - Err(ProtocolError::UserAlreadyExists) - } - // if not, add to store - else { - let _ = Account::create(&user_id, false, &self.store)?; - Ok(()) - } - } - - pub fn del_user( - &self, - admin_user: PubKey, - user_id: PubKey, - sig: Sig, - ) -> Result<(), ProtocolError> { - // TODO implement del_user - Ok(()) - } - pub fn add_client( - &self, - user: PubKey, - client_id: PubKey, - sig: Sig, - ) -> Result<(), ProtocolError> { - // TODO implement add_client - Ok(()) - } - - pub fn del_client( - &self, - user: PubKey, - client_id: PubKey, - sig: Sig, - ) -> Result<(), ProtocolError> { - // TODO implement del_client - Ok(()) - } - - pub fn connect_overlay(&self, user: PubKey, overlay: OverlayId) -> Result<(), ProtocolError> { - // TODO check that the broker has already joined this overlay. if not, send OverlayNotJoined - Err(ProtocolError::OverlayNotJoined) - } - - pub fn del_object( - &self, - user: PubKey, - overlay: Digest, - id: ObjectId, - ) -> Result<(), ProtocolError> { - self.get_repostore_from_overlay_id(&overlay, |store| { - // TODO, only admin users can delete on a store on this broker - let obj = Object::load(id, None, store); - if obj.is_err() { - return Err(ProtocolError::NotFound); - } - let o = obj.ok().unwrap(); - let mut deduplicated: HashSet = HashSet::new(); - for block in o.blocks() { - let id = block.id(); - if deduplicated.get(&id).is_none() { - store.del(&id)?; - deduplicated.insert(id); - } - } - Ok(()) - }) - } - - pub fn pin_object( - &self, - user: PubKey, - overlay: OverlayId, - id: ObjectId, - ) -> Result<(), ProtocolError> { - self.get_repostore_from_overlay_id(&overlay, |store| { - // TODO, store the user who pins, and manage reference counting on how many users pin/unpin - let obj = Object::load(id, None, store); - if obj.is_err() { - return Err(ProtocolError::NotFound); - } - let o = obj.ok().unwrap(); - let mut deduplicated: HashSet = HashSet::new(); - for block in o.blocks() { - let id = block.id(); - if deduplicated.get(&id).is_none() { - store.pin(&id)?; - deduplicated.insert(id); - } - } - Ok(()) - }) - } - - pub fn unpin_object( - &self, - user: PubKey, - overlay: OverlayId, - id: ObjectId, - ) -> Result<(), ProtocolError> { - self.get_repostore_from_overlay_id(&overlay, |store| { - // TODO, store the user who pins, and manage reference counting on how many users pin/unpin - let obj = Object::load(id, None, store); - if obj.is_err() { - return Err(ProtocolError::NotFound); - } - let o = obj.ok().unwrap(); - let mut deduplicated: HashSet = HashSet::new(); - for block in o.blocks() { - let id = block.id(); - if deduplicated.get(&id).is_none() { - store.unpin(&id)?; - deduplicated.insert(id); - } - } - Ok(()) - }) - } - - pub fn copy_object( - &self, - user: PubKey, - overlay: OverlayId, - id: ObjectId, - expiry: Option, - ) -> Result { - // self.get_repostore_from_overlay_id(&overlay, |store| { - // //let obj = Object::from_store(id, None, store); - // //Ok(Object::copy(id, expiry, store)?) - // }); - todo!(); - } - - pub fn put_block( - &self, - user: PubKey, - overlay: OverlayId, - block: &Block, - ) -> Result<(), ProtocolError> { - self.get_repostore_from_overlay_id(&overlay, |store| { - let _ = store.put(block)?; - Ok(()) - }) - } - - pub fn get_block( - &self, - user: PubKey, - overlay: OverlayId, - id: BlockId, - include_children: bool, - topic: Option, - ) -> Result, ProtocolError> { - self.get_repostore_from_overlay_id(&overlay, |store| { - let (s, r) = async_channel::unbounded::(); - if !include_children { - let block = store.get(&id)?; - s.send_blocking(block) - .map_err(|_e| ProtocolError::IoError)?; - Ok(r) - } else { - let obj = Object::load(id, None, store); - // TODO return partial blocks when some are missing ? - if obj.is_err() { - //&& obj.err().unwrap().len() == 1 && obj.err().unwrap()[0] == id { - return Err(ProtocolError::NotFound); - } - // TODO use a task to send non blocking (streaming) - let o = obj.ok().unwrap(); - //log_debug!("{} BLOCKS ", o.blocks().len()); - let mut deduplicated: HashSet = HashSet::new(); - for block in o.blocks() { - let id = block.id(); - if deduplicated.get(&id).is_none() { - s.send_blocking(block.clone()) - .map_err(|_e| ProtocolError::IoError)?; - deduplicated.insert(id); - } - } - Ok(r) - } - }) - } - - pub fn sync_branch( - &self, - user: PubKey, - overlay: &OverlayId, - heads: &Vec, - known_heads: &Vec, - known_commits: &BloomFilter, - ) -> Result, ProtocolError> { - //log_debug!("heads {:?}", heads); - //log_debug!("known_heads {:?}", known_heads); - //log_debug!("known_commits {:?}", known_commits); - - self.get_repostore_from_overlay_id(&overlay, |store| { - let (s, r) = async_channel::unbounded::(); - - let res = Branch::sync_req(heads, known_heads, known_commits, store) - .map_err(|e| ProtocolError::ObjectParseError)?; - - // todo, use a task to send non blocking (streaming) - log_debug!("SYNCING {} COMMITS", res.len()); - - let mut deduplicated: HashSet = HashSet::new(); - - for objectid in res { - let object = Object::load(objectid, None, store)?; - - for block in object.blocks() { - let id = block.id(); - if deduplicated.get(&id).is_none() { - s.send_blocking(block.clone()) - .map_err(|_e| ProtocolError::IoError)?; - deduplicated.insert(id); - } - } - } - Ok(r) - }) - } - - // fn compute_repostore_id(&self, overlay: OverlayId, repo_id: Option) -> RepoStoreId { - // match self.mode { - // ConfigMode::Core => RepoStoreId::Overlay(overlay), - // ConfigMode::Local => RepoStoreId::Repo(repo_id.unwrap()), - // } - // } - - pub fn join_overlay( - &self, - user: PubKey, - overlay_id: OverlayId, - repo_id: Option, - secret: SymKey, - peers: &Vec, - ) -> Result<(), ProtocolError> { - // check if this overlay already exists - //log_debug!("SEARCHING OVERLAY"); - let overlay_res = Overlay::open(&overlay_id, &self.store); - let overlay = match overlay_res { - Err(StorageError::NotFound) => { - // we have to add it - if self.mode == ConfigMode::Local && repo_id.is_none() { - return Err(ProtocolError::RepoIdRequired); - } - let over = Overlay::create( - &overlay_id, - &secret, - if self.mode == ConfigMode::Local { - repo_id - } else { - None - }, - &self.store, - )?; - // we need to add an encryption key for the repostore. - let mut random_buf = [0u8; 32]; - getrandom::getrandom(&mut random_buf).unwrap(); - let key = SymKey::ChaCha20Key(random_buf); - - let _ = RepoStoreInfo::create(&overlay_id, &key, &self.store)?; // TODO in case of error, delete the previously created Overlay - //log_debug!("KEY ADDED"); - over - } - Err(e) => return Err(e.into()), - Ok(overlay) => overlay, - }; - //log_debug!("OVERLAY FOUND"); - // add the peers to the overlay - for advert in peers { - Peer::update_or_create(advert, &self.store)?; - overlay.add_peer(&advert.peer())?; - } - //log_debug!("PEERS ADDED"); - - // now adding the overlay_id to the account - let account = Account::open(&user, &self.store)?; // TODO in case of error, delete the previously created Overlay - account.add_overlay(&overlay_id)?; - //log_debug!("USER <-> OVERLAY"); - - //TODO: connect to peers - - Ok(()) - } -} diff --git a/ng-broker/src/server_storage.rs b/ng-broker/src/server_storage.rs index bcb0cb0..73caac1 100644 --- a/ng-broker/src/server_storage.rs +++ b/ng-broker/src/server_storage.rs @@ -15,18 +15,18 @@ use std::io::Write; use std::path::{Path, PathBuf}; use std::sync::Mutex; -use crate::broker_store::account::Account; -use crate::broker_store::invitation::Invitation; -use crate::broker_store::wallet::Wallet; +use crate::broker_storage::account::Account; +use crate::broker_storage::invitation::Invitation; +use crate::broker_storage::wallet::Wallet; use crate::types::*; use ng_net::errors::{ProtocolError, ServerError}; use ng_net::server_storage::*; use ng_net::types::{BootstrapContentV0, InvitationCode, InvitationV0}; -use ng_repo::kcv_store::KCVStore; +use ng_repo::errors::StorageError; +use ng_repo::kcv_storage::KCVStore; use ng_repo::log::*; -use ng_repo::store::StorageError; use ng_repo::types::{PeerId, PubKey, SymKey}; -use ng_stores_rocksdb::kcv_store::RocksdbKCVStore; +use ng_storage_rocksdb::kcv_storage::RocksdbKCVStore; pub struct RocksdbServerStorage { wallet_storage: RocksdbKCVStore, diff --git a/ng-client-ws/README.md b/ng-client-ws/README.md index 0eb5805..192a0e8 100644 --- a/ng-client-ws/README.md +++ b/ng-client-ws/README.md @@ -48,7 +48,7 @@ additional terms or conditions. NextGraph received funding through the [NGI Assure Fund](https://nlnet.nl/project/NextGraph/index.html), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's [Next Generation Internet](https://ngi.eu/) programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 957073. -[rustc-image]: https://img.shields.io/badge/rustc-1.64+-blue.svg +[rustc-image]: https://img.shields.io/badge/rustc-1.74+-blue.svg [license-image]: https://img.shields.io/badge/license-Apache2.0-blue.svg [license-link]: https://git.nextgraph.org/NextGraph/nextgraph-rs/raw/branch/master/LICENSE-APACHE2 [license-image2]: https://img.shields.io/badge/license-MIT-blue.svg diff --git a/ng-net/Cargo.toml b/ng-net/Cargo.toml index e115220..c239d8b 100644 --- a/ng-net/Cargo.toml +++ b/ng-net/Cargo.toml @@ -48,4 +48,4 @@ features = ["js"] [target.'cfg(not(target_arch = "wasm32"))'.dependencies] getrandom = "0.2.7" default-net = { git = "https://git.nextgraph.org/NextGraph/default-net.git" } -# ng-stores-rocksdb = { path = "../ng-stores-rocksdb", version = "0.1.0" } +# ng-storage-rocksdb = { path = "../ng-storage-rocksdb", version = "0.1.0" } diff --git a/ng-net/README.md b/ng-net/README.md index 4fabb6a..adf3a9f 100644 --- a/ng-net/README.md +++ b/ng-net/README.md @@ -48,7 +48,7 @@ additional terms or conditions. NextGraph received funding through the [NGI Assure Fund](https://nlnet.nl/project/NextGraph/index.html), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's [Next Generation Internet](https://ngi.eu/) programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 957073. -[rustc-image]: https://img.shields.io/badge/rustc-1.64+-blue.svg +[rustc-image]: https://img.shields.io/badge/rustc-1.74+-blue.svg [license-image]: https://img.shields.io/badge/license-Apache2.0-blue.svg [license-link]: https://git.nextgraph.org/NextGraph/nextgraph-rs/raw/branch/master/LICENSE-APACHE2 [license-image2]: https://img.shields.io/badge/license-MIT-blue.svg diff --git a/ng-net/src/broker.rs b/ng-net/src/broker.rs index 32a38f9..e26229c 100644 --- a/ng-net/src/broker.rs +++ b/ng-net/src/broker.rs @@ -22,11 +22,11 @@ use async_std::sync::{Arc, RwLock}; use either::Either; use futures::channel::mpsc; use futures::SinkExt; +use ng_repo::block_storage::HashMapBlockStorage; use ng_repo::errors::NgError; use ng_repo::errors::ObjectParseError; use ng_repo::log::*; use ng_repo::object::Object; -use ng_repo::store::HashMapRepoStore; use ng_repo::types::*; use ng_repo::utils::generate_keypair; use once_cell::sync::Lazy; @@ -88,9 +88,6 @@ pub struct Broker<'a> { tauri_streams: HashMap>, disconnections_sender: Sender, disconnections_receiver: Option>, - //last_seq_function: Option>, - //base_path: Option, - //in_memory: bool, local_broker: Option>, } @@ -346,7 +343,7 @@ impl<'a> Broker<'a> { let blockstream = self .get_block_from_store_with_block_id(nuri, obj_ref.id, true) .await?; - let store = Box::new(HashMapRepoStore::from_block_stream(blockstream).await); + let store = Box::new(HashMapBlockStorage::from_block_stream(blockstream).await); Object::load(obj_ref.id, Some(obj_ref.key), &store) .map_err(|e| match e { @@ -451,7 +448,7 @@ impl<'a> Broker<'a> { // #[cfg(not(target_arch = "wasm32"))] // pub fn test_storage(&self, path: PathBuf) { - // use ng_stores_rocksdb::kcv_store::RocksdbKCVStore; + // use ng_storage_rocksdb::kcv_store::RocksdbKCVStore; // let key: [u8; 32] = [0; 32]; // let test_storage = RocksdbKCVStore::open(&path, key); diff --git a/ng-net/src/broker_connection.rs b/ng-net/src/broker_connection.rs index 5970b14..883a3fb 100644 --- a/ng-net/src/broker_connection.rs +++ b/ng-net/src/broker_connection.rs @@ -266,7 +266,7 @@ where topic: Option, ) -> Result { let mut blockstream = self.get_block(id, true, topic).await?; - let mut store = HashMapRepoStore::new(); + let mut store = HashMapBlockStorage::new(); while let Some(block) = blockstream.next().await { store.put(&block).unwrap(); } diff --git a/ng-net/src/errors.rs b/ng-net/src/errors.rs index 3bcefe2..9659dbe 100644 --- a/ng-net/src/errors.rs +++ b/ng-net/src/errors.rs @@ -8,8 +8,7 @@ // according to those terms. use core::fmt; -use ng_repo::errors::ObjectParseError; -use ng_repo::store::StorageError; +use ng_repo::errors::{ObjectParseError, StorageError}; use num_enum::IntoPrimitive; use num_enum::TryFromPrimitive; use std::convert::From; diff --git a/ng-net/src/types.rs b/ng-net/src/types.rs index e7d75ee..ae0209b 100644 --- a/ng-net/src/types.rs +++ b/ng-net/src/types.rs @@ -154,6 +154,11 @@ pub struct BrokerServerV0 { pub peer_id: PubKey, } +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] +pub enum BrokerServer { + V0(BrokerServerV0), +} + impl BrokerServerV0 { pub fn new_localhost(peer_id: PubKey) -> Self { BrokerServerV0 { @@ -2516,6 +2521,7 @@ impl AdminResponse { /// /// When client will disconnect, the subscriptions and publisherAdvert of the topics will be removed, /// except if a PinRepo occurred before or after the OpenRepo +/// replied with a RepoOpened #[derive(Clone, Debug, Serialize, Deserialize)] pub struct OpenRepoV0 { /// Repo Hash @@ -2536,6 +2542,7 @@ pub struct OpenRepoV0 { pub allowed_peers: Vec, /// Maximum number of peers to connect to for this overlay (only valid for an inner (RW/WO) overlay) + /// 0 means automatic/unlimited pub max_peer_count: u16, /// list of topics that should be subscribed to @@ -2636,7 +2643,7 @@ pub struct RefreshPinRepoV0 { /// The userId of banned user is revealed to the local broker where it was attached, which is a breach of privacy deemed acceptable /// as only a broker that already knew the userid will enforce it, and /// that broker might be interested to know that the offending user was banned from a repo, as only malicious users are banned. - /// The broker might also discard this information, and just proceeed with the flush without much ado. + /// The broker might also discard this information, and just proceed with the flush without much ado. /// Of course, if the broker is controlled by the malicious user, it might not proceed with the ban/flush. But who cares. That broker will keep old data forever, but it is a malicious broker anyway. pub flush_topics: Vec<(TopicId, Sig)>, } @@ -2731,6 +2738,8 @@ impl RepoPinStatus { } /// Request subscription to a `Topic` of an already opened or pinned Repo +/// +/// replied with a list of TopicSubRes containing the current heads that should be used to do a TopicSync #[derive(Clone, Copy, Debug, Serialize, Deserialize)] pub struct TopicSubV0 { /// Topic to subscribe @@ -3005,11 +3014,29 @@ impl BlocksFound { } } +/// Topic subscription response V0 +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct TopicSubResV0 { + /// Topic subscribed + pub topic: PubKey, + pub known_heads: Vec, +} + +/// Topic subscription response +/// +/// it is a stream of blocks and or events. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum TopicSubRes { + V0(TopicSubResV0), +} + /// Content of `ClientResponseV0` #[derive(Clone, Debug, Serialize, Deserialize)] pub enum ClientResponseContentV0 { EmptyResponse, Block(Block), + RepoOpened(Vec), + TopicSubRes(TopicSubRes), TopicSyncRes(TopicSyncRes), BlocksFound(BlocksFound), RepoPinStatus(RepoPinStatus), diff --git a/ng-repo/README.md b/ng-repo/README.md index 38b8a74..5940bb6 100644 --- a/ng-repo/README.md +++ b/ng-repo/README.md @@ -48,7 +48,7 @@ additional terms or conditions. NextGraph received funding through the [NGI Assure Fund](https://nlnet.nl/project/NextGraph/index.html), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's [Next Generation Internet](https://ngi.eu/) programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 957073. -[rustc-image]: https://img.shields.io/badge/rustc-1.64+-blue.svg +[rustc-image]: https://img.shields.io/badge/rustc-1.74+-blue.svg [license-image]: https://img.shields.io/badge/license-Apache2.0-blue.svg [license-link]: https://git.nextgraph.org/NextGraph/nextgraph-rs/raw/branch/master/LICENSE-APACHE2 [license-image2]: https://img.shields.io/badge/license-MIT-blue.svg diff --git a/ng-repo/src/store.rs b/ng-repo/src/block_storage.rs similarity index 85% rename from ng-repo/src/store.rs rename to ng-repo/src/block_storage.rs index 94d0770..51d7d63 100644 --- a/ng-repo/src/store.rs +++ b/ng-repo/src/block_storage.rs @@ -11,6 +11,7 @@ use futures::StreamExt; +use crate::errors::*; use crate::types::*; use crate::utils::Receiver; use std::sync::RwLock; @@ -20,7 +21,7 @@ use std::{ mem::size_of_val, }; -pub trait RepoStore: Send + Sync { +pub trait BlockStorage: Send + Sync { /// Load a block from the storage. fn get(&self, id: &BlockId) -> Result; @@ -34,29 +35,6 @@ pub trait RepoStore: Send + Sync { fn len(&self) -> Result; } -#[derive(Debug, PartialEq, Eq, Clone)] -pub enum StorageError { - NotFound, - InvalidValue, - DifferentValue, - BackendError, - SerializationError, - AlreadyExists, - DataCorruption, -} - -impl core::fmt::Display for StorageError { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - write!(f, "{:?}", self) - } -} - -impl From for StorageError { - fn from(_e: serde_bare::error::Error) -> Self { - StorageError::SerializationError - } -} - /* LMDB values: const MIN_SIZE: usize = 4072; @@ -102,13 +80,13 @@ pub const fn store_max_value_size() -> usize { } /// Store with a HashMap backend -pub struct HashMapRepoStore { +pub struct HashMapBlockStorage { blocks: RwLock>, } -impl HashMapRepoStore { - pub fn new() -> HashMapRepoStore { - HashMapRepoStore { +impl HashMapBlockStorage { + pub fn new() -> HashMapBlockStorage { + HashMapBlockStorage { blocks: RwLock::new(HashMap::new()), } } @@ -135,7 +113,7 @@ impl HashMapRepoStore { } } -impl RepoStore for HashMapRepoStore { +impl BlockStorage for HashMapBlockStorage { fn get(&self, id: &BlockId) -> Result { match self.blocks.read().unwrap().get(id) { Some(block) => { diff --git a/ng-repo/src/branch.rs b/ng-repo/src/branch.rs index cd6f0b2..079f4d4 100644 --- a/ng-repo/src/branch.rs +++ b/ng-repo/src/branch.rs @@ -14,9 +14,9 @@ use zeroize::{Zeroize, ZeroizeOnDrop}; // use fastbloom_rs::{BloomFilter as Filter, Membership}; +use crate::block_storage::*; use crate::errors::*; use crate::object::*; -use crate::store::*; use crate::types::*; use crate::utils::encrypt_in_place; @@ -33,6 +33,7 @@ impl BranchV0 { let topic = topic_priv.to_pub(); BranchV0 { id, + content_type: BranchContentType::None, repo, root_branch_readcap_id, topic, @@ -96,18 +97,18 @@ impl Branch { target_heads: &[ObjectId], known_heads: &[ObjectId], //their_filter: &BloomFilter, - store: &Box, + store: &Box, ) -> Result, ObjectParseError> { //log_debug!(">> sync_req"); //log_debug!(" target_heads: {:?}", target_heads); //log_debug!(" known_heads: {:?}", known_heads); - /// Load causal past of a Commit `cobj` in a `Branch` from the `RepoStore`, + /// Load causal past of a Commit `cobj` in a `Branch` from the `BlockStorage`, /// and collect in `visited` the ObjectIds encountered on the way, stopping at any commit already belonging to `theirs` or the root of DAG. /// optionally collecting the missing objects/blocks that couldn't be found locally on the way fn load_causal_past( cobj: &Object, - store: &Box, + store: &Box, theirs: &HashSet, visited: &mut HashSet, missing: &mut Option<&mut HashSet>, @@ -179,16 +180,16 @@ mod test { //use fastbloom_rs::{BloomFilter as Filter, FilterBuilder, Membership}; struct Test<'a> { - storage: Box, + storage: Box, } impl<'a> Test<'a> { - fn storage(s: impl RepoStore + 'a) -> Self { + fn storage(s: impl BlockStorage + 'a) -> Self { Test { storage: Box::new(s), } } - fn s(&self) -> &Box { + fn s(&self) -> &Box { &self.storage } } @@ -207,7 +208,7 @@ mod test { header: Option, store_pubkey: &StoreRepo, store_secret: &ReadCapSecret, - store: &Box, + store: &Box, ) -> ObjectRef { let max_object_size = 4000; let mut obj = Object::new( @@ -234,7 +235,7 @@ mod test { body_ref: ObjectRef, store_pubkey: &StoreRepo, store_secret: &ReadCapSecret, - store: &Box, + store: &Box, ) -> ObjectRef { let header = CommitHeader::new_with_deps_and_acks( deps.iter().map(|r| r.id).collect(), @@ -280,7 +281,7 @@ mod test { branch: BranchV0, store_pubkey: &StoreRepo, store_secret: &ReadCapSecret, - store: &Box, + store: &Box, ) -> ObjectRef { let body: CommitBodyV0 = CommitBodyV0::Branch(Branch::V0(branch)); //log_debug!("body: {:?}", body); @@ -297,7 +298,7 @@ mod test { header: Option, store_pubkey: &StoreRepo, store_secret: &ReadCapSecret, - store: &Box, + store: &Box, ) -> ObjectRef { let content = [7u8; 777].to_vec(); let body = CommitBodyV0::AsyncTransaction(Transaction::V0(content)); @@ -311,7 +312,7 @@ mod test { ) } - let hashmap_storage = HashMapRepoStore::new(); + let hashmap_storage = HashMapBlockStorage::new(); let t = Test::storage(hashmap_storage); // repo diff --git a/ng-repo/src/commit.rs b/ng-repo/src/commit.rs index 5a094c1..dc2ff5e 100644 --- a/ng-repo/src/commit.rs +++ b/ng-repo/src/commit.rs @@ -15,11 +15,11 @@ use once_cell::sync::OnceCell; use crate::errors::NgError; +use crate::block_storage::*; use crate::errors::*; use crate::log::*; use crate::object::*; use crate::repo::Repo; -use crate::store::*; use crate::types::*; use crate::utils::*; use std::collections::HashSet; @@ -129,7 +129,7 @@ impl CommitV0 { block_size: usize, store_pubkey: &StoreRepo, store_secret: &ReadCapSecret, - store: &Box, + store: &Box, ) -> Result { if self.id.is_some() && self.key.is_some() { return Ok(ObjectRef::from_id_key( @@ -224,7 +224,7 @@ impl Commit { body: CommitBody, store_pubkey: &StoreRepo, store_secret: &ReadCapSecret, - storage: &Box, + storage: &Box, ) -> Result { Self::new_with_body_and_save( author_privkey, @@ -263,7 +263,7 @@ impl Commit { block_size: usize, store_pubkey: &StoreRepo, store_secret: &ReadCapSecret, - storage: &Box, + storage: &Box, ) -> Result { let (body_ref, mut saved_body) = body.clone() @@ -307,7 +307,7 @@ impl Commit { block_size: usize, store_pubkey: &StoreRepo, store_secret: &ReadCapSecret, - store: &Box, + store: &Box, ) -> Result { match self { Commit::V0(v0) => v0.save(block_size, store_pubkey, store_secret, store), @@ -323,7 +323,7 @@ impl Commit { /// Load commit from store pub fn load( commit_ref: ObjectRef, - store: &Box, + store: &Box, with_body: bool, ) -> Result { let (id, key) = (commit_ref.id, commit_ref.key); @@ -356,7 +356,7 @@ impl Commit { /// Load commit body from store pub fn load_body( &self, - store: &Box, + store: &Box, ) -> Result<&CommitBody, CommitLoadError> { if self.body().is_some() { return Ok(self.body().unwrap()); @@ -457,7 +457,7 @@ impl Commit { pub fn owners_signature_required( &self, - store: &Box, + store: &Box, ) -> Result { match self.load_body(store)? { CommitBody::V0(CommitBodyV0::UpdateRootBranch(new_root)) => { @@ -635,15 +635,15 @@ impl Commit { /// or a list of missing blocks pub fn verify_full_object_refs_of_branch_at_commit( &self, - store: &Box, + store: &Box, ) -> Result, CommitLoadError> { //log_debug!(">> verify_full_object_refs_of_branch_at_commit: #{}", self.seq()); - /// Load `Commit`s of a `Branch` from the `RepoStore` starting from the given `Commit`, + /// Load `Commit`s of a `Branch` from the `BlockStorage` starting from the given `Commit`, /// and collect missing `ObjectId`s fn load_direct_object_refs( commit: &Commit, - store: &Box, + store: &Box, visited: &mut HashSet, missing: &mut HashSet, ) -> Result<(), CommitLoadError> { @@ -772,7 +772,7 @@ impl CommitBody { block_size: usize, store_pubkey: &StoreRepo, store_secret: &ReadCapSecret, - store: &Box, + store: &Box, ) -> Result<(ObjectRef, Vec), StorageError> { let obj = Object::new( ObjectContent::V0(ObjectContentV0::CommitBody(self)), @@ -1458,16 +1458,16 @@ mod test { use crate::log::*; struct Test<'a> { - storage: Box, + storage: Box, } impl<'a> Test<'a> { - fn storage(s: impl RepoStore + 'a) -> Self { + fn storage(s: impl BlockStorage + 'a) -> Self { Test { storage: Box::new(s), } } - fn s(&self) -> &Box { + fn s(&self) -> &Box { &self.storage } } @@ -1511,7 +1511,7 @@ mod test { let max_object_size = 0; let (store_repo, store_secret) = StoreRepo::dummy_public_v0(); - let hashmap_storage = HashMapRepoStore::new(); + let hashmap_storage = HashMapBlockStorage::new(); let storage = Box::new(hashmap_storage); let commit_ref = commit @@ -1579,7 +1579,7 @@ mod test { &store_secret, ); - let hashmap_storage = HashMapRepoStore::new(); + let hashmap_storage = HashMapBlockStorage::new(); let storage = Box::new(hashmap_storage); _ = obj.save(&storage).expect("save object"); @@ -1612,7 +1612,7 @@ mod test { let max_object_size = 0; let (store_repo, store_secret) = StoreRepo::dummy_public_v0(); - let hashmap_storage = HashMapRepoStore::new(); + let hashmap_storage = HashMapBlockStorage::new(); let storage = Box::new(hashmap_storage); let commit = Commit::new_with_body_and_save( @@ -1676,7 +1676,7 @@ mod test { .unwrap(); log_debug!("{}", commit); - let hashmap_storage = HashMapRepoStore::new(); + let hashmap_storage = HashMapBlockStorage::new(); let t = Test::storage(hashmap_storage); let repo = @@ -1735,7 +1735,7 @@ mod test { let max_object_size = 0; let (store_repo, store_secret) = StoreRepo::dummy_public_v0(); - let hashmap_storage = HashMapRepoStore::new(); + let hashmap_storage = HashMapBlockStorage::new(); let t = Test::storage(hashmap_storage); let commit = Commit::new_with_body_and_save( @@ -1808,7 +1808,7 @@ mod test { log_debug!("{}", commit); - let hashmap_storage = HashMapRepoStore::new(); + let hashmap_storage = HashMapBlockStorage::new(); let t = Test::storage(hashmap_storage); let repo = Repo::new_with_member( diff --git a/ng-repo/src/errors.rs b/ng-repo/src/errors.rs index 6475f3a..9ddc986 100644 --- a/ng-repo/src/errors.rs +++ b/ng-repo/src/errors.rs @@ -10,7 +10,7 @@ //! Errors use crate::commit::{CommitLoadError, CommitVerifyError}; -use crate::store::StorageError; + use crate::types::BlockId; use core::fmt; use std::error::Error; @@ -149,3 +149,26 @@ pub enum ObjectParseError { /// Error deserializing content of the object ObjectDeserializeError, } + +#[derive(Debug, PartialEq, Eq, Clone)] +pub enum StorageError { + NotFound, + InvalidValue, + DifferentValue, + BackendError, + SerializationError, + AlreadyExists, + DataCorruption, +} + +impl core::fmt::Display for StorageError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "{:?}", self) + } +} + +impl From for StorageError { + fn from(_e: serde_bare::error::Error) -> Self { + StorageError::SerializationError + } +} diff --git a/ng-repo/src/event.rs b/ng-repo/src/event.rs index e567fd9..243a83f 100644 --- a/ng-repo/src/event.rs +++ b/ng-repo/src/event.rs @@ -8,9 +8,9 @@ //! Event, a message sent in the PUB/SUB +use crate::block_storage::*; use crate::errors::*; use crate::object::*; -use crate::store::*; use crate::types::*; use crate::utils::*; use core::fmt; @@ -61,7 +61,7 @@ impl Event { topic_id: TopicId, branch_read_cap_secret: ReadCapSecret, topic_priv_key: &BranchWriteCapSecret, - storage: &'a Box, + storage: &'a Box, ) -> Result { Ok(Event::V0(EventV0::new( publisher, @@ -85,7 +85,7 @@ impl EventV0 { topic_id: TopicId, branch_read_cap_secret: ReadCapSecret, topic_priv_key: &BranchWriteCapSecret, - storage: &'a Box, + storage: &'a Box, ) -> Result { let mut blocks = vec![]; for bid in commit.blocks().iter() { diff --git a/ng-repo/src/file.rs b/ng-repo/src/file.rs index 8ac50fd..e0d0e3b 100644 --- a/ng-repo/src/file.rs +++ b/ng-repo/src/file.rs @@ -17,10 +17,10 @@ use chacha20::cipher::{KeyIvInit, StreamCipher}; use chacha20::ChaCha20; use zeroize::Zeroize; +use crate::block_storage::*; use crate::errors::*; use crate::log::*; use crate::object::*; -use crate::store::*; use crate::types::*; /// File errors @@ -83,7 +83,7 @@ impl<'a> File<'a> { pub fn open( id: ObjectId, key: SymKey, - storage: &'a Box, + storage: &'a Box, ) -> Result, FileError> { let root_block = storage.get(&id)?; @@ -133,8 +133,8 @@ impl ReadFile for SmallFileV0 { /// A RandomAccessFile in memory. This is not used to serialize data pub struct RandomAccessFile<'a> { - //storage: Arc<&'a dyn RepoStore>, - storage: &'a Box, + //storage: Arc<&'a dyn BlockStorage>, + storage: &'a Box, /// accurate once saved or opened meta: RandomAccessFileMeta, @@ -263,7 +263,7 @@ impl<'a> RandomAccessFile<'a> { conv_key: &[u8; blake3::OUT_LEN], children: Vec, already_existing: &mut HashMap, - storage: &Box, + storage: &Box, ) -> Result<(BlockId, BlockKey), StorageError> { let key_hash = blake3::keyed_hash(conv_key, &content); @@ -294,7 +294,7 @@ impl<'a> RandomAccessFile<'a> { conv_key: &[u8; blake3::OUT_LEN], children: Vec<(BlockId, BlockKey)>, already_existing: &mut HashMap, - storage: &Box, + storage: &Box, ) -> Result<(BlockId, BlockKey), StorageError> { let mut ids: Vec = Vec::with_capacity(children.len()); let mut keys: Vec = Vec::with_capacity(children.len()); @@ -314,7 +314,7 @@ impl<'a> RandomAccessFile<'a> { leaves: &[(BlockId, BlockKey)], conv_key: &ChaCha20Key, arity: u16, - storage: &'a Box, + storage: &'a Box, ) -> Result<(BlockId, BlockKey), StorageError> { let mut parents: Vec<(BlockId, BlockKey)> = vec![]; let mut chunks = leaves.chunks(arity as usize); @@ -347,7 +347,7 @@ impl<'a> RandomAccessFile<'a> { blocks: &[(BlockId, BlockKey)], meta: &mut RandomAccessFileMeta, conv_key: &ChaCha20Key, - storage: &'a Box, + storage: &'a Box, ) -> Result<((BlockId, BlockKey), (BlockId, BlockKey)), FileError> { let leaf_blocks_nbr = blocks.len(); let arity = meta.arity(); @@ -406,7 +406,7 @@ impl<'a> RandomAccessFile<'a> { metadata: Vec, store: &StoreRepo, store_secret: &ReadCapSecret, - storage: &'a Box, + storage: &'a Box, ) -> Result, FileError> { //let max_block_size = store_max_value_size(); let valid_block_size = store_valid_value_size(block_size) - BLOCK_EXTRA; @@ -477,7 +477,7 @@ impl<'a> RandomAccessFile<'a> { metadata: Vec, store: &StoreRepo, store_secret: &ReadCapSecret, - storage: &'a Box, + storage: &'a Box, ) -> Self { let valid_block_size = store_valid_value_size(block_size) - BLOCK_EXTRA; @@ -617,7 +617,7 @@ impl<'a> RandomAccessFile<'a> { pub fn open( id: ObjectId, key: SymKey, - storage: &'a Box, + storage: &'a Box, ) -> Result, FileError> { // load root block let root_block = storage.get(&id)?; @@ -735,16 +735,16 @@ mod test { use std::io::Read; struct Test<'a> { - storage: Box, + storage: Box, } impl<'a> Test<'a> { - fn storage(s: impl RepoStore + 'a) -> Self { + fn storage(s: impl BlockStorage + 'a) -> Self { Test { storage: Box::new(s), } } - fn s(&self) -> &Box { + fn s(&self) -> &Box { &self.storage } } @@ -755,10 +755,10 @@ mod test { let block_size = store_max_value_size(); //store_valid_value_size(0) - let hashmap_storage = HashMapRepoStore::new(); + let hashmap_storage = HashMapBlockStorage::new(); let t = Test::storage(hashmap_storage); - //let storage: Arc<&dyn RepoStore> = Arc::new(&hashmap_storage); + //let storage: Arc<&dyn BlockStorage> = Arc::new(&hashmap_storage); ////// 1 MB of data! let data_size = block_size - BLOCK_EXTRA; @@ -834,7 +834,7 @@ mod test { const MAX_ARITY_LEAVES: usize = 15887; const MAX_DATA_PAYLOAD_SIZE: usize = 1048564; - let hashmap_storage = HashMapRepoStore::new(); + let hashmap_storage = HashMapBlockStorage::new(); let t = Test::storage(hashmap_storage); ////// 16 GB of data! @@ -873,7 +873,7 @@ mod test { const MAX_ARITY_LEAVES: usize = 15887; const MAX_DATA_PAYLOAD_SIZE: usize = 1048564; - let hashmap_storage = HashMapRepoStore::new(); + let hashmap_storage = HashMapBlockStorage::new(); let t = Test::storage(hashmap_storage); ////// 16 GB of data! @@ -911,7 +911,7 @@ mod test { pub fn test_depth_3() { const MAX_ARITY_LEAVES: usize = 61; const MAX_DATA_PAYLOAD_SIZE: usize = 4084; - let hashmap_storage = HashMapRepoStore::new(); + let hashmap_storage = HashMapBlockStorage::new(); let t = Test::storage(hashmap_storage); ////// 900 MB of data! @@ -981,7 +981,7 @@ mod test { * MAX_ARITY_LEAVES * MAX_DATA_PAYLOAD_SIZE; - let hashmap_storage = HashMapRepoStore::new(); + let hashmap_storage = HashMapBlockStorage::new(); let t = Test::storage(hashmap_storage); let (store_repo, store_secret) = StoreRepo::dummy_public_v0(); @@ -1022,7 +1022,7 @@ mod test { .read_to_end(&mut img_buffer) .expect("read of test.jpg"); - let hashmap_storage = HashMapRepoStore::new(); + let hashmap_storage = HashMapBlockStorage::new(); let t = Test::storage(hashmap_storage); let (store_repo, store_secret) = StoreRepo::dummy_public_v0(); @@ -1097,7 +1097,7 @@ mod test { .read_to_end(&mut img_buffer) .expect("read of test.jpg"); - let hashmap_storage = HashMapRepoStore::new(); + let hashmap_storage = HashMapBlockStorage::new(); let t = Test::storage(hashmap_storage); let (store_repo, store_secret) = StoreRepo::dummy_public_v0(); @@ -1174,7 +1174,7 @@ mod test { .read_to_end(&mut img_buffer) .expect("read of test.jpg"); - let hashmap_storage = HashMapRepoStore::new(); + let hashmap_storage = HashMapBlockStorage::new(); let t = Test::storage(hashmap_storage); let (store_repo, store_secret) = StoreRepo::dummy_public_v0(); @@ -1259,7 +1259,7 @@ mod test { let first_block_content = img_buffer[0..4084].to_vec(); - let hashmap_storage = HashMapRepoStore::new(); + let hashmap_storage = HashMapBlockStorage::new(); let t = Test::storage(hashmap_storage); let (store_repo, store_secret) = StoreRepo::dummy_public_v0(); @@ -1342,7 +1342,7 @@ mod test { let chunk_nbr = data_size / 5000000; let last_chunk = data_size % 5000000; - let hashmap_storage = HashMapRepoStore::new(); + let hashmap_storage = HashMapBlockStorage::new(); let t = Test::storage(hashmap_storage); let (store_repo, store_secret) = StoreRepo::dummy_public_v0(); @@ -1396,7 +1396,7 @@ mod test { .read_to_end(&mut img_buffer) .expect("read of test.jpg"); - let hashmap_storage = HashMapRepoStore::new(); + let hashmap_storage = HashMapBlockStorage::new(); let t = Test::storage(hashmap_storage); let (store_repo, store_secret) = StoreRepo::dummy_public_v0(); @@ -1464,7 +1464,7 @@ mod test { log_debug!("{}", obj); - let hashmap_storage = HashMapRepoStore::new(); + let hashmap_storage = HashMapBlockStorage::new(); let t = Test::storage(hashmap_storage); let _ = obj.save_in_test(t.s()).expect("save"); @@ -1490,7 +1490,7 @@ mod test { let max_object_size = store_max_value_size(); let (store_repo, store_secret) = StoreRepo::dummy_public_v0(); - let hashmap_storage = HashMapRepoStore::new(); + let hashmap_storage = HashMapBlockStorage::new(); let t = Test::storage(hashmap_storage); log_debug!("creating empty file"); @@ -1533,7 +1533,7 @@ mod test { let f = std::fs::File::open("[enter path of a big file here]").expect("open of a big file"); let mut reader = BufReader::new(f); - let hashmap_storage = HashMapRepoStore::new(); + let hashmap_storage = HashMapBlockStorage::new(); let t = Test::storage(hashmap_storage); let (store_repo, store_secret) = StoreRepo::dummy_public_v0(); @@ -1587,7 +1587,7 @@ mod test { let f = std::fs::File::open("[enter path of a big file here]").expect("open of a big file"); let mut reader = BufReader::new(f); - let hashmap_storage = HashMapRepoStore::new(); + let hashmap_storage = HashMapBlockStorage::new(); let t = Test::storage(hashmap_storage); let (store_repo, store_secret) = StoreRepo::dummy_public_v0(); diff --git a/ng-repo/src/kcv_store.rs b/ng-repo/src/kcv_storage.rs similarity index 99% rename from ng-repo/src/kcv_store.rs rename to ng-repo/src/kcv_storage.rs index e3a705d..9ce4a69 100644 --- a/ng-repo/src/kcv_store.rs +++ b/ng-repo/src/kcv_storage.rs @@ -8,7 +8,7 @@ //! KeyColumnValue Store abstraction -use crate::store::StorageError; +use crate::errors::StorageError; // TODO:remove mut on self for trait WriteTransaction methods diff --git a/ng-repo/src/lib.rs b/ng-repo/src/lib.rs index 74648e5..2982d87 100644 --- a/ng-repo/src/lib.rs +++ b/ng-repo/src/lib.rs @@ -8,7 +8,7 @@ pub mod types; -pub mod store; +pub mod block_storage; pub mod block; @@ -30,7 +30,7 @@ pub mod utils; pub mod errors; -pub mod kcv_store; +pub mod kcv_storage; pub mod os_info; diff --git a/ng-repo/src/object.rs b/ng-repo/src/object.rs index deac4ff..3fec9de 100644 --- a/ng-repo/src/object.rs +++ b/ng-repo/src/object.rs @@ -18,9 +18,9 @@ use chacha20::cipher::{KeyIvInit, StreamCipher}; use chacha20::ChaCha20; use zeroize::Zeroize; +use crate::block_storage::*; use crate::errors::*; use crate::log::*; -use crate::store::*; use crate::types::*; pub const BLOCK_EXTRA: usize = 12; // 8 is the smallest extra + BLOCK_MAX_DATA_EXTRA @@ -418,17 +418,17 @@ impl Object { } } - /// Load an Object from RepoStore + /// Load an Object from BlockStorage /// /// Returns Ok(Object) or an Err(ObjectParseError::MissingBlocks(Vec)) of missing BlockIds pub fn load( id: ObjectId, key: Option, - store: &Box, + store: &Box, ) -> Result { fn load_tree( parents: Vec, - store: &Box, + store: &Box, blocks: &mut Vec, missing: &mut Vec, block_contents: &mut HashMap, @@ -517,7 +517,10 @@ impl Object { } /// Save blocks of the object and the blocks of the header object in the store - pub fn save(&self, store: &Box) -> Result, StorageError> { + pub fn save( + &self, + store: &Box, + ) -> Result, StorageError> { let mut deduplicated: HashSet = HashSet::new(); //.chain(self.header_blocks.iter()) for block_id in self.blocks.iter() { @@ -543,7 +546,7 @@ impl Object { #[cfg(test)] pub fn save_in_test( &mut self, - store: &Box, + store: &Box, ) -> Result, StorageError> { assert!(self.already_saved == false); self.already_saved = true; @@ -963,7 +966,7 @@ mod test { use std::io::Read; use std::io::Write; - // Those constants are calculated with RepoStore::get_max_value_size + // Those constants are calculated with BlockStorage::get_max_value_size /// Maximum arity of branch containing max number of leaves // const MAX_ARITY_LEAVES: usize = 15887; // /// Maximum arity of root branch @@ -1064,7 +1067,7 @@ mod test { } Err(e) => panic!("Object parse error: {:?}", e), } - let store = Box::new(HashMapRepoStore::new()); + let store = Box::new(HashMapBlockStorage::new()); obj.save_in_test(&store).expect("Object save error"); diff --git a/ng-repo/src/repo.rs b/ng-repo/src/repo.rs index 8ea7c9d..e86053b 100644 --- a/ng-repo/src/repo.rs +++ b/ng-repo/src/repo.rs @@ -9,11 +9,11 @@ //! Repository +use crate::block_storage::*; use crate::errors::*; use crate::event::*; use crate::log::*; use crate::object::Object; -use crate::store::*; use crate::types::*; use crate::utils::generate_keypair; use crate::utils::sign; @@ -84,7 +84,7 @@ pub struct Repo<'a> { pub members: HashMap, - storage: &'a Box, + storage: &'a Box, } impl<'a> fmt::Display for Repo<'a> { @@ -112,7 +112,7 @@ impl<'a> Repo<'a> { peer_last_seq_num: &mut u64, store_repo: &StoreRepo, store_secret: &ReadCapSecret, - storage: &'a Box, + storage: &'a Box, ) -> Result<(Self, Vec), NgError> { let mut events = Vec::with_capacity(6); @@ -223,6 +223,7 @@ impl<'a> Repo<'a> { let main_branch_commit_body = CommitBody::V0(CommitBodyV0::Branch(Branch::V0(BranchV0 { id: main_branch_pub_key, + content_type: BranchContentType::None, repo: repository_commit_ref.clone(), root_branch_readcap_id: root_branch_commit.id().unwrap(), topic: main_branch_topic_pub_key, @@ -467,7 +468,7 @@ impl<'a> Repo<'a> { member: &UserId, perms: &[PermissionV0], overlay: OverlayId, - storage: &'a Box, + storage: &'a Box, ) -> Self { let mut members = HashMap::new(); let permissions = HashMap::from_iter( @@ -510,7 +511,7 @@ impl<'a> Repo<'a> { } } - pub fn get_storage(&self) -> &Box { + pub fn get_storage(&self) -> &Box { self.storage } } @@ -522,16 +523,16 @@ mod test { use crate::repo::*; struct Test<'a> { - storage: Box, + storage: Box, } impl<'a> Test<'a> { - fn storage(s: impl RepoStore + 'a) -> Self { + fn storage(s: impl BlockStorage + 'a) -> Self { Test { storage: Box::new(s), } } - fn s(&self) -> &Box { + fn s(&self) -> &Box { &self.storage } } @@ -546,7 +547,7 @@ mod test { let mut peer_last_seq_num = 10; let (store_repo, store_secret) = StoreRepo::dummy_public_v0(); - let hashmap_storage = HashMapRepoStore::new(); + let hashmap_storage = HashMapBlockStorage::new(); let t = Test::storage(hashmap_storage); let (repo, events) = Repo::new_default( diff --git a/ng-repo/src/types.rs b/ng-repo/src/types.rs index 7c9bec0..bb719c9 100644 --- a/ng-repo/src/types.rs +++ b/ng-repo/src/types.rs @@ -165,6 +165,11 @@ impl PubKey { pub fn nil() -> Self { PubKey::Ed25519PubKey([0u8; 32]) } + + pub fn to_hash_string(&self) -> String { + let hash = blake3::hash(self.slice()); + base64_url::encode(&hash.as_bytes()) + } } impl fmt::Display for PubKey { @@ -597,6 +602,22 @@ impl StoreOverlay { StoreOverlay::Own(_) => unimplemented!(), } } + + pub fn overlay_id_for_write_purpose( + &self, + store_overlay_branch_readcap_secret: ReadCapSecret, + ) -> OverlayId { + match self { + StoreOverlay::V0(StoreOverlayV0::PublicStore(id)) + | StoreOverlay::V0(StoreOverlayV0::ProtectedStore(id)) + | StoreOverlay::V0(StoreOverlayV0::PrivateStore(id)) + | StoreOverlay::V0(StoreOverlayV0::Group(id)) => { + OverlayId::inner(id, store_overlay_branch_readcap_secret) + } + StoreOverlay::V0(StoreOverlayV0::Dialog(d)) => unimplemented!(), + StoreOverlay::Own(_) => unimplemented!(), + } + } } impl From<&StoreRepo> for StoreOverlay { @@ -1115,6 +1136,21 @@ pub enum Quorum { V0(QuorumV0), } +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub enum BranchContentType { + GraphOnly, + YMap, + YXml, + YText, + Automerge, + //Rdfs, + //Owl, + //Shacl, + //Shex, + None, // this is used by Store, Overlay and User BranchTypes + //Chat, +} + /// Branch definition /// /// First commit in a branch, signed by branch key @@ -1128,6 +1164,8 @@ pub struct BranchV0 { /// Branch public key ID pub id: PubKey, + pub content_type: BranchContentType, + /// Reference to the repository commit pub repo: ObjectRef, @@ -1176,12 +1214,12 @@ pub enum Branch { #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] pub enum BranchType { - Main, + Main, // Main is also transactional Chat, Store, Overlay, User, - Transactional, + Transactional, // this could have been called OtherTransaction, but for the sake of simplicity, we use Transaction for any branch that is not the Main one. } impl fmt::Display for BranchType { @@ -1594,6 +1632,7 @@ pub enum WalletUpdate { /// /// DEPS to the previous ones. /// this is used to speedup joining the overlay of such stores, for new devices on new brokers +/// so they don't have to read the whole pub/sub of the StoreRepo in order to get the last ReadCap #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] pub struct StoreUpdateV0 { // id of the store. diff --git a/ng-repo/src/utils.rs b/ng-repo/src/utils.rs index 0075523..ac441e6 100644 --- a/ng-repo/src/utils.rs +++ b/ng-repo/src/utils.rs @@ -22,6 +22,10 @@ use time::OffsetDateTime; use web_time::{Duration, SystemTime, UNIX_EPOCH}; use zeroize::Zeroize; +pub fn derive_key(context: &str, key_material: &[u8]) -> [u8; 32] { + blake3::derive_key(context, key_material) +} + pub fn ed_keypair_from_priv_bytes(secret_key: [u8; 32]) -> (PrivKey, PubKey) { let sk = SecretKey::from_bytes(&secret_key).unwrap(); let pk: PublicKey = (&sk).into(); @@ -200,4 +204,4 @@ pub fn display_timestamp(ts: &Timestamp) -> String { .unwrap() } -pub type Receiver = mpsc::UnboundedReceiver; +pub(crate) type Receiver = mpsc::UnboundedReceiver; diff --git a/ng-sdk-js/js/browser.js b/ng-sdk-js/js/browser.js index 098ccc8..b53572d 100644 --- a/ng-sdk-js/js/browser.js +++ b/ng-sdk-js/js/browser.js @@ -31,6 +31,10 @@ export function session_save(key,value) { } } +export function is_browser() { + return true; +} + function convert_error(e) { if ( e == "The operation is insecure." || @@ -56,6 +60,17 @@ export function session_get(key) { } +export function session_remove(key) { + + try { + return sessionStorage.removeItem(key); + + } catch(e) { + console.error(e); + } + +} + export function local_save(key,value) { try { localStorage.setItem(key, value); diff --git a/ng-sdk-js/js/node.js b/ng-sdk-js/js/node.js index 625e9ea..4771cdb 100644 --- a/ng-sdk-js/js/node.js +++ b/ng-sdk-js/js/node.js @@ -140,6 +140,12 @@ module.exports.client_details = function () { }); }; + +module.exports.is_browser = function() { + return false; +} + + module.exports.session_save = function(key,value) { } @@ -148,6 +154,10 @@ module.exports.session_get = function(key) { } +module.exports.session_remove = function(key) { + +} + module.exports.local_save = function(key,value) { } diff --git a/ng-sdk-js/src/lib.rs b/ng-sdk-js/src/lib.rs index 4a58582..e9afeaf 100644 --- a/ng-sdk-js/src/lib.rs +++ b/ng-sdk-js/src/lib.rs @@ -236,10 +236,28 @@ pub async fn session_start(wallet_name: String, user_js: JsValue) -> Result(user_js) .map_err(|_| "Deserialization error of user_id")?; - let config = SessionConfig::V0(SessionConfigV0 { - user_id, - wallet_name, - }); + let config = SessionConfig::new_in_memory(&user_id, &wallet_name); + let res = nextgraph::local_broker::session_start(config) + .await + .map_err(|e: NgError| e.to_string())?; + + Ok(serde_wasm_bindgen::to_value(&res).unwrap()) +} + +#[cfg(target_arch = "wasm32")] +#[wasm_bindgen] +pub async fn session_start_remote( + wallet_name: String, + user_js: JsValue, + peer_id_js: JsValue, +) -> Result { + let user_id = serde_wasm_bindgen::from_value::(user_js) + .map_err(|_| "Deserialization error of user_id")?; + + let peer_id = serde_wasm_bindgen::from_value::>(peer_id_js) + .map_err(|_| "Deserialization error of peer_id")?; + + let config = SessionConfig::new_remote(&user_id, &wallet_name, peer_id); let res = nextgraph::local_broker::session_start(config) .await .map_err(|e: NgError| e.to_string())?; @@ -274,8 +292,10 @@ pub async fn add_in_memory_wallet(lws_js: JsValue) -> Result<(), String> { extern "C" { fn session_save(key: String, value: String) -> Option; fn session_get(key: String) -> Option; + fn session_remove(key: String); fn local_save(key: String, value: String) -> Option; fn local_get(key: String) -> Option; + fn is_browser() -> bool; } #[cfg(wasmpack_target = "nodejs")] @@ -283,8 +303,10 @@ extern "C" { extern "C" { fn session_save(key: String, value: String) -> Option; fn session_get(key: String) -> Option; + fn session_remove(key: String); fn local_save(key: String, value: String) -> Option; fn local_get(key: String) -> Option; + fn is_browser() -> bool; } #[cfg(target_arch = "wasm32")] @@ -313,14 +335,22 @@ fn session_write(key: String, value: String) -> Result<(), NgError> { } } +#[cfg(target_arch = "wasm32")] +fn session_del(key: String) -> Result<(), NgError> { + session_remove(key); + Ok(()) +} + #[cfg(target_arch = "wasm32")] static INIT_LOCAL_BROKER: Lazy> = Lazy::new(|| { Box::new(|| { LocalBrokerConfig::JsStorage(JsStorageConfig { local_read: Box::new(local_read), local_write: Box::new(local_write), - session_read: Box::new(session_read), - session_write: Box::new(session_write), + session_read: Arc::new(Box::new(session_read)), + session_write: Arc::new(Box::new(session_write)), + session_del: Arc::new(Box::new(session_del)), + is_browser: is_browser(), }) }) }); @@ -354,6 +384,7 @@ pub async fn wallet_get_file(wallet_name: String) -> Result { #[cfg(target_arch = "wasm32")] #[wasm_bindgen] pub async fn wallet_read_file(js_file: JsValue) -> Result { + init_local_broker_with_lazy(&INIT_LOCAL_BROKER).await; let mut file = serde_wasm_bindgen::from_value::(js_file) .map_err(|_| "Deserialization error of file".to_string())?; diff --git a/ng-stores-lmdb/Cargo.toml b/ng-storage-lmdb/Cargo.toml similarity index 96% rename from ng-stores-lmdb/Cargo.toml rename to ng-storage-lmdb/Cargo.toml index 7d8604f..aff09f5 100644 --- a/ng-stores-lmdb/Cargo.toml +++ b/ng-storage-lmdb/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "ng-stores-lmdb" +name = "ng-storage-lmdb" # version = "0.1.0" description = "Stores based on LMDB for NextGraph" version.workspace = true diff --git a/ng-stores-lmdb/src/repo_store.rs b/ng-storage-lmdb/src/block_storage.rs similarity index 97% rename from ng-stores-lmdb/src/repo_store.rs rename to ng-storage-lmdb/src/block_storage.rs index fdf2112..ab94721 100644 --- a/ng-stores-lmdb/src/repo_store.rs +++ b/ng-storage-lmdb/src/block_storage.rs @@ -28,7 +28,7 @@ use serde::{Deserialize, Serialize}; use serde_bare::error::Error; #[derive(Debug)] -pub struct LmdbRepoStore { +pub struct LmdbBlockStorage { /// the main store where all the repo blocks are stored main_store: SingleStore, /// store for the pin boolean, recently_used timestamp, and synced boolean @@ -49,7 +49,7 @@ struct BlockMeta { pub synced: bool, } -impl RepoStore for LmdbRepoStore { +impl BlockStorage for LmdbBlockStorage { /// Retrieves a block from the storage backend. fn get(&self, block_id: &BlockId) -> Result { let lock = self.environment.read().unwrap(); @@ -203,10 +203,10 @@ impl RepoStore for LmdbRepoStore { } } -impl LmdbRepoStore { - /// Opens the store and returns a RepoStore object that should be kept and used to call put/get/delete/pin +impl LmdbBlockStorage { + /// Opens the store and returns a BlockStorage object that should be kept and used to call put/get/delete/pin /// The key is the encryption key for the data at rest. - pub fn open<'a>(path: &Path, key: [u8; 32]) -> Result { + pub fn open<'a>(path: &Path, key: [u8; 32]) -> Result { let mut manager = Manager::::singleton().write().unwrap(); let shared_rkv = manager .get_or_create(path, |path| { @@ -232,7 +232,7 @@ impl LmdbRepoStore { let expiry_store = env.open_multi_integer("expiry", opts).unwrap(); let recently_used_store = env.open_multi_integer("recently_used", opts).unwrap(); - Ok(LmdbRepoStore { + Ok(LmdbBlockStorage { environment: shared_rkv.clone(), main_store, meta_store, @@ -495,7 +495,7 @@ impl LmdbRepoStore { #[cfg(test)] mod test { - use crate::repo_store::LmdbRepoStore; + use crate::repo_store::LmdbBlockStorage; use ng_repo::log::*; use ng_repo::store::*; use ng_repo::types::*; @@ -515,7 +515,7 @@ mod test { let key: [u8; 32] = [0; 32]; fs::create_dir_all(root.path()).unwrap(); log_debug!("{}", root.path().to_str().unwrap()); - let mut store = LmdbRepoStore::open(root.path(), key).unwrap(); + let mut store = LmdbBlockStorage::open(root.path(), key).unwrap(); let mut now = now_timestamp(); now -= 200; // TODO: fix the LMDB bug that is triggered with x max set to 86 !!! @@ -548,7 +548,7 @@ mod test { let key: [u8; 32] = [0; 32]; fs::create_dir_all(root.path()).unwrap(); log_debug!("{}", root.path().to_str().unwrap()); - let mut store = LmdbRepoStore::open(root.path(), key).unwrap(); + let mut store = LmdbBlockStorage::open(root.path(), key).unwrap(); let mut now = now_timestamp(); now -= 200; // TODO: fix the LMDB bug that is triggered with x max set to 86 !!! @@ -605,7 +605,7 @@ mod test { let key: [u8; 32] = [0; 32]; fs::create_dir_all(root.path()).unwrap(); log_debug!("{}", root.path().to_str().unwrap()); - let mut store = LmdbRepoStore::open(root.path(), key).unwrap(); + let mut store = LmdbBlockStorage::open(root.path(), key).unwrap(); let now = now_timestamp(); let list = [ @@ -659,7 +659,7 @@ mod test { let key: [u8; 32] = [0; 32]; fs::create_dir_all(root.path()).unwrap(); log_debug!("{}", root.path().to_str().unwrap()); - let mut store = LmdbRepoStore::open(root.path(), key).unwrap(); + let mut store = LmdbBlockStorage::open(root.path(), key).unwrap(); let now = now_timestamp(); let list = [ @@ -706,7 +706,7 @@ mod test { let key: [u8; 32] = [0; 32]; fs::create_dir_all(root.path()).unwrap(); log_debug!("{}", root.path().to_str().unwrap()); - let store = LmdbRepoStore::open(root.path(), key).unwrap(); + let store = LmdbBlockStorage::open(root.path(), key).unwrap(); store.remove_expired().unwrap(); } @@ -720,7 +720,7 @@ mod test { log_debug!("{}", root.path().to_str().unwrap()); - let mut store = LmdbRepoStore::open(root.path(), key).unwrap(); + let mut store = LmdbBlockStorage::open(root.path(), key).unwrap(); let block = Block::new( Vec::new(), diff --git a/ng-stores-lmdb/src/kcv_store.rs b/ng-storage-lmdb/src/kcv_storage.rs similarity index 100% rename from ng-stores-lmdb/src/kcv_store.rs rename to ng-storage-lmdb/src/kcv_storage.rs diff --git a/ng-stores-lmdb/src/lib.rs b/ng-storage-lmdb/src/lib.rs similarity index 100% rename from ng-stores-lmdb/src/lib.rs rename to ng-storage-lmdb/src/lib.rs diff --git a/ng-stores-rocksdb/Cargo.toml b/ng-storage-rocksdb/Cargo.toml similarity index 95% rename from ng-stores-rocksdb/Cargo.toml rename to ng-storage-rocksdb/Cargo.toml index 0c87739..9a35d54 100644 --- a/ng-stores-rocksdb/Cargo.toml +++ b/ng-storage-rocksdb/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "ng-stores-rocksdb" +name = "ng-storage-rocksdb" # version = "0.1.0" description = "Stores based on RocksDB for NextGraph" version.workspace = true diff --git a/ng-stores-rocksdb/README.md b/ng-storage-rocksdb/README.md similarity index 97% rename from ng-stores-rocksdb/README.md rename to ng-storage-rocksdb/README.md index f39fd3c..49b3696 100644 --- a/ng-stores-rocksdb/README.md +++ b/ng-storage-rocksdb/README.md @@ -1,4 +1,4 @@ -# ng-stores-rocksdb +# ng-storage-rocksdb ![MSRV][rustc-image] [![Apache 2.0 Licensed][license-image]][license-link] @@ -48,7 +48,7 @@ additional terms or conditions. NextGraph received funding through the [NGI Assure Fund](https://nlnet.nl/project/NextGraph/index.html), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's [Next Generation Internet](https://ngi.eu/) programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 957073. -[rustc-image]: https://img.shields.io/badge/rustc-1.64+-blue.svg +[rustc-image]: https://img.shields.io/badge/rustc-1.74+-blue.svg [license-image]: https://img.shields.io/badge/license-Apache2.0-blue.svg [license-link]: https://git.nextgraph.org/NextGraph/nextgraph-rs/raw/branch/master/LICENSE-APACHE2 [license-image2]: https://img.shields.io/badge/license-MIT-blue.svg diff --git a/ng-stores-rocksdb/src/repo_store.rs b/ng-storage-rocksdb/src/block_storage.rs similarity index 97% rename from ng-stores-rocksdb/src/repo_store.rs rename to ng-storage-rocksdb/src/block_storage.rs index 9f94cad..4a1599b 100644 --- a/ng-stores-rocksdb/src/repo_store.rs +++ b/ng-storage-rocksdb/src/block_storage.rs @@ -7,7 +7,7 @@ // notice may not be copied, modified, or distributed except // according to those terms. -use ng_repo::store::*; +use ng_repo::errors::StorageError; use ng_repo::types::*; use ng_repo::utils::*; @@ -19,7 +19,7 @@ use serde::{Deserialize, Serialize}; use serde_bare::error::Error; /* #[derive(Debug)] -pub struct LmdbRepoStore { +pub struct LmdbBlockStorage { /// the main store where all the repo blocks are stored main_store: SingleStore, /// store for the pin boolean, recently_used timestamp, and synced boolean @@ -40,7 +40,7 @@ struct BlockMeta { pub synced: bool, } -impl RepoStore for LmdbRepoStore { +impl BlockStorage for LmdbBlockStorage { /// Retrieves a block from the storage backend. fn get(&self, block_id: &BlockId) -> Result { let lock = self.environment.read().unwrap(); @@ -194,10 +194,10 @@ impl RepoStore for LmdbRepoStore { } } -impl LmdbRepoStore { - /// Opens the store and returns a RepoStore object that should be kept and used to call put/get/delete/pin +impl LmdbBlockStorage { + /// Opens the store and returns a BlockStorage object that should be kept and used to call put/get/delete/pin /// The key is the encryption key for the data at rest. - pub fn open<'a>(path: &Path, key: [u8; 32]) -> Result { + pub fn open<'a>(path: &Path, key: [u8; 32]) -> Result { let mut manager = Manager::::singleton().write().unwrap(); let shared_rkv = manager .get_or_create(path, |path| { @@ -223,7 +223,7 @@ impl LmdbRepoStore { let expiry_store = env.open_multi_integer("expiry", opts).unwrap(); let recently_used_store = env.open_multi_integer("recently_used", opts).unwrap(); - Ok(LmdbRepoStore { + Ok(LmdbBlockStorage { environment: shared_rkv.clone(), main_store, meta_store, @@ -487,8 +487,8 @@ impl LmdbRepoStore { #[cfg(test)] mod test { + use ng_repo::block_storage::*; use ng_repo::log::*; - use ng_repo::store::*; use ng_repo::types::*; use ng_repo::utils::*; #[allow(unused_imports)] @@ -504,7 +504,7 @@ mod test { let key: [u8; 32] = [0; 32]; fs::create_dir_all(root.path()).unwrap(); log_debug!("{}", root.path().to_str().unwrap()); - let mut store = LmdbRepoStore::open(root.path(), key).unwrap(); + let mut store = LmdbBlockStorage::open(root.path(), key).unwrap(); let mut now = now_timestamp(); now -= 200; // TODO: fix the LMDB bug that is triggered with x max set to 86 !!! @@ -537,7 +537,7 @@ mod test { let key: [u8; 32] = [0; 32]; fs::create_dir_all(root.path()).unwrap(); log_debug!("{}", root.path().to_str().unwrap()); - let mut store = LmdbRepoStore::open(root.path(), key).unwrap(); + let mut store = LmdbBlockStorage::open(root.path(), key).unwrap(); let mut now = now_timestamp(); now -= 200; // TODO: fix the LMDB bug that is triggered with x max set to 86 !!! @@ -595,7 +595,7 @@ mod test { let key: [u8; 32] = [0; 32]; fs::create_dir_all(root.path()).unwrap(); log_debug!("{}", root.path().to_str().unwrap()); - let mut store = LmdbRepoStore::open(root.path(), key).unwrap(); + let mut store = LmdbBlockStorage::open(root.path(), key).unwrap(); let now = now_timestamp(); let list = [ @@ -649,7 +649,7 @@ mod test { let key: [u8; 32] = [0; 32]; fs::create_dir_all(root.path()).unwrap(); log_debug!("{}", root.path().to_str().unwrap()); - let mut store = LmdbRepoStore::open(root.path(), key).unwrap(); + let mut store = LmdbBlockStorage::open(root.path(), key).unwrap(); let now = now_timestamp(); let list = [ @@ -696,7 +696,7 @@ mod test { let key: [u8; 32] = [0; 32]; fs::create_dir_all(root.path()).unwrap(); log_debug!("{}", root.path().to_str().unwrap()); - let store = LmdbRepoStore::open(root.path(), key).unwrap(); + let store = LmdbBlockStorage::open(root.path(), key).unwrap(); store.remove_expired().unwrap(); } @@ -710,7 +710,7 @@ mod test { log_debug!("{}", root.path().to_str().unwrap()); - let mut store = LmdbRepoStore::open(root.path(), key).unwrap(); + let mut store = LmdbBlockStorage::open(root.path(), key).unwrap(); let block = Block::new( Vec::new(), diff --git a/ng-stores-rocksdb/src/kcv_store.rs b/ng-storage-rocksdb/src/kcv_storage.rs similarity index 99% rename from ng-stores-rocksdb/src/kcv_store.rs rename to ng-storage-rocksdb/src/kcv_storage.rs index 4aaa170..f569f6f 100644 --- a/ng-stores-rocksdb/src/kcv_store.rs +++ b/ng-storage-rocksdb/src/kcv_storage.rs @@ -7,9 +7,9 @@ // notice may not be copied, modified, or distributed except // according to those terms. -use ng_repo::kcv_store::*; -use ng_repo::store::*; +use ng_repo::kcv_storage::*; +use ng_repo::errors::*; use ng_repo::log::*; use std::path::Path; diff --git a/ng-stores-rocksdb/src/lib.rs b/ng-storage-rocksdb/src/lib.rs similarity index 62% rename from ng-stores-rocksdb/src/lib.rs rename to ng-storage-rocksdb/src/lib.rs index 7a62c33..cf018ea 100644 --- a/ng-stores-rocksdb/src/lib.rs +++ b/ng-storage-rocksdb/src/lib.rs @@ -1,5 +1,5 @@ #[cfg(not(target_arch = "wasm32"))] -pub mod repo_store; +pub mod block_storage; #[cfg(not(target_arch = "wasm32"))] -pub mod kcv_store; +pub mod kcv_storage; diff --git a/ng-verifier/Cargo.toml b/ng-verifier/Cargo.toml index 150c45b..8421b83 100644 --- a/ng-verifier/Cargo.toml +++ b/ng-verifier/Cargo.toml @@ -23,3 +23,9 @@ chacha20 = "0.9.0" serde = { version = "1.0", features = ["derive"] } serde_bare = "0.5.0" serde_bytes = "0.11.7" +oxigraph = { git = "https://git.nextgraph.org/NextGraph/oxigraph.git", branch="main" } +automerge = "0.5.9" +yrs = "0.18.2" + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +ng-storage-rocksdb = { path = "../ng-storage-rocksdb", version = "0.1.0" } \ No newline at end of file diff --git a/ng-verifier/README.md b/ng-verifier/README.md index 712c6f3..2133d01 100644 --- a/ng-verifier/README.md +++ b/ng-verifier/README.md @@ -48,7 +48,7 @@ additional terms or conditions. NextGraph received funding through the [NGI Assure Fund](https://nlnet.nl/project/NextGraph/index.html), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's [Next Generation Internet](https://ngi.eu/) programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 957073. -[rustc-image]: https://img.shields.io/badge/rustc-1.64+-blue.svg +[rustc-image]: https://img.shields.io/badge/rustc-1.74+-blue.svg [license-image]: https://img.shields.io/badge/license-Apache2.0-blue.svg [license-link]: https://git.nextgraph.org/NextGraph/nextgraph-rs/raw/branch/master/LICENSE-APACHE2 [license-image2]: https://img.shields.io/badge/license-MIT-blue.svg diff --git a/ng-verifier/src/lib.rs b/ng-verifier/src/lib.rs index e69de29..f8ff6a9 100644 --- a/ng-verifier/src/lib.rs +++ b/ng-verifier/src/lib.rs @@ -0,0 +1,6 @@ +pub mod types; + +pub mod user_storage; + +#[cfg(not(target_family = "wasm"))] +pub mod rocksdb_user_storage; diff --git a/ng-verifier/src/rocksdb_user_storage.rs b/ng-verifier/src/rocksdb_user_storage.rs new file mode 100644 index 0000000..d171005 --- /dev/null +++ b/ng-verifier/src/rocksdb_user_storage.rs @@ -0,0 +1,39 @@ +// Copyright (c) 2022-2024 Niko Bonnieure, Par le Peuple, NextGraph.org developers +// All rights reserved. +// Licensed under the Apache License, Version 2.0 +// +// or the MIT license , +// at your option. All files in the project carrying such +// notice may not be copied, modified, or distributed except +// according to those terms. + +//! RocksDb Backend for UserStorage trait + +use crate::types::*; +use crate::user_storage::*; +use ng_repo::{errors::StorageError, types::*}; +use ng_storage_rocksdb::kcv_storage::RocksdbKCVStore; +use std::path::PathBuf; +use std::{ + cmp::{max, min}, + collections::HashMap, + mem::size_of_val, +}; + +pub(crate) struct RocksDbUserStorage { + user_storage: RocksdbKCVStore, +} + +impl RocksDbUserStorage { + pub fn open(path: &PathBuf, master_key: [u8; 32]) -> Result { + Ok(RocksDbUserStorage { + user_storage: RocksdbKCVStore::open(path, master_key)?, + }) + } +} + +impl UserStorage for RocksDbUserStorage { + fn repo_id_to_store_overlay(&self, id: &RepoId) -> Result { + unimplemented!(); + } +} diff --git a/ng-verifier/src/types.rs b/ng-verifier/src/types.rs index e69de29..43dadb4 100644 --- a/ng-verifier/src/types.rs +++ b/ng-verifier/src/types.rs @@ -0,0 +1,268 @@ +// Copyright (c) 2022-2024 Niko Bonnieure, Par le Peuple, NextGraph.org developers +// All rights reserved. +// Licensed under the Apache License, Version 2.0 +// +// or the MIT license , +// at your option. All files in the project carrying such +// notice may not be copied, modified, or distributed except +// according to those terms. + +//! Types for Verifier + +use core::fmt; +//use oxigraph::io::{RdfFormat, RdfParser, RdfSerializer}; +use oxigraph::store::Store; +//use oxigraph::model::GroundQuad; +#[cfg(not(target_family = "wasm"))] +use crate::rocksdb_user_storage::RocksDbUserStorage; +use crate::user_storage::{InMemoryUserStorage, UserStorage}; +use std::path::PathBuf; + +use ng_net::{ + types::*, + utils::{Receiver, Sender}, +}; +use ng_repo::{ + errors::{NgError, StorageError}, + types::*, +}; +use serde::{Deserialize, Serialize}; +//use yrs::{StateVector, Update}; + +#[derive(Debug, Clone)] +pub enum VerifierType { + /// nothing will be saved on disk during the session + Memory, + /// will save all user data locally, with RocksDb backend + RocksDb, + /// the verifier will be remote. a Noise connection will be opened + /// optional peerId to connect to. If None, will try any that has the flag `can_verify` + Remote(Option), + /// IndexedDb based rocksdb compiled to WASM... not ready yet. obviously. only works in the browser + WebRocksDb, +} + +impl VerifierType { + pub fn is_memory(&self) -> bool { + match self { + Self::Memory => true, + _ => false, + } + } +} + +//type LastSeqFn = fn(PubKey, u16) -> Result; +pub type LastSeqFn = dyn Fn(PubKey, u16) -> Result + 'static + Sync + Send; + +// peer_id: PubKey, seq_num:u64, event_ser: vec, +pub type OutboxWriteFn = + dyn Fn(PubKey, u64, Vec) -> Result<(), NgError> + 'static + Sync + Send; + +// peer_id: PubKey, +pub type OutboxReadFn = dyn Fn(PubKey) -> Result>, NgError> + 'static + Sync + Send; + +pub struct JsSaveSessionConfig { + pub last_seq_function: Box, + pub outbox_write_function: Box, + pub outbox_read_function: Box, +} + +impl fmt::Debug for JsSaveSessionConfig { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "JsSaveSessionConfig") + } +} + +#[derive(Debug)] +pub enum VerifierConfigType { + /// nothing will be saved on disk during the session + Memory, + JsSaveSession(JsSaveSessionConfig), + /// will save all user data locally, with RocksDb backend + RocksDb(PathBuf), + /// the verifier will be remote. a Noise connection will be opened + /// optional peerId to connect to. If None, will try any that has the flag `can_verify` + /// // TODO: Pass the AppConfig + Remote(Option), + /// IndexedDb based rocksdb compiled to WASM... not ready yet. obviously. only works in the browser + WebRocksDb, +} + +#[derive(Debug)] +pub struct VerifierConfig { + pub config_type: VerifierConfigType, + /// not used for Memory + pub user_master_key: [u8; 32], + /// not used for Memory + pub peer_priv_key: PrivKey, + pub user_priv_key: PrivKey, + pub private_store_read_cap: ObjectRef, +} + +pub type CancelFn = Box; + +pub struct Verifier { + pub config: VerifierConfig, + pub connected_server_id: Option, + graph_dataset: Option, + user_storage: Option>, +} + +impl fmt::Debug for Verifier { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!(f, "Verifier\nconfig: {:?}", self.config)?; + writeln!(f, "connected_server_id: {:?}", self.connected_server_id) + } +} + +impl Verifier { + pub fn new(config: VerifierConfig) -> Result { + let (graph, user) = match &config.config_type { + VerifierConfigType::Memory | VerifierConfigType::JsSaveSession(_) => ( + Some(Store::new().unwrap()), + Some(Box::new(InMemoryUserStorage::new()) as Box), + ), + #[cfg(not(target_family = "wasm"))] + VerifierConfigType::RocksDb(path) => ( + // FIXME BIG TIME: we are reusing the same encryption key here. + // this is very temporary, until we remove the code in oxi_rocksdb of oxigraph, + // and have oxigraph use directly the UserStorage + Some(Store::open_with_key(path, config.user_master_key).unwrap()), + Some( + Box::new(RocksDbUserStorage::open(path, config.user_master_key)?) + as Box, + ), + ), + VerifierConfigType::Remote(_) => (None, None), + _ => unimplemented!(), // can be WebRocksDb or RocksDb on wasm platforms + }; + Ok(Verifier { + config, + connected_server_id: None, + graph_dataset: graph, + user_storage: user, + }) + } + + pub fn doc_fetch( + &self, + nuri: String, + payload: Option, + ) -> Result<(Receiver, CancelFn), NgError> { + unimplemented!(); + } +} + +// +// APP PROTOCOL (between APP and VERIFIER) +// + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct AppRequestV0 {} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum AppRequest { + V0(AppRequestV0), +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct GraphUpdate { + sparql_update: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum DiscreteUpdate { + /// A yrs::Update + YMap(Vec), + YXml(Vec), + YText(Vec), + /// An automerge::Patch + Automerge(Vec), +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct AppUpdate { + heads: Vec, + graph: Option, + discrete: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct AppCreate { + store: StoreRepo, + content_type: BranchContentType, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct AppDelete { + /// Nuri of doc to delete + nuri: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum AppRequestPayloadV0 { + Create(AppCreate), + Update(AppUpdate), + Delete(AppDelete), +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum AppRequestPayload { + V0(AppRequestPayloadV0), +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum DiscretePatch { + /// A yrs::Update + YMap(Vec), + YXml(Vec), + YText(Vec), + /// An automerge::Patch + Automerge(Vec), +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct GraphPatch { + /// oxigraph::model::GroundQuad serialized in turtle with oxrdfio + pub adds: Vec, + pub removes: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum DiscreteState { + /// A yrs::StateVector + YMap(Vec), + YXml(Vec), + YText(Vec), + // the output of Automerge::save() + Automerge(Vec), +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct GraphState { + pub tuples: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct AppState { + heads: Vec, + graph: Option, // there is always a graph present in the branch. but it might not have been asked in the request + discrete: Option, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct AppPatch { + heads: Vec, + graph: Option, + discrete: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum AppResponseV0 { + State(AppState), + Patch(AppPatch), +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum AppResponse { + V0(AppResponseV0), +} diff --git a/ng-verifier/src/user_storage.rs b/ng-verifier/src/user_storage.rs new file mode 100644 index 0000000..a1a2a08 --- /dev/null +++ b/ng-verifier/src/user_storage.rs @@ -0,0 +1,46 @@ +// Copyright (c) 2022-2024 Niko Bonnieure, Par le Peuple, NextGraph.org developers +// All rights reserved. +// Licensed under the Apache License, Version 2.0 +// +// or the MIT license , +// at your option. All files in the project carrying such +// notice may not be copied, modified, or distributed except +// according to those terms. + +//! Storage of user application data (RDF, content of rich-text document, etc) + +use ng_repo::{errors::StorageError, types::*}; + +use crate::types::*; +use std::{ + cmp::{max, min}, + collections::HashMap, + mem::size_of_val, +}; + +pub trait UserStorage: Send + Sync { + /// Gets the StoreRepo for a given RepoId + fn repo_id_to_store_overlay(&self, id: &RepoId) -> Result; +} + +pub(crate) struct InMemoryUserStorage { + repo_id_to_store_overlay: HashMap, +} + +impl InMemoryUserStorage { + pub fn new() -> Self { + InMemoryUserStorage { + repo_id_to_store_overlay: HashMap::new(), + } + } +} + +impl UserStorage for InMemoryUserStorage { + fn repo_id_to_store_overlay(&self, id: &RepoId) -> Result { + Ok(self + .repo_id_to_store_overlay + .get(&id) + .ok_or(StorageError::NotFound)? + .to_owned()) + } +} diff --git a/ng-wallet/README.md b/ng-wallet/README.md index 69485e9..e34735b 100644 --- a/ng-wallet/README.md +++ b/ng-wallet/README.md @@ -48,7 +48,7 @@ additional terms or conditions. NextGraph received funding through the [NGI Assure Fund](https://nlnet.nl/project/NextGraph/index.html), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's [Next Generation Internet](https://ngi.eu/) programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 957073. -[rustc-image]: https://img.shields.io/badge/rustc-1.64+-blue.svg +[rustc-image]: https://img.shields.io/badge/rustc-1.74+-blue.svg [license-image]: https://img.shields.io/badge/license-Apache2.0-blue.svg [license-link]: https://git.nextgraph.org/NextGraph/nextgraph-rs/raw/branch/master/LICENSE-APACHE2 [license-image2]: https://img.shields.io/badge/license-MIT-blue.svg diff --git a/ng-wallet/src/types.rs b/ng-wallet/src/types.rs index 37f2399..5c65bc4 100644 --- a/ng-wallet/src/types.rs +++ b/ng-wallet/src/types.rs @@ -140,6 +140,12 @@ impl SessionWalletStorageV0 { // } } +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SessionInfo { + pub session_id: u8, + pub user: UserId, +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub struct SessionPeerStorageV0 { pub user: UserId, @@ -173,7 +179,7 @@ pub struct LocalClientStorageV0 { impl LocalClientStorageV0 { fn crypt(text: &mut Vec, client: ClientId, wallet_privkey: PrivKey) { let client_ser = serde_bare::to_vec(&client).unwrap(); - let mut wallet_privkey_ser = serde_bare::to_vec(&wallet_privkey).unwrap(); + let wallet_privkey_ser = serde_bare::to_vec(&wallet_privkey).unwrap(); let mut key_material = [client_ser, wallet_privkey_ser].concat(); let mut key: [u8; 32] = blake3::derive_key( @@ -462,6 +468,17 @@ impl SensitiveWallet { Self::V0(v0) => v0.client = Some(client), } } + pub fn individual_site(&self, user_id: &UserId) -> Option<&(PrivKey, ReadCap)> { + match self { + Self::V0(v0) => match v0.sites.get(&user_id.to_string()) { + Some(site) => match &site.site_type { + SiteType::Individual(creds) => Some(creds), + _ => None, + }, + None => None, + }, + } + } pub fn has_user(&self, user_id: &UserId) -> bool { match self { Self::V0(v0) => v0.sites.get(&user_id.to_string()).is_some(), diff --git a/ngaccount/Cargo.toml b/ngaccount/Cargo.toml index 14b7e2a..7c0648d 100644 --- a/ngaccount/Cargo.toml +++ b/ngaccount/Cargo.toml @@ -32,4 +32,4 @@ base64-url = "2.0.0" serde_json = "1.0.96" bytes = "1.0" anyhow = "1.0.71" -duration-str = "0.5.1" \ No newline at end of file +duration-str = "0.7.1" \ No newline at end of file diff --git a/ngcli/Cargo.toml b/ngcli/Cargo.toml index 45e7980..962afcc 100644 --- a/ngcli/Cargo.toml +++ b/ngcli/Cargo.toml @@ -37,4 +37,4 @@ blake3 = "1.3.1" serde = { version = "1.0", features = ["derive"] } serde_bare = "0.5.0" serde_bytes = "0.11.7" -duration-str = "0.5.1" \ No newline at end of file +duration-str = "0.7.1" \ No newline at end of file diff --git a/ngcli/README.md b/ngcli/README.md index 5d41a28..f06c43e 100644 --- a/ngcli/README.md +++ b/ngcli/README.md @@ -50,7 +50,7 @@ additional terms or conditions. NextGraph received funding through the [NGI Assure Fund](https://nlnet.nl/project/NextGraph/index.html), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's [Next Generation Internet](https://ngi.eu/) programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 957073. -[rustc-image]: https://img.shields.io/badge/rustc-1.64+-blue.svg +[rustc-image]: https://img.shields.io/badge/rustc-1.74+-blue.svg [license-image]: https://img.shields.io/badge/license-Apache2.0-blue.svg [license-link]: https://git.nextgraph.org/NextGraph/nextgraph-rs/raw/branch/master/LICENSE-APACHE2 [license-image2]: https://img.shields.io/badge/license-MIT-blue.svg diff --git a/ngcli/src/main.rs b/ngcli/src/main.rs index d27bbce..a836b31 100644 --- a/ngcli/src/main.rs +++ b/ngcli/src/main.rs @@ -12,7 +12,9 @@ use ed25519_dalek::*; use duration_str::parse; use futures::{future, pin_mut, stream, SinkExt, StreamExt}; use ng_net::actors::*; -use ng_repo::store::{store_max_value_size, store_valid_value_size, HashMapRepoStore, RepoStore}; +use ng_repo::block_storage::{ + store_max_value_size, store_valid_value_size, BlockStorage, HashMapBlockStorage, +}; use rand::rngs::OsRng; use serde::{Deserialize, Serialize}; use serde_json::{from_str, to_string_pretty}; diff --git a/ngcli/src/old.rs b/ngcli/src/old.rs index 1c7f541..df79027 100644 --- a/ngcli/src/old.rs +++ b/ngcli/src/old.rs @@ -10,7 +10,7 @@ async fn test_sync(cnx: &mut impl BrokerConnection, user_pub_key: PubKey, userpr expiry: Option, repo_pubkey: PubKey, repo_secret: SymKey, - store: &mut impl RepoStore, + store: &mut impl BlockStorage, ) -> ObjectRef { let max_object_size = 4000; let obj = Object::new( @@ -38,7 +38,7 @@ async fn test_sync(cnx: &mut impl BrokerConnection, user_pub_key: PubKey, userpr body_ref: ObjectRef, repo_pubkey: PubKey, repo_secret: SymKey, - store: &mut impl RepoStore, + store: &mut impl BlockStorage, ) -> ObjectRef { let mut obj_deps: Vec = vec![]; obj_deps.extend(deps.iter().map(|r| r.id)); @@ -80,7 +80,7 @@ async fn test_sync(cnx: &mut impl BrokerConnection, user_pub_key: PubKey, userpr branch: Branch, repo_pubkey: PubKey, repo_secret: SymKey, - store: &mut impl RepoStore, + store: &mut impl BlockStorage, ) -> ObjectRef { let deps = vec![]; let expiry = None; @@ -100,7 +100,7 @@ async fn test_sync(cnx: &mut impl BrokerConnection, user_pub_key: PubKey, userpr deps: Vec, repo_pubkey: PubKey, repo_secret: SymKey, - store: &mut impl RepoStore, + store: &mut impl BlockStorage, ) -> ObjectRef { let expiry = None; let content = [7u8; 777].to_vec(); @@ -120,7 +120,7 @@ async fn test_sync(cnx: &mut impl BrokerConnection, user_pub_key: PubKey, userpr deps: Vec, repo_pubkey: PubKey, repo_secret: SymKey, - store: &mut impl RepoStore, + store: &mut impl BlockStorage, ) -> ObjectRef { let expiry = None; let body = CommitBody::Ack(Ack::V0()); @@ -135,7 +135,7 @@ async fn test_sync(cnx: &mut impl BrokerConnection, user_pub_key: PubKey, userpr ) } - let mut store = HashMapRepoStore::new(); + let mut store = HashMapBlockStorage::new(); let mut rng = OsRng {}; // repo @@ -346,7 +346,7 @@ async fn test_sync(cnx: &mut impl BrokerConnection, user_pub_key: PubKey, userpr // Now emptying the local store of the client, and adding only 1 commit into it (br) // we also have received an commit (t5) but we don't know what to do with it... - let mut store = HashMapRepoStore::new(); + let mut store = HashMapBlockStorage::new(); let br = add_commit( branch_body, diff --git a/ngd/README.md b/ngd/README.md index de3e264..34197df 100644 --- a/ngd/README.md +++ b/ngd/README.md @@ -108,7 +108,7 @@ additional terms or conditions. NextGraph received funding through the [NGI Assure Fund](https://nlnet.nl/project/NextGraph/index.html), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's [Next Generation Internet](https://ngi.eu/) programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 957073. -[rustc-image]: https://img.shields.io/badge/rustc-1.64+-blue.svg +[rustc-image]: https://img.shields.io/badge/rustc-1.74+-blue.svg [license-image]: https://img.shields.io/badge/license-Apache2.0-blue.svg [license-link]: https://git.nextgraph.org/NextGraph/nextgraph-rs/raw/branch/master/LICENSE-APACHE2 [license-image2]: https://img.shields.io/badge/license-MIT-blue.svg diff --git a/ngone/Cargo.toml b/ngone/Cargo.toml index 9794dd3..e92dccd 100644 --- a/ngone/Cargo.toml +++ b/ngone/Cargo.toml @@ -20,7 +20,7 @@ warp-embed = "0.4" rust-embed = "6" log = "0.4" env_logger = "0.10" -ng-stores-rocksdb = { path = "../ng-stores-rocksdb" } +ng-storage-rocksdb = { path = "../ng-storage-rocksdb" } ng-repo = { path = "../ng-repo", features = ["server_log_output"] } ng-net = { path = "../ng-net" } ng-wallet = { path = "../ng-wallet" } diff --git a/ngone/src/main.rs b/ngone/src/main.rs index 7a617ad..64d5d73 100644 --- a/ngone/src/main.rs +++ b/ngone/src/main.rs @@ -12,7 +12,7 @@ extern crate slice_as_array; mod store; mod types; -use ng_repo::store::StorageError; +use ng_repo::errors::StorageError; use warp::reply::Response; use warp::{Filter, Reply}; @@ -28,7 +28,7 @@ use ng_net::types::{APP_NG_ONE_URL, NG_ONE_URL}; use ng_repo::log::*; use ng_repo::types::*; use ng_repo::utils::{generate_keypair, sign, verify}; -use ng_stores_rocksdb::kcv_store::RocksdbKCVStore; +use ng_storage_rocksdb::kcv_storage::RocksdbKCVStore; use ng_wallet::types::*; #[derive(RustEmbed)] diff --git a/ngone/src/store/dynpeer.rs b/ngone/src/store/dynpeer.rs index 074489e..0490d85 100644 --- a/ngone/src/store/dynpeer.rs +++ b/ngone/src/store/dynpeer.rs @@ -10,8 +10,8 @@ //! ng-one bootstrap use ng_net::types::*; -use ng_repo::kcv_store::KCVStore; -use ng_repo::store::*; +use ng_repo::errors::StorageError; +use ng_repo::kcv_storage::KCVStore; use ng_repo::types::PubKey; use serde::{Deserialize, Serialize}; diff --git a/ngone/src/store/wallet_record.rs b/ngone/src/store/wallet_record.rs index 01aa3f3..fc5aee3 100644 --- a/ngone/src/store/wallet_record.rs +++ b/ngone/src/store/wallet_record.rs @@ -9,8 +9,8 @@ //! ng-wallet -use ng_repo::kcv_store::KCVStore; -use ng_repo::store::*; +use ng_repo::errors::StorageError; +use ng_repo::kcv_storage::KCVStore; use ng_repo::types::*; use ng_wallet::types::*; use serde::{Deserialize, Serialize};