From 9e05125f1782fd4606b2d0e6c940b8a5c36d5030 Mon Sep 17 00:00:00 2001 From: Niko PLP Date: Sun, 19 May 2024 09:16:54 +0300 Subject: [PATCH] fix sync_req and some docs --- README.md | 16 +++++------ nextgraph/README.md | 2 +- nextgraph/src/lib.rs | 2 +- nextgraph/src/local_broker.rs | 3 ++- ng-net/src/broker.rs | 4 +-- .../src/storage/backend/oxi_rocksdb.rs | 4 ++- ng-repo/src/branch.rs | 27 +++++++++++++++---- ng-sdk-js/README.md | 26 ++++-------------- ng-sdk-js/app-node/index.js | 2 +- ng-sdk-js/app-node/package-lock.json | 4 +-- ng-sdk-js/app-node/package.json | 2 +- ng-sdk-js/prepare-node.js | 4 +-- ng-sdk-js/src/lib.rs | 10 +++---- ng-verifier/src/verifier.rs | 16 +++++------ 14 files changed, 61 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index 46abe75..cc37a03 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ [![project chat](https://img.shields.io/badge/zulip-join_chat-brightgreen.svg)](https://forum.nextgraph.org) [![Crates.io Version](https://img.shields.io/crates/v/nextgraph)](https://crates.io/crates/nextgraph) [![docs.rs](https://img.shields.io/docsrs/nextgraph)](https://docs.rs/nextgraph) -[![NPM Version](https://img.shields.io/npm/v/ng-sdk-node)](https://www.npmjs.com/package/ng-sdk-node) +[![NPM Version](https://img.shields.io/npm/v/nextgraph)](https://www.npmjs.com/package/nextgraph) Rust implementation of NextGraph @@ -103,19 +103,19 @@ Please test by following this order (as we need to generate some files locally) ``` cargo test --package nextgraph -r --lib -- local_broker::test::gen_wallet_for_test --show-output --nocapture -cargo test +cargo test -r cargo test --package nextgraph -r --lib -- local_broker::test::import_session_for_test_to_disk --show-output --nocapture --ignored ``` Test a single crate: ``` -cargo test --package ng-repo --lib -- branch::test --show-output --nocapture -cargo test --package ng-wallet --lib -- branch::test --show-output --nocapture -cargo test --package ng-verifier --lib -- branch::test --show-output --nocapture -cargo test --package ng-sdk-js --lib -- branch::test --show-output --nocapture -cargo test --package ng-broker --lib -- branch::test --show-output --nocapture -cargo test --package ng-client-ws --lib -- branch::test --show-output --nocapture +cargo test --package ng-repo --lib -- --show-output --nocapture +cargo test --package ng-wallet --lib -- --show-output --nocapture +cargo test --package ng-verifier --lib -- --show-output --nocapture +cargo test --package ng-sdk-js --lib -- --show-output --nocapture +cargo test --package ng-broker --lib -- --show-output --nocapture +cargo test --package ng-client-ws --lib -- --show-output --nocapture ``` Test WASM websocket diff --git a/nextgraph/README.md b/nextgraph/README.md index 2ddc162..383b2c6 100644 --- a/nextgraph/README.md +++ b/nextgraph/README.md @@ -10,7 +10,7 @@ [![project chat](https://img.shields.io/badge/zulip-join_chat-brightgreen.svg)](https://forum.nextgraph.org) [![Crates.io Version](https://img.shields.io/crates/v/nextgraph)](https://crates.io/crates/nextgraph) [![docs.rs](https://img.shields.io/docsrs/nextgraph)](https://docs.rs/nextgraph) -[![NPM Version](https://img.shields.io/npm/v/ng-sdk-node)](https://www.npmjs.com/package/ng-sdk-node) +[![NPM Version](https://img.shields.io/npm/v/nextgraph)](https://www.npmjs.com/package/nextgraph) Rust client library for NextGraph framework diff --git a/nextgraph/src/lib.rs b/nextgraph/src/lib.rs index 01e54cb..bdf844b 100644 --- a/nextgraph/src/lib.rs +++ b/nextgraph/src/lib.rs @@ -23,7 +23,7 @@ //! //! The Rust API is used internally in the CLI, and for all the Tauri-based Apps. //! -//! The same API is also made available in Javascript for the browser (and is used by our webapp) and for nodejs. See the npm package [ng-sdk-js](https://www.npmjs.com/package/ng-sdk-js) or [ng-sdk-node](https://www.npmjs.com/package/ng-sdk-node) +//! The same API is also made available in Javascript for the browser (and is used by our webapp) and for nodejs. See the npm package [ng-sdk-js](https://www.npmjs.com/package/ng-sdk-js) or [nextgraph](https://www.npmjs.com/package/nextgraph) //! //! The library requires `async-std` minimal version 1.12.0 //! diff --git a/nextgraph/src/local_broker.rs b/nextgraph/src/local_broker.rs index 4537191..6f3b370 100644 --- a/nextgraph/src/local_broker.rs +++ b/nextgraph/src/local_broker.rs @@ -1931,7 +1931,8 @@ pub async fn session_stop(user_id: &UserId) -> Result<(), NgError> { match broker.opened_sessions.remove(user_id) { Some(id) => { let _ = broker.get_session(id)?; - broker.opened_sessions_list[id as usize].take(); + let real_id = LocalBroker::to_real_session_id(id); + broker.opened_sessions_list[real_id as usize].take(); // TODO: change the logic here once it will be possible to have several users connected at the same time Broker::close_all_connections().await; } diff --git a/ng-net/src/broker.rs b/ng-net/src/broker.rs index f5a5e93..e24bc91 100644 --- a/ng-net/src/broker.rs +++ b/ng-net/src/broker.rs @@ -1109,9 +1109,9 @@ impl Broker { }; for client in peers_for_local_dispatch { - //log_debug!("dispatch_event peer {:?}", client); + log_debug!("dispatch_event peer {:?}", client); if let Some(fsm) = self.get_fsm_for_client(&client) { - //log_debug!("ForwardedEvent peer {:?}", client); + log_debug!("ForwardedEvent peer {:?}", client); let _ = fsm .lock() .await diff --git a/ng-oxigraph/src/storage/backend/oxi_rocksdb.rs b/ng-oxigraph/src/storage/backend/oxi_rocksdb.rs index 7b3c0b7..37e18ee 100644 --- a/ng-oxigraph/src/storage/backend/oxi_rocksdb.rs +++ b/ng-oxigraph/src/storage/backend/oxi_rocksdb.rs @@ -260,9 +260,11 @@ impl Db { !write_options.is_null(), "rocksdb_writeoptions_create returned null" ); - rocksdb_writeoptions_set_sync(write_options, 1); + if in_memory { rocksdb_writeoptions_disable_WAL(write_options, 1); // No need for WAL + } else { + rocksdb_writeoptions_set_sync(write_options, 1); } let transaction_options = rocksdb_transaction_options_create(); diff --git a/ng-repo/src/branch.rs b/ng-repo/src/branch.rs index ab709c9..0f4bf3e 100644 --- a/ng-repo/src/branch.rs +++ b/ng-repo/src/branch.rs @@ -84,11 +84,16 @@ impl DagNode { fn collapse( id: &ObjectId, dag: &HashMap, + dag_ids: &HashSet, already_in: &mut HashSet, ) -> Vec { let this = dag.get(id).unwrap(); - - if this.past.len() > 1 && !this.past.is_subset(already_in) { + let intersec = this + .past + .intersection(dag_ids) + .cloned() + .collect::>(); + if intersec.len() > 1 && !intersec.is_subset(already_in) { // we postpone it // log_debug!("postponed {}", id); vec![] @@ -96,7 +101,8 @@ impl DagNode { let mut res = vec![*id]; already_in.insert(*id); for child in this.future.iter() { - res.append(&mut Self::collapse(child, dag, already_in)); + log_debug!("child of {} : {}", id, child); + res.append(&mut Self::collapse(child, dag, dag_ids, already_in)); } res } @@ -317,10 +323,21 @@ impl Branch { let sub_dag_to_send_size = visited.len(); let mut result = Vec::with_capacity(sub_dag_to_send_size); + let dag_ids: HashSet = visited.keys().cloned().collect(); for first in first_generation { - result.append(&mut DagNode::collapse(first, &visited, &mut already_in)); + result.append(&mut DagNode::collapse( + first, + &visited, + &dag_ids, + &mut already_in, + )); } - + // log_debug!( + // "DAG {} {} {}", + // result.len(), + // sub_dag_to_send_size, + // already_in.len() + // ); if result.len() != sub_dag_to_send_size || already_in.len() != sub_dag_to_send_size { return Err(ObjectParseError::MalformedDag); } diff --git a/ng-sdk-js/README.md b/ng-sdk-js/README.md index c13e25f..3905332 100644 --- a/ng-sdk-js/README.md +++ b/ng-sdk-js/README.md @@ -30,7 +30,7 @@ npm i ng-sdk-js The API is divided in 4 parts: -- the wallet API that gives access to the local data, once the user has opened their wallet +- the wallet API that lets user open and change their wallet and use its credentials - the LocalVerifier API to open the documents locally - the RemoteVerifier API that is connecting to the ngd server and runs the verifier on the server. - a special mode of operation for ngd called `Headless` where all the users of that server have given full control of their data, to the server. @@ -46,22 +46,6 @@ NextGraph daemon (ngd) is normally used only as a Broker of encrypted messages, The verifier is the service that opens the encrypted data and "materialize" it. In local-first/CRDT terminology, this means that the many commits that form the DAG of operations, are reduced in order to obtain the current state of a document, that can then be read or edited locally by the user. Usually, the verifier runs locally in the native NextGraph app, and the materialized state is persisted locally (with encryption at rest). The web version of the app (available at https://app.nextgraph.one) is not persisting the materialized state yet, because the "UserStorage for Web" feature is not ready yet. Programmers can also run a local verifier with the wallet API in Rust or nodeJS (not documented), or use the CLI to create a local materialized state. It is also possible to run a remote verifier on ngd, and the user has to give their credentials to the server (partially or fully) so the server can decrypt the data and process it. Obviously this breaks the end-to-end-encryption. But depending on the use-cases, it can be useful to have the verifier run on some server. -Here are 3 main use-cases for the remote verifier : - -- A specific user wants to run a remote verifier on the server instead of running their verifier locally. This is the case for end-users on platforms that are not supported by Tauri which powers all the native apps. - The end-user on those platforms has to run a local ngd daemon instead, and access the app in their browser of choice, at the url http://localhost:1440 . Here the breaking of E2EE is acceptable, as the decrypted data will reside locally, on the machine of the user. - As the web app cannot save decrypted user data yet, it has to reprocess all the encrypted commits at every load. - In order to avoid this, running a remote verifier on the local ngd is a solution, as the ngd can save the decrypted user's data locally, if the user gave permission for it. - The API for that use case is `session_start_remote` and the credentials (usually stored in the user's wallet) are extracted from the wallet and passed to ngd. - The rest of the "session APIs" can be used in the same manner as with a local Verifier. This present JS library connects to the server transparently and opens a RemoteVerifier there. - The remote session can be detached, which means that even after the session is closed, or when the client disconnects from ngd, the Verifier still runs in the daemon. - This "detached" feature is useful if we want some automatic actions that only the Verifier can do, be performed in the background (signing by example, is a background task). -- The second use case is what we call a Headless server (because it doesn't have any wallets connecting to it). It departs a bit from the general architecture of NextGraph, as it is meant for backward compatibility with the web 2.0 federation, based on domain names and without E2EE. - This mode of operation allows users to delegate all their trust to the server. In the future, we will provide the possibility to delegate access only to some parts of the User's data. - In Headless mode, the server can be used in a traditional federated way, where the server can see the user's data in clear, and act accordingly. We have in mind here to offer bridges to existing federated protocols like ActivityPub and Solid (via the project ActivityPods) at first, and later add other protocols like ATproto, Nostr, XMPP, and even SMTP ! Any web 2.0 federated protocol could be bridged. At the same time, the bridging ngd server would still be a fully-fledged ngd daemon, thus offering all the advantages of NextGraph to its users, who could decide to port their data somewhere else, restrict the access of the server to their own data, interact and collaborate with other users (of the federation or of the whole NextGraph network) in a secure and private way, and use the local-first NG app and access their own data offline. -- A third use case will be to be able to run some services (in nodeJS or Rust) that have received partial access to the user's data, and can process it accordingly. By example, an AI service like jan.ai, or a SPARQL REST endpoint, an LDP endpoint, an endpoint to fetch data that will be displayed by a headless framework like Astro or any other REST/HTTP endpoint to access some of the user's data. - -All of those use cases are handled with the present nodeJS library, using the API described below. ## APIs @@ -102,7 +86,7 @@ entrust the credentials of user to an ngd server. coming soon - once at first with some metadata information in `file.V0.FileMeta` - one or more times with all the blobs of data, in `file.V0.FileBinary` - finally, one more time with `file.V0 == 'EndOfStream'`. See the example on how to reconstruct a buffer out of this. -- `ng.session_headless_stop(session_id, force_close)` stops the session, but doesn't close the remote verifier, except if force_close is true. if false, the verifier is detached from the session and continues to run on the server. a a new session can then be reattached to it, by calling session_headless_start with the same user_id. +- `ng.session_headless_stop(session_id, force_close)` stops the session, but doesn't close the remote verifier, except if force_close is true. if false, the verifier is detached from the session and continues to run on the server. A new session can then be reattached to it, by calling session_headless_start with the same user_id. Here is the format of the config object to be supplied in the calls to `init_headless` and `admin_create_user`: @@ -148,7 +132,7 @@ ngd -v --save-key -l 1440 -d --admin ` option. But the goal of ngd is to be a broker that connects to other brokers on the internet, so it should have a public interface configured at some point. In another terminal, same current working directory: @@ -173,7 +157,7 @@ you can now save the configs on both the server and client # stop the running server by entering ctrl+C on its terminal. ngd -l 1440 -d --save-config # in the other terminal -ngcli -s 127.0.0.1,1440, -d -u --save-config +ngcli -s 127.0.0.1,1440, -u --save-config ``` From now on, you can just use `ngd` and `ngcli` commands without the need to specify the above options, as the config has been saved to disk. Except if you changed the base directory, in which case you have to supply the `--base` option at every call. @@ -190,7 +174,7 @@ ngcli gen-key That's it. The broker is configured. You can create an entry in systemd/init.d for your system to start the daemon at every boot. Don't forget to change the working directory to where your data is, or use `--base` option. -If you have configured a domain, then the web app can be accessed at https://app.your-domain.com by example. +If you have configured a domain, then the web app can be accessed at https://app.server.com by example. --- diff --git a/ng-sdk-js/app-node/index.js b/ng-sdk-js/app-node/index.js index 94504e6..82a55ce 100644 --- a/ng-sdk-js/app-node/index.js +++ b/ng-sdk-js/app-node/index.js @@ -9,7 +9,7 @@ const WebSocket = require("ws"); // shim to insert WebSocket in global -const ng = require("ng-sdk-node"); +const ng = require("nextgraph"); global.WebSocket = WebSocket; let config = { diff --git a/ng-sdk-js/app-node/package-lock.json b/ng-sdk-js/app-node/package-lock.json index e5b3c7e..6ae012a 100644 --- a/ng-sdk-js/app-node/package-lock.json +++ b/ng-sdk-js/app-node/package-lock.json @@ -9,12 +9,12 @@ "version": "0.1.0", "license": "(MIT OR Apache-2.0)", "dependencies": { - "ng-sdk-node": "^0.1.0", + "nextgraph": "^0.1.0", "ws": "^8.13.0" } }, "../pkg-node": { - "name": "ng-sdk-node", + "name": "nextgraph", "version": "0.1.0", "license": "MIT/Apache-2.0" }, diff --git a/ng-sdk-js/app-node/package.json b/ng-sdk-js/app-node/package.json index 0387c06..5108a89 100644 --- a/ng-sdk-js/app-node/package.json +++ b/ng-sdk-js/app-node/package.json @@ -13,7 +13,7 @@ "author": "Niko PLP ", "license": "(MIT OR Apache-2.0)", "dependencies": { - "ng-sdk-node": "^0.1.0", + "nextgraph": "^0.1.0", "ws": "^8.13.0" } } diff --git a/ng-sdk-js/prepare-node.js b/ng-sdk-js/prepare-node.js index 0d1fede..531b867 100644 --- a/ng-sdk-js/prepare-node.js +++ b/ng-sdk-js/prepare-node.js @@ -5,7 +5,7 @@ const PATH_README = './pkg-node/README.md'; const pkg_json = fs.readFileSync(PATH); let pkg = JSON.parse(pkg_json) -pkg.name = "ng-sdk-node"; +pkg.name = "nextgraph"; pkg.description = "nodejs SDK of NextGraph"; fs.writeFileSync(PATH, JSON.stringify(pkg, null, 2), 'utf8'); @@ -13,7 +13,7 @@ fs.readFile(PATH_README, 'utf8', function (err,data) { if (err) { return console.log(err); } - var result = data.replace(/ng-sdk-js/g, 'ng-sdk-node'); + var result = data.replace(/ng-sdk-js/g, 'nextgraph'); fs.writeFile(PATH_README, result, 'utf8', function (err) { if (err) return console.log(err); diff --git a/ng-sdk-js/src/lib.rs b/ng-sdk-js/src/lib.rs index e8f50d2..264fc7e 100644 --- a/ng-sdk-js/src/lib.rs +++ b/ng-sdk-js/src/lib.rs @@ -10,6 +10,7 @@ */ #![cfg(target_arch = "wasm32")] +#![allow(unused_imports)] mod model; @@ -20,7 +21,6 @@ use std::sync::Arc; use once_cell::sync::Lazy; use serde::{Deserialize, Serialize}; -#[allow(unused_imports)] use serde_json::json; // use js_sys::Reflect; use async_std::stream::StreamExt; @@ -30,25 +30,19 @@ use wasm_bindgen::prelude::*; use wasm_bindgen_futures::future_to_promise; use wasm_bindgen_futures::JsFuture; -#[allow(unused_imports)] use ng_repo::errors::{NgError, ProtocolError}; use ng_repo::log::*; use ng_repo::types::*; -#[allow(unused_imports)] use ng_repo::utils::{decode_key, decode_priv_key}; use ng_net::app_protocol::*; use ng_net::broker::*; -#[allow(unused_imports)] use ng_net::types::{BindAddress, ClientInfo, ClientInfoV0, ClientType, CreateAccountBSP, IP}; -#[allow(unused_imports)] use ng_net::utils::{ decode_invitation_string, parse_ip_and_port_for, retrieve_local_bootstrap, retrieve_local_url, spawn_and_log_error, Receiver, ResultSend, }; -#[allow(unused_imports)] use ng_net::{actor::*, actors::admin::*}; -#[allow(unused_imports)] use ng_net::{WS_PORT, WS_PORT_REVERSE_PROXY}; use ng_client_ws::remote_ws_wasm::ConnectionWebSocket; @@ -738,6 +732,7 @@ async fn do_upload_done( Ok(reference) } +#[cfg(wasmpack_target = "nodejs")] async fn do_upload_done_( upload_id: u32, session_id: u64, @@ -871,6 +866,7 @@ async fn do_upload_chunk( .map_err(|e: NgError| e.to_string()) } +#[cfg(wasmpack_target = "nodejs")] async fn do_upload_chunk_( session_id: u64, upload_id: u32, diff --git a/ng-verifier/src/verifier.rs b/ng-verifier/src/verifier.rs index 37b5504..7dd2f16 100644 --- a/ng-verifier/src/verifier.rs +++ b/ng-verifier/src/verifier.rs @@ -878,9 +878,9 @@ impl Verifier { let user = self.config.user_priv_key.to_pub(); let broker = BROKER.read().await; - //log_info!("looping on branches {:?}", branches); + log_info!("looping on branches {:?}", branches); for (repo, branch, publisher) in branches { - //log_info!("open_branch_ repo {} branch {}", repo, branch); + log_info!("open_branch_ repo {} branch {}", repo, branch); let _e = self .open_branch_( &repo, @@ -892,12 +892,12 @@ impl Verifier { false, ) .await; - // log_info!( - // "END OF open_branch_ repo {} branch {} with {:?}", - // repo, - // branch, - // _e - // ); + log_info!( + "END OF open_branch_ repo {} branch {} with {:?}", + repo, + branch, + _e + ); // discarding error. } Ok(())