diff --git a/ng-app/src/lib/Test.svelte b/ng-app/src/lib/Test.svelte index 81052e5..5652f36 100644 --- a/ng-app/src/lib/Test.svelte +++ b/ng-app/src/lib/Test.svelte @@ -65,11 +65,10 @@ final_blob = new Blob([final_blob, blob.V0.FileBinary], { type: content_type, }); - } else { - var imageUrl = URL.createObjectURL(final_blob); - - resolve(imageUrl); } + } else if (blob.V0 == "EndOfStream") { + var imageUrl = URL.createObjectURL(final_blob); + resolve(imageUrl); } }); } catch (e) { diff --git a/ng-net/src/app_protocol.rs b/ng-net/src/app_protocol.rs index bd1e454..13da847 100644 --- a/ng-net/src/app_protocol.rs +++ b/ng-net/src/app_protocol.rs @@ -9,13 +9,22 @@ //! App Protocol (between LocalBroker and Verifier) +use lazy_static::lazy_static; +use regex::Regex; use serde::{Deserialize, Serialize}; use ng_repo::errors::NgError; use ng_repo::types::*; +use ng_repo::utils::{decode_id, decode_sym_key}; use crate::types::*; +lazy_static! { + #[doc(hidden)] + static ref RE_FILE_READ_CAP: Regex = + Regex::new(r"^did:ng:j:([A-Za-z0-9-_%.]*):k:([A-Za-z0-9-_%.]*)$").unwrap(); +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub enum AppFetchContentV0 { Get, // does not subscribe. more to be detailed @@ -127,8 +136,31 @@ impl NuriV0 { locator: vec![], } } - pub fn new(_from: String) -> Self { - todo!(); + pub fn new_from(from: String) -> Result { + let c = RE_FILE_READ_CAP.captures(&from); + + if c.is_some() + && c.as_ref().unwrap().get(1).is_some() + && c.as_ref().unwrap().get(2).is_some() + { + let cap = c.unwrap(); + let j = cap.get(1).unwrap().as_str(); + let k = cap.get(2).unwrap().as_str(); + let id = decode_id(j)?; + let key = decode_sym_key(k)?; + Ok(Self { + target: NuriTargetV0::PrivateStore, + entire_store: false, + object: Some(id), + branch: None, + overlay: None, + access: vec![NgAccessV0::Key(key)], + topic: None, + locator: vec![], + }) + } else { + Err(NgError::InvalidNuri) + } } } @@ -439,6 +471,7 @@ pub enum AppResponseV0 { True, False, Error(String), + EndOfStream, } #[derive(Clone, Debug, Serialize, Deserialize)] diff --git a/ng-repo/src/errors.rs b/ng-repo/src/errors.rs index 1aacc14..8776820 100644 --- a/ng-repo/src/errors.rs +++ b/ng-repo/src/errors.rs @@ -80,6 +80,7 @@ pub enum NgError { ConfigError(String), LocalBrokerIsHeadless, LocalBrokerIsNotHeadless, + InvalidNuri, } impl Error for NgError {} diff --git a/ng-repo/src/repo.rs b/ng-repo/src/repo.rs index cca77ed..e10cebc 100644 --- a/ng-repo/src/repo.rs +++ b/ng-repo/src/repo.rs @@ -102,7 +102,7 @@ pub struct BranchInfo { pub commits_nbr: u64, } -/// In memory Repository representation. With helper functions that access the underlying UserStore and keeps proxy of the values +/// In memory Repository representation. With helper functions that access the underlying UserStorage and keeps proxy of the values #[derive(Debug)] pub struct Repo { pub id: RepoId, diff --git a/ng-repo/src/utils.rs b/ng-repo/src/utils.rs index 96d81e3..ebb35e0 100644 --- a/ng-repo/src/utils.rs +++ b/ng-repo/src/utils.rs @@ -65,6 +65,16 @@ pub fn decode_priv_key(key_string: &str) -> Result { Ok(serde_bare::from_slice(&vec).map_err(|_| NgError::InvalidKey)?) } +pub fn decode_sym_key(key_string: &str) -> Result { + let vec = base64_url::decode(key_string).map_err(|_| NgError::InvalidKey)?; + Ok(serde_bare::from_slice(&vec).map_err(|_| NgError::InvalidKey)?) +} + +pub fn decode_id(key_string: &str) -> Result { + let vec = base64_url::decode(key_string).map_err(|_| NgError::InvalidKey)?; + Ok(serde_bare::from_slice(&vec).map_err(|_| NgError::InvalidKey)?) +} + pub fn ed_privkey_to_ed_pubkey(privkey: &PrivKey) -> PubKey { // SecretKey is zeroized on drop (3 lines below) se we are safe let sk = SecretKey::from_bytes(privkey.slice()).unwrap(); diff --git a/ng-sdk-js/README.md b/ng-sdk-js/README.md index 6cf3dec..c13e25f 100644 --- a/ng-sdk-js/README.md +++ b/ng-sdk-js/README.md @@ -28,7 +28,171 @@ Read our [getting started guide](https://docs.nextgraph.org/en/getting-started/) npm i ng-sdk-js ``` -## How to use it +The API is divided in 4 parts: + +- the wallet API that gives access to the local data, once the user has opened their wallet +- the LocalVerifier API to open the documents locally +- the RemoteVerifier API that is connecting to the ngd server and runs the verifier on the server. +- a special mode of operation for ngd called `Headless` where all the users of that server have given full control of their data, to the server. + +All of those API share a common `Session API` (all the functions that have a session_id as first argument) + +The wallet API is not documented as it will be deprecated as soon as we will have an Authorization/Capability Delegation mechanism between the NextGraph apps and the Wallet. +Still, this API will always be available as it is used internally by the NextGraph app, and could be used also by the owner of a wallet, to access its data with nodeJS or Rust. + +## Headless server (runs the verifiers of the users on the server) + +NextGraph daemon (ngd) is normally used only as a Broker of encrypted messages, but it can also be configured to run the verifiers of some or all of the users' data. +The verifier is the service that opens the encrypted data and "materialize" it. In local-first/CRDT terminology, this means that the many commits that form the DAG of operations, are reduced in order to obtain the current state of a document, that can then be read or edited locally by the user. Usually, the verifier runs locally in the native NextGraph app, and the materialized state is persisted locally (with encryption at rest). The web version of the app (available at https://app.nextgraph.one) is not persisting the materialized state yet, because the "UserStorage for Web" feature is not ready yet. Programmers can also run a local verifier with the wallet API in Rust or nodeJS (not documented), or use the CLI to create a local materialized state. + +It is also possible to run a remote verifier on ngd, and the user has to give their credentials to the server (partially or fully) so the server can decrypt the data and process it. Obviously this breaks the end-to-end-encryption. But depending on the use-cases, it can be useful to have the verifier run on some server. +Here are 3 main use-cases for the remote verifier : + +- A specific user wants to run a remote verifier on the server instead of running their verifier locally. This is the case for end-users on platforms that are not supported by Tauri which powers all the native apps. + The end-user on those platforms has to run a local ngd daemon instead, and access the app in their browser of choice, at the url http://localhost:1440 . Here the breaking of E2EE is acceptable, as the decrypted data will reside locally, on the machine of the user. + As the web app cannot save decrypted user data yet, it has to reprocess all the encrypted commits at every load. + In order to avoid this, running a remote verifier on the local ngd is a solution, as the ngd can save the decrypted user's data locally, if the user gave permission for it. + The API for that use case is `session_start_remote` and the credentials (usually stored in the user's wallet) are extracted from the wallet and passed to ngd. + The rest of the "session APIs" can be used in the same manner as with a local Verifier. This present JS library connects to the server transparently and opens a RemoteVerifier there. + The remote session can be detached, which means that even after the session is closed, or when the client disconnects from ngd, the Verifier still runs in the daemon. + This "detached" feature is useful if we want some automatic actions that only the Verifier can do, be performed in the background (signing by example, is a background task). +- The second use case is what we call a Headless server (because it doesn't have any wallets connecting to it). It departs a bit from the general architecture of NextGraph, as it is meant for backward compatibility with the web 2.0 federation, based on domain names and without E2EE. + This mode of operation allows users to delegate all their trust to the server. In the future, we will provide the possibility to delegate access only to some parts of the User's data. + In Headless mode, the server can be used in a traditional federated way, where the server can see the user's data in clear, and act accordingly. We have in mind here to offer bridges to existing federated protocols like ActivityPub and Solid (via the project ActivityPods) at first, and later add other protocols like ATproto, Nostr, XMPP, and even SMTP ! Any web 2.0 federated protocol could be bridged. At the same time, the bridging ngd server would still be a fully-fledged ngd daemon, thus offering all the advantages of NextGraph to its users, who could decide to port their data somewhere else, restrict the access of the server to their own data, interact and collaborate with other users (of the federation or of the whole NextGraph network) in a secure and private way, and use the local-first NG app and access their own data offline. +- A third use case will be to be able to run some services (in nodeJS or Rust) that have received partial access to the user's data, and can process it accordingly. By example, an AI service like jan.ai, or a SPARQL REST endpoint, an LDP endpoint, an endpoint to fetch data that will be displayed by a headless framework like Astro or any other REST/HTTP endpoint to access some of the user's data. + +All of those use cases are handled with the present nodeJS library, using the API described below. + +## APIs + +The nodeJS API is limited for now, to the following functions. + +All the functions are async. you must use them with `await` (or `.then()`). + +They all can throw errors. You must enclose them in `try {} catch(e) {}` + +See the example [here](https://git.nextgraph.org/NextGraph/nextgraph-rs/src/branch/master/ng-sdk-js/app-node). + +## Wallet API + +open and modify the wallet. +not documented yet. We don't really want developers to use it, as the opening of a wallet is a sensitive operation, that shouldn't be necessary for developers to create apps and ask permission to access the data of users. +We will provide an adhoc API for Permission/Capability delegation so the wallet API will be deprecated. + +## LocalVerifier API + +can manipulate partial access to the user's data. coming soon + +## RemoteVerifier API + +entrust the credentials of user to an ngd server. coming soon + +## Headless API + +- `ng.init_headless(config)` must be called before any other call. +- `ng.admin_create_user(config)` creates a new user on the server, and populates their 3P stores. returns the user_id +- `ng.session_headless_start(user_id)` starts a new session for the user. returns the session info, including the session_id +- `ng.sparql_query(session_id, "[SPARQL query]")` returns or: + - for SELECT queries: a JSON Sparql Query Result as a Javascript object. [SPARQL Query Results JSON Format](https://www.w3.org/TR/sparql11-results-json/) + - for CONSTRUCT queries: a list of quads in the format [RDF-JS data model](http://rdf.js.org/data-model-spec/) that can be used as ingress to RDFjs lib. + - for ASK queries: a boolean +- `ng.sparql_update(session_id, "[SPARQL update]")` returns nothing, but can throw an error. +- `ng.file_put_to_private_store(session_id,"[filename]","[mimetype]")` returns the Nuri (NextGraph URI) of the file, as a string. +- `ng.file_get_from_private_store(session_id, "[nuri]", callback)` returns a cancel function. The `callback(file)` function will be called as follow + - once at first with some metadata information in `file.V0.FileMeta` + - one or more times with all the blobs of data, in `file.V0.FileBinary` + - finally, one more time with `file.V0 == 'EndOfStream'`. See the example on how to reconstruct a buffer out of this. +- `ng.session_headless_stop(session_id, force_close)` stops the session, but doesn't close the remote verifier, except if force_close is true. if false, the verifier is detached from the session and continues to run on the server. a a new session can then be reattached to it, by calling session_headless_start with the same user_id. + +Here is the format of the config object to be supplied in the calls to `init_headless` and `admin_create_user`: + +```js +config = { + server_peer_id: "[your server ID]", + admin_user_key: "[your admin key]", + client_peer_key: "[the client key]", + server_addr: "[IP and PORT of the server]", // this one is optional. it will default to localhost:1440. Format is: A.A.A.A:P for IPv4 or [AAAA:::]:P for IpV6 +}; +``` + +Alternatively, you can use the environnement variables: + +``` +NG_HEADLESS_SERVER_PEER_ID +NG_HEADLESS_ADMIN_USER_KEY +NG_HEADLESS_CLIENT_PEER_KEY +NG_HEADLESS_SERVER_ADDR +``` + +If you supply both, the values passed in the API function call takes precedence over the env vars. + +In order to generate those keys, you will have first to run the `ngd` server, by following those instructions. + +## Install and configure ngd + +The binaries can be obtained from the [release page](https://git.nextgraph.org/NextGraph/nextgraph-rs/releases). + +You can also, [compile](https://git.nextgraph.org/NextGraph/nextgraph-rs#build-release-binaries) them from source. + +The current directory will be used to save all the config, keys and storage data, in a subfolder called `.ng`. +If you prefer to change the base directory, use the argument `--base [PATH]` when using `ngd` and/or `ngcli` commands. +Use `--help` to see a full list of options and commands on those 2 binaries. + +```bash +ngcli gen-key +# this will output 2 keys. keep both keys +# the private key is the NG_HEADLESS_ADMIN_USER_KEY value you need for the config of the above API calls. +ngd -v --save-key -l 1440 -d --admin +# In the terminal output of the server, find the line `PeerId of node` and keep the value. You will need it for the next step, as PEER_ID_OF_NODE. +# and it is also the value you need to give to NG_HEADLESS_SERVER_PEER_ID in the config for the above API calls. +``` + +`SERVER_DOMAIN` can be anything you want. If you run a web server with some content at `server.com`, then the NextGraph web app could be served at the subdomain `app.server.com` or `ng.server.com`. +This is what you should enter in `SERVER_DOMAIN`. You also have to setup your reverse proxy (haproxy, nginx, etc...) to forward incoming TLS connections to ngd. ngd listens for TCP connections on localhost port 1440 by default. The header `X-Forwarded-For` must be set by your reverse proxy. ngd does not handle TLS. Your reverse proxy has to handle the TLS terminated connections, and forward a TCP connection to ngd. +You can use ngd in your internal network (Docker, etc...) without exposing it to the internet. In this case, remove the `-d ` option. But the goal of ngd is to be a broker that connects to other brokers on the internet, so it should have a public interface configured at some point. + +In another terminal, same current working directory: + +```bash +ngcli --save-key -s 127.0.0.1,1440, -u admin add-user -a +``` + +you should see a message `User added successfully`. + +to check that the admin user has been created : + +```bash +ngcli -s 127.0.0.1,1440, -u admin list-users -a +``` + +should return your UserId + +you can now save the configs on both the server and client + +```bash +# stop the running server by entering ctrl+C on its terminal. +ngd -l 1440 -d --save-config +# in the other terminal +ngcli -s 127.0.0.1,1440, -d -u --save-config +``` + +From now on, you can just use `ngd` and `ngcli` commands without the need to specify the above options, as the config has been saved to disk. Except if you changed the base directory, in which case you have to supply the `--base` option at every call. + +The 2 API functions that need a config, also need a `NG_HEADLESS_CLIENT_PEER_KEY` that we haven't created yet. + +You should create it with another call to: + +```bash +ngcli gen-key +# the private key is what goes to NG_HEADLESS_CLIENT_PEER_KEY . it identifies the client (the process that is using this library. a nodeJS process) +# the public key will go to the ngd config for authorization (but this is not implemented yet. just keep it somewhere for now) +``` + +That's it. The broker is configured. You can create an entry in systemd/init.d for your system to start the daemon at every boot. Don't forget to change the working directory to where your data is, or use `--base` option. + +If you have configured a domain, then the web app can be accessed at https://app.your-domain.com by example. + +--- ## License diff --git a/ng-sdk-js/app-node/index.js b/ng-sdk-js/app-node/index.js index cb60663..94504e6 100644 --- a/ng-sdk-js/app-node/index.js +++ b/ng-sdk-js/app-node/index.js @@ -41,8 +41,29 @@ ng.init_headless(config).then( async() => { let result = await ng.sparql_update(session.session_id, "INSERT DATA { }"); console.log(result); - // the 2nd argument `false` means do not `force_close` the dataset. - // it stays in memory even when the session is stopped. (not all the dataset is in memory. just some metadata) + let file_nuri = await ng.file_put_to_private_store(session.session_id,"LICENSE-MIT","text/plain"); + console.log(file_nuri); + + //let file_nuri = "did:ng:j:AD_d4njVMAtIDEU1G-RDxfOLIOZyOrB_1Rb7B6XykIEJ:k:APV-_Xtk03PW_Mbl4OaYpLrmEkBDVtn81lpto8sxc_tb"; + var bufs = []; + let cancel = await ng.file_get_from_private_store(session.session_id, file_nuri, async (file) => { + if (file.V0.FileMeta) { + //skip + } else if (file.V0.FileBinary) { + if (file.V0.FileBinary.byteLength > 0) { + bufs.push(file.V0.FileBinary); + } + } else if (file.V0 == 'EndOfStream') { + //console.log("end of file"); + var buf = Buffer.concat(bufs); + // if the file contains some UTF8 text + console.log(buf.toString('utf8')); + } + }); + + // the 2nd argument `false` means: do not `force_close` the dataset. + // it will be detached, which means it stays in memory even when the session is stopped. + // (not all the dataset is in memory anyway! just some metadata) // if you set this to true, the dataset is closed and removed from memory on the server. // next time you will open a session for this user, the dataset will be loaded again. let res = await ng.session_headless_stop(session.session_id, false); diff --git a/ng-sdk-js/js/node.js b/ng-sdk-js/js/node.js index 393e233..4f8a1d7 100644 --- a/ng-sdk-js/js/node.js +++ b/ng-sdk-js/js/node.js @@ -123,6 +123,32 @@ module.exports.get_env_vars = function () { }; } +const path = require('path'); +const fs = require('fs'); + +module.exports.upload_file = async ( filename, callback, end) => { + let readStream = fs.createReadStream(filename,{ highWaterMark: 1048564 }); + + return new Promise(async (resolve, reject) => { + readStream.on('data', async function(chunk) { + try { + let ret = await callback(chunk); + } + catch (e) { + readStream.destroy(); + reject(e); + } + }).on('end', async function() { + let reference = await end(path.basename(filename)); + resolve(reference); + }).on('error', async function(e) { + reject(e.message); + + }); + }) +} + + module.exports.client_details = function () { const process = require('process'); let arch = osnode.machine? osnode.machine() : process.arch; diff --git a/ng-sdk-js/src/lib.rs b/ng-sdk-js/src/lib.rs index 81b4427..e8f50d2 100644 --- a/ng-sdk-js/src/lib.rs +++ b/ng-sdk-js/src/lib.rs @@ -27,6 +27,7 @@ use async_std::stream::StreamExt; use js_sys::Array; use oxrdf::Triple; use wasm_bindgen::prelude::*; +use wasm_bindgen_futures::future_to_promise; use wasm_bindgen_futures::JsFuture; #[allow(unused_imports)] @@ -122,13 +123,13 @@ pub fn wallet_gen_shuffle_for_pin() -> Vec { #[wasm_bindgen] pub fn wallet_open_with_pazzle( - js_wallet: JsValue, + wallet: JsValue, pazzle: Vec, - js_pin: JsValue, + pin: JsValue, ) -> Result { - let encrypted_wallet = serde_wasm_bindgen::from_value::(js_wallet) + let encrypted_wallet = serde_wasm_bindgen::from_value::(wallet) .map_err(|_| "Deserialization error of wallet")?; - let pin = serde_wasm_bindgen::from_value::<[u8; 4]>(js_pin) + let pin = serde_wasm_bindgen::from_value::<[u8; 4]>(pin) .map_err(|_| "Deserialization error of pin")?; let res = nextgraph::local_broker::wallet_open_with_pazzle(&encrypted_wallet, pazzle, pin); match res { @@ -140,10 +141,10 @@ pub fn wallet_open_with_pazzle( } #[wasm_bindgen] -pub fn wallet_update(js_wallet_id: JsValue, js_operations: JsValue) -> Result { - let _wallet = serde_wasm_bindgen::from_value::(js_wallet_id) +pub fn wallet_update(wallet_id: JsValue, operations: JsValue) -> Result { + let _wallet = serde_wasm_bindgen::from_value::(wallet_id) .map_err(|_| "Deserialization error of WalletId")?; - let _operations = serde_wasm_bindgen::from_value::>(js_operations) + let _operations = serde_wasm_bindgen::from_value::>(operations) .map_err(|_| "Deserialization error of operations")?; unimplemented!(); // match res { @@ -270,8 +271,8 @@ pub async fn sparql_update(session_id: JsValue, sparql: String) -> Result<(), St #[cfg(wasmpack_target = "nodejs")] #[wasm_bindgen] -pub async fn admin_create_user(js_config: JsValue) -> Result { - let config = HeadLessConfigStrings::load(js_config)?; +pub async fn admin_create_user(config: JsValue) -> Result { + let config = HeadLessConfigStrings::load(config)?; let admin_user_key = config .admin_user_key .ok_or("No admin_user_key found in config nor env var.".to_string())?; @@ -350,6 +351,12 @@ extern "C" { fn local_get(key: String) -> Option; fn is_browser() -> bool; fn storage_clear(); + #[wasm_bindgen(catch)] + async fn upload_file( + filename: String, + cb_chunk: &Closure js_sys::Promise>, + cb_end: &Closure js_sys::Promise>, + ) -> Result; } fn local_read(key: String) -> Result { @@ -398,9 +405,9 @@ static INIT_LOCAL_BROKER: Lazy> = Lazy::new(|| { }); #[wasm_bindgen] -pub async fn wallet_create(js_params: JsValue) -> Result { +pub async fn wallet_create(params: JsValue) -> Result { init_local_broker_with_lazy(&INIT_LOCAL_BROKER).await; - let mut params = serde_wasm_bindgen::from_value::(js_params) + let mut params = serde_wasm_bindgen::from_value::(params) .map_err(|_| "Deserialization error of args")?; params.result_with_wallet_file = true; let res = nextgraph::local_broker::wallet_create_v0(params).await; @@ -422,9 +429,9 @@ pub async fn wallet_get_file(wallet_name: String) -> Result { } #[wasm_bindgen] -pub async fn wallet_read_file(js_file: JsValue) -> Result { +pub async fn wallet_read_file(file: JsValue) -> Result { init_local_broker_with_lazy(&INIT_LOCAL_BROKER).await; - let file = serde_wasm_bindgen::from_value::(js_file) + let file = serde_wasm_bindgen::from_value::(file) .map_err(|_| "Deserialization error of file".to_string())?; let wallet = nextgraph::local_broker::wallet_read_file(file.into_vec()) @@ -436,9 +443,9 @@ pub async fn wallet_read_file(js_file: JsValue) -> Result { #[wasm_bindgen] pub async fn wallet_was_opened( - js_opened_wallet: JsValue, //SensitiveWallet + opened_wallet: JsValue, //SensitiveWallet ) -> Result { - let opened_wallet = serde_wasm_bindgen::from_value::(js_opened_wallet) + let opened_wallet = serde_wasm_bindgen::from_value::(opened_wallet) .map_err(|_| "Deserialization error of SensitiveWallet".to_string())?; let client = nextgraph::local_broker::wallet_was_opened(opened_wallet) @@ -450,13 +457,13 @@ pub async fn wallet_was_opened( #[wasm_bindgen] pub async fn wallet_import( - js_encrypted_wallet: JsValue, //Wallet, - js_opened_wallet: JsValue, //SensitiveWallet + encrypted_wallet: JsValue, //Wallet, + opened_wallet: JsValue, //SensitiveWallet in_memory: bool, ) -> Result { - let encrypted_wallet = serde_wasm_bindgen::from_value::(js_encrypted_wallet) + let encrypted_wallet = serde_wasm_bindgen::from_value::(encrypted_wallet) .map_err(|_| "Deserialization error of Wallet".to_string())?; - let opened_wallet = serde_wasm_bindgen::from_value::(js_opened_wallet) + let opened_wallet = serde_wasm_bindgen::from_value::(opened_wallet) .map_err(|_| "Deserialization error of SensitiveWallet".to_string())?; let client = nextgraph::local_broker::wallet_import(encrypted_wallet, opened_wallet, in_memory) @@ -564,13 +571,13 @@ pub async fn test() { #[wasm_bindgen] pub async fn app_request_stream( // js_session_id: JsValue, - js_request: JsValue, + request: JsValue, callback: &js_sys::Function, ) -> Result { // let session_id: u64 = serde_wasm_bindgen::from_value::(js_session_id) // .map_err(|_| "Deserialization error of session_id".to_string())?; - let request = serde_wasm_bindgen::from_value::(js_request) + let request = serde_wasm_bindgen::from_value::(request) .map_err(|_| "Deserialization error of AppRequest".to_string())?; let (reader, cancel) = nextgraph::local_broker::app_request_stream(request) @@ -616,10 +623,10 @@ pub async fn app_request_stream( } #[wasm_bindgen] -pub async fn app_request(js_request: JsValue) -> Result { +pub async fn app_request(request: JsValue) -> Result { // let session_id: u64 = serde_wasm_bindgen::from_value::(js_session_id) // .map_err(|_| "Deserialization error of session_id".to_string())?; - let request = serde_wasm_bindgen::from_value::(js_request) + let request = serde_wasm_bindgen::from_value::(request) .map_err(|_| "Deserialization error of AppRequest".to_string())?; let response = nextgraph::local_broker::app_request(request) @@ -630,28 +637,148 @@ pub async fn app_request(js_request: JsValue) -> Result { } #[wasm_bindgen] -pub async fn upload_chunk( - js_session_id: JsValue, - js_upload_id: JsValue, - js_chunk: JsValue, - js_nuri: JsValue, +pub async fn file_get_from_private_store( + session_id: JsValue, + nuri: String, + callback: &js_sys::Function, ) -> Result { - //log_debug!("upload_chunk {:?}", js_nuri); - let session_id: u64 = serde_wasm_bindgen::from_value::(js_session_id) + let session_id: u64 = serde_wasm_bindgen::from_value::(session_id) .map_err(|_| "Deserialization error of session_id".to_string())?; - let upload_id: u32 = serde_wasm_bindgen::from_value::(js_upload_id) + + let nuri = NuriV0::new_from(nuri).map_err(|_| "Deserialization error of Nuri".to_string())?; + + let mut request = AppRequest::new(AppRequestCommandV0::FileGet, nuri.clone(), None); + request.set_session_id(session_id); + + let (reader, cancel) = nextgraph::local_broker::app_request_stream(request) + .await + .map_err(|e: NgError| e.to_string())?; + + async fn inner_task( + mut reader: Receiver, + callback: js_sys::Function, + ) -> ResultSend<()> { + while let Some(app_response) = reader.next().await { + let response_js = serde_wasm_bindgen::to_value(&app_response).unwrap(); + let this = JsValue::null(); + match callback.call1(&this, &response_js) { + Ok(jsval) => { + let promise_res: Result = jsval.dyn_into(); + match promise_res { + Ok(promise) => { + let _ = JsFuture::from(promise).await; + } + Err(_) => {} + } + } + Err(e) => { + log_err!( + "JS callback for fetch_file_from_private_store failed with {:?}", + e + ); + } + } + } + Ok(()) + } + + spawn_and_log_error(inner_task(reader, callback.clone())); + + let cb = Closure::once(move || { + log_info!("cancelling"); + //sender.close_channel() + cancel(); + }); + //Closure::wrap(Box::new(move |sender| sender.close_channel()) as Box)>); + let ret = cb.as_ref().clone(); + cb.forget(); + Ok(ret) +} + +async fn do_upload_done( + upload_id: u32, + session_id: u64, + nuri: NuriV0, + filename: String, +) -> Result { + let mut request = AppRequest::new( + AppRequestCommandV0::FilePut, + nuri.clone(), + Some(AppRequestPayload::V0( + AppRequestPayloadV0::RandomAccessFilePutChunk((upload_id, serde_bytes::ByteBuf::new())), + )), + ); + request.set_session_id(session_id); + + let response = nextgraph::local_broker::app_request(request) + .await + .map_err(|e: NgError| e.to_string())?; + + let reference = match response { + AppResponse::V0(AppResponseV0::FileUploaded(refe)) => refe, + _ => return Err("invalid response".to_string()), + }; + + let mut request = AppRequest::new( + AppRequestCommandV0::FilePut, + nuri, + Some(AppRequestPayload::V0(AppRequestPayloadV0::AddFile( + DocAddFile { + filename: Some(filename), + object: reference.clone(), + }, + ))), + ); + request.set_session_id(session_id); + + nextgraph::local_broker::app_request(request) + .await + .map_err(|e: NgError| e.to_string())?; + + Ok(reference) +} + +async fn do_upload_done_( + upload_id: u32, + session_id: u64, + nuri: NuriV0, + filename: String, +) -> Result { + let response = do_upload_done(upload_id, session_id, nuri, filename) + .await + .map_err(|e| { + let ee: JsValue = e.into(); + ee + })?; + + Ok(serde_wasm_bindgen::to_value(&response).unwrap()) +} + +#[wasm_bindgen] +pub async fn upload_done( + upload_id: JsValue, + session_id: JsValue, + nuri: JsValue, + filename: String, +) -> Result { + let upload_id: u32 = serde_wasm_bindgen::from_value::(upload_id) .map_err(|_| "Deserialization error of upload_id".to_string())?; - let chunk: serde_bytes::ByteBuf = - serde_wasm_bindgen::from_value::(js_chunk) - .map_err(|_| "Deserialization error of chunk".to_string())?; - let nuri: NuriV0 = serde_wasm_bindgen::from_value::(js_nuri) + let nuri: NuriV0 = serde_wasm_bindgen::from_value::(nuri) .map_err(|_| "Deserialization error of nuri".to_string())?; + let session_id: u64 = serde_wasm_bindgen::from_value::(session_id) + .map_err(|_| "Deserialization error of session_id".to_string())?; + let reference = do_upload_done(upload_id, session_id, nuri, filename).await?; + + Ok(serde_wasm_bindgen::to_value(&reference).unwrap()) +} + +async fn do_upload_start(session_id: u64, nuri: NuriV0, mimetype: String) -> Result { let mut request = AppRequest::new( AppRequestCommandV0::FilePut, nuri, Some(AppRequestPayload::V0( - AppRequestPayloadV0::RandomAccessFilePutChunk((upload_id, chunk)), + AppRequestPayloadV0::RandomAccessFilePut(mimetype), )), ); request.set_session_id(session_id); @@ -660,6 +787,125 @@ pub async fn upload_chunk( .await .map_err(|e: NgError| e.to_string())?; + match response { + AppResponse::V0(AppResponseV0::FileUploading(upload_id)) => Ok(upload_id), + _ => Err("invalid response".to_string()), + } +} + +#[wasm_bindgen] +pub async fn upload_start( + session_id: JsValue, + nuri: JsValue, + mimetype: String, +) -> Result { + let session_id: u64 = serde_wasm_bindgen::from_value::(session_id) + .map_err(|_| "Deserialization error of session_id".to_string())?; + let nuri: NuriV0 = serde_wasm_bindgen::from_value::(nuri) + .map_err(|_| "Deserialization error of nuri".to_string())?; + + let upload_id = do_upload_start(session_id, nuri, mimetype).await?; + + Ok(serde_wasm_bindgen::to_value(&upload_id).unwrap()) +} + +#[cfg(wasmpack_target = "nodejs")] +#[wasm_bindgen] +pub async fn file_put_to_private_store( + session_id: JsValue, + filename: String, + mimetype: String, +) -> Result { + let target = NuriV0::new_private_store_target(); + + let session_id: u64 = serde_wasm_bindgen::from_value::(session_id) + .map_err(|_| "Deserialization error of session_id".to_string())?; + + let upload_id = do_upload_start(session_id, target.clone(), mimetype).await?; + let target_for_chunk = target.clone(); + let cb_chunk = Closure::new(move |chunk| { + let chunk_res = serde_wasm_bindgen::from_value::(chunk); + match chunk_res { + Err(_e) => { + js_sys::Promise::reject(&JsValue::from_str("Deserialization error of chunk")) + } + Ok(chunk) => future_to_promise(do_upload_chunk_( + session_id, + upload_id, + chunk, + target_for_chunk.clone(), + )), + } + }); + + let cb_end = Closure::new(move |file| { + future_to_promise(do_upload_done_(upload_id, session_id, target.clone(), file)) + }); + + let reference = upload_file(filename, &cb_chunk, &cb_end) + .await + .map_err(|e| e.as_string().unwrap())?; + let reference = serde_wasm_bindgen::from_value::(reference) + .map_err(|_| "Deserialization error of reference".to_string())?; + let nuri = format!("did:ng{}", reference.nuri()); + Ok(nuri) +} + +async fn do_upload_chunk( + session_id: u64, + upload_id: u32, + chunk: serde_bytes::ByteBuf, + nuri: NuriV0, +) -> Result { + let mut request = AppRequest::new( + AppRequestCommandV0::FilePut, + nuri, + Some(AppRequestPayload::V0( + AppRequestPayloadV0::RandomAccessFilePutChunk((upload_id, chunk)), + )), + ); + request.set_session_id(session_id); + + nextgraph::local_broker::app_request(request) + .await + .map_err(|e: NgError| e.to_string()) +} + +async fn do_upload_chunk_( + session_id: u64, + upload_id: u32, + chunk: serde_bytes::ByteBuf, + nuri: NuriV0, +) -> Result { + let response = do_upload_chunk(session_id, upload_id, chunk, nuri) + .await + .map_err(|e| { + let ee: JsValue = e.into(); + ee + })?; + + Ok(serde_wasm_bindgen::to_value(&response).unwrap()) +} + +#[wasm_bindgen] +pub async fn upload_chunk( + session_id: JsValue, + upload_id: JsValue, + chunk: JsValue, + nuri: JsValue, +) -> Result { + //log_debug!("upload_chunk {:?}", js_nuri); + let session_id: u64 = serde_wasm_bindgen::from_value::(session_id) + .map_err(|_| "Deserialization error of session_id".to_string())?; + let upload_id: u32 = serde_wasm_bindgen::from_value::(upload_id) + .map_err(|_| "Deserialization error of upload_id".to_string())?; + let chunk: serde_bytes::ByteBuf = serde_wasm_bindgen::from_value::(chunk) + .map_err(|_| "Deserialization error of chunk".to_string())?; + let nuri: NuriV0 = serde_wasm_bindgen::from_value::(nuri) + .map_err(|_| "Deserialization error of nuri".to_string())?; + + let response = do_upload_chunk(session_id, upload_id, chunk, nuri).await?; + Ok(serde_wasm_bindgen::to_value(&response).unwrap()) } @@ -757,9 +1003,9 @@ struct HeadLessConfigStrings { #[cfg(wasmpack_target = "nodejs")] impl HeadLessConfigStrings { - fn load(js_config: JsValue) -> Result { - let string_config = if js_config.is_object() { - serde_wasm_bindgen::from_value::(js_config) + fn load(config: JsValue) -> Result { + let string_config = if config.is_object() { + serde_wasm_bindgen::from_value::(config) .map_err(|_| "Deserialization error of config object".to_string())? } else { HeadLessConfigStrings { @@ -837,10 +1083,10 @@ pub struct HeadlessConfig { #[cfg(wasmpack_target = "nodejs")] #[wasm_bindgen] -pub async fn init_headless(js_config: JsValue) -> Result<(), String> { +pub async fn init_headless(config: JsValue) -> Result<(), String> { //log_info!("{:?}", js_config); - let config = HeadLessConfigStrings::load(js_config)?; + let config = HeadLessConfigStrings::load(config)?; let _ = config .client_peer_key .as_ref() diff --git a/ng-verifier/src/request_processor.rs b/ng-verifier/src/request_processor.rs index f3da9d6..cc9ae33 100644 --- a/ng-verifier/src/request_processor.rs +++ b/ng-verifier/src/request_processor.rs @@ -86,9 +86,8 @@ impl Verifier { if res.is_err() { //log_info!("ERR={:?}", res.unwrap_err()); - let _ = tx - .send(AppResponse::V0(AppResponseV0::FileBinary(vec![]))) - .await; + let _ = tx.send(AppResponse::V0(AppResponseV0::EndOfStream)).await; + tx.close_channel(); break; } let res = res.unwrap();