headless verifier (without wallet) running in ngd and answers to query from nodejs

Cargo.lock

@@ -3189,6 +3189,7 @@ name = "ng-broker"
 version = "0.1.0"
 dependencies = [
  "async-std",
+ "async-trait",
  "async-tungstenite",
  "blake3",
  "default-net",
@@ -3199,6 +3200,7 @@ dependencies = [
  "ng-net",
  "ng-repo",
  "ng-storage-rocksdb",
+ "ng-verifier",
  "once_cell",
  "rust-embed",
  "serde",
@@ -3240,14 +3242,17 @@ dependencies = [
  "either",
  "futures",
  "getrandom 0.2.10",
+ "lazy_static",
  "ng-repo",
  "noise-protocol",
  "noise-rust-crypto",
  "once_cell",
+ "regex",
  "reqwest",
  "serde",
  "serde_bare",
  "serde_bytes",
+ "serde_json",
  "unique_id",
  "url",
  "web-time",

nextgraph/src/lib.rs

@@ -58,6 +58,9 @@ pub mod net {
 pub mod verifier {
     pub use ng_verifier::site::*;
     pub use ng_verifier::types::*;
+    pub mod protocol {
+        pub use ng_net::app_protocol::*;
+    }
 }
 
 pub mod wallet {

ng-app/src-tauri/src/lib.rs

@@ -20,7 +20,9 @@ use tauri::{path::BaseDirectory, App, Manager};
 use ng_repo::errors::NgError;
 use ng_repo::log::*;
 use ng_repo::types::*;
+use ng_repo::utils::decode_key;
 
+use ng_net::app_protocol::*;
 use ng_net::types::{ClientInfo, CreateAccountBSP, Invitation};
 use ng_net::utils::{decode_invitation_string, spawn_and_log_error, Receiver, ResultSend};
 
@@ -28,7 +30,6 @@ use ng_wallet::types::*;
 use ng_wallet::*;
 
 use nextgraph::local_broker::*;
-use nextgraph::verifier::*;
 
 #[cfg(mobile)]
 mod mobile;
@@ -183,11 +184,12 @@ async fn session_start(
     wallet_name: String,
     user: PubKey,
     _app: tauri::AppHandle,
-) -> Result<SessionInfo, String> {
+) -> Result<SessionInfoString, String> {
     let config = SessionConfig::new_save(&user, &wallet_name);
     nextgraph::local_broker::session_start(config)
         .await
         .map_err(|e: NgError| e.to_string())
+        .map(|s| s.into())
 }
 
 #[tauri::command(rename_all = "snake_case")]
@@ -196,11 +198,12 @@ async fn session_start_remote(
     user: PubKey,
     peer_id: Option<PubKey>,
     _app: tauri::AppHandle,
-) -> Result<SessionInfo, String> {
+) -> Result<SessionInfoString, String> {
     let config = SessionConfig::new_remote(&user, &wallet_name, peer_id);
     nextgraph::local_broker::session_start(config)
         .await
         .map_err(|e: NgError| e.to_string())
+        .map(|s| s.into())
 }
 
 #[tauri::command(rename_all = "snake_case")]
@@ -240,18 +243,17 @@ async fn decode_invitation(invite: String) -> Option<Invitation> {
 
 #[tauri::command(rename_all = "snake_case")]
 async fn app_request_stream(
-    session_id: u64,
     request: AppRequest,
     stream_id: &str,
     app: tauri::AppHandle,
 ) -> Result<(), String> {
-    log_debug!("app request stream {} {:?}", stream_id, request);
+    //log_debug!("app request stream {} {:?}", stream_id, request);
     let main_window = app.get_window("main").unwrap();
 
     let reader;
     {
         let cancel;
-        (reader, cancel) = nextgraph::local_broker::app_request_stream(session_id, request)
+        (reader, cancel) = nextgraph::local_broker::app_request_stream(request)
             .await
             .map_err(|e| e.to_string())?;
 
@@ -284,33 +286,29 @@ async fn app_request_stream(
 
 #[tauri::command(rename_all = "snake_case")]
 async fn doc_fetch_private_subscribe() -> Result<AppRequest, String> {
-    let request = AppRequest::V0(AppRequestV0 {
-        command: AppRequestCommandV0::Fetch(AppFetchContentV0::get_or_subscribe(true)),
-        nuri: NuriV0::new_private_store_target(),
-        payload: None,
-    });
+    let request = AppRequest::new(
+        AppRequestCommandV0::Fetch(AppFetchContentV0::get_or_subscribe(true)),
+        NuriV0::new_private_store_target(),
+        None,
+    );
     Ok(request)
 }
 
 #[tauri::command(rename_all = "snake_case")]
 async fn doc_fetch_repo_subscribe(repo_id: String) -> Result<AppRequest, String> {
-    let request = AppRequest::V0(AppRequestV0 {
-        command: AppRequestCommandV0::Fetch(AppFetchContentV0::get_or_subscribe(true)),
-        nuri: NuriV0::new_repo_target_from_string(repo_id).map_err(|e| e.to_string())?,
-        payload: None,
-    });
+    let request = AppRequest::new(
+        AppRequestCommandV0::Fetch(AppFetchContentV0::get_or_subscribe(true)),
+        NuriV0::new_repo_target_from_string(repo_id).map_err(|e| e.to_string())?,
+        None,
+    );
     Ok(request)
 }
 
 #[tauri::command(rename_all = "snake_case")]
-async fn app_request(
-    session_id: u64,
-    request: AppRequest,
-    _app: tauri::AppHandle,
-) -> Result<AppResponse, String> {
-    log_debug!("app request {:?}", request);
+async fn app_request(request: AppRequest, _app: tauri::AppHandle) -> Result<AppResponse, String> {
+    //log_debug!("app request {:?}", request);
 
-    nextgraph::local_broker::app_request(session_id, request)
+    nextgraph::local_broker::app_request(request)
         .await
         .map_err(|e| e.to_string())
 }
@@ -325,22 +323,23 @@ async fn upload_chunk(
 ) -> Result<AppResponse, String> {
     //log_debug!("upload_chunk {:?}", chunk);
 
-    let request = AppRequest::V0(AppRequestV0 {
-        command: AppRequestCommandV0::FilePut,
+    let mut request = AppRequest::new(
+        AppRequestCommandV0::FilePut,
         nuri,
-        payload: Some(AppRequestPayload::V0(
+        Some(AppRequestPayload::V0(
             AppRequestPayloadV0::RandomAccessFilePutChunk((upload_id, chunk)),
         )),
-    });
+    );
+    request.set_session_id(session_id);
 
-    nextgraph::local_broker::app_request(session_id, request)
+    nextgraph::local_broker::app_request(request)
         .await
         .map_err(|e| e.to_string())
 }
 
 #[tauri::command(rename_all = "snake_case")]
 async fn cancel_stream(stream_id: &str) -> Result<(), String> {
-    log_debug!("cancel stream {}", stream_id);
+    //log_debug!("cancel stream {}", stream_id);
     Ok(
         nextgraph::local_broker::tauri_stream_cancel(stream_id.to_string())
             .await
@@ -381,14 +380,16 @@ async fn disconnections_subscribe(app: tauri::AppHandle) -> Result<(), String> {
 }
 
 #[tauri::command(rename_all = "snake_case")]
-async fn session_stop(user_id: UserId) -> Result<(), String> {
+async fn session_stop(user_id: String) -> Result<(), String> {
+    let user_id = decode_key(&user_id).map_err(|_| "Invalid user_id")?;
     nextgraph::local_broker::session_stop(&user_id)
         .await
         .map_err(|e: NgError| e.to_string())
 }
 
 #[tauri::command(rename_all = "snake_case")]
-async fn user_disconnect(user_id: UserId) -> Result<(), String> {
+async fn user_disconnect(user_id: String) -> Result<(), String> {
+    let user_id = decode_key(&user_id).map_err(|_| "Invalid user_id")?;
     nextgraph::local_broker::user_disconnect(&user_id)
         .await
         .map_err(|e: NgError| e.to_string())
@@ -412,9 +413,10 @@ struct ConnectionInfo {
 #[tauri::command(rename_all = "snake_case")]
 async fn user_connect(
     info: ClientInfo,
-    user_id: UserId,
+    user_id: String,
     _location: Option<String>,
 ) -> Result<HashMap<String, ConnectionInfo>, String> {
+    let user_id = decode_key(&user_id).map_err(|_| "Invalid user_id")?;
     let mut opened_connections: HashMap<String, ConnectionInfo> = HashMap::new();
 
     let results = nextgraph::local_broker::user_connect_with_device_info(info, &user_id, None)

ng-app/src/api.ts

@@ -31,7 +31,7 @@ const mapping = {
     "decode_invitation": ["invite"],
     "user_connect": ["info","user_id","location"],
     "user_disconnect": ["user_id"],
-    "app_request": ["session_id","request"],
+    "app_request": ["request"],
     "test": [ ],
     "doc_fetch_private_subscribe": [],
     "doc_fetch_repo_subscribe": ["repo_id"],
@@ -120,9 +120,9 @@ const handler = {
                 let stream_id = (lastStreamId += 1).toString();
                 //console.log("stream_id",stream_id);
                 let { getCurrent } = await import("@tauri-apps/plugin-window");
-                let session_id = args[0];
-                let request = args[1];
-                let callback = args[2];
+                //let session_id = args[0];
+                let request = args[0];
+                let callback = args[1];
 
                 let unlisten = await getCurrent().listen(stream_id, (event) => {
                     //console.log(event.payload);
@@ -131,7 +131,7 @@ const handler = {
                     }
                     callback(event.payload).then(()=> {})
                 })
-                await tauri.invoke("app_request_stream",{session_id, stream_id, request});
+                await tauri.invoke("app_request_stream",{stream_id, request});
                 
                 return () => {
                     unlisten();

ng-app/src/lib/Test.svelte

@@ -49,15 +49,13 @@
           V0: {
             command: "FileGet",
             nuri,
+            session_id: $active_session.session_id,
           },
         };
 
         let final_blob;
         let content_type;
-        let unsub = await ng.app_request_stream(
-          $active_session.session_id,
-          file_request,
-          async (blob) => {
+        let unsub = await ng.app_request_stream(file_request, async (blob) => {
           //console.log("GOT APP RESPONSE", blob);
           if (blob.V0.FileMeta) {
             content_type = blob.V0.FileMeta.content_type;
@@ -73,8 +71,7 @@
               resolve(imageUrl);
             }
           }
-          }
-        );
+        });
       } catch (e) {
         console.error(e);
         resolve(false);
@@ -166,13 +163,11 @@
             RandomAccessFilePut: image.type,
           },
         },
+        session_id: $active_session.session_id,
       },
     };
 
-    let start_res = await ng.app_request(
-      $active_session.session_id,
-      start_request
-    );
+    let start_res = await ng.app_request(start_request);
     let upload_id = start_res.V0.FileUploading;
 
     uploadFile(upload_id, nuri, image, async (reference) => {
@@ -189,10 +184,11 @@
                 },
               },
             },
+            session_id: $active_session.session_id,
           },
         };
 
-        await ng.app_request($active_session.session_id, request);
+        await ng.app_request(request);
       }
     });
     fileinput.value = "";

ng-app/src/routes/User.svelte

@@ -322,7 +322,7 @@
                   } else {
                     $connections[personal_site].error = "Stopped";
                     personal_site_status.since = new Date();
-                    await ng.user_disconnect(personal_site_id);
+                    await ng.user_disconnect(personal_site);
                   }
                 }}
                 checked={personal_site_status &&

ng-app/src/store.ts

@@ -117,7 +117,7 @@ export const reconnect = async function() {
     console.log("attempting to connect...");
     try {
         let info = await ng.client_info()
-        //console.log("Connecting with",client,info);
+        //console.log("Connecting with",get(active_session).user);
         connections.set(await ng.user_connect( 
             info,
             get(active_session).user,
@@ -235,7 +235,9 @@ export const branch_subs = function(nuri) {
                             unsub();
                             unsub = () => {};
                             set([]);
-                            unsub = await ng.app_request_stream(session.session_id, await ng.doc_fetch_repo_subscribe(nuri), 
+                            let req= await ng.doc_fetch_repo_subscribe(nuri);
+                            req.V0.session_id = session.session_id;
+                            unsub = await ng.app_request_stream(req, 
                             async (commit) => {
                                 //console.log("GOT APP RESPONSE", commit);
                                 update( (old) => {old.unshift(commit); return old;} )

ng-broker/Cargo.toml

@@ -23,12 +23,14 @@ futures = "0.3.24"
 once_cell = "1.17.1"
 either = { version = "1.8.1", features=["serde"] }
 async-std = {  version = "1.12.0", features = ["attributes"] }
+async-trait = "0.1.64"
 rust-embed= { version = "6.7.0", features=["include-exclude"] }
 async-tungstenite = {  git = "https://git.nextgraph.org/NextGraph/async-tungstenite.git", branch = "nextgraph", features = ["async-std-runtime"] }
 blake3 = "1.3.1"
 ng-repo = { path = "../ng-repo", version = "0.1.0" }
 ng-net = { path = "../ng-net", version = "0.1.0" }
 ng-client-ws = { path = "../ng-client-ws", version = "0.1.0" }
+ng-verifier = { path = "../ng-verifier", version = "0.1.0" }
 ng-storage-rocksdb = { path = "../ng-storage-rocksdb", version = "0.1.0" }
 
 [target.'cfg(target_arch = "wasm32")'.dependencies.getrandom]

ng-broker/src/rocksdb_server_storage.rs

@@ -160,6 +160,12 @@ impl RocksDbServerStorage {
         })
     }
 
+    pub(crate) fn get_block_storage(
+        &self,
+    ) -> Arc<std::sync::RwLock<dyn BlockStorage + Send + Sync>> {
+        Arc::clone(&self.block_storage)
+    }
+
     pub(crate) fn next_seq_for_peer(&self, peer: &PeerId, seq: u64) -> Result<(), ServerError> {
         // for now we don't use the hashmap.
         // TODO: let's see if the lock is even needed
@@ -194,15 +200,38 @@ impl RocksDbServerStorage {
         log_debug!("get_user {user_id}");
         Ok(Account::open(&user_id, &self.accounts_storage)?.is_admin()?)
     }
+    /// returns the crednetials, storage_master_key, and peer_priv_key
+    pub(crate) fn get_user_credentials(
+        &self,
+        user_id: &PubKey,
+    ) -> Result<Credentials, ProtocolError> {
+        log_debug!("get_user_credentials {user_id}");
+        let acc = Account::open(user_id, &self.accounts_storage)?;
+        Ok(acc.get_credentials()?)
+    }
     pub(crate) fn add_user(&self, user_id: PubKey, is_admin: bool) -> Result<(), ProtocolError> {
         log_debug!("add_user {user_id} is admin {is_admin}");
         Account::create(&user_id, is_admin, &self.accounts_storage)?;
         Ok(())
     }
+    pub(crate) fn add_user_credentials(
+        &self,
+        user_id: &PubKey,
+        credentials: &Credentials,
+    ) -> Result<(), ProtocolError> {
+        log_debug!("add_user_credentials {user_id}");
+        let acc = Account::create(&user_id, false, &self.accounts_storage)?;
+        acc.add_credentials(credentials)?;
+        //let storage_key = SymKey::random();
+        //let peer_priv_key = PrivKey::random_ed();
+        //acc.add_user_keys(&storage_key, &peer_priv_key)?;
+        Ok(())
+    }
     pub(crate) fn del_user(&self, user_id: PubKey) -> Result<(), ProtocolError> {
         log_debug!("del_user {user_id}");
         let acc = Account::open(&user_id, &self.accounts_storage)?;
         acc.del()?;
+        // TODO: stop the verifier, if any
         Ok(())
     }
     pub(crate) fn list_users(&self, admins: bool) -> Result<Vec<PubKey>, ProtocolError> {

ng-broker/src/server_broker.rs

@@ -11,18 +11,37 @@
 
 //! Implementation of the Server Broker
 
-use std::collections::{HashMap, HashSet};
+use std::{
+    collections::{HashMap, HashSet},
+    path::PathBuf,
+    sync::Arc,
+};
 
+use async_std::sync::{Mutex, RwLock};
 use either::Either;
+use futures::StreamExt;
 use serde::{Deserialize, Serialize};
 
 use ng_repo::{
+    block_storage::BlockStorage,
     errors::{NgError, ProtocolError, ServerError},
     log::*,
     types::*,
 };
 
-use ng_net::{server_broker::IServerBroker, types::*};
+use ng_net::{
+    app_protocol::*,
+    broker::{ClientPeerId, BROKER},
+    connection::NoiseFSM,
+    server_broker::IServerBroker,
+    types::*,
+};
+
+use ng_verifier::{
+    site::SiteV0,
+    types::{BrokerPeerId, VerifierConfig, VerifierConfigType},
+    verifier::Verifier,
+};
 
 use crate::rocksdb_server_storage::RocksDbServerStorage;
 
@@ -104,31 +123,53 @@ impl From<OverlayAccess> for OverlayType {
     }
 }
 
-pub struct OverlayInfo {
+#[allow(dead_code)]
+pub(crate) struct OverlayInfo {
     pub overlay_type: OverlayType,
     pub overlay_topic: Option<TopicId>,
     pub topics: HashMap<TopicId, TopicInfo>,
     pub repos: HashMap<RepoHash, RepoInfo>,
 }
 
-pub struct ServerBroker {
-    storage: RocksDbServerStorage,
+struct DetachableVerifier {
+    detach: bool,
+    attached: Option<(DirectPeerId, u64)>,
+    verifier: Verifier,
+}
 
+pub struct ServerBrokerState {
     #[allow(dead_code)]
     overlays: HashMap<OverlayId, OverlayInfo>,
     #[allow(dead_code)]
     inner_overlays: HashMap<OverlayId, Option<OverlayId>>,
 
-    local_subscriptions: HashMap<(OverlayId, TopicId), HashSet<PubKey>>,
+    local_subscriptions: HashMap<(OverlayId, TopicId), HashMap<PubKey, Option<UserId>>>,
+
+    verifiers: HashMap<UserId, Arc<RwLock<DetachableVerifier>>>,
+    remote_apps: HashMap<(DirectPeerId, u64), UserId>,
+}
+
+pub struct ServerBroker {
+    storage: RocksDbServerStorage,
+
+    state: RwLock<ServerBrokerState>,
+
+    path_users: PathBuf,
 }
 
 impl ServerBroker {
-    pub(crate) fn new(storage: RocksDbServerStorage) -> Self {
+    pub(crate) fn new(storage: RocksDbServerStorage, path_users: PathBuf) -> Self {
         ServerBroker {
             storage: storage,
+            state: RwLock::new(ServerBrokerState {
                 overlays: HashMap::new(),
                 inner_overlays: HashMap::new(),
                 local_subscriptions: HashMap::new(),
+                verifiers: HashMap::new(),
+                remote_apps: HashMap::new(),
+            }),
+
+            path_users,
         }
     }
 
@@ -136,53 +177,116 @@ impl ServerBroker {
         Ok(())
     }
 
-    fn add_subscription(
-        &mut self,
+    async fn add_subscription(
+        &self,
         overlay: OverlayId,
         topic: TopicId,
-        peer: PubKey,
+        peer: ClientPeerId,
     ) -> Result<(), ServerError> {
-        let peers_set = self
+        let mut lock = self.state.write().await;
+        let peers_map = lock
             .local_subscriptions
             .entry((overlay, topic))
-            .or_insert(HashSet::with_capacity(1));
+            .or_insert(HashMap::with_capacity(1));
 
         log_debug!(
-            "SUBSCRIBING PEER {} TOPIC {} OVERLAY {}",
+            "SUBSCRIBING PEER {:?} TOPIC {} OVERLAY {}",
             peer,
             topic,
             overlay
         );
 
-        if !peers_set.insert(peer) {
+        if peers_map.insert(*peer.key(), peer.value()).is_some() {
             //return Err(ServerError::PeerAlreadySubscribed);
         }
         Ok(())
     }
 
     #[allow(dead_code)]
-    fn remove_subscription(
-        &mut self,
+    async fn remove_subscription(
+        &self,
         overlay: &OverlayId,
         topic: &TopicId,
         peer: &PubKey,
     ) -> Result<(), ServerError> {
-        let peers_set = self
+        let mut lock = self.state.write().await;
+        let peers_set = lock
             .local_subscriptions
             .get_mut(&(*overlay, *topic))
             .ok_or(ServerError::SubscriptionNotFound)?;
 
-        if !peers_set.remove(peer) {
+        if peers_set.remove(peer).is_none() {
             return Err(ServerError::SubscriptionNotFound);
         }
         Ok(())
     }
+
+    async fn new_verifier_from_credentials(
+        &self,
+        user_id: &UserId,
+        credentials: Credentials,
+        local_peer_id: DirectPeerId,
+        partial_credentials: bool,
+    ) -> Result<Verifier, NgError> {
+        let block_storage = self.get_block_storage();
+        let mut path = self.get_path_users();
+        let user_hash: Digest = user_id.into();
+        path.push(user_hash.to_string());
+        std::fs::create_dir_all(path.clone()).unwrap();
+        let peer_id_dh = credentials.peer_priv_key.to_pub().to_dh_from_ed();
+        let mut verifier = Verifier::new(
+            VerifierConfig {
+                config_type: VerifierConfigType::RocksDb(path),
+                user_master_key: *credentials.user_master_key.slice(),
+                peer_priv_key: credentials.peer_priv_key,
+                user_priv_key: credentials.user_key,
+                private_store_read_cap: if partial_credentials {
+                    None
+                } else {
+                    Some(credentials.read_cap)
+                },
+                private_store_id: if partial_credentials {
+                    None
+                } else {
+                    Some(credentials.private_store)
+                },
+                protected_store_id: if partial_credentials {
+                    None
+                } else {
+                    Some(credentials.protected_store)
+                },
+                public_store_id: if partial_credentials {
+                    None
+                } else {
+                    Some(credentials.public_store)
+                },
+            },
+            block_storage,
+        )?;
+        if !partial_credentials {
+            verifier.connected_broker = BrokerPeerId::Local(local_peer_id);
+            // start the local transport connection
+            let mut lock = BROKER.write().await;
+            lock.connect_local(peer_id_dh, *user_id)?;
+        }
+        Ok(verifier)
+    }
 }
 
 //TODO: the purpose of this trait is to have a level of indirection so we can keep some data in memory (cache) and avoid hitting the storage backend (rocksdb) at every call.
 //for now this cache is not implemented, but the structs are ready (see above), and it would just require to change slightly the implementation of the trait functions here below.
-
+#[async_trait::async_trait]
 impl IServerBroker for ServerBroker {
+    fn get_block_storage(
+        &self,
+    ) -> std::sync::Arc<std::sync::RwLock<dyn BlockStorage + Send + Sync>> {
+        self.storage.get_block_storage()
+    }
+
+    fn get_path_users(&self) -> PathBuf {
+        self.path_users.clone()
+    }
+
     fn has_block(&self, overlay_id: &OverlayId, block_id: &BlockId) -> Result<(), ServerError> {
         self.storage.has_block(overlay_id, block_id)
     }
@@ -199,13 +303,59 @@ impl IServerBroker for ServerBroker {
         self.storage.add_block(overlay_id, block)?;
         Ok(())
     }
+    async fn create_user(&self, broker_id: &DirectPeerId) -> Result<UserId, ProtocolError> {
+        let user_privkey = PrivKey::random_ed();
+        let user_id = user_privkey.to_pub();
+        let mut creds = Credentials::new_partial(&user_privkey);
+        let mut verifier = self
+            .new_verifier_from_credentials(&user_id, creds.clone(), *broker_id, true)
+            .await?;
+        let _site = SiteV0::create_personal(user_privkey.clone(), &mut verifier)
+            .await
+            .map_err(|e| {
+                log_err!("create_personal failed with {e}");
+                ProtocolError::BrokerError
+            })?;
+
+        // update credentials from config of verifier.
+        verifier.complement_credentials(&mut creds);
+        //verifier.close().await;
+        // save credentials and user
+        self.add_user_credentials(&user_id, &creds)?;
+
+        verifier.connected_broker = BrokerPeerId::Local(*broker_id);
+
+        // start the local transport connection
+        {
+            let mut lock = BROKER.write().await;
+            let peer_id_dh = creds.peer_priv_key.to_pub().to_dh_from_ed();
+            lock.connect_local(peer_id_dh, user_id)?;
+        }
+        let _res = verifier.send_outbox().await;
+        if _res.is_err() {
+            log_err!("{:?}", _res);
+        }
+
+        Ok(user_id)
+    }
 
     fn get_user(&self, user_id: PubKey) -> Result<bool, ProtocolError> {
         self.storage.get_user(user_id)
     }
+    fn add_user_credentials(
+        &self,
+        user_id: &PubKey,
+        credentials: &Credentials,
+    ) -> Result<(), ProtocolError> {
+        self.storage.add_user_credentials(user_id, credentials)
+    }
+    fn get_user_credentials(&self, user_id: &PubKey) -> Result<Credentials, ProtocolError> {
+        self.storage.get_user_credentials(user_id)
+    }
     fn add_user(&self, user_id: PubKey, is_admin: bool) -> Result<(), ProtocolError> {
         self.storage.add_user(user_id, is_admin)
     }
+
     fn del_user(&self, user_id: PubKey) -> Result<(), ProtocolError> {
         self.storage.del_user(user_id)
     }
@@ -234,6 +384,201 @@ impl IServerBroker for ServerBroker {
     fn remove_invitation(&self, invite_code: [u8; 32]) -> Result<(), ProtocolError> {
         self.storage.remove_invitation(invite_code)
     }
+
+    async fn app_process_request(
+        &self,
+        req: AppRequest,
+        request_id: i64,
+        fsm: &Mutex<NoiseFSM>,
+    ) -> Result<(), ServerError> {
+        // get the session
+        let remote = {
+            fsm.lock()
+                .await
+                .remote_peer()
+                .ok_or(ServerError::SessionNotFound)?
+        };
+
+        let session_id = (remote, req.session_id());
+        let session_lock = {
+            let lock = self.state.read().await;
+            let user_id = lock
+                .remote_apps
+                .get(&session_id)
+                .ok_or(ServerError::SessionNotFound)?
+                .to_owned();
+
+            Arc::clone(
+                lock.verifiers
+                    .get(&user_id)
+                    .ok_or(ServerError::SessionNotFound)?,
+            )
+        };
+
+        let mut session = session_lock.write().await;
+
+        if session.attached.is_none() || session.attached.unwrap() != session_id {
+            return Err(ServerError::SessionDetached);
+        }
+
+        if req.command().is_stream() {
+            let res = session.verifier.app_request_stream(req).await;
+
+            match res {
+                Err(e) => {
+                    let error: ServerError = e.into();
+                    let error_res: AppMessage = error.into();
+                    fsm.lock()
+                        .await
+                        .send_in_reply_to(error_res.into(), request_id)
+                        .await?;
+                }
+                Ok((mut receiver, _cancel)) => {
+                    //TODO: implement cancel
+                    let mut some_sent = false;
+                    while let Some(response) = receiver.next().await {
+                        some_sent = true;
+                        let mut msg: AppMessage = response.into();
+                        msg.set_result(ServerError::PartialContent.into());
+                        fsm.lock()
+                            .await
+                            .send_in_reply_to(msg.into(), request_id)
+                            .await?;
+                    }
+                    let end: Result<EmptyAppResponse, ServerError> = if some_sent {
+                        Err(ServerError::EndOfStream)
+                    } else {
+                        Err(ServerError::EmptyStream)
+                    };
+                    fsm.lock()
+                        .await
+                        .send_in_reply_to(end.into(), request_id)
+                        .await?;
+                }
+            }
+        } else {
+            let res = session
+                .verifier
+                .app_request(req)
+                .await
+                .map_err(|e| e.into());
+
+            fsm.lock()
+                .await
+                .send_in_reply_to(res.into(), request_id)
+                .await?;
+        }
+
+        Ok(())
+    }
+
+    async fn app_session_start(
+        &self,
+        req: AppSessionStart,
+        remote: DirectPeerId,
+        local_peer_id: DirectPeerId,
+    ) -> Result<AppSessionStartResponse, ServerError> {
+        let user_id = req.user_id();
+        let id = (remote, req.session_id());
+        let verifier_lock_res = {
+            let lock = self.state.read().await;
+            lock.verifiers.get(user_id).map(|l| Arc::clone(l))
+        };
+        let verifier_lock = match verifier_lock_res {
+            Some(session_lock) => {
+                let mut session = session_lock.write().await;
+                if let Some((peer_id, session_id)) = session.attached {
+                    if peer_id != remote || session_id == req.session_id() {
+                        // remove the previous session
+                        let mut write_lock = self.state.write().await;
+                        let _ = write_lock.remote_apps.remove(&(peer_id, session_id));
+                    }
+                }
+                session.attached = Some(id);
+                Arc::clone(&session_lock)
+            }
+            None => {
+                // we create and load a new verifier
+
+                let credentials = if req.credentials().is_none() {
+                    // headless do not have credentials. we fetch them from server_storage
+                    self.storage.get_user_credentials(user_id)?
+                } else {
+                    req.credentials().clone().unwrap()
+                };
+
+                if *user_id != credentials.user_key.to_pub() {
+                    log_debug!("InvalidRequest");
+                    return Err(ServerError::InvalidRequest);
+                }
+
+                let verifier = self
+                    .new_verifier_from_credentials(user_id, credentials, local_peer_id, false)
+                    .await;
+                if verifier.is_err() {
+                    log_err!(
+                        "new_verifier failed with: {:?}",
+                        verifier.as_ref().unwrap_err()
+                    );
+                }
+                let mut verifier = verifier?;
+
+                // TODO : key.zeroize();
+
+                //load verifier from local_storage
+                let _ = verifier.load();
+                //TODO: save opened_branches in user_storage, so that when we open again the verifier, the syncing can work
+                verifier.sync().await;
+
+                let session = DetachableVerifier {
+                    detach: true,
+                    attached: Some(id),
+                    verifier,
+                };
+                let mut write_lock = self.state.write().await;
+                Arc::clone(
+                    write_lock
+                        .verifiers
+                        .entry(*user_id)
+                        .or_insert(Arc::new(RwLock::new(session))),
+                )
+            }
+        };
+        let verifier = &verifier_lock.read().await.verifier;
+        let res = AppSessionStartResponse::V0(AppSessionStartResponseV0 {
+            private_store: *verifier.private_store_id(),
+            protected_store: *verifier.protected_store_id(),
+            public_store: *verifier.public_store_id(),
+        });
+        let mut write_lock = self.state.write().await;
+        if let Some(previous_user) = write_lock.remote_apps.insert(id, *user_id) {
+            // weird. another session was opened for this id.
+            // we have to stop it otherwise it would be dangling.
+            if previous_user != *user_id {
+                if let Some(previous_session_lock) = write_lock
+                    .verifiers
+                    .get(&previous_user)
+                    .map(|v| Arc::clone(v))
+                {
+                    let mut previous_session = previous_session_lock.write().await;
+                    if previous_session.detach {
+                        previous_session.attached = None;
+                    } else {
+                        // we stop it and drop it
+                        let verifier = write_lock.verifiers.remove(&previous_user);
+                        verifier.unwrap().read().await.verifier.close().await;
+                    }
+                }
+            }
+        }
+        Ok(res)
+    }
+
+    fn app_session_stop(&self, _req: AppSessionStop) -> Result<EmptyAppResponse, ServerError> {
+        //TODO
+        Ok(EmptyAppResponse(()))
+    }
+
     fn get_repo_pin_status(
         &self,
         overlay: &OverlayId,
@@ -243,8 +588,8 @@ impl IServerBroker for ServerBroker {
         self.storage.get_repo_pin_status(overlay, repo, user)
     }
 
-    fn pin_repo_write(
-        &mut self,
+    async fn pin_repo_write(
+        &self,
         overlay: &OverlayAccess,
         repo: &RepoHash,
         user_id: &UserId,
@@ -252,7 +597,7 @@ impl IServerBroker for ServerBroker {
         rw_topics: &Vec<PublisherAdvert>,
         overlay_root_topic: &Option<TopicId>,
         expose_outer: bool,
-        peer: &PubKey,
+        peer: &ClientPeerId,
     ) -> Result<RepoOpened, ServerError> {
         let res = self.storage.pin_repo_write(
             overlay,
@@ -263,23 +608,25 @@ impl IServerBroker for ServerBroker {
             overlay_root_topic,
             expose_outer,
         )?;
+
         for topic in res.iter() {
             self.add_subscription(
                 *overlay.overlay_id_for_client_protocol_purpose(),
                 *topic.topic_id(),
-                *peer,
-            )?;
+                peer.clone(),
+            )
+            .await?;
         }
         Ok(res)
     }
 
-    fn pin_repo_read(
-        &mut self,
+    async fn pin_repo_read(
+        &self,
         overlay: &OverlayId,
         repo: &RepoHash,
         user_id: &UserId,
         ro_topics: &Vec<TopicId>,
-        peer: &PubKey,
+        peer: &ClientPeerId,
     ) -> Result<RepoOpened, ServerError> {
         let res = self
             .storage
@@ -287,24 +634,26 @@ impl IServerBroker for ServerBroker {
 
         for topic in res.iter() {
             // TODO: those outer subscriptions are not handled yet. they will not emit events.
-            self.add_subscription(*overlay, *topic.topic_id(), *peer)?;
+            self.add_subscription(*overlay, *topic.topic_id(), peer.clone())
+                .await?;
         }
         Ok(res)
     }
 
-    fn topic_sub(
-        &mut self,
+    async fn topic_sub(
+        &self,
         overlay: &OverlayId,
         repo: &RepoHash,
         topic: &TopicId,
         user: &UserId,
         publisher: Option<&PublisherAdvert>,
-        peer: &PubKey,
+        peer: &ClientPeerId,
     ) -> Result<TopicSubRes, ServerError> {
         let res = self
             .storage
             .topic_sub(overlay, repo, topic, user, publisher)?;
-        self.add_subscription(*overlay, *topic, *peer)?;
+        self.add_subscription(*overlay, *topic, peer.clone())
+            .await?;
         Ok(res)
     }
 
@@ -312,9 +661,11 @@ impl IServerBroker for ServerBroker {
         self.storage.get_commit(overlay, id)
     }
 
-    fn remove_all_subscriptions_of_peer(&mut self, remote_peer: &PubKey) {
-        for ((overlay, topic), peers) in self.local_subscriptions.iter_mut() {
-            if peers.remove(remote_peer) {
+    async fn remove_all_subscriptions_of_client(&self, client: &ClientPeerId) {
+        let remote_peer = client.key();
+        let mut lock = self.state.write().await;
+        for ((overlay, topic), peers) in lock.local_subscriptions.iter_mut() {
+            if peers.remove(remote_peer).is_some() {
                 log_debug!(
                     "subscription of peer {} to topic {} in overlay {} removed",
                     remote_peer,
@@ -325,13 +676,13 @@ impl IServerBroker for ServerBroker {
         }
     }
 
-    fn dispatch_event(
+    async fn dispatch_event(
         &self,
         overlay: &OverlayId,
         event: Event,
         user_id: &UserId,
         remote_peer: &PubKey,
-    ) -> Result<HashSet<&PubKey>, ServerError> {
+    ) -> Result<Vec<ClientPeerId>, ServerError> {
         let topic = self.storage.save_event(overlay, event, user_id)?;
 
         // log_debug!(
@@ -340,15 +691,18 @@ impl IServerBroker for ServerBroker {
         //     topic,
         //     self.local_subscriptions
         // );
-
-        let mut set = self
+        let lock = self.state.read().await;
+        let mut map = lock
             .local_subscriptions
             .get(&(*overlay, topic))
-            .map(|set| set.iter().collect())
-            .unwrap_or(HashSet::new());
-
-        set.remove(remote_peer);
-        Ok(set)
+            .map(|map| map.iter().collect())
+            .unwrap_or(HashMap::new());
+
+        map.remove(remote_peer);
+        Ok(map
+            .iter()
+            .map(|(k, v)| ClientPeerId::new_from(k, v))
+            .collect())
     }
 
     fn topic_sync_req(

ng-broker/src/server_storage/admin/account.rs

@@ -10,6 +10,7 @@
 //! User account Storage (Object Key/Col/Value Mapping)
 
 use std::collections::hash_map::DefaultHasher;
+use std::fmt;
 use std::hash::Hash;
 use std::hash::Hasher;
 use std::time::SystemTime;
@@ -30,6 +31,12 @@ pub struct Account<'a> {
     storage: &'a dyn KCVStorage,
 }
 
+impl<'a> fmt::Debug for Account<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "Account {}", self.id)
+    }
+}
+
 impl<'a> Account<'a> {
     const PREFIX_ACCOUNT: u8 = b'a';
     const PREFIX_CLIENT: u8 = b'c';
@@ -38,8 +45,15 @@ impl<'a> Account<'a> {
     // propertie's client suffixes
     const INFO: u8 = b'i';
     const LAST_SEEN: u8 = b'l';
+    const CREDENTIALS: u8 = b'c';
+    //const USER_KEYS: u8 = b'k';
 
-    const ALL_CLIENT_PROPERTIES: [u8; 2] = [Self::INFO, Self::LAST_SEEN];
+    const ALL_CLIENT_PROPERTIES: [u8; 3] = [
+        Self::INFO,
+        Self::LAST_SEEN,
+        Self::CREDENTIALS,
+        //Self::USER_KEYS,
+    ];
 
     pub fn open(id: &UserId, storage: &'a dyn KCVStorage) -> Result<Account<'a>, StorageError> {
         let opening = Account {
@@ -160,26 +174,72 @@ impl<'a> Account<'a> {
         })
     }
 
-    // pub fn has_client(&self, client: &ClientId) -> Result<(), StorageError> {
-    //     self.storage.has_property_value(
-    //         Self::PREFIX,
-    //         &to_vec(&self.id)?,
-    //         Some(Self::CLIENT),
-    //         to_vec(client)?,
-    //     )
-    // }
+    pub fn add_credentials(&self, credentials: &Credentials) -> Result<(), StorageError> {
+        if !self.exists() {
+            return Err(StorageError::BackendError);
+        }
+        self.storage.put(
+            Self::PREFIX_ACCOUNT,
+            &to_vec(&self.id)?,
+            Some(Self::CREDENTIALS),
+            &to_vec(credentials)?,
+            &None,
+        )
+    }
+
+    pub fn remove_credentials(&self) -> Result<(), StorageError> {
+        self.storage.del(
+            Self::PREFIX_ACCOUNT,
+            &to_vec(&self.id)?,
+            Some(Self::CREDENTIALS),
+            &None,
+        )
+    }
+
+    pub fn get_credentials(&self) -> Result<Credentials, StorageError> {
+        Ok(from_slice(&self.storage.get(
+            Self::PREFIX_ACCOUNT,
+            &to_vec(&self.id)?,
+            Some(Self::CREDENTIALS),
+            &None,
+        )?)?)
+    }
 
-    // pub fn add_overlay(&self, overlay: &OverlayId) -> Result<(), StorageError> {
+    // pub fn add_user_keys(
+    //     &self,
+    //     storage_key: &SymKey,
+    //     peer_priv_key: &PrivKey,
+    // ) -> Result<(), StorageError> {
     //     if !self.exists() {
     //         return Err(StorageError::BackendError);
     //     }
     //     self.storage.put(
-    //         Self::PREFIX,
+    //         Self::PREFIX_ACCOUNT,
     //         &to_vec(&self.id)?,
-    //         Some(Self::OVERLAY),
-    //         to_vec(overlay)?,
+    //         Some(Self::USER_KEYS),
+    //         &to_vec(&(storage_key.clone(), peer_priv_key.clone()))?,
+    //         &None,
+    //     )
+    // }
+
+    // pub fn remove_user_keys(&self) -> Result<(), StorageError> {
+    //     self.storage.del(
+    //         Self::PREFIX_ACCOUNT,
+    //         &to_vec(&self.id)?,
+    //         Some(Self::USER_KEYS),
+    //         &None,
     //     )
     // }
+
+    // pub fn get_user_keys(&self) -> Result<(SymKey, PrivKey), StorageError> {
+    //     Ok(from_slice(&self.storage.get(
+    //         Self::PREFIX_ACCOUNT,
+    //         &to_vec(&self.id)?,
+    //         Some(Self::USER_KEYS),
+    //         &None,
+    //     )?)?)
+    // }
+
     // pub fn remove_overlay(&self, overlay: &OverlayId) -> Result<(), StorageError> {
     //     self.storage.del_property_value(
     //         Self::PREFIX,

ng-broker/src/server_storage/core/overlay.rs

@@ -70,7 +70,11 @@ impl<'a> OverlayStorage<'a> {
         }
     }
 
-    pub fn load(id: &OverlayId, storage: &'a dyn KCVStorage) -> Result<OverlayInfo, StorageError> {
+    #[allow(dead_code)]
+    pub(crate) fn load(
+        id: &OverlayId,
+        storage: &'a dyn KCVStorage,
+    ) -> Result<OverlayInfo, StorageError> {
         let mut opening = OverlayStorage::new(id, storage);
         let props = opening.load_props()?;
         let existential = col(&Self::TYPE, &props)?;

ng-broker/src/server_ws.rs

@@ -764,8 +764,11 @@ pub async fn run_server_v0(
     // and we need those infos for permission checking.
     {
         //let root = tempfile::Builder::new().prefix("ngd").tempdir().unwrap();
+        let mut path_users = path.clone();
+        path_users.push("users");
         path.push("storage");
         std::fs::create_dir_all(path.clone()).unwrap();
+        std::fs::create_dir_all(path_users.clone()).unwrap();
 
         // opening the server storage (that contains the encryption keys for each store/overlay )
         let server_storage = RocksDbServerStorage::open(
@@ -781,7 +784,7 @@ pub async fn run_server_v0(
             NgError::BrokerConfigError(format!("Error while opening server storage: {}", e))
         })?;
 
-        let server_broker = ServerBroker::new(server_storage);
+        let server_broker = ServerBroker::new(server_storage, path_users);
 
         let mut broker = BROKER.write().await;
         broker.set_server_broker(server_broker);

ng-net/Cargo.toml

@@ -20,6 +20,8 @@ maintenance = { status = "actively-developed" }
 serde = { version = "1.0", features = ["derive"] }
 serde_bare = "0.5.0"
 serde_bytes = "0.11.7"
+serde_json = "1.0"
+lazy_static = "1.4.0"
 once_cell = "1.17.1"
 either = "1.8.1"
 futures = "0.3.24"
@@ -31,6 +33,7 @@ noise-protocol = "0.2.0-rc1"
 noise-rust-crypto = "0.6.0-rc.1"
 ed25519-dalek = "1.0.1"
 url = "2.4.0"
+regex = "1.8.4"
 base64-url = "2.0.0"
 web-time = "0.2.0"
 ng-repo = { path = "../ng-repo", version = "0.1.0" }

ng-net/src/actor.rs

@@ -127,7 +127,7 @@ impl<
         let mut receiver = self.receiver.take().unwrap();
         match receiver.next().await {
             Some(ConnectionCommand::Msg(msg)) => {
-                if let ProtocolMessage::ClientMessage(ref bm) = msg {
+                if let Some(bm) = msg.is_streamable() {
                     if bm.result() == Into::<u16>::into(ServerError::PartialContent)
                         && TypeId::of::<B>() != TypeId::of::<()>()
                     {
@@ -150,7 +150,7 @@ impl<
                                 while let Some(ConnectionCommand::Msg(msg)) =
                                     actor_receiver.next().await
                                 {
-                                    if let ProtocolMessage::ClientMessage(ref bm) = msg {
+                                    if let Some(bm) = msg.is_streamable() {
                                         if bm.result()
                                             == Into::<u16>::into(ServerError::EndOfStream)
                                         {

ng-net/src/actors/admin/add_invitation.rs

@@ -103,20 +103,29 @@ impl EActor for Actor<'_, AddInvitation, AdminResponse> {
         fsm: Arc<Mutex<NoiseFSM>>,
     ) -> Result<(), ProtocolError> {
         let req = AddInvitation::try_from(msg)?;
+        let (url, bootstrap, sb) = {
             let broker = BROKER.read().await;
-        broker
-            .get_server_broker()?
+            let url = if req.tos_url() {
+                broker.get_registration_url().map(|s| s.clone())
+            } else {
+                None
+            };
+            (
+                url,
+                broker.get_bootstrap()?.clone(),
+                broker.get_server_broker()?,
+            )
+        };
+        {
+            sb.read()
+                .await
                 .add_invitation(req.code(), req.expiry(), req.memo())?;
-
+        }
         let invitation = crate::types::Invitation::V0(InvitationV0::new(
-            broker.get_bootstrap()?.clone(),
+            bootstrap,
             Some(req.code().get_symkey()),
             None,
-            if req.tos_url() {
-                broker.get_registration_url().map(|s| s.clone())
-            } else {
-                None
-            },
+            url,
         ));
         let response: AdminResponseV0 = invitation.into();
         fsm.lock().await.send(response.into()).await?;

ng-net/src/actors/admin/add_user.rs

@@ -94,8 +94,11 @@ impl EActor for Actor<'_, AddUser, AdminResponse> {
         fsm: Arc<Mutex<NoiseFSM>>,
     ) -> Result<(), ProtocolError> {
         let req = AddUser::try_from(msg)?;
-        let broker = BROKER.read().await;
+
+        let res = {
             let mut is_admin = req.is_admin();
+            let sb = {
+                let broker = BROKER.read().await;
                 if let Some(ServerConfig {
                     admin_user: Some(admin_user),
                     ..
@@ -105,7 +108,12 @@ impl EActor for Actor<'_, AddUser, AdminResponse> {
                         is_admin = true;
                     }
                 }
-        let res = broker.get_server_broker()?.add_user(req.user(), is_admin);
+                broker.get_server_broker()?
+            };
+
+            let lock = sb.read().await;
+            lock.add_user(req.user(), is_admin)
+        };
         let response: AdminResponseV0 = res.into();
         fsm.lock().await.send(response.into()).await?;
         Ok(())

ng-net/src/actors/admin/create_user.rs

@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2022-2024 Niko Bonnieure, Par le Peuple, NextGraph.org developers
+ * All rights reserved.
+ * Licensed under the Apache License, Version 2.0
+ * <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
+ * or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
+ * at your option. All files in the project carrying such
+ * notice may not be copied, modified, or distributed except
+ * according to those terms.
+*/
+
+use std::sync::Arc;
+
+use async_std::sync::Mutex;
+use ng_repo::types::UserId;
+use serde::{Deserialize, Serialize};
+
+use ng_repo::errors::*;
+use ng_repo::log::*;
+
+use super::super::StartProtocol;
+
+use crate::broker::BROKER;
+use crate::connection::NoiseFSM;
+use crate::types::*;
+use crate::{actor::*, types::ProtocolMessage};
+
+/// Create user and keeps credentials in the server (for use with headless API)
+#[doc(hidden)]
+#[derive(Clone, Copy, Debug, Serialize, Deserialize)]
+pub struct CreateUserV0 {}
+
+/// Create user
+#[doc(hidden)]
+#[derive(Clone, Copy, Debug, Serialize, Deserialize)]
+pub enum CreateUser {
+    V0(CreateUserV0),
+}
+
+impl TryFrom<ProtocolMessage> for CreateUser {
+    type Error = ProtocolError;
+    fn try_from(msg: ProtocolMessage) -> Result<Self, Self::Error> {
+        if let ProtocolMessage::Start(StartProtocol::Admin(AdminRequest::V0(AdminRequestV0 {
+            content: AdminRequestContentV0::CreateUser(a),
+            ..
+        }))) = msg
+        {
+            Ok(a)
+        } else {
+            log_debug!("INVALID {:?}", msg);
+            Err(ProtocolError::InvalidValue)
+        }
+    }
+}
+
+impl From<CreateUser> for ProtocolMessage {
+    fn from(_msg: CreateUser) -> ProtocolMessage {
+        unimplemented!();
+    }
+}
+
+impl From<UserId> for ProtocolMessage {
+    fn from(_msg: UserId) -> ProtocolMessage {
+        unimplemented!();
+    }
+}
+
+impl TryFrom<ProtocolMessage> for UserId {
+    type Error = ProtocolError;
+    fn try_from(_msg: ProtocolMessage) -> Result<Self, Self::Error> {
+        unimplemented!();
+    }
+}
+
+impl From<CreateUser> for AdminRequestContentV0 {
+    fn from(msg: CreateUser) -> AdminRequestContentV0 {
+        AdminRequestContentV0::CreateUser(msg)
+    }
+}
+
+impl CreateUser {
+    pub fn get_actor(&self) -> Box<dyn EActor> {
+        Actor::<CreateUser, UserId>::new_responder(0)
+    }
+}
+
+impl Actor<'_, CreateUser, UserId> {}
+
+#[async_trait::async_trait]
+impl EActor for Actor<'_, CreateUser, UserId> {
+    async fn respond(
+        &mut self,
+        msg: ProtocolMessage,
+        fsm: Arc<Mutex<NoiseFSM>>,
+    ) -> Result<(), ProtocolError> {
+        let _req = CreateUser::try_from(msg)?;
+
+        let res = {
+            let (broker_id, sb) = {
+                let b = BROKER.read().await;
+                (b.get_server_peer_id(), b.get_server_broker()?)
+            };
+            let lock = sb.read().await;
+            lock.create_user(&broker_id).await
+        };
+
+        let response: AdminResponseV0 = res.into();
+        fsm.lock().await.send(response.into()).await?;
+        Ok(())
+    }
+}

ng-net/src/actors/admin/del_user.rs

@@ -85,8 +85,8 @@ impl EActor for Actor<'_, DelUser, AdminResponse> {
         fsm: Arc<Mutex<NoiseFSM>>,
     ) -> Result<(), ProtocolError> {
         let req = DelUser::try_from(msg)?;
-        let broker = BROKER.read().await;
-        let res = broker.get_server_broker()?.del_user(req.user());
+        let sb = { BROKER.read().await.get_server_broker()? };
+        let res = { sb.read().await.del_user(req.user()) };
         let response: AdminResponseV0 = res.into();
         fsm.lock().await.send(response.into()).await?;
         Ok(())

ng-net/src/actors/admin/list_invitations.rs

@@ -101,11 +101,12 @@ impl EActor for Actor<'_, ListInvitations, AdminResponse> {
         fsm: Arc<Mutex<NoiseFSM>>,
     ) -> Result<(), ProtocolError> {
         let req = ListInvitations::try_from(msg)?;
-        let res = BROKER.read().await.get_server_broker()?.list_invitations(
-            req.admin(),
-            req.unique(),
-            req.multi(),
-        );
+        let sb = { BROKER.read().await.get_server_broker()? };
+        let res = {
+            sb.read()
+                .await
+                .list_invitations(req.admin(), req.unique(), req.multi())
+        };
         let response: AdminResponseV0 = res.into();
         fsm.lock().await.send(response.into()).await?;
         Ok(())

ng-net/src/actors/admin/list_users.rs

@@ -85,11 +85,9 @@ impl EActor for Actor<'_, ListUsers, AdminResponse> {
         fsm: Arc<Mutex<NoiseFSM>>,
     ) -> Result<(), ProtocolError> {
         let req = ListUsers::try_from(msg)?;
-        let res = BROKER
-            .read()
-            .await
-            .get_server_broker()?
-            .list_users(req.admins());
+        let sb = { BROKER.read().await.get_server_broker()? };
+        let res = { sb.read().await.list_users(req.admins()) };
+
         let response: AdminResponseV0 = res.into();
         fsm.lock().await.send(response.into()).await?;
         Ok(())

ng-net/src/actors/admin/mod.rs

@@ -12,3 +12,6 @@ pub use add_invitation::*;
 
 pub mod list_invitations;
 pub use list_invitations::*;
+
+pub mod create_user;
+pub use create_user::*;

ng-net/src/actors/app/mod.rs

@@ -0,0 +1,3 @@
+pub mod request;
+
+pub mod session;

ng-net/src/actors/app/request.rs

@@ -0,0 +1,133 @@
+/*
+ * Copyright (c) 2022-2024 Niko Bonnieure, Par le Peuple, NextGraph.org developers
+ * All rights reserved.
+ * Licensed under the Apache License, Version 2.0
+ * <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
+ * or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
+ * at your option. All files in the project carrying such
+ * notice may not be copied, modified, or distributed except
+ * according to those terms.
+*/
+
+use std::sync::Arc;
+
+use async_std::sync::Mutex;
+
+use ng_repo::errors::*;
+use ng_repo::log::*;
+
+use crate::app_protocol::*;
+use crate::broker::BROKER;
+use crate::connection::NoiseFSM;
+use crate::types::*;
+use crate::{actor::*, types::ProtocolMessage};
+
+impl AppRequest {
+    pub fn get_actor(&self, id: i64) -> Box<dyn EActor> {
+        Actor::<AppRequest, AppResponse>::new_responder(id)
+    }
+}
+
+impl TryFrom<ProtocolMessage> for AppRequest {
+    type Error = ProtocolError;
+    fn try_from(msg: ProtocolMessage) -> Result<Self, Self::Error> {
+        if let AppMessageContentV0::Request(req) = msg.try_into()? {
+            Ok(req)
+        } else {
+            log_debug!("INVALID AppMessageContentV0::Request");
+            Err(ProtocolError::InvalidValue)
+        }
+    }
+}
+
+impl From<AppRequest> for ProtocolMessage {
+    fn from(request: AppRequest) -> ProtocolMessage {
+        AppMessageContentV0::Request(request).into()
+    }
+}
+
+impl From<AppMessageContentV0> for ProtocolMessage {
+    fn from(content: AppMessageContentV0) -> ProtocolMessage {
+        AppMessage::V0(AppMessageV0 {
+            content,
+            id: 0,
+            result: 0,
+        })
+        .into()
+    }
+}
+
+impl TryFrom<ProtocolMessage> for AppResponse {
+    type Error = ProtocolError;
+    fn try_from(msg: ProtocolMessage) -> Result<Self, Self::Error> {
+        if let AppMessageContentV0::Response(res) = msg.try_into()? {
+            Ok(res)
+        } else {
+            log_debug!("INVALID AppMessageContentV0::Response");
+            Err(ProtocolError::InvalidValue)
+        }
+    }
+}
+
+impl TryFrom<ProtocolMessage> for AppMessageContentV0 {
+    type Error = ProtocolError;
+    fn try_from(msg: ProtocolMessage) -> Result<Self, Self::Error> {
+        if let ProtocolMessage::AppMessage(AppMessage::V0(AppMessageV0 {
+            content, result, ..
+        })) = msg
+        {
+            let err = ServerError::try_from(result).unwrap();
+            if !err.is_err() {
+                Ok(content)
+            } else {
+                Err(ProtocolError::ServerError)
+            }
+        } else {
+            log_debug!("INVALID AppMessageContentV0");
+            Err(ProtocolError::InvalidValue)
+        }
+    }
+}
+
+impl From<AppResponse> for AppMessage {
+    fn from(response: AppResponse) -> AppMessage {
+        AppMessage::V0(AppMessageV0 {
+            content: AppMessageContentV0::Response(response),
+            id: 0,
+            result: 0,
+        })
+    }
+}
+
+impl From<AppResponse> for ProtocolMessage {
+    fn from(response: AppResponse) -> ProtocolMessage {
+        response.into()
+    }
+}
+
+impl Actor<'_, AppRequest, AppResponse> {}
+
+#[async_trait::async_trait]
+impl EActor for Actor<'_, AppRequest, AppResponse> {
+    async fn respond(
+        &mut self,
+        msg: ProtocolMessage,
+        fsm: Arc<Mutex<NoiseFSM>>,
+    ) -> Result<(), ProtocolError> {
+        let req = AppRequest::try_from(msg)?;
+        let res = {
+            let sb = { BROKER.read().await.get_server_broker()? };
+            let lock = sb.read().await;
+            lock.app_process_request(req, self.id(), &fsm).await
+        };
+        if res.is_err() {
+            let server_err: ServerError = res.unwrap_err().into();
+            let app_message: AppMessage = server_err.into();
+            fsm.lock()
+                .await
+                .send_in_reply_to(app_message.into(), self.id())
+                .await?;
+        }
+        Ok(())
+    }
+}

ng-net/src/actors/app/session.rs

@@ -0,0 +1,176 @@
+/*
+ * Copyright (c) 2022-2024 Niko Bonnieure, Par le Peuple, NextGraph.org developers
+ * All rights reserved.
+ * Licensed under the Apache License, Version 2.0
+ * <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
+ * or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
+ * at your option. All files in the project carrying such
+ * notice may not be copied, modified, or distributed except
+ * according to those terms.
+*/
+
+use std::sync::Arc;
+
+use async_std::sync::Mutex;
+
+use ng_repo::errors::*;
+use ng_repo::log::*;
+
+use crate::app_protocol::*;
+use crate::broker::BROKER;
+use crate::connection::NoiseFSM;
+use crate::types::*;
+use crate::{actor::*, types::ProtocolMessage};
+
+impl AppSessionStart {
+    pub fn get_actor(&self, id: i64) -> Box<dyn EActor> {
+        Actor::<AppSessionStart, AppSessionStartResponse>::new_responder(id)
+    }
+}
+
+impl TryFrom<ProtocolMessage> for AppSessionStart {
+    type Error = ProtocolError;
+    fn try_from(msg: ProtocolMessage) -> Result<Self, Self::Error> {
+        if let AppMessageContentV0::SessionStart(req) = msg.try_into()? {
+            Ok(req)
+        } else {
+            log_debug!("INVALID AppMessageContentV0::SessionStart");
+            Err(ProtocolError::InvalidValue)
+        }
+    }
+}
+
+impl From<AppSessionStart> for ProtocolMessage {
+    fn from(request: AppSessionStart) -> ProtocolMessage {
+        AppMessageContentV0::SessionStart(request).into()
+    }
+}
+
+impl TryFrom<ProtocolMessage> for AppSessionStartResponse {
+    type Error = ProtocolError;
+    fn try_from(msg: ProtocolMessage) -> Result<Self, Self::Error> {
+        if let AppMessageContentV0::Response(AppResponse::V0(AppResponseV0::SessionStart(res))) =
+            msg.try_into()?
+        {
+            Ok(res)
+        } else {
+            log_debug!("INVALID AppSessionStartResponse");
+            Err(ProtocolError::InvalidValue)
+        }
+    }
+}
+
+impl From<AppSessionStartResponse> for AppMessage {
+    fn from(response: AppSessionStartResponse) -> AppMessage {
+        AppResponse::V0(AppResponseV0::SessionStart(response)).into()
+    }
+}
+
+impl From<AppSessionStartResponse> for ProtocolMessage {
+    fn from(response: AppSessionStartResponse) -> ProtocolMessage {
+        response.into()
+    }
+}
+
+impl Actor<'_, AppSessionStart, AppSessionStartResponse> {}
+
+#[async_trait::async_trait]
+impl EActor for Actor<'_, AppSessionStart, AppSessionStartResponse> {
+    async fn respond(
+        &mut self,
+        msg: ProtocolMessage,
+        fsm: Arc<Mutex<NoiseFSM>>,
+    ) -> Result<(), ProtocolError> {
+        let req = AppSessionStart::try_from(msg)?;
+        let res = {
+            let lock = fsm.lock().await;
+            let remote = lock.remote_peer();
+
+            //TODO: if fsm.get_user_id is some, check that user_priv_key in credentials matches.
+            //TODO: if no user in fsm (headless), check user in request is allowed
+            if remote.is_none() {
+                Err(ServerError::BrokerError)
+            } else {
+                let (sb, broker_id) = {
+                    let b = BROKER.read().await;
+                    (b.get_server_broker()?, b.get_server_peer_id())
+                };
+                let lock = sb.read().await;
+                lock.app_session_start(req, remote.unwrap(), broker_id)
+                    .await
+            }
+        };
+        let app_message: AppMessage = match res {
+            Err(e) => e.into(),
+            Ok(o) => o.into(),
+        };
+        fsm.lock()
+            .await
+            .send_in_reply_to(app_message.into(), self.id())
+            .await?;
+        Ok(())
+    }
+}
+
+///////////////////////
+
+impl AppSessionStop {
+    pub fn get_actor(&self, id: i64) -> Box<dyn EActor> {
+        Actor::<AppSessionStop, ()>::new_responder(id)
+    }
+}
+
+impl TryFrom<ProtocolMessage> for AppSessionStop {
+    type Error = ProtocolError;
+    fn try_from(msg: ProtocolMessage) -> Result<Self, Self::Error> {
+        if let AppMessageContentV0::SessionStop(req) = msg.try_into()? {
+            Ok(req)
+        } else {
+            log_debug!("INVALID AppMessageContentV0::SessionStop");
+            Err(ProtocolError::InvalidValue)
+        }
+    }
+}
+
+impl From<AppSessionStop> for ProtocolMessage {
+    fn from(request: AppSessionStop) -> ProtocolMessage {
+        AppMessageContentV0::SessionStop(request).into()
+    }
+}
+
+impl From<Result<EmptyAppResponse, ServerError>> for ProtocolMessage {
+    fn from(res: Result<EmptyAppResponse, ServerError>) -> ProtocolMessage {
+        match res {
+            Ok(_a) => ServerError::Ok.into(),
+            Err(err) => AppMessage::V0(AppMessageV0 {
+                id: 0,
+                result: err.into(),
+                content: AppMessageContentV0::EmptyResponse,
+            }),
+        }
+        .into()
+    }
+}
+
+impl Actor<'_, AppSessionStop, ()> {}
+
+#[async_trait::async_trait]
+impl EActor for Actor<'_, AppSessionStop, ()> {
+    async fn respond(
+        &mut self,
+        msg: ProtocolMessage,
+        fsm: Arc<Mutex<NoiseFSM>>,
+    ) -> Result<(), ProtocolError> {
+        let req = AppSessionStop::try_from(msg)?;
+        let res = {
+            let sb = { BROKER.read().await.get_server_broker()? };
+            let lock = sb.read().await;
+            lock.app_session_stop(req)
+        };
+        fsm.lock()
+            .await
+            .send_in_reply_to(res.into(), self.id())
+            .await?;
+        Ok(())
+    }
+}

ng-net/src/actors/client/blocks_exist.rs

@@ -76,7 +76,7 @@ impl EActor for Actor<'_, BlocksExist, BlocksFound> {
         fsm: Arc<Mutex<NoiseFSM>>,
     ) -> Result<(), ProtocolError> {
         let req = BlocksExist::try_from(msg)?;
-        let broker = BROKER.read().await;
+        let sb = { BROKER.read().await.get_server_broker()? };
 
         let overlay = req.overlay().clone();
         let mut found = vec![];
@@ -84,7 +84,7 @@ impl EActor for Actor<'_, BlocksExist, BlocksFound> {
         match req {
             BlocksExist::V0(v0) => {
                 for block_id in v0.blocks {
-                    let r = broker.get_server_broker()?.has_block(&overlay, &block_id);
+                    let r = sb.read().await.has_block(&overlay, &block_id);
                     if r.is_err() {
                         missing.push(block_id);
                     } else {

ng-net/src/actors/client/blocks_get.rs

@@ -12,6 +12,7 @@
 use std::sync::Arc;
 
 use async_recursion::async_recursion;
+use async_std::sync::RwLock;
 use async_std::sync::{Mutex, MutexGuard};
 
 use ng_repo::errors::*;
@@ -71,15 +72,14 @@ impl EActor for Actor<'_, BlocksGet, Block> {
         fsm: Arc<Mutex<NoiseFSM>>,
     ) -> Result<(), ProtocolError> {
         let req = BlocksGet::try_from(msg)?;
-        let broker = BROKER.read().await;
-        let server = broker.get_server_broker()?;
+        let server = { BROKER.read().await.get_server_broker()? };
         let mut lock = fsm.lock().await;
         let mut something_was_sent = false;
 
         #[async_recursion]
         async fn process_children(
             children: &Vec<BlockId>,
-            server: &Box<dyn IServerBroker + Send + Sync>,
+            server: &RwLock<dyn IServerBroker + Send + Sync>,
             overlay: &OverlayId,
             lock: &mut MutexGuard<'_, NoiseFSM>,
             req_id: i64,
@@ -87,7 +87,7 @@ impl EActor for Actor<'_, BlocksGet, Block> {
             something_was_sent: &mut bool,
         ) {
             for block_id in children {
-                if let Ok(block) = server.get_block(overlay, block_id) {
+                if let Ok(block) = { server.read().await.get_block(overlay, block_id) } {
                     let grand_children = block.children().to_vec();
                     if let Err(_) = lock.send_in_reply_to(block.into(), req_id).await {
                         break;
@@ -110,7 +110,7 @@ impl EActor for Actor<'_, BlocksGet, Block> {
         }
         process_children(
             req.ids(),
-            server,
+            &server,
             req.overlay(),
             &mut lock,
             self.id(),

ng-net/src/actors/client/blocks_put.rs

@@ -57,13 +57,13 @@ impl EActor for Actor<'_, BlocksPut, ()> {
         fsm: Arc<Mutex<NoiseFSM>>,
     ) -> Result<(), ProtocolError> {
         let req = BlocksPut::try_from(msg)?;
-        let broker = BROKER.read().await;
+        let sb = { BROKER.read().await.get_server_broker()? };
         let mut res: Result<(), ServerError> = Ok(());
         let overlay = req.overlay().clone();
         match req {
             BlocksPut::V0(v0) => {
                 for block in v0.blocks {
-                    let r = broker.get_server_broker()?.put_block(&overlay, block);
+                    let r = sb.read().await.put_block(&overlay, block);
                     if r.is_err() {
                         res = r;
                         break;

ng-net/src/actors/client/commit_get.rs

@@ -90,10 +90,8 @@ impl EActor for Actor<'_, CommitGet, Block> {
         fsm: Arc<Mutex<NoiseFSM>>,
     ) -> Result<(), ProtocolError> {
         let req = CommitGet::try_from(msg)?;
-        let broker = BROKER.read().await;
-        let blocks_res = broker
-            .get_server_broker()?
-            .get_commit(req.overlay(), req.id());
+        let broker = { BROKER.read().await.get_server_broker()? };
+        let blocks_res = { broker.read().await.get_commit(req.overlay(), req.id()) };
         // IF NEEDED, the get_commit could be changed to return a stream, and then the send_in_reply_to would be also totally async
         match blocks_res {
             Ok(blocks) => {

ng-net/src/actors/client/event.rs

@@ -78,10 +78,8 @@ impl EActor for Actor<'_, PublishEvent, ()> {
         #[cfg(not(target_arch = "wasm32"))]
         {
             let req = PublishEvent::try_from(_msg)?;
-
             // send a ProtocolError if invalid signatures (will disconnect the client)
             req.event().verify()?;
-
             let overlay = req.overlay().clone();
             let (user_id, remote_peer) = {
                 let fsm = _fsm.lock().await;
@@ -91,15 +89,32 @@ impl EActor for Actor<'_, PublishEvent, ()> {
                 )
             };
             let res = {
-                let mut broker = BROKER.write().await;
+                let broker = BROKER.read().await;
                 broker
                     .dispatch_event(&overlay, req.take_event(), &user_id, &remote_peer)
                     .await
             };
+            if res.is_err() {
+                let res: Result<(), ServerError> = Err(res.unwrap_err());
                 _fsm.lock()
                     .await
                     .send_in_reply_to(res.into(), self.id())
                     .await?;
+            } else {
+                let broker = { BROKER.read().await.get_server_broker()? };
+                for client in res.unwrap() {
+                    broker
+                        .read()
+                        .await
+                        .remove_all_subscriptions_of_client(&client)
+                        .await;
+                }
+                let finalres: Result<(), ServerError> = Ok(());
+                _fsm.lock()
+                    .await
+                    .send_in_reply_to(finalres.into(), self.id())
+                    .await?;
+            }
         }
         Ok(())
     }

ng-net/src/actors/client/pin_repo.rs

@@ -109,20 +109,19 @@ impl EActor for Actor<'_, PinRepo, RepoOpened> {
     ) -> Result<(), ProtocolError> {
         let req = PinRepo::try_from(msg)?;
 
-        let mut broker = BROKER.write().await;
+        let (sb, server_peer_id) = {
+            let b = BROKER.read().await;
+            (b.get_server_broker()?, b.get_server_peer_id())
+        };
 
         // check the validity of the PublisherAdvert(s). this will return a ProtocolError (will close the connection)
-        let server_peer_id = broker.get_config().unwrap().peer_id;
         for pub_ad in req.rw_topics() {
             pub_ad.verify_for_broker(&server_peer_id)?;
         }
 
         let (user_id, remote_peer) = {
             let fsm = fsm.lock().await;
-            (
-                fsm.user_id()?,
-                fsm.remote_peer().ok_or(ProtocolError::ActorError)?,
-            )
+            (fsm.user_id()?, fsm.get_client_peer_id()?)
         };
 
         let result = {
@@ -135,13 +134,16 @@ impl EActor for Actor<'_, PinRepo, RepoOpened> {
                     {
                         Err(ServerError::InvalidRequest)
                     } else {
-                        broker.get_server_broker_mut()?.pin_repo_read(
+                        sb.read()
+                            .await
+                            .pin_repo_read(
                                 req.overlay(),
                                 req.hash(),
                                 &user_id,
                                 req.ro_topics(),
                                 &remote_peer,
                             )
+                            .await
                     }
                 }
                 OverlayAccess::ReadWrite((w, r)) => {
@@ -154,7 +156,9 @@ impl EActor for Actor<'_, PinRepo, RepoOpened> {
                         // TODO add a check on "|| overlay_root_topic.is_none()"  because it should be mandatory to have one (not sent by client at the moment)
                         Err(ServerError::InvalidRequest)
                     } else {
-                        broker.get_server_broker_mut()?.pin_repo_write(
+                        sb.read()
+                            .await
+                            .pin_repo_write(
                                 req.overlay_access(),
                                 req.hash(),
                                 &user_id,
@@ -164,13 +168,16 @@ impl EActor for Actor<'_, PinRepo, RepoOpened> {
                                 req.expose_outer(),
                                 &remote_peer,
                             )
+                            .await
                     }
                 }
                 OverlayAccess::WriteOnly(w) => {
                     if !w.is_inner() || req.overlay() != w || req.expose_outer() {
                         Err(ServerError::InvalidRequest)
                     } else {
-                        broker.get_server_broker_mut()?.pin_repo_write(
+                        sb.read()
+                            .await
+                            .pin_repo_write(
                                 req.overlay_access(),
                                 req.hash(),
                                 &user_id,
@@ -180,6 +187,7 @@ impl EActor for Actor<'_, PinRepo, RepoOpened> {
                                 false,
                                 &remote_peer,
                             )
+                            .await
                     }
                 }
             }

ng-net/src/actors/client/repo_pin_status.rs

@@ -79,12 +79,14 @@ impl EActor for Actor<'_, RepoPinStatusReq, RepoPinStatus> {
         fsm: Arc<Mutex<NoiseFSM>>,
     ) -> Result<(), ProtocolError> {
         let req = RepoPinStatusReq::try_from(msg)?;
-        let broker = BROKER.read().await;
-        let res = broker.get_server_broker()?.get_repo_pin_status(
+        let sb = { BROKER.read().await.get_server_broker()? };
+        let res = {
+            sb.read().await.get_repo_pin_status(
                 req.overlay(),
                 req.hash(),
                 &fsm.lock().await.user_id()?,
-        );
+            )
+        };
         fsm.lock()
             .await
             .send_in_reply_to(res.into(), self.id())

ng-net/src/actors/client/topic_sub.rs

@@ -101,30 +101,34 @@ impl EActor for Actor<'_, TopicSub, TopicSubRes> {
     ) -> Result<(), ProtocolError> {
         let req = TopicSub::try_from(msg)?;
 
-        let mut broker = BROKER.write().await;
+        let (sb, server_peer_id) = {
+            let b = BROKER.read().await;
+            (b.get_server_broker()?, b.get_server_peer_id())
+        };
 
         // check the validity of the PublisherAdvert. this will return a ProtocolError (will close the connection)
         if let Some(advert) = req.publisher() {
-            let server_peer_id = broker.get_config().unwrap().peer_id;
             advert.verify_for_broker(&server_peer_id)?;
         }
 
         let (user_id, remote_peer) = {
             let fsm = fsm.lock().await;
-            (
-                fsm.user_id()?,
-                fsm.remote_peer().ok_or(ProtocolError::ActorError)?,
-            )
+            (fsm.user_id()?, fsm.get_client_peer_id()?)
         };
 
-        let res = broker.get_server_broker_mut()?.topic_sub(
+        let res = {
+            sb.read()
+                .await
+                .topic_sub(
                     req.overlay(),
                     req.hash(),
                     req.topic(),
                     &user_id,
                     req.publisher(),
                     &remote_peer,
-        );
+                )
+                .await
+        };
 
         fsm.lock()
             .await

ng-net/src/actors/client/topic_sync_req.rs

@@ -107,15 +107,17 @@ impl EActor for Actor<'_, TopicSyncReq, TopicSyncRes> {
     ) -> Result<(), ProtocolError> {
         let req = TopicSyncReq::try_from(msg)?;
 
-        let broker = BROKER.read().await;
+        let sb = { BROKER.read().await.get_server_broker()? };
 
-        let res = broker.get_server_broker()?.topic_sync_req(
+        let res = {
+            sb.read().await.topic_sync_req(
                 req.overlay(),
                 req.topic(),
                 req.known_heads(),
                 req.target_heads(),
                 req.known_commits(),
-        );
+            )
+        };
 
         // IF NEEDED, the topic_sync_req could be changed to return a stream, and then the send_in_reply_to would be also totally async
         match res {

ng-net/src/actors/mod.rs

@@ -19,3 +19,5 @@ pub use connecting::*;
 pub mod client;
 
 pub mod admin;
+
+pub mod app;

ng-net/src/actors/start.rs

@@ -17,12 +17,13 @@ use serde::{Deserialize, Serialize};
 
 use ng_repo::errors::*;
 use ng_repo::log::*;
+use ng_repo::types::UserId;
 
 use crate::actors::noise::Noise;
 use crate::connection::NoiseFSM;
 use crate::types::{
-    AdminRequest, CoreBrokerConnect, CoreBrokerConnectResponse, CoreMessage, CoreMessageV0,
-    CoreResponse, CoreResponseContentV0, CoreResponseV0, ExtResponse,
+    AdminRequest, ClientInfo, CoreBrokerConnect, CoreBrokerConnectResponse, CoreMessage,
+    CoreMessageV0, CoreResponse, CoreResponseContentV0, CoreResponseV0, ExtResponse,
 };
 use crate::{actor::*, types::ProtocolMessage};
 
@@ -36,6 +37,8 @@ pub enum StartProtocol {
     Ext(ExtHello),
     Core(CoreHello),
     Admin(AdminRequest),
+    App(AppHello),
+    AppResponse(AppHelloResponse),
 }
 
 impl StartProtocol {
@@ -45,6 +48,8 @@ impl StartProtocol {
             StartProtocol::Core(a) => a.type_id(),
             StartProtocol::Ext(a) => a.type_id(),
             StartProtocol::Admin(a) => a.type_id(),
+            StartProtocol::App(a) => a.type_id(),
+            StartProtocol::AppResponse(a) => a.type_id(),
         }
     }
     pub fn get_actor(&self) -> Box<dyn EActor> {
@@ -53,6 +58,8 @@ impl StartProtocol {
             StartProtocol::Core(a) => a.get_actor(),
             StartProtocol::Ext(a) => a.get_actor(),
             StartProtocol::Admin(a) => a.get_actor(),
+            StartProtocol::App(a) => a.get_actor(),
+            StartProtocol::AppResponse(_) => panic!("AppResponse is not a request"),
         }
     }
 }
@@ -256,3 +263,64 @@ impl EActor for Actor<'_, ExtHello, ExtResponse> {
         Ok(())
     }
 }
+
+// ///////////// APP HELLO ///////////////
+
+/// App Hello (finalizes the Noise handshake and sends info about device, and the user_id.
+/// not signing any nonce because anyway, in the next message "AppSessionRequest", the user_priv_key will be sent and checked again)
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct AppHello {
+    // contains the 3rd Noise handshake message "s,se"
+    pub noise: Noise,
+
+    pub user: Option<UserId>, // None for Headless
+
+    pub info: ClientInfo,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct AppHelloResponse {
+    pub result: u16,
+}
+
+impl AppHello {
+    pub fn get_actor(&self) -> Box<dyn EActor> {
+        Actor::<AppHello, AppHelloResponse>::new_responder(0)
+    }
+}
+
+impl From<AppHello> for ProtocolMessage {
+    fn from(msg: AppHello) -> ProtocolMessage {
+        ProtocolMessage::Start(StartProtocol::App(msg))
+    }
+}
+
+impl From<AppHelloResponse> for ProtocolMessage {
+    fn from(msg: AppHelloResponse) -> ProtocolMessage {
+        ProtocolMessage::Start(StartProtocol::AppResponse(msg))
+    }
+}
+
+impl TryFrom<ProtocolMessage> for AppHelloResponse {
+    type Error = ProtocolError;
+    fn try_from(msg: ProtocolMessage) -> Result<Self, Self::Error> {
+        if let ProtocolMessage::Start(StartProtocol::AppResponse(res)) = msg {
+            Ok(res)
+        } else {
+            Err(ProtocolError::InvalidValue)
+        }
+    }
+}
+
+impl Actor<'_, AppHello, AppHelloResponse> {}
+
+#[async_trait::async_trait]
+impl EActor for Actor<'_, AppHello, AppHelloResponse> {
+    async fn respond(
+        &mut self,
+        _msg: ProtocolMessage,
+        _fsm: Arc<Mutex<NoiseFSM>>,
+    ) -> Result<(), ProtocolError> {
+        Ok(())
+    }
+}

ng-net/src/app_protocol.rs

@@ -0,0 +1,400 @@
+// Copyright (c) 2022-2024 Niko Bonnieure, Par le Peuple, NextGraph.org developers
+// All rights reserved.
+// Licensed under the Apache License, Version 2.0
+// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
+// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
+// at your option. All files in the project carrying such
+// notice may not be copied, modified, or distributed except
+// according to those terms.
+
+//! App Protocol (between LocalBroker and Verifier)
+
+use serde::{Deserialize, Serialize};
+
+use ng_repo::errors::NgError;
+use ng_repo::types::*;
+
+use crate::types::*;
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum AppFetchContentV0 {
+    Get,       // does not subscribe. more to be detailed
+    Subscribe, // more to be detailed
+    Update,
+    //Invoke,
+    ReadQuery,  // more to be detailed
+    WriteQuery, // more to be detailed
+}
+
+impl AppFetchContentV0 {
+    pub fn get_or_subscribe(subscribe: bool) -> Self {
+        if !subscribe {
+            AppFetchContentV0::Get
+        } else {
+            AppFetchContentV0::Subscribe
+        }
+    }
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum NgAccessV0 {
+    ReadCap(ReadCap),
+    Token(Digest),
+    #[serde(with = "serde_bytes")]
+    ExtRequest(Vec<u8>),
+    Key(BlockKey),
+    Inbox(Digest),
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum TargetBranchV0 {
+    Chat,
+    Stream,
+    Context,
+    Ontology,
+    BranchId(BranchId),
+    Named(String),          // branch or commit
+    Commits(Vec<ObjectId>), // only possible if access to their branch is given. must belong to the same branch.
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum NuriTargetV0 {
+    UserSite, // targets the whole data set of the user
+
+    PublicStore,
+    ProtectedStore,
+    PrivateStore,
+    AllDialogs,
+    Dialog(String), // shortname of a Dialog
+    AllGroups,
+    Group(String), // shortname of a Group
+
+    Repo(RepoId),
+
+    Identity(UserId),
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct NuriV0 {
+    pub target: NuriTargetV0,
+    pub entire_store: bool, // If it is a store, will (try to) include all the docs belonging to the store
+
+    pub object: Option<ObjectId>, // used only for FileGet. // cannot be used for queries. only to download an object (file,commit..)
+    pub branch: Option<TargetBranchV0>, // if None, the main branch is chosen
+    pub overlay: Option<OverlayLink>,
+
+    pub access: Vec<NgAccessV0>,
+    pub topic: Option<TopicId>,
+    pub locator: Vec<PeerAdvert>,
+}
+
+impl NuriV0 {
+    pub fn new_repo_target_from_string(repo_id_string: String) -> Result<Self, NgError> {
+        let repo_id: RepoId = repo_id_string.as_str().try_into()?;
+        Ok(Self {
+            target: NuriTargetV0::Repo(repo_id),
+            entire_store: false,
+            object: None,
+            branch: None,
+            overlay: None,
+            access: vec![],
+            topic: None,
+            locator: vec![],
+        })
+    }
+
+    pub fn new_private_store_target() -> Self {
+        Self {
+            target: NuriTargetV0::PrivateStore,
+            entire_store: false,
+            object: None,
+            branch: None,
+            overlay: None,
+            access: vec![],
+            topic: None,
+            locator: vec![],
+        }
+    }
+    pub fn new(_from: String) -> Self {
+        todo!();
+    }
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum AppRequestCommandV0 {
+    Fetch(AppFetchContentV0),
+    Pin,
+    UnPin,
+    Delete,
+    Create,
+    FileGet, // needs the Nuri of branch/doc/store AND ObjectId
+    FilePut, // needs the Nuri of branch/doc/store
+}
+
+impl AppRequestCommandV0 {
+    pub fn is_stream(&self) -> bool {
+        match self {
+            Self::FilePut | Self::Create | Self::Delete | Self::UnPin | Self::Pin => false,
+            Self::Fetch(_) | Self::FileGet => true,
+        }
+    }
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct AppRequestV0 {
+    pub command: AppRequestCommandV0,
+
+    pub nuri: NuriV0,
+
+    pub payload: Option<AppRequestPayload>,
+
+    pub session_id: u64,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum AppRequest {
+    V0(AppRequestV0),
+}
+
+impl AppRequest {
+    pub fn set_session_id(&mut self, session_id: u64) {
+        match self {
+            Self::V0(v0) => v0.session_id = session_id,
+        }
+    }
+    pub fn session_id(&self) -> u64 {
+        match self {
+            Self::V0(v0) => v0.session_id,
+        }
+    }
+    pub fn command(&self) -> &AppRequestCommandV0 {
+        match self {
+            Self::V0(v0) => &v0.command,
+        }
+    }
+    pub fn new(
+        command: AppRequestCommandV0,
+        nuri: NuriV0,
+        payload: Option<AppRequestPayload>,
+    ) -> Self {
+        AppRequest::V0(AppRequestV0 {
+            command,
+            nuri,
+            payload,
+            session_id: 0,
+        })
+    }
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct AppSessionStopV0 {
+    pub session_id: u64,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum AppSessionStop {
+    V0(AppSessionStopV0),
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct AppSessionStartV0 {
+    pub session_id: u64,
+
+    pub credentials: Option<Credentials>,
+
+    pub user_id: UserId,
+
+    pub detach: bool,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum AppSessionStart {
+    V0(AppSessionStartV0),
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct AppSessionStartResponseV0 {
+    pub private_store: RepoId,
+    pub protected_store: RepoId,
+    pub public_store: RepoId,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum AppSessionStartResponse {
+    V0(AppSessionStartResponseV0),
+}
+
+impl AppSessionStart {
+    pub fn session_id(&self) -> u64 {
+        match self {
+            Self::V0(v0) => v0.session_id,
+        }
+    }
+    pub fn credentials(&self) -> &Option<Credentials> {
+        match self {
+            Self::V0(v0) => &v0.credentials,
+        }
+    }
+    pub fn user_id(&self) -> &UserId {
+        match self {
+            Self::V0(v0) => &v0.user_id,
+        }
+    }
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum DocQuery {
+    V0(String), // Sparql
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct GraphUpdate {
+    add: Vec<String>,
+    remove: Vec<String>,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum DiscreteUpdate {
+    /// A yrs::Update
+    #[serde(with = "serde_bytes")]
+    YMap(Vec<u8>),
+    #[serde(with = "serde_bytes")]
+    YXml(Vec<u8>),
+    #[serde(with = "serde_bytes")]
+    YText(Vec<u8>),
+    /// An automerge::Patch
+    #[serde(with = "serde_bytes")]
+    Automerge(Vec<u8>),
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct DocUpdate {
+    heads: Vec<ObjectId>,
+    graph: Option<GraphUpdate>,
+    discrete: Option<DiscreteUpdate>,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct DocAddFile {
+    pub filename: Option<String>,
+    pub object: ObjectRef,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct DocCreate {
+    store: StoreRepo,
+    content_type: BranchContentType,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct DocDelete {
+    /// Nuri of doc to delete
+    nuri: String,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum AppRequestPayloadV0 {
+    Create(DocCreate),
+    Query(DocQuery),
+    Update(DocUpdate),
+    AddFile(DocAddFile),
+    //RemoveFile
+    Delete(DocDelete),
+    //Invoke(InvokeArguments),
+    SmallFilePut(SmallFile),
+    RandomAccessFilePut(String),                           // content_type
+    RandomAccessFilePutChunk((u32, serde_bytes::ByteBuf)), // end the upload with an empty vec
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum AppRequestPayload {
+    V0(AppRequestPayloadV0),
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum DiscretePatch {
+    /// A yrs::Update
+    #[serde(with = "serde_bytes")]
+    YMap(Vec<u8>),
+    #[serde(with = "serde_bytes")]
+    YXml(Vec<u8>),
+    #[serde(with = "serde_bytes")]
+    YText(Vec<u8>),
+    /// An automerge::Patch
+    #[serde(with = "serde_bytes")]
+    Automerge(Vec<u8>),
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct GraphPatch {
+    /// oxigraph::model::GroundQuad serialized to n-quads with oxrdfio
+    pub adds: Vec<String>,
+    pub removes: Vec<String>,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum DiscreteState {
+    /// A yrs::StateVector
+    #[serde(with = "serde_bytes")]
+    YMap(Vec<u8>),
+    #[serde(with = "serde_bytes")]
+    YXml(Vec<u8>),
+    #[serde(with = "serde_bytes")]
+    YText(Vec<u8>),
+    // the output of Automerge::save()
+    #[serde(with = "serde_bytes")]
+    Automerge(Vec<u8>),
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct GraphState {
+    pub tuples: Vec<String>,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct AppState {
+    heads: Vec<ObjectId>,
+    graph: Option<GraphState>, // there is always a graph present in the branch. but it might not have been asked in the request
+    discrete: Option<DiscreteState>,
+}
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct AppPatch {
+    heads: Vec<ObjectId>,
+    graph: Option<GraphPatch>,
+    discrete: Option<DiscretePatch>,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct FileName {
+    pub heads: Vec<ObjectId>,
+    pub name: Option<String>,
+    pub reference: ObjectRef,
+    pub nuri: String,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct FileMetaV0 {
+    pub content_type: String,
+    pub size: u64,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum AppResponseV0 {
+    SessionStart(AppSessionStartResponse),
+    State(AppState),
+    Patch(AppPatch),
+    Text(String),
+    File(FileName),
+    FileUploading(u32),
+    FileUploaded(ObjectRef),
+    #[serde(with = "serde_bytes")]
+    FileBinary(Vec<u8>),
+    FileMeta(FileMetaV0),
+    QueryResult, // see sparesults
+    Ok,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum AppResponse {
+    V0(AppResponseV0),
+}

ng-net/src/broker.rs

@@ -16,6 +16,8 @@ use std::collections::HashMap;
 use std::collections::HashSet;
 
 use async_std::stream::StreamExt;
+#[cfg(not(target_arch = "wasm32"))]
+use async_std::sync::Mutex;
 use async_std::sync::{Arc, RwLock};
 use either::Either;
 use futures::channel::mpsc;
@@ -34,13 +36,57 @@ use crate::types::*;
 use crate::utils::spawn_and_log_error;
 use crate::utils::{Receiver, ResultSend, Sender};
 
+#[doc(hidden)]
+#[derive(Debug, Clone)]
+pub enum ClientPeerId {
+    Local((UserId, DirectPeerId)),
+    Remote(DirectPeerId),
+}
+
+impl ClientPeerId {
+    pub fn key(&self) -> &DirectPeerId {
+        match self {
+            Self::Remote(dpi) => dpi,
+            Self::Local((_user, dpi)) => dpi,
+        }
+    }
+    pub fn value(&self) -> Option<UserId> {
+        match self {
+            Self::Remote(_) => None,
+            Self::Local((user, _)) => Some(*user),
+        }
+    }
+    pub fn new_from(peer: &DirectPeerId, local_user: &Option<UserId>) -> Self {
+        match local_user {
+            Some(user) => ClientPeerId::Local((*user, *peer)),
+            None => ClientPeerId::Remote(*peer),
+        }
+    }
+}
+
 #[derive(Debug)]
 enum PeerConnection {
     Core(BindAddress),
     Client(ConnectionBase),
+    Local(LocalTransport),
     NONE,
 }
 
+#[derive(Debug)]
+struct LocalTransport {
+    #[allow(dead_code)]
+    client_peer_id: DirectPeerId,
+    client_cnx: ConnectionBase,
+    server_cnx: ConnectionBase,
+}
+
+impl LocalTransport {
+    async fn close(&mut self) {
+        self.client_cnx.close().await;
+        self.server_cnx.close().await;
+    }
+}
+
 #[derive(Debug)]
 struct BrokerPeerInfo {
     #[allow(dead_code)]
@@ -81,8 +127,8 @@ pub static BROKER: Lazy<Arc<RwLock<Broker>>> = Lazy::new(|| Arc::new(RwLock::new
 
 pub struct Broker {
     direct_connections: HashMap<BindAddress, DirectConnection>,
-    /// tuple of optional userId and peer key in montgomery form. userId is always None on the server side.
-    peers: HashMap<(Option<PubKey>, X25519PubKey), BrokerPeerInfo>,
+    /// tuple of optional userId and peer key in montgomery form. userId is always None on the server side (except for local transport).
+    peers: HashMap<(Option<PubKey>, Option<X25519PubKey>), BrokerPeerInfo>,
     /// (local,remote) -> ConnectionBase
     anonymous_connections: HashMap<(BindAddress, BindAddress), ConnectionBase>,
 
@@ -90,7 +136,7 @@ pub struct Broker {
     shutdown: Option<Receiver<ProtocolError>>,
     shutdown_sender: Sender<ProtocolError>,
     closing: bool,
-    server_broker: Option<Box<dyn IServerBroker + Send + Sync>>,
+    server_broker: Option<Arc<RwLock<dyn IServerBroker + Send + Sync>>>,
 
     //local_broker: Option<Box<dyn ILocalBroker + Send + Sync + 'a>>,
     local_broker: Option<Arc<RwLock<dyn ILocalBroker>>>,
@@ -100,7 +146,7 @@ pub struct Broker {
     #[cfg(not(target_arch = "wasm32"))]
     bind_addresses: HashMap<BindAddress, String>,
     #[cfg(not(target_arch = "wasm32"))]
-    users_peers: HashMap<UserId, HashSet<X25519PubKey>>,
+    users_peers: HashMap<UserId, HashSet<Option<X25519PubKey>>>,
 }
 
 impl Broker {
@@ -123,6 +169,10 @@ impl Broker {
     //     }
     // }
 
+    pub fn get_server_peer_id(&self) -> DirectPeerId {
+        self.config.as_ref().unwrap().peer_id
+    }
+
     pub(crate) fn get_config(&self) -> Option<&ServerConfig> {
         self.config.as_ref()
     }
@@ -143,7 +193,7 @@ impl Broker {
     #[doc(hidden)]
     pub fn set_server_broker(&mut self, broker: impl IServerBroker + 'static) {
         //log_debug!("set_server_broker");
-        self.server_broker = Some(Box::new(broker));
+        self.server_broker = Some(Arc::new(RwLock::new(broker)));
     }
 
     #[doc(hidden)]
@@ -174,23 +224,26 @@ impl Broker {
         (copy_listeners, copy_bind_addresses)
     }
 
-    pub(crate) fn get_server_broker(
+    #[doc(hidden)]
+    pub fn get_server_broker(
         &self,
-    ) -> Result<&Box<dyn IServerBroker + Send + Sync>, ProtocolError> {
+    ) -> Result<Arc<RwLock<dyn IServerBroker + Send + Sync>>, ProtocolError> {
         //log_debug!("GET STORAGE {:?}", self.server_storage);
+        Ok(Arc::clone(
             self.server_broker
                 .as_ref()
-            .ok_or(ProtocolError::BrokerError)
+                .ok_or(ProtocolError::BrokerError)?,
+        ))
     }
 
-    pub(crate) fn get_server_broker_mut(
-        &mut self,
-    ) -> Result<&mut Box<dyn IServerBroker + Send + Sync>, ProtocolError> {
-        //log_debug!("GET STORAGE {:?}", self.server_storage);
-        self.server_broker
-            .as_mut()
-            .ok_or(ProtocolError::BrokerError)
-    }
+    // pub(crate) fn get_server_broker_mut(
+    //     &mut self,
+    // ) -> Result<&mut Box<dyn IServerBroker + Send + Sync>, ProtocolError> {
+    //     //log_debug!("GET STORAGE {:?}", self.server_storage);
+    //     self.server_broker
+    //         .as_mut()
+    //         .ok_or(ProtocolError::BrokerError)
+    // }
 
     //Option<Arc<RwLock<dyn ILocalBroker>>>,
     pub(crate) fn get_local_broker(&self) -> Result<Arc<RwLock<dyn ILocalBroker>>, ProtocolError> {
@@ -202,7 +255,7 @@ impl Broker {
     }
 
     #[cfg(not(target_arch = "wasm32"))]
-    pub(crate) fn authorize(
+    pub(crate) async fn authorize(
         &self,
         bind_addresses: &(BindAddress, BindAddress),
         auth: Authorization,
@@ -230,7 +283,8 @@ impl Broker {
             Authorization::Client(user_and_registration) => {
                 if user_and_registration.1.is_some() {
                     // user wants to register
-                    let storage = self.get_server_broker()?;
+                    let lock = self.get_server_broker()?;
+                    let storage = lock.read().await;
                     if storage.get_user(user_and_registration.0).is_ok() {
                         return Ok(());
                     }
@@ -274,8 +328,7 @@ impl Broker {
                                         storage.remove_invitation(code)?;
                                     }
                                 }
-                                self.get_server_broker()?
-                                    .add_user(user_and_registration.0, is_admin)?;
+                                storage.add_user(user_and_registration.0, is_admin)?;
                                 Ok(())
                             }
                         };
@@ -298,7 +351,7 @@ impl Broker {
                             return Ok(());
                         }
                     }
-                    let found = self.get_server_broker()?.get_user(admin_user);
+                    let found = self.get_server_broker()?.read().await.get_user(admin_user);
                     if found.is_ok() && found.unwrap() {
                         return Ok(());
                     }
@@ -310,7 +363,7 @@ impl Broker {
     }
 
     fn reconnecting(&mut self, peer_id: X25519PrivKey, user: Option<PubKey>) {
-        let peerinfo = self.peers.get_mut(&(user, peer_id));
+        let peerinfo = self.peers.get_mut(&(user, Some(peer_id)));
         match peerinfo {
             Some(info) => match &info.connected {
                 PeerConnection::NONE => {}
@@ -321,11 +374,24 @@ impl Broker {
                     self.direct_connections.remove(&ip);
                     info.connected = PeerConnection::NONE;
                 }
+                PeerConnection::Local(_) => {
+                    panic!("local transport connections cannot disconnect. shouldn't reconnect")
+                }
             },
             None => {}
         }
     }
+
     async fn remove_peer_id(&mut self, peer_id: X25519PrivKey, user: Option<PubKey>) {
+        self.remove_peer_id_(Some(peer_id), user).await
+    }
+
+    #[allow(dead_code)]
+    async fn remove_local_transport(&mut self, user: PubKey) {
+        self.remove_peer_id_(None, Some(user)).await
+    }
+
+    async fn remove_peer_id_(&mut self, peer_id: Option<X25519PrivKey>, user: Option<PubKey>) {
         let removed = self.peers.remove(&(user, peer_id));
         match removed {
             Some(info) => match info.connected {
@@ -336,19 +402,43 @@ impl Broker {
                         // server side
                         if let Some(fsm) = _cb.fsm {
                             if let Ok(user) = fsm.lock().await.user_id() {
-                                let _ = self.remove_user_peer(&user, &peer_id);
+                                let _ = self
+                                    .remove_user_peer(&user, &Some(peer_id.to_owned().unwrap()));
                             }
                         }
-                        let peer = PubKey::X25519PubKey(peer_id);
+                        let peer = PubKey::X25519PubKey(peer_id.unwrap());
                         log_debug!("unsubscribing peer {}", peer);
-                        self.get_server_broker_mut()
+                        self.get_server_broker()
                             .unwrap()
-                            .remove_all_subscriptions_of_peer(&peer);
+                            .read()
+                            .await
+                            .remove_all_subscriptions_of_client(&ClientPeerId::new_from(
+                                &peer, &user,
+                            ))
+                            .await;
                     }
                 }
                 PeerConnection::Core(ip) => {
                     self.direct_connections.remove(&ip);
                 }
+                PeerConnection::Local(_lt) => {
+                    #[cfg(not(target_arch = "wasm32"))]
+                    if peer_id.is_none() && user.is_some() {
+                        // server side
+                        let _ = self.remove_user_peer(user.as_ref().unwrap(), &None);
+
+                        log_debug!("unsubscribing local peer {}", _lt.client_peer_id);
+                        self.get_server_broker()
+                            .unwrap()
+                            .read()
+                            .await
+                            .remove_all_subscriptions_of_client(&ClientPeerId::new_from(
+                                &_lt.client_peer_id,
+                                &user,
+                            ))
+                            .await;
+                    }
+                }
             },
             None => {}
         }
@@ -489,12 +579,14 @@ impl Broker {
             anonymous = Vec::from_iter(broker.anonymous_connections.keys().cloned());
         }
         for peer_id in peer_ids {
+            if peer_id.1.is_some() {
                 BROKER
                     .write()
                     .await
                     .close_peer_connection_x(peer_id.1, peer_id.0)
                     .await;
             }
+        }
         for anon in anonymous {
             BROKER.write().await.close_anonymous(anon.1, anon.0).await;
         }
@@ -577,7 +669,11 @@ impl Broker {
     }
 
     #[cfg(not(target_arch = "wasm32"))]
-    fn add_user_peer(&mut self, user: UserId, peer: X25519PrivKey) -> Result<(), ProtocolError> {
+    fn add_user_peer(
+        &mut self,
+        user: UserId,
+        peer: Option<X25519PrivKey>,
+    ) -> Result<(), ProtocolError> {
         let peers_set = self
             .users_peers
             .entry(user)
@@ -593,7 +689,7 @@ impl Broker {
     fn remove_user_peer(
         &mut self,
         user: &UserId,
-        peer: &X25519PrivKey,
+        peer: &Option<X25519PrivKey>,
     ) -> Result<(), ProtocolError> {
         let peers_set = self
             .users_peers
@@ -603,6 +699,54 @@ impl Broker {
         if !peers_set.remove(peer) {
             return Err(ProtocolError::PeerNotConnected);
         }
+        if peers_set.is_empty() {
+            let _ = self.users_peers.remove(user);
+        }
+        Ok(())
+    }
+
+    #[cfg(not(target_arch = "wasm32"))]
+    pub(crate) async fn attach_and_authorize_app(
+        &mut self,
+        remote_bind_address: BindAddress,
+        local_bind_address: BindAddress,
+        remote_peer_id: X25519PrivKey,
+        user: &Option<UserId>,
+        _info: &ClientInfo,
+    ) -> Result<(), ProtocolError> {
+        let already = self.peers.get(&(None, Some(remote_peer_id)));
+        if already.is_some() {
+            match already.unwrap().connected {
+                PeerConnection::NONE => {}
+                _ => {
+                    return Err(ProtocolError::PeerAlreadyConnected);
+                }
+            };
+        }
+
+        //TODO: check permissions for user/remote_bind_address or headless if no user
+
+        //TODO: keep the info
+
+        let mut connection = self
+            .anonymous_connections
+            .remove(&(local_bind_address, remote_bind_address))
+            .ok_or(ProtocolError::BrokerError)?;
+
+        connection.reset_shutdown(remote_peer_id).await;
+
+        if user.is_some() {
+            self.add_user_peer(user.unwrap(), Some(remote_peer_id))?;
+        }
+
+        let connected = PeerConnection::Client(connection);
+
+        let bpi = BrokerPeerInfo {
+            last_peer_advert: None,
+            connected,
+        };
+        self.peers.insert((None, Some(remote_peer_id)), bpi);
+
         Ok(())
     }
 
@@ -618,7 +762,7 @@ impl Broker {
     ) -> Result<(), ProtocolError> {
         log_debug!("ATTACH PEER_ID {:?}", remote_peer_id);
 
-        let already = self.peers.get(&(None, remote_peer_id));
+        let already = self.peers.get(&(None, Some(remote_peer_id)));
         if already.is_some() {
             match already.unwrap().connected {
                 PeerConnection::NONE => {}
@@ -653,7 +797,8 @@ impl Broker {
             self.authorize(
                 &(local_bind_address, remote_bind_address),
                 Authorization::Client((client.user.clone(), client.registration.clone())),
-            )?;
+            )
+            .await?;
 
             // TODO add client to storage
             false
@@ -668,7 +813,7 @@ impl Broker {
         let connected = if !is_core {
             let user = client.unwrap().user;
             fsm.set_user_id(user);
-            self.add_user_peer(user, remote_peer_id)?;
+            self.add_user_peer(user, Some(remote_peer_id))?;
 
             PeerConnection::Client(connection)
         } else {
@@ -685,7 +830,7 @@ impl Broker {
             last_peer_advert: None,
             connected,
         };
-        self.peers.insert((None, remote_peer_id), bpi);
+        self.peers.insert((None, Some(remote_peer_id)), bpi);
 
         Ok(())
     }
@@ -741,6 +886,27 @@ impl Broker {
         connection.admin::<A>().await
     }
 
+    #[doc(hidden)]
+    pub fn connect_local(&mut self, peer_pubk: PubKey, user: UserId) -> Result<(), ProtocolError> {
+        if self.closing {
+            return Err(ProtocolError::Closing);
+        }
+
+        let (client_cnx, server_cnx) = ConnectionBase::create_local_transport_pipe(user, peer_pubk);
+
+        let bpi = BrokerPeerInfo {
+            last_peer_advert: None,
+            connected: PeerConnection::Local(LocalTransport {
+                client_peer_id: peer_pubk,
+                client_cnx,
+                server_cnx,
+            }),
+        };
+
+        self.peers.insert((Some(user), None), bpi);
+        Ok(())
+    }
+
     pub async fn connect(
         &mut self,
         cnx: Arc<Box<dyn IConnect>>,
@@ -760,7 +926,7 @@ impl Broker {
         if config.is_keep_alive() {
             let already = self
                 .peers
-                .get(&(config.get_user(), *remote_peer_id_dh.slice()));
+                .get(&(config.get_user(), Some(*remote_peer_id_dh.slice())));
             if already.is_some() {
                 match already.unwrap().connected {
                     PeerConnection::NONE => {}
@@ -799,7 +965,7 @@ impl Broker {
                 self.direct_connections.insert(config.addr, dc);
                 PeerConnection::Core(config.addr)
             }
-            StartConfig::Client(_config) => PeerConnection::Client(connection),
+            StartConfig::Client(_) | StartConfig::App(_) => PeerConnection::Client(connection),
             _ => unimplemented!(),
         };
 
@@ -809,7 +975,7 @@ impl Broker {
         };
 
         self.peers
-            .insert((config.get_user(), *remote_peer_id_dh.slice()), bpi);
+            .insert((config.get_user(), Some(*remote_peer_id_dh.slice())), bpi);
 
         async fn watch_close(
             mut join: Receiver<Either<NetError, X25519PrivKey>>,
@@ -831,6 +997,7 @@ impl Broker {
                     let mut broker = BROKER.write().await;
                     broker.reconnecting(remote_peer_id, config.get_user());
                     // TODO: deal with cycle error https://users.rust-lang.org/t/recursive-async-method-causes-cycle-error/84628/5
+                    // there is async_recursion now. use that
                     // use a channel and send the reconnect job to it.
                     // create a spawned loop to read the channel and process the reconnection requests.
                     // let result = broker
@@ -874,48 +1041,77 @@ impl Broker {
         B: TryFrom<ProtocolMessage, Error = ProtocolError> + std::fmt::Debug + Sync + Send + 'static,
     >(
         &self,
-        user: &UserId,
-        remote_peer_id: &DirectPeerId,
+        user: &Option<UserId>,
+        remote_peer_id: &Option<DirectPeerId>, // None means local
         msg: A,
     ) -> Result<SoS<B>, NgError> {
         let bpi = self
             .peers
-            .get(&(Some(*user), remote_peer_id.to_dh_slice()))
+            .get(&(*user, remote_peer_id.map(|rpi| rpi.to_dh_slice())))
             .ok_or(NgError::ConnectionNotFound)?;
-        if let PeerConnection::Client(cnx) = &bpi.connected {
-            cnx.request(msg).await
+        match &bpi.connected {
+            PeerConnection::Client(cnx) => cnx.request(msg).await,
+            PeerConnection::Local(lt) => lt.client_cnx.request(msg).await,
+            _ => Err(NgError::BrokerError),
+        }
+    }
+
+    #[cfg(not(target_arch = "wasm32"))]
+    fn get_fsm_for_client(&self, client: &ClientPeerId) -> Option<Arc<Mutex<NoiseFSM>>> {
+        match client {
+            ClientPeerId::Local((user, _)) => {
+                if let Some(BrokerPeerInfo {
+                    connected:
+                        PeerConnection::Local(LocalTransport {
+                            server_cnx: ConnectionBase { fsm: Some(fsm), .. },
+                            ..
+                        }),
+                    ..
+                }) = self.peers.get(&(Some(*user), None))
+                {
+                    Some(Arc::clone(fsm))
                 } else {
-            Err(NgError::BrokerError)
+                    None
+                }
+            }
+            ClientPeerId::Remote(peer) => {
+                if let Some(BrokerPeerInfo {
+                    connected: PeerConnection::Client(ConnectionBase { fsm: Some(fsm), .. }),
+                    ..
+                }) = self.peers.get(&(None, Some(peer.to_dh())))
+                {
+                    Some(Arc::clone(fsm))
+                } else {
+                    None
+                }
+            }
         }
     }
 
     #[cfg(not(target_arch = "wasm32"))]
     pub(crate) async fn dispatch_event(
-        &mut self,
+        &self,
         overlay: &OverlayId,
         event: Event,
         user_id: &UserId,
         remote_peer: &PubKey,
-    ) -> Result<(), ServerError> {
+    ) -> Result<Vec<ClientPeerId>, ServerError> {
         // TODO: deal with subscriptions on the outer overlay. for now we assume everything is on the inner overlay
 
-        let peers_for_local_dispatch: Vec<PubKey> = self
-            .get_server_broker()?
-            .dispatch_event(overlay, event.clone(), user_id, remote_peer)?
-            .into_iter()
-            .cloned()
-            .collect();
+        let mut clients_to_remove = vec![];
 
-        //log_debug!("dispatch_event {:?}", peers_for_local_dispatch);
+        let peers_for_local_dispatch = {
+            self.get_server_broker()?
+                .read()
+                .await
+                .dispatch_event(overlay, event.clone(), user_id, remote_peer)
+                .await?
+        };
 
-        for peer in peers_for_local_dispatch {
-            //log_debug!("dispatch_event peer {:?}", peer);
-            if let Some(BrokerPeerInfo {
-                connected: PeerConnection::Client(ConnectionBase { fsm: Some(fsm), .. }),
-                ..
-            }) = self.peers.get(&(None, peer.to_dh()))
-            {
-                //log_debug!("ForwardedEvent peer {:?}", peer);
+        for client in peers_for_local_dispatch {
+            //log_debug!("dispatch_event peer {:?}", client);
+            if let Some(fsm) = self.get_fsm_for_client(&client) {
+                //log_debug!("ForwardedEvent peer {:?}", client);
                 let _ = fsm
                     .lock()
                     .await
@@ -929,15 +1125,19 @@ impl Broker {
                     .await;
             } else {
                 // we remove the peer from all local_subscriptions
-                self.get_server_broker_mut()?
-                    .remove_all_subscriptions_of_peer(&peer);
+                clients_to_remove.push(client);
             }
         }
 
-        Ok(())
+        Ok(clients_to_remove)
     }
 
-    async fn close_peer_connection_x(&mut self, peer_id: X25519PubKey, user: Option<PubKey>) {
+    #[doc(hidden)]
+    pub async fn close_peer_connection_x(
+        &mut self,
+        peer_id: Option<X25519PubKey>,
+        user: Option<PubKey>,
+    ) {
         if let Some(peer) = self.peers.get_mut(&(user, peer_id)) {
             match &mut peer.connected {
                 PeerConnection::Core(_) => {
@@ -948,13 +1148,24 @@ impl Broker {
                     cb.close().await;
                 }
                 PeerConnection::NONE => {}
+                PeerConnection::Local(lt) => {
+                    assert!(peer_id.is_none());
+                    assert!(user.is_some());
+                    lt.close().await;
+                    if self.peers.remove(&(user, None)).is_some() {
+                        log_debug!(
+                            "Local transport connection closed ! {}",
+                            user.unwrap().to_string()
+                        );
+                    }
+                }
             }
             //self.peers.remove(peer_id); // this is done in the watch_close instead
         }
     }
 
     pub async fn close_peer_connection(&mut self, peer_id: &DirectPeerId, user: Option<PubKey>) {
-        self.close_peer_connection_x(peer_id.to_dh_slice(), user)
+        self.close_peer_connection_x(Some(peer_id.to_dh_slice()), user)
             .await
     }
 

ng-net/src/connection.rs

@@ -38,7 +38,7 @@ use ng_repo::utils::verify;
 
 use crate::actor::{Actor, SoS};
 use crate::actors::*;
-use crate::broker::BROKER;
+use crate::broker::{ClientPeerId, BROKER};
 use crate::types::*;
 use crate::utils::*;
 
@@ -117,6 +117,8 @@ pub enum FSMstate {
     ServerHello,
     ClientAuth,
     AuthResult,
+    AppHello,
+    AppHello2,
     Closing,
 }
 
@@ -138,6 +140,7 @@ pub struct NoiseFSM {
     local: Option<PrivKey>,
     remote: Option<PubKey>,
 
+    #[allow(dead_code)]
     nonce_for_hello: Vec<u8>,
     config: Option<StartConfig>,
 
@@ -191,6 +194,13 @@ pub struct AdminConfig {
     pub request: AdminRequestContentV0,
 }
 
+#[derive(Debug, Clone)]
+pub struct AppConfig {
+    pub addr: BindAddress,
+    pub info: ClientInfo,
+    pub user_priv: Option<PrivKey>,
+}
+
 #[derive(Debug, Clone)]
 pub enum StartConfig {
     Probe,
@@ -199,6 +209,7 @@ pub enum StartConfig {
     Ext(ExtConfig),
     Core(CoreConfig),
     Admin(AdminConfig),
+    App(AppConfig),
 }
 
 impl StartConfig {
@@ -207,18 +218,20 @@ impl StartConfig {
             Self::Client(config) => config.url.clone(),
             Self::Admin(config) => format!("ws://{}:{}", config.addr.ip, config.addr.port),
             Self::Core(config) => format!("ws://{}:{}", config.addr.ip, config.addr.port),
+            Self::App(config) => format!("ws://{}:{}", config.addr.ip, config.addr.port),
             _ => unimplemented!(),
         }
     }
     pub fn get_user(&self) -> Option<PubKey> {
         match self {
             Self::Client(config) => Some(config.user_priv.to_pub()),
+            Self::App(config) => config.user_priv.as_ref().map(|k| k.to_pub()),
             _ => None,
         }
     }
     pub fn is_keep_alive(&self) -> bool {
         match self {
-            StartConfig::Core(_) | StartConfig::Client(_) => true,
+            StartConfig::Core(_) | StartConfig::Client(_) | StartConfig::App(_) => true,
             _ => false,
         }
     }
@@ -272,6 +285,21 @@ impl NoiseFSM {
         &self.remote
     }
 
+    pub fn get_client_peer_id(&self) -> Result<ClientPeerId, ProtocolError> {
+        Ok(match self.state {
+            FSMstate::Local0 => {
+                ClientPeerId::new_from(
+                    &self.remote_peer().ok_or(ProtocolError::ActorError)?,
+                    &Some(self.user.unwrap()),
+                )
+                // the unwrap and Some is on purpose. to enforce that we do have a user
+            }
+            _ => {
+                ClientPeerId::new_from(&self.remote_peer().ok_or(ProtocolError::ActorError)?, &None)
+            }
+        })
+    }
+
     #[cfg(not(target_arch = "wasm32"))]
     pub(crate) fn set_user_id(&mut self, user: UserId) {
         if self.user.is_none() {
@@ -387,6 +415,7 @@ impl NoiseFSM {
         return Ok(StepReply::NONE);
     }
 
+    #[allow(dead_code)]
     fn process_server_noise3(&mut self, noise: &Noise) -> Result<(), ProtocolError> {
         let handshake = self.noise_handshake_state.as_mut().unwrap();
 
@@ -436,22 +465,22 @@ impl NoiseFSM {
         match self.state {
             FSMstate::Closing => {}
             // TODO verify that ID is zero
-            FSMstate::Local0 => {
-                // CLIENT LOCAL
-                if !self.dir.is_server() && msg_opt.is_none() {
-                    self.state = FSMstate::ClientHello;
-                    //Box::new(Actor::<ClientHello, ServerHello>::new(0, true));
-                    return Ok(StepReply::NONE);
-                }
-                // SERVER LOCAL
-                else if let Some(msg) = msg_opt.as_ref() {
-                    if self.dir.is_server() && msg.type_id() == ClientHello::Local.type_id() {
-                        self.state = FSMstate::ServerHello;
-                        //Box::new(Actor::<ClientHello, ServerHello>::new(msg.id(), false));
-                        return Ok(StepReply::NONE);
-                    }
-                }
-            }
+            // FSMstate::Local0 => {
+            //     // CLIENT LOCAL
+            //     if !self.dir.is_server() && msg_opt.is_none() {
+            //         self.state = FSMstate::ClientHello;
+            //         //Box::new(Actor::<ClientHello, ServerHello>::new(0, true));
+            //         return Ok(StepReply::NONE);
+            //     }
+            //     // SERVER LOCAL
+            //     else if let Some(msg) = msg_opt.as_ref() {
+            //         if self.dir.is_server() && msg.type_id() == ClientHello::Local.type_id() {
+            //             self.state = FSMstate::ServerHello;
+            //             //Box::new(Actor::<ClientHello, ServerHello>::new(msg.id(), false));
+            //             return Ok(StepReply::NONE);
+            //         }
+            //     }
+            // }
             FSMstate::Start => {
                 if !self.dir.is_server() && msg_opt.is_none() {
                     // CLIENT START
@@ -521,6 +550,7 @@ impl NoiseFSM {
                                                 .ok_or(ProtocolError::BrokerError)?,
                                             Authorization::Discover,
                                         )
+                                        .await
                                         .is_ok()
                                     {
                                         probe_response.peer_id = Some(
@@ -610,12 +640,21 @@ impl NoiseFSM {
                                 StartConfig::Core(_core_config) => {
                                     todo!();
                                 }
-                                StartConfig::Admin(_admin_config) => {
+                                StartConfig::Admin(_) => {
                                     let noise = Noise::V0(NoiseV0 { data: payload });
                                     self.send(noise.into()).await?;
                                     self.state = FSMstate::Noise3;
                                     next_step = StepReply::ReEnter;
                                 }
+                                StartConfig::App(app_config) => {
+                                    let app_hello = AppHello {
+                                        noise: Noise::V0(NoiseV0 { data: payload }),
+                                        user: app_config.user_priv.as_ref().map(|k| k.to_pub()),
+                                        info: app_config.info.clone(),
+                                    };
+                                    self.send(app_hello.into()).await?;
+                                    self.state = FSMstate::AppHello;
+                                }
                                 _ => return Err(ProtocolError::InvalidState),
                             }
 
@@ -629,8 +668,38 @@ impl NoiseFSM {
                     }
                 }
             }
+            FSMstate::AppHello => {
+                if let Some(msg) = msg_opt.as_ref() {
+                    if !self.dir.is_server() {
+                        if let ProtocolMessage::Start(StartProtocol::AppResponse(hello_response)) =
+                            msg
+                        {
+                            if hello_response.result != 0 {
+                                return Err(ProtocolError::AccessDenied);
+                            }
+
+                            self.state = FSMstate::AppHello2;
+
+                            log_debug!("AUTHENTICATION SUCCESSFUL ! waiting for APP requests on the client side");
+
+                            // we notify the actor "Connecting" that the connection is ready
+                            let mut lock = self.actors.lock().await;
+                            let exists = lock.remove(&0);
+                            match exists {
+                                Some(mut actor_sender) => {
+                                    let _ = actor_sender.send(ConnectionCommand::ReEnter).await;
+                                }
+                                _ => {}
+                            }
+
+                            return Ok(StepReply::NONE);
+                        }
+                    }
+                }
+            }
             FSMstate::Noise2 => {
                 // SERVER second round NOISE
+                #[cfg(not(target_arch = "wasm32"))]
                 if let Some(msg) = msg_opt.as_ref() {
                     if self.dir.is_server() {
                         if let ProtocolMessage::Start(StartProtocol::Client(ClientHello::Noise3(
@@ -657,6 +726,42 @@ impl NoiseFSM {
 
                             self.state = FSMstate::Noise3;
 
+                            return Ok(StepReply::NONE);
+                        } else if let ProtocolMessage::Start(StartProtocol::App(app_hello)) = msg {
+                            self.process_server_noise3(&app_hello.noise)?;
+
+                            let (local_bind_address, remote_bind_address) =
+                                self.bind_addresses.ok_or(ProtocolError::BrokerError)?;
+                            let result = BROKER
+                                .write()
+                                .await
+                                .attach_and_authorize_app(
+                                    remote_bind_address,
+                                    local_bind_address,
+                                    *self.remote.unwrap().slice(),
+                                    &app_hello.user,
+                                    &app_hello.info,
+                                )
+                                .await
+                                .err()
+                                .unwrap_or(ProtocolError::NoError);
+
+                            let hello_response = AppHelloResponse {
+                                result: result.clone() as u16,
+                            };
+                            self.send(hello_response.into()).await?;
+
+                            if result.is_err() {
+                                return Err(result);
+                            }
+                            if app_hello.user.is_some() {
+                                self.set_user_id(app_hello.user.unwrap());
+                            }
+
+                            log_debug!("AUTHENTICATION SUCCESSFUL ! waiting for APP requests on the server side");
+
+                            self.state = FSMstate::AppHello2;
+
                             return Ok(StepReply::NONE);
                         }
                     }
@@ -709,11 +814,18 @@ impl NoiseFSM {
                             //     todo!();
                             // }
                             StartProtocol::Admin(AdminRequest::V0(req)) => {
-                                BROKER.read().await.authorize(
-                                    &self.bind_addresses.ok_or(ProtocolError::BrokerError)?,
+                                {
+                                    BROKER
+                                        .read()
+                                        .await
+                                        .authorize(
+                                            &self
+                                                .bind_addresses
+                                                .ok_or(ProtocolError::BrokerError)?,
                                             Authorization::Admin(req.admin_user),
-                                )?;
-
+                                        )
+                                        .await?;
+                                }
                                 // PROCESS AdminRequest and send back AdminResponse
                                 let ser = serde_bare::to_vec(&req.content)?;
 
@@ -859,7 +971,24 @@ impl NoiseFSM {
                     }
                 }
             }
-            FSMstate::AuthResult => {
+            FSMstate::AppHello2 => {
+                if let Some(msg) = msg_opt {
+                    if msg.type_id() != TypeId::of::<AppMessage>() {
+                        return Err(ProtocolError::AccessDenied);
+                    }
+                    match msg.id() {
+                        Some(id) => {
+                            if self.dir.is_server() && id > 0 || !self.dir.is_server() && id < 0 {
+                                return Ok(StepReply::Responder(msg));
+                            } else if id != 0 {
+                                return Ok(StepReply::Response(msg));
+                            }
+                        }
+                        None => return Err(ProtocolError::InvalidMessage),
+                    }
+                }
+            }
+            FSMstate::AuthResult | FSMstate::Local0 => {
                 if let Some(msg) = msg_opt {
                     if msg.type_id() != TypeId::of::<ClientMessage>() {
                         return Err(ProtocolError::AccessDenied);
@@ -902,7 +1031,7 @@ pub struct ConnectionBase {
     sender: Option<Receiver<ConnectionCommand>>,
     receiver: Option<Sender<ConnectionCommand>>,
     sender_tx: Option<Sender<ConnectionCommand>>,
-    receiver_tx: Option<Sender<ConnectionCommand>>,
+    //receiver_tx: Option<Sender<ConnectionCommand>>,
     shutdown: Option<Receiver<Either<NetError, X25519PrivKey>>>,
     shutdown_sender: Option<Sender<Either<NetError, X25519PrivKey>>>,
     dir: ConnectionDir,
@@ -913,13 +1042,69 @@ pub struct ConnectionBase {
 }
 
 impl ConnectionBase {
+    pub fn create_local_transport_pipe(user: UserId, client_peer_id: DirectPeerId) -> (Self, Self) {
+        let mut client_cnx = Self::new(ConnectionDir::Client, TransportProtocol::Local);
+        let mut server_cnx = Self::new(ConnectionDir::Server, TransportProtocol::Local);
+
+        let (sender_tx, sender_rx) = mpsc::unbounded();
+        let (receiver_tx, receiver_rx) = mpsc::unbounded();
+
+        // SETTING UP THE CLIENT
+        client_cnx.sender_tx = Some(sender_tx.clone());
+
+        let fsm = Arc::new(Mutex::new(NoiseFSM::new(
+            None,
+            client_cnx.tp,
+            client_cnx.dir.clone(),
+            Arc::clone(&client_cnx.actors),
+            sender_tx.clone(),
+            None,
+            None,
+        )));
+        client_cnx.fsm = Some(Arc::clone(&fsm));
+
+        spawn_and_log_error(Self::read_loop(
+            receiver_tx.clone(),
+            receiver_rx,
+            sender_tx.clone(),
+            Arc::clone(&client_cnx.actors),
+            fsm,
+        ));
+
+        // SETTING UP THE SERVER
+        server_cnx.sender_tx = Some(receiver_tx.clone());
+
+        let mut fsm_mut = NoiseFSM::new(
+            None,
+            server_cnx.tp,
+            server_cnx.dir.clone(),
+            Arc::clone(&server_cnx.actors),
+            receiver_tx.clone(),
+            None,
+            Some(client_peer_id),
+        );
+        fsm_mut.user = Some(user);
+        let fsm = Arc::new(Mutex::new(fsm_mut));
+        server_cnx.fsm = Some(Arc::clone(&fsm));
+
+        spawn_and_log_error(Self::read_loop(
+            sender_tx,
+            sender_rx,
+            receiver_tx,
+            Arc::clone(&server_cnx.actors),
+            fsm,
+        ));
+
+        (client_cnx, server_cnx)
+    }
+
     pub fn new(dir: ConnectionDir, tp: TransportProtocol) -> Self {
         Self {
             fsm: None,
             receiver: None,
             sender: None,
             sender_tx: None,
-            receiver_tx: None,
+            //receiver_tx: None,
             shutdown: None,
             shutdown_sender: None,
             next_request_id: SequenceGenerator::new(1),
@@ -1114,7 +1299,7 @@ impl ConnectionBase {
         res
     }
 
-    // FIXME: why not use the FSm instead? looks like this is sending messages to the wire, unencrypted.
+    // FIXME: why not use the FSM instead? looks like this is sending messages to the wire, unencrypted.
     // Only final errors are sent this way. but it looks like even those error should be encrypted
     pub async fn send(&mut self, cmd: ConnectionCommand) {
         let _ = self.sender_tx.as_mut().unwrap().send(cmd).await;
@@ -1277,7 +1462,7 @@ impl ConnectionBase {
         self.sender = Some(sender_rx);
         self.receiver = Some(receiver_tx.clone());
         self.sender_tx = Some(sender_tx.clone());
-        self.receiver_tx = Some(receiver_tx.clone());
+        //self.receiver_tx = Some(receiver_tx.clone());
 
         let fsm = Arc::new(Mutex::new(NoiseFSM::new(
             bind_addresses,

ng-net/src/lib.rs

@@ -11,6 +11,9 @@
 
 pub mod types;
 
+#[doc(hidden)]
+pub mod app_protocol;
+
 pub mod broker;
 
 pub mod server_broker;

ng-net/src/server_broker.rs

@@ -11,18 +11,35 @@
 
 //! Trait for ServerBroker
 
-use std::collections::HashSet;
+use std::path::PathBuf;
+use std::sync::Arc;
 
+use async_std::sync::Mutex;
+
+use ng_repo::block_storage::BlockStorage;
 use ng_repo::errors::*;
 use ng_repo::types::*;
 
+use crate::app_protocol::{AppRequest, AppSessionStart, AppSessionStartResponse, AppSessionStop};
+use crate::broker::ClientPeerId;
+use crate::connection::NoiseFSM;
 use crate::types::*;
 
+#[async_trait::async_trait]
 pub trait IServerBroker: Send + Sync {
+    fn get_path_users(&self) -> PathBuf;
+    fn get_block_storage(&self) -> Arc<std::sync::RwLock<dyn BlockStorage + Send + Sync>>;
     fn put_block(&self, overlay_id: &OverlayId, block: Block) -> Result<(), ServerError>;
     fn has_block(&self, overlay_id: &OverlayId, block_id: &BlockId) -> Result<(), ServerError>;
     fn get_block(&self, overlay_id: &OverlayId, block_id: &BlockId) -> Result<Block, ServerError>;
+    async fn create_user(&self, broker_id: &DirectPeerId) -> Result<UserId, ProtocolError>;
     fn get_user(&self, user_id: PubKey) -> Result<bool, ProtocolError>;
+    fn get_user_credentials(&self, user_id: &PubKey) -> Result<Credentials, ProtocolError>;
+    fn add_user_credentials(
+        &self,
+        user_id: &PubKey,
+        credentials: &Credentials,
+    ) -> Result<(), ProtocolError>;
     fn add_user(&self, user_id: PubKey, is_admin: bool) -> Result<(), ProtocolError>;
     fn del_user(&self, user_id: PubKey) -> Result<(), ProtocolError>;
     fn list_users(&self, admins: bool) -> Result<Vec<PubKey>, ProtocolError>;
@@ -41,6 +58,20 @@ pub trait IServerBroker: Send + Sync {
     fn get_invitation_type(&self, invite: [u8; 32]) -> Result<u8, ProtocolError>;
     fn remove_invitation(&self, invite: [u8; 32]) -> Result<(), ProtocolError>;
 
+    async fn app_process_request(
+        &self,
+        req: AppRequest,
+        request_id: i64,
+        fsm: &Mutex<NoiseFSM>,
+    ) -> Result<(), ServerError>;
+    async fn app_session_start(
+        &self,
+        req: AppSessionStart,
+        remote_peer_id: DirectPeerId,
+        local_peer_id: DirectPeerId,
+    ) -> Result<AppSessionStartResponse, ServerError>;
+    fn app_session_stop(&self, req: AppSessionStop) -> Result<EmptyAppResponse, ServerError>;
+
     fn next_seq_for_peer(&self, peer: &PeerId, seq: u64) -> Result<(), ServerError>;
 
     fn get_repo_pin_status(
@@ -50,8 +81,8 @@ pub trait IServerBroker: Send + Sync {
         user_id: &UserId,
     ) -> Result<RepoPinStatus, ServerError>;
 
-    fn pin_repo_write(
-        &mut self,
+    async fn pin_repo_write(
+        &self,
         overlay: &OverlayAccess,
         repo: &RepoHash,
         user_id: &UserId,
@@ -59,39 +90,39 @@ pub trait IServerBroker: Send + Sync {
         rw_topics: &Vec<PublisherAdvert>,
         overlay_root_topic: &Option<TopicId>,
         expose_outer: bool,
-        peer: &PubKey,
+        peer: &ClientPeerId,
     ) -> Result<RepoOpened, ServerError>;
 
-    fn pin_repo_read(
-        &mut self,
+    async fn pin_repo_read(
+        &self,
         overlay: &OverlayId,
         repo: &RepoHash,
         user_id: &UserId,
         ro_topics: &Vec<TopicId>,
-        peer: &PubKey,
+        peer: &ClientPeerId,
     ) -> Result<RepoOpened, ServerError>;
 
-    fn topic_sub(
-        &mut self,
+    async fn topic_sub(
+        &self,
         overlay: &OverlayId,
         repo: &RepoHash,
         topic: &TopicId,
         user_id: &UserId,
         publisher: Option<&PublisherAdvert>,
-        peer: &PubKey,
+        peer: &ClientPeerId,
     ) -> Result<TopicSubRes, ServerError>;
 
     fn get_commit(&self, overlay: &OverlayId, id: &ObjectId) -> Result<Vec<Block>, ServerError>;
 
-    fn dispatch_event(
+    async fn dispatch_event(
         &self,
         overlay: &OverlayId,
         event: Event,
         user_id: &UserId,
         remote_peer: &PubKey,
-    ) -> Result<HashSet<&PubKey>, ServerError>;
+    ) -> Result<Vec<ClientPeerId>, ServerError>;
 
-    fn remove_all_subscriptions_of_peer(&mut self, remote_peer: &PubKey);
+    async fn remove_all_subscriptions_of_client(&self, client: &ClientPeerId);
 
     fn topic_sync_req(
         &self,

ng-net/src/types.rs

@@ -27,6 +27,7 @@ use ng_repo::store::Store;
 use ng_repo::types::*;
 use ng_repo::utils::{sign, verify};
 
+use crate::app_protocol::*;
 use crate::utils::{
     get_domain_without_port_443, is_ipv4_private, is_ipv6_private, is_private_ip, is_public_ip,
     is_public_ipv4, is_public_ipv6,
@@ -34,6 +35,32 @@ use crate::utils::{
 use crate::WS_PORT_ALTERNATE;
 use crate::{actor::EActor, actors::admin::*, actors::*};
 
+#[derive(Debug, Clone, Serialize, Deserialize)]
+/// used to initiate a session at a local broker V0
+pub struct Credentials {
+    pub user_key: PrivKey,
+    pub read_cap: ReadCap,
+    pub private_store: RepoId,
+    pub protected_store: RepoId,
+    pub public_store: RepoId,
+    pub user_master_key: SymKey,
+    pub peer_priv_key: PrivKey,
+}
+
+impl Credentials {
+    pub fn new_partial(user_priv_key: &PrivKey) -> Self {
+        Credentials {
+            user_key: user_priv_key.clone(),
+            read_cap: ReadCap::nil(),
+            private_store: RepoId::nil(),
+            protected_store: RepoId::nil(),
+            public_store: RepoId::nil(),
+            user_master_key: SymKey::random(),
+            peer_priv_key: PrivKey::random_ed(),
+        }
+    }
+}
+
 //
 //  Network common types
 //
@@ -104,6 +131,12 @@ impl BindAddress {
             if self.port == 0 { 80 } else { self.port }
         )
     }
+    pub fn new_localhost_with_port(port: u16) -> Self {
+        BindAddress {
+            ip: LOOPBACK_IPV4.clone(),
+            port,
+        }
+    }
 }
 
 impl From<&SocketAddr> for BindAddress {
@@ -1325,6 +1358,8 @@ pub type ClientId = PubKey;
 /// IPv4 address
 pub type IPv4 = [u8; 4];
 
+const LOOPBACK_IPV4: IP = IP::IPv4([127, 0, 0, 1]);
+
 /// IPv6 address
 pub type IPv6 = [u8; 16];
 
@@ -2443,6 +2478,76 @@ pub enum CoreMessage {
     V0(CoreMessageV0),
 }
 
+/// AppMessageContentV0
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum AppMessageContentV0 {
+    Request(AppRequest),
+    Response(AppResponse),
+    SessionStop(AppSessionStop),
+    SessionStart(AppSessionStart),
+    EmptyResponse,
+}
+
+/// AppMessageV0
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct AppMessageV0 {
+    pub content: AppMessageContentV0,
+
+    pub id: i64,
+
+    pub result: u16,
+}
+
+/// App message
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum AppMessage {
+    V0(AppMessageV0),
+}
+
+impl IStreamable for AppMessage {
+    fn result(&self) -> u16 {
+        match self {
+            AppMessage::V0(v0) => v0.result,
+        }
+    }
+    fn set_result(&mut self, result: u16) {
+        match self {
+            AppMessage::V0(v0) => v0.result = result,
+        }
+    }
+}
+
+impl AppMessage {
+    pub fn get_actor(&self) -> Box<dyn EActor> {
+        match self {
+            AppMessage::V0(AppMessageV0 { content: o, id, .. }) => match o {
+                AppMessageContentV0::Request(req) => req.get_actor(*id),
+                AppMessageContentV0::SessionStop(req) => req.get_actor(*id),
+                AppMessageContentV0::SessionStart(req) => req.get_actor(*id),
+                AppMessageContentV0::Response(_) | AppMessageContentV0::EmptyResponse => {
+                    panic!("it is not a request");
+                }
+            },
+        }
+    }
+    pub fn id(&self) -> Option<i64> {
+        match self {
+            AppMessage::V0(v0) => Some(v0.id),
+        }
+    }
+    pub fn set_id(&mut self, id: i64) {
+        match self {
+            AppMessage::V0(r) => r.id = id,
+        }
+    }
+}
+
+impl From<AppMessage> for ProtocolMessage {
+    fn from(msg: AppMessage) -> ProtocolMessage {
+        ProtocolMessage::AppMessage(msg)
+    }
+}
+
 //
 // ADMIN PROTOCOL
 //
@@ -2455,6 +2560,8 @@ pub enum AdminRequestContentV0 {
     ListUsers(ListUsers),
     ListInvitations(ListInvitations),
     AddInvitation(AddInvitation),
+    #[doc(hidden)]
+    CreateUser(CreateUser),
 }
 impl AdminRequestContentV0 {
     pub fn type_id(&self) -> TypeId {
@@ -2464,6 +2571,7 @@ impl AdminRequestContentV0 {
             Self::ListUsers(a) => a.type_id(),
             Self::ListInvitations(a) => a.type_id(),
             Self::AddInvitation(a) => a.type_id(),
+            Self::CreateUser(a) => a.type_id(),
         }
     }
     pub fn get_actor(&self) -> Box<dyn EActor> {
@@ -2473,6 +2581,7 @@ impl AdminRequestContentV0 {
             Self::ListUsers(a) => a.get_actor(),
             Self::ListInvitations(a) => a.get_actor(),
             Self::AddInvitation(a) => a.get_actor(),
+            Self::CreateUser(a) => a.get_actor(),
         }
     }
 }
@@ -2557,6 +2666,7 @@ pub enum AdminResponseContentV0 {
     Users(Vec<PubKey>),
     Invitations(Vec<(InvitationCode, u32, Option<String>)>),
     Invitation(Invitation),
+    UserId(UserId),
 }
 
 /// Response to an `AdminRequest` V0
@@ -2592,6 +2702,25 @@ impl From<Result<(), ProtocolError>> for AdminResponseV0 {
     }
 }
 
+impl From<Result<PubKey, ProtocolError>> for AdminResponseV0 {
+    fn from(res: Result<PubKey, ProtocolError>) -> AdminResponseV0 {
+        match res {
+            Err(e) => AdminResponseV0 {
+                id: 0,
+                result: e.into(),
+                content: AdminResponseContentV0::EmptyResponse,
+                padding: vec![],
+            },
+            Ok(id) => AdminResponseV0 {
+                id: 0,
+                result: 0,
+                content: AdminResponseContentV0::UserId(id),
+                padding: vec![],
+            },
+        }
+    }
+}
+
 impl From<Result<Vec<PubKey>, ProtocolError>> for AdminResponseV0 {
     fn from(res: Result<Vec<PubKey>, ProtocolError>) -> AdminResponseV0 {
         match res {
@@ -3508,6 +3637,19 @@ impl From<ServerError> for ClientResponse {
     }
 }
 
+#[derive(Debug)]
+pub struct EmptyAppResponse(pub ());
+
+impl From<ServerError> for AppMessage {
+    fn from(err: ServerError) -> AppMessage {
+        AppMessage::V0(AppMessageV0 {
+            id: 0,
+            result: err.into(),
+            content: AppMessageContentV0::EmptyResponse,
+        })
+    }
+}
+
 impl<A> From<Result<A, ServerError>> for ProtocolMessage
 where
     A: Into<ProtocolMessage> + std::fmt::Debug,
@@ -3615,12 +3757,44 @@ pub struct ClientMessageV0 {
     pub padding: Vec<u8>,
 }
 
+pub trait IStreamable {
+    fn result(&self) -> u16;
+    fn set_result(&mut self, result: u16);
+}
+
 /// Broker message for an overlay
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub enum ClientMessage {
     V0(ClientMessageV0),
 }
 
+impl IStreamable for ClientMessage {
+    fn result(&self) -> u16 {
+        match self {
+            ClientMessage::V0(o) => match &o.content {
+                ClientMessageContentV0::ClientResponse(r) => r.result(),
+                ClientMessageContentV0::ClientRequest(_)
+                | ClientMessageContentV0::ForwardedEvent(_)
+                | ClientMessageContentV0::ForwardedBlock(_) => {
+                    panic!("it is not a response");
+                }
+            },
+        }
+    }
+    fn set_result(&mut self, result: u16) {
+        match self {
+            ClientMessage::V0(o) => match &mut o.content {
+                ClientMessageContentV0::ClientResponse(r) => r.set_result(result),
+                ClientMessageContentV0::ClientRequest(_)
+                | ClientMessageContentV0::ForwardedEvent(_)
+                | ClientMessageContentV0::ForwardedBlock(_) => {
+                    panic!("it is not a response");
+                }
+            },
+        }
+    }
+}
+
 impl ClientMessage {
     pub fn content_v0(&self) -> &ClientMessageContentV0 {
         match self {
@@ -3686,18 +3860,7 @@ impl ClientMessage {
             },
         }
     }
-    pub fn result(&self) -> u16 {
-        match self {
-            ClientMessage::V0(o) => match &o.content {
-                ClientMessageContentV0::ClientResponse(r) => r.result(),
-                ClientMessageContentV0::ClientRequest(_)
-                | ClientMessageContentV0::ForwardedEvent(_)
-                | ClientMessageContentV0::ForwardedBlock(_) => {
-                    panic!("it is not a response");
-                }
-            },
-        }
-    }
+
     pub fn block<'a>(&self) -> Option<&Block> {
         match self {
             ClientMessage::V0(o) => match &o.content {
@@ -3947,6 +4110,7 @@ pub enum ProtocolMessage {
     //AdminRequest(AdminRequest),
     AdminResponse(AdminResponse),
     ClientMessage(ClientMessage),
+    AppMessage(AppMessage),
     CoreMessage(CoreMessage),
 }
 
@@ -3959,6 +4123,12 @@ impl TryFrom<&ProtocolMessage> for ServerError {
                 return Ok(ServerError::try_from(res).unwrap());
             }
         }
+        if let ProtocolMessage::AppMessage(ref bm) = msg {
+            let res = bm.result();
+            if res != 0 {
+                return Ok(ServerError::try_from(res).unwrap());
+            }
+        }
         Err(NgError::NotAServerError)
     }
 }
@@ -3969,6 +4139,7 @@ impl ProtocolMessage {
             ProtocolMessage::ExtRequest(ext_req) => Some(ext_req.id()),
             ProtocolMessage::ExtResponse(ext_res) => Some(ext_res.id()),
             ProtocolMessage::ClientMessage(client_msg) => client_msg.id(),
+            ProtocolMessage::AppMessage(app_msg) => app_msg.id(),
             _ => None,
         }
     }
@@ -3977,6 +4148,7 @@ impl ProtocolMessage {
             ProtocolMessage::ExtRequest(ext_req) => ext_req.set_id(id),
             ProtocolMessage::ExtResponse(ext_res) => ext_res.set_id(id),
             ProtocolMessage::ClientMessage(client_msg) => client_msg.set_id(id),
+            ProtocolMessage::AppMessage(app_msg) => app_msg.set_id(id),
             _ => panic!("cannot set ID"),
         }
     }
@@ -3991,6 +4163,7 @@ impl ProtocolMessage {
             ProtocolMessage::ExtResponse(a) => a.type_id(),
             ProtocolMessage::ClientMessage(a) => a.type_id(),
             ProtocolMessage::CoreMessage(a) => a.type_id(),
+            ProtocolMessage::AppMessage(a) => a.type_id(),
             //ProtocolMessage::AdminRequest(a) => a.type_id(),
             ProtocolMessage::AdminResponse(a) => a.type_id(),
             ProtocolMessage::Probe(a) => a.type_id(),
@@ -4002,11 +4175,26 @@ impl ProtocolMessage {
         }
     }
 
+    pub(crate) fn is_streamable(&self) -> Option<&dyn IStreamable> {
+        match self {
+            ProtocolMessage::ClientMessage(s) => Some(s as &dyn IStreamable),
+            ProtocolMessage::AppMessage(s) => Some(s as &dyn IStreamable),
+            // ProtocolMessage::ServerHello(a) => a.get_actor(),
+            // ProtocolMessage::ClientAuth(a) => a.get_actor(),
+            // ProtocolMessage::AuthResult(a) => a.get_actor(),
+            // ProtocolMessage::ExtRequest(a) => a.get_actor(),
+            // ProtocolMessage::ExtResponse(a) => a.get_actor(),
+            // ProtocolMessage::BrokerMessage(a) => a.get_actor(),
+            _ => None,
+        }
+    }
+
     pub fn get_actor(&self) -> Box<dyn EActor> {
         match self {
             //ProtocolMessage::Noise(a) => a.get_actor(),
             ProtocolMessage::Start(a) => a.get_actor(),
             ProtocolMessage::ClientMessage(a) => a.get_actor(),
+            ProtocolMessage::AppMessage(a) => a.get_actor(),
             // ProtocolMessage::ServerHello(a) => a.get_actor(),
             // ProtocolMessage::ClientAuth(a) => a.get_actor(),
             // ProtocolMessage::AuthResult(a) => a.get_actor(),

ng-net/src/utils.rs

@@ -14,13 +14,15 @@ use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
 use async_std::task;
 use ed25519_dalek::*;
 use futures::{channel::mpsc, Future};
+use lazy_static::lazy_static;
 use noise_protocol::U8Array;
 use noise_protocol::DH;
 use noise_rust_crypto::sensitive::Sensitive;
+use regex::Regex;
 use url::Host;
 use url::Url;
 
-#[cfg(target_arch = "wasm32")]
+#[allow(unused_imports)]
 use ng_repo::errors::*;
 use ng_repo::types::PubKey;
 use ng_repo::{log::*, types::PrivKey};
@@ -28,6 +30,7 @@ use ng_repo::{log::*, types::PrivKey};
 use crate::types::*;
 #[cfg(target_arch = "wasm32")]
 use crate::NG_BOOTSTRAP_LOCAL_PATH;
+use crate::WS_PORT;
 
 #[doc(hidden)]
 #[cfg(target_arch = "wasm32")]
@@ -72,6 +75,99 @@ pub fn decode_invitation_string(string: String) -> Option<Invitation> {
     Invitation::try_from(string).ok()
 }
 
+lazy_static! {
+    #[doc(hidden)]
+    static ref RE_IPV6_WITH_PORT: Regex =
+        Regex::new(r"^\[([0-9a-fA-F:]{3,39})\](\:\d{1,5})?$").unwrap();
+}
+
+#[doc(hidden)]
+pub fn parse_ipv4_and_port_for(
+    string: String,
+    for_option: &str,
+    default_port: u16,
+) -> Result<(Ipv4Addr, u16), NgError> {
+    let parts: Vec<&str> = string.split(":").collect();
+    let ipv4 = parts[0].parse::<Ipv4Addr>().map_err(|_| {
+        NgError::ConfigError(format!(
+            "The <IPv4:PORT> value submitted for the {} option is invalid.",
+            for_option
+        ))
+    })?;
+
+    let port = if parts.len() > 1 {
+        match serde_json::from_str::<u16>(parts[1]) {
+            Err(_) => default_port,
+            Ok(p) => {
+                if p == 0 {
+                    default_port
+                } else {
+                    p
+                }
+            }
+        }
+    } else {
+        default_port
+    };
+    Ok((ipv4, port))
+}
+
+#[doc(hidden)]
+pub fn parse_ip_and_port_for(string: String, for_option: &str) -> Result<BindAddress, NgError> {
+    let bind = parse_ip_and_port_for_(string, for_option)?;
+    Ok(BindAddress {
+        ip: (&bind.0).into(),
+        port: bind.1,
+    })
+}
+
+fn parse_ip_and_port_for_(string: String, for_option: &str) -> Result<(IpAddr, u16), NgError> {
+    let c = RE_IPV6_WITH_PORT.captures(&string);
+    let ipv6;
+    let port;
+    if c.is_some() && c.as_ref().unwrap().get(1).is_some() {
+        let cap = c.unwrap();
+        let ipv6_str = cap.get(1).unwrap().as_str();
+        port = match cap.get(2) {
+            None => WS_PORT,
+            Some(p) => {
+                let mut chars = p.as_str().chars();
+                chars.next();
+                match serde_json::from_str::<u16>(chars.as_str()) {
+                    Err(_) => WS_PORT,
+                    Ok(p) => {
+                        if p == 0 {
+                            WS_PORT
+                        } else {
+                            p
+                        }
+                    }
+                }
+            }
+        };
+        let ipv6 = ipv6_str.parse::<Ipv6Addr>().map_err(|_| {
+            NgError::ConfigError(format!(
+                "The <[IPv6]:PORT> value submitted for the {} option is invalid.",
+                for_option
+            ))
+        })?;
+        Ok((IpAddr::V6(ipv6), port))
+    } else {
+        // we try just an IPV6 without port
+        let ipv6_res = string.parse::<Ipv6Addr>();
+        if ipv6_res.is_err() {
+            // let's try IPv4
+
+            parse_ipv4_and_port_for(string, for_option, WS_PORT)
+                .map(|ipv4| (IpAddr::V4(ipv4.0), ipv4.1))
+        } else {
+            ipv6 = ipv6_res.unwrap();
+            port = WS_PORT;
+            Ok((IpAddr::V6(ipv6), port))
+        }
+    }
+}
+
 pub fn check_is_local_url(bootstrap: &BrokerServerV0, location: &String) -> Option<String> {
     if location.starts_with(APP_NG_ONE_URL) {
         match &bootstrap.server_type {

ng-repo/src/errors.rs

@@ -77,6 +77,8 @@ pub enum NgError {
     FileError(FileError),
     InternalError,
     OxiGraphError(String),
+    ConfigError(String),
+    LocalBrokerIsHeadless,
 }
 
 impl Error for NgError {}
@@ -252,6 +254,9 @@ pub enum ServerError {
     ProtocolError,
     PeerAlreadySubscribed,
     SubscriptionNotFound,
+    SessionNotFound,
+    SessionDetached,
+    OxiGraphError,
 }
 
 impl From<StorageError> for ServerError {
@@ -277,6 +282,7 @@ impl From<NgError> for ServerError {
     fn from(e: NgError) -> Self {
         match e {
             NgError::InvalidSignature => ServerError::InvalidSignature,
+            NgError::OxiGraphError(_) => ServerError::OxiGraphError,
             _ => ServerError::OtherError,
         }
     }
@@ -319,6 +325,7 @@ pub enum VerifierError {
     BranchNotOpened,
     DoubleBranchSubscription,
     InvalidCommit,
+    LocallyConnected,
 }
 
 impl From<NgError> for VerifierError {
@@ -421,6 +428,7 @@ pub enum ProtocolError {
     WhereIsTheMagic,
 
     InvalidNonce,
+    InvalidMessage,
 } //MAX 949 ProtocolErrors
 
 impl From<NetError> for ProtocolError {

ng-repo/src/types.rs

@@ -230,7 +230,6 @@ impl PrivKey {
         }
     }
 
-    #[deprecated(note = "**Don't use nil method**")]
     pub fn nil() -> PrivKey {
         PrivKey::Ed25519PrivKey([0u8; 32])
     }
@@ -691,7 +690,7 @@ pub enum StoreRepo {
 
 impl fmt::Display for StoreRepo {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        writeln!(
+        write!(
             f,
             "StoreRepo V0 {} {}",
             match self {

ng-sdk-js/app-node/index.js

@@ -6,14 +6,33 @@
 // at your option. All files in the project carrying such
 // notice may not be copied, modified, or distributed except
 // according to those terms.
+
 const WebSocket = require("ws");
 // shim to insert WebSocket in global
 const ng = require("ng-sdk-node");
 global.WebSocket = WebSocket;
 
-const test = require("./test")
-console.log("FROM INDEX");
-ng.test();
-//test.random();
-console.log(ng.start());
+let config = {
+    server_peer_id: "ALyGZgFaDDALXLppJZLS2TrMScG0TQIS68RzRcPv99aN",
+    admin_user_key:"ABI7mSYq1jzulcatewG6ssaFuCjYVVxF_esEmV33oBW4",
+    client_peer_key:"APbhJBuWUUmwZbuYwVm88eJ0b_ZulpSMOlA-9Zwi-S0Q"
+};
+
+ng.init_headless(config).then( async() => {
+    
+    try {
+        //let user_id = "AC6ukMzC_ig85A0y-ivFOI_VXBB_EQJjTz2XnMn9d0nT";
+        
+        let user_id = await ng.admin_create_user(config);
+        console.log("user created: ",user_id);
+        
+        let session = await ng.session_headless_start(user_id);
+        console.log(session);
 
+    } catch (e) {
+        console.error(e);
+    }
+})
+.catch(err => {
+    console.error(err);
+});

ng-sdk-js/app-node/test.js

@@ -1,6 +0,0 @@
-const ng = require("ng-sdk-node");
-
-module.exports.random = function () {
-    console.log("FROM TEST");
-    ng.test()
-  };

ng-sdk-js/js/node.js

@@ -113,6 +113,16 @@ module.exports.version = function () {
   return require('../../../package.json').version;
 }
 
+module.exports.get_env_vars = function () {
+
+  return {
+    server_addr: process.env.NG_HEADLESS_SERVER_ADDR,
+    server_peer_id: process.env.NG_HEADLESS_SERVER_PEER_ID,
+    client_peer_key: process.env.NG_HEADLESS_CLIENT_PEER_KEY,
+    admin_user_key: process.env.NG_HEADLESS_ADMIN_USER_KEY
+  };
+}
+
 module.exports.client_details = function () {
   const process = require('process');
   let arch = osnode.machine? osnode.machine() : process.arch;

ng-sdk-js/src/lib.rs

@@ -25,15 +25,26 @@ use async_std::stream::StreamExt;
 use wasm_bindgen::prelude::*;
 use wasm_bindgen_futures::JsFuture;
 
-use ng_repo::errors::NgError;
+#[allow(unused_imports)]
+use ng_repo::errors::{NgError, ProtocolError};
 use ng_repo::log::*;
 use ng_repo::types::*;
+#[allow(unused_imports)]
+use ng_repo::utils::{decode_key, decode_priv_key};
 
+use ng_net::app_protocol::*;
 use ng_net::broker::*;
-use ng_net::types::{ClientInfo, ClientInfoV0, ClientType, CreateAccountBSP, IP};
-use ng_net::utils::{decode_invitation_string, spawn_and_log_error, Receiver, ResultSend};
-use ng_net::utils::{retrieve_local_bootstrap, retrieve_local_url};
-use ng_net::WS_PORT;
+#[allow(unused_imports)]
+use ng_net::types::{BindAddress, ClientInfo, ClientInfoV0, ClientType, CreateAccountBSP, IP};
+#[allow(unused_imports)]
+use ng_net::utils::{
+    decode_invitation_string, parse_ip_and_port_for, retrieve_local_bootstrap, retrieve_local_url,
+    spawn_and_log_error, Receiver, ResultSend,
+};
+#[allow(unused_imports)]
+use ng_net::{actor::*, actors::admin::*};
+#[allow(unused_imports)]
+use ng_net::{WS_PORT, WS_PORT_REVERSE_PROXY};
 
 use ng_client_ws::remote_ws_wasm::ConnectionWebSocket;
 
@@ -41,7 +52,6 @@ use ng_wallet::types::*;
 use ng_wallet::*;
 
 use nextgraph::local_broker::*;
-use nextgraph::verifier::*;
 
 #[wasm_bindgen]
 pub async fn get_local_bootstrap(location: String, invite: JsValue) -> JsValue {
@@ -141,7 +151,7 @@ pub async fn get_wallets() -> Result<JsValue, JsValue> {
     init_local_broker_with_lazy(&INIT_LOCAL_BROKER).await;
 
     let res = wallets_get_all().await.map_err(|e| {
-        log_err!("{}", e.to_string());
+        log_err!("wallets_get_all error {}", e.to_string());
     });
     if res.is_ok() {
         return Ok(serde_wasm_bindgen::to_value(&res.unwrap()).unwrap());
@@ -155,13 +165,47 @@ pub async fn session_start(wallet_name: String, user_js: JsValue) -> Result<JsVa
         .map_err(|_| "Deserialization error of user_id")?;
 
     let config = SessionConfig::new_save(&user_id, &wallet_name);
-    let res = nextgraph::local_broker::session_start(config)
+    let res: SessionInfoString = nextgraph::local_broker::session_start(config)
         .await
-        .map_err(|e: NgError| e.to_string())?;
+        .map_err(|e: NgError| e.to_string())?
+        .into();
 
     Ok(serde_wasm_bindgen::to_value(&res).unwrap())
 }
 
+#[cfg(wasmpack_target = "nodejs")]
+#[wasm_bindgen]
+pub async fn session_headless_start(user_js: String) -> Result<JsValue, String> {
+    let user_id = decode_key(&user_js).map_err(|_| "Invalid user_id")?;
+
+    let config = SessionConfig::new_headless(user_id);
+    let res: SessionInfoString = nextgraph::local_broker::session_start(config)
+        .await
+        .map_err(|e: NgError| e.to_string())?
+        .into();
+
+    Ok(serde_wasm_bindgen::to_value(&res).unwrap())
+}
+
+#[cfg(wasmpack_target = "nodejs")]
+#[wasm_bindgen]
+pub async fn admin_create_user(js_config: JsValue) -> Result<JsValue, String> {
+    let config = HeadLessConfigStrings::load(js_config)?;
+    let admin_user_key = config
+        .admin_user_key
+        .ok_or("No admin_user_key found in config nor env var.".to_string())?;
+
+    let res = nextgraph::local_broker::admin_create_user(
+        config.server_peer_id,
+        admin_user_key,
+        config.server_addr,
+    )
+    .await
+    .map_err(|e: ProtocolError| e.to_string())?;
+
+    Ok(serde_wasm_bindgen::to_value(&res.to_string()).unwrap())
+}
+
 #[wasm_bindgen]
 pub async fn session_start_remote(
     wallet_name: String,
@@ -175,9 +219,10 @@ pub async fn session_start_remote(
         .map_err(|_| "Deserialization error of peer_id")?;
 
     let config = SessionConfig::new_remote(&user_id, &wallet_name, peer_id);
-    let res = nextgraph::local_broker::session_start(config)
+    let res: SessionInfoString = nextgraph::local_broker::session_start(config)
         .await
-        .map_err(|e: NgError| e.to_string())?;
+        .map_err(|e: NgError| e.to_string())?
+        .into();
 
     Ok(serde_wasm_bindgen::to_value(&res).unwrap())
 }
@@ -344,12 +389,8 @@ pub async fn wallet_import(
 #[wasm_bindgen(module = "/js/node.js")]
 extern "C" {
     fn client_details() -> String;
-}
-
-#[cfg(wasmpack_target = "nodejs")]
-#[wasm_bindgen(module = "/js/node.js")]
-extern "C" {
     fn version() -> String;
+    fn get_env_vars() -> JsValue;
 }
 
 #[cfg(wasmpack_target = "nodejs")]
@@ -441,17 +482,17 @@ pub async fn test() {
 
 #[wasm_bindgen]
 pub async fn app_request_stream(
-    js_session_id: JsValue,
+    // js_session_id: JsValue,
     js_request: JsValue,
     callback: &js_sys::Function,
 ) -> Result<JsValue, String> {
-    let session_id: u64 = serde_wasm_bindgen::from_value::<u64>(js_session_id)
-        .map_err(|_| "Deserialization error of session_id".to_string())?;
+    // let session_id: u64 = serde_wasm_bindgen::from_value::<u64>(js_session_id)
+    //     .map_err(|_| "Deserialization error of session_id".to_string())?;
 
     let request = serde_wasm_bindgen::from_value::<AppRequest>(js_request)
         .map_err(|_| "Deserialization error of AppRequest".to_string())?;
 
-    let (reader, cancel) = nextgraph::local_broker::app_request_stream(session_id, request)
+    let (reader, cancel) = nextgraph::local_broker::app_request_stream(request)
         .await
         .map_err(|e: NgError| e.to_string())?;
 
@@ -495,13 +536,13 @@ pub async fn app_request_stream(
 }
 
 #[wasm_bindgen]
-pub async fn app_request(js_session_id: JsValue, js_request: JsValue) -> Result<JsValue, String> {
-    let session_id: u64 = serde_wasm_bindgen::from_value::<u64>(js_session_id)
-        .map_err(|_| "Deserialization error of session_id".to_string())?;
+pub async fn app_request(js_request: JsValue) -> Result<JsValue, String> {
+    // let session_id: u64 = serde_wasm_bindgen::from_value::<u64>(js_session_id)
+    //     .map_err(|_| "Deserialization error of session_id".to_string())?;
     let request = serde_wasm_bindgen::from_value::<AppRequest>(js_request)
         .map_err(|_| "Deserialization error of AppRequest".to_string())?;
 
-    let response = nextgraph::local_broker::app_request(session_id, request)
+    let response = nextgraph::local_broker::app_request(request)
         .await
         .map_err(|e: NgError| e.to_string())?;
 
@@ -526,15 +567,16 @@ pub async fn upload_chunk(
     let nuri: NuriV0 = serde_wasm_bindgen::from_value::<NuriV0>(js_nuri)
         .map_err(|_| "Deserialization error of nuri".to_string())?;
 
-    let request = AppRequest::V0(AppRequestV0 {
-        command: AppRequestCommandV0::FilePut,
+    let mut request = AppRequest::new(
+        AppRequestCommandV0::FilePut,
         nuri,
-        payload: Some(AppRequestPayload::V0(
+        Some(AppRequestPayload::V0(
             AppRequestPayloadV0::RandomAccessFilePutChunk((upload_id, chunk)),
         )),
-    });
+    );
+    request.set_session_id(session_id);
 
-    let response = nextgraph::local_broker::app_request(session_id, request)
+    let response = nextgraph::local_broker::app_request(request)
         .await
         .map_err(|e: NgError| e.to_string())?;
 
@@ -543,21 +585,21 @@ pub async fn upload_chunk(
 
 #[wasm_bindgen]
 pub async fn doc_fetch_private_subscribe() -> Result<JsValue, String> {
-    let request = AppRequest::V0(AppRequestV0 {
-        command: AppRequestCommandV0::Fetch(AppFetchContentV0::get_or_subscribe(true)),
-        nuri: NuriV0::new_private_store_target(),
-        payload: None,
-    });
+    let request = AppRequest::new(
+        AppRequestCommandV0::Fetch(AppFetchContentV0::get_or_subscribe(true)),
+        NuriV0::new_private_store_target(),
+        None,
+    );
     Ok(serde_wasm_bindgen::to_value(&request).unwrap())
 }
 
 #[wasm_bindgen]
 pub async fn doc_fetch_repo_subscribe(repo_id: String) -> Result<JsValue, String> {
-    let request = AppRequest::V0(AppRequestV0 {
-        command: AppRequestCommandV0::Fetch(AppFetchContentV0::get_or_subscribe(true)),
-        nuri: NuriV0::new_repo_target_from_string(repo_id).map_err(|e| e.to_string())?,
-        payload: None,
-    });
+    let request = AppRequest::new(
+        AppRequestCommandV0::Fetch(AppFetchContentV0::get_or_subscribe(true)),
+        NuriV0::new_repo_target_from_string(repo_id).map_err(|e| e.to_string())?,
+        None,
+    );
     Ok(serde_wasm_bindgen::to_value(&request).unwrap())
 }
 
@@ -624,6 +666,114 @@ pub async fn probe() {
     let _ = Broker::join_shutdown_with_timeout(std::time::Duration::from_secs(5)).await;
 }
 
+#[cfg(wasmpack_target = "nodejs")]
+#[derive(Serialize, Deserialize)]
+struct HeadLessConfigStrings {
+    server_addr: Option<String>,
+    server_peer_id: Option<String>,
+    client_peer_key: Option<String>,
+    admin_user_key: Option<String>,
+}
+
+#[cfg(wasmpack_target = "nodejs")]
+impl HeadLessConfigStrings {
+    fn load(js_config: JsValue) -> Result<HeadlessConfig, String> {
+        let string_config = if js_config.is_object() {
+            serde_wasm_bindgen::from_value::<HeadLessConfigStrings>(js_config)
+                .map_err(|_| "Deserialization error of config object".to_string())?
+        } else {
+            HeadLessConfigStrings {
+                server_addr: None,
+                server_peer_id: None,
+                client_peer_key: None,
+                admin_user_key: None,
+            }
+        };
+        let var_env_config =
+            serde_wasm_bindgen::from_value::<HeadLessConfigStrings>(get_env_vars())
+                .map_err(|_| "Deserialization error of env vars".to_string())?;
+
+        let server_addr = if let Some(s) = string_config.server_addr {
+            parse_ip_and_port_for(s, "server_addr").map_err(|e: NgError| e.to_string())?
+        } else {
+            if let Some(s) = var_env_config.server_addr {
+                parse_ip_and_port_for(s, "server_addr from var env")
+                    .map_err(|e: NgError| e.to_string())?
+            } else {
+                BindAddress::new_localhost_with_port(WS_PORT_REVERSE_PROXY)
+            }
+        };
+
+        let server_peer_id = if let Some(s) = string_config.server_peer_id {
+            Some(decode_key(&s).map_err(|e: NgError| e.to_string())?)
+        } else {
+            if let Some(s) = var_env_config.server_peer_id {
+                Some(decode_key(&s).map_err(|e: NgError| e.to_string())?)
+            } else {
+                None
+            }
+        }
+        .ok_or("No server_peer_id found in config nor env var.".to_string())?;
+
+        let client_peer_key = if let Some(s) = string_config.client_peer_key {
+            Some(decode_priv_key(&s).map_err(|e: NgError| e.to_string())?)
+        } else {
+            if let Some(s) = var_env_config.client_peer_key {
+                Some(decode_priv_key(&s).map_err(|e: NgError| e.to_string())?)
+            } else {
+                None
+            }
+        };
+
+        let admin_user_key = if let Some(s) = string_config.admin_user_key {
+            Some(decode_priv_key(&s).map_err(|e: NgError| e.to_string())?)
+        } else {
+            if let Some(s) = var_env_config.admin_user_key {
+                Some(decode_priv_key(&s).map_err(|e: NgError| e.to_string())?)
+            } else {
+                None
+            }
+        };
+
+        Ok(HeadlessConfig {
+            server_addr,
+            server_peer_id,
+            client_peer_key,
+            admin_user_key,
+        })
+    }
+}
+/*
+#[doc(hidden)]
+#[derive(Debug)]
+pub struct HeadlessConfig {
+    // parse_ip_and_port_for(string, "verifier_server")
+    pub server_addr: Option<BindAddress>,
+    // decode_key(string)
+    pub server_peer_id: PubKey,
+    // decode_priv_key(string)
+    pub client_peer_key: PrivKey,
+}*/
+
+#[cfg(wasmpack_target = "nodejs")]
+#[wasm_bindgen]
+pub async fn init_headless(js_config: JsValue) -> Result<(), String> {
+    //log_info!("{:?}", js_config);
+
+    let config = HeadLessConfigStrings::load(js_config)?;
+    let _ = config
+        .client_peer_key
+        .as_ref()
+        .ok_or("No client_peer_key found in config nor env var.".to_string())?;
+
+    init_local_broker(Box::new(move || {
+        LocalBrokerConfig::Headless(config.clone())
+    }))
+    .await;
+
+    Ok(())
+}
+
 #[wasm_bindgen]
 pub async fn start() {
     async fn inner_task() -> ResultSend<()> {
@@ -633,9 +783,8 @@ pub async fn start() {
 }
 
 #[wasm_bindgen]
-pub async fn session_stop(user_id_js: JsValue) -> Result<(), String> {
-    let user_id = serde_wasm_bindgen::from_value::<UserId>(user_id_js)
-        .map_err(|_| "serde error on user_id")?;
+pub async fn session_stop(user_id_js: String) -> Result<(), String> {
+    let user_id = decode_key(&user_id_js).map_err(|_| "Invalid user_id")?;
 
     nextgraph::local_broker::session_stop(&user_id)
         .await
@@ -643,9 +792,8 @@ pub async fn session_stop(user_id_js: JsValue) -> Result<(), String> {
 }
 
 #[wasm_bindgen]
-pub async fn user_disconnect(user_id_js: JsValue) -> Result<(), String> {
-    let user_id = serde_wasm_bindgen::from_value::<UserId>(user_id_js)
-        .map_err(|_| "serde error on user_id")?;
+pub async fn user_disconnect(user_id_js: String) -> Result<(), String> {
+    let user_id = decode_key(&user_id_js).map_err(|_| "Invalid user_id")?;
 
     nextgraph::local_broker::user_disconnect(&user_id)
         .await
@@ -662,13 +810,12 @@ pub async fn wallet_close(wallet_name: String) -> Result<(), String> {
 #[wasm_bindgen]
 pub async fn user_connect(
     client_info_js: JsValue,
-    user_id_js: JsValue,
+    user_id_js: String,
     location: Option<String>,
 ) -> Result<JsValue, String> {
     let info = serde_wasm_bindgen::from_value::<ClientInfo>(client_info_js)
         .map_err(|_| "serde error on info")?;
-    let user_id = serde_wasm_bindgen::from_value::<UserId>(user_id_js)
-        .map_err(|_| "serde error on user_id")?;
+    let user_id = decode_key(&user_id_js).map_err(|_| "Invalid user_id")?;
 
     #[derive(Serialize, Deserialize)]
     struct ConnectionInfo {

ng-verifier/src/commits/mod.rs

@@ -20,7 +20,8 @@ use ng_repo::repo::{BranchInfo, Repo};
 use ng_repo::store::Store;
 use ng_repo::types::*;
 
-use crate::types::*;
+use ng_net::app_protocol::*;
+
 use crate::verifier::Verifier;
 
 #[async_trait::async_trait]

ng-verifier/src/request_processor.rs

@@ -23,38 +23,38 @@ use ng_repo::types::BranchId;
 use ng_repo::types::StoreRepo;
 use ng_repo::types::*;
 
+use ng_net::app_protocol::*;
 use ng_net::utils::ResultSend;
 use ng_net::utils::{spawn_and_log_error, Receiver, Sender};
 
 use crate::types::*;
 use crate::verifier::*;
 
-impl AppRequestCommandV0 {
+impl Verifier {
     pub(crate) async fn process_stream(
-        &self,
-        verifier: &mut Verifier,
+        &mut self,
+        command: &AppRequestCommandV0,
         nuri: &NuriV0,
         _payload: &Option<AppRequestPayload>,
     ) -> Result<(Receiver<AppResponse>, CancelFn), NgError> {
-        match self {
-            Self::Fetch(fetch) => match fetch {
+        match command {
+            AppRequestCommandV0::Fetch(fetch) => match fetch {
                 AppFetchContentV0::Subscribe => {
-                    let (_, branch_id, _) =
-                        Self::open_for_target(verifier, &nuri.target, false).await?;
-                    Ok(verifier.create_branch_subscription(branch_id).await?)
+                    let (_, branch_id, _) = self.open_for_target(&nuri.target, false).await?;
+                    Ok(self.create_branch_subscription(branch_id).await?)
                 }
                 _ => unimplemented!(),
             },
-            Self::FileGet => {
+            AppRequestCommandV0::FileGet => {
                 if nuri.access.len() < 1 || nuri.object.is_none() {
                     return Err(NgError::InvalidArgument);
                 }
-                let (repo_id, _, store_repo) = Self::resolve_target(verifier, &nuri.target)?;
+                let (repo_id, _, store_repo) = self.resolve_target(&nuri.target)?;
                 let access = nuri.access.get(0).unwrap();
                 if let NgAccessV0::Key(key) = access {
-                    let repo = verifier.get_repo(&repo_id, &store_repo)?;
+                    let repo = self.get_repo(&repo_id, &store_repo)?;
                     let obj_id = nuri.object.unwrap();
-                    if let Some(mut stream) = verifier
+                    if let Some(mut stream) = self
                         .fetch_blocks_if_needed(&obj_id, &repo_id, &store_repo)
                         .await?
                     {
@@ -117,14 +117,14 @@ impl AppRequestCommandV0 {
     }
 
     fn resolve_target(
-        verifier: &mut Verifier,
+        &mut self,
         target: &NuriTargetV0,
     ) -> Result<(RepoId, BranchId, StoreRepo), NgError> {
         match target {
             NuriTargetV0::PrivateStore => {
-                let repo_id = verifier.config.private_store_id.unwrap();
+                let repo_id = self.config.private_store_id.unwrap();
                 let (branch, store_repo) = {
-                    let repo = verifier.repos.get(&repo_id).ok_or(NgError::RepoNotFound)?;
+                    let repo = self.repos.get(&repo_id).ok_or(NgError::RepoNotFound)?;
                     let branch = repo.main_branch().ok_or(NgError::BranchNotFound)?;
                     (branch.id, repo.store.get_store_repo().clone())
                 };
@@ -132,7 +132,7 @@ impl AppRequestCommandV0 {
             }
             NuriTargetV0::Repo(repo_id) => {
                 let (branch, store_repo) = {
-                    let repo = verifier.repos.get(repo_id).ok_or(NgError::RepoNotFound)?;
+                    let repo = self.repos.get(repo_id).ok_or(NgError::RepoNotFound)?;
                     let branch = repo.main_branch().ok_or(NgError::BranchNotFound)?;
                     (branch.id, repo.store.get_store_repo().clone())
                 };
@@ -143,35 +143,32 @@ impl AppRequestCommandV0 {
     }
 
     async fn open_for_target(
-        verifier: &mut Verifier,
+        &mut self,
         target: &NuriTargetV0,
         as_publisher: bool,
     ) -> Result<(RepoId, BranchId, StoreRepo), NgError> {
-        let (repo_id, branch, store_repo) = Self::resolve_target(verifier, target)?;
-        verifier
-            .open_branch(&repo_id, &branch, as_publisher)
-            .await?;
+        let (repo_id, branch, store_repo) = self.resolve_target(target)?;
+        self.open_branch(&repo_id, &branch, as_publisher).await?;
         Ok((repo_id, branch, store_repo))
     }
 
     pub(crate) async fn process(
-        &self,
-        verifier: &mut Verifier,
+        &mut self,
+        command: &AppRequestCommandV0,
         nuri: NuriV0,
         payload: Option<AppRequestPayload>,
     ) -> Result<AppResponse, NgError> {
-        match self {
-            Self::FilePut => match payload {
+        match command {
+            AppRequestCommandV0::FilePut => match payload {
                 None => return Err(NgError::InvalidPayload),
                 Some(AppRequestPayload::V0(v0)) => match v0 {
                     AppRequestPayloadV0::AddFile(add) => {
                         let (repo_id, branch, store_repo) =
-                            Self::open_for_target(verifier, &nuri.target, true).await?;
+                            self.open_for_target(&nuri.target, true).await?;
                         //log_info!("GOT ADD FILE {:?}", add);
 
-                        if verifier.connected_server_id.is_some() {
-                            verifier
-                                .put_all_blocks_of_file(&add.object, &repo_id, &store_repo)
+                        if self.connected_broker.is_some() {
+                            self.put_all_blocks_of_file(&add.object, &repo_id, &store_repo)
                                 .await?;
                         }
 
@@ -180,8 +177,7 @@ impl AppRequestCommandV0 {
                             metadata: vec![],
                         }));
 
-                        verifier
-                            .new_commit(
+                        self.new_commit(
                             add_file_commit_body,
                             &repo_id,
                             &branch,
@@ -196,17 +192,16 @@ impl AppRequestCommandV0 {
                         unimplemented!();
                     }
                     AppRequestPayloadV0::RandomAccessFilePut(content_type) => {
-                        let (repo_id, _, store_repo) =
-                            Self::resolve_target(verifier, &nuri.target)?;
-                        let repo = verifier.get_repo(&repo_id, &store_repo)?;
-                        let id = verifier.start_upload(content_type, Arc::clone(&repo.store));
+                        let (repo_id, _, store_repo) = self.resolve_target(&nuri.target)?;
+                        let repo = self.get_repo(&repo_id, &store_repo)?;
+                        let id = self.start_upload(content_type, Arc::clone(&repo.store));
                         return Ok(AppResponse::V0(AppResponseV0::FileUploading(id)));
                     }
                     AppRequestPayloadV0::RandomAccessFilePutChunk((id, chunk)) => {
                         if chunk.len() > 0 {
-                            verifier.continue_upload(id, &chunk)?;
+                            self.continue_upload(id, &chunk)?;
                         } else {
-                            let reference = verifier.finish_upload(id)?;
+                            let reference = self.finish_upload(id)?;
                             return Ok(AppResponse::V0(AppResponseV0::FileUploaded(reference)));
                         }
                     }

ng-verifier/src/rocksdb_user_storage.rs

@@ -15,6 +15,7 @@ use std::sync::{Arc, RwLock};
 
 use either::Either::{Left, Right};
 
+use ng_net::app_protocol::FileName;
 use ng_repo::block_storage::BlockStorage;
 use ng_repo::log::*;
 use ng_repo::repo::{BranchInfo, Repo};
@@ -23,7 +24,6 @@ use ng_repo::{errors::StorageError, types::*};
 
 use ng_storage_rocksdb::kcv_storage::RocksDbKCVStorage;
 
-use crate::types::*;
 use crate::user_storage::branch::*;
 use crate::user_storage::repo::*;
 use crate::user_storage::*;

ng-verifier/src/site.rs

@@ -221,7 +221,7 @@ impl SiteV0 {
             .new_events(commits, private_repo_id, &private_store_repo)
             .await?;
 
-        Ok(Self {
+        let site = Self {
             site_type: SiteType::Individual((user_priv_key, private_repo_read_cap)),
             id: site_pubkey,
             name: site_name,
@@ -230,7 +230,14 @@ impl SiteV0 {
             private,
             cores: vec![],
             bootstraps: vec![],
-        })
+        };
+
+        verifier.config.private_store_read_cap = site.get_individual_site_private_store_read_cap();
+        verifier.config.private_store_id = Some(site.private.id);
+        verifier.config.protected_store_id = Some(site.protected.id);
+        verifier.config.public_store_id = Some(site.public.id);
+
+        Ok(site)
     }
 
     pub async fn create_individual(
@@ -246,10 +253,7 @@ impl SiteV0 {
         verifier: &mut Verifier,
     ) -> Result<Self, NgError> {
         let site = Self::create_individual_(user_priv_key, verifier, SiteName::Personal).await?;
-        verifier.config.private_store_read_cap = site.get_individual_site_private_store_read_cap();
-        verifier.config.private_store_id = Some(site.private.id);
-        verifier.config.protected_store_id = Some(site.protected.id);
-        verifier.config.public_store_id = Some(site.public.id);
+
         Ok(site)
     }
 

ng-verifier/src/types.rs

@@ -18,7 +18,7 @@ use serde::{Deserialize, Serialize};
 //use oxigraph::model::GroundQuad;
 //use yrs::{StateVector, Update};
 
-use ng_repo::{errors::NgError, types::*};
+use ng_repo::{errors::*, types::*};
 
 use ng_net::types::*;
 
@@ -65,6 +65,13 @@ impl VerifierType {
             _ => false,
         }
     }
+
+    pub fn is_remote(&self) -> bool {
+        match self {
+            Self::Remote(_) => true,
+            _ => false,
+        }
+    }
 }
 #[doc(hidden)]
 //type LastSeqFn = fn(peer_id: PubKey, qty: u16) -> Result<u64, NgError>;
@@ -90,6 +97,7 @@ impl fmt::Debug for JsSaveSessionConfig {
     }
 }
 
+#[doc(hidden)]
 #[derive(Debug)]
 pub enum VerifierConfigType {
     /// nothing will be saved on disk during the session
@@ -104,6 +112,8 @@ pub enum VerifierConfigType {
     Remote(Option<PubKey>),
     /// IndexedDb based rocksdb compiled to WASM... not ready yet. obviously. only works in the browser
     WebRocksDb,
+    /// headless
+    Headless(Credentials),
 }
 
 impl VerifierConfigType {
@@ -147,291 +157,78 @@ pub struct VerifierConfig {
 #[doc(hidden)]
 pub type CancelFn = Box<dyn FnOnce() + Sync + Send>;
 
-//
-// APP PROTOCOL (between APP and VERIFIER)
-//
-
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub enum AppFetchContentV0 {
-    Get,       // does not subscribe. more to be detailed
-    Subscribe, // more to be detailed
-    Update,
-    //Invoke,
-    ReadQuery,  // more to be detailed
-    WriteQuery, // more to be detailed
-}
-
-impl AppFetchContentV0 {
-    pub fn get_or_subscribe(subscribe: bool) -> Self {
-        if !subscribe {
-            AppFetchContentV0::Get
-        } else {
-            AppFetchContentV0::Subscribe
-        }
-    }
-}
-
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub enum NgAccessV0 {
-    ReadCap(ReadCap),
-    Token(Digest),
-    #[serde(with = "serde_bytes")]
-    ExtRequest(Vec<u8>),
-    Key(BlockKey),
-    Inbox(Digest),
-}
-
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub enum TargetBranchV0 {
-    Chat,
-    Stream,
-    Context,
-    Ontology,
-    BranchId(BranchId),
-    Named(String),          // branch or commit
-    Commits(Vec<ObjectId>), // only possible if access to their branch is given. must belong to the same branch.
-}
-
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub enum NuriTargetV0 {
-    UserSite, // targets the whole data set of the user
-
-    PublicStore,
-    ProtectedStore,
-    PrivateStore,
-    AllDialogs,
-    Dialog(String), // shortname of a Dialog
-    AllGroups,
-    Group(String), // shortname of a Group
-
-    Repo(RepoId),
-
-    Identity(UserId),
-}
-
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub struct NuriV0 {
-    pub target: NuriTargetV0,
-    pub entire_store: bool, // If it is a store, will (try to) include all the docs belonging to the store
-
-    pub object: Option<ObjectId>, // used only for FileGet. // cannot be used for queries. only to download an object (file,commit..)
-    pub branch: Option<TargetBranchV0>, // if None, the main branch is chosen
-    pub overlay: Option<OverlayLink>,
-
-    pub access: Vec<NgAccessV0>,
-    pub topic: Option<TopicId>,
-    pub locator: Vec<PeerAdvert>,
-}
-
-impl NuriV0 {
-    pub fn new_repo_target_from_string(repo_id_string: String) -> Result<Self, NgError> {
-        let repo_id: RepoId = repo_id_string.as_str().try_into()?;
-        Ok(Self {
-            target: NuriTargetV0::Repo(repo_id),
-            entire_store: false,
-            object: None,
-            branch: None,
-            overlay: None,
-            access: vec![],
-            topic: None,
-            locator: vec![],
-        })
-    }
-
-    pub fn new_private_store_target() -> Self {
-        Self {
-            target: NuriTargetV0::PrivateStore,
-            entire_store: false,
-            object: None,
-            branch: None,
-            overlay: None,
-            access: vec![],
-            topic: None,
-            locator: vec![],
-        }
-    }
-    pub fn new(_from: String) -> Self {
-        todo!();
-    }
+#[doc(hidden)]
+#[derive(Debug, Clone)]
+pub enum BrokerPeerId {
+    Local(DirectPeerId),
+    Direct(DirectPeerId),
+    None,
 }
 
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub enum AppRequestCommandV0 {
-    Fetch(AppFetchContentV0),
-    Pin,
-    UnPin,
-    Delete,
-    Create,
-    FileGet, // needs the Nuri of branch/doc/store AND ObjectId
-    FilePut, // needs the Nuri of branch/doc/store
+impl From<&BrokerPeerId> for Option<PubKey> {
+    fn from(bpi: &BrokerPeerId) -> Option<PubKey> {
+        match bpi {
+            BrokerPeerId::Local(_) => None,
+            BrokerPeerId::Direct(d) => Some(*d),
+            BrokerPeerId::None => panic!("cannot connect to a broker without a peerid"),
         }
-
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub struct AppRequestV0 {
-    pub command: AppRequestCommandV0,
-
-    pub nuri: NuriV0,
-
-    pub payload: Option<AppRequestPayload>,
     }
-
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub enum AppRequest {
-    V0(AppRequestV0),
 }
 
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub enum DocQuery {
-    V0(String), // Sparql
+impl From<BrokerPeerId> for Option<PubKey> {
+    fn from(bpi: BrokerPeerId) -> Option<PubKey> {
+        (&bpi).into()
     }
-
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub struct GraphUpdate {
-    add: Vec<String>,
-    remove: Vec<String>,
 }
 
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub enum DiscreteUpdate {
-    /// A yrs::Update
-    #[serde(with = "serde_bytes")]
-    YMap(Vec<u8>),
-    #[serde(with = "serde_bytes")]
-    YXml(Vec<u8>),
-    #[serde(with = "serde_bytes")]
-    YText(Vec<u8>),
-    /// An automerge::Patch
-    #[serde(with = "serde_bytes")]
-    Automerge(Vec<u8>),
+impl BrokerPeerId {
+    pub fn new_direct(peer: DirectPeerId) -> Self {
+        Self::Direct(peer)
     }
-
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub struct DocUpdate {
-    heads: Vec<ObjectId>,
-    graph: Option<GraphUpdate>,
-    discrete: Option<DiscreteUpdate>,
+    pub fn is_some(&self) -> bool {
+        match self {
+            BrokerPeerId::Local(_) | BrokerPeerId::Direct(_) => true,
+            _ => false,
         }
-
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub struct DocAddFile {
-    pub filename: Option<String>,
-    pub object: ObjectRef,
     }
-
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub struct DocCreate {
-    store: StoreRepo,
-    content_type: BranchContentType,
+    pub fn is_none(&self) -> bool {
+        !self.is_some()
     }
-
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub struct DocDelete {
-    /// Nuri of doc to delete
-    nuri: String,
+    pub fn connected_or_err(&self) -> Result<Option<PubKey>, NgError> {
+        match self {
+            BrokerPeerId::None => Err(NgError::NotConnected),
+            _ => Ok(self.into()),
         }
-
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub enum AppRequestPayloadV0 {
-    Create(DocCreate),
-    Query(DocQuery),
-    Update(DocUpdate),
-    AddFile(DocAddFile),
-    //RemoveFile
-    Delete(DocDelete),
-    //Invoke(InvokeArguments),
-    SmallFilePut(SmallFile),
-    RandomAccessFilePut(String),                           // content_type
-    RandomAccessFilePutChunk((u32, serde_bytes::ByteBuf)), // end the upload with an empty vec
     }
-
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub enum AppRequestPayload {
-    V0(AppRequestPayloadV0),
+    pub fn broker_peer_id(&self) -> &DirectPeerId {
+        match self {
+            BrokerPeerId::Local(p) | BrokerPeerId::Direct(p) => p,
+            _ => panic!("dont call broker_peer_id on a BrokerPeerId::None"),
         }
-
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub enum DiscretePatch {
-    /// A yrs::Update
-    #[serde(with = "serde_bytes")]
-    YMap(Vec<u8>),
-    #[serde(with = "serde_bytes")]
-    YXml(Vec<u8>),
-    #[serde(with = "serde_bytes")]
-    YText(Vec<u8>),
-    /// An automerge::Patch
-    #[serde(with = "serde_bytes")]
-    Automerge(Vec<u8>),
     }
-
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub struct GraphPatch {
-    /// oxigraph::model::GroundQuad serialized to n-quads with oxrdfio
-    pub adds: Vec<String>,
-    pub removes: Vec<String>,
+    pub fn is_local(&self) -> bool {
+        match self {
+            BrokerPeerId::Local(_) => true,
+            _ => false,
         }
-
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub enum DiscreteState {
-    /// A yrs::StateVector
-    #[serde(with = "serde_bytes")]
-    YMap(Vec<u8>),
-    #[serde(with = "serde_bytes")]
-    YXml(Vec<u8>),
-    #[serde(with = "serde_bytes")]
-    YText(Vec<u8>),
-    // the output of Automerge::save()
-    #[serde(with = "serde_bytes")]
-    Automerge(Vec<u8>),
     }
-
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub struct GraphState {
-    pub tuples: Vec<String>,
+    pub fn is_direct(&self) -> bool {
+        match self {
+            BrokerPeerId::Direct(_) => true,
+            _ => false,
         }
-
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub struct AppState {
-    heads: Vec<ObjectId>,
-    graph: Option<GraphState>, // there is always a graph present in the branch. but it might not have been asked in the request
-    discrete: Option<DiscreteState>,
     }
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub struct AppPatch {
-    heads: Vec<ObjectId>,
-    graph: Option<GraphPatch>,
-    discrete: Option<DiscretePatch>,
+    pub fn is_direct_or_err(&self) -> Result<(), NgError> {
+        match self {
+            BrokerPeerId::Direct(_) => Ok(()),
+            _ => Err(NgError::NotConnected),
         }
-
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub struct FileName {
-    pub heads: Vec<ObjectId>,
-    pub name: Option<String>,
-    pub reference: ObjectRef,
-    pub nuri: String,
     }
 
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub struct FileMetaV0 {
-    pub content_type: String,
-    pub size: u64,
+    pub fn to_direct_if_not_local(&self, peer: DirectPeerId) -> Result<Self, VerifierError> {
+        match self {
+            BrokerPeerId::Local(_) => Err(VerifierError::LocallyConnected),
+            _ => Ok(BrokerPeerId::Direct(peer)),
         }
-
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub enum AppResponseV0 {
-    State(AppState),
-    Patch(AppPatch),
-    Text(String),
-    File(FileName),
-    FileUploading(u32),
-    FileUploaded(ObjectRef),
-    #[serde(with = "serde_bytes")]
-    FileBinary(Vec<u8>),
-    FileMeta(FileMetaV0),
-    QueryResult, // see sparesults
-    Ok,
     }
-
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub enum AppResponse {
-    V0(AppResponseV0),
 }

ng-verifier/src/user_storage/branch.rs

@@ -22,7 +22,7 @@ use ng_repo::log::*;
 use ng_repo::repo::BranchInfo;
 use ng_repo::types::*;
 
-use crate::types::FileName;
+use ng_net::app_protocol::FileName;
 
 pub struct BranchStorage<'a> {
     storage: &'a dyn KCVStorage,

ng-verifier/src/user_storage/storage.rs

@@ -14,6 +14,7 @@ use std::{
     sync::{Arc, RwLock},
 };
 
+use ng_net::app_protocol::FileName;
 use ng_repo::{
     block_storage::BlockStorage,
     errors::StorageError,
@@ -22,8 +23,6 @@ use ng_repo::{
     types::*,
 };
 
-use crate::types::*;
-
 pub trait UserStorage: Send + Sync {
     //fn repo_id_to_store_overlay(&self, id: &RepoId) -> Result<StoreOverlay, StorageError>;
 

ng-verifier/src/verifier.rs

@@ -47,6 +47,7 @@ use ng_repo::{
 };
 
 use ng_net::actor::SoS;
+use ng_net::app_protocol::*;
 use ng_net::broker::{Broker, BROKER};
 use ng_net::{
     connection::NoiseFSM,
@@ -76,7 +77,7 @@ use crate::user_storage::UserStorage;
 
 pub struct Verifier {
     pub(crate) config: VerifierConfig,
-    pub(crate) connected_server_id: Option<PubKey>,
+    pub connected_broker: BrokerPeerId,
     #[allow(dead_code)]
     graph_dataset: Option<oxigraph::store::Store>,
     pub(crate) user_storage: Option<Arc<Box<dyn UserStorage>>>,
@@ -103,7 +104,7 @@ pub struct Verifier {
 impl fmt::Debug for Verifier {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         writeln!(f, "Verifier\nconfig: {:?}", self.config)?;
-        writeln!(f, "connected_server_id: {:?}", self.connected_server_id)?;
+        writeln!(f, "connected_broker: {:?}", self.connected_broker)?;
         writeln!(f, "stores: {:?}", self.stores)?;
         writeln!(f, "repos: {:?}", self.repos)
     }
@@ -116,10 +117,35 @@ struct EventOutboxStorage {
 }
 
 impl Verifier {
+    pub fn complement_credentials(&self, creds: &mut Credentials) {
+        creds.private_store = self.private_store_id().clone();
+        creds.protected_store = self.protected_store_id().clone();
+        creds.public_store = self.public_store_id().clone();
+        creds.read_cap = self.config.private_store_read_cap.to_owned().unwrap();
+    }
+
     pub(crate) fn user_privkey(&self) -> &PrivKey {
         &self.config.user_priv_key
     }
 
+    pub fn private_store_id(&self) -> &RepoId {
+        self.config.private_store_id.as_ref().unwrap()
+    }
+    pub fn protected_store_id(&self) -> &RepoId {
+        self.config.protected_store_id.as_ref().unwrap()
+    }
+    pub fn public_store_id(&self) -> &RepoId {
+        self.config.public_store_id.as_ref().unwrap()
+    }
+
+    pub async fn close(&self) {
+        BROKER
+            .write()
+            .await
+            .close_peer_connection_x(None, Some(self.config.user_priv_key.to_pub()))
+            .await;
+    }
+
     pub(crate) fn start_upload(&mut self, content_type: String, store: Arc<Store>) -> u32 {
         let mut first_available: u32 = 0;
         for upload in self.uploads.keys() {
@@ -253,7 +279,7 @@ impl Verifier {
                 protected_store_id: None,
                 public_store_id: None,
             },
-            connected_server_id: None,
+            connected_broker: BrokerPeerId::None,
             graph_dataset: None,
             user_storage: None,
             block_storage: Some(block_storage),
@@ -280,11 +306,10 @@ impl Verifier {
         // );
         if self.is_persistent() && self.user_storage.is_some() && self.block_storage.is_some() {
             let user_storage = Arc::clone(self.user_storage.as_ref().unwrap());
-            //log_info!("LOADING ...");
             let stores = user_storage.get_all_store_and_repo_ids()?;
 
             for (store, repos) in stores.iter() {
-                //log_info!("LOADING STORE: {}", store);
+                log_info!("LOADING STORE: {}", store);
                 let repo = user_storage
                     .load_store(store, Arc::clone(self.block_storage.as_ref().unwrap()))?;
                 self.stores.insert(
@@ -742,12 +767,12 @@ impl Verifier {
 
         self.update_branch_current_heads(&repo_id, &branch_id, past, commit_ref)?;
 
-        if self.connected_server_id.is_some() {
+        if self.connected_broker.is_some() {
             // send the event to the server already
+            let connected_broker = self.connected_broker.clone();
             let broker = BROKER.read().await;
             let user = self.config.user_priv_key.to_pub();
-            let remote = self.connected_server_id.to_owned().unwrap();
-            self.send_event(event, &broker, &user, &remote, overlay)
+            self.send_event(event, &broker, &Some(user), &connected_broker, overlay)
                 .await?;
         } else {
             match &self.config.config_type {
@@ -792,20 +817,52 @@ impl Verifier {
     }
 
     pub fn connection_lost(&mut self) {
-        self.connected_server_id = None;
+        self.connected_broker = BrokerPeerId::None;
         // for (_, repo) in self.repos.iter_mut() {
         //     repo.opened_branches = HashMap::new();
         // }
     }
 
+    pub async fn sync(&mut self) {
+        let mut branches = vec![];
+        {
+            for (id, repo) in self.repos.iter_mut() {
+                for (branch, publisher) in repo.opened_branches.iter() {
+                    branches.push((*id, *branch, *publisher));
+                }
+                repo.opened_branches = HashMap::new();
+            }
+        }
+        let connected_broker = self.connected_broker.clone();
+        let user = self.config.user_priv_key.to_pub();
+        let broker = BROKER.read().await;
+        //log_info!("looping on branches {:?}", branches);
+        for (repo, branch, publisher) in branches {
+            //log_info!("open_branch_ repo {} branch {}", repo, branch);
+            let _e = self
+                .open_branch_(
+                    &repo,
+                    &branch,
+                    publisher,
+                    &broker,
+                    &Some(user),
+                    &connected_broker,
+                    false,
+                )
+                .await;
+        }
+    }
+
     pub async fn connection_opened(&mut self, peer: DirectPeerId) -> Result<(), NgError> {
-        self.connected_server_id = Some(peer);
+        self.connected_broker = self.connected_broker.to_direct_if_not_local(peer)?;
         log_info!("CONNECTION ESTABLISHED WITH peer {}", peer);
         if let Err(e) = self.bootstrap().await {
-            self.connected_server_id = None;
+            self.connected_broker = BrokerPeerId::None;
             return Err(e);
         }
 
+        let connected_broker = self.connected_broker.clone();
+
         let mut branches = vec![];
         {
             for (id, repo) in self.repos.iter_mut() {
@@ -825,7 +882,15 @@ impl Verifier {
         for (repo, branch, publisher) in branches {
             //log_info!("open_branch_ repo {} branch {}", repo, branch);
             let _e = self
-                .open_branch_(&repo, &branch, publisher, &broker, &user, &peer, false)
+                .open_branch_(
+                    &repo,
+                    &branch,
+                    publisher,
+                    &broker,
+                    &Some(user),
+                    &connected_broker,
+                    false,
+                )
                 .await;
             // log_info!(
             //     "END OF open_branch_ repo {} branch {} with {:?}",
@@ -844,24 +909,21 @@ impl Verifier {
         branch: &BranchId,
         as_publisher: bool,
     ) -> Result<(), NgError> {
-        let remote = match self.connected_server_id.as_ref() {
-            Some(r) => r.clone(),
-            None => {
+        if !self.connected_broker.is_some() {
             let repo = self.repos.get_mut(repo_id).ok_or(NgError::RepoNotFound)?;
             repo.opened_branches.insert(*branch, as_publisher);
             return Ok(());
         }
-        };
 
         let user = self.config.user_priv_key.to_pub();
-
+        let connected_broker = self.connected_broker.clone();
         self.open_branch_(
             repo_id,
             branch,
             as_publisher,
             &BROKER.read().await,
-            &user,
-            &remote,
+            &Some(user),
+            &connected_broker,
             false,
         )
         .await
@@ -872,16 +934,15 @@ impl Verifier {
 
         let broker = BROKER.read().await;
         let user = self.config.user_priv_key.to_pub();
-        let remote = self
-            .connected_server_id
-            .to_owned()
-            .ok_or(NgError::NotConnected)?;
+        let remote = self.connected_broker.connected_or_err()?;
 
         let msg = BlocksPut::V0(BlocksPutV0 {
             blocks,
             overlay: Some(overlay),
         });
-        broker.request::<BlocksPut, ()>(&user, &remote, msg).await?;
+        broker
+            .request::<BlocksPut, ()>(&Some(user), &remote, msg)
+            .await?;
         Ok(())
     }
 
@@ -894,17 +955,14 @@ impl Verifier {
 
         let broker = BROKER.read().await;
         let user = self.config.user_priv_key.to_pub();
-        let remote = self
-            .connected_server_id
-            .to_owned()
-            .ok_or(NgError::NotConnected)?;
+        let remote = self.connected_broker.connected_or_err()?;
 
         let msg = BlocksExist::V0(BlocksExistV0 {
             blocks,
             overlay: Some(overlay),
         });
         if let SoS::Single(found) = broker
-            .request::<BlocksExist, BlocksFound>(&user, &remote, msg)
+            .request::<BlocksExist, BlocksFound>(&Some(user), &remote, msg)
             .await?
         {
             Ok(found)
@@ -919,8 +977,8 @@ impl Verifier {
         branch: &BranchId,
         as_publisher: bool,
         broker: &RwLockReadGuard<'static, Broker>,
-        user: &UserId,
-        remote: &DirectPeerId,
+        user: &Option<UserId>,
+        remote_broker: &BrokerPeerId,
         force: bool,
     ) -> Result<(), NgError> {
         let (need_open, mut need_sub, overlay) = {
@@ -937,6 +995,8 @@ impl Verifier {
         };
         //log_info!("need_open {} need_sub {}", need_open, need_sub);
 
+        let remote = remote_broker.into();
+
         if need_open {
             // TODO: implement OpenRepo. for now we always do a Pinning because OpenRepo is not implemented on the broker.
             let msg = RepoPinStatusReq::V0(RepoPinStatusReqV0 {
@@ -944,7 +1004,7 @@ impl Verifier {
                 overlay: Some(overlay),
             });
             match broker
-                .request::<RepoPinStatusReq, RepoPinStatus>(user, remote, msg)
+                .request::<RepoPinStatusReq, RepoPinStatus>(user, &remote, msg)
                 .await
             {
                 Err(NgError::ServerError(ServerError::False))
@@ -954,12 +1014,12 @@ impl Verifier {
                         let repo = self.repos.get(repo_id).ok_or(NgError::RepoNotFound)?;
                         let topic_id = repo.branch(branch).unwrap().topic;
                         //TODO: only pin the requested branch.
-                        let pin_req = PinRepo::from_repo(repo, remote);
+                        let pin_req = PinRepo::from_repo(repo, remote_broker.broker_peer_id());
                         (pin_req, topic_id)
                     };
 
                     match broker
-                        .request::<PinRepo, RepoOpened>(user, remote, pin_req)
+                        .request::<PinRepo, RepoOpened>(user, &remote, pin_req)
                         .await
                     {
                         Ok(SoS::Single(opened)) => {
@@ -971,7 +1031,7 @@ impl Verifier {
                                     self.do_sync_req_if_needed(
                                         broker,
                                         user,
-                                        remote,
+                                        &remote,
                                         branch,
                                         repo_id,
                                         topic.known_heads(),
@@ -989,7 +1049,6 @@ impl Verifier {
                 Err(e) => return Err(e),
                 Ok(SoS::Single(pin_status)) => {
                     // checking that the branch is subscribed as publisher
-
                     let repo = self.repos.get(repo_id).ok_or(NgError::RepoNotFound)?;
                     let branch_info = repo.branch(branch)?;
                     let topic_id = &branch_info.topic;
@@ -1007,7 +1066,7 @@ impl Verifier {
                                 self.do_sync_req_if_needed(
                                     broker,
                                     user,
-                                    remote,
+                                    &remote,
                                     branch,
                                     repo_id,
                                     topic.known_heads(),
@@ -1024,7 +1083,6 @@ impl Verifier {
         }
         if need_sub {
             // we subscribe
-
             let repo = self.repos.get(repo_id).ok_or(NgError::RepoNotFound)?;
             let branch_info = repo.branch(branch)?;
 
@@ -1033,7 +1091,7 @@ impl Verifier {
                     // we need to subscribe as publisher, but we cant
                     return Err(NgError::PermissionDenied);
                 }
-                Some(remote)
+                Some(remote_broker.broker_peer_id())
             } else {
                 None
             };
@@ -1041,7 +1099,7 @@ impl Verifier {
             let topic_sub = TopicSub::new(repo, branch_info, broker_id);
 
             match broker
-                .request::<TopicSub, TopicSubRes>(user, remote, topic_sub)
+                .request::<TopicSub, TopicSubRes>(user, &remote, topic_sub)
                 .await
             {
                 Ok(SoS::Single(sub)) => {
@@ -1051,7 +1109,7 @@ impl Verifier {
                     self.do_sync_req_if_needed(
                         broker,
                         user,
-                        remote,
+                        &remote,
                         branch,
                         repo_id,
                         sub.known_heads(),
@@ -1072,8 +1130,8 @@ impl Verifier {
         &mut self,
         event: Event,
         broker: &RwLockReadGuard<'static, Broker>,
-        user: &UserId,
-        remote: &DirectPeerId,
+        user: &Option<UserId>,
+        remote: &BrokerPeerId,
         overlay: OverlayId,
     ) -> Result<(), NgError> {
         assert!(overlay.is_inner());
@@ -1087,7 +1145,7 @@ impl Verifier {
             .await?;
 
         let _ = broker
-            .request::<PublishEvent, ()>(user, remote, PublishEvent::new(event, overlay))
+            .request::<PublishEvent, ()>(user, &remote.into(), PublishEvent::new(event, overlay))
             .await?;
 
         Ok(())
@@ -1300,8 +1358,8 @@ impl Verifier {
     async fn do_sync_req_if_needed(
         &mut self,
         broker: &RwLockReadGuard<'static, Broker>,
-        user: &UserId,
-        remote: &DirectPeerId,
+        user: &Option<UserId>,
+        remote: &Option<DirectPeerId>,
         branch_id: &BranchId,
         repo_id: &RepoId,
         remote_heads: &Vec<ObjectId>,
@@ -1408,8 +1466,8 @@ impl Verifier {
     async fn do_sync_req(
         &mut self,
         broker: &RwLockReadGuard<'static, Broker>,
-        user: &UserId,
-        remote: &DirectPeerId,
+        user: &Option<UserId>,
+        remote: &Option<DirectPeerId>,
         topic: &TopicId,
         branch_id: &BranchId,
         branch_secret: &ReadCapSecret,
@@ -1440,8 +1498,8 @@ impl Verifier {
     async fn load_store_from_read_cap<'a>(
         &mut self,
         broker: &RwLockReadGuard<'static, Broker>,
-        user: &UserId,
-        remote: &DirectPeerId,
+        user: &Option<UserId>,
+        remote: &Option<DirectPeerId>,
         store: Arc<Store>,
     ) -> Result<(), NgError> {
         // first we fetch the read_cap commit of private store repo.
@@ -1450,8 +1508,8 @@ impl Verifier {
             None,
             &store.overlay_for_read_on_client_protocol(),
             &broker,
-            &user,
-            &remote,
+            user,
+            remote,
         )
         .await?;
 
@@ -1468,8 +1526,8 @@ impl Verifier {
                     let repo_id = store.id();
                     self.do_sync_req(
                         &broker,
-                        &user,
-                        &remote,
+                        user,
+                        remote,
                         topic,
                         repo_id,
                         store.get_store_readcap_secret(),
@@ -1500,8 +1558,8 @@ impl Verifier {
                         }
                         self.do_sync_req(
                             &broker,
-                            &user,
-                            &remote,
+                            user,
+                            remote,
                             &topic,
                             &branch_id,
                             &secret,
@@ -1527,8 +1585,8 @@ impl Verifier {
         topic_id: Option<TopicId>,
         overlay: &OverlayId,
         broker: &RwLockReadGuard<'static, Broker>,
-        user: &UserId,
-        remote: &DirectPeerId,
+        user: &Option<UserId>,
+        remote: &Option<DirectPeerId>,
     ) -> Result<Commit, NgError> {
         let msg = CommitGet::V0(CommitGetV0 {
             id: commit_ref.id,
@@ -1568,8 +1626,8 @@ impl Verifier {
         let overlay = repo.store.overlay_for_read_on_client_protocol();
 
         let broker = BROKER.read().await;
-        let user = self.config.user_priv_key.to_pub();
-        let remote = self.connected_server_id.to_owned();
+        let user = Some(self.config.user_priv_key.to_pub());
+        let remote = &self.connected_broker;
 
         match repo.store.has(id) {
             Err(StorageError::NotFound) => {
@@ -1583,7 +1641,7 @@ impl Verifier {
                     overlay: Some(overlay),
                 });
                 match broker
-                    .request::<BlocksGet, Block>(&user, remote.as_ref().unwrap(), msg)
+                    .request::<BlocksGet, Block>(&user, &remote.into(), msg)
                     .await
                 {
                     Ok(SoS::Stream(blockstream)) => Ok(Some(blockstream)),
@@ -1599,14 +1657,12 @@ impl Verifier {
     async fn bootstrap_from_remote(&mut self) -> Result<(), NgError> {
         if self.need_bootstrap() {
             let broker = BROKER.read().await;
-            let user = self.config.user_priv_key.to_pub();
-            let remote = self
-                .connected_server_id
-                .to_owned()
-                .ok_or(NgError::NotConnected)?;
+            let user = Some(self.config.user_priv_key.to_pub());
+            self.connected_broker.is_direct_or_err()?;
 
             let private_store_id = self.config.private_store_id.to_owned().unwrap();
             let private_store = self.create_private_store_from_credentials()?;
+            let remote = (&self.connected_broker).into();
 
             self.load_store_from_read_cap(&broker, &user, &remote, private_store)
                 .await?;
@@ -1779,22 +1835,22 @@ impl Verifier {
     //     ret
     // }
 
-    async fn send_outbox(&mut self) -> Result<(), NgError> {
+    pub async fn send_outbox(&mut self) -> Result<(), NgError> {
         let ret = self.take_events_from_outbox();
-        // if ret.is_err() {
-        //     log_err!("send_outbox {:}", ret.as_ref().unwrap_err());
-        // }
+        if ret.is_err() {
+            log_info!(
+                "take_events_from_outbox returned {:}",
+                ret.as_ref().unwrap_err()
+            );
+        }
         let events: Vec<EventOutboxStorage> = ret.unwrap_or(vec![]);
         if events.is_empty() {
             return Ok(());
         }
         let broker = BROKER.read().await;
-        let user = self.config.user_priv_key.to_pub();
-        let remote = self
-            .connected_server_id
-            .as_ref()
-            .ok_or(NgError::NotConnected)?
-            .clone();
+        let user = Some(self.config.user_priv_key.to_pub());
+        self.connected_broker.connected_or_err()?;
+        let remote = self.connected_broker.clone();
 
         // for all the events, check that they are valid (topic exists, current_heads match with event)
         let mut need_replay = false;
@@ -1986,12 +2042,12 @@ impl Verifier {
                 )
             }
             VerifierConfigType::Remote(_) => (None, None, None),
-            _ => unimplemented!(), // can be WebRocksDb or RocksDb on wasm platforms
+            _ => unimplemented!(), // can be WebRocksDb or RocksDb on wasm platforms, or Headless
         };
         let peer_id = config.peer_priv_key.to_pub();
         let mut verif = Verifier {
             config,
-            connected_server_id: None,
+            connected_broker: BrokerPeerId::None,
             graph_dataset: graph,
             user_storage: user.map(|u| Arc::new(u)),
             block_storage: block,
@@ -2021,13 +2077,16 @@ impl Verifier {
         req: AppRequest,
     ) -> Result<(Receiver<AppResponse>, CancelFn), NgError> {
         match req {
-            AppRequest::V0(v0) => v0.command.process_stream(self, &v0.nuri, &v0.payload).await,
+            AppRequest::V0(v0) => {
+                self.process_stream(&v0.command, &v0.nuri, &v0.payload)
+                    .await
+            }
         }
     }
 
     pub async fn app_request(&mut self, req: AppRequest) -> Result<AppResponse, NgError> {
         match req {
-            AppRequest::V0(v0) => v0.command.process(self, v0.nuri, v0.payload).await,
+            AppRequest::V0(v0) => self.process(&v0.command, v0.nuri, v0.payload).await,
         }
     }
 

ng-wallet/src/types.rs

@@ -129,6 +129,23 @@ impl SessionWalletStorageV0 {
     // }
 }
 
+#[derive(Serialize, Deserialize)]
+pub struct SessionInfoString {
+    pub session_id: u64,
+    pub user: String,
+    pub private_store_id: String,
+}
+
+impl From<SessionInfo> for SessionInfoString {
+    fn from(f: SessionInfo) -> Self {
+        SessionInfoString {
+            session_id: f.session_id,
+            private_store_id: f.private_store_id,
+            user: f.user.to_string(),
+        }
+    }
+}
+
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct SessionInfo {
     pub session_id: u64,

ngcli/src/main.rs

@@ -243,7 +243,7 @@ async fn main_inner() -> Result<(), NgcliError> {
 
     if let Some(_matches) = matches.subcommand_matches("gen-key") {
         let (privkey, pubkey) = generate_keypair();
-        println!("Your UserId is: {pubkey}");
+        println!("Your Public key is:  {pubkey}");
         println!("Your Private key is: {privkey}");
         return Ok(());
     }

ngd/src/cli.rs

@@ -74,7 +74,7 @@ pub(crate) struct Cli {
     #[arg(long, conflicts_with("private"))]
     pub public_without_clients: bool,
 
-    /// When --public is used with a public IPV6, this option will bind the IPV6 to the private interface. This is how DMZ work for IpV6
+    /// When --public is used with a public IPV6, this option will bind the IPV6 to the private interface. This is how DMZ works for IpV6
     #[arg(long, requires("public"), conflicts_with("no_ipv6"))]
     pub bind_public_ipv6: bool,
 

ngd/src/main.rs

@@ -37,10 +37,7 @@ use ng_repo::{
 };
 
 use ng_net::types::*;
-use ng_net::utils::is_private_ip;
-use ng_net::utils::is_public_ip;
-use ng_net::utils::is_public_ipv4;
-use ng_net::utils::is_public_ipv6;
+use ng_net::utils::*;
 use ng_net::{WS_PORT, WS_PORT_REVERSE_PROXY};
 
 use ng_broker::interfaces::*;
@@ -68,12 +65,6 @@ lazy_static! {
     .unwrap();
 }
 
-lazy_static! {
-    #[doc(hidden)]
-    static ref RE_IPV6_WITH_PORT: Regex =
-        Regex::new(r"^\[([0-9a-fA-F:]{3,39})\](\:\d{1,5})?$").unwrap();
-}
-
 pub static DEFAULT_PORT: u16 = WS_PORT;
 
 pub static DEFAULT_TLS_PORT: u16 = 443;
@@ -123,83 +114,6 @@ fn parse_ipv6_for(string: String, for_option: &str) -> Result<Ipv6Addr, NgdError
     })
 }
 
-fn parse_ipv4_and_port_for(
-    string: String,
-    for_option: &str,
-    default_port: u16,
-) -> Result<(Ipv4Addr, u16), NgdError> {
-    let parts: Vec<&str> = string.split(":").collect();
-    let ipv4 = parts[0].parse::<Ipv4Addr>().map_err(|_| {
-        NgdError::OtherConfigError(format!(
-            "The <IPv4:PORT> value submitted for the {} option is invalid.",
-            for_option
-        ))
-    })?;
-
-    let port = if parts.len() > 1 {
-        match from_str::<u16>(parts[1]) {
-            Err(_) => default_port,
-            Ok(p) => {
-                if p == 0 {
-                    default_port
-                } else {
-                    p
-                }
-            }
-        }
-    } else {
-        default_port
-    };
-    Ok((ipv4, port))
-}
-
-fn parse_ip_and_port_for(string: String, for_option: &str) -> Result<(IpAddr, u16), NgdError> {
-    let c = RE_IPV6_WITH_PORT.captures(&string);
-    let ipv6;
-    let port;
-    if c.is_some() && c.as_ref().unwrap().get(1).is_some() {
-        let cap = c.unwrap();
-        let ipv6_str = cap.get(1).unwrap().as_str();
-        port = match cap.get(2) {
-            None => DEFAULT_PORT,
-            Some(p) => {
-                let mut chars = p.as_str().chars();
-                chars.next();
-                match from_str::<u16>(chars.as_str()) {
-                    Err(_) => DEFAULT_PORT,
-                    Ok(p) => {
-                        if p == 0 {
-                            DEFAULT_PORT
-                        } else {
-                            p
-                        }
-                    }
-                }
-            }
-        };
-        let ipv6 = ipv6_str.parse::<Ipv6Addr>().map_err(|_| {
-            NgdError::OtherConfigError(format!(
-                "The <[IPv6]:PORT> value submitted for the {} option is invalid.",
-                for_option
-            ))
-        })?;
-        Ok((IpAddr::V6(ipv6), port))
-    } else {
-        // we try just an IPV6 without port
-        let ipv6_res = string.parse::<Ipv6Addr>();
-        if ipv6_res.is_err() {
-            // let's try IPv4
-
-            parse_ipv4_and_port_for(string, for_option, DEFAULT_PORT)
-                .map(|ipv4| (IpAddr::V4(ipv4.0), ipv4.1))
-        } else {
-            ipv6 = ipv6_res.unwrap();
-            port = DEFAULT_PORT;
-            Ok((IpAddr::V6(ipv6), port))
-        }
-    }
-}
-
 fn parse_triple_interface_and_port_for(
     string: &String,
     for_option: &str,
@@ -331,7 +245,10 @@ impl From<NgdError> for std::io::Error {
 
 impl From<NgError> for NgdError {
     fn from(err: NgError) -> NgdError {
-        Self::NgError(err)
+        match err {
+            NgError::ConfigError(c) => Self::OtherConfigError(c),
+            _ => Self::NgError(err),
+        }
     }
 }
 
@@ -920,14 +837,10 @@ async fn main_inner() -> Result<(), NgdError> {
 
                 if first_char == '[' || first_char.is_numeric() {
                     // an IPv6 or IPv4
-                    let bind = parse_ip_and_port_for(parts[0].to_string(), "--forward")?;
-                    let bind_addr = BindAddress {
-                        ip: (&bind.0).into(),
-                        port: bind.1,
-                    };
-                    if is_private_ip(&bind.0) {
+                    let bind_addr = parse_ip_and_port_for(parts[0].to_string(), "--forward")?;
+                    if bind_addr.ip.is_private() {
                         BrokerServerTypeV0::BoxPrivate(vec![bind_addr])
-                    } else if is_public_ip(&bind.0) {
+                    } else if bind_addr.ip.is_public() {
                         BrokerServerTypeV0::Public(vec![bind_addr])
                     } else {
                         return Err(NgdError::OtherConfigErrorStr(