parent
cf7629c3b6
commit
e26de1feee
@ -0,0 +1,6 @@ |
||||
*~ |
||||
!.github |
||||
\#* |
||||
/target |
||||
/result* |
||||
.DS_Store |
||||
@ -1,10 +1,10 @@ |
||||
[package] |
||||
name = "ng-app-web" |
||||
name = "ng-app-js-sdk" |
||||
version = "0.1.0" |
||||
edition = "2021" |
||||
license = "MIT/Apache-2.0" |
||||
authors = ["Niko PLP <niko@nextgraph.org>"] |
||||
description = "Web app client of NextGraph" |
||||
description = "JS app sdk of NextGraph" |
||||
repository = "https://git.nextgraph.org/NextGraph/nextgraph-rs" |
||||
|
||||
[package.metadata.wasm-pack.profile.release] |
||||
@ -0,0 +1,104 @@ |
||||
# ng-app-js |
||||
|
||||
JS/WASM module of NextGraph (SDK and apps) |
||||
|
||||
## NextGraph |
||||
|
||||
> NextGraph brings about the convergence between P2P and Semantic Web technologies, towards a decentralized, secure and privacy-preserving cloud, based on CRDTs. |
||||
> |
||||
> This open source ecosystem provides solutions for end-users and software developers alike, wishing to use or create **decentralized** apps featuring: **live collaboration** on rich-text documents, peer to peer communication with end-to-end encryption, offline-first, **local-first**, portable and interoperable data, total ownership of data and software, security and privacy. Centered on repositories containing **semantic data** (RDF), **rich text**, and structured data formats like **JSON**, synced between peers belonging to permissioned groups of users, it offers strong eventual consistency, thanks to the use of **CRDTs**. Documents can be linked together, signed, shared securely, queried using the **SPARQL** language and organized into sites and containers. |
||||
> |
||||
> More info here [https://nextgraph.org](https://nextgraph.org) |
||||
|
||||
## JS/WASM module |
||||
|
||||
This module is part of the SDK of NextGraph. |
||||
|
||||
It is composed of |
||||
- the npm package `ng-app-js` which is the SDK |
||||
- the plain JS web app `app-web` |
||||
- the React web app `app-react` |
||||
- the node-js app `app-node` |
||||
|
||||
## Support |
||||
|
||||
Documentation can be found here [https://docs.nextgraph.org](https://docs.nextgraph.org) |
||||
|
||||
And our community forum where you can ask questions is here [https://forum.nextgraph.org](https://forum.nextgraph.org) |
||||
|
||||
## For developers |
||||
|
||||
Read our [getting started guide](https://docs.nextgraph.org/en/getting-started/). |
||||
|
||||
``` |
||||
npm i ng-app-js-sdk |
||||
``` |
||||
|
||||
## For contributors |
||||
|
||||
``` |
||||
wasm-pack build --target bundler |
||||
cd pkg |
||||
// if you have access to npm registry and want to publish the package |
||||
// npm publish --access=public |
||||
|
||||
cd .. |
||||
wasm-pack build -t nodejs -d pkg-node |
||||
node prepare-node.js |
||||
cd pkg-node |
||||
// if you have access to npm registry and want to publish the package |
||||
// npm publish --access=public |
||||
``` |
||||
|
||||
### Plain JS web app |
||||
|
||||
``` |
||||
cd ../app-web |
||||
// for local development |
||||
npm install --no-save ../pkg |
||||
// or, for install from npm registry: npm install |
||||
npm start |
||||
``` |
||||
|
||||
Open this URL in browser : [http://localhost:8080](http://localhost:8080) |
||||
|
||||
### React web app |
||||
|
||||
``` |
||||
cd ../app-react |
||||
// for local development |
||||
npm install --no-save ../pkg |
||||
// or, for install from npm registry: npm install |
||||
npm run dev |
||||
``` |
||||
|
||||
Open this URL in browser : [http://localhost:8080](http://localhost:8080) |
||||
|
||||
### NodeJS app |
||||
|
||||
``` |
||||
cd ../app-node |
||||
// for local development |
||||
npm install --no-save ../pkg-node |
||||
// or, for install from npm registry: npm install |
||||
npm run start |
||||
``` |
||||
|
||||
### Contributions license |
||||
|
||||
Unless you explicitly state otherwise, any contribution intentionally submitted |
||||
for inclusion in the work by you shall be dual licensed as below, without any |
||||
additional terms or conditions.s |
||||
|
||||
## License |
||||
|
||||
Licensed under either of |
||||
* Apache License, Version 2.0 ([LICENSE-APACHE2](LICENSE-APACHE2) or http://www.apache.org/licenses/LICENSE-2.0) |
||||
* MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) |
||||
at your option. |
||||
|
||||
`SPDX-License-Identifier: Apache-2.0 OR MIT` |
||||
|
||||
--- |
||||
|
||||
NextGraph received funding through the [NGI Assure Fund](https://nlnet.nl/project/NextGraph/index.html), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's [Next Generation Internet](https://ngi.eu/) programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 957073. |
||||
@ -0,0 +1 @@ |
||||
node_modules |
||||
@ -0,0 +1,12 @@ |
||||
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers
|
||||
// All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0
|
||||
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
|
||||
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
|
||||
// at your option. All files in the project carrying such
|
||||
// notice may not be copied, modified, or distributed except
|
||||
// according to those terms.
|
||||
|
||||
const ng = require("ng-app-node-sdk"); |
||||
|
||||
console.log(ng.change("you")); |
||||
@ -0,0 +1,18 @@ |
||||
{ |
||||
"name": "ng-app-node", |
||||
"version": "0.1.0", |
||||
"description": "NodeJS app for NextGraph", |
||||
"main": "index.js", |
||||
"scripts": { |
||||
"start": "node index.js" |
||||
}, |
||||
"repository": { |
||||
"type": "git", |
||||
"url": "git+https://git.nextgraph.org/NextGraph/nextgraph-rs.git" |
||||
}, |
||||
"author": "Niko PLP <niko@nextgraph.org>", |
||||
"license": "(MIT OR Apache-2.0)", |
||||
"dependencies": { |
||||
"ng-app-node-sdk": "^0.1.0" |
||||
} |
||||
} |
||||
@ -0,0 +1,3 @@ |
||||
{ |
||||
"presets": ["@babel/preset-env", "@babel/preset-react"] |
||||
} |
||||
@ -0,0 +1,3 @@ |
||||
node_modules |
||||
dist |
||||
build |
||||
@ -0,0 +1,41 @@ |
||||
# ng-app-react |
||||
|
||||
React app client of NextGraph |
||||
|
||||
## NextGraph |
||||
|
||||
> NextGraph brings about the convergence between P2P and Semantic Web technologies, towards a decentralized, secure and privacy-preserving cloud, based on CRDTs. |
||||
> |
||||
> This open source ecosystem provides solutions for end-users and software developers alike, wishing to use or create **decentralized** apps featuring: **live collaboration** on rich-text documents, peer to peer communication with end-to-end encryption, offline-first, **local-first**, portable and interoperable data, total ownership of data and software, security and privacy. Centered on repositories containing **semantic data** (RDF), **rich text**, and structured data formats like **JSON**, synced between peers belonging to permissioned groups of users, it offers strong eventual consistency, thanks to the use of **CRDTs**. Documents can be linked together, signed, shared securely, queried using the **SPARQL** language and organized into sites and containers. |
||||
> |
||||
> More info here [https://nextgraph.org](https://nextgraph.org) |
||||
|
||||
## For contributors |
||||
|
||||
Build the JS SDK |
||||
|
||||
``` |
||||
cd .. |
||||
wasm-pack build --target bundler |
||||
``` |
||||
|
||||
``` |
||||
cd app-react |
||||
npm install --no-save ../pkg |
||||
npm run dev |
||||
``` |
||||
|
||||
Open this URL in browser : [http://localhost:8080](http://localhost:8080) |
||||
|
||||
## License |
||||
|
||||
Licensed under either of |
||||
* Apache License, Version 2.0 ([LICENSE-APACHE2](LICENSE-APACHE2) or http://www.apache.org/licenses/LICENSE-2.0) |
||||
* MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) |
||||
at your option. |
||||
|
||||
`SPDX-License-Identifier: Apache-2.0 OR MIT` |
||||
|
||||
--- |
||||
|
||||
NextGraph received funding through the [NGI Assure Fund](https://nlnet.nl/project/NextGraph/index.html), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's [Next Generation Internet](https://ngi.eu/) programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 957073. |
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,31 @@ |
||||
{ |
||||
"name": "ng-app-react", |
||||
"version": "0.1.0", |
||||
"description": "React based web application of NextGraph", |
||||
"main": "src/index.jsx", |
||||
"scripts": { |
||||
"dev": "webpack server" |
||||
}, |
||||
"keywords": [], |
||||
"author": "Niko PLP <niko@nextgraph.org>", |
||||
"license": "MIT/Apache-2.0", |
||||
"repository": { |
||||
"type": "git", |
||||
"url": "https://git.nextgraph.org/NextGraph/nextgraph-rs" |
||||
}, |
||||
"dependencies": { |
||||
"react": "^18.2.0", |
||||
"react-dom": "^18.2.0", |
||||
"ng-app-js-sdk": "^0.1.0" |
||||
}, |
||||
"devDependencies": { |
||||
"@babel/preset-env": "^7.20.2", |
||||
"@babel/preset-react": "^7.18.6", |
||||
"babel-core": "^6.26.3", |
||||
"babel-loader": "^9.1.2", |
||||
"html-webpack-plugin": "^5.5.0", |
||||
"webpack": "^5.75.0", |
||||
"webpack-cli": "^5.0.1", |
||||
"webpack-dev-server": "^4.11.1" |
||||
} |
||||
} |
||||
@ -0,0 +1,23 @@ |
||||
<!-- |
||||
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers |
||||
// All rights reserved. |
||||
// Licensed under the Apache License, Version 2.0 |
||||
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0> |
||||
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>, |
||||
// at your option. All files in the project carrying such |
||||
// notice may not be copied, modified, or distributed except |
||||
// according to those terms. |
||||
--> |
||||
|
||||
<!DOCTYPE html> |
||||
<html lang="en"> |
||||
<head> |
||||
<meta charset="UTF-8" /> |
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" /> |
||||
<title>NextGraph</title> |
||||
</head> |
||||
|
||||
<body> |
||||
<div id="root"></div> |
||||
</body> |
||||
</html> |
||||
@ -0,0 +1,38 @@ |
||||
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers |
||||
// All rights reserved. |
||||
// Licensed under the Apache License, Version 2.0 |
||||
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0> |
||||
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>, |
||||
// at your option. All files in the project carrying such |
||||
// notice may not be copied, modified, or distributed except |
||||
// according to those terms. |
||||
|
||||
import React, { useState } from "react"; |
||||
import ReactDOM from "react-dom"; |
||||
|
||||
const ng_sdk = import("ng-app-js-sdk"); |
||||
|
||||
ng_sdk.then((ng) => { |
||||
const App = () => { |
||||
const [name, setName] = useState(""); |
||||
const handleChange = (e) => { |
||||
setName(ng.change(e.target.value)); |
||||
}; |
||||
const handleClick = () => { |
||||
console.log(name); |
||||
ng.greet(name); |
||||
}; |
||||
|
||||
return ( |
||||
<> |
||||
<div> |
||||
I say: {name}<br/> |
||||
<input type="text" onChange={handleChange} /> |
||||
<button onClick={handleClick}>Say hello!</button> |
||||
</div> |
||||
</> |
||||
); |
||||
}; |
||||
|
||||
ReactDOM.render(<App />, document.getElementById("root")); |
||||
}); |
||||
@ -0,0 +1,40 @@ |
||||
const HtmlWebpackPlugin = require("html-webpack-plugin"); |
||||
const path = require("path"); |
||||
|
||||
module.exports = { |
||||
entry: "./src/index.jsx", |
||||
output: { |
||||
path: path.resolve(__dirname, "dist"), |
||||
filename: "bundle.[hash].js", |
||||
}, |
||||
devServer: { |
||||
compress: true, |
||||
port: 8080, |
||||
hot: true, |
||||
static: "./dist", |
||||
historyApiFallback: true, |
||||
open: true, |
||||
}, |
||||
module: { |
||||
rules: [ |
||||
{ |
||||
test: /\.(js|jsx)$/, |
||||
exclude: /node_modules/, |
||||
use: { |
||||
loader: "babel-loader", |
||||
}, |
||||
}, |
||||
], |
||||
}, |
||||
plugins: [ |
||||
new HtmlWebpackPlugin({ |
||||
template: __dirname + "/public/index.html", |
||||
filename: "index.html", |
||||
}), |
||||
], |
||||
experiments: { |
||||
asyncWebAssembly: true |
||||
}, |
||||
mode: "development", |
||||
devtool: "inline-source-map", |
||||
}; |
||||
@ -0,0 +1,2 @@ |
||||
node_modules |
||||
dist |
||||
@ -0,0 +1,201 @@ |
||||
Apache License |
||||
Version 2.0, January 2004 |
||||
http://www.apache.org/licenses/ |
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION |
||||
|
||||
1. Definitions. |
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction, |
||||
and distribution as defined by Sections 1 through 9 of this document. |
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by |
||||
the copyright owner that is granting the License. |
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all |
||||
other entities that control, are controlled by, or are under common |
||||
control with that entity. For the purposes of this definition, |
||||
"control" means (i) the power, direct or indirect, to cause the |
||||
direction or management of such entity, whether by contract or |
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the |
||||
outstanding shares, or (iii) beneficial ownership of such entity. |
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity |
||||
exercising permissions granted by this License. |
||||
|
||||
"Source" form shall mean the preferred form for making modifications, |
||||
including but not limited to software source code, documentation |
||||
source, and configuration files. |
||||
|
||||
"Object" form shall mean any form resulting from mechanical |
||||
transformation or translation of a Source form, including but |
||||
not limited to compiled object code, generated documentation, |
||||
and conversions to other media types. |
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or |
||||
Object form, made available under the License, as indicated by a |
||||
copyright notice that is included in or attached to the work |
||||
(an example is provided in the Appendix below). |
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object |
||||
form, that is based on (or derived from) the Work and for which the |
||||
editorial revisions, annotations, elaborations, or other modifications |
||||
represent, as a whole, an original work of authorship. For the purposes |
||||
of this License, Derivative Works shall not include works that remain |
||||
separable from, or merely link (or bind by name) to the interfaces of, |
||||
the Work and Derivative Works thereof. |
||||
|
||||
"Contribution" shall mean any work of authorship, including |
||||
the original version of the Work and any modifications or additions |
||||
to that Work or Derivative Works thereof, that is intentionally |
||||
submitted to Licensor for inclusion in the Work by the copyright owner |
||||
or by an individual or Legal Entity authorized to submit on behalf of |
||||
the copyright owner. For the purposes of this definition, "submitted" |
||||
means any form of electronic, verbal, or written communication sent |
||||
to the Licensor or its representatives, including but not limited to |
||||
communication on electronic mailing lists, source code control systems, |
||||
and issue tracking systems that are managed by, or on behalf of, the |
||||
Licensor for the purpose of discussing and improving the Work, but |
||||
excluding communication that is conspicuously marked or otherwise |
||||
designated in writing by the copyright owner as "Not a Contribution." |
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity |
||||
on behalf of whom a Contribution has been received by Licensor and |
||||
subsequently incorporated within the Work. |
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of |
||||
this License, each Contributor hereby grants to You a perpetual, |
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable |
||||
copyright license to reproduce, prepare Derivative Works of, |
||||
publicly display, publicly perform, sublicense, and distribute the |
||||
Work and such Derivative Works in Source or Object form. |
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of |
||||
this License, each Contributor hereby grants to You a perpetual, |
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable |
||||
(except as stated in this section) patent license to make, have made, |
||||
use, offer to sell, sell, import, and otherwise transfer the Work, |
||||
where such license applies only to those patent claims licensable |
||||
by such Contributor that are necessarily infringed by their |
||||
Contribution(s) alone or by combination of their Contribution(s) |
||||
with the Work to which such Contribution(s) was submitted. If You |
||||
institute patent litigation against any entity (including a |
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work |
||||
or a Contribution incorporated within the Work constitutes direct |
||||
or contributory patent infringement, then any patent licenses |
||||
granted to You under this License for that Work shall terminate |
||||
as of the date such litigation is filed. |
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the |
||||
Work or Derivative Works thereof in any medium, with or without |
||||
modifications, and in Source or Object form, provided that You |
||||
meet the following conditions: |
||||
|
||||
(a) You must give any other recipients of the Work or |
||||
Derivative Works a copy of this License; and |
||||
|
||||
(b) You must cause any modified files to carry prominent notices |
||||
stating that You changed the files; and |
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works |
||||
that You distribute, all copyright, patent, trademark, and |
||||
attribution notices from the Source form of the Work, |
||||
excluding those notices that do not pertain to any part of |
||||
the Derivative Works; and |
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its |
||||
distribution, then any Derivative Works that You distribute must |
||||
include a readable copy of the attribution notices contained |
||||
within such NOTICE file, excluding those notices that do not |
||||
pertain to any part of the Derivative Works, in at least one |
||||
of the following places: within a NOTICE text file distributed |
||||
as part of the Derivative Works; within the Source form or |
||||
documentation, if provided along with the Derivative Works; or, |
||||
within a display generated by the Derivative Works, if and |
||||
wherever such third-party notices normally appear. The contents |
||||
of the NOTICE file are for informational purposes only and |
||||
do not modify the License. You may add Your own attribution |
||||
notices within Derivative Works that You distribute, alongside |
||||
or as an addendum to the NOTICE text from the Work, provided |
||||
that such additional attribution notices cannot be construed |
||||
as modifying the License. |
||||
|
||||
You may add Your own copyright statement to Your modifications and |
||||
may provide additional or different license terms and conditions |
||||
for use, reproduction, or distribution of Your modifications, or |
||||
for any such Derivative Works as a whole, provided Your use, |
||||
reproduction, and distribution of the Work otherwise complies with |
||||
the conditions stated in this License. |
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise, |
||||
any Contribution intentionally submitted for inclusion in the Work |
||||
by You to the Licensor shall be under the terms and conditions of |
||||
this License, without any additional terms or conditions. |
||||
Notwithstanding the above, nothing herein shall supersede or modify |
||||
the terms of any separate license agreement you may have executed |
||||
with Licensor regarding such Contributions. |
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade |
||||
names, trademarks, service marks, or product names of the Licensor, |
||||
except as required for reasonable and customary use in describing the |
||||
origin of the Work and reproducing the content of the NOTICE file. |
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or |
||||
agreed to in writing, Licensor provides the Work (and each |
||||
Contributor provides its Contributions) on an "AS IS" BASIS, |
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or |
||||
implied, including, without limitation, any warranties or conditions |
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A |
||||
PARTICULAR PURPOSE. You are solely responsible for determining the |
||||
appropriateness of using or redistributing the Work and assume any |
||||
risks associated with Your exercise of permissions under this License. |
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory, |
||||
whether in tort (including negligence), contract, or otherwise, |
||||
unless required by applicable law (such as deliberate and grossly |
||||
negligent acts) or agreed to in writing, shall any Contributor be |
||||
liable to You for damages, including any direct, indirect, special, |
||||
incidental, or consequential damages of any character arising as a |
||||
result of this License or out of the use or inability to use the |
||||
Work (including but not limited to damages for loss of goodwill, |
||||
work stoppage, computer failure or malfunction, or any and all |
||||
other commercial damages or losses), even if such Contributor |
||||
has been advised of the possibility of such damages. |
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing |
||||
the Work or Derivative Works thereof, You may choose to offer, |
||||
and charge a fee for, acceptance of support, warranty, indemnity, |
||||
or other liability obligations and/or rights consistent with this |
||||
License. However, in accepting such obligations, You may act only |
||||
on Your own behalf and on Your sole responsibility, not on behalf |
||||
of any other Contributor, and only if You agree to indemnify, |
||||
defend, and hold each Contributor harmless for any liability |
||||
incurred by, or claims asserted against, such Contributor by reason |
||||
of your accepting any such warranty or additional liability. |
||||
|
||||
END OF TERMS AND CONDITIONS |
||||
|
||||
APPENDIX: How to apply the Apache License to your work. |
||||
|
||||
To apply the Apache License to your work, attach the following |
||||
boilerplate notice, with the fields enclosed by brackets "[]" |
||||
replaced with your own identifying information. (Don't include |
||||
the brackets!) The text should be enclosed in the appropriate |
||||
comment syntax for the file format. We also recommend that a |
||||
file or class name and description of purpose be included on the |
||||
same "printed page" as the copyright notice for easier |
||||
identification within third-party archives. |
||||
|
||||
Copyright 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers |
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); |
||||
you may not use this file except in compliance with the License. |
||||
You may obtain a copy of the License at |
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
Unless required by applicable law or agreed to in writing, software |
||||
distributed under the License is distributed on an "AS IS" BASIS, |
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
See the License for the specific language governing permissions and |
||||
limitations under the License. |
||||
@ -0,0 +1,25 @@ |
||||
Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers |
||||
|
||||
Permission is hereby granted, free of charge, to any |
||||
person obtaining a copy of this software and associated |
||||
documentation files (the "Software"), to deal in the |
||||
Software without restriction, including without |
||||
limitation the rights to use, copy, modify, merge, |
||||
publish, distribute, sublicense, and/or sell copies of |
||||
the Software, and to permit persons to whom the Software |
||||
is furnished to do so, subject to the following |
||||
conditions: |
||||
|
||||
The above copyright notice and this permission notice |
||||
shall be included in all copies or substantial portions |
||||
of the Software. |
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF |
||||
ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED |
||||
TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A |
||||
PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT |
||||
SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY |
||||
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION |
||||
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR |
||||
IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER |
||||
DEALINGS IN THE SOFTWARE. |
||||
@ -0,0 +1,15 @@ |
||||
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers
|
||||
// All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0
|
||||
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
|
||||
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
|
||||
// at your option. All files in the project carrying such
|
||||
// notice may not be copied, modified, or distributed except
|
||||
// according to those terms.
|
||||
|
||||
|
||||
// A dependency graph that contains any wasm must all be imported
|
||||
// asynchronously. This `bootstrap.js` file does the single async import, so
|
||||
// that no one else needs to worry about it again.
|
||||
import("./index.js") |
||||
.catch(e => console.error("Error importing `index.js`:", e)); |
||||
@ -0,0 +1,22 @@ |
||||
|
||||
<!-- |
||||
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers |
||||
// All rights reserved. |
||||
// Licensed under the Apache License, Version 2.0 |
||||
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0> |
||||
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>, |
||||
// at your option. All files in the project carrying such |
||||
// notice may not be copied, modified, or distributed except |
||||
// according to those terms. |
||||
--> |
||||
<!DOCTYPE html> |
||||
<html> |
||||
<head> |
||||
<meta charset="utf-8"> |
||||
<title>NextGraph</title> |
||||
</head> |
||||
<body> |
||||
<noscript>This page contains webassembly and javascript content, please enable javascript in your browser.</noscript> |
||||
<script src="./bootstrap.js"></script> |
||||
</body> |
||||
</html> |
||||
@ -0,0 +1,12 @@ |
||||
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers
|
||||
// All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0
|
||||
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
|
||||
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
|
||||
// at your option. All files in the project carrying such
|
||||
// notice may not be copied, modified, or distributed except
|
||||
// according to those terms.
|
||||
|
||||
import * as ng from "ng-app-js-sdk"; |
||||
|
||||
console.log(ng.change("you")); |
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,35 @@ |
||||
{ |
||||
"name": "ng-app-web", |
||||
"version": "0.1.0", |
||||
"description": "Web app client of NextGraph", |
||||
"main": "index.js", |
||||
"scripts": { |
||||
"build": "webpack --config webpack.config.js", |
||||
"start": "webpack-dev-server" |
||||
}, |
||||
"repository": { |
||||
"type": "git", |
||||
"url": "git+https://git.nextgraph.org/NextGraph/nextgraph-rs.git" |
||||
}, |
||||
"keywords": [ |
||||
"webassembly", |
||||
"wasm", |
||||
"rust", |
||||
"webpack" |
||||
], |
||||
"author": "Niko PLP <niko@nextgraph.org>", |
||||
"license": "(MIT OR Apache-2.0)", |
||||
"bugs": { |
||||
"url": "https://git.nextgraph.org/NextGraph/nextgraph-rs/issues" |
||||
}, |
||||
"homepage": "https://docs.nextgraph.org", |
||||
"dependencies": { |
||||
"ng-app-js-sdk": "^0.1.0" |
||||
}, |
||||
"devDependencies": { |
||||
"webpack": "^4.29.3", |
||||
"webpack-cli": "^3.1.0", |
||||
"webpack-dev-server": "^3.1.5", |
||||
"copy-webpack-plugin": "^5.0.0" |
||||
} |
||||
} |
||||
@ -0,0 +1,14 @@ |
||||
const CopyWebpackPlugin = require("copy-webpack-plugin"); |
||||
const path = require('path'); |
||||
|
||||
module.exports = { |
||||
entry: "./bootstrap.js", |
||||
output: { |
||||
path: path.resolve(__dirname, "dist"), |
||||
filename: "bootstrap.js", |
||||
}, |
||||
mode: "development", |
||||
plugins: [ |
||||
new CopyWebpackPlugin(['index.html']) |
||||
], |
||||
}; |
||||
@ -0,0 +1,27 @@ |
||||
<!-- |
||||
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers |
||||
// All rights reserved. |
||||
// Licensed under the Apache License, Version 2.0 |
||||
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0> |
||||
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>, |
||||
// at your option. All files in the project carrying such |
||||
// notice may not be copied, modified, or distributed except |
||||
// according to those terms. |
||||
--> |
||||
|
||||
<!DOCTYPE html> |
||||
<html lang="en-US"> |
||||
<head> |
||||
<meta charset="utf-8" /> |
||||
<title>NextGraph web sdk test</title> |
||||
</head> |
||||
<body> |
||||
<p>run <code>python3 -m http.server</code> to use it</p> |
||||
<script type="module"> |
||||
import init, { greet } from "./pkg/ng_app_js_sdk.js"; |
||||
init().then(() => { |
||||
greet("WebAssembly"); |
||||
}); |
||||
</script> |
||||
</body> |
||||
</html> |
||||
@ -0,0 +1,9 @@ |
||||
const fs = require('fs'); |
||||
|
||||
const PATH = './pkg-node/package.json'; |
||||
|
||||
const pkg_json = fs.readFileSync(PATH); |
||||
let pkg = JSON.parse(pkg_json) |
||||
pkg.name = "ng-app-node-sdk"; |
||||
pkg.description = "nodejs app sdk of NextGraph"; |
||||
fs.writeFileSync(PATH, JSON.stringify(pkg, null, 2), 'utf8'); |
||||
@ -0,0 +1,16 @@ |
||||
use wasm_bindgen::prelude::*; |
||||
|
||||
#[wasm_bindgen] |
||||
extern { |
||||
pub fn alert(s: &str); |
||||
} |
||||
|
||||
#[wasm_bindgen] |
||||
pub fn greet(name: &str) { |
||||
alert(&format!("I say: {}", name)); |
||||
} |
||||
|
||||
#[wasm_bindgen] |
||||
pub fn change(name: &str) -> JsValue { |
||||
JsValue::from_str(&format!("Hellooo, {}!", name)) |
||||
} |
||||
@ -1,15 +0,0 @@ |
||||
<!DOCTYPE html> |
||||
<html lang="en-US"> |
||||
<head> |
||||
<meta charset="utf-8" /> |
||||
<title>hello-wasm example</title> |
||||
</head> |
||||
<body> |
||||
<script type="module"> |
||||
import init, { greet } from "./pkg/ng_app_web.js"; |
||||
init().then(() => { |
||||
greet("WebAssembly"); |
||||
}); |
||||
</script> |
||||
</body> |
||||
</html> |
||||
@ -1,11 +0,0 @@ |
||||
use wasm_bindgen::prelude::*; |
||||
|
||||
#[wasm_bindgen] |
||||
extern { |
||||
pub fn alert(s: &str); |
||||
} |
||||
|
||||
#[wasm_bindgen] |
||||
pub fn greet(name: &str) { |
||||
alert(&format!("Hello, {}!", name)); |
||||
} |
||||
@ -0,0 +1,24 @@ |
||||
[package] |
||||
name = "ngcli" |
||||
version = "0.1.0" |
||||
edition = "2021" |
||||
license = "MIT/Apache-2.0" |
||||
authors = ["Niko PLP <niko@nextgraph.org>"] |
||||
description = "CLI command-line interpreter of NextGraph" |
||||
repository = "https://git.nextgraph.org/NextGraph/nextgraph-rs" |
||||
|
||||
[dependencies] |
||||
debug_print = "1.0.0" |
||||
p2p-repo = { path = "../p2p-repo" } |
||||
p2p-net = { path = "../p2p-net" } |
||||
p2p-client = { path = "../p2p-client" } |
||||
p2p-broker = { path = "../p2p-broker" } |
||||
p2p-stores-lmdb = { path = "../p2p-stores-lmdb" } |
||||
async-std = { version = "1.7.0", features = ["attributes"] } |
||||
futures = "0.3.24" |
||||
xactor = "0.7.11" |
||||
tempfile = "3" |
||||
fastbloom-rs = "0.3.1" |
||||
rand = "0.7" |
||||
ed25519-dalek = "1.0.1" |
||||
assert_cmd = "2.0.5" |
||||
@ -0,0 +1,636 @@ |
||||
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers
|
||||
// All rights reserved.
|
||||
// This code is partly derived from work written by TG x Thoth from P2Pcollab.
|
||||
// Copyright 2022 TG x Thoth
|
||||
// Licensed under the Apache License, Version 2.0
|
||||
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
|
||||
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
|
||||
// at your option. All files in the project carrying such
|
||||
// notice may not be copied, modified, or distributed except
|
||||
// according to those terms.
|
||||
|
||||
use debug_print::*; |
||||
use ed25519_dalek::*; |
||||
use fastbloom_rs::{BloomFilter as Filter, FilterBuilder, Membership}; |
||||
use futures::{future, pin_mut, stream, SinkExt, StreamExt}; |
||||
use p2p_repo::object::Object; |
||||
use p2p_repo::store::{store_max_value_size, store_valid_value_size, HashMapRepoStore, RepoStore}; |
||||
use p2p_broker::broker_store_config::ConfigMode; |
||||
use p2p_stores_lmdb::broker_store::LmdbBrokerStore; |
||||
use p2p_stores_lmdb::repo_store::LmdbRepoStore; |
||||
use rand::rngs::OsRng; |
||||
use std::collections::HashMap; |
||||
|
||||
use p2p_repo::types::*; |
||||
use p2p_repo::utils::{generate_keypair, now_timestamp}; |
||||
use p2p_broker::server_ws::*; |
||||
use p2p_broker::server::*; |
||||
use p2p_net::errors::*; |
||||
use p2p_net::types::*; |
||||
use p2p_net::broker_connection::*; |
||||
use p2p_client::connection_remote::*; |
||||
use p2p_client::connection_ws::*; |
||||
use p2p_broker::connection_local::*; |
||||
|
||||
fn block_size() -> usize { |
||||
store_max_value_size() |
||||
//store_valid_value_size(0)
|
||||
} |
||||
|
||||
async fn test_sync(cnx: &mut impl BrokerConnection, user_pub_key: PubKey, userpriv_key: PrivKey) { |
||||
fn add_obj( |
||||
content: ObjectContent, |
||||
deps: Vec<ObjectId>, |
||||
expiry: Option<Timestamp>, |
||||
repo_pubkey: PubKey, |
||||
repo_secret: SymKey, |
||||
store: &mut impl RepoStore, |
||||
) -> ObjectRef { |
||||
let max_object_size = 4000; |
||||
let obj = Object::new( |
||||
content, |
||||
deps, |
||||
expiry, |
||||
max_object_size, |
||||
repo_pubkey, |
||||
repo_secret, |
||||
); |
||||
//println!(">>> add_obj");
|
||||
println!(" id: {}", obj.id()); |
||||
//println!(" deps: {:?}", obj.deps());
|
||||
obj.save(store).unwrap(); |
||||
obj.reference().unwrap() |
||||
} |
||||
|
||||
fn add_commit( |
||||
branch: ObjectRef, |
||||
author_privkey: PrivKey, |
||||
author_pubkey: PubKey, |
||||
seq: u32, |
||||
deps: Vec<ObjectRef>, |
||||
acks: Vec<ObjectRef>, |
||||
body_ref: ObjectRef, |
||||
repo_pubkey: PubKey, |
||||
repo_secret: SymKey, |
||||
store: &mut impl RepoStore, |
||||
) -> ObjectRef { |
||||
let mut obj_deps: Vec<ObjectId> = vec![]; |
||||
obj_deps.extend(deps.iter().map(|r| r.id)); |
||||
obj_deps.extend(acks.iter().map(|r| r.id)); |
||||
|
||||
let obj_ref = ObjectRef { |
||||
id: ObjectId::Blake3Digest32([1; 32]), |
||||
key: SymKey::ChaCha20Key([2; 32]), |
||||
}; |
||||
let refs = vec![obj_ref]; |
||||
let metadata = vec![5u8; 55]; |
||||
let expiry = None; |
||||
|
||||
let commit = Commit::new( |
||||
author_privkey, |
||||
author_pubkey, |
||||
seq, |
||||
branch, |
||||
deps, |
||||
acks, |
||||
refs, |
||||
metadata, |
||||
body_ref, |
||||
expiry, |
||||
) |
||||
.unwrap(); |
||||
//println!("commit: {}", commit.id().unwrap());
|
||||
add_obj( |
||||
ObjectContent::Commit(commit), |
||||
obj_deps, |
||||
expiry, |
||||
repo_pubkey, |
||||
repo_secret, |
||||
store, |
||||
) |
||||
} |
||||
|
||||
fn add_body_branch( |
||||
branch: Branch, |
||||
repo_pubkey: PubKey, |
||||
repo_secret: SymKey, |
||||
store: &mut impl RepoStore, |
||||
) -> ObjectRef { |
||||
let deps = vec![]; |
||||
let expiry = None; |
||||
let body = CommitBody::Branch(branch); |
||||
//println!("body: {:?}", body);
|
||||
add_obj( |
||||
ObjectContent::CommitBody(body), |
||||
deps, |
||||
expiry, |
||||
repo_pubkey, |
||||
repo_secret, |
||||
store, |
||||
) |
||||
} |
||||
|
||||
fn add_body_trans( |
||||
deps: Vec<ObjectId>, |
||||
repo_pubkey: PubKey, |
||||
repo_secret: SymKey, |
||||
store: &mut impl RepoStore, |
||||
) -> ObjectRef { |
||||
let expiry = None; |
||||
let content = [7u8; 777].to_vec(); |
||||
let body = CommitBody::Transaction(Transaction::V0(content)); |
||||
//println!("body: {:?}", body);
|
||||
add_obj( |
||||
ObjectContent::CommitBody(body), |
||||
deps, |
||||
expiry, |
||||
repo_pubkey, |
||||
repo_secret, |
||||
store, |
||||
) |
||||
} |
||||
|
||||
fn add_body_ack( |
||||
deps: Vec<ObjectId>, |
||||
repo_pubkey: PubKey, |
||||
repo_secret: SymKey, |
||||
store: &mut impl RepoStore, |
||||
) -> ObjectRef { |
||||
let expiry = None; |
||||
let body = CommitBody::Ack(Ack::V0()); |
||||
//println!("body: {:?}", body);
|
||||
add_obj( |
||||
ObjectContent::CommitBody(body), |
||||
deps, |
||||
expiry, |
||||
repo_pubkey, |
||||
repo_secret, |
||||
store, |
||||
) |
||||
} |
||||
|
||||
let mut store = HashMapRepoStore::new(); |
||||
let mut rng = OsRng {}; |
||||
|
||||
// repo
|
||||
|
||||
let repo_keypair: Keypair = Keypair::generate(&mut rng); |
||||
// println!(
|
||||
// "repo private key: ({}) {:?}",
|
||||
// repo_keypair.secret.as_bytes().len(),
|
||||
// repo_keypair.secret.as_bytes()
|
||||
// );
|
||||
// println!(
|
||||
// "repo public key: ({}) {:?}",
|
||||
// repo_keypair.public.as_bytes().len(),
|
||||
// repo_keypair.public.as_bytes()
|
||||
// );
|
||||
let _repo_privkey = PrivKey::Ed25519PrivKey(repo_keypair.secret.to_bytes()); |
||||
let repo_pubkey = PubKey::Ed25519PubKey(repo_keypair.public.to_bytes()); |
||||
let repo_secret = SymKey::ChaCha20Key([9; 32]); |
||||
|
||||
let repolink = RepoLink::V0(RepoLinkV0 { |
||||
id: repo_pubkey, |
||||
secret: repo_secret, |
||||
peers: vec![], |
||||
}); |
||||
|
||||
// branch
|
||||
|
||||
let branch_keypair: Keypair = Keypair::generate(&mut rng); |
||||
//println!("branch public key: {:?}", branch_keypair.public.as_bytes());
|
||||
let branch_pubkey = PubKey::Ed25519PubKey(branch_keypair.public.to_bytes()); |
||||
|
||||
let member_keypair: Keypair = Keypair::generate(&mut rng); |
||||
//println!("member public key: {:?}", member_keypair.public.as_bytes());
|
||||
let member_privkey = PrivKey::Ed25519PrivKey(member_keypair.secret.to_bytes()); |
||||
let member_pubkey = PubKey::Ed25519PubKey(member_keypair.public.to_bytes()); |
||||
|
||||
let metadata = [66u8; 64].to_vec(); |
||||
let commit_types = vec![CommitType::Ack, CommitType::Transaction]; |
||||
let secret = SymKey::ChaCha20Key([0; 32]); |
||||
|
||||
let member = MemberV0::new(member_pubkey, commit_types, metadata.clone()); |
||||
let members = vec![member]; |
||||
let mut quorum = HashMap::new(); |
||||
quorum.insert(CommitType::Transaction, 3); |
||||
let ack_delay = RelTime::Minutes(3); |
||||
let tags = [99u8; 32].to_vec(); |
||||
let branch = Branch::new( |
||||
branch_pubkey, |
||||
branch_pubkey, |
||||
secret, |
||||
members, |
||||
quorum, |
||||
ack_delay, |
||||
tags, |
||||
metadata, |
||||
); |
||||
//println!("branch: {:?}", branch);
|
||||
|
||||
println!("branch deps/acks:"); |
||||
println!(""); |
||||
println!(" br"); |
||||
println!(" / \\"); |
||||
println!(" t1 t2"); |
||||
println!(" / \\ / \\"); |
||||
println!(" a3 t4<--t5-->(t1)"); |
||||
println!(" / \\"); |
||||
println!(" a6 a7"); |
||||
println!(""); |
||||
|
||||
// commit bodies
|
||||
|
||||
let branch_body = add_body_branch( |
||||
branch.clone(), |
||||
repo_pubkey.clone(), |
||||
repo_secret.clone(), |
||||
&mut store, |
||||
); |
||||
let ack_body = add_body_ack(vec![], repo_pubkey, repo_secret, &mut store); |
||||
let trans_body = add_body_trans(vec![], repo_pubkey, repo_secret, &mut store); |
||||
|
||||
// create & add commits to store
|
||||
|
||||
println!(">> br"); |
||||
let br = add_commit( |
||||
branch_body, |
||||
member_privkey, |
||||
member_pubkey, |
||||
0, |
||||
vec![], |
||||
vec![], |
||||
branch_body, |
||||
repo_pubkey, |
||||
repo_secret, |
||||
&mut store, |
||||
); |
||||
|
||||
println!(">> t1"); |
||||
let t1 = add_commit( |
||||
branch_body, |
||||
member_privkey, |
||||
member_pubkey, |
||||
1, |
||||
vec![br], |
||||
vec![], |
||||
trans_body, |
||||
repo_pubkey, |
||||
repo_secret, |
||||
&mut store, |
||||
); |
||||
|
||||
println!(">> t2"); |
||||
let t2 = add_commit( |
||||
branch_body, |
||||
member_privkey, |
||||
member_pubkey, |
||||
2, |
||||
vec![br], |
||||
vec![], |
||||
trans_body, |
||||
repo_pubkey, |
||||
repo_secret, |
||||
&mut store, |
||||
); |
||||
|
||||
println!(">> a3"); |
||||
let a3 = add_commit( |
||||
branch_body, |
||||
member_privkey, |
||||
member_pubkey, |
||||
3, |
||||
vec![t1], |
||||
vec![], |
||||
ack_body, |
||||
repo_pubkey, |
||||
repo_secret, |
||||
&mut store, |
||||
); |
||||
|
||||
println!(">> t4"); |
||||
let t4 = add_commit( |
||||
branch_body, |
||||
member_privkey, |
||||
member_pubkey, |
||||
4, |
||||
vec![t2], |
||||
vec![t1], |
||||
trans_body, |
||||
repo_pubkey, |
||||
repo_secret, |
||||
&mut store, |
||||
); |
||||
|
||||
println!(">> t5"); |
||||
let t5 = add_commit( |
||||
branch_body, |
||||
member_privkey, |
||||
member_pubkey, |
||||
5, |
||||
vec![t1, t2], |
||||
vec![t4], |
||||
trans_body, |
||||
repo_pubkey, |
||||
repo_secret, |
||||
&mut store, |
||||
); |
||||
|
||||
println!(">> a6"); |
||||
let a6 = add_commit( |
||||
branch_body, |
||||
member_privkey, |
||||
member_pubkey, |
||||
6, |
||||
vec![t4], |
||||
vec![], |
||||
ack_body, |
||||
repo_pubkey, |
||||
repo_secret, |
||||
&mut store, |
||||
); |
||||
|
||||
println!(">> a7"); |
||||
let a7 = add_commit( |
||||
branch_body, |
||||
member_privkey, |
||||
member_pubkey, |
||||
7, |
||||
vec![t4], |
||||
vec![], |
||||
ack_body, |
||||
repo_pubkey, |
||||
repo_secret, |
||||
&mut store, |
||||
); |
||||
|
||||
let mut public_overlay_cnx = cnx |
||||
.overlay_connect(&repolink, true) |
||||
.await |
||||
.expect("overlay_connect failed"); |
||||
|
||||
// Sending everything to the broker
|
||||
for (v) in store.get_all() { |
||||
//debug_println!("SENDING {}", k);
|
||||
let _ = public_overlay_cnx |
||||
.put_block(&v) |
||||
.await |
||||
.expect("put_block failed"); |
||||
} |
||||
|
||||
// Now emptying the local store of the client, and adding only 1 commit into it (br)
|
||||
// we also have received an commit (t5) but we don't know what to do with it...
|
||||
let mut store = HashMapRepoStore::new(); |
||||
|
||||
let br = add_commit( |
||||
branch_body, |
||||
member_privkey, |
||||
member_pubkey, |
||||
0, |
||||
vec![], |
||||
vec![], |
||||
branch_body, |
||||
repo_pubkey, |
||||
repo_secret, |
||||
&mut store, |
||||
); |
||||
|
||||
let t5 = add_commit( |
||||
branch_body, |
||||
member_privkey, |
||||
member_pubkey, |
||||
5, |
||||
vec![t1, t2], |
||||
vec![t4], |
||||
trans_body, |
||||
repo_pubkey, |
||||
repo_secret, |
||||
&mut store, |
||||
); |
||||
|
||||
debug_println!("LOCAL STORE HAS {} BLOCKS", store.get_len()); |
||||
|
||||
// Let's pretend that we know that the head of the branch in the broker is at commits a6 and a7.
|
||||
// normally it would be the pub/sub that notifies us of those heads.
|
||||
// now we want to synchronize with the broker.
|
||||
|
||||
let mut filter = Filter::new(FilterBuilder::new(10, 0.01)); |
||||
for commit_ref in [br, t5] { |
||||
match commit_ref.id { |
||||
ObjectId::Blake3Digest32(d) => filter.add(&d), |
||||
} |
||||
} |
||||
let cfg = filter.config(); |
||||
|
||||
let known_commits = BloomFilter { |
||||
k: cfg.hashes, |
||||
f: filter.get_u8_array().to_vec(), |
||||
}; |
||||
|
||||
let known_heads = [br.id]; |
||||
|
||||
let remote_heads = [a6.id, a7.id]; |
||||
|
||||
let mut synced_blocks_stream = public_overlay_cnx |
||||
.sync_branch(remote_heads.to_vec(), known_heads.to_vec(), known_commits) |
||||
.await |
||||
.expect("sync_branch failed"); |
||||
|
||||
let mut i = 0; |
||||
while let Some(b) = synced_blocks_stream.next().await { |
||||
debug_println!("GOT BLOCK {}", b.id()); |
||||
store.put(&b); |
||||
i += 1; |
||||
} |
||||
|
||||
debug_println!("SYNCED {} BLOCKS", i); |
||||
|
||||
debug_println!("LOCAL STORE HAS {} BLOCKS", store.get_len()); |
||||
|
||||
// now the client can verify the DAG and each commit. Then update its list of heads.
|
||||
} |
||||
|
||||
async fn test(cnx: &mut impl BrokerConnection, pub_key: PubKey, priv_key: PrivKey) -> Result<(), ProtocolError>{ |
||||
|
||||
cnx.add_user(PubKey::Ed25519PubKey([1; 32]), priv_key).await?; |
||||
|
||||
cnx.add_user(pub_key, priv_key).await?; |
||||
//.expect("add_user 2 (myself) failed");
|
||||
|
||||
assert_eq!( |
||||
cnx.add_user(PubKey::Ed25519PubKey([1; 32]), priv_key).await.err().unwrap(), |
||||
ProtocolError::UserAlreadyExists |
||||
); |
||||
|
||||
let repo = RepoLink::V0(RepoLinkV0 { |
||||
id: PubKey::Ed25519PubKey([1; 32]), |
||||
secret: SymKey::ChaCha20Key([0; 32]), |
||||
peers: vec![], |
||||
}); |
||||
let mut public_overlay_cnx = cnx |
||||
.overlay_connect(&repo, true) |
||||
.await?; |
||||
|
||||
let my_block_id = public_overlay_cnx |
||||
.put_block(&Block::new( |
||||
vec![], |
||||
ObjectDeps::ObjectIdList(vec![]), |
||||
None, |
||||
vec![27; 150], |
||||
None, |
||||
)) |
||||
.await?; |
||||
|
||||
debug_println!("added block_id to store {}", my_block_id); |
||||
|
||||
let object_id = public_overlay_cnx |
||||
.put_object( |
||||
ObjectContent::File(File::V0(FileV0 { |
||||
content_type: vec![], |
||||
metadata: vec![], |
||||
content: vec![48; 69000], |
||||
})), |
||||
vec![], |
||||
None, |
||||
block_size(), |
||||
repo.id(), |
||||
repo.secret(), |
||||
) |
||||
.await?; |
||||
|
||||
debug_println!("added object_id to store {}", object_id); |
||||
|
||||
let mut my_block_stream = public_overlay_cnx |
||||
.get_block(my_block_id, true, None) |
||||
.await?; |
||||
//.expect("get_block failed");
|
||||
|
||||
while let Some(b) = my_block_stream.next().await { |
||||
debug_println!("GOT BLOCK {}", b.id()); |
||||
} |
||||
|
||||
let mut my_object_stream = public_overlay_cnx |
||||
.get_block(object_id, true, None) |
||||
.await?; |
||||
//.expect("get_block for object failed");
|
||||
|
||||
while let Some(b) = my_object_stream.next().await { |
||||
debug_println!("GOT BLOCK {}", b.id()); |
||||
} |
||||
|
||||
let object = public_overlay_cnx |
||||
.get_object(object_id, None) |
||||
.await?; |
||||
//.expect("get_object failed");
|
||||
|
||||
debug_println!("GOT OBJECT with ID {}", object.id()); |
||||
|
||||
// let object_id = public_overlay_cnx
|
||||
// .copy_object(object_id, Some(now_timestamp() + 60))
|
||||
// .await
|
||||
// .expect("copy_object failed");
|
||||
|
||||
// debug_println!("COPIED OBJECT to OBJECT ID {}", object_id);
|
||||
|
||||
public_overlay_cnx |
||||
.delete_object(object_id) |
||||
.await?; |
||||
//.expect("delete_object failed");
|
||||
|
||||
let res = public_overlay_cnx |
||||
.get_object(object_id, None) |
||||
.await |
||||
.unwrap_err(); |
||||
|
||||
debug_println!("result from get object after delete: {}", res); |
||||
assert_eq!(res, ProtocolError::NotFound); |
||||
|
||||
//TODO test pin/unpin
|
||||
|
||||
// TEST BRANCH SYNC
|
||||
|
||||
test_sync(cnx, pub_key, priv_key).await; |
||||
|
||||
Ok(()) |
||||
} |
||||
|
||||
async fn test_local_connection() { |
||||
debug_println!("===== TESTING LOCAL API ====="); |
||||
|
||||
let root = tempfile::Builder::new() |
||||
.prefix("node-daemon") |
||||
.tempdir() |
||||
.unwrap(); |
||||
let master_key: [u8; 32] = [0; 32]; |
||||
std::fs::create_dir_all(root.path()).unwrap(); |
||||
println!("{}", root.path().to_str().unwrap()); |
||||
let store = LmdbBrokerStore::open(root.path(), master_key); |
||||
|
||||
let mut server = BrokerServer::new(store, ConfigMode::Local).expect("starting broker"); |
||||
|
||||
let (priv_key, pub_key) = generate_keypair(); |
||||
|
||||
let mut cnx = server.local_connection(pub_key); |
||||
|
||||
test(&mut cnx, pub_key, priv_key).await; |
||||
} |
||||
|
||||
async fn test_remote_connection(url: &str) { |
||||
debug_println!("===== TESTING REMOTE API ====="); |
||||
|
||||
let (priv_key, pub_key) = generate_keypair(); |
||||
let cnx_res = BrokerConnectionWebSocket::open(url, priv_key, pub_key).await; |
||||
match cnx_res { |
||||
Ok(mut cnx) => { |
||||
if let Err(e) = test(&mut cnx, pub_key, priv_key).await { |
||||
debug_println!("error: {:?}", e) |
||||
} |
||||
else { |
||||
cnx.close().await; |
||||
|
||||
} } |
||||
Err(e) => { |
||||
|
||||
} |
||||
} |
||||
|
||||
} |
||||
|
||||
#[xactor::main] |
||||
async fn main() -> std::io::Result<()> { |
||||
debug_println!("Starting nextgraph CLI..."); |
||||
|
||||
test_local_connection().await; |
||||
|
||||
test_remote_connection("ws://127.0.0.1:3012").await; |
||||
|
||||
Ok(()) |
||||
} |
||||
|
||||
#[cfg(test)] |
||||
mod test { |
||||
|
||||
use crate::{test_local_connection, test_remote_connection}; |
||||
|
||||
#[async_std::test] |
||||
pub async fn test_local_cnx() { |
||||
xactor::block_on(test_local_connection()); |
||||
} |
||||
|
||||
use async_std::task; |
||||
use p2p_broker::server_ws::*; |
||||
|
||||
#[async_std::test] |
||||
pub async fn test_remote_cnx() -> Result<(), Box<dyn std::error::Error>> { |
||||
|
||||
let thr = task::spawn(run_server_accept_one("127.0.0.1:3012")); |
||||
|
||||
std::thread::sleep(std::time::Duration::from_secs(2)); |
||||
|
||||
xactor::block_on(test_remote_connection("ws://127.0.0.1:3012")); |
||||
|
||||
xactor::block_on(thr); |
||||
|
||||
Ok(()) |
||||
} |
||||
} |
||||
@ -0,0 +1,12 @@ |
||||
[package] |
||||
name = "ngd" |
||||
version = "0.1.0" |
||||
edition = "2021" |
||||
license = "MIT/Apache-2.0" |
||||
authors = ["Niko PLP <niko@nextgraph.org>"] |
||||
description = "Daemon of NextGraph" |
||||
repository = "https://git.nextgraph.org/NextGraph/nextgraph-rs" |
||||
|
||||
[dependencies] |
||||
p2p-broker = { path = "../p2p-broker" } |
||||
async-std = { version = "1.7.0", features = ["attributes"] } |
||||
@ -0,0 +1,18 @@ |
||||
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers
|
||||
// All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0
|
||||
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
|
||||
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
|
||||
// at your option. All files in the project carrying such
|
||||
// notice may not be copied, modified, or distributed except
|
||||
// according to those terms.
|
||||
|
||||
use p2p_broker::server_ws::run_server; |
||||
|
||||
|
||||
#[async_std::main] |
||||
async fn main() -> std::io::Result<()> { |
||||
println!("Starting NextGraph daemon..."); |
||||
|
||||
run_server("127.0.0.1:3012").await |
||||
} |
||||
@ -0,0 +1,27 @@ |
||||
[package] |
||||
name = "p2p-broker" |
||||
version = "0.1.0" |
||||
edition = "2021" |
||||
license = "MIT/Apache-2.0" |
||||
authors = ["Niko PLP <niko@nextgraph.org>"] |
||||
description = "P2P Broker module of NextGraph" |
||||
repository = "https://git.nextgraph.org/NextGraph/nextgraph-rs" |
||||
|
||||
[dependencies] |
||||
debug_print = "1.0.0" |
||||
p2p-repo = { path = "../p2p-repo" } |
||||
p2p-net = { path = "../p2p-net" } |
||||
p2p-stores-lmdb = { path = "../p2p-stores-lmdb" } |
||||
chacha20 = "0.9.0" |
||||
serde = { version = "1.0", features = ["derive"] } |
||||
serde_bare = "0.5.0" |
||||
serde_bytes = "0.11.7" |
||||
async-std = { version = "1.7.0", features = ["attributes"] } |
||||
futures = "0.3.24" |
||||
rust-fsm = "0.6.0" |
||||
getrandom = "0.2.7" |
||||
async-channel = "1.7.1" |
||||
tempfile = "3" |
||||
hex = "0.4.3" |
||||
async-trait = "0.1.57" |
||||
async-tungstenite = { version = "0.17.2", features = ["async-std-runtime","async-native-tls"] } |
||||
@ -0,0 +1,178 @@ |
||||
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers
|
||||
// All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0
|
||||
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
|
||||
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
|
||||
// at your option. All files in the project carrying such
|
||||
// notice may not be copied, modified, or distributed except
|
||||
// according to those terms.
|
||||
|
||||
use std::pin::Pin; |
||||
|
||||
use debug_print::*; |
||||
use futures::future::BoxFuture; |
||||
use futures::future::OptionFuture; |
||||
use futures::FutureExt; |
||||
use p2p_repo::types::*; |
||||
use p2p_repo::utils::*; |
||||
use p2p_net::errors::*; |
||||
use p2p_net::types::*; |
||||
use rust_fsm::*; |
||||
|
||||
state_machine! { |
||||
derive(Debug) |
||||
AuthProtocolClient(Ready) |
||||
|
||||
Ready(ClientHelloSent) => ClientHelloSent, |
||||
ClientHelloSent(ServerHelloReceived) => ServerHelloReceived, |
||||
ServerHelloReceived(ClientAuthSent) => ClientAuthSent, |
||||
ClientAuthSent(AuthResultReceived) => AuthResult, |
||||
AuthResult => { |
||||
Ok => BrokerProtocol, |
||||
Error => Closed, |
||||
}, |
||||
} |
||||
|
||||
state_machine! { |
||||
derive(Debug) |
||||
AuthProtocolServer(Ready) |
||||
|
||||
Ready(ClientHelloReceived) => ClientHelloReceived, |
||||
ClientHelloReceived(ServerHelloSent) => ServerHelloSent, |
||||
ServerHelloSent(ClientAuthReceived) => ClientAuthReceived, |
||||
ClientAuthReceived => { |
||||
Ok => AuthResultOk, |
||||
Error => AuthResultError, |
||||
}, |
||||
AuthResultOk(AuthResultSent) => BrokerProtocol, |
||||
AuthResultError(AuthResultSent) => Closed, |
||||
} |
||||
|
||||
pub struct AuthProtocolHandler { |
||||
machine: StateMachine<AuthProtocolServer>, |
||||
nonce: Option<Vec<u8>>, |
||||
user: Option<PubKey>, |
||||
} |
||||
|
||||
impl AuthProtocolHandler { |
||||
pub fn new() -> AuthProtocolHandler { |
||||
AuthProtocolHandler { |
||||
machine: StateMachine::new(), |
||||
nonce: None, |
||||
user: None, |
||||
} |
||||
} |
||||
|
||||
pub fn get_user(&self) -> Option<PubKey> { |
||||
self.user |
||||
} |
||||
|
||||
pub fn handle_init(&mut self, client_hello: ClientHello) -> Result<Vec<u8>, ProtocolError> { |
||||
let _ = self |
||||
.machine |
||||
.consume(&AuthProtocolServerInput::ClientHelloReceived) |
||||
.map_err(|_e| ProtocolError::InvalidState)?; |
||||
|
||||
let mut random_buf = [0u8; 32]; |
||||
getrandom::getrandom(&mut random_buf).unwrap(); |
||||
let nonce = random_buf.to_vec(); |
||||
let reply = ServerHello::V0(ServerHelloV0 { |
||||
nonce: nonce.clone(), |
||||
}); |
||||
self.nonce = Some(nonce); |
||||
|
||||
let _ = self |
||||
.machine |
||||
.consume(&AuthProtocolServerInput::ServerHelloSent) |
||||
.map_err(|_e| ProtocolError::InvalidState)?; |
||||
|
||||
//debug_println!("sending nonce to client: {:?}", self.nonce);
|
||||
|
||||
Ok(serde_bare::to_vec(&reply).unwrap()) |
||||
} |
||||
|
||||
pub fn handle_incoming( |
||||
&mut self, |
||||
frame: Vec<u8>, |
||||
) -> ( |
||||
Result<Vec<u8>, ProtocolError>, |
||||
Pin<Box<OptionFuture<BoxFuture<'static, u16>>>>, |
||||
) { |
||||
fn prepare_reply(res: Result<Vec<u8>, ProtocolError>) -> AuthResult { |
||||
let (result, metadata) = match res { |
||||
Ok(m) => (0, m), |
||||
Err(e) => (e.into(), vec![]), |
||||
}; |
||||
AuthResult::V0(AuthResultV0 { result, metadata }) |
||||
} |
||||
|
||||
fn process_state( |
||||
handler: &mut AuthProtocolHandler, |
||||
frame: Vec<u8>, |
||||
) -> Result<Vec<u8>, ProtocolError> { |
||||
match handler.machine.state() { |
||||
&AuthProtocolServerState::ServerHelloSent => { |
||||
let message = serde_bare::from_slice::<ClientAuth>(&frame)?; |
||||
let _ = handler |
||||
.machine |
||||
.consume(&AuthProtocolServerInput::ClientAuthReceived) |
||||
.map_err(|_e| ProtocolError::InvalidState)?; |
||||
|
||||
// verifying client auth
|
||||
|
||||
debug_println!("verifying client auth"); |
||||
|
||||
let _ = verify( |
||||
&serde_bare::to_vec(&message.content_v0()).unwrap(), |
||||
message.sig(), |
||||
message.user(), |
||||
) |
||||
.map_err(|_e| ProtocolError::AccessDenied)?; |
||||
|
||||
// debug_println!(
|
||||
// "matching nonce : {:?} {:?}",
|
||||
// message.nonce(),
|
||||
// handler.nonce.as_ref().unwrap()
|
||||
// );
|
||||
|
||||
if message.nonce() != handler.nonce.as_ref().unwrap() { |
||||
let _ = handler |
||||
.machine |
||||
.consume(&AuthProtocolServerInput::Error) |
||||
.map_err(|_e| ProtocolError::InvalidState); |
||||
|
||||
return Err(ProtocolError::AccessDenied); |
||||
} |
||||
|
||||
// TODO check that the device has been registered for this user. if not, return AccessDenied
|
||||
|
||||
// all is good, we advance the FSM and send back response
|
||||
let _ = handler |
||||
.machine |
||||
.consume(&AuthProtocolServerInput::Ok) |
||||
.map_err(|_e| ProtocolError::InvalidState)?; |
||||
|
||||
handler.user = Some(message.user()); |
||||
|
||||
Ok(vec![]) // without any metadata
|
||||
} |
||||
_ => Err(ProtocolError::InvalidState), |
||||
} |
||||
} |
||||
|
||||
let res = process_state(self, frame); |
||||
let is_err = res.as_ref().err().cloned(); |
||||
let reply = prepare_reply(res); |
||||
let reply_ser: Result<Vec<u8>, ProtocolError> = Ok(serde_bare::to_vec(&reply).unwrap()); |
||||
if is_err.is_some() { |
||||
( |
||||
reply_ser, |
||||
Box::pin(OptionFuture::from(Some( |
||||
async move { reply.result() }.boxed(), |
||||
))), |
||||
) |
||||
} else { |
||||
(reply_ser, Box::pin(OptionFuture::from(None))) |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,197 @@ |
||||
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers
|
||||
// All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0
|
||||
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
|
||||
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
|
||||
// at your option. All files in the project carrying such
|
||||
// notice may not be copied, modified, or distributed except
|
||||
// according to those terms.
|
||||
|
||||
//! User account
|
||||
|
||||
use p2p_repo::broker_store::BrokerStore; |
||||
use p2p_repo::store::*; |
||||
use p2p_repo::types::*; |
||||
use p2p_net::types::*; |
||||
use serde_bare::to_vec; |
||||
|
||||
pub struct Account<'a> { |
||||
/// User ID
|
||||
id: UserId, |
||||
store: &'a dyn BrokerStore, |
||||
} |
||||
|
||||
impl<'a> Account<'a> { |
||||
const PREFIX: u8 = b"u"[0]; |
||||
|
||||
// propertie's suffixes
|
||||
const CLIENT: u8 = b"c"[0]; |
||||
const ADMIN: u8 = b"a"[0]; |
||||
const OVERLAY: u8 = b"o"[0]; |
||||
|
||||
const ALL_PROPERTIES: [u8; 3] = [Self::CLIENT, Self::ADMIN, Self::OVERLAY]; |
||||
|
||||
const SUFFIX_FOR_EXIST_CHECK: u8 = Self::ADMIN; |
||||
|
||||
pub fn open(id: &UserId, store: &'a dyn BrokerStore) -> Result<Account<'a>, StorageError> { |
||||
let opening = Account { |
||||
id: id.clone(), |
||||
store, |
||||
}; |
||||
if !opening.exists() { |
||||
return Err(StorageError::NotFound); |
||||
} |
||||
Ok(opening) |
||||
} |
||||
pub fn create( |
||||
id: &UserId, |
||||
admin: bool, |
||||
store: &'a dyn BrokerStore, |
||||
) -> Result<Account<'a>, StorageError> { |
||||
let acc = Account { |
||||
id: id.clone(), |
||||
store, |
||||
}; |
||||
if acc.exists() { |
||||
return Err(StorageError::BackendError); |
||||
} |
||||
store.put( |
||||
Self::PREFIX, |
||||
&to_vec(&id)?, |
||||
Some(Self::ADMIN), |
||||
to_vec(&admin)?, |
||||
)?; |
||||
Ok(acc) |
||||
} |
||||
pub fn exists(&self) -> bool { |
||||
self.store |
||||
.get( |
||||
Self::PREFIX, |
||||
&to_vec(&self.id).unwrap(), |
||||
Some(Self::SUFFIX_FOR_EXIST_CHECK), |
||||
) |
||||
.is_ok() |
||||
} |
||||
pub fn id(&self) -> UserId { |
||||
self.id |
||||
} |
||||
pub fn add_client(&self, client: &ClientId) -> Result<(), StorageError> { |
||||
if !self.exists() { |
||||
return Err(StorageError::BackendError); |
||||
} |
||||
self.store.put( |
||||
Self::PREFIX, |
||||
&to_vec(&self.id)?, |
||||
Some(Self::CLIENT), |
||||
to_vec(client)?, |
||||
) |
||||
} |
||||
pub fn remove_client(&self, client: &ClientId) -> Result<(), StorageError> { |
||||
self.store.del_property_value( |
||||
Self::PREFIX, |
||||
&to_vec(&self.id)?, |
||||
Some(Self::CLIENT), |
||||
to_vec(client)?, |
||||
) |
||||
} |
||||
|
||||
pub fn has_client(&self, client: &ClientId) -> Result<(), StorageError> { |
||||
self.store.has_property_value( |
||||
Self::PREFIX, |
||||
&to_vec(&self.id)?, |
||||
Some(Self::CLIENT), |
||||
to_vec(client)?, |
||||
) |
||||
} |
||||
|
||||
pub fn add_overlay(&self, overlay: &OverlayId) -> Result<(), StorageError> { |
||||
if !self.exists() { |
||||
return Err(StorageError::BackendError); |
||||
} |
||||
self.store.put( |
||||
Self::PREFIX, |
||||
&to_vec(&self.id)?, |
||||
Some(Self::OVERLAY), |
||||
to_vec(overlay)?, |
||||
) |
||||
} |
||||
pub fn remove_overlay(&self, overlay: &OverlayId) -> Result<(), StorageError> { |
||||
self.store.del_property_value( |
||||
Self::PREFIX, |
||||
&to_vec(&self.id)?, |
||||
Some(Self::OVERLAY), |
||||
to_vec(overlay)?, |
||||
) |
||||
} |
||||
|
||||
pub fn has_overlay(&self, overlay: &OverlayId) -> Result<(), StorageError> { |
||||
self.store.has_property_value( |
||||
Self::PREFIX, |
||||
&to_vec(&self.id)?, |
||||
Some(Self::OVERLAY), |
||||
to_vec(overlay)?, |
||||
) |
||||
} |
||||
|
||||
pub fn is_admin(&self) -> Result<bool, StorageError> { |
||||
if self |
||||
.store |
||||
.has_property_value( |
||||
Self::PREFIX, |
||||
&to_vec(&self.id)?, |
||||
Some(Self::ADMIN), |
||||
to_vec(&true)?, |
||||
) |
||||
.is_ok() |
||||
{ |
||||
return Ok(true); |
||||
} |
||||
Ok(false) |
||||
} |
||||
|
||||
pub fn del(&self) -> Result<(), StorageError> { |
||||
self.store |
||||
.del_all(Self::PREFIX, &to_vec(&self.id)?, &Self::ALL_PROPERTIES) |
||||
} |
||||
} |
||||
|
||||
#[cfg(test)] |
||||
mod test { |
||||
|
||||
use p2p_repo::store::*; |
||||
use p2p_repo::types::*; |
||||
use p2p_repo::utils::*; |
||||
use p2p_stores_lmdb::broker_store::LmdbBrokerStore; |
||||
use std::fs; |
||||
use tempfile::Builder; |
||||
|
||||
use crate::broker_store_account::Account; |
||||
|
||||
#[test] |
||||
pub fn test_account() { |
||||
let path_str = "test-env"; |
||||
let root = Builder::new().prefix(path_str).tempdir().unwrap(); |
||||
let key: [u8; 32] = [0; 32]; |
||||
fs::create_dir_all(root.path()).unwrap(); |
||||
println!("{}", root.path().to_str().unwrap()); |
||||
let mut store = LmdbBrokerStore::open(root.path(), key); |
||||
|
||||
let user_id = PubKey::Ed25519PubKey([1; 32]); |
||||
|
||||
let account = Account::create(&user_id, true, &store).unwrap(); |
||||
println!("account created {}", account.id()); |
||||
|
||||
let account2 = Account::open(&user_id, &store).unwrap(); |
||||
println!("account opened {}", account2.id()); |
||||
|
||||
let client_id = PubKey::Ed25519PubKey([56; 32]); |
||||
let client_id_not_added = PubKey::Ed25519PubKey([57; 32]); |
||||
|
||||
account2.add_client(&client_id).unwrap(); |
||||
|
||||
assert!(account2.is_admin().unwrap()); |
||||
|
||||
account.has_client(&client_id).unwrap(); |
||||
assert!(account.has_client(&client_id_not_added).is_err()); |
||||
} |
||||
} |
||||
@ -0,0 +1,103 @@ |
||||
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers
|
||||
// All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0
|
||||
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
|
||||
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
|
||||
// at your option. All files in the project carrying such
|
||||
// notice may not be copied, modified, or distributed except
|
||||
// according to those terms.
|
||||
|
||||
//! Broker Config, persisted to store
|
||||
|
||||
use p2p_repo::broker_store::BrokerStore; |
||||
use p2p_repo::store::*; |
||||
use p2p_repo::types::*; |
||||
use p2p_net::types::*; |
||||
use serde::{Deserialize, Serialize}; |
||||
use serde_bare::{from_slice, to_vec}; |
||||
|
||||
// TODO: versioning V0
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] |
||||
pub enum ConfigMode { |
||||
Local, |
||||
Core, |
||||
} |
||||
|
||||
pub struct Config<'a> { |
||||
store: &'a dyn BrokerStore, |
||||
} |
||||
|
||||
impl<'a> Config<'a> { |
||||
const PREFIX: u8 = b"c"[0]; |
||||
|
||||
const KEY: [u8; 5] = *b"onfig"; |
||||
|
||||
// propertie's suffixes
|
||||
const MODE: u8 = b"m"[0]; |
||||
|
||||
const ALL_PROPERTIES: [u8; 1] = [Self::MODE]; |
||||
|
||||
const SUFFIX_FOR_EXIST_CHECK: u8 = Self::MODE; |
||||
|
||||
pub fn open(store: &'a dyn BrokerStore) -> Result<Config<'a>, StorageError> { |
||||
let opening = Config { store }; |
||||
if !opening.exists() { |
||||
return Err(StorageError::NotFound); |
||||
} |
||||
Ok(opening) |
||||
} |
||||
pub fn get_or_create( |
||||
mode: &ConfigMode, |
||||
store: &'a dyn BrokerStore, |
||||
) -> Result<Config<'a>, StorageError> { |
||||
match Self::open(store) { |
||||
Err(e) => { |
||||
if e == StorageError::NotFound { |
||||
Self::create(mode, store) |
||||
} else { |
||||
Err(StorageError::BackendError) |
||||
} |
||||
} |
||||
Ok(p) => { |
||||
if &p.mode().unwrap() != mode { |
||||
return Err(StorageError::InvalidValue); |
||||
} |
||||
Ok(p) |
||||
} |
||||
} |
||||
} |
||||
pub fn create( |
||||
mode: &ConfigMode, |
||||
store: &'a dyn BrokerStore, |
||||
) -> Result<Config<'a>, StorageError> { |
||||
let acc = Config { store }; |
||||
if acc.exists() { |
||||
return Err(StorageError::BackendError); |
||||
} |
||||
store.put( |
||||
Self::PREFIX, |
||||
&to_vec(&Self::KEY)?, |
||||
Some(Self::MODE), |
||||
to_vec(&mode)?, |
||||
)?; |
||||
Ok(acc) |
||||
} |
||||
pub fn exists(&self) -> bool { |
||||
self.store |
||||
.get( |
||||
Self::PREFIX, |
||||
&to_vec(&Self::KEY).unwrap(), |
||||
Some(Self::SUFFIX_FOR_EXIST_CHECK), |
||||
) |
||||
.is_ok() |
||||
} |
||||
pub fn mode(&self) -> Result<ConfigMode, StorageError> { |
||||
match self |
||||
.store |
||||
.get(Self::PREFIX, &to_vec(&Self::KEY)?, Some(Self::MODE)) |
||||
{ |
||||
Ok(ver) => Ok(from_slice::<ConfigMode>(&ver)?), |
||||
Err(e) => Err(e), |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,219 @@ |
||||
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers
|
||||
// All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0
|
||||
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
|
||||
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
|
||||
// at your option. All files in the project carrying such
|
||||
// notice may not be copied, modified, or distributed except
|
||||
// according to those terms.
|
||||
|
||||
//! Overlay
|
||||
|
||||
use p2p_repo::broker_store::BrokerStore; |
||||
use p2p_repo::store::*; |
||||
use p2p_repo::types::*; |
||||
use p2p_net::types::*; |
||||
use p2p_repo::utils::now_timestamp; |
||||
use serde::{Deserialize, Serialize}; |
||||
use serde_bare::{from_slice, to_vec}; |
||||
|
||||
// TODO: versioning V0
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] |
||||
pub struct OverlayMeta { |
||||
pub users: u32, |
||||
pub last_used: Timestamp, |
||||
} |
||||
|
||||
pub struct Overlay<'a> { |
||||
/// Overlay ID
|
||||
id: OverlayId, |
||||
store: &'a dyn BrokerStore, |
||||
} |
||||
|
||||
impl<'a> Overlay<'a> { |
||||
const PREFIX: u8 = b"o"[0]; |
||||
|
||||
// propertie's suffixes
|
||||
const SECRET: u8 = b"s"[0]; |
||||
const PEER: u8 = b"p"[0]; |
||||
const TOPIC: u8 = b"t"[0]; |
||||
const META: u8 = b"m"[0]; |
||||
const REPO: u8 = b"r"[0]; |
||||
|
||||
const ALL_PROPERTIES: [u8; 5] = [ |
||||
Self::SECRET, |
||||
Self::PEER, |
||||
Self::TOPIC, |
||||
Self::META, |
||||
Self::REPO, |
||||
]; |
||||
|
||||
const SUFFIX_FOR_EXIST_CHECK: u8 = Self::SECRET; |
||||
|
||||
pub fn open(id: &OverlayId, store: &'a dyn BrokerStore) -> Result<Overlay<'a>, StorageError> { |
||||
let opening = Overlay { |
||||
id: id.clone(), |
||||
store, |
||||
}; |
||||
if !opening.exists() { |
||||
return Err(StorageError::NotFound); |
||||
} |
||||
Ok(opening) |
||||
} |
||||
pub fn create( |
||||
id: &OverlayId, |
||||
secret: &SymKey, |
||||
repo: Option<PubKey>, |
||||
store: &'a dyn BrokerStore, |
||||
) -> Result<Overlay<'a>, StorageError> { |
||||
let acc = Overlay { |
||||
id: id.clone(), |
||||
store, |
||||
}; |
||||
if acc.exists() { |
||||
return Err(StorageError::BackendError); |
||||
} |
||||
store.put( |
||||
Self::PREFIX, |
||||
&to_vec(&id)?, |
||||
Some(Self::SECRET), |
||||
to_vec(&secret)?, |
||||
)?; |
||||
if repo.is_some() { |
||||
store.put( |
||||
Self::PREFIX, |
||||
&to_vec(&id)?, |
||||
Some(Self::REPO), |
||||
to_vec(&repo.unwrap())?, |
||||
)?; |
||||
//TODO if failure, should remove the previously added SECRET property
|
||||
} |
||||
let meta = OverlayMeta { |
||||
users: 1, |
||||
last_used: now_timestamp(), |
||||
}; |
||||
store.put( |
||||
Self::PREFIX, |
||||
&to_vec(&id)?, |
||||
Some(Self::META), |
||||
to_vec(&meta)?, |
||||
)?; |
||||
//TODO if failure, should remove the previously added SECRET and REPO properties
|
||||
Ok(acc) |
||||
} |
||||
pub fn exists(&self) -> bool { |
||||
self.store |
||||
.get( |
||||
Self::PREFIX, |
||||
&to_vec(&self.id).unwrap(), |
||||
Some(Self::SUFFIX_FOR_EXIST_CHECK), |
||||
) |
||||
.is_ok() |
||||
} |
||||
pub fn id(&self) -> OverlayId { |
||||
self.id |
||||
} |
||||
pub fn add_peer(&self, peer: &PeerId) -> Result<(), StorageError> { |
||||
if !self.exists() { |
||||
return Err(StorageError::BackendError); |
||||
} |
||||
self.store.put( |
||||
Self::PREFIX, |
||||
&to_vec(&self.id)?, |
||||
Some(Self::PEER), |
||||
to_vec(peer)?, |
||||
) |
||||
} |
||||
pub fn remove_peer(&self, peer: &PeerId) -> Result<(), StorageError> { |
||||
self.store.del_property_value( |
||||
Self::PREFIX, |
||||
&to_vec(&self.id)?, |
||||
Some(Self::PEER), |
||||
to_vec(peer)?, |
||||
) |
||||
} |
||||
|
||||
pub fn has_peer(&self, peer: &PeerId) -> Result<(), StorageError> { |
||||
self.store.has_property_value( |
||||
Self::PREFIX, |
||||
&to_vec(&self.id)?, |
||||
Some(Self::PEER), |
||||
to_vec(peer)?, |
||||
) |
||||
} |
||||
|
||||
pub fn add_topic(&self, topic: &TopicId) -> Result<(), StorageError> { |
||||
if !self.exists() { |
||||
return Err(StorageError::BackendError); |
||||
} |
||||
self.store.put( |
||||
Self::PREFIX, |
||||
&to_vec(&self.id)?, |
||||
Some(Self::TOPIC), |
||||
to_vec(topic)?, |
||||
) |
||||
} |
||||
pub fn remove_topic(&self, topic: &TopicId) -> Result<(), StorageError> { |
||||
self.store.del_property_value( |
||||
Self::PREFIX, |
||||
&to_vec(&self.id)?, |
||||
Some(Self::TOPIC), |
||||
to_vec(topic)?, |
||||
) |
||||
} |
||||
|
||||
pub fn has_topic(&self, topic: &TopicId) -> Result<(), StorageError> { |
||||
self.store.has_property_value( |
||||
Self::PREFIX, |
||||
&to_vec(&self.id)?, |
||||
Some(Self::TOPIC), |
||||
to_vec(topic)?, |
||||
) |
||||
} |
||||
|
||||
pub fn secret(&self) -> Result<SymKey, StorageError> { |
||||
match self |
||||
.store |
||||
.get(Self::PREFIX, &to_vec(&self.id)?, Some(Self::SECRET)) |
||||
{ |
||||
Ok(secret) => Ok(from_slice::<SymKey>(&secret)?), |
||||
Err(e) => Err(e), |
||||
} |
||||
} |
||||
|
||||
pub fn metadata(&self) -> Result<OverlayMeta, StorageError> { |
||||
match self |
||||
.store |
||||
.get(Self::PREFIX, &to_vec(&self.id)?, Some(Self::META)) |
||||
{ |
||||
Ok(meta) => Ok(from_slice::<OverlayMeta>(&meta)?), |
||||
Err(e) => Err(e), |
||||
} |
||||
} |
||||
pub fn set_metadata(&self, meta: &OverlayMeta) -> Result<(), StorageError> { |
||||
if !self.exists() { |
||||
return Err(StorageError::BackendError); |
||||
} |
||||
self.store.replace( |
||||
Self::PREFIX, |
||||
&to_vec(&self.id)?, |
||||
Some(Self::META), |
||||
to_vec(meta)?, |
||||
) |
||||
} |
||||
|
||||
pub fn repo(&self) -> Result<PubKey, StorageError> { |
||||
match self |
||||
.store |
||||
.get(Self::PREFIX, &to_vec(&self.id)?, Some(Self::REPO)) |
||||
{ |
||||
Ok(repo) => Ok(from_slice::<PubKey>(&repo)?), |
||||
Err(e) => Err(e), |
||||
} |
||||
} |
||||
|
||||
pub fn del(&self) -> Result<(), StorageError> { |
||||
self.store |
||||
.del_all(Self::PREFIX, &to_vec(&self.id)?, &Self::ALL_PROPERTIES) |
||||
} |
||||
} |
||||
@ -0,0 +1,163 @@ |
||||
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers
|
||||
// All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0
|
||||
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
|
||||
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
|
||||
// at your option. All files in the project carrying such
|
||||
// notice may not be copied, modified, or distributed except
|
||||
// according to those terms.
|
||||
|
||||
//! Peer
|
||||
|
||||
use p2p_repo::broker_store::BrokerStore; |
||||
use p2p_repo::store::*; |
||||
use p2p_repo::types::*; |
||||
use p2p_net::types::*; |
||||
use serde::{Deserialize, Serialize}; |
||||
use serde_bare::{from_slice, to_vec}; |
||||
|
||||
pub struct Peer<'a> { |
||||
/// Topic ID
|
||||
id: PeerId, |
||||
store: &'a dyn BrokerStore, |
||||
} |
||||
|
||||
impl<'a> Peer<'a> { |
||||
const PREFIX: u8 = b"p"[0]; |
||||
|
||||
// propertie's suffixes
|
||||
const VERSION: u8 = b"v"[0]; |
||||
const ADVERT: u8 = b"a"[0]; |
||||
|
||||
const ALL_PROPERTIES: [u8; 2] = [Self::VERSION, Self::ADVERT]; |
||||
|
||||
const SUFFIX_FOR_EXIST_CHECK: u8 = Self::VERSION; |
||||
|
||||
pub fn open(id: &PeerId, store: &'a dyn BrokerStore) -> Result<Peer<'a>, StorageError> { |
||||
let opening = Peer { |
||||
id: id.clone(), |
||||
store, |
||||
}; |
||||
if !opening.exists() { |
||||
return Err(StorageError::NotFound); |
||||
} |
||||
Ok(opening) |
||||
} |
||||
pub fn update_or_create( |
||||
advert: &PeerAdvert, |
||||
store: &'a dyn BrokerStore, |
||||
) -> Result<Peer<'a>, StorageError> { |
||||
let id = advert.peer(); |
||||
match Self::open(id, store) { |
||||
Err(e) => { |
||||
if e == StorageError::NotFound { |
||||
Self::create(advert, store) |
||||
} else { |
||||
Err(StorageError::BackendError) |
||||
} |
||||
} |
||||
Ok(p) => { |
||||
p.update_advert(advert)?; |
||||
Ok(p) |
||||
} |
||||
} |
||||
} |
||||
pub fn create( |
||||
advert: &PeerAdvert, |
||||
store: &'a dyn BrokerStore, |
||||
) -> Result<Peer<'a>, StorageError> { |
||||
let id = advert.peer(); |
||||
let acc = Peer { |
||||
id: id.clone(), |
||||
store, |
||||
}; |
||||
if acc.exists() { |
||||
return Err(StorageError::BackendError); |
||||
} |
||||
store.put( |
||||
Self::PREFIX, |
||||
&to_vec(&id)?, |
||||
Some(Self::VERSION), |
||||
to_vec(&advert.version())?, |
||||
)?; |
||||
store.put( |
||||
Self::PREFIX, |
||||
&to_vec(&id)?, |
||||
Some(Self::ADVERT), |
||||
to_vec(&advert)?, |
||||
)?; |
||||
Ok(acc) |
||||
} |
||||
pub fn exists(&self) -> bool { |
||||
self.store |
||||
.get( |
||||
Self::PREFIX, |
||||
&to_vec(&self.id).unwrap(), |
||||
Some(Self::SUFFIX_FOR_EXIST_CHECK), |
||||
) |
||||
.is_ok() |
||||
} |
||||
pub fn id(&self) -> PeerId { |
||||
self.id |
||||
} |
||||
pub fn version(&self) -> Result<u32, StorageError> { |
||||
match self |
||||
.store |
||||
.get(Self::PREFIX, &to_vec(&self.id)?, Some(Self::VERSION)) |
||||
{ |
||||
Ok(ver) => Ok(from_slice::<u32>(&ver)?), |
||||
Err(e) => Err(e), |
||||
} |
||||
} |
||||
pub fn set_version(&self, version: u32) -> Result<(), StorageError> { |
||||
if !self.exists() { |
||||
return Err(StorageError::BackendError); |
||||
} |
||||
self.store.replace( |
||||
Self::PREFIX, |
||||
&to_vec(&self.id)?, |
||||
Some(Self::VERSION), |
||||
to_vec(&version)?, |
||||
) |
||||
} |
||||
pub fn update_advert(&self, advert: &PeerAdvert) -> Result<(), StorageError> { |
||||
if advert.peer() != &self.id { |
||||
return Err(StorageError::InvalidValue); |
||||
} |
||||
let current_advert = self.advert().map_err(|e| StorageError::BackendError)?; |
||||
if current_advert.version() >= advert.version() { |
||||
return Ok(()); |
||||
} |
||||
self.store.replace( |
||||
Self::PREFIX, |
||||
&to_vec(&self.id)?, |
||||
Some(Self::ADVERT), |
||||
to_vec(advert)?, |
||||
) |
||||
} |
||||
pub fn advert(&self) -> Result<PeerAdvert, StorageError> { |
||||
match self |
||||
.store |
||||
.get(Self::PREFIX, &to_vec(&self.id)?, Some(Self::ADVERT)) |
||||
{ |
||||
Ok(advert) => Ok(from_slice::<PeerAdvert>(&advert)?), |
||||
Err(e) => Err(e), |
||||
} |
||||
} |
||||
pub fn set_advert(&self, advert: &PeerAdvert) -> Result<(), StorageError> { |
||||
if !self.exists() { |
||||
return Err(StorageError::BackendError); |
||||
} |
||||
self.store.replace( |
||||
Self::PREFIX, |
||||
&to_vec(&self.id)?, |
||||
Some(Self::ADVERT), |
||||
to_vec(advert)?, |
||||
) |
||||
} |
||||
|
||||
pub fn del(&self) -> Result<(), StorageError> { |
||||
self.store |
||||
.del_all(Self::PREFIX, &to_vec(&self.id)?, &Self::ALL_PROPERTIES) |
||||
} |
||||
} |
||||
@ -0,0 +1,103 @@ |
||||
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers
|
||||
// All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0
|
||||
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
|
||||
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
|
||||
// at your option. All files in the project carrying such
|
||||
// notice may not be copied, modified, or distributed except
|
||||
// according to those terms.
|
||||
|
||||
//! RepoStore information about each RepoStore
|
||||
//! It contains the symKeys to open the RepoStores
|
||||
//! A repoStore is identified by its repo pubkey if in local mode
|
||||
//! In core mode, it is identified by the overlayid.
|
||||
|
||||
use p2p_repo::broker_store::BrokerStore; |
||||
use p2p_repo::store::*; |
||||
use p2p_repo::types::*; |
||||
use p2p_net::types::*; |
||||
use serde::{Deserialize, Serialize}; |
||||
use serde_bare::{from_slice, to_vec}; |
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] |
||||
pub enum RepoStoreId { |
||||
Overlay(OverlayId), |
||||
Repo(PubKey), |
||||
} |
||||
|
||||
impl From<RepoStoreId> for String { |
||||
fn from(id: RepoStoreId) -> Self { |
||||
hex::encode(to_vec(&id).unwrap()) |
||||
} |
||||
} |
||||
|
||||
pub struct RepoStoreInfo<'a> { |
||||
/// RepoStore ID
|
||||
id: RepoStoreId, |
||||
store: &'a dyn BrokerStore, |
||||
} |
||||
|
||||
impl<'a> RepoStoreInfo<'a> { |
||||
const PREFIX: u8 = b"r"[0]; |
||||
|
||||
// propertie's suffixes
|
||||
const KEY: u8 = b"k"[0]; |
||||
|
||||
const ALL_PROPERTIES: [u8; 1] = [Self::KEY]; |
||||
|
||||
const SUFFIX_FOR_EXIST_CHECK: u8 = Self::KEY; |
||||
|
||||
pub fn open( |
||||
id: &RepoStoreId, |
||||
store: &'a dyn BrokerStore, |
||||
) -> Result<RepoStoreInfo<'a>, StorageError> { |
||||
let opening = RepoStoreInfo { |
||||
id: id.clone(), |
||||
store, |
||||
}; |
||||
if !opening.exists() { |
||||
return Err(StorageError::NotFound); |
||||
} |
||||
Ok(opening) |
||||
} |
||||
pub fn create( |
||||
id: &RepoStoreId, |
||||
key: &SymKey, |
||||
store: &'a dyn BrokerStore, |
||||
) -> Result<RepoStoreInfo<'a>, StorageError> { |
||||
let acc = RepoStoreInfo { |
||||
id: id.clone(), |
||||
store, |
||||
}; |
||||
if acc.exists() { |
||||
return Err(StorageError::BackendError); |
||||
} |
||||
store.put(Self::PREFIX, &to_vec(&id)?, Some(Self::KEY), to_vec(key)?)?; |
||||
Ok(acc) |
||||
} |
||||
pub fn exists(&self) -> bool { |
||||
self.store |
||||
.get( |
||||
Self::PREFIX, |
||||
&to_vec(&self.id).unwrap(), |
||||
Some(Self::SUFFIX_FOR_EXIST_CHECK), |
||||
) |
||||
.is_ok() |
||||
} |
||||
pub fn id(&self) -> &RepoStoreId { |
||||
&self.id |
||||
} |
||||
pub fn key(&self) -> Result<SymKey, StorageError> { |
||||
match self |
||||
.store |
||||
.get(Self::PREFIX, &to_vec(&self.id)?, Some(Self::KEY)) |
||||
{ |
||||
Ok(k) => Ok(from_slice::<SymKey>(&k)?), |
||||
Err(e) => Err(e), |
||||
} |
||||
} |
||||
pub fn del(&self) -> Result<(), StorageError> { |
||||
self.store |
||||
.del_all(Self::PREFIX, &to_vec(&self.id)?, &Self::ALL_PROPERTIES) |
||||
} |
||||
} |
||||
@ -0,0 +1,136 @@ |
||||
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers
|
||||
// All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0
|
||||
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
|
||||
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
|
||||
// at your option. All files in the project carrying such
|
||||
// notice may not be copied, modified, or distributed except
|
||||
// according to those terms.
|
||||
|
||||
//! Topic
|
||||
|
||||
use p2p_repo::broker_store::BrokerStore; |
||||
use p2p_repo::store::*; |
||||
use p2p_repo::types::*; |
||||
use p2p_net::types::*; |
||||
use serde::{Deserialize, Serialize}; |
||||
use serde_bare::{from_slice, to_vec}; |
||||
|
||||
// TODO: versioning V0
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] |
||||
pub struct TopicMeta { |
||||
pub users: u32, |
||||
} |
||||
|
||||
pub struct Topic<'a> { |
||||
/// Topic ID
|
||||
id: TopicId, |
||||
store: &'a dyn BrokerStore, |
||||
} |
||||
|
||||
impl<'a> Topic<'a> { |
||||
const PREFIX: u8 = b"t"[0]; |
||||
|
||||
// propertie's suffixes
|
||||
const ADVERT: u8 = b"a"[0]; |
||||
const HEAD: u8 = b"h"[0]; |
||||
const META: u8 = b"m"[0]; |
||||
|
||||
const ALL_PROPERTIES: [u8; 3] = [Self::ADVERT, Self::HEAD, Self::META]; |
||||
|
||||
const SUFFIX_FOR_EXIST_CHECK: u8 = Self::META; |
||||
|
||||
pub fn open(id: &TopicId, store: &'a dyn BrokerStore) -> Result<Topic<'a>, StorageError> { |
||||
let opening = Topic { |
||||
id: id.clone(), |
||||
store, |
||||
}; |
||||
if !opening.exists() { |
||||
return Err(StorageError::NotFound); |
||||
} |
||||
Ok(opening) |
||||
} |
||||
pub fn create(id: &TopicId, store: &'a dyn BrokerStore) -> Result<Topic<'a>, StorageError> { |
||||
let acc = Topic { |
||||
id: id.clone(), |
||||
store, |
||||
}; |
||||
if acc.exists() { |
||||
return Err(StorageError::BackendError); |
||||
} |
||||
let meta = TopicMeta { users: 0 }; |
||||
store.put( |
||||
Self::PREFIX, |
||||
&to_vec(&id)?, |
||||
Some(Self::META), |
||||
to_vec(&meta)?, |
||||
)?; |
||||
Ok(acc) |
||||
} |
||||
pub fn exists(&self) -> bool { |
||||
self.store |
||||
.get( |
||||
Self::PREFIX, |
||||
&to_vec(&self.id).unwrap(), |
||||
Some(Self::SUFFIX_FOR_EXIST_CHECK), |
||||
) |
||||
.is_ok() |
||||
} |
||||
pub fn id(&self) -> TopicId { |
||||
self.id |
||||
} |
||||
pub fn add_head(&self, head: &ObjectId) -> Result<(), StorageError> { |
||||
if !self.exists() { |
||||
return Err(StorageError::BackendError); |
||||
} |
||||
self.store.put( |
||||
Self::PREFIX, |
||||
&to_vec(&self.id)?, |
||||
Some(Self::HEAD), |
||||
to_vec(head)?, |
||||
) |
||||
} |
||||
pub fn remove_head(&self, head: &ObjectId) -> Result<(), StorageError> { |
||||
self.store.del_property_value( |
||||
Self::PREFIX, |
||||
&to_vec(&self.id)?, |
||||
Some(Self::HEAD), |
||||
to_vec(head)?, |
||||
) |
||||
} |
||||
|
||||
pub fn has_head(&self, head: &ObjectId) -> Result<(), StorageError> { |
||||
self.store.has_property_value( |
||||
Self::PREFIX, |
||||
&to_vec(&self.id)?, |
||||
Some(Self::HEAD), |
||||
to_vec(head)?, |
||||
) |
||||
} |
||||
|
||||
pub fn metadata(&self) -> Result<TopicMeta, StorageError> { |
||||
match self |
||||
.store |
||||
.get(Self::PREFIX, &to_vec(&self.id)?, Some(Self::META)) |
||||
{ |
||||
Ok(meta) => Ok(from_slice::<TopicMeta>(&meta)?), |
||||
Err(e) => Err(e), |
||||
} |
||||
} |
||||
pub fn set_metadata(&self, meta: &TopicMeta) -> Result<(), StorageError> { |
||||
if !self.exists() { |
||||
return Err(StorageError::BackendError); |
||||
} |
||||
self.store.replace( |
||||
Self::PREFIX, |
||||
&to_vec(&self.id)?, |
||||
Some(Self::META), |
||||
to_vec(meta)?, |
||||
) |
||||
} |
||||
|
||||
pub fn del(&self) -> Result<(), StorageError> { |
||||
self.store |
||||
.del_all(Self::PREFIX, &to_vec(&self.id)?, &Self::ALL_PROPERTIES) |
||||
} |
||||
} |
||||
@ -0,0 +1,148 @@ |
||||
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers
|
||||
// All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0
|
||||
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
|
||||
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
|
||||
// at your option. All files in the project carrying such
|
||||
// notice may not be copied, modified, or distributed except
|
||||
// according to those terms.
|
||||
|
||||
//! Connection to a Broker, can be local or remote.
|
||||
//! If remote, it will use a Stream and Sink of framed messages
|
||||
|
||||
use futures::{ |
||||
ready, |
||||
stream::Stream, |
||||
task::{Context, Poll}, |
||||
Future, |
||||
select, FutureExt, |
||||
}; |
||||
use futures::channel::mpsc; |
||||
use std::pin::Pin; |
||||
use std::{collections::HashSet, fmt::Debug}; |
||||
|
||||
use crate::server::BrokerServer; |
||||
use debug_print::*; |
||||
use futures::{pin_mut, stream, Sink, SinkExt, StreamExt}; |
||||
use p2p_repo::object::*; |
||||
use p2p_repo::store::*; |
||||
use p2p_repo::types::*; |
||||
use p2p_repo::utils::*; |
||||
use p2p_net::errors::*; |
||||
use p2p_net::types::*; |
||||
use p2p_net::broker_connection::*; |
||||
use std::collections::HashMap; |
||||
|
||||
|
||||
pub struct BrokerConnectionLocal<'a> { |
||||
broker: &'a mut BrokerServer, |
||||
user: PubKey, |
||||
} |
||||
|
||||
#[async_trait::async_trait] |
||||
impl<'a> BrokerConnection for BrokerConnectionLocal<'a> { |
||||
type OC = BrokerConnectionLocal<'a>; |
||||
type BlockStream = async_channel::Receiver<Block>; |
||||
|
||||
async fn close(&mut self) {} |
||||
|
||||
async fn add_user( |
||||
&mut self, |
||||
user_id: PubKey, |
||||
admin_user_pk: PrivKey, |
||||
) -> Result<(), ProtocolError> { |
||||
let op_content = AddUserContentV0 { user: user_id }; |
||||
let sig = sign(admin_user_pk, self.user, &serde_bare::to_vec(&op_content)?)?; |
||||
|
||||
self.broker.add_user(self.user, user_id, sig) |
||||
} |
||||
|
||||
async fn process_overlay_request( |
||||
&mut self, |
||||
overlay: OverlayId, |
||||
request: BrokerOverlayRequestContentV0, |
||||
) -> Result<(), ProtocolError> { |
||||
match request { |
||||
BrokerOverlayRequestContentV0::OverlayConnect(_) => { |
||||
self.broker.connect_overlay(self.user, overlay) |
||||
} |
||||
BrokerOverlayRequestContentV0::OverlayJoin(j) => { |
||||
self.broker |
||||
.join_overlay(self.user, overlay, j.repo_pubkey(), j.secret(), j.peers()) |
||||
} |
||||
BrokerOverlayRequestContentV0::ObjectPin(op) => { |
||||
self.broker.pin_object(self.user, overlay, op.id()) |
||||
} |
||||
BrokerOverlayRequestContentV0::ObjectUnpin(op) => { |
||||
self.broker.unpin_object(self.user, overlay, op.id()) |
||||
} |
||||
BrokerOverlayRequestContentV0::ObjectDel(op) => { |
||||
self.broker.del_object(self.user, overlay, op.id()) |
||||
} |
||||
BrokerOverlayRequestContentV0::BlockPut(b) => { |
||||
self.broker.put_block(self.user, overlay, b.block()) |
||||
} |
||||
_ => Err(ProtocolError::InvalidState), |
||||
} |
||||
} |
||||
|
||||
async fn process_overlay_request_objectid_response( |
||||
&mut self, |
||||
overlay: OverlayId, |
||||
request: BrokerOverlayRequestContentV0, |
||||
) -> Result<ObjectId, ProtocolError> { |
||||
match request { |
||||
BrokerOverlayRequestContentV0::ObjectCopy(oc) => { |
||||
self.broker |
||||
.copy_object(self.user, overlay, oc.id(), oc.expiry()) |
||||
} |
||||
_ => Err(ProtocolError::InvalidState), |
||||
} |
||||
} |
||||
|
||||
async fn process_overlay_request_stream_response( |
||||
&mut self, |
||||
overlay: OverlayId, |
||||
request: BrokerOverlayRequestContentV0, |
||||
) -> Result<Pin<Box<Self::BlockStream>>, ProtocolError> { |
||||
match request { |
||||
|
||||
BrokerOverlayRequestContentV0::BlockGet(b) => self |
||||
.broker |
||||
.get_block(self.user, overlay, b.id(), b.include_children(), b.topic()) |
||||
.map(|r| Box::pin(r)), |
||||
BrokerOverlayRequestContentV0::BranchSyncReq(b) => self |
||||
.broker |
||||
.sync_branch( |
||||
self.user, |
||||
&overlay, |
||||
b.heads(), |
||||
b.known_heads(), |
||||
b.known_commits(), |
||||
) |
||||
.map(|r| Box::pin(r)), |
||||
_ => Err(ProtocolError::InvalidState), |
||||
} |
||||
} |
||||
|
||||
async fn del_user(&mut self, user_id: PubKey, admin_user_pk: PrivKey) {} |
||||
|
||||
async fn add_client(&mut self, user_id: PubKey, admin_user_pk: PrivKey) {} |
||||
|
||||
async fn del_client(&mut self, user_id: PubKey, admin_user_pk: PrivKey) {} |
||||
|
||||
async fn overlay_connect( |
||||
&mut self, |
||||
repo_link: &RepoLink, |
||||
public: bool, |
||||
) -> Result<OverlayConnectionClient<BrokerConnectionLocal<'a>>, ProtocolError> { |
||||
let overlay = self.process_overlay_connect(repo_link, public).await?; |
||||
Ok(OverlayConnectionClient::create(self, overlay, repo_link.clone())) |
||||
} |
||||
} |
||||
|
||||
impl<'a> BrokerConnectionLocal<'a> { |
||||
pub fn new(broker: &'a mut BrokerServer, user: PubKey) -> BrokerConnectionLocal<'a> { |
||||
BrokerConnectionLocal { broker, user } |
||||
} |
||||
} |
||||
@ -0,0 +1,20 @@ |
||||
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers
|
||||
pub mod broker_store_account; |
||||
|
||||
pub mod broker_store_config; |
||||
|
||||
pub mod broker_store_overlay; |
||||
|
||||
pub mod broker_store_peer; |
||||
|
||||
pub mod broker_store_repostoreinfo; |
||||
|
||||
pub mod broker_store_topic; |
||||
|
||||
pub mod connection_local; |
||||
|
||||
pub mod server; |
||||
|
||||
pub mod server_ws; |
||||
|
||||
pub mod auth; |
||||
@ -0,0 +1,891 @@ |
||||
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers
|
||||
// All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0
|
||||
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
|
||||
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
|
||||
// at your option. All files in the project carrying such
|
||||
// notice may not be copied, modified, or distributed except
|
||||
// according to those terms.
|
||||
|
||||
//! A Broker server
|
||||
|
||||
use std::collections::HashMap; |
||||
use std::collections::HashSet; |
||||
use std::pin::Pin; |
||||
use std::sync::Arc; |
||||
use std::sync::RwLock; |
||||
|
||||
use crate::broker_store_account::Account; |
||||
use crate::auth::*; |
||||
use crate::broker_store_config::Config; |
||||
use crate::broker_store_config::ConfigMode; |
||||
use crate::connection_local::BrokerConnectionLocal; |
||||
use crate::broker_store_overlay::Overlay; |
||||
use crate::broker_store_peer::Peer; |
||||
use crate::broker_store_repostoreinfo::RepoStoreId; |
||||
use crate::broker_store_repostoreinfo::RepoStoreInfo; |
||||
use async_std::task; |
||||
use debug_print::*; |
||||
use futures::future::BoxFuture; |
||||
use futures::future::OptionFuture; |
||||
use futures::FutureExt; |
||||
use futures::Stream; |
||||
use p2p_repo::object::Object; |
||||
use p2p_repo::store::RepoStore; |
||||
use p2p_repo::store::StorageError; |
||||
use p2p_repo::types::*; |
||||
use p2p_repo::utils::*; |
||||
use p2p_net::errors::*; |
||||
use p2p_net::types::*; |
||||
use p2p_stores_lmdb::broker_store::LmdbBrokerStore; |
||||
use p2p_stores_lmdb::repo_store::LmdbRepoStore; |
||||
|
||||
#[derive(Debug, Eq, PartialEq, Clone)] |
||||
pub enum BrokerError { |
||||
CannotStart, |
||||
MismatchedMode, |
||||
OverlayNotFound, |
||||
} |
||||
|
||||
impl From<BrokerError> for ProtocolError { |
||||
fn from(e: BrokerError) -> Self { |
||||
match e { |
||||
BrokerError::CannotStart => ProtocolError::OverlayNotFound, |
||||
BrokerError::OverlayNotFound => ProtocolError::OverlayNotFound, |
||||
_ => ProtocolError::BrokerError, |
||||
} |
||||
} |
||||
} |
||||
|
||||
impl From<p2p_repo::store::StorageError> for BrokerError { |
||||
fn from(e: p2p_repo::store::StorageError) -> Self { |
||||
match e { |
||||
p2p_repo::store::StorageError::InvalidValue => BrokerError::MismatchedMode, |
||||
_ => BrokerError::CannotStart, |
||||
} |
||||
} |
||||
} |
||||
|
||||
#[derive(Debug)] |
||||
enum ProtocolType { |
||||
Start, |
||||
Auth, |
||||
Broker, |
||||
Ext, |
||||
P2P, |
||||
} |
||||
|
||||
pub struct ProtocolHandler { |
||||
broker: Arc<BrokerServer>, |
||||
protocol: ProtocolType, |
||||
auth_protocol: Option<AuthProtocolHandler>, |
||||
broker_protocol: Option<BrokerProtocolHandler>, |
||||
ext_protocol: Option<ExtProtocolHandler>, |
||||
r: Option<async_channel::Receiver<Vec<u8>>>, |
||||
s: async_channel::Sender<Vec<u8>>, |
||||
} |
||||
|
||||
impl ProtocolHandler { |
||||
pub fn async_frames_receiver(&mut self) -> async_channel::Receiver<Vec<u8>> { |
||||
self.r.take().unwrap() |
||||
} |
||||
|
||||
/// Handle incoming message
|
||||
pub async fn handle_incoming( |
||||
&mut self, |
||||
frame: Vec<u8>, |
||||
) -> ( |
||||
Result<Vec<u8>, ProtocolError>, |
||||
OptionFuture<BoxFuture<'static, u16>>, |
||||
) { |
||||
//debug_println!("SERVER PROTOCOL {:?}", &self.protocol);
|
||||
match &self.protocol { |
||||
ProtocolType::Start => { |
||||
let message = serde_bare::from_slice::<StartProtocol>(&frame); |
||||
match message { |
||||
Ok(StartProtocol::Auth(b)) => { |
||||
self.protocol = ProtocolType::Auth; |
||||
self.auth_protocol = Some(AuthProtocolHandler::new()); |
||||
return ( |
||||
self.auth_protocol.as_mut().unwrap().handle_init(b), |
||||
OptionFuture::from(None), |
||||
); |
||||
} |
||||
Ok(StartProtocol::Ext(ext)) => { |
||||
self.protocol = ProtocolType::Ext; |
||||
self.ext_protocol = Some(ExtProtocolHandler {}); |
||||
let reply = self.ext_protocol.as_ref().unwrap().handle_incoming(ext); |
||||
return ( |
||||
Ok(serde_bare::to_vec(&reply).unwrap()), |
||||
OptionFuture::from(None), |
||||
); |
||||
} |
||||
Err(e) => { |
||||
return (Err(ProtocolError::SerializationError),OptionFuture::from(None)) |
||||
} |
||||
} |
||||
} |
||||
ProtocolType::Auth => { |
||||
let res = self.auth_protocol.as_mut().unwrap().handle_incoming(frame); |
||||
match res.1.await { |
||||
None => { |
||||
// we switch to Broker protocol
|
||||
self.protocol = ProtocolType::Broker; |
||||
self.broker_protocol = Some(BrokerProtocolHandler { |
||||
user: self.auth_protocol.as_ref().unwrap().get_user().unwrap(), |
||||
broker: Arc::clone(&self.broker), |
||||
async_frames_sender: self.s.clone(), |
||||
}); |
||||
self.auth_protocol = None; |
||||
(res.0, OptionFuture::from(None)) |
||||
} |
||||
Some(e) => (res.0, OptionFuture::from(Some(async move { e }.boxed()))), |
||||
} |
||||
} |
||||
ProtocolType::Broker => { |
||||
let message = serde_bare::from_slice::<BrokerMessage>(&frame); |
||||
match (message) { |
||||
Ok(message) => { |
||||
let reply = self |
||||
.broker_protocol |
||||
.as_ref() |
||||
.unwrap() |
||||
.handle_incoming(message) |
||||
.await; |
||||
(Ok(serde_bare::to_vec(&reply.0).unwrap()), reply.1) |
||||
} |
||||
Err(e_) => { |
||||
(Err(ProtocolError::SerializationError),OptionFuture::from(None)) |
||||
} |
||||
} |
||||
} |
||||
ProtocolType::Ext => { |
||||
// Ext protocol is not accepting 2 extrequest in the same connection.
|
||||
// closing the connection
|
||||
(Err(ProtocolError::InvalidState), OptionFuture::from(None)) |
||||
} |
||||
ProtocolType::P2P => { |
||||
unimplemented!() |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
pub struct ExtProtocolHandler {} |
||||
|
||||
impl ExtProtocolHandler { |
||||
pub fn handle_incoming(&self, msg: ExtRequest) -> ExtResponse { |
||||
unimplemented!() |
||||
} |
||||
} |
||||
|
||||
pub struct BrokerProtocolHandler { |
||||
broker: Arc<BrokerServer>, |
||||
user: PubKey, |
||||
async_frames_sender: async_channel::Sender<Vec<u8>>, |
||||
} |
||||
use std::{thread, time}; |
||||
|
||||
impl BrokerProtocolHandler { |
||||
fn prepare_reply_broker_message( |
||||
res: Result<(), ProtocolError>, |
||||
id: u64, |
||||
padding_size: usize, |
||||
) -> BrokerMessage { |
||||
let result = match res { |
||||
Ok(_) => 0, |
||||
Err(e) => e.into(), |
||||
}; |
||||
let msg = BrokerMessage::V0(BrokerMessageV0 { |
||||
padding: vec![0; padding_size], |
||||
content: BrokerMessageContentV0::BrokerResponse(BrokerResponse::V0(BrokerResponseV0 { |
||||
id, |
||||
result, |
||||
})), |
||||
}); |
||||
msg |
||||
} |
||||
|
||||
fn prepare_reply_broker_overlay_message( |
||||
res: Result<(), ProtocolError>, |
||||
id: u64, |
||||
overlay: OverlayId, |
||||
block: Option<Block>, |
||||
padding_size: usize, |
||||
) -> BrokerMessage { |
||||
let result = match res { |
||||
Ok(_) => 0, |
||||
Err(e) => e.into(), |
||||
}; |
||||
let content = match block { |
||||
Some(b) => Some(BrokerOverlayResponseContentV0::Block(b)), |
||||
None => None, |
||||
}; |
||||
let msg = BrokerMessage::V0(BrokerMessageV0 { |
||||
padding: vec![0; padding_size], |
||||
content: BrokerMessageContentV0::BrokerOverlayMessage(BrokerOverlayMessage::V0( |
||||
BrokerOverlayMessageV0 { |
||||
overlay, |
||||
content: BrokerOverlayMessageContentV0::BrokerOverlayResponse( |
||||
BrokerOverlayResponse::V0(BrokerOverlayResponseV0 { |
||||
id, |
||||
result, |
||||
content, |
||||
}), |
||||
), |
||||
}, |
||||
)), |
||||
}); |
||||
msg |
||||
} |
||||
|
||||
fn prepare_reply_broker_overlay_message_stream( |
||||
res: Result<Block, ProtocolError>, |
||||
id: u64, |
||||
overlay: OverlayId, |
||||
padding_size: usize, |
||||
) -> BrokerMessage { |
||||
let result: u16 = match &res { |
||||
Ok(r) => ProtocolError::PartialContent.into(), |
||||
Err(e) => (*e).clone().into(), |
||||
}; |
||||
let content = match res { |
||||
Ok(r) => Some(BrokerOverlayResponseContentV0::Block(r)), |
||||
Err(_) => None, |
||||
}; |
||||
let msg = BrokerMessage::V0(BrokerMessageV0 { |
||||
padding: vec![0; padding_size], |
||||
content: BrokerMessageContentV0::BrokerOverlayMessage(BrokerOverlayMessage::V0( |
||||
BrokerOverlayMessageV0 { |
||||
overlay, |
||||
content: BrokerOverlayMessageContentV0::BrokerOverlayResponse( |
||||
BrokerOverlayResponse::V0(BrokerOverlayResponseV0 { |
||||
id, |
||||
result, |
||||
content, |
||||
}), |
||||
), |
||||
}, |
||||
)), |
||||
}); |
||||
msg |
||||
} |
||||
|
||||
async fn send_block_stream_response_to_client( |
||||
&self, |
||||
res: Result<async_channel::Receiver<Block>, ProtocolError>, |
||||
id: u64, |
||||
overlay: OverlayId, |
||||
padding_size: usize, |
||||
) -> (BrokerMessage, OptionFuture<BoxFuture<'static, u16>>) { |
||||
// return an error or the first block, and setup a spawner for the remaining blocks to be sent.
|
||||
let one_reply: ( |
||||
Result<Block, ProtocolError>, |
||||
OptionFuture<BoxFuture<'static, u16>>, |
||||
) = match res { |
||||
Err(e) => (Err(e), OptionFuture::from(None)), |
||||
Ok(stream) => { |
||||
let one = stream |
||||
.recv_blocking() |
||||
.map_err(|e| ProtocolError::EndOfStream); |
||||
|
||||
if one.is_ok() { |
||||
let sender = self.async_frames_sender.clone(); |
||||
let a = OptionFuture::from(Some( |
||||
async move { |
||||
while let Ok(next) = stream.recv().await { |
||||
let msg = Self::prepare_reply_broker_overlay_message_stream( |
||||
Ok(next), |
||||
id, |
||||
overlay, |
||||
padding_size, |
||||
); |
||||
let res = sender.send(serde_bare::to_vec(&msg).unwrap()).await; |
||||
if res.is_err() { |
||||
break; |
||||
} |
||||
} |
||||
// sending end of stream
|
||||
let msg = Self::prepare_reply_broker_overlay_message_stream( |
||||
Err(ProtocolError::EndOfStream), |
||||
id, |
||||
overlay, |
||||
padding_size, |
||||
); |
||||
let _ = sender.send(serde_bare::to_vec(&msg).unwrap()).await; |
||||
0 |
||||
} |
||||
.boxed(), |
||||
)); |
||||
(one, a) |
||||
} else { |
||||
(one, OptionFuture::from(None)) |
||||
} |
||||
} |
||||
}; |
||||
return ( |
||||
Self::prepare_reply_broker_overlay_message_stream( |
||||
one_reply.0, |
||||
id, |
||||
overlay, |
||||
padding_size, |
||||
), |
||||
one_reply.1, |
||||
); |
||||
} |
||||
|
||||
pub async fn handle_incoming( |
||||
&self, |
||||
msg: BrokerMessage, |
||||
) -> (BrokerMessage, OptionFuture<BoxFuture<'static, u16>>) { |
||||
|
||||
let padding_size = 20; // TODO randomize, if config of server contains padding_max
|
||||
|
||||
let id = msg.id(); |
||||
let content = msg.content(); |
||||
match content { |
||||
BrokerMessageContentV0::BrokerRequest(req) => ( |
||||
Self::prepare_reply_broker_message( |
||||
match req.content_v0() { |
||||
BrokerRequestContentV0::AddUser(cmd) => { |
||||
self.broker.add_user(self.user, cmd.user(), cmd.sig()) |
||||
} |
||||
BrokerRequestContentV0::DelUser(cmd) => { |
||||
self.broker.del_user(self.user, cmd.user(), cmd.sig()) |
||||
} |
||||
BrokerRequestContentV0::AddClient(cmd) => { |
||||
self.broker.add_client(self.user, cmd.client(), cmd.sig()) |
||||
} |
||||
BrokerRequestContentV0::DelClient(cmd) => { |
||||
self.broker.del_client(self.user, cmd.client(), cmd.sig()) |
||||
} |
||||
}, |
||||
id, |
||||
padding_size, |
||||
), |
||||
OptionFuture::from(None), |
||||
), |
||||
BrokerMessageContentV0::BrokerResponse(res) => ( |
||||
Self::prepare_reply_broker_message( |
||||
Err(ProtocolError::InvalidState), |
||||
id, |
||||
padding_size, |
||||
), |
||||
OptionFuture::from(None), |
||||
), |
||||
BrokerMessageContentV0::BrokerOverlayMessage(omsg) => { |
||||
let overlay = omsg.overlay_id(); |
||||
let block = None; |
||||
let mut res = Err(ProtocolError::InvalidState); |
||||
|
||||
if omsg.is_request() { |
||||
match omsg.overlay_request().content_v0() { |
||||
BrokerOverlayRequestContentV0::OverlayConnect(_) => { |
||||
res = self.broker.connect_overlay(self.user, overlay) |
||||
} |
||||
BrokerOverlayRequestContentV0::OverlayJoin(j) => { |
||||
res = self.broker.join_overlay( |
||||
self.user, |
||||
overlay, |
||||
j.repo_pubkey(), |
||||
j.secret(), |
||||
j.peers(), |
||||
) |
||||
} |
||||
BrokerOverlayRequestContentV0::ObjectDel(op) => { |
||||
res = self.broker.del_object(self.user, overlay, op.id()) |
||||
} |
||||
BrokerOverlayRequestContentV0::ObjectPin(op) => { |
||||
res = self.broker.pin_object(self.user, overlay, op.id()) |
||||
} |
||||
BrokerOverlayRequestContentV0::ObjectUnpin(op) => { |
||||
res = self.broker.unpin_object(self.user, overlay, op.id()) |
||||
} |
||||
BrokerOverlayRequestContentV0::BlockPut(b) => { |
||||
res = self.broker.put_block(self.user, overlay, b.block()) |
||||
} |
||||
BrokerOverlayRequestContentV0::BranchSyncReq(b) => { |
||||
let res = self.broker.sync_branch( |
||||
self.user, |
||||
&overlay, |
||||
b.heads(), |
||||
b.known_heads(), |
||||
b.known_commits(), |
||||
); |
||||
return self |
||||
.send_block_stream_response_to_client( |
||||
res, |
||||
id, |
||||
overlay, |
||||
padding_size, |
||||
) |
||||
.await; |
||||
} |
||||
BrokerOverlayRequestContentV0::BlockGet(b) => { |
||||
let res = self.broker.get_block( |
||||
self.user, |
||||
overlay, |
||||
b.id(), |
||||
b.include_children(), |
||||
b.topic(), |
||||
); |
||||
return self |
||||
.send_block_stream_response_to_client( |
||||
res, |
||||
id, |
||||
overlay, |
||||
padding_size, |
||||
) |
||||
.await; |
||||
} |
||||
_ => {} |
||||
} |
||||
} |
||||
|
||||
( |
||||
Self::prepare_reply_broker_overlay_message( |
||||
res, |
||||
id, |
||||
overlay, |
||||
block, |
||||
padding_size, |
||||
), |
||||
OptionFuture::from(None), |
||||
) |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
const REPO_STORES_SUBDIR: &str = "repos"; |
||||
|
||||
pub struct BrokerServer { |
||||
store: LmdbBrokerStore, |
||||
mode: ConfigMode, |
||||
repo_stores: Arc<RwLock<HashMap<RepoStoreId, LmdbRepoStore>>>, |
||||
// only used in ConfigMode::Local
|
||||
// try to change it to this version below in order to avoid double hashmap lookup in local mode. but hard to do...
|
||||
//overlayid_to_repostore: HashMap<RepoStoreId, &'a LmdbRepoStore>,
|
||||
overlayid_to_repostore: Arc<RwLock<HashMap<OverlayId, RepoStoreId>>>, |
||||
} |
||||
|
||||
impl BrokerServer { |
||||
pub fn new(store: LmdbBrokerStore, mode: ConfigMode) -> Result<BrokerServer, BrokerError> { |
||||
let mut configmode: ConfigMode; |
||||
{ |
||||
let config = Config::get_or_create(&mode, &store)?; |
||||
configmode = config.mode()?; |
||||
} |
||||
Ok(BrokerServer { |
||||
store, |
||||
mode: configmode, |
||||
repo_stores: Arc::new(RwLock::new(HashMap::new())), |
||||
overlayid_to_repostore: Arc::new(RwLock::new(HashMap::new())), |
||||
}) |
||||
} |
||||
|
||||
fn open_or_create_repostore<F, R>( |
||||
&self, |
||||
repostore_id: RepoStoreId, |
||||
f: F, |
||||
) -> Result<R, ProtocolError> |
||||
where |
||||
F: FnOnce(&LmdbRepoStore) -> Result<R, ProtocolError>, |
||||
{ |
||||
// first let's find it in the BrokerStore.repostoreinfo table in order to get the encryption key
|
||||
let info = RepoStoreInfo::open(&repostore_id, &self.store) |
||||
.map_err(|e| BrokerError::OverlayNotFound)?; |
||||
let key = info.key()?; |
||||
let mut path = self.store.path(); |
||||
path.push(REPO_STORES_SUBDIR); |
||||
path.push::<String>(repostore_id.clone().into()); |
||||
std::fs::create_dir_all(path.clone()).map_err(|_e| ProtocolError::WriteError )?; |
||||
println!("path for repo store: {}", path.to_str().unwrap()); |
||||
let repo = LmdbRepoStore::open(&path, *key.slice()); |
||||
let mut writer = self.repo_stores.write().expect("write repo_store hashmap"); |
||||
writer.insert(repostore_id.clone(), repo); |
||||
|
||||
f(writer.get(&repostore_id).unwrap()) |
||||
} |
||||
|
||||
fn get_repostore_from_overlay_id<F, R>( |
||||
&self, |
||||
overlay_id: &OverlayId, |
||||
f: F, |
||||
) -> Result<R, ProtocolError> |
||||
where |
||||
F: FnOnce(&LmdbRepoStore) -> Result<R, ProtocolError>, |
||||
{ |
||||
if self.mode == ConfigMode::Core { |
||||
let repostore_id = RepoStoreId::Overlay(*overlay_id); |
||||
let reader = self.repo_stores.read().expect("read repo_store hashmap"); |
||||
let rep = reader.get(&repostore_id); |
||||
match rep { |
||||
Some(repo) => return f(repo), |
||||
None => { |
||||
// we need to open/create it
|
||||
// TODO: last_access
|
||||
return self.open_or_create_repostore(repostore_id, |repo| f(repo)); |
||||
} |
||||
} |
||||
} else { |
||||
// it is ConfigMode::Local
|
||||
{ |
||||
let reader = self |
||||
.overlayid_to_repostore |
||||
.read() |
||||
.expect("read overlayid_to_repostore hashmap"); |
||||
match reader.get(&overlay_id) { |
||||
Some(repostoreid) => { |
||||
let reader = self.repo_stores.read().expect("read repo_store hashmap"); |
||||
match reader.get(repostoreid) { |
||||
Some(repo) => return f(repo), |
||||
None => return Err(ProtocolError::BrokerError), |
||||
} |
||||
} |
||||
None => {} |
||||
}; |
||||
} |
||||
|
||||
// we need to open/create it
|
||||
// first let's find it in the BrokerStore.overlay table to retrieve its repo_pubkey
|
||||
debug_println!("searching for overlayId {}", overlay_id); |
||||
let overlay = Overlay::open(overlay_id, &self.store)?; |
||||
debug_println!("found overlayId {}", overlay_id); |
||||
let repo_id = overlay.repo()?; |
||||
let repostore_id = RepoStoreId::Repo(repo_id); |
||||
let mut writer = self |
||||
.overlayid_to_repostore |
||||
.write() |
||||
.expect("write overlayid_to_repostore hashmap"); |
||||
writer.insert(*overlay_id, repostore_id.clone()); |
||||
// now opening/creating the RepoStore
|
||||
// TODO: last_access
|
||||
return self.open_or_create_repostore(repostore_id, |repo| f(repo)); |
||||
} |
||||
} |
||||
|
||||
pub fn local_connection(&mut self, user: PubKey) -> BrokerConnectionLocal { |
||||
BrokerConnectionLocal::new(self, user) |
||||
} |
||||
|
||||
pub fn protocol_handler(self: Arc<Self>) -> ProtocolHandler { |
||||
let (s, r) = async_channel::unbounded::<Vec<u8>>(); |
||||
return ProtocolHandler { |
||||
broker: Arc::clone(&self), |
||||
protocol: ProtocolType::Start, |
||||
auth_protocol: None, |
||||
broker_protocol: None, |
||||
ext_protocol: None, |
||||
r: Some(r), |
||||
s, |
||||
}; |
||||
} |
||||
|
||||
pub fn add_user( |
||||
&self, |
||||
admin_user: PubKey, |
||||
user_id: PubKey, |
||||
sig: Sig, |
||||
) -> Result<(), ProtocolError> { |
||||
debug_println!("ADDING USER {}", user_id); |
||||
// TODO add is_admin boolean
|
||||
// TODO check that admin_user is indeed an admin
|
||||
|
||||
// verify signature
|
||||
let op_content = AddUserContentV0 { user: user_id }; |
||||
let _ = verify(&serde_bare::to_vec(&op_content).unwrap(), sig, admin_user)?; |
||||
|
||||
// check user_id is not already present
|
||||
let account = Account::open(&user_id, &self.store); |
||||
if account.is_ok() { |
||||
Err(ProtocolError::UserAlreadyExists) |
||||
} |
||||
// if not, add to store
|
||||
else { |
||||
let _ = Account::create(&user_id, false, &self.store)?; |
||||
Ok(()) |
||||
} |
||||
} |
||||
|
||||
pub fn del_user( |
||||
&self, |
||||
admin_user: PubKey, |
||||
user_id: PubKey, |
||||
sig: Sig, |
||||
) -> Result<(), ProtocolError> { |
||||
// TODO implement del_user
|
||||
Ok(()) |
||||
} |
||||
pub fn add_client( |
||||
&self, |
||||
user: PubKey, |
||||
client_id: PubKey, |
||||
sig: Sig, |
||||
) -> Result<(), ProtocolError> { |
||||
// TODO implement add_client
|
||||
Ok(()) |
||||
} |
||||
|
||||
pub fn del_client( |
||||
&self, |
||||
user: PubKey, |
||||
client_id: PubKey, |
||||
sig: Sig, |
||||
) -> Result<(), ProtocolError> { |
||||
// TODO implement del_client
|
||||
Ok(()) |
||||
} |
||||
|
||||
pub fn connect_overlay(&self, user: PubKey, overlay: OverlayId) -> Result<(), ProtocolError> { |
||||
// TODO check that the broker has already joined this overlay. if not, send OverlayNotJoined
|
||||
Err(ProtocolError::OverlayNotJoined) |
||||
} |
||||
|
||||
pub fn del_object( |
||||
&self, |
||||
user: PubKey, |
||||
overlay: Digest, |
||||
id: ObjectId, |
||||
) -> Result<(), ProtocolError> { |
||||
self.get_repostore_from_overlay_id(&overlay, |store| { |
||||
// TODO, only admin users can delete on a store on this broker
|
||||
let obj = Object::load(id, None, store); |
||||
if obj.is_err() { |
||||
return Err(ProtocolError::NotFound); |
||||
} |
||||
let o = obj.ok().unwrap(); |
||||
let mut deduplicated: HashSet<ObjectId> = HashSet::new(); |
||||
for block in o.blocks() { |
||||
let id = block.id(); |
||||
if deduplicated.get(&id).is_none() { |
||||
store.del(&id)?; |
||||
deduplicated.insert(id); |
||||
} |
||||
} |
||||
Ok(()) |
||||
}) |
||||
} |
||||
|
||||
pub fn pin_object( |
||||
&self, |
||||
user: PubKey, |
||||
overlay: OverlayId, |
||||
id: ObjectId, |
||||
) -> Result<(), ProtocolError> { |
||||
self.get_repostore_from_overlay_id(&overlay, |store| { |
||||
// TODO, store the user who pins, and manage reference counting on how many users pin/unpin
|
||||
let obj = Object::load(id, None, store); |
||||
if obj.is_err() { |
||||
return Err(ProtocolError::NotFound); |
||||
} |
||||
let o = obj.ok().unwrap(); |
||||
let mut deduplicated: HashSet<ObjectId> = HashSet::new(); |
||||
for block in o.blocks() { |
||||
let id = block.id(); |
||||
if deduplicated.get(&id).is_none() { |
||||
store.pin(&id)?; |
||||
deduplicated.insert(id); |
||||
} |
||||
} |
||||
Ok(()) |
||||
}) |
||||
} |
||||
|
||||
pub fn unpin_object( |
||||
&self, |
||||
user: PubKey, |
||||
overlay: OverlayId, |
||||
id: ObjectId, |
||||
) -> Result<(), ProtocolError> { |
||||
self.get_repostore_from_overlay_id(&overlay, |store| { |
||||
// TODO, store the user who pins, and manage reference counting on how many users pin/unpin
|
||||
let obj = Object::load(id, None, store); |
||||
if obj.is_err() { |
||||
return Err(ProtocolError::NotFound); |
||||
} |
||||
let o = obj.ok().unwrap(); |
||||
let mut deduplicated: HashSet<ObjectId> = HashSet::new(); |
||||
for block in o.blocks() { |
||||
let id = block.id(); |
||||
if deduplicated.get(&id).is_none() { |
||||
store.unpin(&id)?; |
||||
deduplicated.insert(id); |
||||
} |
||||
} |
||||
Ok(()) |
||||
}) |
||||
} |
||||
|
||||
pub fn copy_object( |
||||
&self, |
||||
user: PubKey, |
||||
overlay: OverlayId, |
||||
id: ObjectId, |
||||
expiry: Option<Timestamp>, |
||||
) -> Result<ObjectId, ProtocolError> { |
||||
// self.get_repostore_from_overlay_id(&overlay, |store| {
|
||||
// //let obj = Object::from_store(id, None, store);
|
||||
// //Ok(Object::copy(id, expiry, store)?)
|
||||
// });
|
||||
todo!(); |
||||
} |
||||
|
||||
pub fn put_block( |
||||
&self, |
||||
user: PubKey, |
||||
overlay: OverlayId, |
||||
block: &Block, |
||||
) -> Result<(), ProtocolError> { |
||||
self.get_repostore_from_overlay_id(&overlay, |store| { |
||||
let _ = store.put(block)?; |
||||
Ok(()) |
||||
}) |
||||
} |
||||
|
||||
pub fn get_block( |
||||
&self, |
||||
user: PubKey, |
||||
overlay: OverlayId, |
||||
id: BlockId, |
||||
include_children: bool, |
||||
topic: Option<PubKey>, |
||||
) -> Result<async_channel::Receiver<Block>, ProtocolError> { |
||||
self.get_repostore_from_overlay_id(&overlay, |store| { |
||||
let (s, r) = async_channel::unbounded::<Block>(); |
||||
if !include_children { |
||||
let block = store.get(&id)?; |
||||
s.send_blocking(block) |
||||
.map_err(|_e| ProtocolError::WriteError)?; |
||||
Ok(r) |
||||
} else { |
||||
let obj = Object::load(id, None, store); |
||||
// TODO return partial blocks when some are missing ?
|
||||
if obj.is_err() { |
||||
//&& obj.err().unwrap().len() == 1 && obj.err().unwrap()[0] == id {
|
||||
return Err(ProtocolError::NotFound); |
||||
} |
||||
// TODO use a task to send non blocking (streaming)
|
||||
let o = obj.ok().unwrap(); |
||||
//debug_println!("{} BLOCKS ", o.blocks().len());
|
||||
let mut deduplicated: HashSet<BlockId> = HashSet::new(); |
||||
for block in o.blocks() { |
||||
let id = block.id(); |
||||
if deduplicated.get(&id).is_none() { |
||||
s.send_blocking(block.clone()) |
||||
.map_err(|_e| ProtocolError::WriteError)?; |
||||
deduplicated.insert(id); |
||||
} |
||||
} |
||||
Ok(r) |
||||
} |
||||
}) |
||||
} |
||||
|
||||
pub fn sync_branch( |
||||
&self, |
||||
user: PubKey, |
||||
overlay: &OverlayId, |
||||
heads: &Vec<ObjectId>, |
||||
known_heads: &Vec<ObjectId>, |
||||
known_commits: &BloomFilter, |
||||
) -> Result<async_channel::Receiver<Block>, ProtocolError> { |
||||
//debug_println!("heads {:?}", heads);
|
||||
//debug_println!("known_heads {:?}", known_heads);
|
||||
//debug_println!("known_commits {:?}", known_commits);
|
||||
|
||||
self.get_repostore_from_overlay_id(&overlay, |store| { |
||||
let (s, r) = async_channel::unbounded::<Block>(); |
||||
|
||||
let res = Branch::sync_req(heads, known_heads, known_commits, store) |
||||
.map_err(|e| ProtocolError::ObjectParseError)?; |
||||
|
||||
// todo, use a task to send non blocking (streaming)
|
||||
debug_println!("SYNCING {} COMMITS", res.len()); |
||||
|
||||
let mut deduplicated: HashSet<BlockId> = HashSet::new(); |
||||
|
||||
for objectid in res { |
||||
let object = Object::load(objectid, None, store)?; |
||||
|
||||
for block in object.blocks() { |
||||
let id = block.id(); |
||||
if deduplicated.get(&id).is_none() { |
||||
s.send_blocking(block.clone()) |
||||
.map_err(|_e| ProtocolError::WriteError)?; |
||||
deduplicated.insert(id); |
||||
} |
||||
} |
||||
} |
||||
Ok(r) |
||||
}) |
||||
} |
||||
|
||||
fn compute_repostore_id(&self, overlay: OverlayId, repo_id: Option<PubKey>) -> RepoStoreId { |
||||
match self.mode { |
||||
ConfigMode::Core => RepoStoreId::Overlay(overlay), |
||||
ConfigMode::Local => RepoStoreId::Repo(repo_id.unwrap()), |
||||
} |
||||
} |
||||
|
||||
pub fn join_overlay( |
||||
&self, |
||||
user: PubKey, |
||||
overlay_id: OverlayId, |
||||
repo_id: Option<PubKey>, |
||||
secret: SymKey, |
||||
peers: &Vec<PeerAdvert>, |
||||
) -> Result<(), ProtocolError> { |
||||
// check if this overlay already exists
|
||||
//debug_println!("SEARCHING OVERLAY");
|
||||
let overlay_res = Overlay::open(&overlay_id, &self.store); |
||||
let overlay = match overlay_res { |
||||
Err(StorageError::NotFound) => { |
||||
// we have to add it
|
||||
if self.mode == ConfigMode::Local && repo_id.is_none() { |
||||
return Err(ProtocolError::RepoIdRequired); |
||||
} |
||||
let over = Overlay::create( |
||||
&overlay_id, |
||||
&secret, |
||||
if self.mode == ConfigMode::Local { |
||||
repo_id |
||||
} else { |
||||
None |
||||
}, |
||||
&self.store, |
||||
)?; |
||||
// we need to add an encryption key for the repostore.
|
||||
let mut random_buf = [0u8; 32]; |
||||
getrandom::getrandom(&mut random_buf).unwrap(); |
||||
let key = SymKey::ChaCha20Key(random_buf); |
||||
|
||||
let _ = RepoStoreInfo::create( |
||||
&self.compute_repostore_id(overlay_id, repo_id), |
||||
&key, |
||||
&self.store, |
||||
)?; // TODO in case of error, delete the previously created Overlay
|
||||
//debug_println!("KEY ADDED");
|
||||
over |
||||
} |
||||
Err(e) => return Err(e.into()), |
||||
Ok(overlay) => overlay, |
||||
}; |
||||
//debug_println!("OVERLAY FOUND");
|
||||
// add the peers to the overlay
|
||||
for advert in peers { |
||||
Peer::update_or_create(advert, &self.store)?; |
||||
overlay.add_peer(&advert.peer())?; |
||||
} |
||||
//debug_println!("PEERS ADDED");
|
||||
|
||||
// now adding the overlay_id to the account
|
||||
let account = Account::open(&user, &self.store)?; // TODO in case of error, delete the previously created Overlay
|
||||
account.add_overlay(&overlay_id)?; |
||||
//debug_println!("USER <-> OVERLAY");
|
||||
|
||||
//TODO: connect to peers
|
||||
|
||||
Ok(()) |
||||
} |
||||
} |
||||
@ -0,0 +1,160 @@ |
||||
/* |
||||
* Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers |
||||
* All rights reserved. |
||||
* Licensed under the Apache License, Version 2.0 |
||||
* <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
|
||||
* or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
|
||||
* at your option. All files in the project carrying such |
||||
* notice may not be copied, modified, or distributed except |
||||
* according to those terms. |
||||
*/ |
||||
|
||||
use async_std::net::{TcpListener, TcpStream}; |
||||
use async_std::sync::Mutex; |
||||
use async_std::task; |
||||
use async_tungstenite::accept_async; |
||||
use async_tungstenite::tungstenite::protocol::Message; |
||||
use debug_print::*; |
||||
use futures::{SinkExt, StreamExt}; |
||||
use crate::broker_store_config::ConfigMode; |
||||
use crate::server::*; |
||||
use p2p_stores_lmdb::broker_store::LmdbBrokerStore; |
||||
use p2p_stores_lmdb::repo_store::LmdbRepoStore; |
||||
use std::fs; |
||||
use std::sync::Arc; |
||||
use tempfile::Builder; |
||||
use std::{thread, time}; |
||||
|
||||
pub async fn connection_loop(tcp: TcpStream, mut handler: ProtocolHandler) -> std::io::Result<()> { |
||||
let mut ws = accept_async(tcp).await.unwrap(); |
||||
let (mut tx, mut rx) = ws.split(); |
||||
|
||||
let mut tx_mutex = Arc::new(Mutex::new(tx)); |
||||
|
||||
// setup the async frames task
|
||||
let receiver = handler.async_frames_receiver(); |
||||
let ws_in_task = Arc::clone(&tx_mutex); |
||||
task::spawn(async move { |
||||
while let Ok(frame) = receiver.recv().await { |
||||
let mut sink = ws_in_task |
||||
.lock() |
||||
.await; |
||||
if sink.send(Message::binary(frame)) |
||||
.await |
||||
.is_err() |
||||
{ |
||||
break; |
||||
} |
||||
} |
||||
debug_println!("end of async frames loop"); |
||||
|
||||
let mut sink = ws_in_task.lock().await; |
||||
let _ = sink.send(Message::Close(None)).await; |
||||
let _ = sink.close().await; |
||||
}); |
||||
|
||||
while let Some(msg) = rx.next().await { |
||||
//debug_println!("RCV: {:?}", msg);
|
||||
let msg = match msg { |
||||
Err(e) => { |
||||
debug_println!("Error on server stream: {:?}", e); |
||||
// Errors returned directly through the AsyncRead/Write API are fatal, generally an error on the underlying
|
||||
// transport. closing connection
|
||||
break; |
||||
} |
||||
Ok(m) => m, |
||||
}; |
||||
//TODO implement PING messages
|
||||
if msg.is_close() { |
||||
debug_println!("CLOSE from CLIENT"); |
||||
break; |
||||
} else if msg.is_binary() { |
||||
//debug_println!("server received binary: {:?}", msg);
|
||||
|
||||
let replies = handler.handle_incoming(msg.into_data()).await; |
||||
|
||||
match replies.0 { |
||||
Err(e) => { |
||||
debug_println!("Protocol Error: {:?}", e); |
||||
// dealing with ProtocolErrors (closing the connection)
|
||||
break; |
||||
} |
||||
Ok(r) => { |
||||
if tx_mutex |
||||
.lock() |
||||
.await |
||||
.send(Message::binary(r)) |
||||
.await |
||||
.is_err() |
||||
{ |
||||
//dealing with sending errors (closing the connection)
|
||||
break; |
||||
} |
||||
} |
||||
} |
||||
match replies.1.await { |
||||
Some(errcode) => { |
||||
if errcode > 0 { |
||||
debug_println!("Close due to error code : {:?}", errcode); |
||||
//closing connection
|
||||
break; |
||||
} |
||||
} |
||||
None => {} |
||||
} |
||||
} |
||||
} |
||||
let mut sink = tx_mutex.lock().await; |
||||
let _ = sink.send(Message::Close(None)).await; |
||||
let _ = sink.close().await; |
||||
debug_println!("end of sync read+write loop"); |
||||
Ok(()) |
||||
} |
||||
|
||||
pub async fn run_server_accept_one(addrs: &str) -> std::io::Result<()> { |
||||
let root = tempfile::Builder::new() |
||||
.prefix("node-daemon") |
||||
.tempdir() |
||||
.unwrap(); |
||||
let master_key: [u8; 32] = [0; 32]; |
||||
std::fs::create_dir_all(root.path()).unwrap(); |
||||
println!("{}", root.path().to_str().unwrap()); |
||||
let store = LmdbBrokerStore::open(root.path(), master_key); |
||||
|
||||
let server: BrokerServer = |
||||
BrokerServer::new(store, ConfigMode::Local).expect("starting broker"); |
||||
|
||||
let socket = TcpListener::bind(addrs).await?; |
||||
debug_println!("Listening on 127.0.0.1:3012"); |
||||
let mut connections = socket.incoming(); |
||||
let server_arc = Arc::new(server); |
||||
let tcp = connections.next().await.unwrap()?; |
||||
let proto_handler = Arc::clone(&server_arc).protocol_handler(); |
||||
let _handle = task::spawn(connection_loop(tcp, proto_handler)); |
||||
|
||||
Ok(()) |
||||
} |
||||
|
||||
|
||||
pub async fn run_server(addrs: &str) -> std::io::Result<()> { |
||||
let root = tempfile::Builder::new() |
||||
.prefix("node-daemon") |
||||
.tempdir() |
||||
.unwrap(); |
||||
let master_key: [u8; 32] = [0; 32]; |
||||
std::fs::create_dir_all(root.path()).unwrap(); |
||||
println!("{}", root.path().to_str().unwrap()); |
||||
let store = LmdbBrokerStore::open(root.path(), master_key); |
||||
|
||||
let server: BrokerServer = |
||||
BrokerServer::new(store, ConfigMode::Local).expect("starting broker"); |
||||
|
||||
let socket = TcpListener::bind(addrs).await?; |
||||
let mut connections = socket.incoming(); |
||||
let server_arc = Arc::new(server); |
||||
while let Some(tcp) = connections.next().await { |
||||
let proto_handler = Arc::clone(&server_arc).protocol_handler(); |
||||
let _handle = task::spawn(connection_loop(tcp.unwrap(), proto_handler)); |
||||
} |
||||
Ok(()) |
||||
} |
||||
@ -0,0 +1,24 @@ |
||||
[package] |
||||
name = "p2p-client" |
||||
version = "0.1.0" |
||||
edition = "2021" |
||||
license = "MIT/Apache-2.0" |
||||
authors = ["Niko PLP <niko@nextgraph.org>"] |
||||
description = "P2P Client module of NextGraph" |
||||
repository = "https://git.nextgraph.org/NextGraph/nextgraph-rs" |
||||
|
||||
[dependencies] |
||||
debug_print = "1.0.0" |
||||
p2p-repo = { path = "../p2p-repo" } |
||||
p2p-net = { path = "../p2p-net" } |
||||
chacha20 = "0.9.0" |
||||
serde = { version = "1.0", features = ["derive"] } |
||||
serde_bare = "0.5.0" |
||||
serde_bytes = "0.11.7" |
||||
xactor = "0.7.11" |
||||
async-trait = "0.1.57" |
||||
async-std = { version = "1.7.0", features = ["attributes"] } |
||||
futures = "0.3.24" |
||||
async-channel = "1.7.1" |
||||
async-oneshot = "0.5.0" |
||||
async-tungstenite = { version = "0.17.2", features = ["async-std-runtime","async-native-tls"] } |
||||
@ -0,0 +1,596 @@ |
||||
/* |
||||
* Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers |
||||
* All rights reserved. |
||||
* Licensed under the Apache License, Version 2.0 |
||||
* <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
|
||||
* or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
|
||||
* at your option. All files in the project carrying such |
||||
* notice may not be copied, modified, or distributed except |
||||
* according to those terms. |
||||
*/ |
||||
|
||||
use async_std::task; |
||||
use async_std::sync::Mutex; |
||||
use futures::{ |
||||
ready, |
||||
stream::Stream, |
||||
task::{Context, Poll}, |
||||
Future, |
||||
select, FutureExt, |
||||
}; |
||||
use futures::channel::mpsc; |
||||
use std::pin::Pin; |
||||
use std::{collections::HashSet, fmt::Debug}; |
||||
|
||||
use async_oneshot::oneshot; |
||||
use debug_print::*; |
||||
use futures::{pin_mut, stream, Sink, SinkExt, StreamExt}; |
||||
use p2p_repo::object::*; |
||||
use p2p_repo::store::*; |
||||
use p2p_repo::types::*; |
||||
use p2p_repo::utils::*; |
||||
use p2p_net::errors::*; |
||||
use p2p_net::types::*; |
||||
use p2p_net::broker_connection::*; |
||||
use std::collections::HashMap; |
||||
use std::sync::{Arc, RwLock}; |
||||
use xactor::{message, spawn, Actor, Addr, Handler, WeakAddr}; |
||||
|
||||
|
||||
#[message] |
||||
struct BrokerMessageXActor(BrokerMessage); |
||||
|
||||
struct BrokerMessageActor { |
||||
r: Option<async_oneshot::Receiver<BrokerMessage>>, |
||||
s: async_oneshot::Sender<BrokerMessage>, |
||||
} |
||||
|
||||
impl Actor for BrokerMessageActor {} |
||||
|
||||
impl BrokerMessageActor { |
||||
fn new() -> BrokerMessageActor { |
||||
let (s, r) = oneshot::<BrokerMessage>(); |
||||
BrokerMessageActor { r: Some(r), s } |
||||
} |
||||
fn resolve(&mut self, msg: BrokerMessage) { |
||||
let _ = self.s.send(msg); |
||||
} |
||||
|
||||
fn receiver(&mut self) -> async_oneshot::Receiver<BrokerMessage> { |
||||
self.r.take().unwrap() |
||||
} |
||||
} |
||||
|
||||
struct BrokerMessageStreamActor { |
||||
r: Option<async_channel::Receiver<Block>>, |
||||
s: async_channel::Sender<Block>, |
||||
error_r: Option<async_oneshot::Receiver<Option<ProtocolError>>>, |
||||
error_s: Option<async_oneshot::Sender<Option<ProtocolError>>>, |
||||
} |
||||
|
||||
impl Actor for BrokerMessageStreamActor {} |
||||
|
||||
impl BrokerMessageStreamActor { |
||||
fn new() -> BrokerMessageStreamActor { |
||||
let (s, r) = async_channel::unbounded::<Block>(); |
||||
let (error_s, error_r) = oneshot::<Option<ProtocolError>>(); |
||||
BrokerMessageStreamActor { |
||||
r: Some(r), |
||||
s, |
||||
error_r: Some(error_r), |
||||
error_s: Some(error_s), |
||||
} |
||||
} |
||||
async fn partial(&mut self, block: Block) -> Result<(), ProtocolError> { |
||||
//debug_println!("GOT PARTIAL {:?}", block.id());
|
||||
self.s |
||||
.send(block) |
||||
.await |
||||
.map_err(|e| ProtocolError::WriteError) |
||||
} |
||||
|
||||
fn receiver(&mut self) -> async_channel::Receiver<Block> { |
||||
self.r.take().unwrap() |
||||
} |
||||
|
||||
fn error_receiver(&mut self) -> async_oneshot::Receiver<Option<ProtocolError>> { |
||||
self.error_r.take().unwrap() |
||||
} |
||||
|
||||
fn send_error(&mut self, err: Option<ProtocolError>) { |
||||
if self.error_s.is_some() { |
||||
let _ = self.error_s.take().unwrap().send(err); |
||||
self.error_s = None; |
||||
} |
||||
} |
||||
|
||||
fn close(&mut self) { |
||||
self.s.close(); |
||||
} |
||||
} |
||||
|
||||
#[async_trait::async_trait] |
||||
impl Handler<BrokerMessageXActor> for BrokerMessageActor { |
||||
async fn handle(&mut self, ctx: &mut xactor::Context<Self>, msg: BrokerMessageXActor) { |
||||
//println!("handling {:?}", msg.0);
|
||||
self.resolve(msg.0); |
||||
ctx.stop(None); |
||||
} |
||||
} |
||||
|
||||
#[async_trait::async_trait] |
||||
impl Handler<BrokerMessageXActor> for BrokerMessageStreamActor { |
||||
async fn handle(&mut self, ctx: &mut xactor::Context<Self>, msg: BrokerMessageXActor) { |
||||
//println!("handling {:?}", msg.0);
|
||||
let res: Result<Option<Block>, ProtocolError> = msg.0.into(); |
||||
match res { |
||||
Err(e) => { |
||||
self.send_error(Some(e)); |
||||
ctx.stop(None); |
||||
self.close(); |
||||
} |
||||
Ok(Some(b)) => { |
||||
self.send_error(None); |
||||
// it must be a partial content
|
||||
let res = self.partial(b).await; |
||||
if let Err(e) = res { |
||||
ctx.stop(None); |
||||
self.close(); |
||||
} |
||||
} |
||||
Ok(None) => { |
||||
self.send_error(None); |
||||
ctx.stop(None); |
||||
self.close(); |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
pub struct ConnectionRemote {} |
||||
|
||||
impl ConnectionRemote { |
||||
pub async fn ext_request< |
||||
B: Stream<Item = Vec<u8>> + StreamExt + Send + Sync, |
||||
A: Sink<Vec<u8>, Error = ProtocolError> + Send, |
||||
>( |
||||
w: A, |
||||
r: B, |
||||
request: ExtRequest, |
||||
) -> Result<ExtResponse, ProtocolError> { |
||||
unimplemented!(); |
||||
} |
||||
|
||||
async fn close<S>(w: S, err: ProtocolError) -> ProtocolError |
||||
where |
||||
S: Sink<Vec<u8>, Error = ProtocolError>, |
||||
{ |
||||
let mut writer = Box::pin(w); |
||||
let _ = writer.send(vec![]); |
||||
let _ = writer.close().await; |
||||
err |
||||
} |
||||
|
||||
pub async fn open_broker_connection< |
||||
B: Stream<Item = Vec<u8>> + StreamExt + Send + Sync + 'static, |
||||
A: Sink<Vec<u8>, Error = ProtocolError> + Send + 'static, |
||||
>( |
||||
w: A, |
||||
r: B, |
||||
user: PubKey, |
||||
user_pk: PrivKey, |
||||
client: PubKey, |
||||
) -> Result<impl BrokerConnection, ProtocolError> { |
||||
let mut writer = Box::pin(w); |
||||
writer |
||||
.send(serde_bare::to_vec(&StartProtocol::Auth(ClientHello::V0()))?) |
||||
.await |
||||
.map_err(|_e| ProtocolError::WriteError)?; |
||||
|
||||
let mut reader = Box::pin(r); |
||||
let answer = reader.next().await; |
||||
if answer.is_none() { |
||||
return Err(Self::close(writer, ProtocolError::InvalidState).await); |
||||
} |
||||
|
||||
let server_hello = serde_bare::from_slice::<ServerHello>(&answer.unwrap())?; |
||||
|
||||
//debug_println!("received nonce from server: {:?}", server_hello.nonce());
|
||||
|
||||
let content = ClientAuthContentV0 { |
||||
user, |
||||
client, |
||||
nonce: server_hello.nonce().clone(), |
||||
}; |
||||
|
||||
let sig = sign(user_pk, user, &serde_bare::to_vec(&content)?) |
||||
.map_err(|_e| ProtocolError::SignatureError)?; |
||||
|
||||
let auth_ser = serde_bare::to_vec(&ClientAuth::V0(ClientAuthV0 { content, sig }))?; |
||||
//debug_println!("AUTH SENT {:?}", auth_ser);
|
||||
writer |
||||
.send(auth_ser) |
||||
.await |
||||
.map_err(|_e| ProtocolError::WriteError)?; |
||||
|
||||
let answer = reader.next().await; |
||||
if answer.is_none() { |
||||
//return Err(ProtocolError::InvalidState);
|
||||
return Err(Self::close(writer, ProtocolError::InvalidState).await); |
||||
} |
||||
|
||||
let auth_result = serde_bare::from_slice::<AuthResult>(&answer.unwrap())?; |
||||
|
||||
match auth_result.result() { |
||||
0 => { |
||||
async fn transform(message: BrokerMessage) -> Result<Vec<u8>, ProtocolError> { |
||||
if message.is_close() { |
||||
Ok(vec![]) |
||||
} else { |
||||
Ok(serde_bare::to_vec(&message)?) |
||||
} |
||||
} |
||||
let messages_stream_write = writer.with(|message| transform(message)); |
||||
|
||||
let mut messages_stream_read = reader.map(|message| { |
||||
if message.len() == 0 { |
||||
BrokerMessage::Close |
||||
} else { |
||||
match serde_bare::from_slice::<BrokerMessage>(&message) { |
||||
Err(e) => BrokerMessage::Close, |
||||
Ok(m) => m |
||||
} |
||||
} |
||||
}); |
||||
|
||||
let cnx = |
||||
BrokerConnectionRemote::open(messages_stream_write, messages_stream_read, user); |
||||
|
||||
Ok(cnx) |
||||
} |
||||
err => Err(Self::close(writer, ProtocolError::try_from(err).unwrap()).await), |
||||
} |
||||
} |
||||
} |
||||
|
||||
pub struct BrokerConnectionRemote<T> |
||||
where |
||||
T: Sink<BrokerMessage> + Send + 'static, |
||||
{ |
||||
writer: Arc<Mutex<Pin<Box<T>>>>, |
||||
user: PubKey, |
||||
actors: Arc<RwLock<HashMap<u64, WeakAddr<BrokerMessageActor>>>>, |
||||
stream_actors: Arc<RwLock<HashMap<u64, WeakAddr<BrokerMessageStreamActor>>>>, |
||||
shutdown: mpsc::UnboundedSender<Void>, |
||||
} |
||||
|
||||
#[async_trait::async_trait] |
||||
impl<T> BrokerConnection for BrokerConnectionRemote<T> |
||||
where |
||||
T: Sink<BrokerMessage> + Send, |
||||
{ |
||||
type OC = BrokerConnectionRemote<T>; |
||||
type BlockStream = async_channel::Receiver<Block>; |
||||
|
||||
async fn close(&mut self) { |
||||
let _ = self.shutdown.close().await; |
||||
let mut w = self.writer.lock().await; |
||||
let _ = w.send(BrokerMessage::Close).await; |
||||
let _ = w.close().await; |
||||
} |
||||
|
||||
async fn process_overlay_request_stream_response( |
||||
&mut self, |
||||
overlay: OverlayId, |
||||
request: BrokerOverlayRequestContentV0, |
||||
) -> Result<Pin<Box<Self::BlockStream>>, ProtocolError> { |
||||
let mut actor = BrokerMessageStreamActor::new(); |
||||
let receiver = actor.receiver(); |
||||
let error_receiver = actor.error_receiver(); |
||||
let mut addr = actor |
||||
.start() |
||||
.await |
||||
.map_err(|_e| ProtocolError::ActorError)?; |
||||
|
||||
let request_id = addr.actor_id(); |
||||
//debug_println!("actor ID {}", request_id);
|
||||
|
||||
{ |
||||
let mut map = self.stream_actors.write().expect("RwLock poisoned"); |
||||
map.insert(request_id, addr.downgrade()); |
||||
} |
||||
|
||||
let mut w = self.writer.lock().await; |
||||
w.send(BrokerMessage::V0(BrokerMessageV0 { |
||||
padding: vec![], //FIXME implement padding
|
||||
content: BrokerMessageContentV0::BrokerOverlayMessage(BrokerOverlayMessage::V0( |
||||
BrokerOverlayMessageV0 { |
||||
overlay, |
||||
content: BrokerOverlayMessageContentV0::BrokerOverlayRequest( |
||||
BrokerOverlayRequest::V0(BrokerOverlayRequestV0 { |
||||
id: request_id, |
||||
content: request, |
||||
}), |
||||
), |
||||
}, |
||||
)), |
||||
})) |
||||
.await |
||||
.map_err(|_e| ProtocolError::WriteError)?; |
||||
|
||||
//debug_println!("waiting for first reply");
|
||||
let reply = error_receiver.await; |
||||
match reply { |
||||
Err(_e) => { |
||||
Err(ProtocolError::Closing) |
||||
} |
||||
Ok(Some(e)) => { |
||||
let mut map = self.stream_actors.write().expect("RwLock poisoned"); |
||||
map.remove(&request_id); |
||||
return Err(e); |
||||
} |
||||
Ok(None) => { |
||||
let stream_actors_in_thread = Arc::clone(&self.stream_actors); |
||||
task::spawn(async move { |
||||
addr.wait_for_stop().await; // TODO add timeout
|
||||
let mut map = stream_actors_in_thread.write().expect("RwLock poisoned"); |
||||
map.remove(&request_id); |
||||
}); |
||||
|
||||
Ok(Box::pin(receiver)) |
||||
} |
||||
} |
||||
} |
||||
|
||||
async fn process_overlay_request_objectid_response( |
||||
&mut self, |
||||
overlay: OverlayId, |
||||
request: BrokerOverlayRequestContentV0, |
||||
) -> Result<ObjectId, ProtocolError> { |
||||
before!(self, request_id, addr, receiver); |
||||
|
||||
self.writer.lock().await |
||||
.send(BrokerMessage::V0(BrokerMessageV0 { |
||||
padding: vec![], // FIXME implement padding
|
||||
content: BrokerMessageContentV0::BrokerOverlayMessage(BrokerOverlayMessage::V0( |
||||
BrokerOverlayMessageV0 { |
||||
overlay, |
||||
content: BrokerOverlayMessageContentV0::BrokerOverlayRequest( |
||||
BrokerOverlayRequest::V0(BrokerOverlayRequestV0 { |
||||
id: request_id, |
||||
content: request, |
||||
}), |
||||
), |
||||
}, |
||||
)), |
||||
})) |
||||
.await |
||||
.map_err(|_e| ProtocolError::WriteError)?; |
||||
|
||||
after!(self, request_id, addr, receiver, reply); |
||||
reply.into() |
||||
} |
||||
|
||||
async fn process_overlay_request( |
||||
&mut self, |
||||
overlay: OverlayId, |
||||
request: BrokerOverlayRequestContentV0, |
||||
) -> Result<(), ProtocolError> { |
||||
before!(self, request_id, addr, receiver); |
||||
|
||||
self.writer.lock().await |
||||
.send(BrokerMessage::V0(BrokerMessageV0 { |
||||
padding: vec![], // FIXME implement padding
|
||||
content: BrokerMessageContentV0::BrokerOverlayMessage(BrokerOverlayMessage::V0( |
||||
BrokerOverlayMessageV0 { |
||||
overlay, |
||||
content: BrokerOverlayMessageContentV0::BrokerOverlayRequest( |
||||
BrokerOverlayRequest::V0(BrokerOverlayRequestV0 { |
||||
id: request_id, |
||||
content: request, |
||||
}), |
||||
), |
||||
}, |
||||
)), |
||||
})) |
||||
.await |
||||
.map_err(|_e| ProtocolError::WriteError)?; |
||||
|
||||
after!(self, request_id, addr, receiver, reply); |
||||
reply.into() |
||||
} |
||||
|
||||
async fn add_user( |
||||
&mut self, |
||||
user_id: PubKey, |
||||
admin_user_pk: PrivKey, |
||||
) -> Result<(), ProtocolError> { |
||||
before!(self, request_id, addr, receiver); |
||||
|
||||
let op_content = AddUserContentV0 { user: user_id }; |
||||
|
||||
let sig = sign( |
||||
admin_user_pk, |
||||
self.user, |
||||
&serde_bare::to_vec(&op_content)?, |
||||
)?; |
||||
|
||||
self.writer.lock().await |
||||
.send(BrokerMessage::V0(BrokerMessageV0 { |
||||
padding: vec![], // TODO implement padding
|
||||
content: BrokerMessageContentV0::BrokerRequest(BrokerRequest::V0( |
||||
BrokerRequestV0 { |
||||
id: request_id, |
||||
content: BrokerRequestContentV0::AddUser(AddUser::V0(AddUserV0 { |
||||
content: op_content, |
||||
sig, |
||||
})), |
||||
}, |
||||
)), |
||||
})) |
||||
.await |
||||
.map_err(|_e| ProtocolError::WriteError)?; |
||||
|
||||
after!(self, request_id, addr, receiver, reply); |
||||
reply.into() |
||||
} |
||||
|
||||
async fn del_user(&mut self, user_id: PubKey, admin_user_pk: PrivKey) {} |
||||
|
||||
async fn add_client(&mut self, client_id: ClientId, user_pk: PrivKey) {} |
||||
|
||||
async fn del_client(&mut self, client_id: ClientId, user_pk: PrivKey) {} |
||||
|
||||
async fn overlay_connect( |
||||
&mut self, |
||||
repo_link: &RepoLink, |
||||
public: bool, |
||||
) -> Result<OverlayConnectionClient<BrokerConnectionRemote<T>>, ProtocolError> { |
||||
let overlay = self.process_overlay_connect(repo_link, public).await?; |
||||
|
||||
Ok(OverlayConnectionClient::create(self, overlay,repo_link.clone() )) |
||||
} |
||||
} |
||||
|
||||
#[derive(Debug)] |
||||
enum Void {} |
||||
|
||||
impl<T> BrokerConnectionRemote<T> |
||||
where |
||||
T: Sink<BrokerMessage> + Send, |
||||
{ |
||||
async fn connection_reader_loop< |
||||
U: Stream<Item = BrokerMessage> + StreamExt + Send + Sync + Unpin + 'static, |
||||
>( |
||||
stream: U, |
||||
actors: Arc<RwLock<HashMap<u64, WeakAddr<BrokerMessageActor>>>>, |
||||
stream_actors: Arc<RwLock<HashMap<u64, WeakAddr<BrokerMessageStreamActor>>>>, |
||||
shutdown: mpsc::UnboundedReceiver<Void>, |
||||
) -> Result<(), ProtocolError> { |
||||
let mut s = stream.fuse(); |
||||
let mut shutdown = shutdown.fuse(); |
||||
loop { |
||||
select! { |
||||
void = shutdown.next().fuse() => match void { |
||||
Some(void) => match void {}, |
||||
None => break, |
||||
}, |
||||
message = s.next().fuse() => match message { |
||||
Some(message) =>
|
||||
{ |
||||
//debug_println!("GOT MESSAGE {:?}", message);
|
||||
|
||||
if message.is_close() { |
||||
// releasing the blocking calls on the actors
|
||||
|
||||
let map = actors.read().expect("RwLock poisoned"); |
||||
for (a) in map.values() { |
||||
if let Some(mut addr) = a.upgrade() { |
||||
let _ = addr.stop(Some(ProtocolError::Closing.into())); |
||||
} |
||||
} |
||||
let map2 = stream_actors.read().expect("RwLock poisoned"); |
||||
for (a) in map2.values() { |
||||
if let Some(mut addr) = a.upgrade() { |
||||
let _ = addr.stop(Some(ProtocolError::Closing.into())); |
||||
} |
||||
} |
||||
return Err(ProtocolError::Closing); |
||||
} |
||||
|
||||
if message.is_request() { |
||||
debug_println!("is request {}", message.id()); |
||||
// closing connection. a client is not supposed to receive requests.
|
||||
return Err(ProtocolError::Closing); |
||||
|
||||
} else if message.is_response() { |
||||
let id = message.id(); |
||||
//debug_println!("is response for {}", id);
|
||||
{ |
||||
let map = actors.read().expect("RwLock poisoned"); |
||||
match map.get(&id) { |
||||
Some(weak_addr) => match weak_addr.upgrade() { |
||||
Some(addr) => { |
||||
addr.send(BrokerMessageXActor(message)) |
||||
.map_err(|e| ProtocolError::Closing)? |
||||
//.expect("sending message back to actor failed");
|
||||
} |
||||
None => { |
||||
debug_println!("ERROR. Addr is dead for ID {}", id); |
||||
return Err(ProtocolError::Closing); |
||||
} |
||||
}, |
||||
None => { |
||||
let map2 = stream_actors.read().expect("RwLock poisoned"); |
||||
match map2.get(&id) { |
||||
Some(weak_addr) => match weak_addr.upgrade() { |
||||
Some(addr) => { |
||||
addr.send(BrokerMessageXActor(message)) |
||||
.map_err(|e| ProtocolError::Closing)? |
||||
//.expect("sending message back to stream actor failed");
|
||||
} |
||||
None => { |
||||
debug_println!( |
||||
"ERROR. Addr is dead for ID {} {:?}", |
||||
id, |
||||
message |
||||
); |
||||
return Err(ProtocolError::Closing); |
||||
} |
||||
}, |
||||
None => { |
||||
debug_println!("Actor ID not found {} {:?}", id, message); |
||||
return Err(ProtocolError::Closing); |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
}, |
||||
None => break, |
||||
} |
||||
} |
||||
} |
||||
Ok(()) |
||||
} |
||||
|
||||
pub fn open<U: Stream<Item = BrokerMessage> + StreamExt + Send + Sync + Unpin + 'static>( |
||||
writer: T, |
||||
reader: U, |
||||
user: PubKey, |
||||
) -> BrokerConnectionRemote<T> { |
||||
let actors: Arc<RwLock<HashMap<u64, WeakAddr<BrokerMessageActor>>>> = |
||||
Arc::new(RwLock::new(HashMap::new())); |
||||
|
||||
let stream_actors: Arc<RwLock<HashMap<u64, WeakAddr<BrokerMessageStreamActor>>>> = |
||||
Arc::new(RwLock::new(HashMap::new())); |
||||
|
||||
let (shutdown_sender, shutdown_receiver) = mpsc::unbounded::<Void>(); |
||||
|
||||
let w = Arc::new(Mutex::new(Box::pin(writer))); |
||||
let ws_in_task = Arc::clone(&w); |
||||
|
||||
let actors_in_thread = Arc::clone(&actors); |
||||
let stream_actors_in_thread = Arc::clone(&stream_actors); |
||||
task::spawn(async move { |
||||
debug_println!("START of reader loop"); |
||||
if let Err(e) = |
||||
Self::connection_reader_loop(reader, actors_in_thread, stream_actors_in_thread, shutdown_receiver) |
||||
.await |
||||
{ |
||||
debug_println!("closing because of {}", e); |
||||
let _ = ws_in_task.lock().await.close().await; |
||||
} |
||||
debug_println!("END of reader loop"); |
||||
}); |
||||
|
||||
BrokerConnectionRemote::<T> { |
||||
writer: Arc::clone(&w), |
||||
user, |
||||
actors: Arc::clone(&actors), |
||||
stream_actors: Arc::clone(&stream_actors), |
||||
shutdown:shutdown_sender , |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,95 @@ |
||||
/* |
||||
* Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers |
||||
* All rights reserved. |
||||
* Licensed under the Apache License, Version 2.0 |
||||
* <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
|
||||
* or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
|
||||
* at your option. All files in the project carrying such |
||||
* notice may not be copied, modified, or distributed except |
||||
* according to those terms. |
||||
*/ |
||||
|
||||
use debug_print::*; |
||||
|
||||
use p2p_repo::types::*; |
||||
use p2p_repo::utils::{generate_keypair, now_timestamp}; |
||||
use p2p_net::errors::*; |
||||
use p2p_net::types::*; |
||||
use p2p_net::broker_connection::*; |
||||
use crate::connection_remote::*; |
||||
use futures::{future, pin_mut, stream, SinkExt, StreamExt}; |
||||
|
||||
use async_tungstenite::async_std::connect_async; |
||||
use async_tungstenite::client_async; |
||||
use async_tungstenite::tungstenite::{Error, Message}; |
||||
|
||||
pub struct BrokerConnectionWebSocket { |
||||
|
||||
} |
||||
|
||||
impl BrokerConnectionWebSocket{ |
||||
|
||||
pub async fn open(url:&str, priv_key: PrivKey, pub_key: PubKey) -> Result<impl BrokerConnection, ProtocolError>
|
||||
{ |
||||
|
||||
let res = connect_async(url).await; |
||||
|
||||
match (res) { |
||||
Ok((ws, _)) => { |
||||
debug_println!("WebSocket handshake completed"); |
||||
|
||||
let (write, read) = ws.split(); |
||||
let mut frames_stream_read = read.map(|msg_res| match msg_res { |
||||
Err(e) => { |
||||
debug_println!("ERROR {:?}", e); |
||||
vec![] |
||||
} |
||||
Ok(message) => { |
||||
if message.is_close() { |
||||
debug_println!("CLOSE FROM SERVER"); |
||||
vec![] |
||||
} else { |
||||
message.into_data() |
||||
} |
||||
} |
||||
}); |
||||
async fn transform(message: Vec<u8>) -> Result<Message, Error> { |
||||
if message.len() == 0 { |
||||
debug_println!("sending CLOSE message to SERVER"); |
||||
Ok(Message::Close(None)) |
||||
} else { |
||||
Ok(Message::binary(message)) |
||||
} |
||||
} |
||||
let frames_stream_write = write |
||||
.with(|message| transform(message)) |
||||
.sink_map_err(|e| ProtocolError::WriteError); |
||||
|
||||
let master_key: [u8; 32] = [0; 32]; |
||||
let mut cnx_res = ConnectionRemote::open_broker_connection( |
||||
frames_stream_write, |
||||
frames_stream_read, |
||||
pub_key, |
||||
priv_key, |
||||
PubKey::Ed25519PubKey([1; 32]), |
||||
) |
||||
.await; |
||||
|
||||
match cnx_res { |
||||
Ok(mut cnx) => { |
||||
Ok(cnx) |
||||
} |
||||
Err(e) => { |
||||
debug_println!("cannot connect {:?}", e); |
||||
Err(e) |
||||
} |
||||
} |
||||
} |
||||
Err(e) => { |
||||
debug_println!("Cannot connect: {:?}", e); |
||||
Err(ProtocolError::ConnectionError) |
||||
} |
||||
} |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,47 @@ |
||||
// All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0
|
||||
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
|
||||
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
|
||||
// at your option. All files in the project carrying such
|
||||
// notice may not be copied, modified, or distributed except
|
||||
// according to those terms.
|
||||
|
||||
#[macro_export] |
||||
macro_rules! before { |
||||
( $self:expr, $request_id:ident, $addr:ident, $receiver:ident ) => { |
||||
let mut actor = BrokerMessageActor::new(); |
||||
let $receiver = actor.receiver(); |
||||
let mut $addr = actor |
||||
.start() |
||||
.await |
||||
.map_err(|_e| ProtocolError::ActorError)?; |
||||
|
||||
let $request_id = $addr.actor_id(); |
||||
//debug_println!("actor ID {}", $request_id);
|
||||
|
||||
{ |
||||
let mut map = $self.actors.write().expect("RwLock poisoned"); |
||||
map.insert($request_id, $addr.downgrade()); |
||||
} |
||||
}; |
||||
} |
||||
|
||||
macro_rules! after { |
||||
( $self:expr, $request_id:ident, $addr:ident, $receiver:ident, $reply:ident ) => { |
||||
//debug_println!("waiting for reply");
|
||||
|
||||
$addr.wait_for_stop().await; // TODO add timeout and close connection if there's no reply
|
||||
let r = $receiver.await; |
||||
if r.is_err() { return Err(ProtocolError::Closing);} |
||||
let $reply = r.unwrap(); |
||||
//debug_println!("reply arrived {:?}", $reply);
|
||||
{ |
||||
let mut map = $self.actors.write().expect("RwLock poisoned"); |
||||
map.remove(&$request_id); |
||||
} |
||||
}; |
||||
} |
||||
|
||||
pub mod connection_remote; |
||||
|
||||
pub mod connection_ws; |
||||
@ -0,0 +1,20 @@ |
||||
[package] |
||||
name = "p2p-net" |
||||
version = "0.1.0" |
||||
edition = "2021" |
||||
license = "MIT/Apache-2.0" |
||||
authors = ["Niko PLP <niko@nextgraph.org>"] |
||||
description = "P2P network module of NextGraph" |
||||
repository = "https://git.nextgraph.org/NextGraph/nextgraph-rs" |
||||
|
||||
[dependencies] |
||||
debug_print = "1.0.0" |
||||
p2p-repo = { path = "../p2p-repo" } |
||||
serde = { version = "1.0", features = ["derive"] } |
||||
serde_bare = "0.5.0" |
||||
serde_bytes = "0.11.7" |
||||
num_enum = "0.5.7" |
||||
async-broadcast = "0.4.1" |
||||
futures = "0.3.24" |
||||
async-trait = "0.1.57" |
||||
blake3 = "1.3.1" |
||||
@ -0,0 +1,337 @@ |
||||
/* |
||||
* Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers |
||||
* All rights reserved. |
||||
* Licensed under the Apache License, Version 2.0 |
||||
* <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
|
||||
* or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
|
||||
* at your option. All files in the project carrying such |
||||
* notice may not be copied, modified, or distributed except |
||||
* according to those terms. |
||||
*/ |
||||
|
||||
//! Connection to a Broker, can be local or remote.
|
||||
//! If remote, it will use a Stream and Sink of framed messages
|
||||
//! This is the trait
|
||||
//!
|
||||
|
||||
use futures::{ |
||||
ready, |
||||
stream::Stream, |
||||
task::{Context, Poll}, |
||||
Future, |
||||
select, FutureExt, |
||||
}; |
||||
use futures::channel::mpsc; |
||||
use std::pin::Pin; |
||||
use std::{collections::HashSet, fmt::Debug}; |
||||
|
||||
use async_broadcast::{broadcast, Receiver}; |
||||
use debug_print::*; |
||||
use futures::{pin_mut, stream, Sink, SinkExt, StreamExt}; |
||||
use p2p_repo::object::*; |
||||
use p2p_repo::store::*; |
||||
use p2p_repo::types::*; |
||||
use p2p_repo::utils::*; |
||||
use crate::errors::*; |
||||
use crate::types::*; |
||||
|
||||
|
||||
#[async_trait::async_trait] |
||||
pub trait BrokerConnection { |
||||
type OC: BrokerConnection; |
||||
type BlockStream: Stream<Item = Block>; |
||||
|
||||
async fn close(&mut self); |
||||
|
||||
async fn add_user( |
||||
&mut self, |
||||
user_id: PubKey, |
||||
admin_user_pk: PrivKey, |
||||
) -> Result<(), ProtocolError>; |
||||
|
||||
async fn del_user(&mut self, user_id: PubKey, admin_user_pk: PrivKey); |
||||
|
||||
async fn add_client(&mut self, client_id: ClientId, user_pk: PrivKey); |
||||
|
||||
async fn del_client(&mut self, client_id: ClientId, user_pk: PrivKey); |
||||
|
||||
async fn overlay_connect( |
||||
&mut self, |
||||
repo: &RepoLink, |
||||
public: bool, |
||||
) -> Result<OverlayConnectionClient<Self::OC>, ProtocolError>; |
||||
|
||||
// TODO: remove those 4 functions from trait. they are used internally only. should not be exposed to end-user
|
||||
async fn process_overlay_request( |
||||
&mut self, |
||||
overlay: OverlayId, |
||||
request: BrokerOverlayRequestContentV0, |
||||
) -> Result<(), ProtocolError>; |
||||
|
||||
async fn process_overlay_request_stream_response( |
||||
&mut self, |
||||
overlay: OverlayId, |
||||
request: BrokerOverlayRequestContentV0, |
||||
) -> Result<Pin<Box<Self::BlockStream>>, ProtocolError>; |
||||
|
||||
async fn process_overlay_request_objectid_response( |
||||
&mut self, |
||||
overlay: OverlayId, |
||||
request: BrokerOverlayRequestContentV0, |
||||
) -> Result<ObjectId, ProtocolError>; |
||||
|
||||
async fn process_overlay_connect( |
||||
&mut self, |
||||
repo_link: &RepoLink, |
||||
public: bool, |
||||
) -> Result<OverlayId, ProtocolError> { |
||||
let overlay: OverlayId = match public { |
||||
true => Digest::Blake3Digest32(*blake3::hash(repo_link.id().slice()).as_bytes()), |
||||
false => { |
||||
let key: [u8; blake3::OUT_LEN] = |
||||
blake3::derive_key("NextGraph OverlayId BLAKE3 key", repo_link.secret().slice()); |
||||
let keyed_hash = blake3::keyed_hash(&key, repo_link.id().slice()); |
||||
Digest::Blake3Digest32(*keyed_hash.as_bytes()) |
||||
} |
||||
}; |
||||
|
||||
let res = self |
||||
.process_overlay_request( |
||||
overlay, |
||||
BrokerOverlayRequestContentV0::OverlayConnect(OverlayConnect::V0()), |
||||
) |
||||
.await; |
||||
|
||||
match res { |
||||
Err(e) => { |
||||
if e == ProtocolError::OverlayNotJoined { |
||||
debug_println!("OverlayNotJoined"); |
||||
let res2 = self |
||||
.process_overlay_request( |
||||
overlay, |
||||
BrokerOverlayRequestContentV0::OverlayJoin(OverlayJoin::V0( |
||||
OverlayJoinV0 { |
||||
secret: repo_link.secret(), |
||||
peers: repo_link.peers(), |
||||
repo_pubkey: Some(repo_link.id()), //TODO if we know we are connecting to a core node, we can pass None here
|
||||
}, |
||||
)), |
||||
) |
||||
.await?; |
||||
} else { |
||||
return Err(e); |
||||
} |
||||
} |
||||
Ok(()) => {} |
||||
} |
||||
|
||||
debug_println!("OverlayConnectionClient ready"); |
||||
Ok(overlay) |
||||
} |
||||
} |
||||
|
||||
pub struct OverlayConnectionClient<'a, T> |
||||
where |
||||
T: BrokerConnection, |
||||
{ |
||||
broker: &'a mut T, |
||||
overlay: OverlayId, |
||||
repo_link: RepoLink, |
||||
} |
||||
|
||||
impl<'a, T> OverlayConnectionClient<'a, T> |
||||
where |
||||
T: BrokerConnection, |
||||
{ |
||||
pub fn create( broker: &'a mut T, overlay: OverlayId, repo_link: RepoLink) -> OverlayConnectionClient<'a, T> { |
||||
OverlayConnectionClient { |
||||
broker, |
||||
repo_link, |
||||
overlay, |
||||
} |
||||
} |
||||
|
||||
pub fn overlay(repo_link: &RepoLink, public: bool) -> OverlayId { |
||||
let overlay: OverlayId = match public { |
||||
true => Digest::Blake3Digest32(*blake3::hash(repo_link.id().slice()).as_bytes()), |
||||
false => { |
||||
let key: [u8; blake3::OUT_LEN] = |
||||
blake3::derive_key("NextGraph OverlayId BLAKE3 key", repo_link.secret().slice()); |
||||
let keyed_hash = blake3::keyed_hash(&key, repo_link.id().slice()); |
||||
Digest::Blake3Digest32(*keyed_hash.as_bytes()) |
||||
} |
||||
}; |
||||
overlay |
||||
} |
||||
|
||||
pub async fn sync_branch( |
||||
&mut self, |
||||
heads: Vec<ObjectId>, |
||||
known_heads: Vec<ObjectId>, |
||||
known_commits: BloomFilter, |
||||
) -> Result<Pin<Box<T::BlockStream>>, ProtocolError> { |
||||
self.broker |
||||
.process_overlay_request_stream_response( |
||||
self.overlay, |
||||
BrokerOverlayRequestContentV0::BranchSyncReq(BranchSyncReq::V0(BranchSyncReqV0 { |
||||
heads, |
||||
known_heads, |
||||
known_commits, |
||||
})), |
||||
) |
||||
.await |
||||
} |
||||
|
||||
pub fn leave(&self) {} |
||||
|
||||
pub fn topic_connect(&self, id: TopicId) -> TopicSubscription<T> { |
||||
let (s, mut r1) = broadcast(128); // FIXME this should be done only once, in the Broker
|
||||
TopicSubscription { |
||||
id, |
||||
overlay_cnx: self, |
||||
event_stream: r1.clone(), |
||||
} |
||||
} |
||||
|
||||
pub async fn delete_object(&mut self, id: ObjectId) -> Result<(), ProtocolError> { |
||||
self.broker |
||||
.process_overlay_request( |
||||
self.overlay, |
||||
BrokerOverlayRequestContentV0::ObjectDel(ObjectDel::V0(ObjectDelV0 { id })), |
||||
) |
||||
.await |
||||
} |
||||
|
||||
pub async fn pin_object(&mut self, id: ObjectId) -> Result<(), ProtocolError> { |
||||
self.broker |
||||
.process_overlay_request( |
||||
self.overlay, |
||||
BrokerOverlayRequestContentV0::ObjectPin(ObjectPin::V0(ObjectPinV0 { id })), |
||||
) |
||||
.await |
||||
} |
||||
|
||||
pub async fn unpin_object(&mut self, id: ObjectId) -> Result<(), ProtocolError> { |
||||
self.broker |
||||
.process_overlay_request( |
||||
self.overlay, |
||||
BrokerOverlayRequestContentV0::ObjectUnpin(ObjectUnpin::V0(ObjectUnpinV0 { id })), |
||||
) |
||||
.await |
||||
} |
||||
|
||||
pub async fn copy_object( |
||||
&mut self, |
||||
id: ObjectId, |
||||
expiry: Option<Timestamp>, |
||||
) -> Result<ObjectId, ProtocolError> { |
||||
self.broker |
||||
.process_overlay_request_objectid_response( |
||||
self.overlay, |
||||
BrokerOverlayRequestContentV0::ObjectCopy(ObjectCopy::V0(ObjectCopyV0 { |
||||
id, |
||||
expiry, |
||||
})), |
||||
) |
||||
.await |
||||
} |
||||
|
||||
pub async fn get_block( |
||||
&mut self, |
||||
id: BlockId, |
||||
include_children: bool, |
||||
topic: Option<PubKey>, |
||||
) -> Result<Pin<Box<T::BlockStream>>, ProtocolError> { |
||||
self.broker |
||||
.process_overlay_request_stream_response( |
||||
self.overlay, |
||||
BrokerOverlayRequestContentV0::BlockGet(BlockGet::V0(BlockGetV0 { |
||||
id, |
||||
include_children, |
||||
topic, |
||||
})), |
||||
) |
||||
.await |
||||
} |
||||
|
||||
pub async fn get_object( |
||||
&mut self, |
||||
id: ObjectId, |
||||
topic: Option<PubKey>, |
||||
) -> Result<Object, ProtocolError> { |
||||
let mut blockstream = self.get_block(id, true, topic).await?; |
||||
let mut store = HashMapRepoStore::new(); |
||||
while let Some(block) = blockstream.next().await { |
||||
store.put(&block).unwrap(); |
||||
} |
||||
Object::load(id, None, &store).map_err(|e| match e { |
||||
ObjectParseError::MissingBlocks(_missing) => ProtocolError::MissingBlocks, |
||||
_ => ProtocolError::ObjectParseError, |
||||
}) |
||||
} |
||||
|
||||
pub async fn put_block(&mut self, block: &Block) -> Result<BlockId, ProtocolError> { |
||||
self.broker |
||||
.process_overlay_request( |
||||
self.overlay, |
||||
BrokerOverlayRequestContentV0::BlockPut(BlockPut::V0(block.clone())), |
||||
) |
||||
.await?; |
||||
Ok(block.id()) |
||||
} |
||||
|
||||
// TODO maybe implement a put_block_with_children ? that would behave like put_object, but taking in a parent Blockk instead of a content
|
||||
|
||||
pub async fn put_object( |
||||
&mut self, |
||||
content: ObjectContent, |
||||
deps: Vec<ObjectId>, |
||||
expiry: Option<Timestamp>, |
||||
max_object_size: usize, |
||||
repo_pubkey: PubKey, |
||||
repo_secret: SymKey, |
||||
) -> Result<ObjectId, ProtocolError> { |
||||
let obj = Object::new( |
||||
content, |
||||
deps, |
||||
expiry, |
||||
max_object_size, |
||||
repo_pubkey, |
||||
repo_secret, |
||||
); |
||||
debug_println!("object has {} blocks", obj.blocks().len()); |
||||
let mut deduplicated: HashSet<ObjectId> = HashSet::new(); |
||||
for block in obj.blocks() { |
||||
let id = block.id(); |
||||
if deduplicated.get(&id).is_none() { |
||||
let _ = self.put_block(block).await?; |
||||
deduplicated.insert(id); |
||||
} |
||||
} |
||||
Ok(obj.id()) |
||||
} |
||||
} |
||||
|
||||
pub struct TopicSubscription<'a, T> |
||||
where |
||||
T: BrokerConnection, |
||||
{ |
||||
id: TopicId, |
||||
overlay_cnx: &'a OverlayConnectionClient<'a, T>, |
||||
event_stream: Receiver<Event>, |
||||
} |
||||
|
||||
impl<'a, T> TopicSubscription<'a, T> |
||||
where |
||||
T: BrokerConnection, |
||||
{ |
||||
pub fn unsubscribe(&self) {} |
||||
|
||||
pub fn disconnect(&self) {} |
||||
|
||||
pub fn get_branch_heads(&self) {} |
||||
|
||||
pub fn get_event_stream(&self) -> &Receiver<Event> { |
||||
&self.event_stream |
||||
} |
||||
} |
||||
@ -0,0 +1,162 @@ |
||||
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers
|
||||
// All rights reserved.
|
||||
// This code is partly derived from work written by TG x Thoth from P2Pcollab.
|
||||
// Copyright 2022 TG x Thoth
|
||||
// Licensed under the Apache License, Version 2.0
|
||||
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
|
||||
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
|
||||
// at your option. All files in the project carrying such
|
||||
// notice may not be copied, modified, or distributed except
|
||||
// according to those terms.
|
||||
|
||||
use crate::types::BrokerMessage; |
||||
use core::fmt; |
||||
use p2p_repo::object::ObjectParseError; |
||||
use p2p_repo::types::Block; |
||||
use p2p_repo::types::ObjectId; |
||||
use num_enum::IntoPrimitive; |
||||
use num_enum::TryFromPrimitive; |
||||
use std::convert::From; |
||||
use std::convert::TryFrom; |
||||
use std::error::Error; |
||||
|
||||
#[derive(Debug, Eq, PartialEq, TryFromPrimitive, IntoPrimitive, Clone)] |
||||
#[repr(u16)] |
||||
pub enum ProtocolError { |
||||
WriteError = 1, |
||||
ActorError, |
||||
InvalidState, |
||||
SignatureError, |
||||
InvalidSignature, |
||||
SerializationError, |
||||
PartialContent, |
||||
AccessDenied, |
||||
OverlayNotJoined, |
||||
OverlayNotFound, |
||||
BrokerError, |
||||
NotFound, |
||||
EndOfStream, |
||||
StoreError, |
||||
MissingBlocks, |
||||
ObjectParseError, |
||||
InvalidValue, |
||||
UserAlreadyExists, |
||||
RepoIdRequired, |
||||
Closing, |
||||
ConnectionError, |
||||
} |
||||
|
||||
impl ProtocolError { |
||||
pub fn is_stream(&self) -> bool { |
||||
*self == ProtocolError::PartialContent || *self == ProtocolError::EndOfStream |
||||
} |
||||
} |
||||
|
||||
impl Error for ProtocolError {} |
||||
|
||||
impl fmt::Display for ProtocolError { |
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { |
||||
write!(f, "{:?}", self) |
||||
} |
||||
} |
||||
|
||||
impl From<p2p_repo::errors::NgError> for ProtocolError { |
||||
fn from(e: p2p_repo::errors::NgError) -> Self { |
||||
match e { |
||||
p2p_repo::errors::NgError::InvalidSignature => ProtocolError::InvalidSignature, |
||||
p2p_repo::errors::NgError::SerializationError => ProtocolError::SerializationError, |
||||
} |
||||
} |
||||
} |
||||
|
||||
impl From<ObjectParseError> for ProtocolError { |
||||
fn from(e: ObjectParseError) -> Self { |
||||
ProtocolError::ObjectParseError |
||||
} |
||||
} |
||||
|
||||
impl From<p2p_repo::store::StorageError> for ProtocolError { |
||||
fn from(e: p2p_repo::store::StorageError) -> Self { |
||||
match e { |
||||
p2p_repo::store::StorageError::NotFound => ProtocolError::NotFound, |
||||
p2p_repo::store::StorageError::InvalidValue => ProtocolError::InvalidValue, |
||||
_ => ProtocolError::StoreError, |
||||
} |
||||
} |
||||
} |
||||
|
||||
impl From<serde_bare::error::Error> for ProtocolError { |
||||
fn from(e: serde_bare::error::Error) -> Self { |
||||
ProtocolError::SerializationError |
||||
} |
||||
} |
||||
|
||||
impl From<BrokerMessage> for Result<(), ProtocolError> { |
||||
fn from(msg: BrokerMessage) -> Self { |
||||
if !msg.is_response() { |
||||
panic!("BrokerMessage is not a response"); |
||||
} |
||||
match msg.result() { |
||||
0 => Ok(()), |
||||
err => Err(ProtocolError::try_from(err).unwrap()), |
||||
} |
||||
} |
||||
} |
||||
|
||||
impl From<BrokerMessage> for Result<ObjectId, ProtocolError> { |
||||
fn from(msg: BrokerMessage) -> Self { |
||||
if !msg.is_response() { |
||||
panic!("BrokerMessage is not a response"); |
||||
} |
||||
match msg.result() { |
||||
0 => Ok(msg.response_object_id()), |
||||
err => Err(ProtocolError::try_from(err).unwrap()), |
||||
} |
||||
} |
||||
} |
||||
|
||||
/// Option represents if a Block is available. cannot be returned here. call BrokerMessage.response_block() to get a reference to it.
|
||||
impl From<BrokerMessage> for Result<Option<u16>, ProtocolError> { |
||||
fn from(msg: BrokerMessage) -> Self { |
||||
if !msg.is_response() { |
||||
panic!("BrokerMessage is not a response"); |
||||
} |
||||
//let partial: u16 = ProtocolError::PartialContent.into();
|
||||
let res = msg.result(); |
||||
if res == 0 || ProtocolError::try_from(res).unwrap().is_stream() { |
||||
if msg.is_overlay() { |
||||
match msg.response_block() { |
||||
Some(_) => Ok(Some(res)), |
||||
None => Ok(None), |
||||
} |
||||
} else { |
||||
Ok(None) |
||||
} |
||||
} else { |
||||
Err(ProtocolError::try_from(res).unwrap()) |
||||
} |
||||
} |
||||
} |
||||
|
||||
/// Option represents if a Block is available. returns a clone.
|
||||
impl From<BrokerMessage> for Result<Option<Block>, ProtocolError> { |
||||
fn from(msg: BrokerMessage) -> Self { |
||||
if !msg.is_response() { |
||||
panic!("BrokerMessage is not a response"); |
||||
} |
||||
//let partial: u16 = ProtocolError::PartialContent.into();
|
||||
let res = msg.result(); |
||||
if res == 0 || ProtocolError::try_from(res).unwrap().is_stream() { |
||||
if msg.is_overlay() { |
||||
match msg.response_block() { |
||||
Some(b) => Ok(Some(b.clone())), |
||||
None => Ok(None), |
||||
} |
||||
} else { |
||||
Ok(None) |
||||
} |
||||
} else { |
||||
Err(ProtocolError::try_from(res).unwrap()) |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,5 @@ |
||||
pub mod types; |
||||
|
||||
pub mod errors; |
||||
|
||||
pub mod broker_connection; |
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,20 @@ |
||||
[package] |
||||
name = "p2p-repo" |
||||
version = "0.1.0" |
||||
edition = "2021" |
||||
license = "MIT/Apache-2.0" |
||||
authors = ["Niko PLP <niko@nextgraph.org>"] |
||||
description = "P2P repository module of NextGraph" |
||||
repository = "https://git.nextgraph.org/NextGraph/nextgraph-rs" |
||||
|
||||
[dependencies] |
||||
blake3 = "1.3.1" |
||||
chacha20 = "0.9.0" |
||||
ed25519-dalek = "1.0.1" |
||||
rand = "0.7" |
||||
serde = { version = "1.0.142", features = ["derive"] } |
||||
serde_bare = "0.5.0" |
||||
serde_bytes = "0.11.7" |
||||
fastbloom-rs = "0.3.1" |
||||
debug_print = "1.0.0" |
||||
hex = "0.4.3" |
||||
@ -0,0 +1,116 @@ |
||||
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers
|
||||
// All rights reserved.
|
||||
// This code is partly derived from work written by TG x Thoth from P2Pcollab.
|
||||
// Copyright 2022 TG x Thoth
|
||||
// Licensed under the Apache License, Version 2.0
|
||||
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
|
||||
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
|
||||
// at your option. All files in the project carrying such
|
||||
// notice may not be copied, modified, or distributed except
|
||||
// according to those terms.
|
||||
|
||||
//! Immutable Block
|
||||
|
||||
use crate::types::*; |
||||
|
||||
impl BlockV0 { |
||||
pub fn new( |
||||
children: Vec<BlockId>, |
||||
deps: ObjectDeps, |
||||
expiry: Option<Timestamp>, |
||||
content: Vec<u8>, |
||||
key: Option<SymKey>, |
||||
) -> BlockV0 { |
||||
let mut b = BlockV0 { |
||||
id: None, |
||||
key, |
||||
children, |
||||
deps, |
||||
expiry, |
||||
content, |
||||
}; |
||||
let block = Block::V0(b.clone()); |
||||
b.id = Some(block.get_id()); |
||||
b |
||||
} |
||||
} |
||||
|
||||
impl Block { |
||||
pub fn new( |
||||
children: Vec<BlockId>, |
||||
deps: ObjectDeps, |
||||
expiry: Option<Timestamp>, |
||||
content: Vec<u8>, |
||||
key: Option<SymKey>, |
||||
) -> Block { |
||||
Block::V0(BlockV0::new(children, deps, expiry, content, key)) |
||||
} |
||||
|
||||
/// Compute the ID
|
||||
pub fn get_id(&self) -> BlockId { |
||||
let ser = serde_bare::to_vec(self).unwrap(); |
||||
let hash = blake3::hash(ser.as_slice()); |
||||
Digest::Blake3Digest32(hash.as_bytes().clone()) |
||||
} |
||||
|
||||
/// Get the already computed ID
|
||||
pub fn id(&self) -> BlockId { |
||||
match self { |
||||
Block::V0(b) => match b.id { |
||||
Some(id) => id, |
||||
None => self.get_id(), |
||||
}, |
||||
} |
||||
} |
||||
|
||||
/// Get the content
|
||||
pub fn content(&self) -> &Vec<u8> { |
||||
match self { |
||||
Block::V0(b) => &b.content, |
||||
} |
||||
} |
||||
|
||||
/// Get the children
|
||||
pub fn children(&self) -> &Vec<BlockId> { |
||||
match self { |
||||
Block::V0(b) => &b.children, |
||||
} |
||||
} |
||||
|
||||
/// Get the dependencies
|
||||
pub fn deps(&self) -> &ObjectDeps { |
||||
match self { |
||||
Block::V0(b) => &b.deps, |
||||
} |
||||
} |
||||
|
||||
/// Get the expiry
|
||||
pub fn expiry(&self) -> Option<Timestamp> { |
||||
match self { |
||||
Block::V0(b) => b.expiry, |
||||
} |
||||
} |
||||
|
||||
pub fn set_expiry(&mut self, expiry: Option<Timestamp>) { |
||||
match self { |
||||
Block::V0(b) => { |
||||
b.id = None; |
||||
b.expiry = expiry |
||||
} |
||||
} |
||||
} |
||||
|
||||
/// Get the key
|
||||
pub fn key(&self) -> Option<SymKey> { |
||||
match self { |
||||
Block::V0(b) => b.key, |
||||
} |
||||
} |
||||
|
||||
/// Set the key
|
||||
pub fn set_key(&mut self, key: Option<SymKey>) { |
||||
match self { |
||||
Block::V0(b) => b.key = key, |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,581 @@ |
||||
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers
|
||||
// All rights reserved.
|
||||
// This code is partly derived from work written by TG x Thoth from P2Pcollab.
|
||||
// Copyright 2022 TG x Thoth
|
||||
// Licensed under the Apache License, Version 2.0
|
||||
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
|
||||
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
|
||||
// at your option. All files in the project carrying such
|
||||
// notice may not be copied, modified, or distributed except
|
||||
// according to those terms.
|
||||
|
||||
//! Branch of a Repository
|
||||
|
||||
use debug_print::*; |
||||
use std::collections::{HashMap, HashSet}; |
||||
|
||||
use fastbloom_rs::{BloomFilter as Filter, Membership}; |
||||
|
||||
use crate::object::*; |
||||
use crate::store::*; |
||||
use crate::types::*; |
||||
|
||||
impl MemberV0 { |
||||
/// New member
|
||||
pub fn new(id: PubKey, commit_types: Vec<CommitType>, metadata: Vec<u8>) -> MemberV0 { |
||||
MemberV0 { |
||||
id, |
||||
commit_types, |
||||
metadata, |
||||
} |
||||
} |
||||
|
||||
/// Check whether this member has permission for the given commit type
|
||||
pub fn has_perm(&self, commit_type: CommitType) -> bool { |
||||
self.commit_types.contains(&commit_type) |
||||
} |
||||
} |
||||
|
||||
impl Member { |
||||
/// New member
|
||||
pub fn new(id: PubKey, commit_types: Vec<CommitType>, metadata: Vec<u8>) -> Member { |
||||
Member::V0(MemberV0::new(id, commit_types, metadata)) |
||||
} |
||||
|
||||
/// Check whether this member has permission for the given commit type
|
||||
pub fn has_perm(&self, commit_type: CommitType) -> bool { |
||||
match self { |
||||
Member::V0(m) => m.has_perm(commit_type), |
||||
} |
||||
} |
||||
} |
||||
|
||||
impl BranchV0 { |
||||
pub fn new( |
||||
id: PubKey, |
||||
topic: PubKey, |
||||
secret: SymKey, |
||||
members: Vec<MemberV0>, |
||||
quorum: HashMap<CommitType, u32>, |
||||
ack_delay: RelTime, |
||||
tags: Vec<u8>, |
||||
metadata: Vec<u8>, |
||||
) -> BranchV0 { |
||||
BranchV0 { |
||||
id, |
||||
topic, |
||||
secret, |
||||
members, |
||||
quorum, |
||||
ack_delay, |
||||
tags, |
||||
metadata, |
||||
} |
||||
} |
||||
} |
||||
|
||||
impl Branch { |
||||
pub fn new( |
||||
id: PubKey, |
||||
topic: PubKey, |
||||
secret: SymKey, |
||||
members: Vec<MemberV0>, |
||||
quorum: HashMap<CommitType, u32>, |
||||
ack_delay: RelTime, |
||||
tags: Vec<u8>, |
||||
metadata: Vec<u8>, |
||||
) -> Branch { |
||||
Branch::V0(BranchV0::new( |
||||
id, topic, secret, members, quorum, ack_delay, tags, metadata, |
||||
)) |
||||
} |
||||
|
||||
/// Get member by ID
|
||||
pub fn get_member(&self, id: &PubKey) -> Option<&MemberV0> { |
||||
match self { |
||||
Branch::V0(b) => { |
||||
for m in b.members.iter() { |
||||
if m.id == *id { |
||||
return Some(m); |
||||
} |
||||
} |
||||
} |
||||
} |
||||
None |
||||
} |
||||
|
||||
/// Branch sync request from another peer
|
||||
///
|
||||
/// Return ObjectIds to send
|
||||
pub fn sync_req( |
||||
our_heads: &[ObjectId], |
||||
their_heads: &[ObjectId], |
||||
their_filter: &BloomFilter, |
||||
store: &impl RepoStore, |
||||
) -> Result<Vec<ObjectId>, ObjectParseError> { |
||||
//debug_println!(">> sync_req");
|
||||
//debug_println!(" our_heads: {:?}", our_heads);
|
||||
//debug_println!(" their_heads: {:?}", their_heads);
|
||||
|
||||
/// Load `Commit` `Object`s of a `Branch` from the `RepoStore` starting from the given `Object`,
|
||||
/// and collect `ObjectId`s starting from `our_heads` towards `their_heads`
|
||||
fn load_branch( |
||||
cobj: &Object, |
||||
store: &impl RepoStore, |
||||
their_heads: &[ObjectId], |
||||
visited: &mut HashSet<ObjectId>, |
||||
missing: &mut HashSet<ObjectId>, |
||||
) -> Result<bool, ObjectParseError> { |
||||
//debug_println!(">>> load_branch: {}", cobj.id());
|
||||
let id = cobj.id(); |
||||
|
||||
// root has no deps
|
||||
let is_root = cobj.deps().len() == 0; |
||||
//debug_println!(" deps: {:?}", cobj.deps());
|
||||
|
||||
// check if this commit object is present in their_heads
|
||||
let mut their_head_found = their_heads.contains(&id); |
||||
|
||||
// load deps, stop at the root or if this is a commit object from their_heads
|
||||
if !is_root && !their_head_found { |
||||
visited.insert(id); |
||||
for id in cobj.deps() { |
||||
match Object::load(*id, None, store) { |
||||
Ok(o) => { |
||||
if !visited.contains(id) { |
||||
if load_branch(&o, store, their_heads, visited, missing)? { |
||||
their_head_found = true; |
||||
} |
||||
} |
||||
} |
||||
Err(ObjectParseError::MissingBlocks(m)) => { |
||||
missing.extend(m); |
||||
} |
||||
Err(e) => return Err(e), |
||||
} |
||||
} |
||||
} |
||||
Ok(their_head_found) |
||||
} |
||||
|
||||
// missing commits from our branch
|
||||
let mut missing = HashSet::new(); |
||||
// our commits
|
||||
let mut ours = HashSet::new(); |
||||
// their commits
|
||||
let mut theirs = HashSet::new(); |
||||
|
||||
// collect all commits reachable from our_heads
|
||||
// up to the root or until encountering a commit from their_heads
|
||||
for id in our_heads { |
||||
let cobj = Object::load(*id, None, store)?; |
||||
let mut visited = HashSet::new(); |
||||
let their_head_found = |
||||
load_branch(&cobj, store, their_heads, &mut visited, &mut missing)?; |
||||
//debug_println!("<<< load_branch: {}", their_head_found);
|
||||
ours.extend(visited); // add if one of their_heads found
|
||||
} |
||||
|
||||
// collect all commits reachable from their_heads
|
||||
for id in their_heads { |
||||
let cobj = Object::load(*id, None, store)?; |
||||
let mut visited = HashSet::new(); |
||||
let their_head_found = load_branch(&cobj, store, &[], &mut visited, &mut missing)?; |
||||
//debug_println!("<<< load_branch: {}", their_head_found);
|
||||
theirs.extend(visited); // add if one of their_heads found
|
||||
} |
||||
|
||||
let mut result = &ours - &theirs; |
||||
|
||||
//debug_println!("!! ours: {:?}", ours);
|
||||
//debug_println!("!! theirs: {:?}", theirs);
|
||||
//debug_println!("!! result: {:?}", result);
|
||||
|
||||
// remove their_commits from result
|
||||
let filter = Filter::from_u8_array(their_filter.f.as_slice(), their_filter.k.into()); |
||||
for id in result.clone() { |
||||
match id { |
||||
Digest::Blake3Digest32(d) => { |
||||
if filter.contains(&d) { |
||||
result.remove(&id); |
||||
} |
||||
} |
||||
} |
||||
} |
||||
//debug_println!("!! result filtered: {:?}", result);
|
||||
Ok(Vec::from_iter(result)) |
||||
} |
||||
} |
||||
|
||||
mod test { |
||||
use std::collections::HashMap; |
||||
|
||||
use ed25519_dalek::*; |
||||
use fastbloom_rs::{BloomFilter as Filter, FilterBuilder, Membership}; |
||||
use rand::rngs::OsRng; |
||||
|
||||
use crate::branch::*; |
||||
use crate::commit::*; |
||||
use crate::object::*; |
||||
use crate::repo; |
||||
use crate::store::*; |
||||
|
||||
#[test] |
||||
pub fn test_branch() { |
||||
fn add_obj( |
||||
content: ObjectContent, |
||||
deps: Vec<ObjectId>, |
||||
expiry: Option<Timestamp>, |
||||
repo_pubkey: PubKey, |
||||
repo_secret: SymKey, |
||||
store: &mut impl RepoStore, |
||||
) -> ObjectRef { |
||||
let max_object_size = 4000; |
||||
let obj = Object::new( |
||||
content, |
||||
deps, |
||||
expiry, |
||||
max_object_size, |
||||
repo_pubkey, |
||||
repo_secret, |
||||
); |
||||
println!(">>> add_obj"); |
||||
println!(" id: {:?}", obj.id()); |
||||
println!(" deps: {:?}", obj.deps()); |
||||
obj.save(store).unwrap(); |
||||
obj.reference().unwrap() |
||||
} |
||||
|
||||
fn add_commit( |
||||
branch: ObjectRef, |
||||
author_privkey: PrivKey, |
||||
author_pubkey: PubKey, |
||||
seq: u32, |
||||
deps: Vec<ObjectRef>, |
||||
acks: Vec<ObjectRef>, |
||||
body_ref: ObjectRef, |
||||
repo_pubkey: PubKey, |
||||
repo_secret: SymKey, |
||||
store: &mut impl RepoStore, |
||||
) -> ObjectRef { |
||||
let mut obj_deps: Vec<ObjectId> = vec![]; |
||||
obj_deps.extend(deps.iter().map(|r| r.id)); |
||||
obj_deps.extend(acks.iter().map(|r| r.id)); |
||||
|
||||
let obj_ref = ObjectRef { |
||||
id: ObjectId::Blake3Digest32([1; 32]), |
||||
key: SymKey::ChaCha20Key([2; 32]), |
||||
}; |
||||
let refs = vec![obj_ref]; |
||||
let metadata = vec![5u8; 55]; |
||||
let expiry = None; |
||||
|
||||
let commit = Commit::new( |
||||
author_privkey, |
||||
author_pubkey, |
||||
seq, |
||||
branch, |
||||
deps, |
||||
acks, |
||||
refs, |
||||
metadata, |
||||
body_ref, |
||||
expiry, |
||||
) |
||||
.unwrap(); |
||||
//println!("commit: {:?}", commit);
|
||||
add_obj( |
||||
ObjectContent::Commit(commit), |
||||
obj_deps, |
||||
expiry, |
||||
repo_pubkey, |
||||
repo_secret, |
||||
store, |
||||
) |
||||
} |
||||
|
||||
fn add_body_branch( |
||||
branch: Branch, |
||||
repo_pubkey: PubKey, |
||||
repo_secret: SymKey, |
||||
store: &mut impl RepoStore, |
||||
) -> ObjectRef { |
||||
let deps = vec![]; |
||||
let expiry = None; |
||||
let body = CommitBody::Branch(branch); |
||||
//println!("body: {:?}", body);
|
||||
add_obj( |
||||
ObjectContent::CommitBody(body), |
||||
deps, |
||||
expiry, |
||||
repo_pubkey, |
||||
repo_secret, |
||||
store, |
||||
) |
||||
} |
||||
|
||||
fn add_body_trans( |
||||
deps: Vec<ObjectId>, |
||||
repo_pubkey: PubKey, |
||||
repo_secret: SymKey, |
||||
store: &mut impl RepoStore, |
||||
) -> ObjectRef { |
||||
let expiry = None; |
||||
let content = [7u8; 777].to_vec(); |
||||
let body = CommitBody::Transaction(Transaction::V0(content)); |
||||
//println!("body: {:?}", body);
|
||||
add_obj( |
||||
ObjectContent::CommitBody(body), |
||||
deps, |
||||
expiry, |
||||
repo_pubkey, |
||||
repo_secret, |
||||
store, |
||||
) |
||||
} |
||||
|
||||
fn add_body_ack( |
||||
deps: Vec<ObjectId>, |
||||
repo_pubkey: PubKey, |
||||
repo_secret: SymKey, |
||||
store: &mut impl RepoStore, |
||||
) -> ObjectRef { |
||||
let expiry = None; |
||||
let body = CommitBody::Ack(Ack::V0()); |
||||
//println!("body: {:?}", body);
|
||||
add_obj( |
||||
ObjectContent::CommitBody(body), |
||||
deps, |
||||
expiry, |
||||
repo_pubkey, |
||||
repo_secret, |
||||
store, |
||||
) |
||||
} |
||||
|
||||
let mut store = HashMapRepoStore::new(); |
||||
let mut rng = OsRng {}; |
||||
|
||||
// repo
|
||||
|
||||
let repo_keypair: Keypair = Keypair::generate(&mut rng); |
||||
println!( |
||||
"repo private key: ({}) {:?}", |
||||
repo_keypair.secret.as_bytes().len(), |
||||
repo_keypair.secret.as_bytes() |
||||
); |
||||
println!( |
||||
"repo public key: ({}) {:?}", |
||||
repo_keypair.public.as_bytes().len(), |
||||
repo_keypair.public.as_bytes() |
||||
); |
||||
let _repo_privkey = PrivKey::Ed25519PrivKey(repo_keypair.secret.to_bytes()); |
||||
let repo_pubkey = PubKey::Ed25519PubKey(repo_keypair.public.to_bytes()); |
||||
let repo_secret = SymKey::ChaCha20Key([9; 32]); |
||||
|
||||
// branch
|
||||
|
||||
let branch_keypair: Keypair = Keypair::generate(&mut rng); |
||||
println!("branch public key: {:?}", branch_keypair.public.as_bytes()); |
||||
let branch_pubkey = PubKey::Ed25519PubKey(branch_keypair.public.to_bytes()); |
||||
|
||||
let member_keypair: Keypair = Keypair::generate(&mut rng); |
||||
println!("member public key: {:?}", member_keypair.public.as_bytes()); |
||||
let member_privkey = PrivKey::Ed25519PrivKey(member_keypair.secret.to_bytes()); |
||||
let member_pubkey = PubKey::Ed25519PubKey(member_keypair.public.to_bytes()); |
||||
|
||||
let metadata = [66u8; 64].to_vec(); |
||||
let commit_types = vec![CommitType::Ack, CommitType::Transaction]; |
||||
let secret = SymKey::ChaCha20Key([0; 32]); |
||||
|
||||
let member = MemberV0::new(member_pubkey, commit_types, metadata.clone()); |
||||
let members = vec![member]; |
||||
let mut quorum = HashMap::new(); |
||||
quorum.insert(CommitType::Transaction, 3); |
||||
let ack_delay = RelTime::Minutes(3); |
||||
let tags = [99u8; 32].to_vec(); |
||||
let branch = Branch::new( |
||||
branch_pubkey, |
||||
branch_pubkey, |
||||
secret, |
||||
members, |
||||
quorum, |
||||
ack_delay, |
||||
tags, |
||||
metadata, |
||||
); |
||||
//println!("branch: {:?}", branch);
|
||||
|
||||
fn print_branch() { |
||||
println!("branch deps/acks:"); |
||||
println!(""); |
||||
println!(" br"); |
||||
println!(" / \\"); |
||||
println!(" t1 t2"); |
||||
println!(" / \\ / \\"); |
||||
println!(" a3 t4<--t5-->(t1)"); |
||||
println!(" / \\"); |
||||
println!(" a6 a7"); |
||||
println!(""); |
||||
} |
||||
|
||||
print_branch(); |
||||
|
||||
// commit bodies
|
||||
|
||||
let branch_body = add_body_branch( |
||||
branch.clone(), |
||||
repo_pubkey.clone(), |
||||
repo_secret.clone(), |
||||
&mut store, |
||||
); |
||||
let ack_body = add_body_ack(vec![], repo_pubkey, repo_secret, &mut store); |
||||
let trans_body = add_body_trans(vec![], repo_pubkey, repo_secret, &mut store); |
||||
|
||||
// create & add commits to store
|
||||
|
||||
println!(">> br"); |
||||
let br = add_commit( |
||||
branch_body, |
||||
member_privkey, |
||||
member_pubkey, |
||||
0, |
||||
vec![], |
||||
vec![], |
||||
branch_body, |
||||
repo_pubkey, |
||||
repo_secret, |
||||
&mut store, |
||||
); |
||||
|
||||
println!(">> t1"); |
||||
let t1 = add_commit( |
||||
branch_body, |
||||
member_privkey, |
||||
member_pubkey, |
||||
1, |
||||
vec![br], |
||||
vec![], |
||||
trans_body, |
||||
repo_pubkey, |
||||
repo_secret, |
||||
&mut store, |
||||
); |
||||
|
||||
println!(">> t2"); |
||||
let t2 = add_commit( |
||||
branch_body, |
||||
member_privkey, |
||||
member_pubkey, |
||||
2, |
||||
vec![br], |
||||
vec![], |
||||
trans_body, |
||||
repo_pubkey, |
||||
repo_secret, |
||||
&mut store, |
||||
); |
||||
|
||||
println!(">> a3"); |
||||
let a3 = add_commit( |
||||
branch_body, |
||||
member_privkey, |
||||
member_pubkey, |
||||
3, |
||||
vec![t1], |
||||
vec![], |
||||
ack_body, |
||||
repo_pubkey, |
||||
repo_secret, |
||||
&mut store, |
||||
); |
||||
|
||||
println!(">> t4"); |
||||
let t4 = add_commit( |
||||
branch_body, |
||||
member_privkey, |
||||
member_pubkey, |
||||
4, |
||||
vec![t2], |
||||
vec![t1], |
||||
trans_body, |
||||
repo_pubkey, |
||||
repo_secret, |
||||
&mut store, |
||||
); |
||||
|
||||
println!(">> t5"); |
||||
let t5 = add_commit( |
||||
branch_body, |
||||
member_privkey, |
||||
member_pubkey, |
||||
5, |
||||
vec![t1, t2], |
||||
vec![t4], |
||||
trans_body, |
||||
repo_pubkey, |
||||
repo_secret, |
||||
&mut store, |
||||
); |
||||
|
||||
println!(">> a6"); |
||||
let a6 = add_commit( |
||||
branch_body, |
||||
member_privkey, |
||||
member_pubkey, |
||||
6, |
||||
vec![t4], |
||||
vec![], |
||||
ack_body, |
||||
repo_pubkey, |
||||
repo_secret, |
||||
&mut store, |
||||
); |
||||
|
||||
println!(">> a7"); |
||||
let a7 = add_commit( |
||||
branch_body, |
||||
member_privkey, |
||||
member_pubkey, |
||||
7, |
||||
vec![t4], |
||||
vec![], |
||||
ack_body, |
||||
repo_pubkey, |
||||
repo_secret, |
||||
&mut store, |
||||
); |
||||
|
||||
let c7 = Commit::load(a7, &store).unwrap(); |
||||
c7.verify(&branch, &store).unwrap(); |
||||
|
||||
let mut filter = Filter::new(FilterBuilder::new(10, 0.01)); |
||||
for commit_ref in [br, t1, t2, a3, t5, a6] { |
||||
match commit_ref.id { |
||||
ObjectId::Blake3Digest32(d) => filter.add(&d), |
||||
} |
||||
} |
||||
let cfg = filter.config(); |
||||
let their_commits = BloomFilter { |
||||
k: cfg.hashes, |
||||
f: filter.get_u8_array().to_vec(), |
||||
}; |
||||
|
||||
print_branch(); |
||||
println!(">> sync_req"); |
||||
println!(" our_heads: [a3, t5, a6, a7]"); |
||||
println!(" their_heads: [a3, t5]"); |
||||
println!(" their_commits: [br, t1, t2, a3, t5, a6]"); |
||||
|
||||
let ids = Branch::sync_req( |
||||
&[a3.id, t5.id, a6.id, a7.id], |
||||
&[a3.id, t5.id], |
||||
&their_commits, |
||||
&store, |
||||
) |
||||
.unwrap(); |
||||
|
||||
assert_eq!(ids.len(), 1); |
||||
assert!(ids.contains(&a7.id)); |
||||
} |
||||
} |
||||
@ -0,0 +1,64 @@ |
||||
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers
|
||||
// Licensed under the Apache License, Version 2.0
|
||||
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
|
||||
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
|
||||
// at your option. All files in the project carrying such
|
||||
// notice may not be copied, modified, or distributed except
|
||||
// according to those terms.
|
||||
|
||||
use crate::store::{StorageError}; |
||||
|
||||
pub trait BrokerStore { |
||||
/// Load a property from the store.
|
||||
fn get(&self, prefix: u8, key: &Vec<u8>, suffix: Option<u8>) -> Result<Vec<u8>, StorageError>; |
||||
|
||||
/// Load all the values of a property from the store.
|
||||
fn get_all( |
||||
&self, |
||||
prefix: u8, |
||||
key: &Vec<u8>, |
||||
suffix: Option<u8>, |
||||
) -> Result<Vec<Vec<u8>>, StorageError>; |
||||
|
||||
/// Check if a specific value exists for a property from the store.
|
||||
fn has_property_value( |
||||
&self, |
||||
prefix: u8, |
||||
key: &Vec<u8>, |
||||
suffix: Option<u8>, |
||||
value: Vec<u8>, |
||||
) -> Result<(), StorageError>; |
||||
|
||||
/// Save a property value to the store.
|
||||
fn put( |
||||
&self, |
||||
prefix: u8, |
||||
key: &Vec<u8>, |
||||
suffix: Option<u8>, |
||||
value: Vec<u8>, |
||||
) -> Result<(), StorageError>; |
||||
|
||||
/// Replace the property of a key (single value) to the store.
|
||||
fn replace( |
||||
&self, |
||||
prefix: u8, |
||||
key: &Vec<u8>, |
||||
suffix: Option<u8>, |
||||
value: Vec<u8>, |
||||
) -> Result<(), StorageError>; |
||||
|
||||
/// Delete a property from the store.
|
||||
fn del(&self, prefix: u8, key: &Vec<u8>, suffix: Option<u8>) -> Result<(), StorageError>; |
||||
|
||||
/// Delete all properties of a key from the store.
|
||||
fn del_all(&self, prefix: u8, key: &Vec<u8>, all_suffixes: &[u8]) -> Result<(), StorageError>; |
||||
|
||||
/// Delete a specific value for a property from the store.
|
||||
fn del_property_value( |
||||
&self, |
||||
prefix: u8, |
||||
key: &Vec<u8>, |
||||
suffix: Option<u8>, |
||||
value: Vec<u8>, |
||||
) -> Result<(), StorageError>; |
||||
} |
||||
@ -0,0 +1,454 @@ |
||||
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers
|
||||
// All rights reserved.
|
||||
// This code is partly derived from work written by TG x Thoth from P2Pcollab.
|
||||
// Copyright 2022 TG x Thoth
|
||||
// Licensed under the Apache License, Version 2.0
|
||||
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
|
||||
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
|
||||
// at your option. All files in the project carrying such
|
||||
// notice may not be copied, modified, or distributed except
|
||||
// according to those terms.
|
||||
|
||||
//! Commit
|
||||
|
||||
use debug_print::*; |
||||
use ed25519_dalek::*; |
||||
|
||||
use std::collections::HashSet; |
||||
use std::iter::FromIterator; |
||||
|
||||
use crate::object::*; |
||||
use crate::store::*; |
||||
use crate::types::*; |
||||
|
||||
#[derive(Debug)] |
||||
pub enum CommitLoadError { |
||||
MissingBlocks(Vec<BlockId>), |
||||
ObjectParseError, |
||||
DeserializeError, |
||||
} |
||||
|
||||
#[derive(Debug)] |
||||
pub enum CommitVerifyError { |
||||
InvalidSignature, |
||||
PermissionDenied, |
||||
BodyLoadError(CommitLoadError), |
||||
DepLoadError(CommitLoadError), |
||||
} |
||||
impl CommitBody { |
||||
/// Get CommitType corresponding to CommitBody
|
||||
pub fn to_type(&self) -> CommitType { |
||||
match self { |
||||
CommitBody::Ack(_) => CommitType::Ack, |
||||
CommitBody::AddBranch(_) => CommitType::AddBranch, |
||||
CommitBody::AddMembers(_) => CommitType::AddMembers, |
||||
CommitBody::Branch(_) => CommitType::Branch, |
||||
CommitBody::EndOfBranch(_) => CommitType::EndOfBranch, |
||||
CommitBody::RemoveBranch(_) => CommitType::RemoveBranch, |
||||
CommitBody::Repository(_) => CommitType::Repository, |
||||
CommitBody::Snapshot(_) => CommitType::Snapshot, |
||||
CommitBody::Transaction(_) => CommitType::Transaction, |
||||
} |
||||
} |
||||
} |
||||
|
||||
impl CommitV0 { |
||||
/// New commit
|
||||
pub fn new( |
||||
author_privkey: PrivKey, |
||||
author_pubkey: PubKey, |
||||
seq: u32, |
||||
branch: ObjectRef, |
||||
deps: Vec<ObjectRef>, |
||||
acks: Vec<ObjectRef>, |
||||
refs: Vec<ObjectRef>, |
||||
metadata: Vec<u8>, |
||||
body: ObjectRef, |
||||
expiry: Option<Timestamp>, |
||||
) -> Result<CommitV0, SignatureError> { |
||||
let content = CommitContentV0 { |
||||
author: author_pubkey, |
||||
seq, |
||||
branch, |
||||
deps, |
||||
acks, |
||||
refs, |
||||
metadata, |
||||
body, |
||||
expiry, |
||||
}; |
||||
let content_ser = serde_bare::to_vec(&content).unwrap(); |
||||
|
||||
// sign commit
|
||||
let kp = match (author_privkey, author_pubkey) { |
||||
(PrivKey::Ed25519PrivKey(sk), PubKey::Ed25519PubKey(pk)) => [sk, pk].concat(), |
||||
}; |
||||
let keypair = Keypair::from_bytes(kp.as_slice())?; |
||||
let sig_bytes = keypair.sign(content_ser.as_slice()).to_bytes(); |
||||
let mut it = sig_bytes.chunks_exact(32); |
||||
let mut ss: Ed25519Sig = [[0; 32], [0; 32]]; |
||||
ss[0].copy_from_slice(it.next().unwrap()); |
||||
ss[1].copy_from_slice(it.next().unwrap()); |
||||
let sig = Sig::Ed25519Sig(ss); |
||||
Ok(CommitV0 { |
||||
content, |
||||
sig, |
||||
id: None, |
||||
key: None, |
||||
}) |
||||
} |
||||
} |
||||
|
||||
impl Commit { |
||||
/// New commit
|
||||
pub fn new( |
||||
author_privkey: PrivKey, |
||||
author_pubkey: PubKey, |
||||
seq: u32, |
||||
branch: ObjectRef, |
||||
deps: Vec<ObjectRef>, |
||||
acks: Vec<ObjectRef>, |
||||
refs: Vec<ObjectRef>, |
||||
metadata: Vec<u8>, |
||||
body: ObjectRef, |
||||
expiry: Option<Timestamp>, |
||||
) -> Result<Commit, SignatureError> { |
||||
CommitV0::new( |
||||
author_privkey, |
||||
author_pubkey, |
||||
seq, |
||||
branch, |
||||
deps, |
||||
acks, |
||||
refs, |
||||
metadata, |
||||
body, |
||||
expiry, |
||||
) |
||||
.map(|c| Commit::V0(c)) |
||||
} |
||||
|
||||
/// Load commit from store
|
||||
pub fn load(commit_ref: ObjectRef, store: &impl RepoStore) -> Result<Commit, CommitLoadError> { |
||||
let (id, key) = (commit_ref.id, commit_ref.key); |
||||
match Object::load(id, Some(key), store) { |
||||
Ok(obj) => { |
||||
let content = obj |
||||
.content() |
||||
.map_err(|_e| CommitLoadError::ObjectParseError)?; |
||||
let mut commit = match content { |
||||
ObjectContent::Commit(c) => c, |
||||
_ => return Err(CommitLoadError::DeserializeError), |
||||
}; |
||||
commit.set_id(id); |
||||
commit.set_key(key); |
||||
Ok(commit) |
||||
} |
||||
Err(ObjectParseError::MissingBlocks(missing)) => { |
||||
Err(CommitLoadError::MissingBlocks(missing)) |
||||
} |
||||
Err(_) => Err(CommitLoadError::ObjectParseError), |
||||
} |
||||
} |
||||
|
||||
/// Load commit body from store
|
||||
pub fn load_body(&self, store: &impl RepoStore) -> Result<CommitBody, CommitLoadError> { |
||||
let content = self.content(); |
||||
let (id, key) = (content.body.id, content.body.key); |
||||
let obj = Object::load(id.clone(), Some(key.clone()), store).map_err(|e| match e { |
||||
ObjectParseError::MissingBlocks(missing) => CommitLoadError::MissingBlocks(missing), |
||||
_ => CommitLoadError::ObjectParseError, |
||||
})?; |
||||
let content = obj |
||||
.content() |
||||
.map_err(|_e| CommitLoadError::ObjectParseError)?; |
||||
match content { |
||||
ObjectContent::CommitBody(body) => Ok(body), |
||||
_ => Err(CommitLoadError::DeserializeError), |
||||
} |
||||
} |
||||
|
||||
/// Get ID of parent `Object`
|
||||
pub fn id(&self) -> Option<ObjectId> { |
||||
match self { |
||||
Commit::V0(c) => c.id, |
||||
} |
||||
} |
||||
|
||||
/// Set ID of parent `Object`
|
||||
pub fn set_id(&mut self, id: ObjectId) { |
||||
match self { |
||||
Commit::V0(c) => c.id = Some(id), |
||||
} |
||||
} |
||||
|
||||
/// Get key of parent `Object`
|
||||
pub fn key(&self) -> Option<SymKey> { |
||||
match self { |
||||
Commit::V0(c) => c.key, |
||||
} |
||||
} |
||||
|
||||
/// Set key of parent `Object`
|
||||
pub fn set_key(&mut self, key: SymKey) { |
||||
match self { |
||||
Commit::V0(c) => c.key = Some(key), |
||||
} |
||||
} |
||||
|
||||
/// Get commit signature
|
||||
pub fn sig(&self) -> &Sig { |
||||
match self { |
||||
Commit::V0(c) => &c.sig, |
||||
} |
||||
} |
||||
|
||||
/// Get commit content
|
||||
pub fn content(&self) -> &CommitContentV0 { |
||||
match self { |
||||
Commit::V0(c) => &c.content, |
||||
} |
||||
} |
||||
|
||||
/// Get acks
|
||||
pub fn acks(&self) -> Vec<ObjectRef> { |
||||
match self { |
||||
Commit::V0(c) => c.content.acks.clone(), |
||||
} |
||||
} |
||||
|
||||
/// Get deps
|
||||
pub fn deps(&self) -> Vec<ObjectRef> { |
||||
match self { |
||||
Commit::V0(c) => c.content.deps.clone(), |
||||
} |
||||
} |
||||
|
||||
/// Get all direct commit dependencies of the commit (`deps`, `acks`)
|
||||
pub fn deps_acks(&self) -> Vec<ObjectRef> { |
||||
match self { |
||||
Commit::V0(c) => [c.content.acks.clone(), c.content.deps.clone()].concat(), |
||||
} |
||||
} |
||||
|
||||
/// Get seq
|
||||
pub fn seq(&self) -> u32 { |
||||
match self { |
||||
Commit::V0(c) => c.content.seq, |
||||
} |
||||
} |
||||
|
||||
/// Verify commit signature
|
||||
pub fn verify_sig(&self) -> Result<(), SignatureError> { |
||||
let c = match self { |
||||
Commit::V0(c) => c, |
||||
}; |
||||
let content_ser = serde_bare::to_vec(&c.content).unwrap(); |
||||
let pubkey = match c.content.author { |
||||
PubKey::Ed25519PubKey(pk) => pk, |
||||
}; |
||||
let pk = PublicKey::from_bytes(&pubkey)?; |
||||
let sig_bytes = match c.sig { |
||||
Sig::Ed25519Sig(ss) => [ss[0], ss[1]].concat(), |
||||
}; |
||||
let sig = Signature::from_bytes(&sig_bytes)?; |
||||
pk.verify_strict(&content_ser, &sig) |
||||
} |
||||
|
||||
/// Verify commit permissions
|
||||
pub fn verify_perm(&self, body: &CommitBody, branch: &Branch) -> Result<(), CommitVerifyError> { |
||||
let content = self.content(); |
||||
match branch.get_member(&content.author) { |
||||
Some(m) => { |
||||
if m.has_perm(body.to_type()) { |
||||
return Ok(()); |
||||
} |
||||
} |
||||
None => (), |
||||
} |
||||
Err(CommitVerifyError::PermissionDenied) |
||||
} |
||||
|
||||
/// Verify if the commit's `body` and dependencies (`deps` & `acks`) are available in the `store`
|
||||
pub fn verify_deps(&self, store: &impl RepoStore) -> Result<Vec<ObjectId>, CommitLoadError> { |
||||
//debug_println!(">> verify_deps: #{}", self.seq());
|
||||
/// Load `Commit`s of a `Branch` from the `RepoStore` starting from the given `Commit`,
|
||||
/// and collect missing `ObjectId`s
|
||||
fn load_branch( |
||||
commit: &Commit, |
||||
store: &impl RepoStore, |
||||
visited: &mut HashSet<ObjectId>, |
||||
missing: &mut HashSet<ObjectId>, |
||||
) -> Result<(), CommitLoadError> { |
||||
//debug_println!(">>> load_branch: #{}", commit.seq());
|
||||
// the commit verify_deps() was called on may not have an ID set,
|
||||
// but the commits loaded from store should have it
|
||||
match commit.id() { |
||||
Some(id) => { |
||||
if visited.contains(&id) { |
||||
return Ok(()); |
||||
} |
||||
visited.insert(id); |
||||
} |
||||
None => (), |
||||
} |
||||
|
||||
// load body & check if it's the Branch commit at the root
|
||||
let is_root = match commit.load_body(store) { |
||||
Ok(body) => body.to_type() == CommitType::Branch, |
||||
Err(CommitLoadError::MissingBlocks(m)) => { |
||||
missing.extend(m); |
||||
false |
||||
} |
||||
Err(e) => return Err(e), |
||||
}; |
||||
debug_println!("!!! is_root: {}", is_root); |
||||
|
||||
// load deps
|
||||
if !is_root { |
||||
for dep in commit.deps_acks() { |
||||
match Commit::load(dep, store) { |
||||
Ok(c) => { |
||||
load_branch(&c, store, visited, missing)?; |
||||
} |
||||
Err(CommitLoadError::MissingBlocks(m)) => { |
||||
missing.extend(m); |
||||
} |
||||
Err(e) => return Err(e), |
||||
} |
||||
} |
||||
} |
||||
Ok(()) |
||||
} |
||||
|
||||
let mut visited = HashSet::new(); |
||||
let mut missing = HashSet::new(); |
||||
load_branch(self, store, &mut visited, &mut missing)?; |
||||
|
||||
if !missing.is_empty() { |
||||
return Err(CommitLoadError::MissingBlocks(Vec::from_iter(missing))); |
||||
} |
||||
Ok(Vec::from_iter(visited)) |
||||
} |
||||
|
||||
/// Verify signature, permissions, and dependencies
|
||||
pub fn verify(&self, branch: &Branch, store: &impl RepoStore) -> Result<(), CommitVerifyError> { |
||||
self.verify_sig() |
||||
.map_err(|_e| CommitVerifyError::InvalidSignature)?; |
||||
let body = self |
||||
.load_body(store) |
||||
.map_err(|e| CommitVerifyError::BodyLoadError(e))?; |
||||
self.verify_perm(&body, branch)?; |
||||
self.verify_deps(store) |
||||
.map_err(|e| CommitVerifyError::DepLoadError(e))?; |
||||
Ok(()) |
||||
} |
||||
} |
||||
|
||||
mod test { |
||||
use std::collections::HashMap; |
||||
|
||||
use ed25519_dalek::*; |
||||
use rand::rngs::OsRng; |
||||
|
||||
use crate::branch::*; |
||||
use crate::commit::*; |
||||
use crate::store::*; |
||||
use crate::types::*; |
||||
|
||||
#[test] |
||||
pub fn test_commit() { |
||||
let mut csprng = OsRng {}; |
||||
let keypair: Keypair = Keypair::generate(&mut csprng); |
||||
println!( |
||||
"private key: ({}) {:?}", |
||||
keypair.secret.as_bytes().len(), |
||||
keypair.secret.as_bytes() |
||||
); |
||||
println!( |
||||
"public key: ({}) {:?}", |
||||
keypair.public.as_bytes().len(), |
||||
keypair.public.as_bytes() |
||||
); |
||||
let ed_priv_key = keypair.secret.to_bytes(); |
||||
let ed_pub_key = keypair.public.to_bytes(); |
||||
let priv_key = PrivKey::Ed25519PrivKey(ed_priv_key); |
||||
let pub_key = PubKey::Ed25519PubKey(ed_pub_key); |
||||
let seq = 3; |
||||
let obj_ref = ObjectRef { |
||||
id: ObjectId::Blake3Digest32([1; 32]), |
||||
key: SymKey::ChaCha20Key([2; 32]), |
||||
}; |
||||
let obj_refs = vec![obj_ref]; |
||||
let branch = obj_ref.clone(); |
||||
let deps = obj_refs.clone(); |
||||
let acks = obj_refs.clone(); |
||||
let refs = obj_refs.clone(); |
||||
let metadata = vec![1, 2, 3]; |
||||
let body_ref = obj_ref.clone(); |
||||
let expiry = Some(2342); |
||||
|
||||
let commit = Commit::new( |
||||
priv_key, pub_key, seq, branch, deps, acks, refs, metadata, body_ref, expiry, |
||||
) |
||||
.unwrap(); |
||||
println!("commit: {:?}", commit); |
||||
|
||||
let store = HashMapRepoStore::new(); |
||||
let metadata = [66u8; 64].to_vec(); |
||||
let commit_types = vec![CommitType::Ack, CommitType::Transaction]; |
||||
let key: [u8; 32] = [0; 32]; |
||||
let secret = SymKey::ChaCha20Key(key); |
||||
let member = MemberV0::new(pub_key, commit_types, metadata.clone()); |
||||
let members = vec![member]; |
||||
let mut quorum = HashMap::new(); |
||||
quorum.insert(CommitType::Transaction, 3); |
||||
let ack_delay = RelTime::Minutes(3); |
||||
let tags = [99u8; 32].to_vec(); |
||||
let branch = Branch::new( |
||||
pub_key.clone(), |
||||
pub_key.clone(), |
||||
secret, |
||||
members, |
||||
quorum, |
||||
ack_delay, |
||||
tags, |
||||
metadata, |
||||
); |
||||
//println!("branch: {:?}", branch);
|
||||
let body = CommitBody::Ack(Ack::V0()); |
||||
//println!("body: {:?}", body);
|
||||
|
||||
match commit.load_body(&store) { |
||||
Ok(_b) => panic!("Body should not exist"), |
||||
Err(CommitLoadError::MissingBlocks(missing)) => { |
||||
assert_eq!(missing.len(), 1); |
||||
} |
||||
Err(e) => panic!("Commit verify error: {:?}", e), |
||||
} |
||||
|
||||
let content = commit.content(); |
||||
println!("content: {:?}", content); |
||||
|
||||
commit.verify_sig().expect("Invalid signature"); |
||||
commit |
||||
.verify_perm(&body, &branch) |
||||
.expect("Permission denied"); |
||||
|
||||
match commit.verify_deps(&store) { |
||||
Ok(_) => panic!("Commit should not be Ok"), |
||||
Err(CommitLoadError::MissingBlocks(missing)) => { |
||||
assert_eq!(missing.len(), 1); |
||||
} |
||||
Err(e) => panic!("Commit verify error: {:?}", e), |
||||
} |
||||
|
||||
match commit.verify(&branch, &store) { |
||||
Ok(_) => panic!("Commit should not be Ok"), |
||||
Err(CommitVerifyError::BodyLoadError(CommitLoadError::MissingBlocks(missing))) => { |
||||
assert_eq!(missing.len(), 1); |
||||
} |
||||
Err(e) => panic!("Commit verify error: {:?}", e), |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,29 @@ |
||||
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers
|
||||
// All rights reserved.
|
||||
// This code is partly derived from work written by TG x Thoth from P2Pcollab.
|
||||
// Copyright 2022 TG x Thoth
|
||||
// Licensed under the Apache License, Version 2.0
|
||||
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
|
||||
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
|
||||
// at your option. All files in the project carrying such
|
||||
// notice may not be copied, modified, or distributed except
|
||||
// according to those terms.
|
||||
|
||||
//! Errors
|
||||
|
||||
pub enum NgError { |
||||
InvalidSignature, |
||||
SerializationError, |
||||
} |
||||
|
||||
impl From<serde_bare::error::Error> for NgError { |
||||
fn from(e: serde_bare::error::Error) -> Self { |
||||
NgError::SerializationError |
||||
} |
||||
} |
||||
|
||||
impl From<ed25519_dalek::ed25519::Error> for NgError { |
||||
fn from(e: ed25519_dalek::ed25519::Error) -> Self { |
||||
NgError::InvalidSignature |
||||
} |
||||
} |
||||
@ -0,0 +1,19 @@ |
||||
pub mod types; |
||||
|
||||
pub mod store; |
||||
|
||||
pub mod block; |
||||
|
||||
pub mod object; |
||||
|
||||
pub mod commit; |
||||
|
||||
pub mod branch; |
||||
|
||||
pub mod repo; |
||||
|
||||
pub mod utils; |
||||
|
||||
pub mod errors; |
||||
|
||||
pub mod broker_store; |
||||
@ -0,0 +1,909 @@ |
||||
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers
|
||||
// All rights reserved.
|
||||
// This code is partly derived from work written by TG x Thoth from P2Pcollab.
|
||||
// Copyright 2022 TG x Thoth
|
||||
// Licensed under the Apache License, Version 2.0
|
||||
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
|
||||
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
|
||||
// at your option. All files in the project carrying such
|
||||
// notice may not be copied, modified, or distributed except
|
||||
// according to those terms.
|
||||
|
||||
//! Merkle hash tree of Objects
|
||||
|
||||
use std::collections::{HashMap, HashSet}; |
||||
|
||||
use debug_print::*; |
||||
|
||||
use chacha20::cipher::{KeyIvInit, StreamCipher}; |
||||
use chacha20::ChaCha20; |
||||
|
||||
use crate::store::*; |
||||
use crate::types::*; |
||||
|
||||
/// Size of a serialized empty Block
|
||||
const EMPTY_BLOCK_SIZE: usize = 12; |
||||
/// Size of a serialized BlockId
|
||||
const BLOCK_ID_SIZE: usize = 33; |
||||
/// Size of serialized SymKey
|
||||
const BLOCK_KEY_SIZE: usize = 33; |
||||
/// Size of serialized Object with deps reference.
|
||||
const EMPTY_ROOT_SIZE_DEPSREF: usize = 77; |
||||
/// Extra size needed if depsRef used instead of deps list.
|
||||
const DEPSREF_OVERLOAD: usize = EMPTY_ROOT_SIZE_DEPSREF - EMPTY_BLOCK_SIZE; |
||||
/// Varint extra bytes when reaching the maximum value we will ever use
|
||||
const BIG_VARINT_EXTRA: usize = 3; |
||||
/// Varint extra bytes when reaching the maximum size of data byte arrays.
|
||||
const DATA_VARINT_EXTRA: usize = 4; |
||||
/// Max extra space used by the deps list
|
||||
const MAX_DEPS_SIZE: usize = 8 * BLOCK_ID_SIZE; |
||||
|
||||
#[derive(Debug)] |
||||
pub struct Object { |
||||
/// Blocks of the Object (nodes of the tree)
|
||||
blocks: Vec<Block>, |
||||
|
||||
/// Dependencies
|
||||
deps: Vec<ObjectId>, |
||||
} |
||||
|
||||
/// Object parsing errors
|
||||
#[derive(Debug)] |
||||
pub enum ObjectParseError { |
||||
/// Missing blocks
|
||||
MissingBlocks(Vec<BlockId>), |
||||
/// Missing root key
|
||||
MissingRootKey, |
||||
/// Invalid BlockId encountered in the tree
|
||||
InvalidBlockId, |
||||
/// Too many or too few children of a block
|
||||
InvalidChildren, |
||||
/// Number of keys does not match number of children of a block
|
||||
InvalidKeys, |
||||
/// Invalid DepList object content
|
||||
InvalidDeps, |
||||
/// Error deserializing content of a block
|
||||
BlockDeserializeError, |
||||
/// Error deserializing content of the object
|
||||
ObjectDeserializeError, |
||||
} |
||||
|
||||
/// Object copy error
|
||||
#[derive(Debug)] |
||||
pub enum ObjectCopyError { |
||||
NotFound, |
||||
ParseError, |
||||
} |
||||
|
||||
impl Object { |
||||
fn convergence_key(repo_pubkey: PubKey, repo_secret: SymKey) -> [u8; blake3::OUT_LEN] { |
||||
let key_material = match (repo_pubkey, repo_secret) { |
||||
(PubKey::Ed25519PubKey(pubkey), SymKey::ChaCha20Key(secret)) => { |
||||
[pubkey, secret].concat() |
||||
} |
||||
}; |
||||
blake3::derive_key("NextGraph Data BLAKE3 key", key_material.as_slice()) |
||||
} |
||||
|
||||
fn make_block( |
||||
content: &[u8], |
||||
conv_key: &[u8; blake3::OUT_LEN], |
||||
children: Vec<ObjectId>, |
||||
deps: ObjectDeps, |
||||
expiry: Option<Timestamp>, |
||||
) -> Block { |
||||
let key_hash = blake3::keyed_hash(conv_key, content); |
||||
let nonce = [0u8; 12]; |
||||
let key = key_hash.as_bytes(); |
||||
let mut cipher = ChaCha20::new(key.into(), &nonce.into()); |
||||
let mut content_enc = Vec::from(content); |
||||
let mut content_enc_slice = &mut content_enc.as_mut_slice(); |
||||
cipher.apply_keystream(&mut content_enc_slice); |
||||
let key = SymKey::ChaCha20Key(key.clone()); |
||||
let block = Block::new(children, deps, expiry, content_enc, Some(key)); |
||||
//debug_println!(">>> make_block:");
|
||||
//debug_println!("!! id: {:?}", obj.id());
|
||||
//debug_println!("!! children: ({}) {:?}", children.len(), children);
|
||||
block |
||||
} |
||||
|
||||
fn make_deps( |
||||
deps_vec: Vec<ObjectId>, |
||||
object_size: usize, |
||||
repo_pubkey: PubKey, |
||||
repo_secret: SymKey, |
||||
) -> ObjectDeps { |
||||
if deps_vec.len() <= 8 { |
||||
ObjectDeps::ObjectIdList(deps_vec) |
||||
} else { |
||||
let dep_list = DepList::V0(deps_vec); |
||||
let dep_obj = Object::new( |
||||
ObjectContent::DepList(dep_list), |
||||
vec![], |
||||
None, |
||||
object_size, |
||||
repo_pubkey, |
||||
repo_secret, |
||||
); |
||||
let dep_ref = ObjectRef { |
||||
id: dep_obj.id(), |
||||
key: dep_obj.key().unwrap(), |
||||
}; |
||||
ObjectDeps::DepListRef(dep_ref) |
||||
} |
||||
} |
||||
|
||||
/// Build tree from leaves, returns parent nodes
|
||||
fn make_tree( |
||||
leaves: &[Block], |
||||
conv_key: &ChaCha20Key, |
||||
root_deps: &ObjectDeps, |
||||
expiry: Option<Timestamp>, |
||||
arity: usize, |
||||
) -> Vec<Block> { |
||||
let mut parents = vec![]; |
||||
let chunks = leaves.chunks(arity); |
||||
let mut it = chunks.peekable(); |
||||
while let Some(nodes) = it.next() { |
||||
let keys = nodes.iter().map(|block| block.key().unwrap()).collect(); |
||||
let children = nodes.iter().map(|block| block.id()).collect(); |
||||
let content = BlockContentV0::InternalNode(keys); |
||||
let content_ser = serde_bare::to_vec(&content).unwrap(); |
||||
let child_deps = ObjectDeps::ObjectIdList(vec![]); |
||||
let deps = if parents.is_empty() && it.peek().is_none() { |
||||
root_deps.clone() |
||||
} else { |
||||
child_deps |
||||
}; |
||||
parents.push(Self::make_block( |
||||
content_ser.as_slice(), |
||||
conv_key, |
||||
children, |
||||
deps, |
||||
expiry, |
||||
)); |
||||
} |
||||
//debug_println!("parents += {}", parents.len());
|
||||
|
||||
if 1 < parents.len() { |
||||
let mut great_parents = |
||||
Self::make_tree(parents.as_slice(), conv_key, root_deps, expiry, arity); |
||||
parents.append(&mut great_parents); |
||||
} |
||||
parents |
||||
} |
||||
|
||||
/// Create new Object from given content
|
||||
///
|
||||
/// The Object is chunked and stored in a Merkle tree
|
||||
/// The arity of the Merkle tree is the maximum that fits in the given `max_object_size`
|
||||
///
|
||||
/// Arguments:
|
||||
/// * `content`: Object content
|
||||
/// * `deps`: Dependencies of the object
|
||||
/// * `block_size`: Desired block size for chunking content, rounded up to nearest valid block size
|
||||
/// * `repo_pubkey`: Repository public key
|
||||
/// * `repo_secret`: Repository secret
|
||||
pub fn new( |
||||
content: ObjectContent, |
||||
deps: Vec<ObjectId>, |
||||
expiry: Option<Timestamp>, |
||||
block_size: usize, |
||||
repo_pubkey: PubKey, |
||||
repo_secret: SymKey, |
||||
) -> Object { |
||||
// create blocks by chunking + encrypting content
|
||||
let valid_block_size = store_valid_value_size(block_size); |
||||
let data_chunk_size = valid_block_size - EMPTY_BLOCK_SIZE - DATA_VARINT_EXTRA; |
||||
|
||||
let mut blocks: Vec<Block> = vec![]; |
||||
let conv_key = Self::convergence_key(repo_pubkey, repo_secret); |
||||
|
||||
let obj_deps = Self::make_deps(deps.clone(), valid_block_size, repo_pubkey, repo_secret); |
||||
|
||||
let content_ser = serde_bare::to_vec(&content).unwrap(); |
||||
|
||||
if EMPTY_BLOCK_SIZE + DATA_VARINT_EXTRA + BLOCK_ID_SIZE * deps.len() + content_ser.len() |
||||
<= valid_block_size |
||||
{ |
||||
// content fits in root node
|
||||
let data_chunk = BlockContentV0::DataChunk(content_ser.clone()); |
||||
let content_ser = serde_bare::to_vec(&data_chunk).unwrap(); |
||||
blocks.push(Self::make_block( |
||||
content_ser.as_slice(), |
||||
&conv_key, |
||||
vec![], |
||||
obj_deps, |
||||
expiry, |
||||
)); |
||||
} else { |
||||
// chunk content and create leaf nodes
|
||||
for chunk in content_ser.chunks(data_chunk_size) { |
||||
let data_chunk = BlockContentV0::DataChunk(chunk.to_vec()); |
||||
let content_ser = serde_bare::to_vec(&data_chunk).unwrap(); |
||||
blocks.push(Self::make_block( |
||||
content_ser.as_slice(), |
||||
&conv_key, |
||||
vec![], |
||||
ObjectDeps::ObjectIdList(vec![]), |
||||
expiry, |
||||
)); |
||||
} |
||||
|
||||
// internal nodes
|
||||
// arity: max number of ObjectRefs that fit inside an InternalNode Object within the object_size limit
|
||||
let arity: usize = |
||||
(valid_block_size - EMPTY_BLOCK_SIZE - BIG_VARINT_EXTRA * 2 - MAX_DEPS_SIZE) |
||||
/ (BLOCK_ID_SIZE + BLOCK_KEY_SIZE); |
||||
let mut parents = |
||||
Self::make_tree(blocks.as_slice(), &conv_key, &obj_deps, expiry, arity); |
||||
blocks.append(&mut parents); |
||||
} |
||||
|
||||
Object { blocks, deps } |
||||
} |
||||
|
||||
pub fn copy( |
||||
&self, |
||||
expiry: Option<Timestamp>, |
||||
repo_pubkey: PubKey, |
||||
repo_secret: SymKey, |
||||
) -> Result<Object, ObjectCopyError> { |
||||
// getting the old object from store
|
||||
let leaves: Vec<Block> = self.leaves().map_err(|_e| ObjectCopyError::ParseError)?; |
||||
|
||||
let conv_key = Self::convergence_key(repo_pubkey, repo_secret); |
||||
let block_size = leaves.first().unwrap().content().len(); |
||||
let valid_block_size = store_valid_value_size(block_size); |
||||
|
||||
let mut blocks: Vec<Block> = vec![]; |
||||
for block in leaves { |
||||
let mut copy = block.clone(); |
||||
copy.set_expiry(expiry); |
||||
blocks.push(copy); |
||||
} |
||||
|
||||
// internal nodes
|
||||
// arity: max number of ObjectRefs that fit inside an InternalNode Object within the object_size limit
|
||||
let arity: usize = |
||||
(valid_block_size - EMPTY_BLOCK_SIZE - BIG_VARINT_EXTRA * 2 - MAX_DEPS_SIZE) |
||||
/ (BLOCK_ID_SIZE + BLOCK_KEY_SIZE); |
||||
let mut parents = Self::make_tree( |
||||
blocks.as_slice(), |
||||
&conv_key, |
||||
self.root().deps(), |
||||
expiry, |
||||
arity, |
||||
); |
||||
blocks.append(&mut parents); |
||||
|
||||
Ok(Object { |
||||
blocks, |
||||
deps: self.deps().clone(), |
||||
}) |
||||
} |
||||
|
||||
/// Load an Object from RepoStore
|
||||
///
|
||||
/// Returns Ok(Object) or an Err(Vec<ObjectId>) of missing BlockIds
|
||||
pub fn load( |
||||
id: ObjectId, |
||||
key: Option<SymKey>, |
||||
store: &impl RepoStore, |
||||
) -> Result<Object, ObjectParseError> { |
||||
fn load_tree( |
||||
parents: Vec<BlockId>, |
||||
store: &impl RepoStore, |
||||
blocks: &mut Vec<Block>, |
||||
missing: &mut Vec<BlockId>, |
||||
) { |
||||
let mut children: Vec<BlockId> = vec![]; |
||||
for id in parents { |
||||
match store.get(&id) { |
||||
Ok(block) => { |
||||
blocks.insert(0, block.clone()); |
||||
match block { |
||||
Block::V0(o) => { |
||||
children.extend(o.children.iter().rev()); |
||||
} |
||||
} |
||||
} |
||||
Err(_) => missing.push(id.clone()), |
||||
} |
||||
} |
||||
if !children.is_empty() { |
||||
load_tree(children, store, blocks, missing); |
||||
} |
||||
} |
||||
|
||||
let mut blocks: Vec<Block> = vec![]; |
||||
let mut missing: Vec<BlockId> = vec![]; |
||||
|
||||
load_tree(vec![id], store, &mut blocks, &mut missing); |
||||
|
||||
if !missing.is_empty() { |
||||
return Err(ObjectParseError::MissingBlocks(missing)); |
||||
} |
||||
|
||||
let root = blocks.last_mut().unwrap(); |
||||
if key.is_some() { |
||||
root.set_key(key); |
||||
} |
||||
|
||||
let deps = match root.deps().clone() { |
||||
ObjectDeps::ObjectIdList(deps_vec) => deps_vec, |
||||
ObjectDeps::DepListRef(deps_ref) => { |
||||
let obj = Object::load(deps_ref.id, Some(deps_ref.key), store)?; |
||||
match obj.content()? { |
||||
ObjectContent::DepList(DepList::V0(deps_vec)) => deps_vec, |
||||
_ => return Err(ObjectParseError::InvalidDeps), |
||||
} |
||||
} |
||||
}; |
||||
|
||||
Ok(Object { blocks, deps }) |
||||
} |
||||
|
||||
/// Save blocks of the object in the store
|
||||
pub fn save(&self, store: &mut impl RepoStore) -> Result<(), StorageError> { |
||||
let mut deduplicated: HashSet<ObjectId> = HashSet::new(); |
||||
for block in &self.blocks { |
||||
let id = block.id(); |
||||
if deduplicated.get(&id).is_none() { |
||||
store.put(block)?; |
||||
deduplicated.insert(id); |
||||
} |
||||
} |
||||
Ok(()) |
||||
} |
||||
|
||||
/// Get the ID of the Object
|
||||
pub fn id(&self) -> ObjectId { |
||||
self.blocks.last().unwrap().id() |
||||
} |
||||
|
||||
/// Get the key for the Object
|
||||
pub fn key(&self) -> Option<SymKey> { |
||||
self.blocks.last().unwrap().key() |
||||
} |
||||
|
||||
/// Get an `ObjectRef` for the root object
|
||||
pub fn reference(&self) -> Option<ObjectRef> { |
||||
if self.key().is_some() { |
||||
Some(ObjectRef { |
||||
id: self.id(), |
||||
key: self.key().unwrap(), |
||||
}) |
||||
} else { |
||||
None |
||||
} |
||||
} |
||||
|
||||
pub fn root(&self) -> &Block { |
||||
self.blocks.last().unwrap() |
||||
} |
||||
|
||||
pub fn expiry(&self) -> Option<Timestamp> { |
||||
self.blocks.last().unwrap().expiry() |
||||
} |
||||
|
||||
pub fn deps(&self) -> &Vec<ObjectId> { |
||||
&self.deps |
||||
} |
||||
|
||||
pub fn blocks(&self) -> &Vec<Block> { |
||||
&self.blocks |
||||
} |
||||
|
||||
pub fn to_hashmap(&self) -> HashMap<BlockId, Block> { |
||||
let mut map: HashMap<BlockId, Block> = HashMap::new(); |
||||
for block in &self.blocks { |
||||
map.insert(block.id(), block.clone()); |
||||
} |
||||
map |
||||
} |
||||
|
||||
/// Collect leaves from the tree
|
||||
fn collect_leaves( |
||||
blocks: &Vec<Block>, |
||||
parents: &Vec<(ObjectId, SymKey)>, |
||||
parent_index: usize, |
||||
leaves: &mut Option<&mut Vec<Block>>, |
||||
obj_content: &mut Option<&mut Vec<u8>>, |
||||
) -> Result<(), ObjectParseError> { |
||||
/*debug_println!(
|
||||
">>> collect_leaves: #{}..{}", |
||||
parent_index, |
||||
parent_index + parents.len() - 1 |
||||
);*/ |
||||
let mut children: Vec<(ObjectId, SymKey)> = vec![]; |
||||
let mut i = parent_index; |
||||
|
||||
for (id, key) in parents { |
||||
//debug_println!("!!! parent: #{}", i);
|
||||
let block = &blocks[i]; |
||||
i += 1; |
||||
|
||||
// verify object ID
|
||||
if *id != block.id() { |
||||
debug_println!("Invalid ObjectId.\nExp: {:?}\nGot: {:?}", *id, block.id()); |
||||
return Err(ObjectParseError::InvalidBlockId); |
||||
} |
||||
|
||||
match block { |
||||
Block::V0(b) => { |
||||
// decrypt content
|
||||
let mut content_dec = b.content.clone(); |
||||
match key { |
||||
SymKey::ChaCha20Key(key) => { |
||||
let nonce = [0u8; 12]; |
||||
let mut cipher = ChaCha20::new(key.into(), &nonce.into()); |
||||
let mut content_dec_slice = &mut content_dec.as_mut_slice(); |
||||
cipher.apply_keystream(&mut content_dec_slice); |
||||
} |
||||
} |
||||
|
||||
// deserialize content
|
||||
let content: BlockContentV0; |
||||
match serde_bare::from_slice(content_dec.as_slice()) { |
||||
Ok(c) => content = c, |
||||
Err(e) => { |
||||
debug_println!("Block deserialize error: {}", e); |
||||
return Err(ObjectParseError::BlockDeserializeError); |
||||
} |
||||
} |
||||
|
||||
// parse content
|
||||
match content { |
||||
BlockContentV0::InternalNode(keys) => { |
||||
if keys.len() != b.children.len() { |
||||
debug_println!( |
||||
"Invalid keys length: got {}, expected {}", |
||||
keys.len(), |
||||
b.children.len() |
||||
); |
||||
debug_println!("!!! children: {:?}", b.children); |
||||
debug_println!("!!! keys: {:?}", keys); |
||||
return Err(ObjectParseError::InvalidKeys); |
||||
} |
||||
|
||||
for (id, key) in b.children.iter().zip(keys.iter()) { |
||||
children.push((id.clone(), key.clone())); |
||||
} |
||||
} |
||||
BlockContentV0::DataChunk(chunk) => { |
||||
if leaves.is_some() { |
||||
let mut leaf = block.clone(); |
||||
leaf.set_key(Some(*key)); |
||||
let l = &mut **leaves.as_mut().unwrap(); |
||||
l.push(leaf); |
||||
} |
||||
if obj_content.is_some() { |
||||
let c = &mut **obj_content.as_mut().unwrap(); |
||||
c.extend_from_slice(chunk.as_slice()); |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
if !children.is_empty() { |
||||
if parent_index < children.len() { |
||||
return Err(ObjectParseError::InvalidChildren); |
||||
} |
||||
match Self::collect_leaves( |
||||
blocks, |
||||
&children, |
||||
parent_index - children.len(), |
||||
leaves, |
||||
obj_content, |
||||
) { |
||||
Ok(_) => (), |
||||
Err(e) => return Err(e), |
||||
} |
||||
} |
||||
Ok(()) |
||||
} |
||||
|
||||
/// Parse the Object and return the leaf Blocks with decryption key set
|
||||
pub fn leaves(&self) -> Result<Vec<Block>, ObjectParseError> { |
||||
let mut leaves: Vec<Block> = vec![]; |
||||
let parents = vec![(self.id(), self.key().unwrap())]; |
||||
match Self::collect_leaves( |
||||
&self.blocks, |
||||
&parents, |
||||
self.blocks.len() - 1, |
||||
&mut Some(&mut leaves), |
||||
&mut None, |
||||
) { |
||||
Ok(_) => Ok(leaves), |
||||
Err(e) => Err(e), |
||||
} |
||||
} |
||||
|
||||
/// Parse the Object and return the decrypted content assembled from Blocks
|
||||
pub fn content(&self) -> Result<ObjectContent, ObjectParseError> { |
||||
if self.key().is_none() { |
||||
return Err(ObjectParseError::MissingRootKey); |
||||
} |
||||
let mut obj_content: Vec<u8> = vec![]; |
||||
let parents = vec![(self.id(), self.key().unwrap())]; |
||||
match Self::collect_leaves( |
||||
&self.blocks, |
||||
&parents, |
||||
self.blocks.len() - 1, |
||||
&mut None, |
||||
&mut Some(&mut obj_content), |
||||
) { |
||||
Ok(_) => { |
||||
let content: ObjectContent; |
||||
match serde_bare::from_slice(obj_content.as_slice()) { |
||||
Ok(c) => Ok(c), |
||||
Err(e) => { |
||||
debug_println!("Object deserialize error: {}", e); |
||||
Err(ObjectParseError::ObjectDeserializeError) |
||||
} |
||||
} |
||||
} |
||||
Err(e) => Err(e), |
||||
} |
||||
} |
||||
} |
||||
|
||||
#[cfg(test)] |
||||
mod test { |
||||
|
||||
use crate::object::*; |
||||
use crate::store::*; |
||||
use crate::types::*; |
||||
|
||||
// Those constants are calculated with RepoStore::get_max_value_size
|
||||
|
||||
/// Maximum arity of branch containing max number of leaves
|
||||
const MAX_ARITY_LEAVES: usize = 31774; |
||||
/// Maximum arity of root branch
|
||||
const MAX_ARITY_ROOT: usize = 31770; |
||||
/// Maximum data that can fit in object.content
|
||||
const MAX_DATA_PAYLOAD_SIZE: usize = 2097112; |
||||
|
||||
/// Test tree API
|
||||
#[test] |
||||
pub fn test_object() { |
||||
let file = File::V0(FileV0 { |
||||
content_type: Vec::from("file/test"), |
||||
metadata: Vec::from("some meta data here"), |
||||
content: [(0..255).collect::<Vec<u8>>().as_slice(); 320].concat(), |
||||
}); |
||||
let content = ObjectContent::File(file); |
||||
|
||||
let deps: Vec<ObjectId> = vec![Digest::Blake3Digest32([9; 32])]; |
||||
let exp = Some(2u32.pow(31)); |
||||
let max_object_size = 0; |
||||
|
||||
let repo_secret = SymKey::ChaCha20Key([0; 32]); |
||||
let repo_pubkey = PubKey::Ed25519PubKey([1; 32]); |
||||
|
||||
let obj = Object::new( |
||||
content.clone(), |
||||
deps.clone(), |
||||
exp, |
||||
max_object_size, |
||||
repo_pubkey, |
||||
repo_secret, |
||||
); |
||||
|
||||
println!("obj.id: {:?}", obj.id()); |
||||
println!("obj.key: {:?}", obj.key()); |
||||
println!("obj.deps: {:?}", obj.deps()); |
||||
println!("obj.blocks.len: {:?}", obj.blocks().len()); |
||||
|
||||
let mut i = 0; |
||||
for node in obj.blocks() { |
||||
println!("#{}: {:?}", i, node.id()); |
||||
i += 1; |
||||
} |
||||
|
||||
assert_eq!(*obj.deps(), deps); |
||||
|
||||
match obj.content() { |
||||
Ok(cnt) => { |
||||
assert_eq!(content, cnt); |
||||
} |
||||
Err(e) => panic!("Object parse error: {:?}", e), |
||||
} |
||||
let mut store = HashMapRepoStore::new(); |
||||
|
||||
obj.save(&mut store).expect("Object save error"); |
||||
|
||||
let obj2 = Object::load(obj.id(), obj.key(), &store).unwrap(); |
||||
|
||||
println!("obj2.id: {:?}", obj2.id()); |
||||
println!("obj2.key: {:?}", obj2.key()); |
||||
println!("obj2.deps: {:?}", obj2.deps()); |
||||
println!("obj2.blocks.len: {:?}", obj2.blocks().len()); |
||||
let mut i = 0; |
||||
for node in obj2.blocks() { |
||||
println!("#{}: {:?}", i, node.id()); |
||||
i += 1; |
||||
} |
||||
|
||||
assert_eq!(*obj2.deps(), deps); |
||||
assert_eq!(*obj2.deps(), deps); |
||||
|
||||
match obj2.content() { |
||||
Ok(cnt) => { |
||||
assert_eq!(content, cnt); |
||||
} |
||||
Err(e) => panic!("Object2 parse error: {:?}", e), |
||||
} |
||||
|
||||
let obj3 = Object::load(obj.id(), None, &store).unwrap(); |
||||
|
||||
println!("obj3.id: {:?}", obj3.id()); |
||||
println!("obj3.key: {:?}", obj3.key()); |
||||
println!("obj3.deps: {:?}", obj3.deps()); |
||||
println!("obj3.blocks.len: {:?}", obj3.blocks().len()); |
||||
let mut i = 0; |
||||
for node in obj3.blocks() { |
||||
println!("#{}: {:?}", i, node.id()); |
||||
i += 1; |
||||
} |
||||
|
||||
assert_eq!(*obj3.deps(), deps); |
||||
|
||||
match obj3.content() { |
||||
Err(ObjectParseError::MissingRootKey) => (), |
||||
Err(e) => panic!("Object3 parse error: {:?}", e), |
||||
Ok(_) => panic!("Object3 should not return content"), |
||||
} |
||||
|
||||
let exp4 = Some(2342); |
||||
let obj4 = obj.copy(exp4, repo_pubkey, repo_secret).unwrap(); |
||||
obj4.save(&mut store).unwrap(); |
||||
|
||||
assert_eq!(obj4.expiry(), exp4); |
||||
assert_eq!(*obj.deps(), deps); |
||||
|
||||
match obj4.content() { |
||||
Ok(cnt) => { |
||||
assert_eq!(content, cnt); |
||||
} |
||||
Err(e) => panic!("Object3 parse error: {:?}", e), |
||||
} |
||||
} |
||||
|
||||
/// Checks that a content that fits the root node, will not be chunked into children nodes
|
||||
#[test] |
||||
pub fn test_depth_1() { |
||||
let deps: Vec<ObjectId> = vec![Digest::Blake3Digest32([9; 32])]; |
||||
|
||||
let empty_file = ObjectContent::File(File::V0(FileV0 { |
||||
content_type: vec![], |
||||
metadata: vec![], |
||||
content: vec![], |
||||
})); |
||||
let empty_file_ser = serde_bare::to_vec(&empty_file).unwrap(); |
||||
println!("empty file size: {}", empty_file_ser.len()); |
||||
|
||||
let size = store_max_value_size() |
||||
- EMPTY_BLOCK_SIZE |
||||
- DATA_VARINT_EXTRA |
||||
- BLOCK_ID_SIZE * deps.len() |
||||
- empty_file_ser.len() |
||||
- DATA_VARINT_EXTRA; |
||||
println!("file size: {}", size); |
||||
|
||||
let content = ObjectContent::File(File::V0(FileV0 { |
||||
content_type: vec![], |
||||
metadata: vec![], |
||||
content: vec![99; size], |
||||
})); |
||||
let content_ser = serde_bare::to_vec(&content).unwrap(); |
||||
println!("content len: {}", content_ser.len()); |
||||
|
||||
let expiry = Some(2u32.pow(31)); |
||||
let max_object_size = store_max_value_size(); |
||||
|
||||
let repo_secret = SymKey::ChaCha20Key([0; 32]); |
||||
let repo_pubkey = PubKey::Ed25519PubKey([1; 32]); |
||||
|
||||
let object = Object::new( |
||||
content, |
||||
deps, |
||||
expiry, |
||||
max_object_size, |
||||
repo_pubkey, |
||||
repo_secret, |
||||
); |
||||
|
||||
println!("root_id: {:?}", object.id()); |
||||
println!("root_key: {:?}", object.key().unwrap()); |
||||
println!("nodes.len: {:?}", object.blocks().len()); |
||||
//println!("root: {:?}", tree.root());
|
||||
//println!("nodes: {:?}", object.blocks);
|
||||
assert_eq!(object.blocks.len(), 1); |
||||
} |
||||
|
||||
#[test] |
||||
pub fn test_block_size() { |
||||
let max_block_size = store_max_value_size(); |
||||
println!("max_object_size: {}", max_block_size); |
||||
|
||||
let id = Digest::Blake3Digest32([0u8; 32]); |
||||
let key = SymKey::ChaCha20Key([0u8; 32]); |
||||
|
||||
let one_key = BlockContentV0::InternalNode(vec![key]); |
||||
let one_key_ser = serde_bare::to_vec(&one_key).unwrap(); |
||||
|
||||
let two_keys = BlockContentV0::InternalNode(vec![key, key]); |
||||
let two_keys_ser = serde_bare::to_vec(&two_keys).unwrap(); |
||||
|
||||
let max_keys = BlockContentV0::InternalNode(vec![key; MAX_ARITY_LEAVES]); |
||||
let max_keys_ser = serde_bare::to_vec(&max_keys).unwrap(); |
||||
|
||||
let data = BlockContentV0::DataChunk(vec![]); |
||||
let data_ser = serde_bare::to_vec(&data).unwrap(); |
||||
|
||||
let data_full = BlockContentV0::DataChunk(vec![0; MAX_DATA_PAYLOAD_SIZE]); |
||||
let data_full_ser = serde_bare::to_vec(&data_full).unwrap(); |
||||
|
||||
let leaf_empty = Block::new( |
||||
vec![], |
||||
ObjectDeps::ObjectIdList(vec![]), |
||||
Some(2342), |
||||
data_ser.clone(), |
||||
None, |
||||
); |
||||
let leaf_empty_ser = serde_bare::to_vec(&leaf_empty).unwrap(); |
||||
|
||||
let leaf_full_data = Block::new( |
||||
vec![], |
||||
ObjectDeps::ObjectIdList(vec![]), |
||||
Some(2342), |
||||
data_full_ser.clone(), |
||||
None, |
||||
); |
||||
let leaf_full_data_ser = serde_bare::to_vec(&leaf_full_data).unwrap(); |
||||
|
||||
let root_depsref = Block::new( |
||||
vec![], |
||||
ObjectDeps::DepListRef(ObjectRef { id: id, key: key }), |
||||
Some(2342), |
||||
data_ser.clone(), |
||||
None, |
||||
); |
||||
|
||||
let root_depsref_ser = serde_bare::to_vec(&root_depsref).unwrap(); |
||||
|
||||
let internal_max = Block::new( |
||||
vec![id; MAX_ARITY_LEAVES], |
||||
ObjectDeps::ObjectIdList(vec![]), |
||||
Some(2342), |
||||
max_keys_ser.clone(), |
||||
None, |
||||
); |
||||
let internal_max_ser = serde_bare::to_vec(&internal_max).unwrap(); |
||||
|
||||
let internal_one = Block::new( |
||||
vec![id; 1], |
||||
ObjectDeps::ObjectIdList(vec![]), |
||||
Some(2342), |
||||
one_key_ser.clone(), |
||||
None, |
||||
); |
||||
let internal_one_ser = serde_bare::to_vec(&internal_one).unwrap(); |
||||
|
||||
let internal_two = Block::new( |
||||
vec![id; 2], |
||||
ObjectDeps::ObjectIdList(vec![]), |
||||
Some(2342), |
||||
two_keys_ser.clone(), |
||||
None, |
||||
); |
||||
let internal_two_ser = serde_bare::to_vec(&internal_two).unwrap(); |
||||
|
||||
let root_one = Block::new( |
||||
vec![id; 1], |
||||
ObjectDeps::ObjectIdList(vec![id; 8]), |
||||
Some(2342), |
||||
one_key_ser.clone(), |
||||
None, |
||||
); |
||||
let root_one_ser = serde_bare::to_vec(&root_one).unwrap(); |
||||
|
||||
let root_two = Block::new( |
||||
vec![id; 2], |
||||
ObjectDeps::ObjectIdList(vec![id; 8]), |
||||
Some(2342), |
||||
two_keys_ser.clone(), |
||||
None, |
||||
); |
||||
let root_two_ser = serde_bare::to_vec(&root_two).unwrap(); |
||||
|
||||
println!( |
||||
"range of valid value sizes {} {}", |
||||
store_valid_value_size(0), |
||||
store_max_value_size() |
||||
); |
||||
|
||||
println!( |
||||
"max_data_payload_of_object: {}", |
||||
max_block_size - EMPTY_BLOCK_SIZE - DATA_VARINT_EXTRA |
||||
); |
||||
|
||||
println!( |
||||
"max_data_payload_depth_1: {}", |
||||
max_block_size - EMPTY_BLOCK_SIZE - DATA_VARINT_EXTRA - MAX_DEPS_SIZE |
||||
); |
||||
|
||||
println!( |
||||
"max_data_payload_depth_2: {}", |
||||
MAX_ARITY_ROOT * MAX_DATA_PAYLOAD_SIZE |
||||
); |
||||
|
||||
println!( |
||||
"max_data_payload_depth_3: {}", |
||||
MAX_ARITY_ROOT * MAX_ARITY_LEAVES * MAX_DATA_PAYLOAD_SIZE |
||||
); |
||||
|
||||
let max_arity_leaves = (max_block_size - EMPTY_BLOCK_SIZE - BIG_VARINT_EXTRA * 2) |
||||
/ (BLOCK_ID_SIZE + BLOCK_KEY_SIZE); |
||||
println!("max_arity_leaves: {}", max_arity_leaves); |
||||
assert_eq!(max_arity_leaves, MAX_ARITY_LEAVES); |
||||
assert_eq!( |
||||
max_block_size - EMPTY_BLOCK_SIZE - DATA_VARINT_EXTRA, |
||||
MAX_DATA_PAYLOAD_SIZE |
||||
); |
||||
let max_arity_root = |
||||
(max_block_size - EMPTY_BLOCK_SIZE - MAX_DEPS_SIZE - BIG_VARINT_EXTRA * 2) |
||||
/ (BLOCK_ID_SIZE + BLOCK_KEY_SIZE); |
||||
println!("max_arity_root: {}", max_arity_root); |
||||
assert_eq!(max_arity_root, MAX_ARITY_ROOT); |
||||
println!("store_max_value_size: {}", leaf_full_data_ser.len()); |
||||
assert_eq!(leaf_full_data_ser.len(), max_block_size); |
||||
println!("leaf_empty: {}", leaf_empty_ser.len()); |
||||
assert_eq!(leaf_empty_ser.len(), EMPTY_BLOCK_SIZE); |
||||
println!("root_depsref: {}", root_depsref_ser.len()); |
||||
assert_eq!(root_depsref_ser.len(), EMPTY_ROOT_SIZE_DEPSREF); |
||||
println!("internal_max: {}", internal_max_ser.len()); |
||||
assert_eq!( |
||||
internal_max_ser.len(), |
||||
EMPTY_BLOCK_SIZE |
||||
+ BIG_VARINT_EXTRA * 2 |
||||
+ MAX_ARITY_LEAVES * (BLOCK_ID_SIZE + BLOCK_KEY_SIZE) |
||||
); |
||||
assert!(internal_max_ser.len() < max_block_size); |
||||
println!("internal_one: {}", internal_one_ser.len()); |
||||
assert_eq!( |
||||
internal_one_ser.len(), |
||||
EMPTY_BLOCK_SIZE + 1 * BLOCK_ID_SIZE + 1 * BLOCK_KEY_SIZE |
||||
); |
||||
println!("internal_two: {}", internal_two_ser.len()); |
||||
assert_eq!( |
||||
internal_two_ser.len(), |
||||
EMPTY_BLOCK_SIZE + 2 * BLOCK_ID_SIZE + 2 * BLOCK_KEY_SIZE |
||||
); |
||||
println!("root_one: {}", root_one_ser.len()); |
||||
assert_eq!( |
||||
root_one_ser.len(), |
||||
EMPTY_BLOCK_SIZE + 8 * BLOCK_ID_SIZE + 1 * BLOCK_ID_SIZE + 1 * BLOCK_KEY_SIZE |
||||
); |
||||
println!("root_two: {}", root_two_ser.len()); |
||||
assert_eq!( |
||||
root_two_ser.len(), |
||||
EMPTY_BLOCK_SIZE + 8 * BLOCK_ID_SIZE + 2 * BLOCK_ID_SIZE + 2 * BLOCK_KEY_SIZE |
||||
); |
||||
|
||||
// let object_size_1 = 4096 * 1 - VALUE_HEADER_SIZE;
|
||||
// let object_size_512 = 4096 * MAX_PAGES_PER_VALUE - VALUE_HEADER_SIZE;
|
||||
// let arity_1: usize =
|
||||
// (object_size_1 - 8 * OBJECT_ID_SIZE) / (OBJECT_ID_SIZE + OBJECT_KEY_SIZE);
|
||||
// let arity_512: usize =
|
||||
// (object_size_512 - 8 * OBJECT_ID_SIZE) / (OBJECT_ID_SIZE + OBJECT_KEY_SIZE);
|
||||
|
||||
// println!("1-page object_size: {}", object_size_1);
|
||||
// println!("512-page object_size: {}", object_size_512);
|
||||
// println!("max arity of 1-page object: {}", arity_1);
|
||||
// println!("max arity of 512-page object: {}", arity_512);
|
||||
} |
||||
} |
||||
@ -0,0 +1,46 @@ |
||||
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers
|
||||
// All rights reserved.
|
||||
// This code is partly derived from work written by TG x Thoth from P2Pcollab.
|
||||
// Copyright 2022 TG x Thoth
|
||||
// Licensed under the Apache License, Version 2.0
|
||||
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
|
||||
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
|
||||
// at your option. All files in the project carrying such
|
||||
// notice may not be copied, modified, or distributed except
|
||||
// according to those terms.
|
||||
|
||||
//! Repository
|
||||
|
||||
use crate::types::*; |
||||
|
||||
impl RepositoryV0 { |
||||
pub fn new( |
||||
id: &PubKey, |
||||
branches: &Vec<ObjectRef>, |
||||
allow_ext_requests: bool, |
||||
metadata: &Vec<u8>, |
||||
) -> RepositoryV0 { |
||||
RepositoryV0 { |
||||
id: id.clone(), |
||||
branches: branches.clone(), |
||||
allow_ext_requests, |
||||
metadata: metadata.clone(), |
||||
} |
||||
} |
||||
} |
||||
|
||||
impl Repository { |
||||
pub fn new( |
||||
id: &PubKey, |
||||
branches: &Vec<ObjectRef>, |
||||
allow_ext_requests: bool, |
||||
metadata: &Vec<u8>, |
||||
) -> Repository { |
||||
Repository::V0(RepositoryV0::new( |
||||
id, |
||||
branches, |
||||
allow_ext_requests, |
||||
metadata, |
||||
)) |
||||
} |
||||
} |
||||
@ -0,0 +1,109 @@ |
||||
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers
|
||||
// All rights reserved.
|
||||
// This code is partly derived from work written by TG x Thoth from P2Pcollab.
|
||||
// Copyright 2022 TG x Thoth
|
||||
// Licensed under the Apache License, Version 2.0
|
||||
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
|
||||
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
|
||||
// at your option. All files in the project carrying such
|
||||
// notice may not be copied, modified, or distributed except
|
||||
// according to those terms.
|
||||
|
||||
//! Block store
|
||||
|
||||
use crate::types::*; |
||||
|
||||
use std::{ |
||||
cmp::min, |
||||
collections::{hash_map::Iter, HashMap}, |
||||
mem::size_of_val, |
||||
}; |
||||
use std::sync::{Arc, RwLock}; |
||||
|
||||
pub trait RepoStore { |
||||
/// Load a block from the store.
|
||||
fn get(&self, id: &BlockId) -> Result<Block, StorageError>; |
||||
|
||||
/// Save a block to the store.
|
||||
fn put(&self, block: &Block) -> Result<BlockId, StorageError>; |
||||
|
||||
/// Delete a block from the store.
|
||||
fn del(&self, id: &BlockId) -> Result<(Block, usize), StorageError>; |
||||
} |
||||
|
||||
#[derive(Debug, PartialEq)] |
||||
pub enum StorageError { |
||||
NotFound, |
||||
InvalidValue, |
||||
BackendError, |
||||
SerializationError, |
||||
} |
||||
|
||||
impl From<serde_bare::error::Error> for StorageError { |
||||
fn from(e: serde_bare::error::Error) -> Self { |
||||
StorageError::SerializationError |
||||
} |
||||
} |
||||
|
||||
const MIN_SIZE: usize = 4072; |
||||
const PAGE_SIZE: usize = 4096; |
||||
const HEADER: usize = PAGE_SIZE - MIN_SIZE; |
||||
const MAX_FACTOR: usize = 512; |
||||
|
||||
/// Returns a valid/optimal value size for the entries of the storage backend.
|
||||
pub fn store_valid_value_size(size: usize) -> usize { |
||||
min( |
||||
((size + HEADER) as f32 / PAGE_SIZE as f32).ceil() as usize, |
||||
MAX_FACTOR, |
||||
) * PAGE_SIZE |
||||
- HEADER |
||||
} |
||||
|
||||
/// Returns the maximum value size for the entries of the storage backend.
|
||||
pub const fn store_max_value_size() -> usize { |
||||
MAX_FACTOR * PAGE_SIZE - HEADER |
||||
} |
||||
|
||||
/// Store with a HashMap backend
|
||||
pub struct HashMapRepoStore { |
||||
blocks: RwLock<HashMap<BlockId, Block>>, |
||||
} |
||||
|
||||
impl HashMapRepoStore { |
||||
pub fn new() -> HashMapRepoStore { |
||||
HashMapRepoStore { |
||||
blocks: RwLock::new(HashMap::new()), |
||||
} |
||||
} |
||||
|
||||
pub fn get_len(&self) -> usize { |
||||
self.blocks.read().unwrap().len() |
||||
} |
||||
|
||||
pub fn get_all(&self) -> Vec<Block> { |
||||
self.blocks.read().unwrap().values().map(|x| x.clone()).collect() |
||||
} |
||||
} |
||||
|
||||
impl RepoStore for HashMapRepoStore { |
||||
fn get(&self, id: &BlockId) -> Result<Block, StorageError> { |
||||
match self.blocks.read().unwrap().get(id) { |
||||
Some(block) => Ok(block.clone()), |
||||
None => Err(StorageError::NotFound), |
||||
} |
||||
} |
||||
|
||||
fn put(&self, block: &Block) -> Result<BlockId, StorageError> { |
||||
let id = block.id(); |
||||
let mut b = block.clone(); |
||||
b.set_key(None); |
||||
self.blocks.write().unwrap().insert(id, b); |
||||
Ok(id) |
||||
} |
||||
|
||||
fn del(&self, id: &BlockId) -> Result<(Block, usize), StorageError> { |
||||
let block = self.blocks.write().unwrap().remove(id).ok_or(StorageError::NotFound)?; |
||||
let size = size_of_val(&block); |
||||
Ok((block, size)) |
||||
} |
||||
} |
||||
@ -0,0 +1,530 @@ |
||||
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers
|
||||
// All rights reserved.
|
||||
// This code is partly derived from work written by TG x Thoth from P2Pcollab.
|
||||
// Copyright 2022 TG x Thoth
|
||||
// Licensed under the Apache License, Version 2.0
|
||||
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
|
||||
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
|
||||
// at your option. All files in the project carrying such
|
||||
// notice may not be copied, modified, or distributed except
|
||||
// according to those terms.
|
||||
|
||||
//! P2P Repo types
|
||||
//!
|
||||
//! Corresponds to the BARE schema
|
||||
|
||||
use core::fmt; |
||||
use serde::{Deserialize, Serialize}; |
||||
use std::collections::HashMap; |
||||
use std::hash::Hash; |
||||
|
||||
//
|
||||
// COMMON TYPES
|
||||
//
|
||||
|
||||
/// 32-byte Blake3 hash digest
|
||||
pub type Blake3Digest32 = [u8; 32]; |
||||
|
||||
/// Hash digest
|
||||
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] |
||||
pub enum Digest { |
||||
Blake3Digest32(Blake3Digest32), |
||||
} |
||||
|
||||
impl fmt::Display for Digest { |
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { |
||||
match self { |
||||
Digest::Blake3Digest32(d) => write!(f, "{}", hex::encode(d)), |
||||
} |
||||
} |
||||
} |
||||
|
||||
/// ChaCha20 symmetric key
|
||||
pub type ChaCha20Key = [u8; 32]; |
||||
|
||||
/// Symmetric cryptographic key
|
||||
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] |
||||
pub enum SymKey { |
||||
ChaCha20Key(ChaCha20Key), |
||||
} |
||||
|
||||
impl SymKey { |
||||
pub fn slice(&self) -> &[u8; 32] { |
||||
match self { |
||||
SymKey::ChaCha20Key(o) => o, |
||||
} |
||||
} |
||||
} |
||||
|
||||
/// Curve25519 public key
|
||||
pub type Ed25519PubKey = [u8; 32]; |
||||
|
||||
/// Curve25519 private key
|
||||
pub type Ed25519PrivKey = [u8; 32]; |
||||
|
||||
/// Public key
|
||||
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] |
||||
pub enum PubKey { |
||||
Ed25519PubKey(Ed25519PubKey), |
||||
} |
||||
|
||||
impl PubKey { |
||||
pub fn slice(&self) -> &[u8; 32] { |
||||
match self { |
||||
PubKey::Ed25519PubKey(o) => o, |
||||
} |
||||
} |
||||
} |
||||
|
||||
impl fmt::Display for PubKey { |
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { |
||||
match self { |
||||
PubKey::Ed25519PubKey(d) => write!(f, "{}", hex::encode(d)), |
||||
} |
||||
} |
||||
} |
||||
|
||||
/// Private key
|
||||
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] |
||||
pub enum PrivKey { |
||||
Ed25519PrivKey(Ed25519PrivKey), |
||||
} |
||||
|
||||
/// Ed25519 signature
|
||||
pub type Ed25519Sig = [[u8; 32]; 2]; |
||||
|
||||
/// Cryptographic signature
|
||||
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] |
||||
pub enum Sig { |
||||
Ed25519Sig(Ed25519Sig), |
||||
} |
||||
|
||||
/// Timestamp: absolute time in minutes since 2022-02-22 22:22 UTC
|
||||
pub type Timestamp = u32; |
||||
|
||||
pub const EPOCH_AS_UNIX_TIMESTAMP: u64 = 1645568520; |
||||
|
||||
/// Relative time (e.g. delay from current time)
|
||||
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] |
||||
pub enum RelTime { |
||||
Seconds(u8), |
||||
Minutes(u8), |
||||
Hours(u8), |
||||
Days(u8), |
||||
} |
||||
|
||||
/// Bloom filter (variable size)
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] |
||||
pub struct BloomFilter { |
||||
/// Number of hash functions
|
||||
pub k: u32, |
||||
|
||||
/// Filter
|
||||
#[serde(with = "serde_bytes")] |
||||
pub f: Vec<u8>, |
||||
} |
||||
|
||||
/// Bloom filter (128 B)
|
||||
///
|
||||
/// (m=1024; k=7; p=0.01; n=107)
|
||||
pub type BloomFilter128 = [[u8; 32]; 4]; |
||||
|
||||
/// Bloom filter (1 KiB)
|
||||
///
|
||||
/// (m=8192; k=7; p=0.01; n=855)
|
||||
pub type BloomFilter1K = [[u8; 32]; 32]; |
||||
|
||||
//
|
||||
// REPOSITORY TYPES
|
||||
//
|
||||
|
||||
/// Block ID:
|
||||
/// BLAKE3 hash over the serialized Object with encrypted content
|
||||
pub type BlockId = Digest; |
||||
|
||||
/// Block reference
|
||||
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] |
||||
pub struct BlockRef { |
||||
/// Object ID
|
||||
pub id: BlockId, |
||||
|
||||
/// Key for decrypting the Object
|
||||
pub key: SymKey, |
||||
} |
||||
|
||||
/// Object ID
|
||||
pub type ObjectId = BlockId; |
||||
|
||||
/// Object reference
|
||||
pub type ObjectRef = BlockRef; |
||||
|
||||
/// Internal node of a Merkle tree
|
||||
pub type InternalNode = Vec<SymKey>; |
||||
|
||||
/// Content of BlockV0: a Merkle tree node
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] |
||||
pub enum BlockContentV0 { |
||||
/// Internal node with references to children
|
||||
InternalNode(InternalNode), |
||||
|
||||
#[serde(with = "serde_bytes")] |
||||
DataChunk(Vec<u8>), |
||||
} |
||||
|
||||
/// List of ObjectId dependencies as encrypted Object content
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] |
||||
pub enum DepList { |
||||
V0(Vec<ObjectId>), |
||||
} |
||||
|
||||
/// Dependencies of an Object
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] |
||||
pub enum ObjectDeps { |
||||
/// List of Object IDs (max. 8),
|
||||
ObjectIdList(Vec<ObjectId>), |
||||
|
||||
/// Reference to an Object that contains a DepList
|
||||
DepListRef(ObjectRef), |
||||
} |
||||
|
||||
/// Immutable block with encrypted content
|
||||
///
|
||||
/// `ObjectContent` is chunked and stored as `Block`s in a Merkle tree.
|
||||
/// A Block is a Merkle tree node.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] |
||||
pub struct BlockV0 { |
||||
/// Block ID
|
||||
#[serde(skip)] |
||||
pub id: Option<BlockId>, |
||||
|
||||
/// Block Key
|
||||
#[serde(skip)] |
||||
pub key: Option<SymKey>, |
||||
|
||||
/// Block IDs for child nodes in the Merkle tree
|
||||
pub children: Vec<BlockId>, |
||||
|
||||
/// Other objects this object depends on (e.g. Commit deps & acks)
|
||||
/// Only set for the root block
|
||||
pub deps: ObjectDeps, |
||||
|
||||
/// Expiry time of this object and all of its children
|
||||
/// when the object should be deleted by all replicas
|
||||
/// Only set for the root block
|
||||
pub expiry: Option<Timestamp>, |
||||
|
||||
/// Encrypted ObjectContentV0
|
||||
///
|
||||
/// Encrypted using convergent encryption with ChaCha20:
|
||||
/// - convergence_key: BLAKE3 derive_key ("NextGraph Data BLAKE3 key",
|
||||
/// repo_pubkey + repo_secret)
|
||||
/// - key: BLAKE3 keyed hash (convergence_key, plain_object_content)
|
||||
/// - nonce: 0
|
||||
#[serde(with = "serde_bytes")] |
||||
pub content: Vec<u8>, |
||||
} |
||||
|
||||
/// Immutable object with encrypted content
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] |
||||
pub enum Block { |
||||
V0(BlockV0), |
||||
} |
||||
|
||||
/// Repository definition
|
||||
///
|
||||
/// Published in root branch, where:
|
||||
/// - branch_pubkey: repo_pubkey
|
||||
/// - branch_secret: BLAKE3 derive_key ("NextGraph Root Branch secret",
|
||||
/// repo_pubkey + repo_secret)
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] |
||||
pub struct RepositoryV0 { |
||||
/// Repo public key ID
|
||||
pub id: PubKey, |
||||
|
||||
/// List of branches
|
||||
pub branches: Vec<ObjectRef>, |
||||
|
||||
/// Whether or not to allow external requests
|
||||
pub allow_ext_requests: bool, |
||||
|
||||
/// App-specific metadata
|
||||
#[serde(with = "serde_bytes")] |
||||
pub metadata: Vec<u8>, |
||||
} |
||||
|
||||
/// Repository definition
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] |
||||
pub enum Repository { |
||||
V0(RepositoryV0), |
||||
} |
||||
|
||||
/// Add a branch to the repository
|
||||
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] |
||||
pub enum AddBranch { |
||||
V0(ObjectRef), |
||||
} |
||||
|
||||
/// Remove a branch from the repository
|
||||
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] |
||||
pub enum RemoveBranch { |
||||
V0(ObjectRef), |
||||
} |
||||
|
||||
/// Commit object types
|
||||
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] |
||||
pub enum CommitType { |
||||
Repository, |
||||
AddBranch, |
||||
RemoveBranch, |
||||
Branch, |
||||
AddMembers, |
||||
EndOfBranch, |
||||
Transaction, |
||||
Snapshot, |
||||
Ack, |
||||
} |
||||
|
||||
/// Member of a Branch
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] |
||||
pub struct MemberV0 { |
||||
/// Member public key ID
|
||||
pub id: PubKey, |
||||
|
||||
/// Commit types the member is allowed to publish in the branch
|
||||
pub commit_types: Vec<CommitType>, |
||||
|
||||
/// App-specific metadata
|
||||
/// (role, permissions, cryptographic material, etc)
|
||||
#[serde(with = "serde_bytes")] |
||||
pub metadata: Vec<u8>, |
||||
} |
||||
|
||||
/// Member of a branch
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] |
||||
pub enum Member { |
||||
V0(MemberV0), |
||||
} |
||||
|
||||
/// Branch definition
|
||||
///
|
||||
/// First commit in a branch, signed by branch key
|
||||
/// In case of a fork, the commit deps indicat
|
||||
/// the previous branch heads.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] |
||||
pub struct BranchV0 { |
||||
/// Branch public key ID
|
||||
pub id: PubKey, |
||||
|
||||
/// Pub/sub topic for publishing events
|
||||
pub topic: PubKey, |
||||
|
||||
/// Branch secret key
|
||||
pub secret: SymKey, |
||||
|
||||
/// Members with permissions
|
||||
pub members: Vec<MemberV0>, |
||||
|
||||
/// Number of acks required for a commit to be valid
|
||||
pub quorum: HashMap<CommitType, u32>, |
||||
|
||||
/// Delay to send explicit acks,
|
||||
/// if not enough implicit acks arrived by then
|
||||
pub ack_delay: RelTime, |
||||
|
||||
/// Tags for organizing branches within the repository
|
||||
#[serde(with = "serde_bytes")] |
||||
pub tags: Vec<u8>, |
||||
|
||||
/// App-specific metadata (validation rules, etc)
|
||||
#[serde(with = "serde_bytes")] |
||||
pub metadata: Vec<u8>, |
||||
} |
||||
|
||||
/// Branch definition
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] |
||||
pub enum Branch { |
||||
V0(BranchV0), |
||||
} |
||||
|
||||
/// Add members to an existing branch
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] |
||||
pub struct AddMembersV0 { |
||||
/// Members to add, with permissions
|
||||
pub members: Vec<MemberV0>, |
||||
|
||||
/// New quorum
|
||||
pub quorum: Option<HashMap<CommitType, u32>>, |
||||
|
||||
/// New ackDelay
|
||||
pub ack_delay: Option<RelTime>, |
||||
} |
||||
|
||||
/// Add members to an existing branch
|
||||
///
|
||||
/// If a member already exists, it overwrites the previous definition,
|
||||
/// in that case this can only be used for adding new permissions,
|
||||
/// not to remove existing ones.
|
||||
/// The quorum and ackDelay can be changed as well
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] |
||||
pub enum AddMembers { |
||||
V0(AddMembersV0), |
||||
} |
||||
|
||||
/// ObjectRef for EndOfBranch
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] |
||||
pub enum PlainOrEncryptedObjectRef { |
||||
Plain(ObjectRef), |
||||
Encrypted(Vec<u8>), |
||||
} |
||||
|
||||
/// End of branch
|
||||
///
|
||||
/// No more commits accepted afterwards, only acks of this commit
|
||||
/// May reference a fork where the branch continues
|
||||
/// with possibly different members, permissions, validation rules.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] |
||||
pub struct EndOfBranchV0 { |
||||
/// (Encrypted) reference to forked branch (optional)
|
||||
pub fork: Option<PlainOrEncryptedObjectRef>, |
||||
|
||||
/// Expiry time when all commits in the branch should be deleted
|
||||
pub expiry: Timestamp, |
||||
} |
||||
|
||||
/// End of branch
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] |
||||
pub enum EndOfBranch { |
||||
V0(EndOfBranchV0), |
||||
} |
||||
|
||||
/// Transaction with CRDT operations
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] |
||||
pub enum Transaction { |
||||
#[serde(with = "serde_bytes")] |
||||
V0(Vec<u8>), |
||||
} |
||||
|
||||
/// Snapshot of a Branch
|
||||
///
|
||||
/// Contains a data structure
|
||||
/// computed from the commits at the specified head.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] |
||||
pub struct SnapshotV0 { |
||||
/// Branch heads the snapshot was made from
|
||||
pub heads: Vec<ObjectId>, |
||||
|
||||
/// Snapshot data structure
|
||||
#[serde(with = "serde_bytes")] |
||||
pub content: Vec<u8>, |
||||
} |
||||
|
||||
/// Snapshot of a Branch
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] |
||||
pub enum Snapshot { |
||||
V0(SnapshotV0), |
||||
} |
||||
|
||||
/// Acknowledgement of another Commit
|
||||
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] |
||||
pub enum Ack { |
||||
V0(), |
||||
} |
||||
|
||||
/// Commit body, corresponds to CommitType
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] |
||||
pub enum CommitBody { |
||||
Repository(Repository), |
||||
AddBranch(AddBranch), |
||||
RemoveBranch(RemoveBranch), |
||||
Branch(Branch), |
||||
AddMembers(AddMembers), |
||||
EndOfBranch(EndOfBranch), |
||||
Transaction(Transaction), |
||||
Snapshot(Snapshot), |
||||
Ack(Ack), |
||||
} |
||||
|
||||
/// Content of a Commit
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] |
||||
pub struct CommitContentV0 { |
||||
/// Commit author
|
||||
pub author: PubKey, |
||||
|
||||
/// Author's commit sequence number in this branch
|
||||
pub seq: u32, |
||||
|
||||
/// Branch the commit belongs to
|
||||
pub branch: ObjectRef, |
||||
|
||||
/// Direct dependencies of this commit
|
||||
pub deps: Vec<ObjectRef>, |
||||
|
||||
/// Not directly dependent heads to acknowledge
|
||||
pub acks: Vec<ObjectRef>, |
||||
|
||||
/// Files the commit references
|
||||
pub refs: Vec<ObjectRef>, |
||||
|
||||
/// App-specific metadata (commit message, creation time, etc)
|
||||
#[serde(with = "serde_bytes")] |
||||
pub metadata: Vec<u8>, |
||||
|
||||
/// Object with a CommitBody inside
|
||||
pub body: ObjectRef, |
||||
|
||||
/// Expiry time of the body object
|
||||
pub expiry: Option<Timestamp>, |
||||
} |
||||
|
||||
/// Commit object
|
||||
/// Signed by branch key, or a member key authorized to publish this commit type
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] |
||||
pub struct CommitV0 { |
||||
/// ID of parent Object
|
||||
#[serde(skip)] |
||||
pub id: Option<ObjectId>, |
||||
|
||||
/// Key of parent Object
|
||||
#[serde(skip)] |
||||
pub key: Option<SymKey>, |
||||
|
||||
/// Commit content
|
||||
pub content: CommitContentV0, |
||||
|
||||
/// Signature over the content by the author
|
||||
pub sig: Sig, |
||||
} |
||||
|
||||
/// Commit Object
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] |
||||
pub enum Commit { |
||||
V0(CommitV0), |
||||
} |
||||
|
||||
/// File Object
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] |
||||
pub struct FileV0 { |
||||
#[serde(with = "serde_bytes")] |
||||
pub content_type: Vec<u8>, |
||||
|
||||
#[serde(with = "serde_bytes")] |
||||
pub metadata: Vec<u8>, |
||||
|
||||
#[serde(with = "serde_bytes")] |
||||
pub content: Vec<u8>, |
||||
} |
||||
|
||||
/// A file stored in an Object
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] |
||||
pub enum File { |
||||
V0(FileV0), |
||||
} |
||||
|
||||
/// Immutable data stored encrypted in a Merkle tree
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] |
||||
pub enum ObjectContent { |
||||
Commit(Commit), |
||||
CommitBody(CommitBody), |
||||
File(File), |
||||
DepList(DepList), |
||||
} |
||||
@ -0,0 +1,79 @@ |
||||
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers
|
||||
// All rights reserved.
|
||||
// This code is partly derived from work written by TG x Thoth from P2Pcollab.
|
||||
// Copyright 2022 TG x Thoth
|
||||
// Licensed under the Apache License, Version 2.0
|
||||
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
|
||||
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
|
||||
// at your option. All files in the project carrying such
|
||||
// notice may not be copied, modified, or distributed except
|
||||
// according to those terms.
|
||||
|
||||
|
||||
use crate::errors::*; |
||||
use crate::types::*; |
||||
|
||||
use ed25519_dalek::*; |
||||
use rand::rngs::OsRng; |
||||
use std::time::{SystemTime, UNIX_EPOCH}; |
||||
|
||||
pub fn sign( |
||||
author_privkey: PrivKey, |
||||
author_pubkey: PubKey, |
||||
content: &Vec<u8>, |
||||
) -> Result<Sig, NgError> { |
||||
let kp = match (author_privkey, author_pubkey) { |
||||
(PrivKey::Ed25519PrivKey(sk), PubKey::Ed25519PubKey(pk)) => [sk, pk].concat(), |
||||
}; |
||||
let keypair = Keypair::from_bytes(kp.as_slice())?; |
||||
let sig_bytes = keypair.sign(content.as_slice()).to_bytes(); |
||||
let mut it = sig_bytes.chunks_exact(32); |
||||
let mut ss: Ed25519Sig = [[0; 32], [0; 32]]; |
||||
ss[0].copy_from_slice(it.next().unwrap()); |
||||
ss[1].copy_from_slice(it.next().unwrap()); |
||||
Ok(Sig::Ed25519Sig(ss)) |
||||
} |
||||
|
||||
pub fn verify(content: &Vec<u8>, sig: Sig, pub_key: PubKey) -> Result<(), NgError> { |
||||
let pubkey = match pub_key { |
||||
PubKey::Ed25519PubKey(pk) => pk, |
||||
}; |
||||
let pk = PublicKey::from_bytes(&pubkey)?; |
||||
let sig_bytes = match sig { |
||||
Sig::Ed25519Sig(ss) => [ss[0], ss[1]].concat(), |
||||
}; |
||||
let sig = Signature::from_bytes(&sig_bytes)?; |
||||
Ok(pk.verify_strict(content, &sig)?) |
||||
} |
||||
|
||||
pub fn generate_keypair() -> (PrivKey, PubKey) { |
||||
let mut csprng = OsRng {}; |
||||
let keypair: Keypair = Keypair::generate(&mut csprng); |
||||
// println!(
|
||||
// "private key: ({}) {:?}",
|
||||
// keypair.secret.as_bytes().len(),
|
||||
// keypair.secret.as_bytes()
|
||||
// );
|
||||
// println!(
|
||||
// "public key: ({}) {:?}",
|
||||
// keypair.public.as_bytes().len(),
|
||||
// keypair.public.as_bytes()
|
||||
// );
|
||||
let ed_priv_key = keypair.secret.to_bytes(); |
||||
let ed_pub_key = keypair.public.to_bytes(); |
||||
let priv_key = PrivKey::Ed25519PrivKey(ed_priv_key); |
||||
let pub_key = PubKey::Ed25519PubKey(ed_pub_key); |
||||
(priv_key, pub_key) |
||||
} |
||||
|
||||
/// returns the NextGraph Timestamp of now.
|
||||
pub fn now_timestamp() -> Timestamp { |
||||
((SystemTime::now() |
||||
.duration_since(UNIX_EPOCH) |
||||
.unwrap() |
||||
.as_secs() |
||||
- EPOCH_AS_UNIX_TIMESTAMP) |
||||
/ 60) |
||||
.try_into() |
||||
.unwrap() |
||||
} |
||||
@ -0,0 +1,233 @@ |
||||
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers
|
||||
// All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0
|
||||
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
|
||||
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
|
||||
// at your option. All files in the project carrying such
|
||||
// notice may not be copied, modified, or distributed except
|
||||
// according to those terms.
|
||||
|
||||
use p2p_repo::broker_store::*; |
||||
use p2p_repo::store::*; |
||||
use p2p_repo::types::*; |
||||
use p2p_repo::utils::*; |
||||
|
||||
use debug_print::*; |
||||
use std::path::Path; |
||||
use std::path::PathBuf; |
||||
use std::sync::{Arc, RwLock}; |
||||
|
||||
use rkv::backend::{ |
||||
BackendDatabaseFlags, BackendFlags, BackendIter, BackendWriteFlags, DatabaseFlags, Lmdb, |
||||
LmdbDatabase, LmdbDatabaseFlags, LmdbEnvironment, LmdbRwTransaction, LmdbWriteFlags, |
||||
}; |
||||
use rkv::{ |
||||
Manager, MultiStore, Rkv, SingleStore, StoreError, StoreOptions, Value, WriteFlags, Writer, |
||||
}; |
||||
|
||||
use serde::{Deserialize, Serialize}; |
||||
use serde_bare::error::Error; |
||||
|
||||
pub struct LmdbBrokerStore { |
||||
/// the main store where all the properties of keys are stored
|
||||
main_store: MultiStore<LmdbDatabase>, |
||||
/// the opened environment so we can create new transactions
|
||||
environment: Arc<RwLock<Rkv<LmdbEnvironment>>>, |
||||
/// path for the storage backend data
|
||||
path: String, |
||||
} |
||||
|
||||
impl BrokerStore for LmdbBrokerStore { |
||||
/// Load a single value property from the store.
|
||||
fn get(&self, prefix: u8, key: &Vec<u8>, suffix: Option<u8>) -> Result<Vec<u8>, StorageError> { |
||||
let property = Self::compute_property(prefix, key, suffix); |
||||
let lock = self.environment.read().unwrap(); |
||||
let reader = lock.read().unwrap(); |
||||
let mut iter = self |
||||
.main_store |
||||
.get(&reader, property) |
||||
.map_err(|e| StorageError::BackendError)?; |
||||
match iter.next() { |
||||
Some(Ok(val)) => Ok(val.1.to_bytes().unwrap()), |
||||
Some(Err(_e)) => Err(StorageError::BackendError), |
||||
None => Err(StorageError::NotFound), |
||||
} |
||||
} |
||||
|
||||
/// Load all the values of a property from the store.
|
||||
fn get_all( |
||||
&self, |
||||
prefix: u8, |
||||
key: &Vec<u8>, |
||||
suffix: Option<u8>, |
||||
) -> Result<Vec<Vec<u8>>, StorageError> { |
||||
let property = Self::compute_property(prefix, key, suffix); |
||||
let lock = self.environment.read().unwrap(); |
||||
let reader = lock.read().unwrap(); |
||||
let mut iter = self |
||||
.main_store |
||||
.get(&reader, property) |
||||
.map_err(|e| StorageError::BackendError)?; |
||||
let mut vector: Vec<Vec<u8>> = vec![]; |
||||
while let res = iter.next() { |
||||
vector.push(match res { |
||||
Some(Ok(val)) => val.1.to_bytes().unwrap(), |
||||
Some(Err(_e)) => return Err(StorageError::BackendError), |
||||
None => { |
||||
break; |
||||
} |
||||
}); |
||||
} |
||||
Ok(vector) |
||||
} |
||||
|
||||
/// Check if a specific value exists for a property from the store.
|
||||
fn has_property_value( |
||||
&self, |
||||
prefix: u8, |
||||
key: &Vec<u8>, |
||||
suffix: Option<u8>, |
||||
value: Vec<u8>, |
||||
) -> Result<(), StorageError> { |
||||
let property = Self::compute_property(prefix, key, suffix); |
||||
let lock = self.environment.read().unwrap(); |
||||
let reader = lock.read().unwrap(); |
||||
let exists = self |
||||
.main_store |
||||
.get_key_value(&reader, property, &Value::Blob(value.as_slice())) |
||||
.map_err(|e| StorageError::BackendError)?; |
||||
if exists { |
||||
Ok(()) |
||||
} else { |
||||
Err(StorageError::NotFound) |
||||
} |
||||
} |
||||
|
||||
/// Save a property value to the store.
|
||||
fn put( |
||||
&self, |
||||
prefix: u8, |
||||
key: &Vec<u8>, |
||||
suffix: Option<u8>, |
||||
value: Vec<u8>, |
||||
) -> Result<(), StorageError> { |
||||
let property = Self::compute_property(prefix, key, suffix); |
||||
let lock = self.environment.read().unwrap(); |
||||
let mut writer = lock.write().unwrap(); |
||||
self.main_store |
||||
.put(&mut writer, property, &Value::Blob(value.as_slice())) |
||||
.map_err(|e| StorageError::BackendError)?; |
||||
|
||||
writer.commit().unwrap(); |
||||
|
||||
Ok(()) |
||||
} |
||||
|
||||
/// Replace the property of a key (single value) to the store.
|
||||
fn replace( |
||||
&self, |
||||
prefix: u8, |
||||
key: &Vec<u8>, |
||||
suffix: Option<u8>, |
||||
value: Vec<u8>, |
||||
) -> Result<(), StorageError> { |
||||
let property = Self::compute_property(prefix, key, suffix); |
||||
let lock = self.environment.read().unwrap(); |
||||
let mut writer = lock.write().unwrap(); |
||||
self.main_store |
||||
.delete_all(&mut writer, property.clone()) |
||||
.map_err(|e| StorageError::BackendError)?; |
||||
|
||||
self.main_store |
||||
.put(&mut writer, property, &Value::Blob(value.as_slice())) |
||||
.map_err(|e| StorageError::BackendError)?; |
||||
|
||||
writer.commit().unwrap(); |
||||
|
||||
Ok(()) |
||||
} |
||||
|
||||
/// Delete a property from the store.
|
||||
fn del(&self, prefix: u8, key: &Vec<u8>, suffix: Option<u8>) -> Result<(), StorageError> { |
||||
let property = Self::compute_property(prefix, key, suffix); |
||||
let lock = self.environment.read().unwrap(); |
||||
let mut writer = lock.write().unwrap(); |
||||
self.main_store |
||||
.delete_all(&mut writer, property) |
||||
.map_err(|e| StorageError::BackendError)?; |
||||
|
||||
writer.commit().unwrap(); |
||||
|
||||
Ok(()) |
||||
} |
||||
|
||||
/// Delete a specific value for a property from the store.
|
||||
fn del_property_value( |
||||
&self, |
||||
prefix: u8, |
||||
key: &Vec<u8>, |
||||
suffix: Option<u8>, |
||||
value: Vec<u8>, |
||||
) -> Result<(), StorageError> { |
||||
let property = Self::compute_property(prefix, key, suffix); |
||||
let lock = self.environment.read().unwrap(); |
||||
let mut writer = lock.write().unwrap(); |
||||
self.main_store |
||||
.delete(&mut writer, property, &Value::Blob(value.as_slice())) |
||||
.map_err(|e| StorageError::BackendError)?; |
||||
|
||||
writer.commit().unwrap(); |
||||
|
||||
Ok(()) |
||||
} |
||||
|
||||
/// Delete all properties of a key from the store.
|
||||
fn del_all(&self, prefix: u8, key: &Vec<u8>, all_suffixes: &[u8]) -> Result<(), StorageError> { |
||||
for suffix in all_suffixes { |
||||
self.del(prefix, key, Some(*suffix))?; |
||||
} |
||||
if all_suffixes.is_empty() { |
||||
self.del(prefix, key, None)?; |
||||
} |
||||
Ok(()) |
||||
} |
||||
} |
||||
|
||||
impl LmdbBrokerStore { |
||||
pub fn path(&self) -> PathBuf { |
||||
PathBuf::from(&self.path) |
||||
} |
||||
|
||||
fn compute_property(prefix: u8, key: &Vec<u8>, suffix: Option<u8>) -> Vec<u8> { |
||||
let mut new: Vec<u8> = Vec::with_capacity(key.len() + 2); |
||||
new.push(prefix); |
||||
new.extend(key); |
||||
if suffix.is_some() { |
||||
new.push(suffix.unwrap()) |
||||
} |
||||
new |
||||
} |
||||
|
||||
/// Opens the store and returns a BrokerStore object that should be kept and used to manipulate Accounts, Overlays, Topics and options
|
||||
/// The key is the encryption key for the data at rest.
|
||||
pub fn open<'a>(path: &Path, key: [u8; 32]) -> LmdbBrokerStore { |
||||
let mut manager = Manager::<LmdbEnvironment>::singleton().write().unwrap(); |
||||
let shared_rkv = manager |
||||
.get_or_create(path, |path| { |
||||
//Rkv::new::<Lmdb>(path) // use this instead to disable encryption
|
||||
Rkv::with_encryption_key_and_mapsize::<Lmdb>(path, key, 2 * 1024 * 1024 * 1024) |
||||
}) |
||||
.unwrap(); |
||||
let env = shared_rkv.read().unwrap(); |
||||
|
||||
println!("created env with LMDB Version: {}", env.version()); |
||||
|
||||
let main_store = env.open_multi("main", StoreOptions::create()).unwrap(); |
||||
|
||||
LmdbBrokerStore { |
||||
environment: shared_rkv.clone(), |
||||
main_store, |
||||
path: path.to_str().unwrap().to_string(), |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,3 @@ |
||||
pub mod repo_store; |
||||
|
||||
pub mod broker_store; |
||||
@ -0,0 +1,997 @@ |
||||
// Copyright (c) 2022-2023 Niko Bonnieure, Par le Peuple, NextGraph.org developers
|
||||
// All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0
|
||||
// <LICENSE-APACHE2 or http://www.apache.org/licenses/LICENSE-2.0>
|
||||
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
|
||||
// at your option. All files in the project carrying such
|
||||
// notice may not be copied, modified, or distributed except
|
||||
// according to those terms.
|
||||
|
||||
use p2p_repo::store::*; |
||||
use p2p_repo::types::*; |
||||
use p2p_repo::utils::*; |
||||
|
||||
use debug_print::*; |
||||
use std::path::Path; |
||||
use std::sync::{Arc, RwLock}; |
||||
|
||||
use rkv::backend::{ |
||||
BackendDatabaseFlags, BackendFlags, BackendIter, BackendWriteFlags, DatabaseFlags, Lmdb, |
||||
LmdbDatabase, LmdbDatabaseFlags, LmdbEnvironment, LmdbRwTransaction, LmdbWriteFlags, |
||||
}; |
||||
use rkv::{ |
||||
Manager, MultiIntegerStore, Rkv, SingleStore, StoreError, StoreOptions, Value, WriteFlags, |
||||
Writer, |
||||
}; |
||||
|
||||
use serde::{Deserialize, Serialize}; |
||||
use serde_bare::error::Error; |
||||
|
||||
pub struct LmdbRepoStore { |
||||
/// the main store where all the repo blocks are stored
|
||||
main_store: SingleStore<LmdbDatabase>, |
||||
/// store for the pin boolean, recently_used timestamp, and synced boolean
|
||||
meta_store: SingleStore<LmdbDatabase>, |
||||
/// store for the expiry timestamp
|
||||
expiry_store: MultiIntegerStore<LmdbDatabase, u32>, |
||||
/// store for the LRU list
|
||||
recently_used_store: MultiIntegerStore<LmdbDatabase, u32>, |
||||
/// the opened environment so we can create new transactions
|
||||
environment: Arc<RwLock<Rkv<LmdbEnvironment>>>, |
||||
} |
||||
|
||||
// TODO: versioning V0
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] |
||||
struct BlockMeta { |
||||
pub pin: bool, |
||||
pub last_used: Timestamp, |
||||
pub synced: bool, |
||||
} |
||||
|
||||
impl RepoStore for LmdbRepoStore { |
||||
/// Retrieves a block from the storage backend.
|
||||
fn get(&self, block_id: &BlockId) -> Result<Block, StorageError> { |
||||
let lock = self.environment.read().unwrap(); |
||||
let reader = lock.read().unwrap(); |
||||
let block_id_ser = serde_bare::to_vec(&block_id).unwrap(); |
||||
let block_ser_res = self.main_store.get(&reader, block_id_ser.clone()); |
||||
match block_ser_res { |
||||
Err(e) => Err(StorageError::BackendError), |
||||
Ok(None) => Err(StorageError::NotFound), |
||||
Ok(Some(block_ser)) => { |
||||
// updating recently_used
|
||||
// first getting the meta for this BlockId
|
||||
let meta_ser = self.meta_store.get(&reader, block_id_ser.clone()).unwrap(); |
||||
match meta_ser { |
||||
Some(meta_value) => { |
||||
let mut meta = |
||||
serde_bare::from_slice::<BlockMeta>(&meta_value.to_bytes().unwrap()) |
||||
.unwrap(); |
||||
if meta.synced { |
||||
let mut writer = lock.write().unwrap(); |
||||
let now = now_timestamp(); |
||||
if !meta.pin { |
||||
// we remove the previous timestamp (last_used) from recently_used_store
|
||||
self.remove_from_lru(&mut writer, &block_id_ser, &meta.last_used) |
||||
.unwrap(); |
||||
// we add an entry to recently_used_store with now
|
||||
self.add_to_lru(&mut writer, &block_id_ser, &now).unwrap(); |
||||
} |
||||
// we save the new meta (with last_used:now)
|
||||
meta.last_used = now; |
||||
let new_meta_ser = serde_bare::to_vec(&meta).unwrap(); |
||||
self.meta_store |
||||
.put( |
||||
&mut writer, |
||||
block_id_ser, |
||||
&Value::Blob(new_meta_ser.as_slice()), |
||||
) |
||||
.unwrap(); |
||||
// commit
|
||||
writer.commit().unwrap(); |
||||
} |
||||
} |
||||
_ => {} // there is no meta. we do nothing since we start to record LRU only once synced == true.
|
||||
} |
||||
|
||||
match serde_bare::from_slice::<Block>(&block_ser.to_bytes().unwrap()) { |
||||
Err(_e) => Err(StorageError::InvalidValue), |
||||
Ok(o) => { |
||||
if o.id() != *block_id { |
||||
debug_println!( |
||||
"Invalid ObjectId.\nExp: {:?}\nGot: {:?}\nContent: {:?}", |
||||
block_id, |
||||
o.id(), |
||||
o |
||||
); |
||||
panic!("CORRUPTION OF DATA !"); |
||||
} |
||||
Ok(o) |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
/// Adds a block in the storage backend.
|
||||
/// The block is persisted to disk.
|
||||
/// Returns the BlockId of the Block.
|
||||
fn put(&self, block: &Block) -> Result<BlockId, StorageError> { |
||||
let block_ser = serde_bare::to_vec(&block).unwrap(); |
||||
|
||||
let block_id = block.id(); |
||||
let block_id_ser = serde_bare::to_vec(&block_id).unwrap(); |
||||
|
||||
let lock = self.environment.read().unwrap(); |
||||
let mut writer = lock.write().unwrap(); |
||||
|
||||
// TODO: check if the block is already in store? if yes, don't put it again.
|
||||
// I didnt do it yet because it is extra cost. surely a get on the store is lighter than a put
|
||||
// but doing a get in additing to a put for every call, is probably even costlier. better to deal with that at the higher level
|
||||
|
||||
self.main_store |
||||
.put( |
||||
&mut writer, |
||||
&block_id_ser, |
||||
&Value::Blob(block_ser.as_slice()), |
||||
) |
||||
.unwrap(); |
||||
|
||||
// if it has an expiry, adding the BlockId to the expiry_store
|
||||
match block.expiry() { |
||||
Some(expiry) => { |
||||
self.expiry_store |
||||
.put(&mut writer, expiry, &Value::Blob(block_id_ser.as_slice())) |
||||
.unwrap(); |
||||
} |
||||
_ => {} |
||||
} |
||||
writer.commit().unwrap(); |
||||
|
||||
Ok(block_id) |
||||
} |
||||
|
||||
/// Removes the block from the storage backend.
|
||||
/// The removed block is returned, so it can be inspected.
|
||||
/// Also returned is the approximate size of of free space that was reclaimed.
|
||||
fn del(&self, block_id: &BlockId) -> Result<(Block, usize), StorageError> { |
||||
let lock = self.environment.read().unwrap(); |
||||
let mut writer = lock.write().unwrap(); |
||||
let block_id_ser = serde_bare::to_vec(&block_id).unwrap(); |
||||
// retrieving the block itself (we need the expiry)
|
||||
let block_ser = self |
||||
.main_store |
||||
.get(&writer, block_id_ser.clone()) |
||||
.unwrap() |
||||
.ok_or(StorageError::NotFound)?; |
||||
let slice = block_ser.to_bytes().unwrap(); |
||||
let block = serde_bare::from_slice::<Block>(&slice).unwrap(); //FIXME propagate error?
|
||||
let meta_res = self.meta_store.get(&writer, block_id_ser.clone()).unwrap(); |
||||
if meta_res.is_some() { |
||||
let meta = serde_bare::from_slice::<BlockMeta>(&meta_res.unwrap().to_bytes().unwrap()) |
||||
.unwrap(); |
||||
if meta.last_used != 0 { |
||||
self.remove_from_lru(&mut writer, &block_id_ser.clone(), &meta.last_used) |
||||
.unwrap(); |
||||
} |
||||
// removing the meta
|
||||
self.meta_store |
||||
.delete(&mut writer, block_id_ser.clone()) |
||||
.unwrap(); |
||||
} |
||||
// delete block from main_store
|
||||
self.main_store |
||||
.delete(&mut writer, block_id_ser.clone()) |
||||
.unwrap(); |
||||
// remove BlockId from expiry_store, if any expiry
|
||||
match block.expiry() { |
||||
Some(expiry) => { |
||||
self.expiry_store |
||||
.delete( |
||||
&mut writer, |
||||
expiry, |
||||
&Value::Blob(block_id_ser.clone().as_slice()), |
||||
) |
||||
.unwrap(); |
||||
} |
||||
_ => {} |
||||
} |
||||
|
||||
writer.commit().unwrap(); |
||||
Ok((block, slice.len())) |
||||
} |
||||
|
||||
} |
||||
|
||||
impl LmdbRepoStore { |
||||
/// Opens the store and returns a RepoStore object that should be kept and used to call put/get/delete/pin
|
||||
/// The key is the encryption key for the data at rest.
|
||||
pub fn open<'a>(path: &Path, key: [u8; 32]) -> LmdbRepoStore { |
||||
let mut manager = Manager::<LmdbEnvironment>::singleton().write().unwrap(); |
||||
let shared_rkv = manager |
||||
.get_or_create(path, |path| { |
||||
//Rkv::new::<Lmdb>(path) // use this instead to disable encryption
|
||||
Rkv::with_encryption_key_and_mapsize::<Lmdb>(path, key, 2 * 1024 * 1024 * 1024) |
||||
}) |
||||
.unwrap(); |
||||
let env = shared_rkv.read().unwrap(); |
||||
|
||||
println!( |
||||
"created env with LMDB Version: {} key: {}", |
||||
env.version(), |
||||
hex::encode(&key) |
||||
); |
||||
|
||||
let main_store = env.open_single("main", StoreOptions::create()).unwrap(); |
||||
let meta_store = env.open_single("meta", StoreOptions::create()).unwrap(); |
||||
let mut opts = StoreOptions::<LmdbDatabaseFlags>::create(); |
||||
opts.flags.set(DatabaseFlags::DUP_FIXED, true); |
||||
let expiry_store = env.open_multi_integer("expiry", opts).unwrap(); |
||||
let recently_used_store = env.open_multi_integer("recently_used", opts).unwrap(); |
||||
|
||||
LmdbRepoStore { |
||||
environment: shared_rkv.clone(), |
||||
main_store, |
||||
meta_store, |
||||
expiry_store, |
||||
recently_used_store, |
||||
} |
||||
} |
||||
|
||||
//FIXME: use BlockId, not ObjectId. this is a block level operation
|
||||
/// Pins the object
|
||||
pub fn pin(&self, object_id: &ObjectId) -> Result<(), StorageError> { |
||||
self.set_pin(object_id, true) |
||||
} |
||||
|
||||
//FIXME: use BlockId, not ObjectId. this is a block level operation
|
||||
/// Unpins the object
|
||||
pub fn unpin(&self, object_id: &ObjectId) -> Result<(), StorageError> { |
||||
self.set_pin(object_id, false) |
||||
} |
||||
|
||||
//FIXME: use BlockId, not ObjectId. this is a block level operation
|
||||
/// Sets the pin for that Object. if add is true, will add the pin. if false, will remove the pin.
|
||||
/// A pin on an object prevents it from being removed when the store is making some disk space by using the LRU.
|
||||
/// A pin does not override the expiry. If expiry is set and is reached, the obejct will be deleted, no matter what.
|
||||
pub fn set_pin(&self, object_id: &ObjectId, add: bool) -> Result<(), StorageError> { |
||||
let lock = self.environment.read().unwrap(); |
||||
let mut writer = lock.write().unwrap(); |
||||
let obj_id_ser = serde_bare::to_vec(&object_id).unwrap(); |
||||
let meta_ser = self.meta_store.get(&writer, &obj_id_ser).unwrap(); |
||||
let mut meta; |
||||
|
||||
// if adding a pin, if there is a meta (if already pinned, return) and is synced, remove the last_used timestamp from recently_used_store
|
||||
// if no meta, create it with pin:true, synced: false
|
||||
// if removing a pin (if pin already removed, return), if synced, add an entry to recently_used_store with the last_used timestamp (as found in meta, dont use now)
|
||||
|
||||
match meta_ser { |
||||
Some(meta_value) => { |
||||
meta = |
||||
serde_bare::from_slice::<BlockMeta>(&meta_value.to_bytes().unwrap()).unwrap(); |
||||
|
||||
if add == meta.pin { |
||||
// pinning while already pinned, or unpinning while already unpinned. NOP
|
||||
return Ok(()); |
||||
}; |
||||
|
||||
meta.pin = add; |
||||
|
||||
if meta.synced { |
||||
if add { |
||||
// we remove the previous timestamp (last_used) from recently_used_store
|
||||
self.remove_from_lru(&mut writer, &obj_id_ser, &meta.last_used) |
||||
.unwrap(); |
||||
} else { |
||||
// we add an entry to recently_used_store with last_used
|
||||
self.add_to_lru(&mut writer, &obj_id_ser, &meta.last_used) |
||||
.unwrap(); |
||||
} |
||||
} |
||||
} |
||||
None => { |
||||
if add { |
||||
meta = BlockMeta { |
||||
pin: true, |
||||
synced: false, |
||||
last_used: 0, |
||||
} |
||||
} else { |
||||
// there is no meta, and user wants to unpin, so let's leave everything as it is.
|
||||
return Ok(()); |
||||
} |
||||
} |
||||
} |
||||
let new_meta_ser = serde_bare::to_vec(&meta).unwrap(); |
||||
self.meta_store |
||||
.put( |
||||
&mut writer, |
||||
obj_id_ser, |
||||
&Value::Blob(new_meta_ser.as_slice()), |
||||
) |
||||
.unwrap(); |
||||
// commit
|
||||
writer.commit().unwrap(); |
||||
|
||||
Ok(()) |
||||
} |
||||
|
||||
//FIXME: use BlockId, not ObjectId. this is a block level operation
|
||||
/// the broker calls this method when the block has been retrieved/synced by enough peers and it
|
||||
/// can now be included in the LRU for potential garbage collection.
|
||||
/// If this method has not been called on a block, it will be kept in the store and will not enter LRU.
|
||||
pub fn has_been_synced(&self, block_id: &BlockId, when: Option<u32>) -> Result<(), Error> { |
||||
let lock = self.environment.read().unwrap(); |
||||
let mut writer = lock.write().unwrap(); |
||||
let block_id_ser = serde_bare::to_vec(&block_id).unwrap(); |
||||
let meta_ser = self.meta_store.get(&writer, block_id_ser.clone()).unwrap(); |
||||
let mut meta; |
||||
let now = match when { |
||||
None => now_timestamp(), |
||||
Some(w) => w, |
||||
}; |
||||
// get the meta. if no meta, it is ok, we will create it after (with pin:false and synced:true)
|
||||
// if already synced, return
|
||||
// update the meta with last_used:now and synced:true
|
||||
// if pinned, save and return
|
||||
// otherwise add an entry to recently_used_store with now
|
||||
|
||||
match meta_ser { |
||||
Some(meta_value) => { |
||||
meta = |
||||
serde_bare::from_slice::<BlockMeta>(&meta_value.to_bytes().unwrap()).unwrap(); |
||||
|
||||
if meta.synced { |
||||
// already synced. NOP
|
||||
return Ok(()); |
||||
}; |
||||
|
||||
meta.synced = true; |
||||
meta.last_used = now; |
||||
|
||||
if !meta.pin { |
||||
// we add an entry to recently_used_store with now
|
||||
println!("adding to LRU"); |
||||
self.add_to_lru(&mut writer, &block_id_ser, &now).unwrap(); |
||||
} |
||||
} |
||||
None => { |
||||
meta = BlockMeta { |
||||
pin: false, |
||||
synced: true, |
||||
last_used: now, |
||||
}; |
||||
println!("adding to LRU also"); |
||||
self.add_to_lru(&mut writer, &block_id_ser, &now).unwrap(); |
||||
} |
||||
} |
||||
let new_meta_ser = serde_bare::to_vec(&meta).unwrap(); |
||||
self.meta_store |
||||
.put( |
||||
&mut writer, |
||||
block_id_ser, |
||||
&Value::Blob(new_meta_ser.as_slice()), |
||||
) |
||||
.unwrap(); |
||||
// commit
|
||||
writer.commit().unwrap(); |
||||
|
||||
Ok(()) |
||||
} |
||||
|
||||
/// Removes all the blocks that have expired.
|
||||
/// The broker should call this method periodically.
|
||||
pub fn remove_expired(&self) -> Result<(), Error> { |
||||
let mut block_ids: Vec<BlockId> = vec![]; |
||||
|
||||
{ |
||||
let lock = self.environment.read().unwrap(); |
||||
let reader = lock.read().unwrap(); |
||||
|
||||
let mut iter = self |
||||
.expiry_store |
||||
.iter_prev_dup_from(&reader, now_timestamp()) |
||||
.unwrap(); |
||||
|
||||
while let Some(Ok(mut sub_iter)) = iter.next() { |
||||
while let Some(Ok(k)) = sub_iter.next() { |
||||
//println!("removing {:?} {:?}", k.0, k.1);
|
||||
let block_id = serde_bare::from_slice::<ObjectId>(k.1).unwrap(); |
||||
block_ids.push(block_id); |
||||
} |
||||
} |
||||
} |
||||
for block_id in block_ids { |
||||
self.del(&block_id).unwrap(); |
||||
} |
||||
Ok(()) |
||||
} |
||||
|
||||
/// Removes some blocks that haven't been used for a while, reclaiming some space on disk.
|
||||
/// The oldest are removed first, until the total amount of data removed is at least equal to size,
|
||||
/// or the LRU list became empty. The approximate size of the storage space that was reclaimed is returned.
|
||||
pub fn remove_least_used(&self, size: usize) -> usize { |
||||
let mut block_ids: Vec<BlockId> = vec![]; |
||||
let mut total: usize = 0; |
||||
|
||||
{ |
||||
let lock = self.environment.read().unwrap(); |
||||
let reader = lock.read().unwrap(); |
||||
|
||||
let mut iter = self.recently_used_store.iter_start(&reader).unwrap(); |
||||
|
||||
while let Some(Ok(entry)) = iter.next() { |
||||
let block_id = |
||||
serde_bare::from_slice::<ObjectId>(entry.1.to_bytes().unwrap().as_slice()) |
||||
.unwrap(); |
||||
block_ids.push(block_id); |
||||
} |
||||
} |
||||
for block_id in block_ids { |
||||
let (block, block_size) = self.del(&block_id).unwrap(); |
||||
println!("removed {:?}", block_id); |
||||
total += block_size; |
||||
if total >= size { |
||||
break; |
||||
} |
||||
} |
||||
total |
||||
} |
||||
|
||||
fn remove_from_lru( |
||||
&self, |
||||
writer: &mut Writer<LmdbRwTransaction>, |
||||
block_id_ser: &Vec<u8>, |
||||
time: &Timestamp, |
||||
) -> Result<(), StoreError> { |
||||
self.recently_used_store |
||||
.delete(writer, *time, &Value::Blob(block_id_ser.as_slice())) |
||||
} |
||||
|
||||
fn add_to_lru( |
||||
&self, |
||||
writer: &mut Writer<LmdbRwTransaction>, |
||||
block_id_ser: &Vec<u8>, |
||||
time: &Timestamp, |
||||
) -> Result<(), StoreError> { |
||||
let mut flag = LmdbWriteFlags::empty(); |
||||
flag.set(WriteFlags::APPEND_DUP, true); |
||||
self.recently_used_store.put_with_flags( |
||||
writer, |
||||
*time, |
||||
&Value::Blob(block_id_ser.as_slice()), |
||||
flag, |
||||
) |
||||
} |
||||
|
||||
fn list_all(&self) { |
||||
let lock = self.environment.read().unwrap(); |
||||
let reader = lock.read().unwrap(); |
||||
println!("MAIN"); |
||||
let mut iter = self.main_store.iter_start(&reader).unwrap(); |
||||
while let Some(Ok(entry)) = iter.next() { |
||||
println!("{:?} {:?}", entry.0, entry.1) |
||||
} |
||||
println!("META"); |
||||
let mut iter2 = self.meta_store.iter_start(&reader).unwrap(); |
||||
while let Some(Ok(entry)) = iter2.next() { |
||||
println!("{:?} {:?}", entry.0, entry.1) |
||||
} |
||||
println!("EXPIRY"); |
||||
let mut iter3 = self.expiry_store.iter_start(&reader).unwrap(); |
||||
while let Some(Ok(entry)) = iter3.next() { |
||||
println!("{:?} {:?}", entry.0, entry.1) |
||||
} |
||||
println!("LRU"); |
||||
let mut iter4 = self.recently_used_store.iter_start(&reader).unwrap(); |
||||
while let Some(Ok(entry)) = iter4.next() { |
||||
println!("{:?} {:?}", entry.0, entry.1) |
||||
} |
||||
} |
||||
} |
||||
#[cfg(test)] |
||||
mod test { |
||||
|
||||
use crate::repo_store::LmdbRepoStore; |
||||
use p2p_repo::store::*; |
||||
use p2p_repo::types::*; |
||||
use p2p_repo::utils::*; |
||||
use rkv::backend::{BackendInfo, BackendStat, Lmdb, LmdbEnvironment}; |
||||
use rkv::{Manager, Rkv, StoreOptions, Value}; |
||||
#[allow(unused_imports)] |
||||
use std::time::Duration; |
||||
#[allow(unused_imports)] |
||||
use std::{fs, thread}; |
||||
use tempfile::Builder; |
||||
|
||||
#[test] |
||||
pub fn test_remove_least_used() { |
||||
let path_str = "test-env"; |
||||
let root = Builder::new().prefix(path_str).tempdir().unwrap(); |
||||
let key: [u8; 32] = [0; 32]; |
||||
fs::create_dir_all(root.path()).unwrap(); |
||||
println!("{}", root.path().to_str().unwrap()); |
||||
let mut store = LmdbRepoStore::open(root.path(), key); |
||||
let mut now = now_timestamp(); |
||||
now -= 200; |
||||
// TODO: fix the LMDB bug that is triggered with x max set to 86 !!!
|
||||
for x in 1..85 { |
||||
let block = Block::new( |
||||
Vec::new(), |
||||
ObjectDeps::ObjectIdList(Vec::new()), |
||||
None, |
||||
vec![x; 10], |
||||
None, |
||||
); |
||||
let block_id = store.put(&block).unwrap(); |
||||
println!("#{} -> objId {:?}", x, block_id); |
||||
store |
||||
.has_been_synced(&block_id, Some(now + x as u32)) |
||||
.unwrap(); |
||||
} |
||||
|
||||
let ret = store.remove_least_used(200); |
||||
println!("removed {}", ret); |
||||
assert_eq!(ret, 208) |
||||
|
||||
//store.list_all();
|
||||
} |
||||
|
||||
#[test] |
||||
pub fn test_set_pin() { |
||||
let path_str = "test-env"; |
||||
let root = Builder::new().prefix(path_str).tempdir().unwrap(); |
||||
let key: [u8; 32] = [0; 32]; |
||||
fs::create_dir_all(root.path()).unwrap(); |
||||
println!("{}", root.path().to_str().unwrap()); |
||||
let mut store = LmdbRepoStore::open(root.path(), key); |
||||
let mut now = now_timestamp(); |
||||
now -= 200; |
||||
// TODO: fix the LMDB bug that is triggered with x max set to 86 !!!
|
||||
for x in 1..100 { |
||||
let block = Block::new( |
||||
Vec::new(), |
||||
ObjectDeps::ObjectIdList(Vec::new()), |
||||
None, |
||||
vec![x; 10], |
||||
None, |
||||
); |
||||
let obj_id = store.put(&block).unwrap(); |
||||
println!("#{} -> objId {:?}", x, obj_id); |
||||
store.set_pin(&obj_id, true).unwrap(); |
||||
store |
||||
.has_been_synced(&obj_id, Some(now + x as u32)) |
||||
.unwrap(); |
||||
} |
||||
|
||||
let ret = store.remove_least_used(200); |
||||
println!("removed {}", ret); |
||||
assert_eq!(ret, 0); |
||||
|
||||
store.list_all(); |
||||
} |
||||
|
||||
#[test] |
||||
pub fn test_get_valid_value_size() { |
||||
assert_eq!(store_valid_value_size(0), 4072); |
||||
assert_eq!(store_valid_value_size(2), 4072); |
||||
assert_eq!(store_valid_value_size(4072), 4072); |
||||
assert_eq!(store_valid_value_size(4072 + 1), 4072 + 4096); |
||||
assert_eq!(store_valid_value_size(4072 + 4096), 4072 + 4096); |
||||
assert_eq!(store_valid_value_size(4072 + 4096 + 1), 4072 + 4096 + 4096); |
||||
assert_eq!( |
||||
store_valid_value_size(4072 + 4096 + 4096), |
||||
4072 + 4096 + 4096 |
||||
); |
||||
assert_eq!( |
||||
store_valid_value_size(4072 + 4096 + 4096 + 1), |
||||
4072 + 4096 + 4096 + 4096 |
||||
); |
||||
assert_eq!(store_valid_value_size(4072 + 4096 * 511), 4072 + 4096 * 511); |
||||
assert_eq!( |
||||
store_valid_value_size(4072 + 4096 * 511 + 1), |
||||
4072 + 4096 * 511 |
||||
); |
||||
} |
||||
|
||||
#[test] |
||||
pub fn test_remove_expired() { |
||||
let path_str = "test-env"; |
||||
let root = Builder::new().prefix(path_str).tempdir().unwrap(); |
||||
let key: [u8; 32] = [0; 32]; |
||||
fs::create_dir_all(root.path()).unwrap(); |
||||
println!("{}", root.path().to_str().unwrap()); |
||||
let mut store = LmdbRepoStore::open(root.path(), key); |
||||
|
||||
let now = now_timestamp(); |
||||
let list = [ |
||||
now - 10, |
||||
now - 6, |
||||
now - 6, |
||||
now - 3, |
||||
now - 2, |
||||
now - 1, //#5 should be removed, and above
|
||||
now + 3, |
||||
now + 4, |
||||
now + 4, |
||||
now + 5, |
||||
now + 10, |
||||
]; |
||||
let mut block_ids: Vec<ObjectId> = Vec::with_capacity(11); |
||||
println!("now {}", now); |
||||
|
||||
let mut i = 0u8; |
||||
for expiry in list { |
||||
//let i: u8 = (expiry + 10 - now).try_into().unwrap();
|
||||
let block = Block::new( |
||||
Vec::new(), |
||||
ObjectDeps::ObjectIdList(Vec::new()), |
||||
Some(expiry), |
||||
[i].to_vec(), |
||||
None, |
||||
); |
||||
let block_id = store.put(&block).unwrap(); |
||||
println!("#{} -> objId {:?}", i, block_id); |
||||
block_ids.push(block_id); |
||||
i += 1; |
||||
} |
||||
|
||||
store.remove_expired().unwrap(); |
||||
|
||||
assert!(store.get(block_ids.get(0).unwrap()).is_err()); |
||||
assert!(store.get(block_ids.get(1).unwrap()).is_err()); |
||||
assert!(store.get(block_ids.get(2).unwrap()).is_err()); |
||||
assert!(store.get(block_ids.get(5).unwrap()).is_err()); |
||||
assert!(store.get(block_ids.get(6).unwrap()).is_ok()); |
||||
assert!(store.get(block_ids.get(7).unwrap()).is_ok()); |
||||
|
||||
//store.list_all();
|
||||
} |
||||
|
||||
#[test] |
||||
pub fn test_remove_all_expired() { |
||||
let path_str = "test-env"; |
||||
let root = Builder::new().prefix(path_str).tempdir().unwrap(); |
||||
let key: [u8; 32] = [0; 32]; |
||||
fs::create_dir_all(root.path()).unwrap(); |
||||
println!("{}", root.path().to_str().unwrap()); |
||||
let mut store = LmdbRepoStore::open(root.path(), key); |
||||
|
||||
let now = now_timestamp(); |
||||
let list = [ |
||||
now - 10, |
||||
now - 6, |
||||
now - 6, |
||||
now - 3, |
||||
now - 2, |
||||
now - 2, //#5 should be removed, and above
|
||||
]; |
||||
let mut block_ids: Vec<ObjectId> = Vec::with_capacity(6); |
||||
println!("now {}", now); |
||||
|
||||
let mut i = 0u8; |
||||
for expiry in list { |
||||
//let i: u8 = (expiry + 10 - now).try_into().unwrap();
|
||||
let block = Block::new( |
||||
Vec::new(), |
||||
ObjectDeps::ObjectIdList(Vec::new()), |
||||
Some(expiry), |
||||
[i].to_vec(), |
||||
None, |
||||
); |
||||
let block_id = store.put(&block).unwrap(); |
||||
println!("#{} -> objId {:?}", i, block_id); |
||||
block_ids.push(block_id); |
||||
i += 1; |
||||
} |
||||
|
||||
store.remove_expired().unwrap(); |
||||
|
||||
assert!(store.get(block_ids.get(0).unwrap()).is_err()); |
||||
assert!(store.get(block_ids.get(1).unwrap()).is_err()); |
||||
assert!(store.get(block_ids.get(2).unwrap()).is_err()); |
||||
assert!(store.get(block_ids.get(3).unwrap()).is_err()); |
||||
assert!(store.get(block_ids.get(4).unwrap()).is_err()); |
||||
assert!(store.get(block_ids.get(5).unwrap()).is_err()); |
||||
} |
||||
|
||||
#[test] |
||||
pub fn test_remove_empty_expired() { |
||||
let path_str = "test-env"; |
||||
let root = Builder::new().prefix(path_str).tempdir().unwrap(); |
||||
let key: [u8; 32] = [0; 32]; |
||||
fs::create_dir_all(root.path()).unwrap(); |
||||
println!("{}", root.path().to_str().unwrap()); |
||||
let store = LmdbRepoStore::open(root.path(), key); |
||||
store.remove_expired().unwrap(); |
||||
} |
||||
|
||||
#[test] |
||||
pub fn test_store_block() { |
||||
let path_str = "test-env"; |
||||
let root = Builder::new().prefix(path_str).tempdir().unwrap(); |
||||
|
||||
let key: [u8; 32] = [0; 32]; |
||||
fs::create_dir_all(root.path()).unwrap(); |
||||
|
||||
println!("{}", root.path().to_str().unwrap()); |
||||
|
||||
let mut store = LmdbRepoStore::open(root.path(), key); |
||||
|
||||
let block = Block::new( |
||||
Vec::new(), |
||||
ObjectDeps::ObjectIdList(Vec::new()), |
||||
None, |
||||
b"abc".to_vec(), |
||||
None, |
||||
); |
||||
|
||||
let block_id = store.put(&block).unwrap(); |
||||
assert_eq!(block_id, block.id()); |
||||
|
||||
println!("ObjectId: {:?}", block_id); |
||||
assert_eq!( |
||||
block_id, |
||||
Digest::Blake3Digest32([ |
||||
155, 83, 186, 17, 95, 10, 80, 31, 111, 24, 250, 64, 8, 145, 71, 193, 103, 246, 202, |
||||
28, 202, 144, 63, 65, 85, 229, 136, 85, 202, 34, 13, 85 |
||||
]) |
||||
); |
||||
|
||||
let block_res = store.get(&block_id).unwrap(); |
||||
|
||||
println!("Block: {:?}", block_res); |
||||
assert_eq!(block_res.id(), block.id()); |
||||
} |
||||
|
||||
#[test] |
||||
pub fn test_lmdb() { |
||||
let path_str = "test-env"; |
||||
let root = Builder::new().prefix(path_str).tempdir().unwrap(); |
||||
|
||||
// we set an encryption key with all zeros... for test purpose only ;)
|
||||
let key: [u8; 32] = [0; 32]; |
||||
{ |
||||
fs::create_dir_all(root.path()).unwrap(); |
||||
|
||||
println!("{}", root.path().to_str().unwrap()); |
||||
|
||||
let mut manager = Manager::<LmdbEnvironment>::singleton().write().unwrap(); |
||||
let shared_rkv = manager |
||||
.get_or_create(root.path(), |path| { |
||||
// Rkv::new::<Lmdb>(path) // use this instead to disable encryption
|
||||
Rkv::with_encryption_key_and_mapsize::<Lmdb>(path, key, 2 * 1024 * 1024 * 1024) |
||||
}) |
||||
.unwrap(); |
||||
let env = shared_rkv.read().unwrap(); |
||||
|
||||
println!("LMDB Version: {}", env.version()); |
||||
|
||||
let store = env.open_single("testdb", StoreOptions::create()).unwrap(); |
||||
|
||||
{ |
||||
// Use a write transaction to mutate the store via a `Writer`. There can be only
|
||||
// one writer for a given environment, so opening a second one will block until
|
||||
// the first completes.
|
||||
let mut writer = env.write().unwrap(); |
||||
|
||||
// Keys are `AsRef<[u8]>`, while values are `Value` enum instances. Use the `Blob`
|
||||
// variant to store arbitrary collections of bytes. Putting data returns a
|
||||
// `Result<(), StoreError>`, where StoreError is an enum identifying the reason
|
||||
// for a failure.
|
||||
// store.put(&mut writer, "int", &Value::I64(1234)).unwrap();
|
||||
// store
|
||||
// .put(&mut writer, "uint", &Value::U64(1234_u64))
|
||||
// .unwrap();
|
||||
// store
|
||||
// .put(&mut writer, "float", &Value::F64(1234.0.into()))
|
||||
// .unwrap();
|
||||
// store
|
||||
// .put(&mut writer, "instant", &Value::Instant(1528318073700))
|
||||
// .unwrap();
|
||||
// store
|
||||
// .put(&mut writer, "boolean", &Value::Bool(true))
|
||||
// .unwrap();
|
||||
// store
|
||||
// .put(&mut writer, "string", &Value::Str("Héllo, wörld!"))
|
||||
// .unwrap();
|
||||
// store
|
||||
// .put(
|
||||
// &mut writer,
|
||||
// "json",
|
||||
// &Value::Json(r#"{"foo":"bar", "number": 1}"#),
|
||||
// )
|
||||
// .unwrap();
|
||||
const EXTRA: usize = 2095; // + 4096 * 524280 + 0;
|
||||
let key: [u8; 33] = [0; 33]; |
||||
let key2: [u8; 33] = [2; 33]; |
||||
let key3: [u8; 33] = [3; 33]; |
||||
let key4: [u8; 33] = [4; 33]; |
||||
//let value: [u8; 1977 + EXTRA] = [1; 1977 + EXTRA];
|
||||
let value = vec![1; 1977 + EXTRA]; |
||||
let value2: [u8; 1977 + 1] = [1; 1977 + 1]; |
||||
let value4: [u8; 953 + 0] = [1; 953 + 0]; |
||||
store.put(&mut writer, key, &Value::Blob(&value2)).unwrap(); |
||||
store.put(&mut writer, key2, &Value::Blob(&value2)).unwrap(); |
||||
// store.put(&mut writer, key3, &Value::Blob(&value)).unwrap();
|
||||
// store.put(&mut writer, key4, &Value::Blob(&value4)).unwrap();
|
||||
|
||||
// You must commit a write transaction before the writer goes out of scope, or the
|
||||
// transaction will abort and the data won't persist.
|
||||
writer.commit().unwrap(); |
||||
let reader = env.read().expect("reader"); |
||||
let stat = store.stat(&reader).unwrap(); |
||||
|
||||
println!("LMDB stat page_size : {}", stat.page_size()); |
||||
println!("LMDB stat depth : {}", stat.depth()); |
||||
println!("LMDB stat branch_pages : {}", stat.branch_pages()); |
||||
println!("LMDB stat leaf_pages : {}", stat.leaf_pages()); |
||||
println!("LMDB stat overflow_pages : {}", stat.overflow_pages()); |
||||
println!("LMDB stat entries : {}", stat.entries()); |
||||
} |
||||
|
||||
// {
|
||||
// // Use a read transaction to query the store via a `Reader`. There can be multiple
|
||||
// // concurrent readers for a store, and readers never block on a writer nor other
|
||||
// // readers.
|
||||
// let reader = env.read().expect("reader");
|
||||
|
||||
// // Keys are `AsRef<u8>`, and the return value is `Result<Option<Value>, StoreError>`.
|
||||
// // println!("Get int {:?}", store.get(&reader, "int").unwrap());
|
||||
// // println!("Get uint {:?}", store.get(&reader, "uint").unwrap());
|
||||
// // println!("Get float {:?}", store.get(&reader, "float").unwrap());
|
||||
// // println!("Get instant {:?}", store.get(&reader, "instant").unwrap());
|
||||
// // println!("Get boolean {:?}", store.get(&reader, "boolean").unwrap());
|
||||
// // println!("Get string {:?}", store.get(&reader, "string").unwrap());
|
||||
// // println!("Get json {:?}", store.get(&reader, "json").unwrap());
|
||||
// println!("Get blob {:?}", store.get(&reader, "blob").unwrap());
|
||||
|
||||
// // Retrieving a non-existent value returns `Ok(None)`.
|
||||
// println!(
|
||||
// "Get non-existent value {:?}",
|
||||
// store.get(&reader, "non-existent").unwrap()
|
||||
// );
|
||||
|
||||
// // A read transaction will automatically close once the reader goes out of scope,
|
||||
// // so isn't necessary to close it explicitly, although you can do so by calling
|
||||
// // `Reader.abort()`.
|
||||
// }
|
||||
|
||||
// {
|
||||
// // Aborting a write transaction rolls back the change(s).
|
||||
// let mut writer = env.write().unwrap();
|
||||
// store.put(&mut writer, "foo", &Value::Blob(b"bar")).unwrap();
|
||||
// writer.abort();
|
||||
// let reader = env.read().expect("reader");
|
||||
// println!(
|
||||
// "It should be None! ({:?})",
|
||||
// store.get(&reader, "foo").unwrap()
|
||||
// );
|
||||
// }
|
||||
|
||||
// {
|
||||
// // Explicitly aborting a transaction is not required unless an early abort is
|
||||
// // desired, since both read and write transactions will implicitly be aborted once
|
||||
// // they go out of scope.
|
||||
// {
|
||||
// let mut writer = env.write().unwrap();
|
||||
// store.put(&mut writer, "foo", &Value::Blob(b"bar")).unwrap();
|
||||
// }
|
||||
// let reader = env.read().expect("reader");
|
||||
// println!(
|
||||
// "It should be None! ({:?})",
|
||||
// store.get(&reader, "foo").unwrap()
|
||||
// );
|
||||
// }
|
||||
|
||||
// {
|
||||
// // Deleting a key/value pair also requires a write transaction.
|
||||
// let mut writer = env.write().unwrap();
|
||||
// store.put(&mut writer, "foo", &Value::Blob(b"bar")).unwrap();
|
||||
// store.put(&mut writer, "bar", &Value::Blob(b"baz")).unwrap();
|
||||
// store.delete(&mut writer, "foo").unwrap();
|
||||
|
||||
// // A write transaction also supports reading, and the version of the store that it
|
||||
// // reads includes the changes it has made regardless of the commit state of that
|
||||
// // transaction.
|
||||
// // In the code above, "foo" and "bar" were put into the store, then "foo" was
|
||||
// // deleted so only "bar" will return a result when the database is queried via the
|
||||
// // writer.
|
||||
// println!(
|
||||
// "It should be None! ({:?})",
|
||||
// store.get(&writer, "foo").unwrap()
|
||||
// );
|
||||
// println!("Get bar ({:?})", store.get(&writer, "bar").unwrap());
|
||||
|
||||
// // But a reader won't see that change until the write transaction is committed.
|
||||
// {
|
||||
// let reader = env.read().expect("reader");
|
||||
// println!("Get foo {:?}", store.get(&reader, "foo").unwrap());
|
||||
// println!("Get bar {:?}", store.get(&reader, "bar").unwrap());
|
||||
// }
|
||||
// writer.commit().unwrap();
|
||||
// {
|
||||
// let reader = env.read().expect("reader");
|
||||
// println!(
|
||||
// "It should be None! ({:?})",
|
||||
// store.get(&reader, "foo").unwrap()
|
||||
// );
|
||||
// println!("Get bar {:?}", store.get(&reader, "bar").unwrap());
|
||||
// }
|
||||
|
||||
// // Committing a transaction consumes the writer, preventing you from reusing it by
|
||||
// // failing at compile time with an error. This line would report "error[E0382]:
|
||||
// // borrow of moved value: `writer`".
|
||||
// // store.put(&mut writer, "baz", &Value::Str("buz")).unwrap();
|
||||
// }
|
||||
|
||||
// {
|
||||
// // Clearing all the entries in the store with a write transaction.
|
||||
// {
|
||||
// let mut writer = env.write().unwrap();
|
||||
// store.put(&mut writer, "foo", &Value::Blob(b"bar")).unwrap();
|
||||
// store.put(&mut writer, "bar", &Value::Blob(b"baz")).unwrap();
|
||||
// writer.commit().unwrap();
|
||||
// }
|
||||
|
||||
// // {
|
||||
// // let mut writer = env.write().unwrap();
|
||||
// // store.clear(&mut writer).unwrap();
|
||||
// // writer.commit().unwrap();
|
||||
// // }
|
||||
|
||||
// // {
|
||||
// // let reader = env.read().expect("reader");
|
||||
// // println!(
|
||||
// // "It should be None! ({:?})",
|
||||
// // store.get(&reader, "foo").unwrap()
|
||||
// // );
|
||||
// // println!(
|
||||
// // "It should be None! ({:?})",
|
||||
// // store.get(&reader, "bar").unwrap()
|
||||
// // );
|
||||
// // }
|
||||
// }
|
||||
|
||||
let stat = env.stat().unwrap(); |
||||
let info = env.info().unwrap(); |
||||
println!("LMDB info map_size : {}", info.map_size()); |
||||
println!("LMDB info last_pgno : {}", info.last_pgno()); |
||||
println!("LMDB info last_txnid : {}", info.last_txnid()); |
||||
println!("LMDB info max_readers : {}", info.max_readers()); |
||||
println!("LMDB info num_readers : {}", info.num_readers()); |
||||
println!("LMDB stat page_size : {}", stat.page_size()); |
||||
println!("LMDB stat depth : {}", stat.depth()); |
||||
println!("LMDB stat branch_pages : {}", stat.branch_pages()); |
||||
println!("LMDB stat leaf_pages : {}", stat.leaf_pages()); |
||||
println!("LMDB stat overflow_pages : {}", stat.overflow_pages()); |
||||
println!("LMDB stat entries : {}", stat.entries()); |
||||
} |
||||
// We reopen the env and data to see if it was well saved to disk.
|
||||
{ |
||||
let mut manager = Manager::<LmdbEnvironment>::singleton().write().unwrap(); |
||||
let shared_rkv = manager |
||||
.get_or_create(root.path(), |path| { |
||||
//Rkv::new::<Lmdb>(path) // use this instead to disable encryption
|
||||
Rkv::with_encryption_key_and_mapsize::<Lmdb>(path, key, 2 * 1024 * 1024 * 1024) |
||||
}) |
||||
.unwrap(); |
||||
let env = shared_rkv.read().unwrap(); |
||||
|
||||
println!("LMDB Version: {}", env.version()); |
||||
|
||||
let mut store = env.open_single("testdb", StoreOptions::default()).unwrap(); //StoreOptions::create()
|
||||
|
||||
{ |
||||
let reader = env.read().expect("reader"); |
||||
println!( |
||||
"It should be baz! ({:?})", |
||||
store.get(&reader, "bar").unwrap() |
||||
); |
||||
} |
||||
} |
||||
// Here the database and environment is closed, but the files are still present in the temp directory.
|
||||
// uncomment this if you need time to copy them somewhere for analysis, before the temp folder get destroyed
|
||||
//thread::sleep(Duration::from_millis(20000));
|
||||
} |
||||
} |
||||
@ -0,0 +1,17 @@ |
||||
[package] |
||||
name = "p2p-verifier" |
||||
version = "0.1.0" |
||||
edition = "2021" |
||||
license = "MIT/Apache-2.0" |
||||
authors = ["Niko PLP <niko@nextgraph.org>"] |
||||
description = "P2P Verifier module of NextGraph" |
||||
repository = "https://git.nextgraph.org/NextGraph/nextgraph-rs" |
||||
|
||||
[dependencies] |
||||
p2p-repo = { path = "../p2p-repo" } |
||||
p2p-net = { path = "../p2p-net" } |
||||
blake3 = "1.3.1" |
||||
chacha20 = "0.9.0" |
||||
serde = { version = "1.0", features = ["derive"] } |
||||
serde_bare = "0.5.0" |
||||
serde_bytes = "0.11.7" |
||||
Loading…
Reference in new issue