Library: adds basic WASI support

Adds tests for WASM+JS and WASI to the CI
pull/352/head
Tpt 2 years ago committed by Thomas Tanon
parent f6e9ceccc1
commit 1fa0633db3
  1. 60
      .github/workflows/tests.yml
  2. 44
      Cargo.lock
  3. 3
      js/Cargo.toml
  4. 9
      lib/Cargo.toml
  5. 2
      lib/oxsdatatypes/Cargo.toml
  6. 4
      lib/oxsdatatypes/src/date_time.rs
  7. 2
      lib/src/sparql/http/dummy.rs
  8. 8
      lib/src/storage/backend/fallback.rs
  9. 8
      lib/src/storage/backend/mod.rs
  10. 2
      lib/src/storage/binary_encoder.rs
  11. 80
      lib/src/storage/mod.rs
  12. 19
      lib/src/store.rs
  13. 25
      lib/tests/store.rs

@ -37,6 +37,28 @@ jobs:
working-directory: ./lib/spargebra working-directory: ./lib/spargebra
- run: cargo clippy --all-targets --all-features - run: cargo clippy --all-targets --all-features
clippy_wasm_js:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
submodules: true
- run: rustup update && rustup target add wasm32-unknown-unknown && rustup component add clippy
- uses: Swatinem/rust-cache@v2
- run: cargo clippy --lib --tests --target wasm32-unknown-unknown
working-directory: ./js
clippy_wasi:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
submodules: true
- run: rustup update && rustup target add wasm32-wasi && rustup component add clippy
- uses: Swatinem/rust-cache@v2
- run: cargo clippy --lib --tests --target wasm32-wasi
working-directory: ./lib
clippy_msv: clippy_msv:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
@ -53,9 +75,31 @@ jobs:
working-directory: ./lib/sparesults working-directory: ./lib/sparesults
- run: cargo clippy -- -D warnings -D clippy::all - run: cargo clippy -- -D warnings -D clippy::all
working-directory: ./lib/spargebra working-directory: ./lib/spargebra
- run: cargo clippy -- -D warnings -D clippy::all - run: cargo clippy --all-targets -- -D warnings -D clippy::all
working-directory: ./server working-directory: ./server
clippy_msv_wasm_js:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
submodules: true
- run: rustup update && rustup override set 1.60.0 && rustup target add wasm32-unknown-unknown && rustup component add clippy
- uses: Swatinem/rust-cache@v2
- run: cargo clippy --lib --tests --target wasm32-unknown-unknown -- -D warnings -D clippy::all
working-directory: ./js
clippy_msv_wasi:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
submodules: true
- run: rustup update && rustup override set 1.60.0 && rustup target add wasm32-wasi && rustup component add clippy
- uses: Swatinem/rust-cache@v2
- run: cargo clippy --lib --tests --target wasm32-wasi -- -D warnings -D clippy::all
working-directory: ./lib
deny: deny:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
@ -100,6 +144,20 @@ jobs:
env: env:
RUST_BACKTRACE: 1 RUST_BACKTRACE: 1
test_wasi:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
submodules: true
- run: rustup update && rustup target add wasm32-wasi
- uses: Swatinem/rust-cache@v2
- uses: taiki-e/install-action@wasmtime
- run: cargo install cargo-wasi || true
- run: cargo wasi test --workspace --exclude oxigraph_js --exclude oxigraph_server --exclude oxigraph_testsuite --exclude oxrocksdb-sys --exclude pyoxigraph
env:
RUST_BACKTRACE: 1
js: js:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:

44
Cargo.lock generated

@ -1002,7 +1002,6 @@ dependencies = [
"sparesults", "sparesults",
"spargebra", "spargebra",
"sysinfo", "sysinfo",
"wasm-bindgen-test",
"zstd", "zstd",
] ]
@ -1014,7 +1013,6 @@ dependencies = [
"js-sys", "js-sys",
"oxigraph", "oxigraph",
"wasm-bindgen", "wasm-bindgen",
"wasm-bindgen-test",
] ]
[[package]] [[package]]
@ -1549,12 +1547,6 @@ dependencies = [
"windows-sys 0.36.1", "windows-sys 0.36.1",
] ]
[[package]]
name = "scoped-tls"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294"
[[package]] [[package]]
name = "scopeguard" name = "scopeguard"
version = "1.1.0" version = "1.1.0"
@ -1965,18 +1957,6 @@ dependencies = [
"wasm-bindgen-shared", "wasm-bindgen-shared",
] ]
[[package]]
name = "wasm-bindgen-futures"
version = "0.4.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d"
dependencies = [
"cfg-if",
"js-sys",
"wasm-bindgen",
"web-sys",
]
[[package]] [[package]]
name = "wasm-bindgen-macro" name = "wasm-bindgen-macro"
version = "0.2.83" version = "0.2.83"
@ -2006,30 +1986,6 @@ version = "0.2.83"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f"
[[package]]
name = "wasm-bindgen-test"
version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09d2fff962180c3fadf677438054b1db62bee4aa32af26a45388af07d1287e1d"
dependencies = [
"console_error_panic_hook",
"js-sys",
"scoped-tls",
"wasm-bindgen",
"wasm-bindgen-futures",
"wasm-bindgen-test-macro",
]
[[package]]
name = "wasm-bindgen-test-macro"
version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4683da3dfc016f704c9f82cf401520c4f1cb3ee440f7f52b3d6ac29506a49ca7"
dependencies = [
"proc-macro2",
"quote",
]
[[package]] [[package]]
name = "web-sys" name = "web-sys"
version = "0.3.60" version = "0.3.60"

@ -18,6 +18,3 @@ oxigraph = { version = "0.3.10", path="../lib" }
wasm-bindgen = "0.2" wasm-bindgen = "0.2"
js-sys = "0.3" js-sys = "0.3"
console_error_panic_hook = "0.1" console_error_panic_hook = "0.1"
[dev-dependencies]
wasm-bindgen-test = "0.3"

@ -42,22 +42,19 @@ oxsdatatypes = { version = "0.1.0", path="oxsdatatypes" }
spargebra = { version = "0.2.3", path="spargebra", features = ["rdf-star", "sep-0002", "sep-0006"] } spargebra = { version = "0.2.3", path="spargebra", features = ["rdf-star", "sep-0002", "sep-0006"] }
sparesults = { version = "0.1.3", path="sparesults", features = ["rdf-star"] } sparesults = { version = "0.1.3", path="sparesults", features = ["rdf-star"] }
[target.'cfg(not(target_arch = "wasm32"))'.dependencies] [target.'cfg(not(target_family = "wasm"))'.dependencies]
libc = "0.2" libc = "0.2"
oxrocksdb-sys = { version = "0.3.10", path="../oxrocksdb-sys" } oxrocksdb-sys = { version = "0.3.10", path="../oxrocksdb-sys" }
oxhttp = { version = "0.1", optional = true } oxhttp = { version = "0.1", optional = true }
[target.'cfg(target_arch = "wasm32")'.dependencies] [target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies]
getrandom = { version = "0.2", features = ["js"] } getrandom = { version = "0.2", features = ["js"] }
[dev-dependencies] [target.'cfg(not(target_family = "wasm"))'.dev-dependencies]
criterion = "0.4" criterion = "0.4"
oxhttp = "0.1" oxhttp = "0.1"
zstd = "0.12" zstd = "0.12"
[target.'cfg(target_arch = "wasm32")'.dev-dependencies]
wasm-bindgen-test = "0.3"
[[bench]] [[bench]]
name = "store" name = "store"
harness = false harness = false

@ -16,7 +16,7 @@ rust-version = "1.60"
[dependencies] [dependencies]
nom = "7" nom = "7"
[target.'cfg(target_arch = "wasm32")'.dependencies] [target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies]
js-sys = "0.3" js-sys = "0.3"
[package.metadata.docs.rs] [package.metadata.docs.rs]

@ -1602,7 +1602,7 @@ impl Timestamp {
} }
} }
#[cfg(target_arch = "wasm32")] #[cfg(all(target_family = "wasm", target_os = "unknown"))]
fn since_unix_epoch() -> Result<Duration, DateTimeError> { fn since_unix_epoch() -> Result<Duration, DateTimeError> {
Ok(Duration::new( Ok(Duration::new(
0, 0,
@ -1611,7 +1611,7 @@ fn since_unix_epoch() -> Result<Duration, DateTimeError> {
)) ))
} }
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(all(target_family = "wasm", target_os = "unknown")))]
fn since_unix_epoch() -> Result<Duration, DateTimeError> { fn since_unix_epoch() -> Result<Duration, DateTimeError> {
use std::time::SystemTime; use std::time::SystemTime;

@ -10,6 +10,7 @@ impl Client {
Self {} Self {}
} }
#[allow(clippy::unused_self)]
pub fn get(&self, _url: &str, _accept: &str) -> Result<(String, Empty)> { pub fn get(&self, _url: &str, _accept: &str) -> Result<(String, Empty)> {
Err(Error::new( Err(Error::new(
ErrorKind::Unsupported, ErrorKind::Unsupported,
@ -17,6 +18,7 @@ impl Client {
)) ))
} }
#[allow(clippy::unused_self, clippy::needless_pass_by_value)]
pub fn post( pub fn post(
&self, &self,
_url: &str, _url: &str,

@ -19,6 +19,7 @@ pub struct ColumnFamilyDefinition {
pub struct Db(Arc<RwLock<HashMap<ColumnFamily, BTreeMap<Vec<u8>, Vec<u8>>>>>); pub struct Db(Arc<RwLock<HashMap<ColumnFamily, BTreeMap<Vec<u8>, Vec<u8>>>>>);
impl Db { impl Db {
#[allow(clippy::unnecessary_wraps)]
pub fn new(column_families: Vec<ColumnFamilyDefinition>) -> Result<Self, StorageError> { pub fn new(column_families: Vec<ColumnFamilyDefinition>) -> Result<Self, StorageError> {
let mut trees = HashMap::new(); let mut trees = HashMap::new();
for cf in column_families { for cf in column_families {
@ -225,7 +226,7 @@ pub struct Transaction<'a>(
); );
impl Transaction<'_> { impl Transaction<'_> {
#[allow(unsafe_code)] #[allow(unsafe_code, clippy::useless_transmute)]
pub fn reader(&self) -> Reader { pub fn reader(&self) -> Reader {
// This transmute is safe because we take a weak reference and the only Rc reference used is guarded by the lifetime. // This transmute is safe because we take a weak reference and the only Rc reference used is guarded by the lifetime.
Reader(InnerReader::Transaction(Rc::downgrade(unsafe { Reader(InnerReader::Transaction(Rc::downgrade(unsafe {
@ -233,6 +234,7 @@ impl Transaction<'_> {
}))) })))
} }
#[allow(clippy::unnecessary_wraps)]
pub fn contains_key_for_update( pub fn contains_key_for_update(
&self, &self,
column_family: &ColumnFamily, column_family: &ColumnFamily,
@ -244,6 +246,7 @@ impl Transaction<'_> {
.map_or(false, |cf| cf.contains_key(key))) .map_or(false, |cf| cf.contains_key(key)))
} }
#[allow(clippy::unnecessary_wraps)]
pub fn insert( pub fn insert(
&mut self, &mut self,
column_family: &ColumnFamily, column_family: &ColumnFamily,
@ -266,6 +269,7 @@ impl Transaction<'_> {
self.insert(column_family, key, &[]) self.insert(column_family, key, &[])
} }
#[allow(clippy::unnecessary_wraps)]
pub fn remove(&mut self, column_family: &ColumnFamily, key: &[u8]) -> Result<(), StorageError> { pub fn remove(&mut self, column_family: &ColumnFamily, key: &[u8]) -> Result<(), StorageError> {
self.0 self.0
.borrow_mut() .borrow_mut()
@ -286,6 +290,7 @@ impl Iter {
Some(&self.current.as_ref()?.0) Some(&self.current.as_ref()?.0)
} }
#[allow(dead_code)]
pub fn value(&self) -> Option<&[u8]> { pub fn value(&self) -> Option<&[u8]> {
Some(&self.current.as_ref()?.1) Some(&self.current.as_ref()?.1)
} }
@ -294,6 +299,7 @@ impl Iter {
self.current = self.iter.next(); self.current = self.iter.next();
} }
#[allow(clippy::unnecessary_wraps, clippy::unused_self)]
pub fn status(&self) -> Result<(), StorageError> { pub fn status(&self) -> Result<(), StorageError> {
Ok(()) Ok(())
} }

@ -1,14 +1,14 @@
//! A storage backend //! A storage backend
//! RocksDB is available, if not in memory //! RocksDB is available, if not in memory
#[cfg(target_arch = "wasm32")] #[cfg(target_family = "wasm")]
pub use fallback::{ColumnFamily, ColumnFamilyDefinition, Db, Iter, Reader, Transaction}; pub use fallback::{ColumnFamily, ColumnFamilyDefinition, Db, Iter, Reader, Transaction};
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_family = "wasm"))]
pub use rocksdb::{ pub use rocksdb::{
ColumnFamily, ColumnFamilyDefinition, Db, Iter, Reader, SstFileWriter, Transaction, ColumnFamily, ColumnFamilyDefinition, Db, Iter, Reader, SstFileWriter, Transaction,
}; };
#[cfg(target_arch = "wasm32")] #[cfg(target_family = "wasm")]
mod fallback; mod fallback;
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_family = "wasm"))]
mod rocksdb; mod rocksdb;

@ -6,7 +6,7 @@ use oxsdatatypes::*;
use std::io::{Cursor, Read}; use std::io::{Cursor, Read};
use std::mem::size_of; use std::mem::size_of;
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_family = "wasm"))]
pub const LATEST_STORAGE_VERSION: u64 = 1; pub const LATEST_STORAGE_VERSION: u64 = 1;
pub const WRITTEN_TERM_MAX_SIZE: usize = size_of::<u8>() + 2 * size_of::<StrHash>(); pub const WRITTEN_TERM_MAX_SIZE: usize = size_of::<u8>() + 2 * size_of::<StrHash>();

@ -1,6 +1,8 @@
use crate::model::{GraphNameRef, NamedOrBlankNodeRef, Quad, QuadRef, TermRef}; #[cfg(not(target_family = "wasm"))]
use crate::model::Quad;
use crate::model::{GraphNameRef, NamedOrBlankNodeRef, QuadRef, TermRef};
use crate::storage::backend::{Reader, Transaction}; use crate::storage::backend::{Reader, Transaction};
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_family = "wasm"))]
use crate::storage::binary_encoder::LATEST_STORAGE_VERSION; use crate::storage::binary_encoder::LATEST_STORAGE_VERSION;
use crate::storage::binary_encoder::{ use crate::storage::binary_encoder::{
decode_term, encode_term, encode_term_pair, encode_term_quad, encode_term_triple, decode_term, encode_term, encode_term_pair, encode_term_quad, encode_term_triple,
@ -9,24 +11,30 @@ use crate::storage::binary_encoder::{
WRITTEN_TERM_MAX_SIZE, WRITTEN_TERM_MAX_SIZE,
}; };
pub use crate::storage::error::{CorruptionError, LoaderError, SerializerError, StorageError}; pub use crate::storage::error::{CorruptionError, LoaderError, SerializerError, StorageError};
use crate::storage::numeric_encoder::{ #[cfg(not(target_family = "wasm"))]
insert_term, Decoder, EncodedQuad, EncodedTerm, StrHash, StrLookup, use crate::storage::numeric_encoder::Decoder;
}; use crate::storage::numeric_encoder::{insert_term, EncodedQuad, EncodedTerm, StrHash, StrLookup};
use backend::{ColumnFamily, ColumnFamilyDefinition, Db, Iter}; use backend::{ColumnFamily, ColumnFamilyDefinition, Db, Iter};
#[cfg(not(target_family = "wasm"))]
use std::cmp::{max, min}; use std::cmp::{max, min};
#[cfg(not(target_family = "wasm"))]
use std::collections::VecDeque; use std::collections::VecDeque;
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_family = "wasm"))]
use std::collections::{HashMap, HashSet}; use std::collections::{HashMap, HashSet};
use std::error::Error; use std::error::Error;
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_family = "wasm"))]
use std::mem::take; use std::mem::take;
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_family = "wasm"))]
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
#[cfg(not(target_family = "wasm"))]
use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::atomic::{AtomicU64, Ordering};
#[cfg(not(target_family = "wasm"))]
use std::sync::Arc; use std::sync::Arc;
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_family = "wasm"))]
use std::thread::spawn; use std::thread::spawn;
#[cfg(not(target_family = "wasm"))]
use std::thread::JoinHandle; use std::thread::JoinHandle;
#[cfg(not(target_family = "wasm"))]
use sysinfo::{System, SystemExt}; use sysinfo::{System, SystemExt};
mod backend; mod backend;
@ -46,15 +54,18 @@ const DSPO_CF: &str = "dspo";
const DPOS_CF: &str = "dpos"; const DPOS_CF: &str = "dpos";
const DOSP_CF: &str = "dosp"; const DOSP_CF: &str = "dosp";
const GRAPHS_CF: &str = "graphs"; const GRAPHS_CF: &str = "graphs";
#[cfg(not(target_family = "wasm"))]
const DEFAULT_CF: &str = "default"; const DEFAULT_CF: &str = "default";
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_family = "wasm"))]
const DEFAULT_BULK_LOAD_BATCH_SIZE: usize = 1_000_000; const DEFAULT_BULK_LOAD_BATCH_SIZE: usize = 1_000_000;
#[cfg(not(target_family = "wasm"))]
const MAX_BULK_LOAD_BATCH_SIZE: usize = 100_000_000; const MAX_BULK_LOAD_BATCH_SIZE: usize = 100_000_000;
/// Low level storage primitives /// Low level storage primitives
#[derive(Clone)] #[derive(Clone)]
pub struct Storage { pub struct Storage {
db: Db, db: Db,
#[cfg(not(target_family = "wasm"))]
default_cf: ColumnFamily, default_cf: ColumnFamily,
id2str_cf: ColumnFamily, id2str_cf: ColumnFamily,
spog_cf: ColumnFamily, spog_cf: ColumnFamily,
@ -74,7 +85,7 @@ impl Storage {
Self::setup(Db::new(Self::column_families())?) Self::setup(Db::new(Self::column_families())?)
} }
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_family = "wasm"))]
pub fn open(path: &Path) -> Result<Self, StorageError> { pub fn open(path: &Path) -> Result<Self, StorageError> {
Self::setup(Db::open(path, Self::column_families())?) Self::setup(Db::open(path, Self::column_families())?)
} }
@ -150,8 +161,10 @@ impl Storage {
] ]
} }
#[allow(clippy::unnecessary_wraps)]
fn setup(db: Db) -> Result<Self, StorageError> { fn setup(db: Db) -> Result<Self, StorageError> {
let this = Self { let this = Self {
#[cfg(not(target_family = "wasm"))]
default_cf: db.column_family(DEFAULT_CF).unwrap(), default_cf: db.column_family(DEFAULT_CF).unwrap(),
id2str_cf: db.column_family(ID2STR_CF).unwrap(), id2str_cf: db.column_family(ID2STR_CF).unwrap(),
spog_cf: db.column_family(SPOG_CF).unwrap(), spog_cf: db.column_family(SPOG_CF).unwrap(),
@ -166,12 +179,12 @@ impl Storage {
graphs_cf: db.column_family(GRAPHS_CF).unwrap(), graphs_cf: db.column_family(GRAPHS_CF).unwrap(),
db, db,
}; };
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_family = "wasm"))]
this.migrate()?; this.migrate()?;
Ok(this) Ok(this)
} }
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_family = "wasm"))]
fn migrate(&self) -> Result<(), StorageError> { fn migrate(&self) -> Result<(), StorageError> {
let mut version = self.ensure_version()?; let mut version = self.ensure_version()?;
if version == 0 { if version == 0 {
@ -211,7 +224,7 @@ impl Storage {
} }
} }
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_family = "wasm"))]
fn ensure_version(&self) -> Result<u64, StorageError> { fn ensure_version(&self) -> Result<u64, StorageError> {
Ok( Ok(
if let Some(version) = self.db.get(&self.default_cf, b"oxversion")? { if let Some(version) = self.db.get(&self.default_cf, b"oxversion")? {
@ -225,7 +238,7 @@ impl Storage {
) )
} }
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_family = "wasm"))]
fn update_version(&self, version: u64) -> Result<(), StorageError> { fn update_version(&self, version: u64) -> Result<(), StorageError> {
self.db self.db
.insert(&self.default_cf, b"oxversion", &version.to_be_bytes())?; .insert(&self.default_cf, b"oxversion", &version.to_be_bytes())?;
@ -252,7 +265,7 @@ impl Storage {
}) })
} }
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_family = "wasm"))]
pub fn flush(&self) -> Result<(), StorageError> { pub fn flush(&self) -> Result<(), StorageError> {
self.db.flush(&self.default_cf)?; self.db.flush(&self.default_cf)?;
self.db.flush(&self.gpos_cf)?; self.db.flush(&self.gpos_cf)?;
@ -267,7 +280,7 @@ impl Storage {
self.db.flush(&self.id2str_cf) self.db.flush(&self.id2str_cf)
} }
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_family = "wasm"))]
pub fn compact(&self) -> Result<(), StorageError> { pub fn compact(&self) -> Result<(), StorageError> {
self.db.compact(&self.default_cf)?; self.db.compact(&self.default_cf)?;
self.db.compact(&self.gpos_cf)?; self.db.compact(&self.gpos_cf)?;
@ -282,7 +295,7 @@ impl Storage {
self.db.compact(&self.id2str_cf) self.db.compact(&self.id2str_cf)
} }
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_family = "wasm"))]
pub fn backup(&self, target_directory: &Path) -> Result<(), StorageError> { pub fn backup(&self, target_directory: &Path) -> Result<(), StorageError> {
self.db.backup(target_directory) self.db.backup(target_directory)
} }
@ -607,7 +620,7 @@ impl StorageReader {
} }
} }
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_family = "wasm"))]
pub fn get_str(&self, key: &StrHash) -> Result<Option<String>, StorageError> { pub fn get_str(&self, key: &StrHash) -> Result<Option<String>, StorageError> {
Ok(self Ok(self
.storage .storage
@ -618,31 +631,31 @@ impl StorageReader {
.map_err(CorruptionError::new)?) .map_err(CorruptionError::new)?)
} }
#[cfg(target_arch = "wasm32")] #[cfg(target_family = "wasm")]
pub fn get_str(&self, key: &StrHash) -> Result<Option<String>, StorageError> { pub fn get_str(&self, key: &StrHash) -> Result<Option<String>, StorageError> {
Ok(self Ok(self
.reader .reader
.get(&self.storage.id2str_cf, &key.to_be_bytes())? .get(&self.storage.id2str_cf, &key.to_be_bytes())?
.map(|v| String::from_utf8(v.into())) .map(String::from_utf8)
.transpose() .transpose()
.map_err(CorruptionError::new)?) .map_err(CorruptionError::new)?)
} }
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_family = "wasm"))]
pub fn contains_str(&self, key: &StrHash) -> Result<bool, StorageError> { pub fn contains_str(&self, key: &StrHash) -> Result<bool, StorageError> {
self.storage self.storage
.db .db
.contains_key(&self.storage.id2str_cf, &key.to_be_bytes()) .contains_key(&self.storage.id2str_cf, &key.to_be_bytes())
} }
#[cfg(target_arch = "wasm32")] #[cfg(target_family = "wasm")]
pub fn contains_str(&self, key: &StrHash) -> Result<bool, StorageError> { pub fn contains_str(&self, key: &StrHash) -> Result<bool, StorageError> {
self.reader self.reader
.contains_key(&self.storage.id2str_cf, &key.to_be_bytes()) .contains_key(&self.storage.id2str_cf, &key.to_be_bytes())
} }
/// Validates that all the storage invariants held in the data /// Validates that all the storage invariants held in the data
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_family = "wasm"))]
pub fn validate(&self) -> Result<(), StorageError> { pub fn validate(&self) -> Result<(), StorageError> {
// triples // triples
let dspo_size = self.dspo_quads(&[]).count(); let dspo_size = self.dspo_quads(&[]).count();
@ -752,6 +765,13 @@ impl StorageReader {
} }
Ok(()) Ok(())
} }
/// Validates that all the storage invariants held in the data
#[cfg(target_family = "wasm")]
#[allow(clippy::unused_self, clippy::unnecessary_wraps)]
pub fn validate(&self) -> Result<(), StorageError> {
Ok(()) //TODO
}
} }
pub struct ChainedDecodingQuadIterator { pub struct ChainedDecodingQuadIterator {
@ -975,7 +995,7 @@ impl<'a> StorageWriter<'a> {
} }
} }
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_family = "wasm"))]
fn insert_str(&mut self, key: &StrHash, value: &str) -> Result<(), StorageError> { fn insert_str(&mut self, key: &StrHash, value: &str) -> Result<(), StorageError> {
if self if self
.storage .storage
@ -991,7 +1011,7 @@ impl<'a> StorageWriter<'a> {
) )
} }
#[cfg(target_arch = "wasm32")] #[cfg(target_family = "wasm")]
fn insert_str(&mut self, key: &StrHash, value: &str) -> Result<(), StorageError> { fn insert_str(&mut self, key: &StrHash, value: &str) -> Result<(), StorageError> {
self.transaction.insert( self.transaction.insert(
&self.storage.id2str_cf, &self.storage.id2str_cf,
@ -1156,7 +1176,7 @@ impl<'a> StorageWriter<'a> {
} }
} }
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_family = "wasm"))]
pub struct StorageBulkLoader { pub struct StorageBulkLoader {
storage: Storage, storage: Storage,
hooks: Vec<Box<dyn Fn(u64)>>, hooks: Vec<Box<dyn Fn(u64)>>,
@ -1164,7 +1184,7 @@ pub struct StorageBulkLoader {
max_memory_size: Option<usize>, max_memory_size: Option<usize>,
} }
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_family = "wasm"))]
impl StorageBulkLoader { impl StorageBulkLoader {
pub fn new(storage: Storage) -> Self { pub fn new(storage: Storage) -> Self {
Self { Self {
@ -1290,7 +1310,7 @@ impl StorageBulkLoader {
} }
} }
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_family = "wasm"))]
struct FileBulkLoader { struct FileBulkLoader {
storage: Storage, storage: Storage,
id2str: HashMap<StrHash, Box<str>>, id2str: HashMap<StrHash, Box<str>>,
@ -1299,7 +1319,7 @@ struct FileBulkLoader {
graphs: HashSet<EncodedTerm>, graphs: HashSet<EncodedTerm>,
} }
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_family = "wasm"))]
impl FileBulkLoader { impl FileBulkLoader {
fn new(storage: Storage) -> Self { fn new(storage: Storage) -> Self {
Self { Self {

@ -33,7 +33,7 @@ use crate::sparql::{
UpdateOptions, UpdateOptions,
}; };
use crate::storage::numeric_encoder::{Decoder, EncodedQuad, EncodedTerm}; use crate::storage::numeric_encoder::{Decoder, EncodedQuad, EncodedTerm};
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_family = "wasm"))]
use crate::storage::StorageBulkLoader; use crate::storage::StorageBulkLoader;
use crate::storage::{ use crate::storage::{
ChainedDecodingQuadIterator, DecodingGraphIterator, Storage, StorageReader, StorageWriter, ChainedDecodingQuadIterator, DecodingGraphIterator, Storage, StorageReader, StorageWriter,
@ -41,7 +41,7 @@ use crate::storage::{
pub use crate::storage::{CorruptionError, LoaderError, SerializerError, StorageError}; pub use crate::storage::{CorruptionError, LoaderError, SerializerError, StorageError};
use std::error::Error; use std::error::Error;
use std::io::{BufRead, Write}; use std::io::{BufRead, Write};
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_family = "wasm"))]
use std::path::Path; use std::path::Path;
use std::{fmt, str}; use std::{fmt, str};
@ -95,7 +95,7 @@ impl Store {
} }
/// Opens a [`Store`] and creates it if it does not exist yet. /// Opens a [`Store`] and creates it if it does not exist yet.
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_family = "wasm"))]
pub fn open(path: impl AsRef<Path>) -> Result<Self, StorageError> { pub fn open(path: impl AsRef<Path>) -> Result<Self, StorageError> {
Ok(Self { Ok(Self {
storage: Storage::open(path.as_ref())?, storage: Storage::open(path.as_ref())?,
@ -717,7 +717,7 @@ impl Store {
/// Flushes all buffers and ensures that all writes are saved on disk. /// Flushes all buffers and ensures that all writes are saved on disk.
/// ///
/// Flushes are automatically done using background threads but might lag a little bit. /// Flushes are automatically done using background threads but might lag a little bit.
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_family = "wasm"))]
pub fn flush(&self) -> Result<(), StorageError> { pub fn flush(&self) -> Result<(), StorageError> {
self.storage.flush() self.storage.flush()
} }
@ -727,7 +727,7 @@ impl Store {
/// Useful to call after a batch upload or another similar operation. /// Useful to call after a batch upload or another similar operation.
/// ///
/// Warning: Can take hours on huge databases. /// Warning: Can take hours on huge databases.
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_family = "wasm"))]
pub fn optimize(&self) -> Result<(), StorageError> { pub fn optimize(&self) -> Result<(), StorageError> {
self.storage.compact() self.storage.compact()
} }
@ -748,7 +748,7 @@ impl Store {
/// This allows cheap regular backups. /// This allows cheap regular backups.
/// ///
/// If you want to move your data to another RDF storage system, you should have a look at the [`Store::dump_dataset`] function instead. /// If you want to move your data to another RDF storage system, you should have a look at the [`Store::dump_dataset`] function instead.
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_family = "wasm"))]
pub fn backup(&self, target_directory: impl AsRef<Path>) -> Result<(), StorageError> { pub fn backup(&self, target_directory: impl AsRef<Path>) -> Result<(), StorageError> {
self.storage.backup(target_directory.as_ref()) self.storage.backup(target_directory.as_ref())
} }
@ -772,7 +772,7 @@ impl Store {
/// assert!(store.contains(QuadRef::new(ex, ex, ex, ex))?); /// assert!(store.contains(QuadRef::new(ex, ex, ex, ex))?);
/// # Result::<_, Box<dyn std::error::Error>>::Ok(()) /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
/// ``` /// ```
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_family = "wasm"))]
pub fn bulk_loader(&self) -> BulkLoader { pub fn bulk_loader(&self) -> BulkLoader {
BulkLoader { BulkLoader {
storage: StorageBulkLoader::new(self.storage.clone()), storage: StorageBulkLoader::new(self.storage.clone()),
@ -782,7 +782,6 @@ impl Store {
/// Validates that all the store invariants held in the data /// Validates that all the store invariants held in the data
#[doc(hidden)] #[doc(hidden)]
#[cfg(not(target_arch = "wasm32"))]
pub fn validate(&self) -> Result<(), StorageError> { pub fn validate(&self) -> Result<(), StorageError> {
self.storage.snapshot().validate() self.storage.snapshot().validate()
} }
@ -1296,13 +1295,13 @@ impl Iterator for GraphNameIter {
/// assert!(store.contains(QuadRef::new(ex, ex, ex, ex))?); /// assert!(store.contains(QuadRef::new(ex, ex, ex, ex))?);
/// # Result::<_, Box<dyn std::error::Error>>::Ok(()) /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
/// ``` /// ```
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_family = "wasm"))]
pub struct BulkLoader { pub struct BulkLoader {
storage: StorageBulkLoader, storage: StorageBulkLoader,
on_parse_error: Option<Box<dyn Fn(ParseError) -> Result<(), ParseError>>>, on_parse_error: Option<Box<dyn Fn(ParseError) -> Result<(), ParseError>>>,
} }
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_family = "wasm"))]
impl BulkLoader { impl BulkLoader {
/// Sets the maximal number of threads to be used by the bulk loader per operation. /// Sets the maximal number of threads to be used by the bulk loader per operation.
/// ///

@ -2,13 +2,21 @@ use oxigraph::io::{DatasetFormat, GraphFormat};
use oxigraph::model::vocab::{rdf, xsd}; use oxigraph::model::vocab::{rdf, xsd};
use oxigraph::model::*; use oxigraph::model::*;
use oxigraph::store::Store; use oxigraph::store::Store;
#[cfg(not(target_family = "wasm"))]
use rand::random; use rand::random;
#[cfg(not(target_family = "wasm"))]
use std::env::temp_dir; use std::env::temp_dir;
use std::error::Error; use std::error::Error;
#[cfg(not(target_family = "wasm"))]
use std::fs::{create_dir, remove_dir_all, File}; use std::fs::{create_dir, remove_dir_all, File};
use std::io::{Cursor, Write}; use std::io::Cursor;
#[cfg(not(target_family = "wasm"))]
use std::io::Write;
#[cfg(not(target_family = "wasm"))]
use std::iter::once; use std::iter::once;
#[cfg(not(target_family = "wasm"))]
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
#[cfg(not(target_family = "wasm"))]
use std::process::Command; use std::process::Command;
const DATA: &str = r#" const DATA: &str = r#"
@ -111,6 +119,7 @@ fn test_load_graph() -> Result<(), Box<dyn Error>> {
} }
#[test] #[test]
#[cfg(not(target_family = "wasm"))]
fn test_bulk_load_graph() -> Result<(), Box<dyn Error>> { fn test_bulk_load_graph() -> Result<(), Box<dyn Error>> {
let store = Store::new()?; let store = Store::new()?;
store.bulk_loader().load_graph( store.bulk_loader().load_graph(
@ -127,6 +136,7 @@ fn test_bulk_load_graph() -> Result<(), Box<dyn Error>> {
} }
#[test] #[test]
#[cfg(not(target_family = "wasm"))]
fn test_bulk_load_graph_lenient() -> Result<(), Box<dyn Error>> { fn test_bulk_load_graph_lenient() -> Result<(), Box<dyn Error>> {
let store = Store::new()?; let store = Store::new()?;
store.bulk_loader().on_parse_error(|_| Ok(())).load_graph( store.bulk_loader().on_parse_error(|_| Ok(())).load_graph(
@ -160,6 +170,7 @@ fn test_load_dataset() -> Result<(), Box<dyn Error>> {
} }
#[test] #[test]
#[cfg(not(target_family = "wasm"))]
fn test_bulk_load_dataset() -> Result<(), Box<dyn Error>> { fn test_bulk_load_dataset() -> Result<(), Box<dyn Error>> {
let store = Store::new().unwrap(); let store = Store::new().unwrap();
store store
@ -238,15 +249,16 @@ fn test_snapshot_isolation_iterator() -> Result<(), Box<dyn Error>> {
store.insert(quad)?; store.insert(quad)?;
let iter = store.iter(); let iter = store.iter();
store.remove(quad)?; store.remove(quad)?;
store.validate()?;
assert_eq!( assert_eq!(
iter.collect::<Result<Vec<_>, _>>()?, iter.collect::<Result<Vec<_>, _>>()?,
vec![quad.into_owned()] vec![quad.into_owned()]
); );
store.validate()?;
Ok(()) Ok(())
} }
#[test] #[test]
#[cfg(not(target_family = "wasm"))]
fn test_bulk_load_on_existing_delete_overrides_the_delete() -> Result<(), Box<dyn Error>> { fn test_bulk_load_on_existing_delete_overrides_the_delete() -> Result<(), Box<dyn Error>> {
let quad = QuadRef::new( let quad = QuadRef::new(
NamedNodeRef::new_unchecked("http://example.com/s"), NamedNodeRef::new_unchecked("http://example.com/s"),
@ -262,6 +274,7 @@ fn test_bulk_load_on_existing_delete_overrides_the_delete() -> Result<(), Box<dy
} }
#[test] #[test]
#[cfg(not(target_family = "wasm"))]
fn test_open_bad_dir() -> Result<(), Box<dyn Error>> { fn test_open_bad_dir() -> Result<(), Box<dyn Error>> {
let dir = TempDir::default(); let dir = TempDir::default();
create_dir(&dir.0)?; create_dir(&dir.0)?;
@ -291,6 +304,7 @@ fn test_bad_stt_open() -> Result<(), Box<dyn Error>> {
} }
#[test] #[test]
#[cfg(not(target_family = "wasm"))]
fn test_backup() -> Result<(), Box<dyn Error>> { fn test_backup() -> Result<(), Box<dyn Error>> {
let quad = QuadRef { let quad = QuadRef {
subject: NamedNodeRef::new_unchecked("http://example.com/s").into(), subject: NamedNodeRef::new_unchecked("http://example.com/s").into(),
@ -314,6 +328,7 @@ fn test_backup() -> Result<(), Box<dyn Error>> {
} }
#[test] #[test]
#[cfg(not(target_family = "wasm"))]
fn test_bad_backup() -> Result<(), Box<dyn Error>> { fn test_bad_backup() -> Result<(), Box<dyn Error>> {
let store_dir = TempDir::default(); let store_dir = TempDir::default();
let backup_dir = TempDir::default(); let backup_dir = TempDir::default();
@ -324,6 +339,7 @@ fn test_bad_backup() -> Result<(), Box<dyn Error>> {
} }
#[test] #[test]
#[cfg(not(target_family = "wasm"))]
fn test_backup_on_in_memory() -> Result<(), Box<dyn Error>> { fn test_backup_on_in_memory() -> Result<(), Box<dyn Error>> {
let backup_dir = TempDir::default(); let backup_dir = TempDir::default();
assert!(Store::new()?.backup(&backup_dir).is_err()); assert!(Store::new()?.backup(&backup_dir).is_err());
@ -354,6 +370,7 @@ fn test_backward_compatibility() -> Result<(), Box<dyn Error>> {
Ok(()) Ok(())
} }
#[cfg(not(target_family = "wasm"))]
fn reset_dir(dir: &str) -> Result<(), Box<dyn Error>> { fn reset_dir(dir: &str) -> Result<(), Box<dyn Error>> {
assert!(Command::new("git") assert!(Command::new("git")
.args(["clean", "-fX", dir]) .args(["clean", "-fX", dir])
@ -366,20 +383,24 @@ fn reset_dir(dir: &str) -> Result<(), Box<dyn Error>> {
Ok(()) Ok(())
} }
#[cfg(not(target_family = "wasm"))]
struct TempDir(PathBuf); struct TempDir(PathBuf);
#[cfg(not(target_family = "wasm"))]
impl Default for TempDir { impl Default for TempDir {
fn default() -> Self { fn default() -> Self {
Self(temp_dir().join(format!("oxigraph-test-{}", random::<u128>()))) Self(temp_dir().join(format!("oxigraph-test-{}", random::<u128>())))
} }
} }
#[cfg(not(target_family = "wasm"))]
impl AsRef<Path> for TempDir { impl AsRef<Path> for TempDir {
fn as_ref(&self) -> &Path { fn as_ref(&self) -> &Path {
&self.0 &self.0
} }
} }
#[cfg(not(target_family = "wasm"))]
impl Drop for TempDir { impl Drop for TempDir {
fn drop(&mut self) { fn drop(&mut self) {
let _ = remove_dir_all(&self.0); let _ = remove_dir_all(&self.0);

Loading…
Cancel
Save