From 6aa732ba0acf9d0b3eb306f16900bd3835aa0d7f Mon Sep 17 00:00:00 2001
From: Tpt <thomaspt@hotmail.fr>
Date: Fri, 6 Jan 2023 18:07:55 +0100
Subject: [PATCH] Library: adds basic WASI support
Adds tests for WASM+JS and WASI to the CI
---
.github/workflows/tests.yml | 60 +++++++++++++++++++++-
Cargo.lock | 44 ----------------
js/Cargo.toml | 3 --
lib/Cargo.toml | 11 ++--
lib/oxsdatatypes/Cargo.toml | 2 +-
lib/oxsdatatypes/src/date_time.rs | 4 +-
lib/src/sparql/http/dummy.rs | 2 +
lib/src/storage/backend/fallback.rs | 8 ++-
lib/src/storage/backend/mod.rs | 8 +--
lib/src/storage/binary_encoder.rs | 2 +-
lib/src/storage/mod.rs | 80 ++++++++++++++++++-----------
lib/src/store.rs | 19 ++++---
lib/tests/store.rs | 25 ++++++++-
13 files changed, 162 insertions(+), 106 deletions(-)
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index c461cf32..70f4571b 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -37,6 +37,28 @@ jobs:
working-directory: ./lib/spargebra
- run: cargo clippy --all-targets --all-features
+ clippy_wasm_js:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ submodules: true
+ - run: rustup update && rustup target add wasm32-unknown-unknown && rustup component add clippy
+ - uses: Swatinem/rust-cache@v2
+ - run: cargo clippy --lib --tests --target wasm32-unknown-unknown
+ working-directory: ./js
+
+ clippy_wasi:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ submodules: true
+ - run: rustup update && rustup target add wasm32-wasi && rustup component add clippy
+ - uses: Swatinem/rust-cache@v2
+ - run: cargo clippy --lib --tests --target wasm32-wasi
+ working-directory: ./lib
+
clippy_msv:
runs-on: ubuntu-latest
steps:
@@ -53,9 +75,31 @@ jobs:
working-directory: ./lib/sparesults
- run: cargo clippy -- -D warnings -D clippy::all
working-directory: ./lib/spargebra
- - run: cargo clippy -- -D warnings -D clippy::all
+ - run: cargo clippy --all-targets -- -D warnings -D clippy::all
working-directory: ./server
+ clippy_msv_wasm_js:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ submodules: true
+ - run: rustup update && rustup override set 1.60.0 && rustup target add wasm32-unknown-unknown && rustup component add clippy
+ - uses: Swatinem/rust-cache@v2
+ - run: cargo clippy --lib --tests --target wasm32-unknown-unknown -- -D warnings -D clippy::all
+ working-directory: ./js
+
+ clippy_msv_wasi:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ submodules: true
+ - run: rustup update && rustup override set 1.60.0 && rustup target add wasm32-wasi && rustup component add clippy
+ - uses: Swatinem/rust-cache@v2
+ - run: cargo clippy --lib --tests --target wasm32-wasi -- -D warnings -D clippy::all
+ working-directory: ./lib
+
deny:
runs-on: ubuntu-latest
steps:
@@ -100,6 +144,20 @@ jobs:
env:
RUST_BACKTRACE: 1
+ test_wasi:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ submodules: true
+ - run: rustup update && rustup target add wasm32-wasi
+ - uses: Swatinem/rust-cache@v2
+ - uses: taiki-e/install-action@wasmtime
+ - run: cargo install cargo-wasi || true
+ - run: cargo wasi test --workspace --exclude oxigraph_js --exclude oxigraph_server --exclude oxigraph_testsuite --exclude oxrocksdb-sys --exclude pyoxigraph
+ env:
+ RUST_BACKTRACE: 1
+
js:
runs-on: ubuntu-latest
steps:
diff --git a/Cargo.lock b/Cargo.lock
index 97300c25..e3b02f2d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1002,7 +1002,6 @@ dependencies = [
"sparesults",
"spargebra",
"sysinfo",
- "wasm-bindgen-test",
"zstd",
]
@@ -1014,7 +1013,6 @@ dependencies = [
"js-sys",
"oxigraph",
"wasm-bindgen",
- "wasm-bindgen-test",
]
[[package]]
@@ -1549,12 +1547,6 @@ dependencies = [
"windows-sys 0.36.1",
]
-[[package]]
-name = "scoped-tls"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294"
-
[[package]]
name = "scopeguard"
version = "1.1.0"
@@ -1965,18 +1957,6 @@ dependencies = [
"wasm-bindgen-shared",
]
-[[package]]
-name = "wasm-bindgen-futures"
-version = "0.4.33"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d"
-dependencies = [
- "cfg-if",
- "js-sys",
- "wasm-bindgen",
- "web-sys",
-]
-
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.83"
@@ -2006,30 +1986,6 @@ version = "0.2.83"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f"
-[[package]]
-name = "wasm-bindgen-test"
-version = "0.3.33"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09d2fff962180c3fadf677438054b1db62bee4aa32af26a45388af07d1287e1d"
-dependencies = [
- "console_error_panic_hook",
- "js-sys",
- "scoped-tls",
- "wasm-bindgen",
- "wasm-bindgen-futures",
- "wasm-bindgen-test-macro",
-]
-
-[[package]]
-name = "wasm-bindgen-test-macro"
-version = "0.3.33"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4683da3dfc016f704c9f82cf401520c4f1cb3ee440f7f52b3d6ac29506a49ca7"
-dependencies = [
- "proc-macro2",
- "quote",
-]
-
[[package]]
name = "web-sys"
version = "0.3.60"
diff --git a/js/Cargo.toml b/js/Cargo.toml
index 0099d90c..0d6cd243 100644
--- a/js/Cargo.toml
+++ b/js/Cargo.toml
@@ -18,6 +18,3 @@ oxigraph = { version = "0.3.10", path="../lib" }
wasm-bindgen = "0.2"
js-sys = "0.3"
console_error_panic_hook = "0.1"
-
-[dev-dependencies]
-wasm-bindgen-test = "0.3"
diff --git a/lib/Cargo.toml b/lib/Cargo.toml
index f0dcc624..0c75bfc3 100644
--- a/lib/Cargo.toml
+++ b/lib/Cargo.toml
@@ -42,22 +42,19 @@ oxsdatatypes = { version = "0.1.0", path="oxsdatatypes" }
spargebra = { version = "0.2.3", path="spargebra", features = ["rdf-star", "sep-0002", "sep-0006"] }
sparesults = { version = "0.1.3", path="sparesults", features = ["rdf-star"] }
-[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+[target.'cfg(not(target_family = "wasm"))'.dependencies]
libc = "0.2"
oxrocksdb-sys = { version = "0.3.10", path="../oxrocksdb-sys" }
oxhttp = { version = "0.1", optional = true }
-[target.'cfg(target_arch = "wasm32")'.dependencies]
-getrandom = {version="0.2", features=["js"]}
+[target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies]
+getrandom = { version = "0.2", features = ["js"] }
-[dev-dependencies]
+[target.'cfg(not(target_family = "wasm"))'.dev-dependencies]
criterion = "0.4"
oxhttp = "0.1"
zstd = "0.12"
-[target.'cfg(target_arch = "wasm32")'.dev-dependencies]
-wasm-bindgen-test = "0.3"
-
[[bench]]
name = "store"
harness = false
diff --git a/lib/oxsdatatypes/Cargo.toml b/lib/oxsdatatypes/Cargo.toml
index 1b6e0053..70312fe6 100644
--- a/lib/oxsdatatypes/Cargo.toml
+++ b/lib/oxsdatatypes/Cargo.toml
@@ -16,7 +16,7 @@ rust-version = "1.60"
[dependencies]
nom = "7"
-[target.'cfg(target_arch = "wasm32")'.dependencies]
+[target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies]
js-sys = "0.3"
[package.metadata.docs.rs]
diff --git a/lib/oxsdatatypes/src/date_time.rs b/lib/oxsdatatypes/src/date_time.rs
index e7f37815..44e7e23b 100644
--- a/lib/oxsdatatypes/src/date_time.rs
+++ b/lib/oxsdatatypes/src/date_time.rs
@@ -1602,7 +1602,7 @@ impl Timestamp {
}
}
-#[cfg(target_arch = "wasm32")]
+#[cfg(all(target_family = "wasm", target_os = "unknown"))]
fn since_unix_epoch() -> Result<Duration, DateTimeError> {
Ok(Duration::new(
0,
@@ -1611,7 +1611,7 @@ fn since_unix_epoch() -> Result<Duration, DateTimeError> {
))
}
-#[cfg(not(target_arch = "wasm32"))]
+#[cfg(not(all(target_family = "wasm", target_os = "unknown")))]
fn since_unix_epoch() -> Result<Duration, DateTimeError> {
use std::time::SystemTime;
diff --git a/lib/src/sparql/http/dummy.rs b/lib/src/sparql/http/dummy.rs
index 951778ae..0517456b 100644
--- a/lib/src/sparql/http/dummy.rs
+++ b/lib/src/sparql/http/dummy.rs
@@ -10,6 +10,7 @@ impl Client {
Self {}
}
+ #[allow(clippy::unused_self)]
pub fn get(&self, _url: &str, _accept: &str) -> Result<(String, Empty)> {
Err(Error::new(
ErrorKind::Unsupported,
@@ -17,6 +18,7 @@ impl Client {
))
}
+ #[allow(clippy::unused_self, clippy::needless_pass_by_value)]
pub fn post(
&self,
_url: &str,
diff --git a/lib/src/storage/backend/fallback.rs b/lib/src/storage/backend/fallback.rs
index 4d8df9ad..4c4a1ded 100644
--- a/lib/src/storage/backend/fallback.rs
+++ b/lib/src/storage/backend/fallback.rs
@@ -19,6 +19,7 @@ pub struct ColumnFamilyDefinition {
pub struct Db(Arc<RwLock<HashMap<ColumnFamily, BTreeMap<Vec<u8>, Vec<u8>>>>>);
impl Db {
+ #[allow(clippy::unnecessary_wraps)]
pub fn new(column_families: Vec<ColumnFamilyDefinition>) -> Result<Self, StorageError> {
let mut trees = HashMap::new();
for cf in column_families {
@@ -225,7 +226,7 @@ pub struct Transaction<'a>(
);
impl Transaction<'_> {
- #[allow(unsafe_code)]
+ #[allow(unsafe_code, clippy::useless_transmute)]
pub fn reader(&self) -> Reader {
// This transmute is safe because we take a weak reference and the only Rc reference used is guarded by the lifetime.
Reader(InnerReader::Transaction(Rc::downgrade(unsafe {
@@ -233,6 +234,7 @@ impl Transaction<'_> {
})))
}
+ #[allow(clippy::unnecessary_wraps)]
pub fn contains_key_for_update(
&self,
column_family: &ColumnFamily,
@@ -244,6 +246,7 @@ impl Transaction<'_> {
.map_or(false, |cf| cf.contains_key(key)))
}
+ #[allow(clippy::unnecessary_wraps)]
pub fn insert(
&mut self,
column_family: &ColumnFamily,
@@ -266,6 +269,7 @@ impl Transaction<'_> {
self.insert(column_family, key, &[])
}
+ #[allow(clippy::unnecessary_wraps)]
pub fn remove(&mut self, column_family: &ColumnFamily, key: &[u8]) -> Result<(), StorageError> {
self.0
.borrow_mut()
@@ -286,6 +290,7 @@ impl Iter {
Some(&self.current.as_ref()?.0)
}
+ #[allow(dead_code)]
pub fn value(&self) -> Option<&[u8]> {
Some(&self.current.as_ref()?.1)
}
@@ -294,6 +299,7 @@ impl Iter {
self.current = self.iter.next();
}
+ #[allow(clippy::unnecessary_wraps, clippy::unused_self)]
pub fn status(&self) -> Result<(), StorageError> {
Ok(())
}
diff --git a/lib/src/storage/backend/mod.rs b/lib/src/storage/backend/mod.rs
index ed6d71ab..0fea9616 100644
--- a/lib/src/storage/backend/mod.rs
+++ b/lib/src/storage/backend/mod.rs
@@ -1,14 +1,14 @@
//! A storage backend
//! RocksDB is available, if not in memory
-#[cfg(target_arch = "wasm32")]
+#[cfg(target_family = "wasm")]
pub use fallback::{ColumnFamily, ColumnFamilyDefinition, Db, Iter, Reader, Transaction};
-#[cfg(not(target_arch = "wasm32"))]
+#[cfg(not(target_family = "wasm"))]
pub use rocksdb::{
ColumnFamily, ColumnFamilyDefinition, Db, Iter, Reader, SstFileWriter, Transaction,
};
-#[cfg(target_arch = "wasm32")]
+#[cfg(target_family = "wasm")]
mod fallback;
-#[cfg(not(target_arch = "wasm32"))]
+#[cfg(not(target_family = "wasm"))]
mod rocksdb;
diff --git a/lib/src/storage/binary_encoder.rs b/lib/src/storage/binary_encoder.rs
index a8d3baee..c5a4177b 100644
--- a/lib/src/storage/binary_encoder.rs
+++ b/lib/src/storage/binary_encoder.rs
@@ -6,7 +6,7 @@ use oxsdatatypes::*;
use std::io::{Cursor, Read};
use std::mem::size_of;
-#[cfg(not(target_arch = "wasm32"))]
+#[cfg(not(target_family = "wasm"))]
pub const LATEST_STORAGE_VERSION: u64 = 1;
pub const WRITTEN_TERM_MAX_SIZE: usize = size_of::<u8>() + 2 * size_of::<StrHash>();
diff --git a/lib/src/storage/mod.rs b/lib/src/storage/mod.rs
index 8685d880..b3bf01a9 100644
--- a/lib/src/storage/mod.rs
+++ b/lib/src/storage/mod.rs
@@ -1,6 +1,8 @@
-use crate::model::{GraphNameRef, NamedOrBlankNodeRef, Quad, QuadRef, TermRef};
+#[cfg(not(target_family = "wasm"))]
+use crate::model::Quad;
+use crate::model::{GraphNameRef, NamedOrBlankNodeRef, QuadRef, TermRef};
use crate::storage::backend::{Reader, Transaction};
-#[cfg(not(target_arch = "wasm32"))]
+#[cfg(not(target_family = "wasm"))]
use crate::storage::binary_encoder::LATEST_STORAGE_VERSION;
use crate::storage::binary_encoder::{
decode_term, encode_term, encode_term_pair, encode_term_quad, encode_term_triple,
@@ -9,24 +11,30 @@ use crate::storage::binary_encoder::{
WRITTEN_TERM_MAX_SIZE,
};
pub use crate::storage::error::{CorruptionError, LoaderError, SerializerError, StorageError};
-use crate::storage::numeric_encoder::{
- insert_term, Decoder, EncodedQuad, EncodedTerm, StrHash, StrLookup,
-};
+#[cfg(not(target_family = "wasm"))]
+use crate::storage::numeric_encoder::Decoder;
+use crate::storage::numeric_encoder::{insert_term, EncodedQuad, EncodedTerm, StrHash, StrLookup};
use backend::{ColumnFamily, ColumnFamilyDefinition, Db, Iter};
+#[cfg(not(target_family = "wasm"))]
use std::cmp::{max, min};
+#[cfg(not(target_family = "wasm"))]
use std::collections::VecDeque;
-#[cfg(not(target_arch = "wasm32"))]
+#[cfg(not(target_family = "wasm"))]
use std::collections::{HashMap, HashSet};
use std::error::Error;
-#[cfg(not(target_arch = "wasm32"))]
+#[cfg(not(target_family = "wasm"))]
use std::mem::take;
-#[cfg(not(target_arch = "wasm32"))]
+#[cfg(not(target_family = "wasm"))]
use std::path::{Path, PathBuf};
+#[cfg(not(target_family = "wasm"))]
use std::sync::atomic::{AtomicU64, Ordering};
+#[cfg(not(target_family = "wasm"))]
use std::sync::Arc;
-#[cfg(not(target_arch = "wasm32"))]
+#[cfg(not(target_family = "wasm"))]
use std::thread::spawn;
+#[cfg(not(target_family = "wasm"))]
use std::thread::JoinHandle;
+#[cfg(not(target_family = "wasm"))]
use sysinfo::{System, SystemExt};
mod backend;
@@ -46,15 +54,18 @@ const DSPO_CF: &str = "dspo";
const DPOS_CF: &str = "dpos";
const DOSP_CF: &str = "dosp";
const GRAPHS_CF: &str = "graphs";
+#[cfg(not(target_family = "wasm"))]
const DEFAULT_CF: &str = "default";
-#[cfg(not(target_arch = "wasm32"))]
+#[cfg(not(target_family = "wasm"))]
const DEFAULT_BULK_LOAD_BATCH_SIZE: usize = 1_000_000;
+#[cfg(not(target_family = "wasm"))]
const MAX_BULK_LOAD_BATCH_SIZE: usize = 100_000_000;
/// Low level storage primitives
#[derive(Clone)]
pub struct Storage {
db: Db,
+ #[cfg(not(target_family = "wasm"))]
default_cf: ColumnFamily,
id2str_cf: ColumnFamily,
spog_cf: ColumnFamily,
@@ -74,7 +85,7 @@ impl Storage {
Self::setup(Db::new(Self::column_families())?)
}
- #[cfg(not(target_arch = "wasm32"))]
+ #[cfg(not(target_family = "wasm"))]
pub fn open(path: &Path) -> Result<Self, StorageError> {
Self::setup(Db::open(path, Self::column_families())?)
}
@@ -150,8 +161,10 @@ impl Storage {
]
}
+ #[allow(clippy::unnecessary_wraps)]
fn setup(db: Db) -> Result<Self, StorageError> {
let this = Self {
+ #[cfg(not(target_family = "wasm"))]
default_cf: db.column_family(DEFAULT_CF).unwrap(),
id2str_cf: db.column_family(ID2STR_CF).unwrap(),
spog_cf: db.column_family(SPOG_CF).unwrap(),
@@ -166,12 +179,12 @@ impl Storage {
graphs_cf: db.column_family(GRAPHS_CF).unwrap(),
db,
};
- #[cfg(not(target_arch = "wasm32"))]
+ #[cfg(not(target_family = "wasm"))]
this.migrate()?;
Ok(this)
}
- #[cfg(not(target_arch = "wasm32"))]
+ #[cfg(not(target_family = "wasm"))]
fn migrate(&self) -> Result<(), StorageError> {
let mut version = self.ensure_version()?;
if version == 0 {
@@ -211,7 +224,7 @@ impl Storage {
}
}
- #[cfg(not(target_arch = "wasm32"))]
+ #[cfg(not(target_family = "wasm"))]
fn ensure_version(&self) -> Result<u64, StorageError> {
Ok(
if let Some(version) = self.db.get(&self.default_cf, b"oxversion")? {
@@ -225,7 +238,7 @@ impl Storage {
)
}
- #[cfg(not(target_arch = "wasm32"))]
+ #[cfg(not(target_family = "wasm"))]
fn update_version(&self, version: u64) -> Result<(), StorageError> {
self.db
.insert(&self.default_cf, b"oxversion", &version.to_be_bytes())?;
@@ -252,7 +265,7 @@ impl Storage {
})
}
- #[cfg(not(target_arch = "wasm32"))]
+ #[cfg(not(target_family = "wasm"))]
pub fn flush(&self) -> Result<(), StorageError> {
self.db.flush(&self.default_cf)?;
self.db.flush(&self.gpos_cf)?;
@@ -267,7 +280,7 @@ impl Storage {
self.db.flush(&self.id2str_cf)
}
- #[cfg(not(target_arch = "wasm32"))]
+ #[cfg(not(target_family = "wasm"))]
pub fn compact(&self) -> Result<(), StorageError> {
self.db.compact(&self.default_cf)?;
self.db.compact(&self.gpos_cf)?;
@@ -282,7 +295,7 @@ impl Storage {
self.db.compact(&self.id2str_cf)
}
- #[cfg(not(target_arch = "wasm32"))]
+ #[cfg(not(target_family = "wasm"))]
pub fn backup(&self, target_directory: &Path) -> Result<(), StorageError> {
self.db.backup(target_directory)
}
@@ -607,7 +620,7 @@ impl StorageReader {
}
}
- #[cfg(not(target_arch = "wasm32"))]
+ #[cfg(not(target_family = "wasm"))]
pub fn get_str(&self, key: &StrHash) -> Result<Option<String>, StorageError> {
Ok(self
.storage
@@ -618,31 +631,31 @@ impl StorageReader {
.map_err(CorruptionError::new)?)
}
- #[cfg(target_arch = "wasm32")]
+ #[cfg(target_family = "wasm")]
pub fn get_str(&self, key: &StrHash) -> Result<Option<String>, StorageError> {
Ok(self
.reader
.get(&self.storage.id2str_cf, &key.to_be_bytes())?
- .map(|v| String::from_utf8(v.into()))
+ .map(String::from_utf8)
.transpose()
.map_err(CorruptionError::new)?)
}
- #[cfg(not(target_arch = "wasm32"))]
+ #[cfg(not(target_family = "wasm"))]
pub fn contains_str(&self, key: &StrHash) -> Result<bool, StorageError> {
self.storage
.db
.contains_key(&self.storage.id2str_cf, &key.to_be_bytes())
}
- #[cfg(target_arch = "wasm32")]
+ #[cfg(target_family = "wasm")]
pub fn contains_str(&self, key: &StrHash) -> Result<bool, StorageError> {
self.reader
.contains_key(&self.storage.id2str_cf, &key.to_be_bytes())
}
/// Validates that all the storage invariants held in the data
- #[cfg(not(target_arch = "wasm32"))]
+ #[cfg(not(target_family = "wasm"))]
pub fn validate(&self) -> Result<(), StorageError> {
// triples
let dspo_size = self.dspo_quads(&[]).count();
@@ -752,6 +765,13 @@ impl StorageReader {
}
Ok(())
}
+
+ /// Validates that all the storage invariants held in the data
+ #[cfg(target_family = "wasm")]
+ #[allow(clippy::unused_self, clippy::unnecessary_wraps)]
+ pub fn validate(&self) -> Result<(), StorageError> {
+ Ok(()) //TODO
+ }
}
pub struct ChainedDecodingQuadIterator {
@@ -975,7 +995,7 @@ impl<'a> StorageWriter<'a> {
}
}
- #[cfg(not(target_arch = "wasm32"))]
+ #[cfg(not(target_family = "wasm"))]
fn insert_str(&mut self, key: &StrHash, value: &str) -> Result<(), StorageError> {
if self
.storage
@@ -991,7 +1011,7 @@ impl<'a> StorageWriter<'a> {
)
}
- #[cfg(target_arch = "wasm32")]
+ #[cfg(target_family = "wasm")]
fn insert_str(&mut self, key: &StrHash, value: &str) -> Result<(), StorageError> {
self.transaction.insert(
&self.storage.id2str_cf,
@@ -1156,7 +1176,7 @@ impl<'a> StorageWriter<'a> {
}
}
-#[cfg(not(target_arch = "wasm32"))]
+#[cfg(not(target_family = "wasm"))]
pub struct StorageBulkLoader {
storage: Storage,
hooks: Vec<Box<dyn Fn(u64)>>,
@@ -1164,7 +1184,7 @@ pub struct StorageBulkLoader {
max_memory_size: Option<usize>,
}
-#[cfg(not(target_arch = "wasm32"))]
+#[cfg(not(target_family = "wasm"))]
impl StorageBulkLoader {
pub fn new(storage: Storage) -> Self {
Self {
@@ -1290,7 +1310,7 @@ impl StorageBulkLoader {
}
}
-#[cfg(not(target_arch = "wasm32"))]
+#[cfg(not(target_family = "wasm"))]
struct FileBulkLoader {
storage: Storage,
id2str: HashMap<StrHash, Box<str>>,
@@ -1299,7 +1319,7 @@ struct FileBulkLoader {
graphs: HashSet<EncodedTerm>,
}
-#[cfg(not(target_arch = "wasm32"))]
+#[cfg(not(target_family = "wasm"))]
impl FileBulkLoader {
fn new(storage: Storage) -> Self {
Self {
diff --git a/lib/src/store.rs b/lib/src/store.rs
index 54a451c5..7e6974f4 100644
--- a/lib/src/store.rs
+++ b/lib/src/store.rs
@@ -33,7 +33,7 @@ use crate::sparql::{
UpdateOptions,
};
use crate::storage::numeric_encoder::{Decoder, EncodedQuad, EncodedTerm};
-#[cfg(not(target_arch = "wasm32"))]
+#[cfg(not(target_family = "wasm"))]
use crate::storage::StorageBulkLoader;
use crate::storage::{
ChainedDecodingQuadIterator, DecodingGraphIterator, Storage, StorageReader, StorageWriter,
@@ -41,7 +41,7 @@ use crate::storage::{
pub use crate::storage::{CorruptionError, LoaderError, SerializerError, StorageError};
use std::error::Error;
use std::io::{BufRead, Write};
-#[cfg(not(target_arch = "wasm32"))]
+#[cfg(not(target_family = "wasm"))]
use std::path::Path;
use std::{fmt, str};
@@ -95,7 +95,7 @@ impl Store {
}
/// Opens a [`Store`] and creates it if it does not exist yet.
- #[cfg(not(target_arch = "wasm32"))]
+ #[cfg(not(target_family = "wasm"))]
pub fn open(path: impl AsRef<Path>) -> Result<Self, StorageError> {
Ok(Self {
storage: Storage::open(path.as_ref())?,
@@ -717,7 +717,7 @@ impl Store {
/// Flushes all buffers and ensures that all writes are saved on disk.
///
/// Flushes are automatically done using background threads but might lag a little bit.
- #[cfg(not(target_arch = "wasm32"))]
+ #[cfg(not(target_family = "wasm"))]
pub fn flush(&self) -> Result<(), StorageError> {
self.storage.flush()
}
@@ -727,7 +727,7 @@ impl Store {
/// Useful to call after a batch upload or another similar operation.
///
/// Warning: Can take hours on huge databases.
- #[cfg(not(target_arch = "wasm32"))]
+ #[cfg(not(target_family = "wasm"))]
pub fn optimize(&self) -> Result<(), StorageError> {
self.storage.compact()
}
@@ -748,7 +748,7 @@ impl Store {
/// This allows cheap regular backups.
///
/// If you want to move your data to another RDF storage system, you should have a look at the [`Store::dump_dataset`] function instead.
- #[cfg(not(target_arch = "wasm32"))]
+ #[cfg(not(target_family = "wasm"))]
pub fn backup(&self, target_directory: impl AsRef<Path>) -> Result<(), StorageError> {
self.storage.backup(target_directory.as_ref())
}
@@ -772,7 +772,7 @@ impl Store {
/// assert!(store.contains(QuadRef::new(ex, ex, ex, ex))?);
/// # Result::<_, Box<dyn std::error::Error>>::Ok(())
/// ```
- #[cfg(not(target_arch = "wasm32"))]
+ #[cfg(not(target_family = "wasm"))]
pub fn bulk_loader(&self) -> BulkLoader {
BulkLoader {
storage: StorageBulkLoader::new(self.storage.clone()),
@@ -782,7 +782,6 @@ impl Store {
/// Validates that all the store invariants held in the data
#[doc(hidden)]
- #[cfg(not(target_arch = "wasm32"))]
pub fn validate(&self) -> Result<(), StorageError> {
self.storage.snapshot().validate()
}
@@ -1296,13 +1295,13 @@ impl Iterator for GraphNameIter {
/// assert!(store.contains(QuadRef::new(ex, ex, ex, ex))?);
/// # Result::<_, Box<dyn std::error::Error>>::Ok(())
/// ```
-#[cfg(not(target_arch = "wasm32"))]
+#[cfg(not(target_family = "wasm"))]
pub struct BulkLoader {
storage: StorageBulkLoader,
on_parse_error: Option<Box<dyn Fn(ParseError) -> Result<(), ParseError>>>,
}
-#[cfg(not(target_arch = "wasm32"))]
+#[cfg(not(target_family = "wasm"))]
impl BulkLoader {
/// Sets the maximal number of threads to be used by the bulk loader per operation.
///
diff --git a/lib/tests/store.rs b/lib/tests/store.rs
index 966d1568..7ee2dfd6 100644
--- a/lib/tests/store.rs
+++ b/lib/tests/store.rs
@@ -2,13 +2,21 @@ use oxigraph::io::{DatasetFormat, GraphFormat};
use oxigraph::model::vocab::{rdf, xsd};
use oxigraph::model::*;
use oxigraph::store::Store;
+#[cfg(not(target_family = "wasm"))]
use rand::random;
+#[cfg(not(target_family = "wasm"))]
use std::env::temp_dir;
use std::error::Error;
+#[cfg(not(target_family = "wasm"))]
use std::fs::{create_dir, remove_dir_all, File};
-use std::io::{Cursor, Write};
+use std::io::Cursor;
+#[cfg(not(target_family = "wasm"))]
+use std::io::Write;
+#[cfg(not(target_family = "wasm"))]
use std::iter::once;
+#[cfg(not(target_family = "wasm"))]
use std::path::{Path, PathBuf};
+#[cfg(not(target_family = "wasm"))]
use std::process::Command;
const DATA: &str = r#"
@@ -111,6 +119,7 @@ fn test_load_graph() -> Result<(), Box<dyn Error>> {
}
#[test]
+#[cfg(not(target_family = "wasm"))]
fn test_bulk_load_graph() -> Result<(), Box<dyn Error>> {
let store = Store::new()?;
store.bulk_loader().load_graph(
@@ -127,6 +136,7 @@ fn test_bulk_load_graph() -> Result<(), Box<dyn Error>> {
}
#[test]
+#[cfg(not(target_family = "wasm"))]
fn test_bulk_load_graph_lenient() -> Result<(), Box<dyn Error>> {
let store = Store::new()?;
store.bulk_loader().on_parse_error(|_| Ok(())).load_graph(
@@ -160,6 +170,7 @@ fn test_load_dataset() -> Result<(), Box<dyn Error>> {
}
#[test]
+#[cfg(not(target_family = "wasm"))]
fn test_bulk_load_dataset() -> Result<(), Box<dyn Error>> {
let store = Store::new().unwrap();
store
@@ -238,15 +249,16 @@ fn test_snapshot_isolation_iterator() -> Result<(), Box<dyn Error>> {
store.insert(quad)?;
let iter = store.iter();
store.remove(quad)?;
- store.validate()?;
assert_eq!(
iter.collect::<Result<Vec<_>, _>>()?,
vec![quad.into_owned()]
);
+ store.validate()?;
Ok(())
}
#[test]
+#[cfg(not(target_family = "wasm"))]
fn test_bulk_load_on_existing_delete_overrides_the_delete() -> Result<(), Box<dyn Error>> {
let quad = QuadRef::new(
NamedNodeRef::new_unchecked("http://example.com/s"),
@@ -262,6 +274,7 @@ fn test_bulk_load_on_existing_delete_overrides_the_delete() -> Result<(), Box<dy
}
#[test]
+#[cfg(not(target_family = "wasm"))]
fn test_open_bad_dir() -> Result<(), Box<dyn Error>> {
let dir = TempDir::default();
create_dir(&dir.0)?;
@@ -291,6 +304,7 @@ fn test_bad_stt_open() -> Result<(), Box<dyn Error>> {
}
#[test]
+#[cfg(not(target_family = "wasm"))]
fn test_backup() -> Result<(), Box<dyn Error>> {
let quad = QuadRef {
subject: NamedNodeRef::new_unchecked("http://example.com/s").into(),
@@ -314,6 +328,7 @@ fn test_backup() -> Result<(), Box<dyn Error>> {
}
#[test]
+#[cfg(not(target_family = "wasm"))]
fn test_bad_backup() -> Result<(), Box<dyn Error>> {
let store_dir = TempDir::default();
let backup_dir = TempDir::default();
@@ -324,6 +339,7 @@ fn test_bad_backup() -> Result<(), Box<dyn Error>> {
}
#[test]
+#[cfg(not(target_family = "wasm"))]
fn test_backup_on_in_memory() -> Result<(), Box<dyn Error>> {
let backup_dir = TempDir::default();
assert!(Store::new()?.backup(&backup_dir).is_err());
@@ -354,6 +370,7 @@ fn test_backward_compatibility() -> Result<(), Box<dyn Error>> {
Ok(())
}
+#[cfg(not(target_family = "wasm"))]
fn reset_dir(dir: &str) -> Result<(), Box<dyn Error>> {
assert!(Command::new("git")
.args(["clean", "-fX", dir])
@@ -366,20 +383,24 @@ fn reset_dir(dir: &str) -> Result<(), Box<dyn Error>> {
Ok(())
}
+#[cfg(not(target_family = "wasm"))]
struct TempDir(PathBuf);
+#[cfg(not(target_family = "wasm"))]
impl Default for TempDir {
fn default() -> Self {
Self(temp_dir().join(format!("oxigraph-test-{}", random::<u128>())))
}
}
+#[cfg(not(target_family = "wasm"))]
impl AsRef<Path> for TempDir {
fn as_ref(&self) -> &Path {
&self.0
}
}
+#[cfg(not(target_family = "wasm"))]
impl Drop for TempDir {
fn drop(&mut self) {
let _ = remove_dir_all(&self.0);