From 391e8d7662301490ef8980ff6dd545ffbeb3ec87 Mon Sep 17 00:00:00 2001 From: Tpt Date: Wed, 3 Jan 2024 17:21:08 +0100 Subject: [PATCH] Fixes and ensures 32bits x86 support --- .github/workflows/artifacts.yml | 2 +- .github/workflows/tests.yml | 15 +++++++++++++- lib/src/storage/backend/rocksdb.rs | 7 ++++--- lib/src/storage/mod.rs | 32 ++++++++++++++++++++---------- oxrocksdb-sys/build.rs | 10 +++++----- 5 files changed, 45 insertions(+), 21 deletions(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 77cafa38..371ad3db 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -24,7 +24,7 @@ jobs: with: target: aarch64-unknown-linux-gnu - run: | - sudo apt update && sudo apt-get install -y g++-aarch64-linux-gnu + sudo apt-get install -y g++-aarch64-linux-gnu mkdir .cargo echo -e "[target.aarch64-unknown-linux-gnu]\nlinker = \"aarch64-linux-gnu-gcc\"" >> .cargo/config.toml - run: cargo build --release --no-default-features --features rustls-native diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 227a70cb..e7401054 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -118,7 +118,7 @@ jobs: - run: cargo install cargo-semver-checks || true - run: cargo semver-checks check-release --exclude oxrocksdb-sys --exclude oxigraph-js --exclude pyoxigraph --exclude oxigraph-testsuite --exclude oxigraph-cli - test_linux: + test_linux_x86_64: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -127,6 +127,19 @@ jobs: - uses: ./.github/actions/setup-rust - run: cargo test + test_linux_i686: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - uses: ./.github/actions/setup-rust + with: + target: i686-unknown-linux-gnu + - run: sudo apt-get install -y g++-multilib + - run: cargo test --target i686-unknown-linux-gnu --no-default-features --features http-client-rustls-native + working-directory: ./lib + test_linux_msv: runs-on: ubuntu-latest steps: diff --git a/lib/src/storage/backend/rocksdb.rs b/lib/src/storage/backend/rocksdb.rs index afa934e1..acac6585 100644 --- a/lib/src/storage/backend/rocksdb.rs +++ b/lib/src/storage/backend/rocksdb.rs @@ -13,6 +13,7 @@ use libc::{self, c_void, free}; use oxrocksdb_sys::*; use rand::random; use std::borrow::Borrow; +#[cfg(unix)] use std::cmp::min; use std::collections::HashMap; use std::env::temp_dir; @@ -1405,7 +1406,7 @@ fn path_to_cstring(path: &Path) -> Result { } #[cfg(unix)] -fn available_file_descriptors() -> io::Result> { +fn available_file_descriptors() -> io::Result> { let mut rlimit = libc::rlimit { rlim_cur: 0, rlim_max: 0, @@ -1418,12 +1419,12 @@ fn available_file_descriptors() -> io::Result> { } #[cfg(windows)] -fn available_file_descriptors() -> io::Result> { +fn available_file_descriptors() -> io::Result> { Ok(Some(512)) // https://docs.microsoft.com/en-us/cpp/c-runtime-library/file-handling } #[cfg(not(any(unix, windows)))] -fn available_file_descriptors() -> io::Result> { +fn available_file_descriptors() -> io::Result> { Ok(None) } diff --git a/lib/src/storage/mod.rs b/lib/src/storage/mod.rs index 01419239..f592aeb9 100644 --- a/lib/src/storage/mod.rs +++ b/lib/src/storage/mod.rs @@ -26,7 +26,7 @@ use std::mem::{swap, take}; #[cfg(not(target_family = "wasm"))] use std::path::{Path, PathBuf}; #[cfg(not(target_family = "wasm"))] -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Mutex; #[cfg(not(target_family = "wasm"))] use std::{io, thread}; @@ -1249,7 +1249,7 @@ impl StorageBulkLoader { ) .into()); } - let done_counter = AtomicU64::new(0); + let done_counter = Mutex::new(0); let mut done_and_displayed_counter = 0; thread::scope(|thread_scope| { let mut threads = VecDeque::with_capacity(num_threads - 1); @@ -1280,7 +1280,7 @@ impl StorageBulkLoader { )?; for thread in threads { map_thread_result(thread.join()).map_err(StorageError::Io)??; - self.on_possible_progress(&done_counter, &mut done_and_displayed_counter); + self.on_possible_progress(&done_counter, &mut done_and_displayed_counter)?; } Ok(()) }) @@ -1291,17 +1291,17 @@ impl StorageBulkLoader { buffer: &mut Vec, threads: &mut VecDeque>>, thread_scope: &'scope thread::Scope<'scope, '_>, - done_counter: &'scope AtomicU64, + done_counter: &'scope Mutex, done_and_displayed_counter: &mut u64, num_threads: usize, batch_size: usize, ) -> Result<(), StorageError> { - self.on_possible_progress(done_counter, done_and_displayed_counter); + self.on_possible_progress(done_counter, done_and_displayed_counter)?; // We avoid to have too many threads if threads.len() >= num_threads { if let Some(thread) = threads.pop_front() { map_thread_result(thread.join()).map_err(StorageError::Io)??; - self.on_possible_progress(done_counter, done_and_displayed_counter); + self.on_possible_progress(done_counter, done_and_displayed_counter)?; } } let mut buffer_to_load = Vec::with_capacity(batch_size); @@ -1313,15 +1313,22 @@ impl StorageBulkLoader { Ok(()) } - fn on_possible_progress(&self, done: &AtomicU64, done_and_displayed: &mut u64) { - let new_counter = done.load(Ordering::Relaxed); - let display_step = u64::try_from(DEFAULT_BULK_LOAD_BATCH_SIZE).unwrap(); + fn on_possible_progress( + &self, + done: &Mutex, + done_and_displayed: &mut u64, + ) -> Result<(), StorageError> { + let new_counter = *done + .lock() + .map_err(|_| io::Error::new(io::ErrorKind::Other, "Mutex poisoned"))?; + let display_step = DEFAULT_BULK_LOAD_BATCH_SIZE as u64; if new_counter / display_step > *done_and_displayed / display_step { for hook in &self.hooks { hook(new_counter); } } *done_and_displayed = new_counter; + Ok(()) } } @@ -1346,11 +1353,14 @@ impl<'a> FileBulkLoader<'a> { } } - fn load(&mut self, quads: Vec, counter: &AtomicU64) -> Result<(), StorageError> { + fn load(&mut self, quads: Vec, counter: &Mutex) -> Result<(), StorageError> { self.encode(quads)?; let size = self.triples.len() + self.quads.len(); self.save()?; - counter.fetch_add(size.try_into().unwrap_or(u64::MAX), Ordering::Relaxed); + *counter + .lock() + .map_err(|_| io::Error::new(io::ErrorKind::Other, "Mutex poisoned"))? += + size.try_into().unwrap_or(u64::MAX); Ok(()) } diff --git a/oxrocksdb-sys/build.rs b/oxrocksdb-sys/build.rs index abae57b5..8771573e 100644 --- a/oxrocksdb-sys/build.rs +++ b/oxrocksdb-sys/build.rs @@ -153,7 +153,7 @@ fn build_rocksdb() { config.define("NOMINMAX", None); config.define("ROCKSDB_WINDOWS_UTF8_FILENAMES", None); - if target == "x86_64-pc-windows-gnu" { + if target.contains("pc-windows-gnu") { // Tell MinGW to create localtime_r wrapper of localtime_s function. config.define("_POSIX_C_SOURCE", Some("1")); // Tell MinGW to use at least Windows Vista headers instead of the ones of Windows XP. @@ -193,10 +193,10 @@ fn build_rocksdb() { if target.contains("msvc") { config.flag("-EHsc").flag("-std:c++17"); } else { - config - .flag("-std=c++17") - .flag("-Wno-invalid-offsetof") - .define("HAVE_UINT128_EXTENSION", Some("1")); + config.flag("-std=c++17").flag("-Wno-invalid-offsetof"); + if target.contains("x86_64") || target.contains("aarch64") { + config.define("HAVE_UINT128_EXTENSION", Some("1")); + } } for file in lib_sources {