diff --git a/.github/workflows/install_rocksdb.sh b/.github/workflows/install_rocksdb.sh new file mode 100644 index 00000000..ac2b712b --- /dev/null +++ b/.github/workflows/install_rocksdb.sh @@ -0,0 +1,11 @@ +if [ -f "rocksdb" ] +then + cd rocksdb || exit +else + git clone https://github.com/facebook/rocksdb.git + cd rocksdb || exit + git checkout v8.0.0 + make shared_lib +fi +sudo make install-shared +sudo ldconfig /usr/local/lib diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f811899f..d570e796 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -130,7 +130,7 @@ jobs: - run: rm Cargo.lock && cargo +nightly update -Z direct-minimal-versions - run: cargo test - address_sanitizer: + test_linux_address_sanitizer: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -143,6 +143,21 @@ jobs: env: RUSTFLAGS: -Z sanitizer=address + test_linux_dynamic_linking: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - run: rustup update + - uses: Swatinem/rust-cache@v2 + - uses: actions/cache@v3 + with: + path: rocksdb + key: ${{ runner.os }}-rocksdb-8.0.0 + - run: bash .github/workflows/install_rocksdb.sh + - run: cargo test --tests --features oxrocksdb-sys/pkg-config + test_windows: runs-on: windows-latest steps: diff --git a/Cargo.lock b/Cargo.lock index e9b6b056..a958c167 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1078,6 +1078,7 @@ dependencies = [ "bindgen", "cc", "libc", + "pkg-config", ] [[package]] diff --git a/oxrocksdb-sys/Cargo.toml b/oxrocksdb-sys/Cargo.toml index 70e03ff3..849a8ab7 100644 --- a/oxrocksdb-sys/Cargo.toml +++ b/oxrocksdb-sys/Cargo.toml @@ -14,9 +14,13 @@ rust-version.workspace = true build = "build.rs" links = "rocksdb" +[features] +pkg-config = ["dep:pkg-config"] + [dependencies] libc = "0.2.147" [build-dependencies] +pkg-config = { version = "0.3.25", optional = true } bindgen = ">=0.60, <0.69" cc = { version = "1.0.73", features = ["parallel"] } diff --git a/oxrocksdb-sys/README.md b/oxrocksdb-sys/README.md index f4587db6..15b9cdbd 100644 --- a/oxrocksdb-sys/README.md +++ b/oxrocksdb-sys/README.md @@ -3,4 +3,9 @@ Oxigraph RocksDB bindings [RocksDB](http://rocksdb.org/) bindings for [Oxigraph](https://oxigraph.org). +By default it builds RocksDB as part of this crate. +It is also possible to dynamically link to RocksDB using the disabled by default `pkg-config` feature. +In this case [pkg-config](https://crates.io/crates/pkg-config) will be used to link to RocksDB. +Refer to this crate documentation if you want to configure the library lookup. + Based on [librocksdb-sys](https://crates.io/crates/librocksdb-sys) under Apache v2 license. diff --git a/oxrocksdb-sys/api/c.cc b/oxrocksdb-sys/api/c.cc index 49c5e55a..9a948ea2 100644 --- a/oxrocksdb-sys/api/c.cc +++ b/oxrocksdb-sys/api/c.cc @@ -1,7 +1,98 @@ -#include "../rocksdb/db/c.cc" - #include "c.h" +#include +#include +#include + +#include + +using ROCKSDB_NAMESPACE::Checkpoint; +using ROCKSDB_NAMESPACE::ColumnFamilyDescriptor; +using ROCKSDB_NAMESPACE::ColumnFamilyHandle; +using ROCKSDB_NAMESPACE::ColumnFamilyOptions; +using ROCKSDB_NAMESPACE::CompactRangeOptions; +using ROCKSDB_NAMESPACE::DB; +using ROCKSDB_NAMESPACE::DBOptions; +using ROCKSDB_NAMESPACE::FlushOptions; +using ROCKSDB_NAMESPACE::IngestExternalFileOptions; +using ROCKSDB_NAMESPACE::Iterator; +using ROCKSDB_NAMESPACE::Options; +using ROCKSDB_NAMESPACE::PinnableSlice; +using ROCKSDB_NAMESPACE::ReadOptions; +using ROCKSDB_NAMESPACE::Slice; +using ROCKSDB_NAMESPACE::SstFileWriter; +using ROCKSDB_NAMESPACE::Status; +using ROCKSDB_NAMESPACE::Transaction; +using ROCKSDB_NAMESPACE::TransactionDB; +using ROCKSDB_NAMESPACE::TransactionDBOptions; +using ROCKSDB_NAMESPACE::WriteOptions; +using std::vector; + +// From RocksDB +extern "C" { +struct rocksdb_t { + DB* rep; +}; + +struct rocksdb_column_family_handle_t { + ColumnFamilyHandle* rep; +}; + +struct rocksdb_compactoptions_t { + CompactRangeOptions rep; + Slice full_history_ts_low; +}; + +struct rocksdb_flushoptions_t { + FlushOptions rep; +}; + +struct rocksdb_ingestexternalfileoptions_t { + IngestExternalFileOptions rep; +}; + +struct rocksdb_iterator_t { + Iterator* rep; +}; + +struct rocksdb_options_t { + Options rep; +}; + +struct rocksdb_pinnableslice_t { + PinnableSlice rep; +}; + +struct rocksdb_readoptions_t { + ReadOptions rep; + // stack variables to set pointers to in ReadOptions + Slice upper_bound; + Slice lower_bound; + Slice timestamp; + Slice iter_start_ts; +}; + +struct rocksdb_sstfilewriter_t { + SstFileWriter* rep; +}; + +struct rocksdb_transaction_t { + Transaction* rep; +}; + +struct rocksdb_transactiondb_t { + TransactionDB* rep; +}; + +struct rocksdb_transactiondb_options_t { + TransactionDBOptions rep; +}; + +struct rocksdb_writeoptions_t { + WriteOptions rep; +}; +} + static bool SaveStatus(rocksdb_status_t* target, const Status source) { target->code = static_cast(source.code()); target->subcode = static_cast(source.subcode()); @@ -42,7 +133,7 @@ rocksdb_t* rocksdb_open_for_read_only_column_families_with_status( const rocksdb_options_t* const* column_family_options, rocksdb_column_family_handle_t** column_family_handles, unsigned char error_if_wal_file_exists, rocksdb_status_t* statusptr) { - std::vector column_families; + vector column_families; for (int i = 0; i < num_column_families; i++) { column_families.emplace_back(ColumnFamilyDescriptor( std::string(column_family_names[i]), @@ -50,7 +141,7 @@ rocksdb_t* rocksdb_open_for_read_only_column_families_with_status( } DB* db; - std::vector handles; + vector handles; if (SaveStatus(statusptr, DB::OpenForReadOnly(DBOptions(db_options->rep), std::string(name), column_families, &handles, &db, @@ -81,14 +172,14 @@ rocksdb_t* rocksdb_open_as_secondary_column_families_with_status( const rocksdb_options_t* const* column_family_options, rocksdb_column_family_handle_t** column_family_handles, rocksdb_status_t* statusptr) { - std::vector column_families; + vector column_families; for (int i = 0; i != num_column_families; ++i) { column_families.emplace_back( std::string(column_family_names[i]), ColumnFamilyOptions(column_family_options[i]->rep)); } DB* db; - std::vector handles; + vector handles; if (SaveStatus(statusptr, DB::OpenAsSecondary( DBOptions(db_options->rep), std::string(name), std::string(secondary_path), column_families, @@ -127,7 +218,7 @@ rocksdb_transactiondb_t* rocksdb_transactiondb_open_column_families_with_status( const rocksdb_options_t* const* column_family_options, rocksdb_column_family_handle_t** column_family_handles, rocksdb_status_t* statusptr) { - std::vector column_families; + vector column_families; for (int i = 0; i < num_column_families; i++) { column_families.emplace_back(ColumnFamilyDescriptor( std::string(column_family_names[i]), @@ -135,7 +226,7 @@ rocksdb_transactiondb_t* rocksdb_transactiondb_open_column_families_with_status( } TransactionDB* txn_db; - std::vector handles; + vector handles; if (SaveStatus(statusptr, TransactionDB::Open(options->rep, txn_db_options->rep, std::string(name), column_families, @@ -205,10 +296,10 @@ void rocksdb_transactiondb_compact_range_cf_opt_with_status( void rocksdb_transactiondb_ingest_external_files_with_status( rocksdb_transactiondb_t* db, const rocksdb_ingestexternalfilearg_t* list, const size_t list_len, rocksdb_status_t* statusptr) { - std::vector args(list_len); + vector args(list_len); for (size_t i = 0; i < list_len; ++i) { args[i].column_family = list[i].column_family->rep; - std::vector files(list[i].external_files_len); + vector files(list[i].external_files_len); for (size_t j = 0; j < list[i].external_files_len; ++j) { files[j] = std::string(list[i].external_files[j]); } diff --git a/oxrocksdb-sys/api/c.h b/oxrocksdb-sys/api/c.h index c0b2887d..022e8f97 100644 --- a/oxrocksdb-sys/api/c.h +++ b/oxrocksdb-sys/api/c.h @@ -1,6 +1,6 @@ #pragma once -#include "../rocksdb/include/rocksdb/c.h" +#include #ifdef __cplusplus extern "C" { diff --git a/oxrocksdb-sys/build.rs b/oxrocksdb-sys/build.rs index ff8a633b..abae57b5 100644 --- a/oxrocksdb-sys/build.rs +++ b/oxrocksdb-sys/build.rs @@ -1,9 +1,14 @@ // Code from https://github.com/rust-rocksdb/rust-rocksdb/blob/eb2d302682418b361a80ad8f4dcf335ade60dcf5/librocksdb-sys/build.rs // License: https://github.com/rust-rocksdb/rust-rocksdb/blob/master/LICENSE -use std::env::{remove_var, set_var, var}; +use std::env::var; +#[cfg(not(feature = "pkg-config"))] +use std::env::{remove_var, set_var}; +#[cfg(not(feature = "pkg-config"))] +use std::path::Path; use std::path::PathBuf; +#[cfg(not(feature = "pkg-config"))] fn link(name: &str, bundled: bool) { let target = var("TARGET").unwrap(); let target: Vec<_> = target.split('-').collect(); @@ -16,8 +21,14 @@ fn link(name: &str, bundled: bool) { } } -fn bindgen_rocksdb() { - bindgen::Builder::default() +fn bindgen_rocksdb_api(includes: &[PathBuf]) { + println!("cargo:rerun-if-changed=api/"); + + let mut builder = bindgen::Builder::default(); + for include in includes { + builder = builder.clang_arg(format!("-I{}", include.display())); + } + builder .header("api/c.h") .ctypes_prefix("libc") .size_t_is_usize(true) @@ -30,6 +41,22 @@ fn bindgen_rocksdb() { .unwrap(); } +fn build_rocksdb_api(includes: &[PathBuf]) { + let target = var("TARGET").unwrap(); + let mut config = cc::Build::new(); + for include in includes { + config.include(include); + } + if target.contains("msvc") { + config.flag("-EHsc").flag("-std:c++17"); + } else { + config.flag("-std=c++17"); + } + config.cpp(true).file("api/c.cc").compile("oxrocksdb_api"); +} + +#[cfg(not(feature = "pkg-config"))] + fn build_rocksdb() { let target = var("TARGET").unwrap(); @@ -38,7 +65,6 @@ fn build_rocksdb() { .cpp(true) .include("rocksdb/include/") .include("rocksdb/") - .file("api/c.cc") .file("api/build_version.cc") .define("NDEBUG", Some("1")) .define("LZ4", Some("1")) @@ -174,15 +200,15 @@ fn build_rocksdb() { } for file in lib_sources { - if file == "db/c.cc" || file == "util/build_version.cc" { - continue; + if file != "util/build_version.cc" { + config.file(&format!("rocksdb/{file}")); } - config.file(&format!("rocksdb/{file}")); } config.compile("rocksdb"); } +#[cfg(not(feature = "pkg-config"))] fn build_lz4() { let mut config = cc::Build::new(); config @@ -196,9 +222,21 @@ fn build_lz4() { config.compile("lz4"); } +#[cfg(not(feature = "pkg-config"))] fn main() { - println!("cargo:rerun-if-changed=api/"); - bindgen_rocksdb(); + let includes = [Path::new("rocksdb/include").to_path_buf()]; build_lz4(); build_rocksdb(); + build_rocksdb_api(&includes); + bindgen_rocksdb_api(&includes); +} + +#[cfg(feature = "pkg-config")] +fn main() { + let library = pkg_config::Config::new() + .atleast_version("8.0.0") + .probe("rocksdb") + .unwrap(); + build_rocksdb_api(&library.include_paths); + bindgen_rocksdb_api(&library.include_paths); }