Allows dynamic RocksDB linking

.github/workflows/install_rocksdb.sh

@@ -0,0 +1,11 @@
+if [ -f "rocksdb" ]
+then
+  cd rocksdb || exit
+else
+  git clone https://github.com/facebook/rocksdb.git
+  cd rocksdb || exit
+  git checkout v8.0.0
+  make shared_lib
+fi
+sudo make install-shared
+sudo ldconfig /usr/local/lib

.github/workflows/tests.yml

@@ -130,7 +130,7 @@ jobs:
       - run: rm Cargo.lock && cargo +nightly update -Z direct-minimal-versions
       - run: cargo test
 
-  address_sanitizer:
+  test_linux_address_sanitizer:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -143,6 +143,21 @@ jobs:
         env:
           RUSTFLAGS: -Z sanitizer=address
 
+  test_linux_dynamic_linking:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+      - run: rustup update
+      - uses: Swatinem/rust-cache@v2
+      - uses: actions/cache@v3
+        with:
+          path: rocksdb
+          key: ${{ runner.os }}-rocksdb-8.0.0
+      - run: bash .github/workflows/install_rocksdb.sh
+      - run: cargo test --tests --features oxrocksdb-sys/pkg-config
+
   test_windows:
     runs-on: windows-latest
     steps:

Cargo.lock

@@ -1078,6 +1078,7 @@ dependencies = [
  "bindgen",
  "cc",
  "libc",
+ "pkg-config",
 ]
 
 [[package]]

oxrocksdb-sys/Cargo.toml

@@ -14,9 +14,13 @@ rust-version.workspace  = true
 build = "build.rs"
 links = "rocksdb"
 
+[features]
+pkg-config = ["dep:pkg-config"]
+
 [dependencies]
 libc = "0.2.147"
 
 [build-dependencies]
+pkg-config = { version = "0.3.25", optional = true }
 bindgen = ">=0.60, <0.69"
 cc = { version = "1.0.73", features = ["parallel"] }

oxrocksdb-sys/README.md

@@ -3,4 +3,9 @@ Oxigraph RocksDB bindings
 
 [RocksDB](http://rocksdb.org/) bindings for [Oxigraph](https://oxigraph.org).
 
+By default it builds RocksDB as part of this crate.
+It is also possible to dynamically link to RocksDB using the disabled by default `pkg-config` feature.
+In this case [pkg-config](https://crates.io/crates/pkg-config) will be used to link to RocksDB.
+Refer to this crate documentation if you want to configure the library lookup.
+
 Based on [librocksdb-sys](https://crates.io/crates/librocksdb-sys) under Apache v2 license.

oxrocksdb-sys/api/c.cc

@@ -1,7 +1,98 @@
-#include "../rocksdb/db/c.cc"
-
 #include "c.h"
 
+#include <rocksdb/db.h>
+#include <rocksdb/utilities/checkpoint.h>
+#include <rocksdb/utilities/transaction_db.h>
+
+#include <vector>
+
+using ROCKSDB_NAMESPACE::Checkpoint;
+using ROCKSDB_NAMESPACE::ColumnFamilyDescriptor;
+using ROCKSDB_NAMESPACE::ColumnFamilyHandle;
+using ROCKSDB_NAMESPACE::ColumnFamilyOptions;
+using ROCKSDB_NAMESPACE::CompactRangeOptions;
+using ROCKSDB_NAMESPACE::DB;
+using ROCKSDB_NAMESPACE::DBOptions;
+using ROCKSDB_NAMESPACE::FlushOptions;
+using ROCKSDB_NAMESPACE::IngestExternalFileOptions;
+using ROCKSDB_NAMESPACE::Iterator;
+using ROCKSDB_NAMESPACE::Options;
+using ROCKSDB_NAMESPACE::PinnableSlice;
+using ROCKSDB_NAMESPACE::ReadOptions;
+using ROCKSDB_NAMESPACE::Slice;
+using ROCKSDB_NAMESPACE::SstFileWriter;
+using ROCKSDB_NAMESPACE::Status;
+using ROCKSDB_NAMESPACE::Transaction;
+using ROCKSDB_NAMESPACE::TransactionDB;
+using ROCKSDB_NAMESPACE::TransactionDBOptions;
+using ROCKSDB_NAMESPACE::WriteOptions;
+using std::vector;
+
+// From RocksDB
+extern "C" {
+struct rocksdb_t {
+  DB* rep;
+};
+
+struct rocksdb_column_family_handle_t {
+  ColumnFamilyHandle* rep;
+};
+
+struct rocksdb_compactoptions_t {
+  CompactRangeOptions rep;
+  Slice full_history_ts_low;
+};
+
+struct rocksdb_flushoptions_t {
+  FlushOptions rep;
+};
+
+struct rocksdb_ingestexternalfileoptions_t {
+  IngestExternalFileOptions rep;
+};
+
+struct rocksdb_iterator_t {
+  Iterator* rep;
+};
+
+struct rocksdb_options_t {
+  Options rep;
+};
+
+struct rocksdb_pinnableslice_t {
+  PinnableSlice rep;
+};
+
+struct rocksdb_readoptions_t {
+  ReadOptions rep;
+  // stack variables to set pointers to in ReadOptions
+  Slice upper_bound;
+  Slice lower_bound;
+  Slice timestamp;
+  Slice iter_start_ts;
+};
+
+struct rocksdb_sstfilewriter_t {
+  SstFileWriter* rep;
+};
+
+struct rocksdb_transaction_t {
+  Transaction* rep;
+};
+
+struct rocksdb_transactiondb_t {
+  TransactionDB* rep;
+};
+
+struct rocksdb_transactiondb_options_t {
+  TransactionDBOptions rep;
+};
+
+struct rocksdb_writeoptions_t {
+  WriteOptions rep;
+};
+}
+
 static bool SaveStatus(rocksdb_status_t* target, const Status source) {
   target->code = static_cast<rocksdb_status_code_t>(source.code());
   target->subcode = static_cast<rocksdb_status_subcode_t>(source.subcode());
@@ -42,7 +133,7 @@ rocksdb_t* rocksdb_open_for_read_only_column_families_with_status(
     const rocksdb_options_t* const* column_family_options,
     rocksdb_column_family_handle_t** column_family_handles,
     unsigned char error_if_wal_file_exists, rocksdb_status_t* statusptr) {
-  std::vector<ColumnFamilyDescriptor> column_families;
+  vector<ColumnFamilyDescriptor> column_families;
   for (int i = 0; i < num_column_families; i++) {
     column_families.emplace_back(ColumnFamilyDescriptor(
         std::string(column_family_names[i]),
@@ -50,7 +141,7 @@ rocksdb_t* rocksdb_open_for_read_only_column_families_with_status(
   }
 
   DB* db;
-  std::vector<ColumnFamilyHandle*> handles;
+  vector<ColumnFamilyHandle*> handles;
   if (SaveStatus(statusptr, DB::OpenForReadOnly(DBOptions(db_options->rep),
                                                 std::string(name),
                                                 column_families, &handles, &db,
@@ -81,14 +172,14 @@ rocksdb_t* rocksdb_open_as_secondary_column_families_with_status(
     const rocksdb_options_t* const* column_family_options,
     rocksdb_column_family_handle_t** column_family_handles,
     rocksdb_status_t* statusptr) {
-  std::vector<ColumnFamilyDescriptor> column_families;
+  vector<ColumnFamilyDescriptor> column_families;
   for (int i = 0; i != num_column_families; ++i) {
     column_families.emplace_back(
         std::string(column_family_names[i]),
         ColumnFamilyOptions(column_family_options[i]->rep));
   }
   DB* db;
-  std::vector<ColumnFamilyHandle*> handles;
+  vector<ColumnFamilyHandle*> handles;
   if (SaveStatus(statusptr, DB::OpenAsSecondary(
                                 DBOptions(db_options->rep), std::string(name),
                                 std::string(secondary_path), column_families,
@@ -127,7 +218,7 @@ rocksdb_transactiondb_t* rocksdb_transactiondb_open_column_families_with_status(
     const rocksdb_options_t* const* column_family_options,
     rocksdb_column_family_handle_t** column_family_handles,
     rocksdb_status_t* statusptr) {
-  std::vector<ColumnFamilyDescriptor> column_families;
+  vector<ColumnFamilyDescriptor> column_families;
   for (int i = 0; i < num_column_families; i++) {
     column_families.emplace_back(ColumnFamilyDescriptor(
         std::string(column_family_names[i]),
@@ -135,7 +226,7 @@ rocksdb_transactiondb_t* rocksdb_transactiondb_open_column_families_with_status(
   }
 
   TransactionDB* txn_db;
-  std::vector<ColumnFamilyHandle*> handles;
+  vector<ColumnFamilyHandle*> handles;
   if (SaveStatus(statusptr,
                  TransactionDB::Open(options->rep, txn_db_options->rep,
                                      std::string(name), column_families,
@@ -205,10 +296,10 @@ void rocksdb_transactiondb_compact_range_cf_opt_with_status(
 void rocksdb_transactiondb_ingest_external_files_with_status(
     rocksdb_transactiondb_t* db, const rocksdb_ingestexternalfilearg_t* list,
     const size_t list_len, rocksdb_status_t* statusptr) {
-  std::vector<rocksdb::IngestExternalFileArg> args(list_len);
+  vector<rocksdb::IngestExternalFileArg> args(list_len);
   for (size_t i = 0; i < list_len; ++i) {
     args[i].column_family = list[i].column_family->rep;
-    std::vector<std::string> files(list[i].external_files_len);
+    vector<std::string> files(list[i].external_files_len);
     for (size_t j = 0; j < list[i].external_files_len; ++j) {
       files[j] = std::string(list[i].external_files[j]);
     }

oxrocksdb-sys/api/c.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "../rocksdb/include/rocksdb/c.h"
+#include <rocksdb/c.h>
 
 #ifdef __cplusplus
 extern "C" {

oxrocksdb-sys/build.rs

@@ -1,9 +1,14 @@
 // Code from https://github.com/rust-rocksdb/rust-rocksdb/blob/eb2d302682418b361a80ad8f4dcf335ade60dcf5/librocksdb-sys/build.rs
 // License: https://github.com/rust-rocksdb/rust-rocksdb/blob/master/LICENSE
 
-use std::env::{remove_var, set_var, var};
+use std::env::var;
+#[cfg(not(feature = "pkg-config"))]
+use std::env::{remove_var, set_var};
+#[cfg(not(feature = "pkg-config"))]
+use std::path::Path;
 use std::path::PathBuf;
 
+#[cfg(not(feature = "pkg-config"))]
 fn link(name: &str, bundled: bool) {
     let target = var("TARGET").unwrap();
     let target: Vec<_> = target.split('-').collect();
@@ -16,8 +21,14 @@ fn link(name: &str, bundled: bool) {
     }
 }
 
-fn bindgen_rocksdb() {
+fn bindgen_rocksdb_api(includes: &[PathBuf]) {
-    bindgen::Builder::default()
+    println!("cargo:rerun-if-changed=api/");
+
+    let mut builder = bindgen::Builder::default();
+    for include in includes {
+        builder = builder.clang_arg(format!("-I{}", include.display()));
+    }
+    builder
         .header("api/c.h")
         .ctypes_prefix("libc")
         .size_t_is_usize(true)
@@ -30,6 +41,22 @@ fn bindgen_rocksdb() {
         .unwrap();
 }
 
+fn build_rocksdb_api(includes: &[PathBuf]) {
+    let target = var("TARGET").unwrap();
+    let mut config = cc::Build::new();
+    for include in includes {
+        config.include(include);
+    }
+    if target.contains("msvc") {
+        config.flag("-EHsc").flag("-std:c++17");
+    } else {
+        config.flag("-std=c++17");
+    }
+    config.cpp(true).file("api/c.cc").compile("oxrocksdb_api");
+}
+
+#[cfg(not(feature = "pkg-config"))]
+
 fn build_rocksdb() {
     let target = var("TARGET").unwrap();
 
@@ -38,7 +65,6 @@ fn build_rocksdb() {
         .cpp(true)
         .include("rocksdb/include/")
         .include("rocksdb/")
-        .file("api/c.cc")
         .file("api/build_version.cc")
         .define("NDEBUG", Some("1"))
         .define("LZ4", Some("1"))
@@ -174,15 +200,15 @@ fn build_rocksdb() {
     }
 
     for file in lib_sources {
-        if file == "db/c.cc" || file == "util/build_version.cc" {
+        if file != "util/build_version.cc" {
-            continue;
-        }
             config.file(&format!("rocksdb/{file}"));
         }
+    }
 
     config.compile("rocksdb");
 }
 
+#[cfg(not(feature = "pkg-config"))]
 fn build_lz4() {
     let mut config = cc::Build::new();
     config
@@ -196,9 +222,21 @@ fn build_lz4() {
     config.compile("lz4");
 }
 
+#[cfg(not(feature = "pkg-config"))]
 fn main() {
-    println!("cargo:rerun-if-changed=api/");
+    let includes = [Path::new("rocksdb/include").to_path_buf()];
-    bindgen_rocksdb();
     build_lz4();
     build_rocksdb();
+    build_rocksdb_api(&includes);
+    bindgen_rocksdb_api(&includes);
+}
+
+#[cfg(feature = "pkg-config")]
+fn main() {
+    let library = pkg_config::Config::new()
+        .atleast_version("8.0.0")
+        .probe("rocksdb")
+        .unwrap();
+    build_rocksdb_api(&library.include_paths);
+    bindgen_rocksdb_api(&library.include_paths);
 }