Use our own Custom rocksdb bindings

Will allow exposing more C++ functions

.gitmodules

@@ -6,4 +6,7 @@
 	url = https://github.com/w3c/rdf-star.git
 [submodule "bench/bsbm-tools"]
 	path = bench/bsbm-tools
-	url = https://github.com/Tpt/bsbm-tools.git
+	url = https://github.com/Tpt/bsbm-tools.git
+[submodule "rocksdb-sys/rocksdb"]
+	path = rocksdb-sys/rocksdb
+	url = https://github.com/facebook/rocksdb/

Cargo.toml

@@ -3,6 +3,7 @@ members = [
     "js",
     "lib",
     "python",
+    "rocksdb-sys",
     "server",
     "spargebra",
     "testsuite",

lib/Cargo.toml

@@ -44,7 +44,7 @@ spargebra = { version = "0.1", path="../spargebra", features = ["rdf-star"] }
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
 libc = "0.2"
-librocksdb-sys = { version = "6.20.3", default-features = false }
+oxrocksdb-sys = { version = "0.3.0-dev", path="../rocksdb-sys" }
 oxhttp = { version = "0.1", optional = true }
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]

lib/src/storage/rocksdb_backend.rs

@@ -4,7 +4,7 @@
 
 use crate::error::invalid_input_error;
 use libc::{self, c_char, c_void};
-use librocksdb_sys::*;
+use oxrocksdb_sys::*;
 use std::borrow::Borrow;
 use std::env::temp_dir;
 use std::ffi::{CStr, CString};

rocksdb-sys/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "oxrocksdb-sys"
+version = "0.3.0-dev"
+authors = ["Tpt <thomas@pellissier-tanon.fr>"]
+license = "GPL-2.0 OR Apache-2.0"
+repository = "https://github.com/oxigraph/oxigraph/tree/v0.3/rocksdb-sys"
+description = """
+Rust bindings for RocksDB for Oxigraph usage.
+"""
+edition = "2021"
+build = "build.rs"
+links = "rocksdb"
+
+[dependencies]
+libc = "0.2"
+
+[build-dependencies]
+bindgen = "0.59"
+cc = { version = "1", features = ["parallel"] }

rocksdb-sys/api/c.cc

@@ -0,0 +1,9 @@
+#include "../rocksdb/db/c.cc"
+#include "c.h"
+
+void rocksdb_transactiondb_flush(
+        rocksdb_t* db,
+        const rocksdb_flushoptions_t* options,
+        char** errptr) {
+    SaveError(errptr, db->rep->Flush(options->rep));
+}

rocksdb-sys/api/c.h

@@ -0,0 +1,6 @@
+#pragma once
+
+#include "../rocksdb/include/rocksdb/c.h"
+
+extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_flush(
+        rocksdb_transactiondb_t* db, const rocksdb_flushoptions_t* options, char** errptr);

rocksdb-sys/build.rs

@@ -0,0 +1,176 @@
+// Code from https://github.com/rust-rocksdb/rust-rocksdb/blob/eb2d302682418b361a80ad8f4dcf335ade60dcf5/librocksdb-sys/build.rs
+// License: https://github.com/rust-rocksdb/rust-rocksdb/blob/master/LICENSE
+
+use std::env;
+use std::fs::File;
+use std::io::Write;
+use std::path::{Path, PathBuf};
+
+fn link(name: &str, bundled: bool) {
+    use std::env::var;
+    let target = var("TARGET").unwrap();
+    let target: Vec<_> = target.split('-').collect();
+    if target.get(2) == Some(&"windows") {
+        println!("cargo:rustc-link-lib=dylib={}", name);
+        if bundled && target.get(3) == Some(&"gnu") {
+            let dir = var("CARGO_MANIFEST_DIR").unwrap();
+            println!("cargo:rustc-link-search=native={}/{}", dir, target[0]);
+        }
+    }
+}
+
+fn bindgen_rocksdb() {
+    bindgen::Builder::default()
+        .header("api/c.h")
+        .ctypes_prefix("libc")
+        .size_t_is_usize(true)
+        .generate()
+        .expect("unable to generate rocksdb bindings")
+        .write_to_file(PathBuf::from(env::var("OUT_DIR").unwrap()).join("bindings.rs"))
+        .expect("unable to write rocksdb bindings");
+}
+
+fn build_rocksdb() {
+    let target = env::var("TARGET").unwrap();
+
+    let mut config = cc::Build::new();
+    config
+        .cpp(true)
+        .include("rocksdb/include/")
+        .include("rocksdb/")
+        .file("api/c.cc")
+        .define("NDEBUG", Some("1"));
+
+    let mut lib_sources = include_str!("rocksdb/src.mk")
+        .split_once("LIB_SOURCES =")
+        .unwrap()
+        .1
+        .split_once("ifeq")
+        .unwrap()
+        .0
+        .split('\\')
+        .map(str::trim)
+        .filter(|p| !p.is_empty())
+        .collect::<Vec<_>>();
+
+    if target.contains("x86_64") {
+        // This is needed to enable hardware CRC32C. Technically, SSE 4.2 is
+        // only available since Intel Nehalem (about 2010) and AMD Bulldozer
+        // (about 2011).
+        let target_feature = env::var("CARGO_CFG_TARGET_FEATURE").unwrap();
+        let target_features: Vec<_> = target_feature.split(',').collect();
+        if target_features.contains(&"sse2") {
+            config.flag_if_supported("-msse2");
+        }
+        if target_features.contains(&"sse4.1") {
+            config.flag_if_supported("-msse4.1");
+        }
+        if target_features.contains(&"sse4.2") {
+            config.flag_if_supported("-msse4.2");
+            config.define("HAVE_SSE42", Some("1"));
+        }
+        if target_features.contains(&"pclmulqdq") && !target.contains("android") {
+            config.define("HAVE_PCLMUL", Some("1"));
+            config.flag_if_supported("-mpclmul");
+        }
+    } else if target.contains("aarch64") {
+        lib_sources.push("util/crc32c_arm64.cc")
+    }
+
+    if target.contains("darwin") {
+        config.define("OS_MACOSX", None);
+        config.define("ROCKSDB_PLATFORM_POSIX", None);
+        config.define("ROCKSDB_LIB_IO_POSIX", None);
+    } else if target.contains("android") {
+        config.define("OS_ANDROID", None);
+        config.define("ROCKSDB_PLATFORM_POSIX", None);
+        config.define("ROCKSDB_LIB_IO_POSIX", None);
+    } else if target.contains("linux") {
+        config.define("OS_LINUX", None);
+        config.define("ROCKSDB_PLATFORM_POSIX", None);
+        config.define("ROCKSDB_LIB_IO_POSIX", None);
+    } else if target.contains("freebsd") {
+        config.define("OS_FREEBSD", None);
+        config.define("ROCKSDB_PLATFORM_POSIX", None);
+        config.define("ROCKSDB_LIB_IO_POSIX", None);
+    } else if target.contains("windows") {
+        link("rpcrt4", false);
+        link("shlwapi", false);
+        config.define("DWIN32", None);
+        config.define("OS_WIN", None);
+        config.define("_MBCS", None);
+        config.define("WIN64", None);
+        config.define("NOMINMAX", None);
+        config.define("WITH_WINDOWS_UTF8_FILENAMES", "ON");
+
+        if target == "x86_64-pc-windows-gnu" {
+            // Tell MinGW to create localtime_r wrapper of localtime_s function.
+            config.define("_POSIX_C_SOURCE", Some("1"));
+            // Tell MinGW to use at least Windows Vista headers instead of the ones of Windows XP.
+            // (This is minimum supported version of rocksdb)
+            config.define("_WIN32_WINNT", Some("_WIN32_WINNT_VISTA"));
+        }
+
+        // Remove POSIX-specific sources
+        lib_sources = lib_sources
+            .iter()
+            .cloned()
+            .filter(|file| {
+                !matches!(
+                    *file,
+                    "port/port_posix.cc"
+                        | "env/env_posix.cc"
+                        | "env/fs_posix.cc"
+                        | "env/io_posix.cc"
+                )
+            })
+            .collect::<Vec<&'static str>>();
+
+        // Add Windows-specific sources
+        lib_sources.push("port/win/port_win.cc");
+        lib_sources.push("port/win/env_win.cc");
+        lib_sources.push("port/win/env_default.cc");
+        lib_sources.push("port/win/win_logger.cc");
+        lib_sources.push("port/win/io_win.cc");
+        lib_sources.push("port/win/win_thread.cc");
+    }
+
+    config.define("ROCKSDB_SUPPORT_THREAD_LOCAL", None);
+
+    if target.contains("msvc") {
+        config.flag("-EHsc");
+    } else {
+        config.flag("-std=c++11");
+    }
+
+    // version file
+    let build_version_cc_path = Path::new("rocksdb/util/build_version.cc");
+    if !build_version_cc_path.exists() {
+        let build_version_cc = include_str!("rocksdb/util/build_version.cc.in")
+            .replace("@GIT_SHA@", "")
+            .replace("@GIT_TAG@", "")
+            .replace("@GIT_MOD@", "0")
+            .replace("@GIT_DATE@", "")
+            .replace("@BUILD_DATE@", "");
+        File::create(build_version_cc_path)
+            .unwrap()
+            .write_all(build_version_cc.as_bytes())
+            .unwrap();
+    }
+
+    for file in lib_sources {
+        if file == "db/c.cc" {
+            continue;
+        }
+        let file = "rocksdb/".to_string() + file;
+        config.file(&file);
+    }
+    config.cpp(true);
+    config.compile("rocksdb");
+}
+
+fn main() {
+    println!("cargo:rerun-if-changed=api/");
+    bindgen_rocksdb();
+    build_rocksdb();
+}

rocksdb-sys/rocksdb

@@ -0,0 +1 @@
+Subproject commit 0103296f39ec3fd89b4cdda9687c63fde90eec24

rocksdb-sys/src/lib.rs

@@ -0,0 +1,3 @@
+#![allow(non_upper_case_globals, non_camel_case_types, non_snake_case)]
+
+include!(concat!(env!("OUT_DIR"), "/bindings.rs"));