From 64b99a4f3b5fad5931d6da71972382af63072fd4 Mon Sep 17 00:00:00 2001 From: Jordan Terrell Date: Tue, 21 Aug 2018 06:38:41 -0500 Subject: [PATCH] Adding Zstandard compression support... --- .gitmodules | 3 +++ librocksdb-sys/Cargo.toml | 1 + librocksdb-sys/build.rs | 38 ++++++++++++++++++++++++++++++++++++++ src/db.rs | 1 + 4 files changed, 43 insertions(+) diff --git a/.gitmodules b/.gitmodules index 0c44d00..2928a19 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "librocksdb-sys/lz4"] path = librocksdb-sys/lz4 url = https://github.com/lz4/lz4.git +[submodule "librocksdb-sys/zstd"] + path = librocksdb-sys/zstd + url = https://github.com/facebook/zstd diff --git a/librocksdb-sys/Cargo.toml b/librocksdb-sys/Cargo.toml index 7d08b87..a449ef2 100644 --- a/librocksdb-sys/Cargo.toml +++ b/librocksdb-sys/Cargo.toml @@ -25,3 +25,4 @@ const-cstr = "0.2" cc = { version = "^1.0", features = ["parallel"] } make-cmd = "0.1" bindgen = "0.37" +glob = "0.2.11" diff --git a/librocksdb-sys/build.rs b/librocksdb-sys/build.rs index 8b1d31d..efe1265 100644 --- a/librocksdb-sys/build.rs +++ b/librocksdb-sys/build.rs @@ -1,5 +1,6 @@ extern crate cc; extern crate bindgen; +extern crate glob; use std::env; use std::fs; @@ -50,11 +51,14 @@ fn build_rocksdb() { config.include("rocksdb/third-party/gtest-1.7.0/fused-src/"); config.include("snappy/"); config.include("lz4/lib/"); + config.include("zstd/lib/"); + config.include("zstd/lib/dictBuilder/"); config.include("."); config.define("NDEBUG", Some("1")); config.define("SNAPPY", Some("1")); config.define("LZ4", Some("1")); + config.define("ZSTD", Some("1")); let mut lib_sources = include_str!("rocksdb_lib_sources.txt") .split(" ") @@ -172,6 +176,34 @@ fn build_lz4() { compiler.compile("liblz4.a"); } +fn build_zstd() { + let mut compiler = cc::Build::new(); + + compiler.include("zstd/lib/"); + compiler.include("zstd/lib/common"); + compiler.include("zstd/lib/legacy"); + + let globs = &[ + "zstd/lib/common/*.c", + "zstd/lib/compress/*.c", + "zstd/lib/decompress/*.c", + "zstd/lib/dictBuilder/*.c", + "zstd/lib/legacy/*.c", + ]; + + for pattern in globs { + for path in glob::glob(pattern).unwrap() { + let path = path.unwrap(); + compiler.file(path); + } + } + + compiler.opt_level(3); + + compiler.define("ZSTD_LIB_DEPRECATED", Some("0")); + compiler.compile("libzstd.a"); +} + fn try_to_find_and_link_lib(lib_name: &str) -> bool { if let Ok(lib_dir) = env::var(&format!("{}_LIB_DIR", lib_name)) { println!("cargo:rustc-link-search=native={}", lib_dir); @@ -190,10 +222,13 @@ fn main() { println!("cargo:rerun-if-changed=rocksdb/"); println!("cargo:rerun-if-changed=snappy/"); println!("cargo:rerun-if-changed=lz4/"); + println!("cargo:rerun-if-changed=zstd/"); fail_on_empty_directory("rocksdb"); fail_on_empty_directory("snappy"); fail_on_empty_directory("lz4"); + fail_on_empty_directory("zstd"); + bindgen_rocksdb(); if !try_to_find_and_link_lib("ROCKSDB") { @@ -205,4 +240,7 @@ fn main() { if !try_to_find_and_link_lib("LZ4") { build_lz4(); } + if !try_to_find_and_link_lib("ZSTD") { + build_zstd(); + } } diff --git a/src/db.rs b/src/db.rs index 78fd8c7..29998b7 100644 --- a/src/db.rs +++ b/src/db.rs @@ -44,6 +44,7 @@ pub enum DBCompressionType { Bz2 = ffi::rocksdb_bz2_compression as isize, Lz4 = ffi::rocksdb_lz4_compression as isize, Lz4hc = ffi::rocksdb_lz4hc_compression as isize, + Zstd = ffi::rocksdb_zstd_compression as isize, } #[derive(Debug, Copy, Clone, PartialEq)]