diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..10d03de --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "rocksdb-sys/snappy"] + path = rocksdb-sys/snappy + url = https://github.com/google/snappy.git +[submodule "rocksdb-sys/rocksdb"] + path = rocksdb-sys/rocksdb + url = https://github.com/facebook/rocksdb.git diff --git a/.travis.yml b/.travis.yml index 80e8355..bb3e1de 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,37 +2,21 @@ language: rust dist: trusty sudo: true -matrix: - include: - - rust: stable - env: ROCKSDB_VERSION=4.13 - - rust: beta - env: ROCKSDB_VERSION=4.13 - - rust: stable - env: ROCKSDB_VERSION=4.5 - - rust: stable - env: ROCKSDB_VERSION=4.1 +rust: + - stable + - beta addons: apt: sources: - - ubuntu-toolchain-r-test + - ubuntu-toolchain-r-test packages: - - gcc-5 - - g++-5 - - libgflags-dev - - libsnappy-dev - - zlib1g-dev - - libbz2-dev - -install: - - curl -L https://github.com/facebook/rocksdb/archive/$ROCKSDB_VERSION.fb.tar.gz | tar xvz -C $HOME/rocksdb - - sudo INSTALL_PATH=/usr make -C $HOME/rocksdb/rocksdb-$ROCKSDB_VERSION.fb install-shared + - g++-5 script: + - cargo test --manifest-path=rocksdb-sys/Cargo.toml - cargo test cache: directories: - $HOME/.cache/ - - $HOME/rocksdb/ diff --git a/Cargo.toml b/Cargo.toml index bdfeb27..08dca65 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,3 +25,4 @@ path = "test/test.rs" [dependencies] libc = "0.2.13" +rocksdb-sys = { path = "rocksdb-sys", version = "0.4.0" } diff --git a/README.md b/README.md index af31e8f..3af7091 100644 --- a/README.md +++ b/README.md @@ -18,20 +18,10 @@ This library has been tested against RocksDB 3.13.1 on linux and OSX. The 0.4.1 - [x] column family operations - [ ] prefix seek - [ ] slicetransform - - [ ] windows support + - [x] windows support Feedback and pull requests welcome! If a particular feature of RocksDB is important to you, please let me know by opening an issue, and I'll prioritize it. -###### Prerequisite: RocksDB - -First, use your system's package manager to install snappy. This is optional, but lets rocksdb take advantage of better compression, and some code may require it. - -```bash -wget https://github.com/facebook/rocksdb/archive/rocksdb-3.8.tar.gz -tar xvf rocksdb-3.8.tar.gz && cd rocksdb-rocksdb-3.8 && make shared_lib -sudo make install -``` - ### Running ###### Cargo.toml ```rust diff --git a/rocksdb-sys/.gitignore b/rocksdb-sys/.gitignore new file mode 100644 index 0000000..a9d37c5 --- /dev/null +++ b/rocksdb-sys/.gitignore @@ -0,0 +1,2 @@ +target +Cargo.lock diff --git a/rocksdb-sys/Cargo.toml b/rocksdb-sys/Cargo.toml new file mode 100644 index 0000000..0089dec --- /dev/null +++ b/rocksdb-sys/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "rocksdb-sys" +version = "0.4.0" +authors = ["Karl Hobley ", "Arkadiy Paronyan "] +license = "MIT/Apache-2.0/BSD" +description = "Native bindings to librocksdb" +readme = "README.md" +repository = "https://github.com/jsgf/rocksdb-sys.git" +keywords = [ "ffi", "rocksdb" ] + +build = "build.rs" +links = "rocksdb" + +[features] +default = [ "static" ] +static = [] + +[dependencies] +libc = "0.2" + +[build-dependencies] +gcc = { version = "0.3", features = ["parallel"] } diff --git a/rocksdb-sys/Makefile b/rocksdb-sys/Makefile new file mode 100644 index 0000000..8e5405a --- /dev/null +++ b/rocksdb-sys/Makefile @@ -0,0 +1,6 @@ +include rocksdb/src.mk + +rocksdb_lib_sources.txt: rocksdb/src.mk + @echo -n ${LIB_SOURCES} > rocksdb_lib_sources.txt + +gen_lib_sources: rocksdb_lib_sources.txt diff --git a/rocksdb-sys/README.md b/rocksdb-sys/README.md new file mode 100644 index 0000000..495ac54 --- /dev/null +++ b/rocksdb-sys/README.md @@ -0,0 +1,10 @@ +RocksDB bindings +================ + +Low-level bindings to RocksDB's C API. + +Based on original work by Tyler Neely +https://github.com/spacejam/rust-rocksdb +and Jeremy Fitzhardinge +https://github.com/jsgf/rocksdb-sys + diff --git a/rocksdb-sys/build.rs b/rocksdb-sys/build.rs new file mode 100644 index 0000000..410a5ed --- /dev/null +++ b/rocksdb-sys/build.rs @@ -0,0 +1,116 @@ +extern crate gcc; + +fn link(name: &str, bundled: bool) { + use std::env::var; + let target = var("TARGET").unwrap(); + let target: Vec<_> = target.split('-').collect(); + if target.get(2) == Some(&"windows") { + println!("cargo:rustc-link-lib=dylib={}", name); + if bundled && target.get(3) == Some(&"gnu") { + let dir = var("CARGO_MANIFEST_DIR").unwrap(); + println!("cargo:rustc-link-search=native={}/{}", dir, target[0]); + } + } +} + +fn build_rocksdb() { + let mut config = gcc::Config::new(); + config.include("rocksdb/include/"); + config.include("rocksdb/"); + config.include("rocksdb/third-party/gtest-1.7.0/fused-src/"); + config.include("snappy/"); + config.include("."); + + config.define("NDEBUG", Some("1")); + config.define("SNAPPY", Some("1")); + + let mut lib_sources = include_str!("rocksdb_lib_sources.txt").split(" ").collect::>(); + + // We have a pregenerated a version of build_version.cc in the local directory + lib_sources = lib_sources.iter().cloned().filter(|file| { + *file != "util/build_version.cc" + }) + .collect::>(); + + if cfg!(target_os = "macos") { + config.define("OS_MACOSX", Some("1")); + config.define("ROCKSDB_PLATFORM_POSIX", Some("1")); + config.define("ROCKSDB_LIB_IO_POSIX", Some("1")); + + } + if cfg!(target_os = "linux") { + config.define("OS_LINUX", Some("1")); + config.define("ROCKSDB_PLATFORM_POSIX", Some("1")); + config.define("ROCKSDB_LIB_IO_POSIX", Some("1")); + //COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp" + } + if cfg!(target_os = "freebsd") { + config.define("OS_FREEBSD", Some("1")); + config.define("ROCKSDB_PLATFORM_POSIX", Some("1")); + config.define("ROCKSDB_LIB_IO_POSIX", Some("1")); + } + + if cfg!(windows) { + link("rpcrt4", false); + config.define("OS_WIN", Some("1")); + + // Remove POSIX-specific sources + lib_sources = lib_sources.iter().cloned().filter(|file| { + match *file { + "port/port_posix.cc" | + "util/env_posix.cc" | + "util/io_posix.cc" => false, + _ => true + } + }) + .collect::>(); + + // Add Windows-specific sources + lib_sources.push("port/win/port_win.cc"); + lib_sources.push("port/win/env_win.cc"); + lib_sources.push("port/win/env_default.cc"); + lib_sources.push("port/win/win_logger.cc"); + lib_sources.push("port/win/io_win.cc"); + } + + if cfg!(target_env = "msvc") { + config.flag("-EHsc"); + } else { + config.flag("-std=c++11"); + } + + for file in lib_sources { + let file = "rocksdb/".to_string() + file; + config.file(&file); + } + + config.file("build_version.cc"); + + config.cpp(true); + config.compile("librocksdb.a"); +} + +fn build_snappy() { + let mut config = gcc::Config::new(); + config.include("snappy/"); + config.include("."); + + config.define("NDEBUG", Some("1")); + + if cfg!(target_env = "msvc") { + config.flag("-EHsc"); + } else { + config.flag("-std=c++11"); + } + + config.file("snappy/snappy.cc"); + config.file("snappy/snappy-sinksource.cc"); + config.file("snappy/snappy-c.cc"); + config.cpp(true); + config.compile("libsnappy.a"); +} + +fn main() { + build_rocksdb(); + build_snappy(); +} diff --git a/rocksdb-sys/build_version.cc b/rocksdb-sys/build_version.cc new file mode 100644 index 0000000..7f600db --- /dev/null +++ b/rocksdb-sys/build_version.cc @@ -0,0 +1,4 @@ +#include "build_version.h" +const char* rocksdb_build_git_sha = "rocksdb_build_git_sha:f201a44b4102308b840b15d9b89122af787476f1"; +const char* rocksdb_build_git_date = "rocksdb_build_git_date:2016-10-27"; +const char* rocksdb_build_compile_date = __DATE__; diff --git a/rocksdb-sys/build_version.h b/rocksdb-sys/build_version.h new file mode 100644 index 0000000..ca1dbf5 --- /dev/null +++ b/rocksdb-sys/build_version.h @@ -0,0 +1,15 @@ +// Copyright (c) 2013, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. +// +#pragma once +#if !defined(IOS_CROSS_COMPILE) +// if we compile with Xcode, we don't run build_detect_vesion, so we don't +// generate these variables +// this variable tells us about the git revision +extern const char* rocksdb_build_git_sha; + +// Date on which the code was compiled: +extern const char* rocksdb_build_compile_date; +#endif diff --git a/rocksdb-sys/rocksdb b/rocksdb-sys/rocksdb new file mode 160000 index 0000000..f201a44 --- /dev/null +++ b/rocksdb-sys/rocksdb @@ -0,0 +1 @@ +Subproject commit f201a44b4102308b840b15d9b89122af787476f1 diff --git a/rocksdb-sys/rocksdb_lib_sources.txt b/rocksdb-sys/rocksdb_lib_sources.txt new file mode 100644 index 0000000..209deb5 --- /dev/null +++ b/rocksdb-sys/rocksdb_lib_sources.txt @@ -0,0 +1 @@ +db/auto_roll_logger.cc db/builder.cc db/c.cc db/column_family.cc db/compacted_db_impl.cc db/compaction.cc db/compaction_iterator.cc db/compaction_job.cc db/compaction_picker.cc db/convenience.cc db/range_del_aggregator.cc db/db_filesnapshot.cc db/dbformat.cc db/db_impl.cc db/db_impl_debug.cc db/db_impl_readonly.cc db/db_impl_experimental.cc db/db_info_dumper.cc db/db_iter.cc db/external_sst_file_ingestion_job.cc db/experimental.cc db/event_helpers.cc db/file_indexer.cc db/filename.cc db/flush_job.cc db/flush_scheduler.cc db/forward_iterator.cc db/internal_stats.cc db/log_reader.cc db/log_writer.cc db/managed_iterator.cc db/memtable_allocator.cc db/memtable.cc db/memtable_list.cc db/merge_helper.cc db/merge_operator.cc db/repair.cc db/snapshot_impl.cc db/table_cache.cc db/table_properties_collector.cc db/transaction_log_impl.cc db/version_builder.cc db/version_edit.cc db/version_set.cc db/wal_manager.cc db/write_batch.cc db/write_batch_base.cc db/write_controller.cc db/write_thread.cc db/xfunc_test_points.cc memtable/hash_cuckoo_rep.cc memtable/hash_linklist_rep.cc memtable/hash_skiplist_rep.cc memtable/skiplistrep.cc memtable/vectorrep.cc port/stack_trace.cc port/port_posix.cc table/adaptive_table_factory.cc table/block_based_filter_block.cc table/block_based_table_builder.cc table/block_based_table_factory.cc table/block_based_table_reader.cc table/block_builder.cc table/block.cc table/block_prefix_index.cc table/bloom_block.cc table/cuckoo_table_builder.cc table/cuckoo_table_factory.cc table/cuckoo_table_reader.cc table/flush_block_policy.cc table/format.cc table/full_filter_block.cc table/get_context.cc table/iterator.cc table/merger.cc table/meta_blocks.cc table/sst_file_writer.cc table/plain_table_builder.cc table/plain_table_factory.cc table/plain_table_index.cc table/plain_table_key_coding.cc table/plain_table_reader.cc table/persistent_cache_helper.cc table/table_properties.cc table/two_level_iterator.cc tools/dump/db_dump_tool.cc util/arena.cc util/bloom.cc util/build_version.cc util/cf_options.cc util/clock_cache.cc util/coding.cc util/comparator.cc util/compaction_job_stats_impl.cc util/concurrent_arena.cc util/crc32c.cc util/db_options.cc util/delete_scheduler.cc util/dynamic_bloom.cc util/env.cc util/env_chroot.cc util/env_hdfs.cc util/env_posix.cc util/event_logger.cc util/file_util.cc util/file_reader_writer.cc util/filter_policy.cc util/hash.cc util/histogram.cc util/histogram_windowing.cc util/instrumented_mutex.cc util/iostats_context.cc util/io_posix.cc util/log_buffer.cc util/logging.cc util/lru_cache.cc util/memenv.cc util/murmurhash.cc util/options.cc util/options_helper.cc util/options_parser.cc util/options_sanity_check.cc util/perf_context.cc util/perf_level.cc util/random.cc util/rate_limiter.cc util/sharded_cache.cc util/slice.cc util/sst_file_manager_impl.cc util/statistics.cc util/status.cc util/status_message.cc util/string_util.cc util/sync_point.cc util/thread_local.cc util/thread_status_impl.cc util/thread_status_updater.cc util/thread_status_updater_debug.cc util/thread_status_util.cc util/thread_status_util_debug.cc util/threadpool_imp.cc util/transaction_test_util.cc util/xfunc.cc util/xxhash.cc utilities/backupable/backupable_db.cc utilities/blob_db/blob_db.cc utilities/convenience/info_log_finder.cc utilities/checkpoint/checkpoint.cc utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc utilities/document/document_db.cc utilities/document/json_document_builder.cc utilities/document/json_document.cc utilities/env_mirror.cc utilities/env_registry.cc utilities/flashcache/flashcache.cc utilities/geodb/geodb_impl.cc utilities/leveldb_options/leveldb_options.cc utilities/memory/memory_util.cc utilities/merge_operators/put.cc utilities/merge_operators/max.cc utilities/merge_operators/string_append/stringappend2.cc utilities/merge_operators/string_append/stringappend.cc utilities/merge_operators/uint64add.cc utilities/option_change_migration/option_change_migration.cc utilities/options/options_util.cc utilities/persistent_cache/persistent_cache_tier.cc utilities/persistent_cache/volatile_tier_impl.cc utilities/persistent_cache/block_cache_tier_file.cc utilities/persistent_cache/block_cache_tier_metadata.cc utilities/persistent_cache/block_cache_tier.cc utilities/redis/redis_lists.cc utilities/simulator_cache/sim_cache.cc utilities/spatialdb/spatial_db.cc utilities/table_properties_collectors/compact_on_deletion_collector.cc utilities/transactions/optimistic_transaction_impl.cc utilities/transactions/optimistic_transaction_db_impl.cc utilities/transactions/transaction_base.cc utilities/transactions/transaction_db_impl.cc utilities/transactions/transaction_db_mutex_impl.cc utilities/transactions/transaction_lock_mgr.cc utilities/transactions/transaction_impl.cc utilities/transactions/transaction_util.cc utilities/ttl/db_ttl_impl.cc utilities/date_tiered/date_tiered_db_impl.cc utilities/write_batch_with_index/write_batch_with_index.cc utilities/write_batch_with_index/write_batch_with_index_internal.cc \ No newline at end of file diff --git a/rocksdb-sys/snappy b/rocksdb-sys/snappy new file mode 160000 index 0000000..efb39e8 --- /dev/null +++ b/rocksdb-sys/snappy @@ -0,0 +1 @@ +Subproject commit efb39e81b8b66b757ec900db33657c47b4750740 diff --git a/rocksdb-sys/snappy-stubs-public.h b/rocksdb-sys/snappy-stubs-public.h new file mode 100644 index 0000000..14592e0 --- /dev/null +++ b/rocksdb-sys/snappy-stubs-public.h @@ -0,0 +1,97 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// Author: sesse@google.com (Steinar H. Gunderson) +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Various type stubs for the open-source version of Snappy. +// +// This file cannot include config.h, as it is included from snappy.h, +// which is a public header. Instead, snappy-stubs-public.h is generated by +// from snappy-stubs-public.h.in at configure time. + +#ifndef THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_ +#define THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_ + +#define HAVE_STDINT_H 1 + + +#include + +#include + +//#include + +#if defined(_MSC_VER) +#define ssize_t intptr_t +#endif + +#define SNAPPY_MAJOR 1 +#define SNAPPY_MINOR 1 +#define SNAPPY_PATCHLEVEL 3 +#define SNAPPY_VERSION \ + ((SNAPPY_MAJOR << 16) | (SNAPPY_MINOR << 8) | SNAPPY_PATCHLEVEL) + +#include + +namespace snappy { + +#if HAVE_STDINT_H +typedef int8_t int8; +typedef uint8_t uint8; +typedef int16_t int16; +typedef uint16_t uint16; +typedef int32_t int32; +typedef uint32_t uint32; +typedef int64_t int64; +typedef uint64_t uint64; +#else +typedef signed char int8; +typedef unsigned char uint8; +typedef short int16; +typedef unsigned short uint16; +typedef int int32; +typedef unsigned int uint32; +typedef long long int64; +typedef unsigned long long uint64; +#endif + +typedef std::string string; + +#define DISALLOW_COPY_AND_ASSIGN(TypeName) \ + TypeName(const TypeName&); \ + void operator=(const TypeName&) + +// Windows does not have an iovec type, yet the concept is universally useful. +// It is simple to define it ourselves, so we put it inside our own namespace. +struct iovec { + void* iov_base; + size_t iov_len; +}; + +} // namespace snappy + +#endif // THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_ diff --git a/src/ffi.rs b/rocksdb-sys/src/ffi.rs similarity index 100% rename from src/ffi.rs rename to rocksdb-sys/src/ffi.rs diff --git a/rocksdb-sys/src/lib.rs b/rocksdb-sys/src/lib.rs new file mode 100644 index 0000000..fee943e --- /dev/null +++ b/rocksdb-sys/src/lib.rs @@ -0,0 +1,18 @@ +// +// Copyright 2014 Tyler Neely +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +pub use ffi as rocksdb_ffi; +pub use ffi::{DBCompactionStyle, DBComparator, DBCompressionType, DBRecoveryMode, new_bloom_filter}; +pub mod ffi; diff --git a/src/lib.rs b/src/lib.rs index c40c469..3eb2aae 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,14 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. // -pub use ffi as rocksdb_ffi; -pub use ffi::{DBCompactionStyle, DBComparator, DBCompressionType, DBRecoveryMode, new_bloom_filter}; +extern crate rocksdb_sys; +pub use rocksdb_sys::rocksdb_ffi as rocksdb_ffi; +pub use rocksdb_ffi::{DBCompactionStyle, DBComparator, new_bloom_filter}; pub use rocksdb::{DB, DBIterator, DBVector, Direction, IteratorMode, Writable, WriteBatch, Error}; pub use rocksdb_options::{BlockBasedOptions, Options, WriteOptions}; pub use merge_operator::MergeOperands; pub mod rocksdb; -pub mod ffi; pub mod rocksdb_options; pub mod merge_operator; pub mod comparator;