diff --git a/build_tools/build_detect_platform b/build_tools/build_detect_platform index 3389d2851..8479e3127 100755 --- a/build_tools/build_detect_platform +++ b/build_tools/build_detect_platform @@ -46,7 +46,7 @@ PLATFORM_CXXFLAGS="-std=c++11" COMMON_FLAGS="-DROCKSDB_PLATFORM_POSIX" # Default to fbcode gcc on internal fb machines -if [ -d /mnt/gvfs/third-party -a -z "$CXX" ]; then +if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then FBCODE_BUILD="true" if [ -z "$USE_CLANG" ]; then CENTOS_VERSION=`rpm -q --qf "%{VERSION}" \ diff --git a/java/Makefile b/java/Makefile index 47b2afb9e..b2f3674f0 100644 --- a/java/Makefile +++ b/java/Makefile @@ -1,4 +1,4 @@ -NATIVE_JAVA_CLASSES = org.rocksdb.RocksDB org.rocksdb.Options org.rocksdb.WriteBatch org.rocksdb.WriteBatchInternal org.rocksdb.WriteBatchTest org.rocksdb.WriteOptions org.rocksdb.BackupableDB org.rocksdb.BackupableDBOptions org.rocksdb.Statistics org.rocksdb.RocksIterator org.rocksdb.VectorMemTableConfig org.rocksdb.SkipListMemTableConfig org.rocksdb.HashLinkedListMemTableConfig org.rocksdb.HashSkipListMemTableConfig org.rocksdb.PlainTableConfig org.rocksdb.BlockBasedTableConfig org.rocksdb.ReadOptions org.rocksdb.Filter org.rocksdb.BloomFilter org.rocksdb.RestoreOptions org.rocksdb.RestoreBackupableDB org.rocksdb.RocksEnv +NATIVE_JAVA_CLASSES = org.rocksdb.RocksDB org.rocksdb.Options org.rocksdb.WriteBatch org.rocksdb.WriteBatchInternal org.rocksdb.WriteBatchTest org.rocksdb.WriteOptions org.rocksdb.BackupableDB org.rocksdb.BackupableDBOptions org.rocksdb.Statistics org.rocksdb.RocksIterator org.rocksdb.VectorMemTableConfig org.rocksdb.SkipListMemTableConfig org.rocksdb.HashLinkedListMemTableConfig org.rocksdb.HashSkipListMemTableConfig org.rocksdb.PlainTableConfig org.rocksdb.BlockBasedTableConfig org.rocksdb.ReadOptions org.rocksdb.Filter org.rocksdb.BloomFilter org.rocksdb.RestoreOptions org.rocksdb.RestoreBackupableDB org.rocksdb.RocksEnv org.rocksdb.GenericRateLimiterConfig NATIVE_INCLUDE = ./include ROCKSDB_JAR = rocksdbjni.jar diff --git a/java/RocksDBSample.java b/java/RocksDBSample.java index 54e5c2086..bd5a85076 100644 --- a/java/RocksDBSample.java +++ b/java/RocksDBSample.java @@ -75,6 +75,10 @@ public class RocksDBSample { // Plain-Table requires mmap read options.setAllowMmapReads(true); assert(options.tableFactoryName().equals("PlainTable")); + + options.setRateLimiterConfig(new GenericRateLimiterConfig(10000000, + 10000, 10)); + options.setRateLimiterConfig(new GenericRateLimiterConfig(10000000)); BlockBasedTableConfig table_options = new BlockBasedTableConfig(); table_options.setBlockCacheSize(64 * SizeUnit.KB) diff --git a/java/org/rocksdb/GenericRateLimiterConfig.java b/java/org/rocksdb/GenericRateLimiterConfig.java new file mode 100644 index 000000000..78b8b37ec --- /dev/null +++ b/java/org/rocksdb/GenericRateLimiterConfig.java @@ -0,0 +1,36 @@ +// Copyright (c) 2014, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. +package org.rocksdb; + +/** + * Config for rate limiter, which is used to control write rate of flush and + * compaction. + */ +public class GenericRateLimiterConfig extends RateLimiterConfig { + private static final long DEFAULT_REFILL_PERIOD_MICROS = (100 * 1000); + private static final int DEFAULT_FAIRNESS = 10; + + public GenericRateLimiterConfig(long rateBytesPerSecond, + long refillPeriodMicros, int fairness) { + rateBytesPerSecond_ = rateBytesPerSecond; + refillPeriodMicros_ = refillPeriodMicros; + fairness_ = fairness; + } + + public GenericRateLimiterConfig(long rateBytesPerSecond) { + this(rateBytesPerSecond, DEFAULT_REFILL_PERIOD_MICROS, DEFAULT_FAIRNESS); + } + + @Override protected long newRateLimiterHandle() { + return newRateLimiterHandle(rateBytesPerSecond_, refillPeriodMicros_, + fairness_); + } + + private native long newRateLimiterHandle(long rateBytesPerSecond, + long refillPeriodMicros, int fairness); + private final long rateBytesPerSecond_; + private final long refillPeriodMicros_; + private final int fairness_; +} diff --git a/java/org/rocksdb/Options.java b/java/org/rocksdb/Options.java index 922bdbdb0..876a06285 100644 --- a/java/org/rocksdb/Options.java +++ b/java/org/rocksdb/Options.java @@ -1104,6 +1104,19 @@ public class Options extends RocksObject { setMemTableFactory(nativeHandle_, config.newMemTableFactoryHandle()); return this; } + + /** + * Use to control write rate of flush and compaction. Flush has higher + * priority than compaction. Rate limiting is disabled if nullptr. + * Default: nullptr + * + * @param config rate limiter config. + * @return the instance of the current Options. + */ + public Options setRateLimiterConfig(RateLimiterConfig config) { + setRateLimiter(nativeHandle_, config.newRateLimiterHandle()); + return this; + } /** * Returns the name of the current mem table representation. @@ -2192,6 +2205,8 @@ public class Options extends RocksObject { private native long statisticsPtr(long optHandle); private native void setMemTableFactory(long handle, long factoryHandle); + private native void setRateLimiter(long handle, + long rateLimiterHandle); private native String memTableFactoryName(long handle); private native void setTableFactory(long handle, long factoryHandle); diff --git a/java/org/rocksdb/RateLimiterConfig.java b/java/org/rocksdb/RateLimiterConfig.java new file mode 100644 index 000000000..22de65921 --- /dev/null +++ b/java/org/rocksdb/RateLimiterConfig.java @@ -0,0 +1,20 @@ +// Copyright (c) 2014, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. +package org.rocksdb; + +/** + * Config for rate limiter, which is used to control write rate of flush and + * compaction. + */ +public abstract class RateLimiterConfig { + /** + * This function should only be called by Options.setRateLimiter(), + * which will create a c++ shared-pointer to the c++ RateLimiter + * that is associated with the Java RateLimtierConifg. + * + * @see Options.setRateLimiter() + */ + abstract protected long newRateLimiterHandle(); +} diff --git a/java/rocksjni/options.cc b/java/rocksjni/options.cc index a72eecd28..705e9ff8c 100644 --- a/java/rocksjni/options.cc +++ b/java/rocksjni/options.cc @@ -21,6 +21,7 @@ #include "rocksdb/memtablerep.h" #include "rocksdb/table.h" #include "rocksdb/slice_transform.h" +#include "rocksdb/rate_limiter.h" /* * Class: org_rocksdb_Options @@ -459,6 +460,17 @@ void Java_org_rocksdb_Options_setMemTableFactory( reinterpret_cast(jfactory_handle)); } +/* + * Class: org_rocksdb_Options + * Method: setRateLimiter + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setRateLimiter( + JNIEnv* env, jobject jobj, jlong jhandle, jlong jrate_limiter_handle) { + reinterpret_cast(jhandle)->rate_limiter.reset( + reinterpret_cast(jrate_limiter_handle)); +} + /* * Class: org_rocksdb_Options * Method: tableCacheNumshardbits diff --git a/java/rocksjni/ratelimiterjni.cc b/java/rocksjni/ratelimiterjni.cc new file mode 100644 index 000000000..5413978a0 --- /dev/null +++ b/java/rocksjni/ratelimiterjni.cc @@ -0,0 +1,24 @@ +// Copyright (c) 2014, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. +// +// This file implements the "bridge" between Java and C++ for RateLimiter. + +#include "rocksjni/portal.h" +#include "include/org_rocksdb_GenericRateLimiterConfig.h" +#include "rocksdb/rate_limiter.h" + +/* + * Class: org_rocksdb_GenericRateLimiterConfig + * Method: newRateLimiterHandle + * Signature: (JJI)J + */ +jlong Java_org_rocksdb_GenericRateLimiterConfig_newRateLimiterHandle( + JNIEnv* env, jobject jobj, jlong jrate_bytes_per_second, + jlong jrefill_period_micros, jint jfairness) { + return reinterpret_cast(rocksdb::NewGenericRateLimiter( + rocksdb::jlong_to_size_t(jrate_bytes_per_second), + rocksdb::jlong_to_size_t(jrefill_period_micros), + static_cast(jfairness))); +} diff --git a/util/env_test.cc b/util/env_test.cc index 1c4d0bba0..3e811a98d 100644 --- a/util/env_test.cc +++ b/util/env_test.cc @@ -17,6 +17,11 @@ #include #endif +#ifdef ROCKSDB_FALLOCATE_PRESENT +#include +#include +#endif + #include "rocksdb/env.h" #include "port/port.h" #include "util/coding.h" @@ -478,6 +483,31 @@ TEST(EnvPosixTest, RandomAccessUniqueID) { #ifdef ROCKSDB_FALLOCATE_PRESENT TEST(EnvPosixTest, AllocateTest) { std::string fname = GetOnDiskTestDir() + "/preallocate_testfile"; + + // Try fallocate in a file to see whether the target file system supports it. + // Skip the test if fallocate is not supported. + std::string fname_test_fallocate = + GetOnDiskTestDir() + "/preallocate_testfile_2"; + int fd = -1; + do { + fd = open(fname_test_fallocate.c_str(), O_CREAT | O_RDWR | O_TRUNC, 0644); + } while (fd < 0 && errno == EINTR); + ASSERT_GT(fd, 0); + + int alloc_status = fallocate(fd, 0, 0, 1); + + int err_number = 0; + if (alloc_status != 0) { + err_number = errno; + fprintf(stderr, "Warning: fallocate() fails, %s\n", strerror(err_number)); + } + close(fd); + ASSERT_OK(env_->DeleteFile(fname_test_fallocate)); + if (alloc_status != 0 && err_number == EOPNOTSUPP) { + // The filesystem containing the file does not support fallocate + return; + } + EnvOptions soptions; soptions.use_mmap_writes = false; unique_ptr wfile; diff --git a/util/options.cc b/util/options.cc index a61d9d633..f0042cbda 100644 --- a/util/options.cc +++ b/util/options.cc @@ -419,7 +419,7 @@ void ColumnFamilyOptions::Dump(Logger* log) const { "max_size_amplification_percent: %u", compaction_options_universal.max_size_amplification_percent); Log(log, - "Options.compaction_options_universal.compression_size_percent: %u", + "Options.compaction_options_universal.compression_size_percent: %d", compaction_options_universal.compression_size_percent); Log(log, "Options.compaction_options_fifo.max_table_files_size: %" PRIu64, compaction_options_fifo.max_table_files_size);