From 3b897cddd78518ea85b2e15f71ba94673e3cab59 Mon Sep 17 00:00:00 2001 From: Igor Canadi Date: Fri, 19 Sep 2014 09:27:16 -0700 Subject: [PATCH 1/4] Enable no-fbcode RocksDB build Summary: I want to use open source build rather than fbcode one. This enables me to run `ROCKSDB_NO_FBCODE=1 make` and run it with my system g++. Test Plan: ROCKSDB_NO_FBCODE=1 make make Reviewers: sdong, ljin, yhchiang Reviewed By: yhchiang Subscribers: leveldb Differential Revision: https://reviews.facebook.net/D23613 --- build_tools/build_detect_platform | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_tools/build_detect_platform b/build_tools/build_detect_platform index 3389d2851..8479e3127 100755 --- a/build_tools/build_detect_platform +++ b/build_tools/build_detect_platform @@ -46,7 +46,7 @@ PLATFORM_CXXFLAGS="-std=c++11" COMMON_FLAGS="-DROCKSDB_PLATFORM_POSIX" # Default to fbcode gcc on internal fb machines -if [ -d /mnt/gvfs/third-party -a -z "$CXX" ]; then +if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then FBCODE_BUILD="true" if [ -z "$USE_CLANG" ]; then CENTOS_VERSION=`rpm -q --qf "%{VERSION}" \ From 976caca09bb0ce6cac15fbf2a3ddd4fba98d1bad Mon Sep 17 00:00:00 2001 From: sdong Date: Fri, 19 Sep 2014 10:37:42 -0700 Subject: [PATCH 2/4] Skip AllocateTest if fallocate() is not supported in the file system Summary: To avoid false positive test failures when the file system doesn't support fallocate. In EnvTest.AllocateTest, we first make a simple fallocate call and check the error codes to rule out the possibility that it is not supported. Skip the test if the error code indicates it is not supported. Test Plan: Run the test and make sure it passes on file systems supporting and not supporting fallocate Reviewers: yhchiang, ljin, igor Reviewed By: igor Subscribers: leveldb Differential Revision: https://reviews.facebook.net/D23667 --- util/env_test.cc | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/util/env_test.cc b/util/env_test.cc index 1c4d0bba0..3e811a98d 100644 --- a/util/env_test.cc +++ b/util/env_test.cc @@ -17,6 +17,11 @@ #include #endif +#ifdef ROCKSDB_FALLOCATE_PRESENT +#include +#include +#endif + #include "rocksdb/env.h" #include "port/port.h" #include "util/coding.h" @@ -478,6 +483,31 @@ TEST(EnvPosixTest, RandomAccessUniqueID) { #ifdef ROCKSDB_FALLOCATE_PRESENT TEST(EnvPosixTest, AllocateTest) { std::string fname = GetOnDiskTestDir() + "/preallocate_testfile"; + + // Try fallocate in a file to see whether the target file system supports it. + // Skip the test if fallocate is not supported. + std::string fname_test_fallocate = + GetOnDiskTestDir() + "/preallocate_testfile_2"; + int fd = -1; + do { + fd = open(fname_test_fallocate.c_str(), O_CREAT | O_RDWR | O_TRUNC, 0644); + } while (fd < 0 && errno == EINTR); + ASSERT_GT(fd, 0); + + int alloc_status = fallocate(fd, 0, 0, 1); + + int err_number = 0; + if (alloc_status != 0) { + err_number = errno; + fprintf(stderr, "Warning: fallocate() fails, %s\n", strerror(err_number)); + } + close(fd); + ASSERT_OK(env_->DeleteFile(fname_test_fallocate)); + if (alloc_status != 0 && err_number == EOPNOTSUPP) { + // The filesystem containing the file does not support fallocate + return; + } + EnvOptions soptions; soptions.use_mmap_writes = false; unique_ptr wfile; From 32f2532a0b18d7ef2bf6e9c1638519a1ab75dd75 Mon Sep 17 00:00:00 2001 From: Mark Callaghan Date: Fri, 19 Sep 2014 13:09:25 -0700 Subject: [PATCH 3/4] Print compression_size_percent as a signed int Summary: compression_size_percent is an int but was printed as an unsigned int. So the default of -1 is displayed as a big number. Test Plan: make check Reviewers: sdong Reviewed By: sdong Subscribers: leveldb Differential Revision: https://reviews.facebook.net/D23679 --- util/options.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/options.cc b/util/options.cc index a61d9d633..f0042cbda 100644 --- a/util/options.cc +++ b/util/options.cc @@ -419,7 +419,7 @@ void ColumnFamilyOptions::Dump(Logger* log) const { "max_size_amplification_percent: %u", compaction_options_universal.max_size_amplification_percent); Log(log, - "Options.compaction_options_universal.compression_size_percent: %u", + "Options.compaction_options_universal.compression_size_percent: %d", compaction_options_universal.compression_size_percent); Log(log, "Options.compaction_options_fifo.max_table_files_size: %" PRIu64, compaction_options_fifo.max_table_files_size); From bfeef94d31293c57690d63ef040e111cea8f0dec Mon Sep 17 00:00:00 2001 From: Ankit Gupta Date: Fri, 19 Sep 2014 16:11:59 -0700 Subject: [PATCH 4/4] Add rate limiter --- java/Makefile | 2 +- java/RocksDBSample.java | 4 +++ .../org/rocksdb/GenericRateLimiterConfig.java | 36 +++++++++++++++++++ java/org/rocksdb/Options.java | 15 ++++++++ java/org/rocksdb/RateLimiterConfig.java | 20 +++++++++++ java/rocksjni/options.cc | 12 +++++++ java/rocksjni/ratelimiterjni.cc | 24 +++++++++++++ 7 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 java/org/rocksdb/GenericRateLimiterConfig.java create mode 100644 java/org/rocksdb/RateLimiterConfig.java create mode 100644 java/rocksjni/ratelimiterjni.cc diff --git a/java/Makefile b/java/Makefile index 47b2afb9e..b2f3674f0 100644 --- a/java/Makefile +++ b/java/Makefile @@ -1,4 +1,4 @@ -NATIVE_JAVA_CLASSES = org.rocksdb.RocksDB org.rocksdb.Options org.rocksdb.WriteBatch org.rocksdb.WriteBatchInternal org.rocksdb.WriteBatchTest org.rocksdb.WriteOptions org.rocksdb.BackupableDB org.rocksdb.BackupableDBOptions org.rocksdb.Statistics org.rocksdb.RocksIterator org.rocksdb.VectorMemTableConfig org.rocksdb.SkipListMemTableConfig org.rocksdb.HashLinkedListMemTableConfig org.rocksdb.HashSkipListMemTableConfig org.rocksdb.PlainTableConfig org.rocksdb.BlockBasedTableConfig org.rocksdb.ReadOptions org.rocksdb.Filter org.rocksdb.BloomFilter org.rocksdb.RestoreOptions org.rocksdb.RestoreBackupableDB org.rocksdb.RocksEnv +NATIVE_JAVA_CLASSES = org.rocksdb.RocksDB org.rocksdb.Options org.rocksdb.WriteBatch org.rocksdb.WriteBatchInternal org.rocksdb.WriteBatchTest org.rocksdb.WriteOptions org.rocksdb.BackupableDB org.rocksdb.BackupableDBOptions org.rocksdb.Statistics org.rocksdb.RocksIterator org.rocksdb.VectorMemTableConfig org.rocksdb.SkipListMemTableConfig org.rocksdb.HashLinkedListMemTableConfig org.rocksdb.HashSkipListMemTableConfig org.rocksdb.PlainTableConfig org.rocksdb.BlockBasedTableConfig org.rocksdb.ReadOptions org.rocksdb.Filter org.rocksdb.BloomFilter org.rocksdb.RestoreOptions org.rocksdb.RestoreBackupableDB org.rocksdb.RocksEnv org.rocksdb.GenericRateLimiterConfig NATIVE_INCLUDE = ./include ROCKSDB_JAR = rocksdbjni.jar diff --git a/java/RocksDBSample.java b/java/RocksDBSample.java index 9ec3d8345..d78a070df 100644 --- a/java/RocksDBSample.java +++ b/java/RocksDBSample.java @@ -75,6 +75,10 @@ public class RocksDBSample { // Plain-Table requires mmap read options.setAllowMmapReads(true); assert(options.tableFactoryName().equals("PlainTable")); + + options.setRateLimiterConfig(new GenericRateLimiterConfig(10000000, + 10000, 10)); + options.setRateLimiterConfig(new GenericRateLimiterConfig(10000000)); BlockBasedTableConfig table_options = new BlockBasedTableConfig(); table_options.setBlockCacheSize(64 * SizeUnit.KB) diff --git a/java/org/rocksdb/GenericRateLimiterConfig.java b/java/org/rocksdb/GenericRateLimiterConfig.java new file mode 100644 index 000000000..78b8b37ec --- /dev/null +++ b/java/org/rocksdb/GenericRateLimiterConfig.java @@ -0,0 +1,36 @@ +// Copyright (c) 2014, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. +package org.rocksdb; + +/** + * Config for rate limiter, which is used to control write rate of flush and + * compaction. + */ +public class GenericRateLimiterConfig extends RateLimiterConfig { + private static final long DEFAULT_REFILL_PERIOD_MICROS = (100 * 1000); + private static final int DEFAULT_FAIRNESS = 10; + + public GenericRateLimiterConfig(long rateBytesPerSecond, + long refillPeriodMicros, int fairness) { + rateBytesPerSecond_ = rateBytesPerSecond; + refillPeriodMicros_ = refillPeriodMicros; + fairness_ = fairness; + } + + public GenericRateLimiterConfig(long rateBytesPerSecond) { + this(rateBytesPerSecond, DEFAULT_REFILL_PERIOD_MICROS, DEFAULT_FAIRNESS); + } + + @Override protected long newRateLimiterHandle() { + return newRateLimiterHandle(rateBytesPerSecond_, refillPeriodMicros_, + fairness_); + } + + private native long newRateLimiterHandle(long rateBytesPerSecond, + long refillPeriodMicros, int fairness); + private final long rateBytesPerSecond_; + private final long refillPeriodMicros_; + private final int fairness_; +} diff --git a/java/org/rocksdb/Options.java b/java/org/rocksdb/Options.java index 922bdbdb0..876a06285 100644 --- a/java/org/rocksdb/Options.java +++ b/java/org/rocksdb/Options.java @@ -1104,6 +1104,19 @@ public class Options extends RocksObject { setMemTableFactory(nativeHandle_, config.newMemTableFactoryHandle()); return this; } + + /** + * Use to control write rate of flush and compaction. Flush has higher + * priority than compaction. Rate limiting is disabled if nullptr. + * Default: nullptr + * + * @param config rate limiter config. + * @return the instance of the current Options. + */ + public Options setRateLimiterConfig(RateLimiterConfig config) { + setRateLimiter(nativeHandle_, config.newRateLimiterHandle()); + return this; + } /** * Returns the name of the current mem table representation. @@ -2192,6 +2205,8 @@ public class Options extends RocksObject { private native long statisticsPtr(long optHandle); private native void setMemTableFactory(long handle, long factoryHandle); + private native void setRateLimiter(long handle, + long rateLimiterHandle); private native String memTableFactoryName(long handle); private native void setTableFactory(long handle, long factoryHandle); diff --git a/java/org/rocksdb/RateLimiterConfig.java b/java/org/rocksdb/RateLimiterConfig.java new file mode 100644 index 000000000..22de65921 --- /dev/null +++ b/java/org/rocksdb/RateLimiterConfig.java @@ -0,0 +1,20 @@ +// Copyright (c) 2014, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. +package org.rocksdb; + +/** + * Config for rate limiter, which is used to control write rate of flush and + * compaction. + */ +public abstract class RateLimiterConfig { + /** + * This function should only be called by Options.setRateLimiter(), + * which will create a c++ shared-pointer to the c++ RateLimiter + * that is associated with the Java RateLimtierConifg. + * + * @see Options.setRateLimiter() + */ + abstract protected long newRateLimiterHandle(); +} diff --git a/java/rocksjni/options.cc b/java/rocksjni/options.cc index a72eecd28..705e9ff8c 100644 --- a/java/rocksjni/options.cc +++ b/java/rocksjni/options.cc @@ -21,6 +21,7 @@ #include "rocksdb/memtablerep.h" #include "rocksdb/table.h" #include "rocksdb/slice_transform.h" +#include "rocksdb/rate_limiter.h" /* * Class: org_rocksdb_Options @@ -459,6 +460,17 @@ void Java_org_rocksdb_Options_setMemTableFactory( reinterpret_cast(jfactory_handle)); } +/* + * Class: org_rocksdb_Options + * Method: setRateLimiter + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setRateLimiter( + JNIEnv* env, jobject jobj, jlong jhandle, jlong jrate_limiter_handle) { + reinterpret_cast(jhandle)->rate_limiter.reset( + reinterpret_cast(jrate_limiter_handle)); +} + /* * Class: org_rocksdb_Options * Method: tableCacheNumshardbits diff --git a/java/rocksjni/ratelimiterjni.cc b/java/rocksjni/ratelimiterjni.cc new file mode 100644 index 000000000..5413978a0 --- /dev/null +++ b/java/rocksjni/ratelimiterjni.cc @@ -0,0 +1,24 @@ +// Copyright (c) 2014, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. +// +// This file implements the "bridge" between Java and C++ for RateLimiter. + +#include "rocksjni/portal.h" +#include "include/org_rocksdb_GenericRateLimiterConfig.h" +#include "rocksdb/rate_limiter.h" + +/* + * Class: org_rocksdb_GenericRateLimiterConfig + * Method: newRateLimiterHandle + * Signature: (JJI)J + */ +jlong Java_org_rocksdb_GenericRateLimiterConfig_newRateLimiterHandle( + JNIEnv* env, jobject jobj, jlong jrate_bytes_per_second, + jlong jrefill_period_micros, jint jfairness) { + return reinterpret_cast(rocksdb::NewGenericRateLimiter( + rocksdb::jlong_to_size_t(jrate_bytes_per_second), + rocksdb::jlong_to_size_t(jrefill_period_micros), + static_cast(jfairness))); +}