From 543c12ab06576ab46b00901f65aab96fe6cb0fbf Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 7 Oct 2015 21:06:28 -0400 Subject: [PATCH] options: add recycle_log_file_num option Signed-off-by: Sage Weil --- db/c.cc | 5 ++++ include/rocksdb/c.h | 2 ++ include/rocksdb/options.h | 10 ++++++++ java/rocksjni/options.cc | 53 +++++++++++++++++++++++++++++++++++++++ util/options.cc | 4 +++ util/options_helper.h | 3 +++ util/options_test.cc | 6 ++++- 7 files changed, 82 insertions(+), 1 deletion(-) diff --git a/db/c.cc b/db/c.cc index 8cd08265e..ec8e4fb99 100644 --- a/db/c.cc +++ b/db/c.cc @@ -1687,6 +1687,11 @@ void rocksdb_options_set_keep_log_file_num(rocksdb_options_t* opt, size_t v) { opt->rep.keep_log_file_num = v; } +void rocksdb_options_set_recycle_log_file_num(rocksdb_options_t* opt, + size_t v) { + opt->rep.recycle_log_file_num = v; +} + void rocksdb_options_set_soft_rate_limit(rocksdb_options_t* opt, double v) { opt->rep.soft_rate_limit = v; } diff --git a/include/rocksdb/c.h b/include/rocksdb/c.h index 782d10b48..76c801e67 100644 --- a/include/rocksdb/c.h +++ b/include/rocksdb/c.h @@ -572,6 +572,8 @@ extern ROCKSDB_LIBRARY_API void rocksdb_options_set_log_file_time_to_roll( rocksdb_options_t*, size_t); extern ROCKSDB_LIBRARY_API void rocksdb_options_set_keep_log_file_num( rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_recycle_log_file_num( + rocksdb_options_t*, size_t); extern ROCKSDB_LIBRARY_API void rocksdb_options_set_soft_rate_limit( rocksdb_options_t*, double); extern ROCKSDB_LIBRARY_API void rocksdb_options_set_hard_rate_limit( diff --git a/include/rocksdb/options.h b/include/rocksdb/options.h index 16aa3782b..a9830221a 100644 --- a/include/rocksdb/options.h +++ b/include/rocksdb/options.h @@ -957,6 +957,16 @@ struct DBOptions { // Default: 1000 size_t keep_log_file_num; + // Recycle log files. + // If non-zero, we will reuse previously written log files for new + // logs, overwriting the old data. The value indicates how many + // such files we will keep around at any point in time for later + // use. This is more efficient because the blocks are already + // allocated and fdatasync does not need to update the inode after + // each write. + // Default: 0 + size_t recycle_log_file_num; + // manifest file is rolled over on reaching this limit. // The older manifest file be deleted. // The default value is MAX_INT so that roll-over does not take place. diff --git a/java/rocksjni/options.cc b/java/rocksjni/options.cc index 216fa5e8a..de3df942c 100644 --- a/java/rocksjni/options.cc +++ b/java/rocksjni/options.cc @@ -576,6 +576,33 @@ void Java_org_rocksdb_Options_setKeepLogFileNum( } } +/* + * Class: org_rocksdb_Options + * Method: recycleLogFiles + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_recycleLogFileNum(JNIEnv* env, jobject jobj, + jlong jhandle) { + return reinterpret_cast(jhandle)->recycle_log_file_num; +} + +/* + * Class: org_rocksdb_Options + * Method: setRecycleLogFiles + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setRecycleLogFiles(JNIEnv* env, jobject jobj, + jlong jhandle, + jlong recycle_log_file_num) { + rocksdb::Status s = rocksdb::check_if_jlong_fits_size_t(recycle_log_file_num); + if (s.ok()) { + reinterpret_cast(jhandle)->recycle_log_file_num = + recycle_log_file_num; + } else { + rocksdb::IllegalArgumentExceptionJni::ThrowNew(env, s); + } +} + /* * Class: org_rocksdb_Options * Method: maxManifestFileSize @@ -3533,6 +3560,32 @@ jlong Java_org_rocksdb_DBOptions_keepLogFileNum( return reinterpret_cast(jhandle)->keep_log_file_num; } +/* + * Class: org_rocksdb_DBOptions + * Method: setRecycleLogFiles + * Signature: (JJ)V + */ +void Java_org_rocksdb_DBOptions_setRecycleLogFileNum( + JNIEnv* env, jobject jobj, jlong jhandle, jlong recycle_log_file_num) { + rocksdb::Status s = rocksdb::check_if_jlong_fits_size_t(recycle_log_file_num); + if (s.ok()) { + reinterpret_cast(jhandle)->recycle_log_file_num = + recycle_log_file_num; + } else { + rocksdb::IllegalArgumentExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_DBOptions + * Method: recycleLogFiles + * Signature: (J)J + */ +jlong Java_org_rocksdb_DBOptions_recycleLogFileNum(JNIEnv* env, jobject jobj, + jlong jhandle) { + return reinterpret_cast(jhandle)->recycle_log_file_num; +} + /* * Class: org_rocksdb_DBOptions * Method: setMaxManifestFileSize diff --git a/util/options.cc b/util/options.cc index 14b69e678..70bcf7063 100644 --- a/util/options.cc +++ b/util/options.cc @@ -231,6 +231,7 @@ DBOptions::DBOptions() max_log_file_size(0), log_file_time_to_roll(0), keep_log_file_num(1000), + recycle_log_file_num(0), max_manifest_file_size(std::numeric_limits::max()), table_cache_numshardbits(4), WAL_ttl_seconds(0), @@ -285,6 +286,7 @@ DBOptions::DBOptions(const Options& options) max_log_file_size(options.max_log_file_size), log_file_time_to_roll(options.log_file_time_to_roll), keep_log_file_num(options.keep_log_file_num), + recycle_log_file_num(options.recycle_log_file_num), max_manifest_file_size(options.max_manifest_file_size), table_cache_numshardbits(options.table_cache_numshardbits), WAL_ttl_seconds(options.WAL_ttl_seconds), @@ -338,6 +340,8 @@ void DBOptions::Dump(Logger* log) const { log_file_time_to_roll); Header(log, " Options.keep_log_file_num: %" ROCKSDB_PRIszt, keep_log_file_num); + Header(log, " Options.recycle_log_file_num: %" ROCKSDB_PRIszt, + recycle_log_file_num); Header(log, " Options.allow_os_buffer: %d", allow_os_buffer); Header(log, " Options.allow_mmap_reads: %d", allow_mmap_reads); Header(log, " Options.allow_fallocate: %d", allow_fallocate); diff --git a/util/options_helper.h b/util/options_helper.h index ec567bc7c..3f6eab40a 100644 --- a/util/options_helper.h +++ b/util/options_helper.h @@ -207,6 +207,9 @@ static std::unordered_map db_options_type_info = { {"keep_log_file_num", {offsetof(struct DBOptions, keep_log_file_num), OptionType::kSizeT, OptionVerificationType::kNormal}}, + {"recycle_log_file_num", + {offsetof(struct DBOptions, recycle_log_file_num), OptionType::kSizeT, + OptionVerificationType::kNormal}}, {"log_file_time_to_roll", {offsetof(struct DBOptions, log_file_time_to_roll), OptionType::kSizeT, OptionVerificationType::kNormal}}, diff --git a/util/options_test.cc b/util/options_test.cc index 48f56cea3..e1849bb3f 100644 --- a/util/options_test.cc +++ b/util/options_test.cc @@ -323,6 +323,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) { {"max_log_file_size", "37"}, {"log_file_time_to_roll", "38"}, {"keep_log_file_num", "39"}, + {"recycle_log_file_num", "5"}, {"max_manifest_file_size", "40"}, {"table_cache_numshardbits", "41"}, {"WAL_ttl_seconds", "43"}, @@ -339,7 +340,8 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) { {"new_table_reader_for_compaction_inputs", "true"}, {"compaction_readahead_size", "100"}, {"bytes_per_sync", "47"}, - {"wal_bytes_per_sync", "48"}, }; + {"wal_bytes_per_sync", "48"}, + }; ColumnFamilyOptions base_cf_opt; ColumnFamilyOptions new_cf_opt; @@ -431,6 +433,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) { ASSERT_EQ(new_db_opt.max_log_file_size, 37U); ASSERT_EQ(new_db_opt.log_file_time_to_roll, 38U); ASSERT_EQ(new_db_opt.keep_log_file_num, 39U); + ASSERT_EQ(new_db_opt.recycle_log_file_num, 5U); ASSERT_EQ(new_db_opt.max_manifest_file_size, static_cast(40)); ASSERT_EQ(new_db_opt.table_cache_numshardbits, 41); ASSERT_EQ(new_db_opt.WAL_ttl_seconds, static_cast(43)); @@ -692,6 +695,7 @@ void RandomInitDBOptions(DBOptions* db_opt, Random* rnd) { db_opt->skip_stats_update_on_db_open = rnd->Uniform(2); db_opt->use_adaptive_mutex = rnd->Uniform(2); db_opt->use_fsync = rnd->Uniform(2); + db_opt->recycle_log_file_num = rnd->Uniform(2); // int options db_opt->max_background_compactions = rnd->Uniform(100);