Deprecate ttl option from CompactionOptionsFIFO (#4965)

Summary:
We introduced ttl option in CompactionOptionsFIFO when ttl-based file
deletion (compaction) was supported only as part of FIFO Compaction. But
with the extension of ttl semantics even to Level compaction,
CompactionOptionsFIFO.ttl can now be deprecated. Instead we will start
using ColumnFamilyOptions.ttl for FIFO compaction as well.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4965

Differential Revision: D14072960

Pulled By: sagar0

fbshipit-source-id: c98cc2ae695a28136295787cd88d36a220fc219e
main
Aubin Sanyal 6 years ago committed by Facebook Github Bot
parent ca89ac2ba9
commit 3231a2e581
  1. 1
      HISTORY.md
  2. 9
      db/compaction_picker_fifo.cc
  3. 2
      db/db_impl_open.cc
  4. 14
      db/db_options_test.cc
  5. 2
      db/db_properties_test.cc
  6. 39
      db/db_test.cc
  7. 5
      db/version_set.cc
  8. 23
      include/rocksdb/advanced_options.h
  9. 24
      java/rocksjni/compaction_options_fifo.cc
  10. 33
      java/src/main/java/org/rocksdb/CompactionOptionsFIFO.java
  11. 9
      java/src/test/java/org/rocksdb/CompactionOptionsFIFOTest.java
  12. 2
      options/cf_options.cc
  13. 2
      options/options.cc
  14. 6
      options/options_helper.cc
  15. 2
      options/options_parser.cc
  16. 2
      options/options_settable_test.cc
  17. 3
      tools/db_bench_tool.cc
  18. 1
      util/testutil.cc

@ -25,6 +25,7 @@
* Remove PlainTable's store_index_in_file feature. When opening an existing DB with index in SST files, the index and bloom filter will still be rebuild while SST files are opened, in the same way as there is no index in the file. * Remove PlainTable's store_index_in_file feature. When opening an existing DB with index in SST files, the index and bloom filter will still be rebuild while SST files are opened, in the same way as there is no index in the file.
* Remove CuckooHash memtable. * Remove CuckooHash memtable.
* The counter stat `number.block.not_compressed` now also counts blocks not compressed due to poor compression ratio. * The counter stat `number.block.not_compressed` now also counts blocks not compressed due to poor compression ratio.
* Remove ttl option from `CompactionOptionsFIFO`. The option has been deprecated and ttl in `ColumnFamilyOptions` is used instead.
* Support SST file ingestion across multiple column families via DB::IngestExternalFiles. See the function's comment about atomicity. * Support SST file ingestion across multiple column families via DB::IngestExternalFiles. See the function's comment about atomicity.
* Remove Lua compaction filter. * Remove Lua compaction filter.

@ -41,7 +41,7 @@ bool FIFOCompactionPicker::NeedsCompaction(
Compaction* FIFOCompactionPicker::PickTTLCompaction( Compaction* FIFOCompactionPicker::PickTTLCompaction(
const std::string& cf_name, const MutableCFOptions& mutable_cf_options, const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
VersionStorageInfo* vstorage, LogBuffer* log_buffer) { VersionStorageInfo* vstorage, LogBuffer* log_buffer) {
assert(mutable_cf_options.compaction_options_fifo.ttl > 0); assert(mutable_cf_options.ttl > 0);
const int kLevel0 = 0; const int kLevel0 = 0;
const std::vector<FileMetaData*>& level_files = vstorage->LevelFiles(kLevel0); const std::vector<FileMetaData*>& level_files = vstorage->LevelFiles(kLevel0);
@ -63,7 +63,7 @@ Compaction* FIFOCompactionPicker::PickTTLCompaction(
inputs[0].level = 0; inputs[0].level = 0;
// avoid underflow // avoid underflow
if (current_time > mutable_cf_options.compaction_options_fifo.ttl) { if (current_time > mutable_cf_options.ttl) {
for (auto ritr = level_files.rbegin(); ritr != level_files.rend(); ++ritr) { for (auto ritr = level_files.rbegin(); ritr != level_files.rend(); ++ritr) {
auto f = *ritr; auto f = *ritr;
if (f->fd.table_reader != nullptr && if (f->fd.table_reader != nullptr &&
@ -71,8 +71,7 @@ Compaction* FIFOCompactionPicker::PickTTLCompaction(
auto creation_time = auto creation_time =
f->fd.table_reader->GetTableProperties()->creation_time; f->fd.table_reader->GetTableProperties()->creation_time;
if (creation_time == 0 || if (creation_time == 0 ||
creation_time >= (current_time - creation_time >= (current_time - mutable_cf_options.ttl)) {
mutable_cf_options.compaction_options_fifo.ttl)) {
break; break;
} }
total_size -= f->compensated_file_size; total_size -= f->compensated_file_size;
@ -201,7 +200,7 @@ Compaction* FIFOCompactionPicker::PickCompaction(
assert(vstorage->num_levels() == 1); assert(vstorage->num_levels() == 1);
Compaction* c = nullptr; Compaction* c = nullptr;
if (mutable_cf_options.compaction_options_fifo.ttl > 0) { if (mutable_cf_options.ttl > 0) {
c = PickTTLCompaction(cf_name, mutable_cf_options, vstorage, log_buffer); c = PickTTLCompaction(cf_name, mutable_cf_options, vstorage, log_buffer);
} }
if (c == nullptr) { if (c == nullptr) {

@ -176,7 +176,7 @@ static Status ValidateOptions(
return s; return s;
} }
if (cfd.options.ttl > 0 || cfd.options.compaction_options_fifo.ttl > 0) { if (cfd.options.ttl > 0) {
if (db_options.max_open_files != -1) { if (db_options.max_open_files != -1) {
return Status::NotSupported( return Status::NotSupported(
"TTL is only supported when files are always " "TTL is only supported when files are always "

@ -628,9 +628,9 @@ TEST_F(DBOptionsTest, SetFIFOCompactionOptions) {
env_->time_elapse_only_sleep_ = false; env_->time_elapse_only_sleep_ = false;
options.env = env_; options.env = env_;
// Test dynamically changing compaction_options_fifo.ttl // Test dynamically changing ttl.
env_->addon_time_.store(0); env_->addon_time_.store(0);
options.compaction_options_fifo.ttl = 1 * 60 * 60; // 1 hour options.ttl = 1 * 60 * 60; // 1 hour
ASSERT_OK(TryReopen(options)); ASSERT_OK(TryReopen(options));
Random rnd(301); Random rnd(301);
@ -648,13 +648,13 @@ TEST_F(DBOptionsTest, SetFIFOCompactionOptions) {
env_->addon_time_.fetch_add(61); env_->addon_time_.fetch_add(61);
// No files should be compacted as ttl is set to 1 hour. // No files should be compacted as ttl is set to 1 hour.
ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.ttl, 3600); ASSERT_EQ(dbfull()->GetOptions().ttl, 3600);
dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr); dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
ASSERT_EQ(NumTableFilesAtLevel(0), 10); ASSERT_EQ(NumTableFilesAtLevel(0), 10);
// Set ttl to 1 minute. So all files should get deleted. // Set ttl to 1 minute. So all files should get deleted.
ASSERT_OK(dbfull()->SetOptions({{"compaction_options_fifo", "{ttl=60;}"}})); ASSERT_OK(dbfull()->SetOptions({{"ttl", "60"}}));
ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.ttl, 60); ASSERT_EQ(dbfull()->GetOptions().ttl, 60);
dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr); dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
ASSERT_OK(dbfull()->TEST_WaitForCompact()); ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(NumTableFilesAtLevel(0), 0); ASSERT_EQ(NumTableFilesAtLevel(0), 0);
@ -662,7 +662,7 @@ TEST_F(DBOptionsTest, SetFIFOCompactionOptions) {
// Test dynamically changing compaction_options_fifo.max_table_files_size // Test dynamically changing compaction_options_fifo.max_table_files_size
env_->addon_time_.store(0); env_->addon_time_.store(0);
options.compaction_options_fifo.max_table_files_size = 500 << 10; // 00KB options.compaction_options_fifo.max_table_files_size = 500 << 10; // 00KB
options.compaction_options_fifo.ttl = 0; options.ttl = 0;
DestroyAndReopen(options); DestroyAndReopen(options);
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
@ -692,7 +692,7 @@ TEST_F(DBOptionsTest, SetFIFOCompactionOptions) {
// Test dynamically changing compaction_options_fifo.allow_compaction // Test dynamically changing compaction_options_fifo.allow_compaction
options.compaction_options_fifo.max_table_files_size = 500 << 10; // 500KB options.compaction_options_fifo.max_table_files_size = 500 << 10; // 500KB
options.compaction_options_fifo.ttl = 0; options.ttl = 0;
options.compaction_options_fifo.allow_compaction = false; options.compaction_options_fifo.allow_compaction = false;
options.level0_file_num_compaction_trigger = 6; options.level0_file_num_compaction_trigger = 6;
DestroyAndReopen(options); DestroyAndReopen(options);

@ -1432,7 +1432,7 @@ TEST_F(DBPropertiesTest, EstimateOldestKeyTime) {
} }
options.compaction_style = kCompactionStyleFIFO; options.compaction_style = kCompactionStyleFIFO;
options.compaction_options_fifo.ttl = 300; options.ttl = 300;
options.compaction_options_fifo.allow_compaction = false; options.compaction_options_fifo.allow_compaction = false;
DestroyAndReopen(options); DestroyAndReopen(options);

@ -3083,7 +3083,7 @@ TEST_F(DBTest, FIFOCompactionWithTTLAndMaxOpenFilesTest) {
Options options; Options options;
options.compaction_style = kCompactionStyleFIFO; options.compaction_style = kCompactionStyleFIFO;
options.create_if_missing = true; options.create_if_missing = true;
options.compaction_options_fifo.ttl = 600; // seconds options.ttl = 600; // seconds
// Check that it is not supported with max_open_files != -1. // Check that it is not supported with max_open_files != -1.
options.max_open_files = 100; options.max_open_files = 100;
@ -3099,7 +3099,7 @@ TEST_F(DBTest, FIFOCompactionWithTTLAndVariousTableFormatsTest) {
Options options; Options options;
options.compaction_style = kCompactionStyleFIFO; options.compaction_style = kCompactionStyleFIFO;
options.create_if_missing = true; options.create_if_missing = true;
options.compaction_options_fifo.ttl = 600; // seconds options.ttl = 600; // seconds
options = CurrentOptions(options); options = CurrentOptions(options);
options.table_factory.reset(NewBlockBasedTableFactory()); options.table_factory.reset(NewBlockBasedTableFactory());
@ -3130,7 +3130,7 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
env_->addon_time_.store(0); env_->addon_time_.store(0);
options.compaction_options_fifo.max_table_files_size = 150 << 10; // 150KB options.compaction_options_fifo.max_table_files_size = 150 << 10; // 150KB
options.compaction_options_fifo.allow_compaction = false; options.compaction_options_fifo.allow_compaction = false;
options.compaction_options_fifo.ttl = 1 * 60 * 60 ; // 1 hour options.ttl = 1 * 60 * 60 ; // 1 hour
options = CurrentOptions(options); options = CurrentOptions(options);
DestroyAndReopen(options); DestroyAndReopen(options);
@ -3165,7 +3165,7 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
{ {
options.compaction_options_fifo.max_table_files_size = 150 << 10; // 150KB options.compaction_options_fifo.max_table_files_size = 150 << 10; // 150KB
options.compaction_options_fifo.allow_compaction = false; options.compaction_options_fifo.allow_compaction = false;
options.compaction_options_fifo.ttl = 1 * 60 * 60; // 1 hour options.ttl = 1 * 60 * 60; // 1 hour
options = CurrentOptions(options); options = CurrentOptions(options);
DestroyAndReopen(options); DestroyAndReopen(options);
@ -3207,7 +3207,7 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
options.write_buffer_size = 10 << 10; // 10KB options.write_buffer_size = 10 << 10; // 10KB
options.compaction_options_fifo.max_table_files_size = 150 << 10; // 150KB options.compaction_options_fifo.max_table_files_size = 150 << 10; // 150KB
options.compaction_options_fifo.allow_compaction = false; options.compaction_options_fifo.allow_compaction = false;
options.compaction_options_fifo.ttl = 1 * 60 * 60; // 1 hour options.ttl = 1 * 60 * 60; // 1 hour
options = CurrentOptions(options); options = CurrentOptions(options);
DestroyAndReopen(options); DestroyAndReopen(options);
@ -3244,7 +3244,7 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
{ {
options.compaction_options_fifo.max_table_files_size = 150 << 10; // 150KB options.compaction_options_fifo.max_table_files_size = 150 << 10; // 150KB
options.compaction_options_fifo.allow_compaction = true; options.compaction_options_fifo.allow_compaction = true;
options.compaction_options_fifo.ttl = 1 * 60 * 60; // 1 hour options.ttl = 1 * 60 * 60; // 1 hour
options.level0_file_num_compaction_trigger = 6; options.level0_file_num_compaction_trigger = 6;
options = CurrentOptions(options); options = CurrentOptions(options);
DestroyAndReopen(options); DestroyAndReopen(options);
@ -3288,7 +3288,7 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
options.write_buffer_size = 20 << 10; // 20K options.write_buffer_size = 20 << 10; // 20K
options.compaction_options_fifo.max_table_files_size = 1500 << 10; // 1.5MB options.compaction_options_fifo.max_table_files_size = 1500 << 10; // 1.5MB
options.compaction_options_fifo.allow_compaction = true; options.compaction_options_fifo.allow_compaction = true;
options.compaction_options_fifo.ttl = 1 * 60 * 60; // 1 hour options.ttl = 1 * 60 * 60; // 1 hour
options.level0_file_num_compaction_trigger = 6; options.level0_file_num_compaction_trigger = 6;
options = CurrentOptions(options); options = CurrentOptions(options);
DestroyAndReopen(options); DestroyAndReopen(options);
@ -4587,7 +4587,7 @@ TEST_F(DBTest, DynamicCompactionOptions) {
ASSERT_LT(NumTableFilesAtLevel(0), 4); ASSERT_LT(NumTableFilesAtLevel(0), 4);
} }
// Test dynamic FIFO copmaction options. // Test dynamic FIFO compaction options.
// This test covers just option parsing and makes sure that the options are // This test covers just option parsing and makes sure that the options are
// correctly assigned. Also look at DBOptionsTest.SetFIFOCompactionOptions // correctly assigned. Also look at DBOptionsTest.SetFIFOCompactionOptions
// test which makes sure that the FIFO compaction funcionality is working // test which makes sure that the FIFO compaction funcionality is working
@ -4601,7 +4601,7 @@ TEST_F(DBTest, DynamicFIFOCompactionOptions) {
// Initial defaults // Initial defaults
ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size,
1024 * 1024 * 1024); 1024 * 1024 * 1024);
ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.ttl, 0); ASSERT_EQ(dbfull()->GetOptions().ttl, 0);
ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction,
false); false);
@ -4609,21 +4609,21 @@ TEST_F(DBTest, DynamicFIFOCompactionOptions) {
{{"compaction_options_fifo", "{max_table_files_size=23;}"}})); {{"compaction_options_fifo", "{max_table_files_size=23;}"}}));
ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size,
23); 23);
ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.ttl, 0); ASSERT_EQ(dbfull()->GetOptions().ttl, 0);
ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction,
false); false);
ASSERT_OK(dbfull()->SetOptions({{"compaction_options_fifo", "{ttl=97}"}})); ASSERT_OK(dbfull()->SetOptions({{"ttl", "97"}}));
ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size,
23); 23);
ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.ttl, 97); ASSERT_EQ(dbfull()->GetOptions().ttl, 97);
ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction,
false); false);
ASSERT_OK(dbfull()->SetOptions({{"compaction_options_fifo", "{ttl=203;}"}})); ASSERT_OK(dbfull()->SetOptions({{"ttl", "203"}}));
ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size,
23); 23);
ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.ttl, 203); ASSERT_EQ(dbfull()->GetOptions().ttl, 203);
ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction,
false); false);
@ -4631,24 +4631,25 @@ TEST_F(DBTest, DynamicFIFOCompactionOptions) {
{{"compaction_options_fifo", "{allow_compaction=true;}"}})); {{"compaction_options_fifo", "{allow_compaction=true;}"}}));
ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size,
23); 23);
ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.ttl, 203); ASSERT_EQ(dbfull()->GetOptions().ttl, 203);
ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction,
true); true);
ASSERT_OK(dbfull()->SetOptions( ASSERT_OK(dbfull()->SetOptions(
{{"compaction_options_fifo", "{max_table_files_size=31;ttl=19;}"}})); {{"compaction_options_fifo", "{max_table_files_size=31;}"}}));
ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size,
31); 31);
ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.ttl, 19); ASSERT_EQ(dbfull()->GetOptions().ttl, 203);
ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction,
true); true);
ASSERT_OK(dbfull()->SetOptions( ASSERT_OK(dbfull()->SetOptions(
{{"compaction_options_fifo", {{"compaction_options_fifo",
"{max_table_files_size=51;ttl=49;allow_compaction=true;}"}})); "{max_table_files_size=51;allow_compaction=true;}"}}));
ASSERT_OK(dbfull()->SetOptions({{"ttl", "49"}}));
ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size,
51); 51);
ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.ttl, 49); ASSERT_EQ(dbfull()->GetOptions().ttl, 49);
ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction,
true); true);
} }

@ -1597,8 +1597,7 @@ uint32_t GetExpiredTtlFilesCount(const ImmutableCFOptions& ioptions,
auto creation_time = auto creation_time =
f->fd.table_reader->GetTableProperties()->creation_time; f->fd.table_reader->GetTableProperties()->creation_time;
if (creation_time > 0 && if (creation_time > 0 &&
creation_time < (current_time - creation_time < (current_time - mutable_cf_options.ttl)) {
mutable_cf_options.compaction_options_fifo.ttl)) {
ttl_expired_files_count++; ttl_expired_files_count++;
} }
} }
@ -1653,7 +1652,7 @@ void VersionStorageInfo::ComputeCompactionScore(
mutable_cf_options.level0_file_num_compaction_trigger, mutable_cf_options.level0_file_num_compaction_trigger,
score); score);
} }
if (mutable_cf_options.compaction_options_fifo.ttl > 0) { if (mutable_cf_options.ttl > 0) {
score = std::max( score = std::max(
static_cast<double>(GetExpiredTtlFilesCount( static_cast<double>(GetExpiredTtlFilesCount(
immutable_cf_options, mutable_cf_options, files_[level])), immutable_cf_options, mutable_cf_options, files_[level])),

@ -62,13 +62,6 @@ struct CompactionOptionsFIFO {
// Default: 1GB // Default: 1GB
uint64_t max_table_files_size; uint64_t max_table_files_size;
// Drop files older than TTL. TTL based deletion will take precedence over
// size based deletion if ttl > 0.
// delete if sst_file_creation_time < (current_time - ttl)
// unit: seconds. Ex: 1 day = 1 * 24 * 60 * 60
// Default: 0 (disabled)
uint64_t ttl = 0;
// If true, try to do compaction to compact smaller files into larger ones. // If true, try to do compaction to compact smaller files into larger ones.
// Minimum files to compact follows options.level0_file_num_compaction_trigger // Minimum files to compact follows options.level0_file_num_compaction_trigger
// and compaction won't trigger if average compact bytes per del file is // and compaction won't trigger if average compact bytes per del file is
@ -78,10 +71,8 @@ struct CompactionOptionsFIFO {
bool allow_compaction = false; bool allow_compaction = false;
CompactionOptionsFIFO() : max_table_files_size(1 * 1024 * 1024 * 1024) {} CompactionOptionsFIFO() : max_table_files_size(1 * 1024 * 1024 * 1024) {}
CompactionOptionsFIFO(uint64_t _max_table_files_size, bool _allow_compaction, CompactionOptionsFIFO(uint64_t _max_table_files_size, bool _allow_compaction)
uint64_t _ttl = 0)
: max_table_files_size(_max_table_files_size), : max_table_files_size(_max_table_files_size),
ttl(_ttl),
allow_compaction(_allow_compaction) {} allow_compaction(_allow_compaction) {}
}; };
@ -542,7 +533,7 @@ struct AdvancedColumnFamilyOptions {
// //
// Dynamically changeable through SetOptions() API // Dynamically changeable through SetOptions() API
// Dynamic change example: // Dynamic change example:
// SetOptions("compaction_options_fifo", "{max_table_files_size=100;ttl=2;}") // SetOptions("compaction_options_fifo", "{max_table_files_size=100;}")
CompactionOptionsFIFO compaction_options_fifo; CompactionOptionsFIFO compaction_options_fifo;
// An iteration->Next() sequentially skips over keys with the same // An iteration->Next() sequentially skips over keys with the same
@ -631,9 +622,13 @@ struct AdvancedColumnFamilyOptions {
// Dynamically changeable through SetOptions() API // Dynamically changeable through SetOptions() API
bool report_bg_io_stats = false; bool report_bg_io_stats = false;
// Non-bottom-level files older than TTL will go through the compaction // Files older than TTL will go through the compaction process.
// process. This needs max_open_files to be set to -1. // Supported in Level and FIFO compaction.
// Enabled only for level compaction for now. // Pre-req: This needs max_open_files to be set to -1.
// In Level: Non-bottom-level files older than TTL will go through the
// compation process.
// In FIFO: Files older than TTL will be deleted.
// unit: seconds. Ex: 1 day = 1 * 24 * 60 * 60
// //
// Default: 0 (disabled) // Default: 0 (disabled)
// //

@ -46,30 +46,6 @@ jlong Java_org_rocksdb_CompactionOptionsFIFO_maxTableFilesSize(JNIEnv* /*env*/,
return static_cast<jlong>(opt->max_table_files_size); return static_cast<jlong>(opt->max_table_files_size);
} }
/*
* Class: org_rocksdb_CompactionOptionsFIFO
* Method: setTtl
* Signature: (JJ)V
*/
void Java_org_rocksdb_CompactionOptionsFIFO_setTtl(JNIEnv* /*env*/,
jobject /*jobj*/,
jlong jhandle, jlong ttl) {
auto* opt = reinterpret_cast<rocksdb::CompactionOptionsFIFO*>(jhandle);
opt->ttl = static_cast<uint64_t>(ttl);
}
/*
* Class: org_rocksdb_CompactionOptionsFIFO
* Method: ttl
* Signature: (J)J
*/
jlong Java_org_rocksdb_CompactionOptionsFIFO_ttl(JNIEnv* /*env*/,
jobject /*jobj*/,
jlong jhandle) {
auto* opt = reinterpret_cast<rocksdb::CompactionOptionsFIFO*>(jhandle);
return static_cast<jlong>(opt->ttl);
}
/* /*
* Class: org_rocksdb_CompactionOptionsFIFO * Class: org_rocksdb_CompactionOptionsFIFO
* Method: setAllowCompaction * Method: setAllowCompaction

@ -42,37 +42,6 @@ public class CompactionOptionsFIFO extends RocksObject {
return maxTableFilesSize(nativeHandle_); return maxTableFilesSize(nativeHandle_);
} }
/**
* Drop files older than TTL. TTL based deletion will take precedence over
* size based deletion if ttl &gt; 0.
* delete if sst_file_creation_time &lt; (current_time - ttl).
* unit: seconds. Ex: 1 day = 1 * 24 * 60 * 60
*
* Default: 0 (disabled)
*
* @param ttl The ttl for the table files in seconds
*
* @return the reference to the current options.
*/
public CompactionOptionsFIFO setTtl(final long ttl) {
setTtl(nativeHandle_, ttl);
return this;
}
/**
* The current ttl value.
* Drop files older than TTL. TTL based deletion will take precedence over
* size based deletion if ttl &gt; 0.
* delete if sst_file_creation_time &lt; (current_time - ttl).
*
* Default: 0 (disabled)
*
* @return the ttl in seconds
*/
public long ttl() {
return ttl(nativeHandle_);
}
/** /**
* If true, try to do compaction to compact smaller files into larger ones. * If true, try to do compaction to compact smaller files into larger ones.
* Minimum files to compact follows options.level0_file_num_compaction_trigger * Minimum files to compact follows options.level0_file_num_compaction_trigger
@ -109,8 +78,6 @@ public class CompactionOptionsFIFO extends RocksObject {
private native void setMaxTableFilesSize(long handle, long maxTableFilesSize); private native void setMaxTableFilesSize(long handle, long maxTableFilesSize);
private native long maxTableFilesSize(long handle); private native long maxTableFilesSize(long handle);
private native void setTtl(long handle, long ttl);
private native long ttl(long handle);
private native void setAllowCompaction(long handle, boolean allowCompaction); private native void setAllowCompaction(long handle, boolean allowCompaction);
private native boolean allowCompaction(long handle); private native boolean allowCompaction(long handle);

@ -24,15 +24,6 @@ public class CompactionOptionsFIFOTest {
} }
} }
@Test
public void ttl() {
final long ttl = 7 * 24 * 60 * 60; // 7 days
try (final CompactionOptionsFIFO opt = new CompactionOptionsFIFO()) {
opt.setTtl(ttl);
assertThat(opt.ttl()).isEqualTo(ttl);
}
}
@Test @Test
public void allowCompaction() { public void allowCompaction() {
final boolean allowCompaction = true; final boolean allowCompaction = true;

@ -216,8 +216,6 @@ void MutableCFOptions::Dump(Logger* log) const {
// FIFO Compaction Options // FIFO Compaction Options
ROCKS_LOG_INFO(log, "compaction_options_fifo.max_table_files_size : %" PRIu64, ROCKS_LOG_INFO(log, "compaction_options_fifo.max_table_files_size : %" PRIu64,
compaction_options_fifo.max_table_files_size); compaction_options_fifo.max_table_files_size);
ROCKS_LOG_INFO(log, "compaction_options_fifo.ttl : %" PRIu64,
compaction_options_fifo.ttl);
ROCKS_LOG_INFO(log, "compaction_options_fifo.allow_compaction : %d", ROCKS_LOG_INFO(log, "compaction_options_fifo.allow_compaction : %d",
compaction_options_fifo.allow_compaction); compaction_options_fifo.allow_compaction);
} }

@ -306,8 +306,6 @@ void ColumnFamilyOptions::Dump(Logger* log) const {
ROCKS_LOG_HEADER(log, ROCKS_LOG_HEADER(log,
"Options.compaction_options_fifo.allow_compaction: %d", "Options.compaction_options_fifo.allow_compaction: %d",
compaction_options_fifo.allow_compaction); compaction_options_fifo.allow_compaction);
ROCKS_LOG_HEADER(log, "Options.compaction_options_fifo.ttl: %" PRIu64,
compaction_options_fifo.ttl);
std::string collector_names; std::string collector_names;
for (const auto& collector_factory : table_properties_collector_factories) { for (const auto& collector_factory : table_properties_collector_factories) {
collector_names.append(collector_factory->Name()); collector_names.append(collector_factory->Name());

@ -1902,9 +1902,9 @@ std::unordered_map<std::string, OptionTypeInfo>
OptionType::kUInt64T, OptionVerificationType::kNormal, true, OptionType::kUInt64T, OptionVerificationType::kNormal, true,
offsetof(struct CompactionOptionsFIFO, max_table_files_size)}}, offsetof(struct CompactionOptionsFIFO, max_table_files_size)}},
{"ttl", {"ttl",
{offset_of(&CompactionOptionsFIFO::ttl), OptionType::kUInt64T, {0, OptionType::kUInt64T,
OptionVerificationType::kNormal, true, OptionVerificationType::kDeprecated, false,
offsetof(struct CompactionOptionsFIFO, ttl)}}, 0}},
{"allow_compaction", {"allow_compaction",
{offset_of(&CompactionOptionsFIFO::allow_compaction), {offset_of(&CompactionOptionsFIFO::allow_compaction),
OptionType::kBoolean, OptionVerificationType::kNormal, true, OptionType::kBoolean, OptionVerificationType::kNormal, true,

@ -574,7 +574,7 @@ bool AreEqualOptions(
CompactionOptionsFIFO rhs = CompactionOptionsFIFO rhs =
*reinterpret_cast<const CompactionOptionsFIFO*>(offset2); *reinterpret_cast<const CompactionOptionsFIFO*>(offset2);
if (lhs.max_table_files_size == rhs.max_table_files_size && if (lhs.max_table_files_size == rhs.max_table_files_size &&
lhs.ttl == rhs.ttl && lhs.allow_compaction == rhs.allow_compaction) { lhs.allow_compaction == rhs.allow_compaction) {
return true; return true;
} }
return false; return false;

@ -448,7 +448,7 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) {
"disable_auto_compactions=false;" "disable_auto_compactions=false;"
"report_bg_io_stats=true;" "report_bg_io_stats=true;"
"ttl=60;" "ttl=60;"
"compaction_options_fifo={max_table_files_size=3;ttl=100;allow_" "compaction_options_fifo={max_table_files_size=3;allow_"
"compaction=false;};", "compaction=false;};",
new_options)); new_options));

@ -3221,9 +3221,10 @@ void VerifyDBFromDB(std::string& truth_db_name) {
options.use_direct_io_for_flush_and_compaction = options.use_direct_io_for_flush_and_compaction =
FLAGS_use_direct_io_for_flush_and_compaction; FLAGS_use_direct_io_for_flush_and_compaction;
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
options.ttl = FLAGS_fifo_compaction_ttl;
options.compaction_options_fifo = CompactionOptionsFIFO( options.compaction_options_fifo = CompactionOptionsFIFO(
FLAGS_fifo_compaction_max_table_files_size_mb * 1024 * 1024, FLAGS_fifo_compaction_max_table_files_size_mb * 1024 * 1024,
FLAGS_fifo_compaction_allow_compaction, FLAGS_fifo_compaction_ttl); FLAGS_fifo_compaction_allow_compaction);
#endif // ROCKSDB_LITE #endif // ROCKSDB_LITE
if (FLAGS_prefix_size != 0) { if (FLAGS_prefix_size != 0) {
options.prefix_extractor.reset( options.prefix_extractor.reset(

@ -351,7 +351,6 @@ void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, Random* rnd) {
cf_opt->target_file_size_base * rnd->Uniform(100); cf_opt->target_file_size_base * rnd->Uniform(100);
cf_opt->compaction_options_fifo.max_table_files_size = cf_opt->compaction_options_fifo.max_table_files_size =
uint_max + rnd->Uniform(10000); uint_max + rnd->Uniform(10000);
cf_opt->compaction_options_fifo.ttl = uint_max + rnd->Uniform(10000);
// unsigned int options // unsigned int options
cf_opt->rate_limit_delay_max_milliseconds = rnd->Uniform(10000); cf_opt->rate_limit_delay_max_milliseconds = rnd->Uniform(10000);

Loading…
Cancel
Save