diff --git a/HISTORY.md b/HISTORY.md index 1ff833b67..63254df95 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,6 +1,7 @@ # Rocksdb Change Log ## Unreleased ### Public API Change +* Changed the default value of periodic_compaction_seconds to `UINT64_MAX` which allows RocksDB to auto-tune periodic compaction scheduling. When using the default value, periodic compactions are now auto-enabled if a compaction filter is used. A value of `0` will turn off the feature completely. * Added an API GetCreationTimeOfOldestFile(uint64_t* creation_time) to get the file_creation_time of the oldest SST file in the DB. diff --git a/db/column_family.cc b/db/column_family.cc index 16688d6ce..f0360eefe 100644 --- a/db/column_family.cc +++ b/db/column_family.cc @@ -30,6 +30,7 @@ #include "memtable/hash_skiplist_rep.h" #include "monitoring/thread_status_util.h" #include "options/options_helper.h" +#include "port/port.h" #include "table/block_based/block_based_table_factory.h" #include "table/merging_iterator.h" #include "util/autovector.h" @@ -342,6 +343,16 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options, result.max_compaction_bytes = result.target_file_size_base * 25; } + // Turn on periodic compactions and set them to occur once every 30 days if + // compaction filters are used and periodic_compaction_seconds is set to the + // default value. + if (result.compaction_style == kCompactionStyleLevel && + (result.compaction_filter != nullptr || + result.compaction_filter_factory != nullptr) && + result.periodic_compaction_seconds == port::kMaxUint64) { + result.periodic_compaction_seconds = 30 * 24 * 60 * 60; + } + return result; } @@ -1180,12 +1191,8 @@ Status ColumnFamilyData::ValidateOptions( } } - if (cf_options.periodic_compaction_seconds > 0) { - if (db_options.max_open_files != -1) { - return Status::NotSupported( - "Periodic Compaction is only supported when files are always " - "kept open (set max_open_files = -1). "); - } + if (cf_options.periodic_compaction_seconds > 0 && + cf_options.periodic_compaction_seconds < port::kMaxUint64) { if (cf_options.table_factory->Name() != BlockBasedTableFactory().Name()) { return Status::NotSupported( "Periodic Compaction is only supported in " diff --git a/db/db_compaction_test.cc b/db/db_compaction_test.cc index dad19921c..bf301d983 100644 --- a/db/db_compaction_test.cc +++ b/db/db_compaction_test.cc @@ -3748,6 +3748,91 @@ TEST_F(DBCompactionTest, LevelPeriodicAndTtlCompaction) { rocksdb::SyncPoint::GetInstance()->DisableProcessing(); } +TEST_F(DBCompactionTest, LevelPeriodicCompactionWithCompactionFilters) { + class TestCompactionFilter : public CompactionFilter { + const char* Name() const override { return "TestCompactionFilter"; } + }; + class TestCompactionFilterFactory : public CompactionFilterFactory { + const char* Name() const override { return "TestCompactionFilterFactory"; } + std::unique_ptr CreateCompactionFilter( + const CompactionFilter::Context& /*context*/) override { + return std::unique_ptr(new TestCompactionFilter()); + } + }; + + const int kNumKeysPerFile = 32; + const int kNumLevelFiles = 2; + const int kValueSize = 100; + + Random rnd(301); + + Options options = CurrentOptions(); + TestCompactionFilter test_compaction_filter; + env_->time_elapse_only_sleep_ = false; + options.env = env_; + env_->addon_time_.store(0); + + enum CompactionFilterType { + kUseCompactionFilter, + kUseCompactionFilterFactory + }; + + for (CompactionFilterType comp_filter_type : + {kUseCompactionFilter, kUseCompactionFilterFactory}) { + // Assert that periodic compactions are not enabled. + ASSERT_EQ(port::kMaxUint64, options.periodic_compaction_seconds); + + if (comp_filter_type == kUseCompactionFilter) { + options.compaction_filter = &test_compaction_filter; + options.compaction_filter_factory.reset(); + } else if (comp_filter_type == kUseCompactionFilterFactory) { + options.compaction_filter = nullptr; + options.compaction_filter_factory.reset( + new TestCompactionFilterFactory()); + } + DestroyAndReopen(options); + + // periodic_compaction_seconds should be set to the sanitized value when + // a compaction filter or a compaction filter factory is used. + ASSERT_EQ(30 * 24 * 60 * 60, + dbfull()->GetOptions().periodic_compaction_seconds); + + int periodic_compactions = 0; + rocksdb::SyncPoint::GetInstance()->SetCallBack( + "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) { + Compaction* compaction = reinterpret_cast(arg); + auto compaction_reason = compaction->compaction_reason(); + if (compaction_reason == CompactionReason::kPeriodicCompaction) { + periodic_compactions++; + } + }); + rocksdb::SyncPoint::GetInstance()->EnableProcessing(); + + for (int i = 0; i < kNumLevelFiles; ++i) { + for (int j = 0; j < kNumKeysPerFile; ++j) { + ASSERT_OK( + Put(Key(i * kNumKeysPerFile + j), RandomString(&rnd, kValueSize))); + } + Flush(); + } + dbfull()->TEST_WaitForCompact(); + + ASSERT_EQ("2", FilesPerLevel()); + ASSERT_EQ(0, periodic_compactions); + + // Add 31 days and do a write + env_->addon_time_.fetch_add(31 * 24 * 60 * 60); + ASSERT_OK(Put("a", "1")); + Flush(); + dbfull()->TEST_WaitForCompact(); + // Assert that the files stay in the same level + ASSERT_EQ("3", FilesPerLevel()); + // The two old files go through the periodic compaction process + ASSERT_EQ(2, periodic_compactions); + + rocksdb::SyncPoint::GetInstance()->DisableProcessing(); + } +} TEST_F(DBCompactionTest, CompactRangeDelayedByL0FileCount) { // Verify that, when `CompactRangeOptions::allow_write_stall == false`, manual diff --git a/db/version_set.cc b/db/version_set.cc index 11264205a..61d140a6f 100644 --- a/db/version_set.cc +++ b/db/version_set.cc @@ -2445,7 +2445,8 @@ void VersionStorageInfo::ComputeCompactionScore( if (mutable_cf_options.ttl > 0) { ComputeExpiredTtlFiles(immutable_cf_options, mutable_cf_options.ttl); } - if (mutable_cf_options.periodic_compaction_seconds > 0) { + if (mutable_cf_options.periodic_compaction_seconds > 0 && + mutable_cf_options.periodic_compaction_seconds < port::kMaxUint64) { ComputeFilesMarkedForPeriodicCompaction( immutable_cf_options, mutable_cf_options.periodic_compaction_seconds); } @@ -2505,7 +2506,8 @@ void VersionStorageInfo::ComputeExpiredTtlFiles( void VersionStorageInfo::ComputeFilesMarkedForPeriodicCompaction( const ImmutableCFOptions& ioptions, const uint64_t periodic_compaction_seconds) { - assert(periodic_compaction_seconds > 0); + assert(periodic_compaction_seconds > 0 && + periodic_compaction_seconds < port::kMaxUint64); files_marked_for_periodic_compaction_.clear(); @@ -2515,6 +2517,13 @@ void VersionStorageInfo::ComputeFilesMarkedForPeriodicCompaction( return; } const uint64_t current_time = static_cast(temp_current_time); + + assert(periodic_compaction_seconds <= current_time); + // Disable periodic compaction if periodic_compaction_seconds > current_time. + // This also help handle the underflow case. + if (periodic_compaction_seconds > current_time) { + return; + } const uint64_t allowed_time_limit = current_time - periodic_compaction_seconds; diff --git a/include/rocksdb/advanced_options.h b/include/rocksdb/advanced_options.h index 2964491f7..77c55d977 100644 --- a/include/rocksdb/advanced_options.h +++ b/include/rocksdb/advanced_options.h @@ -670,10 +670,18 @@ struct AdvancedColumnFamilyOptions { // Only supported in Level compaction. // Pre-req: max_open_file == -1. // unit: seconds. Ex: 7 days = 7 * 24 * 60 * 60 - // Default: 0 (disabled) + // + // Values: + // 0: Turn off Periodic compactions. + // UINT64_MAX (i.e 0xffffffffffffffff): Let RocksDB control this feature + // as needed. For now, RocksDB will change this value to 30 days + // (i.e 30 * 24 * 60 * 60) so that every file goes through the compaction + // process at least once every 30 days if not compacted sooner. + // + // Default: UINT64_MAX (allow RocksDB to auto-tune) // // Dynamically changeable through SetOptions() API - uint64_t periodic_compaction_seconds = 0; + uint64_t periodic_compaction_seconds = 0xffffffffffffffff; // If this option is set then 1 in N blocks are compressed // using a fast (lz4) and slow (zstd) compression algorithm. diff --git a/utilities/blob_db/blob_db_impl.cc b/utilities/blob_db/blob_db_impl.cc index b7338f230..d3a4d8e4a 100644 --- a/utilities/blob_db/blob_db_impl.cc +++ b/utilities/blob_db/blob_db_impl.cc @@ -137,6 +137,9 @@ Status BlobDBImpl::Open(std::vector* handles) { cf_options_.compaction_filter_factory != nullptr) { return Status::NotSupported("Blob DB doesn't support compaction filter."); } + // BlobDB does not support Periodic Compactions. So disable periodic + // compactions irrespective of the user set value. + cf_options_.periodic_compaction_seconds = 0; Status s;