From 72712f4e2840ae6a4c819d76daacb58432ba2041 Mon Sep 17 00:00:00 2001 From: Anand Ananthabhotla Date: Thu, 20 Sep 2018 15:08:20 -0700 Subject: [PATCH] Allow dynamic modification of window size and deletion trigger (#4403) Summary: Make the CompactOnDeletionCollectorFactory class public, and provide methods to update the window size and deletion trigger params. These will take effect on subsequent created SST files. Pull Request resolved: https://github.com/facebook/rocksdb/pull/4403 Differential Revision: D9976857 Pulled By: anand1976 fbshipit-source-id: 31dbf0511c12fa2bb9b2a7ba620079e0ee09cf48 --- db/db_table_properties_test.cc | 53 +++++++++++++++++-- .../utilities/table_properties_collectors.h | 50 ++++++++++++++++- .../compact_on_deletion_collector.cc | 12 +++-- .../compact_on_deletion_collector.h | 32 ----------- 4 files changed, 105 insertions(+), 42 deletions(-) diff --git a/db/db_table_properties_test.cc b/db/db_table_properties_test.cc index b12a56802..5a54fd81c 100644 --- a/db/db_table_properties_test.cc +++ b/db/db_table_properties_test.cc @@ -252,13 +252,14 @@ TEST_F(DBTablePropertiesTest, GetColumnFamilyNameProperty) { } TEST_F(DBTablePropertiesTest, DeletionTriggeredCompactionMarking) { - const int kNumKeys = 1000; - const int kWindowSize = 100; - const int kNumDelsTrigger = 90; + int kNumKeys = 1000; + int kWindowSize = 100; + int kNumDelsTrigger = 90; + std::shared_ptr compact_on_del = + NewCompactOnDeletionCollectorFactory(kWindowSize, kNumDelsTrigger); Options opts = CurrentOptions(); - opts.table_properties_collector_factories.emplace_back( - NewCompactOnDeletionCollectorFactory(kWindowSize, kNumDelsTrigger)); + opts.table_properties_collector_factories.emplace_back(compact_on_del); Reopen(opts); // add an L1 file to prevent tombstones from dropping due to obsolescence @@ -280,6 +281,48 @@ TEST_F(DBTablePropertiesTest, DeletionTriggeredCompactionMarking) { dbfull()->TEST_WaitForCompact(); ASSERT_EQ(0, NumTableFilesAtLevel(0)); ASSERT_GT(NumTableFilesAtLevel(1), 0); + + // Change the window size and deletion trigger and ensure new values take + // effect + kWindowSize = 50; + kNumDelsTrigger = 40; + static_cast + (compact_on_del.get())->SetWindowSize(kWindowSize); + static_cast + (compact_on_del.get())->SetDeletionTrigger(kNumDelsTrigger); + for (int i = 0; i < kNumKeys; ++i) { + if (i >= kNumKeys - kWindowSize && + i < kNumKeys - kWindowSize + kNumDelsTrigger) { + Delete(Key(i)); + } else { + Put(Key(i), "val"); + } + } + Flush(); + + dbfull()->TEST_WaitForCompact(); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + ASSERT_GT(NumTableFilesAtLevel(1), 0); + + // Change the window size to disable delete triggered compaction + kWindowSize = 0; + static_cast + (compact_on_del.get())->SetWindowSize(kWindowSize); + static_cast + (compact_on_del.get())->SetDeletionTrigger(kNumDelsTrigger); + for (int i = 0; i < kNumKeys; ++i) { + if (i >= kNumKeys - kWindowSize && + i < kNumKeys - kWindowSize + kNumDelsTrigger) { + Delete(Key(i)); + } else { + Put(Key(i), "val"); + } + } + Flush(); + + dbfull()->TEST_WaitForCompact(); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + } } // namespace rocksdb diff --git a/include/rocksdb/utilities/table_properties_collectors.h b/include/rocksdb/utilities/table_properties_collectors.h index 0f8827037..c74f89bc9 100644 --- a/include/rocksdb/utilities/table_properties_collectors.h +++ b/include/rocksdb/utilities/table_properties_collectors.h @@ -5,12 +5,60 @@ #pragma once #ifndef ROCKSDB_LITE +#include #include #include "rocksdb/table_properties.h" namespace rocksdb { +// A factory of a table property collector that marks a SST +// file as need-compaction when it observe at least "D" deletion +// entries in any "N" consecutive entires. +class CompactOnDeletionCollectorFactory + : public TablePropertiesCollectorFactory { + public: + virtual ~CompactOnDeletionCollectorFactory() {} + + virtual TablePropertiesCollector* CreateTablePropertiesCollector( + TablePropertiesCollectorFactory::Context context) override; + + // Change the value of sliding_window_size "N" + // Setting it to 0 disables the delete triggered compaction + void SetWindowSize(size_t sliding_window_size) { + sliding_window_size_.store(sliding_window_size); + } + + // Change the value of deletion_trigger "D" + void SetDeletionTrigger(size_t deletion_trigger) { + deletion_trigger_.store(deletion_trigger); + } + + virtual const char* Name() const override { + return "CompactOnDeletionCollector"; + } + + private: + friend std::shared_ptr + NewCompactOnDeletionCollectorFactory( + size_t sliding_window_size, + size_t deletion_trigger); + // A factory of a table property collector that marks a SST + // file as need-compaction when it observe at least "D" deletion + // entries in any "N" consecutive entires. + // + // @param sliding_window_size "N" + // @param deletion_trigger "D" + CompactOnDeletionCollectorFactory( + size_t sliding_window_size, + size_t deletion_trigger) : + sliding_window_size_(sliding_window_size), + deletion_trigger_(deletion_trigger) {} + + std::atomic sliding_window_size_; + std::atomic deletion_trigger_; +}; + // Creates a factory of a table property collector that marks a SST // file as need-compaction when it observe at least "D" deletion // entries in any "N" consecutive entires. @@ -20,7 +68,7 @@ namespace rocksdb { // than the specified size. // @param deletion_trigger "D". Note that even when "N" is changed, // the specified number for "D" will not be changed. -extern std::shared_ptr +extern std::shared_ptr NewCompactOnDeletionCollectorFactory( size_t sliding_window_size, size_t deletion_trigger); diff --git a/utilities/table_properties_collectors/compact_on_deletion_collector.cc b/utilities/table_properties_collectors/compact_on_deletion_collector.cc index fdb7389a7..eaf2c501d 100644 --- a/utilities/table_properties_collectors/compact_on_deletion_collector.cc +++ b/utilities/table_properties_collectors/compact_on_deletion_collector.cc @@ -20,7 +20,6 @@ CompactOnDeletionCollector::CompactOnDeletionCollector( deletion_trigger_(deletion_trigger), need_compaction_(false), finished_(false) { - assert(bucket_size_ > 0U); memset(num_deletions_in_buckets_, 0, sizeof(size_t) * kNumBuckets); } @@ -35,6 +34,11 @@ Status CompactOnDeletionCollector::AddUserKey(const Slice& /*key*/, SequenceNumber /*seq*/, uint64_t /*file_size*/) { assert(!finished_); + if (bucket_size_ == 0) { + // This collector is effectively disabled + return Status::OK(); + } + if (need_compaction_) { // If the output file already needs to be compacted, skip the check. return Status::OK(); @@ -71,14 +75,14 @@ TablePropertiesCollector* CompactOnDeletionCollectorFactory::CreateTablePropertiesCollector( TablePropertiesCollectorFactory::Context /*context*/) { return new CompactOnDeletionCollector( - sliding_window_size_, deletion_trigger_); + sliding_window_size_.load(), deletion_trigger_.load()); } -std::shared_ptr +std::shared_ptr NewCompactOnDeletionCollectorFactory( size_t sliding_window_size, size_t deletion_trigger) { - return std::shared_ptr( + return std::shared_ptr( new CompactOnDeletionCollectorFactory( sliding_window_size, deletion_trigger)); } diff --git a/utilities/table_properties_collectors/compact_on_deletion_collector.h b/utilities/table_properties_collectors/compact_on_deletion_collector.h index 5406ecea2..96c8db896 100644 --- a/utilities/table_properties_collectors/compact_on_deletion_collector.h +++ b/utilities/table_properties_collectors/compact_on_deletion_collector.h @@ -9,38 +9,6 @@ #include "rocksdb/utilities/table_properties_collectors.h" namespace rocksdb { -// A factory of a table property collector that marks a SST -// file as need-compaction when it observe at least "D" deletion -// entries in any "N" consecutive entires. -class CompactOnDeletionCollectorFactory - : public TablePropertiesCollectorFactory { - public: - // A factory of a table property collector that marks a SST - // file as need-compaction when it observe at least "D" deletion - // entries in any "N" consecutive entires. - // - // @param sliding_window_size "N" - // @param deletion_trigger "D" - CompactOnDeletionCollectorFactory( - size_t sliding_window_size, - size_t deletion_trigger) : - sliding_window_size_(sliding_window_size), - deletion_trigger_(deletion_trigger) {} - - virtual ~CompactOnDeletionCollectorFactory() {} - - virtual TablePropertiesCollector* CreateTablePropertiesCollector( - TablePropertiesCollectorFactory::Context context) override; - - virtual const char* Name() const override { - return "CompactOnDeletionCollector"; - } - - private: - size_t sliding_window_size_; - size_t deletion_trigger_; -}; - class CompactOnDeletionCollector : public TablePropertiesCollector { public: CompactOnDeletionCollector(