From 2e5f76429402f674b111821517e1382760e3cfdd Mon Sep 17 00:00:00 2001 From: slk Date: Thu, 23 Dec 2021 11:02:43 -0800 Subject: [PATCH] Make IncreaseFullHistoryTsLow to a public API (#9221) Summary: As (https://github.com/facebook/rocksdb/issues/9210) discussed, the **full_history_ts_low** is a member of CompactRangeOptions currently, which means a CF's fullHistoryTsLow is advanced only when users submit a CompactRange request. However, users may want to advance the fllHistoryTsLow without an immediate compact. This merge make IncreaseFullHistoryTsLow to a public API so users can advance each CF's fullHistoryTsLow seperately. Pull Request resolved: https://github.com/facebook/rocksdb/pull/9221 Reviewed By: akankshamahajan15 Differential Revision: D33201106 Pulled By: riversand963 fbshipit-source-id: 9cb1d013ba93260f72e16353e693ffee167b47ee --- HISTORY.md | 2 ++ db/db_impl/db_impl.cc | 24 +++++++++++++ db/db_impl/db_impl.h | 13 ++++++- db/db_impl/db_impl_compaction_flush.cc | 26 ++++++++++++-- db/db_test.cc | 10 ++++++ db/db_with_timestamp_basic_test.cc | 45 ++++++++++++++++++++++++ include/rocksdb/db.h | 9 +++++ include/rocksdb/utilities/stackable_db.h | 10 ++++++ 8 files changed, 136 insertions(+), 3 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index d12b1ba44..e20b00a1a 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -17,6 +17,8 @@ ### Public API change * Extend WriteBatch::AssignTimestamp and AssignTimestamps API so that both functions can accept an optional `checker` argument that performs additional checking on timestamp sizes. * Introduce a new EventListener callback that will be called upon the end of automatic error recovery. +* Add IncreaseFullHistoryTsLow API so users can advance each column family's full_history_ts_low seperately. +* Add GetFullHistoryTsLow API so users can query current full_history_low value of specified column family. ### Performance Improvements * Replaced map property `TableProperties::properties_offsets` with uint64_t property `external_sst_file_global_seqno_offset` to save table properties's memory. diff --git a/db/db_impl/db_impl.cc b/db/db_impl/db_impl.cc index bc1b88164..1e074732a 100644 --- a/db/db_impl/db_impl.cc +++ b/db/db_impl/db_impl.cc @@ -1496,6 +1496,30 @@ bool DBImpl::SetPreserveDeletesSequenceNumber(SequenceNumber seqnum) { } } +Status DBImpl::GetFullHistoryTsLow(ColumnFamilyHandle* column_family, + std::string* ts_low) { + if (ts_low == nullptr) { + return Status::InvalidArgument("ts_low is nullptr"); + } + ColumnFamilyData* cfd = nullptr; + if (column_family == nullptr) { + cfd = default_cf_handle_->cfd(); + } else { + auto cfh = static_cast_with_check(column_family); + assert(cfh != nullptr); + cfd = cfh->cfd(); + } + assert(cfd != nullptr && cfd->user_comparator() != nullptr); + if (cfd->user_comparator()->timestamp_size() == 0) { + return Status::InvalidArgument( + "Timestamp is not enabled in this column family"); + } + InstrumentedMutexLock l(&mutex_); + *ts_low = cfd->GetFullHistoryTsLow(); + assert(cfd->user_comparator()->timestamp_size() == ts_low->size()); + return Status::OK(); +} + InternalIterator* DBImpl::NewInternalIterator(const ReadOptions& read_options, Arena* arena, RangeDelAggregator* range_del_agg, diff --git a/db/db_impl/db_impl.h b/db/db_impl/db_impl.h index f46d41ce2..25d93221c 100644 --- a/db/db_impl/db_impl.h +++ b/db/db_impl/db_impl.h @@ -355,6 +355,16 @@ class DBImpl : public DB { virtual bool SetPreserveDeletesSequenceNumber(SequenceNumber seqnum) override; + // IncreaseFullHistoryTsLow(ColumnFamilyHandle*, std::string) will acquire + // and release db_mutex + Status IncreaseFullHistoryTsLow(ColumnFamilyHandle* column_family, + std::string ts_low) override; + + // GetFullHistoryTsLow(ColumnFamilyHandle*, std::string*) will acquire and + // release db_mutex + Status GetFullHistoryTsLow(ColumnFamilyHandle* column_family, + std::string* ts_low) override; + virtual Status GetDbIdentity(std::string& identity) const override; virtual Status GetDbIdentityFromIdentityFile(std::string* identity) const; @@ -1987,7 +1997,8 @@ class DBImpl : public DB { Status DisableFileDeletionsWithLock(); - Status IncreaseFullHistoryTsLow(ColumnFamilyData* cfd, std::string ts_low); + Status IncreaseFullHistoryTsLowImpl(ColumnFamilyData* cfd, + std::string ts_low); // Lock over the persistent DB state. Non-nullptr iff successfully acquired. FileLock* db_lock_; diff --git a/db/db_impl/db_impl_compaction_flush.cc b/db/db_impl/db_impl_compaction_flush.cc index 80410f671..4152163d9 100644 --- a/db/db_impl/db_impl_compaction_flush.cc +++ b/db/db_impl/db_impl_compaction_flush.cc @@ -917,8 +917,29 @@ Status DBImpl::CompactRange(const CompactRangeOptions& options, end_with_ts); } -Status DBImpl::IncreaseFullHistoryTsLow(ColumnFamilyData* cfd, +Status DBImpl::IncreaseFullHistoryTsLow(ColumnFamilyHandle* column_family, std::string ts_low) { + ColumnFamilyData* cfd = nullptr; + if (column_family == nullptr) { + cfd = default_cf_handle_->cfd(); + } else { + auto cfh = static_cast_with_check(column_family); + assert(cfh != nullptr); + cfd = cfh->cfd(); + } + assert(cfd != nullptr && cfd->user_comparator() != nullptr); + if (cfd->user_comparator()->timestamp_size() == 0) { + return Status::InvalidArgument( + "Timestamp is not enabled in this column family"); + } + if (cfd->user_comparator()->timestamp_size() != ts_low.size()) { + return Status::InvalidArgument("ts_low size mismatch"); + } + return IncreaseFullHistoryTsLowImpl(cfd, ts_low); +} + +Status DBImpl::IncreaseFullHistoryTsLowImpl(ColumnFamilyData* cfd, + std::string ts_low) { VersionEdit edit; edit.SetColumnFamily(cfd->GetID()); edit.SetFullHistoryTsLow(ts_low); @@ -926,6 +947,7 @@ Status DBImpl::IncreaseFullHistoryTsLow(ColumnFamilyData* cfd, InstrumentedMutexLock l(&mutex_); std::string current_ts_low = cfd->GetFullHistoryTsLow(); const Comparator* ucmp = cfd->user_comparator(); + assert(ucmp->timestamp_size() == ts_low.size() && !ts_low.empty()); if (!current_ts_low.empty() && ucmp->CompareTimestamp(ts_low, current_ts_low) < 0) { return Status::InvalidArgument( @@ -956,7 +978,7 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options, return Status::InvalidArgument( "Cannot specify compaction range with full_history_ts_low"); } - Status s = IncreaseFullHistoryTsLow(cfd, ts_low); + Status s = IncreaseFullHistoryTsLowImpl(cfd, ts_low); if (!s.ok()) { LogFlush(immutable_db_options_.info_log); return s; diff --git a/db/db_test.cc b/db/db_test.cc index e5720007f..051d99f2d 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -3214,6 +3214,16 @@ class ModelDB : public DB { return true; } + Status IncreaseFullHistoryTsLow(ColumnFamilyHandle* /*cf*/, + std::string /*ts_low*/) override { + return Status::OK(); + } + + Status GetFullHistoryTsLow(ColumnFamilyHandle* /*cf*/, + std::string* /*ts_low*/) override { + return Status::OK(); + } + ColumnFamilyHandle* DefaultColumnFamily() const override { return nullptr; } private: diff --git a/db/db_with_timestamp_basic_test.cc b/db/db_with_timestamp_basic_test.cc index 488af7b46..cb53b8186 100644 --- a/db/db_with_timestamp_basic_test.cc +++ b/db/db_with_timestamp_basic_test.cc @@ -435,6 +435,51 @@ TEST_F(DBBasicTestWithTimestamp, UpdateFullHistoryTsLow) { Close(); } +TEST_F(DBBasicTestWithTimestamp, UpdateFullHistoryTsLowWithPublicAPI) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + std::string ts_low_str = Timestamp(9, 0); + ASSERT_OK( + db_->IncreaseFullHistoryTsLow(db_->DefaultColumnFamily(), ts_low_str)); + std::string result_ts_low; + ASSERT_OK(db_->GetFullHistoryTsLow(nullptr, &result_ts_low)); + ASSERT_TRUE(test_cmp.CompareTimestamp(ts_low_str, result_ts_low) == 0); + // test increase full_history_low backward + std::string ts_low_str_back = Timestamp(8, 0); + auto s = db_->IncreaseFullHistoryTsLow(db_->DefaultColumnFamily(), + ts_low_str_back); + ASSERT_EQ(s, Status::InvalidArgument()); + // test IncreaseFullHistoryTsLow with a timestamp whose length is longger + // than the cf's timestamp size + std::string ts_low_str_long(Timestamp(0, 0).size() + 1, 'a'); + s = db_->IncreaseFullHistoryTsLow(db_->DefaultColumnFamily(), + ts_low_str_long); + ASSERT_EQ(s, Status::InvalidArgument()); + // test IncreaseFullHistoryTsLow with a timestamp which is null + std::string ts_low_str_null = ""; + s = db_->IncreaseFullHistoryTsLow(db_->DefaultColumnFamily(), + ts_low_str_null); + ASSERT_EQ(s, Status::InvalidArgument()); + // test IncreaseFullHistoryTsLow for a column family that does not enable + // timestamp + options.comparator = BytewiseComparator(); + DestroyAndReopen(options); + ts_low_str = Timestamp(10, 0); + s = db_->IncreaseFullHistoryTsLow(db_->DefaultColumnFamily(), ts_low_str); + ASSERT_EQ(s, Status::InvalidArgument()); + // test GetFullHistoryTsLow for a column family that does not enable + // timestamp + std::string current_ts_low; + s = db_->GetFullHistoryTsLow(db_->DefaultColumnFamily(), ¤t_ts_low); + ASSERT_EQ(s, Status::InvalidArgument()); + Close(); +} + TEST_F(DBBasicTestWithTimestamp, GetApproximateSizes) { Options options = CurrentOptions(); options.write_buffer_size = 100000000; // Large write buffer diff --git a/include/rocksdb/db.h b/include/rocksdb/db.h index 4b70606fa..750380f46 100644 --- a/include/rocksdb/db.h +++ b/include/rocksdb/db.h @@ -1356,6 +1356,15 @@ class DB { // times have the same effect as calling it once. virtual Status DisableFileDeletions() = 0; + // Increase the full_history_ts of column family. The new ts_low value should + // be newer than current full_history_ts value. + virtual Status IncreaseFullHistoryTsLow(ColumnFamilyHandle* column_family, + std::string ts_low) = 0; + + // Get current full_history_ts value. + virtual Status GetFullHistoryTsLow(ColumnFamilyHandle* column_family, + std::string* ts_low) = 0; + // Allow compactions to delete obsolete files. // If force == true, the call to EnableFileDeletions() will guarantee that // file deletions are enabled after the call, even if DisableFileDeletions() diff --git a/include/rocksdb/utilities/stackable_db.h b/include/rocksdb/utilities/stackable_db.h index eabe60d63..4a7a984d0 100644 --- a/include/rocksdb/utilities/stackable_db.h +++ b/include/rocksdb/utilities/stackable_db.h @@ -419,6 +419,16 @@ class StackableDB : public DB { return db_->SetPreserveDeletesSequenceNumber(seqnum); } + Status IncreaseFullHistoryTsLow(ColumnFamilyHandle* column_family, + std::string ts_low) override { + return db_->IncreaseFullHistoryTsLow(column_family, ts_low); + } + + Status GetFullHistoryTsLow(ColumnFamilyHandle* column_family, + std::string* ts_low) override { + return db_->GetFullHistoryTsLow(column_family, ts_low); + } + virtual Status GetSortedWalFiles(VectorLogPtr& files) override { return db_->GetSortedWalFiles(files); }