Make IncreaseFullHistoryTsLow to a public API (#9221)

Summary:
As (https://github.com/facebook/rocksdb/issues/9210) discussed, the **full_history_ts_low** is a member of CompactRangeOptions currently, which means a CF's fullHistoryTsLow is advanced only when users submit a CompactRange request.
However, users may want to advance the fllHistoryTsLow without an immediate compact.
This merge make IncreaseFullHistoryTsLow to a public API so users can advance each CF's fullHistoryTsLow seperately.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9221

Reviewed By: akankshamahajan15

Differential Revision: D33201106

Pulled By: riversand963

fbshipit-source-id: 9cb1d013ba93260f72e16353e693ffee167b47ee
main
slk 3 years ago committed by Facebook GitHub Bot
parent 538d2365e9
commit 2e5f764294
  1. 2
      HISTORY.md
  2. 24
      db/db_impl/db_impl.cc
  3. 13
      db/db_impl/db_impl.h
  4. 26
      db/db_impl/db_impl_compaction_flush.cc
  5. 10
      db/db_test.cc
  6. 45
      db/db_with_timestamp_basic_test.cc
  7. 9
      include/rocksdb/db.h
  8. 10
      include/rocksdb/utilities/stackable_db.h

@ -17,6 +17,8 @@
### Public API change
* Extend WriteBatch::AssignTimestamp and AssignTimestamps API so that both functions can accept an optional `checker` argument that performs additional checking on timestamp sizes.
* Introduce a new EventListener callback that will be called upon the end of automatic error recovery.
* Add IncreaseFullHistoryTsLow API so users can advance each column family's full_history_ts_low seperately.
* Add GetFullHistoryTsLow API so users can query current full_history_low value of specified column family.
### Performance Improvements
* Replaced map property `TableProperties::properties_offsets` with uint64_t property `external_sst_file_global_seqno_offset` to save table properties's memory.

@ -1496,6 +1496,30 @@ bool DBImpl::SetPreserveDeletesSequenceNumber(SequenceNumber seqnum) {
}
}
Status DBImpl::GetFullHistoryTsLow(ColumnFamilyHandle* column_family,
std::string* ts_low) {
if (ts_low == nullptr) {
return Status::InvalidArgument("ts_low is nullptr");
}
ColumnFamilyData* cfd = nullptr;
if (column_family == nullptr) {
cfd = default_cf_handle_->cfd();
} else {
auto cfh = static_cast_with_check<ColumnFamilyHandleImpl>(column_family);
assert(cfh != nullptr);
cfd = cfh->cfd();
}
assert(cfd != nullptr && cfd->user_comparator() != nullptr);
if (cfd->user_comparator()->timestamp_size() == 0) {
return Status::InvalidArgument(
"Timestamp is not enabled in this column family");
}
InstrumentedMutexLock l(&mutex_);
*ts_low = cfd->GetFullHistoryTsLow();
assert(cfd->user_comparator()->timestamp_size() == ts_low->size());
return Status::OK();
}
InternalIterator* DBImpl::NewInternalIterator(const ReadOptions& read_options,
Arena* arena,
RangeDelAggregator* range_del_agg,

@ -355,6 +355,16 @@ class DBImpl : public DB {
virtual bool SetPreserveDeletesSequenceNumber(SequenceNumber seqnum) override;
// IncreaseFullHistoryTsLow(ColumnFamilyHandle*, std::string) will acquire
// and release db_mutex
Status IncreaseFullHistoryTsLow(ColumnFamilyHandle* column_family,
std::string ts_low) override;
// GetFullHistoryTsLow(ColumnFamilyHandle*, std::string*) will acquire and
// release db_mutex
Status GetFullHistoryTsLow(ColumnFamilyHandle* column_family,
std::string* ts_low) override;
virtual Status GetDbIdentity(std::string& identity) const override;
virtual Status GetDbIdentityFromIdentityFile(std::string* identity) const;
@ -1987,7 +1997,8 @@ class DBImpl : public DB {
Status DisableFileDeletionsWithLock();
Status IncreaseFullHistoryTsLow(ColumnFamilyData* cfd, std::string ts_low);
Status IncreaseFullHistoryTsLowImpl(ColumnFamilyData* cfd,
std::string ts_low);
// Lock over the persistent DB state. Non-nullptr iff successfully acquired.
FileLock* db_lock_;

@ -917,7 +917,28 @@ Status DBImpl::CompactRange(const CompactRangeOptions& options,
end_with_ts);
}
Status DBImpl::IncreaseFullHistoryTsLow(ColumnFamilyData* cfd,
Status DBImpl::IncreaseFullHistoryTsLow(ColumnFamilyHandle* column_family,
std::string ts_low) {
ColumnFamilyData* cfd = nullptr;
if (column_family == nullptr) {
cfd = default_cf_handle_->cfd();
} else {
auto cfh = static_cast_with_check<ColumnFamilyHandleImpl>(column_family);
assert(cfh != nullptr);
cfd = cfh->cfd();
}
assert(cfd != nullptr && cfd->user_comparator() != nullptr);
if (cfd->user_comparator()->timestamp_size() == 0) {
return Status::InvalidArgument(
"Timestamp is not enabled in this column family");
}
if (cfd->user_comparator()->timestamp_size() != ts_low.size()) {
return Status::InvalidArgument("ts_low size mismatch");
}
return IncreaseFullHistoryTsLowImpl(cfd, ts_low);
}
Status DBImpl::IncreaseFullHistoryTsLowImpl(ColumnFamilyData* cfd,
std::string ts_low) {
VersionEdit edit;
edit.SetColumnFamily(cfd->GetID());
@ -926,6 +947,7 @@ Status DBImpl::IncreaseFullHistoryTsLow(ColumnFamilyData* cfd,
InstrumentedMutexLock l(&mutex_);
std::string current_ts_low = cfd->GetFullHistoryTsLow();
const Comparator* ucmp = cfd->user_comparator();
assert(ucmp->timestamp_size() == ts_low.size() && !ts_low.empty());
if (!current_ts_low.empty() &&
ucmp->CompareTimestamp(ts_low, current_ts_low) < 0) {
return Status::InvalidArgument(
@ -956,7 +978,7 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options,
return Status::InvalidArgument(
"Cannot specify compaction range with full_history_ts_low");
}
Status s = IncreaseFullHistoryTsLow(cfd, ts_low);
Status s = IncreaseFullHistoryTsLowImpl(cfd, ts_low);
if (!s.ok()) {
LogFlush(immutable_db_options_.info_log);
return s;

@ -3214,6 +3214,16 @@ class ModelDB : public DB {
return true;
}
Status IncreaseFullHistoryTsLow(ColumnFamilyHandle* /*cf*/,
std::string /*ts_low*/) override {
return Status::OK();
}
Status GetFullHistoryTsLow(ColumnFamilyHandle* /*cf*/,
std::string* /*ts_low*/) override {
return Status::OK();
}
ColumnFamilyHandle* DefaultColumnFamily() const override { return nullptr; }
private:

@ -435,6 +435,51 @@ TEST_F(DBBasicTestWithTimestamp, UpdateFullHistoryTsLow) {
Close();
}
TEST_F(DBBasicTestWithTimestamp, UpdateFullHistoryTsLowWithPublicAPI) {
Options options = CurrentOptions();
options.env = env_;
options.create_if_missing = true;
const size_t kTimestampSize = Timestamp(0, 0).size();
TestComparator test_cmp(kTimestampSize);
options.comparator = &test_cmp;
DestroyAndReopen(options);
std::string ts_low_str = Timestamp(9, 0);
ASSERT_OK(
db_->IncreaseFullHistoryTsLow(db_->DefaultColumnFamily(), ts_low_str));
std::string result_ts_low;
ASSERT_OK(db_->GetFullHistoryTsLow(nullptr, &result_ts_low));
ASSERT_TRUE(test_cmp.CompareTimestamp(ts_low_str, result_ts_low) == 0);
// test increase full_history_low backward
std::string ts_low_str_back = Timestamp(8, 0);
auto s = db_->IncreaseFullHistoryTsLow(db_->DefaultColumnFamily(),
ts_low_str_back);
ASSERT_EQ(s, Status::InvalidArgument());
// test IncreaseFullHistoryTsLow with a timestamp whose length is longger
// than the cf's timestamp size
std::string ts_low_str_long(Timestamp(0, 0).size() + 1, 'a');
s = db_->IncreaseFullHistoryTsLow(db_->DefaultColumnFamily(),
ts_low_str_long);
ASSERT_EQ(s, Status::InvalidArgument());
// test IncreaseFullHistoryTsLow with a timestamp which is null
std::string ts_low_str_null = "";
s = db_->IncreaseFullHistoryTsLow(db_->DefaultColumnFamily(),
ts_low_str_null);
ASSERT_EQ(s, Status::InvalidArgument());
// test IncreaseFullHistoryTsLow for a column family that does not enable
// timestamp
options.comparator = BytewiseComparator();
DestroyAndReopen(options);
ts_low_str = Timestamp(10, 0);
s = db_->IncreaseFullHistoryTsLow(db_->DefaultColumnFamily(), ts_low_str);
ASSERT_EQ(s, Status::InvalidArgument());
// test GetFullHistoryTsLow for a column family that does not enable
// timestamp
std::string current_ts_low;
s = db_->GetFullHistoryTsLow(db_->DefaultColumnFamily(), &current_ts_low);
ASSERT_EQ(s, Status::InvalidArgument());
Close();
}
TEST_F(DBBasicTestWithTimestamp, GetApproximateSizes) {
Options options = CurrentOptions();
options.write_buffer_size = 100000000; // Large write buffer

@ -1356,6 +1356,15 @@ class DB {
// times have the same effect as calling it once.
virtual Status DisableFileDeletions() = 0;
// Increase the full_history_ts of column family. The new ts_low value should
// be newer than current full_history_ts value.
virtual Status IncreaseFullHistoryTsLow(ColumnFamilyHandle* column_family,
std::string ts_low) = 0;
// Get current full_history_ts value.
virtual Status GetFullHistoryTsLow(ColumnFamilyHandle* column_family,
std::string* ts_low) = 0;
// Allow compactions to delete obsolete files.
// If force == true, the call to EnableFileDeletions() will guarantee that
// file deletions are enabled after the call, even if DisableFileDeletions()

@ -419,6 +419,16 @@ class StackableDB : public DB {
return db_->SetPreserveDeletesSequenceNumber(seqnum);
}
Status IncreaseFullHistoryTsLow(ColumnFamilyHandle* column_family,
std::string ts_low) override {
return db_->IncreaseFullHistoryTsLow(column_family, ts_low);
}
Status GetFullHistoryTsLow(ColumnFamilyHandle* column_family,
std::string* ts_low) override {
return db_->GetFullHistoryTsLow(column_family, ts_low);
}
virtual Status GetSortedWalFiles(VectorLogPtr& files) override {
return db_->GetSortedWalFiles(files);
}

Loading…
Cancel
Save