Expose the start of the expiration range for TTL blob files through LiveFileMetaData (#7365)

Summary:
The patch adds support for exposing the start of the expiration range
for TTL blob files through the `GetLiveFilesMetaData` API. This can be
used for monitoring purposes, i.e. to make sure TTL blob files are
deleted in a timely manner. The patch also fixes a couple of uninitialized
variable issues.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7365

Test Plan: `make check`

Reviewed By: pdillinger

Differential Revision: D23605465

Pulled By: ltamasi

fbshipit-source-id: 97a9612bf5f4b058423debdd3f28f576bb23a70f
main
Levi Tamasi 4 years ago committed by Facebook GitHub Bot
parent ec5add398c
commit 5ce246c716
  1. 6
      include/rocksdb/metadata.h
  2. 3
      utilities/blob_db/blob_db_impl_filesnapshot.cc
  3. 35
      utilities/blob_db/blob_db_test.cc
  4. 4
      utilities/blob_db/blob_file.h

@ -62,7 +62,9 @@ struct SstFileMetaData {
being_compacted(false),
num_entries(0),
num_deletions(0),
oldest_blob_file_number(0) {}
oldest_blob_file_number(0),
oldest_ancester_time(0),
file_creation_time(0) {}
SstFileMetaData(const std::string& _file_name, uint64_t _file_number,
const std::string& _path, size_t _size,
@ -117,6 +119,8 @@ struct SstFileMetaData {
// oldest SST file that is the compaction ancester of this file.
// The timestamp is provided Env::GetCurrentTime().
// 0 if the information is not available.
//
// Note: for TTL blob files, it contains the start of the expiration range.
uint64_t oldest_ancester_time;
// Timestamp when the SST file is created, provided by Env::GetCurrentTime().
// 0 if the information is not available.

@ -98,6 +98,9 @@ void BlobDBImpl::GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata) {
// Path should be relative to db_name, but begin with slash.
filemetadata.name = BlobFileName("", bdb_options_.blob_dir, file_number);
filemetadata.file_number = file_number;
if (blob_file->HasTTL()) {
filemetadata.oldest_ancester_time = blob_file->GetExpirationRange().first;
}
auto cfh =
static_cast_with_check<ColumnFamilyHandleImpl>(DefaultColumnFamily());
filemetadata.column_family_name = cfh->GetName();

@ -956,29 +956,50 @@ TEST_F(BlobDBTest, ColumnFamilyNotSupported) {
TEST_F(BlobDBTest, GetLiveFilesMetaData) {
Random rnd(301);
BlobDBOptions bdb_options;
bdb_options.blob_dir = "blob_dir";
bdb_options.path_relative = true;
bdb_options.ttl_range_secs = 10;
bdb_options.min_blob_size = 0;
bdb_options.disable_background_tasks = true;
Open(bdb_options);
Options options;
options.env = mock_env_.get();
Open(bdb_options, options);
std::map<std::string, std::string> data;
for (size_t i = 0; i < 100; i++) {
PutRandom("key" + ToString(i), &rnd, &data);
}
constexpr uint64_t expiration = 1000ULL;
PutRandomUntil("key100", expiration, &rnd, &data);
std::vector<LiveFileMetaData> metadata;
blob_db_->GetLiveFilesMetaData(&metadata);
ASSERT_EQ(1U, metadata.size());
ASSERT_EQ(2U, metadata.size());
// Path should be relative to db_name, but begin with slash.
std::string filename = "/blob_dir/000001.blob";
ASSERT_EQ(filename, metadata[0].name);
const std::string filename1("/blob_dir/000001.blob");
ASSERT_EQ(filename1, metadata[0].name);
ASSERT_EQ(1, metadata[0].file_number);
ASSERT_EQ("default", metadata[0].column_family_name);
ASSERT_EQ(0, metadata[0].oldest_ancester_time);
ASSERT_EQ(kDefaultColumnFamilyName, metadata[0].column_family_name);
const std::string filename2("/blob_dir/000002.blob");
ASSERT_EQ(filename2, metadata[1].name);
ASSERT_EQ(2, metadata[1].file_number);
ASSERT_EQ(expiration, metadata[1].oldest_ancester_time);
ASSERT_EQ(kDefaultColumnFamilyName, metadata[1].column_family_name);
std::vector<std::string> livefile;
uint64_t mfs;
ASSERT_OK(blob_db_->GetLiveFiles(livefile, &mfs, false));
ASSERT_EQ(4U, livefile.size());
ASSERT_EQ(filename, livefile[3]);
ASSERT_EQ(5U, livefile.size());
ASSERT_EQ(filename1, livefile[3]);
ASSERT_EQ(filename2, livefile[4]);
VerifyDB(data);
}

@ -189,7 +189,9 @@ class BlobFile {
// All Get functions which are not atomic, will need ReadLock on the mutex
ExpirationRange GetExpirationRange() const { return expiration_range_; }
const ExpirationRange& GetExpirationRange() const {
return expiration_range_;
}
void ExtendExpirationRange(uint64_t expiration) {
expiration_range_.first = std::min(expiration_range_.first, expiration);

Loading…
Cancel
Save