From 5ce246c716bbe7787150e3ced964b27a93995f6b Mon Sep 17 00:00:00 2001 From: Levi Tamasi Date: Thu, 10 Sep 2020 11:32:02 -0700 Subject: [PATCH] Expose the start of the expiration range for TTL blob files through LiveFileMetaData (#7365) Summary: The patch adds support for exposing the start of the expiration range for TTL blob files through the `GetLiveFilesMetaData` API. This can be used for monitoring purposes, i.e. to make sure TTL blob files are deleted in a timely manner. The patch also fixes a couple of uninitialized variable issues. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7365 Test Plan: `make check` Reviewed By: pdillinger Differential Revision: D23605465 Pulled By: ltamasi fbshipit-source-id: 97a9612bf5f4b058423debdd3f28f576bb23a70f --- include/rocksdb/metadata.h | 6 +++- .../blob_db/blob_db_impl_filesnapshot.cc | 3 ++ utilities/blob_db/blob_db_test.cc | 35 +++++++++++++++---- utilities/blob_db/blob_file.h | 4 ++- 4 files changed, 39 insertions(+), 9 deletions(-) diff --git a/include/rocksdb/metadata.h b/include/rocksdb/metadata.h index f1a9ee60e..9a64a7a8f 100644 --- a/include/rocksdb/metadata.h +++ b/include/rocksdb/metadata.h @@ -62,7 +62,9 @@ struct SstFileMetaData { being_compacted(false), num_entries(0), num_deletions(0), - oldest_blob_file_number(0) {} + oldest_blob_file_number(0), + oldest_ancester_time(0), + file_creation_time(0) {} SstFileMetaData(const std::string& _file_name, uint64_t _file_number, const std::string& _path, size_t _size, @@ -117,6 +119,8 @@ struct SstFileMetaData { // oldest SST file that is the compaction ancester of this file. // The timestamp is provided Env::GetCurrentTime(). // 0 if the information is not available. + // + // Note: for TTL blob files, it contains the start of the expiration range. uint64_t oldest_ancester_time; // Timestamp when the SST file is created, provided by Env::GetCurrentTime(). // 0 if the information is not available. diff --git a/utilities/blob_db/blob_db_impl_filesnapshot.cc b/utilities/blob_db/blob_db_impl_filesnapshot.cc index bfd81c51d..7ec4443d4 100644 --- a/utilities/blob_db/blob_db_impl_filesnapshot.cc +++ b/utilities/blob_db/blob_db_impl_filesnapshot.cc @@ -98,6 +98,9 @@ void BlobDBImpl::GetLiveFilesMetaData(std::vector* metadata) { // Path should be relative to db_name, but begin with slash. filemetadata.name = BlobFileName("", bdb_options_.blob_dir, file_number); filemetadata.file_number = file_number; + if (blob_file->HasTTL()) { + filemetadata.oldest_ancester_time = blob_file->GetExpirationRange().first; + } auto cfh = static_cast_with_check(DefaultColumnFamily()); filemetadata.column_family_name = cfh->GetName(); diff --git a/utilities/blob_db/blob_db_test.cc b/utilities/blob_db/blob_db_test.cc index 99d1d609b..b5472ae8d 100644 --- a/utilities/blob_db/blob_db_test.cc +++ b/utilities/blob_db/blob_db_test.cc @@ -956,29 +956,50 @@ TEST_F(BlobDBTest, ColumnFamilyNotSupported) { TEST_F(BlobDBTest, GetLiveFilesMetaData) { Random rnd(301); + BlobDBOptions bdb_options; bdb_options.blob_dir = "blob_dir"; bdb_options.path_relative = true; + bdb_options.ttl_range_secs = 10; bdb_options.min_blob_size = 0; bdb_options.disable_background_tasks = true; - Open(bdb_options); + + Options options; + options.env = mock_env_.get(); + + Open(bdb_options, options); + std::map data; for (size_t i = 0; i < 100; i++) { PutRandom("key" + ToString(i), &rnd, &data); } + + constexpr uint64_t expiration = 1000ULL; + PutRandomUntil("key100", expiration, &rnd, &data); + std::vector metadata; blob_db_->GetLiveFilesMetaData(&metadata); - ASSERT_EQ(1U, metadata.size()); + + ASSERT_EQ(2U, metadata.size()); // Path should be relative to db_name, but begin with slash. - std::string filename = "/blob_dir/000001.blob"; - ASSERT_EQ(filename, metadata[0].name); + const std::string filename1("/blob_dir/000001.blob"); + ASSERT_EQ(filename1, metadata[0].name); ASSERT_EQ(1, metadata[0].file_number); - ASSERT_EQ("default", metadata[0].column_family_name); + ASSERT_EQ(0, metadata[0].oldest_ancester_time); + ASSERT_EQ(kDefaultColumnFamilyName, metadata[0].column_family_name); + + const std::string filename2("/blob_dir/000002.blob"); + ASSERT_EQ(filename2, metadata[1].name); + ASSERT_EQ(2, metadata[1].file_number); + ASSERT_EQ(expiration, metadata[1].oldest_ancester_time); + ASSERT_EQ(kDefaultColumnFamilyName, metadata[1].column_family_name); + std::vector livefile; uint64_t mfs; ASSERT_OK(blob_db_->GetLiveFiles(livefile, &mfs, false)); - ASSERT_EQ(4U, livefile.size()); - ASSERT_EQ(filename, livefile[3]); + ASSERT_EQ(5U, livefile.size()); + ASSERT_EQ(filename1, livefile[3]); + ASSERT_EQ(filename2, livefile[4]); VerifyDB(data); } diff --git a/utilities/blob_db/blob_file.h b/utilities/blob_db/blob_file.h index 9b00dccd1..fb990dafd 100644 --- a/utilities/blob_db/blob_file.h +++ b/utilities/blob_db/blob_file.h @@ -189,7 +189,9 @@ class BlobFile { // All Get functions which are not atomic, will need ReadLock on the mutex - ExpirationRange GetExpirationRange() const { return expiration_range_; } + const ExpirationRange& GetExpirationRange() const { + return expiration_range_; + } void ExtendExpirationRange(uint64_t expiration) { expiration_range_.first = std::min(expiration_range_.first, expiration);