From ec880436c1627b2f6f79b508a6face5671e3117f Mon Sep 17 00:00:00 2001 From: Vijay Nadimpalli Date: Fri, 25 Oct 2019 11:52:24 -0700 Subject: [PATCH] API to get file_creation_time of the oldest file in the DB (#5948) Summary: Adding a new API to db.h that allows users to get file_creation_time of the oldest file in the DB. Pull Request resolved: https://github.com/facebook/rocksdb/pull/5948 Test Plan: Added unit test. Differential Revision: D18056151 Pulled By: vjnadimpalli fbshipit-source-id: 448ec9d34cb6772e1e5a62db399ace00dcbfbb5d --- db/db_impl/db_impl.cc | 20 +++++ db/db_impl/db_impl.h | 2 + db/db_test.cc | 105 +++++++++++++++++++++++ db/version_set.cc | 19 ++++ db/version_set.h | 4 + include/rocksdb/db.h | 12 +++ include/rocksdb/utilities/stackable_db.h | 5 ++ 7 files changed, 167 insertions(+) diff --git a/db/db_impl/db_impl.cc b/db/db_impl/db_impl.cc index 72cfb0be9..14d9523f7 100644 --- a/db/db_impl/db_impl.cc +++ b/db/db_impl/db_impl.cc @@ -4270,6 +4270,26 @@ Status DBImpl::ReserveFileNumbersBeforeIngestion( dummy_sv_ctx.Clean(); return s; } + +Status DBImpl::GetCreationTimeOfOldestFile(uint64_t* creation_time) { + if (mutable_db_options_.max_open_files == -1) { + uint64_t oldest_time = port::kMaxUint64; + for (auto cfd : *versions_->GetColumnFamilySet()) { + uint64_t ctime; + cfd->current()->GetCreationTimeOfOldestFile(&ctime); + if (ctime < oldest_time) { + oldest_time = ctime; + } + if (oldest_time == 0) { + break; + } + } + *creation_time = oldest_time; + return Status::OK(); + } else { + return Status::NotSupported("This API only works if max_open_files = -1"); + } +} #endif // ROCKSDB_LITE } // namespace rocksdb diff --git a/db/db_impl/db_impl.h b/db/db_impl/db_impl.h index 4a9b243d9..d3bf37b7d 100644 --- a/db/db_impl/db_impl.h +++ b/db/db_impl/db_impl.h @@ -349,6 +349,8 @@ class DBImpl : public DB { virtual Status GetSortedWalFiles(VectorLogPtr& files) override; virtual Status GetCurrentWalFile( std::unique_ptr* current_log_file) override; + virtual Status GetCreationTimeOfOldestFile( + uint64_t* creation_time) override; virtual Status GetUpdatesSince( SequenceNumber seq_number, std::unique_ptr* iter, diff --git a/db/db_test.cc b/db/db_test.cc index 04785c3c1..89553b3a7 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -2798,6 +2798,11 @@ class ModelDB : public DB { return Status::OK(); } + virtual Status GetCreationTimeOfOldestFile( + uint64_t* /*creation_time*/) override { + return Status::NotSupported(); + } + Status DeleteFile(std::string /*name*/) override { return Status::OK(); } Status GetUpdatesSince( @@ -6271,6 +6276,106 @@ TEST_F(DBTest, LargeBlockSizeTest) { ASSERT_NOK(TryReopenWithColumnFamilies({"default", "pikachu"}, options)); } +TEST_F(DBTest, CreationTimeOfOldestFile) { + const int kNumKeysPerFile = 32; + const int kNumLevelFiles = 2; + const int kValueSize = 100; + + Options options = CurrentOptions(); + options.max_open_files = -1; + env_->time_elapse_only_sleep_ = false; + options.env = env_; + + env_->addon_time_.store(0); + DestroyAndReopen(options); + + bool set_file_creation_time_to_zero = true; + int idx = 0; + + int64_t time_1 = 0; + env_->GetCurrentTime(&time_1); + const uint64_t uint_time_1 = static_cast(time_1); + + // Add 50 hours + env_->addon_time_.fetch_add(50 * 60 * 60); + + int64_t time_2 = 0; + env_->GetCurrentTime(&time_2); + const uint64_t uint_time_2 = static_cast(time_2); + + rocksdb::SyncPoint::GetInstance()->SetCallBack( + "PropertyBlockBuilder::AddTableProperty:Start", [&](void* arg) { + TableProperties* props = reinterpret_cast(arg); + if (set_file_creation_time_to_zero) { + if (idx == 0) { + props->file_creation_time = 0; + idx++; + } else if (idx == 1) { + props->file_creation_time = uint_time_1; + idx = 0; + } + } else { + if (idx == 0) { + props->file_creation_time = uint_time_1; + idx++; + } else if (idx == 1) { + props->file_creation_time = uint_time_2; + } + } + }); + rocksdb::SyncPoint::GetInstance()->EnableProcessing(); + + Random rnd(301); + for (int i = 0; i < kNumLevelFiles; ++i) { + for (int j = 0; j < kNumKeysPerFile; ++j) { + ASSERT_OK( + Put(Key(i * kNumKeysPerFile + j), RandomString(&rnd, kValueSize))); + } + Flush(); + } + + // At this point there should be 2 files, oen with file_creation_time = 0 and + // the other non-zero. GetCreationTimeOfOldestFile API should return 0. + uint64_t creation_time; + Status s1 = dbfull()->GetCreationTimeOfOldestFile(&creation_time); + ASSERT_EQ(0, creation_time); + ASSERT_EQ(s1, Status::OK()); + + // Testing with non-zero file creation time. + set_file_creation_time_to_zero = false; + options = CurrentOptions(); + options.max_open_files = -1; + env_->time_elapse_only_sleep_ = false; + options.env = env_; + + env_->addon_time_.store(0); + DestroyAndReopen(options); + + for (int i = 0; i < kNumLevelFiles; ++i) { + for (int j = 0; j < kNumKeysPerFile; ++j) { + ASSERT_OK( + Put(Key(i * kNumKeysPerFile + j), RandomString(&rnd, kValueSize))); + } + Flush(); + } + + // At this point there should be 2 files with non-zero file creation time. + // GetCreationTimeOfOldestFile API should return non-zero value. + uint64_t ctime; + Status s2 = dbfull()->GetCreationTimeOfOldestFile(&ctime); + ASSERT_EQ(uint_time_1, ctime); + ASSERT_EQ(s2, Status::OK()); + + // Testing with max_open_files != -1 + options = CurrentOptions(); + options.max_open_files = 10; + DestroyAndReopen(options); + Status s3 = dbfull()->GetCreationTimeOfOldestFile(&ctime); + ASSERT_EQ(s3, Status::NotSupported()); + + rocksdb::SyncPoint::GetInstance()->DisableProcessing(); +} + } // namespace rocksdb #ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS diff --git a/db/version_set.cc b/db/version_set.cc index ac66a5cb3..11264205a 100644 --- a/db/version_set.cc +++ b/db/version_set.cc @@ -1481,6 +1481,25 @@ uint64_t Version::GetSstFilesSize() { return sst_files_size; } +void Version::GetCreationTimeOfOldestFile(uint64_t* creation_time) { + uint64_t oldest_time = port::kMaxUint64; + for (int level = 0; level < storage_info_.num_non_empty_levels_; level++) { + for (FileMetaData* meta : storage_info_.LevelFiles(level)) { + assert(meta->fd.table_reader != nullptr); + uint64_t file_creation_time = + meta->fd.table_reader->GetTableProperties()->file_creation_time; + if (file_creation_time == 0) { + *creation_time = file_creation_time; + return; + } + if (file_creation_time < oldest_time) { + oldest_time = file_creation_time; + } + } + } + *creation_time = oldest_time; +} + uint64_t VersionStorageInfo::GetEstimatedActiveKeys() const { // Estimation will be inaccurate when: // (1) there exist merge keys diff --git a/db/version_set.h b/db/version_set.h index 24919a602..6b9d71c37 100644 --- a/db/version_set.h +++ b/db/version_set.h @@ -672,6 +672,10 @@ class Version { uint64_t GetSstFilesSize(); + // Retrieves the file_creation_time of the oldest file in the DB. + // Prerequisite for this API is max_open_files = -1 + void GetCreationTimeOfOldestFile(uint64_t* creation_time); + const MutableCFOptions& GetMutableCFOptions() { return mutable_cf_options_; } private: diff --git a/include/rocksdb/db.h b/include/rocksdb/db.h index b03c12e00..e73ae9c20 100644 --- a/include/rocksdb/db.h +++ b/include/rocksdb/db.h @@ -1140,6 +1140,18 @@ class DB { virtual Status GetCurrentWalFile( std::unique_ptr* current_log_file) = 0; + // Retrieves the creation time of the oldest file in the DB. + // This API only works if max_open_files = -1, if it is not then + // Status returned is Status::NotSupported() + // The file creation time is set using the env provided to the DB. + // If the DB was created from a very old release then its possible that + // the SST files might not have file_creation_time property and even after + // moving to a newer release its possible that some files never got compacted + // and may not have file_creation_time property. In both the cases + // file_creation_time is considered 0 which means this API will return + // creation_time = 0 as there wouldn't be a timestamp lower than 0. + virtual Status GetCreationTimeOfOldestFile(uint64_t* creation_time) = 0; + // Note: this API is not yet consistent with WritePrepared transactions. // Sets iter to an iterator that is positioned at a write-batch containing // seq_number. If the sequence number is non existent, it returns an iterator diff --git a/include/rocksdb/utilities/stackable_db.h b/include/rocksdb/utilities/stackable_db.h index e6618cf45..c0cb7e316 100644 --- a/include/rocksdb/utilities/stackable_db.h +++ b/include/rocksdb/utilities/stackable_db.h @@ -383,6 +383,11 @@ class StackableDB : public DB { return db_->GetCurrentWalFile(current_log_file); } + virtual Status GetCreationTimeOfOldestFile( + uint64_t* creation_time) override { + return db_->GetCreationTimeOfOldestFile(creation_time); + } + virtual Status DeleteFile(std::string name) override { return db_->DeleteFile(name); }