diff --git a/db/db_impl/db_impl_open.cc b/db/db_impl/db_impl_open.cc index 808459b77..9004946f4 100644 --- a/db/db_impl/db_impl_open.cc +++ b/db/db_impl/db_impl_open.cc @@ -1930,21 +1930,24 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname, // vast majority of all files), we'll pass the size to SstFileManager. // For all other files SstFileManager will query the size from filesystem. - std::vector metadata; - - // TODO: Once GetLiveFilesMetaData supports blob files, update the logic - // below to get known_file_sizes for blob files. - impl->mutex_.Lock(); - impl->versions_->GetLiveFilesMetaData(&metadata); - impl->mutex_.Unlock(); + std::vector metadata; + impl->GetAllColumnFamilyMetaData(&metadata); std::unordered_map known_file_sizes; for (const auto& md : metadata) { - std::string name = md.name; - if (!name.empty() && name[0] == '/') { - name = name.substr(1); + for (const auto& lmd : md.levels) { + for (const auto& fmd : lmd.files) { + known_file_sizes[fmd.relative_filename] = fmd.size; + } + } + for (const auto& bmd : md.blob_files) { + std::string name = bmd.blob_file_name; + // The BlobMetaData.blob_file_name may start with "/". + if (!name.empty() && name[0] == '/') { + name = name.substr(1); + } + known_file_sizes[name] = bmd.blob_file_size; } - known_file_sizes[name] = md.size; } std::vector paths; diff --git a/db/db_sst_test.cc b/db/db_sst_test.cc index 24aeea52d..3fb0f99a1 100644 --- a/db/db_sst_test.cc +++ b/db/db_sst_test.cc @@ -8,6 +8,7 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. #include "db/db_test_util.h" +#include "env/mock_env.h" #include "file/sst_file_manager_impl.h" #include "port/port.h" #include "port/stack_trace.h" @@ -961,7 +962,6 @@ TEST_F(DBSSTTest, OpenDBWithExistingTrash) { ASSERT_NOK(env_->FileExists(dbname_ + "/" + "003.sst.trash")); } - // Create a DB with 2 db_paths, and generate multiple files in the 2 // db_paths using CompactRangeOptions, make sure that files that were // deleted from first db_path were deleted using DeleteScheduler and @@ -1233,7 +1233,9 @@ TEST_F(DBSSTTest, CancellingCompactionsWorks) { ASSERT_GT(completed_compactions, 0); ASSERT_EQ(sfm->GetCompactionsReservedSize(), 0); // Make sure the stat is bumped - ASSERT_GT(dbfull()->immutable_db_options().statistics.get()->getTickerCount(COMPACTION_CANCELLED), 0); + ASSERT_GT(dbfull()->immutable_db_options().statistics.get()->getTickerCount( + COMPACTION_CANCELLED), + 0); ASSERT_EQ(0, dbfull()->immutable_db_options().statistics.get()->getTickerCount( FILES_MARKED_TRASH)); @@ -1629,6 +1631,45 @@ TEST_F(DBSSTTest, GetTotalSstFilesSize) { ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); } +TEST_F(DBSSTTest, OpenDBWithoutGetFileSizeInvocations) { + Options options = CurrentOptions(); + std::unique_ptr env{MockEnv::Create(Env::Default())}; + options.env = env.get(); + options.disable_auto_compactions = true; + options.compression = kNoCompression; + options.enable_blob_files = true; + options.blob_file_size = 32; // create one blob per file + options.skip_checking_sst_file_sizes_on_db_open = true; + + DestroyAndReopen(options); + // Generate 5 files in L0 + for (int i = 0; i < 5; i++) { + for (int j = 0; j < 10; j++) { + std::string val = "val_file_" + std::to_string(i); + ASSERT_OK(Put(Key(j), val)); + } + ASSERT_OK(Flush()); + } + Close(); + + bool is_get_file_size_called = false; + SyncPoint::GetInstance()->SetCallBack( + "MockFileSystem::GetFileSize:CheckFileType", [&](void* arg) { + std::string* filename = reinterpret_cast(arg); + if (filename->find(".blob") != std::string::npos) { + is_get_file_size_called = true; + } + }); + + SyncPoint::GetInstance()->EnableProcessing(); + Reopen(options); + ASSERT_FALSE(is_get_file_size_called); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + Destroy(options); +} + TEST_F(DBSSTTest, GetTotalSstFilesSizeVersionsFilesShared) { Options options = CurrentOptions(); options.disable_auto_compactions = true; diff --git a/env/mock_env.cc b/env/mock_env.cc index 5bdbe4f70..ebdeb1b06 100644 --- a/env/mock_env.cc +++ b/env/mock_env.cc @@ -878,6 +878,7 @@ IOStatus MockFileSystem::GetFileSize(const std::string& fname, uint64_t* file_size, IODebugContext* /*dbg*/) { auto fn = NormalizeMockPath(fname); + TEST_SYNC_POINT_CALLBACK("MockFileSystem::GetFileSize:CheckFileType", &fn); MutexLock lock(&mutex_); auto iter = file_map_.find(fn); if (iter == file_map_.end()) { diff --git a/include/rocksdb/db.h b/include/rocksdb/db.h index 1a6401442..2ea4ffbd5 100644 --- a/include/rocksdb/db.h +++ b/include/rocksdb/db.h @@ -1516,8 +1516,8 @@ class DB { GetColumnFamilyMetaData(DefaultColumnFamily(), metadata); } - // Obtains the LSM-tree meta data of all column families of the DB, - // including metadata for each live table (SST) file in the DB. + // Obtains the LSM-tree meta data of all column families of the DB, including + // metadata for each live table (SST) file and each blob file in the DB. virtual void GetAllColumnFamilyMetaData( std::vector* /*metadata*/) {} diff --git a/include/rocksdb/metadata.h b/include/rocksdb/metadata.h index 6040e56aa..0cdffcd5f 100644 --- a/include/rocksdb/metadata.h +++ b/include/rocksdb/metadata.h @@ -232,7 +232,7 @@ struct ColumnFamilyMetaData { uint64_t blob_file_size = 0; // The number of blob files in this column family. size_t blob_file_count = 0; - // The metadata of the blobs in this column family + // The metadata of the blobs in this column family. std::vector blob_files; };