Clean up VersionStorageInfo a bit (#9494)

Summary:
The patch does some cleanup in and around `VersionStorageInfo`:
* Renames the method `PrepareApply` to `PrepareAppend` in `Version`
to make it clear that it is to be called before appending the `Version` to
`VersionSet` (via `AppendVersion`), not before applying any `VersionEdit`s.
* Introduces a helper method `VersionStorageInfo::PrepareForVersionAppend`
(called by `Version::PrepareAppend`) that encapsulates the population of the
various derived data structures in `VersionStorageInfo`, and turns the
methods computing the derived structures (`UpdateNumNonEmptyLevels`,
`CalculateBaseBytes` etc.) into private helpers.
* Changes `Version::PrepareAppend` so it only calls `UpdateAccumulatedStats`
if the `update_stats` flag is set. (Earlier, this was checked by the callee.)
Related to this, it also moves the call to `ComputeCompensatedSizes` to
`VersionStorageInfo::PrepareForVersionAppend`.
* Updates and cleans up `version_builder_test`, `version_set_test`, and
`compaction_picker_test` so `PrepareForVersionAppend` is called anytime
a new `VersionStorageInfo` is set up or saved. This cleanup also involves
splitting `VersionStorageInfoTest.MaxBytesForLevelDynamic`
into multiple smaller test cases.
* Fixes up a bunch of comments that were outdated or just plain incorrect.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9494

Test Plan: Ran `make check` and the crash test script for a while.

Reviewed By: riversand963

Differential Revision: D33971666

Pulled By: ltamasi

fbshipit-source-id: fda52faac7783041126e4f8dec0fe01bdcadf65a
main
Levi Tamasi 3 years ago committed by Facebook GitHub Bot
parent bec9ab4316
commit 42e0751b3a
  1. 29
      db/compaction/compaction_picker_test.cc
  2. 90
      db/version_builder_test.cc
  3. 1
      db/version_edit.h
  4. 4
      db/version_edit_handler.cc
  5. 145
      db/version_set.cc
  6. 81
      db/version_set.h
  7. 115
      db/version_set_test.cc
  8. 4
      tools/ldb_cmd_test.cc

@ -76,7 +76,7 @@ class CompactionPickerTest : public testing::Test {
options_.num_levels = num_levels; options_.num_levels = num_levels;
vstorage_.reset(new VersionStorageInfo(&icmp_, ucmp_, options_.num_levels, vstorage_.reset(new VersionStorageInfo(&icmp_, ucmp_, options_.num_levels,
style, nullptr, false)); style, nullptr, false));
vstorage_->CalculateBaseBytes(ioptions_, mutable_cf_options_); vstorage_->PrepareForVersionAppend(ioptions_, mutable_cf_options_);
} }
// Create a new VersionStorageInfo object so we can add mode files and then // Create a new VersionStorageInfo object so we can add mode files and then
@ -148,35 +148,10 @@ class CompactionPickerTest : public testing::Test {
ASSERT_OK(builder.SaveTo(temp_vstorage_.get())); ASSERT_OK(builder.SaveTo(temp_vstorage_.get()));
vstorage_ = std::move(temp_vstorage_); vstorage_ = std::move(temp_vstorage_);
} }
vstorage_->CalculateBaseBytes(ioptions_, mutable_cf_options_); vstorage_->PrepareForVersionAppend(ioptions_, mutable_cf_options_);
vstorage_->UpdateFilesByCompactionPri(ioptions_, mutable_cf_options_);
vstorage_->UpdateNumNonEmptyLevels();
vstorage_->GenerateFileIndexer();
vstorage_->GenerateLevelFilesBrief();
vstorage_->ComputeCompactionScore(ioptions_, mutable_cf_options_); vstorage_->ComputeCompactionScore(ioptions_, mutable_cf_options_);
vstorage_->GenerateLevel0NonOverlapping();
vstorage_->ComputeFilesMarkedForCompaction();
vstorage_->SetFinalized(); vstorage_->SetFinalized();
} }
void AddFileToVersionStorage(int level, uint32_t file_number,
const char* smallest, const char* largest,
uint64_t file_size = 1, uint32_t path_id = 0,
SequenceNumber smallest_seq = 100,
SequenceNumber largest_seq = 100,
size_t compensated_file_size = 0,
bool marked_for_compact = false) {
VersionStorageInfo* base_vstorage = vstorage_.release();
vstorage_.reset(new VersionStorageInfo(&icmp_, ucmp_, options_.num_levels,
kCompactionStyleUniversal,
base_vstorage, false));
Add(level, file_number, smallest, largest, file_size, path_id, smallest_seq,
largest_seq, compensated_file_size, marked_for_compact);
VersionBuilder builder(FileOptions(), &ioptions_, nullptr, base_vstorage,
nullptr);
builder.SaveTo(vstorage_.get());
UpdateVersionStorageInfo();
}
private: private:
std::unique_ptr<VersionStorageInfo> temp_vstorage_; std::unique_ptr<VersionStorageInfo> temp_vstorage_;

@ -152,15 +152,14 @@ class VersionBuilderTest : public testing::Test {
return meta; return meta;
} }
void UpdateVersionStorageInfo() { void UpdateVersionStorageInfo(VersionStorageInfo* vstorage) {
vstorage_.UpdateFilesByCompactionPri(ioptions_, mutable_cf_options_); assert(vstorage);
vstorage_.UpdateNumNonEmptyLevels();
vstorage_.GenerateFileIndexer(); vstorage->PrepareForVersionAppend(ioptions_, mutable_cf_options_);
vstorage_.GenerateLevelFilesBrief(); vstorage->SetFinalized();
vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_);
vstorage_.GenerateLevel0NonOverlapping();
vstorage_.SetFinalized();
} }
void UpdateVersionStorageInfo() { UpdateVersionStorageInfo(&vstorage_); }
}; };
void UnrefFilesInVersion(VersionStorageInfo* new_vstorage) { void UnrefFilesInVersion(VersionStorageInfo* new_vstorage) {
@ -187,6 +186,7 @@ TEST_F(VersionBuilderTest, ApplyAndSaveTo) {
Add(3, 27U, "171", "179", 100U); Add(3, 27U, "171", "179", 100U);
Add(3, 28U, "191", "220", 100U); Add(3, 28U, "191", "220", 100U);
Add(3, 29U, "221", "300", 100U); Add(3, 29U, "221", "300", 100U);
UpdateVersionStorageInfo(); UpdateVersionStorageInfo();
VersionEdit version_edit; VersionEdit version_edit;
@ -210,6 +210,8 @@ TEST_F(VersionBuilderTest, ApplyAndSaveTo) {
ASSERT_OK(version_builder.Apply(&version_edit)); ASSERT_OK(version_builder.Apply(&version_edit));
ASSERT_OK(version_builder.SaveTo(&new_vstorage)); ASSERT_OK(version_builder.SaveTo(&new_vstorage));
UpdateVersionStorageInfo(&new_vstorage);
ASSERT_EQ(400U, new_vstorage.NumLevelBytes(2)); ASSERT_EQ(400U, new_vstorage.NumLevelBytes(2));
ASSERT_EQ(300U, new_vstorage.NumLevelBytes(3)); ASSERT_EQ(300U, new_vstorage.NumLevelBytes(3));
@ -228,6 +230,7 @@ TEST_F(VersionBuilderTest, ApplyAndSaveToDynamic) {
Add(5, 26U, "150", "170", 100U); Add(5, 26U, "150", "170", 100U);
Add(5, 27U, "171", "179", 100U); Add(5, 27U, "171", "179", 100U);
UpdateVersionStorageInfo(); UpdateVersionStorageInfo();
VersionEdit version_edit; VersionEdit version_edit;
@ -252,6 +255,8 @@ TEST_F(VersionBuilderTest, ApplyAndSaveToDynamic) {
ASSERT_OK(version_builder.Apply(&version_edit)); ASSERT_OK(version_builder.Apply(&version_edit));
ASSERT_OK(version_builder.SaveTo(&new_vstorage)); ASSERT_OK(version_builder.SaveTo(&new_vstorage));
UpdateVersionStorageInfo(&new_vstorage);
ASSERT_EQ(0U, new_vstorage.NumLevelBytes(0)); ASSERT_EQ(0U, new_vstorage.NumLevelBytes(0));
ASSERT_EQ(100U, new_vstorage.NumLevelBytes(3)); ASSERT_EQ(100U, new_vstorage.NumLevelBytes(3));
ASSERT_EQ(300U, new_vstorage.NumLevelBytes(4)); ASSERT_EQ(300U, new_vstorage.NumLevelBytes(4));
@ -272,6 +277,7 @@ TEST_F(VersionBuilderTest, ApplyAndSaveToDynamic2) {
Add(5, 26U, "150", "170", 100U); Add(5, 26U, "150", "170", 100U);
Add(5, 27U, "171", "179", 100U); Add(5, 27U, "171", "179", 100U);
UpdateVersionStorageInfo(); UpdateVersionStorageInfo();
VersionEdit version_edit; VersionEdit version_edit;
@ -299,6 +305,8 @@ TEST_F(VersionBuilderTest, ApplyAndSaveToDynamic2) {
ASSERT_OK(version_builder.Apply(&version_edit)); ASSERT_OK(version_builder.Apply(&version_edit));
ASSERT_OK(version_builder.SaveTo(&new_vstorage)); ASSERT_OK(version_builder.SaveTo(&new_vstorage));
UpdateVersionStorageInfo(&new_vstorage);
ASSERT_EQ(0U, new_vstorage.NumLevelBytes(0)); ASSERT_EQ(0U, new_vstorage.NumLevelBytes(0));
ASSERT_EQ(100U, new_vstorage.NumLevelBytes(4)); ASSERT_EQ(100U, new_vstorage.NumLevelBytes(4));
ASSERT_EQ(200U, new_vstorage.NumLevelBytes(5)); ASSERT_EQ(200U, new_vstorage.NumLevelBytes(5));
@ -353,6 +361,8 @@ TEST_F(VersionBuilderTest, ApplyMultipleAndSaveTo) {
ASSERT_OK(version_builder.Apply(&version_edit)); ASSERT_OK(version_builder.Apply(&version_edit));
ASSERT_OK(version_builder.SaveTo(&new_vstorage)); ASSERT_OK(version_builder.SaveTo(&new_vstorage));
UpdateVersionStorageInfo(&new_vstorage);
ASSERT_EQ(500U, new_vstorage.NumLevelBytes(2)); ASSERT_EQ(500U, new_vstorage.NumLevelBytes(2));
UnrefFilesInVersion(&new_vstorage); UnrefFilesInVersion(&new_vstorage);
@ -423,6 +433,8 @@ TEST_F(VersionBuilderTest, ApplyDeleteAndSaveTo) {
ASSERT_OK(version_builder.Apply(&version_edit2)); ASSERT_OK(version_builder.Apply(&version_edit2));
ASSERT_OK(version_builder.SaveTo(&new_vstorage)); ASSERT_OK(version_builder.SaveTo(&new_vstorage));
UpdateVersionStorageInfo(&new_vstorage);
ASSERT_EQ(300U, new_vstorage.NumLevelBytes(2)); ASSERT_EQ(300U, new_vstorage.NumLevelBytes(2));
UnrefFilesInVersion(&new_vstorage); UnrefFilesInVersion(&new_vstorage);
@ -433,8 +445,11 @@ TEST_F(VersionBuilderTest, ApplyFileDeletionIncorrectLevel) {
constexpr uint64_t file_number = 2345; constexpr uint64_t file_number = 2345;
constexpr char smallest[] = "bar"; constexpr char smallest[] = "bar";
constexpr char largest[] = "foo"; constexpr char largest[] = "foo";
constexpr uint64_t file_size = 100;
Add(level, file_number, smallest, largest); Add(level, file_number, smallest, largest, file_size);
UpdateVersionStorageInfo();
EnvOptions env_options; EnvOptions env_options;
constexpr TableCache* table_cache = nullptr; constexpr TableCache* table_cache = nullptr;
@ -457,6 +472,8 @@ TEST_F(VersionBuilderTest, ApplyFileDeletionIncorrectLevel) {
} }
TEST_F(VersionBuilderTest, ApplyFileDeletionNotInLSMTree) { TEST_F(VersionBuilderTest, ApplyFileDeletionNotInLSMTree) {
UpdateVersionStorageInfo();
EnvOptions env_options; EnvOptions env_options;
constexpr TableCache* table_cache = nullptr; constexpr TableCache* table_cache = nullptr;
constexpr VersionSet* version_set = nullptr; constexpr VersionSet* version_set = nullptr;
@ -497,6 +514,8 @@ TEST_F(VersionBuilderTest, ApplyFileDeletionAndAddition) {
largest_seq, num_entries, num_deletions, sampled, smallest_seqno, largest_seq, num_entries, num_deletions, sampled, smallest_seqno,
largest_seqno); largest_seqno);
UpdateVersionStorageInfo();
EnvOptions env_options; EnvOptions env_options;
constexpr TableCache* table_cache = nullptr; constexpr TableCache* table_cache = nullptr;
constexpr VersionSet* version_set = nullptr; constexpr VersionSet* version_set = nullptr;
@ -531,6 +550,9 @@ TEST_F(VersionBuilderTest, ApplyFileDeletionAndAddition) {
force_consistency_checks); force_consistency_checks);
ASSERT_OK(builder.SaveTo(&new_vstorage)); ASSERT_OK(builder.SaveTo(&new_vstorage));
UpdateVersionStorageInfo(&new_vstorage);
ASSERT_EQ(new_vstorage.GetFileLocation(file_number).GetLevel(), level); ASSERT_EQ(new_vstorage.GetFileLocation(file_number).GetLevel(), level);
UnrefFilesInVersion(&new_vstorage); UnrefFilesInVersion(&new_vstorage);
@ -541,8 +563,11 @@ TEST_F(VersionBuilderTest, ApplyFileAdditionAlreadyInBase) {
constexpr uint64_t file_number = 2345; constexpr uint64_t file_number = 2345;
constexpr char smallest[] = "bar"; constexpr char smallest[] = "bar";
constexpr char largest[] = "foo"; constexpr char largest[] = "foo";
constexpr uint64_t file_size = 10000;
Add(level, file_number, smallest, largest, file_size);
Add(level, file_number, smallest, largest); UpdateVersionStorageInfo();
EnvOptions env_options; EnvOptions env_options;
constexpr TableCache* table_cache = nullptr; constexpr TableCache* table_cache = nullptr;
@ -555,7 +580,6 @@ TEST_F(VersionBuilderTest, ApplyFileAdditionAlreadyInBase) {
constexpr int new_level = 2; constexpr int new_level = 2;
constexpr uint32_t path_id = 0; constexpr uint32_t path_id = 0;
constexpr uint64_t file_size = 10000;
constexpr SequenceNumber smallest_seqno = 100; constexpr SequenceNumber smallest_seqno = 100;
constexpr SequenceNumber largest_seqno = 1000; constexpr SequenceNumber largest_seqno = 1000;
constexpr bool marked_for_compaction = false; constexpr bool marked_for_compaction = false;
@ -576,6 +600,8 @@ TEST_F(VersionBuilderTest, ApplyFileAdditionAlreadyInBase) {
} }
TEST_F(VersionBuilderTest, ApplyFileAdditionAlreadyApplied) { TEST_F(VersionBuilderTest, ApplyFileAdditionAlreadyApplied) {
UpdateVersionStorageInfo();
EnvOptions env_options; EnvOptions env_options;
constexpr TableCache* table_cache = nullptr; constexpr TableCache* table_cache = nullptr;
constexpr VersionSet* version_set = nullptr; constexpr VersionSet* version_set = nullptr;
@ -625,6 +651,8 @@ TEST_F(VersionBuilderTest, ApplyFileAdditionAlreadyApplied) {
} }
TEST_F(VersionBuilderTest, ApplyFileAdditionAndDeletion) { TEST_F(VersionBuilderTest, ApplyFileAdditionAndDeletion) {
UpdateVersionStorageInfo();
constexpr int level = 1; constexpr int level = 1;
constexpr uint64_t file_number = 2345; constexpr uint64_t file_number = 2345;
constexpr uint32_t path_id = 0; constexpr uint32_t path_id = 0;
@ -666,12 +694,17 @@ TEST_F(VersionBuilderTest, ApplyFileAdditionAndDeletion) {
force_consistency_checks); force_consistency_checks);
ASSERT_OK(builder.SaveTo(&new_vstorage)); ASSERT_OK(builder.SaveTo(&new_vstorage));
UpdateVersionStorageInfo(&new_vstorage);
ASSERT_FALSE(new_vstorage.GetFileLocation(file_number).IsValid()); ASSERT_FALSE(new_vstorage.GetFileLocation(file_number).IsValid());
UnrefFilesInVersion(&new_vstorage); UnrefFilesInVersion(&new_vstorage);
} }
TEST_F(VersionBuilderTest, ApplyBlobFileAddition) { TEST_F(VersionBuilderTest, ApplyBlobFileAddition) {
UpdateVersionStorageInfo();
EnvOptions env_options; EnvOptions env_options;
constexpr TableCache* table_cache = nullptr; constexpr TableCache* table_cache = nullptr;
constexpr VersionSet* version_set = nullptr; constexpr VersionSet* version_set = nullptr;
@ -705,6 +738,8 @@ TEST_F(VersionBuilderTest, ApplyBlobFileAddition) {
ASSERT_OK(builder.SaveTo(&new_vstorage)); ASSERT_OK(builder.SaveTo(&new_vstorage));
UpdateVersionStorageInfo(&new_vstorage);
const auto& new_blob_files = new_vstorage.GetBlobFiles(); const auto& new_blob_files = new_vstorage.GetBlobFiles();
ASSERT_EQ(new_blob_files.size(), 1); ASSERT_EQ(new_blob_files.size(), 1);
@ -741,6 +776,8 @@ TEST_F(VersionBuilderTest, ApplyBlobFileAdditionAlreadyInBase) {
checksum_value, BlobFileMetaData::LinkedSsts(), garbage_blob_count, checksum_value, BlobFileMetaData::LinkedSsts(), garbage_blob_count,
garbage_blob_bytes); garbage_blob_bytes);
UpdateVersionStorageInfo();
EnvOptions env_options; EnvOptions env_options;
constexpr TableCache* table_cache = nullptr; constexpr TableCache* table_cache = nullptr;
constexpr VersionSet* version_set = nullptr; constexpr VersionSet* version_set = nullptr;
@ -761,6 +798,8 @@ TEST_F(VersionBuilderTest, ApplyBlobFileAdditionAlreadyInBase) {
TEST_F(VersionBuilderTest, ApplyBlobFileAdditionAlreadyApplied) { TEST_F(VersionBuilderTest, ApplyBlobFileAdditionAlreadyApplied) {
// Attempt to add the same blob file twice using version edits. // Attempt to add the same blob file twice using version edits.
UpdateVersionStorageInfo();
EnvOptions env_options; EnvOptions env_options;
constexpr TableCache* table_cache = nullptr; constexpr TableCache* table_cache = nullptr;
constexpr VersionSet* version_set = nullptr; constexpr VersionSet* version_set = nullptr;
@ -813,6 +852,8 @@ TEST_F(VersionBuilderTest, ApplyBlobFileGarbageFileInBase) {
// Add dummy table file to ensure the blob file is referenced. // Add dummy table file to ensure the blob file is referenced.
AddDummyFile(table_file_number, blob_file_number); AddDummyFile(table_file_number, blob_file_number);
UpdateVersionStorageInfo();
EnvOptions env_options; EnvOptions env_options;
constexpr TableCache* table_cache = nullptr; constexpr TableCache* table_cache = nullptr;
constexpr VersionSet* version_set = nullptr; constexpr VersionSet* version_set = nullptr;
@ -837,6 +878,8 @@ TEST_F(VersionBuilderTest, ApplyBlobFileGarbageFileInBase) {
ASSERT_OK(builder.SaveTo(&new_vstorage)); ASSERT_OK(builder.SaveTo(&new_vstorage));
UpdateVersionStorageInfo(&new_vstorage);
const auto& new_blob_files = new_vstorage.GetBlobFiles(); const auto& new_blob_files = new_vstorage.GetBlobFiles();
ASSERT_EQ(new_blob_files.size(), 1); ASSERT_EQ(new_blob_files.size(), 1);
@ -862,6 +905,8 @@ TEST_F(VersionBuilderTest, ApplyBlobFileGarbageFileInBase) {
TEST_F(VersionBuilderTest, ApplyBlobFileGarbageFileAdditionApplied) { TEST_F(VersionBuilderTest, ApplyBlobFileGarbageFileAdditionApplied) {
// Increase the amount of garbage for a blob file added using a version edit. // Increase the amount of garbage for a blob file added using a version edit.
UpdateVersionStorageInfo();
EnvOptions env_options; EnvOptions env_options;
constexpr TableCache* table_cache = nullptr; constexpr TableCache* table_cache = nullptr;
constexpr VersionSet* version_set = nullptr; constexpr VersionSet* version_set = nullptr;
@ -905,6 +950,8 @@ TEST_F(VersionBuilderTest, ApplyBlobFileGarbageFileAdditionApplied) {
ASSERT_OK(builder.SaveTo(&new_vstorage)); ASSERT_OK(builder.SaveTo(&new_vstorage));
UpdateVersionStorageInfo(&new_vstorage);
const auto& new_blob_files = new_vstorage.GetBlobFiles(); const auto& new_blob_files = new_vstorage.GetBlobFiles();
ASSERT_EQ(new_blob_files.size(), 1); ASSERT_EQ(new_blob_files.size(), 1);
@ -928,6 +975,8 @@ TEST_F(VersionBuilderTest, ApplyBlobFileGarbageFileNotFound) {
// Attempt to increase the amount of garbage for a blob file that is // Attempt to increase the amount of garbage for a blob file that is
// neither in the base version, nor was it added using a version edit. // neither in the base version, nor was it added using a version edit.
UpdateVersionStorageInfo();
EnvOptions env_options; EnvOptions env_options;
constexpr TableCache* table_cache = nullptr; constexpr TableCache* table_cache = nullptr;
constexpr VersionSet* version_set = nullptr; constexpr VersionSet* version_set = nullptr;
@ -953,6 +1002,8 @@ TEST_F(VersionBuilderTest, BlobFileGarbageOverflow) {
// Test that VersionEdits that would result in the count/total size of garbage // Test that VersionEdits that would result in the count/total size of garbage
// exceeding the count/total size of all blobs are rejected. // exceeding the count/total size of all blobs are rejected.
UpdateVersionStorageInfo();
EnvOptions env_options; EnvOptions env_options;
constexpr TableCache* table_cache = nullptr; constexpr TableCache* table_cache = nullptr;
constexpr VersionSet* version_set = nullptr; constexpr VersionSet* version_set = nullptr;
@ -1032,6 +1083,8 @@ TEST_F(VersionBuilderTest, SaveBlobFilesTo) {
AddDummyFile(table_file_number, blob_file_number); AddDummyFile(table_file_number, blob_file_number);
} }
UpdateVersionStorageInfo();
EnvOptions env_options; EnvOptions env_options;
constexpr TableCache* table_cache = nullptr; constexpr TableCache* table_cache = nullptr;
constexpr VersionSet* version_set = nullptr; constexpr VersionSet* version_set = nullptr;
@ -1069,6 +1122,8 @@ TEST_F(VersionBuilderTest, SaveBlobFilesTo) {
ASSERT_OK(builder.SaveTo(&new_vstorage)); ASSERT_OK(builder.SaveTo(&new_vstorage));
UpdateVersionStorageInfo(&new_vstorage);
const auto& new_blob_files = new_vstorage.GetBlobFiles(); const auto& new_blob_files = new_vstorage.GetBlobFiles();
ASSERT_EQ(new_blob_files.size(), 3); ASSERT_EQ(new_blob_files.size(), 3);
@ -1115,6 +1170,8 @@ TEST_F(VersionBuilderTest, SaveBlobFilesTo) {
ASSERT_OK(second_builder.SaveTo(&newer_vstorage)); ASSERT_OK(second_builder.SaveTo(&newer_vstorage));
UpdateVersionStorageInfo(&newer_vstorage);
const auto& newer_blob_files = newer_vstorage.GetBlobFiles(); const auto& newer_blob_files = newer_vstorage.GetBlobFiles();
ASSERT_EQ(newer_blob_files.size(), 2); ASSERT_EQ(newer_blob_files.size(), 2);
@ -1149,6 +1206,8 @@ TEST_F(VersionBuilderTest, SaveBlobFilesToConcurrentJobs) {
BlobFileMetaData::LinkedSsts{base_table_file_number}, BlobFileMetaData::LinkedSsts{base_table_file_number},
garbage_blob_count, garbage_blob_bytes); garbage_blob_count, garbage_blob_bytes);
UpdateVersionStorageInfo();
EnvOptions env_options; EnvOptions env_options;
constexpr TableCache* table_cache = nullptr; constexpr TableCache* table_cache = nullptr;
constexpr VersionSet* version_set = nullptr; constexpr VersionSet* version_set = nullptr;
@ -1194,6 +1253,8 @@ TEST_F(VersionBuilderTest, SaveBlobFilesToConcurrentJobs) {
ASSERT_OK(builder.SaveTo(&new_vstorage)); ASSERT_OK(builder.SaveTo(&new_vstorage));
UpdateVersionStorageInfo(&new_vstorage);
const auto& new_blob_files = new_vstorage.GetBlobFiles(); const auto& new_blob_files = new_vstorage.GetBlobFiles();
ASSERT_EQ(new_blob_files.size(), 2); ASSERT_EQ(new_blob_files.size(), 2);
@ -1293,6 +1354,8 @@ TEST_F(VersionBuilderTest, CheckConsistencyForBlobFiles) {
ASSERT_OK(builder.SaveTo(&new_vstorage)); ASSERT_OK(builder.SaveTo(&new_vstorage));
UpdateVersionStorageInfo(&new_vstorage);
UnrefFilesInVersion(&new_vstorage); UnrefFilesInVersion(&new_vstorage);
} }
@ -1581,6 +1644,8 @@ TEST_F(VersionBuilderTest, MaintainLinkedSstsForBlobFiles) {
ASSERT_OK(builder.SaveTo(&new_vstorage)); ASSERT_OK(builder.SaveTo(&new_vstorage));
UpdateVersionStorageInfo(&new_vstorage);
{ {
const auto& blob_files = new_vstorage.GetBlobFiles(); const auto& blob_files = new_vstorage.GetBlobFiles();
ASSERT_EQ(blob_files.size(), 5); ASSERT_EQ(blob_files.size(), 5);
@ -1605,6 +1670,7 @@ TEST_F(VersionBuilderTest, MaintainLinkedSstsForBlobFiles) {
TEST_F(VersionBuilderTest, CheckConsistencyForFileDeletedTwice) { TEST_F(VersionBuilderTest, CheckConsistencyForFileDeletedTwice) {
Add(0, 1U, "150", "200", 100U); Add(0, 1U, "150", "200", 100U);
UpdateVersionStorageInfo(); UpdateVersionStorageInfo();
VersionEdit version_edit; VersionEdit version_edit;
@ -1622,6 +1688,8 @@ TEST_F(VersionBuilderTest, CheckConsistencyForFileDeletedTwice) {
ASSERT_OK(version_builder.Apply(&version_edit)); ASSERT_OK(version_builder.Apply(&version_edit));
ASSERT_OK(version_builder.SaveTo(&new_vstorage)); ASSERT_OK(version_builder.SaveTo(&new_vstorage));
UpdateVersionStorageInfo(&new_vstorage);
VersionBuilder version_builder2(env_options, &ioptions_, table_cache, VersionBuilder version_builder2(env_options, &ioptions_, table_cache,
&new_vstorage, version_set); &new_vstorage, version_set);
VersionStorageInfo new_vstorage2(&icmp_, ucmp_, options_.num_levels, VersionStorageInfo new_vstorage2(&icmp_, ucmp_, options_.num_levels,

@ -396,7 +396,6 @@ class VersionEdit {
const DeletedFiles& GetDeletedFiles() const { return deleted_files_; } const DeletedFiles& GetDeletedFiles() const { return deleted_files_; }
// Add the specified table file at the specified level. // Add the specified table file at the specified level.
// REQUIRES: This version has not been saved (see VersionSet::SaveTo)
// REQUIRES: "smallest" and "largest" are smallest and largest keys in file // REQUIRES: "smallest" and "largest" are smallest and largest keys in file
// REQUIRES: "oldest_blob_file_number" is the number of the oldest blob file // REQUIRES: "oldest_blob_file_number" is the number of the oldest blob file
// referred to by this file if any, kInvalidBlobFileNumber otherwise. // referred to by this file if any, kInvalidBlobFileNumber otherwise.

@ -532,7 +532,7 @@ Status VersionEditHandler::MaybeCreateVersion(const VersionEdit& /*edit*/,
s = builder->SaveTo(v->storage_info()); s = builder->SaveTo(v->storage_info());
if (s.ok()) { if (s.ok()) {
// Install new version // Install new version
v->PrepareApply( v->PrepareAppend(
*cfd->GetLatestMutableCFOptions(), *cfd->GetLatestMutableCFOptions(),
!(version_set_->db_options_->skip_stats_update_on_db_open)); !(version_set_->db_options_->skip_stats_update_on_db_open));
version_set_->AppendVersion(cfd, v); version_set_->AppendVersion(cfd, v);
@ -806,7 +806,7 @@ Status VersionEditHandlerPointInTime::MaybeCreateVersion(
version_set_->current_version_number_++); version_set_->current_version_number_++);
s = builder->SaveTo(version->storage_info()); s = builder->SaveTo(version->storage_info());
if (s.ok()) { if (s.ok()) {
version->PrepareApply( version->PrepareAppend(
*cfd->GetLatestMutableCFOptions(), *cfd->GetLatestMutableCFOptions(),
!version_set_->db_options_->skip_stats_update_on_db_open); !version_set_->db_options_->skip_stats_update_on_db_open);
auto v_iter = versions_.find(cfd->GetID()); auto v_iter = versions_.find(cfd->GetID());

@ -2402,21 +2402,30 @@ void VersionStorageInfo::GenerateLevelFilesBrief() {
} }
} }
void Version::PrepareApply( void VersionStorageInfo::PrepareForVersionAppend(
const MutableCFOptions& mutable_cf_options, const ImmutableOptions& immutable_options,
bool update_stats) { const MutableCFOptions& mutable_cf_options) {
ComputeCompensatedSizes();
UpdateNumNonEmptyLevels();
CalculateBaseBytes(immutable_options, mutable_cf_options);
UpdateFilesByCompactionPri(immutable_options, mutable_cf_options);
GenerateFileIndexer();
GenerateLevelFilesBrief();
GenerateLevel0NonOverlapping();
GenerateBottommostFiles();
}
void Version::PrepareAppend(const MutableCFOptions& mutable_cf_options,
bool update_stats) {
TEST_SYNC_POINT_CALLBACK( TEST_SYNC_POINT_CALLBACK(
"Version::PrepareApply:forced_check", "Version::PrepareAppend:forced_check",
reinterpret_cast<void*>(&storage_info_.force_consistency_checks_)); reinterpret_cast<void*>(&storage_info_.force_consistency_checks_));
UpdateAccumulatedStats(update_stats);
storage_info_.UpdateNumNonEmptyLevels(); if (update_stats) {
storage_info_.CalculateBaseBytes(*cfd_->ioptions(), mutable_cf_options); UpdateAccumulatedStats();
storage_info_.UpdateFilesByCompactionPri(*cfd_->ioptions(), }
mutable_cf_options);
storage_info_.GenerateFileIndexer(); storage_info_.PrepareForVersionAppend(*cfd_->ioptions(), mutable_cf_options);
storage_info_.GenerateLevelFilesBrief();
storage_info_.GenerateLevel0NonOverlapping();
storage_info_.GenerateBottommostFiles();
} }
bool Version::MaybeInitializeFileMetaData(FileMetaData* file_meta) { bool Version::MaybeInitializeFileMetaData(FileMetaData* file_meta) {
@ -2470,59 +2479,54 @@ void VersionStorageInfo::RemoveCurrentStats(FileMetaData* file_meta) {
} }
} }
void Version::UpdateAccumulatedStats(bool update_stats) { void Version::UpdateAccumulatedStats() {
if (update_stats) { // maximum number of table properties loaded from files.
// maximum number of table properties loaded from files. const int kMaxInitCount = 20;
const int kMaxInitCount = 20; int init_count = 0;
int init_count = 0; // here only the first kMaxInitCount files which haven't been
// here only the first kMaxInitCount files which haven't been // initialized from file will be updated with num_deletions.
// initialized from file will be updated with num_deletions. // The motivation here is to cap the maximum I/O per Version creation.
// The motivation here is to cap the maximum I/O per Version creation. // The reason for choosing files from lower-level instead of higher-level
// The reason for choosing files from lower-level instead of higher-level // is that such design is able to propagate the initialization from
// is that such design is able to propagate the initialization from // lower-level to higher-level: When the num_deletions of lower-level
// lower-level to higher-level: When the num_deletions of lower-level // files are updated, it will make the lower-level files have accurate
// files are updated, it will make the lower-level files have accurate // compensated_file_size, making lower-level to higher-level compaction
// compensated_file_size, making lower-level to higher-level compaction // will be triggered, which creates higher-level files whose num_deletions
// will be triggered, which creates higher-level files whose num_deletions // will be updated here.
// will be updated here. for (int level = 0;
for (int level = 0; level < storage_info_.num_levels_ && init_count < kMaxInitCount;
level < storage_info_.num_levels_ && init_count < kMaxInitCount; ++level) {
++level) { for (auto* file_meta : storage_info_.files_[level]) {
for (auto* file_meta : storage_info_.files_[level]) { if (MaybeInitializeFileMetaData(file_meta)) {
if (MaybeInitializeFileMetaData(file_meta)) { // each FileMeta will be initialized only once.
// each FileMeta will be initialized only once. storage_info_.UpdateAccumulatedStats(file_meta);
storage_info_.UpdateAccumulatedStats(file_meta); // when option "max_open_files" is -1, all the file metadata has
// when option "max_open_files" is -1, all the file metadata has // already been read, so MaybeInitializeFileMetaData() won't incur
// already been read, so MaybeInitializeFileMetaData() won't incur // any I/O cost. "max_open_files=-1" means that the table cache passed
// any I/O cost. "max_open_files=-1" means that the table cache passed // to the VersionSet and then to the ColumnFamilySet has a size of
// to the VersionSet and then to the ColumnFamilySet has a size of // TableCache::kInfiniteCapacity
// TableCache::kInfiniteCapacity if (vset_->GetColumnFamilySet()->get_table_cache()->GetCapacity() ==
if (vset_->GetColumnFamilySet()->get_table_cache()->GetCapacity() == TableCache::kInfiniteCapacity) {
TableCache::kInfiniteCapacity) { continue;
continue; }
} if (++init_count >= kMaxInitCount) {
if (++init_count >= kMaxInitCount) { break;
break;
}
} }
} }
} }
// In case all sampled-files contain only deletion entries, then we }
// load the table-property of a file in higher-level to initialize // In case all sampled-files contain only deletion entries, then we
// that value. // load the table-property of a file in higher-level to initialize
for (int level = storage_info_.num_levels_ - 1; // that value.
storage_info_.accumulated_raw_value_size_ == 0 && level >= 0; for (int level = storage_info_.num_levels_ - 1;
--level) { storage_info_.accumulated_raw_value_size_ == 0 && level >= 0; --level) {
for (int i = static_cast<int>(storage_info_.files_[level].size()) - 1; for (int i = static_cast<int>(storage_info_.files_[level].size()) - 1;
storage_info_.accumulated_raw_value_size_ == 0 && i >= 0; --i) { storage_info_.accumulated_raw_value_size_ == 0 && i >= 0; --i) {
if (MaybeInitializeFileMetaData(storage_info_.files_[level][i])) { if (MaybeInitializeFileMetaData(storage_info_.files_[level][i])) {
storage_info_.UpdateAccumulatedStats(storage_info_.files_[level][i]); storage_info_.UpdateAccumulatedStats(storage_info_.files_[level][i]);
}
} }
} }
} }
storage_info_.ComputeCompensatedSizes();
} }
void VersionStorageInfo::ComputeCompensatedSizes() { void VersionStorageInfo::ComputeCompensatedSizes() {
@ -3062,17 +3066,9 @@ void VersionStorageInfo::AddBlobFile(
blob_files_.emplace_hint(it, blob_file_number, std::move(blob_file_meta)); blob_files_.emplace_hint(it, blob_file_number, std::move(blob_file_meta));
} }
// Version::PrepareApply() need to be called before calling the function, or
// following functions called:
// 1. UpdateNumNonEmptyLevels();
// 2. CalculateBaseBytes();
// 3. UpdateFilesByCompactionPri();
// 4. GenerateFileIndexer();
// 5. GenerateLevelFilesBrief();
// 6. GenerateLevel0NonOverlapping();
// 7. GenerateBottommostFiles();
void VersionStorageInfo::SetFinalized() { void VersionStorageInfo::SetFinalized() {
finalized_ = true; finalized_ = true;
#ifndef NDEBUG #ifndef NDEBUG
if (compaction_style_ != kCompactionStyleLevel) { if (compaction_style_ != kCompactionStyleLevel) {
// Not level based compaction. // Not level based compaction.
@ -4418,8 +4414,10 @@ Status VersionSet::ProcessManifestWrites(
if (s.ok()) { if (s.ok()) {
if (!first_writer.edit_list.front()->IsColumnFamilyManipulation()) { if (!first_writer.edit_list.front()->IsColumnFamilyManipulation()) {
constexpr bool update_stats = true;
for (int i = 0; i < static_cast<int>(versions.size()); ++i) { for (int i = 0; i < static_cast<int>(versions.size()); ++i) {
versions[i]->PrepareApply(*mutable_cf_options_ptrs[i], true); versions[i]->PrepareAppend(*mutable_cf_options_ptrs[i], update_stats);
} }
} }
@ -5914,9 +5912,10 @@ ColumnFamilyData* VersionSet::CreateColumnFamily(
*new_cfd->GetLatestMutableCFOptions(), io_tracer_, *new_cfd->GetLatestMutableCFOptions(), io_tracer_,
current_version_number_++); current_version_number_++);
// Fill level target base information. constexpr bool update_stats = false;
v->storage_info()->CalculateBaseBytes(*new_cfd->ioptions(),
*new_cfd->GetLatestMutableCFOptions()); v->PrepareAppend(*new_cfd->GetLatestMutableCFOptions(), update_stats);
AppendVersion(new_cfd, v); AppendVersion(new_cfd, v);
// GetLatestMutableCFOptions() is safe here without mutex since the // GetLatestMutableCFOptions() is safe here without mutex since the
// cfd is not available to client // cfd is not available to client

@ -129,14 +129,11 @@ class VersionStorageInfo {
void AddBlobFile(std::shared_ptr<BlobFileMetaData> blob_file_meta); void AddBlobFile(std::shared_ptr<BlobFileMetaData> blob_file_meta);
void SetFinalized(); void PrepareForVersionAppend(const ImmutableOptions& immutable_options,
const MutableCFOptions& mutable_cf_options);
// Update num_non_empty_levels_.
void UpdateNumNonEmptyLevels();
void GenerateFileIndexer() { // REQUIRES: PrepareForVersionAppend has been called
file_indexer_.UpdateIndex(&arena_, num_non_empty_levels_, files_); void SetFinalized();
}
// Update the accumulated stats from a file-meta. // Update the accumulated stats from a file-meta.
void UpdateAccumulatedStats(FileMetaData* file_meta); void UpdateAccumulatedStats(FileMetaData* file_meta);
@ -144,8 +141,6 @@ class VersionStorageInfo {
// Decrease the current stat from a to-be-deleted file-meta // Decrease the current stat from a to-be-deleted file-meta
void RemoveCurrentStats(FileMetaData* file_meta); void RemoveCurrentStats(FileMetaData* file_meta);
void ComputeCompensatedSizes();
// Updates internal structures that keep track of compaction scores // Updates internal structures that keep track of compaction scores
// We use compaction scores to figure out which compaction to do next // We use compaction scores to figure out which compaction to do next
// REQUIRES: db_mutex held!! // REQUIRES: db_mutex held!!
@ -192,24 +187,10 @@ class VersionStorageInfo {
double blob_garbage_collection_age_cutoff, double blob_garbage_collection_age_cutoff,
double blob_garbage_collection_force_threshold); double blob_garbage_collection_force_threshold);
// Generate level_files_brief_ from files_
void GenerateLevelFilesBrief();
// Sort all files for this version based on their file size and
// record results in files_by_compaction_pri_. The largest files are listed
// first.
void UpdateFilesByCompactionPri(const ImmutableOptions& immutable_options,
const MutableCFOptions& mutable_cf_options);
void GenerateLevel0NonOverlapping();
bool level0_non_overlapping() const { bool level0_non_overlapping() const {
return level0_non_overlapping_; return level0_non_overlapping_;
} }
// Check whether each file in this version is bottommost (i.e., nothing in its
// key-range could possibly exist in an older file/level).
// REQUIRES: This version has not been saved
void GenerateBottommostFiles();
// Updates the oldest snapshot and related internal state, like the bottommost // Updates the oldest snapshot and related internal state, like the bottommost
// files marked for compaction. // files marked for compaction.
// REQUIRES: DB mutex held // REQUIRES: DB mutex held
@ -270,14 +251,13 @@ class VersionStorageInfo {
int num_levels() const { return num_levels_; } int num_levels() const { return num_levels_; }
// REQUIRES: This version has been saved (see VersionSet::SaveTo) // REQUIRES: PrepareForVersionAppend has been called
int num_non_empty_levels() const { int num_non_empty_levels() const {
assert(finalized_); assert(finalized_);
return num_non_empty_levels_; return num_non_empty_levels_;
} }
// REQUIRES: This version has been finalized. // REQUIRES: PrepareForVersionAppend has been called
// (CalculateBaseBytes() is called)
// This may or may not return number of level files. It is to keep backward // This may or may not return number of level files. It is to keep backward
// compatible behavior in universal compaction. // compatible behavior in universal compaction.
int l0_delay_trigger_count() const { return l0_delay_trigger_count_; } int l0_delay_trigger_count() const { return l0_delay_trigger_count_; }
@ -372,13 +352,13 @@ class VersionStorageInfo {
return level_files_brief_[level]; return level_files_brief_[level];
} }
// REQUIRES: This version has been saved (see VersionSet::SaveTo) // REQUIRES: PrepareForVersionAppend has been called
const std::vector<int>& FilesByCompactionPri(int level) const { const std::vector<int>& FilesByCompactionPri(int level) const {
assert(finalized_); assert(finalized_);
return files_by_compaction_pri_[level]; return files_by_compaction_pri_[level];
} }
// REQUIRES: This version has been saved (see VersionSet::SaveTo) // REQUIRES: ComputeCompactionScore has been called
// REQUIRES: DB mutex held during access // REQUIRES: DB mutex held during access
const autovector<std::pair<int, FileMetaData*>>& FilesMarkedForCompaction() const autovector<std::pair<int, FileMetaData*>>& FilesMarkedForCompaction()
const { const {
@ -386,14 +366,14 @@ class VersionStorageInfo {
return files_marked_for_compaction_; return files_marked_for_compaction_;
} }
// REQUIRES: This version has been saved (see VersionSet::SaveTo) // REQUIRES: ComputeCompactionScore has been called
// REQUIRES: DB mutex held during access // REQUIRES: DB mutex held during access
const autovector<std::pair<int, FileMetaData*>>& ExpiredTtlFiles() const { const autovector<std::pair<int, FileMetaData*>>& ExpiredTtlFiles() const {
assert(finalized_); assert(finalized_);
return expired_ttl_files_; return expired_ttl_files_;
} }
// REQUIRES: This version has been saved (see VersionSet::SaveTo) // REQUIRES: ComputeCompactionScore has been called
// REQUIRES: DB mutex held during access // REQUIRES: DB mutex held during access
const autovector<std::pair<int, FileMetaData*>>& const autovector<std::pair<int, FileMetaData*>>&
FilesMarkedForPeriodicCompaction() const { FilesMarkedForPeriodicCompaction() const {
@ -405,7 +385,7 @@ class VersionStorageInfo {
files_marked_for_periodic_compaction_.emplace_back(level, f); files_marked_for_periodic_compaction_.emplace_back(level, f);
} }
// REQUIRES: This version has been saved (see VersionSet::SaveTo) // REQUIRES: ComputeCompactionScore has been called
// REQUIRES: DB mutex held during access // REQUIRES: DB mutex held during access
const autovector<std::pair<int, FileMetaData*>>& const autovector<std::pair<int, FileMetaData*>>&
BottommostFilesMarkedForCompaction() const { BottommostFilesMarkedForCompaction() const {
@ -413,7 +393,7 @@ class VersionStorageInfo {
return bottommost_files_marked_for_compaction_; return bottommost_files_marked_for_compaction_;
} }
// REQUIRES: This version has been saved (see VersionSet::SaveTo) // REQUIRES: ComputeCompactionScore has been called
// REQUIRES: DB mutex held during access // REQUIRES: DB mutex held during access
const autovector<std::pair<int, FileMetaData*>>& FilesMarkedForForcedBlobGC() const autovector<std::pair<int, FileMetaData*>>& FilesMarkedForForcedBlobGC()
const { const {
@ -436,7 +416,7 @@ class VersionStorageInfo {
return next_file_to_compact_by_size_[level]; return next_file_to_compact_by_size_[level];
} }
// REQUIRES: This version has been saved (see VersionSet::SaveTo) // REQUIRES: PrepareForVersionAppend has been called
const FileIndexer& file_indexer() const { const FileIndexer& file_indexer() const {
assert(finalized_); assert(finalized_);
return file_indexer_; return file_indexer_;
@ -498,10 +478,6 @@ class VersionStorageInfo {
// Returns maximum total bytes of data on a given level. // Returns maximum total bytes of data on a given level.
uint64_t MaxBytesForLevel(int level) const; uint64_t MaxBytesForLevel(int level) const;
// Must be called after any change to MutableCFOptions.
void CalculateBaseBytes(const ImmutableOptions& ioptions,
const MutableCFOptions& options);
// Returns an estimate of the amount of live data in bytes. // Returns an estimate of the amount of live data in bytes.
uint64_t EstimateLiveDataSize() const; uint64_t EstimateLiveDataSize() const;
@ -530,6 +506,21 @@ class VersionStorageInfo {
int last_level, int last_l0_idx); int last_level, int last_l0_idx);
private: private:
void ComputeCompensatedSizes();
void UpdateNumNonEmptyLevels();
void CalculateBaseBytes(const ImmutableOptions& ioptions,
const MutableCFOptions& options);
void UpdateFilesByCompactionPri(const ImmutableOptions& immutable_options,
const MutableCFOptions& mutable_cf_options);
void GenerateFileIndexer() {
file_indexer_.UpdateIndex(&arena_, num_non_empty_levels_, files_);
}
void GenerateLevelFilesBrief();
void GenerateLevel0NonOverlapping();
void GenerateBottommostFiles();
const InternalKeyComparator* internal_comparator_; const InternalKeyComparator* internal_comparator_;
const Comparator* user_comparator_; const Comparator* user_comparator_;
int num_levels_; // Number of levels int num_levels_; // Number of levels
@ -619,7 +610,7 @@ class VersionStorageInfo {
// Level that should be compacted next and its compaction score. // Level that should be compacted next and its compaction score.
// Score < 1 means compaction is not strictly needed. These fields // Score < 1 means compaction is not strictly needed. These fields
// are initialized by Finalize(). // are initialized by ComputeCompactionScore.
// The most critical level to be compacted is listed first // The most critical level to be compacted is listed first
// These are used to pick the best compaction level // These are used to pick the best compaction level
std::vector<double> compaction_score_; std::vector<double> compaction_score_;
@ -667,7 +658,6 @@ class Version {
// yield the contents of this Version when merged together. // yield the contents of this Version when merged together.
// @param read_options Must outlive any iterator built by // @param read_options Must outlive any iterator built by
// `merger_iter_builder`. // `merger_iter_builder`.
// REQUIRES: This version has been saved (see VersionSet::SaveTo).
void AddIterators(const ReadOptions& read_options, void AddIterators(const ReadOptions& read_options,
const FileOptions& soptions, const FileOptions& soptions,
MergeIteratorBuilder* merger_iter_builder, MergeIteratorBuilder* merger_iter_builder,
@ -740,10 +730,11 @@ class Version {
void MultiGetBlob(const ReadOptions& read_options, MultiGetRange& range, void MultiGetBlob(const ReadOptions& read_options, MultiGetRange& range,
std::unordered_map<uint64_t, BlobReadRequests>& blob_rqs); std::unordered_map<uint64_t, BlobReadRequests>& blob_rqs);
// Loads some stats information from files. Call without mutex held. It needs // Loads some stats information from files (if update_stats is set) and
// to be called before applying the version to the version set. // populates derived data structures. Call without mutex held. It needs to be
void PrepareApply(const MutableCFOptions& mutable_cf_options, // called before appending the version to the version set.
bool update_stats); void PrepareAppend(const MutableCFOptions& mutable_cf_options,
bool update_stats);
// Reference count management (so Versions do not disappear out from // Reference count management (so Versions do not disappear out from
// under live iterators) // under live iterators)
@ -849,7 +840,7 @@ class Version {
// Update the accumulated stats associated with the current version. // Update the accumulated stats associated with the current version.
// This accumulated stats will be used in compaction. // This accumulated stats will be used in compaction.
void UpdateAccumulatedStats(bool update_stats); void UpdateAccumulatedStats();
ColumnFamilyData* cfd_; // ColumnFamilyData to which this Version belongs ColumnFamilyData* cfd_; // ColumnFamilyData to which this Version belongs
Logger* info_log_; Logger* info_log_;
@ -1296,7 +1287,7 @@ class VersionSet {
new Version(cfd, this, file_options_, mutable_cf_options, io_tracer_); new Version(cfd, this, file_options_, mutable_cf_options, io_tracer_);
constexpr bool update_stats = false; constexpr bool update_stats = false;
version->PrepareApply(mutable_cf_options, update_stats); version->PrepareAppend(mutable_cf_options, update_stats);
AppendVersion(cfd, version); AppendVersion(cfd, version);
} }

@ -178,15 +178,8 @@ class VersionStorageInfoTestBase : public testing::Test {
vstorage_.AddBlobFile(std::move(meta)); vstorage_.AddBlobFile(std::move(meta));
} }
void Finalize() { void UpdateVersionStorageInfo() {
vstorage_.UpdateNumNonEmptyLevels(); vstorage_.PrepareForVersionAppend(ioptions_, mutable_cf_options_);
vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_);
vstorage_.UpdateFilesByCompactionPri(ioptions_, mutable_cf_options_);
vstorage_.GenerateFileIndexer();
vstorage_.GenerateLevelFilesBrief();
vstorage_.GenerateLevel0NonOverlapping();
vstorage_.GenerateBottommostFiles();
vstorage_.SetFinalized(); vstorage_.SetFinalized();
} }
@ -217,10 +210,12 @@ TEST_F(VersionStorageInfoTest, MaxBytesForLevelStatic) {
ioptions_.level_compaction_dynamic_level_bytes = false; ioptions_.level_compaction_dynamic_level_bytes = false;
mutable_cf_options_.max_bytes_for_level_base = 10; mutable_cf_options_.max_bytes_for_level_base = 10;
mutable_cf_options_.max_bytes_for_level_multiplier = 5; mutable_cf_options_.max_bytes_for_level_multiplier = 5;
Add(4, 100U, "1", "2");
Add(5, 101U, "1", "2");
vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_); Add(4, 100U, "1", "2", 100U);
Add(5, 101U, "1", "2", 100U);
UpdateVersionStorageInfo();
ASSERT_EQ(vstorage_.MaxBytesForLevel(1), 10U); ASSERT_EQ(vstorage_.MaxBytesForLevel(1), 10U);
ASSERT_EQ(vstorage_.MaxBytesForLevel(2), 50U); ASSERT_EQ(vstorage_.MaxBytesForLevel(2), 50U);
ASSERT_EQ(vstorage_.MaxBytesForLevel(3), 250U); ASSERT_EQ(vstorage_.MaxBytesForLevel(3), 250U);
@ -229,40 +224,84 @@ TEST_F(VersionStorageInfoTest, MaxBytesForLevelStatic) {
ASSERT_EQ(0, logger_->log_count); ASSERT_EQ(0, logger_->log_count);
} }
TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamic) { TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamic_1) {
ioptions_.level_compaction_dynamic_level_bytes = true; ioptions_.level_compaction_dynamic_level_bytes = true;
mutable_cf_options_.max_bytes_for_level_base = 1000; mutable_cf_options_.max_bytes_for_level_base = 1000;
mutable_cf_options_.max_bytes_for_level_multiplier = 5; mutable_cf_options_.max_bytes_for_level_multiplier = 5;
Add(5, 1U, "1", "2", 500U); Add(5, 1U, "1", "2", 500U);
vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_); UpdateVersionStorageInfo();
ASSERT_EQ(0, logger_->log_count); ASSERT_EQ(0, logger_->log_count);
ASSERT_EQ(vstorage_.base_level(), 5); ASSERT_EQ(vstorage_.base_level(), 5);
}
TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamic_2) {
ioptions_.level_compaction_dynamic_level_bytes = true;
mutable_cf_options_.max_bytes_for_level_base = 1000;
mutable_cf_options_.max_bytes_for_level_multiplier = 5;
Add(5, 1U, "1", "2", 500U);
Add(5, 2U, "3", "4", 550U); Add(5, 2U, "3", "4", 550U);
vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_);
UpdateVersionStorageInfo();
ASSERT_EQ(0, logger_->log_count); ASSERT_EQ(0, logger_->log_count);
ASSERT_EQ(vstorage_.MaxBytesForLevel(4), 1000U); ASSERT_EQ(vstorage_.MaxBytesForLevel(4), 1000U);
ASSERT_EQ(vstorage_.base_level(), 4); ASSERT_EQ(vstorage_.base_level(), 4);
}
TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamic_3) {
ioptions_.level_compaction_dynamic_level_bytes = true;
mutable_cf_options_.max_bytes_for_level_base = 1000;
mutable_cf_options_.max_bytes_for_level_multiplier = 5;
Add(5, 1U, "1", "2", 500U);
Add(5, 2U, "3", "4", 550U);
Add(4, 3U, "3", "4", 550U); Add(4, 3U, "3", "4", 550U);
vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_);
UpdateVersionStorageInfo();
ASSERT_EQ(0, logger_->log_count); ASSERT_EQ(0, logger_->log_count);
ASSERT_EQ(vstorage_.MaxBytesForLevel(4), 1000U); ASSERT_EQ(vstorage_.MaxBytesForLevel(4), 1000U);
ASSERT_EQ(vstorage_.base_level(), 4); ASSERT_EQ(vstorage_.base_level(), 4);
}
TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamic_4) {
ioptions_.level_compaction_dynamic_level_bytes = true;
mutable_cf_options_.max_bytes_for_level_base = 1000;
mutable_cf_options_.max_bytes_for_level_multiplier = 5;
Add(5, 1U, "1", "2", 500U);
Add(5, 2U, "3", "4", 550U);
Add(4, 3U, "3", "4", 550U);
Add(3, 4U, "3", "4", 250U); Add(3, 4U, "3", "4", 250U);
Add(3, 5U, "5", "7", 300U); Add(3, 5U, "5", "7", 300U);
vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_);
UpdateVersionStorageInfo();
ASSERT_EQ(1, logger_->log_count); ASSERT_EQ(1, logger_->log_count);
ASSERT_EQ(vstorage_.MaxBytesForLevel(4), 1005U); ASSERT_EQ(vstorage_.MaxBytesForLevel(4), 1005U);
ASSERT_EQ(vstorage_.MaxBytesForLevel(3), 1000U); ASSERT_EQ(vstorage_.MaxBytesForLevel(3), 1000U);
ASSERT_EQ(vstorage_.base_level(), 3); ASSERT_EQ(vstorage_.base_level(), 3);
}
TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamic_5) {
ioptions_.level_compaction_dynamic_level_bytes = true;
mutable_cf_options_.max_bytes_for_level_base = 1000;
mutable_cf_options_.max_bytes_for_level_multiplier = 5;
Add(5, 1U, "1", "2", 500U);
Add(5, 2U, "3", "4", 550U);
Add(4, 3U, "3", "4", 550U);
Add(3, 4U, "3", "4", 250U);
Add(3, 5U, "5", "7", 300U);
Add(1, 6U, "3", "4", 5U); Add(1, 6U, "3", "4", 5U);
Add(1, 7U, "8", "9", 5U); Add(1, 7U, "8", "9", 5U);
logger_->log_count = 0;
vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_); UpdateVersionStorageInfo();
ASSERT_EQ(1, logger_->log_count); ASSERT_EQ(1, logger_->log_count);
ASSERT_GT(vstorage_.MaxBytesForLevel(4), 1005U); ASSERT_GT(vstorage_.MaxBytesForLevel(4), 1005U);
ASSERT_GT(vstorage_.MaxBytesForLevel(3), 1005U); ASSERT_GT(vstorage_.MaxBytesForLevel(3), 1005U);
@ -275,6 +314,7 @@ TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamicLotsOfData) {
ioptions_.level_compaction_dynamic_level_bytes = true; ioptions_.level_compaction_dynamic_level_bytes = true;
mutable_cf_options_.max_bytes_for_level_base = 100; mutable_cf_options_.max_bytes_for_level_base = 100;
mutable_cf_options_.max_bytes_for_level_multiplier = 2; mutable_cf_options_.max_bytes_for_level_multiplier = 2;
Add(0, 1U, "1", "2", 50U); Add(0, 1U, "1", "2", 50U);
Add(1, 2U, "1", "2", 50U); Add(1, 2U, "1", "2", 50U);
Add(2, 3U, "1", "2", 500U); Add(2, 3U, "1", "2", 500U);
@ -282,7 +322,8 @@ TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamicLotsOfData) {
Add(4, 5U, "1", "2", 1700U); Add(4, 5U, "1", "2", 1700U);
Add(5, 6U, "1", "2", 500U); Add(5, 6U, "1", "2", 500U);
vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_); UpdateVersionStorageInfo();
ASSERT_EQ(vstorage_.MaxBytesForLevel(4), 800U); ASSERT_EQ(vstorage_.MaxBytesForLevel(4), 800U);
ASSERT_EQ(vstorage_.MaxBytesForLevel(3), 400U); ASSERT_EQ(vstorage_.MaxBytesForLevel(3), 400U);
ASSERT_EQ(vstorage_.MaxBytesForLevel(2), 200U); ASSERT_EQ(vstorage_.MaxBytesForLevel(2), 200U);
@ -296,12 +337,14 @@ TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamicLargeLevel) {
ioptions_.level_compaction_dynamic_level_bytes = true; ioptions_.level_compaction_dynamic_level_bytes = true;
mutable_cf_options_.max_bytes_for_level_base = 10U * kOneGB; mutable_cf_options_.max_bytes_for_level_base = 10U * kOneGB;
mutable_cf_options_.max_bytes_for_level_multiplier = 10; mutable_cf_options_.max_bytes_for_level_multiplier = 10;
Add(0, 1U, "1", "2", 50U); Add(0, 1U, "1", "2", 50U);
Add(3, 4U, "1", "2", 32U * kOneGB); Add(3, 4U, "1", "2", 32U * kOneGB);
Add(4, 5U, "1", "2", 500U * kOneGB); Add(4, 5U, "1", "2", 500U * kOneGB);
Add(5, 6U, "1", "2", 3000U * kOneGB); Add(5, 6U, "1", "2", 3000U * kOneGB);
vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_); UpdateVersionStorageInfo();
ASSERT_EQ(vstorage_.MaxBytesForLevel(5), 3000U * kOneGB); ASSERT_EQ(vstorage_.MaxBytesForLevel(5), 3000U * kOneGB);
ASSERT_EQ(vstorage_.MaxBytesForLevel(4), 300U * kOneGB); ASSERT_EQ(vstorage_.MaxBytesForLevel(4), 300U * kOneGB);
ASSERT_EQ(vstorage_.MaxBytesForLevel(3), 30U * kOneGB); ASSERT_EQ(vstorage_.MaxBytesForLevel(3), 30U * kOneGB);
@ -325,7 +368,8 @@ TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamicWithLargeL0_1) {
Add(3, 6U, "1", "2", 40000U); Add(3, 6U, "1", "2", 40000U);
Add(2, 7U, "1", "2", 8000U); Add(2, 7U, "1", "2", 8000U);
vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_); UpdateVersionStorageInfo();
ASSERT_EQ(0, logger_->log_count); ASSERT_EQ(0, logger_->log_count);
ASSERT_EQ(2, vstorage_.base_level()); ASSERT_EQ(2, vstorage_.base_level());
// level multiplier should be 3.5 // level multiplier should be 3.5
@ -351,7 +395,8 @@ TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamicWithLargeL0_2) {
Add(3, 6U, "1", "2", 40000U); Add(3, 6U, "1", "2", 40000U);
Add(2, 7U, "1", "2", 8000U); Add(2, 7U, "1", "2", 8000U);
vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_); UpdateVersionStorageInfo();
ASSERT_EQ(0, logger_->log_count); ASSERT_EQ(0, logger_->log_count);
ASSERT_EQ(2, vstorage_.base_level()); ASSERT_EQ(2, vstorage_.base_level());
// level multiplier should be 3.5 // level multiplier should be 3.5
@ -383,7 +428,8 @@ TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamicWithLargeL0_3) {
Add(3, 6U, "1", "2", 40000U); Add(3, 6U, "1", "2", 40000U);
Add(2, 7U, "1", "2", 8000U); Add(2, 7U, "1", "2", 8000U);
vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_); UpdateVersionStorageInfo();
ASSERT_EQ(0, logger_->log_count); ASSERT_EQ(0, logger_->log_count);
ASSERT_EQ(2, vstorage_.base_level()); ASSERT_EQ(2, vstorage_.base_level());
// level multiplier should be 3.5 // level multiplier should be 3.5
@ -406,6 +452,9 @@ TEST_F(VersionStorageInfoTest, EstimateLiveDataSize) {
Add(4, 5U, "4", "5", 1U); // Inside range of last level Add(4, 5U, "4", "5", 1U); // Inside range of last level
Add(4, 6U, "6", "7", 1U); // Inside range of last level Add(4, 6U, "6", "7", 1U); // Inside range of last level
Add(5, 7U, "4", "7", 10U); Add(5, 7U, "4", "7", 10U);
UpdateVersionStorageInfo();
ASSERT_EQ(10U, vstorage_.EstimateLiveDataSize()); ASSERT_EQ(10U, vstorage_.EstimateLiveDataSize());
} }
@ -417,6 +466,9 @@ TEST_F(VersionStorageInfoTest, EstimateLiveDataSize2) {
Add(1, 5U, "5", "6", 1U); Add(1, 5U, "5", "6", 1U);
Add(2, 6U, "2", "3", 1U); Add(2, 6U, "2", "3", 1U);
Add(3, 7U, "7", "8", 1U); Add(3, 7U, "7", "8", 1U);
UpdateVersionStorageInfo();
ASSERT_EQ(4U, vstorage_.EstimateLiveDataSize()); ASSERT_EQ(4U, vstorage_.EstimateLiveDataSize());
} }
@ -430,8 +482,8 @@ TEST_F(VersionStorageInfoTest, GetOverlappingInputs) {
// Two files that do not overlap. // Two files that do not overlap.
Add(1, 5U, {"g", 0, kTypeValue}, {"h", 0, kTypeValue}, 1); Add(1, 5U, {"g", 0, kTypeValue}, {"h", 0, kTypeValue}, 1);
Add(1, 6U, {"i", 0, kTypeValue}, {"j", 0, kTypeValue}, 1); Add(1, 6U, {"i", 0, kTypeValue}, {"j", 0, kTypeValue}, 1);
vstorage_.UpdateNumNonEmptyLevels();
vstorage_.GenerateLevelFilesBrief(); UpdateVersionStorageInfo();
ASSERT_EQ("1,2", GetOverlappingFiles( ASSERT_EQ("1,2", GetOverlappingFiles(
1, {"a", 0, kTypeValue}, {"b", 0, kTypeValue})); 1, {"a", 0, kTypeValue}, {"b", 0, kTypeValue}));
@ -459,6 +511,8 @@ TEST_F(VersionStorageInfoTest, FileLocationAndMetaDataByNumber) {
Add(2, 7U, "1", "2", 8000U); Add(2, 7U, "1", "2", 8000U);
UpdateVersionStorageInfo();
ASSERT_EQ(vstorage_.GetFileLocation(11U), ASSERT_EQ(vstorage_.GetFileLocation(11U),
VersionStorageInfo::FileLocation(0, 0)); VersionStorageInfo::FileLocation(0, 0));
ASSERT_NE(vstorage_.GetFileMetaDataByNumber(11U), nullptr); ASSERT_NE(vstorage_.GetFileMetaDataByNumber(11U), nullptr);
@ -477,7 +531,7 @@ TEST_F(VersionStorageInfoTest, FileLocationAndMetaDataByNumber) {
TEST_F(VersionStorageInfoTest, ForcedBlobGCEmpty) { TEST_F(VersionStorageInfoTest, ForcedBlobGCEmpty) {
// No SST or blob files in VersionStorageInfo // No SST or blob files in VersionStorageInfo
Finalize(); UpdateVersionStorageInfo();
constexpr double age_cutoff = 0.5; constexpr double age_cutoff = 0.5;
constexpr double force_threshold = 0.75; constexpr double force_threshold = 0.75;
@ -573,7 +627,7 @@ TEST_F(VersionStorageInfoTest, ForcedBlobGC) {
garbage_blob_bytes); garbage_blob_bytes);
} }
Finalize(); UpdateVersionStorageInfo();
assert(vstorage_.num_levels() > 0); assert(vstorage_.num_levels() > 0);
const auto& level_files = vstorage_.LevelFiles(level); const auto& level_files = vstorage_.LevelFiles(level);
@ -677,8 +731,9 @@ TEST_F(VersionStorageInfoTimestampTest, GetOverlappingInputs) {
/*largest=*/ /*largest=*/
{PackUserKeyAndTimestamp("d", /*ts=*/1), /*s=*/0, kTypeValue}, {PackUserKeyAndTimestamp("d", /*ts=*/1), /*s=*/0, kTypeValue},
/*file_size=*/100); /*file_size=*/100);
vstorage_.UpdateNumNonEmptyLevels();
vstorage_.GenerateLevelFilesBrief(); UpdateVersionStorageInfo();
ASSERT_EQ( ASSERT_EQ(
"1,2", "1,2",
GetOverlappingFiles( GetOverlappingFiles(

@ -788,7 +788,7 @@ TEST_F(LdbCmdTest, DisableConsistencyChecks) {
char* argv[] = {arg1, arg2, arg3}; char* argv[] = {arg1, arg2, arg3};
SyncPoint::GetInstance()->SetCallBack( SyncPoint::GetInstance()->SetCallBack(
"Version::PrepareApply:forced_check", [&](void* arg) { "Version::PrepareAppend:forced_check", [&](void* arg) {
bool* forced = reinterpret_cast<bool*>(arg); bool* forced = reinterpret_cast<bool*>(arg);
ASSERT_TRUE(*forced); ASSERT_TRUE(*forced);
}); });
@ -808,7 +808,7 @@ TEST_F(LdbCmdTest, DisableConsistencyChecks) {
char* argv[] = {arg1, arg2, arg3}; char* argv[] = {arg1, arg2, arg3};
SyncPoint::GetInstance()->SetCallBack( SyncPoint::GetInstance()->SetCallBack(
"Version::PrepareApply:forced_check", [&](void* arg) { "Version::PrepareAppend:forced_check", [&](void* arg) {
bool* forced = reinterpret_cast<bool*>(arg); bool* forced = reinterpret_cast<bool*>(arg);
ASSERT_TRUE(*forced); ASSERT_TRUE(*forced);
}); });

Loading…
Cancel
Save