clang-format for db/compaction (#10882)

Summary: Pull Request resolved: https://github.com/facebook/rocksdb/pull/10882

Reviewed By: riversand963

Differential Revision: D40724867

Pulled By: jay-zhuang

fbshipit-source-id: 7f387724f8cd07d8d2b90566a515a4e9078d21f1
main
Jay Zhuang 2 years ago committed by Facebook GitHub Bot
parent a1a1dc6659
commit b36ec37a4b
  1. 16
      db/compaction/compaction_iterator.cc
  2. 5
      db/compaction/compaction_iterator_test.cc
  3. 15
      db/compaction/compaction_job.cc
  4. 6
      db/compaction/compaction_job.h
  5. 271
      db/compaction/compaction_job_stats_test.cc
  6. 7
      db/compaction/compaction_job_test.cc
  7. 7
      db/compaction/compaction_picker_universal.cc

@ -825,8 +825,8 @@ void CompactionIterator::NextFromInput() {
cmp_with_history_ts_low_ < 0)) && cmp_with_history_ts_low_ < 0)) &&
bottommost_level_) { bottommost_level_) {
// Handle the case where we have a delete key at the bottom most level // Handle the case where we have a delete key at the bottom most level
// We can skip outputting the key iff there are no subsequent puts for this // We can skip outputting the key iff there are no subsequent puts for
// key // this key
assert(!compaction_ || compaction_->KeyNotExistsBeyondOutputLevel( assert(!compaction_ || compaction_->KeyNotExistsBeyondOutputLevel(
ikey_.user_key, &level_ptrs_)); ikey_.user_key, &level_ptrs_));
ParsedInternalKey next_ikey; ParsedInternalKey next_ikey;
@ -853,8 +853,8 @@ void CompactionIterator::NextFromInput() {
DefinitelyNotInSnapshot(next_ikey.sequence, prev_snapshot))) { DefinitelyNotInSnapshot(next_ikey.sequence, prev_snapshot))) {
AdvanceInputIter(); AdvanceInputIter();
} }
// If you find you still need to output a row with this key, we need to output the // If you find you still need to output a row with this key, we need to
// delete too // output the delete too
if (input_.Valid() && if (input_.Valid() &&
(ParseInternalKey(input_.key(), &next_ikey, allow_data_in_errors_) (ParseInternalKey(input_.key(), &next_ikey, allow_data_in_errors_)
.ok()) && .ok()) &&
@ -1212,8 +1212,8 @@ inline SequenceNumber CompactionIterator::findEarliestVisibleSnapshot(
ROCKS_LOG_FATAL(info_log_, ROCKS_LOG_FATAL(info_log_,
"No snapshot left in findEarliestVisibleSnapshot"); "No snapshot left in findEarliestVisibleSnapshot");
} }
auto snapshots_iter = std::lower_bound( auto snapshots_iter =
snapshots_->begin(), snapshots_->end(), in); std::lower_bound(snapshots_->begin(), snapshots_->end(), in);
assert(prev_snapshot != nullptr); assert(prev_snapshot != nullptr);
if (snapshots_iter == snapshots_->begin()) { if (snapshots_iter == snapshots_->begin()) {
*prev_snapshot = 0; *prev_snapshot = 0;
@ -1228,8 +1228,8 @@ inline SequenceNumber CompactionIterator::findEarliestVisibleSnapshot(
} }
} }
if (snapshot_checker_ == nullptr) { if (snapshot_checker_ == nullptr) {
return snapshots_iter != snapshots_->end() return snapshots_iter != snapshots_->end() ? *snapshots_iter
? *snapshots_iter : kMaxSequenceNumber; : kMaxSequenceNumber;
} }
bool has_released_snapshot = !released_snapshots_.empty(); bool has_released_snapshot = !released_snapshots_.empty();
for (; snapshots_iter != snapshots_->end(); ++snapshots_iter) { for (; snapshots_iter != snapshots_->end(); ++snapshots_iter) {

@ -203,7 +203,8 @@ class TestSnapshotChecker : public SnapshotChecker {
public: public:
explicit TestSnapshotChecker( explicit TestSnapshotChecker(
SequenceNumber last_committed_sequence, SequenceNumber last_committed_sequence,
const std::unordered_map<SequenceNumber, SequenceNumber>& snapshots = {{}}) const std::unordered_map<SequenceNumber, SequenceNumber>& snapshots =
{{}})
: last_committed_sequence_(last_committed_sequence), : last_committed_sequence_(last_committed_sequence),
snapshots_(snapshots) {} snapshots_(snapshots) {}
@ -1038,7 +1039,7 @@ TEST_F(CompactionIteratorWithSnapshotCheckerTest,
TEST_F(CompactionIteratorWithSnapshotCheckerTest, TEST_F(CompactionIteratorWithSnapshotCheckerTest,
NotRemoveDeletionIfValuePresentToEarlierSnapshot) { NotRemoveDeletionIfValuePresentToEarlierSnapshot) {
AddSnapshot(2,1); AddSnapshot(2, 1);
RunTest({test::KeyStr("a", 4, kTypeDeletion), RunTest({test::KeyStr("a", 4, kTypeDeletion),
test::KeyStr("a", 1, kTypeValue), test::KeyStr("b", 3, kTypeValue)}, test::KeyStr("a", 1, kTypeValue), test::KeyStr("b", 3, kTypeValue)},
{"", "", ""}, {"", "", ""},

@ -714,11 +714,12 @@ Status CompactionJob::Run() {
break; break;
} }
// Verify that the table is usable // Verify that the table is usable
// We set for_compaction to false and don't OptimizeForCompactionTableRead // We set for_compaction to false and don't
// here because this is a special case after we finish the table building // OptimizeForCompactionTableRead here because this is a special case
// No matter whether use_direct_io_for_flush_and_compaction is true, // after we finish the table building No matter whether
// we will regard this verification as user reads since the goal is // use_direct_io_for_flush_and_compaction is true, we will regard this
// to cache it here for further user reads // verification as user reads since the goal is to cache it here for
// further user reads
ReadOptions read_options; ReadOptions read_options;
InternalIterator* iter = cfd->table_cache()->NewIterator( InternalIterator* iter = cfd->table_cache()->NewIterator(
read_options, file_options_, cfd->internal_comparator(), read_options, file_options_, cfd->internal_comparator(),
@ -764,8 +765,8 @@ Status CompactionJob::Run() {
} }
}; };
for (size_t i = 1; i < compact_->sub_compact_states.size(); i++) { for (size_t i = 1; i < compact_->sub_compact_states.size(); i++) {
thread_pool.emplace_back(verify_table, thread_pool.emplace_back(
std::ref(compact_->sub_compact_states[i].status)); verify_table, std::ref(compact_->sub_compact_states[i].status));
} }
verify_table(compact_->sub_compact_states[0].status); verify_table(compact_->sub_compact_states[0].status);
for (auto& thread : thread_pool) { for (auto& thread : thread_pool) {

@ -261,12 +261,12 @@ class CompactionJob {
Status OpenCompactionOutputFile(SubcompactionState* sub_compact, Status OpenCompactionOutputFile(SubcompactionState* sub_compact,
CompactionOutputs& outputs); CompactionOutputs& outputs);
void UpdateCompactionJobStats( void UpdateCompactionJobStats(
const InternalStats::CompactionStats& stats) const; const InternalStats::CompactionStats& stats) const;
void RecordDroppedKeys(const CompactionIterationStats& c_iter_stats, void RecordDroppedKeys(const CompactionIterationStats& c_iter_stats,
CompactionJobStats* compaction_job_stats = nullptr); CompactionJobStats* compaction_job_stats = nullptr);
void UpdateCompactionInputStatsHelper( void UpdateCompactionInputStatsHelper(int* num_files, uint64_t* bytes_read,
int* num_files, uint64_t* bytes_read, int input_level); int input_level);
void NotifyOnSubcompactionBegin(SubcompactionState* sub_compact); void NotifyOnSubcompactionBegin(SubcompactionState* sub_compact);

@ -155,9 +155,8 @@ class CompactionJobStatsTest : public testing::Test,
ASSERT_OK(TryReopenWithColumnFamilies(cfs, options)); ASSERT_OK(TryReopenWithColumnFamilies(cfs, options));
} }
Status TryReopenWithColumnFamilies( Status TryReopenWithColumnFamilies(const std::vector<std::string>& cfs,
const std::vector<std::string>& cfs, const std::vector<Options>& options) {
const std::vector<Options>& options) {
Close(); Close();
EXPECT_EQ(cfs.size(), options.size()); EXPECT_EQ(cfs.size(), options.size());
std::vector<ColumnFamilyDescriptor> column_families; std::vector<ColumnFamilyDescriptor> column_families;
@ -175,9 +174,7 @@ class CompactionJobStatsTest : public testing::Test,
return TryReopenWithColumnFamilies(cfs, v_opts); return TryReopenWithColumnFamilies(cfs, v_opts);
} }
void Reopen(const Options& options) { void Reopen(const Options& options) { ASSERT_OK(TryReopen(options)); }
ASSERT_OK(TryReopen(options));
}
void Close() { void Close() {
for (auto h : handles_) { for (auto h : handles_) {
@ -226,9 +223,7 @@ class CompactionJobStatsTest : public testing::Test,
return db_->Put(wo, handles_[cf], k, v); return db_->Put(wo, handles_[cf], k, v);
} }
Status Delete(const std::string& k) { Status Delete(const std::string& k) { return db_->Delete(WriteOptions(), k); }
return db_->Delete(WriteOptions(), k);
}
Status Delete(int cf, const std::string& k) { Status Delete(int cf, const std::string& k) {
return db_->Delete(WriteOptions(), handles_[cf], k); return db_->Delete(WriteOptions(), handles_[cf], k);
@ -338,21 +333,21 @@ class CompactionJobStatsTest : public testing::Test,
} }
} }
static void SetDeletionCompactionStats( static void SetDeletionCompactionStats(CompactionJobStats* stats,
CompactionJobStats *stats, uint64_t input_deletions, uint64_t input_deletions,
uint64_t expired_deletions, uint64_t records_replaced) { uint64_t expired_deletions,
uint64_t records_replaced) {
stats->num_input_deletion_records = input_deletions; stats->num_input_deletion_records = input_deletions;
stats->num_expired_deletion_records = expired_deletions; stats->num_expired_deletion_records = expired_deletions;
stats->num_records_replaced = records_replaced; stats->num_records_replaced = records_replaced;
} }
void MakeTableWithKeyValues( void MakeTableWithKeyValues(Random* rnd, uint64_t smallest, uint64_t largest,
Random* rnd, uint64_t smallest, uint64_t largest, int key_size, int value_size, uint64_t interval,
int key_size, int value_size, uint64_t interval, double ratio, int cf = 0) {
double ratio, int cf = 0) {
for (auto key = smallest; key < largest; key += interval) { for (auto key = smallest; key < largest; key += interval) {
ASSERT_OK(Put(cf, Slice(Key(key, key_size)), ASSERT_OK(Put(cf, Slice(Key(key, key_size)),
Slice(RandomString(rnd, value_size, ratio)))); Slice(RandomString(rnd, value_size, ratio))));
} }
ASSERT_OK(Flush(cf)); ASSERT_OK(Flush(cf));
} }
@ -361,9 +356,9 @@ class CompactionJobStatsTest : public testing::Test,
// rounds of keys are inserted into the database, as per the behavior // rounds of keys are inserted into the database, as per the behavior
// of the DeletionStatsTest. // of the DeletionStatsTest.
void SelectivelyDeleteKeys(uint64_t smallest, uint64_t largest, void SelectivelyDeleteKeys(uint64_t smallest, uint64_t largest,
uint64_t interval, int deletion_interval, int key_size, uint64_t interval, int deletion_interval,
uint64_t cutoff_key_num, CompactionJobStats* stats, int cf = 0) { int key_size, uint64_t cutoff_key_num,
CompactionJobStats* stats, int cf = 0) {
// interval needs to be >= 2 so that deletion entries can be inserted // interval needs to be >= 2 so that deletion entries can be inserted
// that are intended to not result in an actual key deletion by using // that are intended to not result in an actual key deletion by using
// an offset of 1 from another existing key // an offset of 1 from another existing key
@ -387,20 +382,19 @@ class CompactionJobStatsTest : public testing::Test,
// Insert some deletions for keys that don't exist that // Insert some deletions for keys that don't exist that
// are both in and out of the key range // are both in and out of the key range
ASSERT_OK(Delete(cf, Key(smallest+1, key_size))); ASSERT_OK(Delete(cf, Key(smallest + 1, key_size)));
deletions_made++; deletions_made++;
ASSERT_OK(Delete(cf, Key(smallest-1, key_size))); ASSERT_OK(Delete(cf, Key(smallest - 1, key_size)));
deletions_made++; deletions_made++;
num_expired++; num_expired++;
ASSERT_OK(Delete(cf, Key(smallest-9, key_size))); ASSERT_OK(Delete(cf, Key(smallest - 9, key_size)));
deletions_made++; deletions_made++;
num_expired++; num_expired++;
ASSERT_OK(Flush(cf)); ASSERT_OK(Flush(cf));
SetDeletionCompactionStats(stats, deletions_made, num_expired, SetDeletionCompactionStats(stats, deletions_made, num_expired, num_deleted);
num_deleted);
} }
}; };
@ -440,25 +434,20 @@ class CompactionJobStatsChecker : public EventListener {
// use ASSERT_GE and ASSERT_LE with a reasonable bias --- // use ASSERT_GE and ASSERT_LE with a reasonable bias ---
// 10% in uncompressed case and 20% when compression is used. // 10% in uncompressed case and 20% when compression is used.
virtual void Verify(const CompactionJobStats& current_stats, virtual void Verify(const CompactionJobStats& current_stats,
const CompactionJobStats& stats) { const CompactionJobStats& stats) {
// time // time
ASSERT_GT(current_stats.elapsed_micros, 0U); ASSERT_GT(current_stats.elapsed_micros, 0U);
ASSERT_EQ(current_stats.num_input_records, ASSERT_EQ(current_stats.num_input_records, stats.num_input_records);
stats.num_input_records); ASSERT_EQ(current_stats.num_input_files, stats.num_input_files);
ASSERT_EQ(current_stats.num_input_files,
stats.num_input_files);
ASSERT_EQ(current_stats.num_input_files_at_output_level, ASSERT_EQ(current_stats.num_input_files_at_output_level,
stats.num_input_files_at_output_level); stats.num_input_files_at_output_level);
ASSERT_EQ(current_stats.num_output_records, ASSERT_EQ(current_stats.num_output_records, stats.num_output_records);
stats.num_output_records); ASSERT_EQ(current_stats.num_output_files, stats.num_output_files);
ASSERT_EQ(current_stats.num_output_files,
stats.num_output_files);
ASSERT_EQ(current_stats.is_full_compaction, stats.is_full_compaction); ASSERT_EQ(current_stats.is_full_compaction, stats.is_full_compaction);
ASSERT_EQ(current_stats.is_manual_compaction, ASSERT_EQ(current_stats.is_manual_compaction, stats.is_manual_compaction);
stats.is_manual_compaction);
// file size // file size
double kFileSizeBias = compression_enabled_ ? 0.20 : 0.10; double kFileSizeBias = compression_enabled_ ? 0.20 : 0.10;
@ -475,18 +464,14 @@ class CompactionJobStatsChecker : public EventListener {
ASSERT_EQ(current_stats.total_input_raw_value_bytes, ASSERT_EQ(current_stats.total_input_raw_value_bytes,
stats.total_input_raw_value_bytes); stats.total_input_raw_value_bytes);
ASSERT_EQ(current_stats.num_records_replaced, ASSERT_EQ(current_stats.num_records_replaced, stats.num_records_replaced);
stats.num_records_replaced);
ASSERT_EQ(current_stats.num_corrupt_keys, ASSERT_EQ(current_stats.num_corrupt_keys, stats.num_corrupt_keys);
stats.num_corrupt_keys);
ASSERT_EQ( ASSERT_EQ(std::string(current_stats.smallest_output_key_prefix),
std::string(current_stats.smallest_output_key_prefix), std::string(stats.smallest_output_key_prefix));
std::string(stats.smallest_output_key_prefix)); ASSERT_EQ(std::string(current_stats.largest_output_key_prefix),
ASSERT_EQ( std::string(stats.largest_output_key_prefix));
std::string(current_stats.largest_output_key_prefix),
std::string(stats.largest_output_key_prefix));
} }
// Add an expected compaction stats, which will be used to // Add an expected compaction stats, which will be used to
@ -497,9 +482,7 @@ class CompactionJobStatsChecker : public EventListener {
expected_stats_.push(stats); expected_stats_.push(stats);
} }
void EnableCompression(bool flag) { void EnableCompression(bool flag) { compression_enabled_ = flag; }
compression_enabled_ = flag;
}
bool verify_next_comp_io_stats() const { return verify_next_comp_io_stats_; } bool verify_next_comp_io_stats() const { return verify_next_comp_io_stats_; }
@ -517,45 +500,37 @@ class CompactionJobDeletionStatsChecker : public CompactionJobStatsChecker {
// Verifies whether two CompactionJobStats match. // Verifies whether two CompactionJobStats match.
void Verify(const CompactionJobStats& current_stats, void Verify(const CompactionJobStats& current_stats,
const CompactionJobStats& stats) override { const CompactionJobStats& stats) override {
ASSERT_EQ( ASSERT_EQ(current_stats.num_input_deletion_records,
current_stats.num_input_deletion_records, stats.num_input_deletion_records);
stats.num_input_deletion_records); ASSERT_EQ(current_stats.num_expired_deletion_records,
ASSERT_EQ( stats.num_expired_deletion_records);
current_stats.num_expired_deletion_records, ASSERT_EQ(current_stats.num_records_replaced, stats.num_records_replaced);
stats.num_expired_deletion_records);
ASSERT_EQ( ASSERT_EQ(current_stats.num_corrupt_keys, stats.num_corrupt_keys);
current_stats.num_records_replaced,
stats.num_records_replaced);
ASSERT_EQ(current_stats.num_corrupt_keys,
stats.num_corrupt_keys);
} }
}; };
namespace { namespace {
uint64_t EstimatedFileSize( uint64_t EstimatedFileSize(uint64_t num_records, size_t key_size,
uint64_t num_records, size_t key_size, size_t value_size, size_t value_size, double compression_ratio = 1.0,
double compression_ratio = 1.0, size_t block_size = 4096,
size_t block_size = 4096, int bloom_bits_per_key = 10) {
int bloom_bits_per_key = 10) {
const size_t kPerKeyOverhead = 8; const size_t kPerKeyOverhead = 8;
const size_t kFooterSize = 512; const size_t kFooterSize = 512;
uint64_t data_size = uint64_t data_size = static_cast<uint64_t>(
static_cast<uint64_t>( num_records *
num_records * (key_size + value_size * compression_ratio + (key_size + value_size * compression_ratio + kPerKeyOverhead));
kPerKeyOverhead));
return data_size + kFooterSize return data_size + kFooterSize +
+ num_records * bloom_bits_per_key / 8 // filter block num_records * bloom_bits_per_key / 8 // filter block
+ data_size * (key_size + 8) / block_size; // index block + data_size * (key_size + 8) / block_size; // index block
} }
namespace { namespace {
void CopyPrefix( void CopyPrefix(const Slice& src, size_t prefix_length, std::string* dst) {
const Slice& src, size_t prefix_length, std::string* dst) {
assert(prefix_length > 0); assert(prefix_length > 0);
size_t length = src.size() > prefix_length ? prefix_length : src.size(); size_t length = src.size() > prefix_length ? prefix_length : src.size();
dst->assign(src.data(), length); dst->assign(src.data(), length);
@ -581,28 +556,24 @@ CompactionJobStats NewManualCompactionJobStats(
stats.num_output_files = num_output_files; stats.num_output_files = num_output_files;
stats.total_input_bytes = stats.total_input_bytes =
EstimatedFileSize( EstimatedFileSize(num_input_records / num_input_files, key_size,
num_input_records / num_input_files, value_size, compression_ratio) *
key_size, value_size, compression_ratio) * num_input_files; num_input_files;
stats.total_output_bytes = stats.total_output_bytes =
EstimatedFileSize( EstimatedFileSize(num_output_records / num_output_files, key_size,
num_output_records / num_output_files, value_size, compression_ratio) *
key_size, value_size, compression_ratio) * num_output_files; num_output_files;
stats.total_input_raw_key_bytes = stats.total_input_raw_key_bytes = num_input_records * (key_size + 8);
num_input_records * (key_size + 8); stats.total_input_raw_value_bytes = num_input_records * value_size;
stats.total_input_raw_value_bytes =
num_input_records * value_size;
stats.is_full_compaction = is_full; stats.is_full_compaction = is_full;
stats.is_manual_compaction = is_manual; stats.is_manual_compaction = is_manual;
stats.num_records_replaced = num_records_replaced; stats.num_records_replaced = num_records_replaced;
CopyPrefix(smallest_key, CopyPrefix(smallest_key, CompactionJobStats::kMaxPrefixLength,
CompactionJobStats::kMaxPrefixLength,
&stats.smallest_output_key_prefix); &stats.smallest_output_key_prefix);
CopyPrefix(largest_key, CopyPrefix(largest_key, CompactionJobStats::kMaxPrefixLength,
CompactionJobStats::kMaxPrefixLength,
&stats.largest_output_key_prefix); &stats.largest_output_key_prefix);
return stats; return stats;
@ -662,13 +633,11 @@ TEST_P(CompactionJobStatsTest, CompactionJobStatsTest) {
// 1st Phase: generate "num_L0_files" L0 files. // 1st Phase: generate "num_L0_files" L0 files.
int num_L0_files = 0; int num_L0_files = 0;
for (uint64_t start_key = key_base; for (uint64_t start_key = key_base; start_key <= key_base * kTestScale;
start_key <= key_base * kTestScale; start_key += key_base) {
start_key += key_base) { MakeTableWithKeyValues(&rnd, start_key, start_key + key_base - 1,
MakeTableWithKeyValues( kKeySize, kValueSize, key_interval,
&rnd, start_key, start_key + key_base - 1, compression_ratio, 1);
kKeySize, kValueSize, key_interval,
compression_ratio, 1);
snprintf(buf, kBufSize, "%d", ++num_L0_files); snprintf(buf, kBufSize, "%d", ++num_L0_files);
ASSERT_EQ(std::string(buf), FilesPerLevel(1)); ASSERT_EQ(std::string(buf), FilesPerLevel(1));
} }
@ -684,13 +653,9 @@ TEST_P(CompactionJobStatsTest, CompactionJobStatsTest) {
start_key += key_base, count++) { start_key += key_base, count++) {
smallest_key = Key(start_key, 10); smallest_key = Key(start_key, 10);
largest_key = Key(start_key + key_base - key_interval, 10); largest_key = Key(start_key + key_base - key_interval, 10);
stats_checker->AddExpectedStats( stats_checker->AddExpectedStats(NewManualCompactionJobStats(
NewManualCompactionJobStats( smallest_key, largest_key, 1, 0, num_keys_per_L0_file, kKeySize,
smallest_key, largest_key, kValueSize, 1, num_keys_per_L0_file, compression_ratio, 0));
1, 0, num_keys_per_L0_file,
kKeySize, kValueSize,
1, num_keys_per_L0_file,
compression_ratio, 0));
ASSERT_EQ(stats_checker->NumberOfUnverifiedStats(), 1U); ASSERT_EQ(stats_checker->NumberOfUnverifiedStats(), 1U);
TEST_Compact(0, 1, smallest_key, largest_key); TEST_Compact(0, 1, smallest_key, largest_key);
snprintf(buf, kBufSize, "%d,%d", num_L0_files - count, count); snprintf(buf, kBufSize, "%d,%d", num_L0_files - count, count);
@ -701,14 +666,10 @@ TEST_P(CompactionJobStatsTest, CompactionJobStatsTest) {
int num_remaining_L0 = num_L0_files - L0_compaction_count; int num_remaining_L0 = num_L0_files - L0_compaction_count;
smallest_key = Key(key_base * (L0_compaction_count + 1), 10); smallest_key = Key(key_base * (L0_compaction_count + 1), 10);
largest_key = Key(key_base * (kTestScale + 1) - key_interval, 10); largest_key = Key(key_base * (kTestScale + 1) - key_interval, 10);
stats_checker->AddExpectedStats( stats_checker->AddExpectedStats(NewManualCompactionJobStats(
NewManualCompactionJobStats( smallest_key, largest_key, num_remaining_L0, 0,
smallest_key, largest_key, num_keys_per_L0_file * num_remaining_L0, kKeySize, kValueSize, 1,
num_remaining_L0, num_keys_per_L0_file * num_remaining_L0, compression_ratio, 0));
0, num_keys_per_L0_file * num_remaining_L0,
kKeySize, kValueSize,
1, num_keys_per_L0_file * num_remaining_L0,
compression_ratio, 0));
ASSERT_EQ(stats_checker->NumberOfUnverifiedStats(), 1U); ASSERT_EQ(stats_checker->NumberOfUnverifiedStats(), 1U);
TEST_Compact(0, 1, smallest_key, largest_key); TEST_Compact(0, 1, smallest_key, largest_key);
@ -719,13 +680,11 @@ TEST_P(CompactionJobStatsTest, CompactionJobStatsTest) {
// 3rd Phase: generate sparse L0 files (wider key-range, same num of keys) // 3rd Phase: generate sparse L0 files (wider key-range, same num of keys)
int sparseness = 2; int sparseness = 2;
for (uint64_t start_key = key_base; for (uint64_t start_key = key_base; start_key <= key_base * kTestScale;
start_key <= key_base * kTestScale; start_key += key_base * sparseness) {
start_key += key_base * sparseness) {
MakeTableWithKeyValues( MakeTableWithKeyValues(
&rnd, start_key, start_key + key_base * sparseness - 1, &rnd, start_key, start_key + key_base * sparseness - 1, kKeySize,
kKeySize, kValueSize, kValueSize, key_base * sparseness / num_keys_per_L0_file,
key_base * sparseness / num_keys_per_L0_file,
compression_ratio, 1); compression_ratio, 1);
snprintf(buf, kBufSize, "%d,%d", ++num_L0_files, num_L1_files); snprintf(buf, kBufSize, "%d,%d", ++num_L0_files, num_L1_files);
ASSERT_EQ(std::string(buf), FilesPerLevel(1)); ASSERT_EQ(std::string(buf), FilesPerLevel(1));
@ -737,21 +696,15 @@ TEST_P(CompactionJobStatsTest, CompactionJobStatsTest) {
// output files without coordinating to see if the output could fit into // output files without coordinating to see if the output could fit into
// a smaller number of files like it does when it runs sequentially // a smaller number of files like it does when it runs sequentially
int num_output_files = options.max_subcompactions > 1 ? 2 : 1; int num_output_files = options.max_subcompactions > 1 ? 2 : 1;
for (uint64_t start_key = key_base; for (uint64_t start_key = key_base; num_L0_files > 1;
num_L0_files > 1;
start_key += key_base * sparseness) { start_key += key_base * sparseness) {
smallest_key = Key(start_key, 10); smallest_key = Key(start_key, 10);
largest_key = largest_key = Key(start_key + key_base * sparseness - key_interval, 10);
Key(start_key + key_base * sparseness - key_interval, 10); stats_checker->AddExpectedStats(NewManualCompactionJobStats(
stats_checker->AddExpectedStats( smallest_key, largest_key, 3, 2, num_keys_per_L0_file * 3, kKeySize,
NewManualCompactionJobStats( kValueSize, num_output_files,
smallest_key, largest_key, num_keys_per_L0_file * 2, // 1/3 of the data will be updated.
3, 2, num_keys_per_L0_file * 3, compression_ratio, num_keys_per_L0_file));
kKeySize, kValueSize,
num_output_files,
num_keys_per_L0_file * 2, // 1/3 of the data will be updated.
compression_ratio,
num_keys_per_L0_file));
ASSERT_EQ(stats_checker->NumberOfUnverifiedStats(), 1U); ASSERT_EQ(stats_checker->NumberOfUnverifiedStats(), 1U);
Compact(1, smallest_key, largest_key); Compact(1, smallest_key, largest_key);
if (options.max_subcompactions == 1) { if (options.max_subcompactions == 1) {
@ -766,14 +719,10 @@ TEST_P(CompactionJobStatsTest, CompactionJobStatsTest) {
// In the first sub-compaction, we expect L0 compaction. // In the first sub-compaction, we expect L0 compaction.
smallest_key = Key(key_base, 10); smallest_key = Key(key_base, 10);
largest_key = Key(key_base * (kTestScale + 1) - key_interval, 10); largest_key = Key(key_base * (kTestScale + 1) - key_interval, 10);
stats_checker->AddExpectedStats( stats_checker->AddExpectedStats(NewManualCompactionJobStats(
NewManualCompactionJobStats( Key(key_base * (kTestScale + 1 - sparseness), 10), largest_key, 2, 1,
Key(key_base * (kTestScale + 1 - sparseness), 10), largest_key, num_keys_per_L0_file * 3, kKeySize, kValueSize, 1,
2, 1, num_keys_per_L0_file * 3, num_keys_per_L0_file * 2, compression_ratio, num_keys_per_L0_file));
kKeySize, kValueSize,
1, num_keys_per_L0_file * 2,
compression_ratio,
num_keys_per_L0_file));
ASSERT_EQ(stats_checker->NumberOfUnverifiedStats(), 1U); ASSERT_EQ(stats_checker->NumberOfUnverifiedStats(), 1U);
Compact(1, smallest_key, largest_key); Compact(1, smallest_key, largest_key);
@ -869,7 +818,7 @@ TEST_P(CompactionJobStatsTest, DeletionStatsTest) {
Options options; Options options;
options.listeners.emplace_back(stats_checker); options.listeners.emplace_back(stats_checker);
options.create_if_missing = true; options.create_if_missing = true;
options.level0_file_num_compaction_trigger = kTestScale+1; options.level0_file_num_compaction_trigger = kTestScale + 1;
options.num_levels = 3; options.num_levels = 3;
options.compression = kNoCompression; options.compression = kNoCompression;
options.max_bytes_for_level_multiplier = 2; options.max_bytes_for_level_multiplier = 2;
@ -881,13 +830,10 @@ TEST_P(CompactionJobStatsTest, DeletionStatsTest) {
// Stage 1: Generate several L0 files and then send them to L2 by // Stage 1: Generate several L0 files and then send them to L2 by
// using CompactRangeOptions and CompactRange(). These files will // using CompactRangeOptions and CompactRange(). These files will
// have a strict subset of the keys from the full key-range // have a strict subset of the keys from the full key-range
for (uint64_t start_key = key_base; for (uint64_t start_key = key_base; start_key <= key_base * kTestScale / 2;
start_key <= key_base * kTestScale / 2; start_key += key_base) {
start_key += key_base) { MakeTableWithKeyValues(&rnd, start_key, start_key + key_base - 1, kKeySize,
MakeTableWithKeyValues( kValueSize, key_interval, compression_ratio, 1);
&rnd, start_key, start_key + key_base - 1,
kKeySize, kValueSize, key_interval,
compression_ratio, 1);
} }
CompactRangeOptions cr_options; CompactRangeOptions cr_options;
@ -897,13 +843,10 @@ TEST_P(CompactionJobStatsTest, DeletionStatsTest) {
ASSERT_GT(NumTableFilesAtLevel(2, 1), 0); ASSERT_GT(NumTableFilesAtLevel(2, 1), 0);
// Stage 2: Generate files including keys from the entire key range // Stage 2: Generate files including keys from the entire key range
for (uint64_t start_key = key_base; for (uint64_t start_key = key_base; start_key <= key_base * kTestScale;
start_key <= key_base * kTestScale; start_key += key_base) {
start_key += key_base) { MakeTableWithKeyValues(&rnd, start_key, start_key + key_base - 1, kKeySize,
MakeTableWithKeyValues( kValueSize, key_interval, compression_ratio, 1);
&rnd, start_key, start_key + key_base - 1,
kKeySize, kValueSize, key_interval,
compression_ratio, 1);
} }
// Send these L0 files to L1 // Send these L0 files to L1
@ -912,16 +855,16 @@ TEST_P(CompactionJobStatsTest, DeletionStatsTest) {
// Add a new record and flush so now there is a L0 file // Add a new record and flush so now there is a L0 file
// with a value too (not just deletions from the next step) // with a value too (not just deletions from the next step)
ASSERT_OK(Put(1, Key(key_base-6, kKeySize), "test")); ASSERT_OK(Put(1, Key(key_base - 6, kKeySize), "test"));
ASSERT_OK(Flush(1)); ASSERT_OK(Flush(1));
// Stage 3: Generate L0 files with some deletions so now // Stage 3: Generate L0 files with some deletions so now
// there are files with the same key range in L0, L1, and L2 // there are files with the same key range in L0, L1, and L2
int deletion_interval = 3; int deletion_interval = 3;
CompactionJobStats first_compaction_stats; CompactionJobStats first_compaction_stats;
SelectivelyDeleteKeys(key_base, largest_key_num, SelectivelyDeleteKeys(key_base, largest_key_num, key_interval,
key_interval, deletion_interval, kKeySize, cutoff_key_num, deletion_interval, kKeySize, cutoff_key_num,
&first_compaction_stats, 1); &first_compaction_stats, 1);
stats_checker->AddExpectedStats(first_compaction_stats); stats_checker->AddExpectedStats(first_compaction_stats);
@ -932,8 +875,7 @@ TEST_P(CompactionJobStatsTest, DeletionStatsTest) {
namespace { namespace {
int GetUniversalCompactionInputUnits(uint32_t num_flushes) { int GetUniversalCompactionInputUnits(uint32_t num_flushes) {
uint32_t compaction_input_units; uint32_t compaction_input_units;
for (compaction_input_units = 1; for (compaction_input_units = 1; num_flushes >= compaction_input_units;
num_flushes >= compaction_input_units;
compaction_input_units *= 2) { compaction_input_units *= 2) {
if ((num_flushes & compaction_input_units) != 0) { if ((num_flushes & compaction_input_units) != 0) {
return compaction_input_units > 1 ? compaction_input_units : 0; return compaction_input_units > 1 ? compaction_input_units : 0;
@ -998,13 +940,10 @@ TEST_P(CompactionJobStatsTest, UniversalCompactionTest) {
} }
ASSERT_EQ(stats_checker->NumberOfUnverifiedStats(), 3U); ASSERT_EQ(stats_checker->NumberOfUnverifiedStats(), 3U);
for (uint64_t start_key = key_base; for (uint64_t start_key = key_base; start_key <= key_base * kTestScale;
start_key <= key_base * kTestScale; start_key += key_base) {
start_key += key_base) { MakeTableWithKeyValues(&rnd, start_key, start_key + key_base - 1, kKeySize,
MakeTableWithKeyValues( kValueSize, key_interval, compression_ratio, 1);
&rnd, start_key, start_key + key_base - 1,
kKeySize, kValueSize, key_interval,
compression_ratio, 1);
ASSERT_OK(static_cast_with_check<DBImpl>(db_)->TEST_WaitForCompact()); ASSERT_OK(static_cast_with_check<DBImpl>(db_)->TEST_WaitForCompact());
} }
ASSERT_EQ(stats_checker->NumberOfUnverifiedStats(), 0U); ASSERT_EQ(stats_checker->NumberOfUnverifiedStats(), 0U);

@ -40,7 +40,7 @@ namespace ROCKSDB_NAMESPACE {
namespace { namespace {
void VerifyInitializationOfCompactionJobStats( void VerifyInitializationOfCompactionJobStats(
const CompactionJobStats& compaction_job_stats) { const CompactionJobStats& compaction_job_stats) {
#if !defined(IOS_CROSS_COMPILE) #if !defined(IOS_CROSS_COMPILE)
ASSERT_EQ(compaction_job_stats.elapsed_micros, 0U); ASSERT_EQ(compaction_job_stats.elapsed_micros, 0U);
@ -499,8 +499,7 @@ class CompactionJobTestBase : public testing::Test {
// This is how the key will look like once it's written in bottommost // This is how the key will look like once it's written in bottommost
// file // file
InternalKey bottommost_internal_key( InternalKey bottommost_internal_key(key, 0, kTypeValue);
key, 0, kTypeValue);
if (corrupt_id(k)) { if (corrupt_id(k)) {
test::CorruptKeyType(&internal_key); test::CorruptKeyType(&internal_key);
@ -620,7 +619,7 @@ class CompactionJobTestBase : public testing::Test {
CompactionInputFiles compaction_level; CompactionInputFiles compaction_level;
compaction_level.level = input_levels[i]; compaction_level.level = input_levels[i];
compaction_level.files.insert(compaction_level.files.end(), compaction_level.files.insert(compaction_level.files.end(),
level_files.begin(), level_files.end()); level_files.begin(), level_files.end());
compaction_input_files.push_back(compaction_level); compaction_input_files.push_back(compaction_level);
num_input_files += level_files.size(); num_input_files += level_files.size();
} }

@ -308,9 +308,10 @@ void UniversalCompactionBuilder::SortedRun::Dump(char* out_buf,
if (file->fd.GetPathId() == 0 || !print_path) { if (file->fd.GetPathId() == 0 || !print_path) {
snprintf(out_buf, out_buf_size, "file %" PRIu64, file->fd.GetNumber()); snprintf(out_buf, out_buf_size, "file %" PRIu64, file->fd.GetNumber());
} else { } else {
snprintf(out_buf, out_buf_size, "file %" PRIu64 snprintf(out_buf, out_buf_size,
"(path " "file %" PRIu64
"%" PRIu32 ")", "(path "
"%" PRIu32 ")",
file->fd.GetNumber(), file->fd.GetPathId()); file->fd.GetNumber(), file->fd.GetPathId());
} }
} else { } else {

Loading…
Cancel
Save