add counter for deletion dropping optimization

Summary:
add this counter stat to track usage of deletion-dropping optimization. if usage is low, we can delete it to prevent bugs like #2726.
Closes https://github.com/facebook/rocksdb/pull/2761

Differential Revision: D5665421

Pulled By: ajkr

fbshipit-source-id: 881befa2d199838dac88709e7b376a43d304e3d4
main
Andrew Kryczka 7 years ago committed by Facebook Github Bot
parent 0d8e992b47
commit 8ace1f79b5
  1. 2
      db/compaction_iteration_stats.h
  2. 7
      db/compaction_iterator.cc
  3. 4
      db/compaction_job.cc
  4. 40
      db/db_compaction_test.cc
  5. 7
      include/rocksdb/statistics.h

@ -16,6 +16,8 @@ struct CompactionIterationStats {
int64_t num_record_drop_obsolete = 0; int64_t num_record_drop_obsolete = 0;
int64_t num_record_drop_range_del = 0; int64_t num_record_drop_range_del = 0;
int64_t num_range_del_drop_obsolete = 0; int64_t num_range_del_drop_obsolete = 0;
// Deletions obsoleted before bottom level due to file gap optimization.
int64_t num_optimized_del_drop_obsolete = 0;
uint64_t total_filter_time = 0; uint64_t total_filter_time = 0;
// Input statistics // Input statistics

@ -111,6 +111,7 @@ void CompactionIterator::ResetRecordCounts() {
iter_stats_.num_record_drop_obsolete = 0; iter_stats_.num_record_drop_obsolete = 0;
iter_stats_.num_record_drop_range_del = 0; iter_stats_.num_record_drop_range_del = 0;
iter_stats_.num_range_del_drop_obsolete = 0; iter_stats_.num_range_del_drop_obsolete = 0;
iter_stats_.num_optimized_del_drop_obsolete = 0;
} }
void CompactionIterator::SeekToFirst() { void CompactionIterator::SeekToFirst() {
@ -426,6 +427,9 @@ void CompactionIterator::NextFromInput() {
// Can compact out this SingleDelete. // Can compact out this SingleDelete.
++iter_stats_.num_record_drop_obsolete; ++iter_stats_.num_record_drop_obsolete;
++iter_stats_.num_single_del_fallthru; ++iter_stats_.num_single_del_fallthru;
if (!bottommost_level_) {
++iter_stats_.num_optimized_del_drop_obsolete;
}
} else { } else {
// Output SingleDelete // Output SingleDelete
valid_ = true; valid_ = true;
@ -467,6 +471,9 @@ void CompactionIterator::NextFromInput() {
// Note: Dropping this Delete will not affect TransactionDB // Note: Dropping this Delete will not affect TransactionDB
// write-conflict checking since it is earlier than any snapshot. // write-conflict checking since it is earlier than any snapshot.
++iter_stats_.num_record_drop_obsolete; ++iter_stats_.num_record_drop_obsolete;
if (!bottommost_level_) {
++iter_stats_.num_optimized_del_drop_obsolete;
}
input_->Next(); input_->Next();
} else if (ikey_.type == kTypeMerge) { } else if (ikey_.type == kTypeMerge) {
if (!merge_helper_->HasOperator()) { if (!merge_helper_->HasOperator()) {

@ -1014,6 +1014,10 @@ void CompactionJob::RecordDroppedKeys(
RecordTick(stats_, COMPACTION_RANGE_DEL_DROP_OBSOLETE, RecordTick(stats_, COMPACTION_RANGE_DEL_DROP_OBSOLETE,
c_iter_stats.num_range_del_drop_obsolete); c_iter_stats.num_range_del_drop_obsolete);
} }
if (c_iter_stats.num_optimized_del_drop_obsolete > 0) {
RecordTick(stats_, COMPACTION_OPTIMIZED_DEL_DROP_OBSOLETE,
c_iter_stats.num_optimized_del_drop_obsolete);
}
} }
Status CompactionJob::FinishCompactionOutputFile( Status CompactionJob::FinishCompactionOutputFile(

@ -2684,6 +2684,46 @@ TEST_P(DBCompactionTestWithParam, IntraL0CompactionDoesNotObsoleteDeletions) {
ASSERT_TRUE(db_->Get(roptions, Key(0), &result).IsNotFound()); ASSERT_TRUE(db_->Get(roptions, Key(0), &result).IsNotFound());
} }
TEST_F(DBCompactionTest, OptimizedDeletionObsoleting) {
// Deletions can be dropped when compacted to non-last level if they fall
// outside the lower-level files' key-ranges.
const int kNumL0Files = 4;
Options options = CurrentOptions();
options.level0_file_num_compaction_trigger = kNumL0Files;
options.statistics = rocksdb::CreateDBStatistics();
DestroyAndReopen(options);
// put key 1 and 3 in separate L1, L2 files.
// So key 0, 2, and 4+ fall outside these levels' key-ranges.
for (int level = 2; level >= 1; --level) {
for (int i = 0; i < 2; ++i) {
Put(Key(2 * i + 1), "val");
Flush();
}
MoveFilesToLevel(level);
ASSERT_EQ(2, NumTableFilesAtLevel(level));
}
// Delete keys in range [1, 4]. These L0 files will be compacted with L1:
// - Tombstones for keys 2 and 4 can be dropped early.
// - Tombstones for keys 1 and 3 must be kept due to L2 files' key-ranges.
for (int i = 0; i < kNumL0Files; ++i) {
Put(Key(0), "val"); // sentinel to prevent trivial move
Delete(Key(i + 1));
Flush();
}
dbfull()->TEST_WaitForCompact();
for (int i = 0; i < kNumL0Files; ++i) {
std::string value;
ASSERT_TRUE(db_->Get(ReadOptions(), Key(i + 1), &value).IsNotFound());
}
ASSERT_EQ(2, options.statistics->getTickerCount(
COMPACTION_OPTIMIZED_DEL_DROP_OBSOLETE));
ASSERT_EQ(2,
options.statistics->getTickerCount(COMPACTION_KEY_DROP_OBSOLETE));
}
INSTANTIATE_TEST_CASE_P(DBCompactionTestWithParam, DBCompactionTestWithParam, INSTANTIATE_TEST_CASE_P(DBCompactionTestWithParam, DBCompactionTestWithParam,
::testing::Values(std::make_tuple(1, true), ::testing::Values(std::make_tuple(1, true),
std::make_tuple(1, false), std::make_tuple(1, false),

@ -105,8 +105,9 @@ enum Tickers : uint32_t {
COMPACTION_KEY_DROP_OBSOLETE, // The key is obsolete. COMPACTION_KEY_DROP_OBSOLETE, // The key is obsolete.
COMPACTION_KEY_DROP_RANGE_DEL, // key was covered by a range tombstone. COMPACTION_KEY_DROP_RANGE_DEL, // key was covered by a range tombstone.
COMPACTION_KEY_DROP_USER, // user compaction function has dropped the key. COMPACTION_KEY_DROP_USER, // user compaction function has dropped the key.
COMPACTION_RANGE_DEL_DROP_OBSOLETE, // all keys in range were deleted. COMPACTION_RANGE_DEL_DROP_OBSOLETE, // all keys in range were deleted.
// Deletions obsoleted before bottom level due to file gap optimization.
COMPACTION_OPTIMIZED_DEL_DROP_OBSOLETE,
// Number of keys written to the database via the Put and Write call's // Number of keys written to the database via the Put and Write call's
NUMBER_KEYS_WRITTEN, NUMBER_KEYS_WRITTEN,
@ -264,7 +265,9 @@ const std::vector<std::pair<Tickers, std::string>> TickersNameMap = {
{COMPACTION_KEY_DROP_RANGE_DEL, "rocksdb.compaction.key.drop.range_del"}, {COMPACTION_KEY_DROP_RANGE_DEL, "rocksdb.compaction.key.drop.range_del"},
{COMPACTION_KEY_DROP_USER, "rocksdb.compaction.key.drop.user"}, {COMPACTION_KEY_DROP_USER, "rocksdb.compaction.key.drop.user"},
{COMPACTION_RANGE_DEL_DROP_OBSOLETE, {COMPACTION_RANGE_DEL_DROP_OBSOLETE,
"rocksdb.compaction.range_del.drop.obsolete"}, "rocksdb.compaction.range_del.drop.obsolete"},
{COMPACTION_OPTIMIZED_DEL_DROP_OBSOLETE,
"rocksdb.compaction.optimized.del.drop.obsolete"},
{NUMBER_KEYS_WRITTEN, "rocksdb.number.keys.written"}, {NUMBER_KEYS_WRITTEN, "rocksdb.number.keys.written"},
{NUMBER_KEYS_READ, "rocksdb.number.keys.read"}, {NUMBER_KEYS_READ, "rocksdb.number.keys.read"},
{NUMBER_KEYS_UPDATED, "rocksdb.number.keys.updated"}, {NUMBER_KEYS_UPDATED, "rocksdb.number.keys.updated"},

Loading…
Cancel
Save