From b3130193266aa732310e094f507e81632eff092d Mon Sep 17 00:00:00 2001 From: Zhongyi Xie Date: Tue, 13 Nov 2018 10:38:45 -0800 Subject: [PATCH] use per-level perfcontext for DB::Get calls (#4617) Summary: this PR adds two more per-level perf context counters to track * number of keys returned in Get call, break down by levels * total processing time at each level during Get call Pull Request resolved: https://github.com/facebook/rocksdb/pull/4617 Differential Revision: D12898024 Pulled By: miasantreble fbshipit-source-id: 6b84ef1c8097c0d9e97bee1a774958f56ab4a6c4 --- HISTORY.md | 2 +- db/db_compaction_test.cc | 2 +- db/version_set.cc | 10 ++++++++++ include/rocksdb/perf_context.h | 9 +++++++++ 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 61424f84f..6bff7668b 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -4,7 +4,7 @@ ## 5.18.0 (11/12/2018) ### New Features * Introduced `Memoryllocator`, which lets the user specify custom allocator for memory in block cache. -* Introduced `PerfContextByLevel` as part of `PerfContext` which allows storing perf context at each level. Also replaced `__thread` with `thread_local` keyword for perf_context. +* Introduced `PerfContextByLevel` as part of `PerfContext` which allows storing perf context at each level. Also replaced `__thread` with `thread_local` keyword for perf_context. Added per-level perf context for bloom filter and `Get` query. * With level_compaction_dynamic_level_bytes = true, level multiplier may be adjusted automatically when Level 0 to 1 compaction is lagged behind. * Introduced DB option `atomic_flush`. If true, RocksDB supports flushing multiple column families and atomically committing the result to MANIFEST. Useful when WAL is disabled. * Added `num_deletions` and `num_merge_operands` members to `TableProperties`. diff --git a/db/db_compaction_test.cc b/db/db_compaction_test.cc index 4bd1d41f3..711d34e55 100644 --- a/db/db_compaction_test.cc +++ b/db/db_compaction_test.cc @@ -8,8 +8,8 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. #include "db/db_test_util.h" -#include "port/stack_trace.h" #include "port/port.h" +#include "port/stack_trace.h" #include "rocksdb/experimental.h" #include "rocksdb/utilities/convenience.h" #include "util/fault_injection_test_env.h" diff --git a/db/version_set.cc b/db/version_set.cc index 995ab54da..3df889a93 100644 --- a/db/version_set.cc +++ b/db/version_set.cc @@ -1218,6 +1218,10 @@ void Version::Get(const ReadOptions& read_options, const LookupKey& k, sample_file_read_inc(f->file_metadata); } + bool timer_enabled = + GetPerfLevel() >= PerfLevel::kEnableTimeExceptForMutex && + get_perf_context()->per_level_perf_context_enabled; + StopWatchNano timer(env_, timer_enabled /* auto_start */); *status = table_cache_->Get( read_options, *internal_comparator(), *f->file_metadata, ikey, &get_context, mutable_cf_options_.prefix_extractor.get(), @@ -1226,6 +1230,10 @@ void Version::Get(const ReadOptions& read_options, const LookupKey& k, fp.IsHitFileLastInLevel()), fp.GetCurrentLevel()); // TODO: examine the behavior for corrupted key + if (timer_enabled) { + PERF_COUNTER_BY_LEVEL_ADD(get_from_table_nanos, timer.ElapsedNanos(), + fp.GetCurrentLevel()); + } if (!status->ok()) { return; } @@ -1241,6 +1249,7 @@ void Version::Get(const ReadOptions& read_options, const LookupKey& k, // Keep searching in other files break; case GetContext::kMerge: + // TODO: update per-level perfcontext user_key_return_count for kMerge break; case GetContext::kFound: if (fp.GetHitFileLevel() == 0) { @@ -1250,6 +1259,7 @@ void Version::Get(const ReadOptions& read_options, const LookupKey& k, } else if (fp.GetHitFileLevel() >= 2) { RecordTick(db_statistics_, GET_HIT_L2_AND_UP); } + PERF_COUNTER_BY_LEVEL_ADD(user_key_return_count, 1, fp.GetHitFileLevel()); return; case GetContext::kDeleted: // Use empty error message for speed diff --git a/include/rocksdb/perf_context.h b/include/rocksdb/perf_context.h index bd7a5de4c..1b11fa3cc 100644 --- a/include/rocksdb/perf_context.h +++ b/include/rocksdb/perf_context.h @@ -17,6 +17,8 @@ namespace rocksdb { // and transparently. // Use SetPerfLevel(PerfLevel::kEnableTime) to enable time stats. +// Break down performance counters by level and store per-level perf context in +// PerfContextByLevel struct PerfContextByLevel { // # of times bloom filter has avoided file reads, i.e., negatives. uint64_t bloom_filter_useful = 0; @@ -26,6 +28,13 @@ struct PerfContextByLevel { // exist. uint64_t bloom_filter_full_true_positive = 0; + // total number of user key returned (only include keys that are found, does + // not include keys that are deleted or merged without a final put + uint64_t user_key_return_count; + + // total nanos spent on reading data from SST files + uint64_t get_from_table_nanos; + void Reset(); // reset all performance counters to zero };