NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add options.compaction_measure_io_stats to print write I/O stats in compactions

Browse Source 
Summary:
Add options.compaction_measure_io_stats to print out / pass to listener accumulated time spent on write calls. Example outputs in info logs:

2015/08/12-16:27:59.463944 7fd428bff700 (Original Log Time 2015/08/12-16:27:59.463922) EVENT_LOG_v1 {"time_micros": 1439422079463897, "job": 6, "event": "compaction_finished", "output_level": 1, "num_output_files": 4, "total_output_size": 6900525, "num_input_records": 111483, "num_output_records": 106877, "file_write_nanos": 15663206, "file_range_sync_nanos": 649588, "file_fsync_nanos": 349614797, "file_prepare_write_nanos": 1505812, "lsm_state": [2, 4, 0, 0, 0, 0, 0]}

Add two more counters in iostats_context.

Also add a parameter of db_bench.

Test Plan: Add a unit test. Also manually verify LOG outputs in db_bench

Subscribers: leveldb, dhruba

Differential Revision: https://reviews.facebook.net/D44115
main
sdong 9 years ago
parent dc9d5634fd
commit 603b6da8b8
16 changed files with 249 additions and 84 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 46
      db/compaction_job.cc
  2. 5
      db/compaction_job.h
  3. 74
      db/compaction_job_stats_test.cc
  4. 9
      db/compaction_job_test.cc
  5. 4
      db/db_bench.cc
  6. 6
      db/db_impl.cc
  7. 15
      include/rocksdb/compaction_job_stats.h
  8. 5
      include/rocksdb/iostats_context.h
  9. 5
      include/rocksdb/options.h
  10. 5
      util/compaction_job_stats_impl.cc
  11. 10
      util/file_reader_writer.cc
  12. 15
      util/iostats_context.cc
  13. 119
      util/mutable_cf_options.h
  14. 10
      util/options.cc
  15. 4
      util/options_helper.cc
  16. 1
      util/options_test.cc

46
db/compaction_job.cc
Unescape View File

@ -96,7 +96,7 @@ CompactionJob::CompactionJob(
Directory* db_directory, Directory* output_directory, Statistics* stats, Directory* db_directory, Directory* output_directory, Statistics* stats,
std::vector<SequenceNumber> existing_snapshots, std::vector<SequenceNumber> existing_snapshots,
std::shared_ptr<Cache> table_cache, EventLogger* event_logger, std::shared_ptr<Cache> table_cache, EventLogger* event_logger,
bool paranoid_file_checks, const std::string& dbname, bool paranoid_file_checks, bool measure_io_stats, const std::string& dbname,
CompactionJobStats* compaction_job_stats) CompactionJobStats* compaction_job_stats)
: job_id_(job_id), : job_id_(job_id),
compact_(new CompactionState(compaction)), compact_(new CompactionState(compaction)),
@ -115,7 +115,8 @@ CompactionJob::CompactionJob(
existing_snapshots_(std::move(existing_snapshots)), existing_snapshots_(std::move(existing_snapshots)),
table_cache_(std::move(table_cache)), table_cache_(std::move(table_cache)),
event_logger_(event_logger), event_logger_(event_logger),
paranoid_file_checks_(paranoid_file_checks) { paranoid_file_checks_(paranoid_file_checks),
measure_io_stats_(measure_io_stats) {
assert(log_buffer_ != nullptr); assert(log_buffer_ != nullptr);
ThreadStatusUtil::SetColumnFamily(compact_->compaction->column_family_data()); ThreadStatusUtil::SetColumnFamily(compact_->compaction->column_family_data());
ThreadStatusUtil::SetThreadOperation(ThreadStatus::OP_COMPACTION); ThreadStatusUtil::SetThreadOperation(ThreadStatus::OP_COMPACTION);
@ -270,6 +271,22 @@ Status CompactionJob::Run() {
} }
Status CompactionJob::SubCompactionRun(Slice* start, Slice* end) { Status CompactionJob::SubCompactionRun(Slice* start, Slice* end) {
PerfLevel prev_perf_level = PerfLevel::kEnableTime;
uint64_t prev_write_nanos = 0;
uint64_t prev_fsync_nanos = 0;
uint64_t prev_range_sync_nanos = 0;
uint64_t prev_prepare_write_nanos = 0;
bool enabled_io_stats = false;
if (measure_io_stats_ && compaction_job_stats_ != nullptr) {
prev_perf_level = GetPerfLevel();
SetPerfLevel(PerfLevel::kEnableTime);
prev_write_nanos = iostats_context.write_nanos;
prev_fsync_nanos = iostats_context.fsync_nanos;
prev_range_sync_nanos = iostats_context.range_sync_nanos;
prev_prepare_write_nanos = iostats_context.prepare_write_nanos;
enabled_io_stats = true;
}
auto* compaction = compact_->compaction; auto* compaction = compact_->compaction;
const uint64_t start_micros = env_->NowMicros(); const uint64_t start_micros = env_->NowMicros();
int64_t imm_micros = 0; // Micros spent doing imm_ compactions int64_t imm_micros = 0; // Micros spent doing imm_ compactions
@ -286,6 +303,21 @@ Status CompactionJob::SubCompactionRun(Slice* start, Slice* end) {
compaction_stats_.micros = env_->NowMicros() - start_micros - imm_micros; compaction_stats_.micros = env_->NowMicros() - start_micros - imm_micros;
MeasureTime(stats_, COMPACTION_TIME, compaction_stats_.micros); MeasureTime(stats_, COMPACTION_TIME, compaction_stats_.micros);
if (enabled_io_stats) {
// Not supporting parallel running yet
compaction_job_stats_->file_write_nanos +=
iostats_context.write_nanos - prev_write_nanos;
compaction_job_stats_->file_fsync_nanos +=
iostats_context.fsync_nanos - prev_fsync_nanos;
compaction_job_stats_->file_range_sync_nanos +=
iostats_context.range_sync_nanos - prev_range_sync_nanos;
compaction_job_stats_->file_prepare_write_nanos +=
iostats_context.prepare_write_nanos - prev_prepare_write_nanos;
if (prev_perf_level != PerfLevel::kEnableTime) {
SetPerfLevel(prev_perf_level);
}
}
return status; return status;
} }
@ -340,6 +372,16 @@ void CompactionJob::Install(Status* status,
<< "total_output_size" << compact_->total_bytes << "total_output_size" << compact_->total_bytes
<< "num_input_records" << compact_->num_input_records << "num_input_records" << compact_->num_input_records
<< "num_output_records" << compact_->num_output_records; << "num_output_records" << compact_->num_output_records;
if (measure_io_stats_ && compaction_job_stats_ != nullptr) {
stream << "file_write_nanos" << compaction_job_stats_->file_write_nanos;
stream << "file_range_sync_nanos"
<< compaction_job_stats_->file_range_sync_nanos;
stream << "file_fsync_nanos" << compaction_job_stats_->file_fsync_nanos;
stream << "file_prepare_write_nanos"
<< compaction_job_stats_->file_prepare_write_nanos;
}
stream << "lsm_state"; stream << "lsm_state";
stream.StartArray(); stream.StartArray();
for (int level = 0; level < vstorage->num_levels(); ++level) { for (int level = 0; level < vstorage->num_levels(); ++level) {

5
db/compaction_job.h
Unescape View File

@ -57,8 +57,8 @@ class CompactionJob {
Directory* db_directory, Directory* output_directory, Directory* db_directory, Directory* output_directory,
Statistics* stats, Statistics* stats,
std::vector<SequenceNumber> existing_snapshots, std::vector<SequenceNumber> existing_snapshots,
std::shared_ptr<Cache> table_cache, std::shared_ptr<Cache> table_cache, EventLogger* event_logger,
EventLogger* event_logger, bool paranoid_file_checks, bool paranoid_file_checks, bool measure_io_stats,
const std::string& dbname, const std::string& dbname,
CompactionJobStats* compaction_job_stats); CompactionJobStats* compaction_job_stats);
@ -153,6 +153,7 @@ class CompactionJob {
EventLogger* event_logger_; EventLogger* event_logger_;
bool paranoid_file_checks_; bool paranoid_file_checks_;
bool measure_io_stats_;
std::vector<Slice> sub_compaction_boundaries_; std::vector<Slice> sub_compaction_boundaries_;
}; };

74
db/compaction_job_stats_test.cc
Unescape View File

@ -417,14 +417,25 @@ class CompactionJobStatsTest : public testing::Test,
// test CompactionJobStatsTest. // test CompactionJobStatsTest.
class CompactionJobStatsChecker : public EventListener { class CompactionJobStatsChecker : public EventListener {
public: public:
CompactionJobStatsChecker() : compression_enabled_(false) {} CompactionJobStatsChecker()
: compression_enabled_(false), verify_next_comp_io_stats_(false) {}
size_t NumberOfUnverifiedStats() { return expected_stats_.size(); } size_t NumberOfUnverifiedStats() { return expected_stats_.size(); }
void set_verify_next_comp_io_stats(bool v) { verify_next_comp_io_stats_ = v; }
// Once a compaction completed, this function will verify the returned // Once a compaction completed, this function will verify the returned
// CompactionJobInfo with the oldest CompactionJobInfo added earlier // CompactionJobInfo with the oldest CompactionJobInfo added earlier
// in "expected_stats_" which has not yet being used for verification. // in "expected_stats_" which has not yet being used for verification.
virtual void OnCompactionCompleted(DB *db, const CompactionJobInfo& ci) { virtual void OnCompactionCompleted(DB *db, const CompactionJobInfo& ci) {
if (verify_next_comp_io_stats_) {
ASSERT_GT(ci.stats.file_write_nanos, 0);
ASSERT_GT(ci.stats.file_range_sync_nanos, 0);
ASSERT_GT(ci.stats.file_fsync_nanos, 0);
ASSERT_GT(ci.stats.file_prepare_write_nanos, 0);
verify_next_comp_io_stats_ = false;
}
std::lock_guard<std::mutex> lock(mutex_); std::lock_guard<std::mutex> lock(mutex_);
if (expected_stats_.size()) { if (expected_stats_.size()) {
Verify(ci.stats, expected_stats_.front()); Verify(ci.stats, expected_stats_.front());
@ -498,10 +509,13 @@ class CompactionJobStatsChecker : public EventListener {
compression_enabled_ = flag; compression_enabled_ = flag;
} }
bool verify_next_comp_io_stats() const { return verify_next_comp_io_stats_; }
private: private:
std::mutex mutex_; std::mutex mutex_;
std::queue<CompactionJobStats> expected_stats_; std::queue<CompactionJobStats> expected_stats_;
bool compression_enabled_; bool compression_enabled_;
bool verify_next_comp_io_stats_;
}; };
// An EventListener which helps verify the compaction statistics in // An EventListener which helps verify the compaction statistics in
@ -643,7 +657,9 @@ TEST_P(CompactionJobStatsTest, CompactionJobStatsTest) {
options.num_levels = 3; options.num_levels = 3;
options.compression = kNoCompression; options.compression = kNoCompression;
options.num_subcompactions = num_subcompactions_; options.num_subcompactions = num_subcompactions_;
options.bytes_per_sync = 512 * 1024;
options.compaction_measure_io_stats = true;
for (int test = 0; test < 2; ++test) { for (int test = 0; test < 2; ++test) {
DestroyAndReopen(options); DestroyAndReopen(options);
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
@ -766,6 +782,7 @@ TEST_P(CompactionJobStatsTest, CompactionJobStatsTest) {
num_keys_per_L0_file)); num_keys_per_L0_file));
ASSERT_EQ(stats_checker->NumberOfUnverifiedStats(), 1U); ASSERT_EQ(stats_checker->NumberOfUnverifiedStats(), 1U);
Compact(1, smallest_key, largest_key); Compact(1, smallest_key, largest_key);
num_L1_files = options.num_subcompactions > 1 ? 7 : 4; num_L1_files = options.num_subcompactions > 1 ? 7 : 4;
char L1_buf[4]; char L1_buf[4];
snprintf(L1_buf, sizeof(L1_buf), "0,%d", num_L1_files); snprintf(L1_buf, sizeof(L1_buf), "0,%d", num_L1_files);
@ -777,6 +794,61 @@ TEST_P(CompactionJobStatsTest, CompactionJobStatsTest) {
} }
stats_checker->EnableCompression(true); stats_checker->EnableCompression(true);
compression_ratio = kCompressionRatio; compression_ratio = kCompressionRatio;
for (int i = 0; i < 5; i++) {
ASSERT_OK(Put(1, Slice(Key(key_base + i, 10)),
Slice(RandomString(&rnd, 512 * 1024, 1))));
}
ASSERT_OK(Flush(1));
reinterpret_cast<DBImpl*>(db_)->TEST_WaitForCompact();
stats_checker->set_verify_next_comp_io_stats(true);
std::atomic<bool> first_prepare_write(true);
rocksdb::SyncPoint::GetInstance()->SetCallBack(
"WritableFileWriter::Append:BeforePrepareWrite", [&](void* arg) {
if (first_prepare_write.load()) {
options.env->SleepForMicroseconds(3);
first_prepare_write.store(false);
}
});
std::atomic<bool> first_flush(true);
rocksdb::SyncPoint::GetInstance()->SetCallBack(
"WritableFileWriter::Flush:BeforeAppend", [&](void* arg) {
if (first_flush.load()) {
options.env->SleepForMicroseconds(3);
first_flush.store(false);
}
});
std::atomic<bool> first_sync(true);
rocksdb::SyncPoint::GetInstance()->SetCallBack(
"WritableFileWriter::SyncInternal:0", [&](void* arg) {
if (first_sync.load()) {
options.env->SleepForMicroseconds(3);
first_sync.store(false);
}
});
std::atomic<bool> first_range_sync(true);
rocksdb::SyncPoint::GetInstance()->SetCallBack(
"WritableFileWriter::RangeSync:0", [&](void* arg) {
if (first_range_sync.load()) {
options.env->SleepForMicroseconds(3);
first_range_sync.store(false);
}
});
rocksdb::SyncPoint::GetInstance()->EnableProcessing();
Compact(1, smallest_key, largest_key);
ASSERT_TRUE(!stats_checker->verify_next_comp_io_stats());
ASSERT_TRUE(!first_prepare_write.load());
ASSERT_TRUE(!first_flush.load());
ASSERT_TRUE(!first_sync.load());
ASSERT_TRUE(!first_range_sync.load());
rocksdb::SyncPoint::GetInstance()->DisableProcessing();
} }
ASSERT_EQ(stats_checker->NumberOfUnverifiedStats(), 0U); ASSERT_EQ(stats_checker->NumberOfUnverifiedStats(), 0U);
} }

9
db/compaction_job_test.cc
Unescape View File

@ -246,10 +246,11 @@ class CompactionJobTest : public testing::Test {
LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, db_options_.info_log.get()); LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, db_options_.info_log.get());
mutex_.Lock(); mutex_.Lock();
EventLogger event_logger(db_options_.info_log.get()); EventLogger event_logger(db_options_.info_log.get());
CompactionJob compaction_job( CompactionJob compaction_job(0, &compaction, db_options_, env_options_,
0, &compaction, db_options_, env_options_, versions_.get(), versions_.get(), &shutting_down_, &log_buffer,
&shutting_down_, &log_buffer, nullptr, nullptr, nullptr, {}, nullptr, nullptr, nullptr, {}, table_cache_,
table_cache_, &event_logger, false, dbname_, &compaction_job_stats_); &event_logger, false, false, dbname_,
&compaction_job_stats_);
VerifyInitializationOfCompactionJobStats(compaction_job_stats_); VerifyInitializationOfCompactionJobStats(compaction_job_stats_);

4
db/db_bench.cc
Unescape View File

@ -465,6 +465,9 @@ DEFINE_int32(transaction_sleep, 0,
"Max microseconds to sleep in between " "Max microseconds to sleep in between "
"reading and writing a value (used in RandomTransaction only). "); "reading and writing a value (used in RandomTransaction only). ");
DEFINE_bool(compaction_measure_io_stats, false,
"Measure times spents on I/Os while in compactions. ");
namespace { namespace {
enum rocksdb::CompressionType StringToCompressionType(const char* ctype) { enum rocksdb::CompressionType StringToCompressionType(const char* ctype) {
assert(ctype); assert(ctype);
@ -2379,6 +2382,7 @@ class Benchmark {
exit(1); exit(1);
} }
options.max_successive_merges = FLAGS_max_successive_merges; options.max_successive_merges = FLAGS_max_successive_merges;
options.compaction_measure_io_stats = FLAGS_compaction_measure_io_stats;
// set universal style compaction configurations, if applicable // set universal style compaction configurations, if applicable
if (FLAGS_universal_size_ratio != 0) { if (FLAGS_universal_size_ratio != 0) {

6
db/db_impl.cc
Unescape View File

@ -1669,7 +1669,8 @@ Status DBImpl::CompactFilesImpl(
&shutting_down_, log_buffer, directories_.GetDbDir(), &shutting_down_, log_buffer, directories_.GetDbDir(),
directories_.GetDataDir(c->output_path_id()), stats_, snapshots_.GetAll(), directories_.GetDataDir(c->output_path_id()), stats_, snapshots_.GetAll(),
table_cache_, &event_logger_, table_cache_, &event_logger_,
c->mutable_cf_options()->paranoid_file_checks, dbname_, c->mutable_cf_options()->paranoid_file_checks,
c->mutable_cf_options()->compaction_measure_io_stats, dbname_,
nullptr); // Here we pass a nullptr for CompactionJobStats because nullptr); // Here we pass a nullptr for CompactionJobStats because
// CompactFiles does not trigger OnCompactionCompleted(), // CompactFiles does not trigger OnCompactionCompleted(),
// which is the only place where CompactionJobStats is // which is the only place where CompactionJobStats is
@ -2683,7 +2684,8 @@ Status DBImpl::BackgroundCompaction(bool* madeProgress, JobContext* job_context,
versions_.get(), &shutting_down_, log_buffer, directories_.GetDbDir(), versions_.get(), &shutting_down_, log_buffer, directories_.GetDbDir(),
directories_.GetDataDir(c->output_path_id()), stats_, directories_.GetDataDir(c->output_path_id()), stats_,
snapshots_.GetAll(), table_cache_, &event_logger_, snapshots_.GetAll(), table_cache_, &event_logger_,
c->mutable_cf_options()->paranoid_file_checks, dbname_, c->mutable_cf_options()->paranoid_file_checks,
c->mutable_cf_options()->compaction_measure_io_stats, dbname_,
&compaction_job_stats); &compaction_job_stats);
compaction_job.Prepare(); compaction_job.Prepare();

15
include/rocksdb/compaction_job_stats.h
Unescape View File

@ -58,6 +58,21 @@ struct CompactionJobStats {
// the key) encountered and written out. // the key) encountered and written out.
uint64_t num_corrupt_keys; uint64_t num_corrupt_keys;
// Following counters are only populated if
// options.compaction_measure_io_stats = true;
// Time spent on file's Append() call.
uint64_t file_write_nanos;
// Time spent on sync file range.
uint64_t file_range_sync_nanos;
// Time spent on file fsync.
uint64_t file_fsync_nanos;
// Time spent on preparing file write (falocate, etc)
uint64_t file_prepare_write_nanos;
// 0-terminated strings storing the first 8 bytes of the smallest and // 0-terminated strings storing the first 8 bytes of the smallest and
// largest key in the output. // largest key in the output.
static const size_t kMaxPrefixLength = 8; static const size_t kMaxPrefixLength = 8;

5
include/rocksdb/iostats_context.h
Unescape View File

@ -38,7 +38,10 @@ struct IOStatsContext {
uint64_t read_nanos; uint64_t read_nanos;
// time spent in sync_file_range(). // time spent in sync_file_range().
uint64_t range_sync_nanos; uint64_t range_sync_nanos;
// time spent in fsync
uint64_t fsync_nanos;
// time spent in preparing write (fallocate etc).
uint64_t prepare_write_nanos;
// time spent in Logger::Logv(). // time spent in Logger::Logv().
uint64_t logger_nanos; uint64_t logger_nanos;
}; };

5
include/rocksdb/options.h
Unescape View File

@ -15,7 +15,6 @@
#include <memory> #include <memory>
#include <vector> #include <vector>
#include <limits> #include <limits>
#include <stdint.h>
#include <unordered_map> #include <unordered_map>
#include "rocksdb/version.h" #include "rocksdb/version.h"
@ -736,6 +735,10 @@ struct ColumnFamilyOptions {
// Default: false // Default: false
bool paranoid_file_checks; bool paranoid_file_checks;
// Measure IO stats in compactions, if true.
// Default: false
bool compaction_measure_io_stats;
// Create ColumnFamilyOptions with default values for all fields // Create ColumnFamilyOptions with default values for all fields
ColumnFamilyOptions(); ColumnFamilyOptions();
// Create ColumnFamilyOptions from Options // Create ColumnFamilyOptions from Options

5
util/compaction_job_stats_impl.cc
Unescape View File

@ -33,6 +33,11 @@ void CompactionJobStats::Reset() {
num_expired_deletion_records = 0; num_expired_deletion_records = 0;
num_corrupt_keys = 0; num_corrupt_keys = 0;
file_write_nanos = 0;
file_range_sync_nanos = 0;
file_fsync_nanos = 0;
file_prepare_write_nanos = 0;
} }
#else #else

10
util/file_reader_writer.cc
Unescape View File

@ -43,7 +43,11 @@ Status WritableFileWriter::Append(const Slice& data) {
TEST_KILL_RANDOM(rocksdb_kill_odds * REDUCE_ODDS2); TEST_KILL_RANDOM(rocksdb_kill_odds * REDUCE_ODDS2);
writable_file_->PrepareWrite(static_cast<size_t>(GetFileSize()), left); {
IOSTATS_TIMER_GUARD(prepare_write_nanos);
TEST_SYNC_POINT("WritableFileWriter::Append:BeforePrepareWrite");
writable_file_->PrepareWrite(static_cast<size_t>(GetFileSize()), left);
}
// if there is no space in the cache, then flush // if there is no space in the cache, then flush
if (cursize_ + left > capacity_) { if (cursize_ + left > capacity_) {
s = Flush(); s = Flush();
@ -105,6 +109,7 @@ Status WritableFileWriter::Flush() {
size_t size = RequestToken(left); size_t size = RequestToken(left);
{ {
IOSTATS_TIMER_GUARD(write_nanos); IOSTATS_TIMER_GUARD(write_nanos);
TEST_SYNC_POINT("WritableFileWriter::Flush:BeforeAppend");
Status s = writable_file_->Append(Slice(src, size)); Status s = writable_file_->Append(Slice(src, size));
if (!s.ok()) { if (!s.ok()) {
return s; return s;
@ -182,6 +187,8 @@ Status WritableFileWriter::SyncWithoutFlush(bool use_fsync) {
Status WritableFileWriter::SyncInternal(bool use_fsync) { Status WritableFileWriter::SyncInternal(bool use_fsync) {
Status s; Status s;
IOSTATS_TIMER_GUARD(fsync_nanos);
TEST_SYNC_POINT("WritableFileWriter::SyncInternal:0");
if (use_fsync) { if (use_fsync) {
s = writable_file_->Fsync(); s = writable_file_->Fsync();
} else { } else {
@ -192,6 +199,7 @@ Status WritableFileWriter::SyncInternal(bool use_fsync) {
Status WritableFileWriter::RangeSync(off_t offset, off_t nbytes) { Status WritableFileWriter::RangeSync(off_t offset, off_t nbytes) {
IOSTATS_TIMER_GUARD(range_sync_nanos); IOSTATS_TIMER_GUARD(range_sync_nanos);
TEST_SYNC_POINT("WritableFileWriter::RangeSync:0");
return writable_file_->RangeSync(offset, nbytes); return writable_file_->RangeSync(offset, nbytes);
} }

15
util/iostats_context.cc
Unescape View File

@ -26,6 +26,8 @@ void IOStatsContext::Reset() {
write_nanos = 0; write_nanos = 0;
read_nanos = 0; read_nanos = 0;
range_sync_nanos = 0; range_sync_nanos = 0;
prepare_write_nanos = 0;
fsync_nanos = 0;
logger_nanos = 0; logger_nanos = 0;
} }
@ -33,15 +35,10 @@ void IOStatsContext::Reset() {
std::string IOStatsContext::ToString() const { std::string IOStatsContext::ToString() const {
std::ostringstream ss; std::ostringstream ss;
ss << OUTPUT(thread_pool_id) ss << OUTPUT(thread_pool_id) << OUTPUT(bytes_read) << OUTPUT(bytes_written)
<< OUTPUT(bytes_read) << OUTPUT(open_nanos) << OUTPUT(allocate_nanos) << OUTPUT(write_nanos)
<< OUTPUT(bytes_written) << OUTPUT(read_nanos) << OUTPUT(range_sync_nanos) << OUTPUT(fsync_nanos)
<< OUTPUT(open_nanos) << OUTPUT(prepare_write_nanos) << OUTPUT(logger_nanos);
<< OUTPUT(allocate_nanos)
<< OUTPUT(write_nanos)
<< OUTPUT(read_nanos)
<< OUTPUT(range_sync_nanos)
<< OUTPUT(logger_nanos);
return ss.str(); return ss.str();
} }

119
util/mutable_cf_options.h
Unescape View File

@ -13,68 +13,70 @@ namespace rocksdb {
struct MutableCFOptions { struct MutableCFOptions {
MutableCFOptions(const Options& options, const ImmutableCFOptions& ioptions) MutableCFOptions(const Options& options, const ImmutableCFOptions& ioptions)
: write_buffer_size(options.write_buffer_size), : write_buffer_size(options.write_buffer_size),
max_write_buffer_number(options.max_write_buffer_number), max_write_buffer_number(options.max_write_buffer_number),
arena_block_size(options.arena_block_size), arena_block_size(options.arena_block_size),
memtable_prefix_bloom_bits(options.memtable_prefix_bloom_bits), memtable_prefix_bloom_bits(options.memtable_prefix_bloom_bits),
memtable_prefix_bloom_probes(options.memtable_prefix_bloom_probes), memtable_prefix_bloom_probes(options.memtable_prefix_bloom_probes),
memtable_prefix_bloom_huge_page_tlb_size( memtable_prefix_bloom_huge_page_tlb_size(
options.memtable_prefix_bloom_huge_page_tlb_size), options.memtable_prefix_bloom_huge_page_tlb_size),
max_successive_merges(options.max_successive_merges), max_successive_merges(options.max_successive_merges),
filter_deletes(options.filter_deletes), filter_deletes(options.filter_deletes),
inplace_update_num_locks(options.inplace_update_num_locks), inplace_update_num_locks(options.inplace_update_num_locks),
disable_auto_compactions(options.disable_auto_compactions), disable_auto_compactions(options.disable_auto_compactions),
soft_rate_limit(options.soft_rate_limit), soft_rate_limit(options.soft_rate_limit),
hard_rate_limit(options.hard_rate_limit), hard_rate_limit(options.hard_rate_limit),
level0_file_num_compaction_trigger( level0_file_num_compaction_trigger(
options.level0_file_num_compaction_trigger), options.level0_file_num_compaction_trigger),
level0_slowdown_writes_trigger(options.level0_slowdown_writes_trigger), level0_slowdown_writes_trigger(options.level0_slowdown_writes_trigger),
level0_stop_writes_trigger(options.level0_stop_writes_trigger), level0_stop_writes_trigger(options.level0_stop_writes_trigger),
max_grandparent_overlap_factor(options.max_grandparent_overlap_factor), max_grandparent_overlap_factor(options.max_grandparent_overlap_factor),
expanded_compaction_factor(options.expanded_compaction_factor), expanded_compaction_factor(options.expanded_compaction_factor),
source_compaction_factor(options.source_compaction_factor), source_compaction_factor(options.source_compaction_factor),
target_file_size_base(options.target_file_size_base), target_file_size_base(options.target_file_size_base),
target_file_size_multiplier(options.target_file_size_multiplier), target_file_size_multiplier(options.target_file_size_multiplier),
max_bytes_for_level_base(options.max_bytes_for_level_base), max_bytes_for_level_base(options.max_bytes_for_level_base),
max_bytes_for_level_multiplier(options.max_bytes_for_level_multiplier), max_bytes_for_level_multiplier(options.max_bytes_for_level_multiplier),
max_bytes_for_level_multiplier_additional( max_bytes_for_level_multiplier_additional(
options.max_bytes_for_level_multiplier_additional), options.max_bytes_for_level_multiplier_additional),
verify_checksums_in_compaction(options.verify_checksums_in_compaction), verify_checksums_in_compaction(options.verify_checksums_in_compaction),
num_subcompactions(options.num_subcompactions), num_subcompactions(options.num_subcompactions),
max_sequential_skip_in_iterations( max_sequential_skip_in_iterations(
options.max_sequential_skip_in_iterations), options.max_sequential_skip_in_iterations),
paranoid_file_checks(options.paranoid_file_checks) paranoid_file_checks(options.paranoid_file_checks),
compaction_measure_io_stats(options.compaction_measure_io_stats)
{ {
RefreshDerivedOptions(ioptions); RefreshDerivedOptions(ioptions);
} }
MutableCFOptions() MutableCFOptions()
: write_buffer_size(0), : write_buffer_size(0),
max_write_buffer_number(0), max_write_buffer_number(0),
arena_block_size(0), arena_block_size(0),
memtable_prefix_bloom_bits(0), memtable_prefix_bloom_bits(0),
memtable_prefix_bloom_probes(0), memtable_prefix_bloom_probes(0),
memtable_prefix_bloom_huge_page_tlb_size(0), memtable_prefix_bloom_huge_page_tlb_size(0),
max_successive_merges(0), max_successive_merges(0),
filter_deletes(false), filter_deletes(false),
inplace_update_num_locks(0), inplace_update_num_locks(0),
disable_auto_compactions(false), disable_auto_compactions(false),
soft_rate_limit(0), soft_rate_limit(0),
hard_rate_limit(0), hard_rate_limit(0),
level0_file_num_compaction_trigger(0), level0_file_num_compaction_trigger(0),
level0_slowdown_writes_trigger(0), level0_slowdown_writes_trigger(0),
level0_stop_writes_trigger(0), level0_stop_writes_trigger(0),
max_grandparent_overlap_factor(0), max_grandparent_overlap_factor(0),
expanded_compaction_factor(0), expanded_compaction_factor(0),
source_compaction_factor(0), source_compaction_factor(0),
target_file_size_base(0), target_file_size_base(0),
target_file_size_multiplier(0), target_file_size_multiplier(0),
max_bytes_for_level_base(0), max_bytes_for_level_base(0),
max_bytes_for_level_multiplier(0), max_bytes_for_level_multiplier(0),
verify_checksums_in_compaction(false), verify_checksums_in_compaction(false),
num_subcompactions(1), num_subcompactions(1),
max_sequential_skip_in_iterations(0), max_sequential_skip_in_iterations(0),
paranoid_file_checks(false) paranoid_file_checks(false),
{} compaction_measure_io_stats(false) {}
// Must be called after any change to MutableCFOptions // Must be called after any change to MutableCFOptions
void RefreshDerivedOptions(const ImmutableCFOptions& ioptions); void RefreshDerivedOptions(const ImmutableCFOptions& ioptions);
@ -128,6 +130,7 @@ struct MutableCFOptions {
// Misc options // Misc options
uint64_t max_sequential_skip_in_iterations; uint64_t max_sequential_skip_in_iterations;
bool paranoid_file_checks; bool paranoid_file_checks;
bool compaction_measure_io_stats;
// Derived options // Derived options
// Per-level target file size. // Per-level target file size.

10
util/options.cc
Unescape View File

@ -121,7 +121,8 @@ ColumnFamilyOptions::ColumnFamilyOptions()
max_successive_merges(0), max_successive_merges(0),
min_partial_merge_operands(2), min_partial_merge_operands(2),
optimize_filters_for_hits(false), optimize_filters_for_hits(false),
paranoid_file_checks(false) { paranoid_file_checks(false),
compaction_measure_io_stats(false) {
assert(memtable_factory.get() != nullptr); assert(memtable_factory.get() != nullptr);
} }
@ -185,7 +186,8 @@ ColumnFamilyOptions::ColumnFamilyOptions(const Options& options)
max_successive_merges(options.max_successive_merges), max_successive_merges(options.max_successive_merges),
min_partial_merge_operands(options.min_partial_merge_operands), min_partial_merge_operands(options.min_partial_merge_operands),
optimize_filters_for_hits(options.optimize_filters_for_hits), optimize_filters_for_hits(options.optimize_filters_for_hits),
paranoid_file_checks(options.paranoid_file_checks) { paranoid_file_checks(options.paranoid_file_checks),
compaction_measure_io_stats(options.compaction_measure_io_stats) {
assert(memtable_factory.get() != nullptr); assert(memtable_factory.get() != nullptr);
if (max_bytes_for_level_multiplier_additional.size() < if (max_bytes_for_level_multiplier_additional.size() <
static_cast<unsigned int>(num_levels)) { static_cast<unsigned int>(num_levels)) {
@ -514,6 +516,10 @@ void ColumnFamilyOptions::Dump(Logger* log) const {
max_successive_merges); max_successive_merges);
Warn(log, " Options.optimize_fllters_for_hits: %d", Warn(log, " Options.optimize_fllters_for_hits: %d",
optimize_filters_for_hits); optimize_filters_for_hits);
Warn(log, " Options.paranoid_file_checks: %d",
paranoid_file_checks);
Warn(log, " Options.compaction_measure_io_stats: %d",
compaction_measure_io_stats);
} // ColumnFamilyOptions::Dump } // ColumnFamilyOptions::Dump
void Options::Dump(Logger* log) const { void Options::Dump(Logger* log) const {

4
util/options_helper.cc
Unescape View File

@ -2,6 +2,7 @@
// This source code is licensed under the BSD-style license found in the // This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant // LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory. // of patent rights can be found in the PATENTS file in the same directory.
#include "util/options_helper.h"
#include <cassert> #include <cassert>
#include <cctype> #include <cctype>
@ -16,7 +17,6 @@
#include "rocksdb/table.h" #include "rocksdb/table.h"
#include "table/block_based_table_factory.h" #include "table/block_based_table_factory.h"
#include "util/logging.h" #include "util/logging.h"
#include "util/options_helper.h"
namespace rocksdb { namespace rocksdb {
@ -449,6 +449,8 @@ bool ParseColumnFamilyOption(const std::string& name, const std::string& value,
new_options->min_partial_merge_operands = ParseUint32(value); new_options->min_partial_merge_operands = ParseUint32(value);
} else if (name == "inplace_update_support") { } else if (name == "inplace_update_support") {
new_options->inplace_update_support = ParseBoolean(name, value); new_options->inplace_update_support = ParseBoolean(name, value);
} else if (name == "compaction_measure_io_stats") {
new_options->compaction_measure_io_stats = ParseBoolean(name, value);
} else if (name == "prefix_extractor") { } else if (name == "prefix_extractor") {
const std::string kFixedPrefixName = "fixed:"; const std::string kFixedPrefixName = "fixed:";
const std::string kCappedPrefixName = "capped:"; const std::string kCappedPrefixName = "capped:";

1
util/options_test.cc
Unescape View File

@ -129,6 +129,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
{"filter_deletes", "0"}, {"filter_deletes", "0"},
{"max_sequential_skip_in_iterations", "24"}, {"max_sequential_skip_in_iterations", "24"},
{"inplace_update_support", "true"}, {"inplace_update_support", "true"},
{"compaction_measure_io_stats", "true"},
{"inplace_update_num_locks", "25"}, {"inplace_update_num_locks", "25"},
{"memtable_prefix_bloom_bits", "26"}, {"memtable_prefix_bloom_bits", "26"},
{"memtable_prefix_bloom_probes", "27"}, {"memtable_prefix_bloom_probes", "27"},

Write Preview
Loading…
Cancel
Save