Summary:
Allow EventListener::OnCompactionCompleted to return CompactionJobStats,
which contains useful information about a compaction.
Example CompactionJobStats returned by OnCompactionCompleted():
smallest_output_key_prefix 05000000
largest_output_key_prefix 06990000
elapsed_time 42419
num_input_records 300
num_input_files 3
num_input_files_at_output_level 2
num_output_records 200
num_output_files 1
actual_bytes_input 167200
actual_bytes_output 110688
total_input_raw_key_bytes 5400
total_input_raw_value_bytes 300000
num_records_replaced 100
is_manual_compaction 1
Test Plan: Developed a mega test in db_test which covers 20 variables in CompactionJobStats.
Reviewers: rven, igor, anthony, sdong
Reviewed By: sdong
Subscribers: tnovak, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D38463
main
Yueh-Hsuan Chiang
10 years ago
parent
3083ed2129
commit
fe5c6321cb
@ -0,0 +1,691 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include <algorithm> |
||||
#include <iostream> |
||||
#include <mutex> |
||||
#include <queue> |
||||
#include <set> |
||||
#include <thread> |
||||
#include <unordered_set> |
||||
#include <utility> |
||||
|
||||
#include "db/db_impl.h" |
||||
#include "db/dbformat.h" |
||||
#include "db/filename.h" |
||||
#include "db/job_context.h" |
||||
#include "db/version_set.h" |
||||
#include "db/write_batch_internal.h" |
||||
#include "port/stack_trace.h" |
||||
#include "rocksdb/cache.h" |
||||
#include "rocksdb/compaction_filter.h" |
||||
#include "rocksdb/db.h" |
||||
#include "rocksdb/env.h" |
||||
#include "rocksdb/experimental.h" |
||||
#include "rocksdb/filter_policy.h" |
||||
#include "rocksdb/options.h" |
||||
#include "rocksdb/perf_context.h" |
||||
#include "rocksdb/slice.h" |
||||
#include "rocksdb/slice_transform.h" |
||||
#include "rocksdb/table.h" |
||||
#include "rocksdb/table_properties.h" |
||||
#include "rocksdb/thread_status.h" |
||||
#include "rocksdb/utilities/checkpoint.h" |
||||
#include "rocksdb/utilities/convenience.h" |
||||
#include "rocksdb/utilities/write_batch_with_index.h" |
||||
#include "table/block_based_table_factory.h" |
||||
#include "table/mock_table.h" |
||||
#include "table/plain_table_factory.h" |
||||
#include "util/compression.h" |
||||
#include "util/hash.h" |
||||
#include "util/hash_linklist_rep.h" |
||||
#include "util/logging.h" |
||||
#include "util/mock_env.h" |
||||
#include "util/mutexlock.h" |
||||
#include "util/rate_limiter.h" |
||||
#include "util/scoped_arena_iterator.h" |
||||
#include "util/statistics.h" |
||||
#include "util/string_util.h" |
||||
#include "util/sync_point.h" |
||||
#include "util/testharness.h" |
||||
#include "util/testutil.h" |
||||
#include "util/thread_status_util.h" |
||||
#include "util/xfunc.h" |
||||
#include "utilities/merge_operators.h" |
||||
|
||||
#if !defined(IOS_CROSS_COMPILE) |
||||
#ifndef ROCKSDB_LITE |
||||
namespace rocksdb { |
||||
|
||||
static std::string RandomString(Random* rnd, int len, double ratio) { |
||||
std::string r; |
||||
test::CompressibleString(rnd, ratio, len, &r); |
||||
return r; |
||||
} |
||||
|
||||
std::string Key(uint64_t key, int length) { |
||||
const int kBufSize = 1000; |
||||
char buf[kBufSize]; |
||||
if (length > kBufSize) { |
||||
length = kBufSize; |
||||
} |
||||
snprintf(buf, kBufSize, "%0*lu", length, key); |
||||
return std::string(buf); |
||||
} |
||||
|
||||
class CompactionJobStatsTest : public testing::Test { |
||||
public: |
||||
std::string dbname_; |
||||
std::string alternative_wal_dir_; |
||||
Env* env_; |
||||
DB* db_; |
||||
std::vector<ColumnFamilyHandle*> handles_; |
||||
|
||||
Options last_options_; |
||||
|
||||
CompactionJobStatsTest() : env_(Env::Default()) { |
||||
env_->SetBackgroundThreads(1, Env::LOW); |
||||
env_->SetBackgroundThreads(1, Env::HIGH); |
||||
dbname_ = test::TmpDir(env_) + "/compaction_job_stats_test"; |
||||
alternative_wal_dir_ = dbname_ + "/wal"; |
||||
Options options; |
||||
options.create_if_missing = true; |
||||
auto delete_options = options; |
||||
delete_options.wal_dir = alternative_wal_dir_; |
||||
EXPECT_OK(DestroyDB(dbname_, delete_options)); |
||||
// Destroy it for not alternative WAL dir is used.
|
||||
EXPECT_OK(DestroyDB(dbname_, options)); |
||||
db_ = nullptr; |
||||
Reopen(options); |
||||
} |
||||
|
||||
~CompactionJobStatsTest() { |
||||
rocksdb::SyncPoint::GetInstance()->DisableProcessing(); |
||||
rocksdb::SyncPoint::GetInstance()->LoadDependency({}); |
||||
rocksdb::SyncPoint::GetInstance()->ClearAllCallBacks(); |
||||
Close(); |
||||
Options options; |
||||
options.db_paths.emplace_back(dbname_, 0); |
||||
options.db_paths.emplace_back(dbname_ + "_2", 0); |
||||
options.db_paths.emplace_back(dbname_ + "_3", 0); |
||||
options.db_paths.emplace_back(dbname_ + "_4", 0); |
||||
EXPECT_OK(DestroyDB(dbname_, options)); |
||||
} |
||||
|
||||
DBImpl* dbfull() { |
||||
return reinterpret_cast<DBImpl*>(db_); |
||||
} |
||||
|
||||
void CreateColumnFamilies(const std::vector<std::string>& cfs, |
||||
const Options& options) { |
||||
ColumnFamilyOptions cf_opts(options); |
||||
size_t cfi = handles_.size(); |
||||
handles_.resize(cfi + cfs.size()); |
||||
for (auto cf : cfs) { |
||||
ASSERT_OK(db_->CreateColumnFamily(cf_opts, cf, &handles_[cfi++])); |
||||
} |
||||
} |
||||
|
||||
void CreateAndReopenWithCF(const std::vector<std::string>& cfs, |
||||
const Options& options) { |
||||
CreateColumnFamilies(cfs, options); |
||||
std::vector<std::string> cfs_plus_default = cfs; |
||||
cfs_plus_default.insert(cfs_plus_default.begin(), kDefaultColumnFamilyName); |
||||
ReopenWithColumnFamilies(cfs_plus_default, options); |
||||
} |
||||
|
||||
void ReopenWithColumnFamilies(const std::vector<std::string>& cfs, |
||||
const std::vector<Options>& options) { |
||||
ASSERT_OK(TryReopenWithColumnFamilies(cfs, options)); |
||||
} |
||||
|
||||
void ReopenWithColumnFamilies(const std::vector<std::string>& cfs, |
||||
const Options& options) { |
||||
ASSERT_OK(TryReopenWithColumnFamilies(cfs, options)); |
||||
} |
||||
|
||||
Status TryReopenWithColumnFamilies( |
||||
const std::vector<std::string>& cfs, |
||||
const std::vector<Options>& options) { |
||||
Close(); |
||||
EXPECT_EQ(cfs.size(), options.size()); |
||||
std::vector<ColumnFamilyDescriptor> column_families; |
||||
for (size_t i = 0; i < cfs.size(); ++i) { |
||||
column_families.push_back(ColumnFamilyDescriptor(cfs[i], options[i])); |
||||
} |
||||
DBOptions db_opts = DBOptions(options[0]); |
||||
return DB::Open(db_opts, dbname_, column_families, &handles_, &db_); |
||||
} |
||||
|
||||
Status TryReopenWithColumnFamilies(const std::vector<std::string>& cfs, |
||||
const Options& options) { |
||||
Close(); |
||||
std::vector<Options> v_opts(cfs.size(), options); |
||||
return TryReopenWithColumnFamilies(cfs, v_opts); |
||||
} |
||||
|
||||
void Reopen(const Options& options) { |
||||
ASSERT_OK(TryReopen(options)); |
||||
} |
||||
|
||||
void Close() { |
||||
for (auto h : handles_) { |
||||
delete h; |
||||
} |
||||
handles_.clear(); |
||||
delete db_; |
||||
db_ = nullptr; |
||||
} |
||||
|
||||
void DestroyAndReopen(const Options& options) { |
||||
// Destroy using last options
|
||||
Destroy(last_options_); |
||||
ASSERT_OK(TryReopen(options)); |
||||
} |
||||
|
||||
void Destroy(const Options& options) { |
||||
Close(); |
||||
ASSERT_OK(DestroyDB(dbname_, options)); |
||||
} |
||||
|
||||
Status ReadOnlyReopen(const Options& options) { |
||||
return DB::OpenForReadOnly(options, dbname_, &db_); |
||||
} |
||||
|
||||
Status TryReopen(const Options& options) { |
||||
Close(); |
||||
last_options_ = options; |
||||
return DB::Open(options, dbname_, &db_); |
||||
} |
||||
|
||||
Status Flush(int cf = 0) { |
||||
if (cf == 0) { |
||||
return db_->Flush(FlushOptions()); |
||||
} else { |
||||
return db_->Flush(FlushOptions(), handles_[cf]); |
||||
} |
||||
} |
||||
|
||||
Status Put(const Slice& k, const Slice& v, WriteOptions wo = WriteOptions()) { |
||||
return db_->Put(wo, k, v); |
||||
} |
||||
|
||||
Status Put(int cf, const Slice& k, const Slice& v, |
||||
WriteOptions wo = WriteOptions()) { |
||||
return db_->Put(wo, handles_[cf], k, v); |
||||
} |
||||
|
||||
Status Delete(const std::string& k) { |
||||
return db_->Delete(WriteOptions(), k); |
||||
} |
||||
|
||||
Status Delete(int cf, const std::string& k) { |
||||
return db_->Delete(WriteOptions(), handles_[cf], k); |
||||
} |
||||
|
||||
std::string Get(const std::string& k, const Snapshot* snapshot = nullptr) { |
||||
ReadOptions options; |
||||
options.verify_checksums = true; |
||||
options.snapshot = snapshot; |
||||
std::string result; |
||||
Status s = db_->Get(options, k, &result); |
||||
if (s.IsNotFound()) { |
||||
result = "NOT_FOUND"; |
||||
} else if (!s.ok()) { |
||||
result = s.ToString(); |
||||
} |
||||
return result; |
||||
} |
||||
|
||||
std::string Get(int cf, const std::string& k, |
||||
const Snapshot* snapshot = nullptr) { |
||||
ReadOptions options; |
||||
options.verify_checksums = true; |
||||
options.snapshot = snapshot; |
||||
std::string result; |
||||
Status s = db_->Get(options, handles_[cf], k, &result); |
||||
if (s.IsNotFound()) { |
||||
result = "NOT_FOUND"; |
||||
} else if (!s.ok()) { |
||||
result = s.ToString(); |
||||
} |
||||
return result; |
||||
} |
||||
|
||||
int NumTableFilesAtLevel(int level, int cf = 0) { |
||||
std::string property; |
||||
if (cf == 0) { |
||||
// default cfd
|
||||
EXPECT_TRUE(db_->GetProperty( |
||||
"rocksdb.num-files-at-level" + NumberToString(level), &property)); |
||||
} else { |
||||
EXPECT_TRUE(db_->GetProperty( |
||||
handles_[cf], "rocksdb.num-files-at-level" + NumberToString(level), |
||||
&property)); |
||||
} |
||||
return atoi(property.c_str()); |
||||
} |
||||
|
||||
// Return spread of files per level
|
||||
std::string FilesPerLevel(int cf = 0) { |
||||
int num_levels = |
||||
(cf == 0) ? db_->NumberLevels() : db_->NumberLevels(handles_[1]); |
||||
std::string result; |
||||
size_t last_non_zero_offset = 0; |
||||
for (int level = 0; level < num_levels; level++) { |
||||
int f = NumTableFilesAtLevel(level, cf); |
||||
char buf[100]; |
||||
snprintf(buf, sizeof(buf), "%s%d", (level ? "," : ""), f); |
||||
result += buf; |
||||
if (f > 0) { |
||||
last_non_zero_offset = result.size(); |
||||
} |
||||
} |
||||
result.resize(last_non_zero_offset); |
||||
return result; |
||||
} |
||||
|
||||
uint64_t Size(const Slice& start, const Slice& limit, int cf = 0) { |
||||
Range r(start, limit); |
||||
uint64_t size; |
||||
if (cf == 0) { |
||||
db_->GetApproximateSizes(&r, 1, &size); |
||||
} else { |
||||
db_->GetApproximateSizes(handles_[1], &r, 1, &size); |
||||
} |
||||
return size; |
||||
} |
||||
|
||||
void Compact(int cf, const Slice& start, const Slice& limit, |
||||
uint32_t target_path_id) { |
||||
ASSERT_OK(db_->CompactRange(handles_[cf], &start, &limit, false, -1, |
||||
target_path_id)); |
||||
} |
||||
|
||||
void Compact(int cf, const Slice& start, const Slice& limit) { |
||||
ASSERT_OK(db_->CompactRange(handles_[cf], &start, &limit)); |
||||
} |
||||
|
||||
void Compact(const Slice& start, const Slice& limit) { |
||||
ASSERT_OK(db_->CompactRange(&start, &limit)); |
||||
} |
||||
|
||||
// Do n memtable compactions, each of which produces an sstable
|
||||
// covering the range [small,large].
|
||||
void MakeTables(int n, const std::string& small, const std::string& large, |
||||
int cf = 0) { |
||||
for (int i = 0; i < n; i++) { |
||||
ASSERT_OK(Put(cf, small, "begin")); |
||||
ASSERT_OK(Put(cf, large, "end")); |
||||
ASSERT_OK(Flush(cf)); |
||||
} |
||||
} |
||||
|
||||
void MakeTableWithKeyValues( |
||||
Random* rnd, uint64_t smallest, uint64_t largest, |
||||
int key_size, int value_size, uint64_t interval, |
||||
double ratio, int cf = 0) { |
||||
for (auto key = smallest; key < largest; key += interval) { |
||||
ASSERT_OK(Put(cf, Slice(Key(key, key_size)), |
||||
Slice(RandomString(rnd, value_size, ratio)))); |
||||
} |
||||
ASSERT_OK(Flush(cf)); |
||||
} |
||||
}; |
||||
|
||||
// An EventListener which helps verify the compaction results in
|
||||
// test CompactionJobStatsTest.
|
||||
class CompactionJobStatsChecker : public EventListener { |
||||
public: |
||||
CompactionJobStatsChecker() : compression_enabled_(false) {} |
||||
|
||||
size_t NumberOfUnverifiedStats() { return expected_stats_.size(); } |
||||
|
||||
// Once a compaction completed, this functionw will verify the returned
|
||||
// CompactionJobInfo with the oldest CompactionJobInfo added earlier
|
||||
// in "expected_stats_" which has not yet being used for verification.
|
||||
virtual void OnCompactionCompleted(DB *db, const CompactionJobInfo& ci) { |
||||
std::lock_guard<std::mutex> lock(mutex_); |
||||
|
||||
if (expected_stats_.size()) { |
||||
Verify(ci.stats, expected_stats_.front()); |
||||
expected_stats_.pop(); |
||||
} |
||||
} |
||||
|
||||
// A helper function which verifies whether two CompactionJobStats
|
||||
// match. The verification of all compaction stats are done by
|
||||
// ASSERT_EQ except the following stats, which we use ASSERT_GE
|
||||
// and ASSERT_LE with a reasonable (< 15%) bias:
|
||||
// 1. write-amplication
|
||||
// 2. actual bytes input and output, which relies on the compression
|
||||
// ratio and the implementation of table formats.
|
||||
void Verify(const CompactionJobStats& current_stats, |
||||
const CompactionJobStats& stats) { |
||||
// time
|
||||
ASSERT_GT(current_stats.elapsed_micros, 0U); |
||||
|
||||
ASSERT_EQ(current_stats.num_input_records, |
||||
stats.num_input_records); |
||||
ASSERT_EQ(current_stats.num_input_files, |
||||
stats.num_input_files); |
||||
ASSERT_EQ(current_stats.num_input_files_at_output_level, |
||||
stats.num_input_files_at_output_level); |
||||
|
||||
ASSERT_EQ(current_stats.num_output_records, |
||||
stats.num_output_records); |
||||
ASSERT_EQ(current_stats.num_output_files, |
||||
stats.num_output_files); |
||||
|
||||
ASSERT_EQ(current_stats.is_manual_compaction, |
||||
stats.is_manual_compaction); |
||||
|
||||
// file size
|
||||
double kFileSizeBias = 0.15; |
||||
ASSERT_GE(current_stats.total_input_bytes * (1.00 + kFileSizeBias), |
||||
stats.total_input_bytes); |
||||
ASSERT_LE(current_stats.total_input_bytes, |
||||
stats.total_input_bytes * (1.00 + kFileSizeBias)); |
||||
ASSERT_GE(current_stats.total_output_bytes * (1.00 + kFileSizeBias), |
||||
stats.total_output_bytes); |
||||
ASSERT_LE(current_stats.total_output_bytes, |
||||
stats.total_output_bytes * (1.00 + kFileSizeBias)); |
||||
ASSERT_EQ(current_stats.total_input_raw_key_bytes, |
||||
stats.total_input_raw_key_bytes); |
||||
ASSERT_EQ(current_stats.total_input_raw_value_bytes, |
||||
stats.total_input_raw_value_bytes); |
||||
|
||||
ASSERT_EQ(current_stats.num_records_replaced, |
||||
stats.num_records_replaced); |
||||
|
||||
ASSERT_EQ( |
||||
std::string(current_stats.smallest_output_key_prefix), |
||||
std::string(stats.smallest_output_key_prefix)); |
||||
ASSERT_EQ( |
||||
std::string(current_stats.largest_output_key_prefix), |
||||
std::string(stats.largest_output_key_prefix)); |
||||
} |
||||
|
||||
// Add an expected compaction stats, which will be used to
|
||||
// verify the CompactionJobStats returned by the OnCompactionCompleted()
|
||||
// callback.
|
||||
void AddExpectedStats(const CompactionJobStats& stats) { |
||||
std::lock_guard<std::mutex> lock(mutex_); |
||||
expected_stats_.push(stats); |
||||
} |
||||
|
||||
void EnableCompression(bool flag) { |
||||
compression_enabled_ = flag; |
||||
} |
||||
|
||||
private: |
||||
std::mutex mutex_; |
||||
std::queue<CompactionJobStats> expected_stats_; |
||||
bool compression_enabled_; |
||||
}; |
||||
|
||||
namespace { |
||||
|
||||
uint64_t EstimatedFileSize( |
||||
uint64_t num_records, size_t key_size, size_t value_size, |
||||
double compression_ratio = 1.0, |
||||
size_t block_size = 4096, |
||||
int bloom_bits_per_key = 10) { |
||||
const size_t kPerKeyOverhead = 8; |
||||
const size_t kFooterSize = 512; |
||||
|
||||
uint64_t data_size = |
||||
num_records * (key_size + value_size * compression_ratio + |
||||
kPerKeyOverhead); |
||||
|
||||
return data_size + kFooterSize |
||||
+ num_records * bloom_bits_per_key / 8 // filter block
|
||||
+ data_size * (key_size + 8) / block_size; // index block
|
||||
} |
||||
|
||||
namespace { |
||||
|
||||
void CopyPrefix( |
||||
char* dst, size_t dst_length, const Slice& src) { |
||||
assert(dst_length > 0); |
||||
size_t length = src.size() > dst_length - 1 ? dst_length - 1 : src.size(); |
||||
memcpy(dst, src.data(), length); |
||||
dst[length] = 0; |
||||
} |
||||
|
||||
} // namespace
|
||||
|
||||
CompactionJobStats NewManualCompactionJobStats( |
||||
const std::string& smallest_key, const std::string& largest_key, |
||||
size_t num_input_files, size_t num_input_files_at_output_level, |
||||
uint64_t num_input_records, size_t key_size, size_t value_size, |
||||
size_t num_output_files, uint64_t num_output_records, |
||||
double compression_ratio, uint64_t num_records_replaced) { |
||||
CompactionJobStats stats; |
||||
stats.Reset(); |
||||
|
||||
stats.num_input_records = num_input_records; |
||||
stats.num_input_files = num_input_files; |
||||
stats.num_input_files_at_output_level = num_input_files_at_output_level; |
||||
|
||||
stats.num_output_records = num_output_records; |
||||
stats.num_output_files = num_output_files; |
||||
|
||||
stats.total_input_bytes = |
||||
EstimatedFileSize( |
||||
num_input_records / num_input_files, |
||||
key_size, value_size, compression_ratio) * num_input_files; |
||||
stats.total_output_bytes = |
||||
EstimatedFileSize( |
||||
num_output_records / num_output_files, |
||||
key_size, value_size, compression_ratio) * num_output_files; |
||||
stats.total_input_raw_key_bytes = |
||||
num_input_records * (key_size + 8); |
||||
stats.total_input_raw_value_bytes = |
||||
num_input_records * value_size; |
||||
|
||||
stats.is_manual_compaction = true; |
||||
|
||||
stats.num_records_replaced = num_records_replaced; |
||||
|
||||
CopyPrefix(stats.smallest_output_key_prefix, |
||||
sizeof(stats.smallest_output_key_prefix), |
||||
smallest_key); |
||||
CopyPrefix(stats.largest_output_key_prefix, |
||||
sizeof(stats.largest_output_key_prefix), |
||||
largest_key); |
||||
|
||||
return stats; |
||||
} |
||||
|
||||
CompressionType GetAnyCompression() { |
||||
if (Snappy_Supported()) { |
||||
return kSnappyCompression; |
||||
} else if (Zlib_Supported()) { |
||||
return kZlibCompression; |
||||
} else if (BZip2_Supported()) { |
||||
return kBZip2Compression; |
||||
} else if (LZ4_Supported()) { |
||||
return kLZ4Compression; |
||||
} |
||||
return kNoCompression; |
||||
} |
||||
|
||||
} // namespace
|
||||
|
||||
TEST_F(CompactionJobStatsTest, CompactionJobStatsTest) { |
||||
Random rnd(301); |
||||
const int kBufSize = 100; |
||||
char buf[kBufSize]; |
||||
uint64_t key_base = 100000000l; |
||||
// Note: key_base must be multiple of num_keys_per_L0_file
|
||||
int num_keys_per_L0_file = 100; |
||||
const int kTestScale = 8; |
||||
const int kKeySize = 10; |
||||
const int kValueSize = 1000; |
||||
const double kCompressionRatio = 0.5; |
||||
double compression_ratio = 1.0; |
||||
uint64_t key_interval = key_base / num_keys_per_L0_file; |
||||
|
||||
// Whenever a compaction completes, this listener will try to
|
||||
// verify whether the returned CompactionJobStats matches
|
||||
// what we expect. The expected CompactionJobStats is added
|
||||
// via AddExpectedStats().
|
||||
auto* stats_checker = new CompactionJobStatsChecker(); |
||||
Options options; |
||||
options.listeners.emplace_back(stats_checker); |
||||
options.create_if_missing = true; |
||||
options.max_background_flushes = 0; |
||||
options.max_mem_compaction_level = 0; |
||||
// just enough setting to hold off auto-compaction.
|
||||
options.level0_file_num_compaction_trigger = kTestScale + 1; |
||||
options.num_levels = 3; |
||||
options.compression = kNoCompression; |
||||
|
||||
for (int test = 0; test < 2; ++test) { |
||||
DestroyAndReopen(options); |
||||
CreateAndReopenWithCF({"pikachu"}, options); |
||||
|
||||
// 1st Phase: generate "num_L0_files" L0 files.
|
||||
int num_L0_files = 0; |
||||
for (uint64_t start_key = key_base; |
||||
start_key <= key_base * kTestScale; |
||||
start_key += key_base) { |
||||
MakeTableWithKeyValues( |
||||
&rnd, start_key, start_key + key_base - 1, |
||||
kKeySize, kValueSize, key_interval, |
||||
kCompressionRatio, 1); |
||||
snprintf(buf, kBufSize, "%d", ++num_L0_files); |
||||
ASSERT_EQ(std::string(buf), FilesPerLevel(1)); |
||||
} |
||||
ASSERT_EQ(std::to_string(num_L0_files), FilesPerLevel(1)); |
||||
|
||||
// 2nd Phase: perform L0 -> L1 compaction.
|
||||
int L0_compaction_count = 6; |
||||
int count = 1; |
||||
std::string smallest_key; |
||||
std::string largest_key; |
||||
for (uint64_t start_key = key_base; |
||||
start_key <= key_base * L0_compaction_count; |
||||
start_key += key_base, count++) { |
||||
smallest_key = Key(start_key, 10); |
||||
largest_key = Key(start_key + key_base - key_interval, 10); |
||||
stats_checker->AddExpectedStats( |
||||
NewManualCompactionJobStats( |
||||
smallest_key, largest_key, |
||||
1, 0, num_keys_per_L0_file, |
||||
kKeySize, kValueSize, |
||||
1, num_keys_per_L0_file, |
||||
compression_ratio, 0)); |
||||
ASSERT_EQ(stats_checker->NumberOfUnverifiedStats(), 1U); |
||||
Compact(1, smallest_key, largest_key); |
||||
snprintf(buf, kBufSize, "%d,%d", num_L0_files - count, count); |
||||
ASSERT_EQ(std::string(buf), FilesPerLevel(1)); |
||||
} |
||||
|
||||
// compact two files into one in the last L0 -> L1 compaction
|
||||
int num_remaining_L0 = num_L0_files - L0_compaction_count; |
||||
smallest_key = Key(key_base * (L0_compaction_count + 1), 10); |
||||
largest_key = Key(key_base * (kTestScale + 1) - key_interval, 10); |
||||
stats_checker->AddExpectedStats( |
||||
NewManualCompactionJobStats( |
||||
smallest_key, largest_key, |
||||
num_remaining_L0, |
||||
0, num_keys_per_L0_file * num_remaining_L0, |
||||
kKeySize, kValueSize, |
||||
1, num_keys_per_L0_file * num_remaining_L0, |
||||
compression_ratio, 0)); |
||||
ASSERT_EQ(stats_checker->NumberOfUnverifiedStats(), 1U); |
||||
Compact(1, smallest_key, largest_key); |
||||
|
||||
int num_L1_files = num_L0_files - num_remaining_L0 + 1; |
||||
num_L0_files = 0; |
||||
snprintf(buf, kBufSize, "%d,%d", num_L0_files, num_L1_files); |
||||
ASSERT_EQ(std::string(buf), FilesPerLevel(1)); |
||||
|
||||
// 3rd Phase: generate sparse L0 files (wider key-range, same num of keys)
|
||||
int sparseness = 2; |
||||
for (uint64_t start_key = key_base; |
||||
start_key <= key_base * kTestScale; |
||||
start_key += key_base * sparseness) { |
||||
MakeTableWithKeyValues( |
||||
&rnd, start_key, start_key + key_base * sparseness - 1, |
||||
kKeySize, kValueSize, |
||||
key_base * sparseness / num_keys_per_L0_file, |
||||
kCompressionRatio, 1); |
||||
snprintf(buf, kBufSize, "%d,%d", ++num_L0_files, num_L1_files); |
||||
ASSERT_EQ(std::string(buf), FilesPerLevel(1)); |
||||
} |
||||
|
||||
// 4th Phase: perform L0 -> L1 compaction again, expect higher write amp
|
||||
for (uint64_t start_key = key_base; |
||||
num_L0_files > 1; |
||||
start_key += key_base * sparseness) { |
||||
smallest_key = Key(start_key, 10); |
||||
largest_key = |
||||
Key(start_key + key_base * sparseness - key_interval, 10); |
||||
stats_checker->AddExpectedStats( |
||||
NewManualCompactionJobStats( |
||||
smallest_key, largest_key, |
||||
3, 2, num_keys_per_L0_file * 3, |
||||
kKeySize, kValueSize, |
||||
1, num_keys_per_L0_file * 2, // 1/3 of the data will be updated.
|
||||
compression_ratio, |
||||
num_keys_per_L0_file)); |
||||
ASSERT_EQ(stats_checker->NumberOfUnverifiedStats(), 1U); |
||||
Compact(1, smallest_key, largest_key); |
||||
snprintf(buf, kBufSize, "%d,%d", |
||||
--num_L0_files, --num_L1_files); |
||||
ASSERT_EQ(std::string(buf), FilesPerLevel(1)); |
||||
} |
||||
|
||||
// 5th Phase: Do a full compaction, which involves in two sub-compactions.
|
||||
// Here we expect to have 1 L0 files and 4 L1 files
|
||||
// In the first sub-compaction, we expect L0 compaction.
|
||||
smallest_key = Key(key_base, 10); |
||||
largest_key = Key(key_base * (kTestScale + 1) - key_interval, 10); |
||||
stats_checker->AddExpectedStats( |
||||
NewManualCompactionJobStats( |
||||
Key(key_base * (kTestScale + 1 - sparseness), 10), largest_key, |
||||
2, 1, num_keys_per_L0_file * 3, |
||||
kKeySize, kValueSize, |
||||
1, num_keys_per_L0_file * 2, |
||||
compression_ratio, |
||||
num_keys_per_L0_file)); |
||||
// In the second sub-compaction, we expect L1 compaction.
|
||||
stats_checker->AddExpectedStats( |
||||
NewManualCompactionJobStats( |
||||
smallest_key, largest_key, |
||||
4, 0, num_keys_per_L0_file * 8, |
||||
kKeySize, kValueSize, |
||||
1, num_keys_per_L0_file * 8, |
||||
compression_ratio, 0)); |
||||
ASSERT_EQ(stats_checker->NumberOfUnverifiedStats(), 2U); |
||||
Compact(1, smallest_key, largest_key); |
||||
ASSERT_EQ("0,1", FilesPerLevel(1)); |
||||
options.compression = GetAnyCompression(); |
||||
if (options.compression == kNoCompression) { |
||||
break; |
||||
} |
||||
stats_checker->EnableCompression(true); |
||||
compression_ratio = kCompressionRatio; |
||||
} |
||||
ASSERT_EQ(stats_checker->NumberOfUnverifiedStats(), 0U); |
||||
} |
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) { |
||||
rocksdb::port::InstallStackTraceHandler(); |
||||
::testing::InitGoogleTest(&argc, argv); |
||||
return RUN_ALL_TESTS(); |
||||
} |
||||
|
||||
#endif // !ROCKSDB_LITE
|
||||
#endif // !defined(IOS_CROSS_COMPILE)
|
||||
@ -0,0 +1,51 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#pragma once |
||||
#include <stddef.h> |
||||
#include <stdint.h> |
||||
|
||||
namespace rocksdb { |
||||
struct CompactionJobStats { |
||||
CompactionJobStats() { Reset(); } |
||||
void Reset(); |
||||
|
||||
// the elapsed time in micro of this compaction.
|
||||
uint64_t elapsed_micros; |
||||
|
||||
// the number of compaction input records.
|
||||
uint64_t num_input_records; |
||||
// the number of compaction input files.
|
||||
size_t num_input_files; |
||||
// the number of compaction input files at the output level.
|
||||
size_t num_input_files_at_output_level; |
||||
|
||||
// the number of compaction output records.
|
||||
uint64_t num_output_records; |
||||
// the number of compaction output files.
|
||||
size_t num_output_files; |
||||
|
||||
// true if the compaction is a manual compaction
|
||||
bool is_manual_compaction; |
||||
|
||||
// the size of the compaction input in bytes.
|
||||
uint64_t total_input_bytes; |
||||
// the size of the compaction output in bytes.
|
||||
uint64_t total_output_bytes; |
||||
|
||||
// number of records being replaced by newer record associated with same key
|
||||
uint64_t num_records_replaced; |
||||
|
||||
// the sum of the uncompressed input keys in bytes.
|
||||
uint64_t total_input_raw_key_bytes; |
||||
// the sum of the uncompressed input values in bytes.
|
||||
uint64_t total_input_raw_value_bytes; |
||||
|
||||
// 0-terminated strings storing the first 8 bytes of the smallest and
|
||||
// largest key in the output.
|
||||
char smallest_output_key_prefix[9]; |
||||
char largest_output_key_prefix[9]; |
||||
}; |
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,44 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#include <cstring> |
||||
#include "include/rocksdb/compaction_job_stats.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
#ifndef ROCKSDB_LITE |
||||
|
||||
void CompactionJobStats::Reset() { |
||||
elapsed_micros = 0; |
||||
|
||||
num_input_files = 0; |
||||
num_input_files_at_output_level = 0; |
||||
num_output_files = 0; |
||||
|
||||
num_input_records = 0; |
||||
num_output_records = 0; |
||||
|
||||
total_input_bytes = 0; |
||||
total_output_bytes = 0; |
||||
|
||||
total_input_raw_key_bytes = 0; |
||||
total_input_raw_value_bytes = 0; |
||||
|
||||
num_records_replaced = 0; |
||||
|
||||
is_manual_compaction = 0; |
||||
|
||||
smallest_output_key_prefix[0] = 0; |
||||
largest_output_key_prefix[0] = 0; |
||||
} |
||||
|
||||
#else |
||||
|
||||
void CompactionJobStats::Reset() { |
||||
} |
||||
|
||||
#endif // !ROCKSDB_LITE
|
||||
|
||||
} // namespace rocksdb
|
||||
Loading…
Reference in new issue