From 116ec527f2e0f790e1220fdea0cbb9cd8ba81f0d Mon Sep 17 00:00:00 2001 From: Dhruba Borthakur Date: Wed, 3 Jul 2013 15:32:49 -0700 Subject: [PATCH] Renamed 'hybrid_compaction' tp be "Universal Compaction'. Summary: All the universal compaction parameters are encapsulated in a new file universal_compaction.h Test Plan: make check --- db/db_bench.cc | 42 +++++++++---------- db/db_impl.cc | 9 ++-- db/version_set.cc | 32 ++++++++------- include/leveldb/options.h | 21 +++++----- include/leveldb/universal_compaction.h | 57 ++++++++++++++++++++++++++ tools/db_stress.cc | 12 +++--- util/ldb_cmd.cc | 2 +- util/options.cc | 18 ++++---- 8 files changed, 128 insertions(+), 65 deletions(-) create mode 100644 include/leveldb/universal_compaction.h diff --git a/db/db_bench.cc b/db/db_bench.cc index f45777801..ecd979ec8 100644 --- a/db/db_bench.cc +++ b/db/db_bench.cc @@ -9,6 +9,7 @@ #include "db/db_impl.h" #include "db/version_set.h" #include "db/db_statistics.h" +#include "leveldb/options.h" #include "leveldb/cache.h" #include "leveldb/db.h" #include "leveldb/env.h" @@ -125,7 +126,7 @@ static int FLAGS_max_write_buffer_number = 0; // The minimum number of write buffers that will be merged together // before writing to storage. This is cheap because it is an // in-memory merge. If this feature is not enabled, then all these -// write buffers are fushed to L0 as seperate files and this increases +// write buffers are fushed to L0 as separate files and this increases // read amplification because a get request has to check in all of these // files. Also, an in-memory merge may result in writing lesser // data to storage if there are duplicate records in each of these @@ -137,14 +138,14 @@ static int FLAGS_min_write_buffer_number_to_merge = 0; // This is initialized to default value of 1 in "main" function. static int FLAGS_max_background_compactions = 0; -// Run database in hybrid mode where all data resides in L0. -static bool FLAGS_hybrid_mode = false; +// style of compaction: level-based vs universal +static leveldb::CompactionStyle FLAGS_compaction_style = leveldb::kCompactionStyleLevel; // Percentage flexibilty while comparing file size. -static int FLAGS_hybrid_size_ratio = 1; +static int FLAGS_universal_size_ratio = 1; // The minimum number of files in a single compaction run. -static int FLAGS_hybrid_min_numfiles_in_single_compaction = 2; +static int FLAGS_compaction_universal_min_merge_width = 2; // Number of bytes to use as a cache of uncompressed data. // Negative means use default settings. @@ -1113,10 +1114,10 @@ unique_ptr GenerateKeyFromInt(int v, const char* suffix = "") options.min_write_buffer_number_to_merge = FLAGS_min_write_buffer_number_to_merge; options.max_background_compactions = FLAGS_max_background_compactions; - options.hybrid_mode = FLAGS_hybrid_mode; - options.hybrid_size_ratio = FLAGS_hybrid_size_ratio; - options.hybrid_min_numfiles_in_single_compaction = - FLAGS_hybrid_min_numfiles_in_single_compaction; + options.compaction_style = FLAGS_compaction_style; + options.compaction_options_universal.size_ratio = FLAGS_universal_size_ratio; + options.compaction_options_universal.min_merge_width = + FLAGS_compaction_universal_min_merge_width; options.block_size = FLAGS_block_size; options.filter_policy = filter_policy_; options.max_open_files = FLAGS_open_files; @@ -1999,12 +2000,11 @@ int main(int argc, char** argv) { FLAGS_open_files = leveldb::Options().max_open_files; FLAGS_max_background_compactions = leveldb::Options().max_background_compactions; - FLAGS_hybrid_mode = - leveldb::Options().hybrid_mode; - FLAGS_hybrid_size_ratio = - leveldb::Options().hybrid_size_ratio; - FLAGS_hybrid_min_numfiles_in_single_compaction = - leveldb::Options().hybrid_min_numfiles_in_single_compaction; + FLAGS_compaction_style = leveldb::Options().compaction_style; + FLAGS_universal_size_ratio = + leveldb::Options().compaction_options_universal.size_ratio; + FLAGS_compaction_universal_min_merge_width = + leveldb::Options().compaction_options_universal.min_merge_width; // Compression test code above refers to FLAGS_block_size FLAGS_block_size = leveldb::Options().block_size; FLAGS_use_os_buffer = leveldb::EnvOptions().use_os_buffer; @@ -2063,13 +2063,13 @@ int main(int argc, char** argv) { FLAGS_min_write_buffer_number_to_merge = n; } else if (sscanf(argv[i], "--max_background_compactions=%d%c", &n, &junk) == 1) { FLAGS_max_background_compactions = n; - } else if (sscanf(argv[i], "--hybrid_mode=%d%c", &n, &junk) == 1) { - FLAGS_hybrid_mode = n; - } else if (sscanf(argv[i], "--hybrid_size_ratio=%d%c", &n, &junk) == 1) { - FLAGS_hybrid_size_ratio = n; - } else if (sscanf(argv[i], "--hybrid_min_numfiles_in_single_compaction=%d%c", + } else if (sscanf(argv[i], "--compaction_style=%d%c", &n, &junk) == 1) { + FLAGS_compaction_style = (leveldb::CompactionStyle)n; + } else if (sscanf(argv[i], "--universal_size_ratio=%d%c", &n, &junk) == 1) { + FLAGS_universal_size_ratio = n; + } else if (sscanf(argv[i], "--universal_min_merge_width=%d%c", &n, &junk) == 1) { - FLAGS_hybrid_min_numfiles_in_single_compaction = n; + FLAGS_compaction_universal_min_merge_width = n; } else if (sscanf(argv[i], "--cache_size=%ld%c", &l, &junk) == 1) { FLAGS_cache_size = l; } else if (sscanf(argv[i], "--block_size=%d%c", &n, &junk) == 1) { diff --git a/db/db_impl.cc b/db/db_impl.cc index d5bd3cbd9..37fe889a5 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -836,7 +836,7 @@ Status DBImpl::WriteLevel0Table(std::vector &mems, VersionEdit* edit, // threads could be concurrently producing compacted files for // that key range. if (base != nullptr && options_.max_background_compactions <= 1 && - !options_.hybrid_mode) { + options_.compaction_style == kCompactionStyleLevel) { level = base->PickLevelForMemTableOutput(min_user_key, max_user_key); } edit->AddFile(level, meta.number, meta.file_size, @@ -1578,7 +1578,8 @@ Status DBImpl::InstallCompactionResults(CompactionState* compact) { for (size_t i = 0; i < compact->outputs.size(); i++) { const CompactionState::Output& out = compact->outputs[i]; compact->compaction->edit()->AddFile( - options_.hybrid_mode? level : level + 1, + (options_.compaction_style == kCompactionStyleUniversal) ? + level : level + 1, out.number, out.file_size, out.smallest, out.largest, out.smallest_seqno, out.largest_seqno); } @@ -1828,9 +1829,9 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) { // If this is the bottommost level (no files in lower levels) // and the earliest snapshot is larger than this seqno // then we can squash the seqno to zero. - // Hybrid mode depends on the sequence number to determine + // Universal mode depends on the sequence number to determine // time-order of files that is needed for compactions. - if (!options_.hybrid_mode && + if (options_.compaction_style == kCompactionStyleLevel && bottommost_level && ikey.sequence < earliest_snapshot && ikey.type != kTypeMerge) { assert(ikey.type != kTypeDeletion); diff --git a/db/version_set.cc b/db/version_set.cc index 0398bf6f6..3cd2b6683 100644 --- a/db/version_set.cc +++ b/db/version_set.cc @@ -384,7 +384,7 @@ void Version::Get(const ReadOptions& options, } if (tmp.empty()) continue; - if (vset_->options_->hybrid_mode) { + if (vset_->options_->compaction_style == kCompactionStyleUniversal) { std::sort(tmp.begin(), tmp.end(), NewestFirstBySeqNo); } else { std::sort(tmp.begin(), tmp.end(), NewestFirst); @@ -1581,7 +1581,7 @@ static bool compareSizeDescending(const VersionSet::Fsize& first, return (first.file->file_size > second.file->file_size); } // A static compator used to sort files based on their seqno -// In hybrid mode: descending seqno +// In universal style : descending seqno static bool compareSeqnoDescending(const VersionSet::Fsize& first, const VersionSet::Fsize& second) { if (first.file->smallest_seqno > second.file->smallest_seqno) { @@ -1596,8 +1596,8 @@ static bool compareSeqnoDescending(const VersionSet::Fsize& first, void VersionSet::UpdateFilesBySize(Version* v) { // No need to sort the highest level because it is never compacted. - int max_level = options_->hybrid_mode? NumberLevels() : - NumberLevels() - 1; + int max_level = (options_->compaction_style == kCompactionStyleUniversal) ? + NumberLevels() : NumberLevels() - 1; for (int level = 0; level < max_level; level++) { @@ -1613,7 +1613,7 @@ void VersionSet::UpdateFilesBySize(Version* v) { } // sort the top number_of_files_to_sort_ based on file size - if (options_->hybrid_mode) { + if (options_->compaction_style == kCompactionStyleUniversal) { int num = temp.size(); std::partial_sort(temp.begin(), temp.begin() + num, temp.end(), compareSeqnoDescending); @@ -2020,7 +2020,11 @@ Compaction* VersionSet::PickCompactionHybrid(int level, double score) { assert (level == 0); // percentage flexibilty while comparing file sizes - uint64_t ratio = (uint64_t)options_->hybrid_size_ratio; + uint64_t ratio = options_->compaction_options_universal.size_ratio; + unsigned int min_merge_width = + options_->compaction_options_universal.min_merge_width; + unsigned int max_merge_width = + options_->compaction_options_universal.max_merge_width; if ((current_->files_[level].size() <= (unsigned int)options_->level0_file_num_compaction_trigger)) { @@ -2043,7 +2047,7 @@ Compaction* VersionSet::PickCompactionHybrid(int level, double score) { bool done = false; assert(file_by_time.size() == current_->files_[level].size()); - unsigned int max_files_to_compact = UINT_MAX; + unsigned int max_files_to_compact = std::min(max_merge_width, UINT_MAX); // Make two pass. The first pass considers a candidate file // only if it is smaller than the total size accumulated so far. @@ -2098,8 +2102,7 @@ Compaction* VersionSet::PickCompactionHybrid(int level, double score) { } // Found a series of consecutive files that need compaction. - if (candidate_count >= (unsigned int) - options_->hybrid_min_numfiles_in_single_compaction) { + if (candidate_count >= (unsigned int)min_merge_width) { for (unsigned int i = loop; i < loop + candidate_count; i++) { int index = file_by_time[i]; FileMetaData* f = current_->files_[level][index]; @@ -2141,8 +2144,8 @@ Compaction* VersionSet::PickCompactionHybrid(int level, double score) { options_->level0_file_num_compaction_trigger + 1) { done = true; // nothing more to do } else { - max_files_to_compact = expected_num_files - - options_->level0_file_num_compaction_trigger; + max_files_to_compact = std::min((int)max_merge_width, + expected_num_files - options_->level0_file_num_compaction_trigger); Log(options_->info_log, "Hybrid: second loop with maxfiles %d", max_files_to_compact); } @@ -2264,8 +2267,8 @@ Compaction* VersionSet::PickCompaction() { current_->vset_->SizeBeingCompacted(size_being_compacted); Finalize(current_, size_being_compacted); - // In hybrid mode compact L0 files back into L0. - if (options_->hybrid_mode) { + // In universal style of compaction, compact L0 files back into L0. + if (options_->compaction_style == kCompactionStyleUniversal) { int level = 0; c = PickCompactionHybrid(level, current_->compaction_score_[level]); return c; @@ -2473,7 +2476,8 @@ Compaction* VersionSet::CompactRange( } } } - int out_level = options_->hybrid_mode ? level : level+1; + int out_level = (options_->compaction_style == kCompactionStyleUniversal) ? + level : level+1; Compaction* c = new Compaction(level, out_level, MaxFileSizeForLevel(level), MaxGrandParentOverlapBytes(level), NumberLevels()); diff --git a/include/leveldb/options.h b/include/leveldb/options.h index b3f85e691..ac16b032c 100644 --- a/include/leveldb/options.h +++ b/include/leveldb/options.h @@ -12,6 +12,7 @@ #include #include "leveldb/slice.h" #include "leveldb/statistics.h" +#include "leveldb/universal_compaction.h" namespace leveldb { @@ -39,6 +40,11 @@ enum CompressionType { kBZip2Compression = 0x3 }; +enum CompactionStyle { + kCompactionStyleLevel = 0x0, // level based compaction style + kCompactionStyleUniversal = 0x1 // Universal compaction style +}; + // Compression options for different compression algorithms like Zlib struct CompressionOptions { int window_bits; @@ -132,7 +138,7 @@ struct Options { int max_write_buffer_number; // The minimum number of write buffers that will be merged together - // before writing to storage. If set to 1, then + // before writing to storage. If set to 1, then // all write buffers are fushed to L0 as individual files and this increases // read amplification because a get request has to check in all of these // files. Also, an in-memory merge may result in writing lesser @@ -476,17 +482,12 @@ struct Options { // Default: 0 uint64_t bytes_per_sync; - // Hybrid Mode. There is only a single level and files in L0 are + // The compaction style // compacted back into L0. Default: false - bool hybrid_mode; - - // Percentage flexibilty while comparing file size. If the candidate file(s) - // size is 1% smaller than the next file's size, then include next file into - // this candidate set. // Default: 1 - int hybrid_size_ratio; + CompactionStyle compaction_style; - // The minimum number of files in a single compaction run. Default: 2 - int hybrid_min_numfiles_in_single_compaction; + // The options needed to support Universal Style compactions + CompactionOptionsUniversal compaction_options_universal; }; // Options that control read operations diff --git a/include/leveldb/universal_compaction.h b/include/leveldb/universal_compaction.h new file mode 100644 index 000000000..df8402ca4 --- /dev/null +++ b/include/leveldb/universal_compaction.h @@ -0,0 +1,57 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_ROCKSDB_UNIVERSAL_COMPACTION_OPTIONS_H +#define STORAGE_ROCKSDB_UNIVERSAL_COMPACTION_OPTIONS_H + +#include +#include +#include +#include +#include +#include +#include "leveldb/slice.h" +#include "leveldb/statistics.h" + +namespace leveldb { + +// +// Algorithm used to make a compaction request stop picking new files +// into a single compaction run +// +enum CompactionStopStyle { + kCompactionStopStyleSimilarSize, // pick files of similar size + kCompactionStopStyleTotalSize // total size of picked files > next file +}; + +class CompactionOptionsUniversal { + public: + + // Percentage flexibilty while comparing file size. If the candidate file(s) + // size is 1% smaller than the next file's size, then include next file into + // this candidate set. // Default: 1 + unsigned int size_ratio; + + // The minimum number of files in a single compaction run. Default: 2 + unsigned int min_merge_width; + + // The maximum number of files in a single compaction run. Default: INT_MAX + unsigned int max_merge_width; + + // The algorithm used to stop picking files into a single compaction run + // Default: kCompactionStopStyleTotalSize + CompactionStopStyle stop_style; + + // Default set of parameters + CompactionOptionsUniversal() : + size_ratio(1), + min_merge_width(2), + max_merge_width(UINT_MAX), + stop_style(kCompactionStopStyleTotalSize) { + } +}; + +} // namespace leveldb + +#endif // STORAGE_ROCKSDB_UNIVERSAL_COMPACTION_OPTIONS_H diff --git a/tools/db_stress.cc b/tools/db_stress.cc index 4c671d84f..cf4951b5c 100644 --- a/tools/db_stress.cc +++ b/tools/db_stress.cc @@ -89,7 +89,7 @@ static int FLAGS_max_write_buffer_number = 0; static int FLAGS_max_background_compactions = 0; // This is initialized to default value of false -static bool FLAGS_hybrid_mode = false; +static leveldb::CompactionStyle FLAGS_compaction_style = leveldb::kCompactionStyleLevel; // Number of bytes to use as a cache of uncompressed data. static long FLAGS_cache_size = 2 * KB * KB * KB; @@ -933,7 +933,7 @@ class StressTest { options.write_buffer_size = FLAGS_write_buffer_size; options.max_write_buffer_number = FLAGS_max_write_buffer_number; options.max_background_compactions = FLAGS_max_background_compactions; - options.hybrid_mode = FLAGS_hybrid_mode; + options.compaction_style = FLAGS_compaction_style; options.block_size = FLAGS_block_size; options.filter_policy = filter_policy_; options.max_open_files = FLAGS_open_files; @@ -1020,8 +1020,8 @@ int main(int argc, char** argv) { FLAGS_open_files = leveldb::Options().max_open_files; FLAGS_max_background_compactions = leveldb::Options().max_background_compactions; - FLAGS_hybrid_mode = - leveldb::Options().hybrid_mode; + FLAGS_compaction_style = + leveldb::Options().compaction_style; FLAGS_level0_file_num_compaction_trigger = leveldb::Options().level0_file_num_compaction_trigger; FLAGS_level0_slowdown_writes_trigger = @@ -1074,8 +1074,8 @@ int main(int argc, char** argv) { FLAGS_max_write_buffer_number = n; } else if (sscanf(argv[i], "--max_background_compactions=%d%c", &n, &junk) == 1) { FLAGS_max_background_compactions = n; - } else if (sscanf(argv[i], "--hybrid_mode=%d%c", &n, &junk) == 1) { - FLAGS_hybrid_mode = n; + } else if (sscanf(argv[i], "--compaction_style=%d%c", &n, &junk) == 1) { + FLAGS_compaction_style = (leveldb::CompactionStyle)n; } else if (sscanf(argv[i], "--cache_size=%ld%c", &l, &junk) == 1) { FLAGS_cache_size = l; } else if (sscanf(argv[i], "--block_size=%d%c", &n, &junk) == 1) { diff --git a/util/ldb_cmd.cc b/util/ldb_cmd.cc index 780d3d2a4..d4fdf8c48 100644 --- a/util/ldb_cmd.cc +++ b/util/ldb_cmd.cc @@ -1070,7 +1070,7 @@ void ApproxSizeCommand::DoCommand() { uint64_t sizes[1]; db_->GetApproximateSizes(ranges, 1, sizes); fprintf(stdout, "%ld\n", sizes[0]); - /* Wierd that GetApproximateSizes() returns void, although documentation + /* Weird that GetApproximateSizes() returns void, although documentation * says that it returns a Status object. if (!st.ok()) { exec_state_ = LDBCommandExecuteResult::FAILED(st.ToString()); diff --git a/util/options.cc b/util/options.cc index a38434749..c58d9614d 100644 --- a/util/options.cc +++ b/util/options.cc @@ -77,9 +77,7 @@ Options::Options() access_hint_on_compaction_start(NORMAL), use_adaptive_mutex(false), bytes_per_sync(0), - hybrid_mode(false), - hybrid_size_ratio(1), - hybrid_min_numfiles_in_single_compaction(2) { + compaction_style(kCompactionStyleLevel) { } static const char* const access_hints[] = { @@ -220,12 +218,14 @@ Options::Dump(Logger* log) const use_adaptive_mutex); Log(log," Options.bytes_per_sync: %ld", bytes_per_sync); - Log(log," Options.hybrid_mode: %d", - hybrid_mode); - Log(log," Options.hybrid_size_ratio: %d", - hybrid_size_ratio); - Log(log,"Options.hybrid_min_numfiles_in_single_compaction: %d", - hybrid_min_numfiles_in_single_compaction); + Log(log," Options.compaction_style: %d", + compaction_style); + Log(log," Options.compaction_options_universal.size_ratio: %d", + compaction_options_universal.size_ratio); + Log(log," Options.compaction_options_universal.min_merge_width: %d", + compaction_options_universal.min_merge_width); + Log(log," Options.compaction_options_universal.max_merge_width: %d", + compaction_options_universal.max_merge_width); } // Options::Dump //