|
|
@ -364,7 +364,8 @@ Compaction* CompactionPicker::CompactRange(Version* version, int input_level, |
|
|
|
return c; |
|
|
|
return c; |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
Compaction* LevelCompactionPicker::PickCompaction(Version* version) { |
|
|
|
Compaction* LevelCompactionPicker::PickCompaction(Version* version, |
|
|
|
|
|
|
|
LogBuffer* log_buffer) { |
|
|
|
Compaction* c = nullptr; |
|
|
|
Compaction* c = nullptr; |
|
|
|
int level = -1; |
|
|
|
int level = -1; |
|
|
|
|
|
|
|
|
|
|
@ -541,31 +542,34 @@ Compaction* LevelCompactionPicker::PickCompactionBySize(Version* version, |
|
|
|
// Universal style of compaction. Pick files that are contiguous in
|
|
|
|
// Universal style of compaction. Pick files that are contiguous in
|
|
|
|
// time-range to compact.
|
|
|
|
// time-range to compact.
|
|
|
|
//
|
|
|
|
//
|
|
|
|
Compaction* UniversalCompactionPicker::PickCompaction(Version* version) { |
|
|
|
Compaction* UniversalCompactionPicker::PickCompaction(Version* version, |
|
|
|
|
|
|
|
LogBuffer* log_buffer) { |
|
|
|
int level = 0; |
|
|
|
int level = 0; |
|
|
|
double score = version->compaction_score_[0]; |
|
|
|
double score = version->compaction_score_[0]; |
|
|
|
|
|
|
|
|
|
|
|
if ((version->files_[level].size() < |
|
|
|
if ((version->files_[level].size() < |
|
|
|
(unsigned int)options_->level0_file_num_compaction_trigger)) { |
|
|
|
(unsigned int)options_->level0_file_num_compaction_trigger)) { |
|
|
|
Log(logger_, "Universal: nothing to do\n"); |
|
|
|
LogToBuffer(log_buffer, "Universal: nothing to do\n"); |
|
|
|
return nullptr; |
|
|
|
return nullptr; |
|
|
|
} |
|
|
|
} |
|
|
|
Version::FileSummaryStorage tmp; |
|
|
|
Version::FileSummaryStorage tmp; |
|
|
|
Log(logger_, "Universal: candidate files(%zu): %s\n", |
|
|
|
LogToBuffer(log_buffer, "Universal: candidate files(%zu): %s\n", |
|
|
|
version->files_[level].size(), version->LevelFileSummary(&tmp, 0)); |
|
|
|
version->files_[level].size(), |
|
|
|
|
|
|
|
version->LevelFileSummary(&tmp, 0)); |
|
|
|
|
|
|
|
|
|
|
|
// Check for size amplification first.
|
|
|
|
// Check for size amplification first.
|
|
|
|
Compaction* c; |
|
|
|
Compaction* c; |
|
|
|
if ((c = PickCompactionUniversalSizeAmp(version, score)) != nullptr) { |
|
|
|
if ((c = PickCompactionUniversalSizeAmp(version, score, log_buffer)) != |
|
|
|
Log(logger_, "Universal: compacting for size amp\n"); |
|
|
|
nullptr) { |
|
|
|
|
|
|
|
LogToBuffer(log_buffer, "Universal: compacting for size amp\n"); |
|
|
|
} else { |
|
|
|
} else { |
|
|
|
|
|
|
|
|
|
|
|
// Size amplification is within limits. Try reducing read
|
|
|
|
// Size amplification is within limits. Try reducing read
|
|
|
|
// amplification while maintaining file size ratios.
|
|
|
|
// amplification while maintaining file size ratios.
|
|
|
|
unsigned int ratio = options_->compaction_options_universal.size_ratio; |
|
|
|
unsigned int ratio = options_->compaction_options_universal.size_ratio; |
|
|
|
|
|
|
|
|
|
|
|
if ((c = PickCompactionUniversalReadAmp(version, score, ratio, UINT_MAX)) != nullptr) { |
|
|
|
if ((c = PickCompactionUniversalReadAmp(version, score, ratio, UINT_MAX, |
|
|
|
Log(logger_, "Universal: compacting for size ratio\n"); |
|
|
|
log_buffer)) != nullptr) { |
|
|
|
|
|
|
|
LogToBuffer(log_buffer, "Universal: compacting for size ratio\n"); |
|
|
|
} else { |
|
|
|
} else { |
|
|
|
// Size amplification and file size ratios are within configured limits.
|
|
|
|
// Size amplification and file size ratios are within configured limits.
|
|
|
|
// If max read amplification is exceeding configured limits, then force
|
|
|
|
// If max read amplification is exceeding configured limits, then force
|
|
|
@ -573,8 +577,9 @@ Compaction* UniversalCompactionPicker::PickCompaction(Version* version) { |
|
|
|
// the number of files to fewer than level0_file_num_compaction_trigger.
|
|
|
|
// the number of files to fewer than level0_file_num_compaction_trigger.
|
|
|
|
unsigned int num_files = version->files_[level].size() - |
|
|
|
unsigned int num_files = version->files_[level].size() - |
|
|
|
options_->level0_file_num_compaction_trigger; |
|
|
|
options_->level0_file_num_compaction_trigger; |
|
|
|
if ((c = PickCompactionUniversalReadAmp(version, score, UINT_MAX, num_files)) != nullptr) { |
|
|
|
if ((c = PickCompactionUniversalReadAmp( |
|
|
|
Log(logger_, "Universal: compacting for file num\n"); |
|
|
|
version, score, UINT_MAX, num_files, log_buffer)) != nullptr) { |
|
|
|
|
|
|
|
LogToBuffer(log_buffer, "Universal: compacting for file num\n"); |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
@ -623,7 +628,7 @@ Compaction* UniversalCompactionPicker::PickCompaction(Version* version) { |
|
|
|
//
|
|
|
|
//
|
|
|
|
Compaction* UniversalCompactionPicker::PickCompactionUniversalReadAmp( |
|
|
|
Compaction* UniversalCompactionPicker::PickCompactionUniversalReadAmp( |
|
|
|
Version* version, double score, unsigned int ratio, |
|
|
|
Version* version, double score, unsigned int ratio, |
|
|
|
unsigned int max_number_of_files_to_compact) { |
|
|
|
unsigned int max_number_of_files_to_compact, LogBuffer* log_buffer) { |
|
|
|
int level = 0; |
|
|
|
int level = 0; |
|
|
|
|
|
|
|
|
|
|
|
unsigned int min_merge_width = |
|
|
|
unsigned int min_merge_width = |
|
|
@ -658,8 +663,9 @@ Compaction* UniversalCompactionPicker::PickCompactionUniversalReadAmp( |
|
|
|
candidate_count = 1; |
|
|
|
candidate_count = 1; |
|
|
|
break; |
|
|
|
break; |
|
|
|
} |
|
|
|
} |
|
|
|
Log(logger_, "Universal: file %lu[%d] being compacted, skipping", |
|
|
|
LogToBuffer(log_buffer, |
|
|
|
(unsigned long)f->number, loop); |
|
|
|
"Universal: file %lu[%d] being compacted, skipping", |
|
|
|
|
|
|
|
(unsigned long)f->number, loop); |
|
|
|
f = nullptr; |
|
|
|
f = nullptr; |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
@ -667,8 +673,8 @@ Compaction* UniversalCompactionPicker::PickCompactionUniversalReadAmp( |
|
|
|
// first candidate to be compacted.
|
|
|
|
// first candidate to be compacted.
|
|
|
|
uint64_t candidate_size = f != nullptr? f->file_size : 0; |
|
|
|
uint64_t candidate_size = f != nullptr? f->file_size : 0; |
|
|
|
if (f != nullptr) { |
|
|
|
if (f != nullptr) { |
|
|
|
Log(logger_, "Universal: Possible candidate file %lu[%d].", |
|
|
|
LogToBuffer(log_buffer, "Universal: Possible candidate file %lu[%d].", |
|
|
|
(unsigned long)f->number, loop); |
|
|
|
(unsigned long)f->number, loop); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
// Check if the suceeding files need compaction.
|
|
|
|
// Check if the suceeding files need compaction.
|
|
|
@ -718,9 +724,10 @@ Compaction* UniversalCompactionPicker::PickCompactionUniversalReadAmp( |
|
|
|
i < loop + candidate_count && i < file_by_time.size(); i++) { |
|
|
|
i < loop + candidate_count && i < file_by_time.size(); i++) { |
|
|
|
int index = file_by_time[i]; |
|
|
|
int index = file_by_time[i]; |
|
|
|
FileMetaData* f = version->files_[level][index]; |
|
|
|
FileMetaData* f = version->files_[level][index]; |
|
|
|
Log(logger_, "Universal: Skipping file %lu[%d] with size %lu %d\n", |
|
|
|
LogToBuffer(log_buffer, |
|
|
|
(unsigned long)f->number, i, (unsigned long)f->file_size, |
|
|
|
"Universal: Skipping file %lu[%d] with size %lu %d\n", |
|
|
|
f->being_compacted); |
|
|
|
(unsigned long)f->number, i, (unsigned long)f->file_size, |
|
|
|
|
|
|
|
f->being_compacted); |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
@ -754,8 +761,8 @@ Compaction* UniversalCompactionPicker::PickCompactionUniversalReadAmp( |
|
|
|
int index = file_by_time[i]; |
|
|
|
int index = file_by_time[i]; |
|
|
|
FileMetaData* f = c->input_version_->files_[level][index]; |
|
|
|
FileMetaData* f = c->input_version_->files_[level][index]; |
|
|
|
c->inputs_[0].push_back(f); |
|
|
|
c->inputs_[0].push_back(f); |
|
|
|
Log(logger_, "Universal: Picking file %lu[%d] with size %lu\n", |
|
|
|
LogToBuffer(log_buffer, "Universal: Picking file %lu[%d] with size %lu\n", |
|
|
|
(unsigned long)f->number, i, (unsigned long)f->file_size); |
|
|
|
(unsigned long)f->number, i, (unsigned long)f->file_size); |
|
|
|
} |
|
|
|
} |
|
|
|
return c; |
|
|
|
return c; |
|
|
|
} |
|
|
|
} |
|
|
@ -767,7 +774,7 @@ Compaction* UniversalCompactionPicker::PickCompactionUniversalReadAmp( |
|
|
|
// min_merge_width and max_merge_width).
|
|
|
|
// min_merge_width and max_merge_width).
|
|
|
|
//
|
|
|
|
//
|
|
|
|
Compaction* UniversalCompactionPicker::PickCompactionUniversalSizeAmp( |
|
|
|
Compaction* UniversalCompactionPicker::PickCompactionUniversalSizeAmp( |
|
|
|
Version* version, double score) { |
|
|
|
Version* version, double score, LogBuffer* log_buffer) { |
|
|
|
int level = 0; |
|
|
|
int level = 0; |
|
|
|
|
|
|
|
|
|
|
|
// percentage flexibilty while reducing size amplification
|
|
|
|
// percentage flexibilty while reducing size amplification
|
|
|
@ -791,17 +798,17 @@ Compaction* UniversalCompactionPicker::PickCompactionUniversalSizeAmp( |
|
|
|
start_index = loop; // Consider this as the first candidate.
|
|
|
|
start_index = loop; // Consider this as the first candidate.
|
|
|
|
break; |
|
|
|
break; |
|
|
|
} |
|
|
|
} |
|
|
|
Log(logger_, "Universal: skipping file %lu[%d] compacted %s", |
|
|
|
LogToBuffer(log_buffer, "Universal: skipping file %lu[%d] compacted %s", |
|
|
|
(unsigned long)f->number, loop, |
|
|
|
(unsigned long)f->number, loop, |
|
|
|
" cannot be a candidate to reduce size amp.\n"); |
|
|
|
" cannot be a candidate to reduce size amp.\n"); |
|
|
|
f = nullptr; |
|
|
|
f = nullptr; |
|
|
|
} |
|
|
|
} |
|
|
|
if (f == nullptr) { |
|
|
|
if (f == nullptr) { |
|
|
|
return nullptr; // no candidate files
|
|
|
|
return nullptr; // no candidate files
|
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
Log(logger_, "Universal: First candidate file %lu[%d] %s", |
|
|
|
LogToBuffer(log_buffer, "Universal: First candidate file %lu[%d] %s", |
|
|
|
(unsigned long)f->number, start_index, " to reduce size amp.\n"); |
|
|
|
(unsigned long)f->number, start_index, " to reduce size amp.\n"); |
|
|
|
|
|
|
|
|
|
|
|
// keep adding up all the remaining files
|
|
|
|
// keep adding up all the remaining files
|
|
|
|
for (unsigned int loop = start_index; loop < file_by_time.size() - 1; |
|
|
|
for (unsigned int loop = start_index; loop < file_by_time.size() - 1; |
|
|
@ -809,7 +816,8 @@ Compaction* UniversalCompactionPicker::PickCompactionUniversalSizeAmp( |
|
|
|
int index = file_by_time[loop]; |
|
|
|
int index = file_by_time[loop]; |
|
|
|
f = version->files_[level][index]; |
|
|
|
f = version->files_[level][index]; |
|
|
|
if (f->being_compacted) { |
|
|
|
if (f->being_compacted) { |
|
|
|
Log(logger_, "Universal: Possible candidate file %lu[%d] %s.", |
|
|
|
LogToBuffer( |
|
|
|
|
|
|
|
log_buffer, "Universal: Possible candidate file %lu[%d] %s.", |
|
|
|
(unsigned long)f->number, loop, |
|
|
|
(unsigned long)f->number, loop, |
|
|
|
" is already being compacted. No size amp reduction possible.\n"); |
|
|
|
" is already being compacted. No size amp reduction possible.\n"); |
|
|
|
return nullptr; |
|
|
|
return nullptr; |
|
|
@ -827,16 +835,18 @@ Compaction* UniversalCompactionPicker::PickCompactionUniversalSizeAmp( |
|
|
|
|
|
|
|
|
|
|
|
// size amplification = percentage of additional size
|
|
|
|
// size amplification = percentage of additional size
|
|
|
|
if (candidate_size * 100 < ratio * earliest_file_size) { |
|
|
|
if (candidate_size * 100 < ratio * earliest_file_size) { |
|
|
|
Log(logger_, |
|
|
|
LogToBuffer(log_buffer, |
|
|
|
"Universal: size amp not needed. newer-files-total-size %lu " |
|
|
|
"Universal: size amp not needed. newer-files-total-size %lu " |
|
|
|
"earliest-file-size %lu", |
|
|
|
"earliest-file-size %lu", |
|
|
|
(unsigned long)candidate_size, (unsigned long)earliest_file_size); |
|
|
|
(unsigned long)candidate_size, |
|
|
|
|
|
|
|
(unsigned long)earliest_file_size); |
|
|
|
return nullptr; |
|
|
|
return nullptr; |
|
|
|
} else { |
|
|
|
} else { |
|
|
|
Log(logger_, |
|
|
|
LogToBuffer(log_buffer, |
|
|
|
"Universal: size amp needed. newer-files-total-size %lu " |
|
|
|
"Universal: size amp needed. newer-files-total-size %lu " |
|
|
|
"earliest-file-size %lu", |
|
|
|
"earliest-file-size %lu", |
|
|
|
(unsigned long)candidate_size, (unsigned long)earliest_file_size); |
|
|
|
(unsigned long)candidate_size, |
|
|
|
|
|
|
|
(unsigned long)earliest_file_size); |
|
|
|
} |
|
|
|
} |
|
|
|
assert(start_index >= 0 && start_index < file_by_time.size() - 1); |
|
|
|
assert(start_index >= 0 && start_index < file_by_time.size() - 1); |
|
|
|
|
|
|
|
|
|
|
@ -850,8 +860,9 @@ Compaction* UniversalCompactionPicker::PickCompactionUniversalSizeAmp( |
|
|
|
int index = file_by_time[loop]; |
|
|
|
int index = file_by_time[loop]; |
|
|
|
f = c->input_version_->files_[level][index]; |
|
|
|
f = c->input_version_->files_[level][index]; |
|
|
|
c->inputs_[0].push_back(f); |
|
|
|
c->inputs_[0].push_back(f); |
|
|
|
Log(logger_, "Universal: size amp picking file %lu[%d] with size %lu", |
|
|
|
LogToBuffer(log_buffer, |
|
|
|
(unsigned long)f->number, index, (unsigned long)f->file_size); |
|
|
|
"Universal: size amp picking file %lu[%d] with size %lu", |
|
|
|
|
|
|
|
(unsigned long)f->number, index, (unsigned long)f->file_size); |
|
|
|
} |
|
|
|
} |
|
|
|
return c; |
|
|
|
return c; |
|
|
|
} |
|
|
|
} |
|
|
|