From 1c27cbfbd1fde4f8b3a6aff8c3e64260fadbe9fd Mon Sep 17 00:00:00 2001 From: Andrew Kryczka Date: Mon, 9 Apr 2018 13:29:51 -0700 Subject: [PATCH] fix intra-L0 FIFO for uncompressed use case Summary: - inflate the argument passed as `max_compact_bytes_per_del_file` by a bit (10%). The intent of this argument is prevent L0 files from being intra-L0 compacted multiple times. Without compression, some intra-L0 compactions exceed this limit (and thus aren't executed), even though none of their files have gone through intra-L0 before. - fix `FindIntraL0Compaction` as it was rejecting some valid intra-L0 compactions. In particular, `compact_bytes_per_del_file` is the work-per-deleted-file for the span [0, span_len), whereas `new_compact_bytes_per_del_file` is the work-per-deleted-file for the span [0, span_len+1). The former is more correct for checking whether we've found an eligible span. Closes https://github.com/facebook/rocksdb/pull/3684 Differential Revision: D7530396 Pulled By: ajkr fbshipit-source-id: cad4f50902bdc428ac9ff6fffb13eb288648d85e --- db/compaction_picker.cc | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/db/compaction_picker.cc b/db/compaction_picker.cc index ef73c0e5c..aca52af3f 100644 --- a/db/compaction_picker.cc +++ b/db/compaction_picker.cc @@ -59,7 +59,7 @@ bool FindIntraL0Compaction(const std::vector& level_files, } if (span_len >= min_files_to_compact && - new_compact_bytes_per_del_file < max_compact_bytes_per_del_file) { + compact_bytes_per_del_file < max_compact_bytes_per_del_file) { assert(comp_inputs != nullptr); comp_inputs->level = 0; for (size_t i = 0; i < span_len; ++i) { @@ -1580,11 +1580,21 @@ Compaction* FIFOCompactionPicker::PickSizeCompaction( if (mutable_cf_options.compaction_options_fifo.allow_compaction && level_files.size() > 0) { CompactionInputFiles comp_inputs; + // try to prevent same files from being compacted multiple times, which + // could produce large files that may never TTL-expire. Achieve this by + // disallowing compactions with files larger than memtable (inflate its + // size by 10% to account for uncompressed L0 files that may have size + // slightly greater than memtable size limit). + size_t max_compact_bytes_per_del_file = + static_cast(MultiplyCheckOverflow( + static_cast(mutable_cf_options.write_buffer_size), + 1.1)); if (FindIntraL0Compaction( level_files, mutable_cf_options - .level0_file_num_compaction_trigger /* min_files_to_compact */, - mutable_cf_options.write_buffer_size, &comp_inputs)) { + .level0_file_num_compaction_trigger /* min_files_to_compact */ + , + max_compact_bytes_per_del_file, &comp_inputs)) { Compaction* c = new Compaction( vstorage, ioptions_, mutable_cf_options, {comp_inputs}, 0, 16 * 1024 * 1024 /* output file size limit */,