diff --git a/docs/static/images/bloom_fp_vs_bpk.png b/docs/static/images/bloom_fp_vs_bpk.png new file mode 100644 index 000000000..e83f4d085 Binary files /dev/null and b/docs/static/images/bloom_fp_vs_bpk.png differ diff --git a/include/rocksdb/filter_policy.h b/include/rocksdb/filter_policy.h index ad6862d5f..fa918c80d 100644 --- a/include/rocksdb/filter_policy.h +++ b/include/rocksdb/filter_policy.h @@ -101,6 +101,9 @@ struct FilterBuildingContext { // The table level at time of constructing the SST file, or -1 if unknown. // (The table file could later be used at a different level.) int level_at_creation = -1; + + // An optional logger for reporting errors, warnings, etc. + Logger* info_log = nullptr; }; // We add a new format of filter block called full filter block diff --git a/table/block_based/block_based_table_builder.cc b/table/block_based/block_based_table_builder.cc index d4e0de5a8..8a756577f 100644 --- a/table/block_based/block_based_table_builder.cc +++ b/table/block_based/block_based_table_builder.cc @@ -425,6 +425,7 @@ struct BlockBasedTableBuilder::Rep { context.column_family_name = column_family_name; context.compaction_style = ioptions.compaction_style; context.level_at_creation = level_at_creation; + context.info_log = ioptions.info_log; filter_builder.reset(CreateFilterBlockBuilder( ioptions, moptions, context, use_delta_encoding_for_index_values, p_index_builder_)); diff --git a/table/block_based/filter_policy.cc b/table/block_based/filter_policy.cc index eb2551f2f..1b49d554d 100644 --- a/table/block_based/filter_policy.cc +++ b/table/block_based/filter_policy.cc @@ -419,7 +419,7 @@ const std::vector BloomFilterPolicy::kAllUserModes = { }; BloomFilterPolicy::BloomFilterPolicy(double bits_per_key, Mode mode) - : mode_(mode) { + : mode_(mode), warned_(false) { // Sanitize bits_per_key if (bits_per_key < 1.0) { bits_per_key = 1.0; @@ -527,6 +527,24 @@ FilterBitsBuilder* BloomFilterPolicy::GetBuilderWithContext( case kFastLocalBloom: return new FastLocalBloomBitsBuilder(millibits_per_key_); case kLegacyBloom: + if (whole_bits_per_key_ >= 14 && context.info_log && + !warned_.load(std::memory_order_relaxed)) { + warned_ = true; + const char* adjective; + if (whole_bits_per_key_ >= 20) { + adjective = "Dramatic"; + } else { + adjective = "Significant"; + } + // For more details, see + // https://github.com/facebook/rocksdb/wiki/RocksDB-Bloom-Filter + ROCKS_LOG_WARN( + context.info_log, + "Using legacy Bloom filter with high (%d) bits/key. " + "%s filter space and/or accuracy improvement is available " + "with format_version>=5.", + whole_bits_per_key_, adjective); + } return new LegacyBloomBitsBuilder(whole_bits_per_key_); } } diff --git a/table/block_based/filter_policy_internal.h b/table/block_based/filter_policy_internal.h index b92980a54..6fe344c48 100644 --- a/table/block_based/filter_policy_internal.h +++ b/table/block_based/filter_policy_internal.h @@ -8,6 +8,7 @@ #pragma once +#include #include #include #include @@ -125,6 +126,10 @@ class BloomFilterPolicy : public FilterPolicy { // implementation) for building new SST filters. Mode mode_; + // Whether relevant warnings have been logged already. (Remember so we + // only report once per BloomFilterPolicy instance, to keep the noise down.) + mutable std::atomic warned_; + // For newer Bloom filter implementation(s) FilterBitsReader* GetBloomBitsReader(const Slice& contents) const; }; diff --git a/table/block_based/mock_block_based_table.h b/table/block_based/mock_block_based_table.h index 52891b1bd..b38de08df 100644 --- a/table/block_based/mock_block_based_table.h +++ b/table/block_based/mock_block_based_table.h @@ -47,6 +47,7 @@ class MockBlockBasedTableTester { context.column_family_name = "mock_cf"; context.compaction_style = ioptions_.compaction_style; context.level_at_creation = kMockLevel; + context.info_log = ioptions_.info_log; return BloomFilterPolicy::GetBuilderFromContext(context); } }; diff --git a/util/filter_bench.cc b/util/filter_bench.cc index 45e042a15..04b419dd3 100644 --- a/util/filter_bench.cc +++ b/util/filter_bench.cc @@ -26,6 +26,7 @@ int main() { #include "util/gflags_compat.h" #include "util/hash.h" #include "util/random.h" +#include "util/stderr_logger.h" #include "util/stop_watch.h" using GFLAGS_NAMESPACE::ParseCommandLineFlags; @@ -112,6 +113,7 @@ using rocksdb::ParsedFullFilterBlock; using rocksdb::PlainTableBloomV1; using rocksdb::Random32; using rocksdb::Slice; +using rocksdb::StderrLogger; using rocksdb::mock::MockBlockBasedTableTester; struct KeyMaker { @@ -243,6 +245,7 @@ struct FilterBench : public MockBlockBasedTableTester { Random32 random_; std::ostringstream fp_rate_report_; Arena arena_; + StderrLogger stderr_logger_; FilterBench() : MockBlockBasedTableTester(new BloomFilterPolicy( @@ -252,6 +255,7 @@ struct FilterBench : public MockBlockBasedTableTester { for (uint32_t i = 0; i < FLAGS_batch_size; ++i) { kms_.emplace_back(FLAGS_key_size < 8 ? 8 : FLAGS_key_size); } + ioptions_.info_log = &stderr_logger_; } void Go();