From 5e067a7b19a5458c6dd910e104f5738543df08c4 Mon Sep 17 00:00:00 2001 From: Igor Canadi Date: Mon, 6 Apr 2015 12:50:44 -0700 Subject: [PATCH] Clean up compression logging Summary: Now we add warnings when user configures compression and the compression is not supported. Test Plan: Configured compression to non-supported values. Observed messages in my log: 2015/03/26-12:17:57.586341 7ffb8a496840 [WARN] Compression type chosen for level 2 is not supported: LZ4. RocksDB will not compress data on level 2. 2015/03/26-12:19:10.768045 7f36f15c5840 [WARN] Compression type chosen is not supported: LZ4. RocksDB will not compress data. Reviewers: rven, sdong, yhchiang Reviewed By: sdong Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D35979 --- db/column_family.cc | 19 +++++++++- db/db_impl.cc | 29 +++++--------- db/db_test.cc | 60 +++++------------------------ include/rocksdb/options.h | 6 +++ table/table_test.cc | 80 +++++---------------------------------- util/compression.h | 28 ++++++++++++++ util/options.cc | 50 ++++++++++++++++++++++-- 7 files changed, 125 insertions(+), 147 deletions(-) diff --git a/db/column_family.cc b/db/column_family.cc index bdfc99864..1b96bbcf7 100644 --- a/db/column_family.cc +++ b/db/column_family.cc @@ -136,7 +136,6 @@ ColumnFamilyOptions SanitizeOptions(const DBOptions& db_options, result.min_write_buffer_number_to_merge = std::min(result.min_write_buffer_number_to_merge, result.max_write_buffer_number - 1); - result.compression_per_level = src.compression_per_level; if (result.max_mem_compaction_level >= result.num_levels) { result.max_mem_compaction_level = result.num_levels - 1; } @@ -155,6 +154,24 @@ ColumnFamilyOptions SanitizeOptions(const DBOptions& db_options, } } + if (!src.compression_per_level.empty()) { + for (size_t level = 0; level < src.compression_per_level.size(); ++level) { + if (!CompressionTypeSupported(src.compression_per_level[level])) { + Log(InfoLogLevel::WARN_LEVEL, db_options.info_log, + "Compression type chosen for level %zu is not supported: %s. " + "RocksDB " + "will not compress data on level %zu.", + level, CompressionTypeToString(src.compression_per_level[level]), + level); + } + } + } else if (!CompressionTypeSupported(src.compression)) { + Log(InfoLogLevel::WARN_LEVEL, db_options.info_log, + "Compression type chosen is not supported: %s. RocksDB will not " + "compress data.", + CompressionTypeToString(src.compression)); + } + if (result.compaction_style == kCompactionStyleFIFO) { result.num_levels = 1; // since we delete level0 files in FIFO compaction when there are too many diff --git a/db/db_impl.cc b/db/db_impl.cc index c4d6cd54b..15116f4db 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -67,6 +67,7 @@ #include "util/autovector.h" #include "util/build_version.h" #include "util/coding.h" +#include "util/compression.h" #include "util/db_info_dumper.h" #include "util/file_util.h" #include "util/hash_skiplist_rep.h" @@ -189,26 +190,14 @@ CompressionType GetCompressionFlush(const ImmutableCFOptions& ioptions) { } void DumpCompressionInfo(Logger* logger) { - Log(InfoLogLevel::INFO_LEVEL, logger, "Snappy " -#ifndef SNAPPY - "NOT " -#endif - "supported"); - Log(InfoLogLevel::INFO_LEVEL, logger, "Zlib " -#ifndef ZLIB - "NOT " -#endif - "supported"); - Log(InfoLogLevel::INFO_LEVEL, logger, "Bzip " -#ifndef BZIP2 - "NOT " -#endif - "supported"); - Log(InfoLogLevel::INFO_LEVEL, logger, "LZ4 " -#ifndef LZ4 - "NOT " -#endif - "supported"); + Log(InfoLogLevel::INFO_LEVEL, logger, "Compression algorithms supported:"); + Log(InfoLogLevel::INFO_LEVEL, logger, "\tSnappy supported: %d", + Snappy_Supported()); + Log(InfoLogLevel::INFO_LEVEL, logger, "\tZlib supported: %d", + Zlib_Supported()); + Log(InfoLogLevel::INFO_LEVEL, logger, "\tBzip supported: %d", + BZip2_Supported()); + Log(InfoLogLevel::INFO_LEVEL, logger, "\tLZ4 supported: %d", LZ4_Supported()); } } // namespace diff --git a/db/db_test.cc b/db/db_test.cc index b2f16db10..3baa73d89 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -60,41 +60,6 @@ namespace rocksdb { -static bool SnappyCompressionSupported(const CompressionOptions& options) { - std::string out; - Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; - return Snappy_Compress(options, in.data(), in.size(), &out); -} - -static bool SnappyCompressionSupported() { - CompressionOptions options; - return SnappyCompressionSupported(options); -} - -static bool ZlibCompressionSupported(const CompressionOptions& options) { - std::string out; - Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; - return Zlib_Compress(options, 2, in.data(), in.size(), &out); -} - -static bool BZip2CompressionSupported(const CompressionOptions& options) { - std::string out; - Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; - return BZip2_Compress(options, 2, in.data(), in.size(), &out); -} - -static bool LZ4CompressionSupported(const CompressionOptions &options) { - std::string out; - Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; - return LZ4_Compress(options, 2, in.data(), in.size(), &out); -} - -static bool LZ4HCCompressionSupported(const CompressionOptions &options) { - std::string out; - Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; - return LZ4HC_Compress(options, 2, in.data(), in.size(), &out); -} - static std::string RandomString(Random* rnd, int len) { std::string r; test::RandomString(rnd, len, &r); @@ -4418,7 +4383,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionStopStyleSimilarSize) { } TEST_F(DBTest, CompressedCache) { - if (!SnappyCompressionSupported()) { + if (!Snappy_Supported()) { return; } int num_iter = 80; @@ -4542,7 +4507,7 @@ static std::string CompressibleString(Random* rnd, int len) { } TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio1) { - if (!SnappyCompressionSupported()) { + if (!Snappy_Supported()) { return; } @@ -4611,7 +4576,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio1) { } TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio2) { - if (!SnappyCompressionSupported()) { + if (!Snappy_Supported()) { return; } Options options; @@ -4655,7 +4620,7 @@ TEST_F(DBTest, FailMoreDbPaths) { } TEST_F(DBTest, UniversalCompactionSecondPathRatio) { - if (!SnappyCompressionSupported()) { + if (!Snappy_Supported()) { return; } Options options; @@ -5348,25 +5313,18 @@ bool MinLevelToCompress(CompressionType& type, Options& options, int wbits, options.level0_file_num_compaction_trigger = 3; options.create_if_missing = true; - if (SnappyCompressionSupported(CompressionOptions(wbits, lev, strategy))) { + if (Snappy_Supported()) { type = kSnappyCompression; fprintf(stderr, "using snappy\n"); - } else if (ZlibCompressionSupported( - CompressionOptions(wbits, lev, strategy))) { + } else if (Zlib_Supported()) { type = kZlibCompression; fprintf(stderr, "using zlib\n"); - } else if (BZip2CompressionSupported( - CompressionOptions(wbits, lev, strategy))) { + } else if (BZip2_Supported()) { type = kBZip2Compression; fprintf(stderr, "using bzip2\n"); - } else if (LZ4CompressionSupported( - CompressionOptions(wbits, lev, strategy))) { + } else if (LZ4_Supported()) { type = kLZ4Compression; fprintf(stderr, "using lz4\n"); - } else if (LZ4HCCompressionSupported( - CompressionOptions(wbits, lev, strategy))) { - type = kLZ4HCCompression; - fprintf(stderr, "using lz4hc\n"); } else { fprintf(stderr, "skipping test, compression disabled\n"); return false; @@ -11129,7 +11087,7 @@ TEST_F(DBTest, DynamicLevelMaxBytesBase2) { } TEST_F(DBTest, DynamicLevelCompressionPerLevel) { - if (!SnappyCompressionSupported()) { + if (!Snappy_Supported()) { return; } const int kNKeys = 120; diff --git a/include/rocksdb/options.h b/include/rocksdb/options.h index 469f9682b..d8a0d885d 100644 --- a/include/rocksdb/options.h +++ b/include/rocksdb/options.h @@ -55,6 +55,12 @@ enum CompressionType : char { kBZip2Compression = 0x3, kLZ4Compression = 0x4, kLZ4HCCompression = 0x5 }; +// returns true if RocksDB was correctly linked with compression library and +// supports the compression type +extern bool CompressionTypeSupported(CompressionType compression_type); +// Returns a human-readable name of the compression type +extern const char* CompressionTypeToString(CompressionType compression_type); + enum CompactionStyle : char { // level based compaction style kCompactionStyleLevel = 0x0, diff --git a/table/table_test.cc b/table/table_test.cc index 1c7ea8b51..6f7b4db2c 100644 --- a/table/table_test.cc +++ b/table/table_test.cc @@ -537,61 +537,6 @@ class DBConstructor: public Constructor { DB* db_; }; -static bool SnappyCompressionSupported() { -#ifdef SNAPPY - std::string out; - Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; - return Snappy_Compress(Options().compression_opts, in.data(), in.size(), - &out); -#else - return false; -#endif -} - -static bool ZlibCompressionSupported() { -#ifdef ZLIB - std::string out; - Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; - return Zlib_Compress(Options().compression_opts, 2, in.data(), in.size(), - &out); -#else - return false; -#endif -} - -static bool BZip2CompressionSupported() { -#ifdef BZIP2 - std::string out; - Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; - return BZip2_Compress(Options().compression_opts, 2, in.data(), in.size(), - &out); -#else - return false; -#endif -} - -static bool LZ4CompressionSupported() { -#ifdef LZ4 - std::string out; - Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; - return LZ4_Compress(Options().compression_opts, 2, in.data(), in.size(), - &out); -#else - return false; -#endif -} - -static bool LZ4HCCompressionSupported() { -#ifdef LZ4 - std::string out; - Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; - return LZ4HC_Compress(Options().compression_opts, 2, in.data(), in.size(), - &out); -#else - return false; -#endif -} - enum TestType { BLOCK_BASED_TABLE_TEST, PLAIN_TABLE_SEMI_FIXED_PREFIX, @@ -623,22 +568,20 @@ static std::vector GenerateArgList() { // Only add compression if it is supported std::vector> compression_types; compression_types.emplace_back(kNoCompression, false); - if (SnappyCompressionSupported()) { + if (Snappy_Supported()) { compression_types.emplace_back(kSnappyCompression, false); } - if (ZlibCompressionSupported()) { + if (Zlib_Supported()) { compression_types.emplace_back(kZlibCompression, false); compression_types.emplace_back(kZlibCompression, true); } - if (BZip2CompressionSupported()) { + if (BZip2_Supported()) { compression_types.emplace_back(kBZip2Compression, false); compression_types.emplace_back(kBZip2Compression, true); } - if (LZ4CompressionSupported()) { + if (LZ4_Supported()) { compression_types.emplace_back(kLZ4Compression, false); compression_types.emplace_back(kLZ4Compression, true); - } - if (LZ4HCCompressionSupported()) { compression_types.emplace_back(kLZ4HCCompression, false); compression_types.emplace_back(kLZ4HCCompression, true); } @@ -1918,13 +1861,13 @@ static void DoCompressionTest(CompressionType comp) { TEST_F(GeneralTableTest, ApproximateOffsetOfCompressed) { std::vector compression_state; - if (!SnappyCompressionSupported()) { + if (!Snappy_Supported()) { fprintf(stderr, "skipping snappy compression tests\n"); } else { compression_state.push_back(kSnappyCompression); } - if (!ZlibCompressionSupported()) { + if (!Zlib_Supported()) { fprintf(stderr, "skipping zlib compression tests\n"); } else { compression_state.push_back(kZlibCompression); @@ -1932,22 +1875,17 @@ TEST_F(GeneralTableTest, ApproximateOffsetOfCompressed) { // TODO(kailiu) DoCompressionTest() doesn't work with BZip2. /* - if (!BZip2CompressionSupported()) { + if (!BZip2_Supported()) { fprintf(stderr, "skipping bzip2 compression tests\n"); } else { compression_state.push_back(kBZip2Compression); } */ - if (!LZ4CompressionSupported()) { - fprintf(stderr, "skipping lz4 compression tests\n"); + if (!LZ4_Supported()) { + fprintf(stderr, "skipping lz4 and lz4hc compression tests\n"); } else { compression_state.push_back(kLZ4Compression); - } - - if (!LZ4HCCompressionSupported()) { - fprintf(stderr, "skipping lz4hc compression tests\n"); - } else { compression_state.push_back(kLZ4HCCompression); } diff --git a/util/compression.h b/util/compression.h index 664036353..36e36d50a 100644 --- a/util/compression.h +++ b/util/compression.h @@ -34,6 +34,34 @@ namespace rocksdb { +inline bool Snappy_Supported() { +#ifdef SNAPPY + return true; +#endif + return false; +} + +inline bool Zlib_Supported() { +#ifdef ZLIB + return true; +#endif + return false; +} + +inline bool BZip2_Supported() { +#ifdef BZIP2 + return true; +#endif + return false; +} + +inline bool LZ4_Supported() { +#ifdef LZ4 + return true; +#endif + return false; +} + // compress_format_version can have two values: // 1 -- decompressed sizes for BZip2 and Zlib are not included in the compressed // block. Also, decompressed sizes for LZ4 are encoded in platform-dependent diff --git a/util/options.cc b/util/options.cc index 223c4cbb2..ad5eecea3 100644 --- a/util/options.cc +++ b/util/options.cc @@ -29,6 +29,7 @@ #include "rocksdb/table.h" #include "rocksdb/table_properties.h" #include "table/block_based_table_factory.h" +#include "util/compression.h" #include "util/statistics.h" #include "util/xfunc.h" @@ -383,11 +384,12 @@ void ColumnFamilyOptions::Dump(Logger* log) const { Log(log, " Options.max_write_buffer_number: %d", max_write_buffer_number); if (!compression_per_level.empty()) { for (unsigned int i = 0; i < compression_per_level.size(); i++) { - Log(log," Options.compression[%d]: %d", - i, compression_per_level[i]); - } + Log(log, " Options.compression[%d]: %s", i, + CompressionTypeToString(compression_per_level[i])); + } } else { - Log(log," Options.compression: %d", compression); + Log(log, " Options.compression: %s", + CompressionTypeToString(compression)); } Log(log," Options.prefix_extractor: %s", prefix_extractor == nullptr ? "nullptr" : prefix_extractor->Name()); @@ -543,6 +545,46 @@ Options::PrepareForBulkLoad() return this; } +const char* CompressionTypeToString(CompressionType compression_type) { + switch (compression_type) { + case kNoCompression: + return "NoCompression"; + case kSnappyCompression: + return "Snappy"; + case kZlibCompression: + return "Zlib"; + case kBZip2Compression: + return "BZip2"; + case kLZ4Compression: + return "LZ4"; + case kLZ4HCCompression: + return "LZ4HC"; + default: + assert(false); + return ""; + } +} + +bool CompressionTypeSupported(CompressionType compression_type) { + switch (compression_type) { + case kNoCompression: + return true; + case kSnappyCompression: + return Snappy_Supported(); + case kZlibCompression: + return Zlib_Supported(); + case kBZip2Compression: + return BZip2_Supported(); + case kLZ4Compression: + return LZ4_Supported(); + case kLZ4HCCompression: + return LZ4_Supported(); + default: + assert(false); + return false; + } +} + #ifndef ROCKSDB_LITE // Optimization functions ColumnFamilyOptions* ColumnFamilyOptions::OptimizeForPointLookup(