Clean up compression logging

Summary: Now we add warnings when user configures compression and the compression is not supported.

Test Plan:
Configured compression to non-supported values. Observed messages in my log:

    2015/03/26-12:17:57.586341 7ffb8a496840 [WARN] Compression type chosen for level 2 is not supported: LZ4. RocksDB will not compress data on level 2.

    2015/03/26-12:19:10.768045 7f36f15c5840 [WARN] Compression type chosen is not supported: LZ4. RocksDB will not compress data.

Reviewers: rven, sdong, yhchiang

Reviewed By: sdong

Subscribers: dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D35979

db/column_family.cc

@@ -136,7 +136,6 @@ ColumnFamilyOptions SanitizeOptions(const DBOptions& db_options,
   result.min_write_buffer_number_to_merge =
       std::min(result.min_write_buffer_number_to_merge,
                result.max_write_buffer_number - 1);
-  result.compression_per_level = src.compression_per_level;
   if (result.max_mem_compaction_level >= result.num_levels) {
     result.max_mem_compaction_level = result.num_levels - 1;
   }
@@ -155,6 +154,24 @@ ColumnFamilyOptions SanitizeOptions(const DBOptions& db_options,
     }
   }
 
+  if (!src.compression_per_level.empty()) {
+    for (size_t level = 0; level < src.compression_per_level.size(); ++level) {
+      if (!CompressionTypeSupported(src.compression_per_level[level])) {
+        Log(InfoLogLevel::WARN_LEVEL, db_options.info_log,
+            "Compression type chosen for level %zu is not supported: %s. "
+            "RocksDB "
+            "will not compress data on level %zu.",
+            level, CompressionTypeToString(src.compression_per_level[level]),
+            level);
+      }
+    }
+  } else if (!CompressionTypeSupported(src.compression)) {
+    Log(InfoLogLevel::WARN_LEVEL, db_options.info_log,
+        "Compression type chosen is not supported: %s. RocksDB will not "
+        "compress data.",
+        CompressionTypeToString(src.compression));
+  }
+
   if (result.compaction_style == kCompactionStyleFIFO) {
     result.num_levels = 1;
     // since we delete level0 files in FIFO compaction when there are too many

db/db_impl.cc

@@ -67,6 +67,7 @@
 #include "util/autovector.h"
 #include "util/build_version.h"
 #include "util/coding.h"
+#include "util/compression.h"
 #include "util/db_info_dumper.h"
 #include "util/file_util.h"
 #include "util/hash_skiplist_rep.h"
@@ -189,26 +190,14 @@ CompressionType GetCompressionFlush(const ImmutableCFOptions& ioptions) {
 }
 
 void DumpCompressionInfo(Logger* logger) {
-  Log(InfoLogLevel::INFO_LEVEL, logger, "Snappy "
+  Log(InfoLogLevel::INFO_LEVEL, logger, "Compression algorithms supported:");
-#ifndef SNAPPY
+  Log(InfoLogLevel::INFO_LEVEL, logger, "\tSnappy supported: %d",
-      "NOT "
+      Snappy_Supported());
-#endif
+  Log(InfoLogLevel::INFO_LEVEL, logger, "\tZlib supported: %d",
-      "supported");
+      Zlib_Supported());
-  Log(InfoLogLevel::INFO_LEVEL, logger, "Zlib "
+  Log(InfoLogLevel::INFO_LEVEL, logger, "\tBzip supported: %d",
-#ifndef ZLIB
+      BZip2_Supported());
-      "NOT "
+  Log(InfoLogLevel::INFO_LEVEL, logger, "\tLZ4 supported: %d", LZ4_Supported());
-#endif
-      "supported");
-  Log(InfoLogLevel::INFO_LEVEL, logger, "Bzip "
-#ifndef BZIP2
-      "NOT "
-#endif
-      "supported");
-  Log(InfoLogLevel::INFO_LEVEL, logger, "LZ4 "
-#ifndef LZ4
-      "NOT "
-#endif
-      "supported");
 }
 
 }  // namespace

db/db_test.cc

@@ -60,41 +60,6 @@
 
 namespace rocksdb {
 
-static bool SnappyCompressionSupported(const CompressionOptions& options) {
-  std::string out;
-  Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
-  return Snappy_Compress(options, in.data(), in.size(), &out);
-}
-
-static bool SnappyCompressionSupported() {
-  CompressionOptions options;
-  return SnappyCompressionSupported(options);
-}
-
-static bool ZlibCompressionSupported(const CompressionOptions& options) {
-  std::string out;
-  Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
-  return Zlib_Compress(options, 2, in.data(), in.size(), &out);
-}
-
-static bool BZip2CompressionSupported(const CompressionOptions& options) {
-  std::string out;
-  Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
-  return BZip2_Compress(options, 2, in.data(), in.size(), &out);
-}
-
-static bool LZ4CompressionSupported(const CompressionOptions &options) {
-  std::string out;
-  Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
-  return LZ4_Compress(options, 2, in.data(), in.size(), &out);
-}
-
-static bool LZ4HCCompressionSupported(const CompressionOptions &options) {
-  std::string out;
-  Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
-  return LZ4HC_Compress(options, 2, in.data(), in.size(), &out);
-}
-
 static std::string RandomString(Random* rnd, int len) {
   std::string r;
   test::RandomString(rnd, len, &r);
@@ -4418,7 +4383,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionStopStyleSimilarSize) {
 }
 
 TEST_F(DBTest, CompressedCache) {
-  if (!SnappyCompressionSupported()) {
+  if (!Snappy_Supported()) {
     return;
   }
   int num_iter = 80;
@@ -4542,7 +4507,7 @@ static std::string CompressibleString(Random* rnd, int len) {
 }
 
 TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio1) {
-  if (!SnappyCompressionSupported()) {
+  if (!Snappy_Supported()) {
     return;
   }
 
@@ -4611,7 +4576,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio1) {
 }
 
 TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio2) {
-  if (!SnappyCompressionSupported()) {
+  if (!Snappy_Supported()) {
     return;
   }
   Options options;
@@ -4655,7 +4620,7 @@ TEST_F(DBTest, FailMoreDbPaths) {
 }
 
 TEST_F(DBTest, UniversalCompactionSecondPathRatio) {
-  if (!SnappyCompressionSupported()) {
+  if (!Snappy_Supported()) {
     return;
   }
   Options options;
@@ -5348,25 +5313,18 @@ bool MinLevelToCompress(CompressionType& type, Options& options, int wbits,
   options.level0_file_num_compaction_trigger = 3;
   options.create_if_missing = true;
 
-  if (SnappyCompressionSupported(CompressionOptions(wbits, lev, strategy))) {
+  if (Snappy_Supported()) {
     type = kSnappyCompression;
     fprintf(stderr, "using snappy\n");
-  } else if (ZlibCompressionSupported(
+  } else if (Zlib_Supported()) {
-               CompressionOptions(wbits, lev, strategy))) {
     type = kZlibCompression;
     fprintf(stderr, "using zlib\n");
-  } else if (BZip2CompressionSupported(
+  } else if (BZip2_Supported()) {
-               CompressionOptions(wbits, lev, strategy))) {
     type = kBZip2Compression;
     fprintf(stderr, "using bzip2\n");
-  } else if (LZ4CompressionSupported(
+  } else if (LZ4_Supported()) {
-                 CompressionOptions(wbits, lev, strategy))) {
     type = kLZ4Compression;
     fprintf(stderr, "using lz4\n");
-  } else if (LZ4HCCompressionSupported(
-                 CompressionOptions(wbits, lev, strategy))) {
-    type = kLZ4HCCompression;
-    fprintf(stderr, "using lz4hc\n");
   } else {
     fprintf(stderr, "skipping test, compression disabled\n");
     return false;
@@ -11129,7 +11087,7 @@ TEST_F(DBTest, DynamicLevelMaxBytesBase2) {
 }
 
 TEST_F(DBTest, DynamicLevelCompressionPerLevel) {
-  if (!SnappyCompressionSupported()) {
+  if (!Snappy_Supported()) {
     return;
   }
   const int kNKeys = 120;

include/rocksdb/options.h

@@ -55,6 +55,12 @@ enum CompressionType : char {
   kBZip2Compression = 0x3, kLZ4Compression = 0x4, kLZ4HCCompression = 0x5
 };
 
+// returns true if RocksDB was correctly linked with compression library and
+// supports the compression type
+extern bool CompressionTypeSupported(CompressionType compression_type);
+// Returns a human-readable name of the compression type
+extern const char* CompressionTypeToString(CompressionType compression_type);
+
 enum CompactionStyle : char {
   // level based compaction style
   kCompactionStyleLevel = 0x0,

table/table_test.cc

@@ -537,61 +537,6 @@ class DBConstructor: public Constructor {
   DB* db_;
 };
 
-static bool SnappyCompressionSupported() {
-#ifdef SNAPPY
-  std::string out;
-  Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
-  return Snappy_Compress(Options().compression_opts, in.data(), in.size(),
-                         &out);
-#else
-  return false;
-#endif
-}
-
-static bool ZlibCompressionSupported() {
-#ifdef ZLIB
-  std::string out;
-  Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
-  return Zlib_Compress(Options().compression_opts, 2, in.data(), in.size(),
-                       &out);
-#else
-  return false;
-#endif
-}
-
-static bool BZip2CompressionSupported() {
-#ifdef BZIP2
-  std::string out;
-  Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
-  return BZip2_Compress(Options().compression_opts, 2, in.data(), in.size(),
-                        &out);
-#else
-  return false;
-#endif
-}
-
-static bool LZ4CompressionSupported() {
-#ifdef LZ4
-  std::string out;
-  Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
-  return LZ4_Compress(Options().compression_opts, 2, in.data(), in.size(),
-                      &out);
-#else
-  return false;
-#endif
-}
-
-static bool LZ4HCCompressionSupported() {
-#ifdef LZ4
-  std::string out;
-  Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
-  return LZ4HC_Compress(Options().compression_opts, 2, in.data(), in.size(),
-                        &out);
-#else
-  return false;
-#endif
-}
-
 enum TestType {
   BLOCK_BASED_TABLE_TEST,
   PLAIN_TABLE_SEMI_FIXED_PREFIX,
@@ -623,22 +568,20 @@ static std::vector<TestArgs> GenerateArgList() {
   // Only add compression if it is supported
   std::vector<std::pair<CompressionType, bool>> compression_types;
   compression_types.emplace_back(kNoCompression, false);
-  if (SnappyCompressionSupported()) {
+  if (Snappy_Supported()) {
     compression_types.emplace_back(kSnappyCompression, false);
   }
-  if (ZlibCompressionSupported()) {
+  if (Zlib_Supported()) {
     compression_types.emplace_back(kZlibCompression, false);
     compression_types.emplace_back(kZlibCompression, true);
   }
-  if (BZip2CompressionSupported()) {
+  if (BZip2_Supported()) {
     compression_types.emplace_back(kBZip2Compression, false);
     compression_types.emplace_back(kBZip2Compression, true);
   }
-  if (LZ4CompressionSupported()) {
+  if (LZ4_Supported()) {
     compression_types.emplace_back(kLZ4Compression, false);
     compression_types.emplace_back(kLZ4Compression, true);
-  }
-  if (LZ4HCCompressionSupported()) {
     compression_types.emplace_back(kLZ4HCCompression, false);
     compression_types.emplace_back(kLZ4HCCompression, true);
   }
@@ -1918,13 +1861,13 @@ static void DoCompressionTest(CompressionType comp) {
 
 TEST_F(GeneralTableTest, ApproximateOffsetOfCompressed) {
   std::vector<CompressionType> compression_state;
-  if (!SnappyCompressionSupported()) {
+  if (!Snappy_Supported()) {
     fprintf(stderr, "skipping snappy compression tests\n");
   } else {
     compression_state.push_back(kSnappyCompression);
   }
 
-  if (!ZlibCompressionSupported()) {
+  if (!Zlib_Supported()) {
     fprintf(stderr, "skipping zlib compression tests\n");
   } else {
     compression_state.push_back(kZlibCompression);
@@ -1932,22 +1875,17 @@ TEST_F(GeneralTableTest, ApproximateOffsetOfCompressed) {
 
   // TODO(kailiu) DoCompressionTest() doesn't work with BZip2.
   /*
-  if (!BZip2CompressionSupported()) {
+  if (!BZip2_Supported()) {
     fprintf(stderr, "skipping bzip2 compression tests\n");
   } else {
     compression_state.push_back(kBZip2Compression);
   }
   */
 
-  if (!LZ4CompressionSupported()) {
+  if (!LZ4_Supported()) {
-    fprintf(stderr, "skipping lz4 compression tests\n");
+    fprintf(stderr, "skipping lz4 and lz4hc compression tests\n");
   } else {
     compression_state.push_back(kLZ4Compression);
-  }
-
-  if (!LZ4HCCompressionSupported()) {
-    fprintf(stderr, "skipping lz4hc compression tests\n");
-  } else {
     compression_state.push_back(kLZ4HCCompression);
   }
 

util/compression.h

@@ -34,6 +34,34 @@
 
 namespace rocksdb {
 
+inline bool Snappy_Supported() {
+#ifdef SNAPPY
+  return true;
+#endif
+  return false;
+}
+
+inline bool Zlib_Supported() {
+#ifdef ZLIB
+  return true;
+#endif
+  return false;
+}
+
+inline bool BZip2_Supported() {
+#ifdef BZIP2
+  return true;
+#endif
+  return false;
+}
+
+inline bool LZ4_Supported() {
+#ifdef LZ4
+  return true;
+#endif
+  return false;
+}
+
 // compress_format_version can have two values:
 // 1 -- decompressed sizes for BZip2 and Zlib are not included in the compressed
 // block. Also, decompressed sizes for LZ4 are encoded in platform-dependent

util/options.cc

@@ -29,6 +29,7 @@
 #include "rocksdb/table.h"
 #include "rocksdb/table_properties.h"
 #include "table/block_based_table_factory.h"
+#include "util/compression.h"
 #include "util/statistics.h"
 #include "util/xfunc.h"
 
@@ -383,11 +384,12 @@ void ColumnFamilyOptions::Dump(Logger* log) const {
   Log(log, " Options.max_write_buffer_number: %d", max_write_buffer_number);
     if (!compression_per_level.empty()) {
       for (unsigned int i = 0; i < compression_per_level.size(); i++) {
-          Log(log,"       Options.compression[%d]: %d",
+        Log(log, "       Options.compression[%d]: %s", i,
-              i, compression_per_level[i]);
+            CompressionTypeToString(compression_per_level[i]));
-       }
+      }
     } else {
-      Log(log,"         Options.compression: %d", compression);
+      Log(log, "         Options.compression: %s",
+          CompressionTypeToString(compression));
     }
     Log(log,"      Options.prefix_extractor: %s",
         prefix_extractor == nullptr ? "nullptr" : prefix_extractor->Name());
@@ -543,6 +545,46 @@ Options::PrepareForBulkLoad()
   return this;
 }
 
+const char* CompressionTypeToString(CompressionType compression_type) {
+  switch (compression_type) {
+    case kNoCompression:
+      return "NoCompression";
+    case kSnappyCompression:
+      return "Snappy";
+    case kZlibCompression:
+      return "Zlib";
+    case kBZip2Compression:
+      return "BZip2";
+    case kLZ4Compression:
+      return "LZ4";
+    case kLZ4HCCompression:
+      return "LZ4HC";
+    default:
+      assert(false);
+      return "";
+  }
+}
+
+bool CompressionTypeSupported(CompressionType compression_type) {
+  switch (compression_type) {
+    case kNoCompression:
+      return true;
+    case kSnappyCompression:
+      return Snappy_Supported();
+    case kZlibCompression:
+      return Zlib_Supported();
+    case kBZip2Compression:
+      return BZip2_Supported();
+    case kLZ4Compression:
+      return LZ4_Supported();
+    case kLZ4HCCompression:
+      return LZ4_Supported();
+    default:
+      assert(false);
+      return false;
+  }
+}
+
 #ifndef ROCKSDB_LITE
 // Optimization functions
 ColumnFamilyOptions* ColumnFamilyOptions::OptimizeForPointLookup(