ldb: file_size and write_buffer_size options.

Summary:
This allows ldb to control the write_buffer_size (which reflects to L0 file size) and file_size (which reflects to L1 file size).  Since the target_file_size_ratio is 1 by default, all other levels will also have the same file size as L1.

As part of the diff, I also cleaned up some unused code and help messages.

Test Plan: ./ldb load --db=/data/users/zshao/test_leveldb --file_size=64000000 --write_buffer_size=32000000 --create_if_missing --input_hex --disable_wal

Reviewers: dhruba, sheki, emayanke

Reviewed By: dhruba

CC: leveldb

Differential Revision: https://reviews.facebook.net/D7569

util/ldb_cmd.cc

@@ -15,6 +15,8 @@ const char* LDBCommand::BLOOM_ARG = "--bloom_bits=";
 const char* LDBCommand::COMPRESSION_TYPE_ARG = "--compression_type=";
 const char* LDBCommand::BLOCK_SIZE = "--block_size=";
 const char* LDBCommand::AUTO_COMPACTION = "--auto_compaction=";
+const char* LDBCommand::WRITE_BUFFER_SIZE_ARG = "--write_buffer_size=";
+const char* LDBCommand::FILE_SIZE_ARG = "--file_size=";
 
 void LDBCommand::parse_open_args(std::vector<std::string>& args) {
   std::vector<std::string> rest_of_args;
@@ -23,7 +25,9 @@ void LDBCommand::parse_open_args(std::vector<std::string>& args) {
     if (arg.find(BLOOM_ARG) == 0
         || arg.find(COMPRESSION_TYPE_ARG) == 0
         || arg.find(BLOCK_SIZE) == 0
-        || arg.find(AUTO_COMPACTION) == 0) {
+        || arg.find(AUTO_COMPACTION) == 0
+        || arg.find(WRITE_BUFFER_SIZE_ARG) == 0
+        || arg.find(FILE_SIZE_ARG) == 0) {
       open_args_.push_back(arg);
     } else {
       rest_of_args.push_back(arg);
@@ -81,6 +85,25 @@ leveldb::Options LDBCommand::PrepareOptionsForOpenDB() {
         exec_state_ = LDBCommandExecuteResult::FAILED(
           "Unknown compression level: " + comp);
       }
+    } else if (arg.find(WRITE_BUFFER_SIZE_ARG) == 0) {
+      std::string write_buffer_str = arg.substr(strlen(WRITE_BUFFER_SIZE_ARG));
+      int write_buffer_size = atoi(write_buffer_str.c_str());
+      if (write_buffer_size == 0) {
+        exec_state_ = LDBCommandExecuteResult::FAILED(
+          std::string("Badly-formatted buffer size: ") + write_buffer_str);
+      }
+      opt.write_buffer_size = write_buffer_size;
+    } else if (arg.find(FILE_SIZE_ARG) == 0) {
+      std::string file_size_str = arg.substr(strlen(FILE_SIZE_ARG));
+      int file_size = atoi(file_size_str.c_str());
+      if (file_size == 0) {
+        exec_state_ = LDBCommandExecuteResult::FAILED(
+          std::string("Badly-formatted file size: ") + file_size_str);
+      }
+      opt.target_file_size_base = file_size;
+    } else {
+      exec_state_ = LDBCommandExecuteResult::FAILED(
+        "Unknown option: " + arg);
     }
   }
 
@@ -347,8 +370,6 @@ void DBDumper::DoCommand() {
 
 const char* ReduceDBLevels::NEW_LEVLES_ARG = "--new_levels=";
 const char* ReduceDBLevels::PRINT_OLD_LEVELS_ARG = "--print_old_levels";
-const char* ReduceDBLevels::COMPRESSION_TYPE_ARG = "--compression=";
-const char* ReduceDBLevels::FILE_SIZE_ARG = "--file_size=";
 
 ReduceDBLevels::ReduceDBLevels(std::string& db_name,
     std::vector<std::string>& args)
@@ -356,8 +377,6 @@ ReduceDBLevels::ReduceDBLevels(std::string& db_name,
   old_levels_(1 << 16),
   new_levels_(-1),
   print_old_levels_(false) {
-  file_size_ = leveldb::Options().target_file_size_base;
-  compression_ = leveldb::Options().compression;
 
   for (unsigned int i = 0; i < args.size(); i++) {
     std::string& arg = args.at(i);
@@ -365,21 +384,6 @@ ReduceDBLevels::ReduceDBLevels(std::string& db_name,
       new_levels_ = atoi(arg.substr(strlen(NEW_LEVLES_ARG)).c_str());
     } else if (arg.find(PRINT_OLD_LEVELS_ARG) == 0) {
       print_old_levels_ = true;
-    } else if (arg.find(COMPRESSION_TYPE_ARG) == 0) {
-      const char* type = arg.substr(strlen(COMPRESSION_TYPE_ARG)).c_str();
-      if (!strcasecmp(type, "none"))
-        compression_ = leveldb::kNoCompression;
-      else if (!strcasecmp(type, "snappy"))
-        compression_ = leveldb::kSnappyCompression;
-      else if (!strcasecmp(type, "zlib"))
-        compression_ = leveldb::kZlibCompression;
-      else if (!strcasecmp(type, "bzip2"))
-        compression_ = leveldb::kBZip2Compression;
-      else
-        exec_state_ = LDBCommandExecuteResult::FAILED(
-            "Invalid compression arg : " + arg);
-    } else if (arg.find(FILE_SIZE_ARG) == 0) {
-      file_size_ = atoi(arg.substr(strlen(FILE_SIZE_ARG)).c_str());
     } else {
       exec_state_ = LDBCommandExecuteResult::FAILED(
           "Unknown argument." + arg);

util/ldb_cmd.h

@@ -123,11 +123,15 @@ public:
     ret.append(LDBCommand::BLOOM_ARG);
     ret.append("<int,e.g.:14>] [");
     ret.append(LDBCommand::COMPRESSION_TYPE_ARG);
-    ret.append("<no|snappy|zlib|bzip2> ");
+    ret.append("<no|snappy|zlib|bzip2>] [");
     ret.append(LDBCommand::BLOCK_SIZE);
-    ret.append("=<block_size_in_bytes> ");
+    ret.append("<block_size_in_bytes>] [");
     ret.append(LDBCommand::AUTO_COMPACTION);
-    ret.append("=<true|false>]");
+    ret.append("<true|false>] [");
+    ret.append(LDBCommand::WRITE_BUFFER_SIZE_ARG);
+    ret.append("<int,e.g.:4194304>] [");
+    ret.append(LDBCommand::FILE_SIZE_ARG);
+    ret.append("<int,e.g.:2097152>] ");
   }
 
   /* Run the command, and return the execute result. */
@@ -209,6 +213,8 @@ private:
   static const char* COMPRESSION_TYPE_ARG;
   static const char* BLOCK_SIZE;
   static const char* AUTO_COMPACTION;
+  static const char* WRITE_BUFFER_SIZE_ARG;
+  static const char* FILE_SIZE_ARG;
   std::vector<std::string> open_args_;
   void parse_open_args(std::vector<std::string>& args);
 };
@@ -296,13 +302,9 @@ private:
   int old_levels_;
   int new_levels_;
   bool print_old_levels_;
-  int file_size_;
-  enum leveldb::CompressionType compression_;
 
   static const char* NEW_LEVLES_ARG;
   static const char* PRINT_OLD_LEVELS_ARG;
-  static const char* COMPRESSION_TYPE_ARG;
-  static const char* FILE_SIZE_ARG;
 
   Status GetOldNumOfLevels(leveldb::Options& opt, int* levels);
 };