[RocksDB] Add bulk_load option to Options and ldb

Summary:
Add a shortcut function to make it easier for people
to efficiently bulk_load data into RocksDB.

Test Plan:
Tried ldb with "--bulk_load" and "--bulk_load --compact" and verified the outcome.
Needs to consult the team on how to test this automatically.

Reviewers: sheki, dhruba, emayanke, heyongqiang

Reviewed By: dhruba

CC: leveldb

Differential Revision: https://reviews.facebook.net/D8907

include/leveldb/options.h

@@ -345,6 +345,15 @@ struct Options {
 
   void Dump(Logger* log) const;
 
+  // Set appropriate parameters for bulk loading.
+  // The reason that this is a function that returns "this" instead of a
+  // constructure is to enable chaining of multiple similar calls in the future.
+  //
+  // All data will be in level 0 without any automatic compaction.
+  // It's recommended to manually call CompactRange(NULL, NULL) before reading
+  // from the database, because otherwise the read can be very slow.
+  Options* PrepareForBulkLoad();
+
   // This method allows an application to modify/delete a key-value at
   // the time of compaction. The compaction process invokes this
   // method for every kv that is being compacted. A return value

util/ldb_cmd.cc

@@ -324,17 +324,23 @@ void CompactorCommand::DoCommand() {
 }
 
 const string DBLoaderCommand::ARG_DISABLE_WAL = "disable_wal";
+const string DBLoaderCommand::ARG_BULK_LOAD = "bulk_load";
+const string DBLoaderCommand::ARG_COMPACT = "compact";
 
 DBLoaderCommand::DBLoaderCommand(const vector<string>& params,
       const map<string, string>& options, const vector<string>& flags) :
     LDBCommand(options, flags, false,
                BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX,
                                     ARG_FROM, ARG_TO, ARG_CREATE_IF_MISSING,
-                                    ARG_DISABLE_WAL})),
+                                    ARG_DISABLE_WAL, ARG_BULK_LOAD,
-    create_if_missing_(false) {
+                                    ARG_COMPACT})),
+    create_if_missing_(false), disable_wal_(false), bulk_load_(false),
+    compact_(false) {
 
   create_if_missing_ = IsFlagPresent(flags, ARG_CREATE_IF_MISSING);
   disable_wal_ = IsFlagPresent(flags, ARG_DISABLE_WAL);
+  bulk_load_ = IsFlagPresent(flags, ARG_BULK_LOAD);
+  compact_ = IsFlagPresent(flags, ARG_COMPACT);
 }
 
 void DBLoaderCommand::Help(string& ret) {
@@ -342,12 +348,17 @@ void DBLoaderCommand::Help(string& ret) {
   ret.append(DBLoaderCommand::Name());
   ret.append(" [--" + ARG_CREATE_IF_MISSING + "]");
   ret.append(" [--" + ARG_DISABLE_WAL + "]");
+  ret.append(" [--" + ARG_BULK_LOAD + "]");
+  ret.append(" [--" + ARG_COMPACT + "]");
   ret.append("\n");
 }
 
 leveldb::Options DBLoaderCommand::PrepareOptionsForOpenDB() {
   leveldb::Options opt = LDBCommand::PrepareOptionsForOpenDB();
   opt.create_if_missing = create_if_missing_;
+  if (bulk_load_) {
+    opt.PrepareForBulkLoad();
+  }
   return opt;
 }
 
@@ -380,6 +391,9 @@ void DBLoaderCommand::DoCommand() {
   if (bad_lines > 0) {
     std::cout << "Warning: " << bad_lines << " bad lines ignored." << std::endl;
   }
+  if (compact_) {
+    db_->CompactRange(NULL, NULL);
+  }
 }
 
 const string DBDumperCommand::ARG_COUNT_ONLY = "count_only";

util/ldb_cmd.h

@@ -372,8 +372,12 @@ public:
 private:
   bool create_if_missing_;
   bool disable_wal_;
+  bool bulk_load_;
+  bool compact_;
 
   static const string ARG_DISABLE_WAL;
+  static const string ARG_BULK_LOAD;
+  static const string ARG_COMPACT;
 };
 
 class ReduceDBLevelsCommand : public LDBCommand {

util/options.cc

@@ -161,4 +161,18 @@ Options::Dump(Logger* log) const
         manifest_preallocation_size);
 }   // Options::Dump
 
+Options*
+Options::PrepareForBulkLoad()
+{
+  level0_file_num_compaction_trigger = (1<<30);
+  level0_slowdown_writes_trigger = (1<<30);
+  level0_stop_writes_trigger = (1<<30);
+  disable_auto_compactions = true;
+  disable_seek_compaction = true;
+  disableDataSync = true;
+  source_compaction_factor = (1<<30);
+
+  return this;
+}
+
 }  // namespace leveldb