Integrate the manifest_dump command with ldb

Summary:
Syntax:

   manifest_dump [--verbose] --num=<manifest_num>

e.g.

$ ./ldb --db=/home/smarlow/tmp/testdb manifest_dump --num=12
manifest_file_number 13 next_file_number 14 last_sequence 3 log_number
11  prev_log_number 0
--- level 0 --- version# 0 ---
 6:116['a1' @ 1 : 1 .. 'a1' @ 1 : 1]
 10:130['a3' @ 2 : 1 .. 'a4' @ 3 : 1]
--- level 1 --- version# 0 ---
--- level 2 --- version# 0 ---
--- level 3 --- version# 0 ---
--- level 4 --- version# 0 ---
--- level 5 --- version# 0 ---
--- level 6 --- version# 0 ---

Test Plan: - Tested on an example DB (see output in summary)

Reviewers: sheki, dhruba

Reviewed By: sheki

CC: leveldb, heyongqiang

Differential Revision: https://reviews.facebook.net/D9609

Makefile

@@ -63,7 +63,6 @@ TESTS = \
 
 
 TOOLS = \
-	manifest_dump \
         sst_dump \
         db_stress \
         ldb \
@@ -229,9 +228,6 @@ leveldb_shell: tools/shell/ShellContext.o tools/shell/ShellState.o tools/shell/L
 DBClientProxy_test: tools/shell/test/DBClientProxyTest.o tools/shell/DBClientProxy.o $(LIBRARY) 
 	$(CXX) tools/shell/test/DBClientProxyTest.o tools/shell/DBClientProxy.o $(LIBRARY) $(EXEC_LDFLAGS) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS)
 
-manifest_dump: tools/manifest_dump.o $(LIBOBJECTS)
-	$(CXX) tools/manifest_dump.o $(LIBOBJECTS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
-
 filelock_test: util/filelock_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) util/filelock_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
 

tools/ldb.cc

@@ -56,6 +56,7 @@ public:
     ReduceDBLevelsCommand::Help(ret);
     DBDumperCommand::Help(ret);
     DBLoaderCommand::Help(ret);
+    ManifestDumpCommand::Help(ret);
 
     fprintf(stderr, "%s\n", ret.c_str());
   }

tools/manifest_dump.cc

@@ -1,81 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/version_set.h"
-
-#include <algorithm>
-#include <stdio.h>
-#include "db/filename.h"
-#include "db/log_reader.h"
-#include "db/log_writer.h"
-#include "db/memtable.h"
-#include "db/table_cache.h"
-#include "leveldb/env.h"
-#include "leveldb/table_builder.h"
-#include "table/merger.h"
-#include "table/two_level_iterator.h"
-#include "util/coding.h"
-#include "util/logging.h"
-#include "util/storage_options.h"
-
-static int verbose = 0;
-static int output_hex = 0;
-
-using namespace leveldb;
-
-//
-// Takes a manifest file and dumps out all metedata
-//
-int main(int argc, char** argv) {
-
-  // parse command line options
-  size_t n;
-  char junk;
-  int foundfile = 0;
-  std::string manifestfile;
-  for (int i = 1; i < argc; i++) {
-    std::string param(argv[i]);
-    if ((n = param.find("--file=")) != std::string::npos) {
-      manifestfile = param.substr(strlen("--file="));
-      foundfile = 1;
-    } else if (sscanf(argv[i], "--verbose=%ld%c", &n, &junk) == 1 &&
-        (n == 0 || n == 1)) {
-      verbose = n;
-    } else if (param == "--output_hex") {
-      output_hex = n;
-    } else {
-      fprintf(stderr, "Unknown or badly-formatted option: %s\n",
-              argv[i]);
-      abort();
-    }
-  }
-  if (!foundfile) {
-    fprintf(stderr,
-      "%s [--verbose=0|1] [--output_hex] [--file=pathname of manifest file]\n",
-      argv[0]);
-    abort();
-  }
-
-  if (verbose) {
-    printf("Processing Manifest file %s\n", manifestfile.c_str());
-  }
-
-  Options options;
-  StorageOptions sopt;
-  std::string file(manifestfile);
-  std::string dbname("dummy");
-  TableCache* tc = new TableCache(dbname, &options, sopt, 10);
-  const InternalKeyComparator* cmp = new InternalKeyComparator(options.comparator);
-
-  VersionSet* versions = new VersionSet(dbname, &options, sopt,
-                                   tc, cmp);
-  Status s = versions->DumpManifest(options, file, verbose, output_hex);
-  if (!s.ok()) {
-    printf("Error in processing file %s %s\n", manifestfile.c_str(),
-           s.ToString().c_str());
-  }
-  if (verbose) {
-    printf("Processing Manifest file %s done\n", manifestfile.c_str());
-  }
-}

util/ldb_cmd.cc

@@ -3,6 +3,8 @@
 // found in the LICENSE file.
 
 #include "util/ldb_cmd.h"
+#include <dirent.h>
+
 #include <sstream>
 #include <string>
 #include <stdexcept>
@@ -10,6 +12,7 @@
 #include "leveldb/write_batch.h"
 #include "db/dbformat.h"
 #include "db/log_reader.h"
+#include "db/filename.h"
 #include "db/write_batch_internal.h"
 
 namespace leveldb {
@@ -123,6 +126,8 @@ LDBCommand* LDBCommand::InitFromCmdLineArgs(const vector<string>& args) {
     return new DBDumperCommand(cmdParams, options, flags);
   } else if (cmd == DBLoaderCommand::Name()) {
     return new DBLoaderCommand(cmdParams, options, flags);
+  } else if (cmd == ManifestDumpCommand::Name()) {
+    return new ManifestDumpCommand(cmdParams, options, flags);
   }
 
   return nullptr;
@@ -410,6 +415,97 @@ void DBLoaderCommand::DoCommand() {
   }
 }
 
+// ----------------------------------------------------------------------------
+
+const string ManifestDumpCommand::ARG_VERBOSE = "verbose";
+const string ManifestDumpCommand::ARG_PATH    = "path";
+
+void ManifestDumpCommand::Help(string& ret) {
+  ret.append("  ");
+  ret.append(ManifestDumpCommand::Name());
+  ret.append(" [--" + ARG_VERBOSE + "]");
+  ret.append(" [--" + ARG_PATH + "=<path_to_manifest_file>]");
+  ret.append("\n");
+}
+
+ManifestDumpCommand::ManifestDumpCommand(const vector<string>& params,
+      const map<string, string>& options, const vector<string>& flags) :
+    LDBCommand(options, flags, false,
+               BuildCmdLineOptions({ARG_VERBOSE,ARG_PATH})),
+    verbose_(false),
+    path_("")
+{
+  verbose_ = IsFlagPresent(flags, ARG_VERBOSE);
+
+  map<string, string>::const_iterator itr = options.find(ARG_PATH);
+  if (itr != options.end()) {
+    path_ = itr->second;
+    if (path_.empty()) {
+      exec_state_ = LDBCommandExecuteResult::FAILED("--path: missing pathname");
+    }
+  }
+}
+
+void ManifestDumpCommand::DoCommand() {
+
+  std::string manifestfile;
+
+  if (!path_.empty()) {
+    manifestfile = path_;
+  } else {
+    bool found = false;
+    // We need to find the manifest file by searching the directory
+    // containing the db for files of the form MANIFEST_[0-9]+
+    DIR* d = opendir(db_path_.c_str());
+    if (d == nullptr) {
+      exec_state_ = LDBCommandExecuteResult::FAILED(
+        db_path_ + " is not a directory");
+      return;
+    }
+    struct dirent* entry;
+    while ((entry = readdir(d)) != nullptr) {
+      unsigned int match;
+      unsigned long long num;
+      if (sscanf(entry->d_name, "MANIFEST-%llu%n", &num, &match)
+          && match == strlen(entry->d_name)) {
+        if (!found) {
+          manifestfile = db_path_ + "/" + std::string(entry->d_name);
+          found = true;
+        } else {
+          exec_state_ = LDBCommandExecuteResult::FAILED(
+            "Multiple MANIFEST files found; use --path to select one");
+          return;
+        }
+      }
+    }
+    closedir(d);
+  }
+
+  if (verbose_) {
+    printf("Processing Manifest file %s\n", manifestfile.c_str());
+  }
+
+  Options options;
+  StorageOptions sopt;
+  std::string file(manifestfile);
+  std::string dbname("dummy");
+  TableCache* tc = new TableCache(dbname, &options, sopt, 10);
+  const InternalKeyComparator* cmp =
+    new InternalKeyComparator(options.comparator);
+
+  VersionSet* versions = new VersionSet(dbname, &options, sopt, tc, cmp);
+  Status s = versions->DumpManifest(options, file, verbose_, is_key_hex_);
+  if (!s.ok()) {
+    printf("Error in processing file %s %s\n", manifestfile.c_str(),
+           s.ToString().c_str());
+  }
+  if (verbose_) {
+    printf("Processing Manifest file %s done\n", manifestfile.c_str());
+  }
+}
+
+// ----------------------------------------------------------------------------
+
 const string DBDumperCommand::ARG_COUNT_ONLY = "count_only";
 const string DBDumperCommand::ARG_STATS = "stats";
 

util/ldb_cmd.h

@@ -377,6 +377,28 @@ private:
   static const string ARG_COMPACT;
 };
 
+class ManifestDumpCommand: public LDBCommand {
+public:
+  static string Name() { return "manifest_dump"; }
+
+  ManifestDumpCommand(const vector<string>& params,
+      const map<string, string>& options, const vector<string>& flags);
+
+  static void Help(string& ret);
+  virtual void DoCommand();
+
+  virtual bool NoDBOpen() {
+    return true;
+  }
+
+private:
+  bool verbose_;
+  string path_;
+
+  static const string ARG_VERBOSE;
+  static const string ARG_PATH;
+};
+
 class ReduceDBLevelsCommand : public LDBCommand {
 public:
   static string Name() { return "reduce_levels"; }