Add option --decode_blob_index to dump_live_files command (#9842)

Summary:
This change only add decode blob index support to dump_live_files command, which is part of a task to add blob support to a few commands.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9842

Reviewed By: ltamasi

Differential Revision: D35650167

Pulled By: jowlyzhang

fbshipit-source-id: a78151b98bc38ac6f52c6e01ca6927a3429ddd14

include/rocksdb/utilities/ldb_cmd.h

@@ -69,6 +69,7 @@ class LDBCommand {
   static const std::string ARG_BLOB_GARBAGE_COLLECTION_AGE_CUTOFF;
   static const std::string ARG_BLOB_GARBAGE_COLLECTION_FORCE_THRESHOLD;
   static const std::string ARG_BLOB_COMPACTION_READAHEAD_SIZE;
+  static const std::string ARG_DECODE_BLOB_INDEX;
 
   struct ParsedParams {
     std::string cmd;

tools/ldb_cmd.cc

@@ -98,6 +98,7 @@ const std::string LDBCommand::ARG_BLOB_GARBAGE_COLLECTION_FORCE_THRESHOLD =
     "blob_garbage_collection_force_threshold";
 const std::string LDBCommand::ARG_BLOB_COMPACTION_READAHEAD_SIZE =
     "blob_compaction_readahead_size";
+const std::string LDBCommand::ARG_DECODE_BLOB_INDEX = "decode_blob_index";
 
 const char* LDBCommand::DELIM = " ==> ";
 
@@ -108,7 +109,7 @@ void DumpWalFile(Options options, std::string wal_file, bool print_header,
                  LDBCommandExecuteResult* exec_state);
 
 void DumpSstFile(Options options, std::string filename, bool output_hex,
-                 bool show_properties);
+                 bool show_properties, bool decode_blob_index);
 };
 
 LDBCommand* LDBCommand::InitFromCmdLineArgs(
@@ -1956,7 +1957,8 @@ void DBDumperCommand::DoCommand() {
                     &exec_state_);
         break;
       case kTableFile:
-        DumpSstFile(options_, path_, is_key_hex_, /* show_properties */ true);
+        DumpSstFile(options_, path_, is_key_hex_, /* show_properties */ true,
+                    /* decode_blob_index */ false);
         break;
       case kDescriptorFile:
         DumpManifestFile(options_, path_, /* verbose_ */ false, is_key_hex_,
@@ -3468,7 +3470,7 @@ void RestoreCommand::DoCommand() {
 namespace {
 
 void DumpSstFile(Options options, std::string filename, bool output_hex,
-                 bool show_properties) {
+                 bool show_properties, bool decode_blob_index) {
   std::string from_key;
   std::string to_key;
   if (filename.length() <= 4 ||
@@ -3477,12 +3479,10 @@ void DumpSstFile(Options options, std::string filename, bool output_hex,
     return;
   }
   // no verification
-  // TODO: add support for decoding blob indexes in ldb as well
   ROCKSDB_NAMESPACE::SstFileDumper dumper(
       options, filename, Temperature::kUnknown,
       2 * 1024 * 1024 /* readahead_size */,
-      /* verify_checksum */ false, output_hex,
+      /* verify_checksum */ false, output_hex, decode_blob_index);
-      /* decode_blob_index */ false);
   Status st = dumper.ReadSequential(true, std::numeric_limits<uint64_t>::max(),
                                     false,            // has_from
                                     from_key, false,  // has_to
@@ -3519,11 +3519,14 @@ DBFileDumperCommand::DBFileDumperCommand(
     const std::vector<std::string>& /*params*/,
     const std::map<std::string, std::string>& options,
     const std::vector<std::string>& flags)
-    : LDBCommand(options, flags, true, BuildCmdLineOptions({})) {}
+    : LDBCommand(options, flags, true,
+                 BuildCmdLineOptions({ARG_DECODE_BLOB_INDEX})),
+      decode_blob_index_(IsFlagPresent(flags, ARG_DECODE_BLOB_INDEX)) {}
 
 void DBFileDumperCommand::Help(std::string& ret) {
   ret.append("  ");
   ret.append(DBFileDumperCommand::Name());
+  ret.append(" [--" + ARG_DECODE_BLOB_INDEX + "] ");
   ret.append("\n");
 }
 
@@ -3568,7 +3571,7 @@ void DBFileDumperCommand::DoCommand() {
     filename = NormalizePath(filename);
     std::cout << filename << " level:" << fileMetadata.level << std::endl;
     std::cout << "------------------------------" << std::endl;
-    DumpSstFile(options_, filename, false, true);
+    DumpSstFile(options_, filename, false, true, decode_blob_index_);
     std::cout << std::endl;
   }
   std::cout << std::endl;

tools/ldb_cmd_impl.h

@@ -44,6 +44,9 @@ class DBFileDumperCommand : public LDBCommand {
   static void Help(std::string& ret);
 
   virtual void DoCommand() override;
+
+ private:
+  bool decode_blob_index_;
 };
 
 class DBLiveFilesMetadataDumperCommand : public LDBCommand {

tools/ldb_test.py

@@ -458,7 +458,7 @@ class LDBTestCase(unittest.TestCase):
 
         dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
         self.assertRunOK("put x1 y1 --create_if_missing", "OK")
-        self.assertRunOK("put x2 y2", "OK")
+        self.assertRunOK("put x2 y2 --enable_blob_files", "OK")
         dumpFilePath = os.path.join(self.TMP_DIR, "dump1")
         self.assertTrue(self.dumpLiveFiles("--db=%s" % dbPath, dumpFilePath))
         self.assertRunOK("delete x1", "OK")
@@ -474,7 +474,7 @@ class LDBTestCase(unittest.TestCase):
             dbPath += "/"
 
         # Call the dump_live_files function with the edited dbPath name.
-        self.assertTrue(self.dumpLiveFiles("--db=%s" % dbPath, dumpFilePath))
+        self.assertTrue(self.dumpLiveFiles("--db=%s --decode_blob_index" % dbPath, dumpFilePath))
 
         # Investigate the output
         with open(dumpFilePath, "r") as tmp:
@@ -493,6 +493,10 @@ class LDBTestCase(unittest.TestCase):
             filenumber = re.findall(r"(?<=MANIFEST-)\d+", manifestFilename)[0]
             self.assertEqual(manifestFilename, dbPath+"MANIFEST-"+filenumber)
 
+        # Check that the blob file index is decoded.
+        decodedBlobIndex = re.findall(r"\[blob ref\]", data)
+        self.assertTrue(len(decodedBlobIndex) >= 1)
+
     def listLiveFilesMetadata(self, params, dumpFile):
         return 0 == run_err_null("./ldb list_live_files_metadata %s > %s" % (
             params, dumpFile))