Add GetAllKeyVersions API

Summary:
- Introduced an include/ file dedicated to db-related debug functions to avoid making db.h more complex
- Added debugging function, `GetAllKeyVersions()`, to return a listing of internal data for a range of user keys. The new `struct KeyVersion` exposes data similar to internal key without exposing any internal type.
- Migrated the "ldb idump" subcommand to use this function
- The API takes an inclusive-exclusive range to match behavior of "ldb idump". This will be quite annoying for users who want to query a single user key's versions :(.
Closes https://github.com/facebook/rocksdb/pull/2232

Differential Revision: D4976007

Pulled By: ajkr

fbshipit-source-id: cab375da53a7595d6575af2b7e3b776aa3ad793e

CMakeLists.txt

@@ -458,6 +458,7 @@ set(SOURCES
         utilities/column_aware_encoding_util.cc
         utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc
         utilities/date_tiered/date_tiered_db_impl.cc
+        utilities/debug.cc
         utilities/document/document_db.cc
         utilities/document/json_document.cc
         utilities/document/json_document_builder.cc

HISTORY.md

@@ -8,6 +8,7 @@
 * Introduce WriteBatch::PopSavePoint to pop the most recent save point explicitly.
 * Support dynamically change `max_open_files` option via SetDBOptions()
 * Added DB::CreateColumnFamilie() and DB::DropColumnFamilies() to bulk create/drop column families.
+* Add debugging function `GetAllKeyVersions` to see internal versions of a range of keys.
 
 ## 5.4.0 (04/11/2017)
 ### Public API Change

TARGETS

@@ -207,6 +207,7 @@ cpp_library(
       "utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc",
       "utilities/convenience/info_log_finder.cc",
       "utilities/date_tiered/date_tiered_db_impl.cc",
+      "utilities/debug.cc",
       "utilities/document/document_db.cc",
       "utilities/document/json_document.cc",
       "utilities/document/json_document_builder.cc",

include/rocksdb/utilities/debug.h

@@ -0,0 +1,41 @@
+// Copyright (c) 2017-present, Facebook, Inc.  All rights reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+// This source code is also licensed under the GPLv2 license found in the
+// COPYING file in the root directory of this source tree.
+
+#pragma once
+
+#ifndef ROCKSDB_LITE
+
+#include "rocksdb/db.h"
+#include "rocksdb/types.h"
+
+namespace rocksdb {
+
+// Data associated with a particular version of a key. A database may internally
+// store multiple versions of a same user key due to snapshots, compaction not
+// happening yet, etc.
+struct KeyVersion {
+  KeyVersion(const std::string& _user_key, const std::string& _value,
+             SequenceNumber _sequence, int _type)
+      : user_key(_user_key), value(_value), sequence(_sequence), type(_type) {}
+
+  std::string user_key;
+  std::string value;
+  SequenceNumber sequence;
+  // TODO(ajkr): we should provide a helper function that converts the int to a
+  // string describing the type for easier debugging.
+  int type;
+};
+
+// Returns listing of all versions of keys in the provided user key range.
+// The range is inclusive-inclusive, i.e., [`begin_key`, `end_key`].
+// The result is inserted into the provided vector, `key_versions`.
+Status GetAllKeyVersions(DB* db, Slice begin_key, Slice end_key,
+                         std::vector<KeyVersion>* key_versions);
+
+}  // namespace rocksdb
+
+#endif  // ROCKSDB_LITE

src.mk

@@ -160,6 +160,7 @@ LIB_SOURCES =                                                   \
   utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc    \
   utilities/convenience/info_log_finder.cc                      \
   utilities/date_tiered/date_tiered_db_impl.cc                  \
+  utilities/debug.cc                                        	\
   utilities/document/document_db.cc                             \
   utilities/document/json_document.cc                           \
   utilities/document/json_document_builder.cc                   \

tools/ldb_cmd.cc

@@ -23,6 +23,7 @@
 #include "rocksdb/table_properties.h"
 #include "rocksdb/utilities/backupable_db.h"
 #include "rocksdb/utilities/checkpoint.h"
+#include "rocksdb/utilities/debug.h"
 #include "rocksdb/utilities/object_registry.h"
 #include "rocksdb/utilities/options_util.h"
 #include "rocksdb/write_batch.h"
@@ -1212,56 +1213,33 @@ void InternalDumpCommand::DoCommand() {
   }
 
   // Cast as DBImpl to get internal iterator
-  DBImpl* idb = dynamic_cast<DBImpl*>(db_);
+  std::vector<KeyVersion> key_versions;
-  if (!idb) {
+  Status st = GetAllKeyVersions(db_, from_, to_, &key_versions);
-    exec_state_ = LDBCommandExecuteResult::Failed("DB is not DBImpl");
+  if (!st.ok()) {
+    exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
     return;
   }
   std::string rtype1, rtype2, row, val;
   rtype2 = "";
   uint64_t c=0;
   uint64_t s1=0,s2=0;
-  // Setup internal key iterator
-  Arena arena;
-  auto icmp = InternalKeyComparator(options_.comparator);
-  RangeDelAggregator range_del_agg(icmp, {} /* snapshots */);
-  ScopedArenaIterator iter(idb->NewInternalIterator(&arena, &range_del_agg));
-  Status st = iter->status();
-  if (!st.ok()) {
-    exec_state_ =
-        LDBCommandExecuteResult::Failed("Iterator error:" + st.ToString());
-  }
-
-  if (has_from_) {
-    InternalKey ikey;
-    ikey.SetMaxPossibleForUserKey(from_);
-    iter->Seek(ikey.Encode());
-  } else {
-    iter->SeekToFirst();
-  }
 
   long long count = 0;
-  for (; iter->Valid(); iter->Next()) {
+  for (auto& key_version : key_versions) {
-    ParsedInternalKey ikey;
+    InternalKey ikey(key_version.user_key, key_version.sequence,
-    if (!ParseInternalKey(iter->key(), &ikey)) {
+                     static_cast<ValueType>(key_version.type));
-      fprintf(stderr, "Internal Key [%s] parse error!\n",
+    if (has_to_ && ikey.user_key() == to_) {
-              iter->key().ToString(true /* in hex*/).data());
+      // GetAllKeyVersions() includes keys with user key `to_`, but idump has
-      // TODO: add error counter
+      // traditionally excluded such keys.
-      continue;
-    }
-
-    // If end marker was specified, we stop before it
-    if (has_to_ && options_.comparator->Compare(ikey.user_key, to_) >= 0) {
       break;
     }
-
     ++count;
     int k;
     if (count_delim_) {
       rtype1 = "";
       s1=0;
-      row = iter->key().ToString();
+      row = ikey.Encode().ToString();
-      val = iter->value().ToString();
+      val = key_version.value;
       for(k=0;row[k]!='\x01' && row[k]!='\0';k++)
         s1++;
       for(k=0;val[k]!='\x01' && val[k]!='\0';k++)
@@ -1278,12 +1256,12 @@ void InternalDumpCommand::DoCommand() {
         c++;
         s2+=s1;
         rtype2=rtype1;
+      }
     }
-  }
 
     if (!count_only_ && !count_delim_) {
       std::string key = ikey.DebugString(is_key_hex_);
-      std::string value = iter->value().ToString(is_value_hex_);
+      std::string value = Slice(key_version.value).ToString(is_value_hex_);
       std::cout << key << " => " << value << "\n";
     }
 

tools/ldb_test.py

@@ -169,10 +169,10 @@ class LDBTestCase(unittest.TestCase):
         print "Running testCountDelimIDump..."
         self.assertRunOK("batchput x.1 x1 --create_if_missing", "OK")
         self.assertRunOK("batchput y.abc abc y.2 2 z.13c pqr", "OK")
-        self.assertRunOK("dump --count_delim", "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8")
+        self.assertRunOK("idump --count_delim", "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8")
-        self.assertRunOK("dump --count_delim=\".\"", "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8")
+        self.assertRunOK("idump --count_delim=\".\"", "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8")
         self.assertRunOK("batchput x,2 x2 x,abc xabc", "OK")
-        self.assertRunOK("dump --count_delim=\",\"", "x => count:2\tsize:14\nx.1 => count:1\tsize:5\ny.2 => count:1\tsize:4\ny.abc => count:1\tsize:8\nz.13c => count:1\tsize:8")
+        self.assertRunOK("idump --count_delim=\",\"", "x => count:2\tsize:14\nx.1 => count:1\tsize:5\ny.2 => count:1\tsize:4\ny.abc => count:1\tsize:8\nz.13c => count:1\tsize:8")
 
     def testInvalidCmdLines(self):
         print "Running testInvalidCmdLines..."
@@ -331,6 +331,18 @@ class LDBTestCase(unittest.TestCase):
         self.assertFalse(self.dumpDb(
             "--db=%s --create_if_missing" % origDbPath, dumpFilePath))
 
+    def testIDumpBasics(self):
+        print "Running testIDumpBasics..."
+        self.assertRunOK("put a val --create_if_missing", "OK")
+        self.assertRunOK("put b val", "OK")
+        self.assertRunOK(
+                "idump", "'a' seq:1, type:1 => val\n"
+                "'b' seq:2, type:1 => val\nInternal keys in range: 2")
+        self.assertRunOK(
+                "idump --input_key_hex --from=%s --to=%s" % (hex(ord('a')),
+                                                             hex(ord('b'))),
+                "'a' seq:1, type:1 => val\nInternal keys in range: 1")
+
     def testMiscAdminTask(self):
         print "Running testMiscAdminTask..."
         # These tests need to be improved; for example with asserts about

utilities/debug.cc

@@ -0,0 +1,57 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//  This source code is also licensed under the GPLv2 license found in the
+//  COPYING file in the root directory of this source tree.
+
+#ifndef ROCKSDB_LITE
+
+#include "rocksdb/utilities/debug.h"
+
+#include "db/db_impl.h"
+
+namespace rocksdb {
+
+Status GetAllKeyVersions(DB* db, Slice begin_key, Slice end_key,
+                         std::vector<KeyVersion>* key_versions) {
+  assert(key_versions != nullptr);
+  key_versions->clear();
+
+  DBImpl* idb = static_cast<DBImpl*>(db->GetRootDB());
+  auto icmp = InternalKeyComparator(idb->GetOptions().comparator);
+  RangeDelAggregator range_del_agg(icmp, {} /* snapshots */);
+  Arena arena;
+  ScopedArenaIterator iter(idb->NewInternalIterator(&arena, &range_del_agg));
+
+  if (!begin_key.empty()) {
+    InternalKey ikey;
+    ikey.SetMaxPossibleForUserKey(begin_key);
+    iter->Seek(ikey.Encode());
+  } else {
+    iter->SeekToFirst();
+  }
+
+  for (; iter->Valid(); iter->Next()) {
+    ParsedInternalKey ikey;
+    if (!ParseInternalKey(iter->key(), &ikey)) {
+      return Status::Corruption("Internal Key [" + iter->key().ToString() +
+                                "] parse error!");
+    }
+
+    if (!end_key.empty() &&
+        icmp.user_comparator()->Compare(ikey.user_key, end_key) > 0) {
+      break;
+    }
+
+    key_versions->emplace_back(ikey.user_key.ToString() /* _user_key */,
+                               iter->value().ToString() /* _value */,
+                               ikey.sequence /* _sequence */,
+                               static_cast<int>(ikey.type) /* _type */);
+  }
+  return Status::OK();
+}
+
+}  // namespace rocksdb
+
+#endif  // ROCKSDB_LITE