Add GetAllKeyVersions API

Summary:
- Introduced an include/ file dedicated to db-related debug functions to avoid making db.h more complex
- Added debugging function, `GetAllKeyVersions()`, to return a listing of internal data for a range of user keys. The new `struct KeyVersion` exposes data similar to internal key without exposing any internal type.
- Migrated the "ldb idump" subcommand to use this function
- The API takes an inclusive-exclusive range to match behavior of "ldb idump". This will be quite annoying for users who want to query a single user key's versions :(.
Closes https://github.com/facebook/rocksdb/pull/2232

Differential Revision: D4976007

Pulled By: ajkr

fbshipit-source-id: cab375da53a7595d6575af2b7e3b776aa3ad793e
main
Andrew Kryczka 8 years ago committed by Facebook Github Bot
parent 1a60982a5a
commit 3fa9a39c68
  1. 1
      CMakeLists.txt
  2. 1
      HISTORY.md
  3. 1
      TARGETS
  4. 41
      include/rocksdb/utilities/debug.h
  5. 1
      src.mk
  6. 52
      tools/ldb_cmd.cc
  7. 18
      tools/ldb_test.py
  8. 57
      utilities/debug.cc

@ -458,6 +458,7 @@ set(SOURCES
utilities/column_aware_encoding_util.cc
utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc
utilities/date_tiered/date_tiered_db_impl.cc
utilities/debug.cc
utilities/document/document_db.cc
utilities/document/json_document.cc
utilities/document/json_document_builder.cc

@ -8,6 +8,7 @@
* Introduce WriteBatch::PopSavePoint to pop the most recent save point explicitly.
* Support dynamically change `max_open_files` option via SetDBOptions()
* Added DB::CreateColumnFamilie() and DB::DropColumnFamilies() to bulk create/drop column families.
* Add debugging function `GetAllKeyVersions` to see internal versions of a range of keys.
## 5.4.0 (04/11/2017)
### Public API Change

@ -207,6 +207,7 @@ cpp_library(
"utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc",
"utilities/convenience/info_log_finder.cc",
"utilities/date_tiered/date_tiered_db_impl.cc",
"utilities/debug.cc",
"utilities/document/document_db.cc",
"utilities/document/json_document.cc",
"utilities/document/json_document_builder.cc",

@ -0,0 +1,41 @@
// Copyright (c) 2017-present, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
// This source code is also licensed under the GPLv2 license found in the
// COPYING file in the root directory of this source tree.
#pragma once
#ifndef ROCKSDB_LITE
#include "rocksdb/db.h"
#include "rocksdb/types.h"
namespace rocksdb {
// Data associated with a particular version of a key. A database may internally
// store multiple versions of a same user key due to snapshots, compaction not
// happening yet, etc.
struct KeyVersion {
KeyVersion(const std::string& _user_key, const std::string& _value,
SequenceNumber _sequence, int _type)
: user_key(_user_key), value(_value), sequence(_sequence), type(_type) {}
std::string user_key;
std::string value;
SequenceNumber sequence;
// TODO(ajkr): we should provide a helper function that converts the int to a
// string describing the type for easier debugging.
int type;
};
// Returns listing of all versions of keys in the provided user key range.
// The range is inclusive-inclusive, i.e., [`begin_key`, `end_key`].
// The result is inserted into the provided vector, `key_versions`.
Status GetAllKeyVersions(DB* db, Slice begin_key, Slice end_key,
std::vector<KeyVersion>* key_versions);
} // namespace rocksdb
#endif // ROCKSDB_LITE

@ -160,6 +160,7 @@ LIB_SOURCES = \
utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc \
utilities/convenience/info_log_finder.cc \
utilities/date_tiered/date_tiered_db_impl.cc \
utilities/debug.cc \
utilities/document/document_db.cc \
utilities/document/json_document.cc \
utilities/document/json_document_builder.cc \

@ -23,6 +23,7 @@
#include "rocksdb/table_properties.h"
#include "rocksdb/utilities/backupable_db.h"
#include "rocksdb/utilities/checkpoint.h"
#include "rocksdb/utilities/debug.h"
#include "rocksdb/utilities/object_registry.h"
#include "rocksdb/utilities/options_util.h"
#include "rocksdb/write_batch.h"
@ -1212,56 +1213,33 @@ void InternalDumpCommand::DoCommand() {
}
// Cast as DBImpl to get internal iterator
DBImpl* idb = dynamic_cast<DBImpl*>(db_);
if (!idb) {
exec_state_ = LDBCommandExecuteResult::Failed("DB is not DBImpl");
std::vector<KeyVersion> key_versions;
Status st = GetAllKeyVersions(db_, from_, to_, &key_versions);
if (!st.ok()) {
exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
return;
}
std::string rtype1, rtype2, row, val;
rtype2 = "";
uint64_t c=0;
uint64_t s1=0,s2=0;
// Setup internal key iterator
Arena arena;
auto icmp = InternalKeyComparator(options_.comparator);
RangeDelAggregator range_del_agg(icmp, {} /* snapshots */);
ScopedArenaIterator iter(idb->NewInternalIterator(&arena, &range_del_agg));
Status st = iter->status();
if (!st.ok()) {
exec_state_ =
LDBCommandExecuteResult::Failed("Iterator error:" + st.ToString());
}
if (has_from_) {
InternalKey ikey;
ikey.SetMaxPossibleForUserKey(from_);
iter->Seek(ikey.Encode());
} else {
iter->SeekToFirst();
}
long long count = 0;
for (; iter->Valid(); iter->Next()) {
ParsedInternalKey ikey;
if (!ParseInternalKey(iter->key(), &ikey)) {
fprintf(stderr, "Internal Key [%s] parse error!\n",
iter->key().ToString(true /* in hex*/).data());
// TODO: add error counter
continue;
}
// If end marker was specified, we stop before it
if (has_to_ && options_.comparator->Compare(ikey.user_key, to_) >= 0) {
for (auto& key_version : key_versions) {
InternalKey ikey(key_version.user_key, key_version.sequence,
static_cast<ValueType>(key_version.type));
if (has_to_ && ikey.user_key() == to_) {
// GetAllKeyVersions() includes keys with user key `to_`, but idump has
// traditionally excluded such keys.
break;
}
++count;
int k;
if (count_delim_) {
rtype1 = "";
s1=0;
row = iter->key().ToString();
val = iter->value().ToString();
row = ikey.Encode().ToString();
val = key_version.value;
for(k=0;row[k]!='\x01' && row[k]!='\0';k++)
s1++;
for(k=0;val[k]!='\x01' && val[k]!='\0';k++)
@ -1278,12 +1256,12 @@ void InternalDumpCommand::DoCommand() {
c++;
s2+=s1;
rtype2=rtype1;
}
}
}
if (!count_only_ && !count_delim_) {
std::string key = ikey.DebugString(is_key_hex_);
std::string value = iter->value().ToString(is_value_hex_);
std::string value = Slice(key_version.value).ToString(is_value_hex_);
std::cout << key << " => " << value << "\n";
}

@ -169,10 +169,10 @@ class LDBTestCase(unittest.TestCase):
print "Running testCountDelimIDump..."
self.assertRunOK("batchput x.1 x1 --create_if_missing", "OK")
self.assertRunOK("batchput y.abc abc y.2 2 z.13c pqr", "OK")
self.assertRunOK("dump --count_delim", "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8")
self.assertRunOK("dump --count_delim=\".\"", "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8")
self.assertRunOK("idump --count_delim", "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8")
self.assertRunOK("idump --count_delim=\".\"", "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8")
self.assertRunOK("batchput x,2 x2 x,abc xabc", "OK")
self.assertRunOK("dump --count_delim=\",\"", "x => count:2\tsize:14\nx.1 => count:1\tsize:5\ny.2 => count:1\tsize:4\ny.abc => count:1\tsize:8\nz.13c => count:1\tsize:8")
self.assertRunOK("idump --count_delim=\",\"", "x => count:2\tsize:14\nx.1 => count:1\tsize:5\ny.2 => count:1\tsize:4\ny.abc => count:1\tsize:8\nz.13c => count:1\tsize:8")
def testInvalidCmdLines(self):
print "Running testInvalidCmdLines..."
@ -331,6 +331,18 @@ class LDBTestCase(unittest.TestCase):
self.assertFalse(self.dumpDb(
"--db=%s --create_if_missing" % origDbPath, dumpFilePath))
def testIDumpBasics(self):
print "Running testIDumpBasics..."
self.assertRunOK("put a val --create_if_missing", "OK")
self.assertRunOK("put b val", "OK")
self.assertRunOK(
"idump", "'a' seq:1, type:1 => val\n"
"'b' seq:2, type:1 => val\nInternal keys in range: 2")
self.assertRunOK(
"idump --input_key_hex --from=%s --to=%s" % (hex(ord('a')),
hex(ord('b'))),
"'a' seq:1, type:1 => val\nInternal keys in range: 1")
def testMiscAdminTask(self):
print "Running testMiscAdminTask..."
# These tests need to be improved; for example with asserts about

@ -0,0 +1,57 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
// This source code is also licensed under the GPLv2 license found in the
// COPYING file in the root directory of this source tree.
#ifndef ROCKSDB_LITE
#include "rocksdb/utilities/debug.h"
#include "db/db_impl.h"
namespace rocksdb {
Status GetAllKeyVersions(DB* db, Slice begin_key, Slice end_key,
std::vector<KeyVersion>* key_versions) {
assert(key_versions != nullptr);
key_versions->clear();
DBImpl* idb = static_cast<DBImpl*>(db->GetRootDB());
auto icmp = InternalKeyComparator(idb->GetOptions().comparator);
RangeDelAggregator range_del_agg(icmp, {} /* snapshots */);
Arena arena;
ScopedArenaIterator iter(idb->NewInternalIterator(&arena, &range_del_agg));
if (!begin_key.empty()) {
InternalKey ikey;
ikey.SetMaxPossibleForUserKey(begin_key);
iter->Seek(ikey.Encode());
} else {
iter->SeekToFirst();
}
for (; iter->Valid(); iter->Next()) {
ParsedInternalKey ikey;
if (!ParseInternalKey(iter->key(), &ikey)) {
return Status::Corruption("Internal Key [" + iter->key().ToString() +
"] parse error!");
}
if (!end_key.empty() &&
icmp.user_comparator()->Compare(ikey.user_key, end_key) > 0) {
break;
}
key_versions->emplace_back(ikey.user_key.ToString() /* _user_key */,
iter->value().ToString() /* _value */,
ikey.sequence /* _sequence */,
static_cast<int>(ikey.type) /* _type */);
}
return Status::OK();
}
} // namespace rocksdb
#endif // ROCKSDB_LITE
Loading…
Cancel
Save