Added JSON manifest dump option to ldb command

Summary:
Added a new flag --json to the ldb manifest_dump command
that prints out the version edits as JSON objects for easier
reading and parsing of information.

Test Plan:
**Sample usage: **
```
./ldb manifest_dump --json --path=path/to/manifest/file
```

**Sample output:**
```
{"EditNumber": 0, "Comparator": "leveldb.BytewiseComparator", "ColumnFamily": 0}
{"EditNumber": 1, "LogNumber": 0, "ColumnFamily": 0}
{"EditNumber": 2, "LogNumber": 4, "PrevLogNumber": 0, "NextFileNumber": 7, "LastSeq": 35356, "AddedFiles": [{"Level": 0, "FileNumber": 5, "FileSize": 1949284, "SmallestIKey": "'", "LargestIKey": "'"}], "ColumnFamily": 0}
...
{"EditNumber": 13, "PrevLogNumber": 0, "NextFileNumber": 36, "LastSeq": 290994, "DeletedFiles": [{"Level": 0, "FileNumber": 17}, {"Level": 0, "FileNumber": 20}, {"Level": 0, "FileNumber": 22}, {"Level": 0, "FileNumber": 24}, {"Level": 1, "FileNumber": 13}, {"Level": 1, "FileNumber": 14}, {"Level": 1, "FileNumber": 15}, {"Level": 1, "FileNumber": 18}], "AddedFiles": [{"Level": 1, "FileNumber": 25, "FileSize": 2114340, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 26, "FileSize": 2115213, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 27, "FileSize": 2114807, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 30, "FileSize": 2115271, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 31, "FileSize": 2115165, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 32, "FileSize": 2114683, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 35, "FileSize": 1757512, "SmallestIKey": "'", "LargestIKey": "'"}], "ColumnFamily": 0}
...
```

Reviewers: sdong, anthony, yhchiang, igor

Reviewed By: igor

Subscribers: dhruba

Differential Revision: https://reviews.facebook.net/D41727
main
Ari Ekmekji 9 years ago
parent a96fcd09b7
commit 74c755c552
  1. 74
      db/version_edit.cc
  2. 1
      db/version_edit.h
  3. 9
      db/version_set.cc
  4. 2
      db/version_set.h
  5. 25
      util/event_logger.h
  6. 17
      util/ldb_cmd.cc
  7. 2
      util/ldb_cmd.h

@ -11,6 +11,7 @@
#include "db/version_set.h" #include "db/version_set.h"
#include "util/coding.h" #include "util/coding.h"
#include "util/event_logger.h"
#include "rocksdb/slice.h" #include "rocksdb/slice.h"
namespace rocksdb { namespace rocksdb {
@ -359,7 +360,7 @@ std::string VersionEdit::DebugString(bool hex_key) const {
AppendNumberTo(&r, prev_log_number_); AppendNumberTo(&r, prev_log_number_);
} }
if (has_next_file_number_) { if (has_next_file_number_) {
r.append("\n NextFile: "); r.append("\n NextFileNumber: ");
AppendNumberTo(&r, next_file_number_); AppendNumberTo(&r, next_file_number_);
} }
if (has_last_sequence_) { if (has_last_sequence_) {
@ -404,4 +405,75 @@ std::string VersionEdit::DebugString(bool hex_key) const {
return r; return r;
} }
std::string VersionEdit::DebugJSON(int edit_num, bool hex_key) const {
JSONWriter jw;
jw << "EditNumber" << edit_num;
if (has_comparator_) {
jw << "Comparator" << comparator_;
}
if (has_log_number_) {
jw << "LogNumber" << log_number_;
}
if (has_prev_log_number_) {
jw << "PrevLogNumber" << prev_log_number_;
}
if (has_next_file_number_) {
jw << "NextFileNumber" << next_file_number_;
}
if (has_last_sequence_) {
jw << "LastSeq" << last_sequence_;
}
if (!deleted_files_.empty()) {
jw << "DeletedFiles";
jw.StartArray();
for (DeletedFileSet::const_iterator iter = deleted_files_.begin();
iter != deleted_files_.end();
++iter) {
jw.StartArrayedObject();
jw << "Level" << iter->first;
jw << "FileNumber" << iter->second;
jw.EndArrayedObject();
}
jw.EndArray();
}
if (!new_files_.empty()) {
jw << "AddedFiles";
jw.StartArray();
for (size_t i = 0; i < new_files_.size(); i++) {
jw.StartArrayedObject();
jw << "Level" << new_files_[i].first;
const FileMetaData& f = new_files_[i].second;
jw << "FileNumber" << f.fd.GetNumber();
jw << "FileSize" << f.fd.GetFileSize();
jw << "SmallestIKey" << f.smallest.DebugString(hex_key);
jw << "LargestIKey" << f.largest.DebugString(hex_key);
jw.EndArrayedObject();
}
jw.EndArray();
}
jw << "ColumnFamily" << column_family_;
if (is_column_family_add_) {
jw << "ColumnFamilyAdd" << column_family_name_;
}
if (is_column_family_drop_) {
jw << "ColumnFamilyDrop" << column_family_name_;
}
if (has_max_column_family_) {
jw << "MaxColumnFamily" << max_column_family_;
}
jw.EndObject();
return jw.Get();
}
} // namespace rocksdb } // namespace rocksdb

@ -227,6 +227,7 @@ class VersionEdit {
} }
std::string DebugString(bool hex_key = false) const; std::string DebugString(bool hex_key = false) const;
std::string DebugJSON(int edit_num, bool hex_key = false) const;
private: private:
friend class VersionSet; friend class VersionSet;

@ -2572,7 +2572,7 @@ Status VersionSet::ReduceNumberOfLevels(const std::string& dbname,
} }
Status VersionSet::DumpManifest(Options& options, std::string& dscname, Status VersionSet::DumpManifest(Options& options, std::string& dscname,
bool verbose, bool hex) { bool verbose, bool hex, bool json) {
// Open the specified manifest file. // Open the specified manifest file.
unique_ptr<SequentialFile> file; unique_ptr<SequentialFile> file;
Status s = options.env->NewSequentialFile(dscname, &file, env_options_); Status s = options.env->NewSequentialFile(dscname, &file, env_options_);
@ -2613,9 +2613,10 @@ Status VersionSet::DumpManifest(Options& options, std::string& dscname,
} }
// Write out each individual edit // Write out each individual edit
if (verbose) { if (verbose && !json) {
printf("*************************Edit[%d] = %s\n", printf("%s\n", edit.DebugString(hex).c_str());
count, edit.DebugString(hex).c_str()); } else if (json) {
printf("%s\n", edit.DebugJSON(count, hex).c_str());
} }
count++; count++;

@ -572,7 +572,7 @@ class VersionSet {
// printf contents (for debugging) // printf contents (for debugging)
Status DumpManifest(Options& options, std::string& manifestFileName, Status DumpManifest(Options& options, std::string& manifestFileName,
bool verbose, bool hex = false); bool verbose, bool hex = false, bool json = false);
#endif // ROCKSDB_LITE #endif // ROCKSDB_LITE

@ -15,11 +15,11 @@
namespace rocksdb { namespace rocksdb {
// JSONWriter doesn't support objects in arrays yet. There wasn't a need for
// that.
class JSONWriter { class JSONWriter {
public: public:
JSONWriter() : state_(kExpectKey), first_element_(true) { stream_ << "{"; } JSONWriter() : state_(kExpectKey), first_element_(true), in_array_(false) {
stream_ << "{";
}
void AddKey(const std::string& key) { void AddKey(const std::string& key) {
assert(state_ == kExpectKey); assert(state_ == kExpectKey);
@ -59,6 +59,7 @@ class JSONWriter {
void StartArray() { void StartArray() {
assert(state_ == kExpectValue); assert(state_ == kExpectValue);
state_ = kInArray; state_ = kInArray;
in_array_ = true;
stream_ << "["; stream_ << "[";
first_element_ = true; first_element_ = true;
} }
@ -66,6 +67,7 @@ class JSONWriter {
void EndArray() { void EndArray() {
assert(state_ == kInArray); assert(state_ == kInArray);
state_ = kExpectKey; state_ = kExpectKey;
in_array_ = false;
stream_ << "]"; stream_ << "]";
first_element_ = false; first_element_ = false;
} }
@ -83,6 +85,21 @@ class JSONWriter {
first_element_ = false; first_element_ = false;
} }
void StartArrayedObject() {
assert(state_ == kInArray && in_array_);
state_ = kExpectValue;
if (!first_element_) {
stream_ << ", ";
}
StartObject();
}
void EndArrayedObject() {
assert(in_array_);
EndObject();
state_ = kInArray;
}
std::string Get() const { return stream_.str(); } std::string Get() const { return stream_.str(); }
JSONWriter& operator<<(const char* val) { JSONWriter& operator<<(const char* val) {
@ -110,9 +127,11 @@ class JSONWriter {
kExpectKey, kExpectKey,
kExpectValue, kExpectValue,
kInArray, kInArray,
kInArrayedObject,
}; };
JSONWriterState state_; JSONWriterState state_;
bool first_element_; bool first_element_;
bool in_array_;
std::ostringstream stream_; std::ostringstream stream_;
}; };

@ -527,7 +527,7 @@ void DBLoaderCommand::DoCommand() {
namespace { namespace {
void DumpManifestFile(std::string file, bool verbose, bool hex) { void DumpManifestFile(std::string file, bool verbose, bool hex, bool json) {
Options options; Options options;
EnvOptions sopt; EnvOptions sopt;
std::string dbname("dummy"); std::string dbname("dummy");
@ -540,7 +540,7 @@ void DumpManifestFile(std::string file, bool verbose, bool hex) {
WriteController wc(options.delayed_write_rate); WriteController wc(options.delayed_write_rate);
WriteBuffer wb(options.db_write_buffer_size); WriteBuffer wb(options.db_write_buffer_size);
VersionSet versions(dbname, &options, sopt, tc.get(), &wb, &wc); VersionSet versions(dbname, &options, sopt, tc.get(), &wb, &wc);
Status s = versions.DumpManifest(options, file, verbose, hex); Status s = versions.DumpManifest(options, file, verbose, hex, json);
if (!s.ok()) { if (!s.ok()) {
printf("Error in processing file %s %s\n", file.c_str(), printf("Error in processing file %s %s\n", file.c_str(),
s.ToString().c_str()); s.ToString().c_str());
@ -550,12 +550,14 @@ void DumpManifestFile(std::string file, bool verbose, bool hex) {
} // namespace } // namespace
const string ManifestDumpCommand::ARG_VERBOSE = "verbose"; const string ManifestDumpCommand::ARG_VERBOSE = "verbose";
const string ManifestDumpCommand::ARG_PATH = "path"; const string ManifestDumpCommand::ARG_JSON = "json";
const string ManifestDumpCommand::ARG_PATH = "path";
void ManifestDumpCommand::Help(string& ret) { void ManifestDumpCommand::Help(string& ret) {
ret.append(" "); ret.append(" ");
ret.append(ManifestDumpCommand::Name()); ret.append(ManifestDumpCommand::Name());
ret.append(" [--" + ARG_VERBOSE + "]"); ret.append(" [--" + ARG_VERBOSE + "]");
ret.append(" [--" + ARG_JSON + "]");
ret.append(" [--" + ARG_PATH + "=<path_to_manifest_file>]"); ret.append(" [--" + ARG_PATH + "=<path_to_manifest_file>]");
ret.append("\n"); ret.append("\n");
} }
@ -563,11 +565,13 @@ void ManifestDumpCommand::Help(string& ret) {
ManifestDumpCommand::ManifestDumpCommand(const vector<string>& params, ManifestDumpCommand::ManifestDumpCommand(const vector<string>& params,
const map<string, string>& options, const vector<string>& flags) : const map<string, string>& options, const vector<string>& flags) :
LDBCommand(options, flags, false, LDBCommand(options, flags, false,
BuildCmdLineOptions({ARG_VERBOSE, ARG_PATH, ARG_HEX})), BuildCmdLineOptions({ARG_VERBOSE, ARG_PATH, ARG_HEX, ARG_JSON})),
verbose_(false), verbose_(false),
json_(false),
path_("") path_("")
{ {
verbose_ = IsFlagPresent(flags, ARG_VERBOSE); verbose_ = IsFlagPresent(flags, ARG_VERBOSE);
json_ = IsFlagPresent(flags, ARG_JSON);
map<string, string>::const_iterator itr = options.find(ARG_PATH); map<string, string>::const_iterator itr = options.find(ARG_PATH);
if (itr != options.end()) { if (itr != options.end()) {
@ -623,7 +627,8 @@ void ManifestDumpCommand::DoCommand() {
printf("Processing Manifest file %s\n", manifestfile.c_str()); printf("Processing Manifest file %s\n", manifestfile.c_str());
} }
DumpManifestFile(manifestfile, verbose_, is_key_hex_); DumpManifestFile(manifestfile, verbose_, is_key_hex_, json_);
if (verbose_) { if (verbose_) {
printf("Processing Manifest file %s done\n", manifestfile.c_str()); printf("Processing Manifest file %s done\n", manifestfile.c_str());
} }
@ -2025,7 +2030,7 @@ void DBFileDumperCommand::DoCommand() {
manifest_filename.resize(manifest_filename.size() - 1); manifest_filename.resize(manifest_filename.size() - 1);
string manifest_filepath = db_->GetName() + "/" + manifest_filename; string manifest_filepath = db_->GetName() + "/" + manifest_filename;
std::cout << manifest_filepath << std::endl; std::cout << manifest_filepath << std::endl;
DumpManifestFile(manifest_filepath, false, false); DumpManifestFile(manifest_filepath, false, false, false);
std::cout << std::endl; std::cout << std::endl;
std::cout << "SST Files" << std::endl; std::cout << "SST Files" << std::endl;

@ -507,9 +507,11 @@ public:
private: private:
bool verbose_; bool verbose_;
bool json_;
string path_; string path_;
static const string ARG_VERBOSE; static const string ARG_VERBOSE;
static const string ARG_JSON;
static const string ARG_PATH; static const string ARG_PATH;
}; };

Loading…
Cancel
Save