From 74c755c552a80ffc574831eca8d9d498345a270c Mon Sep 17 00:00:00 2001 From: Ari Ekmekji Date: Fri, 17 Jul 2015 10:07:40 -0700 Subject: [PATCH] Added JSON manifest dump option to ldb command Summary: Added a new flag --json to the ldb manifest_dump command that prints out the version edits as JSON objects for easier reading and parsing of information. Test Plan: **Sample usage: ** ``` ./ldb manifest_dump --json --path=path/to/manifest/file ``` **Sample output:** ``` {"EditNumber": 0, "Comparator": "leveldb.BytewiseComparator", "ColumnFamily": 0} {"EditNumber": 1, "LogNumber": 0, "ColumnFamily": 0} {"EditNumber": 2, "LogNumber": 4, "PrevLogNumber": 0, "NextFileNumber": 7, "LastSeq": 35356, "AddedFiles": [{"Level": 0, "FileNumber": 5, "FileSize": 1949284, "SmallestIKey": "'", "LargestIKey": "'"}], "ColumnFamily": 0} ... {"EditNumber": 13, "PrevLogNumber": 0, "NextFileNumber": 36, "LastSeq": 290994, "DeletedFiles": [{"Level": 0, "FileNumber": 17}, {"Level": 0, "FileNumber": 20}, {"Level": 0, "FileNumber": 22}, {"Level": 0, "FileNumber": 24}, {"Level": 1, "FileNumber": 13}, {"Level": 1, "FileNumber": 14}, {"Level": 1, "FileNumber": 15}, {"Level": 1, "FileNumber": 18}], "AddedFiles": [{"Level": 1, "FileNumber": 25, "FileSize": 2114340, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 26, "FileSize": 2115213, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 27, "FileSize": 2114807, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 30, "FileSize": 2115271, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 31, "FileSize": 2115165, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 32, "FileSize": 2114683, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 35, "FileSize": 1757512, "SmallestIKey": "'", "LargestIKey": "'"}], "ColumnFamily": 0} ... ``` Reviewers: sdong, anthony, yhchiang, igor Reviewed By: igor Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D41727 --- db/version_edit.cc | 74 ++++++++++++++++++++++++++++++++++++++++++++- db/version_edit.h | 1 + db/version_set.cc | 9 +++--- db/version_set.h | 2 +- util/event_logger.h | 25 +++++++++++++-- util/ldb_cmd.cc | 17 +++++++---- util/ldb_cmd.h | 2 ++ 7 files changed, 115 insertions(+), 15 deletions(-) diff --git a/db/version_edit.cc b/db/version_edit.cc index f7b288870..0c9efe419 100644 --- a/db/version_edit.cc +++ b/db/version_edit.cc @@ -11,6 +11,7 @@ #include "db/version_set.h" #include "util/coding.h" +#include "util/event_logger.h" #include "rocksdb/slice.h" namespace rocksdb { @@ -359,7 +360,7 @@ std::string VersionEdit::DebugString(bool hex_key) const { AppendNumberTo(&r, prev_log_number_); } if (has_next_file_number_) { - r.append("\n NextFile: "); + r.append("\n NextFileNumber: "); AppendNumberTo(&r, next_file_number_); } if (has_last_sequence_) { @@ -404,4 +405,75 @@ std::string VersionEdit::DebugString(bool hex_key) const { return r; } +std::string VersionEdit::DebugJSON(int edit_num, bool hex_key) const { + JSONWriter jw; + jw << "EditNumber" << edit_num; + + if (has_comparator_) { + jw << "Comparator" << comparator_; + } + if (has_log_number_) { + jw << "LogNumber" << log_number_; + } + if (has_prev_log_number_) { + jw << "PrevLogNumber" << prev_log_number_; + } + if (has_next_file_number_) { + jw << "NextFileNumber" << next_file_number_; + } + if (has_last_sequence_) { + jw << "LastSeq" << last_sequence_; + } + + if (!deleted_files_.empty()) { + jw << "DeletedFiles"; + jw.StartArray(); + + for (DeletedFileSet::const_iterator iter = deleted_files_.begin(); + iter != deleted_files_.end(); + ++iter) { + jw.StartArrayedObject(); + jw << "Level" << iter->first; + jw << "FileNumber" << iter->second; + jw.EndArrayedObject(); + } + + jw.EndArray(); + } + + if (!new_files_.empty()) { + jw << "AddedFiles"; + jw.StartArray(); + + for (size_t i = 0; i < new_files_.size(); i++) { + jw.StartArrayedObject(); + jw << "Level" << new_files_[i].first; + const FileMetaData& f = new_files_[i].second; + jw << "FileNumber" << f.fd.GetNumber(); + jw << "FileSize" << f.fd.GetFileSize(); + jw << "SmallestIKey" << f.smallest.DebugString(hex_key); + jw << "LargestIKey" << f.largest.DebugString(hex_key); + jw.EndArrayedObject(); + } + + jw.EndArray(); + } + + jw << "ColumnFamily" << column_family_; + + if (is_column_family_add_) { + jw << "ColumnFamilyAdd" << column_family_name_; + } + if (is_column_family_drop_) { + jw << "ColumnFamilyDrop" << column_family_name_; + } + if (has_max_column_family_) { + jw << "MaxColumnFamily" << max_column_family_; + } + + jw.EndObject(); + + return jw.Get(); +} + } // namespace rocksdb diff --git a/db/version_edit.h b/db/version_edit.h index a978d4780..ea3a308b9 100644 --- a/db/version_edit.h +++ b/db/version_edit.h @@ -227,6 +227,7 @@ class VersionEdit { } std::string DebugString(bool hex_key = false) const; + std::string DebugJSON(int edit_num, bool hex_key = false) const; private: friend class VersionSet; diff --git a/db/version_set.cc b/db/version_set.cc index 3527d2cb0..cb7b3d192 100644 --- a/db/version_set.cc +++ b/db/version_set.cc @@ -2572,7 +2572,7 @@ Status VersionSet::ReduceNumberOfLevels(const std::string& dbname, } Status VersionSet::DumpManifest(Options& options, std::string& dscname, - bool verbose, bool hex) { + bool verbose, bool hex, bool json) { // Open the specified manifest file. unique_ptr file; Status s = options.env->NewSequentialFile(dscname, &file, env_options_); @@ -2613,9 +2613,10 @@ Status VersionSet::DumpManifest(Options& options, std::string& dscname, } // Write out each individual edit - if (verbose) { - printf("*************************Edit[%d] = %s\n", - count, edit.DebugString(hex).c_str()); + if (verbose && !json) { + printf("%s\n", edit.DebugString(hex).c_str()); + } else if (json) { + printf("%s\n", edit.DebugJSON(count, hex).c_str()); } count++; diff --git a/db/version_set.h b/db/version_set.h index 778e537f5..b728a46c4 100644 --- a/db/version_set.h +++ b/db/version_set.h @@ -572,7 +572,7 @@ class VersionSet { // printf contents (for debugging) Status DumpManifest(Options& options, std::string& manifestFileName, - bool verbose, bool hex = false); + bool verbose, bool hex = false, bool json = false); #endif // ROCKSDB_LITE diff --git a/util/event_logger.h b/util/event_logger.h index aecb6b55f..53a40c255 100644 --- a/util/event_logger.h +++ b/util/event_logger.h @@ -15,11 +15,11 @@ namespace rocksdb { -// JSONWriter doesn't support objects in arrays yet. There wasn't a need for -// that. class JSONWriter { public: - JSONWriter() : state_(kExpectKey), first_element_(true) { stream_ << "{"; } + JSONWriter() : state_(kExpectKey), first_element_(true), in_array_(false) { + stream_ << "{"; + } void AddKey(const std::string& key) { assert(state_ == kExpectKey); @@ -59,6 +59,7 @@ class JSONWriter { void StartArray() { assert(state_ == kExpectValue); state_ = kInArray; + in_array_ = true; stream_ << "["; first_element_ = true; } @@ -66,6 +67,7 @@ class JSONWriter { void EndArray() { assert(state_ == kInArray); state_ = kExpectKey; + in_array_ = false; stream_ << "]"; first_element_ = false; } @@ -83,6 +85,21 @@ class JSONWriter { first_element_ = false; } + void StartArrayedObject() { + assert(state_ == kInArray && in_array_); + state_ = kExpectValue; + if (!first_element_) { + stream_ << ", "; + } + StartObject(); + } + + void EndArrayedObject() { + assert(in_array_); + EndObject(); + state_ = kInArray; + } + std::string Get() const { return stream_.str(); } JSONWriter& operator<<(const char* val) { @@ -110,9 +127,11 @@ class JSONWriter { kExpectKey, kExpectValue, kInArray, + kInArrayedObject, }; JSONWriterState state_; bool first_element_; + bool in_array_; std::ostringstream stream_; }; diff --git a/util/ldb_cmd.cc b/util/ldb_cmd.cc index f80b1fb68..1b3dd8b8a 100644 --- a/util/ldb_cmd.cc +++ b/util/ldb_cmd.cc @@ -527,7 +527,7 @@ void DBLoaderCommand::DoCommand() { namespace { -void DumpManifestFile(std::string file, bool verbose, bool hex) { +void DumpManifestFile(std::string file, bool verbose, bool hex, bool json) { Options options; EnvOptions sopt; std::string dbname("dummy"); @@ -540,7 +540,7 @@ void DumpManifestFile(std::string file, bool verbose, bool hex) { WriteController wc(options.delayed_write_rate); WriteBuffer wb(options.db_write_buffer_size); VersionSet versions(dbname, &options, sopt, tc.get(), &wb, &wc); - Status s = versions.DumpManifest(options, file, verbose, hex); + Status s = versions.DumpManifest(options, file, verbose, hex, json); if (!s.ok()) { printf("Error in processing file %s %s\n", file.c_str(), s.ToString().c_str()); @@ -550,12 +550,14 @@ void DumpManifestFile(std::string file, bool verbose, bool hex) { } // namespace const string ManifestDumpCommand::ARG_VERBOSE = "verbose"; -const string ManifestDumpCommand::ARG_PATH = "path"; +const string ManifestDumpCommand::ARG_JSON = "json"; +const string ManifestDumpCommand::ARG_PATH = "path"; void ManifestDumpCommand::Help(string& ret) { ret.append(" "); ret.append(ManifestDumpCommand::Name()); ret.append(" [--" + ARG_VERBOSE + "]"); + ret.append(" [--" + ARG_JSON + "]"); ret.append(" [--" + ARG_PATH + "=]"); ret.append("\n"); } @@ -563,11 +565,13 @@ void ManifestDumpCommand::Help(string& ret) { ManifestDumpCommand::ManifestDumpCommand(const vector& params, const map& options, const vector& flags) : LDBCommand(options, flags, false, - BuildCmdLineOptions({ARG_VERBOSE, ARG_PATH, ARG_HEX})), + BuildCmdLineOptions({ARG_VERBOSE, ARG_PATH, ARG_HEX, ARG_JSON})), verbose_(false), + json_(false), path_("") { verbose_ = IsFlagPresent(flags, ARG_VERBOSE); + json_ = IsFlagPresent(flags, ARG_JSON); map::const_iterator itr = options.find(ARG_PATH); if (itr != options.end()) { @@ -623,7 +627,8 @@ void ManifestDumpCommand::DoCommand() { printf("Processing Manifest file %s\n", manifestfile.c_str()); } - DumpManifestFile(manifestfile, verbose_, is_key_hex_); + DumpManifestFile(manifestfile, verbose_, is_key_hex_, json_); + if (verbose_) { printf("Processing Manifest file %s done\n", manifestfile.c_str()); } @@ -2025,7 +2030,7 @@ void DBFileDumperCommand::DoCommand() { manifest_filename.resize(manifest_filename.size() - 1); string manifest_filepath = db_->GetName() + "/" + manifest_filename; std::cout << manifest_filepath << std::endl; - DumpManifestFile(manifest_filepath, false, false); + DumpManifestFile(manifest_filepath, false, false, false); std::cout << std::endl; std::cout << "SST Files" << std::endl; diff --git a/util/ldb_cmd.h b/util/ldb_cmd.h index 3e61eb894..456751ec9 100644 --- a/util/ldb_cmd.h +++ b/util/ldb_cmd.h @@ -507,9 +507,11 @@ public: private: bool verbose_; + bool json_; string path_; static const string ARG_VERBOSE; + static const string ARG_JSON; static const string ARG_PATH; };