From f5cdf931a032a93688eb26532c4153b6950fd640 Mon Sep 17 00:00:00 2001 From: Abhishek Kona Date: Mon, 12 Nov 2012 16:45:45 -0800 Subject: [PATCH 1/5] LDB can read WAL. Summary: Add option to read WAL and print a summary for each record. facebook task => #1885013 E.G. Output : ./ldb dump_wal --walfile=/tmp/leveldbtest-5907/dbbench/026122.log --header Sequence,Count,ByteSize 49981,1,100033 49981,1,100033 49982,1,100033 49981,1,100033 49982,1,100033 49983,1,100033 49981,1,100033 49982,1,100033 49983,1,100033 49984,1,100033 49981,1,100033 49982,1,100033 Test Plan: Works run ./ldb read_wal --wal-file=/tmp/leveldbtest-5907/dbbench/000078.log --header Reviewers: dhruba, heyongqiang Reviewed By: dhruba CC: emayanke, leveldb, zshao Differential Revision: https://reviews.facebook.net/D6675 --- tools/ldb.cc | 12 ++++++--- util/ldb_cmd.cc | 68 +++++++++++++++++++++++++++++++++++++++++++++++++ util/ldb_cmd.h | 27 ++++++++++++++++++-- 3 files changed, 102 insertions(+), 5 deletions(-) diff --git a/tools/ldb.cc b/tools/ldb.cc index d594d500e..c4d7c714b 100644 --- a/tools/ldb.cc +++ b/tools/ldb.cc @@ -26,6 +26,10 @@ public: ret.append(" reduce_levels "); ReduceDBLevels::Help(ret); + ret.append("\n---dump_wal----:\n"); + ret.append(exec_name); + ret.append(" dump_wal "); + WALDumper::Help(ret); fprintf(stderr, "%s\n", ret.c_str()); } @@ -46,15 +50,17 @@ public: } LDBCommand* cmdObj = NULL; - if (strncmp(cmd, "compact", strlen("compact")) == 0) { + if (strcmp(cmd, "compact") == 0) { // run compactor cmdObj = new Compactor(db_name, args); - } else if (strncmp(cmd, "dump", strlen("dump")) == 0) { + } else if (strcmp(cmd, "dump") == 0) { // run dump cmdObj = new DBDumper(db_name, args); - } else if (strncmp(cmd, "reduce_levels", strlen("reduce_levels")) == 0) { + } else if (strcmp(cmd, "reduce_levels") == 0) { // reduce db levels cmdObj = new ReduceDBLevels(db_name, args); + } else if (strcmp(cmd, "dump_wal") == 0) { + cmdObj = new WALDumper(args); } else { fprintf(stderr, "Unknown command: %s\n", cmd); PrintHelp(argv[0]); diff --git a/util/ldb_cmd.cc b/util/ldb_cmd.cc index 4b266a7df..60c3f9b43 100644 --- a/util/ldb_cmd.cc +++ b/util/ldb_cmd.cc @@ -2,6 +2,11 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. + +#include "leveldb/write_batch.h" +#include "db/dbformat.h" +#include "db/log_reader.h" +#include "db/write_batch_internal.h" #include "util/ldb_cmd.h" namespace leveldb { @@ -340,4 +345,67 @@ void ReduceDBLevels::DoCommand() { } } +const char* WALDumper::WAL_FILE_ARG = "--walfile="; +WALDumper::WALDumper(std::vector& args) : + LDBCommand(args), print_header_(false) { + wal_file_.clear(); + for (unsigned int i = 0; i < args.size(); i++) { + std::string& arg = args.at(i); + if (arg.find("--header") == 0) { + print_header_ = true; + } else if (arg.find(WAL_FILE_ARG) == 0) { + wal_file_ = arg.substr(strlen(WAL_FILE_ARG)); + } else { + exec_state_ = LDBCommandExecuteResult::FAILED("Unknown argument " + arg); + } + } + if (wal_file_.empty()) { + exec_state_ = LDBCommandExecuteResult::FAILED("Argument --walfile reqd."); + } +} + +void WALDumper::Help(std::string& ret) { + ret.append("--walfile write_ahead_log "); + ret.append("[--header print's a header] "); +} + +void WALDumper::DoCommand() { + struct StdErrReporter : public log::Reader::Reporter { + virtual void Corruption(size_t bytes, const Status& s) { + std::cerr<<"Corruption detected in log file "<NewSequentialFile(wal_file_, &file); + if (!status.ok()) { + exec_state_ = LDBCommandExecuteResult::FAILED("Failed to open WAL file " + + status.ToString()); + } else { + StdErrReporter reporter; + log::Reader reader(file, &reporter, true, 0); + std::string scratch; + WriteBatch batch; + Slice record; + std::stringstream row; + if (print_header_) { + std::cout<<"Sequence,Count,ByteSize\n"; + } + while(reader.ReadRecord(&record, &scratch)) { + row.clear(); + if (record.size() < 12) { + reporter.Corruption( + record.size(), Status::Corruption("log record too small")); + } else { + WriteBatchInternal::SetContents(&batch, record); + row< #include "leveldb/db.h" +#include "leveldb/env.h" #include "leveldb/options.h" #include "leveldb/iterator.h" #include "leveldb/slice.h" @@ -41,7 +42,6 @@ public: std::string ret; switch (state_) { case EXEC_SUCCEED: - ret.append("Succeeded."); break; case EXEC_FAILED: ret.append("Failed."); @@ -97,6 +97,11 @@ public: db_(NULL) { } + LDBCommand(std::vector& args) : + db_path_(""), + db_(NULL) { + } + virtual leveldb::Options PrepareOptionsForOpenDB() { leveldb::Options opt; opt.create_if_missing = false; @@ -262,6 +267,24 @@ private: Status GetOldNumOfLevels(leveldb::Options& opt, int* levels); }; -} +class WALDumper : public LDBCommand { +public: + + WALDumper (std::vector& args); + + ~WALDumper() {} + + virtual bool NoDBOpen() { + return true; + } + + static void Help(std::string& ret); + virtual void DoCommand(); +private: + bool print_header_; + std::string wal_file_; + static const char* WAL_FILE_ARG; +}; +} #endif From de278a6de9dfbe8b126da7ef5401b3a41ee225b4 Mon Sep 17 00:00:00 2001 From: amayank Date: Fri, 16 Nov 2012 12:55:21 -0800 Subject: [PATCH 2/5] Fix a coding error in db_test.cc Summary: The new function MinLevelToCompress in db_test.cc was incomplete. It needs to tell the calling function-TEST whether the test has to be skipped or not Test Plan: make all;./db_test Reviewers: dhruba, heyongqiang Reviewed By: dhruba CC: sheki Differential Revision: https://reviews.facebook.net/D6771 --- db/db_test.cc | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/db/db_test.cc b/db/db_test.cc index 88557f6df..514fd0d5f 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -1104,7 +1104,8 @@ void MinLevelHelper(DBTest* self, Options& options) { ASSERT_EQ(self->NumTableFilesAtLevel(1), 1); } -void MinLevelToCompress(CompressionType& type, Options& options, int wbits, +// returns false if the calling-Test should be skipped +bool MinLevelToCompress(CompressionType& type, Options& options, int wbits, int lev, int strategy) { fprintf(stderr, "Test with compression options : window_bits = %d, level = %d, strategy = %d}\n", wbits, lev, strategy); options.write_buffer_size = 100<<10; //100KB @@ -1126,7 +1127,7 @@ void MinLevelToCompress(CompressionType& type, Options& options, int wbits, fprintf(stderr, "using bzip2\n"); } else { fprintf(stderr, "skipping test, compression disabled\n"); - return; + return false; } options.compression_per_level = new CompressionType[options.num_levels]; @@ -1137,11 +1138,14 @@ void MinLevelToCompress(CompressionType& type, Options& options, int wbits, for (int i = 1; i < options.num_levels; i++) { options.compression_per_level[i] = type; } + return true; } TEST(DBTest, MinLevelToCompress1) { Options options = CurrentOptions(); CompressionType type; - MinLevelToCompress(type, options, -14, -1, 0); + if (!MinLevelToCompress(type, options, -14, -1, 0)) { + return; + } Reopen(&options); MinLevelHelper(this, options); @@ -1159,7 +1163,9 @@ TEST(DBTest, MinLevelToCompress1) { TEST(DBTest, MinLevelToCompress2) { Options options = CurrentOptions(); CompressionType type; - MinLevelToCompress(type, options, 15, -1, 0); + if (!MinLevelToCompress(type, options, 15, -1, 0)) { + return; + } Reopen(&options); MinLevelHelper(this, options); From b648401adbb50f7c56f274f5b18500168be8a088 Mon Sep 17 00:00:00 2001 From: Abhishek Kona Date: Mon, 19 Nov 2012 10:46:36 -0800 Subject: [PATCH 3/5] Fix LDB dumpwal to print the messages as in the file. Summary: StringStream.clear() does not clear the stream. It sets some flags. Who knew? Fixing that is not printing the stuff again and again. Test Plan: ran it on a local db Reviewers: dhruba, emayanke Reviewed By: dhruba Differential Revision: https://reviews.facebook.net/D6795 --- util/ldb_cmd.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/util/ldb_cmd.cc b/util/ldb_cmd.cc index 60c3f9b43..acbef77ec 100644 --- a/util/ldb_cmd.cc +++ b/util/ldb_cmd.cc @@ -390,10 +390,10 @@ void WALDumper::DoCommand() { Slice record; std::stringstream row; if (print_header_) { - std::cout<<"Sequence,Count,ByteSize\n"; + std::cout<<"Sequence,Count,ByteSize,Physical Offset\n"; } while(reader.ReadRecord(&record, &scratch)) { - row.clear(); + row.str(""); if (record.size() < 12) { reporter.Corruption( record.size(), Status::Corruption("log record too small")); @@ -401,7 +401,8 @@ void WALDumper::DoCommand() { WriteBatchInternal::SetContents(&batch, record); row< Date: Mon, 19 Nov 2012 11:54:13 -0800 Subject: [PATCH 4/5] Enhance manifest_dump to print each individual edit. Summary: The manifest file contains a series of edits. If the verbose option is switched on, then print each individual edit in the manifest file. This helps in debugging. Test Plan: make clean manifest_dump Reviewers: emayanke, sheki Reviewed By: sheki CC: leveldb Differential Revision: https://reviews.facebook.net/D6807 --- db/version_set.cc | 11 ++++++++++- db/version_set.h | 3 ++- tools/manifest_dump.cc | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/db/version_set.cc b/db/version_set.cc index f8c4d71e5..0882f9f4a 100644 --- a/db/version_set.cc +++ b/db/version_set.cc @@ -1099,7 +1099,8 @@ Status VersionSet::Recover() { return s; } -Status VersionSet::DumpManifest(Options& options, std::string& dscname) { +Status VersionSet::DumpManifest(Options& options, std::string& dscname, + bool verbose) { struct LogReporter : public log::Reader::Reporter { Status* status; virtual void Corruption(size_t bytes, const Status& s) { @@ -1122,6 +1123,7 @@ Status VersionSet::DumpManifest(Options& options, std::string& dscname) { uint64_t last_sequence = 0; uint64_t log_number = 0; uint64_t prev_log_number = 0; + int count = 0; VersionSet::Builder builder(this, current_); { @@ -1142,6 +1144,13 @@ Status VersionSet::DumpManifest(Options& options, std::string& dscname) { } } + // Write out each individual edit + if (verbose) { + printf("*************************Edit[%d] = %s\n", + count, edit.DebugString().c_str()); + } + count++; + if (s.ok()) { builder.Apply(&edit); } diff --git a/db/version_set.h b/db/version_set.h index fa216623d..56b8be1a5 100644 --- a/db/version_set.h +++ b/db/version_set.h @@ -286,7 +286,8 @@ class VersionSet { const char* LevelSummary(LevelSummaryStorage* scratch) const; // printf contents (for debugging) - Status DumpManifest(Options& options, std::string& manifestFileName); + Status DumpManifest(Options& options, std::string& manifestFileName, + bool verbose); // Return a human-readable short (single-line) summary of the data size // of files per level. Uses *scratch as backing store. diff --git a/tools/manifest_dump.cc b/tools/manifest_dump.cc index d8b0a9e0c..2cdf8c7fe 100644 --- a/tools/manifest_dump.cc +++ b/tools/manifest_dump.cc @@ -60,7 +60,7 @@ int main(int argc, char** argv) { VersionSet* versions = new VersionSet(dbname, &options, tc, cmp); - Status s = versions->DumpManifest(options, file); + Status s = versions->DumpManifest(options, file, verbose); if (!s.ok()) { printf("Error in processing file %s %s\n", manifestfile.c_str(), s.ToString().c_str()); From 74054fa99369ee284fa8c8a9399504eb9c06a915 Mon Sep 17 00:00:00 2001 From: Dhruba Borthakur Date: Mon, 19 Nov 2012 13:16:46 -0800 Subject: [PATCH 5/5] Fix compilation error while compiling unit tests with OPT=-g Summary: Fix compilation error while compiling with OPT=-g Test Plan: make clean check OPT=-g Reviewers: CC: Task ID: # Blame Rev: --- db/db_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/db_test.cc b/db/db_test.cc index 514fd0d5f..2b2b9165d 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -1212,7 +1212,7 @@ static bool keep_filter(void* arg, int level, const Slice& key, } static bool delete_filter(void*argv, int level, const Slice& key, const Slice& value, Slice** new_value) { - assert(arg == NULL); + assert(argv == NULL); cfilter_count++; return true; }