From 2aa514ec8c1b5aabe23e539b90cf6a316c842a83 Mon Sep 17 00:00:00 2001 From: Dhruba Borthakur Date: Fri, 17 Aug 2012 10:48:40 -0700 Subject: [PATCH] Utility to dump manifest contents. Summary: ./manifest_dump --file=/tmp/dbbench/MANIFEST-000002 Output looks like manifest_file_number 30 next_file_number 31 last_sequence 388082 log_number 28 prev_log_number 0 --- level 0 --- --- level 1 --- --- level 2 --- 5:3244155['0000000000000000' @ 1 : 1 .. '0000000000028220' @ 28221 : 1] 7:3244177['0000000000028221' @ 28222 : 1 .. '0000000000056441' @ 56442 : 1] 9:3244156['0000000000056442' @ 56443 : 1 .. '0000000000084662' @ 84663 : 1] 11:3244178['0000000000084663' @ 84664 : 1 .. '0000000000112883' @ 112884 : 1] 13:3244158['0000000000112884' @ 112885 : 1 .. '0000000000141104' @ 141105 : 1] 15:3244176['0000000000141105' @ 141106 : 1 .. '0000000000169325' @ 169326 : 1] 17:3244156['0000000000169326' @ 169327 : 1 .. '0000000000197546' @ 197547 : 1] 19:3244178['0000000000197547' @ 197548 : 1 .. '0000000000225767' @ 225768 : 1] 21:3244155['0000000000225768' @ 225769 : 1 .. '0000000000253988' @ 253989 : 1] 23:3244179['0000000000253989' @ 253990 : 1 .. '0000000000282209' @ 282210 : 1] 25:3244157['0000000000282210' @ 282211 : 1 .. '0000000000310430' @ 310431 : 1] 27:3244176['0000000000310431' @ 310432 : 1 .. '0000000000338651' @ 338652 : 1] 29:3244156['0000000000338652' @ 338653 : 1 .. '0000000000366872' @ 366873 : 1] --- level 3 --- --- level 4 --- --- level 5 --- --- level 6 --- Test Plan: run on test directory created by dbbench Reviewers: heyongqiang Reviewed By: heyongqiang CC: hustliubo Differential Revision: https://reviews.facebook.net/D4743 --- Makefile | 10 +++- db/version_set.cc | 113 +++++++++++++++++++++++++++++++++++++++++ db/version_set.h | 3 ++ tools/manifest_dump.cc | 71 ++++++++++++++++++++++++++ 4 files changed, 195 insertions(+), 2 deletions(-) create mode 100644 tools/manifest_dump.cc diff --git a/Makefile b/Makefile index e56e4201a..5ddbd5429 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,10 @@ TESTS = \ version_set_test \ write_batch_test -PROGRAMS = db_bench $(TESTS) +TOOLS = \ + manifest_dump + +PROGRAMS = db_bench $(TESTS) $(TOOLS) BENCHMARKS = db_bench_sqlite3 db_bench_tree_db LIBRARY = libleveldb.a @@ -81,7 +84,7 @@ endif all: $(SHARED) $(LIBRARY) $(THRIFTSERVER) -check: all $(PROGRAMS) $(TESTS) +check: all $(PROGRAMS) $(TESTS) $(TOOLS) for t in $(TESTS); do echo "***** Running $$t"; ./$$t || exit 1; done clean: @@ -168,6 +171,9 @@ leveldb_server: thrift/server.o $(LIBRARY) leveldb_server_test: thrift/test/simpletest.o $(LIBRARY) $(CXX) thrift/test/simpletest.o $(LIBRARY) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) +manifest_dump: tools/manifest_dump.o $(LIBOBJECTS) + $(CXX) tools/manifest_dump.o $(LIBOBJECTS) -o $@ $(LDFLAGS) + ifeq ($(PLATFORM), IOS) # For iOS, create universal object files to be used on both the simulator and # a device. diff --git a/db/version_set.cc b/db/version_set.cc index a51500c46..c75959aaf 100644 --- a/db/version_set.cc +++ b/db/version_set.cc @@ -939,6 +939,119 @@ Status VersionSet::Recover() { return s; } +Status VersionSet::DumpManifest(Options& options, std::string& dscname) { + struct LogReporter : public log::Reader::Reporter { + Status* status; + virtual void Corruption(size_t bytes, const Status& s) { + if (this->status->ok()) *this->status = s; + } + }; + + // Open the specified manifest file. + SequentialFile* file; + Status s = options.env->NewSequentialFile(dscname, &file); + if (!s.ok()) { + return s; + } + + bool have_log_number = false; + bool have_prev_log_number = false; + bool have_next_file = false; + bool have_last_sequence = false; + uint64_t next_file = 0; + uint64_t last_sequence = 0; + uint64_t log_number = 0; + uint64_t prev_log_number = 0; + VersionSet::Builder builder(this, current_); + + { + LogReporter reporter; + reporter.status = &s; + log::Reader reader(file, &reporter, true/*checksum*/, 0/*initial_offset*/); + Slice record; + std::string scratch; + while (reader.ReadRecord(&record, &scratch) && s.ok()) { + VersionEdit edit(NumberLevels()); + s = edit.DecodeFrom(record); + if (s.ok()) { + if (edit.has_comparator_ && + edit.comparator_ != icmp_.user_comparator()->Name()) { + s = Status::InvalidArgument( + edit.comparator_ + "does not match existing comparator ", + icmp_.user_comparator()->Name()); + } + } + + if (s.ok()) { + builder.Apply(&edit); + } + + if (edit.has_log_number_) { + log_number = edit.log_number_; + have_log_number = true; + } + + if (edit.has_prev_log_number_) { + prev_log_number = edit.prev_log_number_; + have_prev_log_number = true; + } + + if (edit.has_next_file_number_) { + next_file = edit.next_file_number_; + have_next_file = true; + } + + if (edit.has_last_sequence_) { + last_sequence = edit.last_sequence_; + have_last_sequence = true; + } + } + } + delete file; + file = NULL; + + if (s.ok()) { + if (!have_next_file) { + s = Status::Corruption("no meta-nextfile entry in descriptor"); + printf("no meta-nextfile entry in descriptor"); + } else if (!have_log_number) { + s = Status::Corruption("no meta-lognumber entry in descriptor"); + printf("no meta-lognumber entry in descriptor"); + } else if (!have_last_sequence) { + printf("no last-sequence-number entry in descriptor"); + s = Status::Corruption("no last-sequence-number entry in descriptor"); + } + + if (!have_prev_log_number) { + prev_log_number = 0; + } + + MarkFileNumberUsed(prev_log_number); + MarkFileNumberUsed(log_number); + } + + if (s.ok()) { + Version* v = new Version(this); + builder.SaveTo(v); + // Install recovered version + Finalize(v); + AppendVersion(v); + manifest_file_number_ = next_file; + next_file_number_ = next_file + 1; + last_sequence_ = last_sequence; + log_number_ = log_number; + prev_log_number_ = prev_log_number; + + printf("manifest_file_number %d next_file_number %d last_sequence %d log_number %d prev_log_number %d\n", + manifest_file_number_, next_file_number_, + last_sequence, log_number, prev_log_number); + printf("%s \n", v->DebugString().c_str()); + } + + + return s; +} + void VersionSet::MarkFileNumberUsed(uint64_t number) { if (next_file_number_ <= number) { next_file_number_ = number + 1; diff --git a/db/version_set.h b/db/version_set.h index a066f01c9..c997724c0 100644 --- a/db/version_set.h +++ b/db/version_set.h @@ -238,6 +238,9 @@ class VersionSet { }; const char* LevelSummary(LevelSummaryStorage* scratch) const; + // printf contents (for debugging) + Status DumpManifest(Options& options, std::string& manifestFileName); + private: class Builder; diff --git a/tools/manifest_dump.cc b/tools/manifest_dump.cc new file mode 100644 index 000000000..5f4d272c6 --- /dev/null +++ b/tools/manifest_dump.cc @@ -0,0 +1,71 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/version_set.h" + +#include +#include +#include "db/filename.h" +#include "db/log_reader.h" +#include "db/log_writer.h" +#include "db/memtable.h" +#include "db/table_cache.h" +#include "leveldb/env.h" +#include "leveldb/table_builder.h" +#include "table/merger.h" +#include "table/two_level_iterator.h" +#include "util/coding.h" +#include "util/logging.h" + +static int verbose = 0; + +using namespace leveldb; + +// +// Takes a manifest file and dumps out all metedata +// +int main(int argc, char** argv) { + + // parse command line options + int n; + char junk; + int foundfile = 0; + std::string manifestfile; + for (int i = 1; i < argc; i++) { + std::string param(argv[i]); + if ((n = param.find("--file=")) != std::string::npos) { + manifestfile = param.substr(strlen("--file=")); + foundfile = 1; + } else if (sscanf(argv[i], "--verbose=%d%c", &n, &junk) == 1 && + (n == 0 || n == 1)) { + verbose = n; + } + } + if (!foundfile) { + fprintf(stderr, "%s [--verbose=0|1] [--file=pathname of manifest file\n", + argv[0]); + abort(); + } + + if (verbose) { + printf("Processing Manifest file %s\n", manifestfile.c_str()); + } + + Options options; + std::string file(manifestfile); + std::string dbname("dummy"); + TableCache* tc = new TableCache(dbname, &options, 10); + const InternalKeyComparator* cmp = new InternalKeyComparator(options.comparator); + + VersionSet* versions = new VersionSet(dbname, &options, + tc, cmp); + Status s = versions->DumpManifest(options, file); + if (!s.ok()) { + printf("Error in processing file %s %s\n", manifestfile.c_str(), + s.ToString().c_str()); + } + if (verbose) { + printf("Processing Manifest file %s done\n", manifestfile.c_str()); + } +}