From eb5a295440837c671a619675dde444a446259ee6 Mon Sep 17 00:00:00 2001 From: Maysam Yabandeh Date: Sat, 7 Apr 2018 21:46:53 -0700 Subject: [PATCH] WritePrepared Txn: add write_committed option to dump_wal Summary: Currently dump_wal cannot print the prepared records from the WAL that is generated by WRITE_PREPARED write policy since the default reaction of the handler is to return NotSupported if markers of WRITE_PREPARED are encountered. This patch enables the admin to pass --write_committed=false option, which will be accordingly passed to the handler. Note that DBFileDumperCommand and DBDumperCommand are still not updated by this patch but firstly they are not urgent and secondly we need to revise this approach later when we also add WRITE_UNPREPARED markers so I leave it for future work. Tested by running it on a WAL generated by WRITE_PREPARED: $ ./ldb dump_wal --walfile=/dev/shm/dbbench/000003.log | grep BEGIN_PREARE | head -1 1,2,70,0,BEGIN_PREARE $ ./ldb dump_wal --walfile=/dev/shm/dbbench/000003.log --write_committed=false | grep BEGIN_PREARE | head -1 1,2,70,0,BEGIN_PREARE PUT(0) : 0x30303031313330313938 PUT(0) : 0x30303032353732313935 END_PREPARE(0x74786E31313535383434323738303738363938313335312D30) Closes https://github.com/facebook/rocksdb/pull/3682 Differential Revision: D7522090 Pulled By: maysamyabandeh fbshipit-source-id: a0332207261c61e18b2f9dfbe9feecd9a1339aca --- include/rocksdb/utilities/ldb_cmd.h | 1 + tools/ldb_cmd.cc | 41 ++++++++++++++++++++--------- tools/ldb_cmd_impl.h | 2 ++ 3 files changed, 31 insertions(+), 13 deletions(-) diff --git a/include/rocksdb/utilities/ldb_cmd.h b/include/rocksdb/utilities/ldb_cmd.h index b9eb1035f..4d91f0a66 100644 --- a/include/rocksdb/utilities/ldb_cmd.h +++ b/include/rocksdb/utilities/ldb_cmd.h @@ -215,6 +215,7 @@ class LDBCommand { LDBOptions ldb_options_; private: + friend class WALDumperCommand; /** * Interpret command line options and flags to determine if the key * should be input/output in hex. diff --git a/tools/ldb_cmd.cc b/tools/ldb_cmd.cc index a22e61359..15c6b074d 100644 --- a/tools/ldb_cmd.cc +++ b/tools/ldb_cmd.cc @@ -82,7 +82,7 @@ const char* LDBCommand::DELIM = " ==> "; namespace { void DumpWalFile(std::string wal_file, bool print_header, bool print_values, - LDBCommandExecuteResult* exec_state); + bool is_write_committed, LDBCommandExecuteResult* exec_state); void DumpSstFile(std::string filename, bool output_hex, bool show_properties); }; @@ -1411,8 +1411,9 @@ void DBDumperCommand::DoCommand() { switch (type) { case kLogFile: + // TODO(myabandeh): allow configuring is_write_commited DumpWalFile(path_, /* print_header_ */ true, /* print_values_ */ true, - &exec_state_); + true /* is_write_commited */, &exec_state_); break; case kTableFile: DumpSstFile(path_, is_key_hex_, /* show_properties */ true); @@ -1853,10 +1854,12 @@ struct StdErrReporter : public log::Reader::Reporter { class InMemoryHandler : public WriteBatch::Handler { public: - InMemoryHandler(std::stringstream& row, bool print_values) - : Handler(), row_(row) { - print_values_ = print_values; - } + InMemoryHandler(std::stringstream& row, bool print_values, + bool write_after_commit = false) + : Handler(), + row_(row), + print_values_(print_values), + write_after_commit_(write_after_commit) {} void commonPutMerge(const Slice& key, const Slice& value) { std::string k = LDBCommand::StringToHex(key.ToString()); @@ -1934,13 +1937,17 @@ class InMemoryHandler : public WriteBatch::Handler { virtual ~InMemoryHandler() {} + protected: + virtual bool WriteAfterCommit() const override { return write_after_commit_; } + private: std::stringstream& row_; bool print_values_; + bool write_after_commit_; }; void DumpWalFile(std::string wal_file, bool print_header, bool print_values, - LDBCommandExecuteResult* exec_state) { + bool is_write_committed, LDBCommandExecuteResult* exec_state) { Env* env_ = Env::Default(); EnvOptions soptions; unique_ptr wal_file_reader; @@ -2000,7 +2007,7 @@ void DumpWalFile(std::string wal_file, bool print_header, bool print_values, row << WriteBatchInternal::Count(&batch) << ","; row << WriteBatchInternal::ByteSize(&batch) << ","; row << reader.LastRecordOffset() << ","; - InMemoryHandler handler(row, print_values); + InMemoryHandler handler(row, print_values, is_write_committed); batch.Iterate(&handler); row << "\n"; } @@ -2012,6 +2019,7 @@ void DumpWalFile(std::string wal_file, bool print_header, bool print_values, } // namespace const std::string WALDumperCommand::ARG_WAL_FILE = "walfile"; +const std::string WALDumperCommand::ARG_WRITE_COMMITTED = "write_committed"; const std::string WALDumperCommand::ARG_PRINT_VALUE = "print_value"; const std::string WALDumperCommand::ARG_PRINT_HEADER = "header"; @@ -2020,10 +2028,11 @@ WALDumperCommand::WALDumperCommand( const std::map& options, const std::vector& flags) : LDBCommand(options, flags, true, - BuildCmdLineOptions( - {ARG_WAL_FILE, ARG_PRINT_HEADER, ARG_PRINT_VALUE})), + BuildCmdLineOptions({ARG_WAL_FILE, ARG_WRITE_COMMITTED, + ARG_PRINT_HEADER, ARG_PRINT_VALUE})), print_header_(false), - print_values_(false) { + print_values_(false), + is_write_committed_(false) { wal_file_.clear(); std::map::const_iterator itr = @@ -2035,6 +2044,8 @@ WALDumperCommand::WALDumperCommand( print_header_ = IsFlagPresent(flags, ARG_PRINT_HEADER); print_values_ = IsFlagPresent(flags, ARG_PRINT_VALUE); + is_write_committed_ = ParseBooleanOption(options, ARG_WRITE_COMMITTED, true); + if (wal_file_.empty()) { exec_state_ = LDBCommandExecuteResult::Failed("Argument " + ARG_WAL_FILE + " must be specified."); @@ -2047,11 +2058,13 @@ void WALDumperCommand::Help(std::string& ret) { ret.append(" --" + ARG_WAL_FILE + "="); ret.append(" [--" + ARG_PRINT_HEADER + "] "); ret.append(" [--" + ARG_PRINT_VALUE + "] "); + ret.append(" [--" + ARG_WRITE_COMMITTED + "=true|false] "); ret.append("\n"); } void WALDumperCommand::DoCommand() { - DumpWalFile(wal_file_, print_header_, print_values_, &exec_state_); + DumpWalFile(wal_file_, print_header_, print_values_, is_write_committed_, + &exec_state_); } // ---------------------------------------------------------------------------- @@ -2912,7 +2925,9 @@ void DBFileDumperCommand::DoCommand() { // TODO(qyang): option.wal_dir should be passed into ldb command std::string filename = db_->GetOptions().wal_dir + wal->PathName(); std::cout << filename << std::endl; - DumpWalFile(filename, true, true, &exec_state_); + // TODO(myabandeh): allow configuring is_write_commited + DumpWalFile(filename, true, true, true /* is_write_commited */, + &exec_state_); } } } diff --git a/tools/ldb_cmd_impl.h b/tools/ldb_cmd_impl.h index 91afd2674..e49e851b3 100644 --- a/tools/ldb_cmd_impl.h +++ b/tools/ldb_cmd_impl.h @@ -276,8 +276,10 @@ class WALDumperCommand : public LDBCommand { bool print_header_; std::string wal_file_; bool print_values_; + bool is_write_committed_; // default will be set to true static const std::string ARG_WAL_FILE; + static const std::string ARG_WRITE_COMMITTED; static const std::string ARG_PRINT_HEADER; static const std::string ARG_PRINT_VALUE; };