Add IO Tracer Parser (#7333)
Summary: Implement a parsing tool io_tracer_parser that takes IO trace file (binary file) with command line argument --io_trace_file and output file with --output_file and dumps the IO trace records in outputfile in human readable form. Also added unit test cases that generates IO trace records and calls io_tracer_parse to parse those records. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7333 Test Plan: make check -j64, Add unit test cases. Reviewed By: anand1976 Differential Revision: D23772360 Pulled By: akankshamahajan15 fbshipit-source-id: 9c20519c189362e6663352d08863326f3e496271main
Akanksha Mahajan
4 years ago
committed by
Facebook GitHub Bot
parent
31d1cea4f3
commit
98ac6b646a
@ -0,0 +1,25 @@ |
||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
//
|
||||
#ifndef ROCKSDB_LITE |
||||
#ifndef GFLAGS |
||||
#include <cstdio> |
||||
int main() { |
||||
fprintf(stderr, "Please install gflags to run rocksdb tools\n"); |
||||
return 1; |
||||
} |
||||
#else // GFLAGS
|
||||
#include "tools/io_tracer_parser_tool.h" |
||||
int main(int argc, char** argv) { |
||||
return ROCKSDB_NAMESPACE::io_tracer_parser(argc, argv); |
||||
} |
||||
#endif // GFLAGS
|
||||
#else // ROCKSDB_LITE
|
||||
#include <stdio.h> |
||||
int main(int /*argc*/, char** /*argv*/) { |
||||
fprintf(stderr, "Not supported in lite mode.\n"); |
||||
return 1; |
||||
} |
||||
#endif // ROCKSDB_LITE
|
||||
@ -0,0 +1,187 @@ |
||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
//
|
||||
|
||||
#ifndef ROCKSDB_LITE |
||||
#ifndef GFLAGS |
||||
#include <cstdio> |
||||
int main() { |
||||
fprintf(stderr, "Please install gflags to run io_tracer_parser_test\n"); |
||||
return 1; |
||||
} |
||||
#else |
||||
|
||||
#include <string> |
||||
#include <vector> |
||||
|
||||
#include "rocksdb/env.h" |
||||
#include "rocksdb/status.h" |
||||
#include "test_util/testharness.h" |
||||
#include "test_util/testutil.h" |
||||
#include "tools/io_tracer_parser_tool.h" |
||||
|
||||
namespace ROCKSDB_NAMESPACE { |
||||
|
||||
namespace { |
||||
const int kMaxArgCount = 100; |
||||
const size_t kArgBufferSize = 100000; |
||||
} // namespace
|
||||
|
||||
class IOTracerParserTest : public testing::Test { |
||||
public: |
||||
IOTracerParserTest() { |
||||
test_path_ = test::PerThreadDBPath("io_tracer_parser_test"); |
||||
env_ = ROCKSDB_NAMESPACE::Env::Default(); |
||||
EXPECT_OK(env_->CreateDirIfMissing(test_path_)); |
||||
trace_file_path_ = test_path_ + "/io_trace_file"; |
||||
dbname_ = test_path_ + "/db"; |
||||
Options options; |
||||
options.create_if_missing = true; |
||||
EXPECT_OK(DB::Open(options, dbname_, &db_)); |
||||
} |
||||
|
||||
~IOTracerParserTest() { |
||||
if (env_->FileExists(trace_file_path_).ok()) { |
||||
EXPECT_OK(env_->DeleteFile(trace_file_path_)); |
||||
} |
||||
if (db_ != nullptr) { |
||||
Options options; |
||||
options.env = env_; |
||||
delete db_; |
||||
db_ = nullptr; |
||||
EXPECT_OK(DestroyDB(dbname_, options)); |
||||
} |
||||
EXPECT_OK(env_->DeleteDir(test_path_)); |
||||
} |
||||
|
||||
void GenerateIOTrace() { |
||||
WriteOptions write_opt; |
||||
TraceOptions trace_opt; |
||||
std::unique_ptr<TraceWriter> trace_writer; |
||||
|
||||
ASSERT_OK(NewFileTraceWriter(env_, env_options_, trace_file_path_, |
||||
&trace_writer)); |
||||
|
||||
ASSERT_OK(db_->StartIOTrace(env_, trace_opt, std::move(trace_writer))); |
||||
|
||||
for (int i = 0; i < 10; i++) { |
||||
ASSERT_OK(db_->Put(write_opt, "key_" + std::to_string(i), |
||||
"value_" + std::to_string(i))); |
||||
ASSERT_OK(db_->Flush(FlushOptions())); |
||||
} |
||||
|
||||
ASSERT_OK(db_->EndIOTrace()); |
||||
ASSERT_OK(env_->FileExists(trace_file_path_)); |
||||
} |
||||
|
||||
void RunIOTracerParserTool() { |
||||
std::vector<std::string> params = {"./io_tracer_parser", |
||||
"-io_trace_file=" + trace_file_path_}; |
||||
|
||||
char arg_buffer[kArgBufferSize]; |
||||
char* argv[kMaxArgCount]; |
||||
int argc = 0; |
||||
int cursor = 0; |
||||
for (const auto& arg : params) { |
||||
ASSERT_LE(cursor + arg.size() + 1, kArgBufferSize); |
||||
ASSERT_LE(argc + 1, kMaxArgCount); |
||||
|
||||
snprintf(arg_buffer + cursor, arg.size() + 1, "%s", arg.c_str()); |
||||
|
||||
argv[argc++] = arg_buffer + cursor; |
||||
cursor += static_cast<int>(arg.size()) + 1; |
||||
} |
||||
ASSERT_EQ(0, ROCKSDB_NAMESPACE::io_tracer_parser(argc, argv)); |
||||
} |
||||
|
||||
DB* db_; |
||||
Env* env_; |
||||
EnvOptions env_options_; |
||||
std::string trace_file_path_; |
||||
std::string output_file_; |
||||
std::string test_path_; |
||||
std::string dbname_; |
||||
}; |
||||
|
||||
TEST_F(IOTracerParserTest, InvalidArguments) { |
||||
{ |
||||
std::vector<std::string> params = {"./io_tracer_parser"}; |
||||
char arg_buffer[kArgBufferSize]; |
||||
char* argv[kMaxArgCount]; |
||||
int argc = 0; |
||||
int cursor = 0; |
||||
for (const auto& arg : params) { |
||||
ASSERT_LE(cursor + arg.size() + 1, kArgBufferSize); |
||||
ASSERT_LE(argc + 1, kMaxArgCount); |
||||
|
||||
snprintf(arg_buffer + cursor, arg.size() + 1, "%s", arg.c_str()); |
||||
|
||||
argv[argc++] = arg_buffer + cursor; |
||||
cursor += static_cast<int>(arg.size()) + 1; |
||||
} |
||||
ASSERT_EQ(1, ROCKSDB_NAMESPACE::io_tracer_parser(argc, argv)); |
||||
} |
||||
} |
||||
|
||||
TEST_F(IOTracerParserTest, DumpAndParseIOTraceRecords) { |
||||
GenerateIOTrace(); |
||||
RunIOTracerParserTool(); |
||||
} |
||||
|
||||
TEST_F(IOTracerParserTest, NoRecordingAfterEndIOTrace) { |
||||
uint64_t file_size = 0; |
||||
// Generate IO trace records and parse them.
|
||||
{ |
||||
GenerateIOTrace(); |
||||
RunIOTracerParserTool(); |
||||
ASSERT_OK(env_->GetFileSize(trace_file_path_, &file_size)); |
||||
} |
||||
// Once DB::EndIOTrace is invoked in GenerateIOTrace(), no new records should
|
||||
// be appended.
|
||||
{ |
||||
WriteOptions write_opt; |
||||
for (int i = 10; i < 20; i++) { |
||||
ASSERT_OK(db_->Put(write_opt, "key_" + std::to_string(i), |
||||
"value_" + std::to_string(i))); |
||||
ASSERT_OK(db_->Flush(FlushOptions())); |
||||
} |
||||
} |
||||
|
||||
uint64_t new_file_size = 0; |
||||
ASSERT_OK(env_->GetFileSize(trace_file_path_, &new_file_size)); |
||||
ASSERT_EQ(file_size, new_file_size); |
||||
} |
||||
|
||||
TEST_F(IOTracerParserTest, NoRecordingBeforeStartIOTrace) { |
||||
{ |
||||
WriteOptions write_opt; |
||||
for (int i = 10; i < 20; i++) { |
||||
ASSERT_OK(db_->Put(write_opt, "key_" + std::to_string(i), |
||||
"value_" + std::to_string(i))); |
||||
ASSERT_OK(db_->Flush(FlushOptions())); |
||||
} |
||||
// IO trace file doesn't exist
|
||||
ASSERT_NOK(env_->FileExists(trace_file_path_)); |
||||
} |
||||
// Generate IO trace records and parse them.
|
||||
{ |
||||
GenerateIOTrace(); |
||||
RunIOTracerParserTool(); |
||||
} |
||||
} |
||||
} // namespace ROCKSDB_NAMESPACE
|
||||
|
||||
int main(int argc, char** argv) { |
||||
::testing::InitGoogleTest(&argc, argv); |
||||
return RUN_ALL_TESTS(); |
||||
} |
||||
#endif // GFLAGS
|
||||
#else |
||||
#include <stdio.h> |
||||
int main(int /*argc*/, char** /*argv*/) { |
||||
fprintf(stderr, "io_tracer_parser_test is not supported in ROCKSDB_LITE\n"); |
||||
return 0; |
||||
} |
||||
#endif // ROCKSDB_LITE
|
||||
@ -0,0 +1,120 @@ |
||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
|
||||
#ifndef ROCKSDB_LITE |
||||
#ifdef GFLAGS |
||||
#include "tools/io_tracer_parser_tool.h" |
||||
|
||||
#include <cinttypes> |
||||
#include <cstdio> |
||||
#include <iomanip> |
||||
#include <memory> |
||||
#include <sstream> |
||||
|
||||
#include "port/lang.h" |
||||
#include "trace_replay/io_tracer.h" |
||||
#include "util/gflags_compat.h" |
||||
|
||||
using GFLAGS_NAMESPACE::ParseCommandLineFlags; |
||||
|
||||
DEFINE_string(io_trace_file, "", "The IO trace file path."); |
||||
|
||||
namespace ROCKSDB_NAMESPACE { |
||||
|
||||
IOTraceRecordParser::IOTraceRecordParser(const std::string& input_file) |
||||
: input_file_(input_file) {} |
||||
|
||||
void IOTraceRecordParser::PrintHumanReadableHeader( |
||||
const IOTraceHeader& header) { |
||||
std::stringstream ss; |
||||
ss << "Start Time: " << header.start_time |
||||
<< "\nRocksDB Major Version: " << header.rocksdb_major_version |
||||
<< "\nRocksDB Minor Version: " << header.rocksdb_minor_version << "\n"; |
||||
fprintf(stdout, "%s", ss.str().c_str()); |
||||
} |
||||
|
||||
void IOTraceRecordParser::PrintHumanReadableIOTraceRecord( |
||||
const IOTraceRecord& record) { |
||||
std::stringstream ss; |
||||
ss << "Access Time : " << std::setw(17) << std::left |
||||
<< record.access_timestamp << ", File Operation: " << std::setw(18) |
||||
<< std::left << record.file_operation.c_str() |
||||
<< ", Latency: " << std::setw(9) << std::left << record.latency |
||||
<< ", IO Status: " << record.io_status.c_str(); |
||||
|
||||
switch (record.trace_type) { |
||||
case TraceType::kIOGeneral: |
||||
break; |
||||
case TraceType::kIOFileNameAndFileSize: |
||||
ss << ", File Size: " << record.file_size; |
||||
FALLTHROUGH_INTENDED; |
||||
case TraceType::kIOFileName: { |
||||
if (!record.file_name.empty()) { |
||||
ss << ", File Name: " << record.file_name.c_str(); |
||||
} |
||||
break; |
||||
} |
||||
case TraceType::kIOLenAndOffset: |
||||
ss << ", Offset: " << record.offset; |
||||
FALLTHROUGH_INTENDED; |
||||
case TraceType::kIOLen: |
||||
ss << ", Length: " << record.len; |
||||
break; |
||||
default: |
||||
assert(false); |
||||
} |
||||
ss << "\n"; |
||||
fprintf(stdout, "%s", ss.str().c_str()); |
||||
} |
||||
|
||||
int IOTraceRecordParser::ReadIOTraceRecords() { |
||||
Status status; |
||||
Env* env(Env::Default()); |
||||
std::unique_ptr<TraceReader> trace_reader; |
||||
std::unique_ptr<IOTraceReader> io_trace_reader; |
||||
|
||||
status = NewFileTraceReader(env, EnvOptions(), input_file_, &trace_reader); |
||||
if (!status.ok()) { |
||||
fprintf(stderr, "%s: %s\n", input_file_.c_str(), status.ToString().c_str()); |
||||
return 1; |
||||
} |
||||
io_trace_reader.reset(new IOTraceReader(std::move(trace_reader))); |
||||
|
||||
// Read the header and dump it in a file.
|
||||
IOTraceHeader header; |
||||
status = io_trace_reader->ReadHeader(&header); |
||||
if (!status.ok()) { |
||||
fprintf(stderr, "%s: %s\n", input_file_.c_str(), status.ToString().c_str()); |
||||
return 1; |
||||
} |
||||
PrintHumanReadableHeader(header); |
||||
|
||||
// Read the records one by one and print them in human readable format.
|
||||
while (status.ok()) { |
||||
IOTraceRecord record; |
||||
status = io_trace_reader->ReadIOOp(&record); |
||||
if (!status.ok()) { |
||||
break; |
||||
} |
||||
PrintHumanReadableIOTraceRecord(record); |
||||
} |
||||
return 0; |
||||
} |
||||
|
||||
int io_tracer_parser(int argc, char** argv) { |
||||
ParseCommandLineFlags(&argc, &argv, true); |
||||
|
||||
if (FLAGS_io_trace_file.empty()) { |
||||
fprintf(stderr, "IO Trace file path is empty\n"); |
||||
return 1; |
||||
} |
||||
|
||||
IOTraceRecordParser io_tracer_parser(FLAGS_io_trace_file); |
||||
return io_tracer_parser.ReadIOTraceRecords(); |
||||
} |
||||
|
||||
} // namespace ROCKSDB_NAMESPACE
|
||||
#endif // GFLAGS
|
||||
#endif // ROCKSDB_LITE
|
||||
@ -0,0 +1,40 @@ |
||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
|
||||
#ifndef ROCKSDB_LITE |
||||
#pragma once |
||||
|
||||
#include <memory> |
||||
|
||||
#include "rocksdb/env.h" |
||||
#include "rocksdb/status.h" |
||||
|
||||
namespace ROCKSDB_NAMESPACE { |
||||
|
||||
struct IOTraceHeader; |
||||
struct IOTraceRecord; |
||||
|
||||
// IOTraceRecordParser class reads the IO trace file (in binary format) and
|
||||
// dumps the human readable records in output_file_.
|
||||
class IOTraceRecordParser { |
||||
public: |
||||
explicit IOTraceRecordParser(const std::string& input_file); |
||||
|
||||
// ReadIOTraceRecords reads the binary trace file records one by one and
|
||||
// invoke PrintHumanReadableIOTraceRecord to dump the records in output_file_.
|
||||
int ReadIOTraceRecords(); |
||||
|
||||
private: |
||||
void PrintHumanReadableHeader(const IOTraceHeader& header); |
||||
void PrintHumanReadableIOTraceRecord(const IOTraceRecord& record); |
||||
|
||||
// Binary file that contains IO trace records.
|
||||
std::string input_file_; |
||||
}; |
||||
|
||||
int io_tracer_parser(int argc, char** argv); |
||||
|
||||
} // namespace ROCKSDB_NAMESPACE
|
||||
#endif // ROCKSDB_LITE
|
||||
Loading…
Reference in new issue