NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Blob DB: blob_dump to show uncompressed values

Browse Source 
Summary:
Make blob_dump tool able to show uncompressed values if the blob file is compressed. Also show total compressed vs. raw size at the end if --show_summary is provided.
Closes https://github.com/facebook/rocksdb/pull/3633

Differential Revision: D7348926

Pulled By: yiwu-arbug

fbshipit-source-id: ca709cb4ed5cf6a550ff2987df8033df81516f8e
main
Yi Wu 6 years ago committed by Facebook Github Bot
parent c827b2dc2a
commit 36a9f22931
3 changed files with 99 additions and 20 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 27
      tools/blob_dump.cc
  2. 79
      utilities/blob_db/blob_dump_tool.cc
  3. 13
      utilities/blob_db/blob_dump_tool.h

27
tools/blob_dump.cc
Unescape View File

@ -27,9 +27,13 @@ int main(int argc, char** argv) {
{"file", required_argument, nullptr, 'f'},
{"show_key", optional_argument, nullptr, 'k'},
{"show_blob", optional_argument, nullptr, 'b'},
{"show_uncompressed_blob", optional_argument, nullptr, 'r'},
{"show_summary", optional_argument, nullptr, 's'},
};
DisplayType show_key = DisplayType::kRaw;
DisplayType show_blob = DisplayType::kNone;
DisplayType show_uncompressed_blob = DisplayType::kNone;
bool show_summary = false;
std::string file;
while (true) {
int c = getopt_long(argc, argv, "hk::b::f:", options, nullptr);
@ -42,7 +46,9 @@ int main(int argc, char** argv) {
fprintf(stdout,
"Usage: blob_dump --file=filename "
"[--show_key[=none|raw|hex|detail]] "
"[--show_blob[=none|raw|hex|detail]]\n");
"[--show_blob[=none|raw|hex|detail]] "
"[--show_uncompressed_blob[=none|raw|hex|detail]] "
"[--show_summary]\n");
return 0;
case 'f':
file = optarg;
@ -64,16 +70,31 @@ int main(int argc, char** argv) {
}
show_blob = display_types.at(arg_str);
} else {
show_blob = DisplayType::kDetail;
show_blob = DisplayType::kHex;
}
break;
case 'r':
if (optarg) {
if (display_types.count(arg_str) == 0) {
fprintf(stderr, "Unrecognized blob display type.\n");
return -1;
}
show_uncompressed_blob = display_types.at(arg_str);
} else {
show_uncompressed_blob = DisplayType::kHex;
}
break;
case 's':
show_summary = true;
break;
default:
fprintf(stderr, "Unrecognized option.\n");
return -1;
}
}
BlobDumpTool tool;
Status s = tool.Run(file, show_key, show_blob);
Status s =
tool.Run(file, show_key, show_blob, show_uncompressed_blob, show_summary);
if (!s.ok()) {
fprintf(stderr, "Failed: %s\n", s.ToString().c_str());
return -1;

79
utilities/blob_db/blob_dump_tool.cc
Unescape View File

@ -17,6 +17,7 @@
#include "port/port.h"
#include "rocksdb/convenience.h"
#include "rocksdb/env.h"
#include "table/format.h"
#include "util/coding.h"
#include "util/string_util.h"
@ -27,7 +28,9 @@ BlobDumpTool::BlobDumpTool()
: reader_(nullptr), buffer_(nullptr), buffer_size_(0) {}
Status BlobDumpTool::Run(const std::string& filename, DisplayType show_key,
DisplayType show_blob) {
DisplayType show_blob,
DisplayType show_uncompressed_blob,
bool show_summary) {
Status s;
Env* env = Env::Default();
s = env->FileExists(filename);
@ -50,7 +53,8 @@ Status BlobDumpTool::Run(const std::string& filename, DisplayType show_key,
reader_.reset(new RandomAccessFileReader(std::move(file), filename));
uint64_t offset = 0;
uint64_t footer_offset = 0;
s = DumpBlobLogHeader(&offset);
CompressionType compression = kNoCompression;
s = DumpBlobLogHeader(&offset, &compression);
if (!s.ok()) {
return s;
}
@ -58,14 +62,30 @@ Status BlobDumpTool::Run(const std::string& filename, DisplayType show_key,
if (!s.ok()) {
return s;
}
if (show_key != DisplayType::kNone) {
uint64_t total_records = 0;
uint64_t total_key_size = 0;
uint64_t total_blob_size = 0;
uint64_t total_uncompressed_blob_size = 0;
if (show_key != DisplayType::kNone || show_summary) {
while (offset < footer_offset) {
s = DumpRecord(show_key, show_blob, &offset);
s = DumpRecord(show_key, show_blob, show_uncompressed_blob, show_summary,
compression, &offset, &total_records, &total_key_size,
&total_blob_size, &total_uncompressed_blob_size);
if (!s.ok()) {
return s;
break;
}
}
}
if (show_summary) {
fprintf(stdout, "Summary:\n");
fprintf(stdout, " total records: %" PRIu64 "\n", total_records);
fprintf(stdout, " total key size: %" PRIu64 "\n", total_key_size);
fprintf(stdout, " total blob size: %" PRIu64 "\n", total_blob_size);
if (compression != kNoCompression) {
fprintf(stdout, " total raw blob size: %" PRIu64 "\n",
total_uncompressed_blob_size);
}
}
return s;
}
@ -89,7 +109,8 @@ Status BlobDumpTool::Read(uint64_t offset, size_t size, Slice* result) {
return s;
}
Status BlobDumpTool::DumpBlobLogHeader(uint64_t* offset) {
Status BlobDumpTool::DumpBlobLogHeader(uint64_t* offset,
CompressionType* compression) {
Slice slice;
Status s = Read(0, BlobLogHeader::kSize, &slice);
if (!s.ok()) {
@ -114,6 +135,7 @@ Status BlobDumpTool::DumpBlobLogHeader(uint64_t* offset) {
fprintf(stdout, " Expiration range : %s\n",
GetString(header.expiration_range).c_str());
*offset = BlobLogHeader::kSize;
*compression = header.compression;
return s;
}
@ -136,7 +158,7 @@ Status BlobDumpTool::DumpBlobLogFooter(uint64_t file_size,
BlobLogFooter footer;
s = footer.DecodeFrom(slice);
if (!s.ok()) {
return s;
return no_footer();
}
fprintf(stdout, "Blob log footer:\n");
fprintf(stdout, " Blob count : %" PRIu64 "\n", footer.blob_count);
@ -146,9 +168,16 @@ Status BlobDumpTool::DumpBlobLogFooter(uint64_t file_size,
}
Status BlobDumpTool::DumpRecord(DisplayType show_key, DisplayType show_blob,
uint64_t* offset) {
fprintf(stdout, "Read record with offset 0x%" PRIx64 " (%" PRIu64 "):\n",
*offset, *offset);
DisplayType show_uncompressed_blob,
bool show_summary, CompressionType compression,
uint64_t* offset, uint64_t* total_records,
uint64_t* total_key_size,
uint64_t* total_blob_size,
uint64_t* total_uncompressed_blob_size) {
if (show_key != DisplayType::kNone) {
fprintf(stdout, "Read record with offset 0x%" PRIx64 " (%" PRIu64 "):\n",
*offset, *offset);
}
Slice slice;
Status s = Read(*offset, BlobLogRecord::kHeaderSize, &slice);
if (!s.ok()) {
@ -161,14 +190,30 @@ Status BlobDumpTool::DumpRecord(DisplayType show_key, DisplayType show_blob,
}
uint64_t key_size = record.key_size;
uint64_t value_size = record.value_size;
fprintf(stdout, " key size : %" PRIu64 "\n", key_size);
fprintf(stdout, " value size : %" PRIu64 "\n", value_size);
fprintf(stdout, " expiration : %" PRIu64 "\n", record.expiration);
if (show_key != DisplayType::kNone) {
fprintf(stdout, " key size : %" PRIu64 "\n", key_size);
fprintf(stdout, " value size : %" PRIu64 "\n", value_size);
fprintf(stdout, " expiration : %" PRIu64 "\n", record.expiration);
}
*offset += BlobLogRecord::kHeaderSize;
s = Read(*offset, key_size + value_size, &slice);
if (!s.ok()) {
return s;
}
// Decompress value
std::string uncompressed_value;
if (compression != kNoCompression &&
(show_uncompressed_blob != DisplayType::kNone || show_summary)) {
BlockContents contents;
s = UncompressBlockContentsForCompressionType(
slice.data() + key_size, value_size, &contents,
2 /*compress_format_version*/, Slice(), compression,
ImmutableCFOptions(Options()));
if (!s.ok()) {
return s;
}
uncompressed_value = contents.data.ToString();
}
if (show_key != DisplayType::kNone) {
fprintf(stdout, " key : ");
DumpSlice(Slice(slice.data(), key_size), show_key);
@ -176,8 +221,16 @@ Status BlobDumpTool::DumpRecord(DisplayType show_key, DisplayType show_blob,
fprintf(stdout, " blob : ");
DumpSlice(Slice(slice.data() + key_size, value_size), show_blob);
}
if (show_uncompressed_blob != DisplayType::kNone) {
fprintf(stdout, " raw blob : ");
DumpSlice(Slice(uncompressed_value), show_uncompressed_blob);
}
}
*offset += key_size + value_size;
*total_records += 1;
*total_key_size += key_size;
*total_blob_size += value_size;
*total_uncompressed_blob_size += uncompressed_value.size();
return s;
}

13
utilities/blob_db/blob_dump_tool.h
Unescape View File

@ -27,8 +27,9 @@ class BlobDumpTool {
BlobDumpTool();
Status Run(const std::string& filename, DisplayType key_type,
DisplayType blob_type);
Status Run(const std::string& filename, DisplayType show_key,
DisplayType show_blob, DisplayType show_uncompressed_blob,
bool show_summary);
private:
std::unique_ptr<RandomAccessFileReader> reader_;
@ -36,10 +37,14 @@ class BlobDumpTool {
size_t buffer_size_;
Status Read(uint64_t offset, size_t size, Slice* result);
Status DumpBlobLogHeader(uint64_t* offset);
Status DumpBlobLogHeader(uint64_t* offset, CompressionType* compression);
Status DumpBlobLogFooter(uint64_t file_size, uint64_t* footer_offset);
Status DumpRecord(DisplayType show_key, DisplayType show_blob,
uint64_t* offset);
DisplayType show_uncompressed_blob, bool show_summary,
CompressionType compression, uint64_t* offset,
uint64_t* total_records, uint64_t* total_key_size,
uint64_t* total_blob_size,
uint64_t* total_uncompressed_blob_size);
void DumpSlice(const Slice s, DisplayType type);
template <class T>

Write Preview
Loading…
Cancel
Save