NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update db_bench and sst_dump to test with block cache mid-point inser?

Browse Source 
Summary:
?tion

Add flags in db_bench to test with block cache mid-point insertion.
Also update sst_dump to dump total block sizes of each type. I find it
useful to look at these test db stats and I don't know if we have them
elsewhere.
Closes https://github.com/facebook/rocksdb/pull/1706

Differential Revision: D4355812

Pulled By: yiwu-arbug

fbshipit-source-id: 3e4a348
main
Yi Wu 8 years ago committed by Facebook Github Bot
parent 653ac1f9c6
commit 640d724808
2 changed files with 68 additions and 26 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 13
      tools/db_bench_tool.cc
  2. 81
      tools/sst_dump_tool.cc

13
tools/db_bench_tool.cc
Unescape View File

@ -362,6 +362,11 @@ DEFINE_int32(cache_numshardbits, 6,
" is 2 ** cache_numshardbits. Negative means use default settings." " is 2 ** cache_numshardbits. Negative means use default settings."
" This is applied only if FLAGS_cache_size is non-negative."); " This is applied only if FLAGS_cache_size is non-negative.");
DEFINE_double(cache_high_pri_pool_ratio, 0.0,
"Ratio of block cache reserve for high pri blocks. "
"If > 0.0, we also enable "
"cache_index_and_filter_blocks_with_high_priority.");
DEFINE_bool(use_clock_cache, false, DEFINE_bool(use_clock_cache, false,
"Replace default LRU block cache with clock cache."); "Replace default LRU block cache with clock cache.");
@ -1993,7 +1998,9 @@ class Benchmark {
} }
return cache; return cache;
} else { } else {
return NewLRUCache((size_t)capacity, FLAGS_cache_numshardbits); return NewLRUCache((size_t)capacity, FLAGS_cache_numshardbits,
false /*strict_capacity_limit*/,
FLAGS_cache_high_pri_pool_ratio);
} }
} }
@ -2884,6 +2891,10 @@ class Benchmark {
FLAGS_cache_index_and_filter_blocks; FLAGS_cache_index_and_filter_blocks;
block_based_options.pin_l0_filter_and_index_blocks_in_cache = block_based_options.pin_l0_filter_and_index_blocks_in_cache =
FLAGS_pin_l0_filter_and_index_blocks_in_cache; FLAGS_pin_l0_filter_and_index_blocks_in_cache;
if (FLAGS_cache_high_pri_pool_ratio > 1e-6) { // > 0.0 + eps
block_based_options.cache_index_and_filter_blocks_with_high_priority =
true;
}
block_based_options.block_cache = cache_; block_based_options.block_cache = cache_;
block_based_options.block_cache_compressed = compressed_cache_; block_based_options.block_cache_compressed = compressed_cache_;
block_based_options.block_size = FLAGS_block_size; block_based_options.block_size = FLAGS_block_size;

81
tools/sst_dump_tool.cc
Unescape View File

@ -408,11 +408,17 @@ int SSTDumpTool::Run(int argc, char** argv) {
bool has_to = false; bool has_to = false;
bool show_properties = false; bool show_properties = false;
bool show_compression_sizes = false; bool show_compression_sizes = false;
bool show_summary = false;
bool set_block_size = false; bool set_block_size = false;
std::string from_key; std::string from_key;
std::string to_key; std::string to_key;
std::string block_size_str; std::string block_size_str;
size_t block_size; size_t block_size;
uint64_t total_num_files = 0;
uint64_t total_num_data_blocks = 0;
uint64_t total_data_block_size = 0;
uint64_t total_index_block_size = 0;
uint64_t total_filter_block_size = 0;
for (int i = 1; i < argc; i++) { for (int i = 1; i < argc; i++) {
if (strncmp(argv[i], "--file=", 7) == 0) { if (strncmp(argv[i], "--file=", 7) == 0) {
dir_or_file = argv[i] + 7; dir_or_file = argv[i] + 7;
@ -438,6 +444,8 @@ int SSTDumpTool::Run(int argc, char** argv) {
show_properties = true; show_properties = true;
} else if (strcmp(argv[i], "--show_compression_sizes") == 0) { } else if (strcmp(argv[i], "--show_compression_sizes") == 0) {
show_compression_sizes = true; show_compression_sizes = true;
} else if (strcmp(argv[i], "--show_summary") == 0) {
show_summary = true;
} else if (strncmp(argv[i], "--set_block_size=", 17) == 0) { } else if (strncmp(argv[i], "--set_block_size=", 17) == 0) {
set_block_size = true; set_block_size = true;
block_size_str = argv[i] + 17; block_size_str = argv[i] + 17;
@ -518,7 +526,7 @@ int SSTDumpTool::Run(int argc, char** argv) {
if (!reader.getStatus().ok()) { if (!reader.getStatus().ok()) {
fprintf(stderr, "%s: %s\n", filename.c_str(), fprintf(stderr, "%s: %s\n", filename.c_str(),
reader.getStatus().ToString().c_str()); reader.getStatus().ToString().c_str());
exit(1); continue;
} }
if (show_compression_sizes) { if (show_compression_sizes) {
@ -559,7 +567,8 @@ int SSTDumpTool::Run(int argc, char** argv) {
break; break;
} }
} }
if (show_properties) {
if (show_properties || show_summary) {
const rocksdb::TableProperties* table_properties; const rocksdb::TableProperties* table_properties;
std::shared_ptr<const rocksdb::TableProperties> std::shared_ptr<const rocksdb::TableProperties>
@ -573,34 +582,56 @@ int SSTDumpTool::Run(int argc, char** argv) {
table_properties = table_properties_from_reader.get(); table_properties = table_properties_from_reader.get();
} }
if (table_properties != nullptr) { if (table_properties != nullptr) {
fprintf(stdout, if (show_properties) {
"Table Properties:\n" fprintf(stdout,
"------------------------------\n" "Table Properties:\n"
" %s", "------------------------------\n"
table_properties->ToString("\n ", ": ").c_str()); " %s",
fprintf(stdout, "# deleted keys: %" PRIu64 "\n", table_properties->ToString("\n ", ": ").c_str());
rocksdb::GetDeletedKeys( fprintf(stdout, "# deleted keys: %" PRIu64 "\n",
table_properties->user_collected_properties)); rocksdb::GetDeletedKeys(
table_properties->user_collected_properties));
bool property_present;
uint64_t merge_operands = rocksdb::GetMergeOperands( bool property_present;
table_properties->user_collected_properties, &property_present); uint64_t merge_operands = rocksdb::GetMergeOperands(
if (property_present) { table_properties->user_collected_properties, &property_present);
fprintf(stdout, " # merge operands: %" PRIu64 "\n", merge_operands); if (property_present) {
} else { fprintf(stdout, " # merge operands: %" PRIu64 "\n",
fprintf(stdout, " # merge operands: UNKNOWN\n"); merge_operands);
} else {
fprintf(stdout, " # merge operands: UNKNOWN\n");
}
} }
total_num_files += 1;
total_num_data_blocks += table_properties->num_data_blocks;
total_data_block_size += table_properties->data_size;
total_index_block_size += table_properties->index_size;
total_filter_block_size += table_properties->filter_size;
} }
fprintf(stdout, if (show_properties) {
"Raw user collected properties\n" fprintf(stdout,
"------------------------------\n"); "Raw user collected properties\n"
for (const auto& kv : table_properties->user_collected_properties) { "------------------------------\n");
std::string prop_name = kv.first; for (const auto& kv : table_properties->user_collected_properties) {
std::string prop_val = Slice(kv.second).ToString(true); std::string prop_name = kv.first;
fprintf(stdout, " # %s: 0x%s\n", prop_name.c_str(), prop_val.c_str()); std::string prop_val = Slice(kv.second).ToString(true);
fprintf(stdout, " # %s: 0x%s\n", prop_name.c_str(),
prop_val.c_str());
}
} }
} }
} }
if (show_summary) {
fprintf(stdout, "total number of files: %" PRIu64 "\n", total_num_files);
fprintf(stdout, "total number of data blocks: %" PRIu64 "\n",
total_num_data_blocks);
fprintf(stdout, "total data block size: %" PRIu64 "\n",
total_data_block_size);
fprintf(stdout, "total index block size: %" PRIu64 "\n",
total_index_block_size);
fprintf(stdout, "total filter block size: %" PRIu64 "\n",
total_filter_block_size);
}
return 0; return 0;
} }
} // namespace rocksdb } // namespace rocksdb

Write Preview
Loading…
Cancel
Save