diff --git a/HISTORY.md b/HISTORY.md index d68655a64..ecb5f7948 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -16,6 +16,9 @@ * DeleteRange now returns `Status::InvalidArgument` if the range's end key comes before its start key according to the user comparator. Previously the behavior was undefined. * ldb now uses options.force_consistency_checks = true by default and "--disable_consistency_checks" is added to disable it. +### New Feature +* sst_dump to add a new --readahead_size argument. Users can specify read size when scanning the data. Sst_dump also tries to prefetch tail part of the SST files so usually some number of I/Os are saved there too. + ## 6.10 (5/2/2020) ### Bug Fixes * Fix wrong result being read from ingested file. May happen when a key in the file happen to be prefix of another key also in the file. The issue can further cause more data corruption. The issue exists with rocksdb >= 5.0.0 since DB::IngestExternalFile() was introduced. diff --git a/db/convenience.cc b/db/convenience.cc index 52592682e..96735d7e5 100644 --- a/db/convenience.cc +++ b/db/convenience.cc @@ -61,7 +61,8 @@ Status VerifySstFileChecksum(const Options& options, s = ioptions.table_factory->NewTableReader( TableReaderOptions(ioptions, options.prefix_extractor.get(), env_options, internal_comparator, false /* skip_filters */, - !kImmortal, -1 /* level */), + !kImmortal, false /* force_direct_prefetch */, + -1 /* level */), std::move(file_reader), file_size, &table_reader, false /* prefetch_index_and_filter_in_cache */); if (!s.ok()) { diff --git a/db/table_cache.cc b/db/table_cache.cc index d4dd73171..d4b0c6518 100644 --- a/db/table_cache.cc +++ b/db/table_cache.cc @@ -123,7 +123,8 @@ Status TableCache::GetTableReader( s = ioptions_.table_factory->NewTableReader( TableReaderOptions(ioptions_, prefix_extractor, file_options, internal_comparator, skip_filters, immortal_tables_, - level, fd.largest_seqno, block_cache_tracer_), + false /* force_direct_prefetch */, level, + fd.largest_seqno, block_cache_tracer_), std::move(file_reader), fd.GetFileSize(), table_reader, prefetch_index_and_filter_in_cache); TEST_SYNC_POINT("TableCache::GetTableReader:0"); diff --git a/file/random_access_file_reader.cc b/file/random_access_file_reader.cc index 8f98aba53..0be3e0546 100644 --- a/file/random_access_file_reader.cc +++ b/file/random_access_file_reader.cc @@ -27,6 +27,8 @@ Status RandomAccessFileReader::Read(const IOOptions& opts, uint64_t offset, AlignedBuf* aligned_buf, bool for_compaction) const { (void)aligned_buf; + + TEST_SYNC_POINT_CALLBACK("RandomAccessFileReader::Read", nullptr); Status s; uint64_t elapsed = 0; { diff --git a/table/block_based/block_based_table_factory.cc b/table/block_based/block_based_table_factory.cc index 2a560cae3..a4c88da53 100644 --- a/table/block_based/block_based_table_factory.cc +++ b/table/block_based/block_based_table_factory.cc @@ -418,7 +418,8 @@ Status BlockBasedTableFactory::NewTableReader( file_size, table_reader, table_reader_options.prefix_extractor, prefetch_index_and_filter_in_cache, table_reader_options.skip_filters, table_reader_options.level, table_reader_options.immortal, - table_reader_options.largest_seqno, &tail_prefetch_stats_, + table_reader_options.largest_seqno, + table_reader_options.force_direct_prefetch, &tail_prefetch_stats_, table_reader_options.block_cache_tracer); } diff --git a/table/block_based/block_based_table_factory.h b/table/block_based/block_based_table_factory.h index 38bf2395b..b8b125658 100644 --- a/table/block_based/block_based_table_factory.h +++ b/table/block_based/block_based_table_factory.h @@ -73,6 +73,8 @@ class BlockBasedTableFactory : public TableFactory { bool IsDeleteRangeSupported() const override { return true; } + TailPrefetchStats* tail_prefetch_stats() { return &tail_prefetch_stats_; } + static const std::string kName; private: diff --git a/table/block_based/block_based_table_reader.cc b/table/block_based/block_based_table_reader.cc index 303484e4a..ad1b7e26a 100644 --- a/table/block_based/block_based_table_reader.cc +++ b/table/block_based/block_based_table_reader.cc @@ -609,7 +609,8 @@ Status BlockBasedTable::Open( const SliceTransform* prefix_extractor, const bool prefetch_index_and_filter_in_cache, const bool skip_filters, const int level, const bool immortal_table, - const SequenceNumber largest_seqno, TailPrefetchStats* tail_prefetch_stats, + const SequenceNumber largest_seqno, const bool force_direct_prefetch, + TailPrefetchStats* tail_prefetch_stats, BlockCacheTracer* const block_cache_tracer) { table_reader->reset(); @@ -622,8 +623,9 @@ Status BlockBasedTable::Open( const bool preload_all = !table_options.cache_index_and_filter_blocks; if (!ioptions.allow_mmap_reads) { - s = PrefetchTail(file.get(), file_size, tail_prefetch_stats, prefetch_all, - preload_all, &prefetch_buffer); + s = PrefetchTail(file.get(), file_size, force_direct_prefetch, + tail_prefetch_stats, prefetch_all, preload_all, + &prefetch_buffer); } else { // Should not prefetch for mmap mode. prefetch_buffer.reset(new FilePrefetchBuffer( @@ -724,8 +726,8 @@ Status BlockBasedTable::Open( Status BlockBasedTable::PrefetchTail( RandomAccessFileReader* file, uint64_t file_size, - TailPrefetchStats* tail_prefetch_stats, const bool prefetch_all, - const bool preload_all, + bool force_direct_prefetch, TailPrefetchStats* tail_prefetch_stats, + const bool prefetch_all, const bool preload_all, std::unique_ptr* prefetch_buffer) { size_t tail_prefetch_size = 0; if (tail_prefetch_stats != nullptr) { @@ -755,7 +757,7 @@ Status BlockBasedTable::PrefetchTail( &tail_prefetch_size); Status s; // TODO should not have this special logic in the future. - if (!file->use_direct_io()) { + if (!file->use_direct_io() && !force_direct_prefetch) { prefetch_buffer->reset(new FilePrefetchBuffer( nullptr, 0, 0, false /* enable */, true /* track_min_offset */)); s = file->Prefetch(prefetch_off, prefetch_len); @@ -768,7 +770,6 @@ Status BlockBasedTable::PrefetchTail( return s; } - Status BlockBasedTable::TryReadPropertiesWithGlobalSeqno( FilePrefetchBuffer* prefetch_buffer, const Slice& handle_value, TableProperties** table_properties) { diff --git a/table/block_based/block_based_table_reader.h b/table/block_based/block_based_table_reader.h index d2aca6cd4..214ceb880 100644 --- a/table/block_based/block_based_table_reader.h +++ b/table/block_based/block_based_table_reader.h @@ -85,6 +85,8 @@ class BlockBasedTable : public TableReader { // @param skip_filters Disables loading/accessing the filter block. Overrides // prefetch_index_and_filter_in_cache, so filter will be skipped if both // are set. + // @param force_direct_prefetch if true, always prefetching to RocksDB + // buffer, rather than calling RandomAccessFile::Prefetch(). static Status Open(const ImmutableCFOptions& ioptions, const EnvOptions& env_options, const BlockBasedTableOptions& table_options, @@ -97,6 +99,7 @@ class BlockBasedTable : public TableReader { bool skip_filters = false, int level = -1, const bool immortal_table = false, const SequenceNumber largest_seqno = 0, + bool force_direct_prefetch = false, TailPrefetchStats* tail_prefetch_stats = nullptr, BlockCacheTracer* const block_cache_tracer = nullptr); @@ -393,10 +396,12 @@ class BlockBasedTable : public TableReader { const SliceTransform* prefix_extractor, BlockCacheLookupContext* lookup_context) const; + // If force_direct_prefetch is true, always prefetching to RocksDB + // buffer, rather than calling RandomAccessFile::Prefetch(). static Status PrefetchTail( RandomAccessFileReader* file, uint64_t file_size, - TailPrefetchStats* tail_prefetch_stats, const bool prefetch_all, - const bool preload_all, + bool force_direct_prefetch, TailPrefetchStats* tail_prefetch_stats, + const bool prefetch_all, const bool preload_all, std::unique_ptr* prefetch_buffer); Status ReadMetaIndexBlock(FilePrefetchBuffer* prefetch_buffer, std::unique_ptr* metaindex_block, diff --git a/table/meta_blocks.cc b/table/meta_blocks.cc index 5d81907a3..3522d6f94 100644 --- a/table/meta_blocks.cc +++ b/table/meta_blocks.cc @@ -358,11 +358,12 @@ Status ReadTableProperties(RandomAccessFileReader* file, uint64_t file_size, const ImmutableCFOptions& ioptions, TableProperties** properties, bool compression_type_missing, - MemoryAllocator* memory_allocator) { + MemoryAllocator* memory_allocator, + FilePrefetchBuffer* prefetch_buffer) { // -- Read metaindex block Footer footer; - auto s = ReadFooterFromFile(file, nullptr /* prefetch_buffer */, file_size, - &footer, table_magic_number); + auto s = ReadFooterFromFile(file, prefetch_buffer, file_size, &footer, + table_magic_number); if (!s.ok()) { return s; } @@ -374,8 +375,8 @@ Status ReadTableProperties(RandomAccessFileReader* file, uint64_t file_size, PersistentCacheOptions cache_options; BlockFetcher block_fetcher( - file, nullptr /* prefetch_buffer */, footer, read_options, - metaindex_handle, &metaindex_contents, ioptions, false /* decompress */, + file, prefetch_buffer, footer, read_options, metaindex_handle, + &metaindex_contents, ioptions, false /* decompress */, false /*maybe_compressed*/, BlockType::kMetaIndex, UncompressionDict::GetEmptyDict(), cache_options, memory_allocator); s = block_fetcher.ReadBlockContents(); @@ -398,11 +399,11 @@ Status ReadTableProperties(RandomAccessFileReader* file, uint64_t file_size, TableProperties table_properties; if (found_properties_block == true) { - s = ReadProperties( - meta_iter->value(), file, nullptr /* prefetch_buffer */, footer, - ioptions, properties, false /* verify_checksum */, - nullptr /* ret_block_hanel */, nullptr /* ret_block_contents */, - compression_type_missing, memory_allocator); + s = ReadProperties(meta_iter->value(), file, prefetch_buffer, footer, + ioptions, properties, false /* verify_checksum */, + nullptr /* ret_block_hanel */, + nullptr /* ret_block_contents */, + compression_type_missing, memory_allocator); } else { s = Status::NotFound(); } diff --git a/table/meta_blocks.h b/table/meta_blocks.h index 0fd140cf0..40a6f33fb 100644 --- a/table/meta_blocks.h +++ b/table/meta_blocks.h @@ -121,7 +121,8 @@ Status ReadTableProperties(RandomAccessFileReader* file, uint64_t file_size, const ImmutableCFOptions& ioptions, TableProperties** properties, bool compression_type_missing = false, - MemoryAllocator* memory_allocator = nullptr); + MemoryAllocator* memory_allocator = nullptr, + FilePrefetchBuffer* prefetch_buffer = nullptr); // Find the meta block from the meta index block. Status FindMetaBlock(InternalIterator* meta_index_iter, diff --git a/table/table_builder.h b/table/table_builder.h index a5c74619e..afbd179b1 100644 --- a/table/table_builder.h +++ b/table/table_builder.h @@ -33,19 +33,20 @@ struct TableReaderOptions { const EnvOptions& _env_options, const InternalKeyComparator& _internal_comparator, bool _skip_filters = false, bool _immortal = false, - int _level = -1, + bool _force_direct_prefetch = false, int _level = -1, BlockCacheTracer* const _block_cache_tracer = nullptr) : TableReaderOptions(_ioptions, _prefix_extractor, _env_options, _internal_comparator, _skip_filters, _immortal, - _level, 0 /* _largest_seqno */, - _block_cache_tracer) {} + _force_direct_prefetch, _level, + 0 /* _largest_seqno */, _block_cache_tracer) {} // @param skip_filters Disables loading/accessing the filter block TableReaderOptions(const ImmutableCFOptions& _ioptions, const SliceTransform* _prefix_extractor, const EnvOptions& _env_options, const InternalKeyComparator& _internal_comparator, - bool _skip_filters, bool _immortal, int _level, + bool _skip_filters, bool _immortal, + bool _force_direct_prefetch, int _level, SequenceNumber _largest_seqno, BlockCacheTracer* const _block_cache_tracer) : ioptions(_ioptions), @@ -54,6 +55,7 @@ struct TableReaderOptions { internal_comparator(_internal_comparator), skip_filters(_skip_filters), immortal(_immortal), + force_direct_prefetch(_force_direct_prefetch), level(_level), largest_seqno(_largest_seqno), block_cache_tracer(_block_cache_tracer) {} @@ -66,6 +68,10 @@ struct TableReaderOptions { bool skip_filters; // Whether the table will be valid as long as the DB is open bool immortal; + // When data prefetching is needed, even if direct I/O is off, read data to + // fetch into RocksDB's buffer, rather than relying + // RandomAccessFile::Prefetch(). + bool force_direct_prefetch; // what level this table/file is on, -1 for "not set, don't know" int level; // largest seqno in the table diff --git a/table/table_test.cc b/table/table_test.cc index 1efa3089d..8e1d97133 100644 --- a/table/table_test.cc +++ b/table/table_test.cc @@ -378,7 +378,7 @@ class TableConstructor: public Constructor { return ioptions.table_factory->NewTableReader( TableReaderOptions(ioptions, moptions.prefix_extractor.get(), soptions, internal_comparator, !kSkipFilters, !kImmortal, - level_, largest_seqno_, &block_cache_tracer_), + false, level_, largest_seqno_, &block_cache_tracer_), std::move(file_reader_), TEST_GetSink()->contents().size(), &table_reader_); } diff --git a/tools/ldb_cmd.cc b/tools/ldb_cmd.cc index c8033709c..abc84a578 100644 --- a/tools/ldb_cmd.cc +++ b/tools/ldb_cmd.cc @@ -3081,7 +3081,8 @@ void DumpSstFile(Options options, std::string filename, bool output_hex, // no verification // TODO: add support for decoding blob indexes in ldb as well ROCKSDB_NAMESPACE::SstFileDumper dumper( - options, filename, /* verify_checksum */ false, output_hex, + options, filename, 2 * 1024 * 1024 /* readahead_size */, + /* verify_checksum */ false, output_hex, /* decode_blob_index */ false); Status st = dumper.ReadSequential(true, std::numeric_limits::max(), false, // has_from diff --git a/tools/sst_dump_test.cc b/tools/sst_dump_test.cc index b8c475c3b..a8a083439 100644 --- a/tools/sst_dump_test.cc +++ b/tools/sst_dump_test.cc @@ -22,7 +22,7 @@ namespace ROCKSDB_NAMESPACE { -const uint32_t optLength = 100; +const uint32_t kOptLength = 100; namespace { static std::string MakeKey(int i) { @@ -121,11 +121,11 @@ class SSTDumpToolTest : public testing::Test { void PopulateCommandArgs(const std::string& file_path, const char* command, char* (&usage)[N]) const { for (int i = 0; i < static_cast(N); ++i) { - usage[i] = new char[optLength]; + usage[i] = new char[kOptLength]; } - snprintf(usage[0], optLength, "./sst_dump"); - snprintf(usage[1], optLength, "%s", command); - snprintf(usage[2], optLength, "--file=%s", file_path.c_str()); + snprintf(usage[0], kOptLength, "./sst_dump"); + snprintf(usage[1], kOptLength, "%s", command); + snprintf(usage[2], kOptLength, "--file=%s", file_path.c_str()); } }; @@ -254,6 +254,38 @@ TEST_F(SSTDumpToolTest, MemEnv) { } } +TEST_F(SSTDumpToolTest, ReadaheadSize) { + Options opts; + opts.env = env(); + std::string file_path = MakeFilePath("rocksdb_sst_test.sst"); + createSST(opts, file_path); + + char* usage[4]; + PopulateCommandArgs(file_path, "--command=verify", usage); + snprintf(usage[3], kOptLength, "--readahead_size=4000000"); + + int num_reads = 0; + SyncPoint::GetInstance()->SetCallBack("RandomAccessFileReader::Read", + [&](void*) { num_reads++; }); + SyncPoint::GetInstance()->EnableProcessing(); + + SSTDumpTool tool; + ASSERT_TRUE(!tool.Run(4, usage, opts)); + + // The file is approximately 10MB. Readahead is 4MB. + // We usually need 3 reads + one metadata read. + // One extra read is needed before opening the file for metadata. + ASSERT_EQ(5, num_reads); + + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->DisableProcessing(); + + cleanup(opts, file_path); + for (int i = 0; i < 4; i++) { + delete[] usage[i]; + } +} + } // namespace ROCKSDB_NAMESPACE #ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS diff --git a/tools/sst_dump_tool.cc b/tools/sst_dump_tool.cc index 412140460..b0c7115dc 100644 --- a/tools/sst_dump_tool.cc +++ b/tools/sst_dump_tool.cc @@ -44,17 +44,19 @@ namespace ROCKSDB_NAMESPACE { SstFileDumper::SstFileDumper(const Options& options, - const std::string& file_path, bool verify_checksum, + const std::string& file_path, + size_t readahead_size, bool verify_checksum, bool output_hex, bool decode_blob_index) : file_name_(file_path), read_num_(0), - verify_checksum_(verify_checksum), output_hex_(output_hex), decode_blob_index_(decode_blob_index), options_(options), ioptions_(options_), moptions_(ColumnFamilyOptions(options_)), + read_options_(verify_checksum, false), internal_comparator_(BytewiseComparator()) { + read_options_.readahead_size = readahead_size; fprintf(stdout, "Process %s\n", file_path.c_str()); init_result_ = GetTableReader(file_name_); } @@ -96,9 +98,18 @@ Status SstFileDumper::GetTableReader(const std::string& file_path) { file_.reset(new RandomAccessFileReader(NewLegacyRandomAccessFileWrapper(file), file_path)); + FilePrefetchBuffer prefetch_buffer(nullptr, 0, 0, true /* enable */, + false /* track_min_offset */); + const uint64_t kSstDumpTailPrefetchSize = 512 * 1024; + uint64_t prefetch_size = (file_size > kSstDumpTailPrefetchSize) + ? kSstDumpTailPrefetchSize + : file_size; + uint64_t prefetch_off = file_size - prefetch_size; + prefetch_buffer.Prefetch(file_.get(), prefetch_off, + static_cast(prefetch_size)); + if (s.ok()) { - s = ReadFooterFromFile(file_.get(), nullptr /* prefetch_buffer */, - file_size, &footer); + s = ReadFooterFromFile(file_.get(), &prefetch_buffer, file_size, &footer); } if (s.ok()) { magic_number = footer.table_magic_number(); @@ -114,7 +125,11 @@ Status SstFileDumper::GetTableReader(const std::string& file_path) { } options_.comparator = &internal_comparator_; // For old sst format, ReadTableProperties might fail but file can be read - if (ReadTableProperties(magic_number, file_.get(), file_size).ok()) { + if (ReadTableProperties(magic_number, file_.get(), file_size, + (magic_number == kBlockBasedTableMagicNumber) + ? &prefetch_buffer + : nullptr) + .ok()) { SetTableOptionsByMagicNumber(magic_number); } else { SetOldTableOptions(); @@ -132,8 +147,10 @@ Status SstFileDumper::NewTableReader( const ImmutableCFOptions& /*ioptions*/, const EnvOptions& /*soptions*/, const InternalKeyComparator& /*internal_comparator*/, uint64_t file_size, std::unique_ptr* /*table_reader*/) { - auto t_opt = TableReaderOptions(ioptions_, moptions_.prefix_extractor.get(), - soptions_, internal_comparator_); + auto t_opt = + TableReaderOptions(ioptions_, moptions_.prefix_extractor.get(), soptions_, + internal_comparator_, false /* skip_filters */, + false /* imortal */, true /* force_direct_prefetch */); // Allow open file with global sequence number for backward compatibility. t_opt.largest_seqno = kMaxSequenceNumber; @@ -152,7 +169,7 @@ Status SstFileDumper::NewTableReader( Status SstFileDumper::VerifyChecksum() { // We could pass specific readahead setting into read options if needed. - return table_reader_->VerifyChecksum(ReadOptions(), + return table_reader_->VerifyChecksum(read_options_, TableReaderCaller::kSSTDumpTool); } @@ -184,7 +201,7 @@ uint64_t SstFileDumper::CalculateCompressedTableSize( TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, dest_writer.get())); std::unique_ptr iter(table_reader_->NewIterator( - ReadOptions(), moptions_.prefix_extractor.get(), /*arena=*/nullptr, + read_options_, moptions_.prefix_extractor.get(), /*arena=*/nullptr, /*skip_filters=*/false, TableReaderCaller::kSSTDumpTool)); for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { table_builder->Add(iter->key(), iter->value()); @@ -234,7 +251,6 @@ int SstFileDumper::ShowCompressionSize( size_t block_size, CompressionType compress_type, const CompressionOptions& compress_opt) { - ReadOptions read_options; Options opts; opts.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); opts.statistics->set_stats_level(StatsLevel::kAll); @@ -301,10 +317,13 @@ int SstFileDumper::ShowCompressionSize( Status SstFileDumper::ReadTableProperties(uint64_t table_magic_number, RandomAccessFileReader* file, - uint64_t file_size) { + uint64_t file_size, + FilePrefetchBuffer* prefetch_buffer) { TableProperties* table_properties = nullptr; Status s = ROCKSDB_NAMESPACE::ReadTableProperties( - file, file_size, table_magic_number, ioptions_, &table_properties); + file, file_size, table_magic_number, ioptions_, &table_properties, + /* compression_type_missing= */ false, + /* memory_allocator= */ nullptr, prefetch_buffer); if (s.ok()) { table_properties_.reset(table_properties); } else { @@ -318,8 +337,16 @@ Status SstFileDumper::SetTableOptionsByMagicNumber( assert(table_properties_); if (table_magic_number == kBlockBasedTableMagicNumber || table_magic_number == kLegacyBlockBasedTableMagicNumber) { - options_.table_factory = std::make_shared(); + BlockBasedTableFactory* bbtf = new BlockBasedTableFactory(); + // To force tail prefetching, we fake reporting two useful reads of 512KB + // from the tail. + // It needs at least two data points to warm up the stats. + bbtf->tail_prefetch_stats()->RecordEffectiveSize(512 * 1024); + bbtf->tail_prefetch_stats()->RecordEffectiveSize(512 * 1024); + + options_.table_factory.reset(bbtf); fprintf(stdout, "Sst file format: block-based\n"); + auto& props = table_properties_->user_collected_properties; auto pos = props.find(BlockBasedTablePropertyNames::kIndexType); if (pos != props.end()) { @@ -373,7 +400,7 @@ Status SstFileDumper::ReadSequential(bool print_kv, uint64_t read_num, } InternalIterator* iter = table_reader_->NewIterator( - ReadOptions(verify_checksum_, false), moptions_.prefix_extractor.get(), + read_options_, moptions_.prefix_extractor.get(), /*arena=*/nullptr, /*skip_filters=*/false, TableReaderCaller::kSSTDumpTool); uint64_t i = 0; @@ -518,6 +545,24 @@ void print_help() { )"); } +// arg_name would include all prefix, e.g. "--my_arg=" +// arg_val is the parses value. +// True if there is a match. False otherwise. +// Woud exit after printing errmsg if cannot be parsed. +bool ParseIntArg(const char* arg, const std::string arg_name, + const std::string err_msg, int64_t* arg_val) { + if (strncmp(arg, arg_name.c_str(), arg_name.size()) == 0) { + std::string input_str = arg + arg_name.size(); + std::istringstream iss(input_str); + iss >> *arg_val; + if (iss.fail()) { + fprintf(stderr, "%s\n", err_msg.c_str()); + exit(1); + } + return true; + } + return false; +} } // namespace int SSTDumpTool::Run(int argc, char** argv, Options options) { @@ -547,6 +592,7 @@ int SSTDumpTool::Run(int argc, char** argv, Options options) { std::string compression_level_from_str; std::string compression_level_to_str; size_t block_size = 0; + size_t readahead_size = 2 * 1024 * 1024; std::vector> compression_types; uint64_t total_num_files = 0; uint64_t total_num_data_blocks = 0; @@ -555,6 +601,9 @@ int SSTDumpTool::Run(int argc, char** argv, Options options) { uint64_t total_filter_block_size = 0; int32_t compress_level_from = CompressionOptions::kDefaultCompressionLevel; int32_t compress_level_to = CompressionOptions::kDefaultCompressionLevel; + + int64_t tmp_val; + for (int i = 1; i < argc; i++) { if (strncmp(argv[i], "--env_uri=", 10) == 0) { env_uri = argv[i] + 10; @@ -586,15 +635,13 @@ int SSTDumpTool::Run(int argc, char** argv, Options options) { show_properties = true; } else if (strcmp(argv[i], "--show_summary") == 0) { show_summary = true; - } else if (strncmp(argv[i], "--set_block_size=", 17) == 0) { + } else if (ParseIntArg(argv[i], "--set_block_size=", + "block size must be numeric", &tmp_val)) { set_block_size = true; - block_size_str = argv[i] + 17; - std::istringstream iss(block_size_str); - iss >> block_size; - if (iss.fail()) { - fprintf(stderr, "block size must be numeric\n"); - exit(1); - } + block_size = static_cast(tmp_val); + } else if (ParseIntArg(argv[i], "--readahead_size=", + "readahead_size must be numeric", &tmp_val)) { + readahead_size = static_cast(tmp_val); } else if (strncmp(argv[i], "--compression_types=", 20) == 0) { std::string compression_types_csv = argv[i] + 20; std::istringstream iss(compression_types_csv); @@ -633,25 +680,16 @@ int SSTDumpTool::Run(int argc, char** argv, Options options) { } fprintf(stdout, "key=%s\n", ikey.DebugString(true).c_str()); return retc; - } else if (strncmp(argv[i], "--compression_level_from=", 25) == 0) { - compression_level_from_str = argv[i] + 25; + } else if (ParseIntArg(argv[i], "--compression_level_from=", + "compression_level_from must be numeric", + &tmp_val)) { has_compression_level_from = true; - std::istringstream iss(compression_level_from_str); - iss >> compress_level_from; - if (iss.fail()) { - fprintf(stderr, "compression_level_from must be numeric\n"); - exit(1); - } - } else if (strncmp(argv[i], "--compression_level_to=", 22) == 0) { - compression_level_to_str = argv[i]+23 ; + compress_level_from = static_cast(tmp_val); + } else if (ParseIntArg(argv[i], "--compression_level_to=", + "compression_level_to must be numeric", &tmp_val)) { has_compression_level_to = true; - std::istringstream iss(compression_level_to_str); - iss >> compress_level_to; - if (iss.fail()) { - fprintf(stderr, "compression_level_to must be numeric\n"); - exit(1); - } - }else { + compress_level_to = static_cast(tmp_val); + } else { fprintf(stderr, "Unrecognized argument '%s'\n\n", argv[i]); print_help(); exit(1); @@ -732,8 +770,9 @@ int SSTDumpTool::Run(int argc, char** argv, Options options) { filename = std::string(dir_or_file) + "/" + filename; } - ROCKSDB_NAMESPACE::SstFileDumper dumper(options, filename, verify_checksum, - output_hex, decode_blob_index); + ROCKSDB_NAMESPACE::SstFileDumper dumper(options, filename, readahead_size, + verify_checksum, output_hex, + decode_blob_index); if (!dumper.getStatus().ok()) { fprintf(stderr, "%s: %s\n", filename.c_str(), dumper.getStatus().ToString().c_str()); diff --git a/tools/sst_dump_tool_imp.h b/tools/sst_dump_tool_imp.h index 3af403383..1d96f46a7 100644 --- a/tools/sst_dump_tool_imp.h +++ b/tools/sst_dump_tool_imp.h @@ -18,8 +18,8 @@ namespace ROCKSDB_NAMESPACE { class SstFileDumper { public: explicit SstFileDumper(const Options& options, const std::string& file_name, - bool verify_checksum, bool output_hex, - bool decode_blob_index); + size_t readahead_size, bool verify_checksum, + bool output_hex, bool decode_blob_index); Status ReadSequential(bool print_kv, uint64_t read_num, bool has_from, const std::string& from_key, bool has_to, @@ -51,7 +51,8 @@ class SstFileDumper { // Get the TableReader implementation for the sst file Status GetTableReader(const std::string& file_path); Status ReadTableProperties(uint64_t table_magic_number, - RandomAccessFileReader* file, uint64_t file_size); + RandomAccessFileReader* file, uint64_t file_size, + FilePrefetchBuffer* prefetch_buffer); uint64_t CalculateCompressedTableSize(const TableBuilderOptions& tb_options, size_t block_size, @@ -70,7 +71,6 @@ class SstFileDumper { std::string file_name_; uint64_t read_num_; - bool verify_checksum_; bool output_hex_; bool decode_blob_index_; EnvOptions soptions_; @@ -85,6 +85,7 @@ class SstFileDumper { const ImmutableCFOptions ioptions_; const MutableCFOptions moptions_; + ReadOptions read_options_; InternalKeyComparator internal_comparator_; std::unique_ptr table_properties_; };