NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

New Statistics to track Compression/Decompression (#1197)

Browse Source 
* Added new statistics and refactored to allow ioptions to be passed around as required to access environment and statistics pointers (and, as a convenient side effect, info_log pointer).

* Prevent incrementing compression counter when compression is turned off in options.

* Prevent incrementing compression counter when compression is turned off in options.

* Added two more supported compression types to test code in db_test.cc

* Prevent incrementing compression counter when compression is turned off in options.

* Added new StatsLevel that excludes compression timing.

* Fixed casting error in coding.h

* Fixed CompressionStatsTest for new StatsLevel.

* Removed unused variable that was breaking the Linux build
main
John Alexander 9 years ago committed by Siying Dong
parent 515b11ffab
commit 9430333f84
19 changed files with 255 additions and 114 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 2
      .gitignore
  2. 75
      db/db_test.cc
  3. 7
      db/plain_table_db_test.cc
  4. 4
      db/table_properties_collector_test.cc
  5. 3
      db/version_set.cc
  6. 22
      include/rocksdb/statistics.h
  7. 17
      table/block_based_table_builder.cc
  8. 110
      table/block_based_table_reader.cc
  9. 6
      table/block_based_table_reader.h
  10. 6
      table/cuckoo_table_builder_test.cc
  11. 2
      table/cuckoo_table_reader.cc
  12. 41
      table/format.cc
  13. 14
      table/format.h
  14. 30
      table/meta_blocks.cc
  15. 13
      table/meta_blocks.h
  16. 6
      table/plain_table_reader.cc
  17. 2
      table/table_test.cc
  18. 3
      tools/sst_dump_tool.cc
  19. 6
      util/coding.h

2
.gitignore vendored
Unescape View File

@ -64,3 +64,5 @@ java/javadoc
scan_build_report/ scan_build_report/
t t
LOG LOG
db_logs/

75
db/db_test.cc
Unescape View File

@ -1189,6 +1189,12 @@ bool MinLevelToCompress(CompressionType& type, Options& options, int wbits,
} else if (LZ4_Supported()) { } else if (LZ4_Supported()) {
type = kLZ4Compression; type = kLZ4Compression;
fprintf(stderr, "using lz4\n"); fprintf(stderr, "using lz4\n");
} else if (XPRESS_Supported()) {
type = kXpressCompression;
fprintf(stderr, "using xpress\n");
} else if (ZSTD_Supported()) {
type = kZSTDNotFinalCompression;
fprintf(stderr, "using ZSTD\n");
} else { } else {
fprintf(stderr, "skipping test, compression disabled\n"); fprintf(stderr, "skipping test, compression disabled\n");
return false; return false;
@ -4685,6 +4691,75 @@ TEST_F(DBTest, EncodeDecompressedBlockSizeTest) {
} }
} }
TEST_F(DBTest, CompressionStatsTest) {
CompressionType type;
if (Snappy_Supported()) {
type = kSnappyCompression;
fprintf(stderr, "using snappy\n");
} else if (Zlib_Supported()) {
type = kZlibCompression;
fprintf(stderr, "using zlib\n");
} else if (BZip2_Supported()) {
type = kBZip2Compression;
fprintf(stderr, "using bzip2\n");
} else if (LZ4_Supported()) {
type = kLZ4Compression;
fprintf(stderr, "using lz4\n");
} else if (XPRESS_Supported()) {
type = kXpressCompression;
fprintf(stderr, "using xpress\n");
} else if (ZSTD_Supported()) {
type = kZSTDNotFinalCompression;
fprintf(stderr, "using ZSTD\n");
} else {
fprintf(stderr, "skipping test, compression disabled\n");
return;
}
Options options = CurrentOptions();
options.compression = type;
options.statistics = rocksdb::CreateDBStatistics();
options.statistics->stats_level_ = StatsLevel::kAll;
DestroyAndReopen(options);
int kNumKeysWritten = 100000;
// Check that compressions occur and are counted when compression is turned on
Random rnd(301);
for (int i = 0; i < kNumKeysWritten; ++i) {
// compressible string
ASSERT_OK(Put(Key(i), RandomString(&rnd, 128) + std::string(128, 'a')));
}
ASSERT_GT(options.statistics->getTickerCount(NUMBER_BLOCK_COMPRESSED), 0);
for (int i = 0; i < kNumKeysWritten; ++i) {
auto r = Get(Key(i));
}
ASSERT_GT(options.statistics->getTickerCount(NUMBER_BLOCK_DECOMPRESSED), 0);
options.compression = kNoCompression;
DestroyAndReopen(options);
uint64_t currentCompressions =
options.statistics->getTickerCount(NUMBER_BLOCK_COMPRESSED);
uint64_t currentDecompressions =
options.statistics->getTickerCount(NUMBER_BLOCK_DECOMPRESSED);
// Check that compressions do not occur when turned off
for (int i = 0; i < kNumKeysWritten; ++i) {
// compressible string
ASSERT_OK(Put(Key(i), RandomString(&rnd, 128) + std::string(128, 'a')));
}
ASSERT_EQ(options.statistics->getTickerCount(NUMBER_BLOCK_COMPRESSED)
- currentCompressions, 0);
for (int i = 0; i < kNumKeysWritten; ++i) {
auto r = Get(Key(i));
}
ASSERT_EQ(options.statistics->getTickerCount(NUMBER_BLOCK_DECOMPRESSED)
- currentDecompressions, 0);
}
TEST_F(DBTest, MutexWaitStatsDisabledByDefault) { TEST_F(DBTest, MutexWaitStatsDisabledByDefault) {
Options options = CurrentOptions(); Options options = CurrentOptions();
options.create_if_missing = true; options.create_if_missing = true;

7
db/plain_table_db_test.cc
Unescape View File

@ -331,20 +331,19 @@ class TestPlainTableFactory : public PlainTableFactory {
TableProperties* props = nullptr; TableProperties* props = nullptr;
auto s = auto s =
ReadTableProperties(file.get(), file_size, kPlainTableMagicNumber, ReadTableProperties(file.get(), file_size, kPlainTableMagicNumber,
table_reader_options.ioptions.env, table_reader_options.ioptions, &props);
table_reader_options.ioptions.info_log, &props);
EXPECT_TRUE(s.ok()); EXPECT_TRUE(s.ok());
if (store_index_in_file_) { if (store_index_in_file_) {
BlockHandle bloom_block_handle; BlockHandle bloom_block_handle;
s = FindMetaBlock(file.get(), file_size, kPlainTableMagicNumber, s = FindMetaBlock(file.get(), file_size, kPlainTableMagicNumber,
table_reader_options.ioptions.env, table_reader_options.ioptions,
BloomBlockBuilder::kBloomBlock, &bloom_block_handle); BloomBlockBuilder::kBloomBlock, &bloom_block_handle);
EXPECT_TRUE(s.ok()); EXPECT_TRUE(s.ok());
BlockHandle index_block_handle; BlockHandle index_block_handle;
s = FindMetaBlock(file.get(), file_size, kPlainTableMagicNumber, s = FindMetaBlock(file.get(), file_size, kPlainTableMagicNumber,
table_reader_options.ioptions.env, table_reader_options.ioptions,
PlainTableIndexBuilder::kPlainTableIndexBlock, PlainTableIndexBuilder::kPlainTableIndexBlock,
&index_block_handle); &index_block_handle);
EXPECT_TRUE(s.ok()); EXPECT_TRUE(s.ok());

4
db/table_properties_collector_test.cc
Unescape View File

@ -276,7 +276,7 @@ void TestCustomizedTablePropertiesCollector(
new test::StringSource(fwf->contents()))); new test::StringSource(fwf->contents())));
TableProperties* props; TableProperties* props;
Status s = ReadTableProperties(fake_file_reader.get(), fwf->contents().size(), Status s = ReadTableProperties(fake_file_reader.get(), fwf->contents().size(),
magic_number, Env::Default(), nullptr, &props); magic_number, ioptions, &props);
std::unique_ptr<TableProperties> props_guard(props); std::unique_ptr<TableProperties> props_guard(props);
ASSERT_OK(s); ASSERT_OK(s);
@ -417,7 +417,7 @@ void TestInternalKeyPropertiesCollector(
TableProperties* props; TableProperties* props;
Status s = Status s =
ReadTableProperties(reader.get(), fwf->contents().size(), magic_number, ReadTableProperties(reader.get(), fwf->contents().size(), magic_number,
Env::Default(), nullptr, &props); ioptions, &props);
ASSERT_OK(s); ASSERT_OK(s);
std::unique_ptr<TableProperties> props_guard(props); std::unique_ptr<TableProperties> props_guard(props);

3
db/version_set.cc
Unescape View File

@ -597,8 +597,7 @@ Status Version::GetTableProperties(std::shared_ptr<const TableProperties>* tp,
new RandomAccessFileReader(std::move(file))); new RandomAccessFileReader(std::move(file)));
s = ReadTableProperties( s = ReadTableProperties(
file_reader.get(), file_meta->fd.GetFileSize(), file_reader.get(), file_meta->fd.GetFileSize(),
Footer::kInvalidTableMagicNumber /* table's magic number */, vset_->env_, Footer::kInvalidTableMagicNumber /* table's magic number */, *ioptions, &raw_table_properties);
ioptions->info_log, &raw_table_properties);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }

22
include/rocksdb/statistics.h
Unescape View File

@ -179,6 +179,11 @@ enum Tickers : uint32_t {
NUMBER_SUPERVERSION_ACQUIRES, NUMBER_SUPERVERSION_ACQUIRES,
NUMBER_SUPERVERSION_RELEASES, NUMBER_SUPERVERSION_RELEASES,
NUMBER_SUPERVERSION_CLEANUPS, NUMBER_SUPERVERSION_CLEANUPS,
// # of compressions/decompressions executed
NUMBER_BLOCK_COMPRESSED,
NUMBER_BLOCK_DECOMPRESSED,
NUMBER_BLOCK_NOT_COMPRESSED, NUMBER_BLOCK_NOT_COMPRESSED,
MERGE_OPERATION_TOTAL_TIME, MERGE_OPERATION_TOTAL_TIME,
FILTER_OPERATION_TOTAL_TIME, FILTER_OPERATION_TOTAL_TIME,
@ -269,6 +274,8 @@ const std::vector<std::pair<Tickers, std::string>> TickersNameMap = {
{NUMBER_SUPERVERSION_ACQUIRES, "rocksdb.number.superversion_acquires"}, {NUMBER_SUPERVERSION_ACQUIRES, "rocksdb.number.superversion_acquires"},
{NUMBER_SUPERVERSION_RELEASES, "rocksdb.number.superversion_releases"}, {NUMBER_SUPERVERSION_RELEASES, "rocksdb.number.superversion_releases"},
{NUMBER_SUPERVERSION_CLEANUPS, "rocksdb.number.superversion_cleanups"}, {NUMBER_SUPERVERSION_CLEANUPS, "rocksdb.number.superversion_cleanups"},
{NUMBER_BLOCK_COMPRESSED, "rocksdb.number.block.compressed"},
{NUMBER_BLOCK_DECOMPRESSED, "rocksdb.number.block.decompressed"},
{NUMBER_BLOCK_NOT_COMPRESSED, "rocksdb.number.block.not_compressed"}, {NUMBER_BLOCK_NOT_COMPRESSED, "rocksdb.number.block.not_compressed"},
{MERGE_OPERATION_TOTAL_TIME, "rocksdb.merge.operation.time.nanos"}, {MERGE_OPERATION_TOTAL_TIME, "rocksdb.merge.operation.time.nanos"},
{FILTER_OPERATION_TOTAL_TIME, "rocksdb.filter.operation.time.nanos"}, {FILTER_OPERATION_TOTAL_TIME, "rocksdb.filter.operation.time.nanos"},
@ -313,6 +320,14 @@ enum Histograms : uint32_t {
BYTES_PER_READ, BYTES_PER_READ,
BYTES_PER_WRITE, BYTES_PER_WRITE,
BYTES_PER_MULTIGET, BYTES_PER_MULTIGET,
// number of bytes compressed/decompressed
// number of bytes is when uncompressed; i.e. before/after respectively
BYTES_COMPRESSED,
BYTES_DECOMPRESSED,
COMPRESSION_TIMES_NANOS,
DECOMPRESSION_TIMES_NANOS,
HISTOGRAM_ENUM_MAX, // TODO(ldemailly): enforce HistogramsNameMap match HISTOGRAM_ENUM_MAX, // TODO(ldemailly): enforce HistogramsNameMap match
}; };
@ -343,6 +358,10 @@ const std::vector<std::pair<Histograms, std::string>> HistogramsNameMap = {
{BYTES_PER_READ, "rocksdb.bytes.per.read"}, {BYTES_PER_READ, "rocksdb.bytes.per.read"},
{BYTES_PER_WRITE, "rocksdb.bytes.per.write"}, {BYTES_PER_WRITE, "rocksdb.bytes.per.write"},
{BYTES_PER_MULTIGET, "rocksdb.bytes.per.multiget"}, {BYTES_PER_MULTIGET, "rocksdb.bytes.per.multiget"},
{BYTES_COMPRESSED, "rocksdb.bytes.compressed"},
{BYTES_DECOMPRESSED, "rocksdb.bytes.decompressed"},
{COMPRESSION_TIMES_NANOS, "rocksdb.compression.times.nanos"},
{DECOMPRESSION_TIMES_NANOS, "rocksdb.decompression.times.nanos"},
}; };
struct HistogramData { struct HistogramData {
@ -357,6 +376,9 @@ enum StatsLevel {
// Collect all stats except the counters requiring to get time inside the // Collect all stats except the counters requiring to get time inside the
// mutex lock. // mutex lock.
kExceptTimeForMutex, kExceptTimeForMutex,
// Collect all stats expect time inside mutex lock AND time spent on
// compression
kExceptDetailedTimers,
// Collect all stats, including measuring duration of mutex operations. // Collect all stats, including measuring duration of mutex operations.
// If getting time is expensive on the platform to run, it can // If getting time is expensive on the platform to run, it can
// reduce scalability to more threads, especially for writes. // reduce scalability to more threads, especially for writes.

17
table/block_based_table_builder.cc
Unescape View File

@ -651,11 +651,16 @@ void BlockBasedTableBuilder::WriteBlock(const Slice& raw_block_contents,
auto type = r->compression_type; auto type = r->compression_type;
Slice block_contents; Slice block_contents;
bool abort_compression = false; bool abort_compression = false;
StopWatchNano timer(r->ioptions.env,
ShouldReportDetailedTime(r->ioptions.env, r->ioptions.statistics));
if (raw_block_contents.size() < kCompressionSizeLimit) { if (raw_block_contents.size() < kCompressionSizeLimit) {
Slice compression_dict; Slice compression_dict;
if (is_data_block && r->compression_dict && r->compression_dict->size()) { if (is_data_block && r->compression_dict && r->compression_dict->size()) {
compression_dict = *r->compression_dict; compression_dict = *r->compression_dict;
} }
block_contents = CompressBlock(raw_block_contents, r->compression_opts, block_contents = CompressBlock(raw_block_contents, r->compression_opts,
&type, r->table_options.format_version, &type, r->table_options.format_version,
compression_dict, &r->compressed_output); compression_dict, &r->compressed_output);
@ -668,7 +673,8 @@ void BlockBasedTableBuilder::WriteBlock(const Slice& raw_block_contents,
BlockContents contents; BlockContents contents;
Status stat = UncompressBlockContentsForCompressionType( Status stat = UncompressBlockContentsForCompressionType(
block_contents.data(), block_contents.size(), &contents, block_contents.data(), block_contents.size(), &contents,
r->table_options.format_version, compression_dict, type); r->table_options.format_version, compression_dict, type,
r->ioptions);
if (stat.ok()) { if (stat.ok()) {
bool compressed_ok = contents.data.compare(raw_block_contents) == 0; bool compressed_ok = contents.data.compare(raw_block_contents) == 0;
@ -698,6 +704,15 @@ void BlockBasedTableBuilder::WriteBlock(const Slice& raw_block_contents,
type = kNoCompression; type = kNoCompression;
block_contents = raw_block_contents; block_contents = raw_block_contents;
} }
else if (type != kNoCompression &&
ShouldReportDetailedTime(r->ioptions.env,
r->ioptions.statistics)) {
MeasureTime(r->ioptions.statistics, COMPRESSION_TIMES_NANOS,
timer.ElapsedNanos());
MeasureTime(r->ioptions.statistics, BYTES_COMPRESSED,
raw_block_contents.size());
RecordTick(r->ioptions.statistics, NUMBER_BLOCK_COMPRESSED);
}
WriteRawBlock(block_contents, type, handle); WriteRawBlock(block_contents, type, handle);
r->compressed_output.clear(); r->compressed_output.clear();

110
table/block_based_table_reader.cc
Unescape View File

@ -60,14 +60,13 @@ namespace {
// dictionary. // dictionary.
Status ReadBlockFromFile(RandomAccessFileReader* file, const Footer& footer, Status ReadBlockFromFile(RandomAccessFileReader* file, const Footer& footer,
const ReadOptions& options, const BlockHandle& handle, const ReadOptions& options, const BlockHandle& handle,
std::unique_ptr<Block>* result, Env* env, std::unique_ptr<Block>* result,
const ImmutableCFOptions &ioptions,
bool do_uncompress, const Slice& compression_dict, bool do_uncompress, const Slice& compression_dict,
const PersistentCacheOptions& cache_options, const PersistentCacheOptions& cache_options) {
Logger* info_log) {
BlockContents contents; BlockContents contents;
Status s = ReadBlockContents(file, footer, options, handle, &contents, env, Status s = ReadBlockContents(file, footer, options, handle, &contents, ioptions,
do_uncompress, compression_dict, cache_options, do_uncompress, compression_dict, cache_options);
info_log);
if (s.ok()) { if (s.ok()) {
result->reset(new Block(std::move(contents))); result->reset(new Block(std::move(contents)));
} }
@ -177,19 +176,18 @@ class BinarySearchIndexReader : public IndexReader {
// On success, index_reader will be populated; otherwise it will remain // On success, index_reader will be populated; otherwise it will remain
// unmodified. // unmodified.
static Status Create(RandomAccessFileReader* file, const Footer& footer, static Status Create(RandomAccessFileReader* file, const Footer& footer,
const BlockHandle& index_handle, Env* env, const BlockHandle& index_handle,
const ImmutableCFOptions &ioptions,
const Comparator* comparator, IndexReader** index_reader, const Comparator* comparator, IndexReader** index_reader,
const PersistentCacheOptions& cache_options, const PersistentCacheOptions& cache_options) {
Statistics* statistics) {
std::unique_ptr<Block> index_block; std::unique_ptr<Block> index_block;
auto s = ReadBlockFromFile(file, footer, ReadOptions(), index_handle, auto s = ReadBlockFromFile(file, footer, ReadOptions(), index_handle,
&index_block, env, true /* decompress */, &index_block, ioptions, true /* decompress */,
Slice() /*compression dict*/, cache_options, Slice() /*compression dict*/, cache_options);
/*info_log*/ nullptr);
if (s.ok()) { if (s.ok()) {
*index_reader = new BinarySearchIndexReader( *index_reader = new BinarySearchIndexReader(
comparator, std::move(index_block), statistics); comparator, std::move(index_block), ioptions.statistics);
} }
return s; return s;
@ -226,15 +224,15 @@ class HashIndexReader : public IndexReader {
public: public:
static Status Create( static Status Create(
const SliceTransform* hash_key_extractor, const Footer& footer, const SliceTransform* hash_key_extractor, const Footer& footer,
RandomAccessFileReader* file, Env* env, const Comparator* comparator, RandomAccessFileReader* file, const ImmutableCFOptions &ioptions,
const BlockHandle& index_handle, InternalIterator* meta_index_iter, const Comparator* comparator, const BlockHandle& index_handle,
IndexReader** index_reader, bool hash_index_allow_collision, InternalIterator* meta_index_iter, IndexReader** index_reader,
const PersistentCacheOptions& cache_options, Statistics* statistics) { bool hash_index_allow_collision,
const PersistentCacheOptions& cache_options) {
std::unique_ptr<Block> index_block; std::unique_ptr<Block> index_block;
auto s = ReadBlockFromFile(file, footer, ReadOptions(), index_handle, auto s = ReadBlockFromFile(file, footer, ReadOptions(), index_handle,
&index_block, env, true /* decompress */, &index_block, ioptions, true /* decompress */,
Slice() /*compression dict*/, cache_options, Slice() /*compression dict*/, cache_options);
/*info_log*/ nullptr);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
@ -245,7 +243,8 @@ class HashIndexReader : public IndexReader {
// So, Create will succeed regardless, from this point on. // So, Create will succeed regardless, from this point on.
auto new_index_reader = auto new_index_reader =
new HashIndexReader(comparator, std::move(index_block), statistics); new HashIndexReader(comparator, std::move(index_block),
ioptions.statistics);
*index_reader = new_index_reader; *index_reader = new_index_reader;
// Get prefixes block // Get prefixes block
@ -269,14 +268,14 @@ class HashIndexReader : public IndexReader {
// Read contents for the blocks // Read contents for the blocks
BlockContents prefixes_contents; BlockContents prefixes_contents;
s = ReadBlockContents(file, footer, ReadOptions(), prefixes_handle, s = ReadBlockContents(file, footer, ReadOptions(), prefixes_handle,
&prefixes_contents, env, true /* decompress */, &prefixes_contents, ioptions, true /* decompress */,
Slice() /*compression dict*/, cache_options); Slice() /*compression dict*/, cache_options);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
BlockContents prefixes_meta_contents; BlockContents prefixes_meta_contents;
s = ReadBlockContents(file, footer, ReadOptions(), prefixes_meta_handle, s = ReadBlockContents(file, footer, ReadOptions(), prefixes_meta_handle,
&prefixes_meta_contents, env, true /* decompress */, &prefixes_meta_contents, ioptions, true /* decompress */,
Slice() /*compression dict*/, cache_options); Slice() /*compression dict*/, cache_options);
if (!s.ok()) { if (!s.ok()) {
// TODO: log error // TODO: log error
@ -547,7 +546,7 @@ Status BlockBasedTable::Open(const ImmutableCFOptions& ioptions,
PersistentCacheOptions(rep->table_options.persistent_cache, PersistentCacheOptions(rep->table_options.persistent_cache,
std::string(rep->persistent_cache_key_prefix, std::string(rep->persistent_cache_key_prefix,
rep->persistent_cache_key_prefix_size), rep->persistent_cache_key_prefix_size),
rep->ioptions.statistics); rep->ioptions.statistics);
// Read meta index // Read meta index
std::unique_ptr<Block> meta; std::unique_ptr<Block> meta;
@ -585,8 +584,7 @@ Status BlockBasedTable::Open(const ImmutableCFOptions& ioptions,
TableProperties* table_properties = nullptr; TableProperties* table_properties = nullptr;
if (s.ok()) { if (s.ok()) {
s = ReadProperties(meta_iter->value(), rep->file.get(), rep->footer, s = ReadProperties(meta_iter->value(), rep->file.get(), rep->footer,
rep->ioptions.env, rep->ioptions.info_log, rep->ioptions, &table_properties);
&table_properties);
} }
if (!s.ok()) { if (!s.ok()) {
@ -613,7 +611,7 @@ Status BlockBasedTable::Open(const ImmutableCFOptions& ioptions,
// true. // true.
unique_ptr<BlockContents> compression_dict_block{new BlockContents()}; unique_ptr<BlockContents> compression_dict_block{new BlockContents()};
s = rocksdb::ReadMetaBlock(rep->file.get(), file_size, s = rocksdb::ReadMetaBlock(rep->file.get(), file_size,
kBlockBasedTableMagicNumber, rep->ioptions.env, kBlockBasedTableMagicNumber, rep->ioptions,
rocksdb::kCompressionDictBlock, rocksdb::kCompressionDictBlock,
compression_dict_block.get()); compression_dict_block.get());
if (!s.ok()) { if (!s.ok()) {
@ -745,9 +743,9 @@ Status BlockBasedTable::ReadMetaBlock(Rep* rep,
std::unique_ptr<Block> meta; std::unique_ptr<Block> meta;
Status s = ReadBlockFromFile( Status s = ReadBlockFromFile(
rep->file.get(), rep->footer, ReadOptions(), rep->file.get(), rep->footer, ReadOptions(),
rep->footer.metaindex_handle(), &meta, rep->ioptions.env, rep->footer.metaindex_handle(), &meta, rep->ioptions,
true /* decompress */, Slice() /*compression dict*/, true /* decompress */, Slice() /*compression dict*/,
rep->persistent_cache_options, rep->ioptions.info_log); rep->persistent_cache_options);
if (!s.ok()) { if (!s.ok()) {
Log(InfoLogLevel::ERROR_LEVEL, rep->ioptions.info_log, Log(InfoLogLevel::ERROR_LEVEL, rep->ioptions.info_log,
@ -764,13 +762,14 @@ Status BlockBasedTable::ReadMetaBlock(Rep* rep,
Status BlockBasedTable::GetDataBlockFromCache( Status BlockBasedTable::GetDataBlockFromCache(
const Slice& block_cache_key, const Slice& compressed_block_cache_key, const Slice& block_cache_key, const Slice& compressed_block_cache_key,
Cache* block_cache, Cache* block_cache_compressed, Statistics* statistics, Cache* block_cache, Cache* block_cache_compressed,
const ReadOptions& read_options, const ImmutableCFOptions &ioptions, const ReadOptions& read_options,
BlockBasedTable::CachableEntry<Block>* block, uint32_t format_version, BlockBasedTable::CachableEntry<Block>* block, uint32_t format_version,
const Slice& compression_dict) { const Slice& compression_dict) {
Status s; Status s;
Block* compressed_block = nullptr; Block* compressed_block = nullptr;
Cache::Handle* block_cache_compressed_handle = nullptr; Cache::Handle* block_cache_compressed_handle = nullptr;
Statistics* statistics = ioptions.statistics;
// Lookup uncompressed cache first // Lookup uncompressed cache first
if (block_cache != nullptr) { if (block_cache != nullptr) {
@ -811,7 +810,8 @@ Status BlockBasedTable::GetDataBlockFromCache(
BlockContents contents; BlockContents contents;
s = UncompressBlockContents(compressed_block->data(), s = UncompressBlockContents(compressed_block->data(),
compressed_block->size(), &contents, compressed_block->size(), &contents,
format_version, compression_dict); format_version, compression_dict,
ioptions);
// Insert uncompressed block into block cache // Insert uncompressed block into block cache
if (s.ok()) { if (s.ok()) {
@ -840,7 +840,7 @@ Status BlockBasedTable::GetDataBlockFromCache(
Status BlockBasedTable::PutDataBlockToCache( Status BlockBasedTable::PutDataBlockToCache(
const Slice& block_cache_key, const Slice& compressed_block_cache_key, const Slice& block_cache_key, const Slice& compressed_block_cache_key,
Cache* block_cache, Cache* block_cache_compressed, Cache* block_cache, Cache* block_cache_compressed,
const ReadOptions& read_options, Statistics* statistics, const ReadOptions& read_options, const ImmutableCFOptions &ioptions,
CachableEntry<Block>* block, Block* raw_block, uint32_t format_version, CachableEntry<Block>* block, Block* raw_block, uint32_t format_version,
const Slice& compression_dict) { const Slice& compression_dict) {
assert(raw_block->compression_type() == kNoCompression || assert(raw_block->compression_type() == kNoCompression ||
@ -849,9 +849,10 @@ Status BlockBasedTable::PutDataBlockToCache(
Status s; Status s;
// Retrieve the uncompressed contents into a new buffer // Retrieve the uncompressed contents into a new buffer
BlockContents contents; BlockContents contents;
Statistics* statistics = ioptions.statistics;
if (raw_block->compression_type() != kNoCompression) { if (raw_block->compression_type() != kNoCompression) {
s = UncompressBlockContents(raw_block->data(), raw_block->size(), &contents, s = UncompressBlockContents(raw_block->data(), raw_block->size(), &contents,
format_version, compression_dict); format_version, compression_dict, ioptions);
} }
if (!s.ok()) { if (!s.ok()) {
delete raw_block; delete raw_block;
@ -913,7 +914,7 @@ FilterBlockReader* BlockBasedTable::ReadFilter(Rep* rep) {
} }
BlockContents block; BlockContents block;
if (!ReadBlockContents(rep->file.get(), rep->footer, ReadOptions(), if (!ReadBlockContents(rep->file.get(), rep->footer, ReadOptions(),
rep->filter_handle, &block, rep->ioptions.env, rep->filter_handle, &block, rep->ioptions,
false /* decompress */, Slice() /*compression dict*/, false /* decompress */, Slice() /*compression dict*/,
rep->persistent_cache_options) rep->persistent_cache_options)
.ok()) { .ok()) {
@ -1148,7 +1149,7 @@ InternalIterator* BlockBasedTable::NewDataBlockIterator(
} }
s = GetDataBlockFromCache( s = GetDataBlockFromCache(
key, ckey, block_cache, block_cache_compressed, statistics, ro, &block, key, ckey, block_cache, block_cache_compressed, rep->ioptions, ro, &block,
rep->table_options.format_version, compression_dict); rep->table_options.format_version, compression_dict);
if (block.value == nullptr && !no_io && ro.fill_cache) { if (block.value == nullptr && !no_io && ro.fill_cache) {
@ -1156,15 +1157,14 @@ InternalIterator* BlockBasedTable::NewDataBlockIterator(
{ {
StopWatch sw(rep->ioptions.env, statistics, READ_BLOCK_GET_MICROS); StopWatch sw(rep->ioptions.env, statistics, READ_BLOCK_GET_MICROS);
s = ReadBlockFromFile(rep->file.get(), rep->footer, ro, handle, s = ReadBlockFromFile(rep->file.get(), rep->footer, ro, handle,
&raw_block, rep->ioptions.env, &raw_block, rep->ioptions,
block_cache_compressed == nullptr, block_cache_compressed == nullptr,
compression_dict, rep->persistent_cache_options, compression_dict, rep->persistent_cache_options);
rep->ioptions.info_log);
} }
if (s.ok()) { if (s.ok()) {
s = PutDataBlockToCache(key, ckey, block_cache, block_cache_compressed, s = PutDataBlockToCache(key, ckey, block_cache, block_cache_compressed,
ro, statistics, &block, raw_block.release(), ro, rep->ioptions, &block, raw_block.release(),
rep->table_options.format_version, rep->table_options.format_version,
compression_dict); compression_dict);
} }
@ -1184,9 +1184,8 @@ InternalIterator* BlockBasedTable::NewDataBlockIterator(
} }
std::unique_ptr<Block> block_value; std::unique_ptr<Block> block_value;
s = ReadBlockFromFile(rep->file.get(), rep->footer, ro, handle, s = ReadBlockFromFile(rep->file.get(), rep->footer, ro, handle,
&block_value, rep->ioptions.env, true /* compress */, &block_value, rep->ioptions, true /* compress */,
compression_dict, rep->persistent_cache_options, compression_dict, rep->persistent_cache_options);
rep->ioptions.info_log);
if (s.ok()) { if (s.ok()) {
block.value = block_value.release(); block.value = block_value.release();
} }
@ -1510,8 +1509,9 @@ bool BlockBasedTable::TEST_KeyInCache(const ReadOptions& options,
handle, cache_key_storage); handle, cache_key_storage);
Slice ckey; Slice ckey;
s = GetDataBlockFromCache(cache_key, ckey, block_cache, nullptr, nullptr, s = GetDataBlockFromCache(cache_key, ckey, block_cache, nullptr,
options, &block, rep_->table_options.format_version, rep_->ioptions, options, &block,
rep_->table_options.format_version,
rep_->compression_dict_block rep_->compression_dict_block
? rep_->compression_dict_block->data ? rep_->compression_dict_block->data
: Slice()); : Slice());
@ -1544,11 +1544,8 @@ Status BlockBasedTable::CreateIndexReader(
} }
auto file = rep_->file.get(); auto file = rep_->file.get();
auto env = rep_->ioptions.env;
auto comparator = &rep_->internal_comparator; auto comparator = &rep_->internal_comparator;
const Footer& footer = rep_->footer; const Footer& footer = rep_->footer;
Statistics* stats = rep_->ioptions.statistics;
if (index_type_on_file == BlockBasedTableOptions::kHashSearch && if (index_type_on_file == BlockBasedTableOptions::kHashSearch &&
rep_->ioptions.prefix_extractor == nullptr) { rep_->ioptions.prefix_extractor == nullptr) {
Log(InfoLogLevel::WARN_LEVEL, rep_->ioptions.info_log, Log(InfoLogLevel::WARN_LEVEL, rep_->ioptions.info_log,
@ -1561,8 +1558,8 @@ Status BlockBasedTable::CreateIndexReader(
switch (index_type_on_file) { switch (index_type_on_file) {
case BlockBasedTableOptions::kBinarySearch: { case BlockBasedTableOptions::kBinarySearch: {
return BinarySearchIndexReader::Create( return BinarySearchIndexReader::Create(
file, footer, footer.index_handle(), env, comparator, index_reader, file, footer, footer.index_handle(), rep_->ioptions, comparator,
rep_->persistent_cache_options, stats); index_reader, rep_->persistent_cache_options);
} }
case BlockBasedTableOptions::kHashSearch: { case BlockBasedTableOptions::kHashSearch: {
std::unique_ptr<Block> meta_guard; std::unique_ptr<Block> meta_guard;
@ -1577,8 +1574,8 @@ Status BlockBasedTable::CreateIndexReader(
"Unable to read the metaindex block." "Unable to read the metaindex block."
" Fall back to binary search index."); " Fall back to binary search index.");
return BinarySearchIndexReader::Create( return BinarySearchIndexReader::Create(
file, footer, footer.index_handle(), env, comparator, file, footer, footer.index_handle(), rep_->ioptions, comparator,
index_reader, rep_->persistent_cache_options, stats); index_reader, rep_->persistent_cache_options);
} }
meta_index_iter = meta_iter_guard.get(); meta_index_iter = meta_iter_guard.get();
} }
@ -1588,10 +1585,9 @@ Status BlockBasedTable::CreateIndexReader(
rep_->internal_prefix_transform.reset( rep_->internal_prefix_transform.reset(
new InternalKeySliceTransform(rep_->ioptions.prefix_extractor)); new InternalKeySliceTransform(rep_->ioptions.prefix_extractor));
return HashIndexReader::Create( return HashIndexReader::Create(
rep_->internal_prefix_transform.get(), footer, file, env, comparator, rep_->internal_prefix_transform.get(), footer, file, rep_->ioptions,
footer.index_handle(), meta_index_iter, index_reader, comparator, footer.index_handle(), meta_index_iter, index_reader,
rep_->hash_index_allow_collision, rep_->persistent_cache_options, rep_->hash_index_allow_collision, rep_->persistent_cache_options);
stats);
} }
default: { default: {
std::string error_message = std::string error_message =
@ -1711,7 +1707,7 @@ Status BlockBasedTable::DumpTable(WritableFile* out_file) {
BlockContents block; BlockContents block;
if (ReadBlockContents( if (ReadBlockContents(
rep_->file.get(), rep_->footer, ReadOptions(), handle, &block, rep_->file.get(), rep_->footer, ReadOptions(), handle, &block,
rep_->ioptions.env, false /*decompress*/, rep_->ioptions, false /*decompress*/,
Slice() /*compression dict*/, rep_->persistent_cache_options) Slice() /*compression dict*/, rep_->persistent_cache_options)
.ok()) { .ok()) {
rep_->filter.reset(new BlockBasedFilterBlockReader( rep_->filter.reset(new BlockBasedFilterBlockReader(

6
table/block_based_table_reader.h
Unescape View File

@ -177,8 +177,8 @@ class BlockBasedTable : public TableReader {
// dictionary. // dictionary.
static Status GetDataBlockFromCache( static Status GetDataBlockFromCache(
const Slice& block_cache_key, const Slice& compressed_block_cache_key, const Slice& block_cache_key, const Slice& compressed_block_cache_key,
Cache* block_cache, Cache* block_cache_compressed, Statistics* statistics, Cache* block_cache, Cache* block_cache_compressed,
const ReadOptions& read_options, const ImmutableCFOptions &ioptions, const ReadOptions& read_options,
BlockBasedTable::CachableEntry<Block>* block, uint32_t format_version, BlockBasedTable::CachableEntry<Block>* block, uint32_t format_version,
const Slice& compression_dict); const Slice& compression_dict);
@ -195,7 +195,7 @@ class BlockBasedTable : public TableReader {
static Status PutDataBlockToCache( static Status PutDataBlockToCache(
const Slice& block_cache_key, const Slice& compressed_block_cache_key, const Slice& block_cache_key, const Slice& compressed_block_cache_key,
Cache* block_cache, Cache* block_cache_compressed, Cache* block_cache, Cache* block_cache_compressed,
const ReadOptions& read_options, Statistics* statistics, const ReadOptions& read_options, const ImmutableCFOptions &ioptions,
CachableEntry<Block>* block, Block* raw_block, uint32_t format_version, CachableEntry<Block>* block, Block* raw_block, uint32_t format_version,
const Slice& compression_dict); const Slice& compression_dict);

6
table/cuckoo_table_builder_test.cc
Unescape View File

@ -49,12 +49,16 @@ class CuckooBuilderTest : public testing::Test {
uint64_t read_file_size; uint64_t read_file_size;
ASSERT_OK(env_->GetFileSize(fname, &read_file_size)); ASSERT_OK(env_->GetFileSize(fname, &read_file_size));
Options options;
options.allow_mmap_reads = true;
ImmutableCFOptions ioptions(options);
// Assert Table Properties. // Assert Table Properties.
TableProperties* props = nullptr; TableProperties* props = nullptr;
unique_ptr<RandomAccessFileReader> file_reader( unique_ptr<RandomAccessFileReader> file_reader(
new RandomAccessFileReader(std::move(read_file))); new RandomAccessFileReader(std::move(read_file)));
ASSERT_OK(ReadTableProperties(file_reader.get(), read_file_size, ASSERT_OK(ReadTableProperties(file_reader.get(), read_file_size,
kCuckooTableMagicNumber, env_, nullptr, kCuckooTableMagicNumber, ioptions,
&props)); &props));
// Check unused bucket. // Check unused bucket.
std::string unused_key = props->user_collected_properties[ std::string unused_key = props->user_collected_properties[

2
table/cuckoo_table_reader.cc
Unescape View File

@ -45,7 +45,7 @@ CuckooTableReader::CuckooTableReader(
} }
TableProperties* props = nullptr; TableProperties* props = nullptr;
status_ = ReadTableProperties(file_.get(), file_size, kCuckooTableMagicNumber, status_ = ReadTableProperties(file_.get(), file_size, kCuckooTableMagicNumber,
ioptions.env, ioptions.info_log, &props); ioptions, &props);
if (!status_.ok()) { if (!status_.ok()) {
return; return;
} }

41
table/format.cc
Unescape View File

@ -23,6 +23,9 @@
#include "util/perf_context_imp.h" #include "util/perf_context_imp.h"
#include "util/string_util.h" #include "util/string_util.h"
#include "util/xxhash.h" #include "util/xxhash.h"
#include "util/statistics.h"
#include "util/stop_watch.h"
namespace rocksdb { namespace rocksdb {
@ -39,6 +42,11 @@ const uint64_t kPlainTableMagicNumber = 0;
#endif #endif
const uint32_t DefaultStackBufferSize = 5000; const uint32_t DefaultStackBufferSize = 5000;
bool ShouldReportDetailedTime(Env* env, Statistics* stats) {
return env != nullptr && stats != nullptr &&
stats->stats_level_ > kExceptDetailedTimers;
}
void BlockHandle::EncodeTo(std::string* dst) const { void BlockHandle::EncodeTo(std::string* dst) const {
// Sanity check that all fields have been set // Sanity check that all fields have been set
assert(offset_ != ~static_cast<uint64_t>(0)); assert(offset_ != ~static_cast<uint64_t>(0));
@ -297,10 +305,10 @@ Status ReadBlock(RandomAccessFileReader* file, const Footer& footer,
Status ReadBlockContents(RandomAccessFileReader* file, const Footer& footer, Status ReadBlockContents(RandomAccessFileReader* file, const Footer& footer,
const ReadOptions& read_options, const ReadOptions& read_options,
const BlockHandle& handle, BlockContents* contents, const BlockHandle& handle, BlockContents* contents,
Env* env, bool decompression_requested, const ImmutableCFOptions &ioptions,
bool decompression_requested,
const Slice& compression_dict, const Slice& compression_dict,
const PersistentCacheOptions& cache_options, const PersistentCacheOptions& cache_options) {
Logger* info_log) {
Status status; Status status;
Slice slice; Slice slice;
size_t n = static_cast<size_t>(handle.size()); size_t n = static_cast<size_t>(handle.size());
@ -318,9 +326,9 @@ Status ReadBlockContents(RandomAccessFileReader* file, const Footer& footer,
return status; return status;
} else { } else {
// uncompressed page is not found // uncompressed page is not found
if (info_log && !status.IsNotFound()) { if (ioptions.info_log && !status.IsNotFound()) {
assert(!status.ok()); assert(!status.ok());
Log(InfoLogLevel::INFO_LEVEL, info_log, Log(InfoLogLevel::INFO_LEVEL, ioptions.info_log,
"Error reading from persistent cache. %s", "Error reading from persistent cache. %s",
status.ToString().c_str()); status.ToString().c_str());
} }
@ -341,9 +349,9 @@ Status ReadBlockContents(RandomAccessFileReader* file, const Footer& footer,
used_buf = heap_buf.get(); used_buf = heap_buf.get();
slice = Slice(heap_buf.get(), n); slice = Slice(heap_buf.get(), n);
} else { } else {
if (info_log && !status.IsNotFound()) { if (ioptions.info_log && !status.IsNotFound()) {
assert(!status.ok()); assert(!status.ok());
Log(InfoLogLevel::INFO_LEVEL, info_log, Log(InfoLogLevel::INFO_LEVEL, ioptions.info_log,
"Error reading from persistent cache. %s", status.ToString().c_str()); "Error reading from persistent cache. %s", status.ToString().c_str());
} }
// cache miss read from device // cache miss read from device
@ -378,7 +386,8 @@ Status ReadBlockContents(RandomAccessFileReader* file, const Footer& footer,
if (decompression_requested && compression_type != kNoCompression) { if (decompression_requested && compression_type != kNoCompression) {
// compressed page, uncompress, update cache // compressed page, uncompress, update cache
status = UncompressBlockContents(slice.data(), n, contents, status = UncompressBlockContents(slice.data(), n, contents,
footer.version(), compression_dict); footer.version(), compression_dict,
ioptions);
} else if (slice.data() != used_buf) { } else if (slice.data() != used_buf) {
// the slice content is not the buffer provided // the slice content is not the buffer provided
*contents = BlockContents(Slice(slice.data(), n), false, compression_type); *contents = BlockContents(Slice(slice.data(), n), false, compression_type);
@ -405,11 +414,13 @@ Status ReadBlockContents(RandomAccessFileReader* file, const Footer& footer,
Status UncompressBlockContentsForCompressionType( Status UncompressBlockContentsForCompressionType(
const char* data, size_t n, BlockContents* contents, const char* data, size_t n, BlockContents* contents,
uint32_t format_version, const Slice& compression_dict, uint32_t format_version, const Slice& compression_dict,
CompressionType compression_type) { CompressionType compression_type, const ImmutableCFOptions &ioptions) {
std::unique_ptr<char[]> ubuf; std::unique_ptr<char[]> ubuf;
assert(compression_type != kNoCompression && "Invalid compression type"); assert(compression_type != kNoCompression && "Invalid compression type");
StopWatchNano timer(ioptions.env,
ShouldReportDetailedTime(ioptions.env, ioptions.statistics));
int decompress_size = 0; int decompress_size = 0;
switch (compression_type) { switch (compression_type) {
case kSnappyCompression: { case kSnappyCompression: {
@ -501,6 +512,13 @@ Status UncompressBlockContentsForCompressionType(
return Status::Corruption("bad block type"); return Status::Corruption("bad block type");
} }
if(ShouldReportDetailedTime(ioptions.env, ioptions.statistics)){
MeasureTime(ioptions.statistics, DECOMPRESSION_TIMES_NANOS,
timer.ElapsedNanos());
MeasureTime(ioptions.statistics, BYTES_DECOMPRESSED, contents->data.size());
RecordTick(ioptions.statistics, NUMBER_BLOCK_DECOMPRESSED);
}
return Status::OK(); return Status::OK();
} }
@ -513,11 +531,12 @@ Status UncompressBlockContentsForCompressionType(
// format_version is the block format as defined in include/rocksdb/table.h // format_version is the block format as defined in include/rocksdb/table.h
Status UncompressBlockContents(const char* data, size_t n, Status UncompressBlockContents(const char* data, size_t n,
BlockContents* contents, uint32_t format_version, BlockContents* contents, uint32_t format_version,
const Slice& compression_dict) { const Slice& compression_dict,
const ImmutableCFOptions &ioptions) {
assert(data[n] != kNoCompression); assert(data[n] != kNoCompression);
return UncompressBlockContentsForCompressionType( return UncompressBlockContentsForCompressionType(
data, n, contents, format_version, compression_dict, data, n, contents, format_version, compression_dict,
(CompressionType)data[n]); (CompressionType)data[n], ioptions);
} }
} // namespace rocksdb } // namespace rocksdb

14
table/format.h
Unescape View File

@ -24,6 +24,8 @@ class Block;
class RandomAccessFile; class RandomAccessFile;
struct ReadOptions; struct ReadOptions;
extern bool ShouldReportDetailedTime(Env* env, Statistics* stats);
// the length of the magic number in bytes. // the length of the magic number in bytes.
const int kMagicNumberLengthByte = 8; const int kMagicNumberLengthByte = 8;
@ -212,10 +214,9 @@ struct BlockContents {
extern Status ReadBlockContents( extern Status ReadBlockContents(
RandomAccessFileReader* file, const Footer& footer, RandomAccessFileReader* file, const Footer& footer,
const ReadOptions& options, const BlockHandle& handle, const ReadOptions& options, const BlockHandle& handle,
BlockContents* contents, Env* env, bool do_uncompress = true, BlockContents* contents, const ImmutableCFOptions &ioptions,
const Slice& compression_dict = Slice(), bool do_uncompress = true, const Slice& compression_dict = Slice(),
const PersistentCacheOptions& cache_options = PersistentCacheOptions(), const PersistentCacheOptions& cache_options = PersistentCacheOptions());
Logger* info_log = nullptr);
// The 'data' points to the raw block contents read in from file. // The 'data' points to the raw block contents read in from file.
// This method allocates a new heap buffer and the raw block // This method allocates a new heap buffer and the raw block
@ -227,7 +228,8 @@ extern Status ReadBlockContents(
extern Status UncompressBlockContents(const char* data, size_t n, extern Status UncompressBlockContents(const char* data, size_t n,
BlockContents* contents, BlockContents* contents,
uint32_t compress_format_version, uint32_t compress_format_version,
const Slice& compression_dict); const Slice& compression_dict,
const ImmutableCFOptions &ioptions);
// This is an extension to UncompressBlockContents that accepts // This is an extension to UncompressBlockContents that accepts
// a specific compression type. This is used by un-wrapped blocks // a specific compression type. This is used by un-wrapped blocks
@ -235,7 +237,7 @@ extern Status UncompressBlockContents(const char* data, size_t n,
extern Status UncompressBlockContentsForCompressionType( extern Status UncompressBlockContentsForCompressionType(
const char* data, size_t n, BlockContents* contents, const char* data, size_t n, BlockContents* contents,
uint32_t compress_format_version, const Slice& compression_dict, uint32_t compress_format_version, const Slice& compression_dict,
CompressionType compression_type); CompressionType compression_type, const ImmutableCFOptions &ioptions);
// Implementation details follow. Clients should ignore, // Implementation details follow. Clients should ignore,

30
table/meta_blocks.cc
Unescape View File

@ -150,7 +150,7 @@ bool NotifyCollectTableCollectorsOnFinish(
} }
Status ReadProperties(const Slice& handle_value, RandomAccessFileReader* file, Status ReadProperties(const Slice& handle_value, RandomAccessFileReader* file,
const Footer& footer, Env* env, Logger* logger, const Footer& footer, const ImmutableCFOptions &ioptions,
TableProperties** table_properties) { TableProperties** table_properties) {
assert(table_properties); assert(table_properties);
@ -165,7 +165,7 @@ Status ReadProperties(const Slice& handle_value, RandomAccessFileReader* file,
read_options.verify_checksums = false; read_options.verify_checksums = false;
Status s; Status s;
s = ReadBlockContents(file, footer, read_options, handle, &block_contents, s = ReadBlockContents(file, footer, read_options, handle, &block_contents,
env, false /* decompress */); ioptions, false /* decompress */);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
@ -219,7 +219,8 @@ Status ReadProperties(const Slice& handle_value, RandomAccessFileReader* file,
auto error_msg = auto error_msg =
"Detect malformed value in properties meta-block:" "Detect malformed value in properties meta-block:"
"\tkey: " + key + "\tval: " + raw_val.ToString(); "\tkey: " + key + "\tval: " + raw_val.ToString();
Log(InfoLogLevel::ERROR_LEVEL, logger, "%s", error_msg.c_str()); Log(InfoLogLevel::ERROR_LEVEL, ioptions.info_log, "%s",
error_msg.c_str());
continue; continue;
} }
*(pos->second) = val; *(pos->second) = val;
@ -251,8 +252,9 @@ Status ReadProperties(const Slice& handle_value, RandomAccessFileReader* file,
} }
Status ReadTableProperties(RandomAccessFileReader* file, uint64_t file_size, Status ReadTableProperties(RandomAccessFileReader* file, uint64_t file_size,
uint64_t table_magic_number, Env* env, uint64_t table_magic_number,
Logger* info_log, TableProperties** properties) { const ImmutableCFOptions &ioptions,
TableProperties** properties) {
// -- Read metaindex block // -- Read metaindex block
Footer footer; Footer footer;
auto s = ReadFooterFromFile(file, file_size, &footer, table_magic_number); auto s = ReadFooterFromFile(file, file_size, &footer, table_magic_number);
@ -265,7 +267,7 @@ Status ReadTableProperties(RandomAccessFileReader* file, uint64_t file_size,
ReadOptions read_options; ReadOptions read_options;
read_options.verify_checksums = false; read_options.verify_checksums = false;
s = ReadBlockContents(file, footer, read_options, metaindex_handle, s = ReadBlockContents(file, footer, read_options, metaindex_handle,
&metaindex_contents, env, false /* decompress */); &metaindex_contents, ioptions, false /* decompress */);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
@ -282,8 +284,7 @@ Status ReadTableProperties(RandomAccessFileReader* file, uint64_t file_size,
TableProperties table_properties; TableProperties table_properties;
if (found_properties_block == true) { if (found_properties_block == true) {
s = ReadProperties(meta_iter->value(), file, footer, env, info_log, s = ReadProperties(meta_iter->value(), file, footer, ioptions, properties);
properties);
} else { } else {
s = Status::NotFound(); s = Status::NotFound();
} }
@ -305,7 +306,8 @@ Status FindMetaBlock(InternalIterator* meta_index_iter,
} }
Status FindMetaBlock(RandomAccessFileReader* file, uint64_t file_size, Status FindMetaBlock(RandomAccessFileReader* file, uint64_t file_size,
uint64_t table_magic_number, Env* env, uint64_t table_magic_number,
const ImmutableCFOptions &ioptions,
const std::string& meta_block_name, const std::string& meta_block_name,
BlockHandle* block_handle) { BlockHandle* block_handle) {
Footer footer; Footer footer;
@ -319,7 +321,7 @@ Status FindMetaBlock(RandomAccessFileReader* file, uint64_t file_size,
ReadOptions read_options; ReadOptions read_options;
read_options.verify_checksums = false; read_options.verify_checksums = false;
s = ReadBlockContents(file, footer, read_options, metaindex_handle, s = ReadBlockContents(file, footer, read_options, metaindex_handle,
&metaindex_contents, env, false /* do decompression */); &metaindex_contents, ioptions, false /* do decompression */);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
@ -332,7 +334,8 @@ Status FindMetaBlock(RandomAccessFileReader* file, uint64_t file_size,
} }
Status ReadMetaBlock(RandomAccessFileReader* file, uint64_t file_size, Status ReadMetaBlock(RandomAccessFileReader* file, uint64_t file_size,
uint64_t table_magic_number, Env* env, uint64_t table_magic_number,
const ImmutableCFOptions &ioptions,
const std::string& meta_block_name, const std::string& meta_block_name,
BlockContents* contents) { BlockContents* contents) {
Status status; Status status;
@ -348,7 +351,8 @@ Status ReadMetaBlock(RandomAccessFileReader* file, uint64_t file_size,
ReadOptions read_options; ReadOptions read_options;
read_options.verify_checksums = false; read_options.verify_checksums = false;
status = ReadBlockContents(file, footer, read_options, metaindex_handle, status = ReadBlockContents(file, footer, read_options, metaindex_handle,
&metaindex_contents, env, false /* decompress */); &metaindex_contents, ioptions,
false /* decompress */);
if (!status.ok()) { if (!status.ok()) {
return status; return status;
} }
@ -368,7 +372,7 @@ Status ReadMetaBlock(RandomAccessFileReader* file, uint64_t file_size,
// Reading metablock // Reading metablock
return ReadBlockContents(file, footer, read_options, block_handle, contents, return ReadBlockContents(file, footer, read_options, block_handle, contents,
env, false /* decompress */); ioptions, false /* decompress */);
} }
} // namespace rocksdb } // namespace rocksdb

13
table/meta_blocks.h
Unescape View File

@ -94,7 +94,7 @@ bool NotifyCollectTableCollectorsOnFinish(
// *table_properties will point to a heap-allocated TableProperties // *table_properties will point to a heap-allocated TableProperties
// object, otherwise value of `table_properties` will not be modified. // object, otherwise value of `table_properties` will not be modified.
Status ReadProperties(const Slice& handle_value, RandomAccessFileReader* file, Status ReadProperties(const Slice& handle_value, RandomAccessFileReader* file,
const Footer& footer, Env* env, Logger* logger, const Footer& footer, const ImmutableCFOptions &ioptions,
TableProperties** table_properties); TableProperties** table_properties);
// Directly read the properties from the properties block of a plain table. // Directly read the properties from the properties block of a plain table.
@ -102,8 +102,9 @@ Status ReadProperties(const Slice& handle_value, RandomAccessFileReader* file,
// *table_properties will point to a heap-allocated TableProperties // *table_properties will point to a heap-allocated TableProperties
// object, otherwise value of `table_properties` will not be modified. // object, otherwise value of `table_properties` will not be modified.
Status ReadTableProperties(RandomAccessFileReader* file, uint64_t file_size, Status ReadTableProperties(RandomAccessFileReader* file, uint64_t file_size,
uint64_t table_magic_number, Env* env, uint64_t table_magic_number,
Logger* info_log, TableProperties** properties); const ImmutableCFOptions &ioptions,
TableProperties** properties);
// Find the meta block from the meta index block. // Find the meta block from the meta index block.
Status FindMetaBlock(InternalIterator* meta_index_iter, Status FindMetaBlock(InternalIterator* meta_index_iter,
@ -112,7 +113,8 @@ Status FindMetaBlock(InternalIterator* meta_index_iter,
// Find the meta block // Find the meta block
Status FindMetaBlock(RandomAccessFileReader* file, uint64_t file_size, Status FindMetaBlock(RandomAccessFileReader* file, uint64_t file_size,
uint64_t table_magic_number, Env* env, uint64_t table_magic_number,
const ImmutableCFOptions &ioptions,
const std::string& meta_block_name, const std::string& meta_block_name,
BlockHandle* block_handle); BlockHandle* block_handle);
@ -120,7 +122,8 @@ Status FindMetaBlock(RandomAccessFileReader* file, uint64_t file_size,
// from `file` and initialize `contents` with contents of this block. // from `file` and initialize `contents` with contents of this block.
// Return Status::OK in case of success. // Return Status::OK in case of success.
Status ReadMetaBlock(RandomAccessFileReader* file, uint64_t file_size, Status ReadMetaBlock(RandomAccessFileReader* file, uint64_t file_size,
uint64_t table_magic_number, Env* env, uint64_t table_magic_number,
const ImmutableCFOptions &ioptions,
const std::string& meta_block_name, const std::string& meta_block_name,
BlockContents* contents); BlockContents* contents);

6
table/plain_table_reader.cc
Unescape View File

@ -128,7 +128,7 @@ Status PlainTableReader::Open(const ImmutableCFOptions& ioptions,
TableProperties* props = nullptr; TableProperties* props = nullptr;
auto s = ReadTableProperties(file.get(), file_size, kPlainTableMagicNumber, auto s = ReadTableProperties(file.get(), file_size, kPlainTableMagicNumber,
ioptions.env, ioptions.info_log, &props); ioptions, &props);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
@ -293,13 +293,13 @@ Status PlainTableReader::PopulateIndex(TableProperties* props,
BlockContents bloom_block_contents; BlockContents bloom_block_contents;
auto s = ReadMetaBlock(file_info_.file.get(), file_size_, auto s = ReadMetaBlock(file_info_.file.get(), file_size_,
kPlainTableMagicNumber, ioptions_.env, kPlainTableMagicNumber, ioptions_,
BloomBlockBuilder::kBloomBlock, &bloom_block_contents); BloomBlockBuilder::kBloomBlock, &bloom_block_contents);
bool index_in_file = s.ok(); bool index_in_file = s.ok();
BlockContents index_block_contents; BlockContents index_block_contents;
s = ReadMetaBlock( s = ReadMetaBlock(
file_info_.file.get(), file_size_, kPlainTableMagicNumber, ioptions_.env, file_info_.file.get(), file_size_, kPlainTableMagicNumber, ioptions_,
PlainTableIndexBuilder::kPlainTableIndexBlock, &index_block_contents); PlainTableIndexBuilder::kPlainTableIndexBlock, &index_block_contents);
index_in_file &= s.ok(); index_in_file &= s.ok();

2
table/table_test.cc
Unescape View File

@ -2064,7 +2064,7 @@ TEST_F(PlainTableTest, BasicPlainTableProperties) {
TableProperties* props = nullptr; TableProperties* props = nullptr;
auto s = ReadTableProperties(file_reader.get(), ss->contents().size(), auto s = ReadTableProperties(file_reader.get(), ss->contents().size(),
kPlainTableMagicNumber, Env::Default(), nullptr, kPlainTableMagicNumber, ioptions,
&props); &props);
std::unique_ptr<TableProperties> props_guard(props); std::unique_ptr<TableProperties> props_guard(props);
ASSERT_OK(s); ASSERT_OK(s);

3
tools/sst_dump_tool.cc
Unescape View File

@ -219,8 +219,7 @@ Status SstFileReader::ReadTableProperties(uint64_t table_magic_number,
uint64_t file_size) { uint64_t file_size) {
TableProperties* table_properties = nullptr; TableProperties* table_properties = nullptr;
Status s = rocksdb::ReadTableProperties(file, file_size, table_magic_number, Status s = rocksdb::ReadTableProperties(file, file_size, table_magic_number,
options_.env, options_.info_log.get(), ioptions_, &table_properties);
&table_properties);
if (s.ok()) { if (s.ok()) {
table_properties_.reset(table_properties); table_properties_.reset(table_properties);
} else { } else {

6
util/coding.h
Unescape View File

@ -154,7 +154,8 @@ inline void EncodeFixed64(char* buf, uint64_t value) {
// Pull the last 8 bits and cast it to a character // Pull the last 8 bits and cast it to a character
inline void PutFixed32(std::string* dst, uint32_t value) { inline void PutFixed32(std::string* dst, uint32_t value) {
#if __BYTE_ORDER__ == __LITTLE_ENDIAN__ #if __BYTE_ORDER__ == __LITTLE_ENDIAN__
dst->append(static_cast<const char*>(&value), sizeof(value)); dst->append(const_cast<const char*>(reinterpret_cast<char*>(&value)),
sizeof(value));
#else #else
char buf[sizeof(value)]; char buf[sizeof(value)];
EncodeFixed32(buf, value); EncodeFixed32(buf, value);
@ -164,7 +165,8 @@ inline void PutFixed32(std::string* dst, uint32_t value) {
inline void PutFixed64(std::string* dst, uint64_t value) { inline void PutFixed64(std::string* dst, uint64_t value) {
#if __BYTE_ORDER__ == __LITTLE_ENDIAN__ #if __BYTE_ORDER__ == __LITTLE_ENDIAN__
dst->append(static_const<const char*>(&value), sizeof(value)); dst->append(const_cast<const char*>(reinterpret_cast<char*>(&value)),
sizeof(value));
#else #else
char buf[sizeof(value)]; char buf[sizeof(value)];
EncodeFixed64(buf, value); EncodeFixed64(buf, value);

Write Preview
Loading…
Cancel
Save