Apply `sample_for_compression` to all block-based tables (#8105)

Summary:
Previously it only applied to block-based tables generated by flush. This restriction
was undocumented and blocked a new use case. Now compression sampling
applies to all block-based tables we generate when it is enabled.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/8105

Test Plan: new unit test

Reviewed By: riversand963

Differential Revision: D27317275

Pulled By: ajkr

fbshipit-source-id: cd9fcc5178d6515e8cb59c6facb5ac01893cb5b0
main
Andrew Kryczka 4 years ago committed by Facebook GitHub Bot
parent 45c65d6dcf
commit c20a7cd6c7
  1. 3
      HISTORY.md
  2. 26
      db/builder.cc
  3. 2
      db/builder.h
  4. 1
      db/compaction/compaction_job.cc
  5. 1
      db/db_impl/db_impl_open.cc
  6. 107
      db/db_properties_test.cc
  7. 3
      db/flush_job.cc
  8. 13
      db/repair.cc
  9. 8
      db/table_properties_collector.cc
  10. 12
      db/table_properties_collector.h
  11. 9
      db/table_properties_collector_test.cc
  12. 2
      db/version_set_test.cc
  13. 6
      include/rocksdb/table_properties.h
  14. 22
      table/block_based/block_based_table_builder.cc
  15. 1
      table/block_based/block_based_table_builder.h
  16. 1
      table/block_based/block_based_table_factory.cc
  17. 6
      table/block_based/block_based_table_reader_test.cc
  18. 5
      table/block_based/data_block_hash_index_test.cc
  19. 6
      table/block_fetcher_test.cc
  20. 8
      table/meta_blocks.cc
  21. 4
      table/meta_blocks.h
  22. 8
      table/sst_file_dumper.cc
  23. 9
      table/sst_file_writer.cc
  24. 6
      table/sst_file_writer_collectors.h
  25. 4
      table/table_builder.h
  26. 5
      table/table_reader_bench.cc
  27. 52
      table/table_test.cc
  28. 5
      tools/sst_dump_test.cc

@ -1,5 +1,8 @@
# Rocksdb Change Log # Rocksdb Change Log
## Unreleased ## Unreleased
### Behavior Changes
* `ColumnFamilyOptions::sample_for_compression` now takes effect for creation of all block-based tables. Previously it only took effect for block-based tables created by flush.
### Bug Fixes ### Bug Fixes
* Use thread-safe `strerror_r()` to get error messages. * Use thread-safe `strerror_r()` to get error messages.

@ -52,8 +52,8 @@ TableBuilder* NewTableBuilder(
int_tbl_prop_collector_factories, int_tbl_prop_collector_factories,
uint32_t column_family_id, const std::string& column_family_name, uint32_t column_family_id, const std::string& column_family_name,
WritableFileWriter* file, const CompressionType compression_type, WritableFileWriter* file, const CompressionType compression_type,
uint64_t sample_for_compression, const CompressionOptions& compression_opts, const CompressionOptions& compression_opts, int level,
int level, const bool skip_filters, const uint64_t creation_time, const bool skip_filters, const uint64_t creation_time,
const uint64_t oldest_key_time, const uint64_t target_file_size, const uint64_t oldest_key_time, const uint64_t target_file_size,
const uint64_t file_creation_time, const std::string& db_id, const uint64_t file_creation_time, const std::string& db_id,
const std::string& db_session_id) { const std::string& db_session_id) {
@ -63,10 +63,10 @@ TableBuilder* NewTableBuilder(
return ioptions.table_factory->NewTableBuilder( return ioptions.table_factory->NewTableBuilder(
TableBuilderOptions(ioptions, moptions, internal_comparator, TableBuilderOptions(ioptions, moptions, internal_comparator,
int_tbl_prop_collector_factories, compression_type, int_tbl_prop_collector_factories, compression_type,
sample_for_compression, compression_opts, compression_opts, skip_filters, column_family_name,
skip_filters, column_family_name, level, level, creation_time, oldest_key_time,
creation_time, oldest_key_time, target_file_size, target_file_size, file_creation_time, db_id,
file_creation_time, db_id, db_session_id), db_session_id),
column_family_id, file); column_family_id, file);
} }
@ -85,11 +85,10 @@ Status BuildTable(
std::vector<SequenceNumber> snapshots, std::vector<SequenceNumber> snapshots,
SequenceNumber earliest_write_conflict_snapshot, SequenceNumber earliest_write_conflict_snapshot,
SnapshotChecker* snapshot_checker, const CompressionType compression, SnapshotChecker* snapshot_checker, const CompressionType compression,
uint64_t sample_for_compression, const CompressionOptions& compression_opts, const CompressionOptions& compression_opts, bool paranoid_file_checks,
bool paranoid_file_checks, InternalStats* internal_stats, InternalStats* internal_stats, TableFileCreationReason reason,
TableFileCreationReason reason, IOStatus* io_status, IOStatus* io_status, const std::shared_ptr<IOTracer>& io_tracer,
const std::shared_ptr<IOTracer>& io_tracer, EventLogger* event_logger, EventLogger* event_logger, int job_id, const Env::IOPriority io_priority,
int job_id, const Env::IOPriority io_priority,
TableProperties* table_properties, int level, const uint64_t creation_time, TableProperties* table_properties, int level, const uint64_t creation_time,
const uint64_t oldest_key_time, Env::WriteLifeTimeHint write_hint, const uint64_t oldest_key_time, Env::WriteLifeTimeHint write_hint,
const uint64_t file_creation_time, const std::string& db_id, const uint64_t file_creation_time, const std::string& db_id,
@ -163,9 +162,8 @@ Status BuildTable(
builder = NewTableBuilder( builder = NewTableBuilder(
ioptions, mutable_cf_options, internal_comparator, ioptions, mutable_cf_options, internal_comparator,
int_tbl_prop_collector_factories, column_family_id, int_tbl_prop_collector_factories, column_family_id,
column_family_name, file_writer.get(), compression, column_family_name, file_writer.get(), compression, compression_opts,
sample_for_compression, compression_opts, level, level, false /* skip_filters */, creation_time, oldest_key_time,
false /* skip_filters */, creation_time, oldest_key_time,
0 /*target_file_size*/, file_creation_time, db_id, db_session_id); 0 /*target_file_size*/, file_creation_time, db_id, db_session_id);
} }

@ -50,7 +50,6 @@ TableBuilder* NewTableBuilder(
int_tbl_prop_collector_factories, int_tbl_prop_collector_factories,
uint32_t column_family_id, const std::string& column_family_name, uint32_t column_family_id, const std::string& column_family_name,
WritableFileWriter* file, const CompressionType compression_type, WritableFileWriter* file, const CompressionType compression_type,
const uint64_t sample_for_compression,
const CompressionOptions& compression_opts, int level, const CompressionOptions& compression_opts, int level,
const bool skip_filters = false, const uint64_t creation_time = 0, const bool skip_filters = false, const uint64_t creation_time = 0,
const uint64_t oldest_key_time = 0, const uint64_t target_file_size = 0, const uint64_t oldest_key_time = 0, const uint64_t target_file_size = 0,
@ -80,7 +79,6 @@ extern Status BuildTable(
std::vector<SequenceNumber> snapshots, std::vector<SequenceNumber> snapshots,
SequenceNumber earliest_write_conflict_snapshot, SequenceNumber earliest_write_conflict_snapshot,
SnapshotChecker* snapshot_checker, const CompressionType compression, SnapshotChecker* snapshot_checker, const CompressionType compression,
const uint64_t sample_for_compression,
const CompressionOptions& compression_opts, bool paranoid_file_checks, const CompressionOptions& compression_opts, bool paranoid_file_checks,
InternalStats* internal_stats, TableFileCreationReason reason, InternalStats* internal_stats, TableFileCreationReason reason,
IOStatus* io_status, const std::shared_ptr<IOTracer>& io_tracer, IOStatus* io_status, const std::shared_ptr<IOTracer>& io_tracer,

@ -1770,7 +1770,6 @@ Status CompactionJob::OpenCompactionOutputFile(
cfd->internal_comparator(), cfd->int_tbl_prop_collector_factories(), cfd->internal_comparator(), cfd->int_tbl_prop_collector_factories(),
cfd->GetID(), cfd->GetName(), sub_compact->outfile.get(), cfd->GetID(), cfd->GetName(), sub_compact->outfile.get(),
sub_compact->compaction->output_compression(), sub_compact->compaction->output_compression(),
0 /*sample_for_compression */,
sub_compact->compaction->output_compression_opts(), sub_compact->compaction->output_compression_opts(),
sub_compact->compaction->output_level(), skip_filters, sub_compact->compaction->output_level(), skip_filters,
oldest_ancester_time, 0 /* oldest_key_time */, oldest_ancester_time, 0 /* oldest_key_time */,

@ -1358,7 +1358,6 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
cfd->GetID(), cfd->GetName(), snapshot_seqs, cfd->GetID(), cfd->GetName(), snapshot_seqs,
earliest_write_conflict_snapshot, snapshot_checker, earliest_write_conflict_snapshot, snapshot_checker,
GetCompressionFlush(*cfd->ioptions(), mutable_cf_options), GetCompressionFlush(*cfd->ioptions(), mutable_cf_options),
mutable_cf_options.sample_for_compression,
mutable_cf_options.compression_opts, paranoid_file_checks, mutable_cf_options.compression_opts, paranoid_file_checks,
cfd->internal_stats(), TableFileCreationReason::kRecovery, &io_s, cfd->internal_stats(), TableFileCreationReason::kRecovery, &io_s,
io_tracer_, &event_logger_, job_id, Env::IO_HIGH, io_tracer_, &event_logger_, job_id, Env::IO_HIGH,

@ -1175,6 +1175,61 @@ class CountingDeleteTabPropCollectorFactory
} }
}; };
class BlockCountingTablePropertiesCollector : public TablePropertiesCollector {
public:
static const std::string kNumSampledBlocksPropertyName;
const char* Name() const override {
return "BlockCountingTablePropertiesCollector";
}
Status Finish(UserCollectedProperties* properties) override {
(*properties)[kNumSampledBlocksPropertyName] =
ToString(num_sampled_blocks_);
return Status::OK();
}
Status AddUserKey(const Slice& /*user_key*/, const Slice& /*value*/,
EntryType /*type*/, SequenceNumber /*seq*/,
uint64_t /*file_size*/) override {
return Status::OK();
}
void BlockAdd(uint64_t /* block_raw_bytes */,
uint64_t block_compressed_bytes_fast,
uint64_t block_compressed_bytes_slow) override {
if (block_compressed_bytes_fast > 0 || block_compressed_bytes_slow > 0) {
num_sampled_blocks_++;
}
}
UserCollectedProperties GetReadableProperties() const override {
return UserCollectedProperties{
{kNumSampledBlocksPropertyName, ToString(num_sampled_blocks_)},
};
}
private:
uint32_t num_sampled_blocks_ = 0;
};
const std::string
BlockCountingTablePropertiesCollector::kNumSampledBlocksPropertyName =
"NumSampledBlocks";
class BlockCountingTablePropertiesCollectorFactory
: public TablePropertiesCollectorFactory {
public:
const char* Name() const override {
return "BlockCountingTablePropertiesCollectorFactory";
}
TablePropertiesCollector* CreateTablePropertiesCollector(
TablePropertiesCollectorFactory::Context /* context */) override {
return new BlockCountingTablePropertiesCollector();
}
};
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
TEST_F(DBPropertiesTest, GetUserDefinedTableProperties) { TEST_F(DBPropertiesTest, GetUserDefinedTableProperties) {
Options options = CurrentOptions(); Options options = CurrentOptions();
@ -1413,6 +1468,58 @@ TEST_F(DBPropertiesTest, NeedCompactHintPersistentTest) {
} }
} }
// Excluded from RocksDB lite tests due to `GetPropertiesOfAllTables()` usage.
TEST_F(DBPropertiesTest, BlockAddForCompressionSampling) {
// Sampled compression requires at least one of the following four types.
if (!Snappy_Supported() && !Zlib_Supported() && !LZ4_Supported() &&
!ZSTD_Supported()) {
return;
}
Options options = CurrentOptions();
options.disable_auto_compactions = true;
options.table_properties_collector_factories.emplace_back(
std::make_shared<BlockCountingTablePropertiesCollectorFactory>());
for (bool sample_for_compression : {false, true}) {
// For simplicity/determinism, sample 100% when enabled, or 0% when disabled
options.sample_for_compression = sample_for_compression ? 1 : 0;
DestroyAndReopen(options);
// Setup the following LSM:
//
// L0_0 ["a", "b"]
// L1_0 ["a", "b"]
//
// L0_0 was created by flush. L1_0 was created by compaction. Each file
// contains one data block.
for (int i = 0; i < 3; ++i) {
ASSERT_OK(Put("a", "val"));
ASSERT_OK(Put("b", "val"));
ASSERT_OK(Flush());
if (i == 1) {
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
}
}
// A `BlockAdd()` should have been seen for files generated by flush or
// compaction when `sample_for_compression` is enabled.
TablePropertiesCollection file_to_props;
ASSERT_OK(db_->GetPropertiesOfAllTables(&file_to_props));
ASSERT_EQ(2, file_to_props.size());
for (const auto& file_and_props : file_to_props) {
auto& user_props = file_and_props.second->user_collected_properties;
ASSERT_TRUE(user_props.find(BlockCountingTablePropertiesCollector::
kNumSampledBlocksPropertyName) !=
user_props.end());
ASSERT_EQ(user_props.at(BlockCountingTablePropertiesCollector::
kNumSampledBlocksPropertyName),
ToString(sample_for_compression ? 1 : 0));
}
}
}
TEST_F(DBPropertiesTest, EstimateNumKeysUnderflow) { TEST_F(DBPropertiesTest, EstimateNumKeysUnderflow) {
Options options = CurrentOptions(); Options options = CurrentOptions();
Reopen(options); Reopen(options);

@ -411,8 +411,7 @@ Status FlushJob::WriteLevel0Table() {
cfd_->internal_comparator(), cfd_->int_tbl_prop_collector_factories(), cfd_->internal_comparator(), cfd_->int_tbl_prop_collector_factories(),
cfd_->GetID(), cfd_->GetName(), existing_snapshots_, cfd_->GetID(), cfd_->GetName(), existing_snapshots_,
earliest_write_conflict_snapshot_, snapshot_checker_, earliest_write_conflict_snapshot_, snapshot_checker_,
output_compression_, mutable_cf_options_.sample_for_compression, output_compression_, mutable_cf_options_.compression_opts,
mutable_cf_options_.compression_opts,
mutable_cf_options_.paranoid_file_checks, cfd_->internal_stats(), mutable_cf_options_.paranoid_file_checks, cfd_->internal_stats(),
TableFileCreationReason::kFlush, &io_s, io_tracer_, event_logger_, TableFileCreationReason::kFlush, &io_s, io_tracer_, event_logger_,
job_context_->job_id, Env::IO_HIGH, &table_properties_, 0 /* level */, job_context_->job_id, Env::IO_HIGH, &table_properties_, 0 /* level */,

@ -447,13 +447,12 @@ class Repairer {
nullptr /* blob_file_additions */, cfd->internal_comparator(), nullptr /* blob_file_additions */, cfd->internal_comparator(),
cfd->int_tbl_prop_collector_factories(), cfd->GetID(), cfd->GetName(), cfd->int_tbl_prop_collector_factories(), cfd->GetID(), cfd->GetName(),
{}, kMaxSequenceNumber, snapshot_checker, kNoCompression, {}, kMaxSequenceNumber, snapshot_checker, kNoCompression,
0 /* sample_for_compression */, CompressionOptions(), false, CompressionOptions(), false, nullptr /* internal_stats */,
nullptr /* internal_stats */, TableFileCreationReason::kRecovery, TableFileCreationReason::kRecovery, &io_s, nullptr /*IOTracer*/,
&io_s, nullptr /*IOTracer*/, nullptr /* event_logger */, nullptr /* event_logger */, 0 /* job_id */, Env::IO_HIGH,
0 /* job_id */, Env::IO_HIGH, nullptr /* table_properties */, nullptr /* table_properties */, -1 /* level */, current_time,
-1 /* level */, current_time, 0 /* oldest_key_time */, write_hint, 0 /* oldest_key_time */, write_hint, 0 /* file_creation_time */,
0 /* file_creation_time */, "DB Repairer" /* db_id */, "DB Repairer" /* db_id */, db_session_id_);
db_session_id_);
ROCKS_LOG_INFO(db_options_.info_log, ROCKS_LOG_INFO(db_options_.info_log,
"Log #%" PRIu64 ": %d ops saved to Table #%" PRIu64 " %s", "Log #%" PRIu64 ": %d ops saved to Table #%" PRIu64 " %s",
log, counter, meta.fd.GetNumber(), log, counter, meta.fd.GetNumber(),

@ -43,10 +43,10 @@ Status UserKeyTablePropertiesCollector::InternalAdd(const Slice& key,
} }
void UserKeyTablePropertiesCollector::BlockAdd( void UserKeyTablePropertiesCollector::BlockAdd(
uint64_t bLockRawBytes, uint64_t blockCompressedBytesFast, uint64_t block_raw_bytes, uint64_t block_compressed_bytes_fast,
uint64_t blockCompressedBytesSlow) { uint64_t block_compressed_bytes_slow) {
return collector_->BlockAdd(bLockRawBytes, blockCompressedBytesFast, return collector_->BlockAdd(block_raw_bytes, block_compressed_bytes_fast,
blockCompressedBytesSlow); block_compressed_bytes_slow);
} }
Status UserKeyTablePropertiesCollector::Finish( Status UserKeyTablePropertiesCollector::Finish(

@ -27,9 +27,9 @@ class IntTblPropCollector {
virtual Status InternalAdd(const Slice& key, const Slice& value, virtual Status InternalAdd(const Slice& key, const Slice& value,
uint64_t file_size) = 0; uint64_t file_size) = 0;
virtual void BlockAdd(uint64_t blockRawBytes, virtual void BlockAdd(uint64_t block_raw_bytes,
uint64_t blockCompressedBytesFast, uint64_t block_compressed_bytes_fast,
uint64_t blockCompressedBytesSlow) = 0; uint64_t block_compressed_bytes_slow) = 0;
virtual UserCollectedProperties GetReadableProperties() const = 0; virtual UserCollectedProperties GetReadableProperties() const = 0;
@ -64,9 +64,9 @@ class UserKeyTablePropertiesCollector : public IntTblPropCollector {
virtual Status InternalAdd(const Slice& key, const Slice& value, virtual Status InternalAdd(const Slice& key, const Slice& value,
uint64_t file_size) override; uint64_t file_size) override;
virtual void BlockAdd(uint64_t blockRawBytes, virtual void BlockAdd(uint64_t block_raw_bytes,
uint64_t blockCompressedBytesFast, uint64_t block_compressed_bytes_fast,
uint64_t blockCompressedBytesSlow) override; uint64_t block_compressed_bytes_slow) override;
virtual Status Finish(UserCollectedProperties* properties) override; virtual Status Finish(UserCollectedProperties* properties) override;

@ -55,8 +55,7 @@ void MakeBuilder(const Options& options, const ImmutableCFOptions& ioptions,
builder->reset(NewTableBuilder( builder->reset(NewTableBuilder(
ioptions, moptions, internal_comparator, int_tbl_prop_collector_factories, ioptions, moptions, internal_comparator, int_tbl_prop_collector_factories,
kTestColumnFamilyId, kTestColumnFamilyName, writable->get(), kTestColumnFamilyId, kTestColumnFamilyName, writable->get(),
options.compression, options.sample_for_compression, options.compression, options.compression_opts, unknown_level));
options.compression_opts, unknown_level));
} }
} // namespace } // namespace
@ -176,9 +175,9 @@ class RegularKeysStartWithAInternal : public IntTblPropCollector {
return Status::OK(); return Status::OK();
} }
void BlockAdd(uint64_t /* blockRawBytes */, void BlockAdd(uint64_t /* block_raw_bytes */,
uint64_t /* blockCompressedBytesFast */, uint64_t /* block_compressed_bytes_fast */,
uint64_t /* blockCompressedBytesSlow */) override { uint64_t /* block_compressed_bytes_slow */) override {
// Nothing to do. // Nothing to do.
return; return;
} }

@ -2781,7 +2781,7 @@ class VersionSetTestMissingFiles : public VersionSetTestBase,
TableBuilderOptions( TableBuilderOptions(
immutable_cf_options_, mutable_cf_options_, *internal_comparator_, immutable_cf_options_, mutable_cf_options_, *internal_comparator_,
&int_tbl_prop_collector_factories, kNoCompression, &int_tbl_prop_collector_factories, kNoCompression,
/*_sample_for_compression=*/0, CompressionOptions(), CompressionOptions(),
/*_skip_filters=*/false, info.column_family, info.level), /*_skip_filters=*/false, info.column_family, info.level),
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
fwriter.get())); fwriter.get()));

@ -99,9 +99,9 @@ class TablePropertiesCollector {
} }
// Called after each new block is cut // Called after each new block is cut
virtual void BlockAdd(uint64_t /* blockRawBytes */, virtual void BlockAdd(uint64_t /* block_raw_bytes */,
uint64_t /* blockCompressedBytesFast */, uint64_t /* block_compressed_bytes_fast */,
uint64_t /* blockCompressedBytesSlow */) { uint64_t /* block_compressed_bytes_slow */) {
// Nothing to do here. Callback registers can override. // Nothing to do here. Callback registers can override.
return; return;
} }

@ -210,9 +210,9 @@ class BlockBasedTableBuilder::BlockBasedTablePropertiesCollector
return Status::OK(); return Status::OK();
} }
virtual void BlockAdd(uint64_t /* blockRawBytes */, virtual void BlockAdd(uint64_t /* block_raw_bytes */,
uint64_t /* blockCompressedBytesFast */, uint64_t /* block_compressed_bytes_fast */,
uint64_t /* blockCompressedBytesSlow */) override { uint64_t /* block_compressed_bytes_slow */) override {
// Intentionally left blank. No interest in collecting stats for // Intentionally left blank. No interest in collecting stats for
// blocks. // blocks.
return; return;
@ -405,7 +405,6 @@ struct BlockBasedTableBuilder::Rep {
int_tbl_prop_collector_factories, int_tbl_prop_collector_factories,
uint32_t _column_family_id, WritableFileWriter* f, uint32_t _column_family_id, WritableFileWriter* f,
const CompressionType _compression_type, const CompressionType _compression_type,
const uint64_t _sample_for_compression,
const CompressionOptions& _compression_opts, const bool skip_filters, const CompressionOptions& _compression_opts, const bool skip_filters,
const int _level_at_creation, const std::string& _column_family_name, const int _level_at_creation, const std::string& _column_family_name,
const uint64_t _creation_time, const uint64_t _oldest_key_time, const uint64_t _creation_time, const uint64_t _oldest_key_time,
@ -431,7 +430,7 @@ struct BlockBasedTableBuilder::Rep {
range_del_block(1 /* block_restart_interval */), range_del_block(1 /* block_restart_interval */),
internal_prefix_transform(_moptions.prefix_extractor.get()), internal_prefix_transform(_moptions.prefix_extractor.get()),
compression_type(_compression_type), compression_type(_compression_type),
sample_for_compression(_sample_for_compression), sample_for_compression(_moptions.sample_for_compression),
compression_opts(_compression_opts), compression_opts(_compression_opts),
compression_dict(), compression_dict(),
compression_ctxs(_compression_opts.parallel_threads), compression_ctxs(_compression_opts.parallel_threads),
@ -841,7 +840,6 @@ BlockBasedTableBuilder::BlockBasedTableBuilder(
int_tbl_prop_collector_factories, int_tbl_prop_collector_factories,
uint32_t column_family_id, WritableFileWriter* file, uint32_t column_family_id, WritableFileWriter* file,
const CompressionType compression_type, const CompressionType compression_type,
const uint64_t sample_for_compression,
const CompressionOptions& compression_opts, const bool skip_filters, const CompressionOptions& compression_opts, const bool skip_filters,
const std::string& column_family_name, const int level_at_creation, const std::string& column_family_name, const int level_at_creation,
const uint64_t creation_time, const uint64_t oldest_key_time, const uint64_t creation_time, const uint64_t oldest_key_time,
@ -859,12 +857,12 @@ BlockBasedTableBuilder::BlockBasedTableBuilder(
sanitized_table_options.format_version = 1; sanitized_table_options.format_version = 1;
} }
rep_ = new Rep( rep_ = new Rep(ioptions, moptions, sanitized_table_options,
ioptions, moptions, sanitized_table_options, internal_comparator, internal_comparator, int_tbl_prop_collector_factories,
int_tbl_prop_collector_factories, column_family_id, file, column_family_id, file, compression_type, compression_opts,
compression_type, sample_for_compression, compression_opts, skip_filters, skip_filters, level_at_creation, column_family_name,
level_at_creation, column_family_name, creation_time, oldest_key_time, creation_time, oldest_key_time, target_file_size,
target_file_size, file_creation_time, db_id, db_session_id); file_creation_time, db_id, db_session_id);
if (rep_->filter_builder != nullptr) { if (rep_->filter_builder != nullptr) {
rep_->filter_builder->StartBlock(0); rep_->filter_builder->StartBlock(0);

@ -46,7 +46,6 @@ class BlockBasedTableBuilder : public TableBuilder {
int_tbl_prop_collector_factories, int_tbl_prop_collector_factories,
uint32_t column_family_id, WritableFileWriter* file, uint32_t column_family_id, WritableFileWriter* file,
const CompressionType compression_type, const CompressionType compression_type,
const uint64_t sample_for_compression,
const CompressionOptions& compression_opts, const bool skip_filters, const CompressionOptions& compression_opts, const bool skip_filters,
const std::string& column_family_name, const int level_at_creation, const std::string& column_family_name, const int level_at_creation,
const uint64_t creation_time = 0, const uint64_t oldest_key_time = 0, const uint64_t creation_time = 0, const uint64_t oldest_key_time = 0,

@ -500,7 +500,6 @@ TableBuilder* BlockBasedTableFactory::NewTableBuilder(
table_options_, table_builder_options.internal_comparator, table_options_, table_builder_options.internal_comparator,
table_builder_options.int_tbl_prop_collector_factories, column_family_id, table_builder_options.int_tbl_prop_collector_factories, column_family_id,
file, table_builder_options.compression_type, file, table_builder_options.compression_type,
table_builder_options.sample_for_compression,
table_builder_options.compression_opts, table_builder_options.compression_opts,
table_builder_options.skip_filters, table_builder_options.skip_filters,
table_builder_options.column_family_name, table_builder_options.level, table_builder_options.column_family_name, table_builder_options.level,

@ -66,9 +66,9 @@ class BlockBasedTableReaderTest
std::vector<std::unique_ptr<IntTblPropCollectorFactory>> factories; std::vector<std::unique_ptr<IntTblPropCollectorFactory>> factories;
std::unique_ptr<TableBuilder> table_builder(table_factory_->NewTableBuilder( std::unique_ptr<TableBuilder> table_builder(table_factory_->NewTableBuilder(
TableBuilderOptions(ioptions, moptions, comparator, &factories, TableBuilderOptions(ioptions, moptions, comparator, &factories,
compression_type, 0 /* sample_for_compression */, compression_type, CompressionOptions(),
CompressionOptions(), false /* skip_filters */, false /* skip_filters */, kDefaultColumnFamilyName,
kDefaultColumnFamilyName, -1 /* level */), -1 /* level */),
0 /* column_family_id */, writer.get())); 0 /* column_family_id */, writer.get()));
// Build table. // Build table.

@ -557,9 +557,8 @@ void TestBoundary(InternalKey& ik1, std::string& v1, InternalKey& ik2,
builder.reset(ioptions.table_factory->NewTableBuilder( builder.reset(ioptions.table_factory->NewTableBuilder(
TableBuilderOptions(ioptions, moptions, internal_comparator, TableBuilderOptions(ioptions, moptions, internal_comparator,
&int_tbl_prop_collector_factories, &int_tbl_prop_collector_factories,
options.compression, options.sample_for_compression, options.compression, CompressionOptions(),
CompressionOptions(), false /* skip_filters */, false /* skip_filters */, column_family_name, level_),
column_family_name, level_),
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
file_writer.get())); file_writer.get()));

@ -100,9 +100,9 @@ class BlockFetcherTest : public testing::Test {
std::vector<std::unique_ptr<IntTblPropCollectorFactory>> factories; std::vector<std::unique_ptr<IntTblPropCollectorFactory>> factories;
std::unique_ptr<TableBuilder> table_builder(table_factory_.NewTableBuilder( std::unique_ptr<TableBuilder> table_builder(table_factory_.NewTableBuilder(
TableBuilderOptions(ioptions, moptions, comparator, &factories, TableBuilderOptions(ioptions, moptions, comparator, &factories,
compression_type, 0 /* sample_for_compression */, compression_type, CompressionOptions(),
CompressionOptions(), false /* skip_filters */, false /* skip_filters */, kDefaultColumnFamilyName,
kDefaultColumnFamilyName, -1 /* level */), -1 /* level */),
0 /* column_family_id */, writer.get())); 0 /* column_family_id */, writer.get()));
// Build table. // Build table.

@ -172,11 +172,11 @@ bool NotifyCollectTableCollectorsOnAdd(
void NotifyCollectTableCollectorsOnBlockAdd( void NotifyCollectTableCollectorsOnBlockAdd(
const std::vector<std::unique_ptr<IntTblPropCollector>>& collectors, const std::vector<std::unique_ptr<IntTblPropCollector>>& collectors,
const uint64_t blockRawBytes, const uint64_t blockCompressedBytesFast, const uint64_t block_raw_bytes, const uint64_t block_compressed_bytes_fast,
const uint64_t blockCompressedBytesSlow) { const uint64_t block_compressed_bytes_slow) {
for (auto& collector : collectors) { for (auto& collector : collectors) {
collector->BlockAdd(blockRawBytes, blockCompressedBytesFast, collector->BlockAdd(block_raw_bytes, block_compressed_bytes_fast,
blockCompressedBytesSlow); block_compressed_bytes_slow);
} }
} }

@ -86,8 +86,8 @@ bool NotifyCollectTableCollectorsOnAdd(
void NotifyCollectTableCollectorsOnBlockAdd( void NotifyCollectTableCollectorsOnBlockAdd(
const std::vector<std::unique_ptr<IntTblPropCollector>>& collectors, const std::vector<std::unique_ptr<IntTblPropCollector>>& collectors,
uint64_t blockRawBytes, uint64_t blockCompressedBytesFast, uint64_t block_raw_bytes, uint64_t block_compressed_bytes_fast,
uint64_t blockCompressedBytesSlow); uint64_t block_compressed_bytes_slow);
// NotifyCollectTableCollectorsOnFinish() triggers the `Finish` event for all // NotifyCollectTableCollectorsOnFinish() triggers the `Finish` event for all
// property collectors. The collected properties will be added to `builder`. // property collectors. The collected properties will be added to `builder`.

@ -274,10 +274,10 @@ Status SstFileDumper::ShowCompressionSize(
std::string column_family_name; std::string column_family_name;
int unknown_level = -1; int unknown_level = -1;
TableBuilderOptions tb_opts( TableBuilderOptions tb_opts(imoptions, moptions, ikc,
imoptions, moptions, ikc, &block_based_table_factories, compress_type, &block_based_table_factories, compress_type,
0 /* sample_for_compression */, compress_opt, false /* skip_filters */, compress_opt, false /* skip_filters */,
column_family_name, unknown_level); column_family_name, unknown_level);
uint64_t num_data_blocks = 0; uint64_t num_data_blocks = 0;
std::chrono::steady_clock::time_point start = std::chrono::steady_clock::time_point start =
std::chrono::steady_clock::now(); std::chrono::steady_clock::now();

@ -210,8 +210,6 @@ Status SstFileWriter::Open(const std::string& file_path) {
compression_type = r->mutable_cf_options.compression; compression_type = r->mutable_cf_options.compression;
compression_opts = r->mutable_cf_options.compression_opts; compression_opts = r->mutable_cf_options.compression_opts;
} }
uint64_t sample_for_compression =
r->mutable_cf_options.sample_for_compression;
std::vector<std::unique_ptr<IntTblPropCollectorFactory>> std::vector<std::unique_ptr<IntTblPropCollectorFactory>>
int_tbl_prop_collector_factories; int_tbl_prop_collector_factories;
@ -252,10 +250,9 @@ Status SstFileWriter::Open(const std::string& file_path) {
} }
TableBuilderOptions table_builder_options( TableBuilderOptions table_builder_options(
r->ioptions, r->mutable_cf_options, r->internal_comparator, r->ioptions, r->mutable_cf_options, r->internal_comparator,
&int_tbl_prop_collector_factories, compression_type, &int_tbl_prop_collector_factories, compression_type, compression_opts,
sample_for_compression, compression_opts, r->skip_filters, r->skip_filters, r->column_family_name, unknown_level,
r->column_family_name, unknown_level, 0 /* creation_time */, 0 /* creation_time */, 0 /* oldest_key_time */, 0 /* target_file_size */,
0 /* oldest_key_time */, 0 /* target_file_size */,
0 /* file_creation_time */, "SST Writer" /* db_id */, db_session_id); 0 /* file_creation_time */, "SST Writer" /* db_id */, db_session_id);
FileTypeSet tmp_set = r->ioptions.checksum_handoff_file_types; FileTypeSet tmp_set = r->ioptions.checksum_handoff_file_types;
r->file_writer.reset(new WritableFileWriter( r->file_writer.reset(new WritableFileWriter(

@ -35,9 +35,9 @@ class SstFileWriterPropertiesCollector : public IntTblPropCollector {
return Status::OK(); return Status::OK();
} }
virtual void BlockAdd(uint64_t /* blockRawBytes */, virtual void BlockAdd(uint64_t /* block_raw_bytes */,
uint64_t /* blockCompressedBytesFast */, uint64_t /* block_compressed_bytes_fast */,
uint64_t /* blockCompressedBytesSlow */) override { uint64_t /* block_compressed_bytes_slow */) override {
// Intentionally left blank. No interest in collecting stats for // Intentionally left blank. No interest in collecting stats for
// blocks. // blocks.
return; return;

@ -92,7 +92,7 @@ struct TableBuilderOptions {
const InternalKeyComparator& _internal_comparator, const InternalKeyComparator& _internal_comparator,
const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>* const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
_int_tbl_prop_collector_factories, _int_tbl_prop_collector_factories,
CompressionType _compression_type, uint64_t _sample_for_compression, CompressionType _compression_type,
const CompressionOptions& _compression_opts, bool _skip_filters, const CompressionOptions& _compression_opts, bool _skip_filters,
const std::string& _column_family_name, int _level, const std::string& _column_family_name, int _level,
const uint64_t _creation_time = 0, const int64_t _oldest_key_time = 0, const uint64_t _creation_time = 0, const int64_t _oldest_key_time = 0,
@ -104,7 +104,6 @@ struct TableBuilderOptions {
internal_comparator(_internal_comparator), internal_comparator(_internal_comparator),
int_tbl_prop_collector_factories(_int_tbl_prop_collector_factories), int_tbl_prop_collector_factories(_int_tbl_prop_collector_factories),
compression_type(_compression_type), compression_type(_compression_type),
sample_for_compression(_sample_for_compression),
compression_opts(_compression_opts), compression_opts(_compression_opts),
skip_filters(_skip_filters), skip_filters(_skip_filters),
column_family_name(_column_family_name), column_family_name(_column_family_name),
@ -122,7 +121,6 @@ struct TableBuilderOptions {
const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>* const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
int_tbl_prop_collector_factories; int_tbl_prop_collector_factories;
CompressionType compression_type; CompressionType compression_type;
uint64_t sample_for_compression;
const CompressionOptions& compression_opts; const CompressionOptions& compression_opts;
bool skip_filters; // only used by BlockBasedTableBuilder bool skip_filters; // only used by BlockBasedTableBuilder
const std::string& column_family_name; const std::string& column_family_name;

@ -102,9 +102,8 @@ void TableReaderBenchmark(Options& opts, EnvOptions& env_options,
tb = opts.table_factory->NewTableBuilder( tb = opts.table_factory->NewTableBuilder(
TableBuilderOptions( TableBuilderOptions(
ioptions, moptions, ikc, &int_tbl_prop_collector_factories, ioptions, moptions, ikc, &int_tbl_prop_collector_factories,
CompressionType::kNoCompression, 0 /* sample_for_compression */, CompressionType::kNoCompression, CompressionOptions(),
CompressionOptions(), false /* skip_filters */, false /* skip_filters */, kDefaultColumnFamilyName, unknown_level),
kDefaultColumnFamilyName, unknown_level),
0 /* column_family_id */, file_writer.get()); 0 /* column_family_id */, file_writer.get());
} else { } else {
s = DB::Open(opts, dbname, &db); s = DB::Open(opts, dbname, &db);

@ -363,9 +363,9 @@ class TableConstructor : public Constructor {
builder.reset(ioptions.table_factory->NewTableBuilder( builder.reset(ioptions.table_factory->NewTableBuilder(
TableBuilderOptions(ioptions, moptions, internal_comparator, TableBuilderOptions(ioptions, moptions, internal_comparator,
&int_tbl_prop_collector_factories, &int_tbl_prop_collector_factories,
options.compression, options.sample_for_compression, options.compression, options.compression_opts,
options.compression_opts, false /* skip_filters */, false /* skip_filters */, column_family_name,
column_family_name, level_), level_),
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
file_writer_.get())); file_writer_.get()));
@ -3325,9 +3325,8 @@ TEST_P(BlockBasedTableTest, NoFileChecksum) {
builder.reset(ioptions.table_factory->NewTableBuilder( builder.reset(ioptions.table_factory->NewTableBuilder(
TableBuilderOptions(ioptions, moptions, *comparator, TableBuilderOptions(ioptions, moptions, *comparator,
&int_tbl_prop_collector_factories, &int_tbl_prop_collector_factories,
options.compression, options.sample_for_compression, options.compression, options.compression_opts,
options.compression_opts, false /* skip_filters */, false /* skip_filters */, column_family_name, level),
column_family_name, level),
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
f.GetFileWriter())); f.GetFileWriter()));
ASSERT_OK(f.ResetTableBuilder(std::move(builder))); ASSERT_OK(f.ResetTableBuilder(std::move(builder)));
@ -3364,9 +3363,8 @@ TEST_P(BlockBasedTableTest, Crc32cFileChecksum) {
builder.reset(ioptions.table_factory->NewTableBuilder( builder.reset(ioptions.table_factory->NewTableBuilder(
TableBuilderOptions(ioptions, moptions, *comparator, TableBuilderOptions(ioptions, moptions, *comparator,
&int_tbl_prop_collector_factories, &int_tbl_prop_collector_factories,
options.compression, options.sample_for_compression, options.compression, options.compression_opts,
options.compression_opts, false /* skip_filters */, false /* skip_filters */, column_family_name, level),
column_family_name, level),
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
f.GetFileWriter())); f.GetFileWriter()));
ASSERT_OK(f.ResetTableBuilder(std::move(builder))); ASSERT_OK(f.ResetTableBuilder(std::move(builder)));
@ -3413,10 +3411,10 @@ TEST_F(PlainTableTest, BasicPlainTableProperties) {
std::string column_family_name; std::string column_family_name;
int unknown_level = -1; int unknown_level = -1;
std::unique_ptr<TableBuilder> builder(factory.NewTableBuilder( std::unique_ptr<TableBuilder> builder(factory.NewTableBuilder(
TableBuilderOptions( TableBuilderOptions(ioptions, moptions, ikc,
ioptions, moptions, ikc, &int_tbl_prop_collector_factories, &int_tbl_prop_collector_factories, kNoCompression,
kNoCompression, 0 /* sample_for_compression */, CompressionOptions(), CompressionOptions(), false /* skip_filters */,
false /* skip_filters */, column_family_name, unknown_level), column_family_name, unknown_level),
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
file_writer.get())); file_writer.get()));
@ -3470,10 +3468,10 @@ TEST_F(PlainTableTest, NoFileChecksum) {
f.CreateWriteableFile(); f.CreateWriteableFile();
std::unique_ptr<TableBuilder> builder(factory.NewTableBuilder( std::unique_ptr<TableBuilder> builder(factory.NewTableBuilder(
TableBuilderOptions( TableBuilderOptions(ioptions, moptions, ikc,
ioptions, moptions, ikc, &int_tbl_prop_collector_factories, &int_tbl_prop_collector_factories, kNoCompression,
kNoCompression, 0 /* sample_for_compression */, CompressionOptions(), CompressionOptions(), false /* skip_filters */,
false /* skip_filters */, column_family_name, unknown_level), column_family_name, unknown_level),
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
f.GetFileWriter())); f.GetFileWriter()));
ASSERT_OK(f.ResetTableBuilder(std::move(builder))); ASSERT_OK(f.ResetTableBuilder(std::move(builder)));
@ -3512,10 +3510,10 @@ TEST_F(PlainTableTest, Crc32cFileChecksum) {
f.SetFileChecksumGenerator(checksum_crc32c_gen1.release()); f.SetFileChecksumGenerator(checksum_crc32c_gen1.release());
std::unique_ptr<TableBuilder> builder(factory.NewTableBuilder( std::unique_ptr<TableBuilder> builder(factory.NewTableBuilder(
TableBuilderOptions( TableBuilderOptions(ioptions, moptions, ikc,
ioptions, moptions, ikc, &int_tbl_prop_collector_factories, &int_tbl_prop_collector_factories, kNoCompression,
kNoCompression, 0 /* sample_for_compression */, CompressionOptions(), CompressionOptions(), false /* skip_filters */,
false /* skip_filters */, column_family_name, unknown_level), column_family_name, unknown_level),
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
f.GetFileWriter())); f.GetFileWriter()));
ASSERT_OK(f.ResetTableBuilder(std::move(builder))); ASSERT_OK(f.ResetTableBuilder(std::move(builder)));
@ -4081,8 +4079,8 @@ TEST_P(BlockBasedTableTest, DISABLED_TableWithGlobalSeqno) {
std::unique_ptr<TableBuilder> builder(options.table_factory->NewTableBuilder( std::unique_ptr<TableBuilder> builder(options.table_factory->NewTableBuilder(
TableBuilderOptions(ioptions, moptions, ikc, TableBuilderOptions(ioptions, moptions, ikc,
&int_tbl_prop_collector_factories, kNoCompression, &int_tbl_prop_collector_factories, kNoCompression,
0 /* sample_for_compression */, CompressionOptions(), CompressionOptions(), false /* skip_filters */,
false /* skip_filters */, column_family_name, -1), column_family_name, -1),
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
file_writer.get())); file_writer.get()));
@ -4267,8 +4265,8 @@ TEST_P(BlockBasedTableTest, BlockAlignTest) {
std::unique_ptr<TableBuilder> builder(options.table_factory->NewTableBuilder( std::unique_ptr<TableBuilder> builder(options.table_factory->NewTableBuilder(
TableBuilderOptions(ioptions, moptions, ikc, TableBuilderOptions(ioptions, moptions, ikc,
&int_tbl_prop_collector_factories, kNoCompression, &int_tbl_prop_collector_factories, kNoCompression,
0 /* sample_for_compression */, CompressionOptions(), CompressionOptions(), false /* skip_filters */,
false /* skip_filters */, column_family_name, -1), column_family_name, -1),
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
file_writer.get())); file_writer.get()));
@ -4362,8 +4360,8 @@ TEST_P(BlockBasedTableTest, PropertiesBlockRestartPointTest) {
std::unique_ptr<TableBuilder> builder(options.table_factory->NewTableBuilder( std::unique_ptr<TableBuilder> builder(options.table_factory->NewTableBuilder(
TableBuilderOptions(ioptions, moptions, ikc, TableBuilderOptions(ioptions, moptions, ikc,
&int_tbl_prop_collector_factories, kNoCompression, &int_tbl_prop_collector_factories, kNoCompression,
0 /* sample_for_compression */, CompressionOptions(), CompressionOptions(), false /* skip_filters */,
false /* skip_filters */, column_family_name, -1), column_family_name, -1),
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
file_writer.get())); file_writer.get()));

@ -113,9 +113,8 @@ class SSTDumpToolTest : public testing::Test {
tb.reset(opts.table_factory->NewTableBuilder( tb.reset(opts.table_factory->NewTableBuilder(
TableBuilderOptions( TableBuilderOptions(
imoptions, moptions, ikc, &int_tbl_prop_collector_factories, imoptions, moptions, ikc, &int_tbl_prop_collector_factories,
CompressionType::kNoCompression, 0 /* sample_for_compression */, CompressionType::kNoCompression, CompressionOptions(),
CompressionOptions(), false /* skip_filters */, column_family_name, false /* skip_filters */, column_family_name, unknown_level),
unknown_level),
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
file_writer.get())); file_writer.get()));

Loading…
Cancel
Save