From 776bd8d5ebd07ffb74cfd7e5419f5c47523bbb1b Mon Sep 17 00:00:00 2001 From: sdong Date: Thu, 8 Oct 2015 16:57:35 -0700 Subject: [PATCH] Pass column family ID to table property collector Summary: Pass column family ID through TablePropertiesCollectorFactory::CreateTablePropertiesCollector() so that users can identify which column family this file is for and handle it differently. Test Plan: Add unit test scenarios in tests related to table properties collectors to verify the information passed in is correct. Reviewers: rven, yhchiang, anthony, kradhakrishnan, igor, IslamAbdelRahman Reviewed By: IslamAbdelRahman Subscribers: yoshinorim, leveldb, dhruba Differential Revision: https://reviews.facebook.net/D48411 --- HISTORY.md | 1 + db/builder.cc | 10 ++- db/builder.h | 6 +- db/compaction_job.cc | 4 +- db/db_impl.cc | 4 +- db/db_test.cc | 90 +++++++++++++++++-- db/flush_job.cc | 5 +- db/repair.cc | 9 +- db/table_properties_collector.h | 13 ++- db/table_properties_collector_test.cc | 12 ++- include/rocksdb/table.h | 2 +- include/rocksdb/table_properties.h | 8 +- table/adaptive_table_factory.cc | 5 +- table/adaptive_table_factory.h | 2 +- table/block_based_table_builder.cc | 12 +-- table/block_based_table_builder.h | 3 +- table/block_based_table_factory.cc | 6 +- table/block_based_table_factory.h | 2 +- table/cuckoo_table_factory.cc | 2 +- table/cuckoo_table_factory.h | 2 +- table/mock_table.cc | 2 +- table/mock_table.h | 2 +- table/plain_table_builder.cc | 10 +-- table/plain_table_builder.h | 10 +-- table/plain_table_factory.cc | 9 +- table/plain_table_factory.h | 2 +- table/sst_file_writer.cc | 7 +- table/table_properties.cc | 3 + table/table_reader_bench.cc | 2 +- table/table_test.cc | 2 + util/sst_dump_test.cc | 1 + util/sst_dump_tool.cc | 4 +- .../compact_on_deletion_collector.cc | 9 +- .../compact_on_deletion_collector.h | 4 +- .../compact_on_deletion_collector_test.cc | 13 +-- 35 files changed, 201 insertions(+), 77 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index ecff88201..7493da314 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -4,6 +4,7 @@ ### Public API Changes * CompactionFilter::Context includes information of Column Family ID * The need-compaction hint given by TablePropertiesCollector::NeedCompact() will be persistent and recoverable after DB recovery. This introduces a breaking format change. If you use this experimental feature, including NewCompactOnDeletionCollectorFactory() in the new version, you may not be able to directly downgrade the DB back to version 4.0 or lower. +* TablePropertiesCollectorFactory::CreateTablePropertiesCollector() now takes an option Context, containing the information of column family ID for the file being written. ## 4.1.0 (10/8/2015) ### New Features diff --git a/db/builder.cc b/db/builder.cc index 3d07a0f30..243f6f38a 100644 --- a/db/builder.cc +++ b/db/builder.cc @@ -40,13 +40,14 @@ TableBuilder* NewTableBuilder( const InternalKeyComparator& internal_comparator, const std::vector>* int_tbl_prop_collector_factories, - WritableFileWriter* file, const CompressionType compression_type, + uint32_t column_family_id, WritableFileWriter* file, + const CompressionType compression_type, const CompressionOptions& compression_opts, const bool skip_filters) { return ioptions.table_factory->NewTableBuilder( TableBuilderOptions(ioptions, internal_comparator, int_tbl_prop_collector_factories, compression_type, compression_opts, skip_filters), - file); + column_family_id, file); } Status BuildTable( @@ -55,7 +56,8 @@ Status BuildTable( FileMetaData* meta, const InternalKeyComparator& internal_comparator, const std::vector>* int_tbl_prop_collector_factories, - std::vector snapshots, const CompressionType compression, + uint32_t column_family_id, std::vector snapshots, + const CompressionType compression, const CompressionOptions& compression_opts, bool paranoid_file_checks, InternalStats* internal_stats, const Env::IOPriority io_priority, TableProperties* table_properties) { @@ -82,7 +84,7 @@ Status BuildTable( builder = NewTableBuilder( ioptions, internal_comparator, int_tbl_prop_collector_factories, - file_writer.get(), compression, compression_opts); + column_family_id, file_writer.get(), compression, compression_opts); } MergeHelper merge(env, internal_comparator.user_comparator(), diff --git a/db/builder.h b/db/builder.h index 09d81bfe4..797e9de60 100644 --- a/db/builder.h +++ b/db/builder.h @@ -37,7 +37,8 @@ TableBuilder* NewTableBuilder( const InternalKeyComparator& internal_comparator, const std::vector>* int_tbl_prop_collector_factories, - WritableFileWriter* file, const CompressionType compression_type, + uint32_t column_family_id, WritableFileWriter* file, + const CompressionType compression_type, const CompressionOptions& compression_opts, const bool skip_filters = false); @@ -52,7 +53,8 @@ extern Status BuildTable( FileMetaData* meta, const InternalKeyComparator& internal_comparator, const std::vector>* int_tbl_prop_collector_factories, - std::vector snapshots, const CompressionType compression, + uint32_t column_family_id, std::vector snapshots, + const CompressionType compression, const CompressionOptions& compression_opts, bool paranoid_file_checks, InternalStats* internal_stats, const Env::IOPriority io_priority = Env::IO_HIGH, diff --git a/db/compaction_job.cc b/db/compaction_job.cc index 4d6656d4e..9ac4b16ec 100644 --- a/db/compaction_job.cc +++ b/db/compaction_job.cc @@ -927,8 +927,8 @@ Status CompactionJob::OpenCompactionOutputFile( cfd->ioptions()->optimize_filters_for_hits && bottommost_level_; sub_compact->builder.reset(NewTableBuilder( *cfd->ioptions(), cfd->internal_comparator(), - cfd->int_tbl_prop_collector_factories(), sub_compact->outfile.get(), - sub_compact->compaction->output_compression(), + cfd->int_tbl_prop_collector_factories(), cfd->GetID(), + sub_compact->outfile.get(), sub_compact->compaction->output_compression(), cfd->ioptions()->compression_opts, skip_filters)); LogFlush(db_options_.info_log); return s; diff --git a/db/db_impl.cc b/db/db_impl.cc index cf4fa74b3..64fee85b0 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -1317,8 +1317,8 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd, s = BuildTable( dbname_, env_, *cfd->ioptions(), env_options_, cfd->table_cache(), iter.get(), &meta, cfd->internal_comparator(), - cfd->int_tbl_prop_collector_factories(), snapshots_.GetAll(), - GetCompressionFlush(*cfd->ioptions()), + cfd->int_tbl_prop_collector_factories(), cfd->GetID(), + snapshots_.GetAll(), GetCompressionFlush(*cfd->ioptions()), cfd->ioptions()->compression_opts, paranoid_file_checks, cfd->internal_stats(), Env::IO_HIGH, &info.table_properties); LogFlush(db_options_.info_log); diff --git a/db/db_test.cc b/db/db_test.cc index 0f9d43ce0..ea455196f 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -821,21 +821,34 @@ class CoutingUserTblPropCollector : public TablePropertiesCollector { class CoutingUserTblPropCollectorFactory : public TablePropertiesCollectorFactory { public: - virtual TablePropertiesCollector* CreateTablePropertiesCollector() override { + explicit CoutingUserTblPropCollectorFactory( + uint32_t expected_column_family_id) + : expected_column_family_id_(expected_column_family_id), + num_created_(0) {} + virtual TablePropertiesCollector* CreateTablePropertiesCollector( + TablePropertiesCollectorFactory::Context context) override { + EXPECT_EQ(expected_column_family_id_, context.column_family_id); + num_created_++; return new CoutingUserTblPropCollector(); } const char* Name() const override { return "CoutingUserTblPropCollectorFactory"; } + void set_expected_column_family_id(uint32_t v) { + expected_column_family_id_ = v; + } + uint32_t expected_column_family_id_; + uint32_t num_created_; }; -TEST_F(DBTest, GetUserDefinedTablaProperties) { +TEST_F(DBTest, GetUserDefinedTableProperties) { Options options = CurrentOptions(); options.level0_file_num_compaction_trigger = (1<<30); options.max_background_flushes = 0; options.table_properties_collector_factories.resize(1); - options.table_properties_collector_factories[0] = - std::make_shared(); + std::shared_ptr collector_factory = + std::make_shared(0); + options.table_properties_collector_factories[0] = collector_factory; Reopen(options); // Create 4 tables for (int table = 0; table < 4; ++table) { @@ -861,6 +874,72 @@ TEST_F(DBTest, GetUserDefinedTablaProperties) { sum += count; } ASSERT_EQ(10u + 11u + 12u + 13u, sum); + + ASSERT_GT(collector_factory->num_created_, 0); + collector_factory->num_created_ = 0; + dbfull()->TEST_CompactRange(0, nullptr, nullptr); + ASSERT_GT(collector_factory->num_created_, 0); +} + +TEST_F(DBTest, UserDefinedTablePropertiesContext) { + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = 3; + options.max_background_flushes = 0; + options.table_properties_collector_factories.resize(1); + std::shared_ptr collector_factory = + std::make_shared(1); + options.table_properties_collector_factories[0] = collector_factory, + CreateAndReopenWithCF({"pikachu"}, options); + // Create 2 files + for (int table = 0; table < 2; ++table) { + for (int i = 0; i < 10 + table; ++i) { + Put(1, ToString(table * 100 + i), "val"); + } + Flush(1); + } + ASSERT_GT(collector_factory->num_created_, 0); + + collector_factory->num_created_ = 0; + // Trigger automatic compactions. + for (int table = 0; table < 3; ++table) { + for (int i = 0; i < 10 + table; ++i) { + Put(1, ToString(table * 100 + i), "val"); + } + Flush(1); + dbfull()->TEST_WaitForCompact(); + } + ASSERT_GT(collector_factory->num_created_, 0); + + collector_factory->num_created_ = 0; + dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1]); + ASSERT_GT(collector_factory->num_created_, 0); + + // Come back to write to default column family + collector_factory->num_created_ = 0; + collector_factory->set_expected_column_family_id(0); // default CF + // Create 4 tables in default column family + for (int table = 0; table < 2; ++table) { + for (int i = 0; i < 10 + table; ++i) { + Put(ToString(table * 100 + i), "val"); + } + Flush(); + } + ASSERT_GT(collector_factory->num_created_, 0); + + collector_factory->num_created_ = 0; + // Trigger automatic compactions. + for (int table = 0; table < 3; ++table) { + for (int i = 0; i < 10 + table; ++i) { + Put(ToString(table * 100 + i), "val"); + } + Flush(); + dbfull()->TEST_WaitForCompact(); + } + ASSERT_GT(collector_factory->num_created_, 0); + + collector_factory->num_created_ = 0; + dbfull()->TEST_CompactRange(0, nullptr, nullptr); + ASSERT_GT(collector_factory->num_created_, 0); } TEST_F(DBTest, LevelLimitReopen) { @@ -8284,7 +8363,8 @@ class CountingDeleteTabPropCollector : public TablePropertiesCollector { class CountingDeleteTabPropCollectorFactory : public TablePropertiesCollectorFactory { public: - virtual TablePropertiesCollector* CreateTablePropertiesCollector() override { + virtual TablePropertiesCollector* CreateTablePropertiesCollector( + TablePropertiesCollectorFactory::Context context) override { return new CountingDeleteTabPropCollector(); } const char* Name() const override { diff --git a/db/flush_job.cc b/db/flush_job.cc index 410108a05..816a141c2 100644 --- a/db/flush_job.cc +++ b/db/flush_job.cc @@ -234,8 +234,9 @@ Status FlushJob::WriteLevel0Table(const autovector& mems, s = BuildTable( dbname_, db_options_.env, *cfd_->ioptions(), env_options_, cfd_->table_cache(), iter.get(), meta, cfd_->internal_comparator(), - cfd_->int_tbl_prop_collector_factories(), existing_snapshots_, - output_compression_, cfd_->ioptions()->compression_opts, + cfd_->int_tbl_prop_collector_factories(), cfd_->GetID(), + existing_snapshots_, output_compression_, + cfd_->ioptions()->compression_opts, mutable_cf_options_.paranoid_file_checks, cfd_->internal_stats(), Env::IO_HIGH, &info.table_properties); LogFlush(db_options_.info_log); diff --git a/db/repair.cc b/db/repair.cc index d1ef6db74..42c702d0b 100644 --- a/db/repair.cc +++ b/db/repair.cc @@ -290,10 +290,11 @@ class Repairer { ro.total_order_seek = true; Arena arena; ScopedArenaIterator iter(mem->NewIterator(ro, &arena)); - status = BuildTable(dbname_, env_, ioptions_, env_options_, table_cache_, - iter.get(), &meta, icmp_, - &int_tbl_prop_collector_factories_, {}, - kNoCompression, CompressionOptions(), false, nullptr); + status = BuildTable( + dbname_, env_, ioptions_, env_options_, table_cache_, iter.get(), + &meta, icmp_, &int_tbl_prop_collector_factories_, + TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, {}, + kNoCompression, CompressionOptions(), false, nullptr); } delete mem->Unref(); delete cf_mems_default; diff --git a/db/table_properties_collector.h b/db/table_properties_collector.h index 51c2ba915..0e5f4e347 100644 --- a/db/table_properties_collector.h +++ b/db/table_properties_collector.h @@ -41,7 +41,8 @@ class IntTblPropCollectorFactory { public: virtual ~IntTblPropCollectorFactory() {} // has to be thread-safe - virtual IntTblPropCollector* CreateIntTblPropCollector() = 0; + virtual IntTblPropCollector* CreateIntTblPropCollector( + uint32_t column_family_id) = 0; // The name of the properties collector can be used for debugging purpose. virtual const char* Name() const = 0; @@ -69,7 +70,8 @@ class InternalKeyPropertiesCollector : public IntTblPropCollector { class InternalKeyPropertiesCollectorFactory : public IntTblPropCollectorFactory { public: - virtual IntTblPropCollector* CreateIntTblPropCollector() override { + virtual IntTblPropCollector* CreateIntTblPropCollector( + uint32_t column_family_id) override { return new InternalKeyPropertiesCollector(); } @@ -114,9 +116,12 @@ class UserKeyTablePropertiesCollectorFactory explicit UserKeyTablePropertiesCollectorFactory( std::shared_ptr user_collector_factory) : user_collector_factory_(user_collector_factory) {} - virtual IntTblPropCollector* CreateIntTblPropCollector() override { + virtual IntTblPropCollector* CreateIntTblPropCollector( + uint32_t column_family_id) override { + TablePropertiesCollectorFactory::Context context; + context.column_family_id = column_family_id; return new UserKeyTablePropertiesCollector( - user_collector_factory_->CreateTablePropertiesCollector()); + user_collector_factory_->CreateTablePropertiesCollector(context)); } virtual const char* Name() const override { diff --git a/db/table_properties_collector_test.cc b/db/table_properties_collector_test.cc index 0eeed8191..7cea86fdd 100644 --- a/db/table_properties_collector_test.cc +++ b/db/table_properties_collector_test.cc @@ -35,6 +35,8 @@ class TablePropertiesTest : public testing::Test, // Utilities test functions namespace { +static const uint32_t kTestColumnFamilyId = 66; + void MakeBuilder(const Options& options, const ImmutableCFOptions& ioptions, const InternalKeyComparator& internal_comparator, const std::vector>* @@ -46,7 +48,8 @@ void MakeBuilder(const Options& options, const ImmutableCFOptions& ioptions, builder->reset(NewTableBuilder( ioptions, internal_comparator, int_tbl_prop_collector_factories, - writable->get(), options.compression, options.compression_opts)); + kTestColumnFamilyId /* column_family_id */, writable->get(), + options.compression, options.compression_opts)); } } // namespace @@ -178,14 +181,17 @@ class RegularKeysStartWithAFactory : public IntTblPropCollectorFactory, public: explicit RegularKeysStartWithAFactory(bool backward_mode) : backward_mode_(backward_mode) {} - virtual TablePropertiesCollector* CreateTablePropertiesCollector() override { + virtual TablePropertiesCollector* CreateTablePropertiesCollector( + TablePropertiesCollectorFactory::Context context) override { + EXPECT_EQ(kTestColumnFamilyId, context.column_family_id); if (!backward_mode_) { return new RegularKeysStartWithA(); } else { return new RegularKeysStartWithABackwardCompatible(); } } - virtual IntTblPropCollector* CreateIntTblPropCollector() override { + virtual IntTblPropCollector* CreateIntTblPropCollector( + uint32_t column_family_id) override { return new RegularKeysStartWithAInternal(); } const char* Name() const override { return "RegularKeysStartWithA"; } diff --git a/include/rocksdb/table.h b/include/rocksdb/table.h index e52b58099..4f62debd2 100644 --- a/include/rocksdb/table.h +++ b/include/rocksdb/table.h @@ -375,7 +375,7 @@ class TableFactory { // to use in this table. virtual TableBuilder* NewTableBuilder( const TableBuilderOptions& table_builder_options, - WritableFileWriter* file) const = 0; + uint32_t column_family_id, WritableFileWriter* file) const = 0; // Sanitizes the specified DB Options and ColumnFamilyOptions. // diff --git a/include/rocksdb/table_properties.h b/include/rocksdb/table_properties.h index 28500749a..25173a942 100644 --- a/include/rocksdb/table_properties.h +++ b/include/rocksdb/table_properties.h @@ -144,9 +144,15 @@ class TablePropertiesCollector { // TablePropertiesCollector for each new table class TablePropertiesCollectorFactory { public: + struct Context { + uint32_t column_family_id; + static const uint32_t kUnknownColumnFamily; + }; + virtual ~TablePropertiesCollectorFactory() {} // has to be thread-safe - virtual TablePropertiesCollector* CreateTablePropertiesCollector() = 0; + virtual TablePropertiesCollector* CreateTablePropertiesCollector( + TablePropertiesCollectorFactory::Context context) = 0; // The name of the properties collector can be used for debugging purpose. virtual const char* Name() const = 0; diff --git a/table/adaptive_table_factory.cc b/table/adaptive_table_factory.cc index c589c07a9..3e2cfa326 100644 --- a/table/adaptive_table_factory.cc +++ b/table/adaptive_table_factory.cc @@ -66,9 +66,10 @@ Status AdaptiveTableFactory::NewTableReader( } TableBuilder* AdaptiveTableFactory::NewTableBuilder( - const TableBuilderOptions& table_builder_options, + const TableBuilderOptions& table_builder_options, uint32_t column_family_id, WritableFileWriter* file) const { - return table_factory_to_write_->NewTableBuilder(table_builder_options, file); + return table_factory_to_write_->NewTableBuilder(table_builder_options, + column_family_id, file); } std::string AdaptiveTableFactory::GetPrintableTableOptions() const { diff --git a/table/adaptive_table_factory.h b/table/adaptive_table_factory.h index dfcae14cf..955ded4d2 100644 --- a/table/adaptive_table_factory.h +++ b/table/adaptive_table_factory.h @@ -40,7 +40,7 @@ class AdaptiveTableFactory : public TableFactory { TableBuilder* NewTableBuilder( const TableBuilderOptions& table_builder_options, - WritableFileWriter* file) const override; + uint32_t column_family_id, WritableFileWriter* file) const override; // Sanitizes the specified DB Options. Status SanitizeOptions(const DBOptions& db_opts, diff --git a/table/block_based_table_builder.cc b/table/block_based_table_builder.cc index e13531529..db6e3a453 100644 --- a/table/block_based_table_builder.cc +++ b/table/block_based_table_builder.cc @@ -474,7 +474,8 @@ struct BlockBasedTableBuilder::Rep { const InternalKeyComparator& icomparator, const std::vector>* int_tbl_prop_collector_factories, - WritableFileWriter* f, const CompressionType _compression_type, + uint32_t column_family_id, WritableFileWriter* f, + const CompressionType _compression_type, const CompressionOptions& _compression_opts, const bool skip_filters) : ioptions(_ioptions), table_options(table_opt), @@ -494,7 +495,7 @@ struct BlockBasedTableBuilder::Rep { table_options, data_block)) { for (auto& collector_factories : *int_tbl_prop_collector_factories) { table_properties_collectors.emplace_back( - collector_factories->CreateIntTblPropCollector()); + collector_factories->CreateIntTblPropCollector(column_family_id)); } table_properties_collectors.emplace_back( new BlockBasedTablePropertiesCollector( @@ -509,7 +510,8 @@ BlockBasedTableBuilder::BlockBasedTableBuilder( const InternalKeyComparator& internal_comparator, const std::vector>* int_tbl_prop_collector_factories, - WritableFileWriter* file, const CompressionType compression_type, + uint32_t column_family_id, WritableFileWriter* file, + const CompressionType compression_type, const CompressionOptions& compression_opts, const bool skip_filters) { BlockBasedTableOptions sanitized_table_options(table_options); if (sanitized_table_options.format_version == 0 && @@ -523,8 +525,8 @@ BlockBasedTableBuilder::BlockBasedTableBuilder( } rep_ = new Rep(ioptions, sanitized_table_options, internal_comparator, - int_tbl_prop_collector_factories, file, compression_type, - compression_opts, skip_filters); + int_tbl_prop_collector_factories, column_family_id, file, + compression_type, compression_opts, skip_filters); if (rep_->filter_block != nullptr) { rep_->filter_block->StartBlock(0); diff --git a/table/block_based_table_builder.h b/table/block_based_table_builder.h index ce868207a..7dc93b754 100644 --- a/table/block_based_table_builder.h +++ b/table/block_based_table_builder.h @@ -40,7 +40,8 @@ class BlockBasedTableBuilder : public TableBuilder { const InternalKeyComparator& internal_comparator, const std::vector>* int_tbl_prop_collector_factories, - WritableFileWriter* file, const CompressionType compression_type, + uint32_t column_family_id, WritableFileWriter* file, + const CompressionType compression_type, const CompressionOptions& compression_opts, const bool skip_filters); // REQUIRES: Either Finish() or Abandon() has been called. diff --git a/table/block_based_table_factory.cc b/table/block_based_table_factory.cc index ea910c6b2..1631652dd 100644 --- a/table/block_based_table_factory.cc +++ b/table/block_based_table_factory.cc @@ -61,13 +61,13 @@ Status BlockBasedTableFactory::NewTableReader( } TableBuilder* BlockBasedTableFactory::NewTableBuilder( - const TableBuilderOptions& table_builder_options, + const TableBuilderOptions& table_builder_options, uint32_t column_family_id, WritableFileWriter* file) const { auto table_builder = new BlockBasedTableBuilder( table_builder_options.ioptions, table_options_, table_builder_options.internal_comparator, - table_builder_options.int_tbl_prop_collector_factories, file, - table_builder_options.compression_type, + table_builder_options.int_tbl_prop_collector_factories, column_family_id, + file, table_builder_options.compression_type, table_builder_options.compression_opts, table_builder_options.skip_filters); diff --git a/table/block_based_table_factory.h b/table/block_based_table_factory.h index 8bdd4cd74..522e31c22 100644 --- a/table/block_based_table_factory.h +++ b/table/block_based_table_factory.h @@ -48,7 +48,7 @@ class BlockBasedTableFactory : public TableFactory { TableBuilder* NewTableBuilder( const TableBuilderOptions& table_builder_options, - WritableFileWriter* file) const override; + uint32_t column_family_id, WritableFileWriter* file) const override; // Sanitizes the specified DB Options. Status SanitizeOptions(const DBOptions& db_opts, diff --git a/table/cuckoo_table_factory.cc b/table/cuckoo_table_factory.cc index 16bf3fbe5..2b9407f2f 100644 --- a/table/cuckoo_table_factory.cc +++ b/table/cuckoo_table_factory.cc @@ -27,7 +27,7 @@ Status CuckooTableFactory::NewTableReader( } TableBuilder* CuckooTableFactory::NewTableBuilder( - const TableBuilderOptions& table_builder_options, + const TableBuilderOptions& table_builder_options, uint32_t column_family_id, WritableFileWriter* file) const { // Ignore the skipFIlters flag. Does not apply to this file format // diff --git a/table/cuckoo_table_factory.h b/table/cuckoo_table_factory.h index 394e834fa..b28342c14 100644 --- a/table/cuckoo_table_factory.h +++ b/table/cuckoo_table_factory.h @@ -62,7 +62,7 @@ class CuckooTableFactory : public TableFactory { TableBuilder* NewTableBuilder( const TableBuilderOptions& table_builder_options, - WritableFileWriter* file) const override; + uint32_t column_family_id, WritableFileWriter* file) const override; // Sanitizes the specified DB Options. Status SanitizeOptions(const DBOptions& db_opts, diff --git a/table/mock_table.cc b/table/mock_table.cc index ff56d6311..d75630374 100644 --- a/table/mock_table.cc +++ b/table/mock_table.cc @@ -74,7 +74,7 @@ Status MockTableFactory::NewTableReader( } TableBuilder* MockTableFactory::NewTableBuilder( - const TableBuilderOptions& table_builder_options, + const TableBuilderOptions& table_builder_options, uint32_t column_family_id, WritableFileWriter* file) const { uint32_t id = GetAndWriteNextID(file->writable_file()); diff --git a/table/mock_table.h b/table/mock_table.h index 322a51d1e..e313fbc08 100644 --- a/table/mock_table.h +++ b/table/mock_table.h @@ -151,7 +151,7 @@ class MockTableFactory : public TableFactory { unique_ptr* table_reader) const override; TableBuilder* NewTableBuilder( const TableBuilderOptions& table_builder_options, - WritableFileWriter* file) const override; + uint32_t column_familly_id, WritableFileWriter* file) const override; // This function will directly create mock table instead of going through // MockTableBuilder. file_contents has to have a format of >* int_tbl_prop_collector_factories, - WritableFileWriter* file, uint32_t user_key_len, EncodingType encoding_type, - size_t index_sparseness, uint32_t bloom_bits_per_key, uint32_t num_probes, - size_t huge_page_tlb_size, double hash_table_ratio, - bool store_index_in_file) + uint32_t column_family_id, WritableFileWriter* file, uint32_t user_key_len, + EncodingType encoding_type, size_t index_sparseness, + uint32_t bloom_bits_per_key, uint32_t num_probes, size_t huge_page_tlb_size, + double hash_table_ratio, bool store_index_in_file) : ioptions_(ioptions), bloom_block_(num_probes), file_(file), @@ -108,7 +108,7 @@ PlainTableBuilder::PlainTableBuilder( for (auto& collector_factories : *int_tbl_prop_collector_factories) { table_properties_collectors_.emplace_back( - collector_factories->CreateIntTblPropCollector()); + collector_factories->CreateIntTblPropCollector(column_family_id)); } } diff --git a/table/plain_table_builder.h b/table/plain_table_builder.h index 75ec3facd..5c0cad977 100644 --- a/table/plain_table_builder.h +++ b/table/plain_table_builder.h @@ -34,11 +34,11 @@ class PlainTableBuilder: public TableBuilder { const ImmutableCFOptions& ioptions, const std::vector>* int_tbl_prop_collector_factories, - WritableFileWriter* file, uint32_t user_key_size, - EncodingType encoding_type, size_t index_sparseness, - uint32_t bloom_bits_per_key, uint32_t num_probes = 6, - size_t huge_page_tlb_size = 0, double hash_table_ratio = 0, - bool store_index_in_file = false); + uint32_t column_family_id, WritableFileWriter* file, + uint32_t user_key_size, EncodingType encoding_type, + size_t index_sparseness, uint32_t bloom_bits_per_key, + uint32_t num_probes = 6, size_t huge_page_tlb_size = 0, + double hash_table_ratio = 0, bool store_index_in_file = false); // REQUIRES: Either Finish() or Abandon() has been called. ~PlainTableBuilder(); diff --git a/table/plain_table_factory.cc b/table/plain_table_factory.cc index 6e86ff54f..4d25f2150 100644 --- a/table/plain_table_factory.cc +++ b/table/plain_table_factory.cc @@ -26,7 +26,7 @@ Status PlainTableFactory::NewTableReader( } TableBuilder* PlainTableFactory::NewTableBuilder( - const TableBuilderOptions& table_builder_options, + const TableBuilderOptions& table_builder_options, uint32_t column_family_id, WritableFileWriter* file) const { // Ignore the skip_filters flag. PlainTable format is optimized for small // in-memory dbs. The skip_filters optimization is not useful for plain @@ -34,9 +34,10 @@ TableBuilder* PlainTableFactory::NewTableBuilder( // return new PlainTableBuilder( table_builder_options.ioptions, - table_builder_options.int_tbl_prop_collector_factories, file, - user_key_len_, encoding_type_, index_sparseness_, bloom_bits_per_key_, 6, - huge_page_tlb_size_, hash_table_ratio_, store_index_in_file_); + table_builder_options.int_tbl_prop_collector_factories, column_family_id, + file, user_key_len_, encoding_type_, index_sparseness_, + bloom_bits_per_key_, 6, huge_page_tlb_size_, hash_table_ratio_, + store_index_in_file_); } std::string PlainTableFactory::GetPrintableTableOptions() const { diff --git a/table/plain_table_factory.h b/table/plain_table_factory.h index 539e7539d..5d1125bb0 100644 --- a/table/plain_table_factory.h +++ b/table/plain_table_factory.h @@ -159,7 +159,7 @@ class PlainTableFactory : public TableFactory { unique_ptr* table) const override; TableBuilder* NewTableBuilder( const TableBuilderOptions& table_builder_options, - WritableFileWriter* file) const override; + uint32_t column_family_id, WritableFileWriter* file) const override; std::string GetPrintableTableOptions() const override; diff --git a/table/sst_file_writer.cc b/table/sst_file_writer.cc index d780f0a4b..1c21a25f7 100644 --- a/table/sst_file_writer.cc +++ b/table/sst_file_writer.cc @@ -57,7 +57,8 @@ class SstFileWriter::SstFileWriterPropertiesCollectorFactory explicit SstFileWriterPropertiesCollectorFactory(int32_t version) : version_(version) {} - virtual IntTblPropCollector* CreateIntTblPropCollector() override { + virtual IntTblPropCollector* CreateIntTblPropCollector( + uint32_t column_family_id) override { return new SstFileWriterPropertiesCollector(version_); } @@ -117,7 +118,9 @@ Status SstFileWriter::Open(const std::string& file_path) { r->file_writer.reset( new WritableFileWriter(std::move(sst_file), r->env_options)); r->builder.reset(r->ioptions.table_factory->NewTableBuilder( - table_builder_options, r->file_writer.get())); + table_builder_options, + TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, + r->file_writer.get())); r->file_info.file_path = file_path; r->file_info.file_size = 0; diff --git a/table/table_properties.cc b/table/table_properties.cc index 86c084385..9193499fa 100644 --- a/table/table_properties.cc +++ b/table/table_properties.cc @@ -12,6 +12,9 @@ namespace rocksdb { +const uint32_t TablePropertiesCollectorFactory::Context::kUnknownColumnFamily = + port::kMaxInt32; + namespace { void AppendProperty( std::string& props, diff --git a/table/table_reader_bench.cc b/table/table_reader_bench.cc index e3baa29ed..b940d89de 100644 --- a/table/table_reader_bench.cc +++ b/table/table_reader_bench.cc @@ -98,7 +98,7 @@ void TableReaderBenchmark(Options& opts, EnvOptions& env_options, TableBuilderOptions(ioptions, ikc, &int_tbl_prop_collector_factories, CompressionType::kNoCompression, CompressionOptions(), false), - file_writer.get()); + 0, file_writer.get()); } else { s = DB::Open(opts, dbname, &db); ASSERT_OK(s); diff --git a/table/table_test.cc b/table/table_test.cc index e21503b8f..928861ae2 100644 --- a/table/table_test.cc +++ b/table/table_test.cc @@ -272,6 +272,7 @@ class TableConstructor: public Constructor { TableBuilderOptions(ioptions, internal_comparator, &int_tbl_prop_collector_factories, options.compression, CompressionOptions(), false), + TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, file_writer_.get())); for (const auto kv : kv_map) { @@ -1824,6 +1825,7 @@ TEST_F(PlainTableTest, BasicPlainTableProperties) { std::unique_ptr builder(factory.NewTableBuilder( TableBuilderOptions(ioptions, ikc, &int_tbl_prop_collector_factories, kNoCompression, CompressionOptions(), false), + TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, file_writer.get())); for (char c = 'a'; c <= 'z'; ++c) { diff --git a/util/sst_dump_test.cc b/util/sst_dump_test.cc index 50e9f1042..b40a3346c 100644 --- a/util/sst_dump_test.cc +++ b/util/sst_dump_test.cc @@ -62,6 +62,7 @@ void createSST(const std::string& file_name, TableBuilderOptions(imoptions, ikc, &int_tbl_prop_collector_factories, CompressionType::kNoCompression, CompressionOptions(), false), + TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, file_writer.get())); // Populate slightly more than 1K keys diff --git a/util/sst_dump_tool.cc b/util/sst_dump_tool.cc index 2e319018a..de7f6f13c 100644 --- a/util/sst_dump_tool.cc +++ b/util/sst_dump_tool.cc @@ -124,7 +124,9 @@ uint64_t SstFileReader::CalculateCompressedTableSize( BlockBasedTableFactory block_based_tf(table_options); unique_ptr table_builder; table_builder.reset(block_based_tf.NewTableBuilder( - tb_options, dest_writer.get())); + tb_options, + TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, + dest_writer.get())); unique_ptr iter(table_reader_->NewIterator(ReadOptions())); for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { if (!iter->status().ok()) { diff --git a/utilities/table_properties_collectors/compact_on_deletion_collector.cc b/utilities/table_properties_collectors/compact_on_deletion_collector.cc index be0e53ae6..2079ccb86 100644 --- a/utilities/table_properties_collectors/compact_on_deletion_collector.cc +++ b/utilities/table_properties_collectors/compact_on_deletion_collector.cc @@ -4,10 +4,10 @@ // of patent rights can be found in the PATENTS file in the same directory. #ifndef ROCKSDB_LITE -#include +#include "utilities/table_properties_collectors/compact_on_deletion_collector.h" +#include #include "rocksdb/utilities/table_properties_collectors.h" -#include "utilities/table_properties_collectors/compact_on_deletion_collector.h" namespace rocksdb { @@ -75,8 +75,9 @@ Status CompactOnDeletionCollector::AddUserKey( return Status::OK(); } -TablePropertiesCollector* CompactOnDeletionCollectorFactory:: - CreateTablePropertiesCollector() { +TablePropertiesCollector* +CompactOnDeletionCollectorFactory::CreateTablePropertiesCollector( + TablePropertiesCollectorFactory::Context context) { return new CompactOnDeletionCollector( sliding_window_size_, deletion_trigger_); } diff --git a/utilities/table_properties_collectors/compact_on_deletion_collector.h b/utilities/table_properties_collectors/compact_on_deletion_collector.h index eb01e439d..3001ce913 100644 --- a/utilities/table_properties_collectors/compact_on_deletion_collector.h +++ b/utilities/table_properties_collectors/compact_on_deletion_collector.h @@ -6,6 +6,7 @@ #pragma once #ifndef ROCKSDB_LITE +#include "rocksdb/utilities/table_properties_collectors.h" namespace rocksdb { // A factory of a table property collector that marks a SST @@ -28,7 +29,8 @@ class CompactOnDeletionCollectorFactory virtual ~CompactOnDeletionCollectorFactory() {} - virtual TablePropertiesCollector* CreateTablePropertiesCollector() override; + virtual TablePropertiesCollector* CreateTablePropertiesCollector( + TablePropertiesCollectorFactory::Context context) override; virtual const char* Name() const override { return "CompactOnDeletionCollector"; diff --git a/utilities/table_properties_collectors/compact_on_deletion_collector_test.cc b/utilities/table_properties_collectors/compact_on_deletion_collector_test.cc index 12f4e2e4f..89e6bbcb8 100644 --- a/utilities/table_properties_collectors/compact_on_deletion_collector_test.cc +++ b/utilities/table_properties_collectors/compact_on_deletion_collector_test.cc @@ -15,6 +15,7 @@ #include #include "rocksdb/table.h" +#include "rocksdb/table_properties.h" #include "rocksdb/utilities/table_properties_collectors.h" #include "util/random.h" #include "utilities/table_properties_collectors/compact_on_deletion_collector.h" @@ -24,6 +25,9 @@ int main(int argc, char** argv) { {1000, 10000, 10000, 127, 128, 129, 255, 256, 257, 2, 10000}; const int kDeletionTriggers[] = {500, 9500, 4323, 47, 61, 128, 250, 250, 250, 2, 2}; + rocksdb::TablePropertiesCollectorFactory::Context context; + context.column_family_id = + rocksdb::TablePropertiesCollectorFactory::Context::kUnknownColumnFamily; std::vector window_sizes; std::vector deletion_triggers; @@ -57,8 +61,7 @@ int main(int argc, char** argv) { std::unique_ptr collector; auto factory = rocksdb::NewCompactOnDeletionCollectorFactory( kWindowSize, kNumDeletionTrigger); - collector.reset( - factory->CreateTablePropertiesCollector()); + collector.reset(factory->CreateTablePropertiesCollector(context)); const int kSample = 10; for (int delete_rate = 0; delete_rate <= kSample; ++delete_rate) { int deletions = 0; @@ -90,8 +93,7 @@ int main(int argc, char** argv) { std::unique_ptr collector; auto factory = rocksdb::NewCompactOnDeletionCollectorFactory( kWindowSize, kNumDeletionTrigger); - collector.reset( - factory->CreateTablePropertiesCollector()); + collector.reset(factory->CreateTablePropertiesCollector(context)); const int kSample = 10; for (int delete_rate = 0; delete_rate <= kSample; ++delete_rate) { int deletions = 0; @@ -138,8 +140,7 @@ int main(int argc, char** argv) { std::unique_ptr collector; auto factory = rocksdb::NewCompactOnDeletionCollectorFactory( kWindowSize, kNumDeletionTrigger); - collector.reset( - factory->CreateTablePropertiesCollector()); + collector.reset(factory->CreateTablePropertiesCollector(context)); assert(collector->NeedCompact() == false); // Insert "kNumDeletionTrigger * 0.95" deletions for every // "kWindowSize" and verify compaction is not needed.