diff --git a/db/builder.cc b/db/builder.cc index 317c9b054..9c98c0db7 100644 --- a/db/builder.cc +++ b/db/builder.cc @@ -41,13 +41,17 @@ TableBuilder* NewTableBuilder( const InternalKeyComparator& internal_comparator, const std::vector>* int_tbl_prop_collector_factories, - uint32_t column_family_id, WritableFileWriter* file, - const CompressionType compression_type, + uint32_t column_family_id, const std::string& column_family_name, + WritableFileWriter* file, const CompressionType compression_type, const CompressionOptions& compression_opts, const bool skip_filters) { + assert((column_family_id == + TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) == + column_family_name.empty()); return ioptions.table_factory->NewTableBuilder( TableBuilderOptions(ioptions, internal_comparator, int_tbl_prop_collector_factories, compression_type, - compression_opts, skip_filters), + compression_opts, skip_filters, + column_family_name), column_family_id, file); } @@ -58,12 +62,16 @@ Status BuildTable( const InternalKeyComparator& internal_comparator, const std::vector>* int_tbl_prop_collector_factories, - uint32_t column_family_id, std::vector snapshots, + uint32_t column_family_id, const std::string& column_family_name, + std::vector snapshots, SequenceNumber earliest_write_conflict_snapshot, const CompressionType compression, const CompressionOptions& compression_opts, bool paranoid_file_checks, InternalStats* internal_stats, const Env::IOPriority io_priority, TableProperties* table_properties, int level) { + assert((column_family_id == + TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) == + column_family_name.empty()); // Reports the IOStats for flush for every following bytes. const size_t kReportFlushIOStatsEvery = 1048576; Status s; @@ -87,7 +95,8 @@ Status BuildTable( builder = NewTableBuilder( ioptions, internal_comparator, int_tbl_prop_collector_factories, - column_family_id, file_writer.get(), compression, compression_opts); + column_family_id, column_family_name, file_writer.get(), compression, + compression_opts); } MergeHelper merge(env, internal_comparator.user_comparator(), diff --git a/db/builder.h b/db/builder.h index 1eba6da9c..bf54aed5d 100644 --- a/db/builder.h +++ b/db/builder.h @@ -33,13 +33,16 @@ class WritableFileWriter; class InternalStats; class InternalIterator; +// @param column_family_name Name of the column family that is also identified +// by column_family_id, or empty string if unknown. It must outlive the +// TableBuilder returned by this function. TableBuilder* NewTableBuilder( const ImmutableCFOptions& options, const InternalKeyComparator& internal_comparator, const std::vector>* int_tbl_prop_collector_factories, - uint32_t column_family_id, WritableFileWriter* file, - const CompressionType compression_type, + uint32_t column_family_id, const std::string& column_family_name, + WritableFileWriter* file, const CompressionType compression_type, const CompressionOptions& compression_opts, const bool skip_filters = false); @@ -48,6 +51,9 @@ TableBuilder* NewTableBuilder( // *meta will be filled with metadata about the generated table. // If no data is present in *iter, meta->file_size will be set to // zero, and no Table file will be produced. +// +// @param column_family_name Name of the column family that is also identified +// by column_family_id, or empty string if unknown. extern Status BuildTable( const std::string& dbname, Env* env, const ImmutableCFOptions& options, const EnvOptions& env_options, TableCache* table_cache, @@ -55,7 +61,8 @@ extern Status BuildTable( const InternalKeyComparator& internal_comparator, const std::vector>* int_tbl_prop_collector_factories, - uint32_t column_family_id, std::vector snapshots, + uint32_t column_family_id, const std::string& column_family_name, + std::vector snapshots, SequenceNumber earliest_write_conflict_snapshot, const CompressionType compression, const CompressionOptions& compression_opts, bool paranoid_file_checks, diff --git a/db/compaction_job.cc b/db/compaction_job.cc index 450625019..487490f57 100644 --- a/db/compaction_job.cc +++ b/db/compaction_job.cc @@ -1019,7 +1019,7 @@ Status CompactionJob::OpenCompactionOutputFile( cfd->ioptions()->optimize_filters_for_hits && bottommost_level_; sub_compact->builder.reset(NewTableBuilder( *cfd->ioptions(), cfd->internal_comparator(), - cfd->int_tbl_prop_collector_factories(), cfd->GetID(), + cfd->int_tbl_prop_collector_factories(), cfd->GetID(), cfd->GetName(), sub_compact->outfile.get(), sub_compact->compaction->output_compression(), cfd->ioptions()->compression_opts, skip_filters)); LogFlush(db_options_.info_log); diff --git a/db/db_impl.cc b/db/db_impl.cc index db5390bca..1dd29eae5 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -1452,8 +1452,8 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd, s = BuildTable( dbname_, env_, *cfd->ioptions(), env_options_, cfd->table_cache(), iter.get(), &meta, cfd->internal_comparator(), - cfd->int_tbl_prop_collector_factories(), cfd->GetID(), snapshot_seqs, - earliest_write_conflict_snapshot, + cfd->int_tbl_prop_collector_factories(), cfd->GetID(), cfd->GetName(), + snapshot_seqs, earliest_write_conflict_snapshot, GetCompressionFlush(*cfd->ioptions()), cfd->ioptions()->compression_opts, paranoid_file_checks, cfd->internal_stats(), Env::IO_HIGH, &info.table_properties); diff --git a/db/db_table_properties_test.cc b/db/db_table_properties_test.cc index 87ba13d86..4501eb4dc 100644 --- a/db/db_table_properties_test.cc +++ b/db/db_table_properties_test.cc @@ -215,6 +215,34 @@ TEST_F(DBTablePropertiesTest, GetPropertiesOfTablesInRange) { TestGetPropertiesOfTablesInRange(std::move(ranges)); } } + +TEST_F(DBTablePropertiesTest, GetColumnFamilyNameProperty) { + std::string kExtraCfName = "pikachu"; + CreateAndReopenWithCF({kExtraCfName}, CurrentOptions()); + + // Create one table per CF, then verify it was created with the column family + // name property. + for (int cf = 0; cf < 2; ++cf) { + Put(cf, "key", "val"); + Flush(cf); + + TablePropertiesCollection fname_to_props; + ASSERT_OK(db_->GetPropertiesOfAllTables(handles_[cf], &fname_to_props)); + ASSERT_EQ(1U, fname_to_props.size()); + + std::string expected_cf_name; + if (cf > 0) { + expected_cf_name = kExtraCfName; + } else { + expected_cf_name = kDefaultColumnFamilyName; + } + ASSERT_EQ(expected_cf_name, + fname_to_props.begin()->second->column_family_name); + ASSERT_EQ(cf, static_cast( + fname_to_props.begin()->second->column_family_id)); + } +} + } // namespace rocksdb #endif // ROCKSDB_LITE diff --git a/db/flush_job.cc b/db/flush_job.cc index b4e5b307f..51c86e6f6 100644 --- a/db/flush_job.cc +++ b/db/flush_job.cc @@ -238,8 +238,9 @@ Status FlushJob::WriteLevel0Table(const autovector& mems, dbname_, db_options_.env, *cfd_->ioptions(), env_options_, cfd_->table_cache(), iter.get(), meta, cfd_->internal_comparator(), cfd_->int_tbl_prop_collector_factories(), cfd_->GetID(), - existing_snapshots_, earliest_write_conflict_snapshot_, - output_compression_, cfd_->ioptions()->compression_opts, + cfd_->GetName(), existing_snapshots_, + earliest_write_conflict_snapshot_, output_compression_, + cfd_->ioptions()->compression_opts, mutable_cf_options_.paranoid_file_checks, cfd_->internal_stats(), Env::IO_HIGH, &table_properties_, 0 /* level */); info.table_properties = table_properties_; diff --git a/db/plain_table_db_test.cc b/db/plain_table_db_test.cc index 8cb070ac6..a268fc627 100644 --- a/db/plain_table_db_test.cc +++ b/db/plain_table_db_test.cc @@ -263,7 +263,9 @@ class TestPlainTableReader : public PlainTableReader { const TableProperties* table_properties, unique_ptr&& file, const ImmutableCFOptions& ioptions, - bool* expect_bloom_not_match, bool store_index_in_file) + bool* expect_bloom_not_match, bool store_index_in_file, + uint32_t column_family_id, + const std::string& column_family_name) : PlainTableReader(ioptions, std::move(file), env_options, icomparator, encoding_type, file_size, table_properties), expect_bloom_not_match_(expect_bloom_not_match) { @@ -276,6 +278,8 @@ class TestPlainTableReader : public PlainTableReader { EXPECT_TRUE(s.ok()); TableProperties* props = const_cast(table_properties); + EXPECT_EQ(column_family_id, static_cast(props->column_family_id)); + EXPECT_EQ(column_family_name, props->column_family_name); if (store_index_in_file) { auto bloom_version_ptr = props->user_collected_properties.find( PlainTablePropertyNames::kBloomVersion); @@ -308,13 +312,17 @@ extern const uint64_t kPlainTableMagicNumber; class TestPlainTableFactory : public PlainTableFactory { public: explicit TestPlainTableFactory(bool* expect_bloom_not_match, - const PlainTableOptions& options) + const PlainTableOptions& options, + uint32_t column_family_id, + std::string column_family_name) : PlainTableFactory(options), bloom_bits_per_key_(options.bloom_bits_per_key), hash_table_ratio_(options.hash_table_ratio), index_sparseness_(options.index_sparseness), store_index_in_file_(options.store_index_in_file), - expect_bloom_not_match_(expect_bloom_not_match) {} + expect_bloom_not_match_(expect_bloom_not_match), + column_family_id_(column_family_id), + column_family_name_(std::move(column_family_name)) {} Status NewTableReader(const TableReaderOptions& table_reader_options, unique_ptr&& file, @@ -354,7 +362,7 @@ class TestPlainTableFactory : public PlainTableFactory { table_reader_options.internal_comparator, encoding_type, file_size, bloom_bits_per_key_, hash_table_ratio_, index_sparseness_, props, std::move(file), table_reader_options.ioptions, expect_bloom_not_match_, - store_index_in_file_)); + store_index_in_file_, column_family_id_, column_family_name_)); *table = std::move(new_reader); return s; @@ -366,6 +374,8 @@ class TestPlainTableFactory : public PlainTableFactory { size_t index_sparseness_; bool store_index_in_file_; bool* expect_bloom_not_match_; + const uint32_t column_family_id_; + const std::string column_family_name_; }; TEST_P(PlainTableDBTest, Flush) { @@ -492,7 +502,8 @@ TEST_P(PlainTableDBTest, Flush2) { plain_table_options.encoding_type = encoding_type; plain_table_options.store_index_in_file = store_index_in_file; options.table_factory.reset(new TestPlainTableFactory( - &expect_bloom_not_match, plain_table_options)); + &expect_bloom_not_match, plain_table_options, + 0 /* column_family_id */, kDefaultColumnFamilyName)); DestroyAndReopen(&options); ASSERT_OK(Put("0000000000000bar", "b")); @@ -561,7 +572,8 @@ TEST_P(PlainTableDBTest, Iterator) { plain_table_options.encoding_type = encoding_type; options.table_factory.reset(new TestPlainTableFactory( - &expect_bloom_not_match, plain_table_options)); + &expect_bloom_not_match, plain_table_options, + 0 /* column_family_id */, kDefaultColumnFamilyName)); } else { PlainTableOptions plain_table_options; plain_table_options.user_key_len = 16; @@ -572,7 +584,8 @@ TEST_P(PlainTableDBTest, Iterator) { plain_table_options.encoding_type = encoding_type; options.table_factory.reset(new TestPlainTableFactory( - &expect_bloom_not_match, plain_table_options)); + &expect_bloom_not_match, plain_table_options, + 0 /* column_family_id */, kDefaultColumnFamilyName)); } DestroyAndReopen(&options); diff --git a/db/repair.cc b/db/repair.cc index 1681fd585..68fef0783 100644 --- a/db/repair.cc +++ b/db/repair.cc @@ -294,9 +294,10 @@ class Repairer { status = BuildTable( dbname_, env_, ioptions_, env_options_, table_cache_, iter.get(), &meta, icmp_, &int_tbl_prop_collector_factories_, - TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, {}, - kMaxSequenceNumber, kNoCompression, CompressionOptions(), false, - nullptr); + TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, + std::string() /* column_family_name */, {}, kMaxSequenceNumber, + kNoCompression, CompressionOptions(), false, + nullptr /* internal_stats */); } delete mem->Unref(); delete cf_mems_default; diff --git a/db/table_properties_collector_test.cc b/db/table_properties_collector_test.cc index d096e6c79..00751d0ce 100644 --- a/db/table_properties_collector_test.cc +++ b/db/table_properties_collector_test.cc @@ -36,6 +36,7 @@ class TablePropertiesTest : public testing::Test, // Utilities test functions namespace { static const uint32_t kTestColumnFamilyId = 66; +static const std::string kTestColumnFamilyName = "test_column_fam"; void MakeBuilder(const Options& options, const ImmutableCFOptions& ioptions, const InternalKeyComparator& internal_comparator, @@ -48,8 +49,8 @@ void MakeBuilder(const Options& options, const ImmutableCFOptions& ioptions, builder->reset(NewTableBuilder( ioptions, internal_comparator, int_tbl_prop_collector_factories, - kTestColumnFamilyId /* column_family_id */, writable->get(), - options.compression, options.compression_opts)); + kTestColumnFamilyId, kTestColumnFamilyName, + writable->get(), options.compression, options.compression_opts)); } } // namespace diff --git a/include/rocksdb/table_properties.h b/include/rocksdb/table_properties.h index 73a825de8..c04e26d18 100644 --- a/include/rocksdb/table_properties.h +++ b/include/rocksdb/table_properties.h @@ -27,47 +27,6 @@ namespace rocksdb { // } typedef std::map UserCollectedProperties; -// TableProperties contains a bunch of read-only properties of its associated -// table. -struct TableProperties { - public: - // the total size of all data blocks. - uint64_t data_size = 0; - // the size of index block. - uint64_t index_size = 0; - // the size of filter block. - uint64_t filter_size = 0; - // total raw key size - uint64_t raw_key_size = 0; - // total raw value size - uint64_t raw_value_size = 0; - // the number of blocks in this table - uint64_t num_data_blocks = 0; - // the number of entries in this table - uint64_t num_entries = 0; - // format version, reserved for backward compatibility - uint64_t format_version = 0; - // If 0, key is variable length. Otherwise number of bytes for each key. - uint64_t fixed_key_len = 0; - - // The name of the filter policy used in this table. - // If no filter policy is used, `filter_policy_name` will be an empty string. - std::string filter_policy_name; - - // user collected properties - UserCollectedProperties user_collected_properties; - UserCollectedProperties readable_properties; - - // convert this object to a human readable form - // @prop_delim: delimiter for each property. - std::string ToString(const std::string& prop_delim = "; ", - const std::string& kv_delim = "=") const; - - // Aggregate the numerical member variables of the specified - // TableProperties. - void Add(const TableProperties& tp); -}; - // table properties' human-readable names in the property block. struct TablePropertiesNames { static const std::string kDataSize; @@ -80,6 +39,8 @@ struct TablePropertiesNames { static const std::string kFormatVersion; static const std::string kFixedKeyLen; static const std::string kFilterPolicy; + static const std::string kColumnFamilyName; + static const std::string kColumnFamilyId; }; extern const std::string kPropertiesBlock; @@ -158,6 +119,55 @@ class TablePropertiesCollectorFactory { virtual const char* Name() const = 0; }; +// TableProperties contains a bunch of read-only properties of its associated +// table. +struct TableProperties { + public: + // the total size of all data blocks. + uint64_t data_size = 0; + // the size of index block. + uint64_t index_size = 0; + // the size of filter block. + uint64_t filter_size = 0; + // total raw key size + uint64_t raw_key_size = 0; + // total raw value size + uint64_t raw_value_size = 0; + // the number of blocks in this table + uint64_t num_data_blocks = 0; + // the number of entries in this table + uint64_t num_entries = 0; + // format version, reserved for backward compatibility + uint64_t format_version = 0; + // If 0, key is variable length. Otherwise number of bytes for each key. + uint64_t fixed_key_len = 0; + // ID of column family for this SST file, corresponding to the CF identified + // by column_family_name. + uint64_t column_family_id = + rocksdb::TablePropertiesCollectorFactory::Context::kUnknownColumnFamily; + + // Name of the column family with which this SST file is associated. + // If column family is unknown, `column_family_name` will be an empty string. + std::string column_family_name; + + // The name of the filter policy used in this table. + // If no filter policy is used, `filter_policy_name` will be an empty string. + std::string filter_policy_name; + + // user collected properties + UserCollectedProperties user_collected_properties; + UserCollectedProperties readable_properties; + + // convert this object to a human readable form + // @prop_delim: delimiter for each property. + std::string ToString(const std::string& prop_delim = "; ", + const std::string& kv_delim = "=") const; + + // Aggregate the numerical member variables of the specified + // TableProperties. + void Add(const TableProperties& tp); +}; + // Extra properties // Below is a list of non-basic properties that are collected by database // itself. Especially some properties regarding to the internal keys (which diff --git a/table/block_based_table_builder.cc b/table/block_based_table_builder.cc index 47d74bc5f..24f4222e1 100644 --- a/table/block_based_table_builder.cc +++ b/table/block_based_table_builder.cc @@ -472,6 +472,8 @@ struct BlockBasedTableBuilder::Rep { std::string compressed_output; std::unique_ptr flush_block_policy; + uint32_t column_family_id; + const std::string& column_family_name; std::vector> table_properties_collectors; @@ -480,9 +482,10 @@ struct BlockBasedTableBuilder::Rep { const InternalKeyComparator& icomparator, const std::vector>* int_tbl_prop_collector_factories, - uint32_t column_family_id, WritableFileWriter* f, + uint32_t _column_family_id, WritableFileWriter* f, const CompressionType _compression_type, - const CompressionOptions& _compression_opts, const bool skip_filters) + const CompressionOptions& _compression_opts, const bool skip_filters, + const std::string& _column_family_name) : ioptions(_ioptions), table_options(table_opt), internal_comparator(icomparator), @@ -500,7 +503,9 @@ struct BlockBasedTableBuilder::Rep { _ioptions, table_options)), flush_block_policy( table_options.flush_block_policy_factory->NewFlushBlockPolicy( - table_options, data_block)) { + table_options, data_block)), + column_family_id(_column_family_id), + column_family_name(_column_family_name) { for (auto& collector_factories : *int_tbl_prop_collector_factories) { table_properties_collectors.emplace_back( collector_factories->CreateIntTblPropCollector(column_family_id)); @@ -520,7 +525,8 @@ BlockBasedTableBuilder::BlockBasedTableBuilder( int_tbl_prop_collector_factories, uint32_t column_family_id, WritableFileWriter* file, const CompressionType compression_type, - const CompressionOptions& compression_opts, const bool skip_filters) { + const CompressionOptions& compression_opts, const bool skip_filters, + const std::string& column_family_name) { BlockBasedTableOptions sanitized_table_options(table_options); if (sanitized_table_options.format_version == 0 && sanitized_table_options.checksum != kCRC32c) { @@ -534,7 +540,8 @@ BlockBasedTableBuilder::BlockBasedTableBuilder( rep_ = new Rep(ioptions, sanitized_table_options, internal_comparator, int_tbl_prop_collector_factories, column_family_id, file, - compression_type, compression_opts, skip_filters); + compression_type, compression_opts, skip_filters, + column_family_name); if (rep_->filter_block != nullptr) { rep_->filter_block->StartBlock(0); @@ -790,6 +797,8 @@ Status BlockBasedTableBuilder::Finish() { // Write properties block. { PropertyBlockBuilder property_block_builder; + r->props.column_family_id = r->column_family_id; + r->props.column_family_name = r->column_family_name; r->props.filter_policy_name = r->table_options.filter_policy != nullptr ? r->table_options.filter_policy->Name() : ""; r->props.index_size = diff --git a/table/block_based_table_builder.h b/table/block_based_table_builder.h index 49fe8dbf4..fed67783e 100644 --- a/table/block_based_table_builder.h +++ b/table/block_based_table_builder.h @@ -42,7 +42,8 @@ class BlockBasedTableBuilder : public TableBuilder { int_tbl_prop_collector_factories, uint32_t column_family_id, WritableFileWriter* file, const CompressionType compression_type, - const CompressionOptions& compression_opts, const bool skip_filters); + const CompressionOptions& compression_opts, const bool skip_filters, + const std::string& column_family_name); // REQUIRES: Either Finish() or Abandon() has been called. ~BlockBasedTableBuilder(); diff --git a/table/block_based_table_factory.cc b/table/block_based_table_factory.cc index c2617b168..ddd837b83 100644 --- a/table/block_based_table_factory.cc +++ b/table/block_based_table_factory.cc @@ -76,7 +76,8 @@ TableBuilder* BlockBasedTableFactory::NewTableBuilder( table_builder_options.int_tbl_prop_collector_factories, column_family_id, file, table_builder_options.compression_type, table_builder_options.compression_opts, - table_builder_options.skip_filters); + table_builder_options.skip_filters, + table_builder_options.column_family_name); return table_builder; } diff --git a/table/cuckoo_table_builder.cc b/table/cuckoo_table_builder.cc index f2f71b78e..0f93b54ed 100644 --- a/table/cuckoo_table_builder.cc +++ b/table/cuckoo_table_builder.cc @@ -52,7 +52,8 @@ CuckooTableBuilder::CuckooTableBuilder( uint32_t max_num_hash_table, uint32_t max_search_depth, const Comparator* user_comparator, uint32_t cuckoo_block_size, bool use_module_hash, bool identity_as_first_hash, - uint64_t (*get_slice_hash)(const Slice&, uint32_t, uint64_t)) + uint64_t (*get_slice_hash)(const Slice&, uint32_t, uint64_t), + uint32_t column_family_id, const std::string& column_family_name) : num_hash_func_(2), file_(file), max_hash_table_ratio_(max_hash_table_ratio), @@ -76,6 +77,8 @@ CuckooTableBuilder::CuckooTableBuilder( properties_.num_data_blocks = 1; properties_.index_size = 0; properties_.filter_size = 0; + properties_.column_family_id = column_family_id; + properties_.column_family_name = column_family_name; } void CuckooTableBuilder::Add(const Slice& key, const Slice& value) { diff --git a/table/cuckoo_table_builder.h b/table/cuckoo_table_builder.h index 8e6873e88..266a71be0 100644 --- a/table/cuckoo_table_builder.h +++ b/table/cuckoo_table_builder.h @@ -27,7 +27,9 @@ class CuckooTableBuilder: public TableBuilder { uint32_t cuckoo_block_size, bool use_module_hash, bool identity_as_first_hash, uint64_t (*get_slice_hash)(const Slice&, uint32_t, - uint64_t)); + uint64_t), + uint32_t column_family_id, + const std::string& column_family_name); // REQUIRES: Either Finish() or Abandon() has been called. ~CuckooTableBuilder() {} diff --git a/table/cuckoo_table_builder_test.cc b/table/cuckoo_table_builder_test.cc index fef7bc3a5..2fda8044e 100644 --- a/table/cuckoo_table_builder_test.cc +++ b/table/cuckoo_table_builder_test.cc @@ -89,6 +89,8 @@ class CuckooBuilderTest : public testing::Test { ASSERT_EQ(props->data_size, expected_unused_bucket.size() * (expected_table_size + expected_cuckoo_block_size - 1)); ASSERT_EQ(props->raw_key_size, keys.size()*props->fixed_key_len); + ASSERT_EQ(props->column_family_id, 0); + ASSERT_EQ(props->column_family_name, kDefaultColumnFamilyName); delete props; // Check contents of the bucket. @@ -148,7 +150,8 @@ TEST_F(CuckooBuilderTest, SuccessWithEmptyFile) { new WritableFileWriter(std::move(writable_file), EnvOptions())); CuckooTableBuilder builder(file_writer.get(), kHashTableRatio, 4, 100, BytewiseComparator(), 1, false, false, - GetSliceHash); + GetSliceHash, 0 /* column_family_id */, + kDefaultColumnFamilyName); ASSERT_OK(builder.status()); ASSERT_EQ(0UL, builder.FileSize()); ASSERT_OK(builder.Finish()); @@ -183,7 +186,8 @@ TEST_F(CuckooBuilderTest, WriteSuccessNoCollisionFullKey) { new WritableFileWriter(std::move(writable_file), EnvOptions())); CuckooTableBuilder builder(file_writer.get(), kHashTableRatio, num_hash_fun, 100, BytewiseComparator(), 1, false, false, - GetSliceHash); + GetSliceHash, 0 /* column_family_id */, + kDefaultColumnFamilyName); ASSERT_OK(builder.status()); for (uint32_t i = 0; i < user_keys.size(); i++) { builder.Add(Slice(keys[i]), Slice(values[i])); @@ -230,7 +234,8 @@ TEST_F(CuckooBuilderTest, WriteSuccessWithCollisionFullKey) { new WritableFileWriter(std::move(writable_file), EnvOptions())); CuckooTableBuilder builder(file_writer.get(), kHashTableRatio, num_hash_fun, 100, BytewiseComparator(), 1, false, false, - GetSliceHash); + GetSliceHash, 0 /* column_family_id */, + kDefaultColumnFamilyName); ASSERT_OK(builder.status()); for (uint32_t i = 0; i < user_keys.size(); i++) { builder.Add(Slice(keys[i]), Slice(values[i])); @@ -276,9 +281,10 @@ TEST_F(CuckooBuilderTest, WriteSuccessWithCollisionAndCuckooBlock) { ASSERT_OK(env_->NewWritableFile(fname, &writable_file, env_options_)); unique_ptr file_writer( new WritableFileWriter(std::move(writable_file), EnvOptions())); - CuckooTableBuilder builder(file_writer.get(), kHashTableRatio, num_hash_fun, - 100, BytewiseComparator(), cuckoo_block_size, - false, false, GetSliceHash); + CuckooTableBuilder builder( + file_writer.get(), kHashTableRatio, num_hash_fun, 100, + BytewiseComparator(), cuckoo_block_size, false, false, GetSliceHash, + 0 /* column_family_id */, kDefaultColumnFamilyName); ASSERT_OK(builder.status()); for (uint32_t i = 0; i < user_keys.size(); i++) { builder.Add(Slice(keys[i]), Slice(values[i])); @@ -330,7 +336,8 @@ TEST_F(CuckooBuilderTest, WithCollisionPathFullKey) { new WritableFileWriter(std::move(writable_file), EnvOptions())); CuckooTableBuilder builder(file_writer.get(), kHashTableRatio, num_hash_fun, 100, BytewiseComparator(), 1, false, false, - GetSliceHash); + GetSliceHash, 0 /* column_family_id */, + kDefaultColumnFamilyName); ASSERT_OK(builder.status()); for (uint32_t i = 0; i < user_keys.size(); i++) { builder.Add(Slice(keys[i]), Slice(values[i])); @@ -379,7 +386,8 @@ TEST_F(CuckooBuilderTest, WithCollisionPathFullKeyAndCuckooBlock) { new WritableFileWriter(std::move(writable_file), EnvOptions())); CuckooTableBuilder builder(file_writer.get(), kHashTableRatio, num_hash_fun, 100, BytewiseComparator(), 2, false, false, - GetSliceHash); + GetSliceHash, 0 /* column_family_id */, + kDefaultColumnFamilyName); ASSERT_OK(builder.status()); for (uint32_t i = 0; i < user_keys.size(); i++) { builder.Add(Slice(keys[i]), Slice(values[i])); @@ -421,7 +429,8 @@ TEST_F(CuckooBuilderTest, WriteSuccessNoCollisionUserKey) { new WritableFileWriter(std::move(writable_file), EnvOptions())); CuckooTableBuilder builder(file_writer.get(), kHashTableRatio, num_hash_fun, 100, BytewiseComparator(), 1, false, false, - GetSliceHash); + GetSliceHash, 0 /* column_family_id */, + kDefaultColumnFamilyName); ASSERT_OK(builder.status()); for (uint32_t i = 0; i < user_keys.size(); i++) { builder.Add(Slice(GetInternalKey(user_keys[i], true)), Slice(values[i])); @@ -464,7 +473,8 @@ TEST_F(CuckooBuilderTest, WriteSuccessWithCollisionUserKey) { new WritableFileWriter(std::move(writable_file), EnvOptions())); CuckooTableBuilder builder(file_writer.get(), kHashTableRatio, num_hash_fun, 100, BytewiseComparator(), 1, false, false, - GetSliceHash); + GetSliceHash, 0 /* column_family_id */, + kDefaultColumnFamilyName); ASSERT_OK(builder.status()); for (uint32_t i = 0; i < user_keys.size(); i++) { builder.Add(Slice(GetInternalKey(user_keys[i], true)), Slice(values[i])); @@ -509,7 +519,8 @@ TEST_F(CuckooBuilderTest, WithCollisionPathUserKey) { new WritableFileWriter(std::move(writable_file), EnvOptions())); CuckooTableBuilder builder(file_writer.get(), kHashTableRatio, num_hash_fun, 2, BytewiseComparator(), 1, false, false, - GetSliceHash); + GetSliceHash, 0 /* column_family_id */, + kDefaultColumnFamilyName); ASSERT_OK(builder.status()); for (uint32_t i = 0; i < user_keys.size(); i++) { builder.Add(Slice(GetInternalKey(user_keys[i], true)), Slice(values[i])); @@ -553,7 +564,8 @@ TEST_F(CuckooBuilderTest, FailWhenCollisionPathTooLong) { new WritableFileWriter(std::move(writable_file), EnvOptions())); CuckooTableBuilder builder(file_writer.get(), kHashTableRatio, num_hash_fun, 2, BytewiseComparator(), 1, false, false, - GetSliceHash); + GetSliceHash, 0 /* column_family_id */, + kDefaultColumnFamilyName); ASSERT_OK(builder.status()); for (uint32_t i = 0; i < user_keys.size(); i++) { builder.Add(Slice(GetInternalKey(user_keys[i], false)), Slice("value")); @@ -580,7 +592,8 @@ TEST_F(CuckooBuilderTest, FailWhenSameKeyInserted) { new WritableFileWriter(std::move(writable_file), EnvOptions())); CuckooTableBuilder builder(file_writer.get(), kHashTableRatio, num_hash_fun, 100, BytewiseComparator(), 1, false, false, - GetSliceHash); + GetSliceHash, 0 /* column_family_id */, + kDefaultColumnFamilyName); ASSERT_OK(builder.status()); builder.Add(Slice(GetInternalKey(user_key, false)), Slice("value1")); diff --git a/table/cuckoo_table_factory.cc b/table/cuckoo_table_factory.cc index bf1561a87..36a938571 100644 --- a/table/cuckoo_table_factory.cc +++ b/table/cuckoo_table_factory.cc @@ -38,7 +38,8 @@ TableBuilder* CuckooTableFactory::NewTableBuilder( table_options_.max_search_depth, table_builder_options.internal_comparator.user_comparator(), table_options_.cuckoo_block_size, table_options_.use_module_hash, - table_options_.identity_as_first_hash, nullptr); + table_options_.identity_as_first_hash, nullptr /* get_slice_hash */, + column_family_id, table_builder_options.column_family_name); } std::string CuckooTableFactory::GetPrintableTableOptions() const { diff --git a/table/cuckoo_table_reader_test.cc b/table/cuckoo_table_reader_test.cc index aa4a93d40..e440af90a 100644 --- a/table/cuckoo_table_reader_test.cc +++ b/table/cuckoo_table_reader_test.cc @@ -98,8 +98,9 @@ class CuckooReaderTest : public testing::Test { unique_ptr file_writer( new WritableFileWriter(std::move(writable_file), env_options)); - CuckooTableBuilder builder(file_writer.get(), 0.9, kNumHashFunc, 100, ucomp, - 2, false, false, GetSliceHash); + CuckooTableBuilder builder( + file_writer.get(), 0.9, kNumHashFunc, 100, ucomp, 2, false, false, + GetSliceHash, 0 /* column_family_id */, kDefaultColumnFamilyName); ASSERT_OK(builder.status()); for (uint32_t key_idx = 0; key_idx < num_items; ++key_idx) { builder.Add(Slice(keys[key_idx]), Slice(values[key_idx])); @@ -405,9 +406,10 @@ void WriteFile(const std::vector& keys, ASSERT_OK(env->NewWritableFile(fname, &writable_file, env_options)); unique_ptr file_writer( new WritableFileWriter(std::move(writable_file), env_options)); - CuckooTableBuilder builder(file_writer.get(), hash_ratio, 64, 1000, - test::Uint64Comparator(), 5, false, - FLAGS_identity_as_first_hash, nullptr); + CuckooTableBuilder builder( + file_writer.get(), hash_ratio, 64, 1000, test::Uint64Comparator(), 5, + false, FLAGS_identity_as_first_hash, nullptr, 0 /* column_family_id */, + kDefaultColumnFamilyName); ASSERT_OK(builder.status()); for (uint64_t key_idx = 0; key_idx < num; ++key_idx) { // Value is just a part of key. diff --git a/table/meta_blocks.cc b/table/meta_blocks.cc index b94d0b6ea..cee9704c7 100644 --- a/table/meta_blocks.cc +++ b/table/meta_blocks.cc @@ -69,11 +69,15 @@ void PropertyBlockBuilder::AddTableProperty(const TableProperties& props) { Add(TablePropertiesNames::kFilterSize, props.filter_size); Add(TablePropertiesNames::kFormatVersion, props.format_version); Add(TablePropertiesNames::kFixedKeyLen, props.fixed_key_len); + Add(TablePropertiesNames::kColumnFamilyId, props.column_family_id); if (!props.filter_policy_name.empty()) { Add(TablePropertiesNames::kFilterPolicy, props.filter_policy_name); } + if (!props.column_family_name.empty()) { + Add(TablePropertiesNames::kColumnFamilyName, props.column_family_name); + } } Slice PropertyBlockBuilder::Finish() { @@ -171,7 +175,10 @@ Status ReadProperties(const Slice& handle_value, RandomAccessFileReader* file, {TablePropertiesNames::kFormatVersion, &new_table_properties->format_version}, {TablePropertiesNames::kFixedKeyLen, - &new_table_properties->fixed_key_len}, }; + &new_table_properties->fixed_key_len}, + {TablePropertiesNames::kColumnFamilyId, + &new_table_properties->column_family_id}, + }; std::string last_key; for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { @@ -203,6 +210,8 @@ Status ReadProperties(const Slice& handle_value, RandomAccessFileReader* file, *(pos->second) = val; } else if (key == TablePropertiesNames::kFilterPolicy) { new_table_properties->filter_policy_name = raw_val.ToString(); + } else if (key == TablePropertiesNames::kColumnFamilyName) { + new_table_properties->column_family_name = raw_val.ToString(); } else { // handle user-collected properties new_table_properties->user_collected_properties.insert( diff --git a/table/plain_table_builder.cc b/table/plain_table_builder.cc index efaf47b07..b438ed86e 100644 --- a/table/plain_table_builder.cc +++ b/table/plain_table_builder.cc @@ -62,8 +62,9 @@ PlainTableBuilder::PlainTableBuilder( int_tbl_prop_collector_factories, uint32_t column_family_id, WritableFileWriter* file, uint32_t user_key_len, EncodingType encoding_type, size_t index_sparseness, - uint32_t bloom_bits_per_key, uint32_t num_probes, size_t huge_page_tlb_size, - double hash_table_ratio, bool store_index_in_file) + uint32_t bloom_bits_per_key, const std::string& column_family_name, + uint32_t num_probes, size_t huge_page_tlb_size, double hash_table_ratio, + bool store_index_in_file) : ioptions_(ioptions), bloom_block_(num_probes), file_(file), @@ -94,6 +95,8 @@ PlainTableBuilder::PlainTableBuilder( // To support roll-back to previous version, now still use version 0 for // plain encoding. properties_.format_version = (encoding_type == kPlain) ? 0 : 1; + properties_.column_family_id = column_family_id; + properties_.column_family_name = column_family_name; if (ioptions_.prefix_extractor) { properties_.user_collected_properties diff --git a/table/plain_table_builder.h b/table/plain_table_builder.h index 02fb8d87a..ba63a82f6 100644 --- a/table/plain_table_builder.h +++ b/table/plain_table_builder.h @@ -6,15 +6,16 @@ #pragma once #ifndef ROCKSDB_LITE #include +#include #include #include "rocksdb/options.h" #include "rocksdb/status.h" -#include "table/table_builder.h" -#include "table/plain_table_key_coding.h" #include "rocksdb/table.h" #include "rocksdb/table_properties.h" #include "table/bloom_block.h" #include "table/plain_table_index.h" +#include "table/plain_table_key_coding.h" +#include "table/table_builder.h" namespace rocksdb { @@ -37,8 +38,9 @@ class PlainTableBuilder: public TableBuilder { uint32_t column_family_id, WritableFileWriter* file, uint32_t user_key_size, EncodingType encoding_type, size_t index_sparseness, uint32_t bloom_bits_per_key, - uint32_t num_probes = 6, size_t huge_page_tlb_size = 0, - double hash_table_ratio = 0, bool store_index_in_file = false); + const std::string& column_family_name, uint32_t num_probes = 6, + size_t huge_page_tlb_size = 0, double hash_table_ratio = 0, + bool store_index_in_file = false); // REQUIRES: Either Finish() or Abandon() has been called. ~PlainTableBuilder(); diff --git a/table/plain_table_factory.cc b/table/plain_table_factory.cc index a2f55b659..7db5e672c 100644 --- a/table/plain_table_factory.cc +++ b/table/plain_table_factory.cc @@ -37,7 +37,8 @@ TableBuilder* PlainTableFactory::NewTableBuilder( table_builder_options.ioptions, table_builder_options.int_tbl_prop_collector_factories, column_family_id, file, table_options_.user_key_len, table_options_.encoding_type, - table_options_.index_sparseness, table_options_.bloom_bits_per_key, 6, + table_options_.index_sparseness, table_options_.bloom_bits_per_key, + table_builder_options.column_family_name, 6, table_options_.huge_page_tlb_size, table_options_.hash_table_ratio, table_options_.store_index_in_file); } diff --git a/table/sst_file_writer.cc b/table/sst_file_writer.cc index 58ca0e84a..ddc3d5821 100644 --- a/table/sst_file_writer.cc +++ b/table/sst_file_writer.cc @@ -83,6 +83,7 @@ struct SstFileWriter::Rep { ImmutableCFOptions ioptions; InternalKeyComparator internal_comparator; ExternalSstFileInfo file_info; + std::string column_family_name; }; SstFileWriter::SstFileWriter(const EnvOptions& env_options, @@ -114,7 +115,8 @@ Status SstFileWriter::Open(const std::string& file_path) { TableBuilderOptions table_builder_options( r->ioptions, r->internal_comparator, &int_tbl_prop_collector_factories, - compression_type, r->ioptions.compression_opts, false); + compression_type, r->ioptions.compression_opts, false /* skip_filters */, + r->column_family_name); r->file_writer.reset( new WritableFileWriter(std::move(sst_file), r->env_options)); r->builder.reset(r->ioptions.table_factory->NewTableBuilder( diff --git a/table/table_builder.h b/table/table_builder.h index 274245f08..fb94b38fe 100644 --- a/table/table_builder.h +++ b/table/table_builder.h @@ -52,21 +52,23 @@ struct TableBuilderOptions { const std::vector>* _int_tbl_prop_collector_factories, CompressionType _compression_type, - const CompressionOptions& _compression_opts, bool _skip_filters) + const CompressionOptions& _compression_opts, bool _skip_filters, + const std::string& _column_family_name) : ioptions(_ioptions), internal_comparator(_internal_comparator), int_tbl_prop_collector_factories(_int_tbl_prop_collector_factories), compression_type(_compression_type), compression_opts(_compression_opts), - skip_filters(_skip_filters) {} + skip_filters(_skip_filters), + column_family_name(_column_family_name) {} const ImmutableCFOptions& ioptions; const InternalKeyComparator& internal_comparator; const std::vector>* int_tbl_prop_collector_factories; CompressionType compression_type; const CompressionOptions& compression_opts; - // This is only used for BlockBasedTableBuilder - bool skip_filters = false; + bool skip_filters; // only used by BlockBasedTableBuilder + const std::string& column_family_name; }; // TableBuilder provides the interface used to build a Table diff --git a/table/table_properties.cc b/table/table_properties.cc index 5bf3e0103..3a41529ff 100644 --- a/table/table_properties.cc +++ b/table/table_properties.cc @@ -108,6 +108,10 @@ const std::string TablePropertiesNames::kFormatVersion = "rocksdb.format.version"; const std::string TablePropertiesNames::kFixedKeyLen = "rocksdb.fixed.key.length"; +const std::string TablePropertiesNames::kColumnFamilyId = + "rocksdb.column.family.id"; +const std::string TablePropertiesNames::kColumnFamilyName = + "rocksdb.column.family.name"; extern const std::string kPropertiesBlock = "rocksdb.properties"; // Old property block name for backward compatibility diff --git a/table/table_reader_bench.cc b/table/table_reader_bench.cc index 1a7d7b5a7..d0d2959f2 100644 --- a/table/table_reader_bench.cc +++ b/table/table_reader_bench.cc @@ -98,8 +98,9 @@ void TableReaderBenchmark(Options& opts, EnvOptions& env_options, tb = opts.table_factory->NewTableBuilder( TableBuilderOptions(ioptions, ikc, &int_tbl_prop_collector_factories, CompressionType::kNoCompression, - CompressionOptions(), false), - 0, file_writer.get()); + CompressionOptions(), false /* skip_filters */, + kDefaultColumnFamilyName), + 0 /* column_family_id */, file_writer.get()); } else { s = DB::Open(opts, dbname, &db); ASSERT_OK(s); diff --git a/table/table_test.cc b/table/table_test.cc index 1ce61cadf..9507d9e77 100644 --- a/table/table_test.cc +++ b/table/table_test.cc @@ -270,10 +270,12 @@ class TableConstructor: public Constructor { unique_ptr builder; std::vector> int_tbl_prop_collector_factories; + std::string column_family_name; builder.reset(ioptions.table_factory->NewTableBuilder( - TableBuilderOptions(ioptions, internal_comparator, - &int_tbl_prop_collector_factories, - options.compression, CompressionOptions(), false), + TableBuilderOptions( + ioptions, internal_comparator, &int_tbl_prop_collector_factories, + options.compression, CompressionOptions(), false /* skip_filters */, + column_family_name), TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, file_writer_.get())); @@ -1941,9 +1943,11 @@ TEST_F(PlainTableTest, BasicPlainTableProperties) { InternalKeyComparator ikc(options.comparator); std::vector> int_tbl_prop_collector_factories; + std::string column_family_name; std::unique_ptr builder(factory.NewTableBuilder( TableBuilderOptions(ioptions, ikc, &int_tbl_prop_collector_factories, - kNoCompression, CompressionOptions(), false), + kNoCompression, CompressionOptions(), + false /* skip_filters */, column_family_name), TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, file_writer.get())); diff --git a/tools/sst_dump_test.cc b/tools/sst_dump_test.cc index 84edaea37..85b7bf04a 100644 --- a/tools/sst_dump_test.cc +++ b/tools/sst_dump_test.cc @@ -58,10 +58,11 @@ void createSST(const std::string& file_name, int_tbl_prop_collector_factories; unique_ptr file_writer( new WritableFileWriter(std::move(file), EnvOptions())); + std::string column_family_name; tb.reset(opts.table_factory->NewTableBuilder( TableBuilderOptions(imoptions, ikc, &int_tbl_prop_collector_factories, CompressionType::kNoCompression, CompressionOptions(), - false), + false /* skip_filters */, column_family_name), TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, file_writer.get())); diff --git a/tools/sst_dump_tool.cc b/tools/sst_dump_tool.cc index 5724b42c1..3e6b16e13 100644 --- a/tools/sst_dump_tool.cc +++ b/tools/sst_dump_tool.cc @@ -203,8 +203,10 @@ int SstFileReader::ShowAllCompressionSizes(size_t block_size) { i = (i == kLZ4HCCompression) ? kZSTDNotFinalCompression : CompressionType(i + 1)) { CompressionOptions compress_opt; + std::string column_family_name; TableBuilderOptions tb_opts(imoptions, ikc, &block_based_table_factories, i, - compress_opt, false); + compress_opt, false /* skip_filters */, + column_family_name); uint64_t file_size = CalculateCompressedTableSize(tb_opts, block_size); fprintf(stdout, "Compression: %s", compress_type.find(i)->second); fprintf(stdout, " Size: %" PRIu64 "\n", file_size);