From 94d04529def513600c05998af6d6904f40f94124 Mon Sep 17 00:00:00 2001 From: Zitan Chen <11285749+gg814@users.noreply.github.com> Date: Wed, 17 Jun 2020 10:55:42 -0700 Subject: [PATCH] Store DB identity and DB session ID in SST files (#6983) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: `db_id` and `db_session_id` are now part of the table properties for all formats and stored in SST files. This adds about 99 bytes to each new SST file. The `TablePropertiesNames` for these two identifiers are `rocksdb.creating.db.identity` and `rocksdb.creating.session.identity`. In addition, SST files generated from SstFileWriter and Repairer have DB identity “SST Writer” and “DB Repairer”, respectively. Their DB session IDs are generated in the same way as `DB::GetDbSessionId`. A table property test is added. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6983 Test Plan: make check and some manual tests. Reviewed By: zhichao-cao Differential Revision: D22048826 Pulled By: gg814 fbshipit-source-id: afdf8c11424a6f509b5c0b06dafad584a80103c9 --- HISTORY.md | 7 ++++- db/builder.cc | 10 ++++--- db/builder.h | 6 ++-- db/compaction/compaction_job.cc | 8 ++++-- db/compaction/compaction_job.h | 6 +++- db/db_impl/db_impl_compaction_flush.cc | 11 +++++--- db/db_impl/db_impl_open.cc | 3 +- db/db_table_properties_test.cc | 20 +++++++++++++ db/event_helpers.cc | 4 ++- db/flush_job.cc | 8 ++++-- db/flush_job.h | 5 +++- db/flush_job_test.cc | 1 - db/repair.cc | 7 ++++- include/rocksdb/table_properties.h | 13 +++++++++ .../block_based/block_based_table_builder.cc | 28 +++++++++++++------ table/block_based/block_based_table_builder.h | 3 +- .../block_based/block_based_table_factory.cc | 3 +- table/cuckoo/cuckoo_table_builder.cc | 5 +++- table/cuckoo/cuckoo_table_builder.h | 17 ++++++----- table/cuckoo/cuckoo_table_factory.cc | 3 +- table/meta_blocks.cc | 6 ++++ table/plain/plain_table_builder.cc | 5 +++- table/plain/plain_table_builder.h | 3 +- table/plain/plain_table_factory.cc | 3 +- table/sst_file_writer.cc | 14 ++++++++-- table/table_builder.h | 10 +++++-- table/table_properties.cc | 8 ++++++ 27 files changed, 167 insertions(+), 50 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index f3794bb07..61d2fd323 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -4,6 +4,12 @@ * Best-efforts recovery ignores CURRENT file completely. If CURRENT file is missing during recovery, best-efforts recovery still proceeds with MANIFEST file(s). * In best-efforts recovery, an error that is not Corruption or IOError::kNotFound or IOError::kPathNotFound will be overwritten silently. Fix this by checking all non-ok cases and return early. +### Public API Change +* `DB::GetDbSessionId(std::string& session_id)` is added. `session_id` stores a unique identifier that gets reset every time the DB is opened. This DB session ID should be unique among all open DB instances on all hosts, and should be unique among re-openings of the same or other DBs. This identifier is recorded in the LOG file on the line starting with "DB Session ID:". + +### New Features +* DB identity (`db_id`) and DB session identity (`db_session_id`) are added to table properties and stored in SST files. SST files generated from SstFileWriter and Repairer have DB identity “SST Writer” and “DB Repairer”, respectively. Their DB session IDs are generated in the same way as `DB::GetDbSessionId`. The session ID for SstFileWriter (resp., Repairer) resets every time `SstFileWriter::Open` (resp., `Repairer::Run`) is called. + ## 6.11 (6/12/2020) ### Bug Fixes * Fix consistency checking error swallowing in some cases when options.force_consistency_checks = true. @@ -30,7 +36,6 @@ * `pin_l0_filter_and_index_blocks_in_cache` no longer applies to L0 files larger than `1.5 * write_buffer_size` to give more predictable memory usage. Such L0 files may exist due to intra-L0 compaction, external file ingestion, or user dynamically changing `write_buffer_size` (note, however, that files that are already pinned will continue being pinned, even after such a dynamic change). * In point-in-time wal recovery mode, fail database recovery in case of IOError while reading the WAL to avoid data loss. * A new method `Env::LowerThreadPoolCPUPriority(Priority, CpuPriority)` is added to `Env` to be able to lower to a specific priority such as `CpuPriority::kIdle`. -* `DB::GetDbSessionId(std::string& session_id)` is added. `session_id` stores a unique identifier that gets reset every time the DB is opened. This DB session ID should be unique among all open DB instances on all hosts, and should be unique among re-openings of the same or other DBs. This identifier is recorded in the `LOG` file on the line starting with `DB Session ID:`. ### New Features * sst_dump to add a new --readahead_size argument. Users can specify read size when scanning the data. Sst_dump also tries to prefetch tail part of the SST files so usually some number of I/Os are saved there too. diff --git a/db/builder.cc b/db/builder.cc index 309a10ff7..6c89562e0 100644 --- a/db/builder.cc +++ b/db/builder.cc @@ -51,7 +51,8 @@ TableBuilder* NewTableBuilder( uint64_t sample_for_compression, const CompressionOptions& compression_opts, int level, const bool skip_filters, const uint64_t creation_time, const uint64_t oldest_key_time, const uint64_t target_file_size, - const uint64_t file_creation_time) { + const uint64_t file_creation_time, const std::string& db_id, + const std::string& db_session_id) { assert((column_family_id == TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) == column_family_name.empty()); @@ -61,7 +62,7 @@ TableBuilder* NewTableBuilder( sample_for_compression, compression_opts, skip_filters, column_family_name, level, creation_time, oldest_key_time, target_file_size, - file_creation_time), + file_creation_time, db_id, db_session_id), column_family_id, file); } @@ -85,7 +86,8 @@ Status BuildTable( EventLogger* event_logger, int job_id, const Env::IOPriority io_priority, TableProperties* table_properties, int level, const uint64_t creation_time, const uint64_t oldest_key_time, Env::WriteLifeTimeHint write_hint, - const uint64_t file_creation_time) { + const uint64_t file_creation_time, const std::string& db_id, + const std::string& db_session_id) { assert((column_family_id == TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) == column_family_name.empty()); @@ -142,7 +144,7 @@ Status BuildTable( column_family_name, file_writer.get(), compression, sample_for_compression, compression_opts_for_flush, level, false /* skip_filters */, creation_time, oldest_key_time, - 0 /*target_file_size*/, file_creation_time); + 0 /*target_file_size*/, file_creation_time, db_id, db_session_id); } MergeHelper merge(env, internal_comparator.user_comparator(), diff --git a/db/builder.h b/db/builder.h index 512310a3c..4b960c0b1 100644 --- a/db/builder.h +++ b/db/builder.h @@ -51,7 +51,8 @@ TableBuilder* NewTableBuilder( const CompressionOptions& compression_opts, int level, const bool skip_filters = false, const uint64_t creation_time = 0, const uint64_t oldest_key_time = 0, const uint64_t target_file_size = 0, - const uint64_t file_creation_time = 0); + const uint64_t file_creation_time = 0, const std::string& db_id = "", + const std::string& db_session_id = ""); // Build a Table file from the contents of *iter. The generated file // will be named according to number specified in meta. On success, the rest of @@ -83,6 +84,7 @@ extern Status BuildTable( TableProperties* table_properties = nullptr, int level = -1, const uint64_t creation_time = 0, const uint64_t oldest_key_time = 0, Env::WriteLifeTimeHint write_hint = Env::WLTH_NOT_SET, - const uint64_t file_creation_time = 0); + const uint64_t file_creation_time = 0, const std::string& db_id = "", + const std::string& db_session_id = ""); } // namespace ROCKSDB_NAMESPACE diff --git a/db/compaction/compaction_job.cc b/db/compaction/compaction_job.cc index aabff4fd6..77fb8db65 100644 --- a/db/compaction/compaction_job.cc +++ b/db/compaction/compaction_job.cc @@ -305,12 +305,15 @@ CompactionJob::CompactionJob( const SnapshotChecker* snapshot_checker, std::shared_ptr table_cache, EventLogger* event_logger, bool paranoid_file_checks, bool measure_io_stats, const std::string& dbname, CompactionJobStats* compaction_job_stats, - Env::Priority thread_pri, const std::atomic* manual_compaction_paused) + Env::Priority thread_pri, const std::atomic* manual_compaction_paused, + const std::string& db_id, const std::string& db_session_id) : job_id_(job_id), compact_(new CompactionState(compaction)), compaction_job_stats_(compaction_job_stats), compaction_stats_(compaction->compaction_reason(), 1), dbname_(dbname), + db_id_(db_id), + db_session_id_(db_session_id), db_options_(db_options), file_options_(file_options), env_(db_options.env), @@ -1554,7 +1557,8 @@ Status CompactionJob::OpenCompactionOutputFile( sub_compact->compaction->output_compression_opts(), sub_compact->compaction->output_level(), skip_filters, oldest_ancester_time, 0 /* oldest_key_time */, - sub_compact->compaction->max_output_file_size(), current_time)); + sub_compact->compaction->max_output_file_size(), current_time, db_id_, + db_session_id_)); LogFlush(db_options_.info_log); return s; } diff --git a/db/compaction/compaction_job.h b/db/compaction/compaction_job.h index 0c7b09a7d..7d7c58ca2 100644 --- a/db/compaction/compaction_job.h +++ b/db/compaction/compaction_job.h @@ -78,7 +78,9 @@ class CompactionJob { const std::string& dbname, CompactionJobStats* compaction_job_stats, Env::Priority thread_pri, - const std::atomic* manual_compaction_paused = nullptr); + const std::atomic* manual_compaction_paused = nullptr, + const std::string& db_id = "", + const std::string& db_session_id = ""); ~CompactionJob(); @@ -152,6 +154,8 @@ class CompactionJob { // DBImpl state const std::string& dbname_; + const std::string db_id_; + const std::string db_session_id_; const ImmutableDBOptions& db_options_; const FileOptions file_options_; diff --git a/db/db_impl/db_impl_compaction_flush.cc b/db/db_impl/db_impl_compaction_flush.cc index 4f1ca1fe3..a58ba97d5 100644 --- a/db/db_impl/db_impl_compaction_flush.cc +++ b/db/db_impl/db_impl_compaction_flush.cc @@ -154,7 +154,8 @@ Status DBImpl::FlushMemTableToOutputFile( GetDataDir(cfd, 0U), GetCompressionFlush(*cfd->ioptions(), mutable_cf_options), stats_, &event_logger_, mutable_cf_options.report_bg_io_stats, - true /* sync_output_directory */, true /* write_manifest */, thread_pri); + true /* sync_output_directory */, true /* write_manifest */, thread_pri, + db_id_, db_session_id_); FileMetaData file_meta; TEST_SYNC_POINT("DBImpl::FlushMemTableToOutputFile:BeforePickMemtables"); @@ -345,7 +346,7 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles( data_dir, GetCompressionFlush(*cfd->ioptions(), mutable_cf_options), stats_, &event_logger_, mutable_cf_options.report_bg_io_stats, false /* sync_output_directory */, false /* write_manifest */, - thread_pri)); + thread_pri, db_id_, db_session_id_)); jobs.back()->PickMemTable(); } @@ -1031,7 +1032,8 @@ Status DBImpl::CompactFilesImpl( snapshot_checker, table_cache_, &event_logger_, c->mutable_cf_options()->paranoid_file_checks, c->mutable_cf_options()->report_bg_io_stats, dbname_, - &compaction_job_stats, Env::Priority::USER, &manual_compaction_paused_); + &compaction_job_stats, Env::Priority::USER, &manual_compaction_paused_, + db_id_, db_session_id_); // Creating a compaction influences the compaction score because the score // takes running compactions into account (by skipping files that are already @@ -2822,7 +2824,8 @@ Status DBImpl::BackgroundCompaction(bool* made_progress, &event_logger_, c->mutable_cf_options()->paranoid_file_checks, c->mutable_cf_options()->report_bg_io_stats, dbname_, &compaction_job_stats, thread_pri, - is_manual ? &manual_compaction_paused_ : nullptr); + is_manual ? &manual_compaction_paused_ : nullptr, db_id_, + db_session_id_); compaction_job.Prepare(); NotifyOnCompactionBegin(c->column_family_data(), c.get(), status, diff --git a/db/db_impl/db_impl_open.cc b/db/db_impl/db_impl_open.cc index dc83a0478..e6f89cdbb 100644 --- a/db/db_impl/db_impl_open.cc +++ b/db/db_impl/db_impl_open.cc @@ -1289,7 +1289,8 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd, mutable_cf_options.compression_opts, paranoid_file_checks, cfd->internal_stats(), TableFileCreationReason::kRecovery, &io_s, &event_logger_, job_id, Env::IO_HIGH, nullptr /* table_properties */, - -1 /* level */, current_time, write_hint); + -1 /* level */, current_time, 0 /* oldest_key_time */, write_hint, + 0 /* file_creation_time */, db_id_, db_session_id_); LogFlush(immutable_db_options_.info_log); ROCKS_LOG_DEBUG(immutable_db_options_.info_log, "[%s] [WriteLevel0TableForRecovery]" diff --git a/db/db_table_properties_test.cc b/db/db_table_properties_test.cc index 96bc37da0..98512e22f 100644 --- a/db/db_table_properties_test.cc +++ b/db/db_table_properties_test.cc @@ -252,6 +252,26 @@ TEST_F(DBTablePropertiesTest, GetColumnFamilyNameProperty) { } } +TEST_F(DBTablePropertiesTest, GetDbIdentifiersProperty) { + CreateAndReopenWithCF({"goku"}, CurrentOptions()); + + for (uint32_t cf = 0; cf < 2; ++cf) { + Put(cf, "key", "val"); + Put(cf, "foo", "bar"); + Flush(cf); + + TablePropertiesCollection fname_to_props; + ASSERT_OK(db_->GetPropertiesOfAllTables(handles_[cf], &fname_to_props)); + ASSERT_EQ(1U, fname_to_props.size()); + + std::string id, sid; + db_->GetDbIdentity(id); + db_->GetDbSessionId(sid); + ASSERT_EQ(id, fname_to_props.begin()->second->db_id); + ASSERT_EQ(sid, fname_to_props.begin()->second->db_session_id); + } +} + class DeletionTriggeredCompactionTestListener : public EventListener { public: void OnCompactionBegin(DB* , const CompactionJobInfo& ci) override { diff --git a/db/event_helpers.cc b/db/event_helpers.cc index 57aa711fc..b7f491409 100644 --- a/db/event_helpers.cc +++ b/db/event_helpers.cc @@ -121,7 +121,9 @@ void EventHelpers::LogAndNotifyTableFileCreationFinished( << table_properties.compression_options << "creation_time" << table_properties.creation_time << "oldest_key_time" << table_properties.oldest_key_time << "file_creation_time" - << table_properties.file_creation_time; + << table_properties.file_creation_time << "db_id" + << table_properties.db_id << "db_session_id" + << table_properties.db_session_id; // user collected properties for (const auto& prop : table_properties.readable_properties) { diff --git a/db/flush_job.cc b/db/flush_job.cc index a1e6720be..adfa52e3e 100644 --- a/db/flush_job.cc +++ b/db/flush_job.cc @@ -97,8 +97,11 @@ FlushJob::FlushJob(const std::string& dbname, ColumnFamilyData* cfd, CompressionType output_compression, Statistics* stats, EventLogger* event_logger, bool measure_io_stats, const bool sync_output_directory, const bool write_manifest, - Env::Priority thread_pri) + Env::Priority thread_pri, const std::string& db_id, + const std::string& db_session_id) : dbname_(dbname), + db_id_(db_id), + db_session_id_(db_session_id), cfd_(cfd), db_options_(db_options), mutable_cf_options_(mutable_cf_options), @@ -393,7 +396,8 @@ Status FlushJob::WriteLevel0Table() { mutable_cf_options_.paranoid_file_checks, cfd_->internal_stats(), TableFileCreationReason::kFlush, &io_s, event_logger_, job_context_->job_id, Env::IO_HIGH, &table_properties_, 0 /* level */, - creation_time, oldest_key_time, write_hint, current_time); + creation_time, oldest_key_time, write_hint, current_time, db_id_, + db_session_id_); if (!io_s.ok()) { io_status_ = io_s; } diff --git a/db/flush_job.h b/db/flush_job.h index ec3a0d981..980d92eb4 100644 --- a/db/flush_job.h +++ b/db/flush_job.h @@ -72,7 +72,8 @@ class FlushJob { CompressionType output_compression, Statistics* stats, EventLogger* event_logger, bool measure_io_stats, const bool sync_output_directory, const bool write_manifest, - Env::Priority thread_pri); + Env::Priority thread_pri, const std::string& db_id = "", + const std::string& db_session_id = ""); ~FlushJob(); @@ -103,6 +104,8 @@ class FlushJob { #endif // !ROCKSDB_LITE const std::string& dbname_; + const std::string db_id_; + const std::string db_session_id_; ColumnFamilyData* cfd_; const ImmutableDBOptions& db_options_; const MutableCFOptions& mutable_cf_options_; diff --git a/db/flush_job_test.cc b/db/flush_job_test.cc index 823a566b9..72cceb522 100644 --- a/db/flush_job_test.cc +++ b/db/flush_job_test.cc @@ -282,7 +282,6 @@ TEST_F(FlushJobTest, FlushMemTablesSingleColumnFamily) { assert(memtable_ids.size() == num_mems); uint64_t smallest_memtable_id = memtable_ids.front(); uint64_t flush_memtable_id = smallest_memtable_id + num_mems_to_flush - 1; - FlushJob flush_job(dbname_, versions_->GetColumnFamilySet()->GetDefault(), db_options_, *cfd->GetLatestMutableCFOptions(), &flush_memtable_id, env_options_, versions_.get(), &mutex_, diff --git a/db/repair.cc b/db/repair.cc index f82db9fbb..da1b3641d 100644 --- a/db/repair.cc +++ b/db/repair.cc @@ -183,6 +183,8 @@ class Repairer { } // Just create a DBImpl temporarily so we can reuse NewDB() DBImpl* db_impl = new DBImpl(db_options_, dbname_); + // Also use this temp DBImpl to get a session id + db_impl->GetDbSessionId(db_session_id_); status = db_impl->NewDB(); delete db_impl; } @@ -229,6 +231,7 @@ class Repairer { }; std::string const dbname_; + std::string db_session_id_; Env* const env_; const EnvOptions env_options_; const DBOptions db_options_; @@ -435,7 +438,9 @@ class Repairer { CompressionOptions(), false, nullptr /* internal_stats */, TableFileCreationReason::kRecovery, &io_s, nullptr /* event_logger */, 0 /* job_id */, Env::IO_HIGH, nullptr /* table_properties */, - -1 /* level */, current_time, write_hint); + -1 /* level */, current_time, 0 /* oldest_key_time */, write_hint, + 0 /* file_creation_time */, "DB Repairer" /* db_id */, + db_session_id_); ROCKS_LOG_INFO(db_options_.info_log, "Log #%" PRIu64 ": %d ops saved to Table #%" PRIu64 " %s", log, counter, meta.fd.GetNumber(), diff --git a/include/rocksdb/table_properties.h b/include/rocksdb/table_properties.h index 213896f35..ba3eca752 100644 --- a/include/rocksdb/table_properties.h +++ b/include/rocksdb/table_properties.h @@ -30,6 +30,8 @@ typedef std::map UserCollectedProperties; // table properties' human-readable names in the property block. struct TablePropertiesNames { + static const std::string kDbId; + static const std::string kDbSessionId; static const std::string kDataSize; static const std::string kIndexSize; static const std::string kIndexPartitions; @@ -193,6 +195,17 @@ struct TableProperties { // Actual SST file creation time. 0 means unknown. uint64_t file_creation_time = 0; + // DB identity + // db_id is an identifier generated the first time the DB is created + // If DB identity is unset or unassigned, `db_id` will be an empty string. + std::string db_id; + + // DB session identity + // db_session_id is an identifier that gets reset every time the DB is opened + // If DB session identity is unset or unassigned, `db_session_id` will be an + // empty string. + std::string db_session_id; + // Name of the column family with which this SST file is associated. // If column family is unknown, `column_family_name` will be an empty string. std::string column_family_name; diff --git a/table/block_based/block_based_table_builder.cc b/table/block_based/block_based_table_builder.cc index 70078a8c9..377077730 100644 --- a/table/block_based/block_based_table_builder.cc +++ b/table/block_based/block_based_table_builder.cc @@ -374,6 +374,10 @@ struct BlockBasedTableBuilder::Rep { const uint64_t target_file_size; uint64_t file_creation_time = 0; + // DB IDs + const std::string db_id; + const std::string db_session_id; + std::vector> table_properties_collectors; std::unique_ptr pc_rep; @@ -447,7 +451,8 @@ struct BlockBasedTableBuilder::Rep { const CompressionOptions& _compression_opts, const bool skip_filters, const int _level_at_creation, const std::string& _column_family_name, const uint64_t _creation_time, const uint64_t _oldest_key_time, - const uint64_t _target_file_size, const uint64_t _file_creation_time) + const uint64_t _target_file_size, const uint64_t _file_creation_time, + const std::string& _db_id, const std::string& _db_session_id) : ioptions(_ioptions), moptions(_moptions), table_options(table_opt), @@ -488,7 +493,9 @@ struct BlockBasedTableBuilder::Rep { creation_time(_creation_time), oldest_key_time(_oldest_key_time), target_file_size(_target_file_size), - file_creation_time(_file_creation_time) { + file_creation_time(_file_creation_time), + db_id(_db_id), + db_session_id(_db_session_id) { for (uint32_t i = 0; i < compression_opts.parallel_threads; i++) { compression_ctxs[i].reset(new CompressionContext(compression_type)); } @@ -698,7 +705,8 @@ BlockBasedTableBuilder::BlockBasedTableBuilder( const CompressionOptions& compression_opts, const bool skip_filters, const std::string& column_family_name, const int level_at_creation, const uint64_t creation_time, const uint64_t oldest_key_time, - const uint64_t target_file_size, const uint64_t file_creation_time) { + const uint64_t target_file_size, const uint64_t file_creation_time, + const std::string& db_id, const std::string& db_session_id) { BlockBasedTableOptions sanitized_table_options(table_options); if (sanitized_table_options.format_version == 0 && sanitized_table_options.checksum != kCRC32c) { @@ -711,12 +719,12 @@ BlockBasedTableBuilder::BlockBasedTableBuilder( sanitized_table_options.format_version = 1; } - rep_ = new Rep(ioptions, moptions, sanitized_table_options, - internal_comparator, int_tbl_prop_collector_factories, - column_family_id, file, compression_type, - sample_for_compression, compression_opts, skip_filters, - level_at_creation, column_family_name, creation_time, - oldest_key_time, target_file_size, file_creation_time); + rep_ = new Rep( + ioptions, moptions, sanitized_table_options, internal_comparator, + int_tbl_prop_collector_factories, column_family_id, file, + compression_type, sample_for_compression, compression_opts, skip_filters, + level_at_creation, column_family_name, creation_time, oldest_key_time, + target_file_size, file_creation_time, db_id, db_session_id); if (rep_->filter_builder != nullptr) { rep_->filter_builder->StartBlock(0); @@ -1445,6 +1453,8 @@ void BlockBasedTableBuilder::WritePropertiesBlock( rep_->props.creation_time = rep_->creation_time; rep_->props.oldest_key_time = rep_->oldest_key_time; rep_->props.file_creation_time = rep_->file_creation_time; + rep_->props.db_id = rep_->db_id; + rep_->props.db_session_id = rep_->db_session_id; // Add basic properties property_block_builder.AddTableProperty(rep_->props); diff --git a/table/block_based/block_based_table_builder.h b/table/block_based/block_based_table_builder.h index ae1ddcb2a..2e3081d26 100644 --- a/table/block_based/block_based_table_builder.h +++ b/table/block_based/block_based_table_builder.h @@ -51,7 +51,8 @@ class BlockBasedTableBuilder : public TableBuilder { const std::string& column_family_name, const int level_at_creation, const uint64_t creation_time = 0, const uint64_t oldest_key_time = 0, const uint64_t target_file_size = 0, - const uint64_t file_creation_time = 0); + const uint64_t file_creation_time = 0, const std::string& db_id = "", + const std::string& db_session_id = ""); // No copying allowed BlockBasedTableBuilder(const BlockBasedTableBuilder&) = delete; diff --git a/table/block_based/block_based_table_factory.cc b/table/block_based/block_based_table_factory.cc index 3bfc0cb2e..ad1d20ca9 100644 --- a/table/block_based/block_based_table_factory.cc +++ b/table/block_based/block_based_table_factory.cc @@ -439,7 +439,8 @@ TableBuilder* BlockBasedTableFactory::NewTableBuilder( table_builder_options.creation_time, table_builder_options.oldest_key_time, table_builder_options.target_file_size, - table_builder_options.file_creation_time); + table_builder_options.file_creation_time, table_builder_options.db_id, + table_builder_options.db_session_id); return table_builder; } diff --git a/table/cuckoo/cuckoo_table_builder.cc b/table/cuckoo/cuckoo_table_builder.cc index 7e5d7a7f3..1e63bf85d 100644 --- a/table/cuckoo/cuckoo_table_builder.cc +++ b/table/cuckoo/cuckoo_table_builder.cc @@ -53,7 +53,8 @@ CuckooTableBuilder::CuckooTableBuilder( const Comparator* user_comparator, uint32_t cuckoo_block_size, bool use_module_hash, bool identity_as_first_hash, uint64_t (*get_slice_hash)(const Slice&, uint32_t, uint64_t), - uint32_t column_family_id, const std::string& column_family_name) + uint32_t column_family_id, const std::string& column_family_name, + const std::string& db_id, const std::string& db_session_id) : num_hash_func_(2), file_(file), max_hash_table_ratio_(max_hash_table_ratio), @@ -79,6 +80,8 @@ CuckooTableBuilder::CuckooTableBuilder( properties_.filter_size = 0; properties_.column_family_id = column_family_id; properties_.column_family_name = column_family_name; + properties_.db_id = db_id; + properties_.db_session_id = db_session_id; } void CuckooTableBuilder::Add(const Slice& key, const Slice& value) { diff --git a/table/cuckoo/cuckoo_table_builder.h b/table/cuckoo/cuckoo_table_builder.h index bf2f78ee3..8e8026487 100644 --- a/table/cuckoo/cuckoo_table_builder.h +++ b/table/cuckoo/cuckoo_table_builder.h @@ -22,15 +22,14 @@ namespace ROCKSDB_NAMESPACE { class CuckooTableBuilder: public TableBuilder { public: - CuckooTableBuilder(WritableFileWriter* file, double max_hash_table_ratio, - uint32_t max_num_hash_func, uint32_t max_search_depth, - const Comparator* user_comparator, - uint32_t cuckoo_block_size, bool use_module_hash, - bool identity_as_first_hash, - uint64_t (*get_slice_hash)(const Slice&, uint32_t, - uint64_t), - uint32_t column_family_id, - const std::string& column_family_name); + CuckooTableBuilder( + WritableFileWriter* file, double max_hash_table_ratio, + uint32_t max_num_hash_func, uint32_t max_search_depth, + const Comparator* user_comparator, uint32_t cuckoo_block_size, + bool use_module_hash, bool identity_as_first_hash, + uint64_t (*get_slice_hash)(const Slice&, uint32_t, uint64_t), + uint32_t column_family_id, const std::string& column_family_name, + const std::string& db_id = "", const std::string& db_session_id = ""); // No copying allowed CuckooTableBuilder(const CuckooTableBuilder&) = delete; void operator=(const CuckooTableBuilder&) = delete; diff --git a/table/cuckoo/cuckoo_table_factory.cc b/table/cuckoo/cuckoo_table_factory.cc index 5ba48f099..46adf8a4f 100644 --- a/table/cuckoo/cuckoo_table_factory.cc +++ b/table/cuckoo/cuckoo_table_factory.cc @@ -40,7 +40,8 @@ TableBuilder* CuckooTableFactory::NewTableBuilder( table_builder_options.internal_comparator.user_comparator(), table_options_.cuckoo_block_size, table_options_.use_module_hash, table_options_.identity_as_first_hash, nullptr /* get_slice_hash */, - column_family_id, table_builder_options.column_family_name); + column_family_id, table_builder_options.column_family_name, + table_builder_options.db_id, table_builder_options.db_session_id); } std::string CuckooTableFactory::GetPrintableTableOptions() const { diff --git a/table/meta_blocks.cc b/table/meta_blocks.cc index 3522d6f94..7ac21f5dc 100644 --- a/table/meta_blocks.cc +++ b/table/meta_blocks.cc @@ -96,6 +96,8 @@ void PropertyBlockBuilder::AddTableProperty(const TableProperties& props) { if (props.file_creation_time > 0) { Add(TablePropertiesNames::kFileCreationTime, props.file_creation_time); } + Add(TablePropertiesNames::kDbId, props.db_id); + Add(TablePropertiesNames::kDbSessionId, props.db_session_id); if (!props.filter_policy_name.empty()) { Add(TablePropertiesNames::kFilterPolicy, props.filter_policy_name); @@ -311,6 +313,10 @@ Status ReadProperties(const Slice& handle_value, RandomAccessFileReader* file, continue; } *(pos->second) = val; + } else if (key == TablePropertiesNames::kDbId) { + new_table_properties->db_id = raw_val.ToString(); + } else if (key == TablePropertiesNames::kDbSessionId) { + new_table_properties->db_session_id = raw_val.ToString(); } else if (key == TablePropertiesNames::kFilterPolicy) { new_table_properties->filter_policy_name = raw_val.ToString(); } else if (key == TablePropertiesNames::kColumnFamilyName) { diff --git a/table/plain/plain_table_builder.cc b/table/plain/plain_table_builder.cc index 56cc64547..0eed5b86b 100644 --- a/table/plain/plain_table_builder.cc +++ b/table/plain/plain_table_builder.cc @@ -64,7 +64,8 @@ PlainTableBuilder::PlainTableBuilder( EncodingType encoding_type, size_t index_sparseness, uint32_t bloom_bits_per_key, const std::string& column_family_name, uint32_t num_probes, size_t huge_page_tlb_size, double hash_table_ratio, - bool store_index_in_file) + bool store_index_in_file, const std::string& db_id, + const std::string& db_session_id) : ioptions_(ioptions), moptions_(moptions), bloom_block_(num_probes), @@ -97,6 +98,8 @@ PlainTableBuilder::PlainTableBuilder( properties_.format_version = (encoding_type == kPlain) ? 0 : 1; properties_.column_family_id = column_family_id; properties_.column_family_name = column_family_name; + properties_.db_id = db_id; + properties_.db_session_id = db_session_id; properties_.prefix_extractor_name = moptions_.prefix_extractor != nullptr ? moptions_.prefix_extractor->Name() : "nullptr"; diff --git a/table/plain/plain_table_builder.h b/table/plain/plain_table_builder.h index cf2c7582b..6ab5d59e3 100644 --- a/table/plain/plain_table_builder.h +++ b/table/plain/plain_table_builder.h @@ -45,7 +45,8 @@ class PlainTableBuilder: public TableBuilder { size_t index_sparseness, uint32_t bloom_bits_per_key, const std::string& column_family_name, uint32_t num_probes = 6, size_t huge_page_tlb_size = 0, double hash_table_ratio = 0, - bool store_index_in_file = false); + bool store_index_in_file = false, const std::string& db_id = "", + const std::string& db_session_id = ""); // No copying allowed PlainTableBuilder(const PlainTableBuilder&) = delete; void operator=(const PlainTableBuilder&) = delete; diff --git a/table/plain/plain_table_factory.cc b/table/plain/plain_table_factory.cc index 98133f4bc..e217640ff 100644 --- a/table/plain/plain_table_factory.cc +++ b/table/plain/plain_table_factory.cc @@ -76,7 +76,8 @@ TableBuilder* PlainTableFactory::NewTableBuilder( table_options_.index_sparseness, table_options_.bloom_bits_per_key, table_builder_options.column_family_name, 6, table_options_.huge_page_tlb_size, table_options_.hash_table_ratio, - table_options_.store_index_in_file); + table_options_.store_index_in_file, table_builder_options.db_id, + table_builder_options.db_session_id); } std::string PlainTableFactory::GetPrintableTableOptions() const { diff --git a/table/sst_file_writer.cc b/table/sst_file_writer.cc index 2c54c46cf..2d0338d23 100644 --- a/table/sst_file_writer.cc +++ b/table/sst_file_writer.cc @@ -237,12 +237,22 @@ Status SstFileWriter::Open(const std::string& file_path) { r->column_family_name = ""; cf_id = TablePropertiesCollectorFactory::Context::kUnknownColumnFamily; } - + // SstFileWriter is used to create sst files that can be added to database + // later. Therefore, no real db_id and db_session_id are associated with it. + // Here we mimic the way db_session_id behaves by resetting the db_session_id + // every time SstFileWriter is used, and in this case db_id is set to be "SST + // Writer". + std::string db_session_id = r->ioptions.env->GenerateUniqueId(); + if (!db_session_id.empty() && db_session_id.back() == '\n') { + db_session_id.pop_back(); + } TableBuilderOptions table_builder_options( r->ioptions, r->mutable_cf_options, r->internal_comparator, &int_tbl_prop_collector_factories, compression_type, sample_for_compression, compression_opts, r->skip_filters, - r->column_family_name, unknown_level); + r->column_family_name, unknown_level, 0 /* creation_time */, + 0 /* oldest_key_time */, 0 /* target_file_size */, + 0 /* file_creation_time */, "SST Writer" /* db_id */, db_session_id); r->file_writer.reset(new WritableFileWriter( NewLegacyWritableFileWrapper(std::move(sst_file)), file_path, r->env_options, r->ioptions.env, nullptr /* stats */, diff --git a/table/table_builder.h b/table/table_builder.h index 1b56d1a9c..36475c143 100644 --- a/table/table_builder.h +++ b/table/table_builder.h @@ -97,7 +97,8 @@ struct TableBuilderOptions { const std::string& _column_family_name, int _level, const uint64_t _creation_time = 0, const int64_t _oldest_key_time = 0, const uint64_t _target_file_size = 0, - const uint64_t _file_creation_time = 0) + const uint64_t _file_creation_time = 0, const std::string& _db_id = "", + const std::string& _db_session_id = "") : ioptions(_ioptions), moptions(_moptions), internal_comparator(_internal_comparator), @@ -111,7 +112,10 @@ struct TableBuilderOptions { creation_time(_creation_time), oldest_key_time(_oldest_key_time), target_file_size(_target_file_size), - file_creation_time(_file_creation_time) {} + file_creation_time(_file_creation_time), + db_id(_db_id), + db_session_id(_db_session_id) {} + const ImmutableCFOptions& ioptions; const MutableCFOptions& moptions; const InternalKeyComparator& internal_comparator; @@ -127,6 +131,8 @@ struct TableBuilderOptions { const int64_t oldest_key_time; const uint64_t target_file_size; const uint64_t file_creation_time; + const std::string db_id; + const std::string db_session_id; }; // TableBuilder provides the interface used to build a Table diff --git a/table/table_properties.cc b/table/table_properties.cc index d1dacd1a5..622f3d45b 100644 --- a/table/table_properties.cc +++ b/table/table_properties.cc @@ -168,6 +168,11 @@ std::string TableProperties::ToString( AppendProperty(result, "file creation time", file_creation_time, prop_delim, kv_delim); + // DB identity and DB session ID + AppendProperty(result, "DB identity", db_id, prop_delim, kv_delim); + AppendProperty(result, "DB session identity", db_session_id, prop_delim, + kv_delim); + return result; } @@ -188,6 +193,9 @@ void TableProperties::Add(const TableProperties& tp) { num_range_deletions += tp.num_range_deletions; } +const std::string TablePropertiesNames::kDbId = "rocksdb.creating.db.identity"; +const std::string TablePropertiesNames::kDbSessionId = + "rocksdb.creating.session.identity"; const std::string TablePropertiesNames::kDataSize = "rocksdb.data.size"; const std::string TablePropertiesNames::kIndexSize =