From d51f88c9e482def78641baceac6aaf6c62c42c13 Mon Sep 17 00:00:00 2001 From: Zhichao Cao Date: Tue, 25 Aug 2020 10:44:39 -0700 Subject: [PATCH] Pass SST file checksum information through OnTableFileCreated (#7108) Summary: When SST file is created, application is able to know the file information through OnTableFileCreated callback in LogAndNotifyTableFileCreationFinished. Since file checksum information can be useful for application when the SST file is created, we add file_checksum and file_checksum_func_name information to TableFileCreationInfo, which will be passed through OnTableFileCreated. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7108 Test Plan: make check, listener_test. Reviewed By: ajkr Differential Revision: D22470240 Pulled By: zhichao-cao fbshipit-source-id: 92c20344d9b986eadfe3480f3769bf4add0dbaae --- HISTORY.md | 2 ++ db/builder.cc | 10 ++++++++-- db/compaction/compaction_job.cc | 10 ++++++++-- db/event_helpers.cc | 9 +++++++-- db/event_helpers.h | 4 +++- db/listener_test.cc | 4 ++++ include/rocksdb/listener.h | 4 ++++ 7 files changed, 36 insertions(+), 7 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 016646df0..90b722562 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -18,6 +18,8 @@ ### Public API Change * Expose kTypeDeleteWithTimestamp in EntryType and update GetEntryType() accordingly. +* Added file_checksum and file_checksum_func_name to TableFileCreationInfo, which can pass the table file checksum information through the OnTableFileCreated callback during flush and compaction. + ## 6.12 (2020-07-28) ### Public API Change diff --git a/db/builder.cc b/db/builder.cc index 0a4adeb6f..8507135a8 100644 --- a/db/builder.cc +++ b/db/builder.cc @@ -106,6 +106,8 @@ Status BuildTable( std::string fname = TableFileName(ioptions.cf_paths, meta->fd.GetNumber(), meta->fd.GetPathId()); + std::string file_checksum = kUnknownFileChecksum; + std::string file_checksum_func_name = kUnknownFileChecksumFuncName; #ifndef ROCKSDB_LITE EventHelpers::NotifyTableFileCreationStarted( ioptions.listeners, dbname, column_family_name, fname, job_id, reason); @@ -133,7 +135,8 @@ Status BuildTable( if (!s.ok()) { EventHelpers::LogAndNotifyTableFileCreationFinished( event_logger, ioptions.listeners, dbname, column_family_name, fname, - job_id, meta->fd, kInvalidBlobFileNumber, tp, reason, s); + job_id, meta->fd, kInvalidBlobFileNumber, tp, reason, s, + file_checksum, file_checksum_func_name); return s; } file->SetIOPriority(io_priority); @@ -232,6 +235,8 @@ Status BuildTable( // Add the checksum information to file metadata. meta->file_checksum = file_writer->GetFileChecksum(); meta->file_checksum_func_name = file_writer->GetFileChecksumFuncName(); + file_checksum = meta->file_checksum; + file_checksum_func_name = meta->file_checksum_func_name; } if (s.ok()) { @@ -292,7 +297,8 @@ Status BuildTable( // Output to event logger and fire events. EventHelpers::LogAndNotifyTableFileCreationFinished( event_logger, ioptions.listeners, dbname, column_family_name, fname, - job_id, meta->fd, meta->oldest_blob_file_number, tp, reason, s); + job_id, meta->fd, meta->oldest_blob_file_number, tp, reason, s, + file_checksum, file_checksum_func_name); return s; } diff --git a/db/compaction/compaction_job.cc b/db/compaction/compaction_job.cc index 5e0e5bd47..e097dbdfd 100644 --- a/db/compaction/compaction_job.cc +++ b/db/compaction/compaction_job.cc @@ -1198,6 +1198,8 @@ Status CompactionJob::FinishCompactionOutputFile( ColumnFamilyData* cfd = sub_compact->compaction->column_family_data(); const Comparator* ucmp = cfd->user_comparator(); + std::string file_checksum = kUnknownFileChecksum; + std::string file_checksum_func_name = kUnknownFileChecksumFuncName; // Check for iterator errors Status s = input_status; @@ -1397,6 +1399,8 @@ Status CompactionJob::FinishCompactionOutputFile( meta->file_checksum = sub_compact->outfile->GetFileChecksum(); meta->file_checksum_func_name = sub_compact->outfile->GetFileChecksumFuncName(); + file_checksum = meta->file_checksum; + file_checksum_func_name = meta->file_checksum_func_name; } if (s.ok()) { s = io_s; @@ -1453,7 +1457,8 @@ Status CompactionJob::FinishCompactionOutputFile( EventHelpers::LogAndNotifyTableFileCreationFinished( event_logger_, cfd->ioptions()->listeners, dbname_, cfd->GetName(), fname, job_id_, output_fd, oldest_blob_file_number, tp, - TableFileCreationReason::kCompaction, s); + TableFileCreationReason::kCompaction, s, file_checksum, + file_checksum_func_name); #ifndef ROCKSDB_LITE // Report new file to SstFileManagerImpl @@ -1582,7 +1587,8 @@ Status CompactionJob::OpenCompactionOutputFile( EventHelpers::LogAndNotifyTableFileCreationFinished( event_logger_, cfd->ioptions()->listeners, dbname_, cfd->GetName(), fname, job_id_, FileDescriptor(), kInvalidBlobFileNumber, - TableProperties(), TableFileCreationReason::kCompaction, s); + TableProperties(), TableFileCreationReason::kCompaction, s, + kUnknownFileChecksum, kUnknownFileChecksumFuncName); return s; } diff --git a/db/event_helpers.cc b/db/event_helpers.cc index b7f491409..0b92d569a 100644 --- a/db/event_helpers.cc +++ b/db/event_helpers.cc @@ -71,14 +71,17 @@ void EventHelpers::LogAndNotifyTableFileCreationFinished( const std::string& db_name, const std::string& cf_name, const std::string& file_path, int job_id, const FileDescriptor& fd, uint64_t oldest_blob_file_number, const TableProperties& table_properties, - TableFileCreationReason reason, const Status& s) { + TableFileCreationReason reason, const Status& s, + const std::string& file_checksum, + const std::string& file_checksum_func_name) { if (s.ok() && event_logger) { JSONWriter jwriter; AppendCurrentTime(&jwriter); jwriter << "cf_name" << cf_name << "job" << job_id << "event" << "table_file_creation" << "file_number" << fd.GetNumber() << "file_size" - << fd.GetFileSize(); + << fd.GetFileSize() << "file_checksum" << file_checksum + << "file_checksum_func_name" << file_checksum_func_name; // table_properties { @@ -154,6 +157,8 @@ void EventHelpers::LogAndNotifyTableFileCreationFinished( info.table_properties = table_properties; info.reason = reason; info.status = s; + info.file_checksum = file_checksum; + info.file_checksum_func_name = file_checksum_func_name; for (auto& listener : listeners) { listener->OnTableFileCreated(info); } diff --git a/db/event_helpers.h b/db/event_helpers.h index 87cc1cb8c..abc00981c 100644 --- a/db/event_helpers.h +++ b/db/event_helpers.h @@ -35,7 +35,9 @@ class EventHelpers { const std::string& db_name, const std::string& cf_name, const std::string& file_path, int job_id, const FileDescriptor& fd, uint64_t oldest_blob_file_number, const TableProperties& table_properties, - TableFileCreationReason reason, const Status& s); + TableFileCreationReason reason, const Status& s, + const std::string& file_checksum, + const std::string& file_checksum_func_name); static void LogAndNotifyTableFileDeletion( EventLogger* event_logger, int job_id, uint64_t file_number, const std::string& file_path, diff --git a/db/listener_test.cc b/db/listener_test.cc index c3df30d6a..9239068ce 100644 --- a/db/listener_test.cc +++ b/db/listener_test.cc @@ -227,6 +227,8 @@ class TestFlushListener : public EventListener { ASSERT_GT(info.table_properties.raw_value_size, 0U); ASSERT_GT(info.table_properties.num_data_blocks, 0U); ASSERT_GT(info.table_properties.num_entries, 0U); + ASSERT_EQ(info.file_checksum, kUnknownFileChecksum); + ASSERT_EQ(info.file_checksum_func_name, kUnknownFileChecksumFuncName); #ifdef ROCKSDB_USING_THREAD_STATUS // Verify the id of the current thread that created this table @@ -751,6 +753,8 @@ class TableFileCreationListener : public EventListener { ASSERT_GT(info.cf_name.size(), 0U); ASSERT_GT(info.file_path.size(), 0U); ASSERT_GT(info.job_id, 0); + ASSERT_EQ(info.file_checksum, kUnknownFileChecksum); + ASSERT_EQ(info.file_checksum_func_name, kUnknownFileChecksumFuncName); if (info.status.ok()) { ASSERT_GT(info.table_properties.data_size, 0U); ASSERT_GT(info.table_properties.raw_key_size, 0U); diff --git a/include/rocksdb/listener.h b/include/rocksdb/listener.h index 19145e816..1c15fc0fe 100644 --- a/include/rocksdb/listener.h +++ b/include/rocksdb/listener.h @@ -58,6 +58,10 @@ struct TableFileCreationInfo : public TableFileCreationBriefInfo { TableProperties table_properties; // The status indicating whether the creation was successful or not. Status status; + // The checksum of the table file being created + std::string file_checksum; + // The checksum function name of checksum generator used for this table file + std::string file_checksum_func_name; }; enum class CompactionReason : int {