From 792d2f906ea70c170f3880ada04ae427051f0d82 Mon Sep 17 00:00:00 2001 From: Levi Tamasi Date: Mon, 31 Aug 2020 12:30:02 -0700 Subject: [PATCH] Log info about generated blob files in BlobFileBuilder (#7324) Summary: The patch adds a log message to `BlobFileBuilder` that is logged upon generating a blob file, similarly to how we log the generation of table files during flush and compaction. The log message contains the column family name, job id, blob file number, and the number and total size of blobs in the new file. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7324 Test Plan: Ran `make check` and checked the actual log messages using a custom `db_bench`. Reviewed By: riversand963 Differential Revision: D23402229 Pulled By: ltamasi fbshipit-source-id: ca42beb4db284b783d1eb2651f321032a45d0c5f --- db/blob/blob_file_builder.cc | 19 +++++++-- db/blob/blob_file_builder.h | 10 ++++- db/blob/blob_file_builder_test.cc | 67 +++++++++++++++++++------------ 3 files changed, 65 insertions(+), 31 deletions(-) diff --git a/db/blob/blob_file_builder.cc b/db/blob/blob_file_builder.cc index 282f2e712..2cc653f9c 100644 --- a/db/blob/blob_file_builder.cc +++ b/db/blob/blob_file_builder.cc @@ -15,6 +15,7 @@ #include "file/filename.h" #include "file/read_write_util.h" #include "file/writable_file_writer.h" +#include "logging/logging.h" #include "options/cf_options.h" #include "rocksdb/slice.h" #include "rocksdb/status.h" @@ -27,19 +28,22 @@ BlobFileBuilder::BlobFileBuilder( VersionSet* versions, Env* env, FileSystem* fs, const ImmutableCFOptions* immutable_cf_options, const MutableCFOptions* mutable_cf_options, const FileOptions* file_options, - uint32_t column_family_id, Env::IOPriority io_priority, + int job_id, uint32_t column_family_id, + const std::string& column_family_name, Env::IOPriority io_priority, Env::WriteLifeTimeHint write_hint, std::vector* blob_file_additions) : BlobFileBuilder([versions]() { return versions->NewFileNumber(); }, env, fs, immutable_cf_options, mutable_cf_options, - file_options, column_family_id, io_priority, write_hint, + file_options, job_id, column_family_id, + column_family_name, io_priority, write_hint, blob_file_additions) {} BlobFileBuilder::BlobFileBuilder( std::function file_number_generator, Env* env, FileSystem* fs, const ImmutableCFOptions* immutable_cf_options, const MutableCFOptions* mutable_cf_options, const FileOptions* file_options, - uint32_t column_family_id, Env::IOPriority io_priority, + int job_id, uint32_t column_family_id, + const std::string& column_family_name, Env::IOPriority io_priority, Env::WriteLifeTimeHint write_hint, std::vector* blob_file_additions) : file_number_generator_(std::move(file_number_generator)), @@ -50,7 +54,9 @@ BlobFileBuilder::BlobFileBuilder( blob_file_size_(mutable_cf_options->blob_file_size), blob_compression_type_(mutable_cf_options->blob_compression_type), file_options_(file_options), + job_id_(job_id), column_family_id_(column_family_id), + column_family_name_(column_family_name), io_priority_(io_priority), write_hint_(write_hint), blob_file_additions_(blob_file_additions), @@ -268,6 +274,13 @@ Status BlobFileBuilder::CloseBlobFile() { std::move(checksum_method), std::move(checksum_value)); + assert(immutable_cf_options_); + ROCKS_LOG_INFO(immutable_cf_options_->info_log, + "[%s] [JOB %d] Generated blob file #%" PRIu64 ": %" PRIu64 + " total blobs, %" PRIu64 " total bytes", + column_family_name_.c_str(), job_id_, blob_file_number, + blob_count_, blob_bytes_); + writer_.reset(); blob_count_ = 0; blob_bytes_ = 0; diff --git a/db/blob/blob_file_builder.h b/db/blob/blob_file_builder.h index 4a0370b89..86018ec01 100644 --- a/db/blob/blob_file_builder.h +++ b/db/blob/blob_file_builder.h @@ -31,7 +31,9 @@ class BlobFileBuilder { BlobFileBuilder(VersionSet* versions, Env* env, FileSystem* fs, const ImmutableCFOptions* immutable_cf_options, const MutableCFOptions* mutable_cf_options, - const FileOptions* file_options, uint32_t column_family_id, + const FileOptions* file_options, int job_id, + uint32_t column_family_id, + const std::string& column_family_name, Env::IOPriority io_priority, Env::WriteLifeTimeHint write_hint, std::vector* blob_file_additions); @@ -40,7 +42,9 @@ class BlobFileBuilder { FileSystem* fs, const ImmutableCFOptions* immutable_cf_options, const MutableCFOptions* mutable_cf_options, - const FileOptions* file_options, uint32_t column_family_id, + const FileOptions* file_options, int job_id, + uint32_t column_family_id, + const std::string& column_family_name, Env::IOPriority io_priority, Env::WriteLifeTimeHint write_hint, std::vector* blob_file_additions); @@ -70,7 +74,9 @@ class BlobFileBuilder { uint64_t blob_file_size_; CompressionType blob_compression_type_; const FileOptions* file_options_; + int job_id_; uint32_t column_family_id_; + std::string column_family_name_; Env::IOPriority io_priority_; Env::WriteLifeTimeHint write_hint_; std::vector* blob_file_additions_; diff --git a/db/blob/blob_file_builder_test.cc b/db/blob/blob_file_builder_test.cc index bafac4a41..e09cc2340 100644 --- a/db/blob/blob_file_builder_test.cc +++ b/db/blob/blob_file_builder_test.cc @@ -131,16 +131,18 @@ TEST_F(BlobFileBuilderTest, BuildAndCheckOneFile) { ImmutableCFOptions immutable_cf_options(options); MutableCFOptions mutable_cf_options(options); + constexpr int job_id = 1; constexpr uint32_t column_family_id = 123; + constexpr char column_family_name[] = "foobar"; constexpr Env::IOPriority io_priority = Env::IO_HIGH; constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM; std::vector blob_file_additions; - BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, &fs_, - &immutable_cf_options, &mutable_cf_options, - &file_options_, column_family_id, io_priority, - write_hint, &blob_file_additions); + BlobFileBuilder builder( + TestFileNumberGenerator(), &mock_env_, &fs_, &immutable_cf_options, + &mutable_cf_options, &file_options_, job_id, column_family_id, + column_family_name, io_priority, write_hint, &blob_file_additions); std::vector> expected_key_value_pairs( number_of_blobs); @@ -202,16 +204,18 @@ TEST_F(BlobFileBuilderTest, BuildAndCheckMultipleFiles) { ImmutableCFOptions immutable_cf_options(options); MutableCFOptions mutable_cf_options(options); + constexpr int job_id = 1; constexpr uint32_t column_family_id = 123; + constexpr char column_family_name[] = "foobar"; constexpr Env::IOPriority io_priority = Env::IO_HIGH; constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM; std::vector blob_file_additions; - BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, &fs_, - &immutable_cf_options, &mutable_cf_options, - &file_options_, column_family_id, io_priority, - write_hint, &blob_file_additions); + BlobFileBuilder builder( + TestFileNumberGenerator(), &mock_env_, &fs_, &immutable_cf_options, + &mutable_cf_options, &file_options_, job_id, column_family_id, + column_family_name, io_priority, write_hint, &blob_file_additions); std::vector> expected_key_value_pairs( number_of_blobs); @@ -276,16 +280,18 @@ TEST_F(BlobFileBuilderTest, InlinedValues) { ImmutableCFOptions immutable_cf_options(options); MutableCFOptions mutable_cf_options(options); + constexpr int job_id = 1; constexpr uint32_t column_family_id = 123; + constexpr char column_family_name[] = "foobar"; constexpr Env::IOPriority io_priority = Env::IO_HIGH; constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM; std::vector blob_file_additions; - BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, &fs_, - &immutable_cf_options, &mutable_cf_options, - &file_options_, column_family_id, io_priority, - write_hint, &blob_file_additions); + BlobFileBuilder builder( + TestFileNumberGenerator(), &mock_env_, &fs_, &immutable_cf_options, + &mutable_cf_options, &file_options_, job_id, column_family_id, + column_family_name, io_priority, write_hint, &blob_file_additions); for (size_t i = 0; i < number_of_blobs; ++i) { const std::string key = std::to_string(i); @@ -323,16 +329,18 @@ TEST_F(BlobFileBuilderTest, Compression) { ImmutableCFOptions immutable_cf_options(options); MutableCFOptions mutable_cf_options(options); + constexpr int job_id = 1; constexpr uint32_t column_family_id = 123; + constexpr char column_family_name[] = "foobar"; constexpr Env::IOPriority io_priority = Env::IO_HIGH; constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM; std::vector blob_file_additions; - BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, &fs_, - &immutable_cf_options, &mutable_cf_options, - &file_options_, column_family_id, io_priority, - write_hint, &blob_file_additions); + BlobFileBuilder builder( + TestFileNumberGenerator(), &mock_env_, &fs_, &immutable_cf_options, + &mutable_cf_options, &file_options_, job_id, column_family_id, + column_family_name, io_priority, write_hint, &blob_file_additions); const std::string key("1"); const std::string uncompressed_value(value_size, 'x'); @@ -393,16 +401,18 @@ TEST_F(BlobFileBuilderTest, CompressionError) { ImmutableCFOptions immutable_cf_options(options); MutableCFOptions mutable_cf_options(options); + constexpr int job_id = 1; constexpr uint32_t column_family_id = 123; + constexpr char column_family_name[] = "foobar"; constexpr Env::IOPriority io_priority = Env::IO_HIGH; constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM; std::vector blob_file_additions; - BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, &fs_, - &immutable_cf_options, &mutable_cf_options, - &file_options_, column_family_id, io_priority, - write_hint, &blob_file_additions); + BlobFileBuilder builder( + TestFileNumberGenerator(), &mock_env_, &fs_, &immutable_cf_options, + &mutable_cf_options, &file_options_, job_id, column_family_id, + column_family_name, io_priority, write_hint, &blob_file_additions); SyncPoint::GetInstance()->SetCallBack("CompressData:TamperWithReturnValue", [](void* arg) { @@ -457,16 +467,18 @@ TEST_F(BlobFileBuilderTest, Checksum) { ImmutableCFOptions immutable_cf_options(options); MutableCFOptions mutable_cf_options(options); + constexpr int job_id = 1; constexpr uint32_t column_family_id = 123; + constexpr char column_family_name[] = "foobar"; constexpr Env::IOPriority io_priority = Env::IO_HIGH; constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM; std::vector blob_file_additions; - BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, &fs_, - &immutable_cf_options, &mutable_cf_options, - &file_options_, column_family_id, io_priority, - write_hint, &blob_file_additions); + BlobFileBuilder builder( + TestFileNumberGenerator(), &mock_env_, &fs_, &immutable_cf_options, + &mutable_cf_options, &file_options_, job_id, column_family_id, + column_family_name, io_priority, write_hint, &blob_file_additions); const std::string key("1"); const std::string value("deadbeef"); @@ -543,7 +555,9 @@ TEST_P(BlobFileBuilderIOErrorTest, IOError) { ImmutableCFOptions immutable_cf_options(options); MutableCFOptions mutable_cf_options(options); + constexpr int job_id = 1; constexpr uint32_t column_family_id = 123; + constexpr char column_family_name[] = "foobar"; constexpr Env::IOPriority io_priority = Env::IO_HIGH; constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM; @@ -551,8 +565,9 @@ TEST_P(BlobFileBuilderIOErrorTest, IOError) { BlobFileBuilder builder(TestFileNumberGenerator(), &fault_injection_env_, &fs_, &immutable_cf_options, &mutable_cf_options, - &file_options_, column_family_id, io_priority, - write_hint, &blob_file_additions); + &file_options_, job_id, column_family_id, + column_family_name, io_priority, write_hint, + &blob_file_additions); SyncPoint::GetInstance()->SetCallBack(sync_point_, [this](void* /* arg */) { fault_injection_env_.SetFilesystemActive(false,