From ea8bb82fc7f6fcbed08291100edba10f09c790cd Mon Sep 17 00:00:00 2001 From: Akanksha Mahajan Date: Tue, 16 Feb 2021 09:47:12 -0800 Subject: [PATCH] Add support for IOTracing in blob files (#7958) Summary: Add support for IOTracing in blob files Pull Request resolved: https://github.com/facebook/rocksdb/pull/7958 Test Plan: Add a new test and checked manually the trace_file for blob files being recorded during read and write. Reviewed By: ltamasi Differential Revision: D26415950 Pulled By: akankshamahajan15 fbshipit-source-id: 49c2859b3a4f8307e7cb69a92704403a4da46d44 --- db/blob/blob_file_builder.cc | 8 +++- db/blob/blob_file_builder.h | 4 ++ db/blob/blob_file_builder_test.cc | 70 +++++++++++++++---------------- db/blob/blob_file_cache.cc | 9 ++-- db/blob/blob_file_cache.h | 5 ++- db/blob/blob_file_cache_test.cc | 8 ++-- db/blob/blob_file_reader.cc | 9 ++-- db/blob/blob_file_reader.h | 5 ++- db/blob/blob_file_reader_test.cc | 50 ++++++++++++---------- db/blob/db_blob_basic_test.cc | 57 +++++++++++++++++++++++++ db/builder.cc | 4 +- db/column_family.cc | 2 +- db/compaction/compaction_job.cc | 2 +- 13 files changed, 156 insertions(+), 77 deletions(-) diff --git a/db/blob/blob_file_builder.cc b/db/blob/blob_file_builder.cc index 722ed21b9..0e023bb77 100644 --- a/db/blob/blob_file_builder.cc +++ b/db/blob/blob_file_builder.cc @@ -21,6 +21,7 @@ #include "rocksdb/slice.h" #include "rocksdb/status.h" #include "test_util/sync_point.h" +#include "trace_replay/io_tracer.h" #include "util/compression.h" namespace ROCKSDB_NAMESPACE { @@ -32,12 +33,13 @@ BlobFileBuilder::BlobFileBuilder( int job_id, uint32_t column_family_id, const std::string& column_family_name, Env::IOPriority io_priority, Env::WriteLifeTimeHint write_hint, + const std::shared_ptr& io_tracer, std::vector* blob_file_paths, std::vector* blob_file_additions) : BlobFileBuilder([versions]() { return versions->NewFileNumber(); }, env, fs, immutable_cf_options, mutable_cf_options, file_options, job_id, column_family_id, - column_family_name, io_priority, write_hint, + column_family_name, io_priority, write_hint, io_tracer, blob_file_paths, blob_file_additions) {} BlobFileBuilder::BlobFileBuilder( @@ -47,6 +49,7 @@ BlobFileBuilder::BlobFileBuilder( int job_id, uint32_t column_family_id, const std::string& column_family_name, Env::IOPriority io_priority, Env::WriteLifeTimeHint write_hint, + const std::shared_ptr& io_tracer, std::vector* blob_file_paths, std::vector* blob_file_additions) : file_number_generator_(std::move(file_number_generator)), @@ -61,6 +64,7 @@ BlobFileBuilder::BlobFileBuilder( column_family_name_(column_family_name), io_priority_(io_priority), write_hint_(write_hint), + io_tracer_(io_tracer), blob_file_paths_(blob_file_paths), blob_file_additions_(blob_file_additions), blob_count_(0), @@ -182,7 +186,7 @@ Status BlobFileBuilder::OpenBlobFileIfNeeded() { Statistics* const statistics = immutable_cf_options_->statistics; std::unique_ptr file_writer(new WritableFileWriter( std::move(file), blob_file_paths_->back(), *file_options_, clock_, - nullptr /*IOTracer*/, statistics, immutable_cf_options_->listeners, + io_tracer_, statistics, immutable_cf_options_->listeners, immutable_cf_options_->file_checksum_gen_factory, tmp_set.Contains(FileType::kBlobFile))); diff --git a/db/blob/blob_file_builder.h b/db/blob/blob_file_builder.h index 524a878c4..70a268c4f 100644 --- a/db/blob/blob_file_builder.h +++ b/db/blob/blob_file_builder.h @@ -26,6 +26,7 @@ class BlobFileAddition; class Status; class Slice; class BlobLogWriter; +class IOTracer; class BlobFileBuilder { public: @@ -37,6 +38,7 @@ class BlobFileBuilder { const std::string& column_family_name, Env::IOPriority io_priority, Env::WriteLifeTimeHint write_hint, + const std::shared_ptr& io_tracer, std::vector* blob_file_paths, std::vector* blob_file_additions); @@ -49,6 +51,7 @@ class BlobFileBuilder { const std::string& column_family_name, Env::IOPriority io_priority, Env::WriteLifeTimeHint write_hint, + const std::shared_ptr& io_tracer, std::vector* blob_file_paths, std::vector* blob_file_additions); @@ -82,6 +85,7 @@ class BlobFileBuilder { std::string column_family_name_; Env::IOPriority io_priority_; Env::WriteLifeTimeHint write_hint_; + std::shared_ptr io_tracer_; std::vector* blob_file_paths_; std::vector* blob_file_additions_; std::unique_ptr writer_; diff --git a/db/blob/blob_file_builder_test.cc b/db/blob/blob_file_builder_test.cc index d6068b64f..06f556aaa 100644 --- a/db/blob/blob_file_builder_test.cc +++ b/db/blob/blob_file_builder_test.cc @@ -140,11 +140,11 @@ TEST_F(BlobFileBuilderTest, BuildAndCheckOneFile) { std::vector blob_file_paths; std::vector blob_file_additions; - BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_, - &immutable_cf_options, &mutable_cf_options, - &file_options_, job_id, column_family_id, - column_family_name, io_priority, write_hint, - &blob_file_paths, &blob_file_additions); + BlobFileBuilder builder( + TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options, + &mutable_cf_options, &file_options_, job_id, column_family_id, + column_family_name, io_priority, write_hint, nullptr /*IOTracer*/, + &blob_file_paths, &blob_file_additions); std::vector> expected_key_value_pairs( number_of_blobs); @@ -223,11 +223,11 @@ TEST_F(BlobFileBuilderTest, BuildAndCheckMultipleFiles) { std::vector blob_file_paths; std::vector blob_file_additions; - BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_, - &immutable_cf_options, &mutable_cf_options, - &file_options_, job_id, column_family_id, - column_family_name, io_priority, write_hint, - &blob_file_paths, &blob_file_additions); + BlobFileBuilder builder( + TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options, + &mutable_cf_options, &file_options_, job_id, column_family_id, + column_family_name, io_priority, write_hint, nullptr /*IOTracer*/, + &blob_file_paths, &blob_file_additions); std::vector> expected_key_value_pairs( number_of_blobs); @@ -308,11 +308,11 @@ TEST_F(BlobFileBuilderTest, InlinedValues) { std::vector blob_file_paths; std::vector blob_file_additions; - BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_, - &immutable_cf_options, &mutable_cf_options, - &file_options_, job_id, column_family_id, - column_family_name, io_priority, write_hint, - &blob_file_paths, &blob_file_additions); + BlobFileBuilder builder( + TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options, + &mutable_cf_options, &file_options_, job_id, column_family_id, + column_family_name, io_priority, write_hint, nullptr /*IOTracer*/, + &blob_file_paths, &blob_file_additions); for (size_t i = 0; i < number_of_blobs; ++i) { const std::string key = std::to_string(i); @@ -360,11 +360,11 @@ TEST_F(BlobFileBuilderTest, Compression) { std::vector blob_file_paths; std::vector blob_file_additions; - BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_, - &immutable_cf_options, &mutable_cf_options, - &file_options_, job_id, column_family_id, - column_family_name, io_priority, write_hint, - &blob_file_paths, &blob_file_additions); + BlobFileBuilder builder( + TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options, + &mutable_cf_options, &file_options_, job_id, column_family_id, + column_family_name, io_priority, write_hint, nullptr /*IOTracer*/, + &blob_file_paths, &blob_file_additions); const std::string key("1"); const std::string uncompressed_value(value_size, 'x'); @@ -442,11 +442,11 @@ TEST_F(BlobFileBuilderTest, CompressionError) { std::vector blob_file_paths; std::vector blob_file_additions; - BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_, - &immutable_cf_options, &mutable_cf_options, - &file_options_, job_id, column_family_id, - column_family_name, io_priority, write_hint, - &blob_file_paths, &blob_file_additions); + BlobFileBuilder builder( + TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options, + &mutable_cf_options, &file_options_, job_id, column_family_id, + column_family_name, io_priority, write_hint, nullptr /*IOTracer*/, + &blob_file_paths, &blob_file_additions); SyncPoint::GetInstance()->SetCallBack("CompressData:TamperWithReturnValue", [](void* arg) { @@ -519,11 +519,11 @@ TEST_F(BlobFileBuilderTest, Checksum) { std::vector blob_file_paths; std::vector blob_file_additions; - BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_, - &immutable_cf_options, &mutable_cf_options, - &file_options_, job_id, column_family_id, - column_family_name, io_priority, write_hint, - &blob_file_paths, &blob_file_additions); + BlobFileBuilder builder( + TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options, + &mutable_cf_options, &file_options_, job_id, column_family_id, + column_family_name, io_priority, write_hint, nullptr /*IOTracer*/, + &blob_file_paths, &blob_file_additions); const std::string key("1"); const std::string value("deadbeef"); @@ -615,11 +615,11 @@ TEST_P(BlobFileBuilderIOErrorTest, IOError) { std::vector blob_file_paths; std::vector blob_file_additions; - BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_, - &immutable_cf_options, &mutable_cf_options, - &file_options_, job_id, column_family_id, - column_family_name, io_priority, write_hint, - &blob_file_paths, &blob_file_additions); + BlobFileBuilder builder( + TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options, + &mutable_cf_options, &file_options_, job_id, column_family_id, + column_family_name, io_priority, write_hint, nullptr /*IOTracer*/, + &blob_file_paths, &blob_file_additions); SyncPoint::GetInstance()->SetCallBack(sync_point_, [this](void* arg) { Status* const s = static_cast(arg); diff --git a/db/blob/blob_file_cache.cc b/db/blob/blob_file_cache.cc index 14a4feec8..979afccbe 100644 --- a/db/blob/blob_file_cache.cc +++ b/db/blob/blob_file_cache.cc @@ -13,6 +13,7 @@ #include "rocksdb/cache.h" #include "rocksdb/slice.h" #include "test_util/sync_point.h" +#include "trace_replay/io_tracer.h" #include "util/hash.h" namespace ROCKSDB_NAMESPACE { @@ -21,13 +22,15 @@ BlobFileCache::BlobFileCache(Cache* cache, const ImmutableCFOptions* immutable_cf_options, const FileOptions* file_options, uint32_t column_family_id, - HistogramImpl* blob_file_read_hist) + HistogramImpl* blob_file_read_hist, + const std::shared_ptr& io_tracer) : cache_(cache), mutex_(kNumberOfMutexStripes, kGetSliceNPHash64UnseededFnPtr), immutable_cf_options_(immutable_cf_options), file_options_(file_options), column_family_id_(column_family_id), - blob_file_read_hist_(blob_file_read_hist) { + blob_file_read_hist_(blob_file_read_hist), + io_tracer_(io_tracer) { assert(cache_); assert(immutable_cf_options_); assert(file_options_); @@ -71,7 +74,7 @@ Status BlobFileCache::GetBlobFileReader( assert(file_options_); const Status s = BlobFileReader::Create( *immutable_cf_options_, *file_options_, column_family_id_, - blob_file_read_hist_, blob_file_number, &reader); + blob_file_read_hist_, blob_file_number, io_tracer_, &reader); if (!s.ok()) { RecordTick(statistics, NO_FILE_ERRORS); return s; diff --git a/db/blob/blob_file_cache.h b/db/blob/blob_file_cache.h index 09f09e87e..ace2d938b 100644 --- a/db/blob/blob_file_cache.h +++ b/db/blob/blob_file_cache.h @@ -20,12 +20,14 @@ class HistogramImpl; class Status; class BlobFileReader; class Slice; +class IOTracer; class BlobFileCache { public: BlobFileCache(Cache* cache, const ImmutableCFOptions* immutable_cf_options, const FileOptions* file_options, uint32_t column_family_id, - HistogramImpl* blob_file_read_hist); + HistogramImpl* blob_file_read_hist, + const std::shared_ptr& io_tracer); BlobFileCache(const BlobFileCache&) = delete; BlobFileCache& operator=(const BlobFileCache&) = delete; @@ -42,6 +44,7 @@ class BlobFileCache { const FileOptions* file_options_; uint32_t column_family_id_; HistogramImpl* blob_file_read_hist_; + std::shared_ptr io_tracer_; static constexpr size_t kNumberOfMutexStripes = 1 << 7; }; diff --git a/db/blob/blob_file_cache_test.cc b/db/blob/blob_file_cache_test.cc index c325eb4cb..15378ca90 100644 --- a/db/blob/blob_file_cache_test.cc +++ b/db/blob/blob_file_cache_test.cc @@ -114,7 +114,7 @@ TEST_F(BlobFileCacheTest, GetBlobFileReader) { BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options, &file_options, column_family_id, - blob_file_read_hist); + blob_file_read_hist, nullptr /*IOTracer*/); // First try: reader should be opened and put in cache CacheHandleGuard first; @@ -159,7 +159,7 @@ TEST_F(BlobFileCacheTest, GetBlobFileReader_Race) { BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options, &file_options, column_family_id, - blob_file_read_hist); + blob_file_read_hist, nullptr /*IOTracer*/); CacheHandleGuard first; CacheHandleGuard second; @@ -207,7 +207,7 @@ TEST_F(BlobFileCacheTest, GetBlobFileReader_IOError) { BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options, &file_options, column_family_id, - blob_file_read_hist); + blob_file_read_hist, nullptr /*IOTracer*/); // Note: there is no blob file with the below number constexpr uint64_t blob_file_number = 123; @@ -248,7 +248,7 @@ TEST_F(BlobFileCacheTest, GetBlobFileReader_CacheFull) { BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options, &file_options, column_family_id, - blob_file_read_hist); + blob_file_read_hist, nullptr /*IOTracer*/); // Insert into cache should fail since it has zero capacity and // strict_capacity_limit is set diff --git a/db/blob/blob_file_reader.cc b/db/blob/blob_file_reader.cc index fd394107d..bc4cede08 100644 --- a/db/blob/blob_file_reader.cc +++ b/db/blob/blob_file_reader.cc @@ -24,6 +24,7 @@ Status BlobFileReader::Create( const ImmutableCFOptions& immutable_cf_options, const FileOptions& file_options, uint32_t column_family_id, HistogramImpl* blob_file_read_hist, uint64_t blob_file_number, + const std::shared_ptr& io_tracer, std::unique_ptr* blob_file_reader) { assert(blob_file_reader); assert(!*blob_file_reader); @@ -34,7 +35,7 @@ Status BlobFileReader::Create( { const Status s = OpenFile(immutable_cf_options, file_options, blob_file_read_hist, - blob_file_number, &file_size, &file_reader); + blob_file_number, io_tracer, &file_size, &file_reader); if (!s.ok()) { return s; } @@ -68,8 +69,8 @@ Status BlobFileReader::Create( Status BlobFileReader::OpenFile( const ImmutableCFOptions& immutable_cf_options, const FileOptions& file_opts, HistogramImpl* blob_file_read_hist, - uint64_t blob_file_number, uint64_t* file_size, - std::unique_ptr* file_reader) { + uint64_t blob_file_number, const std::shared_ptr& io_tracer, + uint64_t* file_size, std::unique_ptr* file_reader) { assert(file_size); assert(file_reader); @@ -118,7 +119,7 @@ Status BlobFileReader::OpenFile( file_reader->reset(new RandomAccessFileReader( std::move(file), blob_file_path, - immutable_cf_options.env->GetSystemClock(), std::shared_ptr(), + immutable_cf_options.env->GetSystemClock(), io_tracer, immutable_cf_options.statistics, BLOB_DB_BLOB_FILE_READ_MICROS, blob_file_read_hist, immutable_cf_options.rate_limiter, immutable_cf_options.listeners)); diff --git a/db/blob/blob_file_reader.h b/db/blob/blob_file_reader.h index 8c7df393d..dee142bc1 100644 --- a/db/blob/blob_file_reader.h +++ b/db/blob/blob_file_reader.h @@ -29,6 +29,7 @@ class BlobFileReader { uint32_t column_family_id, HistogramImpl* blob_file_read_hist, uint64_t blob_file_number, + const std::shared_ptr& io_tracer, std::unique_ptr* reader); BlobFileReader(const BlobFileReader&) = delete; @@ -47,7 +48,9 @@ class BlobFileReader { static Status OpenFile(const ImmutableCFOptions& immutable_cf_options, const FileOptions& file_opts, HistogramImpl* blob_file_read_hist, - uint64_t blob_file_number, uint64_t* file_size, + uint64_t blob_file_number, + const std::shared_ptr& io_tracer, + uint64_t* file_size, std::unique_ptr* file_reader); static Status ReadHeader(const RandomAccessFileReader* file_reader, diff --git a/db/blob/blob_file_reader_test.cc b/db/blob/blob_file_reader_test.cc index f31fb3dae..1d0b78bac 100644 --- a/db/blob/blob_file_reader_test.cc +++ b/db/blob/blob_file_reader_test.cc @@ -142,9 +142,9 @@ TEST_F(BlobFileReaderTest, CreateReaderAndGetBlob) { std::unique_ptr reader; - ASSERT_OK(BlobFileReader::Create(immutable_cf_options, FileOptions(), - column_family_id, blob_file_read_hist, - blob_file_number, &reader)); + ASSERT_OK(BlobFileReader::Create( + immutable_cf_options, FileOptions(), column_family_id, + blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader)); // Make sure the blob can be retrieved with and without checksum verification ReadOptions read_options; @@ -282,7 +282,8 @@ TEST_F(BlobFileReaderTest, Malformed) { ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(), column_family_id, blob_file_read_hist, - blob_file_number, &reader) + blob_file_number, nullptr /*IOTracer*/, + &reader) .IsCorruption()); } @@ -315,7 +316,8 @@ TEST_F(BlobFileReaderTest, TTL) { ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(), column_family_id, blob_file_read_hist, - blob_file_number, &reader) + blob_file_number, nullptr /*IOTracer*/, + &reader) .IsCorruption()); } @@ -353,7 +355,8 @@ TEST_F(BlobFileReaderTest, ExpirationRangeInHeader) { ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(), column_family_id, blob_file_read_hist, - blob_file_number, &reader) + blob_file_number, nullptr /*IOTracer*/, + &reader) .IsCorruption()); } @@ -391,7 +394,8 @@ TEST_F(BlobFileReaderTest, ExpirationRangeInFooter) { ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(), column_family_id, blob_file_read_hist, - blob_file_number, &reader) + blob_file_number, nullptr /*IOTracer*/, + &reader) .IsCorruption()); } @@ -429,7 +433,7 @@ TEST_F(BlobFileReaderTest, IncorrectColumnFamily) { ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(), incorrect_column_family_id, blob_file_read_hist, blob_file_number, - &reader) + nullptr /*IOTracer*/, &reader) .IsCorruption()); } @@ -460,9 +464,9 @@ TEST_F(BlobFileReaderTest, BlobCRCError) { std::unique_ptr reader; - ASSERT_OK(BlobFileReader::Create(immutable_cf_options, FileOptions(), - column_family_id, blob_file_read_hist, - blob_file_number, &reader)); + ASSERT_OK(BlobFileReader::Create( + immutable_cf_options, FileOptions(), column_family_id, + blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader)); SyncPoint::GetInstance()->SetCallBack( "BlobFileReader::VerifyBlob:CheckBlobCRC", [](void* arg) { @@ -516,9 +520,9 @@ TEST_F(BlobFileReaderTest, Compression) { std::unique_ptr reader; - ASSERT_OK(BlobFileReader::Create(immutable_cf_options, FileOptions(), - column_family_id, blob_file_read_hist, - blob_file_number, &reader)); + ASSERT_OK(BlobFileReader::Create( + immutable_cf_options, FileOptions(), column_family_id, + blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader)); // Make sure the blob can be retrieved with and without checksum verification ReadOptions read_options; @@ -576,9 +580,9 @@ TEST_F(BlobFileReaderTest, UncompressionError) { std::unique_ptr reader; - ASSERT_OK(BlobFileReader::Create(immutable_cf_options, FileOptions(), - column_family_id, blob_file_read_hist, - blob_file_number, &reader)); + ASSERT_OK(BlobFileReader::Create( + immutable_cf_options, FileOptions(), column_family_id, + blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader)); SyncPoint::GetInstance()->SetCallBack( "BlobFileReader::UncompressBlobIfNeeded:TamperWithResult", [](void* arg) { @@ -661,9 +665,9 @@ TEST_P(BlobFileReaderIOErrorTest, IOError) { std::unique_ptr reader; - const Status s = BlobFileReader::Create(immutable_cf_options, FileOptions(), - column_family_id, blob_file_read_hist, - blob_file_number, &reader); + const Status s = BlobFileReader::Create( + immutable_cf_options, FileOptions(), column_family_id, + blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader); const bool fail_during_create = (sync_point_ != "BlobFileReader::GetBlob:ReadFromFile"); @@ -741,9 +745,9 @@ TEST_P(BlobFileReaderDecodingErrorTest, DecodingError) { std::unique_ptr reader; - const Status s = BlobFileReader::Create(immutable_cf_options, FileOptions(), - column_family_id, blob_file_read_hist, - blob_file_number, &reader); + const Status s = BlobFileReader::Create( + immutable_cf_options, FileOptions(), column_family_id, + blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader); const bool fail_during_create = sync_point_ != "BlobFileReader::GetBlob:TamperWithResult"; diff --git a/db/blob/db_blob_basic_test.cc b/db/blob/db_blob_basic_test.cc index c1ac3e79f..61555e0a6 100644 --- a/db/blob/db_blob_basic_test.cc +++ b/db/blob/db_blob_basic_test.cc @@ -210,6 +210,63 @@ TEST_F(DBBlobBasicTest, GetBlob_IndexWithInvalidFileNumber) { .IsCorruption()); } +#ifndef ROCKSDB_LITE +TEST_F(DBBlobBasicTest, GenerateIOTracing) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.min_blob_size = 0; + std::string trace_file = dbname_ + "/io_trace_file"; + + Reopen(options); + { + // Create IO trace file + std::unique_ptr trace_writer; + ASSERT_OK( + NewFileTraceWriter(env_, EnvOptions(), trace_file, &trace_writer)); + ASSERT_OK(db_->StartIOTrace(env_, TraceOptions(), std::move(trace_writer))); + + constexpr char key[] = "key"; + constexpr char blob_value[] = "blob_value"; + + ASSERT_OK(Put(key, blob_value)); + ASSERT_OK(Flush()); + ASSERT_EQ(Get(key), blob_value); + + ASSERT_OK(db_->EndIOTrace()); + ASSERT_OK(env_->FileExists(trace_file)); + } + { + // Parse trace file to check file opertions related to blob files are + // recorded. + std::unique_ptr trace_reader; + ASSERT_OK( + NewFileTraceReader(env_, EnvOptions(), trace_file, &trace_reader)); + IOTraceReader reader(std::move(trace_reader)); + + IOTraceHeader header; + ASSERT_OK(reader.ReadHeader(&header)); + ASSERT_EQ(kMajorVersion, static_cast(header.rocksdb_major_version)); + ASSERT_EQ(kMinorVersion, static_cast(header.rocksdb_minor_version)); + + // Read records. + int blob_files_op_count = 0; + Status status; + while (true) { + IOTraceRecord record; + status = reader.ReadIOOp(&record); + if (!status.ok()) { + break; + } + if (record.file_name.find("blob") != std::string::npos) { + blob_files_op_count++; + } + } + // Assuming blob files will have Append, Close and then Read operations. + ASSERT_GT(blob_files_op_count, 2); + } +} +#endif // !ROCKSDB_LITE + class DBBlobBasicIOErrorTest : public DBBlobBasicTest, public testing::WithParamInterface { protected: diff --git a/db/builder.cc b/db/builder.cc index 21ff3a800..3065aac04 100644 --- a/db/builder.cc +++ b/db/builder.cc @@ -179,8 +179,8 @@ Status BuildTable( ? new BlobFileBuilder(versions, env, fs, &ioptions, &mutable_cf_options, &file_options, job_id, column_family_id, column_family_name, - io_priority, write_hint, &blob_file_paths, - blob_file_additions) + io_priority, write_hint, io_tracer, + &blob_file_paths, blob_file_additions) : nullptr); CompactionIterator c_iter( diff --git a/db/column_family.cc b/db/column_family.cc index b4ea8355e..e0b70687b 100644 --- a/db/column_family.cc +++ b/db/column_family.cc @@ -562,7 +562,7 @@ ColumnFamilyData::ColumnFamilyData( block_cache_tracer, io_tracer)); blob_file_cache_.reset( new BlobFileCache(_table_cache, ioptions(), soptions(), id_, - internal_stats_->GetBlobFileReadHist())); + internal_stats_->GetBlobFileReadHist(), io_tracer)); if (ioptions_.compaction_style == kCompactionStyleLevel) { compaction_picker_.reset( diff --git a/db/compaction/compaction_job.cc b/db/compaction/compaction_job.cc index 2fd14675e..31e6faf55 100644 --- a/db/compaction/compaction_job.cc +++ b/db/compaction/compaction_job.cc @@ -978,7 +978,7 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) { sub_compact->compaction->immutable_cf_options(), mutable_cf_options, &file_options_, job_id_, cfd->GetID(), cfd->GetName(), Env::IOPriority::IO_LOW, write_hint_, - &blob_file_paths, &sub_compact->blob_file_additions) + io_tracer_, &blob_file_paths, &sub_compact->blob_file_additions) : nullptr); TEST_SYNC_POINT("CompactionJob::Run():Inprogress");