Add support for IOTracing in blob files (#7958)

Summary:
Add support for IOTracing in blob files

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7958

Test Plan:
Add a new test and checked manually the trace_file for blob
files being recorded during read and write.

Reviewed By: ltamasi

Differential Revision: D26415950

Pulled By: akankshamahajan15

fbshipit-source-id: 49c2859b3a4f8307e7cb69a92704403a4da46d44

db/blob/blob_file_builder.cc

@@ -21,6 +21,7 @@
 #include "rocksdb/slice.h"
 #include "rocksdb/status.h"
 #include "test_util/sync_point.h"
+#include "trace_replay/io_tracer.h"
 #include "util/compression.h"
 
 namespace ROCKSDB_NAMESPACE {
@@ -32,12 +33,13 @@ BlobFileBuilder::BlobFileBuilder(
     int job_id, uint32_t column_family_id,
     const std::string& column_family_name, Env::IOPriority io_priority,
     Env::WriteLifeTimeHint write_hint,
+    const std::shared_ptr<IOTracer>& io_tracer,
     std::vector<std::string>* blob_file_paths,
     std::vector<BlobFileAddition>* blob_file_additions)
     : BlobFileBuilder([versions]() { return versions->NewFileNumber(); }, env,
                       fs, immutable_cf_options, mutable_cf_options,
                       file_options, job_id, column_family_id,
-                      column_family_name, io_priority, write_hint,
+                      column_family_name, io_priority, write_hint, io_tracer,
                       blob_file_paths, blob_file_additions) {}
 
 BlobFileBuilder::BlobFileBuilder(
@@ -47,6 +49,7 @@ BlobFileBuilder::BlobFileBuilder(
     int job_id, uint32_t column_family_id,
     const std::string& column_family_name, Env::IOPriority io_priority,
     Env::WriteLifeTimeHint write_hint,
+    const std::shared_ptr<IOTracer>& io_tracer,
     std::vector<std::string>* blob_file_paths,
     std::vector<BlobFileAddition>* blob_file_additions)
     : file_number_generator_(std::move(file_number_generator)),
@@ -61,6 +64,7 @@ BlobFileBuilder::BlobFileBuilder(
       column_family_name_(column_family_name),
       io_priority_(io_priority),
       write_hint_(write_hint),
+      io_tracer_(io_tracer),
       blob_file_paths_(blob_file_paths),
       blob_file_additions_(blob_file_additions),
       blob_count_(0),
@@ -182,7 +186,7 @@ Status BlobFileBuilder::OpenBlobFileIfNeeded() {
   Statistics* const statistics = immutable_cf_options_->statistics;
   std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
       std::move(file), blob_file_paths_->back(), *file_options_, clock_,
-      nullptr /*IOTracer*/, statistics, immutable_cf_options_->listeners,
+      io_tracer_, statistics, immutable_cf_options_->listeners,
       immutable_cf_options_->file_checksum_gen_factory,
       tmp_set.Contains(FileType::kBlobFile)));
 

db/blob/blob_file_builder.h

@@ -26,6 +26,7 @@ class BlobFileAddition;
 class Status;
 class Slice;
 class BlobLogWriter;
+class IOTracer;
 
 class BlobFileBuilder {
  public:
@@ -37,6 +38,7 @@ class BlobFileBuilder {
                   const std::string& column_family_name,
                   Env::IOPriority io_priority,
                   Env::WriteLifeTimeHint write_hint,
+                  const std::shared_ptr<IOTracer>& io_tracer,
                   std::vector<std::string>* blob_file_paths,
                   std::vector<BlobFileAddition>* blob_file_additions);
 
@@ -49,6 +51,7 @@ class BlobFileBuilder {
                   const std::string& column_family_name,
                   Env::IOPriority io_priority,
                   Env::WriteLifeTimeHint write_hint,
+                  const std::shared_ptr<IOTracer>& io_tracer,
                   std::vector<std::string>* blob_file_paths,
                   std::vector<BlobFileAddition>* blob_file_additions);
 
@@ -82,6 +85,7 @@ class BlobFileBuilder {
   std::string column_family_name_;
   Env::IOPriority io_priority_;
   Env::WriteLifeTimeHint write_hint_;
+  std::shared_ptr<IOTracer> io_tracer_;
   std::vector<std::string>* blob_file_paths_;
   std::vector<BlobFileAddition>* blob_file_additions_;
   std::unique_ptr<BlobLogWriter> writer_;

db/blob/blob_file_builder_test.cc

@@ -140,10 +140,10 @@ TEST_F(BlobFileBuilderTest, BuildAndCheckOneFile) {
   std::vector<std::string> blob_file_paths;
   std::vector<BlobFileAddition> blob_file_additions;
 
-  BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_,
-                          &immutable_cf_options, &mutable_cf_options,
-                          &file_options_, job_id, column_family_id,
-                          column_family_name, io_priority, write_hint,
+  BlobFileBuilder builder(
+      TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options,
+      &mutable_cf_options, &file_options_, job_id, column_family_id,
+      column_family_name, io_priority, write_hint, nullptr /*IOTracer*/,
       &blob_file_paths, &blob_file_additions);
 
   std::vector<std::pair<std::string, std::string>> expected_key_value_pairs(
@@ -223,10 +223,10 @@ TEST_F(BlobFileBuilderTest, BuildAndCheckMultipleFiles) {
   std::vector<std::string> blob_file_paths;
   std::vector<BlobFileAddition> blob_file_additions;
 
-  BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_,
-                          &immutable_cf_options, &mutable_cf_options,
-                          &file_options_, job_id, column_family_id,
-                          column_family_name, io_priority, write_hint,
+  BlobFileBuilder builder(
+      TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options,
+      &mutable_cf_options, &file_options_, job_id, column_family_id,
+      column_family_name, io_priority, write_hint, nullptr /*IOTracer*/,
       &blob_file_paths, &blob_file_additions);
 
   std::vector<std::pair<std::string, std::string>> expected_key_value_pairs(
@@ -308,10 +308,10 @@ TEST_F(BlobFileBuilderTest, InlinedValues) {
   std::vector<std::string> blob_file_paths;
   std::vector<BlobFileAddition> blob_file_additions;
 
-  BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_,
-                          &immutable_cf_options, &mutable_cf_options,
-                          &file_options_, job_id, column_family_id,
-                          column_family_name, io_priority, write_hint,
+  BlobFileBuilder builder(
+      TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options,
+      &mutable_cf_options, &file_options_, job_id, column_family_id,
+      column_family_name, io_priority, write_hint, nullptr /*IOTracer*/,
       &blob_file_paths, &blob_file_additions);
 
   for (size_t i = 0; i < number_of_blobs; ++i) {
@@ -360,10 +360,10 @@ TEST_F(BlobFileBuilderTest, Compression) {
   std::vector<std::string> blob_file_paths;
   std::vector<BlobFileAddition> blob_file_additions;
 
-  BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_,
-                          &immutable_cf_options, &mutable_cf_options,
-                          &file_options_, job_id, column_family_id,
-                          column_family_name, io_priority, write_hint,
+  BlobFileBuilder builder(
+      TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options,
+      &mutable_cf_options, &file_options_, job_id, column_family_id,
+      column_family_name, io_priority, write_hint, nullptr /*IOTracer*/,
       &blob_file_paths, &blob_file_additions);
 
   const std::string key("1");
@@ -442,10 +442,10 @@ TEST_F(BlobFileBuilderTest, CompressionError) {
   std::vector<std::string> blob_file_paths;
   std::vector<BlobFileAddition> blob_file_additions;
 
-  BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_,
-                          &immutable_cf_options, &mutable_cf_options,
-                          &file_options_, job_id, column_family_id,
-                          column_family_name, io_priority, write_hint,
+  BlobFileBuilder builder(
+      TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options,
+      &mutable_cf_options, &file_options_, job_id, column_family_id,
+      column_family_name, io_priority, write_hint, nullptr /*IOTracer*/,
       &blob_file_paths, &blob_file_additions);
 
   SyncPoint::GetInstance()->SetCallBack("CompressData:TamperWithReturnValue",
@@ -519,10 +519,10 @@ TEST_F(BlobFileBuilderTest, Checksum) {
   std::vector<std::string> blob_file_paths;
   std::vector<BlobFileAddition> blob_file_additions;
 
-  BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_,
-                          &immutable_cf_options, &mutable_cf_options,
-                          &file_options_, job_id, column_family_id,
-                          column_family_name, io_priority, write_hint,
+  BlobFileBuilder builder(
+      TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options,
+      &mutable_cf_options, &file_options_, job_id, column_family_id,
+      column_family_name, io_priority, write_hint, nullptr /*IOTracer*/,
       &blob_file_paths, &blob_file_additions);
 
   const std::string key("1");
@@ -615,10 +615,10 @@ TEST_P(BlobFileBuilderIOErrorTest, IOError) {
   std::vector<std::string> blob_file_paths;
   std::vector<BlobFileAddition> blob_file_additions;
 
-  BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_,
-                          &immutable_cf_options, &mutable_cf_options,
-                          &file_options_, job_id, column_family_id,
-                          column_family_name, io_priority, write_hint,
+  BlobFileBuilder builder(
+      TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options,
+      &mutable_cf_options, &file_options_, job_id, column_family_id,
+      column_family_name, io_priority, write_hint, nullptr /*IOTracer*/,
       &blob_file_paths, &blob_file_additions);
 
   SyncPoint::GetInstance()->SetCallBack(sync_point_, [this](void* arg) {

db/blob/blob_file_cache.cc

@@ -13,6 +13,7 @@
 #include "rocksdb/cache.h"
 #include "rocksdb/slice.h"
 #include "test_util/sync_point.h"
+#include "trace_replay/io_tracer.h"
 #include "util/hash.h"
 
 namespace ROCKSDB_NAMESPACE {
@@ -21,13 +22,15 @@ BlobFileCache::BlobFileCache(Cache* cache,
                              const ImmutableCFOptions* immutable_cf_options,
                              const FileOptions* file_options,
                              uint32_t column_family_id,
-                             HistogramImpl* blob_file_read_hist)
+                             HistogramImpl* blob_file_read_hist,
+                             const std::shared_ptr<IOTracer>& io_tracer)
     : cache_(cache),
       mutex_(kNumberOfMutexStripes, kGetSliceNPHash64UnseededFnPtr),
       immutable_cf_options_(immutable_cf_options),
       file_options_(file_options),
       column_family_id_(column_family_id),
-      blob_file_read_hist_(blob_file_read_hist) {
+      blob_file_read_hist_(blob_file_read_hist),
+      io_tracer_(io_tracer) {
   assert(cache_);
   assert(immutable_cf_options_);
   assert(file_options_);
@@ -71,7 +74,7 @@ Status BlobFileCache::GetBlobFileReader(
     assert(file_options_);
     const Status s = BlobFileReader::Create(
         *immutable_cf_options_, *file_options_, column_family_id_,
-        blob_file_read_hist_, blob_file_number, &reader);
+        blob_file_read_hist_, blob_file_number, io_tracer_, &reader);
     if (!s.ok()) {
       RecordTick(statistics, NO_FILE_ERRORS);
       return s;

db/blob/blob_file_cache.h

@@ -20,12 +20,14 @@ class HistogramImpl;
 class Status;
 class BlobFileReader;
 class Slice;
+class IOTracer;
 
 class BlobFileCache {
  public:
   BlobFileCache(Cache* cache, const ImmutableCFOptions* immutable_cf_options,
                 const FileOptions* file_options, uint32_t column_family_id,
-                HistogramImpl* blob_file_read_hist);
+                HistogramImpl* blob_file_read_hist,
+                const std::shared_ptr<IOTracer>& io_tracer);
 
   BlobFileCache(const BlobFileCache&) = delete;
   BlobFileCache& operator=(const BlobFileCache&) = delete;
@@ -42,6 +44,7 @@ class BlobFileCache {
   const FileOptions* file_options_;
   uint32_t column_family_id_;
   HistogramImpl* blob_file_read_hist_;
+  std::shared_ptr<IOTracer> io_tracer_;
 
   static constexpr size_t kNumberOfMutexStripes = 1 << 7;
 };

db/blob/blob_file_cache_test.cc

@@ -114,7 +114,7 @@ TEST_F(BlobFileCacheTest, GetBlobFileReader) {
 
   BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options,
                                 &file_options, column_family_id,
-                                blob_file_read_hist);
+                                blob_file_read_hist, nullptr /*IOTracer*/);
 
   // First try: reader should be opened and put in cache
   CacheHandleGuard<BlobFileReader> first;
@@ -159,7 +159,7 @@ TEST_F(BlobFileCacheTest, GetBlobFileReader_Race) {
 
   BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options,
                                 &file_options, column_family_id,
-                                blob_file_read_hist);
+                                blob_file_read_hist, nullptr /*IOTracer*/);
 
   CacheHandleGuard<BlobFileReader> first;
   CacheHandleGuard<BlobFileReader> second;
@@ -207,7 +207,7 @@ TEST_F(BlobFileCacheTest, GetBlobFileReader_IOError) {
 
   BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options,
                                 &file_options, column_family_id,
-                                blob_file_read_hist);
+                                blob_file_read_hist, nullptr /*IOTracer*/);
 
   // Note: there is no blob file with the below number
   constexpr uint64_t blob_file_number = 123;
@@ -248,7 +248,7 @@ TEST_F(BlobFileCacheTest, GetBlobFileReader_CacheFull) {
 
   BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options,
                                 &file_options, column_family_id,
-                                blob_file_read_hist);
+                                blob_file_read_hist, nullptr /*IOTracer*/);
 
   // Insert into cache should fail since it has zero capacity and
   // strict_capacity_limit is set

db/blob/blob_file_reader.cc

@@ -24,6 +24,7 @@ Status BlobFileReader::Create(
     const ImmutableCFOptions& immutable_cf_options,
     const FileOptions& file_options, uint32_t column_family_id,
     HistogramImpl* blob_file_read_hist, uint64_t blob_file_number,
+    const std::shared_ptr<IOTracer>& io_tracer,
     std::unique_ptr<BlobFileReader>* blob_file_reader) {
   assert(blob_file_reader);
   assert(!*blob_file_reader);
@@ -34,7 +35,7 @@ Status BlobFileReader::Create(
   {
     const Status s =
         OpenFile(immutable_cf_options, file_options, blob_file_read_hist,
-                 blob_file_number, &file_size, &file_reader);
+                 blob_file_number, io_tracer, &file_size, &file_reader);
     if (!s.ok()) {
       return s;
     }
@@ -68,8 +69,8 @@ Status BlobFileReader::Create(
 Status BlobFileReader::OpenFile(
     const ImmutableCFOptions& immutable_cf_options,
     const FileOptions& file_opts, HistogramImpl* blob_file_read_hist,
-    uint64_t blob_file_number, uint64_t* file_size,
-    std::unique_ptr<RandomAccessFileReader>* file_reader) {
+    uint64_t blob_file_number, const std::shared_ptr<IOTracer>& io_tracer,
+    uint64_t* file_size, std::unique_ptr<RandomAccessFileReader>* file_reader) {
   assert(file_size);
   assert(file_reader);
 
@@ -118,7 +119,7 @@ Status BlobFileReader::OpenFile(
 
   file_reader->reset(new RandomAccessFileReader(
       std::move(file), blob_file_path,
-      immutable_cf_options.env->GetSystemClock(), std::shared_ptr<IOTracer>(),
+      immutable_cf_options.env->GetSystemClock(), io_tracer,
       immutable_cf_options.statistics, BLOB_DB_BLOB_FILE_READ_MICROS,
       blob_file_read_hist, immutable_cf_options.rate_limiter,
       immutable_cf_options.listeners));

db/blob/blob_file_reader.h

@@ -29,6 +29,7 @@ class BlobFileReader {
                        uint32_t column_family_id,
                        HistogramImpl* blob_file_read_hist,
                        uint64_t blob_file_number,
+                       const std::shared_ptr<IOTracer>& io_tracer,
                        std::unique_ptr<BlobFileReader>* reader);
 
   BlobFileReader(const BlobFileReader&) = delete;
@@ -47,7 +48,9 @@ class BlobFileReader {
   static Status OpenFile(const ImmutableCFOptions& immutable_cf_options,
                          const FileOptions& file_opts,
                          HistogramImpl* blob_file_read_hist,
-                         uint64_t blob_file_number, uint64_t* file_size,
+                         uint64_t blob_file_number,
+                         const std::shared_ptr<IOTracer>& io_tracer,
+                         uint64_t* file_size,
                          std::unique_ptr<RandomAccessFileReader>* file_reader);
 
   static Status ReadHeader(const RandomAccessFileReader* file_reader,

db/blob/blob_file_reader_test.cc

@@ -142,9 +142,9 @@ TEST_F(BlobFileReaderTest, CreateReaderAndGetBlob) {
 
   std::unique_ptr<BlobFileReader> reader;
 
-  ASSERT_OK(BlobFileReader::Create(immutable_cf_options, FileOptions(),
-                                   column_family_id, blob_file_read_hist,
-                                   blob_file_number, &reader));
+  ASSERT_OK(BlobFileReader::Create(
+      immutable_cf_options, FileOptions(), column_family_id,
+      blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader));
 
   // Make sure the blob can be retrieved with and without checksum verification
   ReadOptions read_options;
@@ -282,7 +282,8 @@ TEST_F(BlobFileReaderTest, Malformed) {
 
   ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(),
                                      column_family_id, blob_file_read_hist,
-                                     blob_file_number, &reader)
+                                     blob_file_number, nullptr /*IOTracer*/,
+                                     &reader)
                   .IsCorruption());
 }
 
@@ -315,7 +316,8 @@ TEST_F(BlobFileReaderTest, TTL) {
 
   ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(),
                                      column_family_id, blob_file_read_hist,
-                                     blob_file_number, &reader)
+                                     blob_file_number, nullptr /*IOTracer*/,
+                                     &reader)
                   .IsCorruption());
 }
 
@@ -353,7 +355,8 @@ TEST_F(BlobFileReaderTest, ExpirationRangeInHeader) {
 
   ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(),
                                      column_family_id, blob_file_read_hist,
-                                     blob_file_number, &reader)
+                                     blob_file_number, nullptr /*IOTracer*/,
+                                     &reader)
                   .IsCorruption());
 }
 
@@ -391,7 +394,8 @@ TEST_F(BlobFileReaderTest, ExpirationRangeInFooter) {
 
   ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(),
                                      column_family_id, blob_file_read_hist,
-                                     blob_file_number, &reader)
+                                     blob_file_number, nullptr /*IOTracer*/,
+                                     &reader)
                   .IsCorruption());
 }
 
@@ -429,7 +433,7 @@ TEST_F(BlobFileReaderTest, IncorrectColumnFamily) {
   ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(),
                                      incorrect_column_family_id,
                                      blob_file_read_hist, blob_file_number,
-                                     &reader)
+                                     nullptr /*IOTracer*/, &reader)
                   .IsCorruption());
 }
 
@@ -460,9 +464,9 @@ TEST_F(BlobFileReaderTest, BlobCRCError) {
 
   std::unique_ptr<BlobFileReader> reader;
 
-  ASSERT_OK(BlobFileReader::Create(immutable_cf_options, FileOptions(),
-                                   column_family_id, blob_file_read_hist,
-                                   blob_file_number, &reader));
+  ASSERT_OK(BlobFileReader::Create(
+      immutable_cf_options, FileOptions(), column_family_id,
+      blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader));
 
   SyncPoint::GetInstance()->SetCallBack(
       "BlobFileReader::VerifyBlob:CheckBlobCRC", [](void* arg) {
@@ -516,9 +520,9 @@ TEST_F(BlobFileReaderTest, Compression) {
 
   std::unique_ptr<BlobFileReader> reader;
 
-  ASSERT_OK(BlobFileReader::Create(immutable_cf_options, FileOptions(),
-                                   column_family_id, blob_file_read_hist,
-                                   blob_file_number, &reader));
+  ASSERT_OK(BlobFileReader::Create(
+      immutable_cf_options, FileOptions(), column_family_id,
+      blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader));
 
   // Make sure the blob can be retrieved with and without checksum verification
   ReadOptions read_options;
@@ -576,9 +580,9 @@ TEST_F(BlobFileReaderTest, UncompressionError) {
 
   std::unique_ptr<BlobFileReader> reader;
 
-  ASSERT_OK(BlobFileReader::Create(immutable_cf_options, FileOptions(),
-                                   column_family_id, blob_file_read_hist,
-                                   blob_file_number, &reader));
+  ASSERT_OK(BlobFileReader::Create(
+      immutable_cf_options, FileOptions(), column_family_id,
+      blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader));
 
   SyncPoint::GetInstance()->SetCallBack(
       "BlobFileReader::UncompressBlobIfNeeded:TamperWithResult", [](void* arg) {
@@ -661,9 +665,9 @@ TEST_P(BlobFileReaderIOErrorTest, IOError) {
 
   std::unique_ptr<BlobFileReader> reader;
 
-  const Status s = BlobFileReader::Create(immutable_cf_options, FileOptions(),
-                                          column_family_id, blob_file_read_hist,
-                                          blob_file_number, &reader);
+  const Status s = BlobFileReader::Create(
+      immutable_cf_options, FileOptions(), column_family_id,
+      blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader);
 
   const bool fail_during_create =
       (sync_point_ != "BlobFileReader::GetBlob:ReadFromFile");
@@ -741,9 +745,9 @@ TEST_P(BlobFileReaderDecodingErrorTest, DecodingError) {
 
   std::unique_ptr<BlobFileReader> reader;
 
-  const Status s = BlobFileReader::Create(immutable_cf_options, FileOptions(),
-                                          column_family_id, blob_file_read_hist,
-                                          blob_file_number, &reader);
+  const Status s = BlobFileReader::Create(
+      immutable_cf_options, FileOptions(), column_family_id,
+      blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader);
 
   const bool fail_during_create =
       sync_point_ != "BlobFileReader::GetBlob:TamperWithResult";

db/blob/db_blob_basic_test.cc

@@ -210,6 +210,63 @@ TEST_F(DBBlobBasicTest, GetBlob_IndexWithInvalidFileNumber) {
                   .IsCorruption());
 }
 
+#ifndef ROCKSDB_LITE
+TEST_F(DBBlobBasicTest, GenerateIOTracing) {
+  Options options = GetDefaultOptions();
+  options.enable_blob_files = true;
+  options.min_blob_size = 0;
+  std::string trace_file = dbname_ + "/io_trace_file";
+
+  Reopen(options);
+  {
+    // Create IO trace file
+    std::unique_ptr<TraceWriter> trace_writer;
+    ASSERT_OK(
+        NewFileTraceWriter(env_, EnvOptions(), trace_file, &trace_writer));
+    ASSERT_OK(db_->StartIOTrace(env_, TraceOptions(), std::move(trace_writer)));
+
+    constexpr char key[] = "key";
+    constexpr char blob_value[] = "blob_value";
+
+    ASSERT_OK(Put(key, blob_value));
+    ASSERT_OK(Flush());
+    ASSERT_EQ(Get(key), blob_value);
+
+    ASSERT_OK(db_->EndIOTrace());
+    ASSERT_OK(env_->FileExists(trace_file));
+  }
+  {
+    // Parse trace file to check file opertions related to blob files are
+    // recorded.
+    std::unique_ptr<TraceReader> trace_reader;
+    ASSERT_OK(
+        NewFileTraceReader(env_, EnvOptions(), trace_file, &trace_reader));
+    IOTraceReader reader(std::move(trace_reader));
+
+    IOTraceHeader header;
+    ASSERT_OK(reader.ReadHeader(&header));
+    ASSERT_EQ(kMajorVersion, static_cast<int>(header.rocksdb_major_version));
+    ASSERT_EQ(kMinorVersion, static_cast<int>(header.rocksdb_minor_version));
+
+    // Read records.
+    int blob_files_op_count = 0;
+    Status status;
+    while (true) {
+      IOTraceRecord record;
+      status = reader.ReadIOOp(&record);
+      if (!status.ok()) {
+        break;
+      }
+      if (record.file_name.find("blob") != std::string::npos) {
+        blob_files_op_count++;
+      }
+    }
+    // Assuming blob files will have Append, Close and then Read operations.
+    ASSERT_GT(blob_files_op_count, 2);
+  }
+}
+#endif  // !ROCKSDB_LITE
+
 class DBBlobBasicIOErrorTest : public DBBlobBasicTest,
                                public testing::WithParamInterface<std::string> {
  protected:

db/builder.cc

@@ -179,8 +179,8 @@ Status BuildTable(
             ? new BlobFileBuilder(versions, env, fs, &ioptions,
                                   &mutable_cf_options, &file_options, job_id,
                                   column_family_id, column_family_name,
-                                  io_priority, write_hint, &blob_file_paths,
-                                  blob_file_additions)
+                                  io_priority, write_hint, io_tracer,
+                                  &blob_file_paths, blob_file_additions)
             : nullptr);
 
     CompactionIterator c_iter(

db/column_family.cc

@@ -562,7 +562,7 @@ ColumnFamilyData::ColumnFamilyData(
                                       block_cache_tracer, io_tracer));
     blob_file_cache_.reset(
         new BlobFileCache(_table_cache, ioptions(), soptions(), id_,
-                          internal_stats_->GetBlobFileReadHist()));
+                          internal_stats_->GetBlobFileReadHist(), io_tracer));
 
     if (ioptions_.compaction_style == kCompactionStyleLevel) {
       compaction_picker_.reset(

db/compaction/compaction_job.cc

@@ -978,7 +978,7 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
                 sub_compact->compaction->immutable_cf_options(),
                 mutable_cf_options, &file_options_, job_id_, cfd->GetID(),
                 cfd->GetName(), Env::IOPriority::IO_LOW, write_hint_,
-                &blob_file_paths, &sub_compact->blob_file_additions)
+                io_tracer_, &blob_file_paths, &sub_compact->blob_file_additions)
           : nullptr);
 
   TEST_SYNC_POINT("CompactionJob::Run():Inprogress");