Add support for IOTracing in blob files (#7958)

Summary:
Add support for IOTracing in blob files

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7958

Test Plan:
Add a new test and checked manually the trace_file for blob
files being recorded during read and write.

Reviewed By: ltamasi

Differential Revision: D26415950

Pulled By: akankshamahajan15

fbshipit-source-id: 49c2859b3a4f8307e7cb69a92704403a4da46d44
main
Akanksha Mahajan 4 years ago committed by Facebook GitHub Bot
parent 9df78a94f1
commit ea8bb82fc7
  1. 8
      db/blob/blob_file_builder.cc
  2. 4
      db/blob/blob_file_builder.h
  3. 70
      db/blob/blob_file_builder_test.cc
  4. 9
      db/blob/blob_file_cache.cc
  5. 5
      db/blob/blob_file_cache.h
  6. 8
      db/blob/blob_file_cache_test.cc
  7. 9
      db/blob/blob_file_reader.cc
  8. 5
      db/blob/blob_file_reader.h
  9. 50
      db/blob/blob_file_reader_test.cc
  10. 57
      db/blob/db_blob_basic_test.cc
  11. 4
      db/builder.cc
  12. 2
      db/column_family.cc
  13. 2
      db/compaction/compaction_job.cc

@ -21,6 +21,7 @@
#include "rocksdb/slice.h" #include "rocksdb/slice.h"
#include "rocksdb/status.h" #include "rocksdb/status.h"
#include "test_util/sync_point.h" #include "test_util/sync_point.h"
#include "trace_replay/io_tracer.h"
#include "util/compression.h" #include "util/compression.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
@ -32,12 +33,13 @@ BlobFileBuilder::BlobFileBuilder(
int job_id, uint32_t column_family_id, int job_id, uint32_t column_family_id,
const std::string& column_family_name, Env::IOPriority io_priority, const std::string& column_family_name, Env::IOPriority io_priority,
Env::WriteLifeTimeHint write_hint, Env::WriteLifeTimeHint write_hint,
const std::shared_ptr<IOTracer>& io_tracer,
std::vector<std::string>* blob_file_paths, std::vector<std::string>* blob_file_paths,
std::vector<BlobFileAddition>* blob_file_additions) std::vector<BlobFileAddition>* blob_file_additions)
: BlobFileBuilder([versions]() { return versions->NewFileNumber(); }, env, : BlobFileBuilder([versions]() { return versions->NewFileNumber(); }, env,
fs, immutable_cf_options, mutable_cf_options, fs, immutable_cf_options, mutable_cf_options,
file_options, job_id, column_family_id, file_options, job_id, column_family_id,
column_family_name, io_priority, write_hint, column_family_name, io_priority, write_hint, io_tracer,
blob_file_paths, blob_file_additions) {} blob_file_paths, blob_file_additions) {}
BlobFileBuilder::BlobFileBuilder( BlobFileBuilder::BlobFileBuilder(
@ -47,6 +49,7 @@ BlobFileBuilder::BlobFileBuilder(
int job_id, uint32_t column_family_id, int job_id, uint32_t column_family_id,
const std::string& column_family_name, Env::IOPriority io_priority, const std::string& column_family_name, Env::IOPriority io_priority,
Env::WriteLifeTimeHint write_hint, Env::WriteLifeTimeHint write_hint,
const std::shared_ptr<IOTracer>& io_tracer,
std::vector<std::string>* blob_file_paths, std::vector<std::string>* blob_file_paths,
std::vector<BlobFileAddition>* blob_file_additions) std::vector<BlobFileAddition>* blob_file_additions)
: file_number_generator_(std::move(file_number_generator)), : file_number_generator_(std::move(file_number_generator)),
@ -61,6 +64,7 @@ BlobFileBuilder::BlobFileBuilder(
column_family_name_(column_family_name), column_family_name_(column_family_name),
io_priority_(io_priority), io_priority_(io_priority),
write_hint_(write_hint), write_hint_(write_hint),
io_tracer_(io_tracer),
blob_file_paths_(blob_file_paths), blob_file_paths_(blob_file_paths),
blob_file_additions_(blob_file_additions), blob_file_additions_(blob_file_additions),
blob_count_(0), blob_count_(0),
@ -182,7 +186,7 @@ Status BlobFileBuilder::OpenBlobFileIfNeeded() {
Statistics* const statistics = immutable_cf_options_->statistics; Statistics* const statistics = immutable_cf_options_->statistics;
std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter( std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
std::move(file), blob_file_paths_->back(), *file_options_, clock_, std::move(file), blob_file_paths_->back(), *file_options_, clock_,
nullptr /*IOTracer*/, statistics, immutable_cf_options_->listeners, io_tracer_, statistics, immutable_cf_options_->listeners,
immutable_cf_options_->file_checksum_gen_factory, immutable_cf_options_->file_checksum_gen_factory,
tmp_set.Contains(FileType::kBlobFile))); tmp_set.Contains(FileType::kBlobFile)));

@ -26,6 +26,7 @@ class BlobFileAddition;
class Status; class Status;
class Slice; class Slice;
class BlobLogWriter; class BlobLogWriter;
class IOTracer;
class BlobFileBuilder { class BlobFileBuilder {
public: public:
@ -37,6 +38,7 @@ class BlobFileBuilder {
const std::string& column_family_name, const std::string& column_family_name,
Env::IOPriority io_priority, Env::IOPriority io_priority,
Env::WriteLifeTimeHint write_hint, Env::WriteLifeTimeHint write_hint,
const std::shared_ptr<IOTracer>& io_tracer,
std::vector<std::string>* blob_file_paths, std::vector<std::string>* blob_file_paths,
std::vector<BlobFileAddition>* blob_file_additions); std::vector<BlobFileAddition>* blob_file_additions);
@ -49,6 +51,7 @@ class BlobFileBuilder {
const std::string& column_family_name, const std::string& column_family_name,
Env::IOPriority io_priority, Env::IOPriority io_priority,
Env::WriteLifeTimeHint write_hint, Env::WriteLifeTimeHint write_hint,
const std::shared_ptr<IOTracer>& io_tracer,
std::vector<std::string>* blob_file_paths, std::vector<std::string>* blob_file_paths,
std::vector<BlobFileAddition>* blob_file_additions); std::vector<BlobFileAddition>* blob_file_additions);
@ -82,6 +85,7 @@ class BlobFileBuilder {
std::string column_family_name_; std::string column_family_name_;
Env::IOPriority io_priority_; Env::IOPriority io_priority_;
Env::WriteLifeTimeHint write_hint_; Env::WriteLifeTimeHint write_hint_;
std::shared_ptr<IOTracer> io_tracer_;
std::vector<std::string>* blob_file_paths_; std::vector<std::string>* blob_file_paths_;
std::vector<BlobFileAddition>* blob_file_additions_; std::vector<BlobFileAddition>* blob_file_additions_;
std::unique_ptr<BlobLogWriter> writer_; std::unique_ptr<BlobLogWriter> writer_;

@ -140,11 +140,11 @@ TEST_F(BlobFileBuilderTest, BuildAndCheckOneFile) {
std::vector<std::string> blob_file_paths; std::vector<std::string> blob_file_paths;
std::vector<BlobFileAddition> blob_file_additions; std::vector<BlobFileAddition> blob_file_additions;
BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_, BlobFileBuilder builder(
&immutable_cf_options, &mutable_cf_options, TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options,
&file_options_, job_id, column_family_id, &mutable_cf_options, &file_options_, job_id, column_family_id,
column_family_name, io_priority, write_hint, column_family_name, io_priority, write_hint, nullptr /*IOTracer*/,
&blob_file_paths, &blob_file_additions); &blob_file_paths, &blob_file_additions);
std::vector<std::pair<std::string, std::string>> expected_key_value_pairs( std::vector<std::pair<std::string, std::string>> expected_key_value_pairs(
number_of_blobs); number_of_blobs);
@ -223,11 +223,11 @@ TEST_F(BlobFileBuilderTest, BuildAndCheckMultipleFiles) {
std::vector<std::string> blob_file_paths; std::vector<std::string> blob_file_paths;
std::vector<BlobFileAddition> blob_file_additions; std::vector<BlobFileAddition> blob_file_additions;
BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_, BlobFileBuilder builder(
&immutable_cf_options, &mutable_cf_options, TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options,
&file_options_, job_id, column_family_id, &mutable_cf_options, &file_options_, job_id, column_family_id,
column_family_name, io_priority, write_hint, column_family_name, io_priority, write_hint, nullptr /*IOTracer*/,
&blob_file_paths, &blob_file_additions); &blob_file_paths, &blob_file_additions);
std::vector<std::pair<std::string, std::string>> expected_key_value_pairs( std::vector<std::pair<std::string, std::string>> expected_key_value_pairs(
number_of_blobs); number_of_blobs);
@ -308,11 +308,11 @@ TEST_F(BlobFileBuilderTest, InlinedValues) {
std::vector<std::string> blob_file_paths; std::vector<std::string> blob_file_paths;
std::vector<BlobFileAddition> blob_file_additions; std::vector<BlobFileAddition> blob_file_additions;
BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_, BlobFileBuilder builder(
&immutable_cf_options, &mutable_cf_options, TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options,
&file_options_, job_id, column_family_id, &mutable_cf_options, &file_options_, job_id, column_family_id,
column_family_name, io_priority, write_hint, column_family_name, io_priority, write_hint, nullptr /*IOTracer*/,
&blob_file_paths, &blob_file_additions); &blob_file_paths, &blob_file_additions);
for (size_t i = 0; i < number_of_blobs; ++i) { for (size_t i = 0; i < number_of_blobs; ++i) {
const std::string key = std::to_string(i); const std::string key = std::to_string(i);
@ -360,11 +360,11 @@ TEST_F(BlobFileBuilderTest, Compression) {
std::vector<std::string> blob_file_paths; std::vector<std::string> blob_file_paths;
std::vector<BlobFileAddition> blob_file_additions; std::vector<BlobFileAddition> blob_file_additions;
BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_, BlobFileBuilder builder(
&immutable_cf_options, &mutable_cf_options, TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options,
&file_options_, job_id, column_family_id, &mutable_cf_options, &file_options_, job_id, column_family_id,
column_family_name, io_priority, write_hint, column_family_name, io_priority, write_hint, nullptr /*IOTracer*/,
&blob_file_paths, &blob_file_additions); &blob_file_paths, &blob_file_additions);
const std::string key("1"); const std::string key("1");
const std::string uncompressed_value(value_size, 'x'); const std::string uncompressed_value(value_size, 'x');
@ -442,11 +442,11 @@ TEST_F(BlobFileBuilderTest, CompressionError) {
std::vector<std::string> blob_file_paths; std::vector<std::string> blob_file_paths;
std::vector<BlobFileAddition> blob_file_additions; std::vector<BlobFileAddition> blob_file_additions;
BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_, BlobFileBuilder builder(
&immutable_cf_options, &mutable_cf_options, TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options,
&file_options_, job_id, column_family_id, &mutable_cf_options, &file_options_, job_id, column_family_id,
column_family_name, io_priority, write_hint, column_family_name, io_priority, write_hint, nullptr /*IOTracer*/,
&blob_file_paths, &blob_file_additions); &blob_file_paths, &blob_file_additions);
SyncPoint::GetInstance()->SetCallBack("CompressData:TamperWithReturnValue", SyncPoint::GetInstance()->SetCallBack("CompressData:TamperWithReturnValue",
[](void* arg) { [](void* arg) {
@ -519,11 +519,11 @@ TEST_F(BlobFileBuilderTest, Checksum) {
std::vector<std::string> blob_file_paths; std::vector<std::string> blob_file_paths;
std::vector<BlobFileAddition> blob_file_additions; std::vector<BlobFileAddition> blob_file_additions;
BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_, BlobFileBuilder builder(
&immutable_cf_options, &mutable_cf_options, TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options,
&file_options_, job_id, column_family_id, &mutable_cf_options, &file_options_, job_id, column_family_id,
column_family_name, io_priority, write_hint, column_family_name, io_priority, write_hint, nullptr /*IOTracer*/,
&blob_file_paths, &blob_file_additions); &blob_file_paths, &blob_file_additions);
const std::string key("1"); const std::string key("1");
const std::string value("deadbeef"); const std::string value("deadbeef");
@ -615,11 +615,11 @@ TEST_P(BlobFileBuilderIOErrorTest, IOError) {
std::vector<std::string> blob_file_paths; std::vector<std::string> blob_file_paths;
std::vector<BlobFileAddition> blob_file_additions; std::vector<BlobFileAddition> blob_file_additions;
BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_, BlobFileBuilder builder(
&immutable_cf_options, &mutable_cf_options, TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options,
&file_options_, job_id, column_family_id, &mutable_cf_options, &file_options_, job_id, column_family_id,
column_family_name, io_priority, write_hint, column_family_name, io_priority, write_hint, nullptr /*IOTracer*/,
&blob_file_paths, &blob_file_additions); &blob_file_paths, &blob_file_additions);
SyncPoint::GetInstance()->SetCallBack(sync_point_, [this](void* arg) { SyncPoint::GetInstance()->SetCallBack(sync_point_, [this](void* arg) {
Status* const s = static_cast<Status*>(arg); Status* const s = static_cast<Status*>(arg);

@ -13,6 +13,7 @@
#include "rocksdb/cache.h" #include "rocksdb/cache.h"
#include "rocksdb/slice.h" #include "rocksdb/slice.h"
#include "test_util/sync_point.h" #include "test_util/sync_point.h"
#include "trace_replay/io_tracer.h"
#include "util/hash.h" #include "util/hash.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
@ -21,13 +22,15 @@ BlobFileCache::BlobFileCache(Cache* cache,
const ImmutableCFOptions* immutable_cf_options, const ImmutableCFOptions* immutable_cf_options,
const FileOptions* file_options, const FileOptions* file_options,
uint32_t column_family_id, uint32_t column_family_id,
HistogramImpl* blob_file_read_hist) HistogramImpl* blob_file_read_hist,
const std::shared_ptr<IOTracer>& io_tracer)
: cache_(cache), : cache_(cache),
mutex_(kNumberOfMutexStripes, kGetSliceNPHash64UnseededFnPtr), mutex_(kNumberOfMutexStripes, kGetSliceNPHash64UnseededFnPtr),
immutable_cf_options_(immutable_cf_options), immutable_cf_options_(immutable_cf_options),
file_options_(file_options), file_options_(file_options),
column_family_id_(column_family_id), column_family_id_(column_family_id),
blob_file_read_hist_(blob_file_read_hist) { blob_file_read_hist_(blob_file_read_hist),
io_tracer_(io_tracer) {
assert(cache_); assert(cache_);
assert(immutable_cf_options_); assert(immutable_cf_options_);
assert(file_options_); assert(file_options_);
@ -71,7 +74,7 @@ Status BlobFileCache::GetBlobFileReader(
assert(file_options_); assert(file_options_);
const Status s = BlobFileReader::Create( const Status s = BlobFileReader::Create(
*immutable_cf_options_, *file_options_, column_family_id_, *immutable_cf_options_, *file_options_, column_family_id_,
blob_file_read_hist_, blob_file_number, &reader); blob_file_read_hist_, blob_file_number, io_tracer_, &reader);
if (!s.ok()) { if (!s.ok()) {
RecordTick(statistics, NO_FILE_ERRORS); RecordTick(statistics, NO_FILE_ERRORS);
return s; return s;

@ -20,12 +20,14 @@ class HistogramImpl;
class Status; class Status;
class BlobFileReader; class BlobFileReader;
class Slice; class Slice;
class IOTracer;
class BlobFileCache { class BlobFileCache {
public: public:
BlobFileCache(Cache* cache, const ImmutableCFOptions* immutable_cf_options, BlobFileCache(Cache* cache, const ImmutableCFOptions* immutable_cf_options,
const FileOptions* file_options, uint32_t column_family_id, const FileOptions* file_options, uint32_t column_family_id,
HistogramImpl* blob_file_read_hist); HistogramImpl* blob_file_read_hist,
const std::shared_ptr<IOTracer>& io_tracer);
BlobFileCache(const BlobFileCache&) = delete; BlobFileCache(const BlobFileCache&) = delete;
BlobFileCache& operator=(const BlobFileCache&) = delete; BlobFileCache& operator=(const BlobFileCache&) = delete;
@ -42,6 +44,7 @@ class BlobFileCache {
const FileOptions* file_options_; const FileOptions* file_options_;
uint32_t column_family_id_; uint32_t column_family_id_;
HistogramImpl* blob_file_read_hist_; HistogramImpl* blob_file_read_hist_;
std::shared_ptr<IOTracer> io_tracer_;
static constexpr size_t kNumberOfMutexStripes = 1 << 7; static constexpr size_t kNumberOfMutexStripes = 1 << 7;
}; };

@ -114,7 +114,7 @@ TEST_F(BlobFileCacheTest, GetBlobFileReader) {
BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options, BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options,
&file_options, column_family_id, &file_options, column_family_id,
blob_file_read_hist); blob_file_read_hist, nullptr /*IOTracer*/);
// First try: reader should be opened and put in cache // First try: reader should be opened and put in cache
CacheHandleGuard<BlobFileReader> first; CacheHandleGuard<BlobFileReader> first;
@ -159,7 +159,7 @@ TEST_F(BlobFileCacheTest, GetBlobFileReader_Race) {
BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options, BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options,
&file_options, column_family_id, &file_options, column_family_id,
blob_file_read_hist); blob_file_read_hist, nullptr /*IOTracer*/);
CacheHandleGuard<BlobFileReader> first; CacheHandleGuard<BlobFileReader> first;
CacheHandleGuard<BlobFileReader> second; CacheHandleGuard<BlobFileReader> second;
@ -207,7 +207,7 @@ TEST_F(BlobFileCacheTest, GetBlobFileReader_IOError) {
BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options, BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options,
&file_options, column_family_id, &file_options, column_family_id,
blob_file_read_hist); blob_file_read_hist, nullptr /*IOTracer*/);
// Note: there is no blob file with the below number // Note: there is no blob file with the below number
constexpr uint64_t blob_file_number = 123; constexpr uint64_t blob_file_number = 123;
@ -248,7 +248,7 @@ TEST_F(BlobFileCacheTest, GetBlobFileReader_CacheFull) {
BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options, BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options,
&file_options, column_family_id, &file_options, column_family_id,
blob_file_read_hist); blob_file_read_hist, nullptr /*IOTracer*/);
// Insert into cache should fail since it has zero capacity and // Insert into cache should fail since it has zero capacity and
// strict_capacity_limit is set // strict_capacity_limit is set

@ -24,6 +24,7 @@ Status BlobFileReader::Create(
const ImmutableCFOptions& immutable_cf_options, const ImmutableCFOptions& immutable_cf_options,
const FileOptions& file_options, uint32_t column_family_id, const FileOptions& file_options, uint32_t column_family_id,
HistogramImpl* blob_file_read_hist, uint64_t blob_file_number, HistogramImpl* blob_file_read_hist, uint64_t blob_file_number,
const std::shared_ptr<IOTracer>& io_tracer,
std::unique_ptr<BlobFileReader>* blob_file_reader) { std::unique_ptr<BlobFileReader>* blob_file_reader) {
assert(blob_file_reader); assert(blob_file_reader);
assert(!*blob_file_reader); assert(!*blob_file_reader);
@ -34,7 +35,7 @@ Status BlobFileReader::Create(
{ {
const Status s = const Status s =
OpenFile(immutable_cf_options, file_options, blob_file_read_hist, OpenFile(immutable_cf_options, file_options, blob_file_read_hist,
blob_file_number, &file_size, &file_reader); blob_file_number, io_tracer, &file_size, &file_reader);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
@ -68,8 +69,8 @@ Status BlobFileReader::Create(
Status BlobFileReader::OpenFile( Status BlobFileReader::OpenFile(
const ImmutableCFOptions& immutable_cf_options, const ImmutableCFOptions& immutable_cf_options,
const FileOptions& file_opts, HistogramImpl* blob_file_read_hist, const FileOptions& file_opts, HistogramImpl* blob_file_read_hist,
uint64_t blob_file_number, uint64_t* file_size, uint64_t blob_file_number, const std::shared_ptr<IOTracer>& io_tracer,
std::unique_ptr<RandomAccessFileReader>* file_reader) { uint64_t* file_size, std::unique_ptr<RandomAccessFileReader>* file_reader) {
assert(file_size); assert(file_size);
assert(file_reader); assert(file_reader);
@ -118,7 +119,7 @@ Status BlobFileReader::OpenFile(
file_reader->reset(new RandomAccessFileReader( file_reader->reset(new RandomAccessFileReader(
std::move(file), blob_file_path, std::move(file), blob_file_path,
immutable_cf_options.env->GetSystemClock(), std::shared_ptr<IOTracer>(), immutable_cf_options.env->GetSystemClock(), io_tracer,
immutable_cf_options.statistics, BLOB_DB_BLOB_FILE_READ_MICROS, immutable_cf_options.statistics, BLOB_DB_BLOB_FILE_READ_MICROS,
blob_file_read_hist, immutable_cf_options.rate_limiter, blob_file_read_hist, immutable_cf_options.rate_limiter,
immutable_cf_options.listeners)); immutable_cf_options.listeners));

@ -29,6 +29,7 @@ class BlobFileReader {
uint32_t column_family_id, uint32_t column_family_id,
HistogramImpl* blob_file_read_hist, HistogramImpl* blob_file_read_hist,
uint64_t blob_file_number, uint64_t blob_file_number,
const std::shared_ptr<IOTracer>& io_tracer,
std::unique_ptr<BlobFileReader>* reader); std::unique_ptr<BlobFileReader>* reader);
BlobFileReader(const BlobFileReader&) = delete; BlobFileReader(const BlobFileReader&) = delete;
@ -47,7 +48,9 @@ class BlobFileReader {
static Status OpenFile(const ImmutableCFOptions& immutable_cf_options, static Status OpenFile(const ImmutableCFOptions& immutable_cf_options,
const FileOptions& file_opts, const FileOptions& file_opts,
HistogramImpl* blob_file_read_hist, HistogramImpl* blob_file_read_hist,
uint64_t blob_file_number, uint64_t* file_size, uint64_t blob_file_number,
const std::shared_ptr<IOTracer>& io_tracer,
uint64_t* file_size,
std::unique_ptr<RandomAccessFileReader>* file_reader); std::unique_ptr<RandomAccessFileReader>* file_reader);
static Status ReadHeader(const RandomAccessFileReader* file_reader, static Status ReadHeader(const RandomAccessFileReader* file_reader,

@ -142,9 +142,9 @@ TEST_F(BlobFileReaderTest, CreateReaderAndGetBlob) {
std::unique_ptr<BlobFileReader> reader; std::unique_ptr<BlobFileReader> reader;
ASSERT_OK(BlobFileReader::Create(immutable_cf_options, FileOptions(), ASSERT_OK(BlobFileReader::Create(
column_family_id, blob_file_read_hist, immutable_cf_options, FileOptions(), column_family_id,
blob_file_number, &reader)); blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader));
// Make sure the blob can be retrieved with and without checksum verification // Make sure the blob can be retrieved with and without checksum verification
ReadOptions read_options; ReadOptions read_options;
@ -282,7 +282,8 @@ TEST_F(BlobFileReaderTest, Malformed) {
ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(), ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(),
column_family_id, blob_file_read_hist, column_family_id, blob_file_read_hist,
blob_file_number, &reader) blob_file_number, nullptr /*IOTracer*/,
&reader)
.IsCorruption()); .IsCorruption());
} }
@ -315,7 +316,8 @@ TEST_F(BlobFileReaderTest, TTL) {
ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(), ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(),
column_family_id, blob_file_read_hist, column_family_id, blob_file_read_hist,
blob_file_number, &reader) blob_file_number, nullptr /*IOTracer*/,
&reader)
.IsCorruption()); .IsCorruption());
} }
@ -353,7 +355,8 @@ TEST_F(BlobFileReaderTest, ExpirationRangeInHeader) {
ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(), ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(),
column_family_id, blob_file_read_hist, column_family_id, blob_file_read_hist,
blob_file_number, &reader) blob_file_number, nullptr /*IOTracer*/,
&reader)
.IsCorruption()); .IsCorruption());
} }
@ -391,7 +394,8 @@ TEST_F(BlobFileReaderTest, ExpirationRangeInFooter) {
ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(), ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(),
column_family_id, blob_file_read_hist, column_family_id, blob_file_read_hist,
blob_file_number, &reader) blob_file_number, nullptr /*IOTracer*/,
&reader)
.IsCorruption()); .IsCorruption());
} }
@ -429,7 +433,7 @@ TEST_F(BlobFileReaderTest, IncorrectColumnFamily) {
ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(), ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(),
incorrect_column_family_id, incorrect_column_family_id,
blob_file_read_hist, blob_file_number, blob_file_read_hist, blob_file_number,
&reader) nullptr /*IOTracer*/, &reader)
.IsCorruption()); .IsCorruption());
} }
@ -460,9 +464,9 @@ TEST_F(BlobFileReaderTest, BlobCRCError) {
std::unique_ptr<BlobFileReader> reader; std::unique_ptr<BlobFileReader> reader;
ASSERT_OK(BlobFileReader::Create(immutable_cf_options, FileOptions(), ASSERT_OK(BlobFileReader::Create(
column_family_id, blob_file_read_hist, immutable_cf_options, FileOptions(), column_family_id,
blob_file_number, &reader)); blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader));
SyncPoint::GetInstance()->SetCallBack( SyncPoint::GetInstance()->SetCallBack(
"BlobFileReader::VerifyBlob:CheckBlobCRC", [](void* arg) { "BlobFileReader::VerifyBlob:CheckBlobCRC", [](void* arg) {
@ -516,9 +520,9 @@ TEST_F(BlobFileReaderTest, Compression) {
std::unique_ptr<BlobFileReader> reader; std::unique_ptr<BlobFileReader> reader;
ASSERT_OK(BlobFileReader::Create(immutable_cf_options, FileOptions(), ASSERT_OK(BlobFileReader::Create(
column_family_id, blob_file_read_hist, immutable_cf_options, FileOptions(), column_family_id,
blob_file_number, &reader)); blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader));
// Make sure the blob can be retrieved with and without checksum verification // Make sure the blob can be retrieved with and without checksum verification
ReadOptions read_options; ReadOptions read_options;
@ -576,9 +580,9 @@ TEST_F(BlobFileReaderTest, UncompressionError) {
std::unique_ptr<BlobFileReader> reader; std::unique_ptr<BlobFileReader> reader;
ASSERT_OK(BlobFileReader::Create(immutable_cf_options, FileOptions(), ASSERT_OK(BlobFileReader::Create(
column_family_id, blob_file_read_hist, immutable_cf_options, FileOptions(), column_family_id,
blob_file_number, &reader)); blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader));
SyncPoint::GetInstance()->SetCallBack( SyncPoint::GetInstance()->SetCallBack(
"BlobFileReader::UncompressBlobIfNeeded:TamperWithResult", [](void* arg) { "BlobFileReader::UncompressBlobIfNeeded:TamperWithResult", [](void* arg) {
@ -661,9 +665,9 @@ TEST_P(BlobFileReaderIOErrorTest, IOError) {
std::unique_ptr<BlobFileReader> reader; std::unique_ptr<BlobFileReader> reader;
const Status s = BlobFileReader::Create(immutable_cf_options, FileOptions(), const Status s = BlobFileReader::Create(
column_family_id, blob_file_read_hist, immutable_cf_options, FileOptions(), column_family_id,
blob_file_number, &reader); blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader);
const bool fail_during_create = const bool fail_during_create =
(sync_point_ != "BlobFileReader::GetBlob:ReadFromFile"); (sync_point_ != "BlobFileReader::GetBlob:ReadFromFile");
@ -741,9 +745,9 @@ TEST_P(BlobFileReaderDecodingErrorTest, DecodingError) {
std::unique_ptr<BlobFileReader> reader; std::unique_ptr<BlobFileReader> reader;
const Status s = BlobFileReader::Create(immutable_cf_options, FileOptions(), const Status s = BlobFileReader::Create(
column_family_id, blob_file_read_hist, immutable_cf_options, FileOptions(), column_family_id,
blob_file_number, &reader); blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader);
const bool fail_during_create = const bool fail_during_create =
sync_point_ != "BlobFileReader::GetBlob:TamperWithResult"; sync_point_ != "BlobFileReader::GetBlob:TamperWithResult";

@ -210,6 +210,63 @@ TEST_F(DBBlobBasicTest, GetBlob_IndexWithInvalidFileNumber) {
.IsCorruption()); .IsCorruption());
} }
#ifndef ROCKSDB_LITE
TEST_F(DBBlobBasicTest, GenerateIOTracing) {
Options options = GetDefaultOptions();
options.enable_blob_files = true;
options.min_blob_size = 0;
std::string trace_file = dbname_ + "/io_trace_file";
Reopen(options);
{
// Create IO trace file
std::unique_ptr<TraceWriter> trace_writer;
ASSERT_OK(
NewFileTraceWriter(env_, EnvOptions(), trace_file, &trace_writer));
ASSERT_OK(db_->StartIOTrace(env_, TraceOptions(), std::move(trace_writer)));
constexpr char key[] = "key";
constexpr char blob_value[] = "blob_value";
ASSERT_OK(Put(key, blob_value));
ASSERT_OK(Flush());
ASSERT_EQ(Get(key), blob_value);
ASSERT_OK(db_->EndIOTrace());
ASSERT_OK(env_->FileExists(trace_file));
}
{
// Parse trace file to check file opertions related to blob files are
// recorded.
std::unique_ptr<TraceReader> trace_reader;
ASSERT_OK(
NewFileTraceReader(env_, EnvOptions(), trace_file, &trace_reader));
IOTraceReader reader(std::move(trace_reader));
IOTraceHeader header;
ASSERT_OK(reader.ReadHeader(&header));
ASSERT_EQ(kMajorVersion, static_cast<int>(header.rocksdb_major_version));
ASSERT_EQ(kMinorVersion, static_cast<int>(header.rocksdb_minor_version));
// Read records.
int blob_files_op_count = 0;
Status status;
while (true) {
IOTraceRecord record;
status = reader.ReadIOOp(&record);
if (!status.ok()) {
break;
}
if (record.file_name.find("blob") != std::string::npos) {
blob_files_op_count++;
}
}
// Assuming blob files will have Append, Close and then Read operations.
ASSERT_GT(blob_files_op_count, 2);
}
}
#endif // !ROCKSDB_LITE
class DBBlobBasicIOErrorTest : public DBBlobBasicTest, class DBBlobBasicIOErrorTest : public DBBlobBasicTest,
public testing::WithParamInterface<std::string> { public testing::WithParamInterface<std::string> {
protected: protected:

@ -179,8 +179,8 @@ Status BuildTable(
? new BlobFileBuilder(versions, env, fs, &ioptions, ? new BlobFileBuilder(versions, env, fs, &ioptions,
&mutable_cf_options, &file_options, job_id, &mutable_cf_options, &file_options, job_id,
column_family_id, column_family_name, column_family_id, column_family_name,
io_priority, write_hint, &blob_file_paths, io_priority, write_hint, io_tracer,
blob_file_additions) &blob_file_paths, blob_file_additions)
: nullptr); : nullptr);
CompactionIterator c_iter( CompactionIterator c_iter(

@ -562,7 +562,7 @@ ColumnFamilyData::ColumnFamilyData(
block_cache_tracer, io_tracer)); block_cache_tracer, io_tracer));
blob_file_cache_.reset( blob_file_cache_.reset(
new BlobFileCache(_table_cache, ioptions(), soptions(), id_, new BlobFileCache(_table_cache, ioptions(), soptions(), id_,
internal_stats_->GetBlobFileReadHist())); internal_stats_->GetBlobFileReadHist(), io_tracer));
if (ioptions_.compaction_style == kCompactionStyleLevel) { if (ioptions_.compaction_style == kCompactionStyleLevel) {
compaction_picker_.reset( compaction_picker_.reset(

@ -978,7 +978,7 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
sub_compact->compaction->immutable_cf_options(), sub_compact->compaction->immutable_cf_options(),
mutable_cf_options, &file_options_, job_id_, cfd->GetID(), mutable_cf_options, &file_options_, job_id_, cfd->GetID(),
cfd->GetName(), Env::IOPriority::IO_LOW, write_hint_, cfd->GetName(), Env::IOPriority::IO_LOW, write_hint_,
&blob_file_paths, &sub_compact->blob_file_additions) io_tracer_, &blob_file_paths, &sub_compact->blob_file_additions)
: nullptr); : nullptr);
TEST_SYNC_POINT("CompactionJob::Run():Inprogress"); TEST_SYNC_POINT("CompactionJob::Run():Inprogress");

Loading…
Cancel
Save