NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add support for IOTracing in blob files (#7958)

Browse Source 
Summary:
Add support for IOTracing in blob files

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7958

Test Plan:
Add a new test and checked manually the trace_file for blob
files being recorded during read and write.

Reviewed By: ltamasi

Differential Revision: D26415950

Pulled By: akankshamahajan15

fbshipit-source-id: 49c2859b3a4f8307e7cb69a92704403a4da46d44
main
Akanksha Mahajan 3 years ago committed by Facebook GitHub Bot
parent 9df78a94f1
commit ea8bb82fc7
13 changed files with 156 additions and 77 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 8
      db/blob/blob_file_builder.cc
  2. 4
      db/blob/blob_file_builder.h
  3. 70
      db/blob/blob_file_builder_test.cc
  4. 9
      db/blob/blob_file_cache.cc
  5. 5
      db/blob/blob_file_cache.h
  6. 8
      db/blob/blob_file_cache_test.cc
  7. 9
      db/blob/blob_file_reader.cc
  8. 5
      db/blob/blob_file_reader.h
  9. 50
      db/blob/blob_file_reader_test.cc
  10. 57
      db/blob/db_blob_basic_test.cc
  11. 4
      db/builder.cc
  12. 2
      db/column_family.cc
  13. 2
      db/compaction/compaction_job.cc

8
db/blob/blob_file_builder.cc
Unescape View File

@ -21,6 +21,7 @@
#include "rocksdb/slice.h" #include "rocksdb/slice.h"
#include "rocksdb/status.h" #include "rocksdb/status.h"
#include "test_util/sync_point.h" #include "test_util/sync_point.h"
#include "trace_replay/io_tracer.h"
#include "util/compression.h" #include "util/compression.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
@ -32,12 +33,13 @@ BlobFileBuilder::BlobFileBuilder(
int job_id, uint32_t column_family_id, int job_id, uint32_t column_family_id,
const std::string& column_family_name, Env::IOPriority io_priority, const std::string& column_family_name, Env::IOPriority io_priority,
Env::WriteLifeTimeHint write_hint, Env::WriteLifeTimeHint write_hint,
const std::shared_ptr<IOTracer>& io_tracer,
std::vector<std::string>* blob_file_paths, std::vector<std::string>* blob_file_paths,
std::vector<BlobFileAddition>* blob_file_additions) std::vector<BlobFileAddition>* blob_file_additions)
: BlobFileBuilder([versions]() { return versions->NewFileNumber(); }, env, : BlobFileBuilder([versions]() { return versions->NewFileNumber(); }, env,
fs, immutable_cf_options, mutable_cf_options, fs, immutable_cf_options, mutable_cf_options,
file_options, job_id, column_family_id, file_options, job_id, column_family_id,
column_family_name, io_priority, write_hint, column_family_name, io_priority, write_hint, io_tracer,
blob_file_paths, blob_file_additions) {} blob_file_paths, blob_file_additions) {}
BlobFileBuilder::BlobFileBuilder( BlobFileBuilder::BlobFileBuilder(
@ -47,6 +49,7 @@ BlobFileBuilder::BlobFileBuilder(
int job_id, uint32_t column_family_id, int job_id, uint32_t column_family_id,
const std::string& column_family_name, Env::IOPriority io_priority, const std::string& column_family_name, Env::IOPriority io_priority,
Env::WriteLifeTimeHint write_hint, Env::WriteLifeTimeHint write_hint,
const std::shared_ptr<IOTracer>& io_tracer,
std::vector<std::string>* blob_file_paths, std::vector<std::string>* blob_file_paths,
std::vector<BlobFileAddition>* blob_file_additions) std::vector<BlobFileAddition>* blob_file_additions)
: file_number_generator_(std::move(file_number_generator)), : file_number_generator_(std::move(file_number_generator)),
@ -61,6 +64,7 @@ BlobFileBuilder::BlobFileBuilder(
column_family_name_(column_family_name), column_family_name_(column_family_name),
io_priority_(io_priority), io_priority_(io_priority),
write_hint_(write_hint), write_hint_(write_hint),
io_tracer_(io_tracer),
blob_file_paths_(blob_file_paths), blob_file_paths_(blob_file_paths),
blob_file_additions_(blob_file_additions), blob_file_additions_(blob_file_additions),
blob_count_(0), blob_count_(0),
@ -182,7 +186,7 @@ Status BlobFileBuilder::OpenBlobFileIfNeeded() {
Statistics* const statistics = immutable_cf_options_->statistics; Statistics* const statistics = immutable_cf_options_->statistics;
std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter( std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
std::move(file), blob_file_paths_->back(), *file_options_, clock_, std::move(file), blob_file_paths_->back(), *file_options_, clock_,
nullptr /*IOTracer*/, statistics, immutable_cf_options_->listeners, io_tracer_, statistics, immutable_cf_options_->listeners,
immutable_cf_options_->file_checksum_gen_factory, immutable_cf_options_->file_checksum_gen_factory,
tmp_set.Contains(FileType::kBlobFile))); tmp_set.Contains(FileType::kBlobFile)));

4
db/blob/blob_file_builder.h
Unescape View File

@ -26,6 +26,7 @@ class BlobFileAddition;
class Status; class Status;
class Slice; class Slice;
class BlobLogWriter; class BlobLogWriter;
class IOTracer;
class BlobFileBuilder { class BlobFileBuilder {
public: public:
@ -37,6 +38,7 @@ class BlobFileBuilder {
const std::string& column_family_name, const std::string& column_family_name,
Env::IOPriority io_priority, Env::IOPriority io_priority,
Env::WriteLifeTimeHint write_hint, Env::WriteLifeTimeHint write_hint,
const std::shared_ptr<IOTracer>& io_tracer,
std::vector<std::string>* blob_file_paths, std::vector<std::string>* blob_file_paths,
std::vector<BlobFileAddition>* blob_file_additions); std::vector<BlobFileAddition>* blob_file_additions);
@ -49,6 +51,7 @@ class BlobFileBuilder {
const std::string& column_family_name, const std::string& column_family_name,
Env::IOPriority io_priority, Env::IOPriority io_priority,
Env::WriteLifeTimeHint write_hint, Env::WriteLifeTimeHint write_hint,
const std::shared_ptr<IOTracer>& io_tracer,
std::vector<std::string>* blob_file_paths, std::vector<std::string>* blob_file_paths,
std::vector<BlobFileAddition>* blob_file_additions); std::vector<BlobFileAddition>* blob_file_additions);
@ -82,6 +85,7 @@ class BlobFileBuilder {
std::string column_family_name_; std::string column_family_name_;
Env::IOPriority io_priority_; Env::IOPriority io_priority_;
Env::WriteLifeTimeHint write_hint_; Env::WriteLifeTimeHint write_hint_;
std::shared_ptr<IOTracer> io_tracer_;
std::vector<std::string>* blob_file_paths_; std::vector<std::string>* blob_file_paths_;
std::vector<BlobFileAddition>* blob_file_additions_; std::vector<BlobFileAddition>* blob_file_additions_;
std::unique_ptr<BlobLogWriter> writer_; std::unique_ptr<BlobLogWriter> writer_;

70
db/blob/blob_file_builder_test.cc
Unescape View File

@ -140,11 +140,11 @@ TEST_F(BlobFileBuilderTest, BuildAndCheckOneFile) {
std::vector<std::string> blob_file_paths; std::vector<std::string> blob_file_paths;
std::vector<BlobFileAddition> blob_file_additions; std::vector<BlobFileAddition> blob_file_additions;
BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_, BlobFileBuilder builder(
&immutable_cf_options, &mutable_cf_options, TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options,
&file_options_, job_id, column_family_id, &mutable_cf_options, &file_options_, job_id, column_family_id,
column_family_name, io_priority, write_hint, column_family_name, io_priority, write_hint, nullptr /*IOTracer*/,
&blob_file_paths, &blob_file_additions); &blob_file_paths, &blob_file_additions);
std::vector<std::pair<std::string, std::string>> expected_key_value_pairs( std::vector<std::pair<std::string, std::string>> expected_key_value_pairs(
number_of_blobs); number_of_blobs);
@ -223,11 +223,11 @@ TEST_F(BlobFileBuilderTest, BuildAndCheckMultipleFiles) {
std::vector<std::string> blob_file_paths; std::vector<std::string> blob_file_paths;
std::vector<BlobFileAddition> blob_file_additions; std::vector<BlobFileAddition> blob_file_additions;
BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_, BlobFileBuilder builder(
&immutable_cf_options, &mutable_cf_options, TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options,
&file_options_, job_id, column_family_id, &mutable_cf_options, &file_options_, job_id, column_family_id,
column_family_name, io_priority, write_hint, column_family_name, io_priority, write_hint, nullptr /*IOTracer*/,
&blob_file_paths, &blob_file_additions); &blob_file_paths, &blob_file_additions);
std::vector<std::pair<std::string, std::string>> expected_key_value_pairs( std::vector<std::pair<std::string, std::string>> expected_key_value_pairs(
number_of_blobs); number_of_blobs);
@ -308,11 +308,11 @@ TEST_F(BlobFileBuilderTest, InlinedValues) {
std::vector<std::string> blob_file_paths; std::vector<std::string> blob_file_paths;
std::vector<BlobFileAddition> blob_file_additions; std::vector<BlobFileAddition> blob_file_additions;
BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_, BlobFileBuilder builder(
&immutable_cf_options, &mutable_cf_options, TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options,
&file_options_, job_id, column_family_id, &mutable_cf_options, &file_options_, job_id, column_family_id,
column_family_name, io_priority, write_hint, column_family_name, io_priority, write_hint, nullptr /*IOTracer*/,
&blob_file_paths, &blob_file_additions); &blob_file_paths, &blob_file_additions);
for (size_t i = 0; i < number_of_blobs; ++i) { for (size_t i = 0; i < number_of_blobs; ++i) {
const std::string key = std::to_string(i); const std::string key = std::to_string(i);
@ -360,11 +360,11 @@ TEST_F(BlobFileBuilderTest, Compression) {
std::vector<std::string> blob_file_paths; std::vector<std::string> blob_file_paths;
std::vector<BlobFileAddition> blob_file_additions; std::vector<BlobFileAddition> blob_file_additions;
BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_, BlobFileBuilder builder(
&immutable_cf_options, &mutable_cf_options, TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options,
&file_options_, job_id, column_family_id, &mutable_cf_options, &file_options_, job_id, column_family_id,
column_family_name, io_priority, write_hint, column_family_name, io_priority, write_hint, nullptr /*IOTracer*/,
&blob_file_paths, &blob_file_additions); &blob_file_paths, &blob_file_additions);
const std::string key("1"); const std::string key("1");
const std::string uncompressed_value(value_size, 'x'); const std::string uncompressed_value(value_size, 'x');
@ -442,11 +442,11 @@ TEST_F(BlobFileBuilderTest, CompressionError) {
std::vector<std::string> blob_file_paths; std::vector<std::string> blob_file_paths;
std::vector<BlobFileAddition> blob_file_additions; std::vector<BlobFileAddition> blob_file_additions;
BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_, BlobFileBuilder builder(
&immutable_cf_options, &mutable_cf_options, TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options,
&file_options_, job_id, column_family_id, &mutable_cf_options, &file_options_, job_id, column_family_id,
column_family_name, io_priority, write_hint, column_family_name, io_priority, write_hint, nullptr /*IOTracer*/,
&blob_file_paths, &blob_file_additions); &blob_file_paths, &blob_file_additions);
SyncPoint::GetInstance()->SetCallBack("CompressData:TamperWithReturnValue", SyncPoint::GetInstance()->SetCallBack("CompressData:TamperWithReturnValue",
[](void* arg) { [](void* arg) {
@ -519,11 +519,11 @@ TEST_F(BlobFileBuilderTest, Checksum) {
std::vector<std::string> blob_file_paths; std::vector<std::string> blob_file_paths;
std::vector<BlobFileAddition> blob_file_additions; std::vector<BlobFileAddition> blob_file_additions;
BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_, BlobFileBuilder builder(
&immutable_cf_options, &mutable_cf_options, TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options,
&file_options_, job_id, column_family_id, &mutable_cf_options, &file_options_, job_id, column_family_id,
column_family_name, io_priority, write_hint, column_family_name, io_priority, write_hint, nullptr /*IOTracer*/,
&blob_file_paths, &blob_file_additions); &blob_file_paths, &blob_file_additions);
const std::string key("1"); const std::string key("1");
const std::string value("deadbeef"); const std::string value("deadbeef");
@ -615,11 +615,11 @@ TEST_P(BlobFileBuilderIOErrorTest, IOError) {
std::vector<std::string> blob_file_paths; std::vector<std::string> blob_file_paths;
std::vector<BlobFileAddition> blob_file_additions; std::vector<BlobFileAddition> blob_file_additions;
BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_, BlobFileBuilder builder(
&immutable_cf_options, &mutable_cf_options, TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options,
&file_options_, job_id, column_family_id, &mutable_cf_options, &file_options_, job_id, column_family_id,
column_family_name, io_priority, write_hint, column_family_name, io_priority, write_hint, nullptr /*IOTracer*/,
&blob_file_paths, &blob_file_additions); &blob_file_paths, &blob_file_additions);
SyncPoint::GetInstance()->SetCallBack(sync_point_, [this](void* arg) { SyncPoint::GetInstance()->SetCallBack(sync_point_, [this](void* arg) {
Status* const s = static_cast<Status*>(arg); Status* const s = static_cast<Status*>(arg);

9
db/blob/blob_file_cache.cc
Unescape View File

@ -13,6 +13,7 @@
#include "rocksdb/cache.h" #include "rocksdb/cache.h"
#include "rocksdb/slice.h" #include "rocksdb/slice.h"
#include "test_util/sync_point.h" #include "test_util/sync_point.h"
#include "trace_replay/io_tracer.h"
#include "util/hash.h" #include "util/hash.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
@ -21,13 +22,15 @@ BlobFileCache::BlobFileCache(Cache* cache,
const ImmutableCFOptions* immutable_cf_options, const ImmutableCFOptions* immutable_cf_options,
const FileOptions* file_options, const FileOptions* file_options,
uint32_t column_family_id, uint32_t column_family_id,
HistogramImpl* blob_file_read_hist) HistogramImpl* blob_file_read_hist,
const std::shared_ptr<IOTracer>& io_tracer)
: cache_(cache), : cache_(cache),
mutex_(kNumberOfMutexStripes, kGetSliceNPHash64UnseededFnPtr), mutex_(kNumberOfMutexStripes, kGetSliceNPHash64UnseededFnPtr),
immutable_cf_options_(immutable_cf_options), immutable_cf_options_(immutable_cf_options),
file_options_(file_options), file_options_(file_options),
column_family_id_(column_family_id), column_family_id_(column_family_id),
blob_file_read_hist_(blob_file_read_hist) { blob_file_read_hist_(blob_file_read_hist),
io_tracer_(io_tracer) {
assert(cache_); assert(cache_);
assert(immutable_cf_options_); assert(immutable_cf_options_);
assert(file_options_); assert(file_options_);
@ -71,7 +74,7 @@ Status BlobFileCache::GetBlobFileReader(
assert(file_options_); assert(file_options_);
const Status s = BlobFileReader::Create( const Status s = BlobFileReader::Create(
*immutable_cf_options_, *file_options_, column_family_id_, *immutable_cf_options_, *file_options_, column_family_id_,
blob_file_read_hist_, blob_file_number, &reader); blob_file_read_hist_, blob_file_number, io_tracer_, &reader);
if (!s.ok()) { if (!s.ok()) {
RecordTick(statistics, NO_FILE_ERRORS); RecordTick(statistics, NO_FILE_ERRORS);
return s; return s;

5
db/blob/blob_file_cache.h
Unescape View File

@ -20,12 +20,14 @@ class HistogramImpl;
class Status; class Status;
class BlobFileReader; class BlobFileReader;
class Slice; class Slice;
class IOTracer;
class BlobFileCache { class BlobFileCache {
public: public:
BlobFileCache(Cache* cache, const ImmutableCFOptions* immutable_cf_options, BlobFileCache(Cache* cache, const ImmutableCFOptions* immutable_cf_options,
const FileOptions* file_options, uint32_t column_family_id, const FileOptions* file_options, uint32_t column_family_id,
HistogramImpl* blob_file_read_hist); HistogramImpl* blob_file_read_hist,
const std::shared_ptr<IOTracer>& io_tracer);
BlobFileCache(const BlobFileCache&) = delete; BlobFileCache(const BlobFileCache&) = delete;
BlobFileCache& operator=(const BlobFileCache&) = delete; BlobFileCache& operator=(const BlobFileCache&) = delete;
@ -42,6 +44,7 @@ class BlobFileCache {
const FileOptions* file_options_; const FileOptions* file_options_;
uint32_t column_family_id_; uint32_t column_family_id_;
HistogramImpl* blob_file_read_hist_; HistogramImpl* blob_file_read_hist_;
std::shared_ptr<IOTracer> io_tracer_;
static constexpr size_t kNumberOfMutexStripes = 1 << 7; static constexpr size_t kNumberOfMutexStripes = 1 << 7;
}; };

8
db/blob/blob_file_cache_test.cc
Unescape View File

@ -114,7 +114,7 @@ TEST_F(BlobFileCacheTest, GetBlobFileReader) {
BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options, BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options,
&file_options, column_family_id, &file_options, column_family_id,
blob_file_read_hist); blob_file_read_hist, nullptr /*IOTracer*/);
// First try: reader should be opened and put in cache // First try: reader should be opened and put in cache
CacheHandleGuard<BlobFileReader> first; CacheHandleGuard<BlobFileReader> first;
@ -159,7 +159,7 @@ TEST_F(BlobFileCacheTest, GetBlobFileReader_Race) {
BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options, BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options,
&file_options, column_family_id, &file_options, column_family_id,
blob_file_read_hist); blob_file_read_hist, nullptr /*IOTracer*/);
CacheHandleGuard<BlobFileReader> first; CacheHandleGuard<BlobFileReader> first;
CacheHandleGuard<BlobFileReader> second; CacheHandleGuard<BlobFileReader> second;
@ -207,7 +207,7 @@ TEST_F(BlobFileCacheTest, GetBlobFileReader_IOError) {
BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options, BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options,
&file_options, column_family_id, &file_options, column_family_id,
blob_file_read_hist); blob_file_read_hist, nullptr /*IOTracer*/);
// Note: there is no blob file with the below number // Note: there is no blob file with the below number
constexpr uint64_t blob_file_number = 123; constexpr uint64_t blob_file_number = 123;
@ -248,7 +248,7 @@ TEST_F(BlobFileCacheTest, GetBlobFileReader_CacheFull) {
BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options, BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options,
&file_options, column_family_id, &file_options, column_family_id,
blob_file_read_hist); blob_file_read_hist, nullptr /*IOTracer*/);
// Insert into cache should fail since it has zero capacity and // Insert into cache should fail since it has zero capacity and
// strict_capacity_limit is set // strict_capacity_limit is set

9
db/blob/blob_file_reader.cc
Unescape View File

@ -24,6 +24,7 @@ Status BlobFileReader::Create(
const ImmutableCFOptions& immutable_cf_options, const ImmutableCFOptions& immutable_cf_options,
const FileOptions& file_options, uint32_t column_family_id, const FileOptions& file_options, uint32_t column_family_id,
HistogramImpl* blob_file_read_hist, uint64_t blob_file_number, HistogramImpl* blob_file_read_hist, uint64_t blob_file_number,
const std::shared_ptr<IOTracer>& io_tracer,
std::unique_ptr<BlobFileReader>* blob_file_reader) { std::unique_ptr<BlobFileReader>* blob_file_reader) {
assert(blob_file_reader); assert(blob_file_reader);
assert(!*blob_file_reader); assert(!*blob_file_reader);
@ -34,7 +35,7 @@ Status BlobFileReader::Create(
{ {
const Status s = const Status s =
OpenFile(immutable_cf_options, file_options, blob_file_read_hist, OpenFile(immutable_cf_options, file_options, blob_file_read_hist,
blob_file_number, &file_size, &file_reader); blob_file_number, io_tracer, &file_size, &file_reader);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
@ -68,8 +69,8 @@ Status BlobFileReader::Create(
Status BlobFileReader::OpenFile( Status BlobFileReader::OpenFile(
const ImmutableCFOptions& immutable_cf_options, const ImmutableCFOptions& immutable_cf_options,
const FileOptions& file_opts, HistogramImpl* blob_file_read_hist, const FileOptions& file_opts, HistogramImpl* blob_file_read_hist,
uint64_t blob_file_number, uint64_t* file_size, uint64_t blob_file_number, const std::shared_ptr<IOTracer>& io_tracer,
std::unique_ptr<RandomAccessFileReader>* file_reader) { uint64_t* file_size, std::unique_ptr<RandomAccessFileReader>* file_reader) {
assert(file_size); assert(file_size);
assert(file_reader); assert(file_reader);
@ -118,7 +119,7 @@ Status BlobFileReader::OpenFile(
file_reader->reset(new RandomAccessFileReader( file_reader->reset(new RandomAccessFileReader(
std::move(file), blob_file_path, std::move(file), blob_file_path,
immutable_cf_options.env->GetSystemClock(), std::shared_ptr<IOTracer>(), immutable_cf_options.env->GetSystemClock(), io_tracer,
immutable_cf_options.statistics, BLOB_DB_BLOB_FILE_READ_MICROS, immutable_cf_options.statistics, BLOB_DB_BLOB_FILE_READ_MICROS,
blob_file_read_hist, immutable_cf_options.rate_limiter, blob_file_read_hist, immutable_cf_options.rate_limiter,
immutable_cf_options.listeners)); immutable_cf_options.listeners));

5
db/blob/blob_file_reader.h
Unescape View File

@ -29,6 +29,7 @@ class BlobFileReader {
uint32_t column_family_id, uint32_t column_family_id,
HistogramImpl* blob_file_read_hist, HistogramImpl* blob_file_read_hist,
uint64_t blob_file_number, uint64_t blob_file_number,
const std::shared_ptr<IOTracer>& io_tracer,
std::unique_ptr<BlobFileReader>* reader); std::unique_ptr<BlobFileReader>* reader);
BlobFileReader(const BlobFileReader&) = delete; BlobFileReader(const BlobFileReader&) = delete;
@ -47,7 +48,9 @@ class BlobFileReader {
static Status OpenFile(const ImmutableCFOptions& immutable_cf_options, static Status OpenFile(const ImmutableCFOptions& immutable_cf_options,
const FileOptions& file_opts, const FileOptions& file_opts,
HistogramImpl* blob_file_read_hist, HistogramImpl* blob_file_read_hist,
uint64_t blob_file_number, uint64_t* file_size, uint64_t blob_file_number,
const std::shared_ptr<IOTracer>& io_tracer,
uint64_t* file_size,
std::unique_ptr<RandomAccessFileReader>* file_reader); std::unique_ptr<RandomAccessFileReader>* file_reader);
static Status ReadHeader(const RandomAccessFileReader* file_reader, static Status ReadHeader(const RandomAccessFileReader* file_reader,

50
db/blob/blob_file_reader_test.cc
Unescape View File

@ -142,9 +142,9 @@ TEST_F(BlobFileReaderTest, CreateReaderAndGetBlob) {
std::unique_ptr<BlobFileReader> reader; std::unique_ptr<BlobFileReader> reader;
ASSERT_OK(BlobFileReader::Create(immutable_cf_options, FileOptions(), ASSERT_OK(BlobFileReader::Create(
column_family_id, blob_file_read_hist, immutable_cf_options, FileOptions(), column_family_id,
blob_file_number, &reader)); blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader));
// Make sure the blob can be retrieved with and without checksum verification // Make sure the blob can be retrieved with and without checksum verification
ReadOptions read_options; ReadOptions read_options;
@ -282,7 +282,8 @@ TEST_F(BlobFileReaderTest, Malformed) {
ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(), ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(),
column_family_id, blob_file_read_hist, column_family_id, blob_file_read_hist,
blob_file_number, &reader) blob_file_number, nullptr /*IOTracer*/,
&reader)
.IsCorruption()); .IsCorruption());
} }
@ -315,7 +316,8 @@ TEST_F(BlobFileReaderTest, TTL) {
ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(), ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(),
column_family_id, blob_file_read_hist, column_family_id, blob_file_read_hist,
blob_file_number, &reader) blob_file_number, nullptr /*IOTracer*/,
&reader)
.IsCorruption()); .IsCorruption());
} }
@ -353,7 +355,8 @@ TEST_F(BlobFileReaderTest, ExpirationRangeInHeader) {
ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(), ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(),
column_family_id, blob_file_read_hist, column_family_id, blob_file_read_hist,
blob_file_number, &reader) blob_file_number, nullptr /*IOTracer*/,
&reader)
.IsCorruption()); .IsCorruption());
} }
@ -391,7 +394,8 @@ TEST_F(BlobFileReaderTest, ExpirationRangeInFooter) {
ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(), ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(),
column_family_id, blob_file_read_hist, column_family_id, blob_file_read_hist,
blob_file_number, &reader) blob_file_number, nullptr /*IOTracer*/,
&reader)
.IsCorruption()); .IsCorruption());
} }
@ -429,7 +433,7 @@ TEST_F(BlobFileReaderTest, IncorrectColumnFamily) {
ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(), ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(),
incorrect_column_family_id, incorrect_column_family_id,
blob_file_read_hist, blob_file_number, blob_file_read_hist, blob_file_number,
&reader) nullptr /*IOTracer*/, &reader)
.IsCorruption()); .IsCorruption());
} }
@ -460,9 +464,9 @@ TEST_F(BlobFileReaderTest, BlobCRCError) {
std::unique_ptr<BlobFileReader> reader; std::unique_ptr<BlobFileReader> reader;
ASSERT_OK(BlobFileReader::Create(immutable_cf_options, FileOptions(), ASSERT_OK(BlobFileReader::Create(
column_family_id, blob_file_read_hist, immutable_cf_options, FileOptions(), column_family_id,
blob_file_number, &reader)); blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader));
SyncPoint::GetInstance()->SetCallBack( SyncPoint::GetInstance()->SetCallBack(
"BlobFileReader::VerifyBlob:CheckBlobCRC", [](void* arg) { "BlobFileReader::VerifyBlob:CheckBlobCRC", [](void* arg) {
@ -516,9 +520,9 @@ TEST_F(BlobFileReaderTest, Compression) {
std::unique_ptr<BlobFileReader> reader; std::unique_ptr<BlobFileReader> reader;
ASSERT_OK(BlobFileReader::Create(immutable_cf_options, FileOptions(), ASSERT_OK(BlobFileReader::Create(
column_family_id, blob_file_read_hist, immutable_cf_options, FileOptions(), column_family_id,
blob_file_number, &reader)); blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader));
// Make sure the blob can be retrieved with and without checksum verification // Make sure the blob can be retrieved with and without checksum verification
ReadOptions read_options; ReadOptions read_options;
@ -576,9 +580,9 @@ TEST_F(BlobFileReaderTest, UncompressionError) {
std::unique_ptr<BlobFileReader> reader; std::unique_ptr<BlobFileReader> reader;
ASSERT_OK(BlobFileReader::Create(immutable_cf_options, FileOptions(), ASSERT_OK(BlobFileReader::Create(
column_family_id, blob_file_read_hist, immutable_cf_options, FileOptions(), column_family_id,
blob_file_number, &reader)); blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader));
SyncPoint::GetInstance()->SetCallBack( SyncPoint::GetInstance()->SetCallBack(
"BlobFileReader::UncompressBlobIfNeeded:TamperWithResult", [](void* arg) { "BlobFileReader::UncompressBlobIfNeeded:TamperWithResult", [](void* arg) {
@ -661,9 +665,9 @@ TEST_P(BlobFileReaderIOErrorTest, IOError) {
std::unique_ptr<BlobFileReader> reader; std::unique_ptr<BlobFileReader> reader;
const Status s = BlobFileReader::Create(immutable_cf_options, FileOptions(), const Status s = BlobFileReader::Create(
column_family_id, blob_file_read_hist, immutable_cf_options, FileOptions(), column_family_id,
blob_file_number, &reader); blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader);
const bool fail_during_create = const bool fail_during_create =
(sync_point_ != "BlobFileReader::GetBlob:ReadFromFile"); (sync_point_ != "BlobFileReader::GetBlob:ReadFromFile");
@ -741,9 +745,9 @@ TEST_P(BlobFileReaderDecodingErrorTest, DecodingError) {
std::unique_ptr<BlobFileReader> reader; std::unique_ptr<BlobFileReader> reader;
const Status s = BlobFileReader::Create(immutable_cf_options, FileOptions(), const Status s = BlobFileReader::Create(
column_family_id, blob_file_read_hist, immutable_cf_options, FileOptions(), column_family_id,
blob_file_number, &reader); blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader);
const bool fail_during_create = const bool fail_during_create =
sync_point_ != "BlobFileReader::GetBlob:TamperWithResult"; sync_point_ != "BlobFileReader::GetBlob:TamperWithResult";

57
db/blob/db_blob_basic_test.cc
Unescape View File

@ -210,6 +210,63 @@ TEST_F(DBBlobBasicTest, GetBlob_IndexWithInvalidFileNumber) {
.IsCorruption()); .IsCorruption());
} }
#ifndef ROCKSDB_LITE
TEST_F(DBBlobBasicTest, GenerateIOTracing) {
Options options = GetDefaultOptions();
options.enable_blob_files = true;
options.min_blob_size = 0;
std::string trace_file = dbname_ + "/io_trace_file";
Reopen(options);
{
// Create IO trace file
std::unique_ptr<TraceWriter> trace_writer;
ASSERT_OK(
NewFileTraceWriter(env_, EnvOptions(), trace_file, &trace_writer));
ASSERT_OK(db_->StartIOTrace(env_, TraceOptions(), std::move(trace_writer)));
constexpr char key[] = "key";
constexpr char blob_value[] = "blob_value";
ASSERT_OK(Put(key, blob_value));
ASSERT_OK(Flush());
ASSERT_EQ(Get(key), blob_value);
ASSERT_OK(db_->EndIOTrace());
ASSERT_OK(env_->FileExists(trace_file));
}
{
// Parse trace file to check file opertions related to blob files are
// recorded.
std::unique_ptr<TraceReader> trace_reader;
ASSERT_OK(
NewFileTraceReader(env_, EnvOptions(), trace_file, &trace_reader));
IOTraceReader reader(std::move(trace_reader));
IOTraceHeader header;
ASSERT_OK(reader.ReadHeader(&header));
ASSERT_EQ(kMajorVersion, static_cast<int>(header.rocksdb_major_version));
ASSERT_EQ(kMinorVersion, static_cast<int>(header.rocksdb_minor_version));
// Read records.
int blob_files_op_count = 0;
Status status;
while (true) {
IOTraceRecord record;
status = reader.ReadIOOp(&record);
if (!status.ok()) {
break;
}
if (record.file_name.find("blob") != std::string::npos) {
blob_files_op_count++;
}
}
// Assuming blob files will have Append, Close and then Read operations.
ASSERT_GT(blob_files_op_count, 2);
}
}
#endif // !ROCKSDB_LITE
class DBBlobBasicIOErrorTest : public DBBlobBasicTest, class DBBlobBasicIOErrorTest : public DBBlobBasicTest,
public testing::WithParamInterface<std::string> { public testing::WithParamInterface<std::string> {
protected: protected:

4
db/builder.cc
Unescape View File

@ -179,8 +179,8 @@ Status BuildTable(
? new BlobFileBuilder(versions, env, fs, &ioptions, ? new BlobFileBuilder(versions, env, fs, &ioptions,
&mutable_cf_options, &file_options, job_id, &mutable_cf_options, &file_options, job_id,
column_family_id, column_family_name, column_family_id, column_family_name,
io_priority, write_hint, &blob_file_paths, io_priority, write_hint, io_tracer,
blob_file_additions) &blob_file_paths, blob_file_additions)
: nullptr); : nullptr);
CompactionIterator c_iter( CompactionIterator c_iter(

2
db/column_family.cc
Unescape View File

@ -562,7 +562,7 @@ ColumnFamilyData::ColumnFamilyData(
block_cache_tracer, io_tracer)); block_cache_tracer, io_tracer));
blob_file_cache_.reset( blob_file_cache_.reset(
new BlobFileCache(_table_cache, ioptions(), soptions(), id_, new BlobFileCache(_table_cache, ioptions(), soptions(), id_,
internal_stats_->GetBlobFileReadHist())); internal_stats_->GetBlobFileReadHist(), io_tracer));
if (ioptions_.compaction_style == kCompactionStyleLevel) { if (ioptions_.compaction_style == kCompactionStyleLevel) {
compaction_picker_.reset( compaction_picker_.reset(

2
db/compaction/compaction_job.cc
Unescape View File

@ -978,7 +978,7 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
sub_compact->compaction->immutable_cf_options(), sub_compact->compaction->immutable_cf_options(),
mutable_cf_options, &file_options_, job_id_, cfd->GetID(), mutable_cf_options, &file_options_, job_id_, cfd->GetID(),
cfd->GetName(), Env::IOPriority::IO_LOW, write_hint_, cfd->GetName(), Env::IOPriority::IO_LOW, write_hint_,
&blob_file_paths, &sub_compact->blob_file_additions) io_tracer_, &blob_file_paths, &sub_compact->blob_file_additions)
: nullptr); : nullptr);
TEST_SYNC_POINT("CompactionJob::Run():Inprogress"); TEST_SYNC_POINT("CompactionJob::Run():Inprogress");

Write Preview
Loading…
Cancel
Save