Dump Blob DB options to info log

Summary:
* Dump blob db options to info log
* Remove BlobDBOptionsImpl to disallow dynamic cast *BlobDBOptions into *BlobDBOptionsImpl. Move options there to be constants or into BlobDBOptions. The dynamic cast is broken after #2645
* Change some of the default options
* Remove blob_db_options.min_blob_size, which is unimplemented. Will implement it soon.
Closes https://github.com/facebook/rocksdb/pull/2671

Differential Revision: D5529912

Pulled By: yiwu-arbug

fbshipit-source-id: dcd58ca981db5bcc7f123b65a0d6f6ae0dc703c7
main
Yi Wu 7 years ago committed by Facebook Github Bot
parent 3218edc573
commit 1900771bd2
  1. 1
      CMakeLists.txt
  2. 1
      TARGETS
  3. 1
      src.mk
  4. 3
      tools/db_bench_tool.cc
  5. 85
      utilities/blob_db/blob_db.cc
  6. 32
      utilities/blob_db/blob_db.h
  7. 50
      utilities/blob_db/blob_db_impl.cc
  8. 48
      utilities/blob_db/blob_db_impl.h
  9. 69
      utilities/blob_db/blob_db_options_impl.cc
  10. 76
      utilities/blob_db/blob_db_options_impl.h
  11. 31
      utilities/blob_db/blob_db_test.cc

@ -476,7 +476,6 @@ set(SOURCES
utilities/backupable/backupable_db.cc utilities/backupable/backupable_db.cc
utilities/blob_db/blob_db.cc utilities/blob_db/blob_db.cc
utilities/blob_db/blob_db_impl.cc utilities/blob_db/blob_db_impl.cc
utilities/blob_db/blob_db_options_impl.cc
utilities/blob_db/blob_dump_tool.cc utilities/blob_db/blob_dump_tool.cc
utilities/blob_db/blob_file.cc utilities/blob_db/blob_file.cc
utilities/blob_db/blob_log_reader.cc utilities/blob_db/blob_log_reader.cc

@ -206,7 +206,6 @@ cpp_library(
"utilities/backupable/backupable_db.cc", "utilities/backupable/backupable_db.cc",
"utilities/blob_db/blob_db.cc", "utilities/blob_db/blob_db.cc",
"utilities/blob_db/blob_db_impl.cc", "utilities/blob_db/blob_db_impl.cc",
"utilities/blob_db/blob_db_options_impl.cc",
"utilities/blob_db/blob_file.cc", "utilities/blob_db/blob_file.cc",
"utilities/blob_db/blob_log_reader.cc", "utilities/blob_db/blob_log_reader.cc",
"utilities/blob_db/blob_log_writer.cc", "utilities/blob_db/blob_log_writer.cc",

@ -154,7 +154,6 @@ LIB_SOURCES = \
utilities/backupable/backupable_db.cc \ utilities/backupable/backupable_db.cc \
utilities/blob_db/blob_db.cc \ utilities/blob_db/blob_db.cc \
utilities/blob_db/blob_db_impl.cc \ utilities/blob_db/blob_db_impl.cc \
utilities/blob_db/blob_db_options_impl.cc \
utilities/blob_db/blob_file.cc \ utilities/blob_db/blob_file.cc \
utilities/blob_db/blob_log_reader.cc \ utilities/blob_db/blob_log_reader.cc \
utilities/blob_db/blob_log_writer.cc \ utilities/blob_db/blob_log_writer.cc \

@ -3311,10 +3311,7 @@ void VerifyDBFromDB(std::string& truth_db_name) {
} else if (FLAGS_use_blob_db) { } else if (FLAGS_use_blob_db) {
blob_db::BlobDBOptions blob_db_options; blob_db::BlobDBOptions blob_db_options;
blob_db::BlobDB* ptr; blob_db::BlobDB* ptr;
s = CreateLoggerFromOptions(db_name, options, &options.info_log);
if (s.ok()) {
s = blob_db::BlobDB::Open(options, blob_db_options, db_name, &ptr); s = blob_db::BlobDB::Open(options, blob_db_options, db_name, &ptr);
}
if (s.ok()) { if (s.ok()) {
db->db = ptr; db->db = ptr;
} }

@ -5,7 +5,14 @@
// //
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
#ifndef __STDC_FORMAT_MACROS
#define __STDC_FORMAT_MACROS
#endif
#include "utilities/blob_db/blob_db.h" #include "utilities/blob_db/blob_db.h"
#include <inttypes.h>
#include "db/write_batch_internal.h" #include "db/write_batch_internal.h"
#include "monitoring/instrumented_mutex.h" #include "monitoring/instrumented_mutex.h"
#include "options/cf_options.h" #include "options/cf_options.h"
@ -94,23 +101,31 @@ Status BlobDB::Open(const Options& options, const BlobDBOptions& bdb_options,
return s; return s;
} }
Status BlobDB::Open(const DBOptions& db_options, Status BlobDB::Open(const DBOptions& db_options_input,
const BlobDBOptions& bdb_options, const std::string& dbname, const BlobDBOptions& bdb_options, const std::string& dbname,
const std::vector<ColumnFamilyDescriptor>& column_families, const std::vector<ColumnFamilyDescriptor>& column_families,
std::vector<ColumnFamilyHandle*>* handles, BlobDB** blob_db, std::vector<ColumnFamilyHandle*>* handles, BlobDB** blob_db,
bool no_base_db) { bool no_base_db) {
*blob_db = nullptr; *blob_db = nullptr;
Status s;
DBOptions db_options(db_options_input);
if (db_options.info_log == nullptr) {
s = CreateLoggerFromOptions(dbname, db_options, &db_options.info_log);
if (!s.ok()) {
return s;
}
}
DBOptions my_db_options(db_options);
FlushBeginListener_t fblistener = FlushBeginListener_t fblistener =
std::make_shared<BlobDBFlushBeginListener>(); std::make_shared<BlobDBFlushBeginListener>();
CompactionListener_t ce_listener = CompactionListener_t ce_listener =
std::make_shared<EvictAllVersionsCompactionListener>(); std::make_shared<EvictAllVersionsCompactionListener>();
ReconcileWalFilter_t rw_filter = std::make_shared<BlobReconcileWalFilter>(); ReconcileWalFilter_t rw_filter = std::make_shared<BlobReconcileWalFilter>();
my_db_options.listeners.emplace_back(fblistener); db_options.listeners.emplace_back(fblistener);
my_db_options.listeners.emplace_back(ce_listener); db_options.listeners.emplace_back(ce_listener);
my_db_options.wal_filter = rw_filter.get(); db_options.wal_filter = rw_filter.get();
{ {
MutexLock l(&listener_mutex); MutexLock l(&listener_mutex);
@ -120,19 +135,25 @@ Status BlobDB::Open(const DBOptions& db_options,
} }
// we need to open blob db first so that recovery can happen // we need to open blob db first so that recovery can happen
BlobDBImpl* bdb = new BlobDBImpl(dbname, bdb_options, my_db_options); BlobDBImpl* bdb = new BlobDBImpl(dbname, bdb_options, db_options);
fblistener->SetImplPtr(bdb); fblistener->SetImplPtr(bdb);
ce_listener->SetImplPtr(bdb); ce_listener->SetImplPtr(bdb);
rw_filter->SetImplPtr(bdb); rw_filter->SetImplPtr(bdb);
Status s = bdb->OpenPhase1(); s = bdb->OpenPhase1();
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
if (no_base_db) return s; if (no_base_db) {
return s;
}
DB* db = nullptr; DB* db = nullptr;
s = DB::Open(my_db_options, dbname, column_families, handles, &db); s = DB::Open(db_options, dbname, column_families, handles, &db);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
// set the implementation pointer // set the implementation pointer
s = bdb->LinkToBaseDB(db); s = bdb->LinkToBaseDB(db);
@ -141,28 +162,36 @@ Status BlobDB::Open(const DBOptions& db_options,
bdb = nullptr; bdb = nullptr;
} }
*blob_db = bdb; *blob_db = bdb;
bdb_options.Dump(db_options.info_log.get());
return s; return s;
} }
BlobDB::BlobDB(DB* db) : StackableDB(db) {} BlobDB::BlobDB(DB* db) : StackableDB(db) {}
//////////////////////////////////////////////////////////////////////////////// void BlobDBOptions::Dump(Logger* log) const {
// ROCKS_LOG_HEADER(log, " blob_db_options.blob_dir: %s",
// blob_dir.c_str());
// std::function<int(double)> fnCaller = ROCKS_LOG_HEADER(log, " blob_db_options.path_relative: %d",
// std::bind(&A::fn, &anInstance, std::placeholders::_1); path_relative);
//////////////////////////////////////////////////////////////////////////////// ROCKS_LOG_HEADER(log, " blob_db_options.is_fifo: %d",
BlobDBOptions::BlobDBOptions() is_fifo);
: blob_dir("blob_dir"), ROCKS_LOG_HEADER(log, " blob_db_options.blob_dir_size: %" PRIu64,
path_relative(true), blob_dir_size);
is_fifo(false), ROCKS_LOG_HEADER(log, " blob_db_options.ttl_range_secs: %" PRIu32,
blob_dir_size(1000ULL * 1024ULL * 1024ULL * 1024ULL), ttl_range_secs);
ttl_range_secs(3600), ROCKS_LOG_HEADER(log, " blob_db_options.bytes_per_sync: %" PRIu64,
min_blob_size(512), bytes_per_sync);
bytes_per_sync(0), ROCKS_LOG_HEADER(log, " blob_db_options.blob_file_size: %" PRIu64,
blob_file_size(256 * 1024 * 1024), blob_file_size);
num_concurrent_simple_blobs(4), ROCKS_LOG_HEADER(log, "blob_db_options.num_concurrent_simple_blobs: %" PRIu32,
compression(kNoCompression) {} num_concurrent_simple_blobs);
ROCKS_LOG_HEADER(log, " blob_db_options.ttl_extractor: %p",
ttl_extractor.get());
ROCKS_LOG_HEADER(log, " blob_db_options.compression: %d",
static_cast<int>(compression));
ROCKS_LOG_HEADER(log, " blob_db_options.disable_background_tasks: %d",
disable_background_tasks);
}
} // namespace blob_db } // namespace blob_db
} // namespace rocksdb } // namespace rocksdb

@ -31,18 +31,18 @@ class TTLExtractor;
struct BlobDBOptions { struct BlobDBOptions {
// name of the directory under main db, where blobs will be stored. // name of the directory under main db, where blobs will be stored.
// default is "blob_dir" // default is "blob_dir"
std::string blob_dir; std::string blob_dir = "blob_dir";
// whether the blob_dir path is relative or absolute. // whether the blob_dir path is relative or absolute.
bool path_relative; bool path_relative = true;
// is the eviction strategy fifo based // is the eviction strategy fifo based
bool is_fifo; bool is_fifo = false;
// maximum size of the blob dir. Once this gets used, up // maximum size of the blob dir. Once this gets used, up
// evict the blob file which is oldest (is_fifo ) // evict the blob file which is oldest (is_fifo )
// 0 means no limits // 0 means no limits
uint64_t blob_dir_size; uint64_t blob_dir_size = 0;
// a new bucket is opened, for ttl_range. So if ttl_range is 600seconds // a new bucket is opened, for ttl_range. So if ttl_range is 600seconds
// (10 minutes), and the first bucket starts at 1471542000 // (10 minutes), and the first bucket starts at 1471542000
@ -50,26 +50,22 @@ struct BlobDBOptions {
// first bucket is 1471542000 - 1471542600 // first bucket is 1471542000 - 1471542600
// second bucket is 1471542600 - 1471543200 // second bucket is 1471542600 - 1471543200
// and so on // and so on
uint32_t ttl_range_secs; uint32_t ttl_range_secs = 3600;
// at what size will the blobs be stored in separate log rather than
// inline
uint64_t min_blob_size;
// at what bytes will the blob files be synced to blob log. // at what bytes will the blob files be synced to blob log.
uint64_t bytes_per_sync; uint64_t bytes_per_sync = 0;
// the target size of each blob file. File will become immutable // the target size of each blob file. File will become immutable
// after it exceeds that size // after it exceeds that size
uint64_t blob_file_size; uint64_t blob_file_size = 256 * 1024 * 1024;
// how many files to use for simple blobs at one time // how many files to use for simple blobs at one time
uint32_t num_concurrent_simple_blobs; uint32_t num_concurrent_simple_blobs = 1;
// Instead of setting TTL explicitly by calling PutWithTTL or PutUntil, // Instead of setting TTL explicitly by calling PutWithTTL or PutUntil,
// applications can set a TTLExtractor which can extract TTL from key-value // applications can set a TTLExtractor which can extract TTL from key-value
// pairs. // pairs.
std::shared_ptr<TTLExtractor> ttl_extractor; std::shared_ptr<TTLExtractor> ttl_extractor = nullptr;
// eviction callback. // eviction callback.
// this function will be called for every blob that is getting // this function will be called for every blob that is getting
@ -78,14 +74,12 @@ struct BlobDBOptions {
gc_evict_cb_fn; gc_evict_cb_fn;
// what compression to use for Blob's // what compression to use for Blob's
CompressionType compression; CompressionType compression = kNoCompression;
// default constructor
BlobDBOptions();
BlobDBOptions(const BlobDBOptions& in) = default; // Disable all background job.
bool disable_background_tasks = false;
virtual ~BlobDBOptions() = default; void Dump(Logger* log) const;
}; };
class BlobDB : public StackableDB { class BlobDB : public StackableDB {

@ -199,12 +199,6 @@ BlobDBImpl::BlobDBImpl(const std::string& dbname,
total_blob_space_(0), total_blob_space_(0),
open_p1_done_(false), open_p1_done_(false),
debug_level_(0) { debug_level_(0) {
const BlobDBOptionsImpl* options_impl =
static_cast_with_check<const BlobDBOptionsImpl, const BlobDBOptions>(
&blob_db_options);
if (options_impl) {
bdb_options_ = *options_impl;
}
blob_dir_ = (bdb_options_.path_relative) blob_dir_ = (bdb_options_.path_relative)
? dbname + "/" + bdb_options_.blob_dir ? dbname + "/" + bdb_options_.blob_dir
: bdb_options_.blob_dir; : bdb_options_.blob_dir;
@ -263,12 +257,6 @@ BlobDBImpl::BlobDBImpl(DB* db, const BlobDBOptions& blob_db_options)
total_periods_write_(0), total_periods_write_(0),
total_periods_ampl_(0), total_periods_ampl_(0),
total_blob_space_(0) { total_blob_space_(0) {
assert(db_impl_ != nullptr);
const BlobDBOptionsImpl* options_impl =
static_cast_with_check<const BlobDBOptionsImpl, const BlobDBOptions>(
&blob_db_options);
bdb_options_ = *options_impl;
if (!bdb_options_.blob_dir.empty()) if (!bdb_options_.blob_dir.empty())
blob_dir_ = (bdb_options_.path_relative) blob_dir_ = (bdb_options_.path_relative)
? db_->GetName() + "/" + bdb_options_.blob_dir ? db_->GetName() + "/" + bdb_options_.blob_dir
@ -304,27 +292,27 @@ Status BlobDBImpl::OpenPhase1() {
void BlobDBImpl::StartBackgroundTasks() { void BlobDBImpl::StartBackgroundTasks() {
// store a call to a member function and object // store a call to a member function and object
tqueue_.add( tqueue_.add(
bdb_options_.reclaim_of_period_millisecs, kReclaimOpenFilesPeriodMillisecs,
std::bind(&BlobDBImpl::ReclaimOpenFiles, this, std::placeholders::_1)); std::bind(&BlobDBImpl::ReclaimOpenFiles, this, std::placeholders::_1));
tqueue_.add(bdb_options_.gc_check_period_millisecs, tqueue_.add(kGCCheckPeriodMillisecs,
std::bind(&BlobDBImpl::RunGC, this, std::placeholders::_1)); std::bind(&BlobDBImpl::RunGC, this, std::placeholders::_1));
tqueue_.add( tqueue_.add(
bdb_options_.deletion_check_period_millisecs, kDeleteCheckPeriodMillisecs,
std::bind(&BlobDBImpl::EvictDeletions, this, std::placeholders::_1)); std::bind(&BlobDBImpl::EvictDeletions, this, std::placeholders::_1));
tqueue_.add( tqueue_.add(
bdb_options_.deletion_check_period_millisecs, kDeleteCheckPeriodMillisecs,
std::bind(&BlobDBImpl::EvictCompacted, this, std::placeholders::_1)); std::bind(&BlobDBImpl::EvictCompacted, this, std::placeholders::_1));
tqueue_.add( tqueue_.add(
bdb_options_.delete_obsf_period_millisecs, kDeleteObsoletedFilesPeriodMillisecs,
std::bind(&BlobDBImpl::DeleteObsFiles, this, std::placeholders::_1)); std::bind(&BlobDBImpl::DeleteObsFiles, this, std::placeholders::_1));
tqueue_.add(bdb_options_.sanity_check_period_millisecs, tqueue_.add(kSanityCheckPeriodMillisecs,
std::bind(&BlobDBImpl::SanityCheck, this, std::placeholders::_1)); std::bind(&BlobDBImpl::SanityCheck, this, std::placeholders::_1));
tqueue_.add(bdb_options_.wa_stats_period_millisecs, tqueue_.add(kWriteAmplificationStatsPeriodMillisecs,
std::bind(&BlobDBImpl::WaStats, this, std::placeholders::_1)); std::bind(&BlobDBImpl::WaStats, this, std::placeholders::_1));
tqueue_.add(bdb_options_.fsync_files_period_millisecs, tqueue_.add(kFSyncFilesPeriodMillisecs,
std::bind(&BlobDBImpl::FsyncFiles, this, std::placeholders::_1)); std::bind(&BlobDBImpl::FsyncFiles, this, std::placeholders::_1));
tqueue_.add( tqueue_.add(
bdb_options_.check_seqf_period_millisecs, kCheckSeqFilesPeriodMillisecs,
std::bind(&BlobDBImpl::CheckSeqFiles, this, std::placeholders::_1)); std::bind(&BlobDBImpl::CheckSeqFiles, this, std::placeholders::_1));
} }
@ -1606,8 +1594,9 @@ std::pair<bool, int64_t> BlobDBImpl::FsyncFiles(bool aborted) {
std::pair<bool, int64_t> BlobDBImpl::ReclaimOpenFiles(bool aborted) { std::pair<bool, int64_t> BlobDBImpl::ReclaimOpenFiles(bool aborted) {
if (aborted) return std::make_pair(false, -1); if (aborted) return std::make_pair(false, -1);
if (open_file_count_.load() < bdb_options_.open_files_trigger) if (open_file_count_.load() < kOpenFilesTrigger) {
return std::make_pair(true, -1); return std::make_pair(true, -1);
}
// in the future, we should sort by last_access_ // in the future, we should sort by last_access_
// instead of closing every file // instead of closing every file
@ -1628,7 +1617,7 @@ std::pair<bool, int64_t> BlobDBImpl::WaStats(bool aborted) {
WriteLock wl(&mutex_); WriteLock wl(&mutex_);
if (all_periods_write_.size() < bdb_options_.wa_num_stats_periods) { if (all_periods_write_.size() < kWriteAmplificationStatsPeriods) {
total_periods_write_ -= (*all_periods_write_.begin()); total_periods_write_ -= (*all_periods_write_.begin());
total_periods_ampl_ = (*all_periods_ampl_.begin()); total_periods_ampl_ = (*all_periods_ampl_.begin());
@ -1868,15 +1857,14 @@ bool BlobDBImpl::ShouldGCFile(std::shared_ptr<BlobFile> bfile, std::time_t tt,
return true; return true;
} }
if (bdb_options_.ttl_range_secs < if (bdb_options_.ttl_range_secs < kPartialExpirationGCRangeSecs) {
bdb_options_.partial_expiration_gc_range_secs) {
*reason = "has ttl but partial expiration not turned on"; *reason = "has ttl but partial expiration not turned on";
return false; return false;
} }
ReadLock lockbfile_r(&bfile->mutex_); ReadLock lockbfile_r(&bfile->mutex_);
bool ret = ((bfile->deleted_size_ * 100.0 / bfile->file_size_.load()) > bool ret = ((bfile->deleted_size_ * 100.0 / bfile->file_size_.load()) >
bdb_options_.partial_expiration_pct); kPartialExpirationPercentage);
if (ret) { if (ret) {
*reason = "deleted blobs beyond threshold"; *reason = "deleted blobs beyond threshold";
} else { } else {
@ -1895,13 +1883,14 @@ bool BlobDBImpl::ShouldGCFile(std::shared_ptr<BlobFile> bfile, std::time_t tt,
ReadLock lockbfile_r(&bfile->mutex_); ReadLock lockbfile_r(&bfile->mutex_);
if ((bfile->deleted_size_ * 100.0 / bfile->file_size_.load()) > if ((bfile->deleted_size_ * 100.0 / bfile->file_size_.load()) >
bdb_options_.partial_expiration_pct) { kPartialExpirationPercentage) {
*reason = "deleted simple blobs beyond threshold"; *reason = "deleted simple blobs beyond threshold";
return true; return true;
} }
// if we haven't reached limits of disk space, don't DELETE // if we haven't reached limits of disk space, don't DELETE
if (total_blob_space_.load() < bdb_options_.blob_dir_size) { if (bdb_options_.blob_dir_size == 0 ||
total_blob_space_.load() < bdb_options_.blob_dir_size) {
*reason = "disk space not exceeded"; *reason = "disk space not exceeded";
return false; return false;
} }
@ -2057,7 +2046,7 @@ void BlobDBImpl::FilterSubsetOfFiles(
uint64_t last_id, size_t files_to_collect) { uint64_t last_id, size_t files_to_collect) {
// 100.0 / 15.0 = 7 // 100.0 / 15.0 = 7
uint64_t next_epoch_increment = static_cast<uint64_t>( uint64_t next_epoch_increment = static_cast<uint64_t>(
std::ceil(100 / static_cast<double>(bdb_options_.gc_file_pct))); std::ceil(100 / static_cast<double>(kGCFilePercentage)));
std::chrono::system_clock::time_point now = std::chrono::system_clock::now(); std::chrono::system_clock::time_point now = std::chrono::system_clock::now();
std::time_t tt = std::chrono::system_clock::to_time_t(now); std::time_t tt = std::chrono::system_clock::to_time_t(now);
@ -2114,8 +2103,7 @@ std::pair<bool, int64_t> BlobDBImpl::RunGC(bool aborted) {
// 15% of files are collected each call to space out the IO and CPU // 15% of files are collected each call to space out the IO and CPU
// consumption. // consumption.
size_t files_to_collect = size_t files_to_collect = (kGCFilePercentage * blob_files.size()) / 100;
(bdb_options_.gc_file_pct * blob_files.size()) / 100;
std::vector<std::shared_ptr<BlobFile>> to_process; std::vector<std::shared_ptr<BlobFile>> to_process;
FilterSubsetOfFiles(blob_files, &to_process, current_epoch_, last_id, FilterSubsetOfFiles(blob_files, &to_process, current_epoch_, last_id,

@ -29,7 +29,6 @@
#include "util/mutexlock.h" #include "util/mutexlock.h"
#include "util/timer_queue.h" #include "util/timer_queue.h"
#include "utilities/blob_db/blob_db.h" #include "utilities/blob_db/blob_db.h"
#include "utilities/blob_db/blob_db_options_impl.h"
#include "utilities/blob_db/blob_log_format.h" #include "utilities/blob_db/blob_log_format.h"
#include "utilities/blob_db/blob_log_reader.h" #include "utilities/blob_db/blob_log_reader.h"
#include "utilities/blob_db/blob_log_writer.h" #include "utilities/blob_db/blob_log_writer.h"
@ -158,6 +157,51 @@ class BlobDBImpl : public BlobDB {
friend class BlobDBIterator; friend class BlobDBIterator;
public: public:
// deletions check period
static constexpr uint32_t kDeleteCheckPeriodMillisecs = 2 * 1000;
// gc percentage each check period
static constexpr uint32_t kGCFilePercentage = 100;
// gc period
static constexpr uint32_t kGCCheckPeriodMillisecs = 60 * 1000;
// sanity check task
static constexpr uint32_t kSanityCheckPeriodMillisecs = 20 * 60 * 1000;
// how many random access open files can we tolerate
static constexpr uint32_t kOpenFilesTrigger = 100;
// how many periods of stats do we keep.
static constexpr uint32_t kWriteAmplificationStatsPeriods = 24;
// what is the length of any period
static constexpr uint32_t kWriteAmplificationStatsPeriodMillisecs =
3600 * 1000;
// we will garbage collect blob files in
// which entire files have expired. However if the
// ttl_range of files is very large say a day, we
// would have to wait for the entire day, before we
// recover most of the space.
static constexpr uint32_t kPartialExpirationGCRangeSecs = 4 * 3600;
// this should be based on allowed Write Amplification
// if 50% of the space of a blob file has been deleted/expired,
static constexpr uint32_t kPartialExpirationPercentage = 75;
// how often should we schedule a job to fsync open files
static constexpr uint32_t kFSyncFilesPeriodMillisecs = 10 * 1000;
// how often to schedule reclaim open files.
static constexpr uint32_t kReclaimOpenFilesPeriodMillisecs = 1 * 1000;
// how often to schedule delete obs files periods
static constexpr uint32_t kDeleteObsoletedFilesPeriodMillisecs = 10 * 1000;
// how often to schedule check seq files period
static constexpr uint32_t kCheckSeqFilesPeriodMillisecs = 10 * 1000;
static constexpr uint64_t kNoExpiration = static constexpr uint64_t kNoExpiration =
std::numeric_limits<uint64_t>::max(); std::numeric_limits<uint64_t>::max();
@ -383,7 +427,7 @@ class BlobDBImpl : public BlobDB {
WriteOptions write_options_; WriteOptions write_options_;
// the options that govern the behavior of Blob Storage // the options that govern the behavior of Blob Storage
BlobDBOptionsImpl bdb_options_; BlobDBOptions bdb_options_;
DBOptions db_options_; DBOptions db_options_;
EnvOptions env_options_; EnvOptions env_options_;

@ -1,69 +0,0 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#ifndef ROCKSDB_LITE
#include "utilities/blob_db/blob_db_options_impl.h"
namespace rocksdb {
namespace blob_db {
BlobDBOptionsImpl::BlobDBOptionsImpl(const BlobDBOptions& in)
: BlobDBOptions(in),
deletion_check_period_millisecs(2 * 1000),
gc_file_pct(20),
gc_check_period_millisecs(60 * 1000),
sanity_check_period_millisecs(20 * 60 * 1000),
open_files_trigger(100),
wa_num_stats_periods(24),
wa_stats_period_millisecs(3600 * 1000),
partial_expiration_gc_range_secs(4 * 3600),
partial_expiration_pct(75),
fsync_files_period_millisecs(10 * 1000),
reclaim_of_period_millisecs(1 * 1000),
delete_obsf_period_millisecs(10 * 1000),
check_seqf_period_millisecs(10 * 1000),
disable_background_tasks(false) {}
BlobDBOptionsImpl::BlobDBOptionsImpl()
: deletion_check_period_millisecs(2 * 1000),
gc_file_pct(20),
gc_check_period_millisecs(60 * 1000),
sanity_check_period_millisecs(20 * 60 * 1000),
open_files_trigger(100),
wa_num_stats_periods(24),
wa_stats_period_millisecs(3600 * 1000),
partial_expiration_gc_range_secs(4 * 3600),
partial_expiration_pct(75),
fsync_files_period_millisecs(10 * 1000),
reclaim_of_period_millisecs(1 * 1000),
delete_obsf_period_millisecs(10 * 1000),
check_seqf_period_millisecs(10 * 1000),
disable_background_tasks(false) {}
BlobDBOptionsImpl& BlobDBOptionsImpl::operator=(const BlobDBOptionsImpl& in) {
BlobDBOptions::operator=(in);
if (this != &in) {
deletion_check_period_millisecs = in.deletion_check_period_millisecs;
gc_file_pct = in.gc_file_pct;
gc_check_period_millisecs = in.gc_check_period_millisecs;
sanity_check_period_millisecs = in.sanity_check_period_millisecs;
open_files_trigger = in.open_files_trigger;
wa_num_stats_periods = in.wa_num_stats_periods;
wa_stats_period_millisecs = in.wa_stats_period_millisecs;
partial_expiration_gc_range_secs = in.partial_expiration_gc_range_secs;
partial_expiration_pct = in.partial_expiration_pct;
fsync_files_period_millisecs = in.fsync_files_period_millisecs;
reclaim_of_period_millisecs = in.reclaim_of_period_millisecs;
delete_obsf_period_millisecs = in.delete_obsf_period_millisecs;
check_seqf_period_millisecs = in.check_seqf_period_millisecs;
disable_background_tasks = in.disable_background_tasks;
}
return *this;
}
} // namespace blob_db
} // namespace rocksdb
#endif // ROCKSDB_LITE

@ -1,76 +0,0 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#pragma once
#ifndef ROCKSDB_LITE
#include "utilities/blob_db/blob_db.h"
namespace rocksdb {
namespace blob_db {
struct BlobDBOptionsImpl : public BlobDBOptions {
// deletions check period
uint32_t deletion_check_period_millisecs;
// gc percentage each check period
uint32_t gc_file_pct;
// gc period
uint32_t gc_check_period_millisecs;
// sanity check task
uint32_t sanity_check_period_millisecs;
// how many random access open files can we tolerate
uint32_t open_files_trigger;
// how many periods of stats do we keep.
uint32_t wa_num_stats_periods;
// what is the length of any period
uint32_t wa_stats_period_millisecs;
// we will garbage collect blob files in
// which entire files have expired. However if the
// ttl_range of files is very large say a day, we
// would have to wait for the entire day, before we
// recover most of the space.
uint32_t partial_expiration_gc_range_secs;
// this should be based on allowed Write Amplification
// if 50% of the space of a blob file has been deleted/expired,
uint32_t partial_expiration_pct;
// how often should we schedule a job to fsync open files
uint32_t fsync_files_period_millisecs;
// how often to schedule reclaim open files.
uint32_t reclaim_of_period_millisecs;
// how often to schedule delete obs files periods
uint32_t delete_obsf_period_millisecs;
// how often to schedule check seq files period
uint32_t check_seqf_period_millisecs;
// Disable all background job.
bool disable_background_tasks;
// default constructor
BlobDBOptionsImpl();
explicit BlobDBOptionsImpl(const BlobDBOptions& in);
BlobDBOptionsImpl& operator=(const BlobDBOptionsImpl& in);
};
} // namespace blob_db
} // namespace rocksdb
#endif // endif ROCKSDB

@ -16,7 +16,6 @@
#include "util/string_util.h" #include "util/string_util.h"
#include "util/testharness.h" #include "util/testharness.h"
#include "utilities/blob_db/blob_db_impl.h" #include "utilities/blob_db/blob_db_impl.h"
#include "utilities/blob_db/blob_db_options_impl.h"
namespace rocksdb { namespace rocksdb {
namespace blob_db { namespace blob_db {
@ -47,7 +46,7 @@ class BlobDBTest : public testing::Test {
~BlobDBTest() { Destroy(); } ~BlobDBTest() { Destroy(); }
void Open(BlobDBOptionsImpl bdb_options = BlobDBOptionsImpl(), void Open(BlobDBOptions bdb_options = BlobDBOptions(),
Options options = Options()) { Options options = Options()) {
options.create_if_missing = true; options.create_if_missing = true;
ASSERT_OK(BlobDB::Open(options, bdb_options, dbname_, &blob_db_)); ASSERT_OK(BlobDB::Open(options, bdb_options, dbname_, &blob_db_));
@ -153,7 +152,7 @@ class BlobDBTest : public testing::Test {
TEST_F(BlobDBTest, Put) { TEST_F(BlobDBTest, Put) {
Random rnd(301); Random rnd(301);
BlobDBOptionsImpl bdb_options; BlobDBOptions bdb_options;
bdb_options.disable_background_tasks = true; bdb_options.disable_background_tasks = true;
Open(bdb_options); Open(bdb_options);
std::map<std::string, std::string> data; std::map<std::string, std::string> data;
@ -167,7 +166,7 @@ TEST_F(BlobDBTest, PutWithTTL) {
Random rnd(301); Random rnd(301);
Options options; Options options;
options.env = mock_env_.get(); options.env = mock_env_.get();
BlobDBOptionsImpl bdb_options; BlobDBOptions bdb_options;
bdb_options.ttl_range_secs = 1000; bdb_options.ttl_range_secs = 1000;
bdb_options.blob_file_size = 256 * 1000 * 1000; bdb_options.blob_file_size = 256 * 1000 * 1000;
bdb_options.disable_background_tasks = true; bdb_options.disable_background_tasks = true;
@ -196,7 +195,7 @@ TEST_F(BlobDBTest, PutUntil) {
Random rnd(301); Random rnd(301);
Options options; Options options;
options.env = mock_env_.get(); options.env = mock_env_.get();
BlobDBOptionsImpl bdb_options; BlobDBOptions bdb_options;
bdb_options.ttl_range_secs = 1000; bdb_options.ttl_range_secs = 1000;
bdb_options.blob_file_size = 256 * 1000 * 1000; bdb_options.blob_file_size = 256 * 1000 * 1000;
bdb_options.disable_background_tasks = true; bdb_options.disable_background_tasks = true;
@ -227,7 +226,7 @@ TEST_F(BlobDBTest, TTLExtrator_NoTTL) {
Random rnd(301); Random rnd(301);
Options options; Options options;
options.env = mock_env_.get(); options.env = mock_env_.get();
BlobDBOptionsImpl bdb_options; BlobDBOptions bdb_options;
bdb_options.ttl_range_secs = 1000; bdb_options.ttl_range_secs = 1000;
bdb_options.blob_file_size = 256 * 1000 * 1000; bdb_options.blob_file_size = 256 * 1000 * 1000;
bdb_options.num_concurrent_simple_blobs = 1; bdb_options.num_concurrent_simple_blobs = 1;
@ -275,7 +274,7 @@ TEST_F(BlobDBTest, TTLExtractor_ExtractTTL) {
ttl_extractor_.reset(new TestTTLExtractor(&rnd)); ttl_extractor_.reset(new TestTTLExtractor(&rnd));
Options options; Options options;
options.env = mock_env_.get(); options.env = mock_env_.get();
BlobDBOptionsImpl bdb_options; BlobDBOptions bdb_options;
bdb_options.ttl_range_secs = 1000; bdb_options.ttl_range_secs = 1000;
bdb_options.blob_file_size = 256 * 1000 * 1000; bdb_options.blob_file_size = 256 * 1000 * 1000;
bdb_options.ttl_extractor = ttl_extractor_; bdb_options.ttl_extractor = ttl_extractor_;
@ -322,7 +321,7 @@ TEST_F(BlobDBTest, TTLExtractor_ExtractExpiration) {
ttl_extractor_.reset(new TestTTLExtractor(&rnd)); ttl_extractor_.reset(new TestTTLExtractor(&rnd));
Options options; Options options;
options.env = mock_env_.get(); options.env = mock_env_.get();
BlobDBOptionsImpl bdb_options; BlobDBOptions bdb_options;
bdb_options.ttl_range_secs = 1000; bdb_options.ttl_range_secs = 1000;
bdb_options.blob_file_size = 256 * 1000 * 1000; bdb_options.blob_file_size = 256 * 1000 * 1000;
bdb_options.ttl_extractor = ttl_extractor_; bdb_options.ttl_extractor = ttl_extractor_;
@ -369,7 +368,7 @@ TEST_F(BlobDBTest, TTLExtractor_ChangeValue) {
Random rnd(301); Random rnd(301);
Options options; Options options;
options.env = mock_env_.get(); options.env = mock_env_.get();
BlobDBOptionsImpl bdb_options; BlobDBOptions bdb_options;
bdb_options.ttl_range_secs = 1000; bdb_options.ttl_range_secs = 1000;
bdb_options.blob_file_size = 256 * 1000 * 1000; bdb_options.blob_file_size = 256 * 1000 * 1000;
bdb_options.ttl_extractor = std::make_shared<TestTTLExtractor>(); bdb_options.ttl_extractor = std::make_shared<TestTTLExtractor>();
@ -404,7 +403,7 @@ TEST_F(BlobDBTest, TTLExtractor_ChangeValue) {
TEST_F(BlobDBTest, StackableDBGet) { TEST_F(BlobDBTest, StackableDBGet) {
Random rnd(301); Random rnd(301);
BlobDBOptionsImpl bdb_options; BlobDBOptions bdb_options;
bdb_options.disable_background_tasks = true; bdb_options.disable_background_tasks = true;
Open(bdb_options); Open(bdb_options);
std::map<std::string, std::string> data; std::map<std::string, std::string> data;
@ -426,7 +425,7 @@ TEST_F(BlobDBTest, StackableDBGet) {
TEST_F(BlobDBTest, WriteBatch) { TEST_F(BlobDBTest, WriteBatch) {
Random rnd(301); Random rnd(301);
BlobDBOptionsImpl bdb_options; BlobDBOptions bdb_options;
bdb_options.disable_background_tasks = true; bdb_options.disable_background_tasks = true;
Open(bdb_options); Open(bdb_options);
std::map<std::string, std::string> data; std::map<std::string, std::string> data;
@ -442,7 +441,7 @@ TEST_F(BlobDBTest, WriteBatch) {
TEST_F(BlobDBTest, Delete) { TEST_F(BlobDBTest, Delete) {
Random rnd(301); Random rnd(301);
BlobDBOptionsImpl bdb_options; BlobDBOptions bdb_options;
bdb_options.disable_background_tasks = true; bdb_options.disable_background_tasks = true;
Open(bdb_options); Open(bdb_options);
std::map<std::string, std::string> data; std::map<std::string, std::string> data;
@ -457,7 +456,7 @@ TEST_F(BlobDBTest, Delete) {
TEST_F(BlobDBTest, DeleteBatch) { TEST_F(BlobDBTest, DeleteBatch) {
Random rnd(301); Random rnd(301);
BlobDBOptionsImpl bdb_options; BlobDBOptions bdb_options;
bdb_options.disable_background_tasks = true; bdb_options.disable_background_tasks = true;
Open(bdb_options); Open(bdb_options);
for (size_t i = 0; i < 100; i++) { for (size_t i = 0; i < 100; i++) {
@ -474,7 +473,7 @@ TEST_F(BlobDBTest, DeleteBatch) {
TEST_F(BlobDBTest, Override) { TEST_F(BlobDBTest, Override) {
Random rnd(301); Random rnd(301);
BlobDBOptionsImpl bdb_options; BlobDBOptions bdb_options;
bdb_options.disable_background_tasks = true; bdb_options.disable_background_tasks = true;
Open(bdb_options); Open(bdb_options);
std::map<std::string, std::string> data; std::map<std::string, std::string> data;
@ -491,7 +490,7 @@ TEST_F(BlobDBTest, Override) {
#ifdef SNAPPY #ifdef SNAPPY
TEST_F(BlobDBTest, Compression) { TEST_F(BlobDBTest, Compression) {
Random rnd(301); Random rnd(301);
BlobDBOptionsImpl bdb_options; BlobDBOptions bdb_options;
bdb_options.disable_background_tasks = true; bdb_options.disable_background_tasks = true;
bdb_options.compression = CompressionType::kSnappyCompression; bdb_options.compression = CompressionType::kSnappyCompression;
Open(bdb_options); Open(bdb_options);
@ -528,7 +527,7 @@ TEST_F(BlobDBTest, DISABLED_MultipleWriters) {
// Test sequence number store in blob file is correct. // Test sequence number store in blob file is correct.
TEST_F(BlobDBTest, SequenceNumber) { TEST_F(BlobDBTest, SequenceNumber) {
Random rnd(301); Random rnd(301);
BlobDBOptionsImpl bdb_options; BlobDBOptions bdb_options;
bdb_options.disable_background_tasks = true; bdb_options.disable_background_tasks = true;
Open(bdb_options); Open(bdb_options);
SequenceNumber sequence = blob_db_->GetLatestSequenceNumber(); SequenceNumber sequence = blob_db_->GetLatestSequenceNumber();

Loading…
Cancel
Save