|
|
@ -8,6 +8,7 @@ |
|
|
|
|
|
|
|
|
|
|
|
#include <mutex> |
|
|
|
#include <mutex> |
|
|
|
#include <string> |
|
|
|
#include <string> |
|
|
|
|
|
|
|
|
|
|
|
#include "rocksdb/db.h" |
|
|
|
#include "rocksdb/db.h" |
|
|
|
#include "rocksdb/env.h" |
|
|
|
#include "rocksdb/env.h" |
|
|
|
#include "rocksdb/options.h" |
|
|
|
#include "rocksdb/options.h" |
|
|
@ -39,8 +40,8 @@ class Compactor : public EventListener { |
|
|
|
// and column family. It is the caller's responsibility to
|
|
|
|
// and column family. It is the caller's responsibility to
|
|
|
|
// destroy the returned CompactionTask. Returns "nullptr"
|
|
|
|
// destroy the returned CompactionTask. Returns "nullptr"
|
|
|
|
// if it cannot find a proper compaction task.
|
|
|
|
// if it cannot find a proper compaction task.
|
|
|
|
virtual CompactionTask* PickCompaction( |
|
|
|
virtual CompactionTask* PickCompaction(DB* db, |
|
|
|
DB* db, const std::string& cf_name) = 0; |
|
|
|
const std::string& cf_name) = 0; |
|
|
|
|
|
|
|
|
|
|
|
// Schedule and run the specified compaction task in background.
|
|
|
|
// Schedule and run the specified compaction task in background.
|
|
|
|
virtual void ScheduleCompaction(CompactionTask* task) = 0; |
|
|
|
virtual void ScheduleCompaction(CompactionTask* task) = 0; |
|
|
@ -48,13 +49,11 @@ class Compactor : public EventListener { |
|
|
|
|
|
|
|
|
|
|
|
// Example structure that describes a compaction task.
|
|
|
|
// Example structure that describes a compaction task.
|
|
|
|
struct CompactionTask { |
|
|
|
struct CompactionTask { |
|
|
|
CompactionTask( |
|
|
|
CompactionTask(DB* _db, Compactor* _compactor, |
|
|
|
DB* _db, Compactor* _compactor, |
|
|
|
|
|
|
|
const std::string& _column_family_name, |
|
|
|
const std::string& _column_family_name, |
|
|
|
const std::vector<std::string>& _input_file_names, |
|
|
|
const std::vector<std::string>& _input_file_names, |
|
|
|
const int _output_level, |
|
|
|
const int _output_level, |
|
|
|
const CompactionOptions& _compact_options, |
|
|
|
const CompactionOptions& _compact_options, bool _retry_on_fail) |
|
|
|
bool _retry_on_fail) |
|
|
|
|
|
|
|
: db(_db), |
|
|
|
: db(_db), |
|
|
|
compactor(_compactor), |
|
|
|
compactor(_compactor), |
|
|
|
column_family_name(_column_family_name), |
|
|
|
column_family_name(_column_family_name), |
|
|
@ -77,15 +76,13 @@ class FullCompactor : public Compactor { |
|
|
|
public: |
|
|
|
public: |
|
|
|
explicit FullCompactor(const Options options) : options_(options) { |
|
|
|
explicit FullCompactor(const Options options) : options_(options) { |
|
|
|
compact_options_.compression = options_.compression; |
|
|
|
compact_options_.compression = options_.compression; |
|
|
|
compact_options_.output_file_size_limit = |
|
|
|
compact_options_.output_file_size_limit = options_.target_file_size_base; |
|
|
|
options_.target_file_size_base; |
|
|
|
|
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
// When flush happens, it determines whether to trigger compaction. If
|
|
|
|
// When flush happens, it determines whether to trigger compaction. If
|
|
|
|
// triggered_writes_stop is true, it will also set the retry flag of
|
|
|
|
// triggered_writes_stop is true, it will also set the retry flag of
|
|
|
|
// compaction-task to true.
|
|
|
|
// compaction-task to true.
|
|
|
|
void OnFlushCompleted( |
|
|
|
void OnFlushCompleted(DB* db, const FlushJobInfo& info) override { |
|
|
|
DB* db, const FlushJobInfo& info) override { |
|
|
|
|
|
|
|
CompactionTask* task = PickCompaction(db, info.cf_name); |
|
|
|
CompactionTask* task = PickCompaction(db, info.cf_name); |
|
|
|
if (task != nullptr) { |
|
|
|
if (task != nullptr) { |
|
|
|
if (info.triggered_writes_stop) { |
|
|
|
if (info.triggered_writes_stop) { |
|
|
@ -97,8 +94,7 @@ class FullCompactor : public Compactor { |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
// Always pick a compaction which includes all files whenever possible.
|
|
|
|
// Always pick a compaction which includes all files whenever possible.
|
|
|
|
CompactionTask* PickCompaction( |
|
|
|
CompactionTask* PickCompaction(DB* db, const std::string& cf_name) override { |
|
|
|
DB* db, const std::string& cf_name) override { |
|
|
|
|
|
|
|
ColumnFamilyMetaData cf_meta; |
|
|
|
ColumnFamilyMetaData cf_meta; |
|
|
|
db->GetColumnFamilyMetaData(&cf_meta); |
|
|
|
db->GetColumnFamilyMetaData(&cf_meta); |
|
|
|
|
|
|
|
|
|
|
@ -111,8 +107,7 @@ class FullCompactor : public Compactor { |
|
|
|
input_file_names.push_back(file.name); |
|
|
|
input_file_names.push_back(file.name); |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
return new CompactionTask( |
|
|
|
return new CompactionTask(db, this, cf_name, input_file_names, |
|
|
|
db, this, cf_name, input_file_names, |
|
|
|
|
|
|
|
options_.num_levels - 1, compact_options_, false); |
|
|
|
options_.num_levels - 1, compact_options_, false); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
@ -127,16 +122,14 @@ class FullCompactor : public Compactor { |
|
|
|
assert(task); |
|
|
|
assert(task); |
|
|
|
assert(task->db); |
|
|
|
assert(task->db); |
|
|
|
Status s = task->db->CompactFiles( |
|
|
|
Status s = task->db->CompactFiles( |
|
|
|
task->compact_options, |
|
|
|
task->compact_options, task->input_file_names, task->output_level); |
|
|
|
task->input_file_names, |
|
|
|
|
|
|
|
task->output_level); |
|
|
|
|
|
|
|
printf("CompactFiles() finished with status %s\n", s.ToString().c_str()); |
|
|
|
printf("CompactFiles() finished with status %s\n", s.ToString().c_str()); |
|
|
|
if (!s.ok() && !s.IsIOError() && task->retry_on_fail) { |
|
|
|
if (!s.ok() && !s.IsIOError() && task->retry_on_fail) { |
|
|
|
// If a compaction task with its retry_on_fail=true failed,
|
|
|
|
// If a compaction task with its retry_on_fail=true failed,
|
|
|
|
// try to schedule another compaction in case the reason
|
|
|
|
// try to schedule another compaction in case the reason
|
|
|
|
// is not an IO error.
|
|
|
|
// is not an IO error.
|
|
|
|
CompactionTask* new_task = task->compactor->PickCompaction( |
|
|
|
CompactionTask* new_task = |
|
|
|
task->db, task->column_family_name); |
|
|
|
task->compactor->PickCompaction(task->db, task->column_family_name); |
|
|
|
task->compactor->ScheduleCompaction(new_task); |
|
|
|
task->compactor->ScheduleCompaction(new_task); |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
@ -173,8 +166,7 @@ int main() { |
|
|
|
// verify the values are still there
|
|
|
|
// verify the values are still there
|
|
|
|
std::string value; |
|
|
|
std::string value; |
|
|
|
for (int i = 1000; i < 99999; ++i) { |
|
|
|
for (int i = 1000; i < 99999; ++i) { |
|
|
|
db->Get(ReadOptions(), std::to_string(i), |
|
|
|
db->Get(ReadOptions(), std::to_string(i), &value); |
|
|
|
&value); |
|
|
|
|
|
|
|
assert(value == std::string(500, 'a' + (i % 26))); |
|
|
|
assert(value == std::string(500, 'a' + (i % 26))); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|