Add remote compaction public API (#8300)
Summary: Pull Request resolved: https://github.com/facebook/rocksdb/pull/8300 Reviewed By: ajkr Differential Revision: D28464726 Pulled By: jay-zhuang fbshipit-source-id: 49e9f4fb791808a6cbf39a7b1a331373f645fc5emain
Jay Zhuang
4 years ago
committed by
Facebook GitHub Bot
parent
311a544c2a
commit
3786181a90
@ -0,0 +1,454 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under both the GPLv2 (found in the
|
||||||
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
|
||||||
|
#ifndef ROCKSDB_LITE |
||||||
|
|
||||||
|
#include "db/db_test_util.h" |
||||||
|
#include "port/stack_trace.h" |
||||||
|
|
||||||
|
namespace ROCKSDB_NAMESPACE { |
||||||
|
|
||||||
|
class MyTestCompactionService : public CompactionService { |
||||||
|
public: |
||||||
|
MyTestCompactionService(const std::string& db_path, |
||||||
|
std::shared_ptr<FileSystem> fs, Options& options) |
||||||
|
: db_path_(db_path), fs_(fs), options_(options) {} |
||||||
|
|
||||||
|
CompactionServiceJobStatus Start(const std::string& compaction_service_input, |
||||||
|
int job_id) override { |
||||||
|
InstrumentedMutexLock l(&mutex_); |
||||||
|
jobs_.emplace(job_id, compaction_service_input); |
||||||
|
CompactionServiceJobStatus s = CompactionServiceJobStatus::kSuccess; |
||||||
|
TEST_SYNC_POINT_CALLBACK("MyTestCompactionService::Start::End", &s); |
||||||
|
return s; |
||||||
|
} |
||||||
|
|
||||||
|
CompactionServiceJobStatus WaitForComplete( |
||||||
|
int job_id, std::string* compaction_service_result) override { |
||||||
|
std::string compaction_input; |
||||||
|
{ |
||||||
|
InstrumentedMutexLock l(&mutex_); |
||||||
|
auto i = jobs_.find(job_id); |
||||||
|
if (i == jobs_.end()) { |
||||||
|
return CompactionServiceJobStatus::kFailure; |
||||||
|
} |
||||||
|
compaction_input = std::move(i->second); |
||||||
|
jobs_.erase(i); |
||||||
|
} |
||||||
|
|
||||||
|
CompactionServiceOptionsOverride options_override; |
||||||
|
options_override.env = options_.env; |
||||||
|
options_override.file_checksum_gen_factory = |
||||||
|
options_.file_checksum_gen_factory; |
||||||
|
options_override.comparator = options_.comparator; |
||||||
|
options_override.merge_operator = options_.merge_operator; |
||||||
|
options_override.compaction_filter = options_.compaction_filter; |
||||||
|
options_override.compaction_filter_factory = |
||||||
|
options_.compaction_filter_factory; |
||||||
|
options_override.prefix_extractor = options_.prefix_extractor; |
||||||
|
options_override.table_factory = options_.table_factory; |
||||||
|
options_override.sst_partitioner_factory = options_.sst_partitioner_factory; |
||||||
|
|
||||||
|
Status s = DB::OpenAndCompact(db_path_, db_path_ + "/" + ToString(job_id), |
||||||
|
compaction_input, compaction_service_result, |
||||||
|
options_override); |
||||||
|
TEST_SYNC_POINT_CALLBACK("MyTestCompactionService::WaitForComplete::End", |
||||||
|
compaction_service_result); |
||||||
|
compaction_num_.fetch_add(1); |
||||||
|
if (s.ok()) { |
||||||
|
return CompactionServiceJobStatus::kSuccess; |
||||||
|
} else { |
||||||
|
return CompactionServiceJobStatus::kFailure; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
int GetCompactionNum() { return compaction_num_.load(); } |
||||||
|
|
||||||
|
private: |
||||||
|
InstrumentedMutex mutex_; |
||||||
|
std::atomic_int compaction_num_{0}; |
||||||
|
std::map<int, std::string> jobs_; |
||||||
|
const std::string db_path_; |
||||||
|
std::shared_ptr<FileSystem> fs_; |
||||||
|
Options options_; |
||||||
|
}; |
||||||
|
|
||||||
|
class CompactionServiceTest : public DBTestBase { |
||||||
|
public: |
||||||
|
explicit CompactionServiceTest() |
||||||
|
: DBTestBase("compaction_service_test", true) {} |
||||||
|
|
||||||
|
protected: |
||||||
|
void GenerateTestData() { |
||||||
|
// Generate 20 files @ L2
|
||||||
|
for (int i = 0; i < 20; i++) { |
||||||
|
for (int j = 0; j < 10; j++) { |
||||||
|
int key_id = i * 10 + j; |
||||||
|
ASSERT_OK(Put(Key(key_id), "value" + ToString(key_id))); |
||||||
|
} |
||||||
|
ASSERT_OK(Flush()); |
||||||
|
} |
||||||
|
MoveFilesToLevel(2); |
||||||
|
|
||||||
|
// Generate 10 files @ L1 overlap with all 20 files @ L2
|
||||||
|
for (int i = 0; i < 10; i++) { |
||||||
|
for (int j = 0; j < 10; j++) { |
||||||
|
int key_id = i * 20 + j * 2; |
||||||
|
ASSERT_OK(Put(Key(key_id), "value_new" + ToString(key_id))); |
||||||
|
} |
||||||
|
ASSERT_OK(Flush()); |
||||||
|
} |
||||||
|
MoveFilesToLevel(1); |
||||||
|
ASSERT_EQ(FilesPerLevel(), "0,10,20"); |
||||||
|
} |
||||||
|
|
||||||
|
void VerifyTestData() { |
||||||
|
for (int i = 0; i < 200; i++) { |
||||||
|
auto result = Get(Key(i)); |
||||||
|
if (i % 2) { |
||||||
|
ASSERT_EQ(result, "value" + ToString(i)); |
||||||
|
} else { |
||||||
|
ASSERT_EQ(result, "value_new" + ToString(i)); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
}; |
||||||
|
|
||||||
|
TEST_F(CompactionServiceTest, BasicCompactions) { |
||||||
|
Options options = CurrentOptions(); |
||||||
|
options.env = env_; |
||||||
|
options.compaction_service = std::make_shared<MyTestCompactionService>( |
||||||
|
dbname_, env_->GetFileSystem(), options); |
||||||
|
|
||||||
|
DestroyAndReopen(options); |
||||||
|
|
||||||
|
for (int i = 0; i < 20; i++) { |
||||||
|
for (int j = 0; j < 10; j++) { |
||||||
|
int key_id = i * 10 + j; |
||||||
|
ASSERT_OK(Put(Key(key_id), "value" + ToString(key_id))); |
||||||
|
} |
||||||
|
ASSERT_OK(Flush()); |
||||||
|
} |
||||||
|
|
||||||
|
for (int i = 0; i < 10; i++) { |
||||||
|
for (int j = 0; j < 10; j++) { |
||||||
|
int key_id = i * 20 + j * 2; |
||||||
|
ASSERT_OK(Put(Key(key_id), "value_new" + ToString(key_id))); |
||||||
|
} |
||||||
|
ASSERT_OK(Flush()); |
||||||
|
} |
||||||
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
||||||
|
|
||||||
|
// verify result
|
||||||
|
for (int i = 0; i < 200; i++) { |
||||||
|
auto result = Get(Key(i)); |
||||||
|
if (i % 2) { |
||||||
|
ASSERT_EQ(result, "value" + ToString(i)); |
||||||
|
} else { |
||||||
|
ASSERT_EQ(result, "value_new" + ToString(i)); |
||||||
|
} |
||||||
|
} |
||||||
|
auto my_cs = |
||||||
|
dynamic_cast<MyTestCompactionService*>(options.compaction_service.get()); |
||||||
|
ASSERT_GE(my_cs->GetCompactionNum(), 1); |
||||||
|
|
||||||
|
// Test failed compaction
|
||||||
|
SyncPoint::GetInstance()->SetCallBack( |
||||||
|
"DBImplSecondary::CompactWithoutInstallation::End", [&](void* status) { |
||||||
|
// override job status
|
||||||
|
Status* s = static_cast<Status*>(status); |
||||||
|
*s = Status::Aborted("MyTestCompactionService failed to compact!"); |
||||||
|
}); |
||||||
|
SyncPoint::GetInstance()->EnableProcessing(); |
||||||
|
|
||||||
|
Status s; |
||||||
|
for (int i = 0; i < 10; i++) { |
||||||
|
for (int j = 0; j < 10; j++) { |
||||||
|
int key_id = i * 20 + j * 2; |
||||||
|
s = Put(Key(key_id), "value_new" + ToString(key_id)); |
||||||
|
if (s.IsAborted()) { |
||||||
|
break; |
||||||
|
} |
||||||
|
} |
||||||
|
if (s.IsAborted()) { |
||||||
|
break; |
||||||
|
} |
||||||
|
s = Flush(); |
||||||
|
if (s.IsAborted()) { |
||||||
|
break; |
||||||
|
} |
||||||
|
s = dbfull()->TEST_WaitForCompact(); |
||||||
|
if (s.IsAborted()) { |
||||||
|
break; |
||||||
|
} |
||||||
|
} |
||||||
|
ASSERT_TRUE(s.IsAborted()); |
||||||
|
} |
||||||
|
|
||||||
|
TEST_F(CompactionServiceTest, ManualCompaction) { |
||||||
|
Options options = CurrentOptions(); |
||||||
|
options.env = env_; |
||||||
|
options.disable_auto_compactions = true; |
||||||
|
options.compaction_service = std::make_shared<MyTestCompactionService>( |
||||||
|
dbname_, env_->GetFileSystem(), options); |
||||||
|
DestroyAndReopen(options); |
||||||
|
GenerateTestData(); |
||||||
|
|
||||||
|
auto my_cs = |
||||||
|
dynamic_cast<MyTestCompactionService*>(options.compaction_service.get()); |
||||||
|
|
||||||
|
std::string start_str = Key(15); |
||||||
|
std::string end_str = Key(45); |
||||||
|
Slice start(start_str); |
||||||
|
Slice end(end_str); |
||||||
|
uint64_t comp_num = my_cs->GetCompactionNum(); |
||||||
|
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &start, &end)); |
||||||
|
ASSERT_GE(my_cs->GetCompactionNum(), comp_num + 1); |
||||||
|
VerifyTestData(); |
||||||
|
|
||||||
|
start_str = Key(120); |
||||||
|
start = start_str; |
||||||
|
comp_num = my_cs->GetCompactionNum(); |
||||||
|
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &start, nullptr)); |
||||||
|
ASSERT_GE(my_cs->GetCompactionNum(), comp_num + 1); |
||||||
|
VerifyTestData(); |
||||||
|
|
||||||
|
end_str = Key(92); |
||||||
|
end = end_str; |
||||||
|
comp_num = my_cs->GetCompactionNum(); |
||||||
|
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, &end)); |
||||||
|
ASSERT_GE(my_cs->GetCompactionNum(), comp_num + 1); |
||||||
|
VerifyTestData(); |
||||||
|
|
||||||
|
comp_num = my_cs->GetCompactionNum(); |
||||||
|
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); |
||||||
|
ASSERT_GE(my_cs->GetCompactionNum(), comp_num + 1); |
||||||
|
VerifyTestData(); |
||||||
|
} |
||||||
|
|
||||||
|
TEST_F(CompactionServiceTest, FailedToStart) { |
||||||
|
Options options = CurrentOptions(); |
||||||
|
options.env = env_; |
||||||
|
options.disable_auto_compactions = true; |
||||||
|
options.compaction_service = std::make_shared<MyTestCompactionService>( |
||||||
|
dbname_, env_->GetFileSystem(), options); |
||||||
|
DestroyAndReopen(options); |
||||||
|
GenerateTestData(); |
||||||
|
|
||||||
|
SyncPoint::GetInstance()->SetCallBack( |
||||||
|
"MyTestCompactionService::Start::End", [&](void* status) { |
||||||
|
// override job status
|
||||||
|
auto s = static_cast<CompactionServiceJobStatus*>(status); |
||||||
|
*s = CompactionServiceJobStatus::kFailure; |
||||||
|
}); |
||||||
|
SyncPoint::GetInstance()->EnableProcessing(); |
||||||
|
|
||||||
|
std::string start_str = Key(15); |
||||||
|
std::string end_str = Key(45); |
||||||
|
Slice start(start_str); |
||||||
|
Slice end(end_str); |
||||||
|
Status s = db_->CompactRange(CompactRangeOptions(), &start, &end); |
||||||
|
ASSERT_TRUE(s.IsIncomplete()); |
||||||
|
} |
||||||
|
|
||||||
|
TEST_F(CompactionServiceTest, InvalidResult) { |
||||||
|
Options options = CurrentOptions(); |
||||||
|
options.env = env_; |
||||||
|
options.disable_auto_compactions = true; |
||||||
|
options.compaction_service = std::make_shared<MyTestCompactionService>( |
||||||
|
dbname_, env_->GetFileSystem(), options); |
||||||
|
DestroyAndReopen(options); |
||||||
|
GenerateTestData(); |
||||||
|
|
||||||
|
SyncPoint::GetInstance()->SetCallBack( |
||||||
|
"MyTestCompactionService::WaitForComplete::End", [&](void* result) { |
||||||
|
// override job status
|
||||||
|
auto result_str = static_cast<std::string*>(result); |
||||||
|
*result_str = "Invalid Str"; |
||||||
|
}); |
||||||
|
SyncPoint::GetInstance()->EnableProcessing(); |
||||||
|
|
||||||
|
std::string start_str = Key(15); |
||||||
|
std::string end_str = Key(45); |
||||||
|
Slice start(start_str); |
||||||
|
Slice end(end_str); |
||||||
|
Status s = db_->CompactRange(CompactRangeOptions(), &start, &end); |
||||||
|
ASSERT_FALSE(s.ok()); |
||||||
|
} |
||||||
|
|
||||||
|
// TODO: support sub-compaction
|
||||||
|
TEST_F(CompactionServiceTest, DISABLED_SubCompaction) { |
||||||
|
Options options = CurrentOptions(); |
||||||
|
options.env = env_; |
||||||
|
options.max_subcompactions = 10; |
||||||
|
options.target_file_size_base = 1 << 10; // 1KB
|
||||||
|
options.disable_auto_compactions = true; |
||||||
|
options.compaction_service = std::make_shared<MyTestCompactionService>( |
||||||
|
dbname_, env_->GetFileSystem(), options); |
||||||
|
|
||||||
|
DestroyAndReopen(options); |
||||||
|
GenerateTestData(); |
||||||
|
|
||||||
|
auto cro = CompactRangeOptions(); |
||||||
|
cro.max_subcompactions = 10; |
||||||
|
db_->CompactRange(cro, nullptr, nullptr); |
||||||
|
} |
||||||
|
|
||||||
|
class PartialDeleteCompactionFilter : public CompactionFilter { |
||||||
|
public: |
||||||
|
CompactionFilter::Decision FilterV2( |
||||||
|
int /*level*/, const Slice& key, ValueType /*value_type*/, |
||||||
|
const Slice& /*existing_value*/, std::string* /*new_value*/, |
||||||
|
std::string* /*skip_until*/) const override { |
||||||
|
int i = std::stoi(key.ToString().substr(3)); |
||||||
|
if (i > 5 && i <= 105) { |
||||||
|
return CompactionFilter::Decision::kRemove; |
||||||
|
} |
||||||
|
return CompactionFilter::Decision::kKeep; |
||||||
|
} |
||||||
|
|
||||||
|
const char* Name() const override { return "PartialDeleteCompactionFilter"; } |
||||||
|
}; |
||||||
|
|
||||||
|
TEST_F(CompactionServiceTest, CompactionFilter) { |
||||||
|
Options options = CurrentOptions(); |
||||||
|
options.env = env_; |
||||||
|
auto delete_comp_filter = PartialDeleteCompactionFilter(); |
||||||
|
options.compaction_filter = &delete_comp_filter; |
||||||
|
options.compaction_service = std::make_shared<MyTestCompactionService>( |
||||||
|
dbname_, env_->GetFileSystem(), options); |
||||||
|
|
||||||
|
DestroyAndReopen(options); |
||||||
|
|
||||||
|
for (int i = 0; i < 20; i++) { |
||||||
|
for (int j = 0; j < 10; j++) { |
||||||
|
int key_id = i * 10 + j; |
||||||
|
ASSERT_OK(Put(Key(key_id), "value" + ToString(key_id))); |
||||||
|
} |
||||||
|
ASSERT_OK(Flush()); |
||||||
|
} |
||||||
|
|
||||||
|
for (int i = 0; i < 10; i++) { |
||||||
|
for (int j = 0; j < 10; j++) { |
||||||
|
int key_id = i * 20 + j * 2; |
||||||
|
ASSERT_OK(Put(Key(key_id), "value_new" + ToString(key_id))); |
||||||
|
} |
||||||
|
ASSERT_OK(Flush()); |
||||||
|
} |
||||||
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
||||||
|
|
||||||
|
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); |
||||||
|
|
||||||
|
// verify result
|
||||||
|
for (int i = 0; i < 200; i++) { |
||||||
|
auto result = Get(Key(i)); |
||||||
|
if (i > 5 && i <= 105) { |
||||||
|
ASSERT_EQ(result, "NOT_FOUND"); |
||||||
|
} else if (i % 2) { |
||||||
|
ASSERT_EQ(result, "value" + ToString(i)); |
||||||
|
} else { |
||||||
|
ASSERT_EQ(result, "value_new" + ToString(i)); |
||||||
|
} |
||||||
|
} |
||||||
|
auto my_cs = |
||||||
|
dynamic_cast<MyTestCompactionService*>(options.compaction_service.get()); |
||||||
|
ASSERT_GE(my_cs->GetCompactionNum(), 1); |
||||||
|
} |
||||||
|
|
||||||
|
TEST_F(CompactionServiceTest, Snapshot) { |
||||||
|
Options options = CurrentOptions(); |
||||||
|
options.env = env_; |
||||||
|
options.compaction_service = std::make_shared<MyTestCompactionService>( |
||||||
|
dbname_, env_->GetFileSystem(), options); |
||||||
|
|
||||||
|
DestroyAndReopen(options); |
||||||
|
|
||||||
|
ASSERT_OK(Put(Key(1), "value1")); |
||||||
|
ASSERT_OK(Put(Key(2), "value1")); |
||||||
|
const Snapshot* s1 = db_->GetSnapshot(); |
||||||
|
ASSERT_OK(Flush()); |
||||||
|
|
||||||
|
ASSERT_OK(Put(Key(1), "value2")); |
||||||
|
ASSERT_OK(Put(Key(3), "value2")); |
||||||
|
ASSERT_OK(Flush()); |
||||||
|
|
||||||
|
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); |
||||||
|
auto my_cs = |
||||||
|
dynamic_cast<MyTestCompactionService*>(options.compaction_service.get()); |
||||||
|
ASSERT_GE(my_cs->GetCompactionNum(), 1); |
||||||
|
ASSERT_EQ("value1", Get(Key(1), s1)); |
||||||
|
ASSERT_EQ("value2", Get(Key(1))); |
||||||
|
db_->ReleaseSnapshot(s1); |
||||||
|
} |
||||||
|
|
||||||
|
TEST_F(CompactionServiceTest, ConcurrentCompaction) { |
||||||
|
Options options = CurrentOptions(); |
||||||
|
options.level0_file_num_compaction_trigger = 100; |
||||||
|
options.env = env_; |
||||||
|
options.compaction_service = std::make_shared<MyTestCompactionService>( |
||||||
|
dbname_, env_->GetFileSystem(), options); |
||||||
|
options.max_background_jobs = 20; |
||||||
|
|
||||||
|
DestroyAndReopen(options); |
||||||
|
GenerateTestData(); |
||||||
|
|
||||||
|
ColumnFamilyMetaData meta; |
||||||
|
db_->GetColumnFamilyMetaData(&meta); |
||||||
|
|
||||||
|
std::vector<std::thread> threads; |
||||||
|
for (const auto& file : meta.levels[1].files) { |
||||||
|
threads.push_back(std::thread([&]() { |
||||||
|
std::string fname = file.db_path + "/" + file.name; |
||||||
|
ASSERT_OK(db_->CompactFiles(CompactionOptions(), {fname}, 2)); |
||||||
|
})); |
||||||
|
} |
||||||
|
|
||||||
|
for (auto& thread : threads) { |
||||||
|
thread.join(); |
||||||
|
} |
||||||
|
ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
||||||
|
|
||||||
|
// verify result
|
||||||
|
for (int i = 0; i < 200; i++) { |
||||||
|
auto result = Get(Key(i)); |
||||||
|
if (i % 2) { |
||||||
|
ASSERT_EQ(result, "value" + ToString(i)); |
||||||
|
} else { |
||||||
|
ASSERT_EQ(result, "value_new" + ToString(i)); |
||||||
|
} |
||||||
|
} |
||||||
|
auto my_cs = |
||||||
|
dynamic_cast<MyTestCompactionService*>(options.compaction_service.get()); |
||||||
|
ASSERT_EQ(my_cs->GetCompactionNum(), 10); |
||||||
|
ASSERT_EQ(FilesPerLevel(), "0,0,10"); |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace ROCKSDB_NAMESPACE
|
||||||
|
|
||||||
|
#ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS |
||||||
|
extern "C" { |
||||||
|
void RegisterCustomObjects(int argc, char** argv); |
||||||
|
} |
||||||
|
#else |
||||||
|
void RegisterCustomObjects(int /*argc*/, char** /*argv*/) {} |
||||||
|
#endif // !ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
|
||||||
|
|
||||||
|
int main(int argc, char** argv) { |
||||||
|
ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); |
||||||
|
::testing::InitGoogleTest(&argc, argv); |
||||||
|
RegisterCustomObjects(argc, argv); |
||||||
|
return RUN_ALL_TESTS(); |
||||||
|
} |
||||||
|
|
||||||
|
#else |
||||||
|
#include <stdio.h> |
||||||
|
|
||||||
|
int main(int /*argc*/, char** /*argv*/) { |
||||||
|
fprintf(stderr, |
||||||
|
"SKIPPED as CompactionService is not supported in ROCKSDB_LITE\n"); |
||||||
|
return 0; |
||||||
|
} |
||||||
|
|
||||||
|
#endif // ROCKSDB_LITE
|
||||||
Loading…
Reference in new issue