|
|
|
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
|
|
|
|
// This source code is licensed under the BSD-style license found in the
|
|
|
|
|
// LICENSE file in the root directory of this source tree. An additional grant
|
|
|
|
|
// of patent rights can be found in the PATENTS file in the same directory.
|
|
|
|
|
|
|
|
|
|
#include <map>
|
|
|
|
|
#include <string>
|
|
|
|
|
|
|
|
|
|
#include "db/compaction_job.h"
|
|
|
|
|
#include "db/column_family.h"
|
|
|
|
|
#include "db/version_set.h"
|
|
|
|
|
#include "db/writebuffer.h"
|
|
|
|
|
#include "rocksdb/cache.h"
|
|
|
|
|
#include "rocksdb/options.h"
|
|
|
|
|
#include "rocksdb/db.h"
|
|
|
|
|
#include "util/string_util.h"
|
|
|
|
|
#include "util/testharness.h"
|
|
|
|
|
#include "util/testutil.h"
|
|
|
|
|
#include "table/mock_table.h"
|
|
|
|
|
|
|
|
|
|
namespace rocksdb {
|
|
|
|
|
|
|
|
|
|
// TODO(icanadi) Make it simpler once we mock out VersionSet
|
|
|
|
|
class CompactionJobTest : public testing::Test {
|
|
|
|
|
public:
|
|
|
|
|
CompactionJobTest()
|
|
|
|
|
: env_(Env::Default()),
|
|
|
|
|
dbname_(test::TmpDir() + "/compaction_job_test"),
|
|
|
|
|
mutable_cf_options_(Options(), ImmutableCFOptions(Options())),
|
|
|
|
|
table_cache_(NewLRUCache(50000, 16)),
|
|
|
|
|
write_buffer_(db_options_.db_write_buffer_size),
|
|
|
|
|
versions_(new VersionSet(dbname_, &db_options_, env_options_,
|
|
|
|
|
table_cache_.get(), &write_buffer_,
|
|
|
|
|
&write_controller_)),
|
|
|
|
|
shutting_down_(false),
|
|
|
|
|
mock_table_factory_(new mock::MockTableFactory()) {
|
rocksdb: Replace ASSERT* with EXPECT* in functions that does not return void value
Summary:
gtest does not use exceptions to fail a unit test by design, and `ASSERT*`s are implemented using `return`. As a consequence we cannot use `ASSERT*` in a function that does not return `void` value ([[ https://code.google.com/p/googletest/wiki/AdvancedGuide#Assertion_Placement | 1]]), and have to fix our existing code. This diff does this in a generic way, with no manual changes.
In order to detect all existing `ASSERT*` that are used in functions that doesn't return void value, I change the code to generate compile errors for such cases.
In `util/testharness.h` I defined `EXPECT*` assertions, the same way as `ASSERT*`, and redefined `ASSERT*` to return `void`. Then executed:
```lang=bash
% USE_CLANG=1 make all -j55 -k 2> build.log
% perl -naF: -e 'print "-- -number=".$F[1]." ".$F[0]."\n" if /: error:/' \
build.log | xargs -L 1 perl -spi -e 's/ASSERT/EXPECT/g if $. == $number'
% make format
```
After that I reverted back change to `ASSERT*` in `util/testharness.h`. But preserved introduced `EXPECT*`, which is the same as `ASSERT*`. This will be deleted once switched to gtest.
This diff is independent and contains manual changes only in `util/testharness.h`.
Test Plan:
Make sure all tests are passing.
```lang=bash
% USE_CLANG=1 make check
```
Reviewers: igor, lgalanis, sdong, yufei.zhu, rven, meyering
Reviewed By: meyering
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D33333
10 years ago
|
|
|
EXPECT_OK(env_->CreateDirIfMissing(dbname_));
|
|
|
|
|
db_options_.db_paths.emplace_back(dbname_,
|
|
|
|
|
std::numeric_limits<uint64_t>::max());
|
|
|
|
|
NewDB();
|
|
|
|
|
std::vector<ColumnFamilyDescriptor> column_families;
|
|
|
|
|
cf_options_.table_factory = mock_table_factory_;
|
|
|
|
|
column_families.emplace_back(kDefaultColumnFamilyName, cf_options_);
|
|
|
|
|
|
rocksdb: Replace ASSERT* with EXPECT* in functions that does not return void value
Summary:
gtest does not use exceptions to fail a unit test by design, and `ASSERT*`s are implemented using `return`. As a consequence we cannot use `ASSERT*` in a function that does not return `void` value ([[ https://code.google.com/p/googletest/wiki/AdvancedGuide#Assertion_Placement | 1]]), and have to fix our existing code. This diff does this in a generic way, with no manual changes.
In order to detect all existing `ASSERT*` that are used in functions that doesn't return void value, I change the code to generate compile errors for such cases.
In `util/testharness.h` I defined `EXPECT*` assertions, the same way as `ASSERT*`, and redefined `ASSERT*` to return `void`. Then executed:
```lang=bash
% USE_CLANG=1 make all -j55 -k 2> build.log
% perl -naF: -e 'print "-- -number=".$F[1]." ".$F[0]."\n" if /: error:/' \
build.log | xargs -L 1 perl -spi -e 's/ASSERT/EXPECT/g if $. == $number'
% make format
```
After that I reverted back change to `ASSERT*` in `util/testharness.h`. But preserved introduced `EXPECT*`, which is the same as `ASSERT*`. This will be deleted once switched to gtest.
This diff is independent and contains manual changes only in `util/testharness.h`.
Test Plan:
Make sure all tests are passing.
```lang=bash
% USE_CLANG=1 make check
```
Reviewers: igor, lgalanis, sdong, yufei.zhu, rven, meyering
Reviewed By: meyering
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D33333
10 years ago
|
|
|
EXPECT_OK(versions_->Recover(column_families, false));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
std::string GenerateFileName(uint64_t file_number) {
|
|
|
|
|
FileMetaData meta;
|
|
|
|
|
std::vector<DbPath> db_paths;
|
|
|
|
|
db_paths.emplace_back(dbname_, std::numeric_limits<uint64_t>::max());
|
|
|
|
|
meta.fd = FileDescriptor(file_number, 0, 0);
|
|
|
|
|
return TableFileName(db_paths, meta.fd.GetNumber(), meta.fd.GetPathId());
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// returns expected result after compaction
|
|
|
|
|
mock::MockFileContents CreateTwoFiles() {
|
|
|
|
|
mock::MockFileContents expected_results;
|
|
|
|
|
const int kKeysPerFile = 10000;
|
|
|
|
|
SequenceNumber sequence_number = 0;
|
|
|
|
|
for (int i = 0; i < 2; ++i) {
|
|
|
|
|
mock::MockFileContents contents;
|
|
|
|
|
SequenceNumber smallest_seqno = 0, largest_seqno = 0;
|
|
|
|
|
InternalKey smallest, largest;
|
|
|
|
|
for (int k = 0; k < kKeysPerFile; ++k) {
|
|
|
|
|
auto key = ToString(i * (kKeysPerFile / 2) + k);
|
|
|
|
|
auto value = ToString(i * kKeysPerFile + k);
|
|
|
|
|
InternalKey internal_key(key, ++sequence_number, kTypeValue);
|
Make Compaction class easier to use
Summary:
The goal of this diff is to make Compaction class easier to use. This should also make new compaction algorithms easier to write (like CompactFiles from @yhchiang and dynamic leveled and multi-leveled universal from @sdong).
Here are couple of things demonstrating that Compaction class is hard to use:
1. we have two constructors of Compaction class
2. there's this thing called grandparents_, but it appears to only be setup for leveled compaction and not compactfiles
3. it's easy to introduce a subtle and dangerous bug like this: D36225
4. SetupBottomMostLevel() is hard to understand and it shouldn't be. See this comment: https://github.com/facebook/rocksdb/blob/afbafeaeaebfd27a0f3e992fee8e0c57d07658fa/db/compaction.cc#L236-L241. It also made it harder for @yhchiang to write CompactFiles, as evidenced by this: https://github.com/facebook/rocksdb/blob/afbafeaeaebfd27a0f3e992fee8e0c57d07658fa/db/compaction_picker.cc#L204-L210
The problem is that we create Compaction object, which holds a lot of state, and then pass it around to some functions. After those functions are done mutating, then we call couple of functions on Compaction object, like SetupBottommostLevel() and MarkFilesBeingCompacted(). It is very hard to see what's happening with all that Compaction's state while it's travelling across different functions. If you're writing a new PickCompaction() function you need to try really hard to understand what are all the functions you need to run on Compaction object and what state you need to setup.
My proposed solution is to make important parts of Compaction immutable after construction. PickCompaction() should calculate compaction inputs and then pass them onto Compaction object once they are finalized. That makes it easy to create a new compaction -- just provide all the parameters to the constructor and you're done. No need to call confusing functions after you created your object.
This diff doesn't fully achieve that goal, but it comes pretty close. Here are some of the changes:
* have one Compaction constructor instead of two.
* inputs_ is constant after construction
* MarkFilesBeingCompacted() is now private to Compaction class and automatically called on construction/destruction.
* SetupBottommostLevel() is gone. Compaction figures it out on its own based on the input.
* CompactionPicker's functions are not passing around Compaction object anymore. They are only passing around the state that they need.
Test Plan:
make check
make asan_check
make valgrind_check
Reviewers: rven, anthony, sdong, yhchiang
Reviewed By: yhchiang
Subscribers: sdong, yhchiang, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D36687
10 years ago
|
|
|
// This is how the key will look like once it's written in bottommost
|
|
|
|
|
// file
|
|
|
|
|
InternalKey bottommost_internal_key(key, 0, kTypeValue);
|
|
|
|
|
if (k == 0) {
|
|
|
|
|
smallest = internal_key;
|
|
|
|
|
smallest_seqno = sequence_number;
|
|
|
|
|
} else if (k == kKeysPerFile - 1) {
|
|
|
|
|
largest = internal_key;
|
|
|
|
|
largest_seqno = sequence_number;
|
|
|
|
|
}
|
|
|
|
|
std::pair<std::string, std::string> key_value(
|
Make Compaction class easier to use
Summary:
The goal of this diff is to make Compaction class easier to use. This should also make new compaction algorithms easier to write (like CompactFiles from @yhchiang and dynamic leveled and multi-leveled universal from @sdong).
Here are couple of things demonstrating that Compaction class is hard to use:
1. we have two constructors of Compaction class
2. there's this thing called grandparents_, but it appears to only be setup for leveled compaction and not compactfiles
3. it's easy to introduce a subtle and dangerous bug like this: D36225
4. SetupBottomMostLevel() is hard to understand and it shouldn't be. See this comment: https://github.com/facebook/rocksdb/blob/afbafeaeaebfd27a0f3e992fee8e0c57d07658fa/db/compaction.cc#L236-L241. It also made it harder for @yhchiang to write CompactFiles, as evidenced by this: https://github.com/facebook/rocksdb/blob/afbafeaeaebfd27a0f3e992fee8e0c57d07658fa/db/compaction_picker.cc#L204-L210
The problem is that we create Compaction object, which holds a lot of state, and then pass it around to some functions. After those functions are done mutating, then we call couple of functions on Compaction object, like SetupBottommostLevel() and MarkFilesBeingCompacted(). It is very hard to see what's happening with all that Compaction's state while it's travelling across different functions. If you're writing a new PickCompaction() function you need to try really hard to understand what are all the functions you need to run on Compaction object and what state you need to setup.
My proposed solution is to make important parts of Compaction immutable after construction. PickCompaction() should calculate compaction inputs and then pass them onto Compaction object once they are finalized. That makes it easy to create a new compaction -- just provide all the parameters to the constructor and you're done. No need to call confusing functions after you created your object.
This diff doesn't fully achieve that goal, but it comes pretty close. Here are some of the changes:
* have one Compaction constructor instead of two.
* inputs_ is constant after construction
* MarkFilesBeingCompacted() is now private to Compaction class and automatically called on construction/destruction.
* SetupBottommostLevel() is gone. Compaction figures it out on its own based on the input.
* CompactionPicker's functions are not passing around Compaction object anymore. They are only passing around the state that they need.
Test Plan:
make check
make asan_check
make valgrind_check
Reviewers: rven, anthony, sdong, yhchiang
Reviewed By: yhchiang
Subscribers: sdong, yhchiang, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D36687
10 years ago
|
|
|
{bottommost_internal_key.Encode().ToString(), value});
|
|
|
|
|
contents.insert(key_value);
|
|
|
|
|
if (i == 1 || k < kKeysPerFile / 2) {
|
|
|
|
|
expected_results.insert(key_value);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
uint64_t file_number = versions_->NewFileNumber();
|
rocksdb: Replace ASSERT* with EXPECT* in functions that does not return void value
Summary:
gtest does not use exceptions to fail a unit test by design, and `ASSERT*`s are implemented using `return`. As a consequence we cannot use `ASSERT*` in a function that does not return `void` value ([[ https://code.google.com/p/googletest/wiki/AdvancedGuide#Assertion_Placement | 1]]), and have to fix our existing code. This diff does this in a generic way, with no manual changes.
In order to detect all existing `ASSERT*` that are used in functions that doesn't return void value, I change the code to generate compile errors for such cases.
In `util/testharness.h` I defined `EXPECT*` assertions, the same way as `ASSERT*`, and redefined `ASSERT*` to return `void`. Then executed:
```lang=bash
% USE_CLANG=1 make all -j55 -k 2> build.log
% perl -naF: -e 'print "-- -number=".$F[1]." ".$F[0]."\n" if /: error:/' \
build.log | xargs -L 1 perl -spi -e 's/ASSERT/EXPECT/g if $. == $number'
% make format
```
After that I reverted back change to `ASSERT*` in `util/testharness.h`. But preserved introduced `EXPECT*`, which is the same as `ASSERT*`. This will be deleted once switched to gtest.
This diff is independent and contains manual changes only in `util/testharness.h`.
Test Plan:
Make sure all tests are passing.
```lang=bash
% USE_CLANG=1 make check
```
Reviewers: igor, lgalanis, sdong, yufei.zhu, rven, meyering
Reviewed By: meyering
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D33333
10 years ago
|
|
|
EXPECT_OK(mock_table_factory_->CreateMockTable(
|
|
|
|
|
env_, GenerateFileName(file_number), std::move(contents)));
|
|
|
|
|
|
|
|
|
|
VersionEdit edit;
|
|
|
|
|
edit.AddFile(0, file_number, 0, 10, smallest, largest, smallest_seqno,
|
|
|
|
|
largest_seqno);
|
|
|
|
|
|
|
|
|
|
mutex_.Lock();
|
|
|
|
|
versions_->LogAndApply(versions_->GetColumnFamilySet()->GetDefault(),
|
|
|
|
|
mutable_cf_options_, &edit, &mutex_);
|
|
|
|
|
mutex_.Unlock();
|
|
|
|
|
}
|
|
|
|
|
versions_->SetLastSequence(sequence_number);
|
|
|
|
|
return expected_results;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void NewDB() {
|
|
|
|
|
VersionEdit new_db;
|
|
|
|
|
new_db.SetLogNumber(0);
|
|
|
|
|
new_db.SetNextFile(2);
|
|
|
|
|
new_db.SetLastSequence(0);
|
|
|
|
|
|
|
|
|
|
const std::string manifest = DescriptorFileName(dbname_, 1);
|
|
|
|
|
unique_ptr<WritableFile> file;
|
|
|
|
|
Status s = env_->NewWritableFile(
|
|
|
|
|
manifest, &file, env_->OptimizeForManifestWrite(env_options_));
|
|
|
|
|
ASSERT_OK(s);
|
|
|
|
|
{
|
|
|
|
|
log::Writer log(std::move(file));
|
|
|
|
|
std::string record;
|
|
|
|
|
new_db.EncodeTo(&record);
|
|
|
|
|
s = log.AddRecord(record);
|
|
|
|
|
}
|
|
|
|
|
ASSERT_OK(s);
|
|
|
|
|
// Make "CURRENT" file that points to the new manifest file.
|
|
|
|
|
s = SetCurrentFile(env_, dbname_, 1, nullptr);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
Env* env_;
|
|
|
|
|
std::string dbname_;
|
|
|
|
|
EnvOptions env_options_;
|
|
|
|
|
MutableCFOptions mutable_cf_options_;
|
|
|
|
|
std::shared_ptr<Cache> table_cache_;
|
|
|
|
|
WriteController write_controller_;
|
|
|
|
|
DBOptions db_options_;
|
|
|
|
|
ColumnFamilyOptions cf_options_;
|
|
|
|
|
WriteBuffer write_buffer_;
|
|
|
|
|
std::unique_ptr<VersionSet> versions_;
|
|
|
|
|
InstrumentedMutex mutex_;
|
|
|
|
|
std::atomic<bool> shutting_down_;
|
|
|
|
|
std::shared_ptr<mock::MockTableFactory> mock_table_factory_;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
TEST_F(CompactionJobTest, Simple) {
|
|
|
|
|
auto cfd = versions_->GetColumnFamilySet()->GetDefault();
|
|
|
|
|
|
|
|
|
|
auto expected_results = CreateTwoFiles();
|
|
|
|
|
|
|
|
|
|
auto files = cfd->current()->storage_info()->LevelFiles(0);
|
|
|
|
|
ASSERT_EQ(2U, files.size());
|
|
|
|
|
|
Make Compaction class easier to use
Summary:
The goal of this diff is to make Compaction class easier to use. This should also make new compaction algorithms easier to write (like CompactFiles from @yhchiang and dynamic leveled and multi-leveled universal from @sdong).
Here are couple of things demonstrating that Compaction class is hard to use:
1. we have two constructors of Compaction class
2. there's this thing called grandparents_, but it appears to only be setup for leveled compaction and not compactfiles
3. it's easy to introduce a subtle and dangerous bug like this: D36225
4. SetupBottomMostLevel() is hard to understand and it shouldn't be. See this comment: https://github.com/facebook/rocksdb/blob/afbafeaeaebfd27a0f3e992fee8e0c57d07658fa/db/compaction.cc#L236-L241. It also made it harder for @yhchiang to write CompactFiles, as evidenced by this: https://github.com/facebook/rocksdb/blob/afbafeaeaebfd27a0f3e992fee8e0c57d07658fa/db/compaction_picker.cc#L204-L210
The problem is that we create Compaction object, which holds a lot of state, and then pass it around to some functions. After those functions are done mutating, then we call couple of functions on Compaction object, like SetupBottommostLevel() and MarkFilesBeingCompacted(). It is very hard to see what's happening with all that Compaction's state while it's travelling across different functions. If you're writing a new PickCompaction() function you need to try really hard to understand what are all the functions you need to run on Compaction object and what state you need to setup.
My proposed solution is to make important parts of Compaction immutable after construction. PickCompaction() should calculate compaction inputs and then pass them onto Compaction object once they are finalized. That makes it easy to create a new compaction -- just provide all the parameters to the constructor and you're done. No need to call confusing functions after you created your object.
This diff doesn't fully achieve that goal, but it comes pretty close. Here are some of the changes:
* have one Compaction constructor instead of two.
* inputs_ is constant after construction
* MarkFilesBeingCompacted() is now private to Compaction class and automatically called on construction/destruction.
* SetupBottommostLevel() is gone. Compaction figures it out on its own based on the input.
* CompactionPicker's functions are not passing around Compaction object anymore. They are only passing around the state that they need.
Test Plan:
make check
make asan_check
make valgrind_check
Reviewers: rven, anthony, sdong, yhchiang
Reviewed By: yhchiang
Subscribers: sdong, yhchiang, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D36687
10 years ago
|
|
|
CompactionInputFiles compaction_input_files;
|
|
|
|
|
compaction_input_files.level = 0;
|
|
|
|
|
compaction_input_files.files.push_back(files[0]);
|
|
|
|
|
compaction_input_files.files.push_back(files[1]);
|
|
|
|
|
std::unique_ptr<Compaction> compaction(new Compaction(
|
|
|
|
|
cfd->current()->storage_info(), *cfd->GetLatestMutableCFOptions(),
|
|
|
|
|
{compaction_input_files}, 1, 1024 * 1024, 10, 0, kNoCompression, {}));
|
|
|
|
|
compaction->SetInputVersion(cfd->current());
|
|
|
|
|
|
|
|
|
|
SnapshotList snapshots;
|
|
|
|
|
int yield_callback_called = 0;
|
|
|
|
|
std::function<uint64_t()> yield_callback = [&]() {
|
|
|
|
|
yield_callback_called++;
|
|
|
|
|
return 0;
|
|
|
|
|
};
|
|
|
|
|
LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, db_options_.info_log.get());
|
|
|
|
|
mutex_.Lock();
|
Include bunch of more events into EventLogger
Summary:
Added these events:
* Recovery start, finish and also when recovery creates a file
* Trivial move
* Compaction start, finish and when compaction creates a file
* Flush start, finish
Also includes small fix to EventLogger
Also added option ROCKSDB_PRINT_EVENTS_TO_STDOUT which is useful when we debug things. I've spent far too much time chasing LOG files.
Still didn't get sst table properties in JSON. They are written very deeply into the stack. I'll address in separate diff.
TODO:
* Write specification. Let's first use this for a while and figure out what's good data to put here, too. After that we'll write spec
* Write tools that parse and analyze LOGs. This can be in python or go. Good intern task.
Test Plan: Ran db_bench with ROCKSDB_PRINT_EVENTS_TO_STDOUT. Here's the output: https://phabricator.fb.com/P19811976
Reviewers: sdong, yhchiang, rven, MarkCallaghan, kradhakrishnan, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D37521
10 years ago
|
|
|
EventLogger event_logger(db_options_.info_log.get());
|
|
|
|
|
CompactionJob compaction_job(
|
|
|
|
|
0, compaction.get(), db_options_, *cfd->GetLatestMutableCFOptions(),
|
|
|
|
|
env_options_, versions_.get(), &shutting_down_, &log_buffer, nullptr,
|
|
|
|
|
nullptr, nullptr, &snapshots, true, table_cache_,
|
|
|
|
|
std::move(yield_callback), &event_logger);
|
|
|
|
|
compaction_job.Prepare();
|
|
|
|
|
mutex_.Unlock();
|
|
|
|
|
ASSERT_OK(compaction_job.Run());
|
|
|
|
|
mutex_.Lock();
|
|
|
|
|
Status s;
|
|
|
|
|
compaction_job.Install(&s, &mutex_);
|
|
|
|
|
ASSERT_OK(s);
|
|
|
|
|
mutex_.Unlock();
|
|
|
|
|
|
|
|
|
|
mock_table_factory_->AssertLatestFile(expected_results);
|
|
|
|
|
ASSERT_EQ(yield_callback_called, 20000);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
} // namespace rocksdb
|
|
|
|
|
|
|
|
|
|
int main(int argc, char** argv) {
|
|
|
|
|
::testing::InitGoogleTest(&argc, argv);
|
|
|
|
|
return RUN_ALL_TESTS();
|
|
|
|
|
}
|
