|
|
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
|
|
// (found in the LICENSE.Apache file in the root directory).
|
|
|
|
|
|
|
|
#include <algorithm>
|
|
|
|
#include <map>
|
|
|
|
#include <string>
|
|
|
|
|
|
|
|
#include "db/column_family.h"
|
|
|
|
#include "db/flush_job.h"
|
|
|
|
#include "db/version_set.h"
|
|
|
|
#include "rocksdb/cache.h"
|
|
|
|
#include "rocksdb/write_buffer_manager.h"
|
|
|
|
#include "table/mock_table.h"
|
Move rate_limiter, write buffering, most perf context instrumentation and most random kill out of Env
Summary: We want to keep Env a think layer for better portability. Less platform dependent codes should be moved out of Env. In this patch, I create a wrapper of file readers and writers, and put rate limiting, write buffering, as well as most perf context instrumentation and random kill out of Env. It will make it easier to maintain multiple Env in the future.
Test Plan: Run all existing unit tests.
Reviewers: anthony, kradhakrishnan, IslamAbdelRahman, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D42321
9 years ago
|
|
|
#include "util/file_reader_writer.h"
|
|
|
|
#include "util/string_util.h"
|
|
|
|
#include "util/testharness.h"
|
|
|
|
#include "util/testutil.h"
|
|
|
|
|
|
|
|
namespace rocksdb {
|
|
|
|
|
|
|
|
// TODO(icanadi) Mock out everything else:
|
|
|
|
// 1. VersionSet
|
|
|
|
// 2. Memtable
|
|
|
|
class FlushJobTest : public testing::Test {
|
|
|
|
public:
|
|
|
|
FlushJobTest()
|
|
|
|
: env_(Env::Default()),
|
|
|
|
dbname_(test::PerThreadDBPath("flush_job_test")),
|
|
|
|
options_(),
|
|
|
|
db_options_(options_),
|
|
|
|
table_cache_(NewLRUCache(50000, 16)),
|
|
|
|
write_buffer_manager_(db_options_.db_write_buffer_size),
|
|
|
|
versions_(new VersionSet(dbname_, &db_options_, env_options_,
|
|
|
|
table_cache_.get(), &write_buffer_manager_,
|
|
|
|
&write_controller_)),
|
|
|
|
shutting_down_(false),
|
|
|
|
mock_table_factory_(new mock::MockTableFactory()) {
|
rocksdb: Replace ASSERT* with EXPECT* in functions that does not return void value
Summary:
gtest does not use exceptions to fail a unit test by design, and `ASSERT*`s are implemented using `return`. As a consequence we cannot use `ASSERT*` in a function that does not return `void` value ([[ https://code.google.com/p/googletest/wiki/AdvancedGuide#Assertion_Placement | 1]]), and have to fix our existing code. This diff does this in a generic way, with no manual changes.
In order to detect all existing `ASSERT*` that are used in functions that doesn't return void value, I change the code to generate compile errors for such cases.
In `util/testharness.h` I defined `EXPECT*` assertions, the same way as `ASSERT*`, and redefined `ASSERT*` to return `void`. Then executed:
```lang=bash
% USE_CLANG=1 make all -j55 -k 2> build.log
% perl -naF: -e 'print "-- -number=".$F[1]." ".$F[0]."\n" if /: error:/' \
build.log | xargs -L 1 perl -spi -e 's/ASSERT/EXPECT/g if $. == $number'
% make format
```
After that I reverted back change to `ASSERT*` in `util/testharness.h`. But preserved introduced `EXPECT*`, which is the same as `ASSERT*`. This will be deleted once switched to gtest.
This diff is independent and contains manual changes only in `util/testharness.h`.
Test Plan:
Make sure all tests are passing.
```lang=bash
% USE_CLANG=1 make check
```
Reviewers: igor, lgalanis, sdong, yufei.zhu, rven, meyering
Reviewed By: meyering
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D33333
10 years ago
|
|
|
EXPECT_OK(env_->CreateDirIfMissing(dbname_));
|
|
|
|
db_options_.db_paths.emplace_back(dbname_,
|
|
|
|
std::numeric_limits<uint64_t>::max());
|
|
|
|
db_options_.statistics = rocksdb::CreateDBStatistics();
|
|
|
|
// TODO(icanadi) Remove this once we mock out VersionSet
|
|
|
|
NewDB();
|
|
|
|
std::vector<ColumnFamilyDescriptor> column_families;
|
|
|
|
cf_options_.table_factory = mock_table_factory_;
|
|
|
|
column_families.emplace_back(kDefaultColumnFamilyName, cf_options_);
|
|
|
|
|
rocksdb: Replace ASSERT* with EXPECT* in functions that does not return void value
Summary:
gtest does not use exceptions to fail a unit test by design, and `ASSERT*`s are implemented using `return`. As a consequence we cannot use `ASSERT*` in a function that does not return `void` value ([[ https://code.google.com/p/googletest/wiki/AdvancedGuide#Assertion_Placement | 1]]), and have to fix our existing code. This diff does this in a generic way, with no manual changes.
In order to detect all existing `ASSERT*` that are used in functions that doesn't return void value, I change the code to generate compile errors for such cases.
In `util/testharness.h` I defined `EXPECT*` assertions, the same way as `ASSERT*`, and redefined `ASSERT*` to return `void`. Then executed:
```lang=bash
% USE_CLANG=1 make all -j55 -k 2> build.log
% perl -naF: -e 'print "-- -number=".$F[1]." ".$F[0]."\n" if /: error:/' \
build.log | xargs -L 1 perl -spi -e 's/ASSERT/EXPECT/g if $. == $number'
% make format
```
After that I reverted back change to `ASSERT*` in `util/testharness.h`. But preserved introduced `EXPECT*`, which is the same as `ASSERT*`. This will be deleted once switched to gtest.
This diff is independent and contains manual changes only in `util/testharness.h`.
Test Plan:
Make sure all tests are passing.
```lang=bash
% USE_CLANG=1 make check
```
Reviewers: igor, lgalanis, sdong, yufei.zhu, rven, meyering
Reviewed By: meyering
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D33333
10 years ago
|
|
|
EXPECT_OK(versions_->Recover(column_families, false));
|
|
|
|
}
|
|
|
|
|
|
|
|
void NewDB() {
|
|
|
|
VersionEdit new_db;
|
|
|
|
new_db.SetLogNumber(0);
|
|
|
|
new_db.SetNextFile(2);
|
|
|
|
new_db.SetLastSequence(0);
|
|
|
|
|
|
|
|
const std::string manifest = DescriptorFileName(dbname_, 1);
|
|
|
|
unique_ptr<WritableFile> file;
|
|
|
|
Status s = env_->NewWritableFile(
|
|
|
|
manifest, &file, env_->OptimizeForManifestWrite(env_options_));
|
|
|
|
ASSERT_OK(s);
|
Move rate_limiter, write buffering, most perf context instrumentation and most random kill out of Env
Summary: We want to keep Env a think layer for better portability. Less platform dependent codes should be moved out of Env. In this patch, I create a wrapper of file readers and writers, and put rate limiting, write buffering, as well as most perf context instrumentation and random kill out of Env. It will make it easier to maintain multiple Env in the future.
Test Plan: Run all existing unit tests.
Reviewers: anthony, kradhakrishnan, IslamAbdelRahman, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D42321
9 years ago
|
|
|
unique_ptr<WritableFileWriter> file_writer(
|
|
|
|
new WritableFileWriter(std::move(file), EnvOptions()));
|
|
|
|
{
|
|
|
|
log::Writer log(std::move(file_writer), 0, false);
|
|
|
|
std::string record;
|
|
|
|
new_db.EncodeTo(&record);
|
|
|
|
s = log.AddRecord(record);
|
|
|
|
}
|
|
|
|
ASSERT_OK(s);
|
|
|
|
// Make "CURRENT" file that points to the new manifest file.
|
|
|
|
s = SetCurrentFile(env_, dbname_, 1, nullptr);
|
|
|
|
}
|
|
|
|
|
|
|
|
Env* env_;
|
|
|
|
std::string dbname_;
|
|
|
|
EnvOptions env_options_;
|
|
|
|
Options options_;
|
|
|
|
ImmutableDBOptions db_options_;
|
|
|
|
std::shared_ptr<Cache> table_cache_;
|
|
|
|
WriteController write_controller_;
|
|
|
|
WriteBufferManager write_buffer_manager_;
|
|
|
|
ColumnFamilyOptions cf_options_;
|
|
|
|
std::unique_ptr<VersionSet> versions_;
|
|
|
|
InstrumentedMutex mutex_;
|
|
|
|
std::atomic<bool> shutting_down_;
|
|
|
|
std::shared_ptr<mock::MockTableFactory> mock_table_factory_;
|
|
|
|
};
|
|
|
|
|
|
|
|
TEST_F(FlushJobTest, Empty) {
|
|
|
|
JobContext job_context(0);
|
|
|
|
auto cfd = versions_->GetColumnFamilySet()->GetDefault();
|
EventLogger
Summary:
Here's my proposal for making our LOGs easier to read by machines.
The idea is to dump all events as JSON objects. JSON is easy to read by humans, but more importantly, it's easy to read by machines. That way, we can parse this, load into SQLite/mongo and then query or visualize.
I started with table_create and table_delete events, but if everybody agrees, I'll continue by adding more events (flush/compaction/etc etc)
Test Plan:
Ran db_bench. Observed:
2015/01/15-14:13:25.788019 1105ef000 EVENT_LOG_v1 {"time_micros": 1421360005788015, "event": "table_file_creation", "file_number": 12, "file_size": 1909699}
2015/01/15-14:13:25.956500 110740000 EVENT_LOG_v1 {"time_micros": 1421360005956498, "event": "table_file_deletion", "file_number": 12}
Reviewers: yhchiang, rven, dhruba, MarkCallaghan, lgalanis, sdong
Reviewed By: sdong
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31647
10 years ago
|
|
|
EventLogger event_logger(db_options_.info_log.get());
|
|
|
|
SnapshotChecker* snapshot_checker = nullptr; // not relavant
|
|
|
|
FlushJob flush_job(
|
|
|
|
dbname_, versions_->GetColumnFamilySet()->GetDefault(), db_options_,
|
|
|
|
*cfd->GetLatestMutableCFOptions(), env_options_, versions_.get(), &mutex_,
|
|
|
|
&shutting_down_, {}, kMaxSequenceNumber, snapshot_checker, &job_context,
|
|
|
|
nullptr, nullptr, nullptr, kNoCompression, nullptr, &event_logger, false);
|
|
|
|
{
|
|
|
|
InstrumentedMutexLock l(&mutex_);
|
|
|
|
flush_job.PickMemTable();
|
|
|
|
ASSERT_OK(flush_job.Run());
|
|
|
|
}
|
|
|
|
job_context.Clean();
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST_F(FlushJobTest, NonEmpty) {
|
|
|
|
JobContext job_context(0);
|
|
|
|
auto cfd = versions_->GetColumnFamilySet()->GetDefault();
|
|
|
|
auto new_mem = cfd->ConstructNewMemtable(*cfd->GetLatestMutableCFOptions(),
|
|
|
|
kMaxSequenceNumber);
|
|
|
|
new_mem->Ref();
|
|
|
|
auto inserted_keys = mock::MakeMockFile();
|
|
|
|
// Test data:
|
|
|
|
// seqno [ 1, 2 ... 8998, 8999, 9000, 9001, 9002 ... 9999 ]
|
|
|
|
// key [ 1001, 1002 ... 9998, 9999, 0, 1, 2 ... 999 ]
|
|
|
|
// range-delete "9995" -> "9999" at seqno 10000
|
|
|
|
for (int i = 1; i < 10000; ++i) {
|
|
|
|
std::string key(ToString((i + 1000) % 10000));
|
|
|
|
std::string value("value" + key);
|
|
|
|
new_mem->Add(SequenceNumber(i), kTypeValue, key, value);
|
|
|
|
if ((i + 1000) % 10000 < 9995) {
|
|
|
|
InternalKey internal_key(key, SequenceNumber(i), kTypeValue);
|
|
|
|
inserted_keys.insert({internal_key.Encode().ToString(), value});
|
|
|
|
}
|
|
|
|
}
|
|
|
|
new_mem->Add(SequenceNumber(10000), kTypeRangeDeletion, "9995", "9999a");
|
|
|
|
InternalKey internal_key("9995", SequenceNumber(10000), kTypeRangeDeletion);
|
|
|
|
inserted_keys.insert({internal_key.Encode().ToString(), "9999a"});
|
Support saving history in memtable_list
Summary:
For transactions, we are using the memtables to validate that there are no write conflicts. But after flushing, we don't have any memtables, and transactions could fail to commit. So we want to someone keep around some extra history to use for conflict checking. In addition, we want to provide a way to increase the size of this history if too many transactions fail to commit.
After chatting with people, it seems like everyone prefers just using Memtables to store this history (instead of a separate history structure). It seems like the best place for this is abstracted inside the memtable_list. I decide to create a separate list in MemtableListVersion as using the same list complicated the flush/installalflushresults logic too much.
This diff adds a new parameter to control how much memtable history to keep around after flushing. However, it sounds like people aren't too fond of adding new parameters. So I am making the default size of flushed+not-flushed memtables be set to max_write_buffers. This should not change the maximum amount of memory used, but make it more likely we're using closer the the limit. (We are now postponing deleting flushed memtables until the max_write_buffer limit is reached). So while we might use more memory on average, we are still obeying the limit set (and you could argue it's better to go ahead and use up memory now instead of waiting for a write stall to happen to test this limit).
However, if people are opposed to this default behavior, we can easily set it to 0 and require this parameter be set in order to use transactions.
Test Plan: Added a xfunc test to play around with setting different values of this parameter in all tests. Added testing in memtablelist_test and planning on adding more testing here.
Reviewers: sdong, rven, igor
Reviewed By: igor
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D37443
9 years ago
|
|
|
|
|
|
|
autovector<MemTable*> to_delete;
|
|
|
|
cfd->imm()->Add(new_mem, &to_delete);
|
|
|
|
for (auto& m : to_delete) {
|
|
|
|
delete m;
|
|
|
|
}
|
|
|
|
|
EventLogger
Summary:
Here's my proposal for making our LOGs easier to read by machines.
The idea is to dump all events as JSON objects. JSON is easy to read by humans, but more importantly, it's easy to read by machines. That way, we can parse this, load into SQLite/mongo and then query or visualize.
I started with table_create and table_delete events, but if everybody agrees, I'll continue by adding more events (flush/compaction/etc etc)
Test Plan:
Ran db_bench. Observed:
2015/01/15-14:13:25.788019 1105ef000 EVENT_LOG_v1 {"time_micros": 1421360005788015, "event": "table_file_creation", "file_number": 12, "file_size": 1909699}
2015/01/15-14:13:25.956500 110740000 EVENT_LOG_v1 {"time_micros": 1421360005956498, "event": "table_file_deletion", "file_number": 12}
Reviewers: yhchiang, rven, dhruba, MarkCallaghan, lgalanis, sdong
Reviewed By: sdong
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31647
10 years ago
|
|
|
EventLogger event_logger(db_options_.info_log.get());
|
|
|
|
SnapshotChecker* snapshot_checker = nullptr; // not relavant
|
|
|
|
FlushJob flush_job(dbname_, versions_->GetColumnFamilySet()->GetDefault(),
|
|
|
|
db_options_, *cfd->GetLatestMutableCFOptions(),
|
|
|
|
env_options_, versions_.get(), &mutex_, &shutting_down_,
|
|
|
|
{}, kMaxSequenceNumber, snapshot_checker, &job_context,
|
|
|
|
nullptr, nullptr, nullptr, kNoCompression,
|
|
|
|
db_options_.statistics.get(), &event_logger, true);
|
|
|
|
|
|
|
|
HistogramData hist;
|
|
|
|
FileMetaData fd;
|
|
|
|
mutex_.Lock();
|
|
|
|
flush_job.PickMemTable();
|
Skip deleted WALs during recovery
Summary:
This patch record min log number to keep to the manifest while flushing SST files to ignore them and any WAL older than them during recovery. This is to avoid scenarios when we have a gap between the WAL files are fed to the recovery procedure. The gap could happen by for example out-of-order WAL deletion. Such gap could cause problems in 2PC recovery where the prepared and commit entry are placed into two separate WAL and gap in the WALs could result into not processing the WAL with the commit entry and hence breaking the 2PC recovery logic.
Before the commit, for 2PC case, we determined which log number to keep in FindObsoleteFiles(). We looked at the earliest logs with outstanding prepare entries, or prepare entries whose respective commit or abort are in memtable. With the commit, the same calculation is done while we apply the SST flush. Just before installing the flush file, we precompute the earliest log file to keep after the flush finishes using the same logic (but skipping the memtables just flushed), record this information to the manifest entry for this new flushed SST file. This pre-computed value is also remembered in memory, and will later be used to determine whether a log file can be deleted. This value is unlikely to change until next flush because the commit entry will stay in memtable. (In WritePrepared, we could have removed the older log files as soon as all prepared entries are committed. It's not yet done anyway. Even if we do it, the only thing we loss with this new approach is earlier log deletion between two flushes, which does not guarantee to happen anyway because the obsolete file clean-up function is only executed after flush or compaction)
This min log number to keep is stored in the manifest using the safely-ignore customized field of AddFile entry, in order to guarantee that the DB generated using newer release can be opened by previous releases no older than 4.2.
Closes https://github.com/facebook/rocksdb/pull/3765
Differential Revision: D7747618
Pulled By: siying
fbshipit-source-id: d00c92105b4f83852e9754a1b70d6b64cb590729
6 years ago
|
|
|
ASSERT_OK(flush_job.Run(nullptr, &fd));
|
|
|
|
mutex_.Unlock();
|
|
|
|
db_options_.statistics->histogramData(FLUSH_TIME, &hist);
|
|
|
|
ASSERT_GT(hist.average, 0.0);
|
|
|
|
|
|
|
|
ASSERT_EQ(ToString(0), fd.smallest.user_key().ToString());
|
|
|
|
ASSERT_EQ("9999a",
|
|
|
|
fd.largest.user_key().ToString()); // range tombstone end key
|
|
|
|
ASSERT_EQ(1, fd.smallest_seqno);
|
|
|
|
ASSERT_EQ(10000, fd.largest_seqno); // range tombstone seqnum 10000
|
|
|
|
mock_table_factory_->AssertSingleFile(inserted_keys);
|
|
|
|
job_context.Clean();
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST_F(FlushJobTest, Snapshots) {
|
|
|
|
JobContext job_context(0);
|
|
|
|
auto cfd = versions_->GetColumnFamilySet()->GetDefault();
|
|
|
|
auto new_mem = cfd->ConstructNewMemtable(*cfd->GetLatestMutableCFOptions(),
|
|
|
|
kMaxSequenceNumber);
|
|
|
|
|
|
|
|
std::vector<SequenceNumber> snapshots;
|
|
|
|
std::set<SequenceNumber> snapshots_set;
|
|
|
|
int keys = 10000;
|
|
|
|
int max_inserts_per_keys = 8;
|
|
|
|
|
|
|
|
Random rnd(301);
|
|
|
|
for (int i = 0; i < keys / 2; ++i) {
|
|
|
|
snapshots.push_back(rnd.Uniform(keys * (max_inserts_per_keys / 2)) + 1);
|
|
|
|
snapshots_set.insert(snapshots.back());
|
|
|
|
}
|
|
|
|
std::sort(snapshots.begin(), snapshots.end());
|
|
|
|
|
|
|
|
new_mem->Ref();
|
|
|
|
SequenceNumber current_seqno = 0;
|
|
|
|
auto inserted_keys = mock::MakeMockFile();
|
|
|
|
for (int i = 1; i < keys; ++i) {
|
|
|
|
std::string key(ToString(i));
|
|
|
|
int insertions = rnd.Uniform(max_inserts_per_keys);
|
|
|
|
for (int j = 0; j < insertions; ++j) {
|
|
|
|
std::string value(test::RandomHumanReadableString(&rnd, 10));
|
|
|
|
auto seqno = ++current_seqno;
|
|
|
|
new_mem->Add(SequenceNumber(seqno), kTypeValue, key, value);
|
|
|
|
// a key is visible only if:
|
|
|
|
// 1. it's the last one written (j == insertions - 1)
|
|
|
|
// 2. there's a snapshot pointing at it
|
|
|
|
bool visible = (j == insertions - 1) ||
|
|
|
|
(snapshots_set.find(seqno) != snapshots_set.end());
|
|
|
|
if (visible) {
|
|
|
|
InternalKey internal_key(key, seqno, kTypeValue);
|
|
|
|
inserted_keys.insert({internal_key.Encode().ToString(), value});
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
autovector<MemTable*> to_delete;
|
|
|
|
cfd->imm()->Add(new_mem, &to_delete);
|
|
|
|
for (auto& m : to_delete) {
|
|
|
|
delete m;
|
|
|
|
}
|
|
|
|
|
|
|
|
EventLogger event_logger(db_options_.info_log.get());
|
|
|
|
SnapshotChecker* snapshot_checker = nullptr; // not relavant
|
|
|
|
FlushJob flush_job(dbname_, versions_->GetColumnFamilySet()->GetDefault(),
|
|
|
|
db_options_, *cfd->GetLatestMutableCFOptions(),
|
|
|
|
env_options_, versions_.get(), &mutex_, &shutting_down_,
|
|
|
|
snapshots, kMaxSequenceNumber, snapshot_checker,
|
|
|
|
&job_context, nullptr, nullptr, nullptr, kNoCompression,
|
|
|
|
db_options_.statistics.get(), &event_logger, true);
|
|
|
|
mutex_.Lock();
|
|
|
|
flush_job.PickMemTable();
|
|
|
|
ASSERT_OK(flush_job.Run());
|
|
|
|
mutex_.Unlock();
|
|
|
|
mock_table_factory_->AssertSingleFile(inserted_keys);
|
|
|
|
HistogramData hist;
|
|
|
|
db_options_.statistics->histogramData(FLUSH_TIME, &hist);
|
|
|
|
ASSERT_GT(hist.average, 0.0);
|
|
|
|
job_context.Clean();
|
|
|
|
}
|
|
|
|
|
|
|
|
} // namespace rocksdb
|
|
|
|
|
|
|
|
int main(int argc, char** argv) {
|
|
|
|
::testing::InitGoogleTest(&argc, argv);
|
|
|
|
return RUN_ALL_TESTS();
|
|
|
|
}
|