Summary: When there are multiple column families, the flush in GetLiveFiles is not atomic, so that there are entries in the wal files which are needed to get a consisten RocksDB. We now add the log files to the checkpoint. Test Plan: CheckpointCF - This test forces more data to be written to the other column families after the flush of the first column family but before the second. Reviewers: igor, yhchiang, IslamAbdelRahman, anthony, kradhakrishnan, sdong Reviewed By: sdong Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D40323main
Venkatesh Radhakrishnan
10 years ago
parent
18cc5018b7
commit
04251e1e3a
@ -0,0 +1,349 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style license that can be
|
||||||
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
|
||||||
|
#include <unistd.h> |
||||||
|
#include <iostream> |
||||||
|
#include <thread> |
||||||
|
#include <utility> |
||||||
|
#include "db/db_impl.h" |
||||||
|
#include "port/stack_trace.h" |
||||||
|
#include "rocksdb/db.h" |
||||||
|
#include "rocksdb/env.h" |
||||||
|
#include "rocksdb/utilities/checkpoint.h" |
||||||
|
#include "util/sync_point.h" |
||||||
|
#include "util/testharness.h" |
||||||
|
#include "util/xfunc.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
class DBTest : public testing::Test { |
||||||
|
protected: |
||||||
|
// Sequence of option configurations to try
|
||||||
|
enum OptionConfig { |
||||||
|
kDefault = 0, |
||||||
|
}; |
||||||
|
int option_config_; |
||||||
|
|
||||||
|
public: |
||||||
|
std::string dbname_; |
||||||
|
std::string alternative_wal_dir_; |
||||||
|
Env* env_; |
||||||
|
DB* db_; |
||||||
|
Options last_options_; |
||||||
|
std::vector<ColumnFamilyHandle*> handles_; |
||||||
|
|
||||||
|
DBTest() : env_(Env::Default()) { |
||||||
|
env_->SetBackgroundThreads(1, Env::LOW); |
||||||
|
env_->SetBackgroundThreads(1, Env::HIGH); |
||||||
|
dbname_ = test::TmpDir(env_) + "/db_test"; |
||||||
|
alternative_wal_dir_ = dbname_ + "/wal"; |
||||||
|
auto options = CurrentOptions(); |
||||||
|
auto delete_options = options; |
||||||
|
delete_options.wal_dir = alternative_wal_dir_; |
||||||
|
EXPECT_OK(DestroyDB(dbname_, delete_options)); |
||||||
|
// Destroy it for not alternative WAL dir is used.
|
||||||
|
EXPECT_OK(DestroyDB(dbname_, options)); |
||||||
|
db_ = nullptr; |
||||||
|
Reopen(options); |
||||||
|
} |
||||||
|
|
||||||
|
~DBTest() { |
||||||
|
rocksdb::SyncPoint::GetInstance()->DisableProcessing(); |
||||||
|
rocksdb::SyncPoint::GetInstance()->LoadDependency({}); |
||||||
|
rocksdb::SyncPoint::GetInstance()->ClearAllCallBacks(); |
||||||
|
Close(); |
||||||
|
Options options; |
||||||
|
options.db_paths.emplace_back(dbname_, 0); |
||||||
|
options.db_paths.emplace_back(dbname_ + "_2", 0); |
||||||
|
options.db_paths.emplace_back(dbname_ + "_3", 0); |
||||||
|
options.db_paths.emplace_back(dbname_ + "_4", 0); |
||||||
|
EXPECT_OK(DestroyDB(dbname_, options)); |
||||||
|
} |
||||||
|
|
||||||
|
// Return the current option configuration.
|
||||||
|
Options CurrentOptions() { |
||||||
|
Options options; |
||||||
|
options.env = env_; |
||||||
|
options.create_if_missing = true; |
||||||
|
return options; |
||||||
|
} |
||||||
|
|
||||||
|
void CreateColumnFamilies(const std::vector<std::string>& cfs, |
||||||
|
const Options& options) { |
||||||
|
ColumnFamilyOptions cf_opts(options); |
||||||
|
size_t cfi = handles_.size(); |
||||||
|
handles_.resize(cfi + cfs.size()); |
||||||
|
for (auto cf : cfs) { |
||||||
|
ASSERT_OK(db_->CreateColumnFamily(cf_opts, cf, &handles_[cfi++])); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
void CreateAndReopenWithCF(const std::vector<std::string>& cfs, |
||||||
|
const Options& options) { |
||||||
|
CreateColumnFamilies(cfs, options); |
||||||
|
std::vector<std::string> cfs_plus_default = cfs; |
||||||
|
cfs_plus_default.insert(cfs_plus_default.begin(), kDefaultColumnFamilyName); |
||||||
|
ReopenWithColumnFamilies(cfs_plus_default, options); |
||||||
|
} |
||||||
|
|
||||||
|
void ReopenWithColumnFamilies(const std::vector<std::string>& cfs, |
||||||
|
const std::vector<Options>& options) { |
||||||
|
ASSERT_OK(TryReopenWithColumnFamilies(cfs, options)); |
||||||
|
} |
||||||
|
|
||||||
|
void ReopenWithColumnFamilies(const std::vector<std::string>& cfs, |
||||||
|
const Options& options) { |
||||||
|
ASSERT_OK(TryReopenWithColumnFamilies(cfs, options)); |
||||||
|
} |
||||||
|
|
||||||
|
Status TryReopenWithColumnFamilies( |
||||||
|
const std::vector<std::string>& cfs, |
||||||
|
const std::vector<Options>& options) { |
||||||
|
Close(); |
||||||
|
EXPECT_EQ(cfs.size(), options.size()); |
||||||
|
std::vector<ColumnFamilyDescriptor> column_families; |
||||||
|
for (size_t i = 0; i < cfs.size(); ++i) { |
||||||
|
column_families.push_back(ColumnFamilyDescriptor(cfs[i], options[i])); |
||||||
|
} |
||||||
|
DBOptions db_opts = DBOptions(options[0]); |
||||||
|
return DB::Open(db_opts, dbname_, column_families, &handles_, &db_); |
||||||
|
} |
||||||
|
|
||||||
|
Status TryReopenWithColumnFamilies(const std::vector<std::string>& cfs, |
||||||
|
const Options& options) { |
||||||
|
Close(); |
||||||
|
std::vector<Options> v_opts(cfs.size(), options); |
||||||
|
return TryReopenWithColumnFamilies(cfs, v_opts); |
||||||
|
} |
||||||
|
|
||||||
|
void Reopen(const Options& options) { |
||||||
|
ASSERT_OK(TryReopen(options)); |
||||||
|
} |
||||||
|
|
||||||
|
void Close() { |
||||||
|
for (auto h : handles_) { |
||||||
|
delete h; |
||||||
|
} |
||||||
|
handles_.clear(); |
||||||
|
delete db_; |
||||||
|
db_ = nullptr; |
||||||
|
} |
||||||
|
|
||||||
|
void DestroyAndReopen(const Options& options) { |
||||||
|
// Destroy using last options
|
||||||
|
Destroy(last_options_); |
||||||
|
ASSERT_OK(TryReopen(options)); |
||||||
|
} |
||||||
|
|
||||||
|
void Destroy(const Options& options) { |
||||||
|
Close(); |
||||||
|
ASSERT_OK(DestroyDB(dbname_, options)); |
||||||
|
} |
||||||
|
|
||||||
|
Status ReadOnlyReopen(const Options& options) { |
||||||
|
return DB::OpenForReadOnly(options, dbname_, &db_); |
||||||
|
} |
||||||
|
|
||||||
|
Status TryReopen(const Options& options) { |
||||||
|
Close(); |
||||||
|
last_options_ = options; |
||||||
|
return DB::Open(options, dbname_, &db_); |
||||||
|
} |
||||||
|
|
||||||
|
Status Flush(int cf = 0) { |
||||||
|
if (cf == 0) { |
||||||
|
return db_->Flush(FlushOptions()); |
||||||
|
} else { |
||||||
|
return db_->Flush(FlushOptions(), handles_[cf]); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
Status Put(const Slice& k, const Slice& v, WriteOptions wo = WriteOptions()) { |
||||||
|
return db_->Put(wo, k, v); |
||||||
|
} |
||||||
|
|
||||||
|
Status Put(int cf, const Slice& k, const Slice& v, |
||||||
|
WriteOptions wo = WriteOptions()) { |
||||||
|
return db_->Put(wo, handles_[cf], k, v); |
||||||
|
} |
||||||
|
|
||||||
|
Status Delete(const std::string& k) { |
||||||
|
return db_->Delete(WriteOptions(), k); |
||||||
|
} |
||||||
|
|
||||||
|
Status Delete(int cf, const std::string& k) { |
||||||
|
return db_->Delete(WriteOptions(), handles_[cf], k); |
||||||
|
} |
||||||
|
|
||||||
|
std::string Get(const std::string& k, const Snapshot* snapshot = nullptr) { |
||||||
|
ReadOptions options; |
||||||
|
options.verify_checksums = true; |
||||||
|
options.snapshot = snapshot; |
||||||
|
std::string result; |
||||||
|
Status s = db_->Get(options, k, &result); |
||||||
|
if (s.IsNotFound()) { |
||||||
|
result = "NOT_FOUND"; |
||||||
|
} else if (!s.ok()) { |
||||||
|
result = s.ToString(); |
||||||
|
} |
||||||
|
return result; |
||||||
|
} |
||||||
|
|
||||||
|
std::string Get(int cf, const std::string& k, |
||||||
|
const Snapshot* snapshot = nullptr) { |
||||||
|
ReadOptions options; |
||||||
|
options.verify_checksums = true; |
||||||
|
options.snapshot = snapshot; |
||||||
|
std::string result; |
||||||
|
Status s = db_->Get(options, handles_[cf], k, &result); |
||||||
|
if (s.IsNotFound()) { |
||||||
|
result = "NOT_FOUND"; |
||||||
|
} else if (!s.ok()) { |
||||||
|
result = s.ToString(); |
||||||
|
} |
||||||
|
return result; |
||||||
|
} |
||||||
|
}; |
||||||
|
|
||||||
|
TEST_F(DBTest, GetSnapshotLink) { |
||||||
|
Options options; |
||||||
|
const std::string snapshot_name = test::TmpDir(env_) + "/snapshot"; |
||||||
|
DB* snapshotDB; |
||||||
|
ReadOptions roptions; |
||||||
|
std::string result; |
||||||
|
Checkpoint* checkpoint; |
||||||
|
|
||||||
|
options = CurrentOptions(); |
||||||
|
delete db_; |
||||||
|
db_ = nullptr; |
||||||
|
ASSERT_OK(DestroyDB(dbname_, options)); |
||||||
|
ASSERT_OK(DestroyDB(snapshot_name, options)); |
||||||
|
env_->DeleteDir(snapshot_name); |
||||||
|
|
||||||
|
// Create a database
|
||||||
|
Status s; |
||||||
|
options.create_if_missing = true; |
||||||
|
ASSERT_OK(DB::Open(options, dbname_, &db_)); |
||||||
|
std::string key = std::string("foo"); |
||||||
|
ASSERT_OK(Put(key, "v1")); |
||||||
|
// Take a snapshot
|
||||||
|
ASSERT_OK(Checkpoint::Create(db_, &checkpoint)); |
||||||
|
ASSERT_OK(checkpoint->CreateCheckpoint(snapshot_name)); |
||||||
|
ASSERT_OK(Put(key, "v2")); |
||||||
|
ASSERT_EQ("v2", Get(key)); |
||||||
|
ASSERT_OK(Flush()); |
||||||
|
ASSERT_EQ("v2", Get(key)); |
||||||
|
// Open snapshot and verify contents while DB is running
|
||||||
|
options.create_if_missing = false; |
||||||
|
ASSERT_OK(DB::Open(options, snapshot_name, &snapshotDB)); |
||||||
|
ASSERT_OK(snapshotDB->Get(roptions, key, &result)); |
||||||
|
ASSERT_EQ("v1", result); |
||||||
|
delete snapshotDB; |
||||||
|
snapshotDB = nullptr; |
||||||
|
delete db_; |
||||||
|
db_ = nullptr; |
||||||
|
|
||||||
|
// Destroy original DB
|
||||||
|
ASSERT_OK(DestroyDB(dbname_, options)); |
||||||
|
|
||||||
|
// Open snapshot and verify contents
|
||||||
|
options.create_if_missing = false; |
||||||
|
dbname_ = snapshot_name; |
||||||
|
ASSERT_OK(DB::Open(options, dbname_, &db_)); |
||||||
|
ASSERT_EQ("v1", Get(key)); |
||||||
|
delete db_; |
||||||
|
db_ = nullptr; |
||||||
|
ASSERT_OK(DestroyDB(dbname_, options)); |
||||||
|
delete checkpoint; |
||||||
|
|
||||||
|
// Restore DB name
|
||||||
|
dbname_ = test::TmpDir(env_) + "/db_test"; |
||||||
|
} |
||||||
|
|
||||||
|
TEST_F(DBTest, CheckpointCF) { |
||||||
|
Options options = CurrentOptions(); |
||||||
|
CreateAndReopenWithCF({"one", "two", "three", "four", "five"}, options); |
||||||
|
rocksdb::SyncPoint::GetInstance()->LoadDependency( |
||||||
|
{{"DBTest::CheckpointCF:2", |
||||||
|
"DBImpl::GetLiveFiles:2"}, |
||||||
|
{"DBImpl::GetLiveFiles:1", |
||||||
|
"DBTest::CheckpointCF:1"}}); |
||||||
|
|
||||||
|
rocksdb::SyncPoint::GetInstance()->EnableProcessing(); |
||||||
|
|
||||||
|
ASSERT_OK(Put(0, "Default", "Default")); |
||||||
|
ASSERT_OK(Put(1, "one", "one")); |
||||||
|
ASSERT_OK(Put(2, "two", "two")); |
||||||
|
ASSERT_OK(Put(3, "three", "three")); |
||||||
|
ASSERT_OK(Put(4, "four", "four")); |
||||||
|
ASSERT_OK(Put(5, "five", "five")); |
||||||
|
|
||||||
|
const std::string snapshot_name = test::TmpDir(env_) + "/snapshot"; |
||||||
|
DB* snapshotDB; |
||||||
|
ReadOptions roptions; |
||||||
|
std::string result; |
||||||
|
std::vector<ColumnFamilyHandle*> cphandles; |
||||||
|
|
||||||
|
ASSERT_OK(DestroyDB(snapshot_name, options)); |
||||||
|
env_->DeleteDir(snapshot_name); |
||||||
|
|
||||||
|
Status s; |
||||||
|
// Take a snapshot
|
||||||
|
std::thread t([&]() { |
||||||
|
Checkpoint* checkpoint; |
||||||
|
ASSERT_OK(Checkpoint::Create(db_, &checkpoint)); |
||||||
|
ASSERT_OK(checkpoint->CreateCheckpoint(snapshot_name)); |
||||||
|
}); |
||||||
|
TEST_SYNC_POINT("DBTest::CheckpointCF:1"); |
||||||
|
ASSERT_OK(Put(0, "Default", "Default1")); |
||||||
|
ASSERT_OK(Put(1, "one", "eleven")); |
||||||
|
ASSERT_OK(Put(2, "two", "twelve")); |
||||||
|
ASSERT_OK(Put(3, "three", "thirteen")); |
||||||
|
ASSERT_OK(Put(4, "four", "fourteen")); |
||||||
|
ASSERT_OK(Put(5, "five", "fifteen")); |
||||||
|
TEST_SYNC_POINT("DBTest::CheckpointCF:2"); |
||||||
|
t.join(); |
||||||
|
rocksdb::SyncPoint::GetInstance()->DisableProcessing(); |
||||||
|
ASSERT_OK(Put(1, "one", "twentyone")); |
||||||
|
ASSERT_OK(Put(2, "two", "twentytwo")); |
||||||
|
ASSERT_OK(Put(3, "three", "twentythree")); |
||||||
|
ASSERT_OK(Put(4, "four", "twentyfour")); |
||||||
|
ASSERT_OK(Put(5, "five", "twentyfive")); |
||||||
|
ASSERT_OK(Flush()); |
||||||
|
|
||||||
|
// Open snapshot and verify contents while DB is running
|
||||||
|
options.create_if_missing = false; |
||||||
|
std::vector<std::string> cfs; |
||||||
|
cfs= {kDefaultColumnFamilyName, "one", "two", "three", "four", "five"}; |
||||||
|
std::vector<ColumnFamilyDescriptor> column_families; |
||||||
|
for (size_t i = 0; i < cfs.size(); ++i) { |
||||||
|
column_families.push_back(ColumnFamilyDescriptor(cfs[i], options)); |
||||||
|
} |
||||||
|
ASSERT_OK(DB::Open(options, snapshot_name, |
||||||
|
column_families, &cphandles, &snapshotDB)); |
||||||
|
ASSERT_OK(snapshotDB->Get(roptions, cphandles[0], "Default", &result)); |
||||||
|
ASSERT_EQ("Default1", result); |
||||||
|
ASSERT_OK(snapshotDB->Get(roptions, cphandles[1], "one", &result)); |
||||||
|
ASSERT_EQ("eleven", result); |
||||||
|
ASSERT_OK(snapshotDB->Get(roptions, cphandles[2], "two", &result)); |
||||||
|
for (auto h : cphandles) { |
||||||
|
delete h; |
||||||
|
} |
||||||
|
cphandles.clear(); |
||||||
|
delete snapshotDB; |
||||||
|
snapshotDB = nullptr; |
||||||
|
ASSERT_OK(DestroyDB(snapshot_name, options)); |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
|
|
||||||
|
int main(int argc, char** argv) { |
||||||
|
rocksdb::port::InstallStackTraceHandler(); |
||||||
|
::testing::InitGoogleTest(&argc, argv); |
||||||
|
return RUN_ALL_TESTS(); |
||||||
|
} |
||||||
Loading…
Reference in new issue