Conflicts: db/db_impl.cc db/db_test.cc db/memtable.cc db/version_set.cc include/rocksdb/statistics.hmain
kailiu
11 years ago
commit
f1cec73a76
@ -0,0 +1,133 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#pragma once |
||||
#include "utilities/stackable_db.h" |
||||
#include "rocksdb/env.h" |
||||
#include "rocksdb/status.h" |
||||
|
||||
#include <string> |
||||
#include <map> |
||||
#include <vector> |
||||
|
||||
namespace rocksdb { |
||||
|
||||
struct BackupableDBOptions { |
||||
// Where to keep the backup files. Has to be different than dbname_
|
||||
// Best to set this to dbname_ + "/backups"
|
||||
// Required
|
||||
std::string backup_dir; |
||||
|
||||
// Backup Env object. It will be used for backup file I/O. If it's
|
||||
// nullptr, backups will be written out using DBs Env. If it's
|
||||
// non-nullptr, backup's I/O will be performed using this object.
|
||||
// If you want to have backups on HDFS, use HDFS Env here!
|
||||
// Default: nullptr
|
||||
Env* backup_env; |
||||
|
||||
// Backup info and error messages will be written to info_log
|
||||
// if non-nullptr.
|
||||
// Default: nullptr
|
||||
Logger* info_log; |
||||
|
||||
// If sync == true, we can guarantee you'll get consistent backup even
|
||||
// on a machine crash/reboot. Backup process is slower with sync enabled.
|
||||
// If sync == false, we don't guarantee anything on machine reboot. However,
|
||||
// chances are some of the backups are consistent.
|
||||
// Default: true
|
||||
bool sync; |
||||
|
||||
// If true, it will delete whatever backups there are already
|
||||
// Default: false
|
||||
bool destroy_old_data; |
||||
|
||||
explicit BackupableDBOptions(const std::string& _backup_dir, |
||||
Env* _backup_env = nullptr, |
||||
Logger* _info_log = nullptr, |
||||
bool _sync = true, |
||||
bool _destroy_old_data = false) : |
||||
backup_dir(_backup_dir), |
||||
backup_env(_backup_env), |
||||
info_log(_info_log), |
||||
sync(_sync), |
||||
destroy_old_data(_destroy_old_data) { } |
||||
}; |
||||
|
||||
class BackupEngine; |
||||
|
||||
typedef uint32_t BackupID; |
||||
|
||||
struct BackupInfo { |
||||
BackupID backup_id; |
||||
int64_t timestamp; |
||||
uint64_t size; |
||||
|
||||
BackupInfo() {} |
||||
BackupInfo(BackupID _backup_id, int64_t _timestamp, uint64_t _size) |
||||
: backup_id(_backup_id), timestamp(_timestamp), size(_size) {} |
||||
}; |
||||
|
||||
// Stack your DB with BackupableDB to be able to backup the DB
|
||||
class BackupableDB : public StackableDB { |
||||
public: |
||||
// BackupableDBOptions have to be the same as the ones used in a previous
|
||||
// incarnation of the DB
|
||||
//
|
||||
// BackupableDB ownes the pointer `DB* db` now. You should not delete it or
|
||||
// use it after the invocation of BackupableDB
|
||||
BackupableDB(DB* db, const BackupableDBOptions& options); |
||||
virtual ~BackupableDB(); |
||||
|
||||
// Captures the state of the database in the latest backup
|
||||
// NOT a thread safe call
|
||||
Status CreateNewBackup(bool flush_before_backup = false); |
||||
// Returns info about backups in backup_info
|
||||
void GetBackupInfo(std::vector<BackupInfo>* backup_info); |
||||
// deletes old backups, keeping latest num_backups_to_keep alive
|
||||
Status PurgeOldBackups(uint32_t num_backups_to_keep); |
||||
// deletes a specific backup
|
||||
Status DeleteBackup(BackupID backup_id); |
||||
|
||||
private: |
||||
BackupEngine* backup_engine_; |
||||
}; |
||||
|
||||
// Use this class to access information about backups and restore from them
|
||||
class RestoreBackupableDB { |
||||
public: |
||||
RestoreBackupableDB(Env* db_env, const BackupableDBOptions& options); |
||||
~RestoreBackupableDB(); |
||||
|
||||
// Returns info about backups in backup_info
|
||||
void GetBackupInfo(std::vector<BackupInfo>* backup_info); |
||||
|
||||
// restore from backup with backup_id
|
||||
// IMPORTANT -- if you restore from some backup that is not the latest,
|
||||
// and you start creating new backups from the new DB, all the backups
|
||||
// that were newer than the backup you restored from will be deleted
|
||||
//
|
||||
// Example: Let's say you have backups 1, 2, 3, 4, 5 and you restore 3.
|
||||
// If you try creating a new backup now, old backups 4 and 5 will be deleted
|
||||
// and new backup with ID 4 will be created.
|
||||
Status RestoreDBFromBackup(BackupID backup_id, const std::string& db_dir, |
||||
const std::string& wal_dir); |
||||
|
||||
// restore from the latest backup
|
||||
Status RestoreDBFromLatestBackup(const std::string& db_dir, |
||||
const std::string& wal_dir); |
||||
// deletes old backups, keeping latest num_backups_to_keep alive
|
||||
Status PurgeOldBackups(uint32_t num_backups_to_keep); |
||||
// deletes a specific backup
|
||||
Status DeleteBackup(BackupID backup_id); |
||||
|
||||
private: |
||||
BackupEngine* backup_engine_; |
||||
}; |
||||
|
||||
} // rocksdb namespace
|
||||
@ -0,0 +1,329 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
#pragma once |
||||
|
||||
#include <algorithm> |
||||
#include <cassert> |
||||
#include <stdexcept> |
||||
#include <iterator> |
||||
#include <vector> |
||||
|
||||
namespace rocksdb { |
||||
|
||||
// A vector that leverages pre-allocated stack-based array to achieve better
|
||||
// performance for array with small amount of items.
|
||||
//
|
||||
// The interface resembles that of vector, but with less features since we aim
|
||||
// to solve the problem that we have in hand, rather than implementing a
|
||||
// full-fledged generic container.
|
||||
//
|
||||
// Currently we don't support:
|
||||
// * reserve()/shrink_to_fit()/resize()
|
||||
// If used correctly, in most cases, people should not touch the
|
||||
// underlying vector at all.
|
||||
// * random insert()/erase(), please only use push_back()/pop_back().
|
||||
// * No move/swap operations. Each autovector instance has a
|
||||
// stack-allocated array and if we want support move/swap operations, we
|
||||
// need to copy the arrays other than just swapping the pointers. In this
|
||||
// case we'll just explicitly forbid these operations since they may
|
||||
// lead users to make false assumption by thinking they are inexpensive
|
||||
// operations.
|
||||
//
|
||||
// Naming style of public methods almost follows that of the STL's.
|
||||
template <class T, size_t kSize = 8> |
||||
class autovector { |
||||
public: |
||||
// General STL-style container member types.
|
||||
typedef T value_type; |
||||
typedef typename std::vector<T>::difference_type difference_type; |
||||
typedef typename std::vector<T>::size_type size_type; |
||||
typedef value_type& reference; |
||||
typedef const value_type& const_reference; |
||||
typedef value_type* pointer; |
||||
typedef const value_type* const_pointer; |
||||
|
||||
// This class is the base for regular/const iterator
|
||||
template <class TAutoVector, class TValueType> |
||||
class iterator_impl { |
||||
public: |
||||
// -- iterator traits
|
||||
typedef iterator_impl<TAutoVector, TValueType> self_type; |
||||
typedef TValueType value_type; |
||||
typedef TValueType& reference; |
||||
typedef TValueType* pointer; |
||||
typedef typename TAutoVector::difference_type difference_type; |
||||
typedef std::random_access_iterator_tag iterator_category; |
||||
|
||||
iterator_impl(TAutoVector* vect, size_t index) |
||||
: vect_(vect) |
||||
, index_(index) { |
||||
}; |
||||
iterator_impl(const iterator_impl&) = default; |
||||
~iterator_impl() { } |
||||
iterator_impl& operator=(const iterator_impl&) = default; |
||||
|
||||
// -- Advancement
|
||||
// iterator++
|
||||
self_type& operator++() { |
||||
++index_; |
||||
return *this; |
||||
} |
||||
|
||||
// ++iterator
|
||||
self_type operator++(int) { |
||||
auto old = *this; |
||||
++index_; |
||||
return old; |
||||
} |
||||
|
||||
// iterator--
|
||||
self_type& operator--() { |
||||
--index_; |
||||
return *this; |
||||
} |
||||
|
||||
// --iterator
|
||||
self_type operator--(int) { |
||||
auto old = *this; |
||||
--index_; |
||||
return old; |
||||
} |
||||
|
||||
self_type operator-(difference_type len) { |
||||
return self_type(vect_, index_ - len); |
||||
} |
||||
|
||||
difference_type operator-(const self_type& other) { |
||||
assert(vect_ == other.vect_); |
||||
return index_ - other.index_; |
||||
} |
||||
|
||||
self_type operator+(difference_type len) { |
||||
return self_type(vect_, index_ + len); |
||||
} |
||||
|
||||
self_type& operator+=(difference_type len) { |
||||
index_ += len; |
||||
return *this; |
||||
} |
||||
|
||||
self_type& operator-=(difference_type len) { |
||||
index_ -= len; |
||||
return *this; |
||||
} |
||||
|
||||
// -- Reference
|
||||
reference operator*() { |
||||
assert(vect_->size() >= index_); |
||||
return (*vect_)[index_]; |
||||
} |
||||
pointer operator->() { |
||||
assert(vect_->size() >= index_); |
||||
return &(*vect_)[index_]; |
||||
} |
||||
|
||||
// -- Logical Operators
|
||||
bool operator==(const self_type& other) const { |
||||
assert(vect_ == other.vect_); |
||||
return index_ == other.index_; |
||||
} |
||||
|
||||
bool operator!=(const self_type& other) const { |
||||
return !(*this == other); |
||||
} |
||||
|
||||
bool operator>(const self_type& other) const { |
||||
assert(vect_ == other.vect_); |
||||
return index_ > other.index_; |
||||
} |
||||
|
||||
bool operator<(const self_type& other) const { |
||||
assert(vect_ == other.vect_); |
||||
return index_ < other.index_; |
||||
} |
||||
|
||||
bool operator>=(const self_type& other) const { |
||||
assert(vect_ == other.vect_); |
||||
return index_ >= other.index_; |
||||
} |
||||
|
||||
bool operator<=(const self_type& other) const { |
||||
assert(vect_ == other.vect_); |
||||
return index_ <= other.index_; |
||||
} |
||||
|
||||
private: |
||||
TAutoVector* vect_ = nullptr; |
||||
size_t index_ = 0; |
||||
}; |
||||
|
||||
typedef iterator_impl<autovector, value_type> iterator; |
||||
typedef iterator_impl<const autovector, const value_type> const_iterator; |
||||
typedef std::reverse_iterator<iterator> reverse_iterator; |
||||
typedef std::reverse_iterator<const_iterator> const_reverse_iterator; |
||||
|
||||
autovector() = default; |
||||
~autovector() = default; |
||||
|
||||
// -- Immutable operations
|
||||
// Indicate if all data resides in in-stack data structure.
|
||||
bool only_in_stack() const { |
||||
// If no element was inserted at all, the vector's capacity will be `0`.
|
||||
return vect_.capacity() == 0; |
||||
} |
||||
|
||||
size_type size() const { |
||||
return num_stack_items_ + vect_.size(); |
||||
} |
||||
|
||||
bool empty() const { |
||||
return size() == 0; |
||||
} |
||||
|
||||
// will not check boundry
|
||||
const_reference operator[](size_type n) const { |
||||
return n < kSize ? values_[n] : vect_[n - kSize]; |
||||
} |
||||
|
||||
reference operator[](size_type n) { |
||||
return n < kSize ? values_[n] : vect_[n - kSize]; |
||||
} |
||||
|
||||
// will check boundry
|
||||
const_reference at(size_type n) const { |
||||
if (n >= size()) { |
||||
throw std::out_of_range("autovector: index out of range"); |
||||
} |
||||
return (*this)[n]; |
||||
} |
||||
|
||||
reference at(size_type n) { |
||||
if (n >= size()) { |
||||
throw std::out_of_range("autovector: index out of range"); |
||||
} |
||||
return (*this)[n]; |
||||
} |
||||
|
||||
reference front() { |
||||
assert(!empty()); |
||||
return *begin(); |
||||
} |
||||
|
||||
const_reference front() const { |
||||
assert(!empty()); |
||||
return *begin(); |
||||
} |
||||
|
||||
reference back() { |
||||
assert(!empty()); |
||||
return *(end() - 1); |
||||
} |
||||
|
||||
const_reference back() const { |
||||
assert(!empty()); |
||||
return *(end() - 1); |
||||
} |
||||
|
||||
// -- Mutable Operations
|
||||
void push_back(T&& item) { |
||||
if (num_stack_items_ < kSize) { |
||||
values_[num_stack_items_++] = std::move(item); |
||||
} else { |
||||
vect_.push_back(item); |
||||
} |
||||
} |
||||
|
||||
void push_back(const T& item) { |
||||
push_back(value_type(item)); |
||||
} |
||||
|
||||
template<class... Args> |
||||
void emplace_back(Args&&... args) { |
||||
push_back(value_type(args...)); |
||||
} |
||||
|
||||
void pop_back() { |
||||
assert(!empty()); |
||||
if (!vect_.empty()) { |
||||
vect_.pop_back(); |
||||
} else { |
||||
--num_stack_items_; |
||||
} |
||||
} |
||||
|
||||
void clear() { |
||||
num_stack_items_ = 0; |
||||
vect_.clear(); |
||||
} |
||||
|
||||
// -- Copy and Assignment
|
||||
autovector& assign(const autovector& other); |
||||
|
||||
autovector(const autovector& other) { |
||||
assign(other); |
||||
} |
||||
|
||||
autovector& operator=(const autovector& other) { |
||||
return assign(other); |
||||
} |
||||
|
||||
// move operation are disallowed since it is very hard to make sure both
|
||||
// autovectors are allocated from the same function stack.
|
||||
autovector& operator=(autovector&& other) = delete; |
||||
autovector(autovector&& other) = delete; |
||||
|
||||
// -- Iterator Operations
|
||||
iterator begin() { |
||||
return iterator(this, 0); |
||||
} |
||||
|
||||
const_iterator begin() const { |
||||
return const_iterator(this, 0); |
||||
} |
||||
|
||||
iterator end() { |
||||
return iterator(this, this->size()); |
||||
} |
||||
|
||||
const_iterator end() const { |
||||
return const_iterator(this, this->size()); |
||||
} |
||||
|
||||
reverse_iterator rbegin() { |
||||
return reverse_iterator(end()); |
||||
} |
||||
|
||||
const_reverse_iterator rbegin() const { |
||||
return const_reverse_iterator(end()); |
||||
} |
||||
|
||||
reverse_iterator rend() { |
||||
return reverse_iterator(begin()); |
||||
} |
||||
|
||||
const_reverse_iterator rend() const { |
||||
return const_reverse_iterator(begin()); |
||||
} |
||||
|
||||
private: |
||||
size_type num_stack_items_ = 0; // current number of items
|
||||
value_type values_[kSize]; // the first `kSize` items
|
||||
// used only if there are more than `kSize` items.
|
||||
std::vector<T> vect_; |
||||
}; |
||||
|
||||
template <class T, size_t kSize> |
||||
autovector<T, kSize>& autovector<T, kSize>::assign(const autovector& other) { |
||||
// copy the internal vector
|
||||
vect_.assign(other.vect_.begin(), other.vect_.end()); |
||||
|
||||
// copy array
|
||||
num_stack_items_ = other.num_stack_items_; |
||||
std::copy(other.values_, other.values_ + num_stack_items_, values_); |
||||
|
||||
return *this; |
||||
} |
||||
|
||||
} // rocksdb
|
||||
@ -0,0 +1,290 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#include <atomic> |
||||
#include <iostream> |
||||
|
||||
#include "rocksdb/env.h" |
||||
#include "util/autovector.h" |
||||
#include "util/testharness.h" |
||||
#include "util/testutil.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
using namespace std; |
||||
|
||||
class AutoVectorTest { }; |
||||
|
||||
const size_t kSize = 8; |
||||
TEST(AutoVectorTest, PushBackAndPopBack) { |
||||
autovector<size_t, kSize> vec; |
||||
ASSERT_TRUE(vec.empty()); |
||||
ASSERT_EQ(0ul, vec.size()); |
||||
|
||||
for (size_t i = 0; i < 1000 * kSize; ++i) { |
||||
vec.push_back(i); |
||||
ASSERT_TRUE(!vec.empty()); |
||||
if (i < kSize) { |
||||
ASSERT_TRUE(vec.only_in_stack()); |
||||
} else { |
||||
ASSERT_TRUE(!vec.only_in_stack()); |
||||
} |
||||
ASSERT_EQ(i + 1, vec.size()); |
||||
ASSERT_EQ(i, vec[i]); |
||||
ASSERT_EQ(i, vec.at(i)); |
||||
} |
||||
|
||||
size_t size = vec.size(); |
||||
while (size != 0) { |
||||
vec.pop_back(); |
||||
// will always be in heap
|
||||
ASSERT_TRUE(!vec.only_in_stack()); |
||||
ASSERT_EQ(--size, vec.size()); |
||||
} |
||||
|
||||
ASSERT_TRUE(vec.empty()); |
||||
} |
||||
|
||||
TEST(AutoVectorTest, EmplaceBack) { |
||||
typedef std::pair<size_t, std::string> ValueType; |
||||
autovector<ValueType, kSize> vec; |
||||
|
||||
for (size_t i = 0; i < 1000 * kSize; ++i) { |
||||
vec.emplace_back(i, std::to_string(i + 123)); |
||||
ASSERT_TRUE(!vec.empty()); |
||||
if (i < kSize) { |
||||
ASSERT_TRUE(vec.only_in_stack()); |
||||
} else { |
||||
ASSERT_TRUE(!vec.only_in_stack()); |
||||
} |
||||
|
||||
ASSERT_EQ(i + 1, vec.size()); |
||||
ASSERT_EQ(i, vec[i].first); |
||||
ASSERT_EQ(std::to_string(i + 123), vec[i].second); |
||||
} |
||||
|
||||
vec.clear(); |
||||
ASSERT_TRUE(vec.empty()); |
||||
ASSERT_TRUE(!vec.only_in_stack()); |
||||
} |
||||
|
||||
void AssertEqual( |
||||
const autovector<size_t, kSize>& a, const autovector<size_t, kSize>& b) { |
||||
ASSERT_EQ(a.size(), b.size()); |
||||
ASSERT_EQ(a.empty(), b.empty()); |
||||
ASSERT_EQ(a.only_in_stack(), b.only_in_stack()); |
||||
for (size_t i = 0; i < a.size(); ++i) { |
||||
ASSERT_EQ(a[i], b[i]); |
||||
} |
||||
} |
||||
|
||||
TEST(AutoVectorTest, CopyAndAssignment) { |
||||
// Test both heap-allocated and stack-allocated cases.
|
||||
for (auto size : { kSize / 2, kSize * 1000 }) { |
||||
autovector<size_t, kSize> vec; |
||||
for (size_t i = 0; i < size; ++i) { |
||||
vec.push_back(i); |
||||
} |
||||
|
||||
{ |
||||
autovector<size_t, kSize> other; |
||||
other = vec; |
||||
AssertEqual(other, vec); |
||||
} |
||||
|
||||
{ |
||||
autovector<size_t, kSize> other(vec); |
||||
AssertEqual(other, vec); |
||||
} |
||||
} |
||||
} |
||||
|
||||
TEST(AutoVectorTest, Iterators) { |
||||
autovector<std::string, kSize> vec; |
||||
for (size_t i = 0; i < kSize * 1000; ++i) { |
||||
vec.push_back(std::to_string(i)); |
||||
} |
||||
|
||||
// basic operator test
|
||||
ASSERT_EQ(vec.front(), *vec.begin()); |
||||
ASSERT_EQ(vec.back(), *(vec.end() - 1)); |
||||
ASSERT_TRUE(vec.begin() < vec.end()); |
||||
|
||||
// non-const iterator
|
||||
size_t index = 0; |
||||
for (const auto& item : vec) { |
||||
ASSERT_EQ(vec[index++], item); |
||||
} |
||||
|
||||
index = vec.size() - 1; |
||||
for (auto pos = vec.rbegin(); pos != vec.rend(); ++pos) { |
||||
ASSERT_EQ(vec[index--], *pos); |
||||
} |
||||
|
||||
// const iterator
|
||||
const auto& cvec = vec; |
||||
index = 0; |
||||
for (const auto& item : cvec) { |
||||
ASSERT_EQ(cvec[index++], item); |
||||
} |
||||
|
||||
index = vec.size() - 1; |
||||
for (auto pos = cvec.rbegin(); pos != cvec.rend(); ++pos) { |
||||
ASSERT_EQ(cvec[index--], *pos); |
||||
} |
||||
|
||||
// forward and backward
|
||||
auto pos = vec.begin(); |
||||
while (pos != vec.end()) { |
||||
auto old_val = *pos; |
||||
auto old = pos++; |
||||
// HACK: make sure -> works
|
||||
ASSERT_TRUE(!old->empty()); |
||||
ASSERT_EQ(old_val, *old); |
||||
ASSERT_TRUE(pos == vec.end() || old_val != *pos); |
||||
} |
||||
|
||||
pos = vec.begin(); |
||||
for (size_t i = 0; i < vec.size(); i += 2) { |
||||
// Cannot use ASSERT_EQ since that macro depends on iostream serialization
|
||||
ASSERT_TRUE(pos + 2 - 2 == pos); |
||||
pos += 2; |
||||
ASSERT_TRUE(pos >= vec.begin()); |
||||
ASSERT_TRUE(pos <= vec.end()); |
||||
|
||||
size_t diff = static_cast<size_t>(pos - vec.begin()); |
||||
ASSERT_EQ(i + 2, diff); |
||||
} |
||||
} |
||||
|
||||
vector<string> GetTestKeys(size_t size) { |
||||
vector<string> keys; |
||||
keys.resize(size); |
||||
|
||||
int index = 0; |
||||
for (auto& key : keys) { |
||||
key = "item-" + to_string(index++); |
||||
} |
||||
return keys; |
||||
} |
||||
|
||||
template<class TVector> |
||||
void BenchmarkVectorCreationAndInsertion( |
||||
string name, size_t ops, size_t item_size, |
||||
const std::vector<typename TVector::value_type>& items) { |
||||
auto env = Env::Default(); |
||||
|
||||
int index = 0; |
||||
auto start_time = env->NowNanos(); |
||||
auto ops_remaining = ops; |
||||
while(ops_remaining--) { |
||||
TVector v; |
||||
for (size_t i = 0; i < item_size; ++i) { |
||||
v.push_back(items[index++]); |
||||
} |
||||
} |
||||
auto elapsed = env->NowNanos() - start_time; |
||||
cout << "created " << ops << " " << name << " instances:\n\t" |
||||
<< "each was inserted with " << item_size << " elements\n\t" |
||||
<< "total time elapsed: " << elapsed << " (ns)" << endl; |
||||
} |
||||
|
||||
template <class TVector> |
||||
size_t BenchmarkSequenceAccess(string name, size_t ops, size_t elem_size) { |
||||
TVector v; |
||||
for (const auto& item : GetTestKeys(elem_size)) { |
||||
v.push_back(item); |
||||
} |
||||
auto env = Env::Default(); |
||||
|
||||
auto ops_remaining = ops; |
||||
auto start_time = env->NowNanos(); |
||||
size_t total = 0; |
||||
while (ops_remaining--) { |
||||
auto end = v.end(); |
||||
for (auto pos = v.begin(); pos != end; ++pos) { |
||||
total += pos->size(); |
||||
} |
||||
} |
||||
auto elapsed = env->NowNanos() - start_time; |
||||
cout << "performed " << ops << " sequence access against " << name << "\n\t" |
||||
<< "size: " << elem_size << "\n\t" |
||||
<< "total time elapsed: " << elapsed << " (ns)" << endl; |
||||
// HACK avoid compiler's optimization to ignore total
|
||||
return total; |
||||
} |
||||
|
||||
// This test case only reports the performance between std::vector<string>
|
||||
// and autovector<string>. We chose string for comparison because in most
|
||||
// o our use cases we used std::vector<string>.
|
||||
TEST(AutoVectorTest, PerfBench) { |
||||
// We run same operations for kOps times in order to get a more fair result.
|
||||
size_t kOps = 100000; |
||||
|
||||
// Creation and insertion test
|
||||
// Test the case when there is:
|
||||
// * no element inserted: internal array of std::vector may not really get
|
||||
// initialize.
|
||||
// * one element inserted: internal array of std::vector must have
|
||||
// initialized.
|
||||
// * kSize elements inserted. This shows the most time we'll spend if we
|
||||
// keep everything in stack.
|
||||
// * 2 * kSize elements inserted. The internal vector of
|
||||
// autovector must have been initialized.
|
||||
cout << "=====================================================" << endl; |
||||
cout << "Creation and Insertion Test (value type: std::string)" << endl; |
||||
cout << "=====================================================" << endl; |
||||
|
||||
// pre-generated unique keys
|
||||
auto string_keys = GetTestKeys(kOps * 2 * kSize); |
||||
for (auto insertions : { 0ul, 1ul, kSize / 2, kSize, 2 * kSize }) { |
||||
BenchmarkVectorCreationAndInsertion<vector<string>>( |
||||
"vector<string>", kOps, insertions, string_keys |
||||
); |
||||
BenchmarkVectorCreationAndInsertion<autovector<string, kSize>>( |
||||
"autovector<string>", kOps, insertions, string_keys |
||||
); |
||||
cout << "-----------------------------------" << endl; |
||||
} |
||||
|
||||
cout << "=====================================================" << endl; |
||||
cout << "Creation and Insertion Test (value type: uint64_t)" << endl; |
||||
cout << "=====================================================" << endl; |
||||
|
||||
// pre-generated unique keys
|
||||
vector<uint64_t> int_keys(kOps * 2 * kSize); |
||||
for (size_t i = 0; i < kOps * 2 * kSize; ++i) { |
||||
int_keys[i] = i; |
||||
} |
||||
for (auto insertions : { 0ul, 1ul, kSize / 2, kSize, 2 * kSize }) { |
||||
BenchmarkVectorCreationAndInsertion<vector<uint64_t>>( |
||||
"vector<uint64_t>", kOps, insertions, int_keys |
||||
); |
||||
BenchmarkVectorCreationAndInsertion<autovector<uint64_t, kSize>>( |
||||
"autovector<uint64_t>", kOps, insertions, int_keys |
||||
); |
||||
cout << "-----------------------------------" << endl; |
||||
} |
||||
|
||||
// Sequence Access Test
|
||||
cout << "=====================================================" << endl; |
||||
cout << "Sequence Access Test" << endl; |
||||
cout << "=====================================================" << endl; |
||||
for (auto elem_size : { kSize / 2, kSize, 2 * kSize }) { |
||||
BenchmarkSequenceAccess<vector<string>>( |
||||
"vector", kOps, elem_size |
||||
); |
||||
BenchmarkSequenceAccess<autovector<string, kSize>>( |
||||
"autovector", kOps, elem_size |
||||
); |
||||
cout << "-----------------------------------" << endl; |
||||
} |
||||
} |
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) { |
||||
return rocksdb::test::RunAllTests(); |
||||
} |
||||
Binary file not shown.
@ -0,0 +1,874 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "utilities/backupable_db.h" |
||||
#include "db/filename.h" |
||||
#include "util/coding.h" |
||||
#include "rocksdb/transaction_log.h" |
||||
|
||||
#define __STDC_FORMAT_MACROS |
||||
|
||||
#include <inttypes.h> |
||||
#include <algorithm> |
||||
#include <vector> |
||||
#include <map> |
||||
#include <string> |
||||
#include <limits> |
||||
|
||||
namespace rocksdb { |
||||
|
||||
// -------- BackupEngine class ---------
|
||||
class BackupEngine { |
||||
public: |
||||
BackupEngine(Env* db_env, const BackupableDBOptions& options); |
||||
~BackupEngine(); |
||||
Status CreateNewBackup(DB* db, bool flush_before_backup = false); |
||||
Status PurgeOldBackups(uint32_t num_backups_to_keep); |
||||
Status DeleteBackup(BackupID backup_id); |
||||
|
||||
void GetBackupInfo(std::vector<BackupInfo>* backup_info); |
||||
Status RestoreDBFromBackup(BackupID backup_id, const std::string &db_dir, |
||||
const std::string &wal_dir); |
||||
Status RestoreDBFromLatestBackup(const std::string &db_dir, |
||||
const std::string &wal_dir) { |
||||
return RestoreDBFromBackup(latest_backup_id_, db_dir, wal_dir); |
||||
} |
||||
|
||||
void DeleteBackupsNewerThan(uint64_t sequence_number); |
||||
|
||||
private: |
||||
class BackupMeta { |
||||
public: |
||||
BackupMeta(const std::string& meta_filename, |
||||
std::unordered_map<std::string, int>* file_refs, Env* env) |
||||
: timestamp_(0), size_(0), meta_filename_(meta_filename), |
||||
file_refs_(file_refs), env_(env) {} |
||||
|
||||
~BackupMeta() {} |
||||
|
||||
void RecordTimestamp() { |
||||
env_->GetCurrentTime(×tamp_); |
||||
} |
||||
int64_t GetTimestamp() const { |
||||
return timestamp_; |
||||
} |
||||
uint64_t GetSize() const { |
||||
return size_; |
||||
} |
||||
void SetSequenceNumber(uint64_t sequence_number) { |
||||
sequence_number_ = sequence_number; |
||||
} |
||||
uint64_t GetSequenceNumber() { |
||||
return sequence_number_; |
||||
} |
||||
|
||||
void AddFile(const std::string& filename, uint64_t size); |
||||
void Delete(); |
||||
|
||||
bool Empty() { |
||||
return files_.empty(); |
||||
} |
||||
|
||||
const std::vector<std::string>& GetFiles() { |
||||
return files_; |
||||
} |
||||
|
||||
Status LoadFromFile(const std::string& backup_dir); |
||||
Status StoreToFile(bool sync); |
||||
|
||||
private: |
||||
int64_t timestamp_; |
||||
// sequence number is only approximate, should not be used
|
||||
// by clients
|
||||
uint64_t sequence_number_; |
||||
uint64_t size_; |
||||
std::string const meta_filename_; |
||||
// files with relative paths (without "/" prefix!!)
|
||||
std::vector<std::string> files_; |
||||
std::unordered_map<std::string, int>* file_refs_; |
||||
Env* env_; |
||||
|
||||
static const size_t max_backup_meta_file_size_ = 10 * 1024 * 1024; // 10MB
|
||||
}; // BackupMeta
|
||||
|
||||
inline std::string GetAbsolutePath( |
||||
const std::string &relative_path = "") const { |
||||
assert(relative_path.size() == 0 || relative_path[0] != '/'); |
||||
return options_.backup_dir + "/" + relative_path; |
||||
} |
||||
inline std::string GetPrivateDirRel() const { |
||||
return "private"; |
||||
} |
||||
inline std::string GetPrivateFileRel(BackupID backup_id, |
||||
const std::string &file = "") const { |
||||
assert(file.size() == 0 || file[0] != '/'); |
||||
return GetPrivateDirRel() + "/" + std::to_string(backup_id) + "/" + file; |
||||
} |
||||
inline std::string GetSharedFileRel(const std::string& file = "") const { |
||||
assert(file.size() == 0 || file[0] != '/'); |
||||
return "shared/" + file; |
||||
} |
||||
inline std::string GetLatestBackupFile(bool tmp = false) const { |
||||
return GetAbsolutePath(std::string("LATEST_BACKUP") + (tmp ? ".tmp" : "")); |
||||
} |
||||
inline std::string GetBackupMetaDir() const { |
||||
return GetAbsolutePath("meta"); |
||||
} |
||||
inline std::string GetBackupMetaFile(BackupID backup_id) const { |
||||
return GetBackupMetaDir() + "/" + std::to_string(backup_id); |
||||
} |
||||
|
||||
Status GetLatestBackupFileContents(uint32_t* latest_backup); |
||||
Status PutLatestBackupFileContents(uint32_t latest_backup); |
||||
// if size_limit == 0, there is no size limit, copy everything
|
||||
Status CopyFile(const std::string& src, |
||||
const std::string& dst, |
||||
Env* src_env, |
||||
Env* dst_env, |
||||
bool sync, |
||||
uint64_t* size = nullptr, |
||||
uint64_t size_limit = 0); |
||||
// if size_limit == 0, there is no size limit, copy everything
|
||||
Status BackupFile(BackupID backup_id, |
||||
BackupMeta* backup, |
||||
bool shared, |
||||
const std::string& src_dir, |
||||
const std::string& src_fname, // starts with "/"
|
||||
uint64_t size_limit = 0); |
||||
// Will delete all the files we don't need anymore
|
||||
// If full_scan == true, it will do the full scan of files/ directory
|
||||
// and delete all the files that are not referenced from backuped_file_refs_
|
||||
void GarbageCollection(bool full_scan); |
||||
|
||||
// backup state data
|
||||
BackupID latest_backup_id_; |
||||
std::map<BackupID, BackupMeta> backups_; |
||||
std::unordered_map<std::string, int> backuped_file_refs_; |
||||
std::vector<BackupID> obsolete_backups_; |
||||
|
||||
// options data
|
||||
BackupableDBOptions options_; |
||||
Env* db_env_; |
||||
Env* backup_env_; |
||||
|
||||
static const size_t copy_file_buffer_size_ = 5 * 1024 * 1024LL; // 5MB
|
||||
}; |
||||
|
||||
BackupEngine::BackupEngine(Env* db_env, const BackupableDBOptions& options) |
||||
: options_(options), |
||||
db_env_(db_env), |
||||
backup_env_(options.backup_env != nullptr ? options.backup_env : db_env_) { |
||||
|
||||
// create all the dirs we need
|
||||
backup_env_->CreateDirIfMissing(GetAbsolutePath()); |
||||
backup_env_->CreateDirIfMissing(GetAbsolutePath(GetSharedFileRel())); |
||||
backup_env_->CreateDirIfMissing(GetAbsolutePath(GetPrivateDirRel())); |
||||
backup_env_->CreateDirIfMissing(GetBackupMetaDir()); |
||||
|
||||
std::vector<std::string> backup_meta_files; |
||||
backup_env_->GetChildren(GetBackupMetaDir(), &backup_meta_files); |
||||
// create backups_ structure
|
||||
for (auto& file : backup_meta_files) { |
||||
BackupID backup_id = 0; |
||||
sscanf(file.c_str(), "%u", &backup_id); |
||||
if (backup_id == 0 || file != std::to_string(backup_id)) { |
||||
// invalid file name, delete that
|
||||
backup_env_->DeleteFile(GetBackupMetaDir() + "/" + file); |
||||
continue; |
||||
} |
||||
assert(backups_.find(backup_id) == backups_.end()); |
||||
backups_.insert(std::make_pair( |
||||
backup_id, BackupMeta(GetBackupMetaFile(backup_id), |
||||
&backuped_file_refs_, backup_env_))); |
||||
} |
||||
|
||||
if (options_.destroy_old_data) { // Destory old data
|
||||
for (auto& backup : backups_) { |
||||
backup.second.Delete(); |
||||
obsolete_backups_.push_back(backup.first); |
||||
} |
||||
backups_.clear(); |
||||
// start from beginning
|
||||
latest_backup_id_ = 0; |
||||
// GarbageCollection() will do the actual deletion
|
||||
} else { // Load data from storage
|
||||
// load the backups if any
|
||||
for (auto& backup : backups_) { |
||||
Status s = backup.second.LoadFromFile(options_.backup_dir); |
||||
if (!s.ok()) { |
||||
Log(options_.info_log, "Backup %u corrupted - deleting -- %s", |
||||
backup.first, s.ToString().c_str()); |
||||
backup.second.Delete(); |
||||
obsolete_backups_.push_back(backup.first); |
||||
} |
||||
} |
||||
// delete obsolete backups from the structure
|
||||
for (auto ob : obsolete_backups_) { |
||||
backups_.erase(ob); |
||||
} |
||||
|
||||
Status s = GetLatestBackupFileContents(&latest_backup_id_); |
||||
// If latest backup file is corrupted or non-existent
|
||||
// set latest backup as the biggest backup we have
|
||||
// or 0 if we have no backups
|
||||
if (!s.ok() || |
||||
backups_.find(latest_backup_id_) == backups_.end()) { |
||||
auto itr = backups_.end(); |
||||
latest_backup_id_ = (itr == backups_.begin()) ? 0 : (--itr)->first; |
||||
} |
||||
} |
||||
|
||||
// delete any backups that claim to be later than latest
|
||||
for (auto itr = backups_.upper_bound(latest_backup_id_); |
||||
itr != backups_.end();) { |
||||
itr->second.Delete(); |
||||
obsolete_backups_.push_back(itr->first); |
||||
itr = backups_.erase(itr); |
||||
} |
||||
|
||||
PutLatestBackupFileContents(latest_backup_id_); // Ignore errors
|
||||
GarbageCollection(true); |
||||
Log(options_.info_log, |
||||
"Initialized BackupEngine, the latest backup is %u.", |
||||
latest_backup_id_); |
||||
} |
||||
|
||||
BackupEngine::~BackupEngine() { |
||||
LogFlush(options_.info_log); |
||||
} |
||||
|
||||
void BackupEngine::DeleteBackupsNewerThan(uint64_t sequence_number) { |
||||
for (auto backup : backups_) { |
||||
if (backup.second.GetSequenceNumber() > sequence_number) { |
||||
Log(options_.info_log, |
||||
"Deleting backup %u because sequence number (%" PRIu64 |
||||
") is newer than %" PRIu64 "", |
||||
backup.first, backup.second.GetSequenceNumber(), sequence_number); |
||||
backup.second.Delete(); |
||||
obsolete_backups_.push_back(backup.first); |
||||
} |
||||
} |
||||
for (auto ob : obsolete_backups_) { |
||||
backups_.erase(backups_.find(ob)); |
||||
} |
||||
auto itr = backups_.end(); |
||||
latest_backup_id_ = (itr == backups_.begin()) ? 0 : (--itr)->first; |
||||
PutLatestBackupFileContents(latest_backup_id_); // Ignore errors
|
||||
GarbageCollection(false); |
||||
} |
||||
|
||||
Status BackupEngine::CreateNewBackup(DB* db, bool flush_before_backup) { |
||||
Status s; |
||||
std::vector<std::string> live_files; |
||||
VectorLogPtr live_wal_files; |
||||
uint64_t manifest_file_size = 0; |
||||
uint64_t sequence_number = db->GetLatestSequenceNumber(); |
||||
|
||||
s = db->DisableFileDeletions(); |
||||
if (s.ok()) { |
||||
// this will return live_files prefixed with "/"
|
||||
s = db->GetLiveFiles(live_files, &manifest_file_size, flush_before_backup); |
||||
} |
||||
// if we didn't flush before backup, we need to also get WAL files
|
||||
if (s.ok() && !flush_before_backup) { |
||||
// returns file names prefixed with "/"
|
||||
s = db->GetSortedWalFiles(live_wal_files); |
||||
} |
||||
if (!s.ok()) { |
||||
db->EnableFileDeletions(); |
||||
return s; |
||||
} |
||||
|
||||
BackupID new_backup_id = latest_backup_id_ + 1; |
||||
assert(backups_.find(new_backup_id) == backups_.end()); |
||||
auto ret = backups_.insert(std::make_pair( |
||||
new_backup_id, BackupMeta(GetBackupMetaFile(new_backup_id), |
||||
&backuped_file_refs_, backup_env_))); |
||||
assert(ret.second == true); |
||||
auto& new_backup = ret.first->second; |
||||
new_backup.RecordTimestamp(); |
||||
new_backup.SetSequenceNumber(sequence_number); |
||||
|
||||
Log(options_.info_log, "Started the backup process -- creating backup %u", |
||||
new_backup_id); |
||||
|
||||
// create private dir
|
||||
s = backup_env_->CreateDir(GetAbsolutePath(GetPrivateFileRel(new_backup_id))); |
||||
|
||||
// copy live_files
|
||||
for (size_t i = 0; s.ok() && i < live_files.size(); ++i) { |
||||
uint64_t number; |
||||
FileType type; |
||||
bool ok = ParseFileName(live_files[i], &number, &type); |
||||
if (!ok) { |
||||
assert(false); |
||||
return Status::Corruption("Can't parse file name. This is very bad"); |
||||
} |
||||
// we should only get sst, manifest and current files here
|
||||
assert(type == kTableFile || |
||||
type == kDescriptorFile || |
||||
type == kCurrentFile); |
||||
|
||||
// rules:
|
||||
// * if it's kTableFile, than it's shared
|
||||
// * if it's kDescriptorFile, limit the size to manifest_file_size
|
||||
s = BackupFile(new_backup_id, |
||||
&new_backup, |
||||
type == kTableFile, /* shared */ |
||||
db->GetName(), /* src_dir */ |
||||
live_files[i], /* src_fname */ |
||||
(type == kDescriptorFile) ? manifest_file_size : 0); |
||||
} |
||||
|
||||
// copy WAL files
|
||||
for (size_t i = 0; s.ok() && i < live_wal_files.size(); ++i) { |
||||
if (live_wal_files[i]->Type() == kAliveLogFile) { |
||||
// we only care about live log files
|
||||
// copy the file into backup_dir/files/<new backup>/
|
||||
s = BackupFile(new_backup_id, |
||||
&new_backup, |
||||
false, /* not shared */ |
||||
db->GetOptions().wal_dir, |
||||
live_wal_files[i]->PathName()); |
||||
} |
||||
} |
||||
|
||||
// we copied all the files, enable file deletions
|
||||
db->EnableFileDeletions(); |
||||
|
||||
if (s.ok()) { |
||||
// persist the backup metadata on the disk
|
||||
s = new_backup.StoreToFile(options_.sync); |
||||
} |
||||
if (s.ok()) { |
||||
// install the newly created backup meta! (atomic)
|
||||
s = PutLatestBackupFileContents(new_backup_id); |
||||
} |
||||
if (!s.ok()) { |
||||
// clean all the files we might have created
|
||||
Log(options_.info_log, "Backup failed -- %s", s.ToString().c_str()); |
||||
backups_.erase(new_backup_id); |
||||
GarbageCollection(true); |
||||
return s; |
||||
} |
||||
|
||||
// here we know that we succeeded and installed the new backup
|
||||
// in the LATEST_BACKUP file
|
||||
latest_backup_id_ = new_backup_id; |
||||
Log(options_.info_log, "Backup DONE. All is good"); |
||||
return s; |
||||
} |
||||
|
||||
Status BackupEngine::PurgeOldBackups(uint32_t num_backups_to_keep) { |
||||
Log(options_.info_log, "Purging old backups, keeping %u", |
||||
num_backups_to_keep); |
||||
while (num_backups_to_keep < backups_.size()) { |
||||
Log(options_.info_log, "Deleting backup %u", backups_.begin()->first); |
||||
backups_.begin()->second.Delete(); |
||||
obsolete_backups_.push_back(backups_.begin()->first); |
||||
backups_.erase(backups_.begin()); |
||||
} |
||||
GarbageCollection(false); |
||||
return Status::OK(); |
||||
} |
||||
|
||||
Status BackupEngine::DeleteBackup(BackupID backup_id) { |
||||
Log(options_.info_log, "Deleting backup %u", backup_id); |
||||
auto backup = backups_.find(backup_id); |
||||
if (backup == backups_.end()) { |
||||
return Status::NotFound("Backup not found"); |
||||
} |
||||
backup->second.Delete(); |
||||
obsolete_backups_.push_back(backup_id); |
||||
backups_.erase(backup); |
||||
GarbageCollection(false); |
||||
return Status::OK(); |
||||
} |
||||
|
||||
void BackupEngine::GetBackupInfo(std::vector<BackupInfo>* backup_info) { |
||||
backup_info->reserve(backups_.size()); |
||||
for (auto& backup : backups_) { |
||||
if (!backup.second.Empty()) { |
||||
backup_info->push_back(BackupInfo( |
||||
backup.first, backup.second.GetTimestamp(), backup.second.GetSize())); |
||||
} |
||||
} |
||||
} |
||||
|
||||
Status BackupEngine::RestoreDBFromBackup(BackupID backup_id, |
||||
const std::string &db_dir, |
||||
const std::string &wal_dir) { |
||||
auto backup_itr = backups_.find(backup_id); |
||||
if (backup_itr == backups_.end()) { |
||||
return Status::NotFound("Backup not found"); |
||||
} |
||||
auto& backup = backup_itr->second; |
||||
if (backup.Empty()) { |
||||
return Status::NotFound("Backup not found"); |
||||
} |
||||
|
||||
Log(options_.info_log, "Restoring backup id %u\n", backup_id); |
||||
|
||||
// just in case. Ignore errors
|
||||
db_env_->CreateDirIfMissing(db_dir); |
||||
db_env_->CreateDirIfMissing(wal_dir); |
||||
|
||||
// delete log files that might have been already in wal_dir.
|
||||
// This is important since they might get replayed to the restored DB,
|
||||
// which will then differ from the backuped DB
|
||||
std::vector<std::string> delete_children; |
||||
db_env_->GetChildren(wal_dir, &delete_children); // ignore errors
|
||||
for (auto f : delete_children) { |
||||
db_env_->DeleteFile(wal_dir + "/" + f); // ignore errors
|
||||
} |
||||
// Also delete all the db_dir children. This is not so important
|
||||
// because obsolete files will be deleted by DBImpl::PurgeObsoleteFiles()
|
||||
delete_children.clear(); |
||||
db_env_->GetChildren(db_dir, &delete_children); // ignore errors
|
||||
for (auto f : delete_children) { |
||||
db_env_->DeleteFile(db_dir + "/" + f); // ignore errors
|
||||
} |
||||
|
||||
Status s; |
||||
for (auto& file : backup.GetFiles()) { |
||||
std::string dst; |
||||
// 1. extract the filename
|
||||
size_t slash = file.find_last_of('/'); |
||||
// file will either be shared/<file> or private/<number>/<file>
|
||||
assert(slash != std::string::npos); |
||||
dst = file.substr(slash + 1); |
||||
|
||||
// 2. find the filetype
|
||||
uint64_t number; |
||||
FileType type; |
||||
bool ok = ParseFileName(dst, &number, &type); |
||||
if (!ok) { |
||||
return Status::Corruption("Backup corrupted"); |
||||
} |
||||
// 3. Construct the final path
|
||||
// kLogFile lives in wal_dir and all the rest live in db_dir
|
||||
dst = ((type == kLogFile) ? wal_dir : db_dir) + |
||||
"/" + dst; |
||||
|
||||
Log(options_.info_log, "Restoring %s to %s\n", file.c_str(), dst.c_str()); |
||||
s = CopyFile(GetAbsolutePath(file), dst, backup_env_, db_env_, false); |
||||
if (!s.ok()) { |
||||
break; |
||||
} |
||||
} |
||||
|
||||
Log(options_.info_log, "Restoring done -- %s\n", s.ToString().c_str()); |
||||
return s; |
||||
} |
||||
|
||||
// latest backup id is an ASCII representation of latest backup id
|
||||
Status BackupEngine::GetLatestBackupFileContents(uint32_t* latest_backup) { |
||||
Status s; |
||||
unique_ptr<SequentialFile> file; |
||||
s = backup_env_->NewSequentialFile(GetLatestBackupFile(), |
||||
&file, |
||||
EnvOptions()); |
||||
if (!s.ok()) { |
||||
return s; |
||||
} |
||||
|
||||
char buf[11]; |
||||
Slice data; |
||||
s = file->Read(10, &data, buf); |
||||
if (!s.ok() || data.size() == 0) { |
||||
return s.ok() ? Status::Corruption("Latest backup file corrupted") : s; |
||||
} |
||||
buf[data.size()] = 0; |
||||
|
||||
*latest_backup = 0; |
||||
sscanf(data.data(), "%u", latest_backup); |
||||
if (backup_env_->FileExists(GetBackupMetaFile(*latest_backup)) == false) { |
||||
s = Status::Corruption("Latest backup file corrupted"); |
||||
} |
||||
return Status::OK(); |
||||
} |
||||
|
||||
// this operation HAS to be atomic
|
||||
// writing 4 bytes to the file is atomic alright, but we should *never*
|
||||
// do something like 1. delete file, 2. write new file
|
||||
// We write to a tmp file and then atomically rename
|
||||
Status BackupEngine::PutLatestBackupFileContents(uint32_t latest_backup) { |
||||
Status s; |
||||
unique_ptr<WritableFile> file; |
||||
EnvOptions env_options; |
||||
env_options.use_mmap_writes = false; |
||||
s = backup_env_->NewWritableFile(GetLatestBackupFile(true), |
||||
&file, |
||||
env_options); |
||||
if (!s.ok()) { |
||||
backup_env_->DeleteFile(GetLatestBackupFile(true)); |
||||
return s; |
||||
} |
||||
|
||||
char file_contents[10]; |
||||
int len = sprintf(file_contents, "%u\n", latest_backup); |
||||
s = file->Append(Slice(file_contents, len)); |
||||
if (s.ok() && options_.sync) { |
||||
file->Sync(); |
||||
} |
||||
if (s.ok()) { |
||||
s = file->Close(); |
||||
} |
||||
if (s.ok()) { |
||||
// atomically replace real file with new tmp
|
||||
s = backup_env_->RenameFile(GetLatestBackupFile(true), |
||||
GetLatestBackupFile(false)); |
||||
} |
||||
return s; |
||||
} |
||||
|
||||
Status BackupEngine::CopyFile(const std::string& src, |
||||
const std::string& dst, |
||||
Env* src_env, |
||||
Env* dst_env, |
||||
bool sync, |
||||
uint64_t* size, |
||||
uint64_t size_limit) { |
||||
Status s; |
||||
unique_ptr<WritableFile> dst_file; |
||||
unique_ptr<SequentialFile> src_file; |
||||
EnvOptions env_options; |
||||
env_options.use_mmap_writes = false; |
||||
if (size != nullptr) { |
||||
*size = 0; |
||||
} |
||||
|
||||
// Check if size limit is set. if not, set it to very big number
|
||||
if (size_limit == 0) { |
||||
size_limit = std::numeric_limits<uint64_t>::max(); |
||||
} |
||||
|
||||
s = src_env->NewSequentialFile(src, &src_file, env_options); |
||||
if (s.ok()) { |
||||
s = dst_env->NewWritableFile(dst, &dst_file, env_options); |
||||
} |
||||
if (!s.ok()) { |
||||
return s; |
||||
} |
||||
|
||||
unique_ptr<char[]> buf(new char[copy_file_buffer_size_]); |
||||
Slice data; |
||||
|
||||
do { |
||||
size_t buffer_to_read = (copy_file_buffer_size_ < size_limit) ? |
||||
copy_file_buffer_size_ : size_limit; |
||||
s = src_file->Read(buffer_to_read, &data, buf.get()); |
||||
size_limit -= data.size(); |
||||
if (size != nullptr) { |
||||
*size += data.size(); |
||||
} |
||||
if (s.ok()) { |
||||
s = dst_file->Append(data); |
||||
} |
||||
} while (s.ok() && data.size() > 0 && size_limit > 0); |
||||
|
||||
if (s.ok() && sync) { |
||||
s = dst_file->Sync(); |
||||
} |
||||
|
||||
return s; |
||||
} |
||||
|
||||
// src_fname will always start with "/"
|
||||
Status BackupEngine::BackupFile(BackupID backup_id, |
||||
BackupMeta* backup, |
||||
bool shared, |
||||
const std::string& src_dir, |
||||
const std::string& src_fname, |
||||
uint64_t size_limit) { |
||||
|
||||
assert(src_fname.size() > 0 && src_fname[0] == '/'); |
||||
std::string dst_relative = src_fname.substr(1); |
||||
if (shared) { |
||||
dst_relative = GetSharedFileRel(dst_relative); |
||||
} else { |
||||
dst_relative = GetPrivateFileRel(backup_id, dst_relative); |
||||
} |
||||
std::string dst_path = GetAbsolutePath(dst_relative); |
||||
Status s; |
||||
uint64_t size; |
||||
|
||||
// if it's shared, we also need to check if it exists -- if it does,
|
||||
// no need to copy it again
|
||||
if (shared && backup_env_->FileExists(dst_path)) { |
||||
backup_env_->GetFileSize(dst_path, &size); // Ignore error
|
||||
Log(options_.info_log, "%s already present", src_fname.c_str()); |
||||
} else { |
||||
Log(options_.info_log, "Copying %s", src_fname.c_str()); |
||||
s = CopyFile(src_dir + src_fname, |
||||
dst_path, |
||||
db_env_, |
||||
backup_env_, |
||||
options_.sync, |
||||
&size, |
||||
size_limit); |
||||
} |
||||
if (s.ok()) { |
||||
backup->AddFile(dst_relative, size); |
||||
} |
||||
return s; |
||||
} |
||||
|
||||
void BackupEngine::GarbageCollection(bool full_scan) { |
||||
Log(options_.info_log, "Starting garbage collection"); |
||||
std::vector<std::string> to_delete; |
||||
for (auto& itr : backuped_file_refs_) { |
||||
if (itr.second == 0) { |
||||
Status s = backup_env_->DeleteFile(GetAbsolutePath(itr.first)); |
||||
Log(options_.info_log, "Deleting %s -- %s", itr.first.c_str(), |
||||
s.ToString().c_str()); |
||||
to_delete.push_back(itr.first); |
||||
} |
||||
} |
||||
for (auto& td : to_delete) { |
||||
backuped_file_refs_.erase(td); |
||||
} |
||||
if (!full_scan) { |
||||
// take care of private dirs -- if full_scan == true, then full_scan will
|
||||
// take care of them
|
||||
for (auto backup_id : obsolete_backups_) { |
||||
std::string private_dir = GetPrivateFileRel(backup_id); |
||||
Status s = backup_env_->DeleteDir(GetAbsolutePath(private_dir)); |
||||
Log(options_.info_log, "Deleting private dir %s -- %s", |
||||
private_dir.c_str(), s.ToString().c_str()); |
||||
} |
||||
} |
||||
obsolete_backups_.clear(); |
||||
|
||||
if (full_scan) { |
||||
Log(options_.info_log, "Starting full scan garbage collection"); |
||||
// delete obsolete shared files
|
||||
std::vector<std::string> shared_children; |
||||
backup_env_->GetChildren(GetAbsolutePath(GetSharedFileRel()), |
||||
&shared_children); |
||||
for (auto& child : shared_children) { |
||||
std::string rel_fname = GetSharedFileRel(child); |
||||
// if it's not refcounted, delete it
|
||||
if (backuped_file_refs_.find(rel_fname) == backuped_file_refs_.end()) { |
||||
// this might be a directory, but DeleteFile will just fail in that
|
||||
// case, so we're good
|
||||
Status s = backup_env_->DeleteFile(GetAbsolutePath(rel_fname)); |
||||
if (s.ok()) { |
||||
Log(options_.info_log, "Deleted %s", rel_fname.c_str()); |
||||
} |
||||
} |
||||
} |
||||
|
||||
// delete obsolete private files
|
||||
std::vector<std::string> private_children; |
||||
backup_env_->GetChildren(GetAbsolutePath(GetPrivateDirRel()), |
||||
&private_children); |
||||
for (auto& child : private_children) { |
||||
BackupID backup_id = 0; |
||||
sscanf(child.c_str(), "%u", &backup_id); |
||||
if (backup_id == 0 || backups_.find(backup_id) != backups_.end()) { |
||||
// it's either not a number or it's still alive. continue
|
||||
continue; |
||||
} |
||||
// here we have to delete the dir and all its children
|
||||
std::string full_private_path = |
||||
GetAbsolutePath(GetPrivateFileRel(backup_id)); |
||||
std::vector<std::string> subchildren; |
||||
backup_env_->GetChildren(full_private_path, &subchildren); |
||||
for (auto& subchild : subchildren) { |
||||
Status s = backup_env_->DeleteFile(full_private_path + subchild); |
||||
if (s.ok()) { |
||||
Log(options_.info_log, "Deleted %s", |
||||
(full_private_path + subchild).c_str()); |
||||
} |
||||
} |
||||
// finally delete the private dir
|
||||
Status s = backup_env_->DeleteDir(full_private_path); |
||||
Log(options_.info_log, "Deleted dir %s -- %s", full_private_path.c_str(), |
||||
s.ToString().c_str()); |
||||
} |
||||
} |
||||
} |
||||
|
||||
// ------- BackupMeta class --------
|
||||
|
||||
void BackupEngine::BackupMeta::AddFile(const std::string& filename, |
||||
uint64_t size) { |
||||
size_ += size; |
||||
files_.push_back(filename); |
||||
auto itr = file_refs_->find(filename); |
||||
if (itr == file_refs_->end()) { |
||||
file_refs_->insert(std::make_pair(filename, 1)); |
||||
} else { |
||||
++itr->second; // increase refcount if already present
|
||||
} |
||||
} |
||||
|
||||
void BackupEngine::BackupMeta::Delete() { |
||||
for (auto& file : files_) { |
||||
auto itr = file_refs_->find(file); |
||||
assert(itr != file_refs_->end()); |
||||
--(itr->second); // decrease refcount
|
||||
} |
||||
files_.clear(); |
||||
// delete meta file
|
||||
env_->DeleteFile(meta_filename_); |
||||
timestamp_ = 0; |
||||
} |
||||
|
||||
// each backup meta file is of the format:
|
||||
// <timestamp>
|
||||
// <seq number>
|
||||
// <number of files>
|
||||
// <file1>
|
||||
// <file2>
|
||||
// ...
|
||||
// TODO: maybe add checksum?
|
||||
Status BackupEngine::BackupMeta::LoadFromFile(const std::string& backup_dir) { |
||||
assert(Empty()); |
||||
Status s; |
||||
unique_ptr<SequentialFile> backup_meta_file; |
||||
s = env_->NewSequentialFile(meta_filename_, &backup_meta_file, EnvOptions()); |
||||
if (!s.ok()) { |
||||
return s; |
||||
} |
||||
|
||||
unique_ptr<char[]> buf(new char[max_backup_meta_file_size_ + 1]); |
||||
Slice data; |
||||
s = backup_meta_file->Read(max_backup_meta_file_size_, &data, buf.get()); |
||||
|
||||
if (!s.ok() || data.size() == max_backup_meta_file_size_) { |
||||
return s.ok() ? Status::IOError("File size too big") : s; |
||||
} |
||||
buf[data.size()] = 0; |
||||
|
||||
uint32_t num_files = 0; |
||||
int bytes_read = 0; |
||||
sscanf(data.data(), "%" PRId64 "%n", ×tamp_, &bytes_read); |
||||
data.remove_prefix(bytes_read + 1); // +1 for '\n'
|
||||
sscanf(data.data(), "%" PRIu64 "%n", &sequence_number_, &bytes_read); |
||||
data.remove_prefix(bytes_read + 1); // +1 for '\n'
|
||||
sscanf(data.data(), "%u%n", &num_files, &bytes_read); |
||||
data.remove_prefix(bytes_read + 1); // +1 for '\n'
|
||||
|
||||
std::vector<std::pair<std::string, uint64_t>> files; |
||||
|
||||
for (uint32_t i = 0; s.ok() && i < num_files; ++i) { |
||||
std::string filename = GetSliceUntil(&data, '\n').ToString(); |
||||
uint64_t size; |
||||
s = env_->GetFileSize(backup_dir + "/" + filename, &size); |
||||
files.push_back(std::make_pair(filename, size)); |
||||
} |
||||
|
||||
if (s.ok()) { |
||||
for (auto file : files) { |
||||
AddFile(file.first, file.second); |
||||
} |
||||
} |
||||
|
||||
return s; |
||||
} |
||||
|
||||
Status BackupEngine::BackupMeta::StoreToFile(bool sync) { |
||||
Status s; |
||||
unique_ptr<WritableFile> backup_meta_file; |
||||
EnvOptions env_options; |
||||
env_options.use_mmap_writes = false; |
||||
s = env_->NewWritableFile(meta_filename_ + ".tmp", &backup_meta_file, |
||||
env_options); |
||||
if (!s.ok()) { |
||||
return s; |
||||
} |
||||
|
||||
unique_ptr<char[]> buf(new char[max_backup_meta_file_size_]); |
||||
int len = 0, buf_size = max_backup_meta_file_size_; |
||||
len += snprintf(buf.get(), buf_size, "%" PRId64 "\n", timestamp_); |
||||
len += snprintf(buf.get() + len, buf_size - len, "%" PRIu64 "\n", |
||||
sequence_number_); |
||||
len += snprintf(buf.get() + len, buf_size - len, "%zu\n", files_.size()); |
||||
for (size_t i = 0; i < files_.size(); ++i) { |
||||
len += snprintf(buf.get() + len, buf_size - len, "%s\n", files_[i].c_str()); |
||||
} |
||||
|
||||
s = backup_meta_file->Append(Slice(buf.get(), (size_t)len)); |
||||
if (s.ok() && sync) { |
||||
s = backup_meta_file->Sync(); |
||||
} |
||||
if (s.ok()) { |
||||
s = backup_meta_file->Close(); |
||||
} |
||||
if (s.ok()) { |
||||
s = env_->RenameFile(meta_filename_ + ".tmp", meta_filename_); |
||||
} |
||||
return s; |
||||
} |
||||
|
||||
// --- BackupableDB methods --------
|
||||
|
||||
BackupableDB::BackupableDB(DB* db, const BackupableDBOptions& options) |
||||
: StackableDB(db), backup_engine_(new BackupEngine(db->GetEnv(), options)) { |
||||
backup_engine_->DeleteBackupsNewerThan(GetLatestSequenceNumber()); |
||||
} |
||||
|
||||
BackupableDB::~BackupableDB() { |
||||
delete backup_engine_; |
||||
} |
||||
|
||||
Status BackupableDB::CreateNewBackup(bool flush_before_backup) { |
||||
return backup_engine_->CreateNewBackup(this, flush_before_backup); |
||||
} |
||||
|
||||
void BackupableDB::GetBackupInfo(std::vector<BackupInfo>* backup_info) { |
||||
backup_engine_->GetBackupInfo(backup_info); |
||||
} |
||||
|
||||
Status BackupableDB::PurgeOldBackups(uint32_t num_backups_to_keep) { |
||||
return backup_engine_->PurgeOldBackups(num_backups_to_keep); |
||||
} |
||||
|
||||
Status BackupableDB::DeleteBackup(BackupID backup_id) { |
||||
return backup_engine_->DeleteBackup(backup_id); |
||||
} |
||||
|
||||
// --- RestoreBackupableDB methods ------
|
||||
|
||||
RestoreBackupableDB::RestoreBackupableDB(Env* db_env, |
||||
const BackupableDBOptions& options) |
||||
: backup_engine_(new BackupEngine(db_env, options)) {} |
||||
|
||||
RestoreBackupableDB::~RestoreBackupableDB() { |
||||
delete backup_engine_; |
||||
} |
||||
|
||||
void |
||||
RestoreBackupableDB::GetBackupInfo(std::vector<BackupInfo>* backup_info) { |
||||
backup_engine_->GetBackupInfo(backup_info); |
||||
} |
||||
|
||||
Status RestoreBackupableDB::RestoreDBFromBackup(BackupID backup_id, |
||||
const std::string& db_dir, |
||||
const std::string& wal_dir) { |
||||
return backup_engine_->RestoreDBFromBackup(backup_id, db_dir, wal_dir); |
||||
} |
||||
|
||||
Status |
||||
RestoreBackupableDB::RestoreDBFromLatestBackup(const std::string& db_dir, |
||||
const std::string& wal_dir) { |
||||
return backup_engine_->RestoreDBFromLatestBackup(db_dir, wal_dir); |
||||
} |
||||
|
||||
Status RestoreBackupableDB::PurgeOldBackups(uint32_t num_backups_to_keep) { |
||||
return backup_engine_->PurgeOldBackups(num_backups_to_keep); |
||||
} |
||||
|
||||
Status RestoreBackupableDB::DeleteBackup(BackupID backup_id) { |
||||
return backup_engine_->DeleteBackup(backup_id); |
||||
} |
||||
|
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,668 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "rocksdb/types.h" |
||||
#include "rocksdb/transaction_log.h" |
||||
#include "utilities/utility_db.h" |
||||
#include "utilities/backupable_db.h" |
||||
#include "util/testharness.h" |
||||
#include "util/random.h" |
||||
#include "util/testutil.h" |
||||
#include "util/auto_roll_logger.h" |
||||
|
||||
#include <string> |
||||
#include <algorithm> |
||||
|
||||
namespace rocksdb { |
||||
|
||||
namespace { |
||||
|
||||
using std::unique_ptr; |
||||
|
||||
class DummyDB : public StackableDB { |
||||
public: |
||||
/* implicit */ |
||||
DummyDB(const Options& options, const std::string& dbname) |
||||
: StackableDB(nullptr), options_(options), dbname_(dbname), |
||||
deletions_enabled_(true), sequence_number_(0) {} |
||||
|
||||
virtual SequenceNumber GetLatestSequenceNumber() const { |
||||
return ++sequence_number_; |
||||
} |
||||
|
||||
virtual const std::string& GetName() const override { |
||||
return dbname_; |
||||
} |
||||
|
||||
virtual Env* GetEnv() const override { |
||||
return options_.env; |
||||
} |
||||
|
||||
virtual const Options& GetOptions() const override { |
||||
return options_; |
||||
} |
||||
|
||||
virtual Status EnableFileDeletions() override { |
||||
ASSERT_TRUE(!deletions_enabled_); |
||||
deletions_enabled_ = true; |
||||
return Status::OK(); |
||||
} |
||||
|
||||
virtual Status DisableFileDeletions() override { |
||||
ASSERT_TRUE(deletions_enabled_); |
||||
deletions_enabled_ = false; |
||||
return Status::OK(); |
||||
} |
||||
|
||||
virtual Status GetLiveFiles(std::vector<std::string>& vec, uint64_t* mfs, |
||||
bool flush_memtable = true) override { |
||||
ASSERT_TRUE(!deletions_enabled_); |
||||
vec = live_files_; |
||||
*mfs = 100; |
||||
return Status::OK(); |
||||
} |
||||
|
||||
class DummyLogFile : public LogFile { |
||||
public: |
||||
/* implicit */ |
||||
DummyLogFile(const std::string& path, bool alive = true) |
||||
: path_(path), alive_(alive) {} |
||||
|
||||
virtual std::string PathName() const override { |
||||
return path_; |
||||
} |
||||
|
||||
virtual uint64_t LogNumber() const { |
||||
// what business do you have calling this method?
|
||||
ASSERT_TRUE(false); |
||||
return 0; |
||||
} |
||||
|
||||
virtual WalFileType Type() const override { |
||||
return alive_ ? kAliveLogFile : kArchivedLogFile; |
||||
} |
||||
|
||||
virtual SequenceNumber StartSequence() const { |
||||
// backupabledb should not need this method
|
||||
ASSERT_TRUE(false); |
||||
return 0; |
||||
} |
||||
|
||||
virtual uint64_t SizeFileBytes() const { |
||||
// backupabledb should not need this method
|
||||
ASSERT_TRUE(false); |
||||
return 0; |
||||
} |
||||
|
||||
private: |
||||
std::string path_; |
||||
bool alive_; |
||||
}; // DummyLogFile
|
||||
|
||||
virtual Status GetSortedWalFiles(VectorLogPtr& files) override { |
||||
ASSERT_TRUE(!deletions_enabled_); |
||||
files.resize(wal_files_.size()); |
||||
for (size_t i = 0; i < files.size(); ++i) { |
||||
files[i].reset( |
||||
new DummyLogFile(wal_files_[i].first, wal_files_[i].second)); |
||||
} |
||||
return Status::OK(); |
||||
} |
||||
|
||||
std::vector<std::string> live_files_; |
||||
// pair<filename, alive?>
|
||||
std::vector<std::pair<std::string, bool>> wal_files_; |
||||
private: |
||||
Options options_; |
||||
std::string dbname_; |
||||
bool deletions_enabled_; |
||||
mutable SequenceNumber sequence_number_; |
||||
}; // DummyDB
|
||||
|
||||
class TestEnv : public EnvWrapper { |
||||
public: |
||||
explicit TestEnv(Env* t) : EnvWrapper(t) {} |
||||
|
||||
class DummySequentialFile : public SequentialFile { |
||||
public: |
||||
DummySequentialFile() : SequentialFile(), rnd_(5) {} |
||||
virtual Status Read(size_t n, Slice* result, char* scratch) { |
||||
size_t read_size = (n > size_left) ? size_left : n; |
||||
for (size_t i = 0; i < read_size; ++i) { |
||||
scratch[i] = rnd_.Next() & 255; |
||||
} |
||||
*result = Slice(scratch, read_size); |
||||
size_left -= read_size; |
||||
return Status::OK(); |
||||
} |
||||
|
||||
virtual Status Skip(uint64_t n) { |
||||
size_left = (n > size_left) ? size_left - n : 0; |
||||
return Status::OK(); |
||||
} |
||||
private: |
||||
size_t size_left = 200; |
||||
Random rnd_; |
||||
}; |
||||
|
||||
Status NewSequentialFile(const std::string& f, |
||||
unique_ptr<SequentialFile>* r, |
||||
const EnvOptions& options) { |
||||
opened_files_.push_back(f); |
||||
if (dummy_sequential_file_) { |
||||
r->reset(new TestEnv::DummySequentialFile()); |
||||
return Status::OK(); |
||||
} else { |
||||
return EnvWrapper::NewSequentialFile(f, r, options); |
||||
} |
||||
} |
||||
|
||||
Status NewWritableFile(const std::string& f, unique_ptr<WritableFile>* r, |
||||
const EnvOptions& options) { |
||||
if (limit_written_files_ <= 0) { |
||||
return Status::IOError("Sorry, can't do this"); |
||||
} |
||||
limit_written_files_--; |
||||
return EnvWrapper::NewWritableFile(f, r, options); |
||||
} |
||||
|
||||
void AssertOpenedFiles(std::vector<std::string>& should_have_opened) { |
||||
sort(should_have_opened.begin(), should_have_opened.end()); |
||||
sort(opened_files_.begin(), opened_files_.end()); |
||||
ASSERT_TRUE(opened_files_ == should_have_opened); |
||||
} |
||||
|
||||
void ClearOpenedFiles() { |
||||
opened_files_.clear(); |
||||
} |
||||
|
||||
void SetLimitWrittenFiles(uint64_t limit) { |
||||
limit_written_files_ = limit; |
||||
} |
||||
|
||||
void SetDummySequentialFile(bool dummy_sequential_file) { |
||||
dummy_sequential_file_ = dummy_sequential_file; |
||||
} |
||||
|
||||
private: |
||||
bool dummy_sequential_file_ = false; |
||||
std::vector<std::string> opened_files_; |
||||
uint64_t limit_written_files_ = 1000000; |
||||
}; // TestEnv
|
||||
|
||||
class FileManager : public EnvWrapper { |
||||
public: |
||||
explicit FileManager(Env* t) : EnvWrapper(t), rnd_(5) {} |
||||
|
||||
Status DeleteRandomFileInDir(const std::string dir) { |
||||
std::vector<std::string> children; |
||||
GetChildren(dir, &children); |
||||
if (children.size() <= 2) { // . and ..
|
||||
return Status::NotFound(""); |
||||
} |
||||
while (true) { |
||||
int i = rnd_.Next() % children.size(); |
||||
if (children[i] != "." && children[i] != "..") { |
||||
return DeleteFile(dir + "/" + children[i]); |
||||
} |
||||
} |
||||
// should never get here
|
||||
assert(false); |
||||
return Status::NotFound(""); |
||||
} |
||||
|
||||
Status CorruptFile(const std::string& fname, uint64_t bytes_to_corrupt) { |
||||
uint64_t size; |
||||
Status s = GetFileSize(fname, &size); |
||||
if (!s.ok()) { |
||||
return s; |
||||
} |
||||
unique_ptr<RandomRWFile> file; |
||||
EnvOptions env_options; |
||||
env_options.use_mmap_writes = false; |
||||
s = NewRandomRWFile(fname, &file, env_options); |
||||
if (!s.ok()) { |
||||
return s; |
||||
} |
||||
|
||||
for (uint64_t i = 0; s.ok() && i < bytes_to_corrupt; ++i) { |
||||
std::string tmp; |
||||
// write one random byte to a random position
|
||||
s = file->Write(rnd_.Next() % size, test::RandomString(&rnd_, 1, &tmp)); |
||||
} |
||||
return s; |
||||
} |
||||
|
||||
Status WriteToFile(const std::string& fname, const std::string& data) { |
||||
unique_ptr<WritableFile> file; |
||||
EnvOptions env_options; |
||||
env_options.use_mmap_writes = false; |
||||
Status s = EnvWrapper::NewWritableFile(fname, &file, env_options); |
||||
if (!s.ok()) { |
||||
return s; |
||||
} |
||||
return file->Append(Slice(data)); |
||||
} |
||||
private: |
||||
Random rnd_; |
||||
}; // FileManager
|
||||
|
||||
// utility functions
|
||||
static void FillDB(DB* db, int from, int to) { |
||||
for (int i = from; i < to; ++i) { |
||||
std::string key = "testkey" + std::to_string(i); |
||||
std::string value = "testvalue" + std::to_string(i); |
||||
|
||||
ASSERT_OK(db->Put(WriteOptions(), Slice(key), Slice(value))); |
||||
} |
||||
} |
||||
|
||||
static void AssertExists(DB* db, int from, int to) { |
||||
for (int i = from; i < to; ++i) { |
||||
std::string key = "testkey" + std::to_string(i); |
||||
std::string value; |
||||
Status s = db->Get(ReadOptions(), Slice(key), &value); |
||||
ASSERT_EQ(value, "testvalue" + std::to_string(i)); |
||||
} |
||||
} |
||||
|
||||
static void AssertEmpty(DB* db, int from, int to) { |
||||
for (int i = from; i < to; ++i) { |
||||
std::string key = "testkey" + std::to_string(i); |
||||
std::string value = "testvalue" + std::to_string(i); |
||||
|
||||
Status s = db->Get(ReadOptions(), Slice(key), &value); |
||||
ASSERT_TRUE(s.IsNotFound()); |
||||
} |
||||
} |
||||
|
||||
class BackupableDBTest { |
||||
public: |
||||
BackupableDBTest() { |
||||
// set up files
|
||||
dbname_ = test::TmpDir() + "/backupable_db"; |
||||
backupdir_ = test::TmpDir() + "/backupable_db_backup"; |
||||
|
||||
// set up envs
|
||||
env_ = Env::Default(); |
||||
test_db_env_.reset(new TestEnv(env_)); |
||||
test_backup_env_.reset(new TestEnv(env_)); |
||||
file_manager_.reset(new FileManager(env_)); |
||||
|
||||
// set up db options
|
||||
options_.create_if_missing = true; |
||||
options_.paranoid_checks = true; |
||||
options_.write_buffer_size = 1 << 17; // 128KB
|
||||
options_.env = test_db_env_.get(); |
||||
options_.wal_dir = dbname_; |
||||
// set up backup db options
|
||||
CreateLoggerFromOptions(dbname_, backupdir_, env_, |
||||
Options(), &logger_); |
||||
backupable_options_.reset(new BackupableDBOptions( |
||||
backupdir_, test_backup_env_.get(), logger_.get(), true)); |
||||
|
||||
// delete old files in db
|
||||
DestroyDB(dbname_, Options()); |
||||
} |
||||
|
||||
DB* OpenDB() { |
||||
DB* db; |
||||
ASSERT_OK(DB::Open(options_, dbname_, &db)); |
||||
return db; |
||||
} |
||||
|
||||
void OpenBackupableDB(bool destroy_old_data = false, bool dummy = false) { |
||||
// reset all the defaults
|
||||
test_backup_env_->SetLimitWrittenFiles(1000000); |
||||
test_db_env_->SetLimitWrittenFiles(1000000); |
||||
test_db_env_->SetDummySequentialFile(dummy); |
||||
|
||||
DB* db; |
||||
if (dummy) { |
||||
dummy_db_ = new DummyDB(options_, dbname_); |
||||
db = dummy_db_; |
||||
} else { |
||||
ASSERT_OK(DB::Open(options_, dbname_, &db)); |
||||
} |
||||
backupable_options_->destroy_old_data = destroy_old_data; |
||||
db_.reset(new BackupableDB(db, *backupable_options_)); |
||||
} |
||||
|
||||
void CloseBackupableDB() { |
||||
db_.reset(nullptr); |
||||
} |
||||
|
||||
void OpenRestoreDB() { |
||||
backupable_options_->destroy_old_data = false; |
||||
restore_db_.reset( |
||||
new RestoreBackupableDB(test_db_env_.get(), *backupable_options_)); |
||||
} |
||||
|
||||
void CloseRestoreDB() { |
||||
restore_db_.reset(nullptr); |
||||
} |
||||
|
||||
// restores backup backup_id and asserts the existence of
|
||||
// [start_exist, end_exist> and not-existence of
|
||||
// [end_exist, end>
|
||||
//
|
||||
// if backup_id == 0, it means restore from latest
|
||||
// if end == 0, don't check AssertEmpty
|
||||
void AssertBackupConsistency(BackupID backup_id, uint32_t start_exist, |
||||
uint32_t end_exist, uint32_t end = 0) { |
||||
bool opened_restore = false; |
||||
if (restore_db_.get() == nullptr) { |
||||
opened_restore = true; |
||||
OpenRestoreDB(); |
||||
} |
||||
if (backup_id > 0) { |
||||
ASSERT_OK(restore_db_->RestoreDBFromBackup(backup_id, dbname_, dbname_)); |
||||
} else { |
||||
ASSERT_OK(restore_db_->RestoreDBFromLatestBackup(dbname_, dbname_)); |
||||
} |
||||
DB* db = OpenDB(); |
||||
AssertExists(db, start_exist, end_exist); |
||||
if (end != 0) { |
||||
AssertEmpty(db, end_exist, end); |
||||
} |
||||
delete db; |
||||
if (opened_restore) { |
||||
CloseRestoreDB(); |
||||
} |
||||
} |
||||
|
||||
// files
|
||||
std::string dbname_; |
||||
std::string backupdir_; |
||||
|
||||
// envs
|
||||
Env* env_; |
||||
unique_ptr<TestEnv> test_db_env_; |
||||
unique_ptr<TestEnv> test_backup_env_; |
||||
unique_ptr<FileManager> file_manager_; |
||||
|
||||
// all the dbs!
|
||||
DummyDB* dummy_db_; // BackupableDB owns dummy_db_
|
||||
unique_ptr<BackupableDB> db_; |
||||
unique_ptr<RestoreBackupableDB> restore_db_; |
||||
|
||||
// options
|
||||
Options options_; |
||||
unique_ptr<BackupableDBOptions> backupable_options_; |
||||
std::shared_ptr<Logger> logger_; |
||||
}; // BackupableDBTest
|
||||
|
||||
void AppendPath(const std::string& path, std::vector<std::string>& v) { |
||||
for (auto& f : v) { |
||||
f = path + f; |
||||
} |
||||
} |
||||
|
||||
// this will make sure that backup does not copy the same file twice
|
||||
TEST(BackupableDBTest, NoDoubleCopy) { |
||||
OpenBackupableDB(true, true); |
||||
|
||||
// should write 5 DB files + LATEST_BACKUP + one meta file
|
||||
test_backup_env_->SetLimitWrittenFiles(7); |
||||
test_db_env_->ClearOpenedFiles(); |
||||
test_db_env_->SetLimitWrittenFiles(0); |
||||
dummy_db_->live_files_ = { "/00010.sst", "/00011.sst", |
||||
"/CURRENT", "/MANIFEST-01" }; |
||||
dummy_db_->wal_files_ = {{"/00011.log", true}, {"/00012.log", false}}; |
||||
ASSERT_OK(db_->CreateNewBackup(false)); |
||||
std::vector<std::string> should_have_openened = dummy_db_->live_files_; |
||||
should_have_openened.push_back("/00011.log"); |
||||
AppendPath(dbname_, should_have_openened); |
||||
test_db_env_->AssertOpenedFiles(should_have_openened); |
||||
|
||||
// should write 4 new DB files + LATEST_BACKUP + one meta file
|
||||
// should not write/copy 00010.sst, since it's already there!
|
||||
test_backup_env_->SetLimitWrittenFiles(6); |
||||
test_db_env_->ClearOpenedFiles(); |
||||
dummy_db_->live_files_ = { "/00010.sst", "/00015.sst", |
||||
"/CURRENT", "/MANIFEST-01" }; |
||||
dummy_db_->wal_files_ = {{"/00011.log", true}, {"/00012.log", false}}; |
||||
ASSERT_OK(db_->CreateNewBackup(false)); |
||||
// should not open 00010.sst - it's already there
|
||||
should_have_openened = { "/00015.sst", "/CURRENT", |
||||
"/MANIFEST-01", "/00011.log" }; |
||||
AppendPath(dbname_, should_have_openened); |
||||
test_db_env_->AssertOpenedFiles(should_have_openened); |
||||
|
||||
ASSERT_OK(db_->DeleteBackup(1)); |
||||
ASSERT_EQ(true, |
||||
test_backup_env_->FileExists(backupdir_ + "/shared/00010.sst")); |
||||
// 00011.sst was only in backup 1, should be deleted
|
||||
ASSERT_EQ(false, |
||||
test_backup_env_->FileExists(backupdir_ + "/shared/00011.sst")); |
||||
ASSERT_EQ(true, |
||||
test_backup_env_->FileExists(backupdir_ + "/shared/00015.sst")); |
||||
|
||||
// MANIFEST file size should be only 100
|
||||
uint64_t size; |
||||
test_backup_env_->GetFileSize(backupdir_ + "/private/2/MANIFEST-01", &size); |
||||
ASSERT_EQ(100UL, size); |
||||
test_backup_env_->GetFileSize(backupdir_ + "/shared/00015.sst", &size); |
||||
ASSERT_EQ(200UL, size); |
||||
|
||||
CloseBackupableDB(); |
||||
} |
||||
|
||||
// test various kind of corruptions that may happen:
|
||||
// 1. Not able to write a file for backup - that backup should fail,
|
||||
// everything else should work
|
||||
// 2. Corrupted/deleted LATEST_BACKUP - everything should work fine
|
||||
// 3. Corrupted backup meta file or missing backuped file - we should
|
||||
// not be able to open that backup, but all other backups should be
|
||||
// fine
|
||||
TEST(BackupableDBTest, CorruptionsTest) { |
||||
const int keys_iteration = 5000; |
||||
Random rnd(6); |
||||
Status s; |
||||
|
||||
OpenBackupableDB(true); |
||||
// create five backups
|
||||
for (int i = 0; i < 5; ++i) { |
||||
FillDB(db_.get(), keys_iteration * i, keys_iteration * (i + 1)); |
||||
ASSERT_OK(db_->CreateNewBackup(!!(rnd.Next() % 2))); |
||||
} |
||||
|
||||
// ---------- case 1. - fail a write -----------
|
||||
// try creating backup 6, but fail a write
|
||||
FillDB(db_.get(), keys_iteration * 5, keys_iteration * 6); |
||||
test_backup_env_->SetLimitWrittenFiles(2); |
||||
// should fail
|
||||
s = db_->CreateNewBackup(!!(rnd.Next() % 2)); |
||||
ASSERT_TRUE(!s.ok()); |
||||
test_backup_env_->SetLimitWrittenFiles(1000000); |
||||
// latest backup should have all the keys
|
||||
CloseBackupableDB(); |
||||
AssertBackupConsistency(0, 0, keys_iteration * 5, keys_iteration * 6); |
||||
|
||||
// ---------- case 2. - corrupt/delete latest backup -----------
|
||||
ASSERT_OK(file_manager_->CorruptFile(backupdir_ + "/LATEST_BACKUP", 2)); |
||||
AssertBackupConsistency(0, 0, keys_iteration * 5); |
||||
ASSERT_OK(file_manager_->DeleteFile(backupdir_ + "/LATEST_BACKUP")); |
||||
AssertBackupConsistency(0, 0, keys_iteration * 5); |
||||
// create backup 6, point LATEST_BACKUP to 5
|
||||
OpenBackupableDB(); |
||||
FillDB(db_.get(), keys_iteration * 5, keys_iteration * 6); |
||||
ASSERT_OK(db_->CreateNewBackup(false)); |
||||
CloseBackupableDB(); |
||||
ASSERT_OK(file_manager_->WriteToFile(backupdir_ + "/LATEST_BACKUP", "5")); |
||||
AssertBackupConsistency(0, 0, keys_iteration * 5, keys_iteration * 6); |
||||
// assert that all 6 data is gone!
|
||||
ASSERT_TRUE(file_manager_->FileExists(backupdir_ + "/meta/6") == false); |
||||
ASSERT_TRUE(file_manager_->FileExists(backupdir_ + "/private/6") == false); |
||||
|
||||
// --------- case 3. corrupted backup meta or missing backuped file ----
|
||||
ASSERT_OK(file_manager_->CorruptFile(backupdir_ + "/meta/5", 3)); |
||||
// since 5 meta is now corrupted, latest backup should be 4
|
||||
AssertBackupConsistency(0, 0, keys_iteration * 4, keys_iteration * 5); |
||||
OpenRestoreDB(); |
||||
s = restore_db_->RestoreDBFromBackup(5, dbname_, dbname_); |
||||
ASSERT_TRUE(!s.ok()); |
||||
CloseRestoreDB(); |
||||
ASSERT_OK(file_manager_->DeleteRandomFileInDir(backupdir_ + "/private/4")); |
||||
// 4 is corrupted, 3 is the latest backup now
|
||||
AssertBackupConsistency(0, 0, keys_iteration * 3, keys_iteration * 5); |
||||
OpenRestoreDB(); |
||||
s = restore_db_->RestoreDBFromBackup(4, dbname_, dbname_); |
||||
CloseRestoreDB(); |
||||
ASSERT_TRUE(!s.ok()); |
||||
|
||||
// new backup should be 4!
|
||||
OpenBackupableDB(); |
||||
FillDB(db_.get(), keys_iteration * 3, keys_iteration * 4); |
||||
ASSERT_OK(db_->CreateNewBackup(!!(rnd.Next() % 2))); |
||||
CloseBackupableDB(); |
||||
AssertBackupConsistency(4, 0, keys_iteration * 4, keys_iteration * 5); |
||||
} |
||||
|
||||
// open DB, write, close DB, backup, restore, repeat
|
||||
TEST(BackupableDBTest, OfflineIntegrationTest) { |
||||
// has to be a big number, so that it triggers the memtable flush
|
||||
const int keys_iteration = 5000; |
||||
const int max_key = keys_iteration * 4 + 10; |
||||
// first iter -- flush before backup
|
||||
// second iter -- don't flush before backup
|
||||
for (int iter = 0; iter < 2; ++iter) { |
||||
// delete old data
|
||||
DestroyDB(dbname_, Options()); |
||||
bool destroy_data = true; |
||||
|
||||
// every iteration --
|
||||
// 1. insert new data in the DB
|
||||
// 2. backup the DB
|
||||
// 3. destroy the db
|
||||
// 4. restore the db, check everything is still there
|
||||
for (int i = 0; i < 5; ++i) { |
||||
// in last iteration, put smaller amount of data,
|
||||
int fill_up_to = std::min(keys_iteration * (i + 1), max_key); |
||||
// ---- insert new data and back up ----
|
||||
OpenBackupableDB(destroy_data); |
||||
destroy_data = false; |
||||
FillDB(db_.get(), keys_iteration * i, fill_up_to); |
||||
ASSERT_OK(db_->CreateNewBackup(iter == 0)); |
||||
CloseBackupableDB(); |
||||
DestroyDB(dbname_, Options()); |
||||
|
||||
// ---- make sure it's empty ----
|
||||
DB* db = OpenDB(); |
||||
AssertEmpty(db, 0, fill_up_to); |
||||
delete db; |
||||
|
||||
// ---- restore the DB ----
|
||||
OpenRestoreDB(); |
||||
if (i >= 3) { // test purge old backups
|
||||
// when i == 4, purge to only 1 backup
|
||||
// when i == 3, purge to 2 backups
|
||||
ASSERT_OK(restore_db_->PurgeOldBackups(5 - i)); |
||||
} |
||||
// ---- make sure the data is there ---
|
||||
AssertBackupConsistency(0, 0, fill_up_to, max_key); |
||||
CloseRestoreDB(); |
||||
} |
||||
} |
||||
} |
||||
|
||||
// open DB, write, backup, write, backup, close, restore
|
||||
TEST(BackupableDBTest, OnlineIntegrationTest) { |
||||
// has to be a big number, so that it triggers the memtable flush
|
||||
const int keys_iteration = 5000; |
||||
const int max_key = keys_iteration * 4 + 10; |
||||
Random rnd(7); |
||||
// delete old data
|
||||
DestroyDB(dbname_, Options()); |
||||
|
||||
OpenBackupableDB(true); |
||||
// write some data, backup, repeat
|
||||
for (int i = 0; i < 5; ++i) { |
||||
if (i == 4) { |
||||
// delete backup number 2, online delete!
|
||||
OpenRestoreDB(); |
||||
ASSERT_OK(restore_db_->DeleteBackup(2)); |
||||
CloseRestoreDB(); |
||||
} |
||||
// in last iteration, put smaller amount of data,
|
||||
// so that backups can share sst files
|
||||
int fill_up_to = std::min(keys_iteration * (i + 1), max_key); |
||||
FillDB(db_.get(), keys_iteration * i, fill_up_to); |
||||
// we should get consistent results with flush_before_backup
|
||||
// set to both true and false
|
||||
ASSERT_OK(db_->CreateNewBackup(!!(rnd.Next() % 2))); |
||||
} |
||||
// close and destroy
|
||||
CloseBackupableDB(); |
||||
DestroyDB(dbname_, Options()); |
||||
|
||||
// ---- make sure it's empty ----
|
||||
DB* db = OpenDB(); |
||||
AssertEmpty(db, 0, max_key); |
||||
delete db; |
||||
|
||||
// ---- restore every backup and verify all the data is there ----
|
||||
OpenRestoreDB(); |
||||
for (int i = 1; i <= 5; ++i) { |
||||
if (i == 2) { |
||||
// we deleted backup 2
|
||||
Status s = restore_db_->RestoreDBFromBackup(2, dbname_, dbname_); |
||||
ASSERT_TRUE(!s.ok()); |
||||
} else { |
||||
int fill_up_to = std::min(keys_iteration * i, max_key); |
||||
AssertBackupConsistency(i, 0, fill_up_to, max_key); |
||||
} |
||||
} |
||||
|
||||
// delete some backups -- this should leave only backups 3 and 5 alive
|
||||
ASSERT_OK(restore_db_->DeleteBackup(4)); |
||||
ASSERT_OK(restore_db_->PurgeOldBackups(2)); |
||||
|
||||
std::vector<BackupInfo> backup_info; |
||||
restore_db_->GetBackupInfo(&backup_info); |
||||
ASSERT_EQ(2UL, backup_info.size()); |
||||
|
||||
// check backup 3
|
||||
AssertBackupConsistency(3, 0, 3 * keys_iteration, max_key); |
||||
// check backup 5
|
||||
AssertBackupConsistency(5, 0, max_key); |
||||
|
||||
CloseRestoreDB(); |
||||
} |
||||
|
||||
TEST(BackupableDBTest, DeleteNewerBackups) { |
||||
// create backups 1, 2, 3, 4, 5
|
||||
OpenBackupableDB(true); |
||||
for (int i = 0; i < 5; ++i) { |
||||
FillDB(db_.get(), 100 * i, 100 * (i + 1)); |
||||
ASSERT_OK(db_->CreateNewBackup(!!(i % 2))); |
||||
} |
||||
CloseBackupableDB(); |
||||
|
||||
// backup 3 is fine
|
||||
AssertBackupConsistency(3, 0, 300, 500); |
||||
// this should delete backups 4 and 5
|
||||
OpenBackupableDB(); |
||||
CloseBackupableDB(); |
||||
// backups 4 and 5 don't exist
|
||||
OpenRestoreDB(); |
||||
Status s = restore_db_->RestoreDBFromBackup(4, dbname_, dbname_); |
||||
ASSERT_TRUE(s.IsNotFound()); |
||||
s = restore_db_->RestoreDBFromBackup(5, dbname_, dbname_); |
||||
ASSERT_TRUE(s.IsNotFound()); |
||||
CloseRestoreDB(); |
||||
} |
||||
|
||||
} // anon namespace
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) { |
||||
return rocksdb::test::RunAllTests(); |
||||
} |
||||
Binary file not shown.
Loading…
Reference in new issue