Refactor expected state in stress/crash test (#8913)
Summary: This is a precursor refactoring to enable an upcoming feature: persistence failure correctness testing. - Changed `--expected_values_path` to `--expected_values_dir` and migrated "db_crashtest.py" to use the new flag. For persistence failure correctness testing there are multiple possible correct states since unsynced data is allowed to be dropped. Making it possible to restore all these possible correct states will eventually involve files containing snapshots of expected values and DB trace files. - The expected values directory is managed by an `ExpectedStateManager` instance. Managing expected state files is separated out of `SharedState` to prevent `SharedState` from becoming too complex when the new files and features (snapshotting, tracing, and restoring) are introduced. - Migrated expected values file access/management out of `SharedState` into a separate class called `ExpectedState`. This is not exposed directly to the test but rather the `ExpectedState` for the latest values file is accessed via a pass-through API on `ExpectedStateManager`. This forces the test to always access the single latest `ExpectedState`. - Changed the initialization of the latest expected values file to use a tempfile followed by rename, and also add cleanup logic for possible stranded tempfiles. Pull Request resolved: https://github.com/facebook/rocksdb/pull/8913 Test Plan: run in several ways; try to make sure it's not obviously broken. - crashtest blackbox without TEST_TMPDIR ``` $ python3 tools/db_crashtest.py blackbox --simple --write_buffer_size=1048576 --target_file_size_base=1048576 --max_bytes_for_level_base=4194304 --max_key=100000 --value_size_mult=33 --compression_type=none --duration=120 --interval=10 --compression_type=none --blob_compression_type=none ``` - crashtest blackbox with TEST_TMPDIR ``` $ TEST_TMPDIR=/dev/shm python3 tools/db_crashtest.py blackbox --simple --write_buffer_size=1048576 --target_file_size_base=1048576 --max_bytes_for_level_base=4194304 --max_key=100000 --value_size_mult=33 --compression_type=none --duration=120 --interval=10 --compression_type=none --blob_compression_type=none ``` - crashtest whitebox with TEST_TMPDIR ``` $ TEST_TMPDIR=/dev/shm python3 tools/db_crashtest.py whitebox --simple --write_buffer_size=1048576 --target_file_size_base=1048576 --max_bytes_for_level_base=4194304 --max_key=100000 --value_size_mult=33 --compression_type=none --duration=120 --interval=10 --compression_type=none --blob_compression_type=none --random_kill_odd=88887 ``` - db_stress without expected_values_dir ``` $ ./db_stress --write_buffer_size=1048576 --target_file_size_base=1048576 --max_bytes_for_level_base=4194304 --max_key=100000 --value_size_mult=33 --compression_type=none --ops_per_thread=10000 --clear_column_family_one_in=0 --destroy_db_initially=true ``` - db_stress with expected_values_dir and manual corruption ``` $ ./db_stress --write_buffer_size=1048576 --target_file_size_base=1048576 --max_bytes_for_level_base=4194304 --max_key=100000 --value_size_mult=33 --compression_type=none --ops_per_thread=10000 --clear_column_family_one_in=0 --destroy_db_initially=true --expected_values_dir=./ // modify one byte in "./LATEST.state" $ ./db_stress --write_buffer_size=1048576 --target_file_size_base=1048576 --max_bytes_for_level_base=4194304 --max_key=100000 --value_size_mult=33 --compression_type=none --ops_per_thread=10000 --clear_column_family_one_in=0 --destroy_db_initially=false --expected_values_dir=./ ... Verification failed for column family 0 key 0000000000000000 (0): Value not found: NotFound: ... ``` Reviewed By: riversand963 Differential Revision: D30921951 Pulled By: ajkr fbshipit-source-id: babfe218062e55d018c9b046536c0289fb78f41cmain
Andrew Kryczka
3 years ago
committed by
Facebook GitHub Bot
parent
6b34eb0ebc
commit
559943cdc0
@ -0,0 +1,244 @@ |
||||
// Copyright (c) 2021-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
|
||||
#ifdef GFLAGS |
||||
|
||||
#include "db_stress_tool/expected_state.h" |
||||
|
||||
#include "db_stress_tool/db_stress_shared_state.h" |
||||
|
||||
namespace ROCKSDB_NAMESPACE { |
||||
|
||||
ExpectedState::ExpectedState(size_t max_key, size_t num_column_families) |
||||
: max_key_(max_key), |
||||
num_column_families_(num_column_families), |
||||
values_(nullptr) {} |
||||
|
||||
void ExpectedState::ClearColumnFamily(int cf) { |
||||
std::fill(&Value(cf, 0 /* key */), &Value(cf + 1, 0 /* key */), |
||||
SharedState::DELETION_SENTINEL); |
||||
} |
||||
|
||||
void ExpectedState::Put(int cf, int64_t key, uint32_t value_base, |
||||
bool pending) { |
||||
if (!pending) { |
||||
// prevent expected-value update from reordering before Write
|
||||
std::atomic_thread_fence(std::memory_order_release); |
||||
} |
||||
Value(cf, key).store(pending ? SharedState::UNKNOWN_SENTINEL : value_base, |
||||
std::memory_order_relaxed); |
||||
if (pending) { |
||||
// prevent Write from reordering before expected-value update
|
||||
std::atomic_thread_fence(std::memory_order_release); |
||||
} |
||||
} |
||||
|
||||
uint32_t ExpectedState::Get(int cf, int64_t key) const { |
||||
return Value(cf, key); |
||||
} |
||||
|
||||
bool ExpectedState::Delete(int cf, int64_t key, bool pending) { |
||||
if (Value(cf, key) == SharedState::DELETION_SENTINEL) { |
||||
return false; |
||||
} |
||||
Put(cf, key, SharedState::DELETION_SENTINEL, pending); |
||||
return true; |
||||
} |
||||
|
||||
bool ExpectedState::SingleDelete(int cf, int64_t key, bool pending) { |
||||
return Delete(cf, key, pending); |
||||
} |
||||
|
||||
int ExpectedState::DeleteRange(int cf, int64_t begin_key, int64_t end_key, |
||||
bool pending) { |
||||
int covered = 0; |
||||
for (int64_t key = begin_key; key < end_key; ++key) { |
||||
if (Delete(cf, key, pending)) { |
||||
++covered; |
||||
} |
||||
} |
||||
return covered; |
||||
} |
||||
|
||||
bool ExpectedState::Exists(int cf, int64_t key) { |
||||
// UNKNOWN_SENTINEL counts as exists. That assures a key for which overwrite
|
||||
// is disallowed can't be accidentally added a second time, in which case
|
||||
// SingleDelete wouldn't be able to properly delete the key. It does allow
|
||||
// the case where a SingleDelete might be added which covers nothing, but
|
||||
// that's not a correctness issue.
|
||||
uint32_t expected_value = Value(cf, key).load(); |
||||
return expected_value != SharedState::DELETION_SENTINEL; |
||||
} |
||||
|
||||
void ExpectedState::Reset() { |
||||
for (size_t i = 0; i < num_column_families_; ++i) { |
||||
for (size_t j = 0; j < max_key_; ++j) { |
||||
Delete(static_cast<int>(i), j, false /* pending */); |
||||
} |
||||
} |
||||
} |
||||
|
||||
FileExpectedState::FileExpectedState(std::string expected_state_file_path, |
||||
size_t max_key, size_t num_column_families) |
||||
: ExpectedState(max_key, num_column_families), |
||||
expected_state_file_path_(expected_state_file_path) {} |
||||
|
||||
Status FileExpectedState::Open(bool create) { |
||||
size_t expected_values_size = GetValuesLen(); |
||||
|
||||
Env* default_env = Env::Default(); |
||||
|
||||
Status status; |
||||
if (create) { |
||||
std::unique_ptr<WritableFile> wfile; |
||||
const EnvOptions soptions; |
||||
status = default_env->NewWritableFile(expected_state_file_path_, &wfile, |
||||
soptions); |
||||
if (status.ok()) { |
||||
std::string buf(expected_values_size, '\0'); |
||||
status = wfile->Append(buf); |
||||
} |
||||
} |
||||
if (status.ok()) { |
||||
status = default_env->NewMemoryMappedFileBuffer( |
||||
expected_state_file_path_, &expected_state_mmap_buffer_); |
||||
} |
||||
if (status.ok()) { |
||||
assert(expected_state_mmap_buffer_->GetLen() == expected_values_size); |
||||
values_ = static_cast<std::atomic<uint32_t>*>( |
||||
expected_state_mmap_buffer_->GetBase()); |
||||
assert(values_ != nullptr); |
||||
if (create) { |
||||
Reset(); |
||||
} |
||||
} else { |
||||
assert(values_ == nullptr); |
||||
} |
||||
return status; |
||||
} |
||||
|
||||
AnonExpectedState::AnonExpectedState(size_t max_key, size_t num_column_families) |
||||
: ExpectedState(max_key, num_column_families) {} |
||||
|
||||
#ifndef NDEBUG |
||||
Status AnonExpectedState::Open(bool create) { |
||||
#else |
||||
Status AnonExpectedState::Open(bool /* create */) { |
||||
#endif |
||||
// AnonExpectedState only supports being freshly created.
|
||||
assert(create); |
||||
values_allocation_.reset( |
||||
new std::atomic<uint32_t>[GetValuesLen() / |
||||
sizeof(std::atomic<uint32_t>)]); |
||||
values_ = &values_allocation_[0]; |
||||
Reset(); |
||||
return Status::OK(); |
||||
} |
||||
|
||||
ExpectedStateManager::ExpectedStateManager(size_t max_key, |
||||
size_t num_column_families) |
||||
: max_key_(max_key), |
||||
num_column_families_(num_column_families), |
||||
latest_(nullptr) {} |
||||
|
||||
ExpectedStateManager::~ExpectedStateManager() {} |
||||
|
||||
const std::string FileExpectedStateManager::kLatestFilename = "LATEST.state"; |
||||
|
||||
FileExpectedStateManager::FileExpectedStateManager( |
||||
size_t max_key, size_t num_column_families, |
||||
std::string expected_state_dir_path) |
||||
: ExpectedStateManager(max_key, num_column_families), |
||||
expected_state_dir_path_(std::move(expected_state_dir_path)) { |
||||
assert(!expected_state_dir_path_.empty()); |
||||
} |
||||
|
||||
Status FileExpectedStateManager::Open() { |
||||
Status s = Clean(); |
||||
|
||||
std::string expected_state_file_path = GetPathForFilename(kLatestFilename); |
||||
bool found = false; |
||||
if (s.ok()) { |
||||
Status exists_status = Env::Default()->FileExists(expected_state_file_path); |
||||
if (exists_status.ok()) { |
||||
found = true; |
||||
} else if (exists_status.IsNotFound()) { |
||||
found = false; |
||||
} else { |
||||
s = exists_status; |
||||
} |
||||
} |
||||
|
||||
if (!found) { |
||||
// Initialize the file in a temp path and then rename it. That way, in case
|
||||
// this process is killed during setup, `Clean()` will take care of removing
|
||||
// the incomplete expected values file.
|
||||
std::string temp_expected_state_file_path = |
||||
GetTempPathForFilename(kLatestFilename); |
||||
FileExpectedState temp_expected_state(temp_expected_state_file_path, |
||||
max_key_, num_column_families_); |
||||
if (s.ok()) { |
||||
s = temp_expected_state.Open(true /* create */); |
||||
} |
||||
if (s.ok()) { |
||||
s = Env::Default()->RenameFile(temp_expected_state_file_path, |
||||
expected_state_file_path); |
||||
} |
||||
} |
||||
|
||||
if (s.ok()) { |
||||
latest_.reset(new FileExpectedState(std::move(expected_state_file_path), |
||||
max_key_, num_column_families_)); |
||||
s = latest_->Open(false /* create */); |
||||
} |
||||
return s; |
||||
} |
||||
|
||||
Status FileExpectedStateManager::Clean() { |
||||
// An incomplete `Open()` could have left behind an invalid temporary file.
|
||||
std::string temp_path = GetTempPathForFilename(kLatestFilename); |
||||
Status s = Env::Default()->FileExists(temp_path); |
||||
if (s.ok()) { |
||||
s = Env::Default()->DeleteFile(temp_path); |
||||
} else if (s.IsNotFound()) { |
||||
s = Status::OK(); |
||||
} |
||||
return s; |
||||
} |
||||
|
||||
std::string FileExpectedStateManager::GetTempPathForFilename( |
||||
const std::string& filename) { |
||||
static const std::string kTempFilenamePrefix = "."; |
||||
static const std::string kTempFilenameSuffix = ".tmp"; |
||||
|
||||
assert(!expected_state_dir_path_.empty()); |
||||
std::string expected_state_dir_path_slash = |
||||
expected_state_dir_path_.back() == '/' ? expected_state_dir_path_ |
||||
: expected_state_dir_path_ + "/"; |
||||
return expected_state_dir_path_slash + kTempFilenamePrefix + filename + |
||||
kTempFilenameSuffix; |
||||
} |
||||
|
||||
std::string FileExpectedStateManager::GetPathForFilename( |
||||
const std::string& filename) { |
||||
assert(!expected_state_dir_path_.empty()); |
||||
std::string expected_state_dir_path_slash = |
||||
expected_state_dir_path_.back() == '/' ? expected_state_dir_path_ |
||||
: expected_state_dir_path_ + "/"; |
||||
return expected_state_dir_path_slash + filename; |
||||
} |
||||
|
||||
AnonExpectedStateManager::AnonExpectedStateManager(size_t max_key, |
||||
size_t num_column_families) |
||||
: ExpectedStateManager(max_key, num_column_families) {} |
||||
|
||||
Status AnonExpectedStateManager::Open() { |
||||
latest_.reset(new AnonExpectedState(max_key_, num_column_families_)); |
||||
return latest_->Open(true /* create */); |
||||
} |
||||
|
||||
} // namespace ROCKSDB_NAMESPACE
|
||||
|
||||
#endif // GFLAGS
|
||||
@ -0,0 +1,215 @@ |
||||
// Copyright (c) 2021-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
|
||||
#ifdef GFLAGS |
||||
|
||||
#pragma once |
||||
|
||||
#include <stdint.h> |
||||
|
||||
#include <atomic> |
||||
#include <memory> |
||||
|
||||
#include "rocksdb/env.h" |
||||
#include "rocksdb/rocksdb_namespace.h" |
||||
|
||||
namespace ROCKSDB_NAMESPACE { |
||||
|
||||
// An `ExpectedState` provides read/write access to expected values for every
|
||||
// key.
|
||||
class ExpectedState { |
||||
public: |
||||
explicit ExpectedState(size_t max_key, size_t num_column_families); |
||||
|
||||
virtual ~ExpectedState() {} |
||||
|
||||
// Requires external locking preventing concurrent execution with any other
|
||||
// member function.
|
||||
virtual Status Open(bool create) = 0; |
||||
|
||||
// Requires external locking covering all keys in `cf`.
|
||||
void ClearColumnFamily(int cf); |
||||
|
||||
// @param pending True if the update may have started but is not yet
|
||||
// guaranteed finished. This is useful for crash-recovery testing when the
|
||||
// process may crash before updating the expected values array.
|
||||
//
|
||||
// Requires external locking covering `key` in `cf`.
|
||||
void Put(int cf, int64_t key, uint32_t value_base, bool pending); |
||||
|
||||
// Requires external locking covering `key` in `cf`.
|
||||
uint32_t Get(int cf, int64_t key) const; |
||||
|
||||
// @param pending See comment above Put()
|
||||
// Returns true if the key was not yet deleted.
|
||||
//
|
||||
// Requires external locking covering `key` in `cf`.
|
||||
bool Delete(int cf, int64_t key, bool pending); |
||||
|
||||
// @param pending See comment above Put()
|
||||
// Returns true if the key was not yet deleted.
|
||||
//
|
||||
// Requires external locking covering `key` in `cf`.
|
||||
bool SingleDelete(int cf, int64_t key, bool pending); |
||||
|
||||
// @param pending See comment above Put()
|
||||
// Returns number of keys deleted by the call.
|
||||
//
|
||||
// Requires external locking covering keys in `[begin_key, end_key)` in `cf`.
|
||||
int DeleteRange(int cf, int64_t begin_key, int64_t end_key, bool pending); |
||||
|
||||
// Requires external locking covering `key` in `cf`.
|
||||
bool Exists(int cf, int64_t key); |
||||
|
||||
private: |
||||
// Requires external locking covering `key` in `cf`.
|
||||
std::atomic<uint32_t>& Value(int cf, int64_t key) const { |
||||
return values_[cf * max_key_ + key]; |
||||
} |
||||
|
||||
const size_t max_key_; |
||||
const size_t num_column_families_; |
||||
|
||||
protected: |
||||
size_t GetValuesLen() const { |
||||
return sizeof(std::atomic<uint32_t>) * num_column_families_ * max_key_; |
||||
} |
||||
|
||||
// Requires external locking preventing concurrent execution with any other
|
||||
// member function.
|
||||
void Reset(); |
||||
|
||||
std::atomic<uint32_t>* values_; |
||||
}; |
||||
|
||||
// A `FileExpectedState` implements `ExpectedState` backed by a file.
|
||||
class FileExpectedState : public ExpectedState { |
||||
public: |
||||
explicit FileExpectedState(std::string expected_state_file_path, |
||||
size_t max_key, size_t num_column_families); |
||||
|
||||
// Requires external locking preventing concurrent execution with any other
|
||||
// member function.
|
||||
Status Open(bool create) override; |
||||
|
||||
private: |
||||
const std::string expected_state_file_path_; |
||||
std::unique_ptr<MemoryMappedFileBuffer> expected_state_mmap_buffer_; |
||||
}; |
||||
|
||||
// An `AnonExpectedState` implements `ExpectedState` backed by a memory
|
||||
// allocation.
|
||||
class AnonExpectedState : public ExpectedState { |
||||
public: |
||||
explicit AnonExpectedState(size_t max_key, size_t num_column_families); |
||||
|
||||
// Requires external locking preventing concurrent execution with any other
|
||||
// member function.
|
||||
Status Open(bool create) override; |
||||
|
||||
private: |
||||
std::unique_ptr<std::atomic<uint32_t>[]> values_allocation_; |
||||
}; |
||||
|
||||
// An `ExpectedStateManager` manages data about the expected state of the
|
||||
// database. It exposes operations for reading and modifying the latest
|
||||
// expected state.
|
||||
class ExpectedStateManager { |
||||
public: |
||||
explicit ExpectedStateManager(size_t max_key, size_t num_column_families); |
||||
|
||||
virtual ~ExpectedStateManager(); |
||||
|
||||
// Requires external locking preventing concurrent execution with any other
|
||||
// member function.
|
||||
virtual Status Open() = 0; |
||||
|
||||
// Requires external locking covering all keys in `cf`.
|
||||
void ClearColumnFamily(int cf) { return latest_->ClearColumnFamily(cf); } |
||||
|
||||
// @param pending True if the update may have started but is not yet
|
||||
// guaranteed finished. This is useful for crash-recovery testing when the
|
||||
// process may crash before updating the expected values array.
|
||||
//
|
||||
// Requires external locking covering `key` in `cf`.
|
||||
void Put(int cf, int64_t key, uint32_t value_base, bool pending) { |
||||
return latest_->Put(cf, key, value_base, pending); |
||||
} |
||||
|
||||
// Requires external locking covering `key` in `cf`.
|
||||
uint32_t Get(int cf, int64_t key) const { return latest_->Get(cf, key); } |
||||
|
||||
// @param pending See comment above Put()
|
||||
// Returns true if the key was not yet deleted.
|
||||
//
|
||||
// Requires external locking covering `key` in `cf`.
|
||||
bool Delete(int cf, int64_t key, bool pending) { |
||||
return latest_->Delete(cf, key, pending); |
||||
} |
||||
|
||||
// @param pending See comment above Put()
|
||||
// Returns true if the key was not yet deleted.
|
||||
//
|
||||
// Requires external locking covering `key` in `cf`.
|
||||
bool SingleDelete(int cf, int64_t key, bool pending) { |
||||
return latest_->SingleDelete(cf, key, pending); |
||||
} |
||||
|
||||
// @param pending See comment above Put()
|
||||
// Returns number of keys deleted by the call.
|
||||
//
|
||||
// Requires external locking covering keys in `[begin_key, end_key)` in `cf`.
|
||||
int DeleteRange(int cf, int64_t begin_key, int64_t end_key, bool pending) { |
||||
return latest_->DeleteRange(cf, begin_key, end_key, pending); |
||||
} |
||||
|
||||
// Requires external locking covering `key` in `cf`.
|
||||
bool Exists(int cf, int64_t key) { return latest_->Exists(cf, key); } |
||||
|
||||
protected: |
||||
const size_t max_key_; |
||||
const size_t num_column_families_; |
||||
std::unique_ptr<ExpectedState> latest_; |
||||
}; |
||||
|
||||
// A `FileExpectedStateManager` implements an `ExpectedStateManager` backed by
|
||||
// a directory of files containing data about the expected state of the
|
||||
// database.
|
||||
class FileExpectedStateManager : public ExpectedStateManager { |
||||
public: |
||||
explicit FileExpectedStateManager(size_t max_key, size_t num_column_families, |
||||
std::string expected_state_dir_path); |
||||
|
||||
// Requires external locking preventing concurrent execution with any other
|
||||
// member function.
|
||||
Status Open() override; |
||||
|
||||
private: |
||||
// Requires external locking preventing concurrent execution with any other
|
||||
// member function.
|
||||
Status Clean(); |
||||
|
||||
std::string GetTempPathForFilename(const std::string& filename); |
||||
std::string GetPathForFilename(const std::string& filename); |
||||
|
||||
static const std::string kLatestFilename; |
||||
|
||||
const std::string expected_state_dir_path_; |
||||
}; |
||||
|
||||
// An `AnonExpectedStateManager` implements an `ExpectedStateManager` backed by
|
||||
// a memory allocation containing data about the expected state of the database.
|
||||
class AnonExpectedStateManager : public ExpectedStateManager { |
||||
public: |
||||
explicit AnonExpectedStateManager(size_t max_key, size_t num_column_families); |
||||
|
||||
// Requires external locking preventing concurrent execution with any other
|
||||
// member function.
|
||||
Status Open() override; |
||||
}; |
||||
|
||||
} // namespace ROCKSDB_NAMESPACE
|
||||
|
||||
#endif // GFLAGS
|
||||
Loading…
Reference in new issue