Summary: Refactored db_bench transaction stress tests so that they can be called from unit tests as well. Test Plan: run new unit test as well as db_bench Reviewers: yhchiang, IslamAbdelRahman, sdong Reviewed By: IslamAbdelRahman Subscribers: andrewkr, dhruba, leveldb Differential Revision: https://reviews.facebook.net/D55203main
agiardullo
9 years ago
parent
e8e6cf0173
commit
790252805d
@ -0,0 +1,237 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
#ifndef ROCKSDB_LITE |
||||||
|
|
||||||
|
#ifndef __STDC_FORMAT_MACROS |
||||||
|
#define __STDC_FORMAT_MACROS |
||||||
|
#endif |
||||||
|
|
||||||
|
#include "util/transaction_test_util.h" |
||||||
|
|
||||||
|
#include <inttypes.h> |
||||||
|
#include <string> |
||||||
|
|
||||||
|
#include "rocksdb/db.h" |
||||||
|
#include "rocksdb/utilities/optimistic_transaction_db.h" |
||||||
|
#include "rocksdb/utilities/transaction.h" |
||||||
|
#include "rocksdb/utilities/transaction_db.h" |
||||||
|
#include "util/random.h" |
||||||
|
#include "util/string_util.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
RandomTransactionInserter::RandomTransactionInserter( |
||||||
|
Random64* rand, const WriteOptions& write_options, |
||||||
|
const ReadOptions& read_options, uint64_t num_keys, uint16_t num_sets) |
||||||
|
: rand_(rand), |
||||||
|
write_options_(write_options), |
||||||
|
read_options_(read_options), |
||||||
|
num_keys_(num_keys), |
||||||
|
num_sets_(num_sets) {} |
||||||
|
|
||||||
|
RandomTransactionInserter::~RandomTransactionInserter() { |
||||||
|
if (txn_ != nullptr) { |
||||||
|
delete txn_; |
||||||
|
} |
||||||
|
if (optimistic_txn_ != nullptr) { |
||||||
|
delete optimistic_txn_; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
bool RandomTransactionInserter::TransactionDBInsert( |
||||||
|
TransactionDB* db, const TransactionOptions& txn_options) { |
||||||
|
txn_ = db->BeginTransaction(write_options_, txn_options, txn_); |
||||||
|
|
||||||
|
return DoInsert(nullptr, txn_, false); |
||||||
|
} |
||||||
|
|
||||||
|
bool RandomTransactionInserter::OptimisticTransactionDBInsert( |
||||||
|
OptimisticTransactionDB* db, |
||||||
|
const OptimisticTransactionOptions& txn_options) { |
||||||
|
optimistic_txn_ = |
||||||
|
db->BeginTransaction(write_options_, txn_options, optimistic_txn_); |
||||||
|
|
||||||
|
return DoInsert(nullptr, optimistic_txn_, true); |
||||||
|
} |
||||||
|
|
||||||
|
bool RandomTransactionInserter::DBInsert(DB* db) { |
||||||
|
return DoInsert(db, nullptr, false); |
||||||
|
} |
||||||
|
|
||||||
|
bool RandomTransactionInserter::DoInsert(DB* db, Transaction* txn, |
||||||
|
bool is_optimistic) { |
||||||
|
Status s; |
||||||
|
WriteBatch batch; |
||||||
|
std::string value; |
||||||
|
|
||||||
|
// pick a random number to use to increment a key in each set
|
||||||
|
uint64_t incr = (rand_->Next() % 100) + 1; |
||||||
|
|
||||||
|
bool unexpected_error = false; |
||||||
|
|
||||||
|
// For each set, pick a key at random and increment it
|
||||||
|
for (uint8_t i = 0; i < num_sets_; i++) { |
||||||
|
uint64_t int_value = 0; |
||||||
|
char prefix_buf[5]; |
||||||
|
// prefix_buf needs to be large enough to hold a uint16 in string form
|
||||||
|
|
||||||
|
// key format: [SET#][random#]
|
||||||
|
std::string rand_key = ToString(rand_->Next() % num_keys_); |
||||||
|
Slice base_key(rand_key); |
||||||
|
|
||||||
|
// Pad prefix appropriately so we can iterate over each set
|
||||||
|
snprintf(prefix_buf, sizeof(prefix_buf), "%.4u", i + 1); |
||||||
|
std::string full_key = std::string(prefix_buf) + base_key.ToString(); |
||||||
|
Slice key(full_key); |
||||||
|
|
||||||
|
if (txn != nullptr) { |
||||||
|
s = txn->GetForUpdate(read_options_, key, &value); |
||||||
|
} else { |
||||||
|
s = db->Get(read_options_, key, &value); |
||||||
|
} |
||||||
|
|
||||||
|
if (s.ok()) { |
||||||
|
// Found key, parse its value
|
||||||
|
int_value = std::stoull(value); |
||||||
|
|
||||||
|
if (int_value == 0 || int_value == ULONG_MAX) { |
||||||
|
unexpected_error = true; |
||||||
|
fprintf(stderr, "Get returned unexpected value: %s\n", value.c_str()); |
||||||
|
s = Status::Corruption(); |
||||||
|
} |
||||||
|
} else if (s.IsNotFound()) { |
||||||
|
// Have not yet written to this key, so assume its value is 0
|
||||||
|
int_value = 0; |
||||||
|
s = Status::OK(); |
||||||
|
} else { |
||||||
|
// Optimistic transactions should never return non-ok status here.
|
||||||
|
// Non-optimistic transactions may return write-coflict/timeout errors.
|
||||||
|
if (is_optimistic || !(s.IsBusy() || s.IsTimedOut() || s.IsTryAgain())) { |
||||||
|
fprintf(stderr, "Get returned an unexpected error: %s\n", |
||||||
|
s.ToString().c_str()); |
||||||
|
unexpected_error = true; |
||||||
|
} |
||||||
|
break; |
||||||
|
} |
||||||
|
|
||||||
|
if (s.ok()) { |
||||||
|
// Increment key
|
||||||
|
std::string sum = ToString(int_value + incr); |
||||||
|
if (txn != nullptr) { |
||||||
|
s = txn->Put(key, sum); |
||||||
|
if (!s.ok()) { |
||||||
|
// Since we did a GetForUpdate, Put should not fail.
|
||||||
|
fprintf(stderr, "Put returned an unexpected error: %s\n", |
||||||
|
s.ToString().c_str()); |
||||||
|
unexpected_error = true; |
||||||
|
} |
||||||
|
} else { |
||||||
|
batch.Put(key, sum); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
if (s.ok()) { |
||||||
|
if (txn != nullptr) { |
||||||
|
s = txn->Commit(); |
||||||
|
|
||||||
|
if (!s.ok()) { |
||||||
|
if (is_optimistic) { |
||||||
|
// Optimistic transactions can have write-conflict errors on commit.
|
||||||
|
// Any other error is unexpected.
|
||||||
|
if (!(s.IsBusy() || s.IsTimedOut() || s.IsTryAgain())) { |
||||||
|
unexpected_error = true; |
||||||
|
} |
||||||
|
} else { |
||||||
|
// Non-optimistic transactions should only fail due to expiration
|
||||||
|
// or write failures. For testing purproses, we do not expect any
|
||||||
|
// write failures.
|
||||||
|
if (!s.IsExpired()) { |
||||||
|
unexpected_error = true; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
if (unexpected_error) { |
||||||
|
fprintf(stderr, "Commit returned an unexpected error: %s\n", |
||||||
|
s.ToString().c_str()); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
} else { |
||||||
|
s = db->Write(write_options_, &batch); |
||||||
|
if (!s.ok()) { |
||||||
|
unexpected_error = true; |
||||||
|
fprintf(stderr, "Write returned an unexpected error: %s\n", |
||||||
|
s.ToString().c_str()); |
||||||
|
} |
||||||
|
} |
||||||
|
} else { |
||||||
|
if (txn != nullptr) { |
||||||
|
txn->Rollback(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
if (s.ok()) { |
||||||
|
success_count_++; |
||||||
|
} else { |
||||||
|
failure_count_++; |
||||||
|
} |
||||||
|
|
||||||
|
last_status_ = s; |
||||||
|
|
||||||
|
// return success if we didn't get any unexpected errors
|
||||||
|
return !unexpected_error; |
||||||
|
} |
||||||
|
|
||||||
|
Status RandomTransactionInserter::Verify(DB* db, uint16_t num_sets) { |
||||||
|
uint64_t prev_total = 0; |
||||||
|
|
||||||
|
// For each set of keys with the same prefix, sum all the values
|
||||||
|
for (uint32_t i = 0; i < num_sets; i++) { |
||||||
|
char prefix_buf[5]; |
||||||
|
snprintf(prefix_buf, sizeof(prefix_buf), "%.4u", i + 1); |
||||||
|
uint64_t total = 0; |
||||||
|
|
||||||
|
Iterator* iter = db->NewIterator(ReadOptions()); |
||||||
|
|
||||||
|
for (iter->Seek(Slice(prefix_buf, 4)); iter->Valid(); iter->Next()) { |
||||||
|
Slice key = iter->key(); |
||||||
|
|
||||||
|
// stop when we reach a different prefix
|
||||||
|
if (key.ToString().compare(0, 4, prefix_buf) != 0) { |
||||||
|
break; |
||||||
|
} |
||||||
|
|
||||||
|
Slice value = iter->value(); |
||||||
|
uint64_t int_value = std::stoull(value.ToString()); |
||||||
|
if (int_value == 0 || int_value == ULONG_MAX) { |
||||||
|
fprintf(stderr, "Iter returned unexpected value: %s\n", |
||||||
|
value.ToString().c_str()); |
||||||
|
return Status::Corruption(); |
||||||
|
} |
||||||
|
|
||||||
|
total += int_value; |
||||||
|
} |
||||||
|
delete iter; |
||||||
|
|
||||||
|
if (i > 0) { |
||||||
|
if (total != prev_total) { |
||||||
|
fprintf(stderr, |
||||||
|
"RandomTransactionVerify found inconsistent totals. " |
||||||
|
"Set[%" PRIu32 "]: %" PRIu64 ", Set[%" PRIu32 "]: %" PRIu64 |
||||||
|
" \n", |
||||||
|
i - 1, prev_total, i, total); |
||||||
|
return Status::Corruption(); |
||||||
|
} |
||||||
|
} |
||||||
|
prev_total = total; |
||||||
|
} |
||||||
|
|
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
|
|
||||||
|
#endif // ROCKSDB_LITE
|
||||||
@ -0,0 +1,111 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
#ifndef ROCKSDB_LITE |
||||||
|
|
||||||
|
#include "rocksdb/options.h" |
||||||
|
#include "rocksdb/utilities/optimistic_transaction_db.h" |
||||||
|
#include "rocksdb/utilities/transaction_db.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class DB; |
||||||
|
class Random64; |
||||||
|
|
||||||
|
// Utility class for stress testing transactions. Can be used to write many
|
||||||
|
// transactions in parallel and then validate that the data written is logically
|
||||||
|
// consistent. This class assumes the input DB is initially empty.
|
||||||
|
//
|
||||||
|
// Each call to TransactionDBInsert()/OptimisticTransactionDBInsert() will
|
||||||
|
// increment the value of a key in #num_sets sets of keys. Regardless of
|
||||||
|
// whether the transaction succeeds, the total sum of values of keys in each
|
||||||
|
// set is an invariant that should remain equal.
|
||||||
|
//
|
||||||
|
// After calling TransactionDBInsert()/OptimisticTransactionDBInsert() many
|
||||||
|
// times, Verify() can be called to validate that the invariant holds.
|
||||||
|
//
|
||||||
|
// To test writing Transaction in parallel, multiple threads can create a
|
||||||
|
// RandomTransactionInserter with similar arguments using the same DB.
|
||||||
|
class RandomTransactionInserter { |
||||||
|
public: |
||||||
|
// num_keys is the number of keys in each set.
|
||||||
|
// num_sets is the number of sets of keys.
|
||||||
|
explicit RandomTransactionInserter( |
||||||
|
Random64* rand, const WriteOptions& write_options = WriteOptions(), |
||||||
|
const ReadOptions& read_options = ReadOptions(), uint64_t num_keys = 1000, |
||||||
|
uint16_t num_sets = 3); |
||||||
|
|
||||||
|
~RandomTransactionInserter(); |
||||||
|
|
||||||
|
// Increment a key in each set using a Transaction on a TransactionDB.
|
||||||
|
//
|
||||||
|
// Returns true if the transaction succeeded OR if any error encountered was
|
||||||
|
// expected (eg a write-conflict). Error status may be obtained by calling
|
||||||
|
// GetLastStatus();
|
||||||
|
bool TransactionDBInsert( |
||||||
|
TransactionDB* db, |
||||||
|
const TransactionOptions& txn_options = TransactionOptions()); |
||||||
|
|
||||||
|
// Increment a key in each set using a Transaction on an
|
||||||
|
// OptimisticTransactionDB
|
||||||
|
//
|
||||||
|
// Returns true if the transaction succeeded OR if any error encountered was
|
||||||
|
// expected (eg a write-conflict). Error status may be obtained by calling
|
||||||
|
// GetLastStatus();
|
||||||
|
bool OptimisticTransactionDBInsert( |
||||||
|
OptimisticTransactionDB* db, |
||||||
|
const OptimisticTransactionOptions& txn_options = |
||||||
|
OptimisticTransactionOptions()); |
||||||
|
// Increment a key in each set without using a transaction. If this function
|
||||||
|
// is called in parallel, then Verify() may fail.
|
||||||
|
//
|
||||||
|
// Returns true if the write succeeds.
|
||||||
|
// Error status may be obtained by calling GetLastStatus().
|
||||||
|
bool DBInsert(DB* db); |
||||||
|
|
||||||
|
// Returns OK if Invariant is true.
|
||||||
|
static Status Verify(DB* db, uint16_t num_sets); |
||||||
|
|
||||||
|
// Returns the status of the previous Insert operation
|
||||||
|
Status GetLastStatus() { return last_status_; } |
||||||
|
|
||||||
|
// Returns the number of successfully written calls to
|
||||||
|
// TransactionDBInsert/OptimisticTransactionDBInsert/DBInsert
|
||||||
|
uint64_t GetSuccessCount() { return success_count_; } |
||||||
|
|
||||||
|
// Returns the number of calls to
|
||||||
|
// TransactionDBInsert/OptimisticTransactionDBInsert/DBInsert that did not
|
||||||
|
// write any data.
|
||||||
|
uint64_t GetFailureCount() { return failure_count_; } |
||||||
|
|
||||||
|
private: |
||||||
|
// Input options
|
||||||
|
Random64* rand_; |
||||||
|
const WriteOptions write_options_; |
||||||
|
const ReadOptions read_options_; |
||||||
|
const uint64_t num_keys_; |
||||||
|
const uint16_t num_sets_; |
||||||
|
|
||||||
|
// Number of successful insert batches performed
|
||||||
|
uint64_t success_count_ = 0; |
||||||
|
|
||||||
|
// Number of failed insert batches attempted
|
||||||
|
uint64_t failure_count_ = 0; |
||||||
|
|
||||||
|
// Status returned by most recent insert operation
|
||||||
|
Status last_status_; |
||||||
|
|
||||||
|
// optimization: re-use allocated transaction objects.
|
||||||
|
Transaction* txn_ = nullptr; |
||||||
|
Transaction* optimistic_txn_ = nullptr; |
||||||
|
|
||||||
|
bool DoInsert(DB* db, Transaction* txn, bool is_optimistic); |
||||||
|
}; |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
|
|
||||||
|
#endif // ROCKSDB_LITE
|
||||||
Loading…
Reference in new issue