Summary: Refactored db_bench transaction stress tests so that they can be called from unit tests as well. Test Plan: run new unit test as well as db_bench Reviewers: yhchiang, IslamAbdelRahman, sdong Reviewed By: IslamAbdelRahman Subscribers: andrewkr, dhruba, leveldb Differential Revision: https://reviews.facebook.net/D55203main
agiardullo
9 years ago
parent
e8e6cf0173
commit
790252805d
@ -0,0 +1,237 @@ |
||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
#ifndef ROCKSDB_LITE |
||||
|
||||
#ifndef __STDC_FORMAT_MACROS |
||||
#define __STDC_FORMAT_MACROS |
||||
#endif |
||||
|
||||
#include "util/transaction_test_util.h" |
||||
|
||||
#include <inttypes.h> |
||||
#include <string> |
||||
|
||||
#include "rocksdb/db.h" |
||||
#include "rocksdb/utilities/optimistic_transaction_db.h" |
||||
#include "rocksdb/utilities/transaction.h" |
||||
#include "rocksdb/utilities/transaction_db.h" |
||||
#include "util/random.h" |
||||
#include "util/string_util.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
RandomTransactionInserter::RandomTransactionInserter( |
||||
Random64* rand, const WriteOptions& write_options, |
||||
const ReadOptions& read_options, uint64_t num_keys, uint16_t num_sets) |
||||
: rand_(rand), |
||||
write_options_(write_options), |
||||
read_options_(read_options), |
||||
num_keys_(num_keys), |
||||
num_sets_(num_sets) {} |
||||
|
||||
RandomTransactionInserter::~RandomTransactionInserter() { |
||||
if (txn_ != nullptr) { |
||||
delete txn_; |
||||
} |
||||
if (optimistic_txn_ != nullptr) { |
||||
delete optimistic_txn_; |
||||
} |
||||
} |
||||
|
||||
bool RandomTransactionInserter::TransactionDBInsert( |
||||
TransactionDB* db, const TransactionOptions& txn_options) { |
||||
txn_ = db->BeginTransaction(write_options_, txn_options, txn_); |
||||
|
||||
return DoInsert(nullptr, txn_, false); |
||||
} |
||||
|
||||
bool RandomTransactionInserter::OptimisticTransactionDBInsert( |
||||
OptimisticTransactionDB* db, |
||||
const OptimisticTransactionOptions& txn_options) { |
||||
optimistic_txn_ = |
||||
db->BeginTransaction(write_options_, txn_options, optimistic_txn_); |
||||
|
||||
return DoInsert(nullptr, optimistic_txn_, true); |
||||
} |
||||
|
||||
bool RandomTransactionInserter::DBInsert(DB* db) { |
||||
return DoInsert(db, nullptr, false); |
||||
} |
||||
|
||||
bool RandomTransactionInserter::DoInsert(DB* db, Transaction* txn, |
||||
bool is_optimistic) { |
||||
Status s; |
||||
WriteBatch batch; |
||||
std::string value; |
||||
|
||||
// pick a random number to use to increment a key in each set
|
||||
uint64_t incr = (rand_->Next() % 100) + 1; |
||||
|
||||
bool unexpected_error = false; |
||||
|
||||
// For each set, pick a key at random and increment it
|
||||
for (uint8_t i = 0; i < num_sets_; i++) { |
||||
uint64_t int_value = 0; |
||||
char prefix_buf[5]; |
||||
// prefix_buf needs to be large enough to hold a uint16 in string form
|
||||
|
||||
// key format: [SET#][random#]
|
||||
std::string rand_key = ToString(rand_->Next() % num_keys_); |
||||
Slice base_key(rand_key); |
||||
|
||||
// Pad prefix appropriately so we can iterate over each set
|
||||
snprintf(prefix_buf, sizeof(prefix_buf), "%.4u", i + 1); |
||||
std::string full_key = std::string(prefix_buf) + base_key.ToString(); |
||||
Slice key(full_key); |
||||
|
||||
if (txn != nullptr) { |
||||
s = txn->GetForUpdate(read_options_, key, &value); |
||||
} else { |
||||
s = db->Get(read_options_, key, &value); |
||||
} |
||||
|
||||
if (s.ok()) { |
||||
// Found key, parse its value
|
||||
int_value = std::stoull(value); |
||||
|
||||
if (int_value == 0 || int_value == ULONG_MAX) { |
||||
unexpected_error = true; |
||||
fprintf(stderr, "Get returned unexpected value: %s\n", value.c_str()); |
||||
s = Status::Corruption(); |
||||
} |
||||
} else if (s.IsNotFound()) { |
||||
// Have not yet written to this key, so assume its value is 0
|
||||
int_value = 0; |
||||
s = Status::OK(); |
||||
} else { |
||||
// Optimistic transactions should never return non-ok status here.
|
||||
// Non-optimistic transactions may return write-coflict/timeout errors.
|
||||
if (is_optimistic || !(s.IsBusy() || s.IsTimedOut() || s.IsTryAgain())) { |
||||
fprintf(stderr, "Get returned an unexpected error: %s\n", |
||||
s.ToString().c_str()); |
||||
unexpected_error = true; |
||||
} |
||||
break; |
||||
} |
||||
|
||||
if (s.ok()) { |
||||
// Increment key
|
||||
std::string sum = ToString(int_value + incr); |
||||
if (txn != nullptr) { |
||||
s = txn->Put(key, sum); |
||||
if (!s.ok()) { |
||||
// Since we did a GetForUpdate, Put should not fail.
|
||||
fprintf(stderr, "Put returned an unexpected error: %s\n", |
||||
s.ToString().c_str()); |
||||
unexpected_error = true; |
||||
} |
||||
} else { |
||||
batch.Put(key, sum); |
||||
} |
||||
} |
||||
} |
||||
|
||||
if (s.ok()) { |
||||
if (txn != nullptr) { |
||||
s = txn->Commit(); |
||||
|
||||
if (!s.ok()) { |
||||
if (is_optimistic) { |
||||
// Optimistic transactions can have write-conflict errors on commit.
|
||||
// Any other error is unexpected.
|
||||
if (!(s.IsBusy() || s.IsTimedOut() || s.IsTryAgain())) { |
||||
unexpected_error = true; |
||||
} |
||||
} else { |
||||
// Non-optimistic transactions should only fail due to expiration
|
||||
// or write failures. For testing purproses, we do not expect any
|
||||
// write failures.
|
||||
if (!s.IsExpired()) { |
||||
unexpected_error = true; |
||||
} |
||||
} |
||||
|
||||
if (unexpected_error) { |
||||
fprintf(stderr, "Commit returned an unexpected error: %s\n", |
||||
s.ToString().c_str()); |
||||
} |
||||
} |
||||
|
||||
} else { |
||||
s = db->Write(write_options_, &batch); |
||||
if (!s.ok()) { |
||||
unexpected_error = true; |
||||
fprintf(stderr, "Write returned an unexpected error: %s\n", |
||||
s.ToString().c_str()); |
||||
} |
||||
} |
||||
} else { |
||||
if (txn != nullptr) { |
||||
txn->Rollback(); |
||||
} |
||||
} |
||||
|
||||
if (s.ok()) { |
||||
success_count_++; |
||||
} else { |
||||
failure_count_++; |
||||
} |
||||
|
||||
last_status_ = s; |
||||
|
||||
// return success if we didn't get any unexpected errors
|
||||
return !unexpected_error; |
||||
} |
||||
|
||||
Status RandomTransactionInserter::Verify(DB* db, uint16_t num_sets) { |
||||
uint64_t prev_total = 0; |
||||
|
||||
// For each set of keys with the same prefix, sum all the values
|
||||
for (uint32_t i = 0; i < num_sets; i++) { |
||||
char prefix_buf[5]; |
||||
snprintf(prefix_buf, sizeof(prefix_buf), "%.4u", i + 1); |
||||
uint64_t total = 0; |
||||
|
||||
Iterator* iter = db->NewIterator(ReadOptions()); |
||||
|
||||
for (iter->Seek(Slice(prefix_buf, 4)); iter->Valid(); iter->Next()) { |
||||
Slice key = iter->key(); |
||||
|
||||
// stop when we reach a different prefix
|
||||
if (key.ToString().compare(0, 4, prefix_buf) != 0) { |
||||
break; |
||||
} |
||||
|
||||
Slice value = iter->value(); |
||||
uint64_t int_value = std::stoull(value.ToString()); |
||||
if (int_value == 0 || int_value == ULONG_MAX) { |
||||
fprintf(stderr, "Iter returned unexpected value: %s\n", |
||||
value.ToString().c_str()); |
||||
return Status::Corruption(); |
||||
} |
||||
|
||||
total += int_value; |
||||
} |
||||
delete iter; |
||||
|
||||
if (i > 0) { |
||||
if (total != prev_total) { |
||||
fprintf(stderr, |
||||
"RandomTransactionVerify found inconsistent totals. " |
||||
"Set[%" PRIu32 "]: %" PRIu64 ", Set[%" PRIu32 "]: %" PRIu64 |
||||
" \n", |
||||
i - 1, prev_total, i, total); |
||||
return Status::Corruption(); |
||||
} |
||||
} |
||||
prev_total = total; |
||||
} |
||||
|
||||
return Status::OK(); |
||||
} |
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
#endif // ROCKSDB_LITE
|
||||
@ -0,0 +1,111 @@ |
||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#pragma once |
||||
|
||||
#ifndef ROCKSDB_LITE |
||||
|
||||
#include "rocksdb/options.h" |
||||
#include "rocksdb/utilities/optimistic_transaction_db.h" |
||||
#include "rocksdb/utilities/transaction_db.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
class DB; |
||||
class Random64; |
||||
|
||||
// Utility class for stress testing transactions. Can be used to write many
|
||||
// transactions in parallel and then validate that the data written is logically
|
||||
// consistent. This class assumes the input DB is initially empty.
|
||||
//
|
||||
// Each call to TransactionDBInsert()/OptimisticTransactionDBInsert() will
|
||||
// increment the value of a key in #num_sets sets of keys. Regardless of
|
||||
// whether the transaction succeeds, the total sum of values of keys in each
|
||||
// set is an invariant that should remain equal.
|
||||
//
|
||||
// After calling TransactionDBInsert()/OptimisticTransactionDBInsert() many
|
||||
// times, Verify() can be called to validate that the invariant holds.
|
||||
//
|
||||
// To test writing Transaction in parallel, multiple threads can create a
|
||||
// RandomTransactionInserter with similar arguments using the same DB.
|
||||
class RandomTransactionInserter { |
||||
public: |
||||
// num_keys is the number of keys in each set.
|
||||
// num_sets is the number of sets of keys.
|
||||
explicit RandomTransactionInserter( |
||||
Random64* rand, const WriteOptions& write_options = WriteOptions(), |
||||
const ReadOptions& read_options = ReadOptions(), uint64_t num_keys = 1000, |
||||
uint16_t num_sets = 3); |
||||
|
||||
~RandomTransactionInserter(); |
||||
|
||||
// Increment a key in each set using a Transaction on a TransactionDB.
|
||||
//
|
||||
// Returns true if the transaction succeeded OR if any error encountered was
|
||||
// expected (eg a write-conflict). Error status may be obtained by calling
|
||||
// GetLastStatus();
|
||||
bool TransactionDBInsert( |
||||
TransactionDB* db, |
||||
const TransactionOptions& txn_options = TransactionOptions()); |
||||
|
||||
// Increment a key in each set using a Transaction on an
|
||||
// OptimisticTransactionDB
|
||||
//
|
||||
// Returns true if the transaction succeeded OR if any error encountered was
|
||||
// expected (eg a write-conflict). Error status may be obtained by calling
|
||||
// GetLastStatus();
|
||||
bool OptimisticTransactionDBInsert( |
||||
OptimisticTransactionDB* db, |
||||
const OptimisticTransactionOptions& txn_options = |
||||
OptimisticTransactionOptions()); |
||||
// Increment a key in each set without using a transaction. If this function
|
||||
// is called in parallel, then Verify() may fail.
|
||||
//
|
||||
// Returns true if the write succeeds.
|
||||
// Error status may be obtained by calling GetLastStatus().
|
||||
bool DBInsert(DB* db); |
||||
|
||||
// Returns OK if Invariant is true.
|
||||
static Status Verify(DB* db, uint16_t num_sets); |
||||
|
||||
// Returns the status of the previous Insert operation
|
||||
Status GetLastStatus() { return last_status_; } |
||||
|
||||
// Returns the number of successfully written calls to
|
||||
// TransactionDBInsert/OptimisticTransactionDBInsert/DBInsert
|
||||
uint64_t GetSuccessCount() { return success_count_; } |
||||
|
||||
// Returns the number of calls to
|
||||
// TransactionDBInsert/OptimisticTransactionDBInsert/DBInsert that did not
|
||||
// write any data.
|
||||
uint64_t GetFailureCount() { return failure_count_; } |
||||
|
||||
private: |
||||
// Input options
|
||||
Random64* rand_; |
||||
const WriteOptions write_options_; |
||||
const ReadOptions read_options_; |
||||
const uint64_t num_keys_; |
||||
const uint16_t num_sets_; |
||||
|
||||
// Number of successful insert batches performed
|
||||
uint64_t success_count_ = 0; |
||||
|
||||
// Number of failed insert batches attempted
|
||||
uint64_t failure_count_ = 0; |
||||
|
||||
// Status returned by most recent insert operation
|
||||
Status last_status_; |
||||
|
||||
// optimization: re-use allocated transaction objects.
|
||||
Transaction* txn_ = nullptr; |
||||
Transaction* optimistic_txn_ = nullptr; |
||||
|
||||
bool DoInsert(DB* db, Transaction* txn, bool is_optimistic); |
||||
}; |
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
#endif // ROCKSDB_LITE
|
||||
Loading…
Reference in new issue