enhance dbstress to simulate hard crash

Summary:
dbstress has an option to reopen the database. Make it such that the
previous handle is not closed before we reopen, this simulates a
situation similar to a process crash.

Added new api to DMImpl to remove the lock file.

Test Plan: run db_stress

Reviewers: emayanke

Reviewed By: emayanke

CC: leveldb

Differential Revision: https://reviews.facebook.net/D6777
main
Dhruba Borthakur 12 years ago
parent c3392c9fe0
commit 62e7583f94
  1. 23
      db/db_impl.cc
  2. 3
      db/db_impl.h
  3. 33
      tools/db_stress.cc

@ -265,6 +265,29 @@ DBImpl::~DBImpl() {
delete logger_; delete logger_;
} }
// Do not flush and close database elegantly. Simulate a crash.
void DBImpl::TEST_Destroy_DBImpl() {
// ensure that no new memtable flushes can occur
flush_on_destroy_ = false;
// wait till all background compactions are done.
mutex_.Lock();
while (bg_compaction_scheduled_ || bg_logstats_scheduled_) {
bg_cv_.Wait();
}
// Prevent new compactions from occuring.
const int LargeNumber = 10000000;
bg_compaction_scheduled_ += LargeNumber;
mutex_.Unlock();
// force release the lock file.
if (db_lock_ != NULL) {
env_->UnlockFile(db_lock_);
}
}
Status DBImpl::NewDB() { Status DBImpl::NewDB() {
VersionEdit new_db(NumberLevels()); VersionEdit new_db(NumberLevels());
new_db.SetComparatorName(user_comparator()->Name()); new_db.SetComparatorName(user_comparator()->Name());

@ -78,6 +78,9 @@ class DBImpl : public DB {
// file at a level >= 1. // file at a level >= 1.
int64_t TEST_MaxNextLevelOverlappingBytes(); int64_t TEST_MaxNextLevelOverlappingBytes();
// Simulate a db crash, no elegant closing of database.
void TEST_Destroy_DBImpl();
protected: protected:
Env* const env_; Env* const env_;

@ -419,7 +419,8 @@ class StressTest {
filter_policy_(FLAGS_bloom_bits >= 0 filter_policy_(FLAGS_bloom_bits >= 0
? NewBloomFilterPolicy(FLAGS_bloom_bits) ? NewBloomFilterPolicy(FLAGS_bloom_bits)
: NULL), : NULL),
db_(NULL) { db_(NULL),
num_times_reopened_(0) {
std::vector<std::string> files; std::vector<std::string> files;
FLAGS_env->GetChildren(FLAGS_db, &files); FLAGS_env->GetChildren(FLAGS_db, &files);
for (unsigned int i = 0; i < files.size(); i++) { for (unsigned int i = 0; i < files.size(); i++) {
@ -457,7 +458,9 @@ class StressTest {
shared.GetCondVar()->Wait(); shared.GetCondVar()->Wait();
} }
fprintf(stdout, "Starting database operations\n"); double now = FLAGS_env->NowMicros();
fprintf(stdout, "%s Starting database operations\n",
FLAGS_env->TimeToString((uint64_t) now/1000000).c_str());
shared.SetStart(); shared.SetStart();
shared.GetCondVar()->SignalAll(); shared.GetCondVar()->SignalAll();
@ -465,7 +468,10 @@ class StressTest {
shared.GetCondVar()->Wait(); shared.GetCondVar()->Wait();
} }
fprintf(stdout, "Starting verification\n"); now = FLAGS_env->NowMicros();
fprintf(stdout, "%s Starting verification\n",
FLAGS_env->TimeToString((uint64_t) now/1000000).c_str());
shared.SetStartVerify(); shared.SetStartVerify();
shared.GetCondVar()->SignalAll(); shared.GetCondVar()->SignalAll();
while (!shared.AllDone()) { while (!shared.AllDone()) {
@ -482,7 +488,10 @@ class StressTest {
delete threads[i]; delete threads[i];
threads[i] = NULL; threads[i] = NULL;
} }
fprintf(stdout, "Verification successful\n"); double now = FLAGS_env->NowMicros();
fprintf(stdout, "%s Verification successful\n",
FLAGS_env->TimeToString((uint64_t) now/1000000).c_str());
PrintStatistics(); PrintStatistics();
} }
@ -542,6 +551,7 @@ class StressTest {
for (long i = 0; i < FLAGS_ops_per_thread; i++) { for (long i = 0; i < FLAGS_ops_per_thread; i++) {
if(i != 0 && (i % (FLAGS_ops_per_thread / (FLAGS_reopen + 1))) == 0) { if(i != 0 && (i % (FLAGS_ops_per_thread / (FLAGS_reopen + 1))) == 0) {
{ {
thread->stats.FinishedSingleOp();
MutexLock l(thread->shared->GetMutex()); MutexLock l(thread->shared->GetMutex());
thread->shared->IncVotedReopen(); thread->shared->IncVotedReopen();
if (thread->shared->AllVotedReopen()) { if (thread->shared->AllVotedReopen()) {
@ -551,6 +561,7 @@ class StressTest {
else { else {
thread->shared->GetCondVar()->Wait(); thread->shared->GetCondVar()->Wait();
} }
thread->stats.Start();
} }
} }
long rand_key = thread->rand.Next() % max_key; long rand_key = thread->rand.Next() % max_key;
@ -667,7 +678,7 @@ class StressTest {
fprintf(stdout, "Num keys per lock : %d\n", fprintf(stdout, "Num keys per lock : %d\n",
1 << FLAGS_log2_keys_per_lock); 1 << FLAGS_log2_keys_per_lock);
char* compression = ""; char* compression = (char *)std::string("").c_str();
switch (FLAGS_compression_type) { switch (FLAGS_compression_type) {
case leveldb::kNoCompression: case leveldb::kNoCompression:
compression = (char *)std::string("none").c_str(); compression = (char *)std::string("none").c_str();
@ -722,7 +733,16 @@ class StressTest {
} }
void Reopen() { void Reopen() {
delete db_; // do not close the db. Just delete the lock file. This
// simulates a crash-recovery kind of situation.
((DBImpl*) db_)->TEST_Destroy_DBImpl();
db_ = NULL;
num_times_reopened_++;
double now = FLAGS_env->NowMicros();
fprintf(stdout, "%s Reopening database for the %dth time\n",
FLAGS_env->TimeToString((uint64_t) now/1000000).c_str(),
num_times_reopened_);
Open(); Open();
} }
@ -739,6 +759,7 @@ class StressTest {
Cache* cache_; Cache* cache_;
const FilterPolicy* filter_policy_; const FilterPolicy* filter_policy_;
DB* db_; DB* db_;
int num_times_reopened_;
}; };
} // namespace leveldb } // namespace leveldb

Loading…
Cancel
Save