Add PauseBackgroundWork() to db_stress (#6148)

Summary:
Worker thread will occasionally call PauseBackgroundWork(),
briefly sleep (to avoid stalling itself) and then call
ContinueBackgroundWork().
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6148

Test Plan:
some running of 'make blackbox_crash_test' with temporary
printf output to confirm code occasionally reached.

Differential Revision: D18913886

Pulled By: pdillinger

fbshipit-source-id: ae9356a803390929f3165dfb6a00194692ba92be

db_stress_tool/db_stress_common.h

@@ -168,6 +168,7 @@ DECLARE_int32(ingest_external_file_width);
 DECLARE_int32(compact_files_one_in);
 DECLARE_int32(compact_range_one_in);
 DECLARE_int32(flush_one_in);
+DECLARE_int32(pause_background_one_in);
 DECLARE_int32(compact_range_width);
 DECLARE_int32(acquire_snapshot_one_in);
 DECLARE_bool(compare_full_db_state_snapshot);

db_stress_tool/db_stress_gflags.cc

@@ -401,6 +401,10 @@ DEFINE_int32(flush_one_in, 0,
              "If non-zero, then Flush() will be called once for every N ops "
              "on average.  0 indicates calls to Flush() are disabled.");
 
+DEFINE_int32(pause_background_one_in, 0,
+             "If non-zero, then PauseBackgroundWork()+Continue will be called "
+             "once for every N ops on average.  0 disables.");
+
 DEFINE_int32(compact_range_width, 10000,
              "The width of the ranges passed to CompactRange().");
 

db_stress_tool/db_stress_test_base.cc

@@ -600,6 +600,28 @@ void StressTest::OperateDb(ThreadState* thread) {
         }
       }
 
+      if (FLAGS_pause_background_one_in > 0 &&
+          thread->rand.OneIn(FLAGS_pause_background_one_in)) {
+        Status status = db_->PauseBackgroundWork();
+        if (!status.ok()) {
+          VerificationAbort(shared, "PauseBackgroundWork status not OK",
+                            status);
+        }
+        // To avoid stalling/deadlocking ourself in this thread, just
+        // sleep here during pause and let other threads do db operations.
+        // Sleep up to ~16 seconds (2**24 microseconds), but very skewed
+        // toward short pause. (1 chance in 25 of pausing >= 1s;
+        // 1 chance in 625 of pausing full 16s.)
+        int pwr2_micros =
+            std::min(thread->rand.Uniform(25), thread->rand.Uniform(25));
+        FLAGS_env->SleepForMicroseconds(1 << pwr2_micros);
+        status = db_->ContinueBackgroundWork();
+        if (!status.ok()) {
+          VerificationAbort(shared, "ContinueBackgroundWork status not OK",
+                            status);
+        }
+      }
+
       std::vector<int64_t> rand_keys = GenerateKeys(rand_key);
 
       if (FLAGS_ingest_external_file_one_in > 0 &&

tools/db_crashtest.py

@@ -52,6 +52,7 @@ default_params = {
     "nooverwritepercent": 1,
     "open_files": lambda : random.choice([-1, 500000]),
     "partition_filters": lambda: random.randint(0, 1),
+    "pause_background_one_in": 1000000,
     "prefixpercent": 5,
     "progress_reports": 0,
     "readpercent": 45,