Re-enable tsan crash white-box test with reduced killing odds

Summary:
Tsan crash white-box test was disabled because it never ends. Re-enable it with reduced killing odds.
Add a parameter in crash test script to allow we pass it through an environment variable.

Test Plan: Run it manually.

Reviewers: IslamAbdelRahman

Reviewed By: IslamAbdelRahman

Subscribers: leveldb, andrewkr, dhruba

Differential Revision: https://reviews.facebook.net/D61053

Makefile

@@ -683,9 +683,15 @@ blackbox_crash_test: db_stress
 	python -u tools/db_crashtest.py --simple blackbox
 	python -u tools/db_crashtest.py blackbox
 
+ifeq ($(CRASH_TEST_KILL_ODD),)
+  CRASH_TEST_KILL_ODD=888887
+endif
+
 whitebox_crash_test: db_stress
-	python -u tools/db_crashtest.py --simple whitebox
-	python -u tools/db_crashtest.py whitebox
+	python -u tools/db_crashtest.py --simple whitebox --random_kill_odd \
+      $(CRASH_TEST_KILL_ODD)
+	python -u tools/db_crashtest.py whitebox  --random_kill_odd \
+      $(CRASH_TEST_KILL_ODD)
 
 asan_check:
 	$(MAKE) clean

build_tools/rocksdb-lego-determinator

@@ -543,7 +543,6 @@ TSAN_UNIT_TEST_COMMANDS="[
 
 #
 # RocksDB crash test under TSAN
-# TODO: run white box crash test too
 #
 TSAN_CRASH_TEST_COMMANDS="[
     {
@@ -555,7 +554,7 @@ TSAN_CRASH_TEST_COMMANDS="[
             {
                 'name':'Compile and run',
                 'timeout': 86400,
-                'shell':'set -o pipefail && $SHM $DEBUG $TSAN make J=1 blackbox_crash_test',
+                'shell':'set -o pipefail && $SHM $DEBUG $TSAN CRASH_TEST_KILL_ODD=1887 make J=1 crash_test',
                 'user':'root',
                 $PARSER
             },

tools/db_crashtest.py

@@ -78,6 +78,7 @@ whitebox_default_params = {
     "test_batches_snapshots": lambda: random.randint(0, 1),
     "write_buffer_size": 4 * 1024 * 1024,
     "subcompactions": lambda: random.randint(1, 4),
+    "random_kill_odd": 888887,
 }
 
 simple_default_params = {
@@ -167,7 +168,8 @@ def gen_cmd(params):
     cmd = './db_stress ' + ' '.join(
         '--{0}={1}'.format(k, v)
         for k, v in finalize_and_sanitize(params).items()
-        if k not in set(['test_type', 'simple', 'duration', 'interval'])
+        if k not in set(['test_type', 'simple', 'duration', 'interval',
+                         'random_kill_odd'])
         and v is not None)
     return cmd
 
@@ -252,7 +254,7 @@ def whitebox_crash_main(args):
 
     total_check_mode = 4
     check_mode = 0
-    kill_random_test = 888887
+    kill_random_test = cmd_params['random_kill_odd']
     kill_mode = 0
 
     while time.time() < exit_time:
@@ -276,6 +278,8 @@ def whitebox_crash_main(args):
                     + "WritableFileWriter::WriteBuffered",
                 })
             elif kill_mode == 2:
+                # TODO: May need to adjust random odds if kill_random_test
+                # is too small.
                 additional_opts.update({
                     "kill_random_test": (kill_random_test / 5000 + 1),
                     "kill_prefix_blacklist": "WritableFileWriter::Append,"