diff --git a/.circleci/config.yml b/.circleci/config.yml index 46c528acd..5a7748f12 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -458,6 +458,17 @@ jobs: - run: DEBUG_LEVEL=0 make -j32 run_microbench - post-steps + build-linux-mini-crashtest: + machine: + image: ubuntu-2004:202111-02 + resource_class: large + steps: + - pre-steps + - install-gflags + - install-compression-libs + - run: make V=1 -j8 CRASH_TEST_EXT_ARGS=--duration=960 blackbox_crash_test_with_atomic_flush + - post-steps + build-windows: executor: windows-2xlarge parameters: @@ -828,6 +839,9 @@ workflows: build-linux-unity-and-headers: jobs: - build-linux-unity-and-headers + build-linux-mini-crashtest: + jobs: + - build-linux-mini-crashtest build-windows-vs2019: jobs: - build-windows: diff --git a/Makefile b/Makefile index afc141bd2..a7eeb7cb5 100644 --- a/Makefile +++ b/Makefile @@ -8,11 +8,7 @@ BASH_EXISTS := $(shell which bash) SHELL := $(shell which bash) -# Default to python3. Some distros like CentOS 8 do not have `python`. -ifeq ($(origin PYTHON), undefined) - PYTHON := $(shell which python3 || which python || echo python3) -endif -export PYTHON +include python.mk CLEAN_FILES = # deliberately empty, so we can append below. CFLAGS += ${EXTRA_CFLAGS} @@ -1031,65 +1027,7 @@ check_some: $(ROCKSDBTESTS_SUBSET) ldb_tests: ldb $(PYTHON) tools/ldb_test.py -crash_test: -# Do not parallelize - $(MAKE) whitebox_crash_test - $(MAKE) blackbox_crash_test - -crash_test_with_atomic_flush: -# Do not parallelize - $(MAKE) whitebox_crash_test_with_atomic_flush - $(MAKE) blackbox_crash_test_with_atomic_flush - -crash_test_with_txn: -# Do not parallelize - $(MAKE) whitebox_crash_test_with_txn - $(MAKE) blackbox_crash_test_with_txn - -crash_test_with_best_efforts_recovery: blackbox_crash_test_with_best_efforts_recovery - -crash_test_with_ts: -# Do not parallelize - $(MAKE) whitebox_crash_test_with_ts - $(MAKE) blackbox_crash_test_with_ts - -blackbox_crash_test: db_stress - $(PYTHON) -u tools/db_crashtest.py --simple blackbox $(CRASH_TEST_EXT_ARGS) - $(PYTHON) -u tools/db_crashtest.py blackbox $(CRASH_TEST_EXT_ARGS) - -blackbox_crash_test_with_atomic_flush: db_stress - $(PYTHON) -u tools/db_crashtest.py --cf_consistency blackbox $(CRASH_TEST_EXT_ARGS) - -blackbox_crash_test_with_txn: db_stress - $(PYTHON) -u tools/db_crashtest.py --txn blackbox $(CRASH_TEST_EXT_ARGS) - -blackbox_crash_test_with_best_efforts_recovery: db_stress - $(PYTHON) -u tools/db_crashtest.py --test_best_efforts_recovery blackbox $(CRASH_TEST_EXT_ARGS) - -blackbox_crash_test_with_ts: db_stress - $(PYTHON) -u tools/db_crashtest.py --enable_ts blackbox $(CRASH_TEST_EXT_ARGS) - -ifeq ($(CRASH_TEST_KILL_ODD),) - CRASH_TEST_KILL_ODD=888887 -endif - -whitebox_crash_test: db_stress - $(PYTHON) -u tools/db_crashtest.py --simple whitebox --random_kill_odd \ - $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS) - $(PYTHON) -u tools/db_crashtest.py whitebox --random_kill_odd \ - $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS) - -whitebox_crash_test_with_atomic_flush: db_stress - $(PYTHON) -u tools/db_crashtest.py --cf_consistency whitebox --random_kill_odd \ - $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS) - -whitebox_crash_test_with_txn: db_stress - $(PYTHON) -u tools/db_crashtest.py --txn whitebox --random_kill_odd \ - $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS) - -whitebox_crash_test_with_ts: db_stress - $(PYTHON) -u tools/db_crashtest.py --enable_ts whitebox --random_kill_odd \ - $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS) +include crash_test.mk asan_check: clean COMPILE_WITH_ASAN=1 $(MAKE) check -j32 diff --git a/crash_test.mk b/crash_test.mk new file mode 100644 index 000000000..26d260977 --- /dev/null +++ b/crash_test.mk @@ -0,0 +1,79 @@ +# This file is used by Meta-internal infrastructure as well as by Makefile + +# When included from Makefile, there are rules to build DB_STRESS_CMD. When +# used directly with `make -f crashtest.mk ...` there will be no rules to +# build DB_STRESS_CMD so it must exist prior. +DB_STRESS_CMD?=./db_stress + +include python.mk + +CRASHTEST_MAKE=$(MAKE) -f crash_test.mk +CRASHTEST_PY=$(PYTHON) -u tools/db_crashtest.py --stress_cmd=$(DB_STRESS_CMD) + +.PHONY: crash_test crash_test_with_atomic_flush crash_test_with_txn \ + crash_test_with_best_efforts_recovery crash_test_with_ts \ + blackbox_crash_test blackbox_crash_test_with_atomic_flush \ + blackbox_crash_test_with_txn blackbox_crash_test_with_ts \ + blackbox_crash_test_with_best_efforts_recovery \ + whitebox_crash_test whitebox_crash_test_with_atomic_flush \ + whitebox_crash_test_with_txn whitebox_crash_test_with_ts + +crash_test: $(DB_STRESS_CMD) +# Do not parallelize + $(CRASHTEST_MAKE) whitebox_crash_test + $(CRASHTEST_MAKE) blackbox_crash_test + +crash_test_with_atomic_flush: $(DB_STRESS_CMD) +# Do not parallelize + $(CRASHTEST_MAKE) whitebox_crash_test_with_atomic_flush + $(CRASHTEST_MAKE) blackbox_crash_test_with_atomic_flush + +crash_test_with_txn: $(DB_STRESS_CMD) +# Do not parallelize + $(CRASHTEST_MAKE) whitebox_crash_test_with_txn + $(CRASHTEST_MAKE) blackbox_crash_test_with_txn + +crash_test_with_best_efforts_recovery: blackbox_crash_test_with_best_efforts_recovery + +crash_test_with_ts: $(DB_STRESS_CMD) +# Do not parallelize + $(CRASHTEST_MAKE) whitebox_crash_test_with_ts + $(CRASHTEST_MAKE) blackbox_crash_test_with_ts + +blackbox_crash_test: $(DB_STRESS_CMD) + $(CRASHTEST_PY) --simple blackbox $(CRASH_TEST_EXT_ARGS) + $(CRASHTEST_PY) blackbox $(CRASH_TEST_EXT_ARGS) + +blackbox_crash_test_with_atomic_flush: $(DB_STRESS_CMD) + $(CRASHTEST_PY) --cf_consistency blackbox $(CRASH_TEST_EXT_ARGS) + +blackbox_crash_test_with_txn: $(DB_STRESS_CMD) + $(CRASHTEST_PY) --txn blackbox $(CRASH_TEST_EXT_ARGS) + +blackbox_crash_test_with_best_efforts_recovery: $(DB_STRESS_CMD) + $(CRASHTEST_PY) --test_best_efforts_recovery blackbox $(CRASH_TEST_EXT_ARGS) + +blackbox_crash_test_with_ts: $(DB_STRESS_CMD) + $(CRASHTEST_PY) --enable_ts blackbox $(CRASH_TEST_EXT_ARGS) + +ifeq ($(CRASH_TEST_KILL_ODD),) + CRASH_TEST_KILL_ODD=888887 +endif + +whitebox_crash_test: $(DB_STRESS_CMD) + $(CRASHTEST_PY) --simple whitebox --random_kill_odd \ + $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS) + $(CRASHTEST_PY) whitebox --random_kill_odd \ + $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS) + +whitebox_crash_test_with_atomic_flush: $(DB_STRESS_CMD) + $(CRASHTEST_PY) --cf_consistency whitebox --random_kill_odd \ + $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS) + +whitebox_crash_test_with_txn: $(DB_STRESS_CMD) + $(CRASHTEST_PY) --txn whitebox --random_kill_odd \ + $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS) + +whitebox_crash_test_with_ts: $(DB_STRESS_CMD) + $(CRASHTEST_PY) --enable_ts whitebox --random_kill_odd \ + $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS) diff --git a/python.mk b/python.mk new file mode 100644 index 000000000..d92cdec47 --- /dev/null +++ b/python.mk @@ -0,0 +1,9 @@ +ifndef PYTHON + +# Default to python3. Some distros like CentOS 8 do not have `python`. +ifeq ($(origin PYTHON), undefined) + PYTHON := $(shell which python3 || which python || echo python3) +endif +export PYTHON + +endif