crash_test Makefile refactoring, add to CircleCI (#9702)

Summary:
some Makefile refactoring to support Meta-internal workflows,
and add a basic crash_test flow to CircleCI

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9702

Test Plan: CI

Reviewed By: riversand963

Differential Revision: D34934315

Pulled By: pdillinger

fbshipit-source-id: 67f17280096d8968d8e44459293f72fb6fe339f3
main
Peter Dillinger 3 years ago committed by Facebook GitHub Bot
parent a88d8795ec
commit fe9a344c55
  1. 14
      .circleci/config.yml
  2. 66
      Makefile
  3. 79
      crash_test.mk
  4. 9
      python.mk

@ -458,6 +458,17 @@ jobs:
- run: DEBUG_LEVEL=0 make -j32 run_microbench - run: DEBUG_LEVEL=0 make -j32 run_microbench
- post-steps - post-steps
build-linux-mini-crashtest:
machine:
image: ubuntu-2004:202111-02
resource_class: large
steps:
- pre-steps
- install-gflags
- install-compression-libs
- run: make V=1 -j8 CRASH_TEST_EXT_ARGS=--duration=960 blackbox_crash_test_with_atomic_flush
- post-steps
build-windows: build-windows:
executor: windows-2xlarge executor: windows-2xlarge
parameters: parameters:
@ -828,6 +839,9 @@ workflows:
build-linux-unity-and-headers: build-linux-unity-and-headers:
jobs: jobs:
- build-linux-unity-and-headers - build-linux-unity-and-headers
build-linux-mini-crashtest:
jobs:
- build-linux-mini-crashtest
build-windows-vs2019: build-windows-vs2019:
jobs: jobs:
- build-windows: - build-windows:

@ -8,11 +8,7 @@
BASH_EXISTS := $(shell which bash) BASH_EXISTS := $(shell which bash)
SHELL := $(shell which bash) SHELL := $(shell which bash)
# Default to python3. Some distros like CentOS 8 do not have `python`. include python.mk
ifeq ($(origin PYTHON), undefined)
PYTHON := $(shell which python3 || which python || echo python3)
endif
export PYTHON
CLEAN_FILES = # deliberately empty, so we can append below. CLEAN_FILES = # deliberately empty, so we can append below.
CFLAGS += ${EXTRA_CFLAGS} CFLAGS += ${EXTRA_CFLAGS}
@ -1031,65 +1027,7 @@ check_some: $(ROCKSDBTESTS_SUBSET)
ldb_tests: ldb ldb_tests: ldb
$(PYTHON) tools/ldb_test.py $(PYTHON) tools/ldb_test.py
crash_test: include crash_test.mk
# Do not parallelize
$(MAKE) whitebox_crash_test
$(MAKE) blackbox_crash_test
crash_test_with_atomic_flush:
# Do not parallelize
$(MAKE) whitebox_crash_test_with_atomic_flush
$(MAKE) blackbox_crash_test_with_atomic_flush
crash_test_with_txn:
# Do not parallelize
$(MAKE) whitebox_crash_test_with_txn
$(MAKE) blackbox_crash_test_with_txn
crash_test_with_best_efforts_recovery: blackbox_crash_test_with_best_efforts_recovery
crash_test_with_ts:
# Do not parallelize
$(MAKE) whitebox_crash_test_with_ts
$(MAKE) blackbox_crash_test_with_ts
blackbox_crash_test: db_stress
$(PYTHON) -u tools/db_crashtest.py --simple blackbox $(CRASH_TEST_EXT_ARGS)
$(PYTHON) -u tools/db_crashtest.py blackbox $(CRASH_TEST_EXT_ARGS)
blackbox_crash_test_with_atomic_flush: db_stress
$(PYTHON) -u tools/db_crashtest.py --cf_consistency blackbox $(CRASH_TEST_EXT_ARGS)
blackbox_crash_test_with_txn: db_stress
$(PYTHON) -u tools/db_crashtest.py --txn blackbox $(CRASH_TEST_EXT_ARGS)
blackbox_crash_test_with_best_efforts_recovery: db_stress
$(PYTHON) -u tools/db_crashtest.py --test_best_efforts_recovery blackbox $(CRASH_TEST_EXT_ARGS)
blackbox_crash_test_with_ts: db_stress
$(PYTHON) -u tools/db_crashtest.py --enable_ts blackbox $(CRASH_TEST_EXT_ARGS)
ifeq ($(CRASH_TEST_KILL_ODD),)
CRASH_TEST_KILL_ODD=888887
endif
whitebox_crash_test: db_stress
$(PYTHON) -u tools/db_crashtest.py --simple whitebox --random_kill_odd \
$(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
$(PYTHON) -u tools/db_crashtest.py whitebox --random_kill_odd \
$(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
whitebox_crash_test_with_atomic_flush: db_stress
$(PYTHON) -u tools/db_crashtest.py --cf_consistency whitebox --random_kill_odd \
$(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
whitebox_crash_test_with_txn: db_stress
$(PYTHON) -u tools/db_crashtest.py --txn whitebox --random_kill_odd \
$(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
whitebox_crash_test_with_ts: db_stress
$(PYTHON) -u tools/db_crashtest.py --enable_ts whitebox --random_kill_odd \
$(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
asan_check: clean asan_check: clean
COMPILE_WITH_ASAN=1 $(MAKE) check -j32 COMPILE_WITH_ASAN=1 $(MAKE) check -j32

@ -0,0 +1,79 @@
# This file is used by Meta-internal infrastructure as well as by Makefile
# When included from Makefile, there are rules to build DB_STRESS_CMD. When
# used directly with `make -f crashtest.mk ...` there will be no rules to
# build DB_STRESS_CMD so it must exist prior.
DB_STRESS_CMD?=./db_stress
include python.mk
CRASHTEST_MAKE=$(MAKE) -f crash_test.mk
CRASHTEST_PY=$(PYTHON) -u tools/db_crashtest.py --stress_cmd=$(DB_STRESS_CMD)
.PHONY: crash_test crash_test_with_atomic_flush crash_test_with_txn \
crash_test_with_best_efforts_recovery crash_test_with_ts \
blackbox_crash_test blackbox_crash_test_with_atomic_flush \
blackbox_crash_test_with_txn blackbox_crash_test_with_ts \
blackbox_crash_test_with_best_efforts_recovery \
whitebox_crash_test whitebox_crash_test_with_atomic_flush \
whitebox_crash_test_with_txn whitebox_crash_test_with_ts
crash_test: $(DB_STRESS_CMD)
# Do not parallelize
$(CRASHTEST_MAKE) whitebox_crash_test
$(CRASHTEST_MAKE) blackbox_crash_test
crash_test_with_atomic_flush: $(DB_STRESS_CMD)
# Do not parallelize
$(CRASHTEST_MAKE) whitebox_crash_test_with_atomic_flush
$(CRASHTEST_MAKE) blackbox_crash_test_with_atomic_flush
crash_test_with_txn: $(DB_STRESS_CMD)
# Do not parallelize
$(CRASHTEST_MAKE) whitebox_crash_test_with_txn
$(CRASHTEST_MAKE) blackbox_crash_test_with_txn
crash_test_with_best_efforts_recovery: blackbox_crash_test_with_best_efforts_recovery
crash_test_with_ts: $(DB_STRESS_CMD)
# Do not parallelize
$(CRASHTEST_MAKE) whitebox_crash_test_with_ts
$(CRASHTEST_MAKE) blackbox_crash_test_with_ts
blackbox_crash_test: $(DB_STRESS_CMD)
$(CRASHTEST_PY) --simple blackbox $(CRASH_TEST_EXT_ARGS)
$(CRASHTEST_PY) blackbox $(CRASH_TEST_EXT_ARGS)
blackbox_crash_test_with_atomic_flush: $(DB_STRESS_CMD)
$(CRASHTEST_PY) --cf_consistency blackbox $(CRASH_TEST_EXT_ARGS)
blackbox_crash_test_with_txn: $(DB_STRESS_CMD)
$(CRASHTEST_PY) --txn blackbox $(CRASH_TEST_EXT_ARGS)
blackbox_crash_test_with_best_efforts_recovery: $(DB_STRESS_CMD)
$(CRASHTEST_PY) --test_best_efforts_recovery blackbox $(CRASH_TEST_EXT_ARGS)
blackbox_crash_test_with_ts: $(DB_STRESS_CMD)
$(CRASHTEST_PY) --enable_ts blackbox $(CRASH_TEST_EXT_ARGS)
ifeq ($(CRASH_TEST_KILL_ODD),)
CRASH_TEST_KILL_ODD=888887
endif
whitebox_crash_test: $(DB_STRESS_CMD)
$(CRASHTEST_PY) --simple whitebox --random_kill_odd \
$(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
$(CRASHTEST_PY) whitebox --random_kill_odd \
$(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
whitebox_crash_test_with_atomic_flush: $(DB_STRESS_CMD)
$(CRASHTEST_PY) --cf_consistency whitebox --random_kill_odd \
$(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
whitebox_crash_test_with_txn: $(DB_STRESS_CMD)
$(CRASHTEST_PY) --txn whitebox --random_kill_odd \
$(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
whitebox_crash_test_with_ts: $(DB_STRESS_CMD)
$(CRASHTEST_PY) --enable_ts whitebox --random_kill_odd \
$(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)

@ -0,0 +1,9 @@
ifndef PYTHON
# Default to python3. Some distros like CentOS 8 do not have `python`.
ifeq ($(origin PYTHON), undefined)
PYTHON := $(shell which python3 || which python || echo python3)
endif
export PYTHON
endif
Loading…
Cancel
Save