NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update a few scripts to be python3 compatible (#6525)

Browse Source 
Summary:
There are a few scripts with python3 compatibility issues that were not
detected by automated tool before. Update them now.

Test Plan (devserver):
python2 tools/ldb_test.py
python3 tools/ldb_test.py

python2 tools/write_stress_runner.py --runtime_sec=30
python3 tools/write_stress_runner.py --runtime_sec=30

python2 tools/db_crashtest.py --simple --interval=2 --duration=10 blackbox
python3 tools/db_crashtest.py --simple --interval=2 --duration=10 blackbox

python2 tools/db_crashtest.py --simple --duration=10 --random_kill_odd=1000 --ops_per_thread=1000 whitebox
python3 tools/db_crashtest.py --simple --duration=10 --random_kill_odd=1000 --ops_per_thread=1000 whitebox
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6525

Reviewed By: cheng-chang

Differential Revision: D20627820

Pulled By: riversand963

fbshipit-source-id: 4b25a7bd4d001c7f868be8b640ef876523be6ca3
main
Yanqin Jin 5 years ago committed by Facebook GitHub Bot
parent 6fd0ed4993
commit ccf7676455
6 changed files with 90 additions and 76 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 25
      Makefile
  2. 7
      coverage/coverage_test.sh
  3. 22
      coverage/parse_gcov_output.py
  4. 57
      tools/db_crashtest.py
  5. 47
      tools/ldb_test.py
  6. 8
      tools/write_stress_runner.py

25
Makefile
Unescape View File

@ -8,6 +8,7 @@
BASH_EXISTS := $(shell which bash) BASH_EXISTS := $(shell which bash)
SHELL := $(shell which bash) SHELL := $(shell which bash)
PYTHON?=$(shell which python)
CLEAN_FILES = # deliberately empty, so we can append below. CLEAN_FILES = # deliberately empty, so we can append below.
CFLAGS += ${EXTRA_CFLAGS} CFLAGS += ${EXTRA_CFLAGS}
@ -948,9 +949,9 @@ check: all
fi fi
rm -rf $(TMPD) rm -rf $(TMPD)
ifneq ($(PLATFORM), OS_AIX) ifneq ($(PLATFORM), OS_AIX)
python tools/check_all_python.py $(PYTHON) tools/check_all_python.py
ifeq ($(filter -DROCKSDB_LITE,$(OPT)),) ifeq ($(filter -DROCKSDB_LITE,$(OPT)),)
python tools/ldb_test.py $(PYTHON) tools/ldb_test.py
sh tools/rocksdb_dump_test.sh sh tools/rocksdb_dump_test.sh
endif endif
endif endif
@ -961,7 +962,7 @@ check_some: $(SUBSET)
.PHONY: ldb_tests .PHONY: ldb_tests
ldb_tests: ldb ldb_tests: ldb
python tools/ldb_test.py $(PYTHON) tools/ldb_test.py
crash_test: whitebox_crash_test blackbox_crash_test crash_test: whitebox_crash_test blackbox_crash_test
@ -970,31 +971,31 @@ crash_test_with_atomic_flush: whitebox_crash_test_with_atomic_flush blackbox_cra
crash_test_with_txn: whitebox_crash_test_with_txn blackbox_crash_test_with_txn crash_test_with_txn: whitebox_crash_test_with_txn blackbox_crash_test_with_txn
blackbox_crash_test: db_stress blackbox_crash_test: db_stress
python -u tools/db_crashtest.py --simple blackbox $(CRASH_TEST_EXT_ARGS) $(PYTHON) -u tools/db_crashtest.py --simple blackbox $(CRASH_TEST_EXT_ARGS)
python -u tools/db_crashtest.py blackbox $(CRASH_TEST_EXT_ARGS) $(PYTHON) -u tools/db_crashtest.py blackbox $(CRASH_TEST_EXT_ARGS)
blackbox_crash_test_with_atomic_flush: db_stress blackbox_crash_test_with_atomic_flush: db_stress
python -u tools/db_crashtest.py --cf_consistency blackbox $(CRASH_TEST_EXT_ARGS) $(PYTHON) -u tools/db_crashtest.py --cf_consistency blackbox $(CRASH_TEST_EXT_ARGS)
blackbox_crash_test_with_txn: db_stress blackbox_crash_test_with_txn: db_stress
python -u tools/db_crashtest.py --txn blackbox $(CRASH_TEST_EXT_ARGS) $(PYTHON) -u tools/db_crashtest.py --txn blackbox $(CRASH_TEST_EXT_ARGS)
ifeq ($(CRASH_TEST_KILL_ODD),) ifeq ($(CRASH_TEST_KILL_ODD),)
CRASH_TEST_KILL_ODD=888887 CRASH_TEST_KILL_ODD=888887
endif endif
whitebox_crash_test: db_stress whitebox_crash_test: db_stress
python -u tools/db_crashtest.py --simple whitebox --random_kill_odd \ $(PYTHON) -u tools/db_crashtest.py --simple whitebox --random_kill_odd \
$(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS) $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
python -u tools/db_crashtest.py whitebox --random_kill_odd \ $(PYTHON) -u tools/db_crashtest.py whitebox --random_kill_odd \
$(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS) $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
whitebox_crash_test_with_atomic_flush: db_stress whitebox_crash_test_with_atomic_flush: db_stress
python -u tools/db_crashtest.py --cf_consistency whitebox --random_kill_odd \ $(PYTHON) -u tools/db_crashtest.py --cf_consistency whitebox --random_kill_odd \
$(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS) $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
whitebox_crash_test_with_txn: db_stress whitebox_crash_test_with_txn: db_stress
python -u tools/db_crashtest.py --txn whitebox --random_kill_odd \ $(PYTHON) -u tools/db_crashtest.py --txn whitebox --random_kill_odd \
$(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS) $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
asan_check: asan_check:
@ -2093,7 +2094,7 @@ jtest_run:
jtest: rocksdbjava jtest: rocksdbjava
cd java;$(MAKE) sample;$(MAKE) test; cd java;$(MAKE) sample;$(MAKE) test;
python tools/check_all_python.py # TODO peterd: find a better place for this check in CI targets $(PYTHON) tools/check_all_python.py # TODO peterd: find a better place for this check in CI targets
jdb_bench: jdb_bench:
cd java;$(MAKE) db_bench; cd java;$(MAKE) db_bench;

7
coverage/coverage_test.sh
Unescape View File

@ -17,16 +17,19 @@ if [ -d /mnt/gvfs/third-party -a -z "$CXX" ]; then
else else
GCOV=$(which gcov) GCOV=$(which gcov)
fi fi
echo -e "Using $GCOV"
COVERAGE_DIR="$PWD/COVERAGE_REPORT" COVERAGE_DIR="$PWD/COVERAGE_REPORT"
mkdir -p $COVERAGE_DIR mkdir -p $COVERAGE_DIR
# Find all gcno files to generate the coverage report # Find all gcno files to generate the coverage report
PYTHON=${1:-`which python`}
echo -e "Using $PYTHON"
GCNO_FILES=`find $ROOT -name "*.gcno"` GCNO_FILES=`find $ROOT -name "*.gcno"`
$GCOV --preserve-paths --relative-only --no-output $GCNO_FILES 2>/dev/null | $GCOV --preserve-paths --relative-only --no-output $GCNO_FILES 2>/dev/null |
# Parse the raw gcov report to more human readable form. # Parse the raw gcov report to more human readable form.
python $ROOT/coverage/parse_gcov_output.py | $PYTHON $ROOT/coverage/parse_gcov_output.py |
# Write the output to both stdout and report file. # Write the output to both stdout and report file.
tee $COVERAGE_DIR/coverage_report_all.txt && tee $COVERAGE_DIR/coverage_report_all.txt &&
echo -e "Generated coverage report for all files: $COVERAGE_DIR/coverage_report_all.txt\n" echo -e "Generated coverage report for all files: $COVERAGE_DIR/coverage_report_all.txt\n"
@ -41,7 +44,7 @@ RECENT_REPORT=$COVERAGE_DIR/coverage_report_recent.txt
echo -e "Recently updated files: $LATEST_FILES\n" > $RECENT_REPORT echo -e "Recently updated files: $LATEST_FILES\n" > $RECENT_REPORT
$GCOV --preserve-paths --relative-only --no-output $GCNO_FILES 2>/dev/null | $GCOV --preserve-paths --relative-only --no-output $GCNO_FILES 2>/dev/null |
python $ROOT/coverage/parse_gcov_output.py -interested-files $LATEST_FILES | $PYTHON $ROOT/coverage/parse_gcov_output.py -interested-files $LATEST_FILES |
tee -a $RECENT_REPORT && tee -a $RECENT_REPORT &&
echo -e "Generated coverage report for recently updated files: $RECENT_REPORT\n" echo -e "Generated coverage report for recently updated files: $RECENT_REPORT\n"

22
coverage/parse_gcov_output.py
Unescape View File

@ -1,10 +1,12 @@
#!/usr/bin/env python2 #!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
from __future__ import print_function
import optparse
import re import re
import sys import sys
from optparse import OptionParser
# the gcov report follows certain pattern. Each file will have two lines # the gcov report follows certain pattern. Each file will have two lines
# of report, from which we can extract the file name, total lines and coverage # of report, from which we can extract the file name, total lines and coverage
# percentage. # percentage.
@ -48,7 +50,7 @@ def parse_gcov_report(gcov_input):
def get_option_parser(): def get_option_parser():
usage = "Parse the gcov output and generate more human-readable code " +\ usage = "Parse the gcov output and generate more human-readable code " +\
"coverage report." "coverage report."
parser = OptionParser(usage) parser = optparse.OptionParser(usage)
parser.add_option( parser.add_option(
"--interested-files", "-i", "--interested-files", "-i",
@ -73,8 +75,8 @@ def display_file_coverage(per_file_coverage, total_coverage):
header_template = \ header_template = \
"%" + str(max_file_name_length) + "s\t%s\t%s" "%" + str(max_file_name_length) + "s\t%s\t%s"
separator = "-" * (max_file_name_length + 10 + 20) separator = "-" * (max_file_name_length + 10 + 20)
print header_template % ("Filename", "Coverage", "Lines") # noqa: E999 T25377293 Grandfathered in print(header_template % ("Filename", "Coverage", "Lines")) # noqa: E999 T25377293 Grandfathered in
print separator print(separator)
# -- Print body # -- Print body
# template for printing coverage report for each file. # template for printing coverage report for each file.
@ -82,12 +84,12 @@ def display_file_coverage(per_file_coverage, total_coverage):
for fname, coverage_info in per_file_coverage.items(): for fname, coverage_info in per_file_coverage.items():
coverage, lines = coverage_info coverage, lines = coverage_info
print record_template % (fname, coverage, lines) print(record_template % (fname, coverage, lines))
# -- Print footer # -- Print footer
if total_coverage: if total_coverage:
print separator print(separator)
print record_template % ("Total", total_coverage[0], total_coverage[1]) print(record_template % ("Total", total_coverage[0], total_coverage[1]))
def report_coverage(): def report_coverage():
parser = get_option_parser() parser = get_option_parser()
@ -111,7 +113,7 @@ def report_coverage():
total_coverage = None total_coverage = None
if not len(per_file_coverage): if not len(per_file_coverage):
print >> sys.stderr, "Cannot find coverage info for the given files." print("Cannot find coverage info for the given files.", file=sys.stderr)
return return
display_file_coverage(per_file_coverage, total_coverage) display_file_coverage(per_file_coverage, total_coverage)

57
tools/db_crashtest.py
Unescape View File

@ -1,5 +1,7 @@
#!/usr/bin/env python2 #!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
from __future__ import absolute_import, division, print_function, unicode_literals
import os import os
import sys import sys
import time import time
@ -132,7 +134,7 @@ def is_direct_io_supported(dbname):
with tempfile.NamedTemporaryFile(dir=dbname) as f: with tempfile.NamedTemporaryFile(dir=dbname) as f:
try: try:
os.open(f.name, os.O_DIRECT) os.open(f.name, os.O_DIRECT)
except: except BaseException:
return False return False
return True return True
@ -297,8 +299,8 @@ def blackbox_crash_main(args, unknown_args):
killtime = time.time() + cmd_params['interval'] killtime = time.time() + cmd_params['interval']
cmd = gen_cmd(dict( cmd = gen_cmd(dict(
cmd_params.items() + list(cmd_params.items())
{'db': dbname}.items()), unknown_args) + list({'db': dbname}.items())), unknown_args)
child = subprocess.Popen(cmd, stderr=subprocess.PIPE) child = subprocess.Popen(cmd, stderr=subprocess.PIPE)
print("Running db_stress with pid=%d: %s\n\n" print("Running db_stress with pid=%d: %s\n\n"
@ -323,7 +325,7 @@ def blackbox_crash_main(args, unknown_args):
time.sleep(1) # time to stabilize after a kill time.sleep(1) # time to stabilize after a kill
while True: while True:
line = child.stderr.readline().strip() line = child.stderr.readline().strip().decode('utf-8')
if line == '': if line == '':
break break
elif not line.startswith('WARNING'): elif not line.startswith('WARNING'):
@ -348,7 +350,7 @@ def whitebox_crash_main(args, unknown_args):
cur_time = time.time() cur_time = time.time()
exit_time = cur_time + cmd_params['duration'] exit_time = cur_time + cmd_params['duration']
half_time = cur_time + cmd_params['duration'] / 2 half_time = cur_time + cmd_params['duration'] // 2
print("Running whitebox-crash-test with \n" print("Running whitebox-crash-test with \n"
+ "total-duration=" + str(cmd_params['duration']) + "\n") + "total-duration=" + str(cmd_params['duration']) + "\n")
@ -374,9 +376,9 @@ def whitebox_crash_main(args, unknown_args):
}) })
elif kill_mode == 1: elif kill_mode == 1:
if cmd_params.get('disable_wal', 0) == 1: if cmd_params.get('disable_wal', 0) == 1:
my_kill_odd = kill_random_test / 50 + 1 my_kill_odd = kill_random_test // 50 + 1
else: else:
my_kill_odd = kill_random_test / 10 + 1 my_kill_odd = kill_random_test // 10 + 1
additional_opts.update({ additional_opts.update({
"kill_random_test": my_kill_odd, "kill_random_test": my_kill_odd,
"kill_prefix_blacklist": "WritableFileWriter::Append," "kill_prefix_blacklist": "WritableFileWriter::Append,"
@ -386,7 +388,7 @@ def whitebox_crash_main(args, unknown_args):
# TODO: May need to adjust random odds if kill_random_test # TODO: May need to adjust random odds if kill_random_test
# is too small. # is too small.
additional_opts.update({ additional_opts.update({
"kill_random_test": (kill_random_test / 5000 + 1), "kill_random_test": (kill_random_test // 5000 + 1),
"kill_prefix_blacklist": "WritableFileWriter::Append," "kill_prefix_blacklist": "WritableFileWriter::Append,"
"WritableFileWriter::WriteBuffered," "WritableFileWriter::WriteBuffered,"
"PosixMmapFile::Allocate,WritableFileWriter::Flush", "PosixMmapFile::Allocate,WritableFileWriter::Flush",
@ -406,7 +408,7 @@ def whitebox_crash_main(args, unknown_args):
# style is quite a bit slower on reads with lot of files # style is quite a bit slower on reads with lot of files
additional_opts = { additional_opts = {
"kill_random_test": None, "kill_random_test": None,
"ops_per_thread": cmd_params['ops_per_thread'] / 5, "ops_per_thread": cmd_params['ops_per_thread'] // 5,
"compaction_style": 2, "compaction_style": 2,
} }
else: else:
@ -416,19 +418,24 @@ def whitebox_crash_main(args, unknown_args):
"ops_per_thread": cmd_params['ops_per_thread'], "ops_per_thread": cmd_params['ops_per_thread'],
} }
cmd = gen_cmd(dict(cmd_params.items() + additional_opts.items() cmd = gen_cmd(dict(list(cmd_params.items())
+ {'db': dbname}.items()), unknown_args) + list(additional_opts.items())
+ list({'db': dbname}.items())), unknown_args)
print "Running:" + ' '.join(cmd) + "\n" # noqa: E999 T25377293 Grandfathered in print("Running:" + ' '.join(cmd) + "\n") # noqa: E999 T25377293 Grandfathered in
popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, popen = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT) stderr=subprocess.STDOUT)
stdoutdata, stderrdata = popen.communicate() stdoutdata, stderrdata = popen.communicate()
if stdoutdata:
stdoutdata = stdoutdata.decode('utf-8')
if stderrdata:
stderrdata = stderrdata.decode('utf-8')
retncode = popen.returncode retncode = popen.returncode
msg = ("check_mode={0}, kill option={1}, exitcode={2}\n".format( msg = ("check_mode={0}, kill option={1}, exitcode={2}\n".format(
check_mode, additional_opts['kill_random_test'], retncode)) check_mode, additional_opts['kill_random_test'], retncode))
print msg print(msg)
print stdoutdata print(stdoutdata)
expected = False expected = False
if additional_opts['kill_random_test'] is None and (retncode == 0): if additional_opts['kill_random_test'] is None and (retncode == 0):
@ -440,19 +447,19 @@ def whitebox_crash_main(args, unknown_args):
expected = True expected = True
if not expected: if not expected:
print "TEST FAILED. See kill option and exit code above!!!\n" print("TEST FAILED. See kill option and exit code above!!!\n")
sys.exit(1) sys.exit(1)
stdoutdata = stdoutdata.lower() stdoutdata = stdoutdata.lower()
errorcount = (stdoutdata.count('error') - errorcount = (stdoutdata.count('error') -
stdoutdata.count('got errors 0 times')) stdoutdata.count('got errors 0 times'))
print "#times error occurred in output is " + str(errorcount) + "\n" print("#times error occurred in output is " + str(errorcount) + "\n")
if (errorcount > 0): if (errorcount > 0):
print "TEST FAILED. Output has 'error'!!!\n" print("TEST FAILED. Output has 'error'!!!\n")
sys.exit(2) sys.exit(2)
if (stdoutdata.find('fail') >= 0): if (stdoutdata.find('fail') >= 0):
print "TEST FAILED. Output has 'fail'!!!\n" print("TEST FAILED. Output has 'fail'!!!\n")
sys.exit(2) sys.exit(2)
# First half of the duration, keep doing kill test. For the next half, # First half of the duration, keep doing kill test. For the next half,
@ -476,12 +483,12 @@ def main():
parser.add_argument("--cf_consistency", action='store_true') parser.add_argument("--cf_consistency", action='store_true')
parser.add_argument("--txn", action='store_true') parser.add_argument("--txn", action='store_true')
all_params = dict(default_params.items() all_params = dict(list(default_params.items())
+ blackbox_default_params.items() + list(blackbox_default_params.items())
+ whitebox_default_params.items() + list(whitebox_default_params.items())
+ simple_default_params.items() + list(simple_default_params.items())
+ blackbox_simple_default_params.items() + list(blackbox_simple_default_params.items())
+ whitebox_simple_default_params.items()) + list(whitebox_simple_default_params.items()))
for k, v in all_params.items(): for k, v in all_params.items():
parser.add_argument("--" + k, type=type(v() if callable(v) else v)) parser.add_argument("--" + k, type=type(v() if callable(v) else v))

47
tools/ldb_test.py
Unescape View File

@ -1,5 +1,7 @@
#!/usr/bin/env python2 #!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
from __future__ import absolute_import, division, print_function, unicode_literals
import os import os
import glob import glob
import os.path import os.path
@ -27,7 +29,7 @@ def my_check_output(*popenargs, **kwargs):
cmd = popenargs[0] cmd = popenargs[0]
raise Exception("Exit code is not 0. It is %d. Command: %s" % raise Exception("Exit code is not 0. It is %d. Command: %s" %
(retcode, cmd)) (retcode, cmd))
return output return output.decode('utf-8')
def run_err_null(cmd): def run_err_null(cmd):
return os.system(cmd + " 2>/dev/null ") return os.system(cmd + " 2>/dev/null ")
@ -52,7 +54,6 @@ class LDBTestCase(unittest.TestCase):
""" """
All command-line params must be specified. All command-line params must be specified.
Allows full flexibility in testing; for example: missing db param. Allows full flexibility in testing; for example: missing db param.
""" """
output = my_check_output("./ldb %s |grep -v \"Created bg thread\"" % output = my_check_output("./ldb %s |grep -v \"Created bg thread\"" %
params, shell=True) params, shell=True)
@ -72,7 +73,6 @@ class LDBTestCase(unittest.TestCase):
""" """
All command-line params must be specified. All command-line params must be specified.
Allows full flexibility in testing; for example: missing db param. Allows full flexibility in testing; for example: missing db param.
""" """
try: try:
@ -87,7 +87,6 @@ class LDBTestCase(unittest.TestCase):
def assertRunOK(self, params, expectedOutput, unexpected=False): def assertRunOK(self, params, expectedOutput, unexpected=False):
""" """
Uses the default test db. Uses the default test db.
""" """
self.assertRunOKFull("%s %s" % (self.dbParam(self.DB_NAME), params), self.assertRunOKFull("%s %s" % (self.dbParam(self.DB_NAME), params),
expectedOutput, unexpected) expectedOutput, unexpected)
@ -99,7 +98,7 @@ class LDBTestCase(unittest.TestCase):
self.assertRunFAILFull("%s %s" % (self.dbParam(self.DB_NAME), params)) self.assertRunFAILFull("%s %s" % (self.dbParam(self.DB_NAME), params))
def testSimpleStringPutGet(self): def testSimpleStringPutGet(self):
print "Running testSimpleStringPutGet..." print("Running testSimpleStringPutGet...")
self.assertRunFAIL("put x1 y1") self.assertRunFAIL("put x1 y1")
self.assertRunOK("put --create_if_missing x1 y1", "OK") self.assertRunOK("put --create_if_missing x1 y1", "OK")
self.assertRunOK("get x1", "y1") self.assertRunOK("get x1", "y1")
@ -157,7 +156,7 @@ class LDBTestCase(unittest.TestCase):
% (inputSst, params)) % (inputSst, params))
def testStringBatchPut(self): def testStringBatchPut(self):
print "Running testStringBatchPut..." print("Running testStringBatchPut...")
self.assertRunOK("batchput x1 y1 --create_if_missing", "OK") self.assertRunOK("batchput x1 y1 --create_if_missing", "OK")
self.assertRunOK("scan", "x1 : y1") self.assertRunOK("scan", "x1 : y1")
self.assertRunOK("batchput x2 y2 x3 y3 \"x4 abc\" \"y4 xyz\"", "OK") self.assertRunOK("batchput x2 y2 x3 y3 \"x4 abc\" \"y4 xyz\"", "OK")
@ -167,7 +166,7 @@ class LDBTestCase(unittest.TestCase):
self.assertRunFAIL("batchput k1 v1 k2") self.assertRunFAIL("batchput k1 v1 k2")
def testCountDelimDump(self): def testCountDelimDump(self):
print "Running testCountDelimDump..." print("Running testCountDelimDump...")
self.assertRunOK("batchput x.1 x1 --create_if_missing", "OK") self.assertRunOK("batchput x.1 x1 --create_if_missing", "OK")
self.assertRunOK("batchput y.abc abc y.2 2 z.13c pqr", "OK") self.assertRunOK("batchput y.abc abc y.2 2 z.13c pqr", "OK")
self.assertRunOK("dump --count_delim", "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8") self.assertRunOK("dump --count_delim", "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8")
@ -176,7 +175,7 @@ class LDBTestCase(unittest.TestCase):
self.assertRunOK("dump --count_delim=\",\"", "x => count:2\tsize:14\nx.1 => count:1\tsize:5\ny.2 => count:1\tsize:4\ny.abc => count:1\tsize:8\nz.13c => count:1\tsize:8") self.assertRunOK("dump --count_delim=\",\"", "x => count:2\tsize:14\nx.1 => count:1\tsize:5\ny.2 => count:1\tsize:4\ny.abc => count:1\tsize:8\nz.13c => count:1\tsize:8")
def testCountDelimIDump(self): def testCountDelimIDump(self):
print "Running testCountDelimIDump..." print("Running testCountDelimIDump...")
self.assertRunOK("batchput x.1 x1 --create_if_missing", "OK") self.assertRunOK("batchput x.1 x1 --create_if_missing", "OK")
self.assertRunOK("batchput y.abc abc y.2 2 z.13c pqr", "OK") self.assertRunOK("batchput y.abc abc y.2 2 z.13c pqr", "OK")
self.assertRunOK("idump --count_delim", "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8") self.assertRunOK("idump --count_delim", "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8")
@ -185,7 +184,7 @@ class LDBTestCase(unittest.TestCase):
self.assertRunOK("idump --count_delim=\",\"", "x => count:2\tsize:14\nx.1 => count:1\tsize:5\ny.2 => count:1\tsize:4\ny.abc => count:1\tsize:8\nz.13c => count:1\tsize:8") self.assertRunOK("idump --count_delim=\",\"", "x => count:2\tsize:14\nx.1 => count:1\tsize:5\ny.2 => count:1\tsize:4\ny.abc => count:1\tsize:8\nz.13c => count:1\tsize:8")
def testInvalidCmdLines(self): def testInvalidCmdLines(self):
print "Running testInvalidCmdLines..." print("Running testInvalidCmdLines...")
# db not specified # db not specified
self.assertRunFAILFull("put 0x6133 0x6233 --hex --create_if_missing") self.assertRunFAILFull("put 0x6133 0x6233 --hex --create_if_missing")
# No param called he # No param called he
@ -195,7 +194,7 @@ class LDBTestCase(unittest.TestCase):
# hex has invalid boolean value # hex has invalid boolean value
def testHexPutGet(self): def testHexPutGet(self):
print "Running testHexPutGet..." print("Running testHexPutGet...")
self.assertRunOK("put a1 b1 --create_if_missing", "OK") self.assertRunOK("put a1 b1 --create_if_missing", "OK")
self.assertRunOK("scan", "a1 : b1") self.assertRunOK("scan", "a1 : b1")
self.assertRunOK("scan --hex", "0x6131 : 0x6231") self.assertRunOK("scan --hex", "0x6131 : 0x6231")
@ -225,7 +224,7 @@ class LDBTestCase(unittest.TestCase):
self.assertRunOK("checkconsistency", "OK") self.assertRunOK("checkconsistency", "OK")
def testTtlPutGet(self): def testTtlPutGet(self):
print "Running testTtlPutGet..." print("Running testTtlPutGet...")
self.assertRunOK("put a1 b1 --ttl --create_if_missing", "OK") self.assertRunOK("put a1 b1 --ttl --create_if_missing", "OK")
self.assertRunOK("scan --hex", "0x6131 : 0x6231", True) self.assertRunOK("scan --hex", "0x6131 : 0x6231", True)
self.assertRunOK("dump --ttl ", "a1 ==> b1", True) self.assertRunOK("dump --ttl ", "a1 ==> b1", True)
@ -240,7 +239,7 @@ class LDBTestCase(unittest.TestCase):
self.assertRunOK("checkconsistency", "OK") self.assertRunOK("checkconsistency", "OK")
def testInvalidCmdLines(self): # noqa: F811 T25377293 Grandfathered in def testInvalidCmdLines(self): # noqa: F811 T25377293 Grandfathered in
print "Running testInvalidCmdLines..." print("Running testInvalidCmdLines...")
# db not specified # db not specified
self.assertRunFAILFull("put 0x6133 0x6233 --hex --create_if_missing") self.assertRunFAILFull("put 0x6133 0x6233 --hex --create_if_missing")
# No param called he # No param called he
@ -251,7 +250,7 @@ class LDBTestCase(unittest.TestCase):
self.assertRunFAIL("put 0x6133 0x6233 --hex=Boo --create_if_missing") self.assertRunFAIL("put 0x6133 0x6233 --hex=Boo --create_if_missing")
def testDumpLoad(self): def testDumpLoad(self):
print "Running testDumpLoad..." print("Running testDumpLoad...")
self.assertRunOK("batchput --create_if_missing x1 y1 x2 y2 x3 y3 x4 y4", self.assertRunOK("batchput --create_if_missing x1 y1 x2 y2 x3 y3 x4 y4",
"OK") "OK")
self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4") self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
@ -342,7 +341,7 @@ class LDBTestCase(unittest.TestCase):
"--db=%s --create_if_missing" % origDbPath, dumpFilePath)) "--db=%s --create_if_missing" % origDbPath, dumpFilePath))
def testIDumpBasics(self): def testIDumpBasics(self):
print "Running testIDumpBasics..." print("Running testIDumpBasics...")
self.assertRunOK("put a val --create_if_missing", "OK") self.assertRunOK("put a val --create_if_missing", "OK")
self.assertRunOK("put b val", "OK") self.assertRunOK("put b val", "OK")
self.assertRunOK( self.assertRunOK(
@ -354,7 +353,7 @@ class LDBTestCase(unittest.TestCase):
"'a' seq:1, type:1 => val\nInternal keys in range: 1") "'a' seq:1, type:1 => val\nInternal keys in range: 1")
def testMiscAdminTask(self): def testMiscAdminTask(self):
print "Running testMiscAdminTask..." print("Running testMiscAdminTask...")
# These tests need to be improved; for example with asserts about # These tests need to be improved; for example with asserts about
# whether compaction or level reduction actually took place. # whether compaction or level reduction actually took place.
self.assertRunOK("batchput --create_if_missing x1 y1 x2 y2 x3 y3 x4 y4", self.assertRunOK("batchput --create_if_missing x1 y1 x2 y2 x3 y3 x4 y4",
@ -390,7 +389,7 @@ class LDBTestCase(unittest.TestCase):
self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4") self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
def testCheckConsistency(self): def testCheckConsistency(self):
print "Running testCheckConsistency..." print("Running testCheckConsistency...")
dbPath = os.path.join(self.TMP_DIR, self.DB_NAME) dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
self.assertRunOK("put x1 y1 --create_if_missing", "OK") self.assertRunOK("put x1 y1 --create_if_missing", "OK")
@ -414,7 +413,7 @@ class LDBTestCase(unittest.TestCase):
params, dumpFile)) params, dumpFile))
def testDumpLiveFiles(self): def testDumpLiveFiles(self):
print "Running testDumpLiveFiles..." print("Running testDumpLiveFiles...")
dbPath = os.path.join(self.TMP_DIR, self.DB_NAME) dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
self.assertRunOK("put x1 y1 --create_if_missing", "OK") self.assertRunOK("put x1 y1 --create_if_missing", "OK")
@ -439,7 +438,7 @@ class LDBTestCase(unittest.TestCase):
return 0 == run_err_null("cp " + src + " " + dest) return 0 == run_err_null("cp " + src + " " + dest)
def testManifestDump(self): def testManifestDump(self):
print "Running testManifestDump..." print("Running testManifestDump...")
dbPath = os.path.join(self.TMP_DIR, self.DB_NAME) dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
self.assertRunOK("put 1 1 --create_if_missing", "OK") self.assertRunOK("put 1 1 --create_if_missing", "OK")
self.assertRunOK("put 2 2", "OK") self.assertRunOK("put 2 2", "OK")
@ -475,7 +474,7 @@ class LDBTestCase(unittest.TestCase):
isPattern=True) isPattern=True)
def testSSTDump(self): def testSSTDump(self):
print "Running testSSTDump..." print("Running testSSTDump...")
dbPath = os.path.join(self.TMP_DIR, self.DB_NAME) dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
self.assertRunOK("put sst1 sst1_val --create_if_missing", "OK") self.assertRunOK("put sst1 sst1_val --create_if_missing", "OK")
@ -495,7 +494,7 @@ class LDBTestCase(unittest.TestCase):
isPattern=True) isPattern=True)
def testWALDump(self): def testWALDump(self):
print "Running testWALDump..." print("Running testWALDump...")
dbPath = os.path.join(self.TMP_DIR, self.DB_NAME) dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
self.assertRunOK("put wal1 wal1_val --create_if_missing", "OK") self.assertRunOK("put wal1 wal1_val --create_if_missing", "OK")
@ -515,7 +514,7 @@ class LDBTestCase(unittest.TestCase):
isPattern=True) isPattern=True)
def testListColumnFamilies(self): def testListColumnFamilies(self):
print "Running testListColumnFamilies..." print("Running testListColumnFamilies...")
self.assertRunOK("put x1 y1 --create_if_missing", "OK") self.assertRunOK("put x1 y1 --create_if_missing", "OK")
cmd = "list_column_families | grep -v \"Column families\"" cmd = "list_column_families | grep -v \"Column families\""
# Test on valid dbPath. # Test on valid dbPath.
@ -524,7 +523,7 @@ class LDBTestCase(unittest.TestCase):
self.assertRunFAIL(cmd) self.assertRunFAIL(cmd)
def testColumnFamilies(self): def testColumnFamilies(self):
print "Running testColumnFamilies..." print("Running testColumnFamilies...")
dbPath = os.path.join(self.TMP_DIR, self.DB_NAME) # noqa: F841 T25377293 Grandfathered in dbPath = os.path.join(self.TMP_DIR, self.DB_NAME) # noqa: F841 T25377293 Grandfathered in
self.assertRunOK("put cf1_1 1 --create_if_missing", "OK") self.assertRunOK("put cf1_1 1 --create_if_missing", "OK")
self.assertRunOK("put cf1_2 2 --create_if_missing", "OK") self.assertRunOK("put cf1_2 2 --create_if_missing", "OK")
@ -559,7 +558,7 @@ class LDBTestCase(unittest.TestCase):
self.assertRunFAIL("drop_column_family four") self.assertRunFAIL("drop_column_family four")
def testIngestExternalSst(self): def testIngestExternalSst(self):
print "Running testIngestExternalSst..." print("Running testIngestExternalSst...")
# Dump, load, write external sst and ingest it in another db # Dump, load, write external sst and ingest it in another db
dbPath = os.path.join(self.TMP_DIR, "db1") dbPath = os.path.join(self.TMP_DIR, "db1")

8
tools/write_stress_runner.py
Unescape View File

@ -1,5 +1,7 @@
#!/usr/bin/env python2 #!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
from __future__ import absolute_import, division, print_function, unicode_literals
import subprocess import subprocess
import argparse import argparse
import random import random
@ -10,7 +12,7 @@ import sys
def generate_runtimes(total_runtime): def generate_runtimes(total_runtime):
# combination of short runtimes and long runtimes, with heavier # combination of short runtimes and long runtimes, with heavier
# weight on short runtimes # weight on short runtimes
possible_runtimes_sec = range(1, 10) + range(1, 20) + [100, 1000] possible_runtimes_sec = list(range(1, 10)) + list(range(1, 20)) + [100, 1000]
runtimes = [] runtimes = []
while total_runtime > 0: while total_runtime > 0:
chosen = random.choice(possible_runtimes_sec) chosen = random.choice(possible_runtimes_sec)
@ -22,7 +24,7 @@ def generate_runtimes(total_runtime):
def main(args): def main(args):
runtimes = generate_runtimes(int(args.runtime_sec)) runtimes = generate_runtimes(int(args.runtime_sec))
print "Going to execute write stress for " + str(runtimes) # noqa: E999 T25377293 Grandfathered in print("Going to execute write stress for " + str(runtimes)) # noqa: E999 T25377293 Grandfathered in
first_time = True first_time = True
for runtime in runtimes: for runtime in runtimes:

Write Preview
Loading…
Cancel
Save