From b0a15e7fb99b9610dc8e2ee97d961b346b12269b Mon Sep 17 00:00:00 2001 From: krad Date: Tue, 12 Jan 2016 16:41:48 -0800 Subject: [PATCH] Mechanism to run CI jobs on local branch via commit_prereq Summary: This patch provides a mechanism to run pre commit tests on the local branch before committing. This can help prevent frequent build breaks. The tests can be run in parallel by specifying the J=<..> environment variable. Test Plan: Run manually Reviewers: sdong rven tec CC: leveldb@ Task ID: #9689218 Blame Rev: --- Makefile | 7 +- build_tools/precommit_checker.py | 198 +++++++++++++++++++++++++++++++ 2 files changed, 201 insertions(+), 4 deletions(-) create mode 100755 build_tools/precommit_checker.py diff --git a/Makefile b/Makefile index 24e6a5d28..8d2b9c41b 100644 --- a/Makefile +++ b/Makefile @@ -1161,11 +1161,10 @@ jtest: rocksdbjava jdb_bench: cd java;$(MAKE) db_bench; -commit-prereq: - $(MAKE) clean && $(MAKE) all check; +commit_prereq: build_tools/rocksdb-lego-determinator \ + build_tools/precommit_checker.py $(MAKE) clean && $(MAKE) jclean && $(MAKE) rocksdbjava; - $(MAKE) clean && USE_CLANG=1 $(MAKE) all; - $(MAKE) clean && OPT=-DROCKSDB_LITE $(MAKE) static_lib; + build_tools/precommit_checker.py unit uint_481 clang_unit tsan asan lite xfunc: for xftest in $(XFUNC_TESTS); do \ diff --git a/build_tools/precommit_checker.py b/build_tools/precommit_checker.py new file mode 100755 index 000000000..ceb5cb4ab --- /dev/null +++ b/build_tools/precommit_checker.py @@ -0,0 +1,198 @@ +#!/usr/local/fbcode/gcc-4.8.1-glibc-2.17-fb/bin/python2.7 + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals +import argparse +import commands +import subprocess +import sys +import re +import os +import time + +# +# Simple logger +# + + +class Log: + + LOG_FILE = "/tmp/precommit-check.log" + + def __init__(self): + self.filename = Log.LOG_FILE + self.f = open(self.filename, 'w+', 0) + + def caption(self, str): + line = "\n##### %s #####\n" % str + if self.f: + self.f.write("%s \n" % line) + else: + print(line) + + def error(self, str): + data = "\n\n##### ERROR ##### %s" % str + if self.f: + self.f.write("%s \n" % data) + else: + print(data) + + def log(self, str): + if self.f: + self.f.write("%s \n" % str) + else: + print(str) + +# +# Shell Environment +# + + +class Env(object): + + def __init__(self, tests): + self.tests = tests + self.log = Log() + + def shell(self, cmd, path=os.getcwd()): + if path: + os.chdir(path) + + self.log.log("==== shell session ===========================") + self.log.log("%s> %s" % (path, cmd)) + status = subprocess.call("cd %s; %s" % (path, cmd), shell=True, + stdout=self.log.f, stderr=self.log.f) + self.log.log("status = %s" % status) + self.log.log("============================================== \n\n") + return status + + def GetOutput(self, cmd, path=os.getcwd()): + if path: + os.chdir(path) + + self.log.log("==== shell session ===========================") + self.log.log("%s> %s" % (path, cmd)) + status, out = commands.getstatusoutput(cmd) + self.log.log("status = %s" % status) + self.log.log("out = %s" % out) + self.log.log("============================================== \n\n") + return status, out + +# +# Pre-commit checker +# + + +class PreCommitChecker(Env): + + def __init__(self, tests): + Env.__init__(self, tests) + + # + # Get commands for a given job from the determinator file + # + def get_commands(self, test): + status, out = self.GetOutput( + "build_tools/rocksdb-lego-determinator %s" % test, ".") + return status, out + + # + # Run a specific CI job + # + def run_test(self, test): + self.log.caption("Running test %s locally" % test) + + # get commands for the CI job determinator + status, cmds = self.get_commands(test) + if status != 0: + self.log.error("Error getting commands for test %s" % test) + return False + + # Parse the JSON to extract the commands to run + cmds = re.findall("'shell':'([^\']*)'", cmds) + + if len(cmds) == 0: + self.log.log("No commands found") + return False + + # Run commands + for cmd in cmds: + # Replace J=<..> with the local environment variable + if "J" in os.environ: + cmd = cmd.replace("J=1", "J=%s" % os.environ["J"]) + cmd = cmd.replace("make ", "make -j%s " % os.environ["J"]) + # Run the command + status = self.shell(cmd, ".") + if status != 0: + self.log.error("Error running command %s for test %s" + % (cmd, test)) + return False + + return True + + # + # Run specified CI jobs + # + def run_tests(self): + if not self.tests: + self.log.error("Invalid args. Please provide tests") + return False + + self.print_separator() + self.print_row("TEST", "RESULT") + self.print_separator() + + for test in self.tests: + start_time = time.time() + self.print_test(test) + result = self.run_test(test) + elapsed_min = (time.time() - start_time) / 60 + if not result: + self.log.error("Error running test %s" % test) + self.print_result("FAIL (%dm)" % elapsed_min) + return False + self.print_result("PASS (%dm)" % elapsed_min) + + self.print_separator() + return True + + # + # Print a line + # + def print_separator(self): + print("".ljust(60, "-")) + + # + # Print two colums + # + def print_row(self, c0, c1): + print("%s%s" % (c0.ljust(40), c1.ljust(20))) + + def print_test(self, test): + print(test.ljust(40), end="") + sys.stdout.flush() + + def print_result(self, result): + print(result.ljust(20)) + +# +# Main +# +parser = argparse.ArgumentParser(description='RocksDB pre-commit checker.') + +# +parser.add_argument('test', nargs='+', + help='CI test(s) to run. e.g: unit punit asan tsan') + +print("Please follow log %s" % Log.LOG_FILE) + +args = parser.parse_args() +checker = PreCommitChecker(args.test) + +if not checker.run_tests(): + print("Error running tests. Please check log file %s" % Log.LOG_FILE) + sys.exit(1) + +sys.exit(0)