From ec84bef24a3fa08a57f9fc242d67d360943532a9 Mon Sep 17 00:00:00 2001 From: Andrew Kryczka Date: Mon, 18 Apr 2016 13:01:10 -0700 Subject: [PATCH] New legocastle output parsing Summary: Added a python script to parse combined stdout/stderr of legocastle steps. Previously we just matched words like 'Failure', which didn't work since even our test names matched that pattern. I went through all the legocastle steps to come up with strict failure regexes for the common failure cases. There is also some more complex logic to present gtest failures, since the test name and failure message are not on the same line. There will definitely be error cases that don't match any of these patterns, so we can iterate on it over time. Test Plan: no end-to-end test. I ran the legocastle steps locally and piped to my script, then verified output, e.g., $ set -o pipefail && TEST_TMPDIR=/dev/shm/rocksdb COMPILE_WITH_ASAN=1 OPT=-g make J=1 asan_check |& /usr/facebook/ops/scripts/asan_symbolize.py -d |& python build_tools/error_filter.py asan ==2058029==ERROR: AddressSanitizer: heap-buffer-overflow on address 0x60200000a414 at pc 0x4c12f6 bp 0x7ffcfb7a0520 sp 0x7ffcfb7a0518 Reviewers: kradhakrishnan Reviewed By: kradhakrishnan Subscribers: andrewkr, dhruba, leveldb Differential Revision: https://reviews.facebook.net/D56691 --- build_tools/error_filter.py | 164 ++++++++++++++++++++++++++ build_tools/rocksdb-lego-determinator | 2 +- 2 files changed, 165 insertions(+), 1 deletion(-) create mode 100644 build_tools/error_filter.py diff --git a/build_tools/error_filter.py b/build_tools/error_filter.py new file mode 100644 index 000000000..f9d177eed --- /dev/null +++ b/build_tools/error_filter.py @@ -0,0 +1,164 @@ +# Copyright (c) 2016-present, Facebook, Inc. All rights reserved. +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. An additional grant +# of patent rights can be found in the PATENTS file in the same directory. + +'''Filter for error messages in test output: + - Receives merged stdout/stderr from test on stdin + - Finds patterns of known error messages for test name (first argument) + - Prints those error messages to stdout +''' + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import re +import sys + + +class ErrorParserBase(object): + def parse_error(self, line): + '''Parses a line of test output. If it contains an error, returns a + formatted message describing the error; otherwise, returns None. + Subclasses must override this method. + ''' + raise NotImplementedError + + +class GTestErrorParser(ErrorParserBase): + '''A parser that remembers the last test that began running so it can print + that test's name upon detecting failure. + ''' + _GTEST_NAME_PATTERN = re.compile(r'\[ RUN \] (\S+)$') + # format: ':: Failure' + _GTEST_FAIL_PATTERN = re.compile(r'(unknown file|\S+:\d+): Failure$') + + def __init__(self): + self._last_gtest_name = 'Unknown test' + + def parse_error(self, line): + gtest_name_match = self._GTEST_NAME_PATTERN.match(line) + if gtest_name_match: + self._last_gtest_name = gtest_name_match.group(1) + return None + gtest_fail_match = self._GTEST_FAIL_PATTERN.match(line) + if gtest_fail_match: + return '%s failed: %s' % ( + self._last_gtest_name, gtest_fail_match.group(1)) + return None + + +class MatchErrorParser(ErrorParserBase): + '''A simple parser that returns the whole line if it matches the pattern. + ''' + def __init__(self, pattern): + self._pattern = re.compile(pattern) + + def parse_error(self, line): + if self._pattern.match(line): + return line + return None + + +class CompilerErrorParser(MatchErrorParser): + def __init__(self): + # format: '::: error: ' + super(CompilerErrorParser, self).__init__(r'\S+:\d+:\d+: error:') + + +class ScanBuildErrorParser(MatchErrorParser): + def __init__(self): + super(ScanBuildErrorParser, self).__init__( + r'scan-build: \d+ bugs found.$') + + +class DbCrashErrorParser(MatchErrorParser): + def __init__(self): + super(DbCrashErrorParser, self).__init__(r'\*\*\*.*\^$|TEST FAILED.') + + +class WriteStressErrorParser(MatchErrorParser): + def __init__(self): + super(WriteStressErrorParser, self).__init__( + r'ERROR: write_stress died with exitcode=\d+') + + +class AsanErrorParser(MatchErrorParser): + def __init__(self): + super(AsanErrorParser, self).__init__( + r'==\d+==ERROR: AddressSanitizer:') + + +class UbsanErrorParser(MatchErrorParser): + def __init__(self): + # format: '::: runtime error: ' + super(UbsanErrorParser, self).__init__(r'\S+:\d+:\d+: runtime error:') + + +class ValgrindErrorParser(MatchErrorParser): + def __init__(self): + # just grab the summary, valgrind doesn't clearly distinguish errors + # from other log messages. + super(ValgrindErrorParser, self).__init__(r'==\d+== ERROR SUMMARY:') + + +class CompatErrorParser(MatchErrorParser): + def __init__(self): + super(CompatErrorParser, self).__init__(r'==== .*[Ee]rror.* ====$') + + +class TsanErrorParser(MatchErrorParser): + def __init__(self): + super(TsanErrorParser, self).__init__(r'WARNING: ThreadSanitizer:') + + +_TEST_NAME_TO_PARSERS = { + 'punit': [CompilerErrorParser, GTestErrorParser], + 'unit': [CompilerErrorParser, GTestErrorParser], + 'unit_481': [CompilerErrorParser, GTestErrorParser], + 'clang_unit': [CompilerErrorParser, GTestErrorParser], + 'clang_analyze': [CompilerErrorParser, ScanBuildErrorParser], + 'code_cov': [CompilerErrorParser, GTestErrorParser], + 'unity': [CompilerErrorParser, GTestErrorParser], + 'lite': [CompilerErrorParser], + 'lite_test': [CompilerErrorParser, GTestErrorParser], + 'stress_crash': [CompilerErrorParser, DbCrashErrorParser], + 'write_stress': [CompilerErrorParser, WriteStressErrorParser], + 'asan': [CompilerErrorParser, GTestErrorParser, AsanErrorParser], + 'asan_crash': [CompilerErrorParser, AsanErrorParser, DbCrashErrorParser], + 'ubsan': [CompilerErrorParser, GTestErrorParser, UbsanErrorParser], + 'ubsan_crash': [CompilerErrorParser, UbsanErrorParser, DbCrashErrorParser], + 'valgrind': [CompilerErrorParser, GTestErrorParser, ValgrindErrorParser], + 'tsan': [CompilerErrorParser, GTestErrorParser, TsanErrorParser], + 'format_compatible': [CompilerErrorParser, CompatErrorParser], + 'run_format_compatible': [CompilerErrorParser, CompatErrorParser], + 'no_compression': [CompilerErrorParser, GTestErrorParser], + 'run_no_compression': [CompilerErrorParser, GTestErrorParser], + 'regression': [CompilerErrorParser], + 'run_regression': [CompilerErrorParser], +} + + +def main(): + if len(sys.argv) != 2: + return 'Usage: %s ' % sys.argv[0] + test_name = sys.argv[1] + if test_name not in _TEST_NAME_TO_PARSERS: + return 'Unknown test name: %s' % test_name + + error_parsers = [] + for parser_cls in _TEST_NAME_TO_PARSERS[test_name]: + error_parsers.append(parser_cls()) + + for line in sys.stdin: + line = line.strip() + for error_parser in error_parsers: + error_msg = error_parser.parse_error(line) + if error_msg is not None: + print(error_msg) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/build_tools/rocksdb-lego-determinator b/build_tools/rocksdb-lego-determinator index e6deecf1f..7f523944e 100755 --- a/build_tools/rocksdb-lego-determinator +++ b/build_tools/rocksdb-lego-determinator @@ -71,7 +71,7 @@ UBSAN="COMPILE_WITH_UBSAN=1" DISABLE_JEMALLOC="DISABLE_JEMALLOC=1" HTTP_PROXY="https_proxy=http://fwdproxy.29.prn1:8080 http_proxy=http://fwdproxy.29.prn1:8080 ftp_proxy=http://fwdproxy.29.prn1:8080" SETUP_JAVA_ENV="export $HTTP_PROXY; export JAVA_HOME=/usr/local/jdk-7u10-64/; export PATH=\$PATH:\$JAVA_HOME/bin" -PARSER="'parser':'egrep \'Failure|^#|Abort|Expected|Actual|GoogleTestFailure|^==\''" +PARSER="'parser':'python build_tools/error_filter.py $1'" ARTIFACTS=" 'artifacts': [ {