Summary:
The main PHP code churn is caused by extracting the common code from `FacebookArcanistConfiguration.php` and `FacebookOldArcanistConfiguration.php` into `RocksDBCommonDeterminator.php`. This is necessary both for reducing the duplication of code and making sure that we can execute the common core logic separately from continuous runs.
The main logic in `RocksDBCommonDeterminator.php` remains quite the same with the exception of some things:
- Adding separation between the cases when a diff is submitted //vs.// when the code is triggered from a continuous run. There are certain actions which we should do in a case of diff only.
- Adding reporting - now the person who authored the diff will receive e-mail notifications if any of the jobs have failed.
- Enabling assertions and making sure that we'll terminate on failure. This is an internal code used by competent engineers, so instead of `if (!condition) { echo "Something"; exit(1); }` for every invariant I think that `assert(condition)` provides better readability with the same behavior. Especially taking into account that we're talking about things which shouldn't ever happen.
Enabling this entire process will be triggered internally and will be a subject of a separate code review. We should discuss the details of triggering continuous RocksDB build and tests on that diff.
Test Plan:
- Make sure that `[p]arc diff` scenario isn't broken by verifying that tests validating this diff will pass.
- Private testing of triggering the continuous build script.
- Once the changes will land then author an internal job which will use the script and verify its validity.
Reviewers: sdong, yhchiang, kradhakrishnan
Reviewed By: kradhakrishnan
Subscribers: andrewkr, dhruba
Differential Revision: https://reviews.facebook.net/D59811
main
Gunnar Kudrjavets
9 years ago
parent
f9bd66779f
commit
a52e4d7d02
@ -0,0 +1,326 @@ |
||||
<?php |
||||
// Copyright 2004-present Facebook. All Rights Reserved. |
||||
// This source code is licensed under the BSD-style license found in the |
||||
// LICENSE file in the root directory of this source tree. An additional grant |
||||
// of patent rights can be found in the PATENTS file in the same directory. |
||||
|
||||
// Name of the environment variables which need to be set by the entity which |
||||
// triggers continuous runs so that code at the end of the file gets executed |
||||
// and Sandcastle run starts. |
||||
define("ENV_POST_RECEIVE_HOOK", "POST_RECEIVE_HOOK"); |
||||
define("ENV_HTTPS_APP_VALUE", "HTTPS_APP_VALUE"); |
||||
define("ENV_HTTPS_TOKEN_VALUE", "HTTPS_TOKEN_VALUE"); |
||||
|
||||
define("PRIMARY_TOKEN_FILE", '/home/krad/.sandcastle'); |
||||
define("SECONDARY_TOKEN_FILE", '$HOME/.sandcastle'); |
||||
define("CONT_RUN_ALIAS", "leveldb"); |
||||
|
||||
////////////////////////////////////////////////////////////////////// |
||||
/* Run tests in sandcastle */ |
||||
function postURL($diffID, $url) { |
||||
assert(strlen($diffID) > 0); |
||||
assert(is_numeric($diffID)); |
||||
assert(strlen($url) > 0); |
||||
|
||||
$cmd = 'echo \'{"diff_id": "' . $diffID . '", ' |
||||
. '"name":"click here for sandcastle tests for D' . $diffID . '", ' |
||||
. '"link":"' . $url . '"}\' | ' |
||||
. 'http_proxy=fwdproxy.any.facebook.com:8080 ' |
||||
. 'https_proxy=fwdproxy.any.facebook.com:8080 arc call-conduit ' |
||||
. 'differential.updateunitresults'; |
||||
shell_exec($cmd); |
||||
} |
||||
|
||||
function buildUpdateTestStatusCmd($diffID, $test, $status) { |
||||
assert(strlen($diffID) > 0); |
||||
assert(is_numeric($diffID)); |
||||
assert(strlen($test) > 0); |
||||
assert(strlen($status) > 0); |
||||
|
||||
$cmd = 'echo \'{"diff_id": "' . $diffID . '", ' |
||||
. '"name":"' . $test . '", ' |
||||
. '"result":"' . $status . '"}\' | ' |
||||
. 'http_proxy=fwdproxy.any.facebook.com:8080 ' |
||||
. 'https_proxy=fwdproxy.any.facebook.com:8080 arc call-conduit ' |
||||
. 'differential.updateunitresults'; |
||||
return $cmd; |
||||
} |
||||
|
||||
function updateTestStatus($diffID, $test) { |
||||
assert(strlen($diffID) > 0); |
||||
assert(is_numeric($diffID)); |
||||
assert(strlen($test) > 0); |
||||
|
||||
shell_exec(buildUpdateTestStatusCmd($diffID, $test, "waiting")); |
||||
} |
||||
|
||||
function getSteps($applyDiff, $diffID, $username, $test) { |
||||
assert(strlen($username) > 0); |
||||
assert(strlen($test) > 0); |
||||
|
||||
if ($applyDiff) { |
||||
assert(strlen($diffID) > 0); |
||||
assert(is_numeric($diffID)); |
||||
|
||||
$arcrc_content = exec("cat ~/.arcrc | gzip -f | base64 -w0"); |
||||
assert(strlen($arcrc_content) > 0); |
||||
|
||||
// Sandcastle machines don't have arc setup. We copy the user certificate |
||||
// and authenticate using that in Sandcastle. |
||||
$setup = array( |
||||
"name" => "Setup arcrc", |
||||
"shell" => "echo " . $arcrc_content . " | base64 --decode" |
||||
. " | gzip -d > ~/.arcrc", |
||||
"user" => "root" |
||||
); |
||||
|
||||
// arc demands certain permission on its config. |
||||
$fix_permission = array( |
||||
"name" => "Fix environment", |
||||
"shell" => "chmod 600 ~/.arcrc", |
||||
"user" => "root" |
||||
); |
||||
|
||||
// Construct the steps in the order of execution. |
||||
$steps[] = $setup; |
||||
$steps[] = $fix_permission; |
||||
} |
||||
|
||||
// fbcode is a sub-repo. We cannot patch until we add it to ignore otherwise |
||||
// Git thinks it is an uncommited change. |
||||
$fix_git_ignore = array( |
||||
"name" => "Fix git ignore", |
||||
"shell" => "echo fbcode >> .git/info/exclude", |
||||
"user" => "root" |
||||
); |
||||
|
||||
$steps[] = $fix_git_ignore; |
||||
|
||||
// This will be the command used to execute particular type of tests. |
||||
$cmd = ""; |
||||
|
||||
if ($applyDiff) { |
||||
// Patch the code (keep your fingures crossed). |
||||
$patch = array( |
||||
"name" => "Patch " . $diffID, |
||||
"shell" => "HTTPS_PROXY=fwdproxy:8080 arc --arcrc-file ~/.arcrc " |
||||
. "patch --nocommit --diff " . $diffID, |
||||
"user" => "root" |
||||
); |
||||
|
||||
$steps[] = $patch; |
||||
|
||||
updateTestStatus($diffID, $test); |
||||
$cmd = buildUpdateTestStatusCmd($diffID, $test, "running") . "; "; |
||||
} |
||||
|
||||
// Run the actual command. |
||||
$cmd = $cmd . "./build_tools/precommit_checker.py " . $test |
||||
. "; exit_code=$?; "; |
||||
|
||||
if ($applyDiff) { |
||||
$cmd = $cmd . "([[ \$exit_code -eq 0 ]] &&" |
||||
. buildUpdateTestStatusCmd($diffID, $test, "pass") . ")" |
||||
. "||" . buildUpdateTestStatusCmd($diffID, $test, "fail") |
||||
. "; "; |
||||
} |
||||
|
||||
$cmd = $cmd . " cat /tmp/precommit-check.log" |
||||
. "; for f in `ls t/log-*`; do echo \$f; cat \$f; done;" |
||||
. "[[ \$exit_code -eq 0 ]]"; |
||||
assert(strlen($cmd) > 0); |
||||
|
||||
$run_test = array( |
||||
"name" => "Run " . $test, |
||||
"shell" => $cmd, |
||||
"user" => "root", |
||||
); |
||||
|
||||
$steps[] = $run_test; |
||||
|
||||
if ($applyDiff) { |
||||
// Clean up the user arc config we are using. |
||||
$cleanup = array( |
||||
"name" => "Arc cleanup", |
||||
"shell" => "rm -f ~/.arcrc", |
||||
"user" => "root" |
||||
); |
||||
|
||||
$steps[] = $cleanup; |
||||
} |
||||
|
||||
assert(count($steps) > 0); |
||||
return $steps; |
||||
} |
||||
|
||||
function getSandcastleConfig() { |
||||
$sandcastle_config = array(); |
||||
|
||||
// This is a case when we're executed from a continuous run. Fetch the values |
||||
// from the environment. |
||||
if (getenv(ENV_POST_RECEIVE_HOOK)) { |
||||
$sandcastle_config[0] = getenv(ENV_HTTPS_APP_VALUE); |
||||
$sandcastle_config[1] = getenv(ENV_HTTPS_TOKEN_VALUE); |
||||
} else { |
||||
// This is a typical `[p]arc diff` case. Fetch the values from the specific |
||||
// configuration files. |
||||
assert(file_exists(PRIMARY_TOKEN_FILE) || |
||||
file_exists(SECONDARY_TOKEN_FILE)); |
||||
|
||||
// Try the primary location first, followed by a secondary. |
||||
if (file_exists(PRIMARY_TOKEN_FILE)) { |
||||
$cmd = 'cat ' . PRIMARY_TOKEN_FILE; |
||||
} else { |
||||
$cmd = 'cat ' . SECONDARY_TOKEN_FILE; |
||||
} |
||||
|
||||
assert(strlen($cmd) > 0); |
||||
$sandcastle_config = explode(':', rtrim(shell_exec($cmd))); |
||||
} |
||||
|
||||
// In this case be very explicit about the implications. |
||||
if (count($sandcastle_config) != 2) { |
||||
echo "Sandcastle configuration files don't contain valid information " . |
||||
"or the necessary environment variables aren't defined. Unable " . |
||||
"to validate the code changes."; |
||||
exit(1); |
||||
} |
||||
|
||||
assert(strlen($sandcastle_config[0]) > 0); |
||||
assert(strlen($sandcastle_config[1]) > 0); |
||||
assert(count($sandcastle_config) > 0); |
||||
|
||||
return $sandcastle_config; |
||||
} |
||||
|
||||
// This function can be called either from `[p]arc diff` command or during |
||||
// the Git post-receive hook. |
||||
function startTestsInSandcastle($applyDiff, $workflow, $diffID) { |
||||
// Default options don't terminate on failure, but that's what we want. In |
||||
// the current case we use assertions intentionally as "terminate on failure |
||||
// invariants". |
||||
assert_options(ASSERT_BAIL, true); |
||||
|
||||
// In case of a diff we'll send notificatios to the author. Else it'll go to |
||||
// the entire team because failures indicate that build quality has regressed. |
||||
$username = $applyDiff ? exec("whoami") : CONT_RUN_ALIAS; |
||||
assert(strlen($username) > 0); |
||||
|
||||
if ($applyDiff) { |
||||
assert($workflow); |
||||
assert(strlen($diffID) > 0); |
||||
assert(is_numeric($diffID)); |
||||
} |
||||
|
||||
if (strcmp(getenv("ROCKSDB_CHECK_ALL"), 1) == 0) { |
||||
// Extract all tests from the CI definition. |
||||
$output = file_get_contents("build_tools/rocksdb-lego-determinator"); |
||||
assert(strlen($output) > 0); |
||||
|
||||
preg_match_all('/[ ]{2}([a-zA-Z0-9_]+)[\)]{1}/', $output, $matches); |
||||
$tests = $matches[1]; |
||||
assert(count($tests) > 0); |
||||
} else { |
||||
// Manually list of tests we want to run in Sandcastle. |
||||
$tests = array( |
||||
"unit", "unit_481", "clang_unit", "tsan", "asan", "lite_test", |
||||
"valgrind" |
||||
); |
||||
} |
||||
|
||||
$send_email_template = array( |
||||
'type' => 'email', |
||||
'triggers' => array('fail'), |
||||
'emails' => array($username . '@fb.com'), |
||||
); |
||||
|
||||
// Construct a job definition for each test and add it to the master plan. |
||||
foreach ($tests as $test) { |
||||
$stepName = "RocksDB diff " . $diffID . " test " . $test; |
||||
|
||||
if (!$applyDiff) { |
||||
$stepName = "RocksDB continuous integration test " . $test; |
||||
} |
||||
|
||||
$arg[] = array( |
||||
"name" => $stepName, |
||||
"report" => array($send_email_template), |
||||
"steps" => getSteps($applyDiff, $diffID, $username, $test) |
||||
); |
||||
} |
||||
|
||||
// We cannot submit the parallel execution master plan to Sandcastle and |
||||
// need supply the job plan as a determinator. So we construct a small job |
||||
// that will spit out the master job plan which Sandcastle will parse and |
||||
// execute. Why compress the job definitions? Otherwise we run over the max |
||||
// string size. |
||||
$cmd = "echo " . base64_encode(json_encode($arg)) |
||||
. " | gzip -f | base64 -w0"; |
||||
assert(strlen($cmd) > 0); |
||||
|
||||
$arg_encoded = shell_exec($cmd); |
||||
assert(strlen($arg_encoded) > 0); |
||||
|
||||
$runName = "Run diff " . $diffID . "for user " . $username; |
||||
|
||||
if (!$applyDiff) { |
||||
$runName = "RocksDB continuous integration build and test run"; |
||||
} |
||||
|
||||
$command = array( |
||||
"name" => $runName, |
||||
"steps" => array() |
||||
); |
||||
|
||||
$command["steps"][] = array( |
||||
"name" => "Generate determinator", |
||||
"shell" => "echo " . $arg_encoded . " | base64 --decode | gzip -d" |
||||
. " | base64 --decode", |
||||
"determinator" => true, |
||||
"user" => "root" |
||||
); |
||||
|
||||
// Submit to Sandcastle. |
||||
$url = 'https://interngraph.intern.facebook.com/sandcastle/generate?' |
||||
.'command=SandcastleUniversalCommand' |
||||
.'&vcs=rocksdb-git&revision=origin%2Fmaster&type=lego' |
||||
.'&user=' . $username . '&alias=rocksdb-precommit' |
||||
.'&command-args=' . urlencode(json_encode($command)); |
||||
|
||||
// Fetch the configuration necessary to submit a successful HTTPS request. |
||||
$sandcastle_config = getSandcastleConfig(); |
||||
|
||||
$app = $sandcastle_config[0]; |
||||
$token = $sandcastle_config[1]; |
||||
|
||||
$cmd = 'https_proxy= HTTPS_PROXY= curl -s -k -F app=' . $app . ' ' |
||||
. '-F token=' . $token . ' "' . $url . '"'; |
||||
|
||||
$output = shell_exec($cmd); |
||||
assert(strlen($output) > 0); |
||||
|
||||
// Extract Sandcastle URL from the response. |
||||
preg_match('/url": "(.+)"/', $output, $sandcastle_url); |
||||
|
||||
assert(count($sandcastle_url) > 0, "Unable to submit Sandcastle request."); |
||||
assert(strlen($sandcastle_url[1]) > 0, "Unable to extract Sandcastle URL."); |
||||
|
||||
if ($applyDiff) { |
||||
echo "\nSandcastle URL: " . $sandcastle_url[1] . "\n"; |
||||
// Ask Phabricator to display it on the diff UI. |
||||
postURL($diffID, $sandcastle_url[1]); |
||||
} else { |
||||
echo "Continuous integration started Sandcastle tests. You can look at "; |
||||
echo "the progress at:\n" . $sandcastle_url[1] . "\n"; |
||||
} |
||||
} |
||||
|
||||
// Continuous run cript will set the environment variable and based on that |
||||
// we'll trigger the execution of tests in Sandcastle. In that case we don't |
||||
// need to apply any diffs and there's no associated workflow either. |
||||
if (getenv(ENV_POST_RECEIVE_HOOK)) { |
||||
startTestsInSandcastle( |
||||
false /* $applyDiff */, |
||||
NULL /* $workflow */, |
||||
NULL /* $diffID */); |
||||
} |
||||
@ -0,0 +1,87 @@ |
||||
#!/bin/bash |
||||
# |
||||
# Copyright (c) 2016, Facebook. All rights reserved. |
||||
# |
||||
# Overall wrapper script for RocksDB continuous builds. The implementation is a |
||||
# trivial pulling scheme. We loop infinitely, check if any new changes have been |
||||
# committed, if yes then trigger a Sandcastle run, and finally go to sleep again |
||||
# for a certain interval. |
||||
# |
||||
|
||||
function log { |
||||
DATE=`date +%Y-%m-%d:%H:%M:%S` |
||||
echo $DATE $@ |
||||
} |
||||
|
||||
function log_err { |
||||
log "ERROR: $@. Error code: $?." |
||||
} |
||||
|
||||
# |
||||
# Execution starts here. |
||||
# |
||||
|
||||
# Path to the determinator from the root of the RocksDB repo. |
||||
CONTRUN_DETERMINATOR=./arcanist_util/config/RocksDBCommonDeterminator.php |
||||
|
||||
# Value of the previous commit. |
||||
PREV_COMMIT= |
||||
|
||||
log "Starting to monitor for new RocksDB changes ..." |
||||
log "Running under `pwd` as `whoami`." |
||||
|
||||
# Paranoia. Make sure that we're using the right branch. |
||||
git checkout master |
||||
|
||||
if [ ! $? -eq 0 ]; then |
||||
log_err "This is not good. Can't checkout master. Bye-bye!" |
||||
exit 1 |
||||
fi |
||||
|
||||
# We'll run forever and let the execution environment terminate us if we'll |
||||
# exceed whatever timeout is set for the job. |
||||
while true; |
||||
do |
||||
# Get the latest changes committed. |
||||
git pull --rebase |
||||
|
||||
if [ $? -eq 0 ]; then |
||||
LAST_COMMIT=`git log -1 | head -1 | grep commit | awk '{ print $2; }'` |
||||
|
||||
log "Last commit is '$LAST_COMMIT', previous commit is '$PREV_COMMIT'." |
||||
|
||||
if [ "$PREV_COMMIT" == "$LAST_COMMIT" ]; then |
||||
log "There were no changes since the last time I checked. Going to sleep." |
||||
else |
||||
if [ ! -z "$LAST_COMMIT" ]; then |
||||
log "New code has been committed or previous commit not know. " \ |
||||
"Will trigger the tests." |
||||
|
||||
PREV_COMMIT=$LAST_COMMIT |
||||
log "Updated previous commit to '$PREV_COMMIT'." |
||||
|
||||
# |
||||
# This is where we'll trigger the Sandcastle run. The values for |
||||
# HTTPS_APP_VALUE and HTTPS_APP_VALUE will be set in the container we're |
||||
# running in. |
||||
# |
||||
POST_RECEIVE_HOOK=1 php $CONTRUN_DETERMINATOR |
||||
|
||||
if [ $? -eq 0 ]; then |
||||
log "Sandcastle run successfully triggered." |
||||
else |
||||
log_err "Failed to trigger Sandcastle run." |
||||
fi |
||||
else |
||||
log_err "Previous commit not updated. Don't know what the last one is." |
||||
fi |
||||
fi |
||||
else |
||||
log_err "git pull --rebase failed. Will skip running tests for now." |
||||
fi |
||||
|
||||
# Always sleep, even if errors happens while trying to determine the latest |
||||
# commit. This will prevent us terminating in case of transient errors. |
||||
log "Will go to sleep for 5 minutes." |
||||
sleep 5m |
||||
done |
||||
Loading…
Reference in new issue