diff --git a/arcanist_util/__phutil_library_map__.php b/arcanist_util/__phutil_library_map__.php index 274ad16e3..e3117382b 100644 --- a/arcanist_util/__phutil_library_map__.php +++ b/arcanist_util/__phutil_library_map__.php @@ -6,33 +6,66 @@ * @phutil-library-version 2 */ -phutil_register_library_map(array( - '__library_version__' => 2, - 'class' => - array( - 'ArcanistCpplintLinter' => 'cpp_linter/ArcanistCpplintLinter.php', - 'BaseDirectoryScopedFormatLinter' => 'cpp_linter/BaseDirectoryScopedFormatLinter.php', - 'FacebookArcanistConfiguration' => 'config/FacebookArcanistConfiguration.php', - 'FacebookFbcodeLintEngine' => 'lint_engine/FacebookFbcodeLintEngine.php', - 'FacebookFbcodeUnitTestEngine' => 'unit_engine/FacebookFbcodeUnitTestEngine.php', - 'FacebookHowtoevenLintEngine' => 'lint_engine/FacebookHowtoevenLintEngine.php', - 'FacebookHowtoevenLinter' => 'cpp_linter/FacebookHowtoevenLinter.php', - 'FbcodeClangFormatLinter' => 'cpp_linter/FbcodeClangFormatLinter.php', - 'FbcodeCppLinter' => 'cpp_linter/FbcodeCppLinter.php', - ), - 'function' => - array( - ), - 'xmap' => - array( - 'ArcanistCpplintLinter' => 'ArcanistLinter', - 'BaseDirectoryScopedFormatLinter' => 'ArcanistLinter', - 'FacebookArcanistConfiguration' => 'ArcanistConfiguration', - 'FacebookFbcodeLintEngine' => 'ArcanistLintEngine', - 'FacebookFbcodeUnitTestEngine' => 'ArcanistBaseUnitTestEngine', - 'FacebookHowtoevenLintEngine' => 'ArcanistLintEngine', - 'FacebookHowtoevenLinter' => 'ArcanistLinter', - 'FbcodeClangFormatLinter' => 'BaseDirectoryScopedFormatLinter', - 'FbcodeCppLinter' => 'ArcanistLinter', - ), -)); +if (class_exists('ArcanistWorkflow')) { + phutil_register_library_map(array( + '__library_version__' => 2, + 'class' => + array( + 'ArcanistCpplintLinter' => 'cpp_linter/ArcanistCpplintLinter.php', + 'BaseDirectoryScopedFormatLinter' => 'cpp_linter/BaseDirectoryScopedFormatLinter.php', + 'FacebookArcanistConfiguration' => 'config/FacebookArcanistConfiguration.php', + 'FacebookFbcodeLintEngine' => 'lint_engine/FacebookFbcodeLintEngine.php', + 'FacebookFbcodeUnitTestEngine' => 'unit_engine/FacebookFbcodeUnitTestEngine.php', + 'FacebookHowtoevenLintEngine' => 'lint_engine/FacebookHowtoevenLintEngine.php', + 'FacebookHowtoevenLinter' => 'cpp_linter/FacebookHowtoevenLinter.php', + 'FbcodeClangFormatLinter' => 'cpp_linter/FbcodeClangFormatLinter.php', + 'FbcodeCppLinter' => 'cpp_linter/FbcodeCppLinter.php', + ), + 'function' => + array( + ), + 'xmap' => + array( + 'ArcanistCpplintLinter' => 'ArcanistLinter', + 'BaseDirectoryScopedFormatLinter' => 'ArcanistLinter', + 'FacebookArcanistConfiguration' => 'ArcanistConfiguration', + 'FacebookFbcodeLintEngine' => 'ArcanistLintEngine', + 'FacebookFbcodeUnitTestEngine' => 'ArcanistBaseUnitTestEngine', + 'FacebookHowtoevenLintEngine' => 'ArcanistLintEngine', + 'FacebookHowtoevenLinter' => 'ArcanistLinter', + 'FbcodeClangFormatLinter' => 'BaseDirectoryScopedFormatLinter', + 'FbcodeCppLinter' => 'ArcanistLinter', + ), + )); +} else { + phutil_register_library_map(array( + '__library_version__' => 2, + 'class' => + array( + 'ArcanistCpplintLinter' => 'cpp_linter/ArcanistCpplintLinter.php', + 'BaseDirectoryScopedFormatLinter' => 'cpp_linter/BaseDirectoryScopedFormatLinter.php', + 'FacebookArcanistConfiguration' => 'config/FacebookOldArcanistConfiguration.php', + 'FacebookFbcodeLintEngine' => 'lint_engine/FacebookFbcodeLintEngine.php', + 'FacebookFbcodeUnitTestEngine' => 'unit_engine/FacebookFbcodeUnitTestEngine.php', + 'FacebookHowtoevenLintEngine' => 'lint_engine/FacebookHowtoevenLintEngine.php', + 'FacebookHowtoevenLinter' => 'cpp_linter/FacebookHowtoevenLinter.php', + 'FbcodeClangFormatLinter' => 'cpp_linter/FbcodeClangFormatLinter.php', + 'FbcodeCppLinter' => 'cpp_linter/FbcodeCppLinter.php', + ), + 'function' => + array( + ), + 'xmap' => + array( + 'ArcanistCpplintLinter' => 'ArcanistLinter', + 'BaseDirectoryScopedFormatLinter' => 'ArcanistLinter', + 'FacebookArcanistConfiguration' => 'ArcanistConfiguration', + 'FacebookFbcodeLintEngine' => 'ArcanistLintEngine', + 'FacebookFbcodeUnitTestEngine' => 'ArcanistBaseUnitTestEngine', + 'FacebookHowtoevenLintEngine' => 'ArcanistLintEngine', + 'FacebookHowtoevenLinter' => 'ArcanistLinter', + 'FbcodeClangFormatLinter' => 'BaseDirectoryScopedFormatLinter', + 'FbcodeCppLinter' => 'ArcanistLinter', + ), + )); +} diff --git a/arcanist_util/config/FacebookArcanistConfiguration.php b/arcanist_util/config/FacebookArcanistConfiguration.php index 8fd27fe40..f0f9653a2 100644 --- a/arcanist_util/config/FacebookArcanistConfiguration.php +++ b/arcanist_util/config/FacebookArcanistConfiguration.php @@ -6,7 +6,7 @@ class FacebookArcanistConfiguration extends ArcanistConfiguration { public function didRunWorkflow($command, - ArcanistBaseWorkflow $workflow, + ArcanistWorkflow $workflow, $error_code) { if ($command == 'diff' && !$workflow->isRawDiffSource()) { $this->startTestsInSandcastle($workflow); diff --git a/arcanist_util/config/FacebookOldArcanistConfiguration.php b/arcanist_util/config/FacebookOldArcanistConfiguration.php new file mode 100644 index 000000000..4e936ce7e --- /dev/null +++ b/arcanist_util/config/FacebookOldArcanistConfiguration.php @@ -0,0 +1,187 @@ +isRawDiffSource()) { + $this->startTestsInSandcastle($workflow); + } + } + + ////////////////////////////////////////////////////////////////////// + /* Run tests in sandcastle */ + function postURL($diffID, $url) { + $cmd = 'echo \'{"diff_id": "' . $diffID . '", ' + . '"name":"click here for sandcastle tests for D' . $diffID . '", ' + . '"link":"' . $url . '"}\' | ' + . 'http_proxy=fwdproxy.any.facebook.com:8080 ' + . 'https_proxy=fwdproxy.any.facebook.com:8080 arc call-conduit ' + . 'differential.updateunitresults'; + shell_exec($cmd); + } + + function updateTestCommand($diffID, $test, $status) { + $cmd = 'echo \'{"diff_id": "' . $diffID . '", ' + . '"name":"' . $test . '", ' + . '"result":"' . $status . '"}\' | ' + . 'http_proxy=fwdproxy.any.facebook.com:8080 ' + . 'https_proxy=fwdproxy.any.facebook.com:8080 arc call-conduit ' + . 'differential.updateunitresults'; + return $cmd; + } + + function updateTest($diffID, $test) { + shell_exec($this->updateTestCommand($diffID, $test, "waiting")); + } + + function getSteps($diffID, $username, $test) { + $arcrc_content = exec("cat ~/.arcrc | gzip -f | base64 -w0"); + + // Sandcastle machines don't have arc setup. We copy the user certificate + // and authenticate using that in sandcastle + $setup = array( + "name" => "Setup arcrc", + "shell" => "echo " . $arcrc_content . " | base64 --decode" + . " | gzip -d > ~/.arcrc", + "user" => "root" + ); + + // arc demands certain permission on its config + $fix_permission = array( + "name" => "Fix environment", + "shell" => "chmod 600 ~/.arcrc", + "user" => "root" + ); + + // fbcode is a sub-repo. We cannot patch until we add it to ignore otherwise + // git thinks it is uncommited change + $fix_git_ignore = array( + "name" => "Fix git ignore", + "shell" => "echo fbcode >> .git/info/exclude", + "user" => "root" + ); + + // Patch the code (keep your fingures crossed) + $patch = array( + "name" => "Patch " . $diffID, + "shell" => "HTTPS_PROXY=fwdproxy:8080 arc --arcrc-file ~/.arcrc " + . "patch --nocommit --diff " . $diffID, + "user" => "root" + ); + + // Clean up the user arc config we are using + $cleanup = array( + "name" => "Arc cleanup", + "shell" => "rm -f ~/.arcrc", + "user" => "root" + ); + + // Construct the steps in the order of execution + $steps[] = $setup; + $steps[] = $fix_permission; + $steps[] = $fix_git_ignore; + $steps[] = $patch; + + // Run the actual command + $this->updateTest($diffID, $test); + $cmd = $this->updateTestCommand($diffID, $test, "running") . ";" + . "./build_tools/precommit_checker.py " . $test + . "; exit_code=$?; ([[ \$exit_code -eq 0 ]] &&" + . $this->updateTestCommand($diffID, $test, "pass") . ")" + . "||" . $this->updateTestCommand($diffID, $test, "fail") + . "; cat /tmp/precommit-check.log" + . "; for f in `ls t/log-*`; do echo \$f; cat \$f; done;" + . "[[ \$exit_code -eq 0 ]]"; + + $run_test = array( + "name" => "Run " . $test, + "shell" => $cmd, + "user" => "root", + ); + + $steps[] = $run_test; + $steps[] = $cleanup; + + return $steps; + } + + function startTestsInSandcastle($workflow) { + // extract information we need from workflow or CLI + $diffID = $workflow->getDiffId(); + $username = exec("whoami"); + + if ($diffID == null || $username == null) { + // there is no diff and we can't extract username + // we cannot schedule sandcasstle job + return; + } + + if (strcmp(getenv("ROCKSDB_CHECK_ALL"), 1) == 0) { + // extract all tests from the CI definition + $output = file_get_contents("build_tools/rocksdb-lego-determinator"); + preg_match_all('/[ ]{2}([a-zA-Z0-9_]+)[\)]{1}/', $output, $matches); + $tests = $matches[1]; + } else { + // manually list of tests we want to run in sandcastle + $tests = array( + "unit", "unit_481", "clang_unit", "tsan", "asan", "lite_test", "valgrind" + ); + } + + // construct a job definition for each test and add it to the master plan + foreach ($tests as $test) { + $arg[] = array( + "name" => "RocksDB diff " . $diffID . " test " . $test, + "steps" => $this->getSteps($diffID, $username, $test) + ); + } + + // we cannot submit the parallel execution master plan to sandcastle + // we need supply the job plan as a determinator + // so we construct a small job that will spit out the master job plan + // which sandcastle will parse and execute + // Why compress ? Otherwise we run over the max string size. + $cmd = "echo " . base64_encode(json_encode($arg)) + . " | gzip -f | base64 -w0"; + $arg_encoded = shell_exec($cmd); + + $command = array( + "name" => "Run diff " . $diffID . "for user " . $username, + "steps" => array() + ); + + $command["steps"][] = array( + "name" => "Generate determinator", + "shell" => "echo " . $arg_encoded . " | base64 --decode | gzip -d" + . " | base64 --decode", + "determinator" => true, + "user" => "root" + ); + + // submit to sandcastle + $url = 'https://interngraph.intern.facebook.com/sandcastle/generate?' + .'command=SandcastleUniversalCommand' + .'&vcs=rocksdb-git&revision=origin%2Fmaster&type=lego' + .'&user=krad&alias=rocksdb-precommit' + .'&command-args=' . urlencode(json_encode($command)); + + $cmd = 'https_proxy= HTTPS_PROXY= curl -s -k -F app=659387027470559 ' + . '-F token=AeO_3f2Ya3TujjnxGD4 "' . $url . '"'; + + $output = shell_exec($cmd); + + // extract sandcastle URL from the response + preg_match('/url": "(.+)"/', $output, $sandcastle_url); + + echo "\nSandcastle URL: " . $sandcastle_url[1] . "\n"; + + // Ask phabricator to display it on the diff UI + $this->postURL($diffID, $sandcastle_url[1]); + } +} diff --git a/arcanist_util/cpp_linter/BaseDirectoryScopedFormatLinter.php b/arcanist_util/cpp_linter/BaseDirectoryScopedFormatLinter.php index 79966e78d..4a7b307dc 100644 --- a/arcanist_util/cpp_linter/BaseDirectoryScopedFormatLinter.php +++ b/arcanist_util/cpp_linter/BaseDirectoryScopedFormatLinter.php @@ -44,7 +44,7 @@ abstract class BaseDirectoryScopedFormatLinter extends ArcanistLinter { $futures[$path] = $this->getFormatFuture($path, $changed); } - foreach (Futures($futures)->limit(8) as $p => $f) { + foreach (id(new FutureIterator($futures))->limit(8) as $p => $f) { $this->rawLintOutput[$p] = $f->resolvex(); } } diff --git a/arcanist_util/lint_engine/FacebookFbcodeLintEngine.php b/arcanist_util/lint_engine/FacebookFbcodeLintEngine.php index 7b12cccdd..88b0748f7 100644 --- a/arcanist_util/lint_engine/FacebookFbcodeLintEngine.php +++ b/arcanist_util/lint_engine/FacebookFbcodeLintEngine.php @@ -42,14 +42,12 @@ class FacebookFbcodeLintEngine extends ArcanistLintEngine { $python_linter = new ArcanistPEP8Linter(); $linters[] = $python_linter; - if (!$this->getCommitHookMode()) { - $cpp_linters = array(); - $cpp_linters[] = $linters[] = new ArcanistCpplintLinter(); - $cpp_linters[] = $linters[] = new FbcodeCppLinter(); + $cpp_linters = array(); + $cpp_linters[] = $linters[] = new ArcanistCpplintLinter(); + $cpp_linters[] = $linters[] = new FbcodeCppLinter(); - $clang_format_linter = new FbcodeClangFormatLinter(); - $linters[] = $clang_format_linter; - } + $clang_format_linter = new FbcodeClangFormatLinter(); + $linters[] = $clang_format_linter; $spelling_linter = new ArcanistSpellingLinter(); $linters[] = $spelling_linter;