Fix use of RocksDBCommonHelper in cont_integration.sh

Reviewed By: mzlee

Differential Revision: D5472936

fbshipit-source-id: cf75858f879f1192b468a9020005634a5afad880

build_tools/RocksDBCommonHelper.php

@@ -0,0 +1,365 @@
+<?php
+// Copyright 2004-present Facebook. All Rights Reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+// Name of the environment variables which need to be set by the entity which
+// triggers continuous runs so that code at the end of the file gets executed
+// and Sandcastle run starts.
+define("ENV_POST_RECEIVE_HOOK", "POST_RECEIVE_HOOK");
+define("ENV_HTTPS_APP_VALUE", "HTTPS_APP_VALUE");
+define("ENV_HTTPS_TOKEN_VALUE", "HTTPS_TOKEN_VALUE");
+
+define("PRIMARY_TOKEN_FILE", '/home/krad/.sandcastle');
+define("CONT_RUN_ALIAS", "leveldb");
+
+//////////////////////////////////////////////////////////////////////
+/*  Run tests in sandcastle */
+function postURL($diffID, $url) {
+  assert(strlen($diffID) > 0);
+  assert(is_numeric($diffID));
+  assert(strlen($url) > 0);
+
+  $cmd = 'echo \'{"diff_id": ' . $diffID . ', '
+         . '"name":"click here for sandcastle tests for D' . $diffID . '", '
+         . '"link":"' . $url . '"}\' | '
+         . 'arc call-conduit '
+         . 'differential.updateunitresults';
+  shell_exec($cmd);
+}
+
+function buildUpdateTestStatusCmd($diffID, $test, $status) {
+  assert(strlen($diffID) > 0);
+  assert(is_numeric($diffID));
+  assert(strlen($test) > 0);
+  assert(strlen($status) > 0);
+
+  $cmd = 'echo \'{"diff_id": ' . $diffID . ', '
+         . '"name":"' . $test . '", '
+         . '"result":"' . $status . '"}\' | '
+         . 'arc call-conduit '
+         . 'differential.updateunitresults';
+  return $cmd;
+}
+
+function updateTestStatus($diffID, $test) {
+  assert(strlen($diffID) > 0);
+  assert(is_numeric($diffID));
+  assert(strlen($test) > 0);
+
+  shell_exec(buildUpdateTestStatusCmd($diffID, $test, "waiting"));
+}
+
+function getSteps($applyDiff, $diffID, $username, $test) {
+  assert(strlen($username) > 0);
+  assert(strlen($test) > 0);
+
+  if ($applyDiff) {
+    assert(strlen($diffID) > 0);
+    assert(is_numeric($diffID));
+
+    $arcrc_content = (PHP_OS == "Darwin" ?
+        exec("cat ~/.arcrc | gzip -f | base64") :
+            exec("cat ~/.arcrc | gzip -f | base64 -w0"));
+    assert(strlen($arcrc_content) > 0);
+
+    // Sandcastle machines don't have arc setup. We copy the user certificate
+    // and authenticate using that in Sandcastle.
+    $setup = array(
+      "name" => "Setup arcrc",
+      "shell" => "echo " . $arcrc_content . " | base64 --decode"
+                 . " | gzip -d > ~/.arcrc",
+      "user" => "root"
+    );
+
+    // arc demands certain permission on its config.
+    // also fix the sticky bit issue in sandcastle
+    $fix_permission = array(
+      "name" => "Fix environment",
+      "shell" => "chmod 600 ~/.arcrc && chmod +t /dev/shm",
+      "user" => "root"
+    );
+
+    // Construct the steps in the order of execution.
+    $steps[] = $setup;
+    $steps[] = $fix_permission;
+  }
+
+  // fbcode is a sub-repo. We cannot patch until we add it to ignore otherwise
+  // Git thinks it is an uncommited change.
+  $fix_git_ignore = array(
+    "name" => "Fix git ignore",
+    "shell" => "echo fbcode >> .git/info/exclude",
+    "user" => "root"
+  );
+
+  // This fixes "FATAL: ThreadSanitizer can not mmap the shadow memory"
+  // Source:
+  // https://github.com/google/sanitizers/wiki/ThreadSanitizerCppManual#FAQ
+  $fix_kernel_issue = array(
+    "name" => "Fix kernel issue with tsan",
+    "shell" => "echo 2 >/proc/sys/kernel/randomize_va_space",
+    "user" => "root"
+  );
+
+  $steps[] = $fix_git_ignore;
+  $steps[] = $fix_kernel_issue;
+
+  // This will be the command used to execute particular type of tests.
+  $cmd = "";
+
+  if ($applyDiff) {
+    // Patch the code (keep your fingures crossed).
+    $patch = array(
+      "name" => "Patch " . $diffID,
+      "shell" => "arc --arcrc-file ~/.arcrc "
+                  . "patch --nocommit --diff " . $diffID,
+      "user" => "root"
+    );
+
+    $steps[] = $patch;
+
+    updateTestStatus($diffID, $test);
+    $cmd = buildUpdateTestStatusCmd($diffID, $test, "running") . "; ";
+  }
+
+  // Run the actual command.
+  $cmd = $cmd . "J=$(nproc) ./build_tools/precommit_checker.py " . $test
+           . "; exit_code=$?; ";
+
+  if ($applyDiff) {
+    $cmd = $cmd . "([[ \$exit_code -eq 0 ]] &&"
+                . buildUpdateTestStatusCmd($diffID, $test, "pass") . ")"
+                . "||" . buildUpdateTestStatusCmd($diffID, $test, "fail")
+                . "; ";
+  }
+
+  // shell command to sort the tests based on exit code and print
+  // the output of the log files.
+  $cat_sorted_logs = "
+    while read code log_file;
+      do echo \"################ cat \$log_file [exit_code : \$code] ################\";
+      cat \$log_file;
+    done < <(tail -n +2 LOG | sort -k7,7n -k4,4gr | awk '{print \$7,\$NF}')";
+
+  // Shell command to cat all log files
+  $cat_all_logs = "for f in `ls t/!(run-*)`; do echo \$f;cat \$f; done";
+
+  // If LOG file exist use it to cat log files sorted by exit code, otherwise
+  // cat everything
+  $logs_cmd = "if [ -f LOG ]; then {$cat_sorted_logs}; else {$cat_all_logs}; fi";
+
+  $cmd = $cmd . " cat /tmp/precommit-check.log"
+              . "; shopt -s extglob; {$logs_cmd}"
+              . "; shopt -u extglob; [[ \$exit_code -eq 0 ]]";
+  assert(strlen($cmd) > 0);
+
+  $run_test = array(
+    "name" => "Run " . $test,
+    "shell" => $cmd,
+    "user" => "root",
+    "parser" => "python build_tools/error_filter.py " . $test,
+  );
+
+  $steps[] = $run_test;
+
+  if ($applyDiff) {
+    // Clean up the user arc config we are using.
+    $cleanup = array(
+      "name" => "Arc cleanup",
+      "shell" => "rm -f ~/.arcrc",
+      "user" => "root"
+    );
+
+    $steps[] = $cleanup;
+  }
+
+  assert(count($steps) > 0);
+  return $steps;
+}
+
+function getSandcastleConfig() {
+  $sandcastle_config = array();
+
+  $cwd = getcwd();
+  $cwd_token_file = "{$cwd}/.sandcastle";
+  // This is a case when we're executed from a continuous run. Fetch the values
+  // from the environment.
+  if (getenv(ENV_POST_RECEIVE_HOOK)) {
+    $sandcastle_config[0] = getenv(ENV_HTTPS_APP_VALUE);
+    $sandcastle_config[1] = getenv(ENV_HTTPS_TOKEN_VALUE);
+  } else {
+    // This is a typical `[p]arc diff` case. Fetch the values from the specific
+    // configuration files.
+    for ($i = 0; $i < 50; $i++) {
+      if (file_exists(PRIMARY_TOKEN_FILE) ||
+          file_exists($cwd_token_file)) {
+        break;
+      }
+      // If we failed to fetch the tokens, sleep for 0.2 second and try again
+      usleep(200000);
+    }
+    assert(file_exists(PRIMARY_TOKEN_FILE) ||
+           file_exists($cwd_token_file));
+
+    // Try the primary location first, followed by a secondary.
+    if (file_exists(PRIMARY_TOKEN_FILE)) {
+      $cmd = 'cat ' . PRIMARY_TOKEN_FILE;
+    } else {
+      $cmd = 'cat ' . $cwd_token_file;
+    }
+
+    assert(strlen($cmd) > 0);
+    $sandcastle_config = explode(':', rtrim(shell_exec($cmd)));
+  }
+
+  // In this case be very explicit about the implications.
+  if (count($sandcastle_config) != 2) {
+    echo "Sandcastle configuration files don't contain valid information " .
+         "or the necessary environment variables aren't defined. Unable " .
+         "to validate the code changes.";
+    exit(1);
+  }
+
+  assert(strlen($sandcastle_config[0]) > 0);
+  assert(strlen($sandcastle_config[1]) > 0);
+  assert(count($sandcastle_config) > 0);
+
+  return $sandcastle_config;
+}
+
+// This function can be called either from `[p]arc diff` command or during
+// the Git post-receive hook.
+ function startTestsInSandcastle($applyDiff, $workflow, $diffID) {
+  // Default options don't terminate on failure, but that's what we want. In
+  // the current case we use assertions intentionally as "terminate on failure
+  // invariants".
+  assert_options(ASSERT_BAIL, true);
+
+  // In case of a diff we'll send notificatios to the author. Else it'll go to
+  // the entire team because failures indicate that build quality has regressed.
+  $username = $applyDiff ? exec("whoami") : CONT_RUN_ALIAS;
+  assert(strlen($username) > 0);
+
+  if ($applyDiff) {
+    assert($workflow);
+    assert(strlen($diffID) > 0);
+    assert(is_numeric($diffID));
+  }
+
+  // List of tests we want to run in Sandcastle.
+  $tests = array("unit", "unit_non_shm", "unit_481", "clang_unit", "tsan",
+                 "asan", "lite_test", "valgrind", "release", "release_481",
+                 "clang_release", "clang_analyze", "code_cov",
+                 "java_build", "no_compression", "unity", "ubsan");
+
+  $send_email_template = array(
+    'type' => 'email',
+    'triggers' => array('fail'),
+    'emails' => array($username . '@fb.com'),
+  );
+
+  // Construct a job definition for each test and add it to the master plan.
+  foreach ($tests as $test) {
+    $stepName = "RocksDB diff " . $diffID . " test " . $test;
+
+    if (!$applyDiff) {
+      $stepName = "RocksDB continuous integration test " . $test;
+    }
+
+    $arg[] = array(
+      "name" => $stepName,
+      "report" => array($send_email_template),
+      "steps" => getSteps($applyDiff, $diffID, $username, $test)
+    );
+  }
+
+  // We cannot submit the parallel execution master plan to Sandcastle and
+  // need supply the job plan as a determinator. So we construct a small job
+  // that will spit out the master job plan which Sandcastle will parse and
+  // execute. Why compress the job definitions? Otherwise we run over the max
+  // string size.
+  $cmd = "echo " . base64_encode(json_encode($arg))
+         . (PHP_OS == "Darwin" ?
+             " | gzip -f | base64" :
+                 " | gzip -f | base64 -w0");
+  assert(strlen($cmd) > 0);
+
+  $arg_encoded = shell_exec($cmd);
+  assert(strlen($arg_encoded) > 0);
+
+  $runName = "Run diff " . $diffID . "for user " . $username;
+
+  if (!$applyDiff) {
+    $runName = "RocksDB continuous integration build and test run";
+  }
+
+  $command = array(
+    "name" => $runName,
+    "steps" => array()
+  );
+
+  $command["steps"][] = array(
+    "name" => "Generate determinator",
+    "shell" => "echo " . $arg_encoded . " | base64 --decode | gzip -d"
+               . " | base64 --decode",
+    "determinator" => true,
+    "user" => "root"
+  );
+
+  // Submit to Sandcastle.
+  $url = 'https://interngraph.intern.facebook.com/sandcastle/create';
+
+  $job = array(
+    'command' => 'SandcastleUniversalCommand',
+    'args' => $command,
+    'capabilities' => array(
+      'vcs' => 'rocksdb-int-git',
+      'type' => 'lego',
+    ),
+    'hash' => 'origin/master',
+    'user' => $username,
+    'alias' => 'rocksdb-precommit',
+    'tags' => array('rocksdb'),
+    'description' => 'Rocksdb precommit job',
+  );
+
+  // Fetch the configuration necessary to submit a successful HTTPS request.
+  $sandcastle_config = getSandcastleConfig();
+
+  $app = $sandcastle_config[0];
+  $token = $sandcastle_config[1];
+
+  $cmd = 'curl -s -k -F app=' . $app . ' '
+          . '-F token=' . $token . ' -F job=\'' . json_encode($job)
+          .'\' "' . $url . '"';
+
+  $output = shell_exec($cmd);
+  assert(strlen($output) > 0);
+
+  // Extract Sandcastle URL from the response.
+  preg_match('/url": "(.+)"/', $output, $sandcastle_url);
+
+  assert(count($sandcastle_url) > 0, "Unable to submit Sandcastle request.");
+  assert(strlen($sandcastle_url[1]) > 0, "Unable to extract Sandcastle URL.");
+
+  if ($applyDiff) {
+    echo "\nSandcastle URL: " . $sandcastle_url[1] . "\n";
+    // Ask Phabricator to display it on the diff UI.
+    postURL($diffID, $sandcastle_url[1]);
+  } else {
+    echo "Continuous integration started Sandcastle tests. You can look at ";
+    echo "the progress at:\n" . $sandcastle_url[1] . "\n";
+  }
+}
+
+// Continuous run cript will set the environment variable and based on that
+// we'll trigger the execution of tests in Sandcastle. In that case we don't
+// need to apply any diffs and there's no associated workflow either.
+if (getenv(ENV_POST_RECEIVE_HOOK)) {
+  startTestsInSandcastle(
+    false /* $applyDiff */,
+    NULL /* $workflow */,
+    NULL /* $diffID */);
+}

build_tools/cont_integration.sh

@@ -67,7 +67,7 @@ function update_repo_status {
 #
 
 # Path to the determinator from the root of the RocksDB repo.
-CONTRUN_DETERMINATOR=./arcanist_util/config/RocksDBCommonHelper.php
+CONTRUN_DETERMINATOR=./build_tools/RocksDBCommonHelper.php
 
 # Value of the previous commit.
 PREV_COMMIT=