Moved FBCode Linter's to LevelDB.

Summary: Added FBCODE like linting support to our codebase.

Test Plan: arc lint  lint's the code.

Reviewers: dhruba

Reviewed By: dhruba

CC: emayanke, leveldb

Differential Revision: https://reviews.facebook.net/D7041

.arcconfig

@@ -1,6 +1,10 @@
 {
   "project_id" : "leveldb",
   "conduit_uri" : "https://reviews.facebook.net/",
-  "copyright_holder" : ""
+  "copyright_holder" : "",
+  "load" : [
+    "linters/src/"
+  ],
+  "lint.engine" : "FacebookFbcodeLintEngine", 
+  "lint.engine.single.linter" : "FbcodeCppLinter" 
 }
-

linters/src/.phutil_module_cache

@@ -0,0 +1 @@
+{"__symbol_cache_version__":8,"b937ad5f80a8bd1156038b730ff56ec5":{"have":{"class":{"FacebookFbcodeLintEngine":71}},"need":{"class":{"ArcanistLintEngine":104,"ArcanistGeneratedLinter":488,"ArcanistNoLintLinter":577,"ArcanistTextLinter":658,"ArcanistPEP8Linter":1227,"FbcodeCppLinter":1715,"PfffCppLinter":1759,"ArcanistSpellingLinter":1875,"ArcanistFilenameLinter":4207,"Filesystem":357,"ArcanistLintSeverity":778}},"xmap":{"FacebookFbcodeLintEngine":["ArcanistLintEngine"]}},"4443484928afb005f585843d07b04190":{"have":{"class":{"FbcodeCppLinter":13}},"need":{"function":{"Futures":1265},"class":{"ArcanistLinter":37,"ExecFuture":934,"ArcanistLintSeverity":1729}},"xmap":{"FbcodeCppLinter":["ArcanistLinter"]}},"02e2a613e371424b2108d2d6cb849d39":{"have":{"class":{"PfffCppLinter":71}},"need":{"function":{"Futures":875},"class":{"ArcanistLinter":93,"ExecFuture":756,"ArcanistLintMessage":1270,"ArcanistLintSeverity":1607}},"xmap":{"PfffCppLinter":["ArcanistLinter"]}}}

linters/src/__phutil_library_init__.php

@@ -0,0 +1,3 @@
+<?php
+
+phutil_register_library('linters', __FILE__);

linters/src/__phutil_library_map__.php

@@ -0,0 +1,26 @@
+<?php
+
+/**
+ * This file is automatically generated. Use 'arc liberate' to rebuild it.
+ * @generated
+ * @phutil-library-version 2
+ */
+
+phutil_register_library_map(array(
+  '__library_version__' => 2,
+  'class' =>
+  array(
+    'FacebookFbcodeLintEngine' => 'lint_engine/FacebookFbcodeLintEngine.php',
+    'FbcodeCppLinter' => 'cpp_linter/FbcodeCppLinter.php',
+    'PfffCppLinter' => 'cpp_linter/PfffCppLinter.php',
+  ),
+  'function' =>
+  array(
+  ),
+  'xmap' =>
+  array(
+    'FacebookFbcodeLintEngine' => 'ArcanistLintEngine',
+    'FbcodeCppLinter' => 'ArcanistLinter',
+    'PfffCppLinter' => 'ArcanistLinter',
+  ),
+));

linters/src/cpp_linter/FbcodeCppLinter.php

@@ -0,0 +1,99 @@
+<?php
+
+class FbcodeCppLinter extends ArcanistLinter {
+  const CPPLINT      = "/home/engshare/tools/cpplint";
+  const LINT_ERROR   = 1;
+  const LINT_WARNING = 2;
+  const C_FLAG = "--c_mode=true";
+  private $rawLintOutput = array();
+
+  public function willLintPaths(array $paths) {
+    $futures = array();
+    $ret_value = 0;
+    $last_line = system("which cpplint", $ret_value);
+    $CPP_LINT = false;
+    if ($ret_value == 0) {
+      $CPP_LINT = $last_line;
+    } else if (file_exists(self::CPPLINT)) {
+      $CPP_LINT = self::CPPLINT;
+    }
+
+    if ($CPP_LINT) {
+      foreach ($paths as $p) {
+        $lpath = $this->getEngine()->getFilePathOnDisk($p);
+        $lpath_file = file($lpath);
+        if (preg_match('/\.(c)$/', $lpath) ||
+            preg_match('/-\*-.*Mode: C[; ].*-\*-/', $lpath_file[0]) ||
+            preg_match('/vim(:.*)*:\s*(set\s+)?filetype=c\s*:/', $lpath_file[0])
+            ) {
+          $futures[$p] = new ExecFuture("%s %s %s 2>&1",
+                             $CPP_LINT, self::C_FLAG,
+                             $this->getEngine()->getFilePathOnDisk($p));
+        } else {
+          $futures[$p] = new ExecFuture("%s %s 2>&1",
+            self::CPPLINT, $this->getEngine()->getFilePathOnDisk($p));
+        }
+      }
+
+      foreach (Futures($futures)->limit(8) as $p => $f) {
+        $this->rawLintOutput[$p] = $f->resolvex();
+      }
+    }
+    return;
+  }
+
+  public function getLinterName() {
+    return "FBCPP";
+  }
+
+  public function lintPath($path) {
+    $msgs = $this->getCppLintOutput($path);
+    foreach ($msgs as $m) {
+      $this->raiseLintAtLine($m['line'], 0, $m['severity'], $m['msg']);
+    }
+  }
+
+  public function getLintSeverityMap() {
+    return array(
+      self::LINT_WARNING => ArcanistLintSeverity::SEVERITY_WARNING,
+      self::LINT_ERROR   => ArcanistLintSeverity::SEVERITY_ERROR
+    );
+  }
+
+  public function getLintNameMap() {
+    return array(
+      self::LINT_WARNING => "CppLint Warning",
+      self::LINT_ERROR   => "CppLint Error"
+    );
+  }
+
+  private function getCppLintOutput($path) {
+    list($output) = $this->rawLintOutput[$path];
+
+    $msgs = array();
+    $current = null;
+    foreach (explode("\n", $output) as $line) {
+      if (preg_match('/[^:]*\((\d+)\):(.*)$/', $line, $matches)) {
+        if ($current) {
+          $msgs[] = $current;
+        }
+        $line = $matches[1];
+        $text = $matches[2];
+        $sev  = preg_match('/.*Warning.*/', $text)
+                  ? self::LINT_WARNING
+                  : self::LINT_ERROR;
+        $current = array('line'     => $line,
+                         'msg'      => $text,
+                         'severity' => $sev);
+      } else if ($current) {
+        $current['msg'] .= ' ' . $line;
+      }
+    }
+    if ($current) {
+      $msgs[] = $current;
+    }
+
+    return $msgs;
+  }
+}
+

linters/src/cpp_linter/PfffCppLinter.php

@@ -0,0 +1,68 @@
+<?php
+// Copyright 2004-present Facebook.  All rights reserved.
+
+class PfffCppLinter extends ArcanistLinter {
+  const PROGRAM      = "/home/engshare/tools/checkCpp";
+
+  public function getLinterName() {
+    return "checkCpp";
+  }
+  public function getLintNameMap() {
+    return array(
+    );
+  }
+
+  public function getLintSeverityMap() {
+    return array(
+    );
+  }
+
+  public function willLintPaths(array $paths) {
+    $program = false;
+    $ret_value = 0;
+    $last_line = system("which checkCpp", $ret_value);
+    if ($ret_value == 0) {
+      $program = $last_line;
+    } else if (file_exists(self::PROGRAM)) {
+      $program = self::PROGRAM;
+    }
+    if ($program) {
+      $futures = array();
+      foreach ($paths as $p) {
+        $futures[$p] = new ExecFuture("%s --lint %s 2>&1",
+          $program, $this->getEngine()->getFilePathOnDisk($p));
+      }
+      foreach (Futures($futures)->limit(8) as $p => $f) {
+
+        list($stdout, $stderr) = $f->resolvex();
+        $raw = json_decode($stdout, true);
+        if (!is_array($raw)) {
+          throw new Exception(
+            "checkCpp returned invalid JSON!".
+            "Stdout: {$stdout} Stderr: {$stderr}"
+          );
+        }
+        foreach($raw as $err) {
+          $this->addLintMessage(
+            ArcanistLintMessage::newFromDictionary(
+              array(
+                'path' => $err['file'],
+                'line' => $err['line'],
+                'char' => 0,
+                'name' => $err['name'],
+                'description' => $err['info'],
+                'code' => $this->getLinterName(),
+                'severity' => ArcanistLintSeverity::SEVERITY_WARNING,
+              )
+            )
+          );
+        }
+      }
+    }
+    return;
+  }
+
+  public function lintPath($path) {
+    return;
+  }
+}

linters/src/lint_engine/FacebookFbcodeLintEngine.php

@@ -0,0 +1,147 @@
+<?php
+// Copyright 2004-present Facebook.  All rights reserved.
+
+class FacebookFbcodeLintEngine extends ArcanistLintEngine {
+
+  public function buildLinters() {
+    $linters = array();
+    $paths = $this->getPaths();
+
+    // Remove all deleted files, which are not checked by the
+    // following linters.
+    foreach ($paths as $key => $path) {
+      if (!Filesystem::pathExists($this->getFilePathOnDisk($path))) {
+        unset($paths[$key]);
+      }
+    }
+
+    $generated_linter = new ArcanistGeneratedLinter();
+    $linters[] = $generated_linter;
+
+    $nolint_linter = new ArcanistNoLintLinter();
+    $linters[] = $nolint_linter;
+
+    $text_linter = new ArcanistTextLinter();
+    $text_linter->setCustomSeverityMap(array(
+      ArcanistTextLinter::LINT_LINE_WRAP
+        => ArcanistLintSeverity::SEVERITY_ADVICE,
+    ));
+    $linters[] = $text_linter;
+
+    $java_text_linter = new ArcanistTextLinter();
+    $java_text_linter->setMaxLineLength(100);
+    $java_text_linter->setCustomSeverityMap(array(
+      ArcanistTextLinter::LINT_LINE_WRAP
+        => ArcanistLintSeverity::SEVERITY_ADVICE,
+    ));
+    $linters[] = $java_text_linter;
+
+    $pep8_options = $this->getPEP8WithTextOptions().',E302';
+
+    $python_linter = new ArcanistPEP8Linter();
+    $python_linter->setConfig(array('options' => $pep8_options));
+    $linters[] = $python_linter;
+
+    $python_2space_linter = new ArcanistPEP8Linter();
+    $python_2space_linter->setConfig(array('options' => $pep8_options.',E111'));
+    $linters[] = $python_2space_linter;
+
+   // Currently we can't run cpplint in commit hook mode, because it
+    // depends on having access to the working directory.
+    if (!$this->getCommitHookMode()) {
+      $cpp_linter = new FbcodeCppLinter();
+      $cpp_linter2 = new PfffCppLinter();
+      $linters[] = $cpp_linter;
+      $linters[] = $cpp_linter2;
+    }
+
+    $spelling_linter = new ArcanistSpellingLinter();
+    $linters[] = $spelling_linter;
+
+    foreach ($paths as $path) {
+      $is_text = false;
+
+      $text_extensions = (
+        '/\.('.
+        'cpp|cxx|c|cc|h|hpp|hxx|tcc|'.
+        'py|rb|hs|pl|pm|tw|'.
+        'php|phpt|css|js|'.
+        'java|'.
+        'thrift|'.
+        'lua|'.
+        'siv|'.
+        'txt'.
+        ')$/'
+      );
+      if (preg_match($text_extensions, $path)) {
+        $is_text = true;
+      }
+      if ($is_text) {
+        $nolint_linter->addPath($path);
+
+        $generated_linter->addPath($path);
+        $generated_linter->addData($path, $this->loadData($path));
+
+        if (preg_match('/\.java$/', $path)) {
+          $java_text_linter->addPath($path);
+          $java_text_linter->addData($path, $this->loadData($path));
+        } else {
+          $text_linter->addPath($path);
+          $text_linter->addData($path, $this->loadData($path));
+        }
+
+        $spelling_linter->addPath($path);
+        $spelling_linter->addData($path, $this->loadData($path));
+      }
+      if (isset($cpp_linter) && isset($cpp_linter2)  &&
+          preg_match('/\.(cpp|c|cc|cxx|h|hh|hpp|hxx|tcc)$/', $path)) {
+        $cpp_linter->addPath($path);
+        $cpp_linter->addData($path, $this->loadData($path));
+        $cpp_linter2->addPath($path);
+        $cpp_linter2->addData($path, $this->loadData($path));
+
+      }
+
+      // Match *.py and contbuild config files
+      if (preg_match('/(\.(py|tw|smcprops)|^contbuild\/configs\/[^\/]*)$/',
+                    $path)) {
+        $space_count = 4;
+        $real_path = $this->getFilePathOnDisk($path);
+        $dir = dirname($real_path);
+        do {
+          if (file_exists($dir.'/.python2space')) {
+            $space_count = 2;
+            break;
+          }
+          $dir = dirname($dir);
+        } while ($dir != '/' && $dir != '.');
+
+        if ($space_count == 4) {
+          $cur_path_linter = $python_linter;
+        } else {
+          $cur_path_linter = $python_2space_linter;
+        }
+        $cur_path_linter->addPath($path);
+        $cur_path_linter->addData($path, $this->loadData($path));
+
+        if (preg_match('/\.tw$/', $path)) {
+          $cur_path_linter->setCustomSeverityMap(array(
+            'E251' => ArcanistLintSeverity::SEVERITY_DISABLED,
+          ));
+        }
+      }
+
+
+
+    }
+
+    $name_linter = new ArcanistFilenameLinter();
+    $linters[] = $name_linter;
+    foreach ($paths as $path) {
+      $name_linter->addPath($path);
+    }
+
+    return $linters;
+  }
+
+}