[Polly] Added arcanist (arc) unit test support

[jdoerfert]

Summary

Thanks to Clemens ([e-mail]) extension of the
arcanist unit test engine, we can run the llvm-lit tests automatically.

Similar to the linters, aracanist will run all tests for each uploaded
commit and asks the commiter for a statements in case unit tests fail.

To use this feature the unit test engine needs to find the polly build
directory. Currently we support the following setups:

  $POLLY_BUILD_DIR (environment variable)
  <root>/build
  <root>.build
  <root>-build
  <root:s/src/build>
  <cwd>

[grosser]

Please make sure Clemens makes clear (on the mailing list or here) that contributing the code is OK.

Otherwise, LGTM.

Tobias

[hammacher]

Confirmed ;)

[hammacher]

Wow, cool. As a newly registered user I can accept Polly revisions...

Sorry if I mixed up the review process.

[grosser]

Thanks Clemens!

I am fine with this. Johannes feel free to commit.

[jdoerfert]

Rebase

[jdoerfert]

r217396.

.arcconfig

 {
   "project_id" : "polly",
   "conduit_uri" : "http://reviews.llvm.org/",
+  "history.immutable" : true,
   "linter.scriptandregex.script": "sh -c './utils/check_format.sh \"$0\" 2> /dev/null || true'",
-  "linter.scriptandregex.regex": "/^(OK:(?P<ignore>.+)|Error:) (?P<message>.+)$/m"
+  "linter.scriptandregex.regex": "/^(OK:(?P<ignore>.+)|Error:) (?P<message>.+)$/m",
+  "load" : [
+    "utils/arcanist/LitTestEngine"
+  ],
+  "unit.engine" : "LitTestEngine"
 }

utils/arcanist/LitTestEngine/__phutil_library_init__.php

+<?php
+
+phutil_register_library('lit-test-engine', __FILE__);

utils/arcanist/LitTestEngine/__phutil_library_map__.php

+<?php
+
+/**
+ * This file is automatically generated. Use 'arc liberate' to rebuild it.
+ * @generated
+ * @phutil-library-version 2
+ */
+
+phutil_register_library_map(array(
+  '__library_version__' => 2,
+  'class' =>
+  array(
+    'LitTestEngine' => 'src/LitTestEngine.php',
+  ),
+  'function' =>
+  array(
+  ),
+  'xmap' =>
+  array(
+    'LitTestEngine' => 'ArcanistBaseUnitTestEngine',
+  ),
+));

utils/arcanist/LitTestEngine/src/LitTestEngine.php

+<?php
+
+/**
+ * llvm-lit wrapper
+ *
+ * To use, set unit.engine in .arcconfig, or use --engine flag
+ * with arc unit.
+ *
+ * This file was authored by Clemens Hammacher <[e-mail]>
+ * and initially used in the Sambamba project (www.sambamba.org).
+ *
+ * @group unitrun
+ */
+final class LitTestEngine extends ArcanistUnitTestEngine {
+
+    protected function supportsRunAllTests() {
+        return true;
+    }
+
+    private function progress($results, $numTests) {
+        static $colors = array(
+            ArcanistUnitTestResult::RESULT_PASS => 'green',
+            ArcanistUnitTestResult::RESULT_FAIL => 'red',
+            ArcanistUnitTestResult::RESULT_SKIP => 'yellow',
+            ArcanistUnitTestResult::RESULT_BROKEN => 'red',
+            ArcanistUnitTestResult::RESULT_UNSOUND => 'yellow',
+            ArcanistUnitTestResult::RESULT_POSTPONED => 'yellow'
+        );
+
+        $s = "[";
+        $lastColor = "";
+        foreach ($results as $result) {
+            $color = $colors[$result->getResult()];
+            if (!$color && $lastColor)
+                $s .= "</bg>";
+            elseif (!$lastColor && $color)
+                $s .= "<bg:$color>";
+            elseif ($lastColor !== $color)
+                $s .= "</bg><bg:$color>";
+            $lastColor = $color;
+            $s .= ".";
+        }
+        if ($lastColor)
+            $s .= "</bg>";
+        $c = count($results);
+        if ($numTests)
+            $s .= str_repeat(" ", $numTests - $c) . " $c/$numTests]";
+        else
+            $s .= " $c]";
+        return phutil_console_format($s);
+    }
+
+    public function run() {
+
+        $projectRoot = $this->getWorkingCopy()->getProjectRoot();
+        $cwd = getcwd();
+        $buildDir = $this->findBuildDirectory($projectRoot, $cwd);
+        print "Using build directory '$buildDir'\n";
+        $makeVars = $this->getMakeVars($buildDir);
+        $lit = $this->findLitExecutable($makeVars);
+        print "Using lit executable '$lit'\n";
+
+        // We have to modify the format string, because llvm-lit does not like a '' argument
+        $cmd = '%s ' . ($this->getEnableAsyncTests() ? '' : '-j1 ') .'%s 2>&1';
+        $litFuture = new ExecFuture($cmd, $lit, $buildDir."/test");
+        $out = "";
+        $results = array();
+        $lastTime = microtime(true);
+        $ready = false;
+        $dots = "";
+        $numTests = 0;
+        while (!$ready) {
+            $ready = $litFuture->isReady();
+            $newout = $litFuture->readStdout();
+            if (strlen($newout) == 0) {
+                usleep(100);
+                continue;
+            }
+            $out .= $newout;
+            if ($ready && strlen($out) > 0 && substr($out, -1) != "\n")
+                $out .= "\n";
+
+            while (($nlPos = strpos($out, "\n")) !== FALSE) {
+                $line = substr($out, 0, $nlPos+1);
+                $out = substr($out, $nlPos+1);
+
+                $res = ArcanistUnitTestResult::RESULT_UNSOUND;
+                if (substr($line, 0, 6) == "PASS: ") {
+                    $res = ArcanistUnitTestResult::RESULT_PASS;
+                } elseif (substr($line, 0, 6) == "FAIL: ") {
+                    $res = ArcanistUnitTestResult::RESULT_FAIL;
+                } elseif (substr($line, 0, 7) == "XPASS: ") {
+                    $res = ArcanistUnitTestResult::RESULT_FAIL;
+                } elseif (substr($line, 0, 7) == "XFAIL: ") {
+                    $res = ArcanistUnitTestResult::RESULT_PASS;
+                } elseif (substr($line, 0, 13) == "UNSUPPORTED: ") {
+                    $res = ArcanistUnitTestResult::RESULT_SKIP;
+                } elseif (!$numTests && preg_match('/Testing: ([0-9]+) tests/', $line, $matches)) {
+                    $numTests = (int)$matches[1];
+                }
+                if ($res == ArcanistUnitTestResult::RESULT_FAIL)
+                    print "\033[1A";
+                if ($res != ArcanistUnitTestResult::RESULT_SKIP && $res != ArcanistUnitTestResult::RESULT_PASS)
+                    print "\r\033[K\033[1A".$line.self::progress($results, $numTests);
+                if ($res == ArcanistUnitTestResult::RESULT_UNSOUND)
+                    continue;
+                $result = new ArcanistUnitTestResult();
+                $result->setName(trim(substr($line, strpos($line, ':') + 1)));
+                $result->setResult($res);
+                $newTime = microtime(true);
+                $result->setDuration($newTime - $lastTime);
+                $lastTime = $newTime;
+                $results[] = $result;
+                $dots .= ".";
+                print "\r\033[K\033[1A".self::progress($results, $numTests);
+            }
+        }
+        list ($out1,$out2) = $litFuture->read();
+        print $out1;
+        if ($out2) {
+            throw new Exception('There was error output, even though it should have been redirected to stdout.');
+        }
+        print "\n";
+
+        return $results;
+    }
+
+    /**
+     * Try to find the build directory of the project, starting from the
+     * project root, and the current working directory.
+     * Also, the environment variable POLLY_BIN_DIR is read to determine the
+     * correct location.
+     *
+     * Search locations are:
+     * $POLLY_BIN_DIR (environment variable)
+     * <root>/build
+     * <root>.build
+     * <root>-build
+     * <root:s/src/build>
+     * <cwd>
+     *
+     * This list might be extended in the future according to other common
+     * setup.
+     *
+     * @param   projectRoot   Directory of the project root
+     * @param   cwd           Current working directory
+     * @return  string        Presumable build directory
+     */
+    public static function findBuildDirectory($projectRoot, $cwd) {
+
+        $projectRoot = rtrim($projectRoot, DIRECTORY_SEPARATOR);
+        $cwd = rtrim($cwd, DIRECTORY_SEPARATOR);
+
+        $tries = array();
+
+        $smbbin_env = getenv("POLLY_BIN_DIR");
+        if ($smbbin_env)
+            $tries[] = $smbbin_env;
+
+        $tries[] = $projectRoot.DIRECTORY_SEPARATOR."build";
+        $tries[] = $projectRoot.".build";
+        $tries[] = $projectRoot."-build";
+
+        // Try to replace each occurence of "src" by "build" (also within path components, like llvm-src)
+        $srcPos = 0;
+        while (($srcPos = strpos($projectRoot, "src", $srcPos)) !== FALSE) {
+            $tries[] = substr($projectRoot, 0, $srcPos)."build".substr($projectRoot, $srcPos+3);
+            $srcPos += 3;
+        }
+
+        $tries[] = $cwd;
+
+        foreach ($tries as $try) {
+            if (is_dir($try) &&
+                file_exists($try.DIRECTORY_SEPARATOR."Makefile") &&
+                (file_exists($try.DIRECTORY_SEPARATOR."test".DIRECTORY_SEPARATOR."lit.site.cfg") ||
+                file_exists($try.DIRECTORY_SEPARATOR."test".DIRECTORY_SEPARATOR."lit.cfg")))
+                return Filesystem::resolvePath($try);
+        }
+
+        throw new Exception("Did not find a build directory for project '$projectRoot', cwd '$cwd'.\n" .
+            "Make sure to have a 'test' directory inside build, which contains lit.cfg or lit.site.cfg.\n" .
+            "You might have to run 'make test' once in the build directory.");
+    }
+
+    /**
+     * Try to find the llvm build directory, based on the make variables
+     * as determined by getMakeVars().
+     *
+     * @param   makeVars      The determined make variables
+     * @return  string        Presumable llvm build directory
+     */
+    public static function getLLVMObjDir($makeVars) {
+        if (!array_key_exists('LLVM_OBJ_ROOT', $makeVars))
+            throw new Exception("Make variables (determined by 'make printvars') does not contain LLVM_OBJ_ROOT.");
+        $llvmObjDir = $makeVars['LLVM_OBJ_ROOT'];
+        if (!is_dir($llvmObjDir))
+            throw new Exception("LLVM_OBJ_ROOT ('$llvmObjDir') is no directory.");
+        return $llvmObjDir;
+    }
+
+    /**
+     * Determine the make variables, by calling 'make printvars' in the
+     * build directory.
+     *
+     * @param   buildDir      The determined build directory
+     * @return  map<str,str>  Map of printed make variables to values
+     */
+    public static function getMakeVars($buildDir) {
+        $litFuture = new ExecFuture('make -C %s printvars', $buildDir);
+        list($stdout, $stderr) = $litFuture->resolvex(10);
+        print $stderr;
+        $makeVars = array();
+
+        foreach (explode("\n", $stdout) as $line) {
+            $components = explode(':', $line);
+            if (count($components) == 3)
+                $makeVars[trim($components[1])] = trim($components[2]);
+        }
+
+        return $makeVars;
+    }
+
+    /**
+     * Return full path to the llvm-lit executable.
+     *
+     * @param   llvmObjDir    The determined llvm build directory
+     * @return  string        Full path to the llvm-lit executable
+     */
+    public static function findLitExecutable($makeVars) {
+        $llvmObjDir = self::getLLVMObjDir($makeVars);
+        $buildMode = array_key_exists('BuildMode', $makeVars) ? $makeVars['BuildMode'] : '';
+
+        if (!$buildMode)
+            throw new Exception("Make variables (determined by 'make printvars') does not contain BuildMode.");
+
+        $lit = $llvmObjDir.DIRECTORY_SEPARATOR.$buildMode.DIRECTORY_SEPARATOR.'bin'.DIRECTORY_SEPARATOR.'llvm-lit';
+        if (!is_executable($lit))
+            throw new Exception("File does not exists or is not executable: $lit");
+        return $lit;
+    }
+}