[Analyzer] Run tests until the end, do not stop at the first failure

[george.karpenkov]

Summary

This is especially important for updating reference results.

[dcoughlin]

This looks good to me.

[dcoughlin]

Oops, wrong button! LGTM!

utils/analyzer/SATestBuild.py

 The list of checkers tested are hardcoded in the Checkers variable.
 For testing additional checkers, use the SA_ADDITIONAL_CHECKERS environment
 variable. It should contain a comma separated list.
 """
 import CmpRuns
 
 import os
 import csv
+from collections import namedtuple
 import sys
 import glob
 import math
 import shutil
 import time
 import plistlib
 import argparse
 from subprocess import check_call, check_output, CalledProcessError

 
 def runCmpResults(Dir, Strictness=0):
     """
     Compare the warnings produced by scan-build.
     Strictness defines the success criteria for the test:
       0 - success if there are no crashes or analyzer failure.
       1 - success if there are no difference in the number of reported bugs.
       2 - success if all the bug reports are identical.
+
+    :return success: Whether tests pass according to the Strictness
+    criteria.
     """
+    TestsPassed = True
     TBegin = time.time()
 
     RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName)
     NewDir = os.path.join(Dir, SBOutputDirName)
 
     # We have to go one level down the directory tree.
     RefList = glob.glob(RefDir + "/*")
     NewList = glob.glob(NewDir + "/*")
 
     # Log folders are also located in the results dir, so ignore them.
     RefLogDir = os.path.join(RefDir, LogFolderName)
     if RefLogDir in RefList:
         RefList.remove(RefLogDir)
     NewList.remove(os.path.join(NewDir, LogFolderName))
 
-    if len(RefList) == 0 or len(NewList) == 0:
-        return False
     assert(len(RefList) == len(NewList))
 
     # There might be more then one folder underneath - one per each scan-build
     # command (Ex: one for configure and one for make).
     if (len(RefList) > 1):
         # Assume that the corresponding folders have the same names.
         RefList.sort()
         NewList.sort()
for P in PairList:
         NumDiffs, ReportsInRef, ReportsInNew = \
             CmpRuns.dumpScanBuildResultsDiff(RefDir, NewDir, Opts, False)
         sys.stdout = OLD_STDOUT
         if (NumDiffs > 0):
             print "Warning: %r differences in diagnostics. See %s" % \
                   (NumDiffs, DiffsPath,)
         if Strictness >= 2 and NumDiffs > 0:
             print "Error: Diffs found in strict mode (2)."
-            sys.exit(-1)
+            TestsPassed = False
         elif Strictness >= 1 and ReportsInRef != ReportsInNew:
             print "Error: The number of results are different in "\
                   "strict mode (1)."
-            sys.exit(-1)
+            TestsPassed = False
 
     print "Diagnostic comparison complete (time: %.2f)." % (
           time.time() - TBegin)
-    return (NumDiffs > 0)
+    return TestsPassed
 
 
 def cleanupReferenceResults(SBOutputDir):
     """
     Delete html, css, and js files from reference results. These can
     include multiple copies of the benchmark source and so get very large.
     """
     Extensions = ["html", "css", "js"]
try:
             print "  Executing: %s" % (CommitCommand,)
         check_call(CommitCommand, shell=True)
     except:
         print "Error: SVN update failed."
         sys.exit(-1)
 
 
 def testProject(ID, ProjectBuildMode, IsReferenceBuild=False, Strictness=0):
+    """
+    Test a given project.
+    :return TestsPassed: Whether tests have passed according
+    to the :param Strictness: criteria.
+    """
     print " \n\n--- Building project %s" % (ID,)
 
     TBegin = time.time()
 
     Dir = getProjectDir(ID)
     if Verbose == 1:
         print "  Build directory: %s." % (Dir,)
 
     # Set the build results directory.
     RelOutputDir = getSBOutputDirName(IsReferenceBuild)
     SBOutputDir = os.path.join(Dir, RelOutputDir)
 
     buildProject(Dir, SBOutputDir, ProjectBuildMode, IsReferenceBuild)
 
     checkBuild(SBOutputDir)
 
     if IsReferenceBuild:
         cleanupReferenceResults(SBOutputDir)
+        TestsPassed = True
     else:
-        runCmpResults(Dir, Strictness)
+        TestsPassed = runCmpResults(Dir, Strictness)
 
     print "Completed tests for project %s (time: %.2f)." % \
           (ID, (time.time() - TBegin))
+    return TestsPassed
 
 
 def isCommentCSVLine(Entries):
     """
     Treat CSV lines starting with a '#' as a comment.
     """
     return len(Entries) > 0 and Entries[0].startswith("#")
 
for I in iterateOverProjects(PMapFile):
             raise Exception()
         if (not ((I[1] == "0") | (I[1] == "1") | (I[1] == "2"))):
             print "Error: Second entry in the ProjectMapFile should be 0" \
                   " (single file), 1 (project), or 2(single file c++11)."
             raise Exception()
 
 
 def testAll(IsReferenceBuild=False, UpdateSVN=False, Strictness=0):
+    TestsPassed = True
     with projectFileHandler() as PMapFile:
         validateProjectFile(PMapFile)
 
         # When we are regenerating the reference results, we might need to
         # update svn. Remove reference results from SVN.
         if UpdateSVN:
             assert(IsReferenceBuild)
             updateSVN("delete", PMapFile)
 
         # Test the projects.
         for (ProjName, ProjBuildMode) in iterateOverProjects(PMapFile):
-            testProject(
+            TestsPassed &= testProject(
                 ProjName, int(ProjBuildMode), IsReferenceBuild, Strictness)
 
         # Re-add reference results to SVN.
         if UpdateSVN:
             updateSVN("add", PMapFile)
 
 
 if __name__ == '__main__':
if __name__ == '__main__':
     UpdateSVN = False
     Strictness = Args.strictness
     if Args.regenerate:
         IsReference = True
     elif Args.update_reference:
         IsReference = True
         UpdateSVN = True
 
-    testAll(IsReference, UpdateSVN, Strictness)
+    TestsPassed = testAll(IsReference, UpdateSVN, Strictness)
+    if not TestsPassed:
+        sys.exit(-1)