[Static Analyzer] Minor improvements to SATest

[xazax.hun]

Summary

The following improvements were made:

• Argument parsing was ported to argparse which is more idiomatic for python scripts nowadays
• Added strictness command line option so build bots can fail when there is a difference from the reference results
• The test should no longer fail when there is an empty line in the build command file

[zaks.anna]

LGTM.

utils/analyzer/SATestBuild.py

410 ↗

(On Diff #28702)

Let's document Strictness here.

cfe/trunk/utils/analyzer/CmpRuns.py

def dumpScanBuildResultsDiff(dirA, dirB, opts, deleteEmpty=True):
 
     TotalReports = len(resultsB.diagnostics)
     print "TOTAL REPORTS: %r" % TotalReports
     print "TOTAL DIFFERENCES: %r" % foundDiffs
     if auxLog:
         print >>auxLog, "('TOTAL NEW REPORTS', %r)" % TotalReports
         print >>auxLog, "('TOTAL DIFFERENCES', %r)" % foundDiffs
         
-    return foundDiffs    
+    return foundDiffs, len(resultsA.diagnostics), len(resultsB.diagnostics)
 
 def main():
     from optparse import OptionParser
     parser = OptionParser("usage: %prog [options] [dir A] [dir B]")
     parser.add_option("", "--rootA", dest="rootA",
                       help="Prefix to ignore on source files for directory A",
                       action="store", type=str, default="")
     parser.add_option("", "--rootB", dest="rootB",

cfe/trunk/utils/analyzer/SATestBuild.py

 import os
 import csv
 import sys
 import glob
 import math
 import shutil
 import time
 import plistlib
+import argparse
 from subprocess import check_call, CalledProcessError
 
 #------------------------------------------------------------------------------
 # Helper functions.
 #------------------------------------------------------------------------------
 
 def detectCPUs():
     """
def runScanBuild(Dir, SBOutputDir, PBuildLogFile):
     # Always use ccc-analyze to ensure that we can locate the failures 
     # directory.
     SBOptions += "--override-compiler "
     try:
         SBCommandFile = open(BuildScriptPath, "r")
         SBPrefix = "scan-build " + SBOptions + " "
         for Command in SBCommandFile:
             Command = Command.strip()
+            if len(Command) == 0:
+                continue;
             # If using 'make', auto imply a -jX argument
             # to speed up analysis.  xcodebuild will
             # automatically use the maximum number of cores.
             if (Command.startswith("make ") or Command == "make") and \
                 "-j" not in Command:
                 Command += " -j%d" % Jobs
             SBCommand = SBPrefix + Command
             if Verbose == 1:        
def checkBuild(SBOutputDir):
     sys.exit(-1)       
 
 # Auxiliary object to discard stdout.
 class Discarder(object):
     def write(self, text):
         pass # do nothing
 
 # Compare the warnings produced by scan-build.
-def runCmpResults(Dir):   
+# Strictness defines the success criteria for the test:
+#   0 - success if there are no crashes or analyzer failure.
+#   1 - success if there are no difference in the number of reported bugs.
+#   2 - success if all the bug reports are identical.
+def runCmpResults(Dir, Strictness = 0):   
     TBegin = time.time() 
 
     RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName)
     NewDir = os.path.join(Dir, SBOutputDirName)
     
     # We have to go one level down the directory tree.
     RefList = glob.glob(RefDir + "/*") 
     NewList = glob.glob(NewDir + "/*")
for P in PairList:
             print "  Comparing Results: %s %s" % (RefDir, NewDir)
     
         DiffsPath = os.path.join(NewDir, DiffsSummaryFileName)
         Opts = CmpRuns.CmpOptions(DiffsPath)
         # Discard everything coming out of stdout (CmpRun produces a lot of them).
         OLD_STDOUT = sys.stdout
         sys.stdout = Discarder()
         # Scan the results, delete empty plist files.
-        NumDiffs = CmpRuns.dumpScanBuildResultsDiff(RefDir, NewDir, Opts, False)
+        NumDiffs, ReportsInRef, ReportsInNew = \
+            CmpRuns.dumpScanBuildResultsDiff(RefDir, NewDir, Opts, False)
         sys.stdout = OLD_STDOUT
         if (NumDiffs > 0) :
             print "Warning: %r differences in diagnostics. See %s" % \
                   (NumDiffs, DiffsPath,)
+        if Strictness >= 2 and NumDiffs > 0:
+            print "Error: Diffs found in strict mode (2)."
+            sys.exit(-1)       
+        elif Strictness >= 1 and ReportsInRef != ReportsInNew:
+            print "Error: The number of results are different in strict mode (1)."
+            sys.exit(-1)       
                     
     print "Diagnostic comparison complete (time: %.2f)." % (time.time()-TBegin) 
     return (NumDiffs > 0)
     
 def updateSVN(Mode, ProjectsMap):
     try:
         ProjectsMap.seek(0)    
         for I in csv.reader(ProjectsMap):
try:
                             "reference results.\""
         if Verbose == 1:        
             print "  Executing: %s" % (CommitCommand,)
         check_call(CommitCommand, shell=True)    
     except:
         print "Error: SVN update failed."
         sys.exit(-1)
         
-def testProject(ID, ProjectBuildMode, IsReferenceBuild=False, Dir=None):
+def testProject(ID, ProjectBuildMode, IsReferenceBuild=False, Dir=None, Strictness = 0):
     print " \n\n--- Building project %s" % (ID,)
 
     TBegin = time.time() 
 
     if Dir is None :
         Dir = getProjectDir(ID)        
     if Verbose == 1:        
         print "  Build directory: %s." % (Dir,)
     
     # Set the build results directory.
     RelOutputDir = getSBOutputDirName(IsReferenceBuild)
     SBOutputDir = os.path.join(Dir, RelOutputDir)
                 
     buildProject(Dir, SBOutputDir, ProjectBuildMode, IsReferenceBuild)
 
     checkBuild(SBOutputDir)
     
     if IsReferenceBuild == False:
-        runCmpResults(Dir)
+        runCmpResults(Dir, Strictness)
         
     print "Completed tests for project %s (time: %.2f)." % \
           (ID, (time.time()-TBegin))
     
-def testAll(IsReferenceBuild = False, UpdateSVN = False):
+def testAll(IsReferenceBuild = False, UpdateSVN = False, Strictness = 0):
     PMapFile = open(getProjectMapPath(), "rb")
     try:        
         # Validate the input.
         for I in csv.reader(PMapFile):
             if (len(I) != 2) :
                 print "Error: Rows in the ProjectMapFile should have 3 entries."
                 raise Exception()
             if (not ((I[1] == "0") | (I[1] == "1") | (I[1] == "2"))):
                 print "Error: Second entry in the ProjectMapFile should be 0" \
                       " (single file), 1 (project), or 2(single file c++11)."
                 raise Exception()              
 
         # When we are regenerating the reference results, we might need to 
         # update svn. Remove reference results from SVN.
         if UpdateSVN == True:
             assert(IsReferenceBuild == True);
             updateSVN("delete",  PMapFile);
             
         # Test the projects.
         PMapFile.seek(0)    
         for I in csv.reader(PMapFile):
-            testProject(I[0], int(I[1]), IsReferenceBuild)
+            testProject(I[0], int(I[1]), IsReferenceBuild, None, Strictness)
 
         # Add reference results to SVN.
         if UpdateSVN == True:
             updateSVN("add",  PMapFile);
 
     except:
         print "Error occurred. Premature termination."
         raise                            
     finally:
         PMapFile.close()    
             
 if __name__ == '__main__':
+    # Parse command line arguments.
+    Parser = argparse.ArgumentParser(description='Test the Clang Static Analyzer.')
+    Parser.add_argument('--strictness', dest='strictness', type=int, default=0,
+                       help='0 to fail on runtime errors, 1 to fail when the number\
+                             of found bugs are different from the reference, 2 to \
+                             fail on any difference from the reference. Default is 0.')
+    Parser.add_argument('-r', dest='regenerate', action='store_true', default=False,
+                        help='Regenerate reference output.')
+    Parser.add_argument('-rs', dest='update_reference', action='store_true',
+                        default=False, help='Regenerate reference output and update svn.')
+    Args = Parser.parse_args()
+
     IsReference = False
     UpdateSVN = False
-    if len(sys.argv) >= 2:
-        if sys.argv[1] == "-r":
-            IsReference = True
-        elif sys.argv[1] == "-rs":
-            IsReference = True
-            UpdateSVN = True
-        else:     
-          print >> sys.stderr, 'Usage: ', sys.argv[0],\
-                             '[-r|-rs]' \
-                             'Use -r to regenerate reference output' \
-                             'Use -rs to regenerate reference output and update svn'
+    Strictness = Args.strictness
+    if Args.regenerate:
+    elif Args.update_reference:
 
-    testAll(IsReference, UpdateSVN)
+    testAll(IsReference, UpdateSVN, Strictness)