[Analyzer] Refactor analyzer testing scripts

[george.karpenkov]

Summary

• Exporting needed function for future reuse
• Idiomatic python: using with file as f instead of try/finally
• Fixing some indentation issues
• No need to reinvent python multiprocessing.getCPUCount()
• Removing a function parameter which is always the same under all invocations
• Adding some docstrings

[george.karpenkov]

Do not import self from CmpRuns

[dcoughlin]

This is great as long as multiprocessing.cpu_count() preserves the behavior we want. Does it?

Also, please make sure to add @NoQ to static analyzer reviews!

utils/analyzer/SATestBuild.py

133 ↗

(On Diff #116296)

Seems reasonable and a nice clean up. Does it work on Windows, Linux, and macOS? The documentation says it is allowed to not be implemented.

[george.karpenkov]

Does it work on Windows, Linux, and macOS?

It definitely works on OS X and Unix.
It's Windows implementation is the same one we have, so it is not worse (apparently it can fail on some obscure Windows versions, but so can the current implementation).

This is great as long as multiprocessing.cpu_count() preserves the behavior we want. Does it?

Yes, the implementation is almost exactly the same as ours:

In [1]: import multiprocessing

In [2]: multiprocessing.cpu_count??
Signature: multiprocessing.cpu_count()
Source:
def cpu_count():
    '''
    Returns the number of CPUs in the system
    '''
    if sys.platform == 'win32':
        try:
            num = int(os.environ['NUMBER_OF_PROCESSORS'])
        except (ValueError, KeyError):
            num = 0
    elif 'bsd' in sys.platform or sys.platform == 'darwin':
        comm = '/sbin/sysctl -n hw.ncpu'
        if sys.platform == 'darwin':
            comm = '/usr' + comm
        try:
            with os.popen(comm) as p:
                num = int(p.read())
        except ValueError:
            num = 0
    else:
        try:
            num = os.sysconf('SC_NPROCESSORS_ONLN')
        except (ValueError, OSError, AttributeError):
            num = 0

    if num >= 1:
        return num
    else:
        raise NotImplementedError('cannot determine number of cpus')

[george.karpenkov]

Also, please make sure to add @NoQ to static analyzer reviews!

OK!

cfe/trunk/utils/analyzer/CmpRuns.py

Usage:
     # Generate a relation from diagnostics in run A to diagnostics in run B
     # to obtain a list of triples (a, b, confidence).
     diff = compareResults(resultsA, resultsB)
 
 """
 
 import os
 import plistlib
-import CmpRuns
 
 # Information about analysis run:
 # path - the analysis output directory
 # root - the name of the root directory, which will be disregarded when
 # determining the source file name
 class SingleRunInfo:
     def __init__(self, path, root="", verboseLog=None):
         self.path = path

cfe/trunk/utils/analyzer/SATestAdd.py

 def addNewProject(ID, BuildMode) :
     CurDir = os.path.abspath(os.curdir)
     Dir = SATestBuild.getProjectDir(ID)
     if not os.path.exists(Dir):
         print "Error: Project directory is missing: %s" % Dir
         sys.exit(-1)
 
     # Build the project.
-    SATestBuild.testProject(ID, BuildMode, IsReferenceBuild=True, Dir=Dir)
+    SATestBuild.testProject(ID, BuildMode, IsReferenceBuild=True)
 
     # Add the project ID to the project map.
     ProjectMapPath = os.path.join(CurDir, SATestBuild.ProjectMapFile)
     if os.path.exists(ProjectMapPath):
         PMapFile = open(ProjectMapPath, "r+b")
     else:
         print "Warning: Creating the Project Map file!!"
         PMapFile = open(ProjectMapPath, "w+b")

cfe/trunk/utils/analyzer/SATestBuild.py

 import sys
 import glob
 import math
 import shutil
 import time
 import plistlib
 import argparse
 from subprocess import check_call, check_output, CalledProcessError
+import multiprocessing
 
 #------------------------------------------------------------------------------
 # Helper functions.
 #------------------------------------------------------------------------------
 
-def detectCPUs():
-    """
-    Detects the number of CPUs on a system. Cribbed from pp.
-    """
-    # Linux, Unix and MacOS:
-    if hasattr(os, "sysconf"):
-        if os.sysconf_names.has_key("SC_NPROCESSORS_ONLN"):
-            # Linux & Unix:
-            ncpus = os.sysconf("SC_NPROCESSORS_ONLN")
-            if isinstance(ncpus, int) and ncpus > 0:
-                return ncpus
-        else: # OSX:
-            return int(capture(['sysctl', '-n', 'hw.ncpu']))
-    # Windows:
-    if os.environ.has_key("NUMBER_OF_PROCESSORS"):
-        ncpus = int(os.environ["NUMBER_OF_PROCESSORS"])
-        if ncpus > 0:
-            return ncpus
-    return 1 # Default
 
 def which(command, paths = None):
    """which(command, [paths]) - Look up the given command in the paths string
    (or the PATH environment variable, if unspecified)."""
 
    if paths is None:
        paths = os.environ.get('PATH','')
 
if 'CC' in os.environ:
     Clang = os.environ['CC']
 else:
     Clang = which("clang", os.environ['PATH'])
 if not Clang:
     print "Error: cannot find 'clang' in PATH"
     sys.exit(-1)
 
 # Number of jobs.
-Jobs = int(math.ceil(detectCPUs() * 0.75))
+Jobs = int(math.ceil(multiprocessing.cpu_count() * 0.75))
 
 # Project map stores info about all the "registered" projects.
 ProjectMapFile = "projectMap.csv"
 
 # Names of the project specific scripts.
 # The script that downloads the project.
 DownloadScript = "download_project.sh"
 # The script that needs to be executed before the build can start.
try:
         for Command in SBCommandFile:
             Command = Command.strip()
             if len(Command) == 0:
                 continue;
             # If using 'make', auto imply a -jX argument
             # to speed up analysis.  xcodebuild will
             # automatically use the maximum number of cores.
             if (Command.startswith("make ") or Command == "make") and \
                     "-j" not in Command:
                 Command += " -j%d" % Jobs
             SBCommand = SBPrefix + Command
             if Verbose == 1:
                 print "  Executing: %s" % (SBCommand,)
             check_call(SBCommand, cwd = SBCwd, stderr=PBuildLogFile,
                                                stdout=PBuildLogFile,
                                                shell=True)
     except:
def buildProject(Dir, SBOutputDir, ProjectBuildMode, IsReferenceBuild):
     if (os.path.exists(SBOutputDir)) :
         RmCommand = "rm -r '%s'" % SBOutputDir
         if Verbose == 1:
             print "  Executing: %s" % (RmCommand,)
             check_call(RmCommand, shell=True)
     assert(not os.path.exists(SBOutputDir))
     os.makedirs(os.path.join(SBOutputDir, LogFolderName))
 
-    # Open the log file.
-    PBuildLogFile = open(BuildLogPath, "wb+")
-
     # Build and analyze the project.
-    try:
+    with open(BuildLogPath, "wb+") as PBuildLogFile:
         if (ProjectBuildMode == 1):
             downloadAndPatch(Dir, PBuildLogFile)
             runCleanupScript(Dir, PBuildLogFile)
             runScanBuild(Dir, SBOutputDir, PBuildLogFile)
         else:
             runAnalyzePreprocessed(Dir, SBOutputDir, ProjectBuildMode)
 
-        if IsReferenceBuild :
+        if IsReferenceBuild:
             runCleanupScript(Dir, PBuildLogFile)
+            normalizeReferenceResults(Dir, SBOutputDir, ProjectBuildMode)
+
+    print "Build complete (time: %.2f). See the log for more details: %s" % \
+           ((time.time()-TBegin), BuildLogPath)
 
-            # Make the absolute paths relative in the reference results.
-            for (DirPath, Dirnames, Filenames) in os.walk(SBOutputDir):
-                for F in Filenames:
-                    if (not F.endswith('plist')):
-                        continue
-                    Plist = os.path.join(DirPath, F)
-                    Data = plistlib.readPlist(Plist)
-                    PathPrefix = Dir
-                    if (ProjectBuildMode == 1):
-                        PathPrefix = os.path.join(Dir, PatchedSourceDirName)
-                    Paths = [SourceFile[len(PathPrefix)+1:]\
-                              if SourceFile.startswith(PathPrefix)\
-                              else SourceFile for SourceFile in Data['files']]
-                    Data['files'] = Paths
-                    plistlib.writePlist(Data, Plist)
+def normalizeReferenceResults(Dir, SBOutputDir, ProjectBuildMode):
+    """
+    Make the absolute paths relative in the reference results.
+    """
 
-    finally:
-        PBuildLogFile.close()
-
-    print "Build complete (time: %.2f). See the log for more details: %s" % \
-           ((time.time()-TBegin), BuildLogPath)
-
 # A plist file is created for each call to the analyzer(each source file).
 # We are only interested on the once that have bug reports, so delete the rest.
 def CleanUpEmptyPlists(SBOutputDir):
     for F in glob.glob(SBOutputDir + "/*/*.plist"):
         P = os.path.join(SBOutputDir, F)
 
         Data = plistlib.readPlist(P)
         # Delete empty reports.
if TotalFailed == 0:
            len(Plists)
         return;
 
     # Create summary file to display when the build fails.
     SummaryPath = os.path.join(SBOutputDir, LogFolderName, FailuresSummaryFileName)
     if (Verbose > 0):
         print "  Creating the failures summary file %s" % (SummaryPath,)
 
-    SummaryLog = open(SummaryPath, "w+")
-    try:
+    with open(SummaryPath, "w+") as SummaryLog:
         SummaryLog.write("Total of %d failures discovered.\n" % (TotalFailed,))
         if TotalFailed > NumOfFailuresInSummary:
             SummaryLog.write("See the first %d below.\n"
                                                    % (NumOfFailuresInSummary,))
         # TODO: Add a line "See the results folder for more."
 
         FailuresCopied = NumOfFailuresInSummary
         Idx = 0
         for FailLogPathI in Failures:
             if Idx >= NumOfFailuresInSummary:
                 break;
             Idx += 1
             SummaryLog.write("\n-- Error #%d -----------\n" % (Idx,));
-            FailLogI = open(FailLogPathI, "r");
-            try:
+            with open(FailLogPathI, "r") as FailLogI:
                 shutil.copyfileobj(FailLogI, SummaryLog);
-            finally:
-                FailLogI.close()
-    finally:
-        SummaryLog.close()
 
     print "Error: analysis failed. See ", SummaryPath
     sys.exit(-1)
 
 # Auxiliary object to discard stdout.
 class Discarder(object):
     def write(self, text):
         pass # do nothing
for E in Extensions:
         for F in glob.glob("%s/*/*.%s" % (SBOutputDir, E)):
             P = os.path.join(SBOutputDir, F)
             RmCommand = "rm '%s'" % P
             check_call(RmCommand, shell=True)
 
     # Remove the log file. It leaks absolute path names.
     removeLogFile(SBOutputDir)
 
-def updateSVN(Mode, ProjectsMap):
+def updateSVN(Mode, PMapFile):
+    """
+    svn delete or svn add (depending on `Mode`) all folders defined in the file
+    handler `PMapFile`.
+    Commit the result to SVN.
+    """
     try:
-        ProjectsMap.seek(0)
-        for I in csv.reader(ProjectsMap):
+        for I in iterateOverProjects(PMapFile):
             ProjName = I[0]
             Path = os.path.join(ProjName, getSBOutputDirName(True))
 
             if Mode == "delete":
                 Command = "svn delete '%s'" % (Path,)
             else:
                 Command = "svn add '%s'" % (Path,)
 
try:
                             "reference results.\""
         if Verbose == 1:
             print "  Executing: %s" % (CommitCommand,)
         check_call(CommitCommand, shell=True)
     except:
         print "Error: SVN update failed."
         sys.exit(-1)
 
-def testProject(ID, ProjectBuildMode, IsReferenceBuild=False, Dir=None, Strictness = 0):
+def testProject(ID, ProjectBuildMode, IsReferenceBuild=False, Strictness = 0):
     print " \n\n--- Building project %s" % (ID,)
 
     TBegin = time.time()
 
-    if Dir is None :
-        Dir = getProjectDir(ID)
     if Verbose == 1:
         print "  Build directory: %s." % (Dir,)
 
     # Set the build results directory.
     RelOutputDir = getSBOutputDirName(IsReferenceBuild)
     SBOutputDir = os.path.join(Dir, RelOutputDir)
 
     buildProject(Dir, SBOutputDir, ProjectBuildMode, IsReferenceBuild)
 
     checkBuild(SBOutputDir)
 
-    if IsReferenceBuild == False:
-        runCmpResults(Dir, Strictness)
-    else:
+    if IsReferenceBuild:
         cleanupReferenceResults(SBOutputDir)
+    else:
+        runCmpResults(Dir, Strictness)
 
     print "Completed tests for project %s (time: %.2f)." % \
           (ID, (time.time()-TBegin))
 
 def isCommentCSVLine(Entries):
     # Treat CSV lines starting with a '#' as a comment.
     return len(Entries) > 0 and Entries[0].startswith("#")
 
-def testAll(IsReferenceBuild = False, UpdateSVN = False, Strictness = 0):
-    PMapFile = open(getProjectMapPath(), "rb")
+def projectFileHandler():
+    return open(getProjectMapPath(), "rb")
+
+def iterateOverProjects(PMapFile):
+    """
+    Iterate over all projects defined in the project file handler `PMapFile`
+    from the start.
+    """
+    PMapFile.seek(0)
     try:
-        # Validate the input.
         for I in csv.reader(PMapFile):
             if (isCommentCSVLine(I)):
                 continue
-            if (len(I) != 2) :
-                print "Error: Rows in the ProjectMapFile should have 3 entries."
-                raise Exception()
-            if (not ((I[1] == "0") | (I[1] == "1") | (I[1] == "2"))):
-                print "Error: Second entry in the ProjectMapFile should be 0" \
-                      " (single file), 1 (project), or 2(single file c++11)."
-                raise Exception()
+            yield I
+    except:
+        print "Error occurred. Premature termination."
+        raise
+
+def validateProjectFile(PMapFile):
+    """
+    Validate project file.
+    """
+    for I in iterateOverProjects(PMapFile):
+            print "Error: Rows in the ProjectMapFile should have 2 entries."
 
+def testAll(IsReferenceBuild = False, UpdateSVN = False, Strictness = 0):
+    with projectFileHandler() as PMapFile:
+        validateProjectFile(PMapFile)
+
         # When we are regenerating the reference results, we might need to
         # update svn. Remove reference results from SVN.
         if UpdateSVN == True:
             assert(IsReferenceBuild == True);
             updateSVN("delete",  PMapFile);
 
         # Test the projects.
-        PMapFile.seek(0)
-        for I in csv.reader(PMapFile):
-            if isCommentCSVLine(I):
-              continue;
-            testProject(I[0], int(I[1]), IsReferenceBuild, None, Strictness)
+        for (ProjName, ProjBuildMode) in iterateOverProjects(PMapFile):
+            testProject(ProjName, int(ProjBuildMode), IsReferenceBuild, Strictness)
 
-        # Add reference results to SVN.
+        # Re-add reference results to SVN.
         if UpdateSVN == True:
             updateSVN("add",  PMapFile);
 
-    except:
-        print "Error occurred. Premature termination."
-        raise
-    finally:
-        PMapFile.close()
-
 if __name__ == '__main__':
     # Parse command line arguments.
     Parser = argparse.ArgumentParser(description='Test the Clang Static Analyzer.')
     Parser.add_argument('--strictness', dest='strictness', type=int, default=0,
                        help='0 to fail on runtime errors, 1 to fail when the number\
                              of found bugs are different from the reference, 2 to \
                              fail on any difference from the reference. Default is 0.')
     Parser.add_argument('-r', dest='regenerate', action='store_true', default=False,