Index: utils/analyzer/SATestBuild.py =================================================================== --- utils/analyzer/SATestBuild.py +++ utils/analyzer/SATestBuild.py @@ -46,6 +46,7 @@ import os import csv +from collections import namedtuple import sys import glob import math @@ -570,7 +571,11 @@ 0 - success if there are no crashes or analyzer failure. 1 - success if there are no difference in the number of reported bugs. 2 - success if all the bug reports are identical. + + :return success: Whether tests pass according to the Strictness + criteria. """ + TestsPassed = True TBegin = time.time() RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName) @@ -586,8 +591,6 @@ RefList.remove(RefLogDir) NewList.remove(os.path.join(NewDir, LogFolderName)) - if len(RefList) == 0 or len(NewList) == 0: - return False assert(len(RefList) == len(NewList)) # There might be more then one folder underneath - one per each scan-build @@ -624,15 +627,15 @@ (NumDiffs, DiffsPath,) if Strictness >= 2 and NumDiffs > 0: print "Error: Diffs found in strict mode (2)." - sys.exit(-1) + TestsPassed = False elif Strictness >= 1 and ReportsInRef != ReportsInNew: print "Error: The number of results are different in "\ "strict mode (1)." - sys.exit(-1) + TestsPassed = False print "Diagnostic comparison complete (time: %.2f)." % ( time.time() - TBegin) - return (NumDiffs > 0) + return TestsPassed def cleanupReferenceResults(SBOutputDir): @@ -686,6 +689,11 @@ def testProject(ID, ProjectBuildMode, IsReferenceBuild=False, Strictness=0): + """ + Test a given project. + :return TestsPassed: Whether tests have passed according + to the :param Strictness: criteria. + """ print " \n\n--- Building project %s" % (ID,) TBegin = time.time() @@ -704,11 +712,13 @@ if IsReferenceBuild: cleanupReferenceResults(SBOutputDir) + TestsPassed = True else: - runCmpResults(Dir, Strictness) + TestsPassed = runCmpResults(Dir, Strictness) print "Completed tests for project %s (time: %.2f)." % \ (ID, (time.time() - TBegin)) + return TestsPassed def isCommentCSVLine(Entries): @@ -749,6 +759,7 @@ def testAll(IsReferenceBuild=False, UpdateSVN=False, Strictness=0): + TestsPassed = True with projectFileHandler() as PMapFile: validateProjectFile(PMapFile) @@ -760,7 +771,7 @@ # Test the projects. for (ProjName, ProjBuildMode) in iterateOverProjects(PMapFile): - testProject( + TestsPassed &= testProject( ProjName, int(ProjBuildMode), IsReferenceBuild, Strictness) # Re-add reference results to SVN. @@ -793,4 +804,6 @@ IsReference = True UpdateSVN = True - testAll(IsReference, UpdateSVN, Strictness) + TestsPassed = testAll(IsReference, UpdateSVN, Strictness) + if not TestsPassed: + sys.exit(-1)