Skip to content

Commit

Permalink
[Analyzer Tests] Run static analyzer integration tests until the end,
Browse files Browse the repository at this point in the history
Do not stop at the first failure.

Differential Revision: https://reviews.llvm.org/D38589

llvm-svn: 314992
George Karpenkov committed Oct 5, 2017
1 parent 9f3e88a commit 1b51cbd
Showing 1 changed file with 20 additions and 8 deletions.
28 changes: 20 additions & 8 deletions clang/utils/analyzer/SATestBuild.py
Original file line number Diff line number Diff line change
@@ -570,7 +570,11 @@ def runCmpResults(Dir, Strictness=0):
0 - success if there are no crashes or analyzer failure.
1 - success if there are no difference in the number of reported bugs.
2 - success if all the bug reports are identical.
:return success: Whether tests pass according to the Strictness
criteria.
"""
TestsPassed = True
TBegin = time.time()

RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName)
@@ -586,8 +590,6 @@ def runCmpResults(Dir, Strictness=0):
RefList.remove(RefLogDir)
NewList.remove(os.path.join(NewDir, LogFolderName))

if len(RefList) == 0 or len(NewList) == 0:
return False
assert(len(RefList) == len(NewList))

# There might be more then one folder underneath - one per each scan-build
@@ -624,15 +626,15 @@ def runCmpResults(Dir, Strictness=0):
(NumDiffs, DiffsPath,)
if Strictness >= 2 and NumDiffs > 0:
print "Error: Diffs found in strict mode (2)."
sys.exit(-1)
TestsPassed = False
elif Strictness >= 1 and ReportsInRef != ReportsInNew:
print "Error: The number of results are different in "\
"strict mode (1)."
sys.exit(-1)
TestsPassed = False

print "Diagnostic comparison complete (time: %.2f)." % (
time.time() - TBegin)
return (NumDiffs > 0)
return TestsPassed


def cleanupReferenceResults(SBOutputDir):
@@ -686,6 +688,11 @@ def updateSVN(Mode, PMapFile):


def testProject(ID, ProjectBuildMode, IsReferenceBuild=False, Strictness=0):
"""
Test a given project.
:return TestsPassed: Whether tests have passed according
to the :param Strictness: criteria.
"""
print " \n\n--- Building project %s" % (ID,)

TBegin = time.time()
@@ -704,11 +711,13 @@ def testProject(ID, ProjectBuildMode, IsReferenceBuild=False, Strictness=0):

if IsReferenceBuild:
cleanupReferenceResults(SBOutputDir)
TestsPassed = True
else:
runCmpResults(Dir, Strictness)
TestsPassed = runCmpResults(Dir, Strictness)

print "Completed tests for project %s (time: %.2f)." % \
(ID, (time.time() - TBegin))
return TestsPassed


def isCommentCSVLine(Entries):
@@ -749,6 +758,7 @@ def validateProjectFile(PMapFile):


def testAll(IsReferenceBuild=False, UpdateSVN=False, Strictness=0):
TestsPassed = True
with projectFileHandler() as PMapFile:
validateProjectFile(PMapFile)

@@ -760,7 +770,7 @@ def testAll(IsReferenceBuild=False, UpdateSVN=False, Strictness=0):

# Test the projects.
for (ProjName, ProjBuildMode) in iterateOverProjects(PMapFile):
testProject(
TestsPassed &= testProject(
ProjName, int(ProjBuildMode), IsReferenceBuild, Strictness)

# Re-add reference results to SVN.
@@ -793,4 +803,6 @@ def testAll(IsReferenceBuild=False, UpdateSVN=False, Strictness=0):
IsReference = True
UpdateSVN = True

testAll(IsReference, UpdateSVN, Strictness)
TestsPassed = testAll(IsReference, UpdateSVN, Strictness)
if not TestsPassed:
sys.exit(-1)

0 comments on commit 1b51cbd

Please sign in to comment.