Modify LitTestCommand class to accept optional argument to parse summary lines only

[chying]

Summary

Modify LitTestCommand class to accept optional argument to parse summary lines only

• Add optional argument parseSummaryOnly with default value False
• If the argument is not specified, the behavior will be all the same as before
• If the argument is passed as True, only the lines after "r'^Failing Tests \(\d*\)$'" will be parsed as test results
• This is supposed to address the test result display problem with ninja command

[sivachandra]

LGTM with nits.

zorg/buildbot/commands/LitTestCommand.py

23 ↗

(On Diff #20651)

May be a better comment would be:

Regular expression for start of summary marker.

24 ↗

(On Diff #20651)

s/kTestResultRE/kSummaryStartRE ?

26 ↗

(On Diff #20651)

s/isSummaryOnly/parseSummaryOnly

38 ↗

(On Diff #20651)

May be a better comment would be:

The current line is parsed only if |parseLine| is True.

39 ↗

(On Diff #20651)

s/parserStarted/parseLine

105 ↗

(On Diff #20651)

Add a comment here to explain why and when this check is required.

zorg/trunk/zorg/buildbot/builders/LLDBBuilder.py

f.addStep(WarningCountingShellCommand(name="compile/ninja",
                                           haltOnFailure=True,
                                           workdir=llvm_builddir))
     # Test
     f.addStep(LitTestCommand(name="test lldb",
                              command=['nice', '-n', '10',
                                       'ninja',
                                       'check-lldb'],
                              description="test lldb",
+                             parseSummaryOnly=True,
                              env=env,
                              workdir='%s' % llvm_builddir))
     return f
 
 def getLLDBxcodebuildFactory(use_cc=None):
     f = buildbot.process.factory.BuildFactory()
     f.addStep(SetProperty(name='get_builddir',
                           command=['pwd'],

zorg/trunk/zorg/buildbot/commands/LitTestCommand.py

class LitLogObserver(LogLineObserver):
 
   # Regular expressions for verbose log start and stop markers. Verbose log
   # output always immediately follow a test.
   kTestVerboseLogStartRE = re.compile(r"""\*{4,80} TEST '(.*)' .*""")
   kTestVerboseLogStopRE = re.compile(r"""\*{10,80}""")
 
   # These are the codes for which we will include the log output in the buildbot
   # step results.
-  failingCodes = set(['FAIL', 'XPASS', 'KPASS', 'UNRESOLVED'])
+  failingCodes = set(['FAIL', 'XPASS', 'KPASS', 'UNRESOLVED', 'TIMEOUT'])
+  # Regular expressions for start of summary marker.
+  kStartSummaryRE = re.compile(r'^Failing Tests \(\d*\)$')
 
-  def __init__(self, maxLogs=None):
+  def __init__(self, maxLogs=None, parseSummaryOnly=False):
     LogLineObserver.__init__(self)
     self.resultCounts = {}
     self.maxLogs = maxLogs
     self.numLogs = 0
 
     # If non-null, a tuple of the last test code and name.
     self.lastTestResult = None
 
     # If non-null, a list of lines in the current log.
     self.activeVerboseLog = None
+    # Current line will be parsed as result steps only if parserStarted is True
+    self.parserStarted = not parseSummaryOnly
 
   def hadFailure(self):
     for code in self.failingCodes:
       if self.resultCounts.get(code):
         return True
 
   def handleVerboseLogLine(self, line):
     # Append to the log.
def outLineReceived(self, line):
     # Otherwise, if we had any previous test consider it finished.
     #
     # FIXME: This assumes that we will always have at least one line following
     # the last test result to properly record each test; we could fix this if
     # buildbot provided us a hook for when the log is done.
     if self.lastTestResult:
       self.testInfoFinished()
 
+    if self.kStartSummaryRE.match(line):
+      self.parserStarted = True;
+
+    #Assign result line only if summary marker has been matched
+    #Or if all lines should be parsed
+    if self.parserStarted is True:
       # Check for a new test status line.
       m = self.kTestLineRE.match(line.strip())
       if m:
         # Remember the last test result and update the result counts.
         self.lastTestResult = (code, name) = m.groups()
         self.resultCounts[code] = self.resultCounts.get(code, 0) + 1
         return
 
 class LitTestCommand(Test):
   resultNames = {'FAIL':'unexpected failures',
                  'PASS':'expected passes',
                  'XFAIL':'expected failures',
                  'XPASS':'unexpected passes',
                  'KFAIL':'known failures',
                  'KPASS':'unknown passes',
                  'UNRESOLVED':'unresolved testcases',
                  'UNTESTED':'untested testcases',
                  'REGRESSED':'runtime performance regression',
                  'IMPROVED':'runtime performance improvement',
                  'UNSUPPORTED':'unsupported tests'}
 
-  def __init__(self, ignore=[], flaky=[], max_logs=20,
+  def __init__(self, ignore=[], flaky=[], max_logs=20, parseSummaryOnly=False,
                *args, **kwargs):
     Test.__init__(self, *args, **kwargs)
     self.maxLogs = int(max_logs)
-    self.logObserver = LitLogObserver(self.maxLogs)
+    self.logObserver = LitLogObserver(self.maxLogs, parseSummaryOnly)
     self.addFactoryArguments(max_logs=max_logs)
+    self.addFactoryArguments(parseSummaryOnly=parseSummaryOnly)
     self.addLogObserver('stdio', self.logObserver)
 
   def evaluateCommand(self, cmd):
     # Always report failure if the command itself failed.
     if cmd.rc != 0:
       return FAILURE
 
     # Otherwise, report failure if there were failures in the log.