diff --git a/zorg/buildbot/commands/NinjaCommand.py b/zorg/buildbot/commands/NinjaCommand.py --- a/zorg/buildbot/commands/NinjaCommand.py +++ b/zorg/buildbot/commands/NinjaCommand.py @@ -3,6 +3,7 @@ from buildbot.process.properties import WithProperties from buildbot.steps.shell import WarningCountingShellCommand +from buildbot.process.logobserver import LineConsumerLogObserver class NinjaCommand(WarningCountingShellCommand): DEFAULT_NINJA = 'ninja' @@ -35,19 +36,16 @@ 'ninja', ) - def __init__(self, options=None, targets=None, ninja=DEFAULT_NINJA, logObserver=None, **kwargs): + def __init__(self, options=None, targets=None, ninja=DEFAULT_NINJA, logObserver=None, maxLogs=20, **kwargs): self.ninja = ninja self.targets = targets + self.maxLogs = maxLogs if options is None: self.options = list() else: self.options = list(options) - if logObserver: - self.logObserver = logObserver - self.addLogObserver('stdio', self.logObserver) - j_opt = re.compile(r'^-j$|^-j\d+$') l_opt = re.compile(r'^-l$|^-l\d+(\.(\d+)?)?$') @@ -93,6 +91,74 @@ # And upcall to let the base class do its work super().__init__(**sanitized_kwargs) + self.logObserver = logObserver or LineConsumerLogObserver(self.parseErrorLogs) + self.addLogObserver('stdio', self.logObserver) + + + # FIXME: The regex depends on NINJA_STATUS to be "%e [%u/%r/%f] " which is only a default set by setupEnvironment. + kTitleLineRE = re.compile(r'(?P[0-9]+\.[0-9]+) \[(?P[0-9]+)\/(?P[0-9]+)\/(?P[0-9]+)\] (?P.+)') + kFailedLineRE = re.compile(r'FAILED: (?P.+)') + + def parseErrorLogs(self): + last_title = None + last_error = None + last_lines = [] + numLogs = 0 + normalLogs = [] + + + def endOfLog(): + nonlocal last_title,last_error,last_lines,numLogs,normalLogs + if last_title is not None: + if last_error is not None: + if numLogs < self.maxLogs: + self.addCompleteLog('FAIL: ' + last_title, '\n'.join(last_lines)) + numLogs += 1 + elif last_lines: + normalLogs.append((last_title, '\n'.join(last_lines))) + last_title = None + last_error = None + last_lines = [] + + try: + stream, line = yield + while True: + title = self.kTitleLineRE.fullmatch(line) + if title and stream=='o': + endOfLog() + secs = float(title.group('secs')) + remaining = int(title.group('remaining')) + active = int(title.group('active')) + completed = int(title.group('completed')) + desc = title.group('desc') + + last_title = desc + self.setProgress('edges', completed) + stream, line = yield + + failheader = self.kFailedLineRE.fullmatch(line) + if failheader and stream=='o': + cmdline = failheader.group('cmdline') + last_error = cmdline + last_lines.append(line) + stream, line = yield + continue + + if stream != 'h': + prefix = '' + if stream == 'o': + prefix = '[stdout] ' + elif stream == 'e': + prefix = '[stderr] ' + last_lines.append(prefix + line) + stream, line = yield + except GeneratorExit: + endOfLog() + if numLogs == 0: # Only emit normal logs if no failures found + for title,content in normalLogs[:self.maxLogs]: + self.addCompleteLog(title, content) + + def setupEnvironment(self, cmd): # First upcall to get everything prepared. super().setupEnvironment(cmd)