diff --git a/.gitignore b/.gitignore --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,8 @@ *~ *.pyc +*.swp +*.orig +twistd.log* lnt/tests/*/Output lnt/tests/*/*/Output lnt/docs/_build diff --git a/buildbot/osuosl/master/config/builders.py b/buildbot/osuosl/master/config/builders.py --- a/buildbot/osuosl/master/config/builders.py +++ b/buildbot/osuosl/master/config/builders.py @@ -2459,18 +2459,24 @@ "-DLLVM_TARGETS_TO_BUILD=X86", "-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=ARC", ])}, +] - +all = [ # BOLT builders managed by Meta {'name' : 'bolt-x86_64-ubuntu-nfc', 'tags' : ["bolt"], 'collapseRequests': False, 'workernames' : ['bolt-worker'], 'builddir': "bolt-x86_64-ubuntu-nfc", - 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( - script="bolt-nfc.py", - depends_on_projects=['llvm', 'bolt'], - warnOnWarnings=True, + 'factory' : BOLTBuilder.getBOLTCmakeBuildFactory( + bolttests=True, + extra_configure_args=[ + "-DLLVM_APPEND_VC_REV=OFF", + "-DLLVM_CCACHE_BUILD=ON", + "-DLLVM_ENABLE_PROJECTS=clang;lld;bolt", + "-DLLVM_TARGETS_TO_BUILD=X86;AArch64", + ], + is_nfc=True, )}, {'name': "bolt-x86_64-ubuntu-clang-bolt-lto-pgo", diff --git a/buildbot/osuosl/master/config/release_builders.py b/buildbot/osuosl/master/config/release_builders.py --- a/buildbot/osuosl/master/config/release_builders.py +++ b/buildbot/osuosl/master/config/release_builders.py @@ -21,8 +21,9 @@ # Release builders. -all = [ +all = [] +''' # Clang builders. {'name' : "llvm-clang-x86_64-win-release", @@ -231,3 +232,4 @@ })}, ] +''' diff --git a/buildbot/osuosl/master/config/status.py b/buildbot/osuosl/master/config/status.py --- a/buildbot/osuosl/master/config/status.py +++ b/buildbot/osuosl/master/config/status.py @@ -7,7 +7,8 @@ # Should be a single e-mail address status_email = str(config.options.get('Master Options', 'status_email')).split(',') -all = [ +all = [] +''' # Report github status for all the release builders, # i.e. those with the "release" tag. @@ -332,6 +333,7 @@ mode = "warnings", builders = ["bolt-x86_64-ubuntu-nfc"]), ] +''' from twisted.python import log diff --git a/buildbot/osuosl/master/config/workers.py b/buildbot/osuosl/master/config/workers.py --- a/buildbot/osuosl/master/config/workers.py +++ b/buildbot/osuosl/master/config/workers.py @@ -8,6 +8,7 @@ password = config.options.get('Worker Passwords', name) return worker.Worker(name, password=password, *args, **kwargs) +''' def get_all(): return [ # FreeBSD @@ -312,4 +313,9 @@ # CSKY T-HEAD workers create_worker("thead-clang-csky", properties={'jobs' : 32}, max_builds=1), +''' +def get_all(): + return [ + # BOLT worker + create_worker("bolt-worker", properties={'jobs' : 16, 'loadaverage': 16}, max_builds=1), ] diff --git a/buildbot/osuosl/master/master.cfg b/buildbot/osuosl/master/master.cfg --- a/buildbot/osuosl/master/master.cfg +++ b/buildbot/osuosl/master/master.cfg @@ -47,7 +47,7 @@ c['change_source'].append( LLVMPoller(repourl='https://github.com/llvm/llvm-project.git', workdir='gitpoller-workdir', - pollinterval=120)) + pollinterval=60*5)) c['change_source'].append( changes.GitPoller(repourl='https://github.com/llvm/llvm-lnt.git', project='lnt', @@ -85,7 +85,7 @@ # has a variety to choose from, like IRC bots. # Configure MailNotifier, IRC, buildbot.reporters.github.GitHubStatusPush, buildbot.reporters.github.GitHubCommentPush -c['services'] = config.status.getReporters() +#c['services'] = config.status.getReporters() ####### PROJECT IDENTITY @@ -97,8 +97,8 @@ c['www'] = dict(port=8011, plugins=dict(waterfall_view={}, console_view={}, grid_view={}), # TODO: badges default_page='console', - auth=config.auth.getAuth(), - authz=config.auth.getAuthz(), + #auth=config.auth.getAuth(), + #authz=config.auth.getAuthz(), #logRotateLength= #maxRotatedFiles= #versions= diff --git a/zorg/buildbot/builders/BOLTBuilder.py b/zorg/buildbot/builders/BOLTBuilder.py --- a/zorg/buildbot/builders/BOLTBuilder.py +++ b/zorg/buildbot/builders/BOLTBuilder.py @@ -1,11 +1,15 @@ -from buildbot.plugins import steps -from zorg.buildbot.commands.CmakeCommand import CmakeCommand +from buildbot.plugins import steps, util +from buildbot.steps.shell import ShellCommand from zorg.buildbot.builders.UnifiedTreeBuilder import getLLVMBuildFactoryAndSourcecodeSteps, addCmakeSteps, addNinjaSteps +from zorg.buildbot.commands.LitTestCommand import LitTestCommand +from zorg.buildbot.commands.CmakeCommand import CmakeCommand +from zorg.buildbot.conditions.FileConditions import FileExists from zorg.buildbot.process.factory import LLVMBuildFactory def getBOLTCmakeBuildFactory( clean = False, bolttests = False, + is_nfc = False, targets = None, checks = None, cache = None, @@ -79,4 +83,49 @@ env=env, **kwargs) + if is_nfc: + f.addSteps([ + ShellCommand( + name='nfc-check-setup', + command=[f"../{f.monorepo_dir}/bolt/utils/nfc-check-setup.py"], + description=('Setup NFC testing'), + warnOnFailure=True, + haltOnFailure=False, + flunkOnFailure=False, + env=env), + ShellCommand( + name='check-bolt-different', + command='rm -f .llvm-bolt.diff; cmp -s bin/llvm-bolt{.old,.new} || touch .llvm-bolt.diff', + description=('Check if llvm-bolt binaries are different'), + haltOnFailure=False, + flunkOnFailure=False, + env=env), + LitTestCommand( + name='nfc-check-bolt', + command=['bin/llvm-lit', '-sv', '-j4', + # bolt-info will always mismatch in NFC mode + '--xfail=bolt-info.test', + # FIXME[aaupov]: https://github.com/llvm/llvm-project/issues/59008 + '--filter-out=X86/bb-with-two-tail-calls.s', + 'tools/bolt/test'], + description=["running", "NFC", "check-bolt"], + descriptionDone=["NFC", "check-bolt", "completed"], + warnOnFailure=True, + haltOnFailure=False, + flunkOnFailure=False, + doStepIf=FileExists('.llvm-bolt.diff'), + env=env), + LitTestCommand( + name='nfc-check-large-bolt', + command=['bin/llvm-lit', '-sv', '-j2', + 'tools/bolttests'], + description=["running", "NFC", "check-large-bolt"], + descriptionDone=["NFC", "check-large-bolt", "completed"], + warnOnFailure=True, + haltOnFailure=False, + flunkOnFailure=False, + doStepIf=FileExists('.llvm-bolt.diff'), + env=env), + ]) + return f diff --git a/zorg/buildbot/builders/annotated/bolt-nfc.py b/zorg/buildbot/builders/annotated/bolt-nfc.py deleted file mode 100644 --- a/zorg/buildbot/builders/annotated/bolt-nfc.py +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/python3 - -import os -import subprocess -import sys -import traceback -import util -from contextlib import contextmanager - -def main(): - source_dir = os.path.join('..', 'llvm-project') - tests_dir = os.path.join('..', 'bolt-tests') - - with step('fetch large-bolt-tests'): - if os.path.exists(tests_dir): - run_command(['git', '-C', tests_dir, 'fetch', 'origin']) - run_command(['git', '-C', tests_dir, 'reset', '--hard', - 'origin/main']) - else: - run_command(['git', 'clone', - 'https://github.com/rafaelauler/bolt-tests', - tests_dir]) - - with step('cmake'): - cmake_args = ['-GNinja', - '-DCMAKE_BUILD_TYPE=Release', - '-DLLVM_APPEND_VC_REV=OFF', - '-DLLVM_CCACHE_BUILD=ON', - '-DLLVM_ENABLE_ASSERTIONS=ON', - '-DLLVM_ENABLE_LLD=ON', - '-DLLVM_ENABLE_PROJECTS=clang;lld;bolt', - '-DLLVM_TARGETS_TO_BUILD=X86;AArch64', - '-DLLVM_EXTERNAL_PROJECTS=bolttests', - '-DLLVM_EXTERNAL_BOLTTESTS_SOURCE_DIR='+tests_dir, - ] - - util.clean_dir('.') - run_command(['cmake', os.path.join(source_dir, 'llvm')] + cmake_args) - - with step('build bolt'): - run_command(['ninja', 'bolt']) - - with step('check-bolt'): - run_command(['ninja', 'check-bolt']) - - with step('check-large-bolt'): - run_command(['ninja', 'check-large-bolt']) - - with step('nfc-check-setup'): - run_command([os.path.join(source_dir, 'bolt', 'utils', - 'nfc-check-setup.py')]) - - with step('nfc-check-bolt', warn_on_fail=True, halt_on_fail=False): - run_command([os.path.join('bin', 'llvm-lit'), '-sv', '-j2', - # bolt-info will always mismatch in NFC mode - '--xfail=bolt-info.test', - # FIXME[aaupov]: https://github.com/llvm/llvm-project/issues/59008 - '--filter-out=X86/bb-with-two-tail-calls.s', - 'tools/bolt/test']) - - with step('nfc-check-large-bolt', warn_on_fail=True, halt_on_fail=False): - run_command([os.path.join('bin', 'llvm-lit'), '-sv', '-j2', - 'tools/bolttests']) - - -@contextmanager -def step(step_name, warn_on_fail=False, halt_on_fail=True): - util.report('@@@BUILD_STEP {}@@@'.format(step_name)) - if halt_on_fail: - util.report('@@@HALT_ON_FAILURE@@@') - try: - yield - except Exception as e: - if isinstance(e, subprocess.CalledProcessError): - util.report( - '{} exited with return code {}.'.format(e.cmd, e.returncode) - ) - util.report('The build step threw an exception...') - traceback.print_exc() - - if warn_on_fail: - util.report('@@@STEP_WARNINGS@@@') - else: - util.report('@@@STEP_FAILURE@@@') - finally: - sys.stdout.flush() - - -def run_command(cmd, directory='.'): - util.report_run_cmd(cmd, cwd=directory) - - -if __name__ == '__main__': - sys.path.append(os.path.dirname(__file__)) - sys.exit(main()) diff --git a/zorg/buildbot/commands/AnnotatedCommand.py b/zorg/buildbot/commands/AnnotatedCommand.py --- a/zorg/buildbot/commands/AnnotatedCommand.py +++ b/zorg/buildbot/commands/AnnotatedCommand.py @@ -119,7 +119,7 @@ yield self.master.data.updates.finishStep(self.stepid, self.results, hidden) # finish unfinished logs - yield self.finishUnfinishedLogs() + yield self._cleanup_logs() @defer.inlineCallbacks def startStep(self, remote, done=False): @@ -131,7 +131,7 @@ yield self.addStep() try: - self.realUpdateSummary() + self.updateSummary() # The "main" step has finished already. # Do only necessary work. #NOTE: we can get 'request finish' flag before the step started. @@ -159,8 +159,8 @@ finally: # update the summary one last time, make sure that completes, # and then don't update it any more. - self.realUpdateSummary() - yield self.realUpdateSummary.stop() + self.updateSummary() + yield self.updateSummary.stop() # Update step status in the database. yield self.finishStep() diff --git a/zorg/buildbot/commands/LitTestCommand.py b/zorg/buildbot/commands/LitTestCommand.py --- a/zorg/buildbot/commands/LitTestCommand.py +++ b/zorg/buildbot/commands/LitTestCommand.py @@ -161,8 +161,8 @@ super().__init__(*args, **kwargs) self.maxLogs = int(max_logs) self.logObserver = LitLogObserver(self.maxLogs, parseSummaryOnly) - self.addFactoryArguments(max_logs=max_logs) - self.addFactoryArguments(parseSummaryOnly=parseSummaryOnly) + #self.addFactoryArguments(max_logs=max_logs) + #self.addFactoryArguments(parseSummaryOnly=parseSummaryOnly) self.addLogObserver('stdio', self.logObserver) def evaluateCommand(self, cmd): diff --git a/zorg/buildbot/util/phasedbuilderutils.py b/zorg/buildbot/util/phasedbuilderutils.py --- a/zorg/buildbot/util/phasedbuilderutils.py +++ b/zorg/buildbot/util/phasedbuilderutils.py @@ -20,8 +20,8 @@ Trigger.__init__(self, **kwargs) self.name = name self.triggeredBuilders = triggeredBuilders - self.addFactoryArguments(name = name, - triggeredBuilders = triggeredBuilders) + #self.addFactoryArguments(name = name, + # triggeredBuilders = triggeredBuilders) def start(self): # Add a log linking to the triggered builders, if supplied. if self.triggeredBuilders: