[BOLT] Enable clang-bolt for host-gcc-built Clang

[Amir]

Summary

Follow-up to D136410 and D137357.

[gkistanova]

Thanks, Amir.

The patch itself looks good.

The bolt-worker seems overloaded, though. It hosts 6 builders, only one build is allowed at any given moment of time. The current waiting queue for each builder is about 2 hours, even if builds themselves are fast. Adding one more builder to the same worker will make it worse.

[Amir]

In D137678#3925207, @gkistanova wrote:

Thanks, Amir.

The patch itself looks good.

The bolt-worker seems overloaded, though. It hosts 6 builders, only one build is allowed at any given moment of time. The current waiting queue for each builder is about 2 hours, even if builds themselves are fast. Adding one more builder to the same worker will make it worse.

bolt-x86_64-ubuntu is going away, but I agree that waiting time is quite high now, largely driven by msvc-wine builder. We'll discuss within the team if we want to keep it.

[gkistanova]

LGTM.

[Amir]

Don't collapse requests, run check-clang

[Amir]

Remove unrelated changes

[gkistanova]

Thanks, Amir!

buildbot/osuosl/master/config/builders.py

 
 from zorg.buildbot.builders import HtmlDocsBuilder
 from zorg.buildbot.builders import DoxygenDocsBuilder
 
 reload(HtmlDocsBuilder)
 reload(DoxygenDocsBuilder)
 
 
+'''
 all = [
 
 # Clang fast builders.
 
     {'name' : "clang-x86_64-debian-fast",
     'tags'  : ["clang", "fast"],
     'collapseRequests': False,
     'workernames':["gribozavr4"],
'factory': UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory(
         extra_configure_args=[
              "-DCMAKE_EXPORT_COMPILE_COMMANDS:BOOL=ON",
              "-DLLVM_ENABLE_ASSERTIONS:BOOL=ON",
              "-DLLVM_TOOL_CLANG_TOOLS_EXTRA_BUILD=0",
              "-DLLVM_ENABLE_LIBPFM=OFF",
              "-DLLVM_TARGETS_TO_BUILD=X86",
              "-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=ARC",
          ])},
+]
+'''
 
-
+all = [
     # BOLT builders managed by Meta
     {'name' : 'bolt-x86_64-ubuntu-nfc',
     'tags'  : ["bolt"],
     'collapseRequests': False,
     'workernames' : ['bolt-worker'],
     'builddir': "bolt-x86_64-ubuntu-nfc",
     'factory' : BOLTBuilder.getBOLTCmakeBuildFactory(
                     bolttests=True,
                     extra_configure_args=[
                         "-DLLVM_APPEND_VC_REV=OFF",
                         "-DLLVM_CCACHE_BUILD=ON",
                         "-DLLVM_ENABLE_PROJECTS=clang;lld;bolt",
                         "-DLLVM_TARGETS_TO_BUILD=X86;AArch64",
                         ],
                     is_nfc=True,
                     )},
 
+    {'name': "bolt-x86_64-ubuntu-clang-bolt-gcc",
+    'tags': ["bolt"],
+    'workernames':["bolt-worker"],
+    'builddir': "bolt-x86_64-ubuntu-clang-bolt-gcc",
+    'factory' : BOLTBuilder.getBOLTCmakeBuildFactory(
+                    bolttests=False,
+                    depends_on_projects=['bolt', 'clang', 'llvm'],
+                    cache='clang/cmake/caches/BOLT.cmake',
+                    targets=['clang++-bolt'],
+                    checks=['check-clang'],
+                    extra_configure_args=[
+                        "-DCMAKE_C_COMPILER=gcc",
+                        "-DCMAKE_CXX_COMPILER=g++",
+                        "-DLLVM_APPEND_VC_REV=OFF",
+                        "-DLLVM_CCACHE_BUILD=ON",
+                        ],
+                    )},
+
+
     {'name': "bolt-x86_64-ubuntu-clang-bolt-lto-pgo",
     'tags': ["bolt"],
     'workernames':["bolt-worker"],
     'builddir': "bolt-x86_64-ubuntu-clang-bolt-lto-pgo",
     'factory' : BOLTBuilder.getBOLTCmakeBuildFactory(
                     bolttests=False,
                     depends_on_projects=['bolt', 'clang', 'lld', 'llvm'],
                     cache='clang/cmake/caches/BOLT-PGO.cmake',
                     targets=['stage2-clang++-bolt'],
                     extra_configure_args=[
                         "-DLLVM_APPEND_VC_REV=OFF",
                         "-DLLVM_ENABLE_LLD=ON",
                         "-DBOOTSTRAP_LLVM_ENABLE_LLD=ON",
                         "-DBOOTSTRAP_BOOTSTRAP_LLVM_ENABLE_LLD=ON",
                         "-DLLVM_CCACHE_BUILD=ON",
                         "-DBOOTSTRAP_LLVM_CCACHE_BUILD=ON",
                         "-DPGO_INSTRUMENT_LTO=Thin",
                         ],
                     )},
+]
+'''
 
     {'name': "bolt-x86_64-ubuntu-dylib",
     'tags': ["bolt"],
     'workernames':["bolt-worker"],
     'builddir': "bolt-x86_64-ubuntu-dylib",
     'factory' : BOLTBuilder.getBOLTCmakeBuildFactory(
                     bolttests=False,
                     depends_on_projects=['bolt', 'llvm'],
'factory' : UnifiedTreeBuilder.getCmakeWithNinjaWithMSVCBuildFactory(
                     extra_configure_args=[
                         "-DLLVM_TARGETS_TO_BUILD=X86;NVPTX",
                         "-DLLVM_DEFAULT_TARGET_TRIPLE=nvptx64-nvidia-cuda",
                         "-DLLVM_ENABLE_ASSERTIONS=ON",
                         "-DLLVM_LIT_ARGS=-vv --threads=32",
                         "-DLLVM_OPTIMIZED_TABLEGEN=ON"])},
 
 ]
+'''

buildbot/osuosl/master/config/release_builders.py

 from zorg.buildbot.builders import AnnotatedBuilder
 from zorg.buildbot.builders import LLDPerformanceTestsuite
 from zorg.buildbot.builders import FuchsiaBuilder
 from zorg.buildbot.builders import XToolchainBuilder
 
 
 # Release builders.
 
-all = [
+all = []
 
+'''
 # Clang builders.
 
     {'name' : "llvm-clang-x86_64-win-release",
     'tags'  : ["clang", "release"],
     'workernames' : ["as-builder-3"],
     'builddir': "llvm-clang-x86_64-win-rel",
     'factory' : UnifiedTreeBuilder.getCmakeWithNinjaWithMSVCBuildFactory(
                     vs="autodetect",
'factory' : OpenMPBuilder.getOpenMPCMakeBuildFactory(
                         '-DLLVM_CCACHE_BUILD=ON',
                     ],
                     env={
                         'PATH':'/home/llvmbb/bin/clang-latest/bin:/home/llvmbb/bin:/usr/local/bin:/usr/local/bin:/usr/bin:/bin',
                         'CC': 'clang', 'CXX': 'clang++',
                     })},
 
 ]
+'''

buildbot/osuosl/master/config/status.py

 from buildbot.process.properties import Interpolate
 from buildbot.plugins import reporters
 
 import config
 from zorg.buildbot.util.InformativeMailNotifier import LLVMInformativeMailNotifier
 
 # Should be a single e-mail address
 status_email = str(config.options.get('Master Options', 'status_email')).split(',')
 
-all = [
+all = []
+'''
 
     # Report github status for all the release builders,
     # i.e. those with the "release" tag.
     reporters.GitHubStatusPush(
         str(config.options.get('GitHub Status', 'token')),
         context = Interpolate("%(prop:buildername)s"),
         verbose = True, # TODO: Turn off the verbosity once this is working reliably.
         builders = [
'''
     reporters.MailNotifier(
         fromaddr = "[e-mail]",
         sendToInterestedUsers = False,
         extraRecipients = ["[e-mail]", "[e-mail]"],
         subject = "Build %(builder)s Failure",
         mode = "failing",
         builders = ["clang-loongarch64-linux"]),
 ]
+'''
 
 
 from twisted.python import log
 
 # Returns a list of Status Targets. The results of each build will be
 # pushed to these targets. buildbot.plugins reporters has a variety
 # to choose from, including email senders, and IRC bots.
 def getReporters():
     if config.options.getboolean('Master Options', 'is_production'):
         log.msg(">>> getReporters: Production mode. All reporters are registered.")
         return all
     else:
         # Staging buildbot does not report issues.
         log.msg(">>> getReporters: Staging buildbot does not report issues.")
         return []

buildbot/osuosl/master/config/workers.py

 # TODO: Rename workers with "slave" as a part of the name.
 # TODO: Refactor to define a list of tuples, each of which describes a worker, then enumerate and create as data driven.
 
 from buildbot.plugins import worker
 import config
 
 def create_worker(name, *args, **kwargs):
     password = config.options.get('Worker Passwords', name)
     return worker.Worker(name, password=password, *args, **kwargs)
 
+'''
 def get_all():
     return [
         # FreeBSD
         create_worker("as-worker-4", properties={'jobs' : 24}, max_builds=2),
 
         # Linux Ubuntu
         create_worker("as-worker-5", properties={'jobs' : 16}),
 
return [
         create_worker("bolt-worker", max_builds=1),
         create_worker("bolt-worker-aarch64", max_builds=1),
 
         # Fedora worker
         create_worker("standalone-build-x86_64", max_builds=1),
 
         # CSKY T-HEAD workers
         create_worker("thead-clang-csky", properties={'jobs' : 32}, max_builds=1),
+'''
+def get_all():
+    return [
+        # BOLT worker
+        create_worker("bolt-worker", properties={'jobs' : 16, 'loadaverage': 16}, max_builds=1),
         ]

buildbot/osuosl/master/master.cfg

 
 ####### BUILDBOT SERVICES
 
 # 'services' is a list of BuildbotService items like reporter targets. The
 # status of each build will be pushed to these targets. buildbot/reporters/*.py
 # has a variety to choose from, like IRC bots.
 
 # Configure MailNotifier, IRC, buildbot.reporters.github.GitHubStatusPush, buildbot.reporters.github.GitHubCommentPush
-c['services'] = config.status.getReporters()
+#c['services'] = config.status.getReporters()
 
 ####### PROJECT IDENTITY
 
 c['title'] = config.options.get('Buildbot', 'title')
 c['titleURL'] = config.options.get('Buildbot', 'title_url')
 c['buildbotURL'] = config.options.get('Buildbot', 'my_url')
 
 # minimalistic config to activate new web UI
 c['www'] = dict(port=8011,
                 plugins=dict(waterfall_view={}, console_view={}, grid_view={}), # TODO: badges
                 default_page='console',
-                auth=config.auth.getAuth(),
-                authz=config.auth.getAuthz(),
+                #auth=config.auth.getAuth(),
+                #authz=config.auth.getAuthz(),
                 #logRotateLength=
                 #maxRotatedFiles=
                 #versions=
             )
 
 ####### DB URL
 
 c['db'] = {

zorg/buildbot/commands/LitTestCommand.py

resultNames = {'FAIL':'unexpected failures',
                  'TIMEOUT':'timeout waiting for results',
                  'NOEXE':'test executable is missing'}
 
   def __init__(self, ignore=[], flaky=[], max_logs=20, parseSummaryOnly=False,
                *args, **kwargs):
     super().__init__(*args, **kwargs)
     self.maxLogs = int(max_logs)
     self.logObserver = LitLogObserver(self.maxLogs, parseSummaryOnly)
-    self.addFactoryArguments(max_logs=max_logs)
-    self.addFactoryArguments(parseSummaryOnly=parseSummaryOnly)
+    #self.addFactoryArguments(max_logs=max_logs)
+    #self.addFactoryArguments(parseSummaryOnly=parseSummaryOnly)
     self.addLogObserver('stdio', self.logObserver)
 
   def evaluateCommand(self, cmd):
     # Always report failure if the command itself failed.
     if cmd.rc != 0:
       return FAILURE
 
     # Otherwise, report failure if there were failures in the log.

zorg/buildbot/util/phasedbuilderutils.py

 
 class NamedTrigger(Trigger):
     """Trigger subclass which allows overriding the trigger name, and also
     allows attaching a log to link to the triggered builds."""
     def __init__(self, name, triggeredBuilders = [], **kwargs):
         Trigger.__init__(self, **kwargs)
         self.name = name
         self.triggeredBuilders = triggeredBuilders
-        self.addFactoryArguments(name = name,
-                                 triggeredBuilders = triggeredBuilders)
+        #self.addFactoryArguments(name = name,
+        #                         triggeredBuilders = triggeredBuilders)
     def start(self):
         # Add a log linking to the triggered builders, if supplied.
         if self.triggeredBuilders:
             logText = StringIO.StringIO()
             for builder in self.triggeredBuilders:
                 print >>logText, ('<b><a href="../../../../../../%s">%s'
                                   '</a></b><br>' % (builder, builder))
             self.addHTMLLog('triggered builds', str(logText.getvalue()))