Index: buildbot/osuosl/master/config/builders.py =================================================================== --- buildbot/osuosl/master/config/builders.py +++ buildbot/osuosl/master/config/builders.py @@ -19,6 +19,7 @@ from zorg.buildbot.builders import AOSPBuilder from zorg.buildbot.builders import AnnotatedBuilder from zorg.buildbot.builders import LLDPerformanceTestsuite +from zorg.buildbot.builders import FuchsiaBuilder # Plain LLVM builders. def _get_llvm_builders(): @@ -1513,6 +1514,16 @@ ] +# Toolchain builders. +def _get_toolchain_builders(): + return [ + {'name': "fuchsia-x86_64-linux", + 'slavenames' :["fuchsia-x86_64-linux-1", "fuchsia-x86_64-linux-2"], + 'builddir': "fuchsia-x86_64-linux", + 'factory': FuchsiaBuilder.getToolchainBuildFactory() + 'category': 'fuchsia'}, + ] + # Experimental and stopped builders def _get_on_demand_builders(): return [ @@ -1678,6 +1689,10 @@ b['category'] = 'libcxx' yield b + for b in _get_toolchain_builders(): + b['category'] = 'toolchain' + yield b + for b in _get_documentation_builders(): yield b Index: buildbot/osuosl/master/config/slaves.py =================================================================== --- buildbot/osuosl/master/config/slaves.py +++ buildbot/osuosl/master/config/slaves.py @@ -255,6 +255,10 @@ # FreeBSD 11 amd64 create_slave("freebsd11-amd64", properties={'jobs': 2}, max_builds=1), + # Debian 9.0 x86_64 64-core GCE instances + create_slave("fuchsia-x86_64-linux-1", properties={'jobs': 64}, max_builds=1), + create_slave("fuchsia-x86_64-linux-2", properties={'jobs': 64}, max_builds=1), + # Defunct. # Debian 86_64, 2 x 6-core Opteron 2.6 GHz #create_slave("osu8", properties={'jobs' : 6}, max_builds=2), Index: buildbot/osuosl/master/config/status.py =================================================================== --- buildbot/osuosl/master/config/status.py +++ buildbot/osuosl/master/config/status.py @@ -264,4 +264,12 @@ builders = ["lldb-x64-windows-ninja"], addLogs=False, num_lines = 15), + InformativeMailNotifier( + fromaddr = "llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers= False, + extraRecipients = ["phosek@google.com"], + subject="Build %(builder)s Failure", + mode = "failing", + builders = ["fuchsia-x86_64-linux"] + addLogs=False), ] Index: buildbot/osuosl/master/master.cfg =================================================================== --- buildbot/osuosl/master/master.cfg +++ buildbot/osuosl/master/master.cfg @@ -247,6 +247,18 @@ config.schedulers.getSingleBranchSchedulers(c['builders'], c['schedulers']) ) +c['schedulers'].append(SingleBranchScheduler(name="fuchsia_scheduler", + treeStableTimer=2*60, + builderNames=get_all_for("fuchsia"), + change_filter=depends_on([ + "llvm", + "cfe", + "lld", + "compiler-rt", + "libcxx", + "libcxxabi", + "libunwind"]))) + ####### PROJECT IDENTITY c['title'] = "LLVM" Index: zorg/buildbot/builders/FuchsiaBuilder.py =================================================================== --- /dev/null +++ zorg/buildbot/builders/FuchsiaBuilder.py @@ -0,0 +1,151 @@ +import buildbot +import buildbot.process.factory +import copy +import os +import platform + +from buildbot.process.properties import WithProperties, Property +from buildbot.steps.shell import ShellCommand, SetProperty +from buildbot.steps.source import Git + +import zorg.buildbot.commands as commands +from zorg.buildbot.commands.CmakeCommand import CmakeCommand +from zorg.buildbot.commands.NinjaCommand import NinjaCommand +from zorg.buildbot.conditions.FileConditions import FileExists, FileDoesNotExist + +def getToolchainBuildFactory( + clean=False, + test=True, + env=None, # Environmental variables for all steps. + extra_configure_args=None): + # Prepare environmental variables. Set here all env we want everywhere. + merged_env = { + "TERM" : "dumb", # Make sure Clang doesn't use color escape sequences. + } + if env is not None: + # Overwrite pre-set items with the given ones, so user can set anything. + merged_env.update(env) + + src_dir = "llvm.src" + obj_dir = "llvm.obj" + install_dir = "llvm.install" + + f = buildbot.process.factory.BuildFactory() + + # Get Fuchsia SDK. + sdk_dir = "fuchsia.sdk" + sdk_platform = { + "Linux": "linux-amd64", + "Darwin": "mac-amd64", + }[platform.system()] + sdk_version = "latest" + + sdk_url = WithProperties( + "https://chrome-infra-packages.appspot.com/dl/" + "fuchsia/sdk/%(sdk_platform)s/+/%(sdk_version)s", + sdk_platform=lambda _: sdk_platform, + sdk_version=lambda _: sdk_version) + + f.addStep(RemoveDirectory(name="clean-sdk", + dir=sdk_dir, + haltOnFailure=True, + description=["remove", "fuchsia sdk"], + doStepIf=FileExists(sdk_dir))) + + f.addStep(ShellCommand(name="fetch-sdk", + command=["curl", "-SLf", "-o", "sdk.cipd", sdk_url], + description=["download", "fuchsia sdk"], + workdir=sdk_dir)) + + f.addStep(ShellCommand(name="extract-sdk", + command=["unzip", "sdk.cipd"] + description=["extract", "fuchsia sdk"], + workdir=sdk_dir)) + + cleanCheckoutRequested = lambda step: step.build.getProperty("clean", default=False) or clean + + # Clean up llvm sources. + f.addStep(RemoveDirectory(name="clean-llvm.src", + dir=src_dir, + haltOnFailure=True, + description=["rm", "src", "dir"], + doStepIf=cleanCheckoutRequested)) + + # Get sources. + f.addStep(Git(repourl="https://github.com/llvm-git-prototype/llvm", + mode="full", + workdir=src_dir)) + + cleanBuildRequested = lambda step: step.build.getProperty("clean", default=step.build.getProperty("clean_obj")) or clean + + # Clean up llvm build. + f.addStep(RemoveDirectory(name="clean-llvm.obj", + dir=obj_dir, + haltOnFailure=True, + description=["rm", "build", "dir"], + doStepIf=cleanBuildRequested)) + + # Configure. + if extra_configure_args is None: + cmake_options = [] + else: + cmake_options = extra_configure_args[:] + + # Some options are required for this stage no matter what. + CmakeCommand.applyRequiredOptions(cmake_options, [ + ("-G", "Ninja"), + ("-DLLVM_ENABLE_PROJECTS=", "clang;lld"), + ("-DLLVM_ENABLE_RUNTIMES=", "compiler-rt;libcxx;libcxxabi;libunwind"), + ]) + + # Set proper defaults. + CmakeCommand.applyDefaultOptions(cmake_options, [ + ("-DBOOTSTRAP_LLVM_ENABLE_LTO=", "OFF"), + ("-DLLVM_ENABLE_LTO=", "OFF"), + ]) + + cmake_options.append( + WithProperties( + "-DCMAKE_INSTALL_PREFIX=%(workdir)s/" + install_dir, + "-DFUCHSIA_SDK=%(workdir)s/" + sdk_dir, + )) + + CmakeCommand.applyRequiredOptions(cmake_options, [ + ("-C", "../" + src_dir + "clang/cmake/caches/Fuchsia.cmake"), + ]) + + f.addStep(CmakeCommand(name="cmake-configure", + options=cmake_options, + path='../' + src_dir, + haltOnFailure=True, + description=["configure"], + workdir=obj_dir, + env=merged_env, + doStepIf=FileDoesNotExist("CMakeCache.txt"))) + + # Build distribution. + f.addStep(NinjaCommand(name="ninja-build", + targets=["stage2-distribution"], + haltOnFailure=True, + description=["build"], + workdir=obj_dir, + env=merged_env)) + + # Test llvm, clang and lld. + f.addStep(NinjaCommand(name="check", + targets=["stage2-check-%s" % p for p in ("llvm", "clang", "lld")], + haltOnFailure=True, + description=["check"], + workdir=obj_dir, + env=merged_env + doStepIf=test)) + + # Install distribution. + f.addStep(NinjaCommand(name="install", + targets=["stage2-install-distribution"], + haltOnFailure=True, + description=["install"], + workdir=obj_dir, + env=merged_env)) + + return f