Index: buildbot/osuosl/master/config/builders.py =================================================================== --- buildbot/osuosl/master/config/builders.py +++ buildbot/osuosl/master/config/builders.py @@ -14,6 +14,8 @@ from zorg.buildbot.builders import ABITestsuitBuilder from zorg.buildbot.builders import ClangLTOBuilder3Stage +from buildbot.process.properties import Property + # Plain LLVM builders. def _get_llvm_builders(): return [ @@ -804,7 +806,21 @@ "-DLLVM_DEFAULT_TARGET_TRIPLE=mipsel-unknown-linux-gnu", "-DLLVM_TARGET_ARCH=Mips"], stage1_upload_directory='clang-cmake-mipsel', - env = {'BOTO_CONFIG': '/var/buildbot/llvmlab-build-artifacts.boto'})}, + env = {'BOTO_CONFIG': '/var/buildbot/llvmlab-build-artifacts.boto'}, + trigger_after_stage1_upload=['lnt-mipsel'])}, + ] + +def _get_triggerable_builders(): + return [ + {'name': "lnt-mipsel", + 'slavenames': ["mips-kl-ci20_001"], + 'builddir': "lnt-mipsel", + 'factory': LNTBuilder.getGCSAndLNTFactory( + nt_flags=[], + env={ + 'BOTO_CONFIG': + '/var/buildbot/llvmlab-build-artifacts.boto'}, + tester_name='lnt-mipsel')} ] def _get_openmp_builders(): @@ -1165,6 +1181,10 @@ b['category'] = 'sanitizer' yield b + for b in _get_triggerable_builders(): + b['category'] = 'triggerable' + yield b + for b in _get_openmp_builders(): b['category'] = 'openmp' yield b Index: buildbot/osuosl/master/config/slaves.py =================================================================== --- buildbot/osuosl/master/config/slaves.py +++ buildbot/osuosl/master/config/slaves.py @@ -113,6 +113,9 @@ # Cavium Octeon II V0.1, MIPS64r2 little endian, Debian Jessie create_slave("mips-kl-erpro001", properties={'jobs' : 2}, max_builds=1), + # Ingenic XBurst V4.15, FPU V0.0, MIPS32r2 little endian, Debian Jessie + create_slave("mips-kl-ci20_001", properties={'jobs' : 1}, max_builds=1), + # Debian Testing x86-64, Intel(R) Core(TM) i7-2700K CPU @ 3.50GHz #create_slave("gribozavr1", properties={'jobs': 8}, max_builds=1), Index: buildbot/osuosl/master/master.cfg =================================================================== --- buildbot/osuosl/master/master.cfg +++ buildbot/osuosl/master/master.cfg @@ -224,6 +224,8 @@ "llvm", "cfe", "llgo"]))) +c['schedulers'].append( + triggerable.Triggerable(name="lnt-mipsel", builderNames=["lnt-mipsel"])) c['schedulers'].extend( config.schedulers.getSingleBranchSchedulers(c['builders'], c['schedulers']) Index: zorg/buildbot/builders/ClangBuilder.py =================================================================== --- zorg/buildbot/builders/ClangBuilder.py +++ zorg/buildbot/builders/ClangBuilder.py @@ -9,6 +9,7 @@ from buildbot.steps.shell import WarningCountingShellCommand from buildbot.steps.source import SVN from buildbot.steps.transfer import FileDownload +from buildbot.steps.trigger import Trigger import zorg.buildbot.util.artifacts as artifacts import zorg.buildbot.builders.Util as builders_util @@ -401,6 +402,8 @@ name="record GCS url for " + package_name, command=['echo', gcs_url], property=gcs_url_property)) + # Prevent a second expansion since the timestamp will change. + gcs_url = Property(gcs_url_property) f.addStep(ShellCommand(name='package ' + package_name, command=['tar', 'cvfJ', '../install.tar.xz', '.'], @@ -416,6 +419,26 @@ workdir=install_prefix, env=env)) +def addTriggerBuilderSteps(f, builder_names, gcs_url_property=None): + """ + Add steps to trigger other builders. + + builder_names - A sequence containing the builders to trigger. + gcs_url_property - The name of a property which containing the GCS URL. It + will be forwarded to the triggered builder. + """ + + set_properties = {'triggeredBuilders': builder_names} + if gcs_url_property: + set_properties[gcs_url_property] = Property(gcs_url_property) + + f.addStep( + phasedbuilderutils.NamedTrigger( + 'trigger builders', + schedulerNames=builder_names, + waitForFinish=False, + set_properties=set_properties)) + def getClangCMakeGCSBuildFactory( clean=True, test=True, @@ -451,7 +474,7 @@ stage1_upload_directory=None, # Triggers - trigger_after_stage1=[]): + trigger_after_stage1_upload=[]): return _getClangCMakeBuildFactory( clean=clean, test=test, cmake=cmake, jobs=jobs, vs=vs, vs_target_arch=vs_target_arch, useTwoStage=useTwoStage, @@ -462,7 +485,7 @@ checkout_clang_tools_extra=checkout_clang_tools_extra, checkout_compiler_rt=checkout_compiler_rt, stage1_upload_directory=stage1_upload_directory, - trigger_after_stage1=trigger_after_stage1) + trigger_after_stage1_upload=trigger_after_stage1_upload) def getClangCMakeBuildFactory( clean=True, @@ -539,7 +562,7 @@ stage1_upload_directory=None, # Triggers - trigger_after_stage1=[]): + trigger_after_stage1_upload=[]): ############# PREPARING f = buildbot.process.factory.BuildFactory() @@ -640,6 +663,10 @@ addGCSUploadSteps(f, 'stage 1', stage1_install, stage1_upload_directory, env, gcs_url_property='stage1_package_gcs_url') + if trigger_after_stage1_upload: + addTriggerBuilderSteps(f, trigger_after_stage1_upload, + gcs_url_property='stage1_package_gcs_url') + # Compute the cmake define flag to set the C and C++ compiler to clang. Use # clang-cl if we used MSVC for stage1. if not vs: Index: zorg/buildbot/builders/LNTBuilder.py =================================================================== --- zorg/buildbot/builders/LNTBuilder.py +++ zorg/buildbot/builders/LNTBuilder.py @@ -6,7 +6,8 @@ import buildbot from buildbot.steps.source.svn import SVN -from buildbot.process.properties import WithProperties +from buildbot.process.properties import WithProperties, Property +from buildbot.steps.shell import ShellCommand import zorg from zorg.buildbot.builders import ClangBuilder @@ -60,6 +61,44 @@ return f +def addGCSDownloadSteps(f, gcs_url, env): + """ + Add steps to download from the Google Cloud Storage bucket. + + f - The BuildFactory to modify. + gcs_url - The GCS URL of the package to download. + env - The environment to use. Set BOTO_CONFIG to use a configuration file + in a non-standard location, and BUCKET to use a different GCS bucket. + """ + f.addStep(ShellCommand(name='download from storage bucket', + command=['gsutil', 'cp', gcs_url, + 'toolchain.tar.xz'], + description='downloading from storage bucket...', + haltOnFailure=True, + workdir='llvm.install', + env=env)) + + f.addStep(ShellCommand(name="remove old toolchain", + command=["rm", "-rf", 'install'], + haltOnFailure=True, + description=["removing old toolchain..."], + workdir=".")) + + f.addStep(ShellCommand(name="unpack toolchain", + command=['tar', 'xvfJ', 'toolchain.tar.xz'], + haltOnFailure=True, + description=["unpacking toolchain..."], + workdir="llvm.install")) + +def getGCSAndLNTFactory(nt_flags, env={}, tester_name=None): + f = buildbot.process.factory.BuildFactory() + addGCSDownloadSteps(f, Property('stage1_package_gcs_url'), env=env) + cc_path = 'llvm.install/bin/clang' + cxx_path = 'llvm.install/bin/clang++' + AddLNTTestsToFactory(f, nt_flags, cc_path=cc_path, + cxx_path=cxx_path, testerName=tester_name); + return f + def AddLNTTestsToFactory(f, nt_flags, cc_path, cxx_path, **kwargs): """ Add the buildbot steps necessary to run an LNT driven test of a compiler.