Index: zorg/trunk/buildbot/osuosl/master/config/builders.py =================================================================== --- zorg/trunk/buildbot/osuosl/master/config/builders.py +++ zorg/trunk/buildbot/osuosl/master/config/builders.py @@ -7,7 +7,6 @@ from zorg.buildbot.builders import ClangAndLLDBuilder from zorg.buildbot.builders import SanitizerBuilder from zorg.buildbot.builders import SanitizerBuilderII -from zorg.buildbot.builders import SanitizerBuilderWindows from zorg.buildbot.builders import OpenMPBuilder from zorg.buildbot.builders import LibcxxAndAbiBuilder from zorg.buildbot.builders import SphinxDocsBuilder @@ -1119,8 +1118,9 @@ {'name': "sanitizer-windows", 'slavenames' :["sanitizer-windows"], 'builddir': "sanitizer-windows", - 'factory': SanitizerBuilderWindows.getSanitizerWindowsBuildFactory( - vs='%VS140COMNTOOLS%')}, + 'factory': AnnotatedBuilder.getAnnotatedBuildFactory( + script="sanitizer-windows.py", + depends_on_projects=["llvm", "clang", "lld", "compiler-rt"])}, ## ARMv7 check-all full (compiler-rt) with CMake builder; Needs x86 for ASAN tests {'name': "clang-cmake-armv7-full", Index: zorg/trunk/zorg/buildbot/builders/SanitizerBuilderWindows.py =================================================================== --- zorg/trunk/zorg/buildbot/builders/SanitizerBuilderWindows.py +++ zorg/trunk/zorg/buildbot/builders/SanitizerBuilderWindows.py @@ -1,117 +0,0 @@ -import os - -import buildbot -import buildbot.process.factory -from buildbot.steps.shell import SetProperty -from buildbot.steps.shell import ShellCommand -from buildbot.steps.slave import RemoveDirectory -from buildbot.steps.source import SVN -from buildbot.process.properties import Property -from buildbot.process.properties import WithProperties -from zorg.buildbot.builders.Util import getVisualStudioEnvironment -from zorg.buildbot.builders.Util import extractSlaveEnvironment -from zorg.buildbot.builders.Util import extractClangVersion -from zorg.buildbot.commands.NinjaCommand import NinjaCommand - -def getSource(f,llvmTopDir='llvm'): - f.addStep(SVN(name='svn-llvm', - mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', - defaultBranch='trunk', - workdir=llvmTopDir)) - f.addStep(SVN(name='svn-clang', - mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/', - defaultBranch='trunk', - workdir='%s/tools/clang' % llvmTopDir)) - f.addStep(SVN(name='svn-lld', - mode='update', baseURL='http://llvm.org/svn/llvm-project/lld/', - defaultBranch='trunk', - workdir='%s/tools/lld' % llvmTopDir)) - f.addStep(SVN(name='svn-compiler-rt', - mode='update', - baseURL='http://llvm.org/svn/llvm-project/compiler-rt/', - defaultBranch='trunk', - workdir='%s/projects/compiler-rt' % llvmTopDir)) - return f - -def getSanitizerWindowsBuildFactory( - clean=False, - cmake='cmake', - - # Default values for VS devenv and build configuration - vs=r"""%VS120COMNTOOLS%""", - config='Release', - target_arch='x86', - - extra_cmake_args=[]): - - ############# PREPARING - f = buildbot.process.factory.BuildFactory() - - # Kill any stale symbolizer processes for the last run. If there are any - # stale processes, the build will fail during linking. This can happen to - # any process, but it is most likely to happen to llvm-symbolizer if its - # pipe isn't closed. - taskkill_cmd = 'taskkill /f /im llvm-symbolizer.exe || exit /b 0' - f.addStep(ShellCommand(name='taskkill', - description='kill stale processes', - command=['cmd', '/c', taskkill_cmd], - haltOnFailure=False)) - - # Determine Slave Environment and Set MSVC environment. - f.addStep(SetProperty( - command=getVisualStudioEnvironment(vs, target_arch), - extract_fn=extractSlaveEnvironment)) - - f = getSource(f,'llvm') - - # Global configurations - build_dir = 'build' - - ############# CLEANING - cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean - f.addStep(RemoveDirectory(name='clean '+build_dir, - dir=build_dir, - haltOnFailure=False, - flunkOnFailure=False, - doStepIf=cleanBuildRequested - )) - - f.addStep(ShellCommand(name='cmake', - command=[cmake, "-G", "Ninja", "../llvm", - "-DCMAKE_BUILD_TYPE="+config, - "-DLLVM_ENABLE_ASSERTIONS=ON"] - + extra_cmake_args, - haltOnFailure=True, - workdir=build_dir, - env=Property('slave_env'))) - - # Build compiler-rt first to speed up detection of Windows-specific - # compiler-time errors in the sanitizer runtimes. - f.addStep(NinjaCommand(name='build compiler-rt', - targets=['compiler-rt'], - haltOnFailure=True, - description='ninja compiler-rt', - workdir=build_dir, - env=Property('slave_env'))) - - # Build Clang and LLD next so that most compilation errors occur in a build - # step. - f.addStep(NinjaCommand(name='build clang lld', - targets=['clang', 'lld'], - haltOnFailure=True, - description='ninja clang lld', - workdir=build_dir, - env=Property('slave_env'))) - - # Only run sanitizer tests. - # Don't build targets that are not required in order to speed up the cycle. - for target in ['check-asan', 'check-asan-dynamic', 'check-sanitizer', - 'check-cfi']: - f.addStep(NinjaCommand(name='run %s' % target, - targets=[ target ], - haltOnFailure=False, - description='ninja %s' % target, - workdir=build_dir, - env=Property('slave_env'))) - - return f Index: zorg/trunk/zorg/buildbot/builders/annotated/annotated_builder.py =================================================================== --- zorg/trunk/zorg/buildbot/builders/annotated/annotated_builder.py +++ zorg/trunk/zorg/buildbot/builders/annotated/annotated_builder.py @@ -231,12 +231,29 @@ def update_sources(self, source_dir, projects, revision, svn='svn'): self.report_build_step('update-sources') self.halt_on_failure() + + # TODO: This needs to be updated to use the monorepo. + # Where to check the project out relative to an LLVM checkout. + checkout_locations = { + 'llvm': '', + 'clang': 'tools/clang', + 'lld': 'tools/lld', + 'compiler-rt': 'projects/compiler-rt', + } + # If the project is named differently in svn, put it here. + svn_locations = { 'clang': 'cfe' } + svn_uri_pattern = 'https://llvm.org/svn/llvm-project/%s/trunk' + try: - for (project, path, uri) in projects: - if path is None: + for project in projects: + # TODO: Fail the build and report an error if we don't know the + # checkout location. + path = checkout_locations[project] + if not path: path = source_dir elif not os.path.isabs(path): path = pjoin(source_dir, path) + uri = svn_uri_pattern % (svn_locations.get(project, project),) util.report("Updating %s to %s at %s from %s" % (project, revision, util.shquote(path), uri)) if os.path.exists(pjoin(path, '.svn')): @@ -252,6 +269,7 @@ def run_steps( self, stages=1, + projects=None, check_targets=None, check_stages=None, extra_cmake_args=None, @@ -263,6 +281,8 @@ jobs=None): """ stages: number of stages to run (default: 1) + projects: which subprojects to check out from SVN + llvm must be first in the list (default: ['llvm', 'clang', 'lld']) check_targets: targets to run during the check phase (default: ['check-all']) check_stages: stages for which to run the check phase (array of bool, default: all True) @@ -290,6 +310,8 @@ revision = os.environ.get('BUILDBOT_REVISION') if stage1_extra_cmake_args is None: stage1_extra_cmake_args = extra_cmake_args + if projects is None: + projects = ['llvm', 'clang', 'lld'] c_compiler, cxx_compiler = self.compiler_binaries(compiler) @@ -323,14 +345,7 @@ cwd = os.getcwd() source_dir = pjoin(cwd, 'llvm.src') build_dir = pjoin(cwd, 'build') - svn_uri_pattern = 'https://llvm.org/svn/llvm-project/%s/trunk' - projects = [] - projects.append(('llvm', None, svn_uri_pattern % ('llvm',))) - projects.append( - ('clang', pjoin('tools', 'clang'), svn_uri_pattern % ('cfe',))) cmake_args = ['-GNinja'] - for p in ['lld']: - projects.append((p, pjoin('tools', p), svn_uri_pattern % (p,))) self.update_sources(source_dir, projects, revision) @@ -363,9 +378,9 @@ if not arch: # First check the wow64 processor architecture, since python is probably # 32-bit, then fall back to PROCESSOR_ARCHITECTURE. - arch = os.environ['PROCESSOR_ARCHITEW6432'].lower() + arch = os.environ.get('PROCESSOR_ARCHITEW6432', '').lower() if not arch: - arch = os.environ['PROCESSOR_ARCHITECTURE'].lower() + arch = os.environ.get('PROCESSOR_ARCHITECTURE', '').lower() else: arch = arch.lower() Index: zorg/trunk/zorg/buildbot/builders/annotated/sanitizer-windows.py =================================================================== --- zorg/trunk/zorg/buildbot/builders/annotated/sanitizer-windows.py +++ zorg/trunk/zorg/buildbot/builders/annotated/sanitizer-windows.py @@ -0,0 +1,67 @@ +#!/usr/bin/python + +import os +import sys +import annotated_builder +import util + + +class SanitizerAnnotatedBuilder(annotated_builder.AnnotatedBuilder): + + """Customizes the 'build' step of the generic AnnotatedBuilder""" + + def build(self, stage_name, build_dir, jobs=None): + # The basic idea here is to run 'ninja compiler-rt ; ninja clang lld'. + # This ensures that portability issues in compiler-rt code are found + # first. Then, we only build clang and lld, the primary dependencies of + # the sanitizer test suites, to keep cycle time low. This means there + # are still some remaining test dependencies (FileCheck) that may be + # compiled during the check step, but there shouldn't be that many. + self.report_build_step('%s build' % (stage_name,)) + self.halt_on_failure() + base_cmd = ['ninja'] + if jobs: + base_cmd += ['-j', str(jobs)] + early_targets = ['compiler-rt'] + late_targets = ['clang', 'lld'] + util.report_run_cmd(base_cmd + early_targets, cwd=build_dir) + util.report_run_cmd(base_cmd + late_targets, cwd=build_dir) + + +def main(argv): + ap = annotated_builder.get_argument_parser() + args = ap.parse_args(argv[1:]) + + projects = ['llvm', 'clang', 'lld', 'compiler-rt'] + stages = 1 + stage1_extra_cmake_args = [ + '-DCMAKE_BUILD_TYPE=Release', + '-DLLVM_ENABLE_PDB=ON', + '-DLLVM_ENABLE_ASSERTIONS=ON', + '-DLLVM_TARGETS_TO_BUILD=X86', + ] + extra_cmake_args = stage1_extra_cmake_args + [ + '-DLLVM_USE_LINKER=lld', + ] + check_targets = ['check-asan', 'check-asan-dynamic', 'check-sanitizer', + 'check-cfi'] + + # These arguments are a bit misleading, they really mean use cl.exe for + # stage1 instead of GCC. + compiler = 'clang-cl' + linker = 'lld-link' + + builder = SanitizerAnnotatedBuilder() + builder.run_steps(stages=stages, + projects=projects, + extra_cmake_args=extra_cmake_args, + stage1_extra_cmake_args=stage1_extra_cmake_args, + check_targets=check_targets, + compiler=compiler, + linker=linker, + jobs=args.jobs) + + +if __name__ == '__main__': + sys.path.append(os.path.dirname(__file__)) + sys.exit(main(sys.argv))