Switch sanitizer-windows bot to annotator scripts

[rnk]

Summary

This is intended to be equivalent to the old SanitizerBuilderWindows
buidlbot factory code. Next, I'll look into enabling libfuzzer testing
on this bot, and make sure it's on VS 2017.

[inglorion]

LGTM, with some suggestions in the comments.

zorg/buildbot/builders/annotated/annotated_builder.py

249 ↗

(On Diff #184165)

You could elide the None here; it's the default.

zorg/buildbot/builders/annotated/sanitizer-windows.py

48 ↗

(On Diff #184165)

You can omit this; check True for all stages is the default. I don't terribly mind making it explicit, so I'm ok if you prefer to leave it in.

[rnk]

Thanks!

zorg/buildbot/builders/annotated/sanitizer-windows.py

48 ↗

(On Diff #184165)

Ah, that I didn't know.

[rnk]

• just raise KeyError instead of using .get()

zorg/buildbot/builders/annotated/annotated_builder.py

249 ↗

(On Diff #184165)

I realized that my "report" call below doesn't actually fail the build. I'm going to just let it throw KeyError, since it indicates a programming error. I wanted to try to make it more usable, but I just don't think this code is going to last very long with the impending monorepo reorg, so let's do the simple thing.

zorg/trunk/buildbot/osuosl/master/config/builders.py

 from zorg.buildbot.builders import ClangBuilder
 from zorg.buildbot.builders import LLVMBuilder
 from zorg.buildbot.builders import PollyBuilder
 from zorg.buildbot.builders import LLDBBuilder
 from zorg.buildbot.builders import LLDBuilder
 from zorg.buildbot.builders import LLGoBuilder
 from zorg.buildbot.builders import ClangAndLLDBuilder
 from zorg.buildbot.builders import SanitizerBuilder
 from zorg.buildbot.builders import SanitizerBuilderII
-from zorg.buildbot.builders import SanitizerBuilderWindows
 from zorg.buildbot.builders import OpenMPBuilder
 from zorg.buildbot.builders import LibcxxAndAbiBuilder
 from zorg.buildbot.builders import SphinxDocsBuilder
 from zorg.buildbot.builders import ABITestsuitBuilder
 from zorg.buildbot.builders import ClangLTOBuilder3Stage
 from zorg.buildbot.builders import ClangLTOBuilder
 from zorg.buildbot.builders import UnifiedTreeBuilder
 from zorg.buildbot.builders import CUDATestsuiteBuilder
def _get_sanitizer_builders():
           {'name': "sanitizer-ppc64le-linux",
            'slavenames' :["ppc64le-sanitizer"],
            'builddir': "sanitizer-ppc64le",
            'factory': SanitizerBuilder.getSanitizerBuildFactory(timeout=1800)},
 
           {'name': "sanitizer-windows",
            'slavenames' :["sanitizer-windows"],
            'builddir': "sanitizer-windows",
-           'factory': SanitizerBuilderWindows.getSanitizerWindowsBuildFactory(
-                        vs='%VS140COMNTOOLS%')},
+           'factory': AnnotatedBuilder.getAnnotatedBuildFactory(
+             script="sanitizer-windows.py",
+             depends_on_projects=["llvm", "clang", "lld", "compiler-rt"])},
 
           ## ARMv7 check-all full (compiler-rt) with CMake builder; Needs x86 for ASAN tests
           {'name': "clang-cmake-armv7-full",
            'slavenames':["linaro-tk1-08"],
            'builddir':"clang-cmake-armv7-full",
            'factory' : ClangBuilder.getClangCMakeBuildFactory(
                         clean=False,
                         checkout_lld=False,

zorg/trunk/zorg/buildbot/builders/SanitizerBuilderWindows.py

-import os
-
-import buildbot
-import buildbot.process.factory
-from buildbot.steps.shell import SetProperty
-from buildbot.steps.shell import ShellCommand
-from buildbot.steps.slave import RemoveDirectory
-from buildbot.steps.source import SVN
-from buildbot.process.properties import Property
-from buildbot.process.properties import WithProperties
-from zorg.buildbot.builders.Util import getVisualStudioEnvironment
-from zorg.buildbot.builders.Util import extractSlaveEnvironment
-from zorg.buildbot.builders.Util import extractClangVersion
-from zorg.buildbot.commands.NinjaCommand import NinjaCommand
-
-def getSource(f,llvmTopDir='llvm'):
-    f.addStep(SVN(name='svn-llvm',
-                  mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/',
-                  defaultBranch='trunk',
-                  workdir=llvmTopDir))
-    f.addStep(SVN(name='svn-clang',
-                  mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/',
-                  defaultBranch='trunk',
-                  workdir='%s/tools/clang' % llvmTopDir))
-    f.addStep(SVN(name='svn-lld',
-                  mode='update', baseURL='http://llvm.org/svn/llvm-project/lld/',
-                  defaultBranch='trunk',
-                  workdir='%s/tools/lld' % llvmTopDir))
-    f.addStep(SVN(name='svn-compiler-rt',
-                  mode='update',
-                  baseURL='http://llvm.org/svn/llvm-project/compiler-rt/',
-                  defaultBranch='trunk',
-                  workdir='%s/projects/compiler-rt' % llvmTopDir))
-    return f
-
-def getSanitizerWindowsBuildFactory(
-            clean=False,
-            cmake='cmake',
-
-            # Default values for VS devenv and build configuration
-            vs=r"""%VS120COMNTOOLS%""",
-            config='Release',
-            target_arch='x86',
-
-            extra_cmake_args=[]):
-
-    ############# PREPARING
-    f = buildbot.process.factory.BuildFactory()
-
-    # Kill any stale symbolizer processes for the last run. If there are any
-    # stale processes, the build will fail during linking. This can happen to
-    # any process, but it is most likely to happen to llvm-symbolizer if its
-    # pipe isn't closed.
-    taskkill_cmd = 'taskkill /f /im llvm-symbolizer.exe || exit /b 0'
-    f.addStep(ShellCommand(name='taskkill',
-                           description='kill stale processes',
-                           command=['cmd', '/c', taskkill_cmd],
-                           haltOnFailure=False))
-
-    # Determine Slave Environment and Set MSVC environment.
-    f.addStep(SetProperty(
-        command=getVisualStudioEnvironment(vs, target_arch),
-        extract_fn=extractSlaveEnvironment))
-
-    f = getSource(f,'llvm')
-
-    # Global configurations
-    build_dir = 'build'
-
-    ############# CLEANING
-    cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
-    f.addStep(RemoveDirectory(name='clean '+build_dir,
-                dir=build_dir,
-                haltOnFailure=False,
-                flunkOnFailure=False,
-                doStepIf=cleanBuildRequested
-                ))
-
-    f.addStep(ShellCommand(name='cmake',
-                           command=[cmake, "-G", "Ninja", "../llvm",
-                                    "-DCMAKE_BUILD_TYPE="+config,
-                                    "-DLLVM_ENABLE_ASSERTIONS=ON"]
-                                   + extra_cmake_args,
-                           haltOnFailure=True,
-                           workdir=build_dir,
-                           env=Property('slave_env')))
-
-    # Build compiler-rt first to speed up detection of Windows-specific
-    # compiler-time errors in the sanitizer runtimes.
-    f.addStep(NinjaCommand(name='build compiler-rt',
-                           targets=['compiler-rt'],
-                           haltOnFailure=True,
-                           description='ninja compiler-rt',
-                           workdir=build_dir,
-                           env=Property('slave_env')))
-
-    # Build Clang and LLD next so that most compilation errors occur in a build
-    # step.
-    f.addStep(NinjaCommand(name='build clang lld',
-                           targets=['clang', 'lld'],
-                           haltOnFailure=True,
-                           description='ninja clang lld',
-                           workdir=build_dir,
-                           env=Property('slave_env')))
-
-    # Only run sanitizer tests.
-    # Don't build targets that are not required in order to speed up the cycle.
-    for target in ['check-asan', 'check-asan-dynamic', 'check-sanitizer',
-                   'check-cfi']:
-      f.addStep(NinjaCommand(name='run %s' % target,
-                             targets=[ target ],
-                             haltOnFailure=False,
-                             description='ninja %s' % target,
-                             workdir=build_dir,
-                             env=Property('slave_env')))
-
-    return f

zorg/trunk/zorg/buildbot/builders/annotated/annotated_builder.py

def set_environment(self, env=None, vs_tools=None, arch=None):
                 util.report('%s=%s' % (var, os.environ[var]))
         except Exception as e:
             self.report_step_exception(e)
             raise
 
     def update_sources(self, source_dir, projects, revision, svn='svn'):
         self.report_build_step('update-sources')
         self.halt_on_failure()
+
+        # TODO: This needs to be updated to use the monorepo.
+        # Where to check the project out relative to an LLVM checkout.
+        checkout_locations = {
+            'llvm': '',
+            'clang': 'tools/clang',
+            'lld': 'tools/lld',
+            'compiler-rt': 'projects/compiler-rt',
+            }
+        # If the project is named differently in svn, put it here.
+        svn_locations = { 'clang': 'cfe' }
+        svn_uri_pattern = 'https://llvm.org/svn/llvm-project/%s/trunk'
+
         try:
-            for (project, path, uri) in projects:
-                if path is None:
+            for project in projects:
+                # TODO: Fail the build and report an error if we don't know the
+                # checkout location.
+                path = checkout_locations[project]
+                if not path:
                     path = source_dir
                 elif not os.path.isabs(path):
                     path = pjoin(source_dir, path)
+                uri = svn_uri_pattern % (svn_locations.get(project, project),)
                 util.report("Updating %s to %s at %s from %s" %
                             (project, revision, util.shquote(path), uri))
                 if os.path.exists(pjoin(path, '.svn')):
                     cmd = [svn, 'up', '-r', revision]
                 else:
                     util.mkdirp(path)
                     cmd = [svn, 'co', '-r', revision, uri, '.']
                 util.report_run_cmd(cmd, cwd=path)
         except Exception as e:
             self.report_step_exception(e)
             raise
 
     def run_steps(
         self,
         stages=1,
+        projects=None,
         check_targets=None,
         check_stages=None,
         extra_cmake_args=None,
         stage1_extra_cmake_args=None,
         revision=None,
         compiler='clang',
         linker='ld.lld',
         env=None,
         jobs=None):
         """
         stages: number of stages to run (default: 1)
+        projects: which subprojects to check out from SVN
+            llvm must be first in the list (default: ['llvm', 'clang', 'lld'])
         check_targets: targets to run during the check phase (default: ['check-all'])
         check_stages: stages for which to run the check phase
             (array of bool, default: all True)
         extra_cmake_args: extra arguments to pass to cmake (default: [])
         stage1_extra_cmake_args: extra arguments to pass to cmake for stage 1
             (default: use extra_cmake_args)
         revision: revision to check out (default: os.environ['BUILDBOT_REVISION'],
             or, if that is unset, the latest revision)
def run_steps(
         if check_stages is None:
             check_stages = [True] * stages
         if extra_cmake_args is None:
             extra_cmake_args = []
         if not revision:
             revision = os.environ.get('BUILDBOT_REVISION')
         if stage1_extra_cmake_args is None:
             stage1_extra_cmake_args = extra_cmake_args
+        if projects is None:
+            projects = ['llvm', 'clang', 'lld']
 
         c_compiler, cxx_compiler = self.compiler_binaries(compiler)
 
         self.set_environment(env)
 
         # On Windows, if we're building clang-cl, make sure stage1 is built with
         # MSVC (cl.exe), and not gcc from mingw. CMake will prefer gcc if it is
         # available.
def run_steps(
                 return 1
         if not revision.isdigit():
             util.report("SVN revision %s is not a positive integer" % (revision,))
 
         # Update sources.
         cwd = os.getcwd()
         source_dir = pjoin(cwd, 'llvm.src')
         build_dir = pjoin(cwd, 'build')
-        svn_uri_pattern = 'https://llvm.org/svn/llvm-project/%s/trunk'
-        projects = []
-        projects.append(('llvm', None, svn_uri_pattern % ('llvm',)))
-        projects.append(
-            ('clang', pjoin('tools', 'clang'), svn_uri_pattern % ('cfe',)))
         cmake_args = ['-GNinja']
-        for p in ['lld']:
-            projects.append((p, pjoin('tools', p), svn_uri_pattern % (p,)))
 
         self.update_sources(source_dir, projects, revision)
 
         # Build and check stages.
         self.build_and_check_stages(
             stages,
             build_dir,
             source_dir,
def get_vcvars(vs_tools, arch):
     This code is following the guidelines from strategy 1 in this blog post:
         https://blogs.msdn.microsoft.com/vcblog/2017/03/06/finding-the-visual-c-compiler-tools-in-visual-studio-2017/
 
     It doesn't work when VS is not installed at the default location.
     """
     if not arch:
         # First check the wow64 processor architecture, since python is probably
         # 32-bit, then fall back to PROCESSOR_ARCHITECTURE.
-        arch = os.environ['PROCESSOR_ARCHITEW6432'].lower()
+        arch = os.environ.get('PROCESSOR_ARCHITEW6432', '').lower()
         if not arch:
-            arch = os.environ['PROCESSOR_ARCHITECTURE'].lower()
+            arch = os.environ.get('PROCESSOR_ARCHITECTURE', '').lower()
     else:
         arch = arch.lower()
 
     # Use vswhere.exe if it exists.
     if os.path.exists(VSWHERE_PATH):
         cmd = [VSWHERE_PATH, "-latest", "-property", "installationPath"]
         vs_path = subprocess.check_output(cmd).strip()
         util.report("Running vswhere to find VS: " + repr(cmd))

zorg/trunk/zorg/buildbot/builders/annotated/sanitizer-windows.py

+#!/usr/bin/python
+
+import os
+import sys
+import annotated_builder
+import util
+
+
+class SanitizerAnnotatedBuilder(annotated_builder.AnnotatedBuilder):
+
+    """Customizes the 'build' step of the generic AnnotatedBuilder"""
+
+    def build(self, stage_name, build_dir, jobs=None):
+        # The basic idea here is to run 'ninja compiler-rt ; ninja clang lld'.
+        # This ensures that portability issues in compiler-rt code are found
+        # first. Then, we only build clang and lld, the primary dependencies of
+        # the sanitizer test suites, to keep cycle time low. This means there
+        # are still some remaining test dependencies (FileCheck) that may be
+        # compiled during the check step, but there shouldn't be that many.
+        self.report_build_step('%s build' % (stage_name,))
+        self.halt_on_failure()
+        base_cmd = ['ninja']
+        if jobs:
+            base_cmd += ['-j', str(jobs)]
+        early_targets = ['compiler-rt']
+        late_targets = ['clang', 'lld']
+        util.report_run_cmd(base_cmd + early_targets, cwd=build_dir)
+        util.report_run_cmd(base_cmd + late_targets, cwd=build_dir)
+
+
+def main(argv):
+    ap = annotated_builder.get_argument_parser()
+    args = ap.parse_args(argv[1:])
+
+    projects = ['llvm', 'clang', 'lld', 'compiler-rt']
+    stages = 1
+    stage1_extra_cmake_args = [
+        '-DCMAKE_BUILD_TYPE=Release',
+        '-DLLVM_ENABLE_PDB=ON',
+        '-DLLVM_ENABLE_ASSERTIONS=ON',
+        '-DLLVM_TARGETS_TO_BUILD=X86',
+    ]
+    extra_cmake_args = stage1_extra_cmake_args + [
+        '-DLLVM_USE_LINKER=lld',
+    ]
+    check_targets = ['check-asan', 'check-asan-dynamic', 'check-sanitizer',
+                     'check-cfi']
+
+    # These arguments are a bit misleading, they really mean use cl.exe for
+    # stage1 instead of GCC.
+    compiler = 'clang-cl'
+    linker = 'lld-link'
+
+    builder = SanitizerAnnotatedBuilder()
+    builder.run_steps(stages=stages,
+                      projects=projects,
+                      extra_cmake_args=extra_cmake_args,
+                      stage1_extra_cmake_args=stage1_extra_cmake_args,
+                      check_targets=check_targets,
+                      compiler=compiler,
+                      linker=linker,
+                      jobs=args.jobs)
+
+
+if __name__ == '__main__':
+    sys.path.append(os.path.dirname(__file__))
+    sys.exit(main(sys.argv))