Replace clang-x86-windows-msvc2015 with an x64 VS 2017 build script

[rnk]

Summary

Use the annotated buildbot steps that the Windows ThinLTO bot uses, and
make them find and prefer VS 2017 by default.

This will rename the builder to clang-x64-windows-msvc.

Once the master restarts, we'll be able to more test and make changes to
the buildbot script. I tested it locally, and it does a two-stage,
64-bit build with VS 2017 as the stage1 compiler.

[rnk]

Thoughts?

[zturner]

I don't really know how any of this stuff works, but do we need the LLVM_ENABLE_PDB=ON?

[rnk]

In D51746#1229744, @zturner wrote:

I don't really know how any of this stuff works, but do we need the LLVM_ENABLE_PDB=ON?

It mirrors our existing configuration which uses RelWithDebInfo, and it helps us find bugs in our handling of debug info. Of course, nothing checks that the info we write is any good, but we do find crash and assert bugs in LLVM's codeview code this way.

[inglorion]

LGTM

zorg/trunk/buildbot/osuosl/master/config/builders.py

return [
                       test=True,
                       useTwoStage=False,
                       runTestSuite=True,
                       testsuite_flags=['--cppflags', '-O0',
                                        '--threads=96', '--build-threads=96'],
                       extra_cmake_args=["-DLLVM_TARGETS_TO_BUILD='ARM;AArch64'"],
                )},
 
-        {'name': 'clang-x86-windows-msvc2015',
+        {'name': 'clang-x64-windows-msvc',
          'slavenames': ['windows-gcebot2'],
-         'builddir': 'clang-x86-windows-msvc2015',
-         'factory' : ClangBuilder.getClangCMakeBuildFactory(
-                        clean=False,
-                        vs='%VS140COMNTOOLS%',
-                        vs_target_arch='x86',
-                        checkout_compiler_rt=False,
-                        checkout_lld=False,
-                        testStage1=True,
-                        useTwoStage=True,
-                        stage1_config='RelWithDebInfo',
-                        stage2_config='RelWithDebInfo',
-                        extra_cmake_args=["-DLLVM_ENABLE_ASSERTIONS=ON"])},
+         'builddir': 'clang-x64-windows-msvc',
+         'factory' : AnnotatedBuilder.getAnnotatedBuildFactory(
+             script="clang-windows.py",
+             depends_on_projects=['llvm', 'clang', 'lld'])},
 
         {'name' : "clang-ppc64be-linux-lnt",
          'slavenames' :["ppc64be-clang-lnt-test"],
          'builddir' :"clang-ppc64be-lnt",
          'factory' : ClangBuilder.getClangCMakeBuildFactory(clean=False,
                                                             checkout_lld=False,
                                                             useTwoStage=False,
                                                             runTestSuite=True,

zorg/trunk/zorg/buildbot/builders/annotated/annotated_builder.py

 from __future__ import print_function
 
 import util
 
 import argparse
 import errno
 import os
 import re
 import shutil
 import subprocess
 import sys
 
 from os.path import join as pjoin
 
+VSWHERE_PATH = "C:/Program Files (x86)/Microsoft Visual Studio/Installer/vswhere.exe"
+
 def get_argument_parser(*args, **kwargs):
     ap = argparse.ArgumentParser(*args, **kwargs)
     ap.add_argument('--jobs', help='Number of concurrent jobs to run')
     return ap
 
 
 class AnnotatedBuilder:
 
class AnnotatedBuilder:
 
     def set_environment(self, env=None, vs_tools=None, arch=None):
         self.report_build_step('set-environment')
         try:
             new_env = {
                 'TERM': 'dumb',
             }
             if os.name == 'nt':
-                if vs_tools is None:
-                    vs_tools = os.path.expandvars('%VS140COMNTOOLS%')
-                if arch is None:
-                    arch = os.environ['PROCESSOR_ARCHITECTURE'].lower()
-                else:
-                    arch = arch.lower()
-                vcvars_path = pjoin(
-                    vs_tools, '..', '..', 'VC', 'vcvarsall.bat')
-                cmd = util.shquote_cmd([vcvars_path, arch]) + ' && set'
-                output = subprocess.check_output(cmd, shell=True)
-                for line in output.splitlines():
-                    var, val = line.split('=', 1)
-                    new_env[var] = val
+                new_env.update(get_vcvars(vs_tools, arch))
 
             if env is not None:
                 new_env.epdate(env)
 
             for (var, val) in new_env.items():
                 os.environ[var] = val
 
             for var in sorted(os.environ.keys()):
def run_steps(
             revision = os.environ.get('BUILDBOT_REVISION')
         if stage1_extra_cmake_args is None:
             stage1_extra_cmake_args = extra_cmake_args
 
         c_compiler, cxx_compiler = self.compiler_binaries(compiler)
 
         self.set_environment(env)
 
+        # On Windows, if we're building clang-cl, make sure stage1 is built with
+        # MSVC (cl.exe), and not gcc from mingw. CMake will prefer gcc if it is
+        # available.
+        if c_compiler == 'clang-cl':
+            stage1_extra_cmake_args += ['-DCMAKE_C_COMPILER=cl',
+                                        '-DCMAKE_CXX_COMPILER=cl']
+
         # Update sources.
         cwd = os.getcwd()
         source_dir = pjoin(cwd, 'llvm.src')
         build_dir = pjoin(cwd, 'build')
         svn_uri_pattern = 'http://llvm.org/svn/llvm-project/%s/trunk'
         projects = []
         projects.append(('llvm', None, svn_uri_pattern % ('llvm',)))
         projects.append(
def run_steps(
             check_stages,
             check_targets,
             stage1_extra_cmake_args,
             jobs)
 
         return 0
 
 
+def get_vcvars(vs_tools, arch):
+    """Get the VC tools environment using vswhere.exe from VS 2017
+
+    This code is following the guidelines from strategy 1 in this blog post:
+        https://blogs.msdn.microsoft.com/vcblog/2017/03/06/finding-the-visual-c-compiler-tools-in-visual-studio-2017/
+
+    It doesn't work when VS is not installed at the default location.
+    """
+    if not arch:
+        arch = os.environ['PROCESSOR_ARCHITECTURE'].lower()
+    else:
+        arch = arch.lower()
+
+    # Use vswhere.exe if it exists.
+    if os.path.exists(VSWHERE_PATH):
+        vs_path = subprocess.check_output([VSWHERE_PATH, "-latest", "-property",
+                                          "installationPath"]).strip()
+        if not os.path.isdir(vs_path):
+            raise ValueError("VS install path does not exist: " + vs_path)
+        vcvars_path = pjoin(vs_path, 'VC', 'Auxiliary', 'Build',
+                            'vcvarsall.bat')
+    elif vs_tools is None:
+        vs_tools = os.path.expandvars('%VS140COMNTOOLS%')
+        vcvars_path = pjoin(vs_tools, '..', '..', 'VC', 'vcvarsall.bat')
+
+    # Newer vcvarsall.bat scripts aren't quiet, so direct them to NUL, aka
+    # Windows /dev/null.
+    cmd = util.shquote_cmd([vcvars_path, arch]) + ' > NUL && set'
+    output = subprocess.check_output(cmd, shell=True)
+    new_env = {}
+    for line in output.splitlines():
+        var, val = line.split('=', 1)
+        new_env[var] = val
+    return new_env
+
+
 def main(argv):
     ap = get_argument_parser()
     args = ap.parse_args(argv[1:])
     builder = AnnotatedBuilder()
     builder.run_steps(jobs=args.jobs)
 
 
 if __name__ == '__main__':
     sys.exit(main(sys.argv))

zorg/trunk/zorg/buildbot/builders/annotated/clang-windows.py

+#!/usr/bin/python
+
+import os
+import sys
+import annotated_builder
+
+def main(argv):
+    ap = annotated_builder.get_argument_parser()
+    args = ap.parse_args(argv[1:])
+
+    stages = 2
+    stage1_extra_cmake_args = [
+        '-DCMAKE_BUILD_TYPE=Release',
+        '-DLLVM_ENABLE_PDB=ON',
+        '-DLLVM_ENABLE_ASSERTIONS=ON',
+        '-DLLVM_TARGETS_TO_BUILD=all',
+    ]
+    extra_cmake_args = stage1_extra_cmake_args + [
+        '-DLLVM_USE_LINKER=lld',
+    ]
+    check_targets = ['check-llvm', 'check-clang', 'check-lld']
+
+    # Check both stage 1 and stage 2.
+    check_stages = [True] * stages
+
+    compiler = 'clang-cl'
+    linker = 'lld-link'
+
+    builder = annotated_builder.AnnotatedBuilder()
+    builder.run_steps(stages=stages,
+                      extra_cmake_args=extra_cmake_args,
+                      stage1_extra_cmake_args=stage1_extra_cmake_args,
+                      check_targets=check_targets,
+                      check_stages=check_stages,
+                      compiler=compiler,
+                      linker=linker,
+                      jobs=args.jobs)
+
+
+if __name__ == '__main__':
+    sys.path.append(os.path.dirname(__file__))
+    sys.exit(main(sys.argv))