Add 'vs' parameter to Clang CMake builder to indicate Windows

[rnk]

Summary

Port the builder steps to Windows, and enable windows-gcebot1. This code
has been tested locally and should work.

[rnk]

Monday ping. :)

I spun up a test instance of the master here:
http://130.211.133.145:8000/builders/clang-x86-win2008-selfhost/

[rnk]

+Rick, since this is related to his change

[gkistanova]

LGTM
Though, I have a question. I'm not very familiar with the Windows you are running, but it seems 'env' is not a standard Windows command, is it? If so, could you add a note for the extra dependency, please?

buildbot/osuosl/master/config/builders.py

#         'factory' : ClangBuilder.getClangBuildFactory(extra_configure_args=['--enable-shared'])},
         {'name': "clang-native-mingw32-win7",
          'slavenames':["as-bldslv7"],
          'builddir':"clang-native-mingw32-win7",
          'factory' : ClangBuilder.getClangBuildFactory(triple='i686-pc-mingw32',
                                                        useTwoStage=True, test=False,
                                                        stage1_config='Release+Asserts',
                                                        stage2_config='Release+Asserts')},
 
-#        # This will ultimately be a self-host bot, even though the config does
-#        # not reflect that today.
-#        {'name': 'clang-x86-win2008-selfhost',
-#         'slavenames': ['windows-gcebot1'],
-#         'builddir': 'clang-x86-win2008-selfhost',
-#         'factory' : ClangBuilder.getClangCMakeBuildFactory(
-#                        triple='i686-pc-windows-msvc',
-#                        clean=False,
-#                        checkout_compiler_rt=False,
-#                        testStage1=True,
-#                        stage1_config='Release',
-#                        extra_cmake_args=["-DLLVM_ENABLE_ASSERTIONS=ON"])},
+        # This will ultimately be a self-host bot, even though the config does
+        # not reflect that today.
+        {'name': 'clang-x86-win2008-selfhost',
+         'slavenames': ['windows-gcebot1'],
+         'builddir': 'clang-x86-win2008-selfhost',
+         'factory' : ClangBuilder.getClangCMakeBuildFactory(
+                        clean=False,
+                        vs='%VS120COMNTOOLS%',
+                        vs_target_arch='x86',
+                        checkout_compiler_rt=False,
+                        testStage1=False,
+                        useTwoStage=True,
+                        stage1_config='Release',
+                        stage2_config='Release',
+                        extra_cmake_args=["-DLLVM_ENABLE_ASSERTIONS=ON"])},
 
         {'name' : "clang-ppc64-elf-linux",
          'slavenames' :["chinook-clangslave1"],
          'builddir' :"clang-ppc64-1",
          'factory' : LNTBuilder.getLNTFactory(triple='ppc64-elf-linux1',
                                               nt_flags=['--multisample=3','--cflag','-mcpu=native'],
                                               jobs=2,  use_pty_in_tests=True,
                                               testerName='O3-plain', run_cxx_tests=True)},

zorg/buildbot/builders/ClangBuilder.py

 import buildbot
 import buildbot.process.factory
 import os
 
-from buildbot.process.properties import WithProperties
+from buildbot.process.properties import WithProperties, Property
 from buildbot.steps.shell import Configure, ShellCommand, SetProperty
 from buildbot.steps.shell import WarningCountingShellCommand
 from buildbot.steps.source import SVN
 from buildbot.steps.transfer import FileDownload
 
 import zorg.buildbot.util.artifacts as artifacts
 import zorg.buildbot.builders.Util as builders_util
 import zorg.buildbot.util.phasedbuilderutils as phasedbuilderutils
if package_dst:
                                workdir=".",
                                warnOnFailure=True,
                                flunkOnFailure=False,
                                haltOnFailure=False,
                                env=merged_env))
 
     return f
 
-# CMake Linux builds
+def addSVNUpdateSteps(f, checkout_clang_tools_extra=True, checkout_compiler_rt=True):
+    # We *must* checkout at least Clang+LLVM
+    f.addStep(SVN(name='svn-llvm',
+                  mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/',
+                  defaultBranch='trunk',
+                  workdir='llvm'))
+    f.addStep(SVN(name='svn-clang',
+                  mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/',
+                  defaultBranch='trunk',
+                  workdir='llvm/tools/clang'))
+
+    # Extra stuff that will be built/tested
+    if checkout_clang_tools_extra:
+        f.addStep(SVN(name='svn-clang-tools-extra',
+                      mode='update', baseURL='http://llvm.org/svn/llvm-project/clang-tools-extra/',
+                      defaultBranch='trunk',
+                      workdir='llvm/tools/clang/tools/extra'))
+    if checkout_compiler_rt:
+        f.addStep(SVN(name='svn-compiler-rt',
+                      mode='update', baseURL='http://llvm.org/svn/llvm-project/compiler-rt/',
+                      defaultBranch='trunk',
+                      workdir='llvm/projects/compiler-rt'))
+
+
 def getClangCMakeBuildFactory(
             clean=True,
             test=True,
             cmake='cmake',
             jobs=None,
 
+            # VS tools environment variable if using MSVC. For example,
+            # %VS120COMNTOOLS% selects the 2013 toolchain.
+            vs=None,
+            vs_target_arch='x86',
+
             # Multi-stage compilation
             useTwoStage=False,
             testStage1=True,
             stage1_config='Release',
             stage2_config='Release',
 
             # Environmental variables for all steps.
             env={},
             extra_cmake_args=[],
 
             # Extra repositories
             checkout_clang_tools_extra=True,
             checkout_compiler_rt=True):
 
     ############# PREPARING
     f = buildbot.process.factory.BuildFactory()
 
-    # We *must* checkout at least Clang+LLVM
-    f.addStep(SVN(name='svn-llvm',
-                  mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/',
-                  defaultBranch='trunk',
-                  workdir='llvm'))
-    f.addStep(SVN(name='svn-clang',
-                  mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/',
-                  defaultBranch='trunk',
-                  workdir='llvm/tools/clang'))
-
-    # Extra stuff that will be built/tested
-    if checkout_clang_tools_extra:
-        f.addStep(SVN(name='svn-clang-tools-extra',
-                      mode='update', baseURL='http://llvm.org/svn/llvm-project/clang-tools-extra/',
-                      defaultBranch='trunk',
-                      workdir='llvm/tools/clang/tools/extra'))
-    if checkout_compiler_rt:
-        f.addStep(SVN(name='svn-compiler-rt',
-                      mode='update', baseURL='http://llvm.org/svn/llvm-project/compiler-rt/',
-                      defaultBranch='trunk',
-                      workdir='llvm/projects/compiler-rt'))
+    addSVNUpdateSteps(f, checkout_clang_tools_extra=checkout_clang_tools_extra,
+                      checkout_compiler_rt=checkout_compiler_rt)
 
     # If jobs not defined, Ninja will choose a suitable value
     jobs_cmd=[]
     lit_args="'-v"
     if jobs is not None:
         jobs_cmd=["-j"+str(jobs)]
         lit_args+=" -j"+str(jobs)+"'"
     else:
         lit_args+="'"
     ninja_cmd=['ninja'] + jobs_cmd
     ninja_install_cmd=['ninja', 'install'] + jobs_cmd
     ninja_check_cmd=['ninja', 'check-all'] + jobs_cmd
+    check_build_cmd = ["sh", "-c",
+                       "test -e build.ninja && echo OK || echo Missing"]
+    if vs:
+        check_build_cmd = ["cmd", "/c", "if exist build.ninja (echo OK) " +
+                           " else (echo Missing & exit 1)"]
 
     # Global configurations
     stage1_build='stage1'
     stage1_install='stage1.install'
     stage2_build='stage2'
 
+    # Set up VS environment, if appropriate.
+    if vs:
+        f.addStep(SetProperty(
+            command=builders_util.getVisualStudioEnvironment(vs, vs_target_arch),
+            extract_fn=builders_util.extractSlaveEnvironment))
+        assert not env, "Can't have custom builder env vars with VS"
+        env = Property('slave_env')
+
+
     ############# CLEANING
     if clean:
         f.addStep(ShellCommand(name='clean stage 1',
                                command=['rm','-rf',stage1_build],
                                warnOnFailure=True,
                                description='cleaning stage 1',
                                descriptionDone='clean',
                                workdir='.',
                                env=env))
     else:
         f.addStep(SetProperty(name="check ninja files 1",
                               workdir=stage1_build,
-                              command=["sh", "-c",
-                                       "test -e build.ninja && echo OK || echo Missing"],
+                              command=check_build_cmd,
                               flunkOnFailure=False,
                               property="exists_ninja_1"))
 
 
     ############# STAGE 1
     f.addStep(ShellCommand(name='cmake stage 1',
                            command=[cmake, "-G", "Ninja", "../llvm",
                                     "-DCMAKE_BUILD_TYPE="+stage1_config,
if clean:
                                warnOnFailure=True,
                                description='cleaning stage 2',
                                descriptionDone='clean',
                                workdir='.',
                                env=env))
     else:
         f.addStep(SetProperty(name="check ninja files 2",
                               workdir=stage2_build,
-                              command=["sh", "-c",
-                                       "test -e build.ninja && echo OK || echo Missing"],
+                              command=check_build_cmd,
                               flunkOnFailure=False,
                               property="exists_ninja_2"))
 
- 
-    f.addStep(ShellCommand(name='cmake stage 2',
-                           command=[cmake, "-G", "Ninja", "../llvm",
-                                    "-DCMAKE_BUILD_TYPE="+stage2_config,
-                                    "-DLLVM_ENABLE_ASSERTIONS=True",
-                                    WithProperties("-DCMAKE_C_COMPILER=%(workdir)s/"+stage1_install+"/bin/clang"),
-                                    WithProperties("-DCMAKE_CXX_COMPILER=%(workdir)s/"+stage1_install+"/bin/clang++"),
-                                    "-DLLVM_LIT_ARGS="+lit_args]
-                                    + extra_cmake_args,
+    # Compute the cmake define flag to set the C and C++ compiler to clang. Use
+    # clang-cl if we used MSVC for stage1.
+    if not vs:
+        cc = 'clang'
+        cxx = 'clang++'
+    else:
+        cc = 'clang-cl.exe'
+        cxx = 'clang-cl.exe'
+    # Set the compiler using the CC and CXX environment variables to work
+    # around backslash string escaping bugs somewhere between buildbot and
+    # cmake.
+    cmake_cmd2 = ['env',
+                  WithProperties('CC=%(workdir)s/'+stage1_install+'/bin/'+cc),
+                  WithProperties('CXX=%(workdir)s/'+stage1_install+'/bin/'+cxx),
+                  cmake, "-G", "Ninja", "../llvm",
+                  "-DLLVM_LIT_ARGS="+lit_args] + extra_cmake_args
+
+    f.addStep(ShellCommand(name='cmake stage 2',
+                           command=cmake_cmd2,
                            haltOnFailure=True,
                            description='cmake stage 2',
                            workdir=stage2_build,
                            doStepIf=lambda step: step.build.getProperty("exists_ninja_2") != "OK",
                            env=env))
- 
     f.addStep(WarningCountingShellCommand(name='build stage 2',
                                           command=ninja_cmd,
                                           haltOnFailure=True,
                                           description='ninja all',
                                           workdir=stage2_build,
                                           env=env))
- 
     if test:
         f.addStep(lit_test_command.LitTestCommand(name='ninja check 2',
                                    command=ninja_check_cmd,
                                    description=["checking stage 2"],
                                    descriptionDone=["stage 2 checked"],
                                    workdir=stage2_build,
                                    env=env))