New MSVC 2013 builders for Clang and LLDB.

[rfoos]

Summary

Updates and parameters needed for cmake build using MSVC 2013 by default.

A new setproperties step which runs the VS script for your desired configuration, and saves the environment to slave env property.

The slave env property is then be used to set the MSVC environment for build steps reflecting what the installed MSVC compiler would do. This method preserves the environment on the buildslave making the builder compatible with a wider range of buildlsaves.

New support for 64 bit MSVC builds of llvm, clang, and LLDB. LLDB builds requiring python are also supported.

Unifies svn update to a single function.

[rfoos]

Merging this into Renato's clang cmake builder.

zorg/buildbot/builders/ClangBuilder.py

 import buildbot.process.factory
 import os
 
-from buildbot.process.properties import WithProperties
+from buildbot.process.properties import WithProperties, Property
 from buildbot.steps.shell import Configure, ShellCommand, SetProperty
 from buildbot.steps.shell import WarningCountingShellCommand
 from buildbot.steps.source import SVN
-from buildbot.steps.transfer import FileDownload
+from buildbot.steps.source.git import Git
+from buildbot.steps.slave import RemoveDirectory
 
 import zorg.buildbot.util.artifacts as artifacts
 import zorg.buildbot.builders.Util as builders_util

 import zorg.buildbot.commands.BatchFileDownload as batch_file_download
 import zorg.buildbot.commands.LitTestCommand as lit_test_command
 
+def slavenv_glob2list(rc, stdout, stderr):
+    '''Extract function for SetPropertyCommand. Loads Slave Environment
+    into a dictionary, and returns slavenv property for ShellCommands.'''
+    if not rc:
+        slavenv_dict = dict(l.strip().split('=',1) 
+            for l in stdout.split('\n') if len(l.split('=',1))==2)
+    return {'slavenv': slavenv_dict}
+
+def getClangTree(f, llvm_srcdir='.',
+        useGit=False, 
+        repo_llvm='http://llvm.org/git/',
+        branch='master'):
+    """ Get or update source tree."""
+    if useGit:
+        f.addStep(Git(name='git-llvm',
+                repourl=repo_llvm + 'llvm',
+                branch=branch,
+                alwaysUseLatest=True,
+                haltOnFailure=True,
+                workdir=llvm_srcdir))
+        f.addStep(Git(name='git-clang',
+                repourl=repo_llvm + 'clang',
+                branch=branch,
+                alwaysUseLatest=True,
+                haltOnFailure=True,
+                workdir='%s/tools/clang' % llvm_srcdir))
+        f.addStep(Git(name='git-clang-tools-extra',
+                repourl=repo_llvm + 'clang-tools-extra',
+                branch=branch,
+                alwaysUseLatest=True,
+                haltOnFailure=True,
+                workdir='%s/tools/clang/tools/extra' % llvm_srcdir))
+    else:
+        svnURL = 'http://llvm.org/svn/llvm-project/'
+        f.addStep(SVN(name='svn-llvm',
+                      mode='update', baseURL = svnURL + 'llvm/',
+                      defaultBranch='trunk',
+                      workdir=llvm_srcdir))
+        f.addStep(SVN(name='svn-clang',
+                      mode='update', baseURL = svnURL + 'cfe/',
+                      defaultBranch='trunk',
+                      workdir='%s/tools/clang' % llvm_srcdir))
+        f.addStep(SVN(name='svn-clang-tools-extra',
+                      mode='update', baseURL = svnURL + 'clang-tools-extra/',
+                      defaultBranch='trunk',
+                      workdir='%s/tools/clang/tools/extra' % llvm_srcdir))
+    return f
+
 def getClangBuildFactory(
             triple=None,
             clean=True,

 
     return f
 
-def getClangMSVCBuildFactory(update=True, clean=True, vcDrive='c', jobs=1, cmake=r"cmake"):
+def getClangMSVCBuildFactory(update=True, clean=False,
+        vcDrive='C',  # deprecated
+        cmake=r"cmake",  # deprecated
+        jobs="%(jobs)s",
+        loadaverage="%(loadaverage)s",
+        cmakeGenerator=r"Visual Studio 12",
+        vsConfiguration=r"RelWithDebInfo",
+        slavenvCmd=r""""%VS120COMNTOOLS%\vsvars32.bat" %PROCESSOR_ARCHITECTURE% & set""",
+        common_cmake_options=None,
+        common_build_options=None,
+        common_test_options=None,
+        enableTargets="X86",
+        env=None,
+        completely_clean=False,
+        useGit=False,
+        *args,**kwargs):
+
+    llvm_srcdir = "llvm.src"
+    llvm_objdir = "llvm.obj"
+
+    if loadaverage is None:
+        loadaverage=jobs
+
     f = buildbot.process.factory.BuildFactory()
 
+    # Determine Slave Environment and Set MSVC environment.
+    f.addStep(SetProperty(
+        command=slavenvCmd,
+        extract_fn=slavenv_glob2list))
+
     if update:
-        f.addStep(SVN(name='svn-llvm',
-                      mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/',
-                      defaultBranch='trunk',
-                      workdir='llvm'))
-        f.addStep(SVN(name='svn-clang',
-                      mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/',
-                      defaultBranch='trunk',
-                      workdir='llvm/tools/clang'))
-        f.addStep(SVN(name='svn-clang-tools-extra',
-                      mode='update', baseURL='http://llvm.org/svn/llvm-project/clang-tools-extra/',
-                      defaultBranch='trunk',
-                      workdir='llvm/tools/clang/tools/extra'))
-
+        f = getClangTree(f,llvm_srcdir=llvm_srcdir,useGit=useGit)
     # Full & fast clean.
     if clean:
-        f.addStep(ShellCommand(name='clean-1',
-                               command=['del','/s/q','build'],
-                               warnOnFailure=True,
-                               description='cleaning',
-                               descriptionDone='clean',
-                               workdir='llvm'))
-        f.addStep(ShellCommand(name='clean-2',
-                               command=['rmdir','/s/q','build'],
-                               warnOnFailure=True,
-                               description='cleaning',
-                               descriptionDone='clean',
-                               workdir='llvm'))
+        f.addStep(RemoveDirectory(dir=llvm_objdir))
 
+    # CMAKE command with user arguments.
+    cmakeCommand = ["cmake", "../%s" % llvm_srcdir,
+        "-G", cmakeGenerator,
+        "-DLLVM_TARGETS_TO_BUILD:=%s" % enableTargets,
+        "-DCMAKE_COLOR_MAKEFILE=OFF",
+        ]
+    # CMAKE command with defaults or user supplied options.
+    default_cmake_options = [
+        "-DLLVM_INCLUDE_EXAMPLES:=OFF",
+        "-DLLVM_LIT_ARGS:=--verbose --no-progress-bar",
+        ]
+    if common_cmake_options is not None:
+        cmakeCommand.extend(common_cmake_options)
+    else:
+        cmakeCommand.extend(default_cmake_options)
+
+    # MSBuild command with user arguments.
+    msbuildCommand = ["MSBuild","/p:Configuration=%s" % vsConfiguration ]
+    # Build with default or user supplied options.
+    default_build_options = [
+        "/maxcpucount",
+        "ALL_BUILD.vcxproj"]
+    if common_build_options is not None:
+        msbuildCommand.extend(common_build_options)
+    else:
+        msbuildCommand.extend(default_build_options)
+
+    # MSBuild command with user arguments.
+    mstestCommand = ["MSBuild","/p:Configuration=%s" % vsConfiguration ]
+    # Test with default or user supplied options.
+    default_test_options = [
+        "/maxcpucount:1",
+        "/verbosity:detailed",
+        "/consoleloggerparameters:ShowCommandLine"]
+    if common_test_options is not None:
+        mstestCommand.extend(common_test_options)
+    else:
+        mstestCommand.extend(default_test_options)
+
     # Create the project files.
-
-    # Use batch files instead of ShellCommand directly, Windows quoting is
-    # borked. FIXME: See buildbot ticket #595 and buildbot ticket #377.
-    f.addStep(batch_file_download.BatchFileDownload(name='cmakegen',
-                                command=[cmake,
-                                         "-DLLVM_TARGETS_TO_BUILD:=X86",
-                                         "-DLLVM_INCLUDE_EXAMPLES:=OFF",
-                                         "-DLLVM_INCLUDE_TESTS:=OFF",
-                                         "-DLLVM_TARGETS_TO_BUILD:=X86",
-                                         "-G",
-                                         "Visual Studio 9 2008",
-                                         ".."],
-                                workdir="llvm\\build"))
     f.addStep(ShellCommand(name='cmake',
-                           command=['cmakegen.bat'],
-                           haltOnFailure=True,
-                           description='cmake gen',
-                           workdir='llvm\\build'))
-
+                       command=cmakeCommand,
+                       env=Property('slavenv'),
+                       workdir=llvm_objdir))
     # Build it.
-    f.addStep(batch_file_download.BatchFileDownload(name='vcbuild',
-                                command=[vcDrive + r""":\Program Files\Microsoft Visual Studio 9.0\VC\VCPackages\vcbuild.exe""",
-                                         "/M%d" % jobs,
-                                         "LLVM.sln",
-                                         "Debug|Win32"],
-                                workdir="llvm\\build"))
-    f.addStep(WarningCountingShellCommand(name='vcbuild',
-                                          command=['vcbuild.bat'],
-                                          haltOnFailure=True,
-                                          description='vcbuild',
-                                          workdir='llvm\\build',
-                                          warningPattern=" warning C.*:"))
-
-    # Build clang-test project.
-    f.addStep(batch_file_download.BatchFileDownload(name='vcbuild_test',
-                                command=[vcDrive + r""":\Program Files\Microsoft Visual Studio 9.0\VC\VCPackages\vcbuild.exe""",
-                                         "clang-test.vcproj",
-                                         "Debug|Win32"],
-                                workdir="llvm\\build\\tools\\clang\\test"))
-    f.addStep(lit_test_command.LitTestCommand(name='test-clang',
-                               command=["vcbuild_test.bat"],
-                               workdir="llvm\\build\\tools\\clang\\test"))
-
+    f.addStep(WarningCountingShellCommand(name='MSBuilder',
+                       command=msbuildCommand,
+                       haltOnFailure=True,
+                       description='MSBuild ALL',
+                       env=Property('slavenv'),
+                       lazylogfiles=True,
+                       workdir=llvm_objdir,
+                       warningPattern=" warning C.*:"))
+    # Test it.
+    TESTROOT = '%(builddir)s/' + llvm_objdir + 'test'
+    f.addStep(WarningCountingShellCommand(name='MSTester-llvm',
+                       command=mstestCommand.extend("check-llvm.vcxproj"),
+                       lazylogfiles=True,
+                       env=Property('slavenv'),
+                       workdir=WithProperties(TESTROOT)))
+    TESTROOT = '%(builddir)s/' + llvm_objdir + 'tools/clang/test'
+    f.addStep(WarningCountingShellCommand(name='MSTester-clang',
+                       command=mstestCommand.extend("check-clang.vcxproj"),
+                       lazylogfiles=True,
+                       env=Property('slavenv'),
+                       workdir=WithProperties(TESTROOT)))
     return f
 
 # Builds on Windows using CMake, MinGW(32|64), and no Microsoft tools.

zorg/buildbot/builders/LLDBBuilder.py

 import buildbot
 import buildbot.process.factory
 from buildbot.steps.source import SVN
+from buildbot.steps.source.git import Git
 from buildbot.steps.shell import Configure, SetProperty
 from buildbot.steps.shell import ShellCommand, WarningCountingShellCommand
-from buildbot.process.properties import WithProperties
+from buildbot.steps.slave import RemoveDirectory
+from buildbot.process.properties import WithProperties, Property
 from zorg.buildbot.commands.LitTestCommand import LitTestCommand
 
+def slavenv_glob2list(rc, stdout, stderr):
+    '''Extract function for SetPropertyCommand. Loads Slave Environment
+    into a dictionary, and returns slavenv property for ShellCommands.
+    This works for both Windows (set) and Linux (printenv)'''
+    if not rc:
+        slavenv_dict = dict(l.strip().split('=', 1) 
+            for l in stdout.split('\n') if len(l.split('=', 1))==2)
+    return {'slavenv': slavenv_dict}
+
+
+def getLLDBTree(f, llvm_srcdir='.', useGit=False):
+    """ Get or update source tree."""
+    if useGit:
+        repo_llvm='http://llvm.org/git/'
+        branch = 'master'
+        f.addStep(Git(name='git-llvm',
+                repourl=repo_llvm + 'llvm',
+                branch=branch,
+                alwaysUseLatest=True,
+                haltOnFailure=True,
+                workdir=llvm_srcdir))
+        f.addStep(Git(name='git-clang',
+                repourl=repo_llvm + 'clang',
+                branch=branch,
+                alwaysUseLatest=True,
+                haltOnFailure=True,
+                workdir='%s/tools/clang' % llvm_srcdir))
+        f.addStep(Git(name='git-lldb',
+                repourl=repo_llvm + 'lldb',
+                branch=branch,
+                alwaysUseLatest=True,
+                haltOnFailure=True,
+                workdir='%s/tools/lldb' % llvm_srcdir))
+    else:
+        # Find out what version of llvm and clang are needed to build this version
+        # of lldb. Right now we will assume they use the same version.
+        # XXX - could this be done directly on the master instead of the slave?
+        f.addStep(SetProperty(command="""svn cat http://llvm.org/svn/llvm-project/lldb/trunk/scripts/build-llvm.pl | grep '^our.*llvm_revision.*=' | cut -d '"' -f 2""",
+                              property='llvmrev'))
+
+        # The SVN build step provides no mechanism to check out a specific revision
+        # based on a property, so just run the commands directly here.
+
+        svn_co = ['svn', 'checkout', '--force']
+        svn_co += ['--revision', WithProperties('%(llvmrev)s')]
+
+        # build llvm svn checkout command
+        svn_co_llvm = svn_co + \
+         [WithProperties('http://llvm.org/svn/llvm-project/llvm/trunk@%(llvmrev)s'),
+                         llvm_srcdir]
+        # build clang svn checkout command
+        svn_co_clang = svn_co + \
+         [WithProperties('http://llvm.org/svn/llvm-project/cfe/trunk@%(llvmrev)s'),
+                         '%s/tools/clang' % llvm_srcdir]
+
+        f.addStep(ShellCommand(name='svn-llvm',
+                               command=svn_co_llvm,
+                               haltOnFailure=True,
+                               workdir='.'))
+        f.addStep(ShellCommand(name='svn-clang',
+                               command=svn_co_clang,
+                               haltOnFailure=True,
+                               workdir='.'))
+
+        f.addStep(SVN(name='svn-lldb',
+                      mode='update',
+                      baseURL='http://llvm.org/svn/llvm-project/lldb/',
+                      defaultBranch='trunk',
+                      always_purge=True,
+                      workdir='%s/tools/lldb' % llvm_srcdir))
+    return f
+
+
 def getLLDBBuildFactory(
             triple,
             outOfDir=False,

 
     return f
 
+
+def getLLDBMSVCBuildFactory(update=True, clean=False,
+        jobs="%(jobs)s",
+        loadaverage="%(loadaverage)s",
+        cmakeGenerator=r"Visual Studio 12 Win64",
+        vsConfiguration=r"RelWithDebInfo",
+        slavenvCmd=r""""%VS120COMNTOOLS%\..\..\VC\vcvarsall.bat" amd64 & set""",
+        common_cmake_options=None,
+        common_build_options=None,
+        common_test_options=None,
+        enableTargets="X86",
+        env=None,
+        completely_clean=False,
+        useGit=False,
+        *args,**kwargs):
+
+    llvm_srcdir = "llvm.src"
+    llvm_objdir = "llvm.obj"
+
+    if loadaverage is None:
+        loadaverage=jobs
+
+    f = buildbot.process.factory.BuildFactory()
+
+    # Determine/Modify Slave Environment for build steps.
+    f.addStep(SetProperty(
+        command=slavenvCmd,
+        extract_fn=slavenv_glob2list))
+
+    if update:
+        f = getLLDBTree(f,llvm_srcdir,useGit=useGit)
+
+    # Full & fast clean.
+    if clean:
+        f.addStep(RemoveDirectory(dir="llvm/build"))
+
+    # CMAKE command with user arguments.
+    cmakeCommand = ["cmake", "../%s" % llvm_srcdir,
+        "-G", cmakeGenerator,
+        "-DLLVM_TARGETS_TO_BUILD:=%s" % enableTargets,
+        "-DCMAKE_COLOR_MAKEFILE=OFF",
+        ]
+    # CMAKE command with defaults or user supplied options.
+    default_cmake_options = [
+        "-DLLVM_INCLUDE_EXAMPLES:=OFF",
+        "-DLLVM_LIT_ARGS:=--verbose --no-progress-bar",
+        ]
+    if common_cmake_options is not None:
+        cmakeCommand.extend(common_cmake_options)
+    else:
+        cmakeCommand.extend(default_cmake_options)
+
+    # MSBuild command with user arguments.
+    msbuildCommand = ["MSBuild","/p:Configuration=%s" % vsConfiguration ]
+
+    # Build with default or user supplied options.
+    default_build_options = [
+        "/maxcpucount",
+        "ALL_BUILD.vcxproj"]
+    if common_build_options is not None:
+        msbuildCommand.extend(common_build_options)
+    else:
+        msbuildCommand.extend(default_build_options)
+
+    # MSBuild command with user arguments.
+    mstestCommand = ["MSBuild","/p:Configuration=%s" % vsConfiguration ]
+    # Test with default or user supplied options.
+    default_test_options = [
+        "/maxcpucount:1",
+        "/verbosity:detailed",
+        "/consoleloggerparameters:ShowCommandLine"]
+    if common_test_options is not None:
+        mstestCommand.extend(common_test_options)
+    else:
+        mstestCommand.extend(default_test_options)
+
+    # Create the project files.
+    f.addStep(ShellCommand(name='cmake',
+                       command=cmakeCommand,
+                       haltOnFailure=True,
+                       description='cmake gen',
+                       env=Property('slavenv'),
+                       workdir=llvm_objdir))
+    # Build it.
+    f.addStep(WarningCountingShellCommand(name='MSBuilder',
+                       command=msbuildCommand,
+                       haltOnFailure=True,
+                       description='MSBuild ALL',
+                       env=Property('slavenv'),
+                       lazylogfiles=True,
+                       workdir=llvm_objdir,
+                       warningPattern=" warning C.*:"))
+    # Test it.
+    TESTROOT = '%(builddir)s/' + llvm_objdir + '/tools/lldb/test'
+    f.addStep(LitTestCommand(name='MSTester',
+                       command=mstestCommand.extend("check-lldb.vcxproj"),
+                       lazylogfiles=True,
+                       env=Property('slavenv'),
+                       workdir=WithProperties(TESTROOT)))
+    return f
+
+
+def getLLDBCmakeBuildFactory(update=True, clean=False,
+        jobs="%(jobs)s",
+        loadaverage="%(loadaverage)s",
+        cmakeGenerator=r"Unix Makefiles",
+        slavenvCmd=r"""printenv""",
+        common_cmake_options=None,
+        common_build_options=None,
+        common_test_options=None,
+        enableTargets="X86",
+        env=None,
+        completely_clean=False,
+        useGit=False,
+        *args,**kwargs):
+
+    llvm_srcdir = "llvm.src"
+    llvm_objdir = "llvm.obj"
+
+    if loadaverage is None:
+        loadaverage=jobs
+
+    f = buildbot.process.factory.BuildFactory()
+
+    # Determine Slave Environment.
+    f.addStep(SetProperty(
+        command=slavenvCmd,
+        extract_fn=slavenv_glob2list))
+
+    if update:
+        f = getLLDBTree(f,llvm_srcdir,useGit=useGit)
+
+    # Full & fast clean.
+    if clean:
+        f.addStep(RemoveDirectory(dir="llvm/build"))
+
+    # CMAKE command with user arguments.
+    cmakeCommand = ["cmake", "../%s" % llvm_srcdir,
+        "-G", cmakeGenerator,
+        "-DLLVM_TARGETS_TO_BUILD:=%s" % enableTargets,
+        "-DCMAKE_COLOR_MAKEFILE=OFF",
+        ]
+    # CMAKE command with defaults or user supplied options.
+    default_cmake_options = [
+        "-DLLVM_INCLUDE_EXAMPLES:=OFF",
+        "-DLLVM_LIT_ARGS:=--verbose --no-progress-bar",
+        ]
+    if common_cmake_options is not None:
+        cmakeCommand.extend(common_cmake_options)
+    else:
+        cmakeCommand.extend(default_cmake_options)
+
+    # Build command with user arguments.
+    if cmakeGenerator == "Ninja":
+        buildCommand = ["ninja"]
+    else:
+        buildCommand = ["make"]
+
+    # Build with default or user supplied options.
+    default_build_options = [
+        WithProperties("-j%s" % jobs),
+        WithProperties("-l%s" % loadaverage)]
+    if common_build_options is not None:
+        buildCommand.extend(common_build_options)
+    else:
+        buildCommand.extend(default_build_options)
+
+    # Test command with user arguments.
+    if cmakeGenerator == "Ninja":
+        testCommand = ["ninja"]
+    else:
+        testCommand = ["make"]
+    # Test with default or user supplied options.
+    default_test_options = [
+        WithProperties("-j%s" % jobs),
+        WithProperties("-l%s" % loadaverage)]
+    if common_test_options is not None:
+        testCommand.extend(common_test_options)
+    else:
+        testCommand.extend(default_test_options)
+
+    # Create the project files.
+    f.addStep(ShellCommand(name='cmake',
+                       command=cmakeCommand,
+                       haltOnFailure=True,
+                       description='cmake gen',
+                       #env=Property('slavenv'),
+                       workdir=llvm_objdir))
+    # Build it.
+    f.addStep(WarningCountingShellCommand(name='Builder',
+                       command=buildCommand,
+                       haltOnFailure=True,
+                       description='Build ALL',
+                       #env=Property('slavenv'),
+                       lazylogfiles=True,
+                       workdir=llvm_objdir,
+                       warningPattern=" warning C.*:"))
+    # Test it.
+    TESTROOT = '%(builddir)s/' + llvm_objdir + '/tools/lldb/test'
+    f.addStep(LitTestCommand(name='test-lldb',
+                       command=testCommand.extend("check-lldb"),
+                       lazylogfiles=True,
+                       env=Property('slavenv'),
+                       workdir=WithProperties(TESTROOT)))
+    return f
+
 def getLLDBxcodebuildFactory(use_cc=None):
     f = buildbot.process.factory.BuildFactory()
     f.addStep(SetProperty(name='get_builddir',