Index: zorg/buildbot/builders/ClangBuilder.py =================================================================== --- zorg/buildbot/builders/ClangBuilder.py +++ zorg/buildbot/builders/ClangBuilder.py @@ -2,11 +2,12 @@ import buildbot.process.factory import os -from buildbot.process.properties import WithProperties +from buildbot.process.properties import WithProperties, Property from buildbot.steps.shell import Configure, ShellCommand, SetProperty from buildbot.steps.shell import WarningCountingShellCommand from buildbot.steps.source import SVN -from buildbot.steps.transfer import FileDownload +from buildbot.steps.source.git import Git +from buildbot.steps.slave import RemoveDirectory import zorg.buildbot.util.artifacts as artifacts import zorg.buildbot.builders.Util as builders_util @@ -15,6 +16,54 @@ import zorg.buildbot.commands.BatchFileDownload as batch_file_download import zorg.buildbot.commands.LitTestCommand as lit_test_command +def slavenv_glob2list(rc, stdout, stderr): + '''Extract function for SetPropertyCommand. Loads Slave Environment + into a dictionary, and returns slavenv property for ShellCommands.''' + if not rc: + slavenv_dict = dict(l.strip().split('=',1) + for l in stdout.split('\n') if len(l.split('=',1))==2) + return {'slavenv': slavenv_dict} + +def getClangTree(f, llvm_srcdir='.', + useGit=False, + repo_llvm='http://llvm.org/git/', + branch='master'): + """ Get or update source tree.""" + if useGit: + f.addStep(Git(name='git-llvm', + repourl=repo_llvm + 'llvm', + branch=branch, + alwaysUseLatest=True, + haltOnFailure=True, + workdir=llvm_srcdir)) + f.addStep(Git(name='git-clang', + repourl=repo_llvm + 'clang', + branch=branch, + alwaysUseLatest=True, + haltOnFailure=True, + workdir='%s/tools/clang' % llvm_srcdir)) + f.addStep(Git(name='git-clang-tools-extra', + repourl=repo_llvm + 'clang-tools-extra', + branch=branch, + alwaysUseLatest=True, + haltOnFailure=True, + workdir='%s/tools/clang/tools/extra' % llvm_srcdir)) + else: + svnURL = 'http://llvm.org/svn/llvm-project/' + f.addStep(SVN(name='svn-llvm', + mode='update', baseURL = svnURL + 'llvm/', + defaultBranch='trunk', + workdir=llvm_srcdir)) + f.addStep(SVN(name='svn-clang', + mode='update', baseURL = svnURL + 'cfe/', + defaultBranch='trunk', + workdir='%s/tools/clang' % llvm_srcdir)) + f.addStep(SVN(name='svn-clang-tools-extra', + mode='update', baseURL = svnURL + 'clang-tools-extra/', + defaultBranch='trunk', + workdir='%s/tools/clang/tools/extra' % llvm_srcdir)) + return f + def getClangBuildFactory( triple=None, clean=True, @@ -557,82 +606,110 @@ return f -def getClangMSVCBuildFactory(update=True, clean=True, vcDrive='c', jobs=1, cmake=r"cmake"): +def getClangMSVCBuildFactory(update=True, clean=False, + vcDrive='C', # deprecated + cmake=r"cmake", # deprecated + jobs="%(jobs)s", + loadaverage="%(loadaverage)s", + cmakeGenerator=r"Visual Studio 12", + vsConfiguration=r"RelWithDebInfo", + slavenvCmd=r""""%VS120COMNTOOLS%\vsvars32.bat" %PROCESSOR_ARCHITECTURE% & set""", + common_cmake_options=None, + common_build_options=None, + common_test_options=None, + enableTargets="X86", + env=None, + completely_clean=False, + useGit=False, + *args,**kwargs): + + llvm_srcdir = "llvm.src" + llvm_objdir = "llvm.obj" + + if loadaverage is None: + loadaverage=jobs + f = buildbot.process.factory.BuildFactory() + # Determine Slave Environment and Set MSVC environment. + f.addStep(SetProperty( + command=slavenvCmd, + extract_fn=slavenv_glob2list)) + if update: - f.addStep(SVN(name='svn-llvm', - mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', - defaultBranch='trunk', - workdir='llvm')) - f.addStep(SVN(name='svn-clang', - mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/', - defaultBranch='trunk', - workdir='llvm/tools/clang')) - f.addStep(SVN(name='svn-clang-tools-extra', - mode='update', baseURL='http://llvm.org/svn/llvm-project/clang-tools-extra/', - defaultBranch='trunk', - workdir='llvm/tools/clang/tools/extra')) - + f = getClangTree(f,llvm_srcdir=llvm_srcdir,useGit=useGit) # Full & fast clean. if clean: - f.addStep(ShellCommand(name='clean-1', - command=['del','/s/q','build'], - warnOnFailure=True, - description='cleaning', - descriptionDone='clean', - workdir='llvm')) - f.addStep(ShellCommand(name='clean-2', - command=['rmdir','/s/q','build'], - warnOnFailure=True, - description='cleaning', - descriptionDone='clean', - workdir='llvm')) + f.addStep(RemoveDirectory(dir=llvm_objdir)) + # CMAKE command with user arguments. + cmakeCommand = ["cmake", "../%s" % llvm_srcdir, + "-G", cmakeGenerator, + "-DLLVM_TARGETS_TO_BUILD:=%s" % enableTargets, + "-DCMAKE_COLOR_MAKEFILE=OFF", + ] + # CMAKE command with defaults or user supplied options. + default_cmake_options = [ + "-DLLVM_INCLUDE_EXAMPLES:=OFF", + "-DLLVM_LIT_ARGS:=--verbose --no-progress-bar", + ] + if common_cmake_options is not None: + cmakeCommand.extend(common_cmake_options) + else: + cmakeCommand.extend(default_cmake_options) + + # MSBuild command with user arguments. + msbuildCommand = ["MSBuild","/p:Configuration=%s" % vsConfiguration ] + # Build with default or user supplied options. + default_build_options = [ + "/maxcpucount", + "ALL_BUILD.vcxproj"] + if common_build_options is not None: + msbuildCommand.extend(common_build_options) + else: + msbuildCommand.extend(default_build_options) + + # MSBuild command with user arguments. + mstestCommand = ["MSBuild","/p:Configuration=%s" % vsConfiguration ] + # Test with default or user supplied options. + default_test_options = [ + "/maxcpucount:1", + "/verbosity:detailed", + "/consoleloggerparameters:ShowCommandLine"] + if common_test_options is not None: + mstestCommand.extend(common_test_options) + else: + mstestCommand.extend(default_test_options) + # Create the project files. - - # Use batch files instead of ShellCommand directly, Windows quoting is - # borked. FIXME: See buildbot ticket #595 and buildbot ticket #377. - f.addStep(batch_file_download.BatchFileDownload(name='cmakegen', - command=[cmake, - "-DLLVM_TARGETS_TO_BUILD:=X86", - "-DLLVM_INCLUDE_EXAMPLES:=OFF", - "-DLLVM_INCLUDE_TESTS:=OFF", - "-DLLVM_TARGETS_TO_BUILD:=X86", - "-G", - "Visual Studio 9 2008", - ".."], - workdir="llvm\\build")) f.addStep(ShellCommand(name='cmake', - command=['cmakegen.bat'], - haltOnFailure=True, - description='cmake gen', - workdir='llvm\\build')) - + command=cmakeCommand, + haltOnFailure=True, + description='cmake gen', + env=Property('slavenv'), + workdir=llvm_objdir)) # Build it. - f.addStep(batch_file_download.BatchFileDownload(name='vcbuild', - command=[vcDrive + r""":\Program Files\Microsoft Visual Studio 9.0\VC\VCPackages\vcbuild.exe""", - "/M%d" % jobs, - "LLVM.sln", - "Debug|Win32"], - workdir="llvm\\build")) - f.addStep(WarningCountingShellCommand(name='vcbuild', - command=['vcbuild.bat'], - haltOnFailure=True, - description='vcbuild', - workdir='llvm\\build', - warningPattern=" warning C.*:")) - - # Build clang-test project. - f.addStep(batch_file_download.BatchFileDownload(name='vcbuild_test', - command=[vcDrive + r""":\Program Files\Microsoft Visual Studio 9.0\VC\VCPackages\vcbuild.exe""", - "clang-test.vcproj", - "Debug|Win32"], - workdir="llvm\\build\\tools\\clang\\test")) - f.addStep(lit_test_command.LitTestCommand(name='test-clang', - command=["vcbuild_test.bat"], - workdir="llvm\\build\\tools\\clang\\test")) - + f.addStep(WarningCountingShellCommand(name='MSBuilder', + command=msbuildCommand, + haltOnFailure=True, + description='MSBuild ALL', + env=Property('slavenv'), + lazylogfiles=True, + workdir=llvm_objdir, + warningPattern=" warning C.*:")) + # Test it. + TESTROOT = '%(builddir)s/' + llvm_objdir + 'test' + f.addStep(WarningCountingShellCommand(name='MSTester-llvm', + command=mstestCommand.extend("check-llvm.vcxproj"), + lazylogfiles=True, + env=Property('slavenv'), + workdir=WithProperties(TESTROOT))) + TESTROOT = '%(builddir)s/' + llvm_objdir + 'tools/clang/test' + f.addStep(WarningCountingShellCommand(name='MSTester-clang', + command=mstestCommand.extend("check-clang.vcxproj"), + lazylogfiles=True, + env=Property('slavenv'), + workdir=WithProperties(TESTROOT))) return f # Builds on Windows using CMake, MinGW(32|64), and no Microsoft tools. Index: zorg/buildbot/builders/LLDBBuilder.py =================================================================== --- zorg/buildbot/builders/LLDBBuilder.py +++ zorg/buildbot/builders/LLDBBuilder.py @@ -3,11 +3,86 @@ import buildbot import buildbot.process.factory from buildbot.steps.source import SVN +from buildbot.steps.source.git import Git from buildbot.steps.shell import Configure, SetProperty from buildbot.steps.shell import ShellCommand, WarningCountingShellCommand -from buildbot.process.properties import WithProperties +from buildbot.steps.slave import RemoveDirectory +from buildbot.process.properties import WithProperties, Property from zorg.buildbot.commands.LitTestCommand import LitTestCommand +def slavenv_glob2list(rc, stdout, stderr): + '''Extract function for SetPropertyCommand. Loads Slave Environment + into a dictionary, and returns slavenv property for ShellCommands. + This works for both Windows (set) and Linux (printenv)''' + if not rc: + slavenv_dict = dict(l.strip().split('=', 1) + for l in stdout.split('\n') if len(l.split('=', 1))==2) + return {'slavenv': slavenv_dict} + + +def getLLDBTree(f, llvm_srcdir='.', useGit=False): + """ Get or update source tree.""" + if useGit: + repo_llvm='http://llvm.org/git/' + branch = 'master' + f.addStep(Git(name='git-llvm', + repourl=repo_llvm + 'llvm', + branch=branch, + alwaysUseLatest=True, + haltOnFailure=True, + workdir=llvm_srcdir)) + f.addStep(Git(name='git-clang', + repourl=repo_llvm + 'clang', + branch=branch, + alwaysUseLatest=True, + haltOnFailure=True, + workdir='%s/tools/clang' % llvm_srcdir)) + f.addStep(Git(name='git-lldb', + repourl=repo_llvm + 'lldb', + branch=branch, + alwaysUseLatest=True, + haltOnFailure=True, + workdir='%s/tools/lldb' % llvm_srcdir)) + else: + # Find out what version of llvm and clang are needed to build this version + # of lldb. Right now we will assume they use the same version. + # XXX - could this be done directly on the master instead of the slave? + f.addStep(SetProperty(command="""svn cat http://llvm.org/svn/llvm-project/lldb/trunk/scripts/build-llvm.pl | grep '^our.*llvm_revision.*=' | cut -d '"' -f 2""", + property='llvmrev')) + + # The SVN build step provides no mechanism to check out a specific revision + # based on a property, so just run the commands directly here. + + svn_co = ['svn', 'checkout', '--force'] + svn_co += ['--revision', WithProperties('%(llvmrev)s')] + + # build llvm svn checkout command + svn_co_llvm = svn_co + \ + [WithProperties('http://llvm.org/svn/llvm-project/llvm/trunk@%(llvmrev)s'), + llvm_srcdir] + # build clang svn checkout command + svn_co_clang = svn_co + \ + [WithProperties('http://llvm.org/svn/llvm-project/cfe/trunk@%(llvmrev)s'), + '%s/tools/clang' % llvm_srcdir] + + f.addStep(ShellCommand(name='svn-llvm', + command=svn_co_llvm, + haltOnFailure=True, + workdir='.')) + f.addStep(ShellCommand(name='svn-clang', + command=svn_co_clang, + haltOnFailure=True, + workdir='.')) + + f.addStep(SVN(name='svn-lldb', + mode='update', + baseURL='http://llvm.org/svn/llvm-project/lldb/', + defaultBranch='trunk', + always_purge=True, + workdir='%s/tools/lldb' % llvm_srcdir)) + return f + + def getLLDBBuildFactory( triple, outOfDir=False, @@ -105,6 +180,212 @@ return f + +def getLLDBMSVCBuildFactory(update=True, clean=False, + jobs="%(jobs)s", + loadaverage="%(loadaverage)s", + cmakeGenerator=r"Visual Studio 12 Win64", + vsConfiguration=r"RelWithDebInfo", + slavenvCmd=r""""%VS120COMNTOOLS%\..\..\VC\vcvarsall.bat" amd64 & set""", + common_cmake_options=None, + common_build_options=None, + common_test_options=None, + enableTargets="X86", + env=None, + completely_clean=False, + useGit=False, + *args,**kwargs): + + llvm_srcdir = "llvm.src" + llvm_objdir = "llvm.obj" + + if loadaverage is None: + loadaverage=jobs + + f = buildbot.process.factory.BuildFactory() + + # Determine/Modify Slave Environment for build steps. + f.addStep(SetProperty( + command=slavenvCmd, + extract_fn=slavenv_glob2list)) + + if update: + f = getLLDBTree(f,llvm_srcdir,useGit=useGit) + + # Full & fast clean. + if clean: + f.addStep(RemoveDirectory(dir="llvm/build")) + + # CMAKE command with user arguments. + cmakeCommand = ["cmake", "../%s" % llvm_srcdir, + "-G", cmakeGenerator, + "-DLLVM_TARGETS_TO_BUILD:=%s" % enableTargets, + "-DCMAKE_COLOR_MAKEFILE=OFF", + ] + # CMAKE command with defaults or user supplied options. + default_cmake_options = [ + "-DLLVM_INCLUDE_EXAMPLES:=OFF", + "-DLLVM_LIT_ARGS:=--verbose --no-progress-bar", + ] + if common_cmake_options is not None: + cmakeCommand.extend(common_cmake_options) + else: + cmakeCommand.extend(default_cmake_options) + + # MSBuild command with user arguments. + msbuildCommand = ["MSBuild","/p:Configuration=%s" % vsConfiguration ] + + # Build with default or user supplied options. + default_build_options = [ + "/maxcpucount", + "ALL_BUILD.vcxproj"] + if common_build_options is not None: + msbuildCommand.extend(common_build_options) + else: + msbuildCommand.extend(default_build_options) + + # MSBuild command with user arguments. + mstestCommand = ["MSBuild","/p:Configuration=%s" % vsConfiguration ] + # Test with default or user supplied options. + default_test_options = [ + "/maxcpucount:1", + "/verbosity:detailed", + "/consoleloggerparameters:ShowCommandLine"] + if common_test_options is not None: + mstestCommand.extend(common_test_options) + else: + mstestCommand.extend(default_test_options) + + # Create the project files. + f.addStep(ShellCommand(name='cmake', + command=cmakeCommand, + haltOnFailure=True, + description='cmake gen', + env=Property('slavenv'), + workdir=llvm_objdir)) + # Build it. + f.addStep(WarningCountingShellCommand(name='MSBuilder', + command=msbuildCommand, + haltOnFailure=True, + description='MSBuild ALL', + env=Property('slavenv'), + lazylogfiles=True, + workdir=llvm_objdir, + warningPattern=" warning C.*:")) + # Test it. + TESTROOT = '%(builddir)s/' + llvm_objdir + '/tools/lldb/test' + f.addStep(LitTestCommand(name='MSTester', + command=mstestCommand.extend("check-lldb.vcxproj"), + lazylogfiles=True, + env=Property('slavenv'), + workdir=WithProperties(TESTROOT))) + return f + + +def getLLDBCmakeBuildFactory(update=True, clean=False, + jobs="%(jobs)s", + loadaverage="%(loadaverage)s", + cmakeGenerator=r"Unix Makefiles", + slavenvCmd=r"""printenv""", + common_cmake_options=None, + common_build_options=None, + common_test_options=None, + enableTargets="X86", + env=None, + completely_clean=False, + useGit=False, + *args,**kwargs): + + llvm_srcdir = "llvm.src" + llvm_objdir = "llvm.obj" + + if loadaverage is None: + loadaverage=jobs + + f = buildbot.process.factory.BuildFactory() + + # Determine Slave Environment. + f.addStep(SetProperty( + command=slavenvCmd, + extract_fn=slavenv_glob2list)) + + if update: + f = getLLDBTree(f,llvm_srcdir,useGit=useGit) + + # Full & fast clean. + if clean: + f.addStep(RemoveDirectory(dir="llvm/build")) + + # CMAKE command with user arguments. + cmakeCommand = ["cmake", "../%s" % llvm_srcdir, + "-G", cmakeGenerator, + "-DLLVM_TARGETS_TO_BUILD:=%s" % enableTargets, + "-DCMAKE_COLOR_MAKEFILE=OFF", + ] + # CMAKE command with defaults or user supplied options. + default_cmake_options = [ + "-DLLVM_INCLUDE_EXAMPLES:=OFF", + "-DLLVM_LIT_ARGS:=--verbose --no-progress-bar", + ] + if common_cmake_options is not None: + cmakeCommand.extend(common_cmake_options) + else: + cmakeCommand.extend(default_cmake_options) + + # Build command with user arguments. + if cmakeGenerator == "Ninja": + buildCommand = ["ninja"] + else: + buildCommand = ["make"] + + # Build with default or user supplied options. + default_build_options = [ + WithProperties("-j%s" % jobs), + WithProperties("-l%s" % loadaverage)] + if common_build_options is not None: + buildCommand.extend(common_build_options) + else: + buildCommand.extend(default_build_options) + + # Test command with user arguments. + if cmakeGenerator == "Ninja": + testCommand = ["ninja"] + else: + testCommand = ["make"] + # Test with default or user supplied options. + default_test_options = [ + WithProperties("-j%s" % jobs), + WithProperties("-l%s" % loadaverage)] + if common_test_options is not None: + testCommand.extend(common_test_options) + else: + testCommand.extend(default_test_options) + + # Create the project files. + f.addStep(ShellCommand(name='cmake', + command=cmakeCommand, + haltOnFailure=True, + description='cmake gen', + #env=Property('slavenv'), + workdir=llvm_objdir)) + # Build it. + f.addStep(WarningCountingShellCommand(name='Builder', + command=buildCommand, + haltOnFailure=True, + description='Build ALL', + #env=Property('slavenv'), + lazylogfiles=True, + workdir=llvm_objdir, + warningPattern=" warning C.*:")) + # Test it. + TESTROOT = '%(builddir)s/' + llvm_objdir + '/tools/lldb/test' + f.addStep(LitTestCommand(name='test-lldb', + command=testCommand.extend("check-lldb"), + lazylogfiles=True, + env=Property('slavenv'), + workdir=WithProperties(TESTROOT))) + return f + def getLLDBxcodebuildFactory(use_cc=None): f = buildbot.process.factory.BuildFactory() f.addStep(SetProperty(name='get_builddir',