Index: zorg/buildbot/builders/ClangBuilder.py =================================================================== --- zorg/buildbot/builders/ClangBuilder.py +++ zorg/buildbot/builders/ClangBuilder.py @@ -2,11 +2,12 @@ import buildbot.process.factory import os -from buildbot.process.properties import WithProperties +from buildbot.process.properties import WithProperties, Property from buildbot.steps.shell import Configure, ShellCommand, SetProperty from buildbot.steps.shell import WarningCountingShellCommand from buildbot.steps.source import SVN from buildbot.steps.transfer import FileDownload +from buildbot.steps.slave import RemoveDirectory import zorg.buildbot.util.artifacts as artifacts import zorg.buildbot.builders.Util as builders_util @@ -15,6 +16,47 @@ import zorg.buildbot.commands.BatchFileDownload as batch_file_download import zorg.buildbot.commands.LitTestCommand as lit_test_command +def slave_env_glob2list(rc, stdout, stderr): + '''Extract function for SetPropertyCommand. Loads Slave Environment + into a dictionary, and returns slave_env property for ShellCommands.''' + if not rc: + slave_env_dict = dict(l.strip().split('=',1) + for l in stdout.split('\n') if len(l.split('=',1))==2) + return {'slave_env': slave_env_dict} + +""" Get or update Clang source tree from SCM.""" +def getClangSource(f, + llvm_srcdir='.', + repourl='http://llvm.org/svn/llvm-project/', + branch='trunk', + mode='update'): + f.addStep(SVN(name='svn-llvm', + mode=mode, baseURL = repourl + 'llvm/', + defaultBranch=branch, + workdir=llvm_srcdir)) + f.addStep(SVN(name='svn-clang', + mode=mode, baseURL = repourl + 'cfe/', + defaultBranch=branch, + workdir='%s/tools/clang' % llvm_srcdir)) + f.addStep(SVN(name='svn-clang-tools-extra', + mode=mode, baseURL = repourl + 'clang-tools-extra/', + defaultBranch=branch, + workdir='%s/tools/clang/tools/extra' % llvm_srcdir)) + return f + +""" Get or update LLDB source tree from SCM.""" +def getLLDBSource(f, + llvm_srcdir='.', + repourl='http://llvm.org/svn/llvm-project/', + branch='trunk', + mode='update'): + f = getClangSource(f,llvm_srcdir,repourl,branch,mode) + f.addStep(SVN(name='svn-lldb', + mode=mode, baseURL = repourl + 'lldb/', + defaultBranch=branch, + workdir='%s/tools/lldb' % llvm_srcdir)) + return f + def getClangBuildFactory( triple=None, clean=True, @@ -444,6 +486,9 @@ return f +# Map Cmake Generator to Project file. +cmakeGenToProjectfile={'Makefiles':'Makefile','Ninja':'build.ninja','Visual':'ALL_BUILD.vcxproj'} + # CMake Linux builds def getClangCMakeBuildFactory( clean=True, @@ -450,7 +495,24 @@ test=True, cmake='cmake', jobs=None, + loadaverage=None, + + # Build and Check + cmakeGenerator='Ninja', + cmakeProjectfile='build.ninja', + build_cmd=['ninja'], + build_install_cmd=['ninja', 'install'], + build_check_cmd=['ninja', 'check-all'], + build_check_dir=None, + # Phase 2 build with built Clang. + cmakeGenerator2='Ninja', + cmakeProjectfile2='build.ninja', + build_cmd2=['ninja'], + build_install_cmd2=['ninja', 'install'], + build_check_cmd2=['ninja', 'check-all'], + build_check_dir2=None, + # Multi-stage compilation useTwoStage=False, testStage1=True, @@ -460,6 +522,8 @@ # Environmental variables for all steps. env={}, extra_cmake_args=[], + slave_envCmd=None, + slave_envCmd2=None, # Extra repositories checkout_clang_tools_extra=True, @@ -469,15 +533,8 @@ f = buildbot.process.factory.BuildFactory() # We *must* checkout at least Clang+LLVM - f.addStep(SVN(name='svn-llvm', - mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', - defaultBranch='trunk', - workdir='llvm')) - f.addStep(SVN(name='svn-clang', - mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/', - defaultBranch='trunk', - workdir='llvm/tools/clang')) - + # For now a function, future a dictionary describing everything. + f = getClangSource(f,'llvm') # Extra stuff that will be built/tested if checkout_clang_tools_extra: f.addStep(SVN(name='svn-clang-tools-extra', @@ -490,17 +547,31 @@ defaultBranch='trunk', workdir='llvm/projects/compiler-rt')) + # Set and return slave environment, if slave_envCmd defined. + if slave_envCmd is not None: + f.addStep(SetProperty( + name='get slave env', + command=slave_envCmd, + extract_fn=slave_env_glob2list)) + env=Property('slave_env') + + # If jobs not defined, Ninja will choose a suitable value jobs_cmd=[] lit_args="'-v" if jobs is not None: jobs_cmd=["-j"+str(jobs)] - lit_args+=" -j"+str(jobs)+"'" - else: - lit_args+="'" - ninja_cmd=['ninja'] + jobs_cmd - ninja_install_cmd=['ninja', 'install'] + jobs_cmd - ninja_check_cmd=['ninja', 'check-all'] + jobs_cmd + lit_args+=" -j"+str(jobs) + if loadaverage is not None: + jobs_cmd.extend(["-l"+str(loadaverage)]) + #lit.py:no such option -l + lit_args+="'" + build_cmd.extend(jobs_cmd) + build_install_cmd.extend(jobs_cmd) + build_check_cmd.extend(jobs_cmd) + build_cmd2.extend(jobs_cmd) + build_install_cmd2.extend(jobs_cmd) + build_check_cmd2.extend(jobs_cmd) # Global configurations stage1_build='stage1' @@ -507,27 +578,37 @@ stage1_install='stage1.install' stage2_build='stage2' + # Set test directories + if build_check_dir is None: + build_check_dir = stage1_build + if build_check_dir2 is None: + build_check_dir2 = stage2_build + + # Project file created by previous cmake run. + if cmakeProjectfile is None: + cmakeProjectfile = 'Missing' + for key,val in cmakeGenToProjectfile.items(): + if key in cmakeGenerator: + cmakeProjectfile = val + ############# CLEANING - if clean: - f.addStep(ShellCommand(name='clean stage 1', - command=['rm','-rf',stage1_build], - warnOnFailure=True, - description='cleaning stage 1', - descriptionDone='clean', - workdir='.', - env=env)) - else: - f.addStep(SetProperty(name="check ninja files 1", - workdir=stage1_build, - command=["sh", "-c", - "test -e build.ninja && echo OK || echo Missing"], - flunkOnFailure=False, - property="exists_ninja_1")) + doCleanIf = lambda step: step.build.getProperty("clean") or clean + f.addStep(RemoveDirectory(name='clean stage 1', + dir=stage1_build, + warnOnFailure=True, + doStepIf=doCleanIf + )) + f.addStep(SetProperty(name="check cmake project files 1", + workdir=stage1_build, + command=["sh", "-c", + "test -e %s && echo OK || echo Missing" % cmakeProjectfile], + flunkOnFailure=False, + property="exists_cmake_projectfile_1")) ############# STAGE 1 f.addStep(ShellCommand(name='cmake stage 1', - command=[cmake, "-G", "Ninja", "../llvm", + command=[cmake, "-G", cmakeGenerator, "../llvm", "-DCMAKE_BUILD_TYPE="+stage1_config, "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS="+lit_args, @@ -536,22 +617,22 @@ haltOnFailure=True, description='cmake stage 1', workdir=stage1_build, - doStepIf=lambda step: step.build.getProperty("exists_ninja_1") != "OK", + doStepIf=lambda step: step.build.getProperty("exists_cmake_projectfile_1") != "OK", env=env)) f.addStep(WarningCountingShellCommand(name='build stage 1', - command=ninja_cmd, + command=build_cmd, haltOnFailure=True, - description='ninja all', + description='build all', workdir=stage1_build, env=env)) if test and testStage1: - f.addStep(lit_test_command.LitTestCommand(name='ninja check 1', - command=ninja_check_cmd, + f.addStep(lit_test_command.LitTestCommand(name='build check 1', + command=build_check_cmd, description=["checking stage 1"], descriptionDone=["stage 1 checked"], - workdir=stage1_build, + workdir=build_check_dir, env=env)) if not useTwoStage: @@ -559,30 +640,34 @@ ############# STAGE 2 f.addStep(ShellCommand(name='install stage 1', - command=ninja_install_cmd, - description='ninja install', + command=build_install_cmd, + description='build install', workdir=stage1_build, env=env)) - if clean: - f.addStep(ShellCommand(name='clean stage 2', - command=['rm','-rf',stage2_build], - warnOnFailure=True, - description='cleaning stage 2', - descriptionDone='clean', - workdir='.', - env=env)) - else: - f.addStep(SetProperty(name="check ninja files 2", - workdir=stage2_build, - command=["sh", "-c", - "test -e build.ninja && echo OK || echo Missing"], - flunkOnFailure=False, - property="exists_ninja_2")) + # Set and return slave environment, if slave_envCmd defined. + if slave_envCmd2 is not None: + f.addStep(SetProperty( + name='get slave env 2', + command=slave_envCmd2, + extract_fn=slave_env_glob2list)) + env=Property('slave_env') + f.addStep(RemoveDirectory(name='clean stage 2', + dir=stage2_build, + warnOnFailure=True, + doStepIf=doCleanIf + )) + f.addStep(SetProperty(name="check cmake project files 2", + workdir=stage2_build, + command=["sh", "-c", + "test -e %s && echo OK || echo Missing" % cmakeProjectfile2], + flunkOnFailure=False, + property="exists_cmake_projectfile_2")) + f.addStep(ShellCommand(name='cmake stage 2', - command=[cmake, "-G", "Ninja", "../llvm", + command=[cmake, "-G", cmakeGenerator2, "../llvm", "-DCMAKE_BUILD_TYPE="+stage2_config, "-DLLVM_ENABLE_ASSERTIONS=True", WithProperties("-DCMAKE_C_COMPILER=%(workdir)s/"+stage1_install+"/bin/clang"), @@ -592,22 +677,22 @@ haltOnFailure=True, description='cmake stage 2', workdir=stage2_build, - doStepIf=lambda step: step.build.getProperty("exists_ninja_2") != "OK", + doStepIf=lambda step: step.build.getProperty("exists_cmake_projectfile_2") != "OK", env=env)) f.addStep(WarningCountingShellCommand(name='build stage 2', - command=ninja_cmd, + command=build_cmd2, haltOnFailure=True, - description='ninja all', + description='build all', workdir=stage2_build, env=env)) if test: - f.addStep(lit_test_command.LitTestCommand(name='ninja check 2', - command=ninja_check_cmd, + f.addStep(lit_test_command.LitTestCommand(name='build check 2', + command=build_check_cmd2, description=["checking stage 2"], descriptionDone=["stage 2 checked"], - workdir=stage2_build, + workdir=build_check_dir2, env=env)) return f