Index: zorg/buildbot/builders/ClangBuilder.py =================================================================== --- zorg/buildbot/builders/ClangBuilder.py +++ zorg/buildbot/builders/ClangBuilder.py @@ -2,11 +2,12 @@ import buildbot.process.factory import os -from buildbot.process.properties import WithProperties +from copy import deepcopy +from buildbot.process.properties import WithProperties, Property from buildbot.steps.shell import Configure, ShellCommand, SetProperty from buildbot.steps.shell import WarningCountingShellCommand -from buildbot.steps.source import SVN from buildbot.steps.transfer import FileDownload +from buildbot.steps.slave import RemoveDirectory import zorg.buildbot.util.artifacts as artifacts import zorg.buildbot.builders.Util as builders_util @@ -15,6 +16,60 @@ import zorg.buildbot.commands.BatchFileDownload as batch_file_download import zorg.buildbot.commands.LitTestCommand as lit_test_command +def slave_env_glob2list(rc, stdout, stderr): + '''Extract function for SetPropertyCommand. Loads Slave Environment + into a dictionary, and returns slave_env property for ShellCommands.''' + if not rc: + slave_env_dict = dict(l.strip().split('=',1) + for l in stdout.split('\n') if len(l.split('=',1))==2) + return {'slave_env': slave_env_dict} + +def SVN(f,llvmsrcdir,project=''): + (scm, workdir, baseURL, branch, mode) = llvmSourceTree[project] + f = f.addStep(buildbot.steps.source.SVN(name=project, + mode=mode, + baseURL=baseURL, + defaultBranch=branch, + workdir=workdir % llvmsrcdir)) + return f + +def Git(f,llvmsrcdir,project=''): + (scm, workdir, baseURL, branch, mode) = llvmSourceTree[project] + f = f.addStep(buildbot.steps.source.Git(name=project, + repourl=baseURL, + branchranch=branch, + alwaysUseLatest=True, + haltOnFailure=True, + workdir=workdir % llvmsrcdir)) + return f + +# SCM, workdir, baseURL, branch, mode +llvmSourceTree = { + 'svn-llvm': [SVN,"%s",'http://llvm.org/svn/llvm-project/llvm/','trunk','update'], + 'svn-clang': [SVN,"%s/tools/clang",'http://llvm.org/svn/llvm-project/cfe/','trunk','update'], + 'svn-clang-tools-extra': [SVN,"%s/tools/clang/tools/extra",'http://llvm.org/svn/llvm-project/clang-tools-extra/','trunk','update'], + 'svn-lldb': [SVN,"%s/tools/lldb",'http://llvm.org/svn/llvm-project/lldb/','trunk','update'], + 'svn-polly': [SVN,"%s/tools/polly",'http://llvm.org/svn/llvm-project/polly/','trunk','update'], + 'svn-libcxx': [SVN,"%s/projects/libcxx",'http://llvm.org/svn/llvm-project/libcxx/','trunk','update'], + 'svn-libcxx-pull': [SVN,"%s/projects/libcxx",'http://llvm.org/svn/llvm-project/libcxx/','trunk','pull'], + 'svn-libcxxabi': [SVN,"%s/projects/libcxxabi",'http://llvm.org/svn/llvm-project/libcxxabi/','trunk','update'], + 'svn-compiler-rt': [SVN,"%s/projects/compiler-rt",'http://llvm.org/svn/llvm-project/compiler-rt/','trunk','update'], + 'svn-dragonegg': [SVN,"dragonegg.src",'http://llvm.org/svn/llvm-project/dragonegg/','trunk','update'], + 'svn-test-suite': [SVN,"test-suite.src",'http://llvm.org/svn/llvm-project/test-suite/','trunk','update'], + 'svn-libiomp5': [SVN,"openmp.src",'http://llvm.org/svn/llvm-project/openmp/','trunk','update'], + 'svn-clang-modern-gdb-tests': [SVN,"clang-tests/src",'http://llvm.org/svn/llvm-project/clang-tests-external/trunk/gdb/7.5/','','update'], + 'git-imath': [Git,"imath.src",'git://codeaurora.org/quic/iml/imath/','isl','update'], + 'git-isl': [Git,"isl.src",'http://repo.or.cz/r/isl.git/','master','update'], +} +# TODO 'svn-lit': [SVN,"lit.src",'http://llvm.org/svn/llvm-project/llvm/trunk/utils/lit','trunk','incremental'], +# Git mirror not needed 'git-llvm': [Git,"%s",'http://llvm.org/git/llvm','master','update'], + +def getLLVMSource(f,llvmsrcdir='.',project_list=[]): + for project in project_list: + if llvmSourceTree[project]: + llvmSourceTree[project][0](f,llvmsrcdir,project) + return f + def getClangBuildFactory( triple=None, clean=True, @@ -125,23 +180,23 @@ haltOnFailure=True, workdir='.')) else: - f.addStep(SVN(name='svn-llvm', + f.addStep(buildbot.steps.source.SVN(name='svn-llvm', mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir=llvm_srcdir)) - f.addStep(SVN(name='svn-clang', + f.addStep(buildbot.steps.source.SVN(name='svn-clang', mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/', defaultBranch='trunk', workdir='%s/tools/clang' % llvm_srcdir)) - f.addStep(SVN(name='svn-clang-tools-extra', + f.addStep(buildbot.steps.source.SVN(name='svn-clang-tools-extra', mode='update', baseURL='http://llvm.org/svn/llvm-project/clang-tools-extra/', defaultBranch='trunk', workdir='%s/tools/clang/tools/extra' % llvm_srcdir)) if checkout_compiler_rt: - f.addStep(SVN(name='svn-compiler-rt', + f.addStep(buildbot.steps.source.SVN(name='svn-compiler-rt', mode='update', baseURL='http://llvm.org/svn/llvm-project/compiler-rt/', defaultBranch='trunk', @@ -444,7 +499,188 @@ return f -# CMake Linux builds +""" CMake Build Factory +name: 'stage', Name: step name. +source: [] List of projects from llvmSourceTree. +'source_dir': 'llvm', +'config': 'Release' Debug, Release, RelWithDebug. +'slave_envCmd': r'', Execute command on slave, get slave env. +env: {}, Update slave environment variables. Superceeded by slave_envCmd. +'cmakeGenerator': 'Ninja', CMake Generator +'cmakeProjectfile': 'build.ninja',Project file after CMake is run. +'extra_cmake_args': [],Extra arguments to CMake. +'build_cmd': ['ninja'], Build source code command. +build_jobs: '-j%(jobs)s -l%(loadaverage)', job control if supported. +'build_dir': 'stage1.build', +'install_cmd': ['ninja', 'install'], +'install_dir': 'stage1.install', +install_cmd_jobs: '-j%(jobs)s -l%(loadaverage)', job control if supported. +'check_cmd': ['ninja', 'check-all'], +'check_dir': '', sub-directory from build_dir to start tests. +check_cmd_jobs: '-j%(jobs)s -l%(loadaverage)', job control if supported. +'useTwoStage': True, Use compiler from previous stage. +'test': '', Execute test step (True) and Ignore Failures (ignoreFail) +clean: False, clean build requested. +""" +def getCMakeBuildFactory( + build=[ + {'name': "stage 1", + 'build_dir': 'stage1.build', + 'install_dir': 'stage1.install', + 'install': True, + 'test': 'ignoreFail', + }, + {'name': "stage 2", + 'usePreviousStageResults': 'stage1.install', + 'build_dir': 'stage2.build', + 'install_dir': 'stage2.install', + 'test': 'ignoreFail', + }, + ], + update=[] # Update each stage [{},...] + ): + + # Apply updates to each stage. [{},{}] + if update: + newBuild = deepcopy(build) + for u,b in zip(update,newBuild): + b.update(u) + else: + newBuild = build + + f = buildbot.process.factory.BuildFactory() + + # Must be initialized before multiple stage settings begin. + env = Property('env') + + # Drive thru build stages. + for stage in newBuild: + # Set the stage name + stageName = stage.get('name','stage') + + # Check for source first. + stage_source = stage.get('source_dir','llvm') + if 'source' in stage: + f = getLLVMSource(f,stage_source,stage['source']) + + # Check for user specified environment updates. + # Note: This is superceeded by slave_envCmd within a stage. + if 'env' in stage: + f = zorg.buildbot.util.phasedbuilderutils.setProperty( + f,new_property='env', new_value=stage['env']) + env=Property('env') + + # Set and return slave environment to be used in all commands. + if 'slave_envCmd' in stage: + f.addStep(SetProperty( + name='get slave env', + command=stage['slave_envCmd'], + extract_fn=slave_env_glob2list)) + env=Property('slave_env') + + # Global configurations + stage_build = stage.get('build_dir','stage.build') + stage_install = stage.get('install_dir','stage.install') + if 'check_dir' in stage: + stage_check = stage_build+"/"+str(stage['check_dir']) + else: + stage_check = stage_build + + ############# CLEANING + cleanBuildRequested = lambda step: step.build.getProperty("clean") or stage.get('clean',False) + f.addStep(RemoveDirectory(name=stageName+': clean', + description='cleaning %s' % stage_build, + dir=stage_build, + warnOnFailure=True, + doStepIf=cleanBuildRequested + )) + cmakeProjectfile = stage.get('cmakeProjectfile','build.ninja') + f.addStep(SetProperty(name=stageName+': check cmake project files', + workdir=stage_build, + command=["sh", "-c", + "test -e %s && echo OK || echo Missing" % cmakeProjectfile], + flunkOnFailure=False, + property="exists_cmake_projectfile")) + + ############# STAGE + cmakeGenerator = stage.get('cmakeGenerator','Ninja') + stage_config = stage.get('config','Release') + lit_args = stage.get('lit_args',"-v") + usePreviousClang = [] + if 'usePreviousStageResults' in stage: + built_clang_dir = stage.get('usePreviousStageResults','stage1.install') + # CMake escapes these variables. Windows \ is fatal. + usePreviousClang=[ + "-DCMAKE_C_COMPILER=../"+built_clang_dir+"/bin/clang", + "-DCMAKE_CXX_COMPILER=../"+built_clang_dir+"/bin/clang++" + ] + extra_cmake_args = stage.get('extra_cmake_args',[]) + f.addStep(ShellCommand(name=stageName+': cmake', + command=["cmake", "-G", cmakeGenerator, "../"+stage_source, + "-DCMAKE_BUILD_TYPE="+stage_config, + "-DLLVM_ENABLE_ASSERTIONS=True", + "-DLLVM_LIT_ARGS="+lit_args, + "-DCMAKE_INSTALL_PREFIX=../"+stage_install] + + usePreviousClang + + extra_cmake_args, + haltOnFailure=True, + description='cmake %s' % stage_config, + workdir=stage_build, + doStepIf=lambda step: step.build.getProperty("exists_cmake_projectfile") != "OK", + env=env)) + + build_cmd = [stage.get('build_cmd',['ninja'])] + if 'build_jobs' in stage: + build_cmd.append(WithProperties(stage['build_jobs'])) + if 'build_loadaverage' in stage: + build_cmd.append(WithProperties(stage['build_loadaverage'])) + f.addStep(WarningCountingShellCommand(name=stageName+': build', + command=build_cmd, + haltOnFailure=True, + description='building %s' % stage_build, + descriptionDone=["stage built"], + workdir=stage_build, + env=env)) + + if 'test' in stage: + check_cmd = stage.get('check_cmd',['ninja', 'check-all']) + if 'check_jobs' in stage: + check_cmd.append(WithProperties(stage['check_jobs'])) + if 'check_loadaverage' in stage: + check_cmd.append(WithProperties(stage['check_loadaverage'])) + ignoreTestFail = bool(stage['test'] != 'ignoreFail') + f.addStep(lit_test_command.LitTestCommand(name=stageName+': check', + command=check_cmd, + description="checking %s" % stage_check, + descriptionDone="stage checked", + workdir=stage_check, + haltOnFailure=ignoreTestFail, + warnOnFailure=ignoreTestFail, + flunkOnFailure=ignoreTestFail, + env=env)) + + ############# STAGE 2 + if 'usePreviousStageResults' or 'install' in stage: + install_cmd = stage.get('install_cmd',['ninja', 'install']) + if 'install_jobs' in stage: + install_cmd.append(WithProperties(stage['install_jobs'])) + if 'install_loadaverage' in stage: + install_cmd.append(WithProperties(stage['install_loadaverage'])) + ignoreInstallFail = True + if 'install' in stage: + ignoreInstallFail = bool(stage['install'] != 'ignoreFail') + f.addStep(ShellCommand(name=stageName+': install', + command=install_cmd, + description='installing %s' % stage_install, + descriptionDone=["stage installed"], + workdir=stage_build, + haltOnFailure=ignoreInstallFail, + warnOnFailure=ignoreInstallFail, + flunkOnFailure=ignoreInstallFail, + env=env)) + return f + +# CMake Linux/Windows builds def getClangCMakeBuildFactory( clean=True, test=True, @@ -469,11 +705,11 @@ f = buildbot.process.factory.BuildFactory() # We *must* checkout at least Clang+LLVM - f.addStep(SVN(name='svn-llvm', + f.addStep(buildbot.steps.source.SVN(name='svn-llvm', mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir='llvm')) - f.addStep(SVN(name='svn-clang', + f.addStep(buildbot.steps.source.SVN(name='svn-clang', mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/', defaultBranch='trunk', workdir='llvm/tools/clang')) @@ -480,12 +716,12 @@ # Extra stuff that will be built/tested if checkout_clang_tools_extra: - f.addStep(SVN(name='svn-clang-tools-extra', + f.addStep(buildbot.steps.source.SVN(name='svn-clang-tools-extra', mode='update', baseURL='http://llvm.org/svn/llvm-project/clang-tools-extra/', defaultBranch='trunk', workdir='llvm/tools/clang/tools/extra')) if checkout_compiler_rt: - f.addStep(SVN(name='svn-compiler-rt', + f.addStep(buildbot.steps.source.SVN(name='svn-compiler-rt', mode='update', baseURL='http://llvm.org/svn/llvm-project/compiler-rt/', defaultBranch='trunk', workdir='llvm/projects/compiler-rt')) @@ -616,15 +852,15 @@ f = buildbot.process.factory.BuildFactory() if update: - f.addStep(SVN(name='svn-llvm', + f.addStep(buildbot.steps.source.SVN(name='svn-llvm', mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir='llvm')) - f.addStep(SVN(name='svn-clang', + f.addStep(buildbot.steps.source.SVN(name='svn-clang', mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/', defaultBranch='trunk', workdir='llvm/tools/clang')) - f.addStep(SVN(name='svn-clang-tools-extra', + f.addStep(buildbot.steps.source.SVN(name='svn-clang-tools-extra', mode='update', baseURL='http://llvm.org/svn/llvm-project/clang-tools-extra/', defaultBranch='trunk', workdir='llvm/tools/clang/tools/extra')) @@ -695,15 +931,15 @@ f = buildbot.process.factory.BuildFactory() if update: - f.addStep(SVN(name='svn-llvm', + f.addStep(buildbot.steps.source.SVN(name='svn-llvm', mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir='llvm')) - f.addStep(SVN(name='svn-clang', + f.addStep(buildbot.steps.source.SVN(name='svn-clang', mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/', defaultBranch='trunk', workdir='llvm/tools/clang')) - f.addStep(SVN(name='svn-clang-tools-extra', + f.addStep(buildbot.steps.source.SVN(name='svn-clang-tools-extra', mode='update', baseURL='http://llvm.org/svn/llvm-project/clang-tools-extra/', defaultBranch='trunk', workdir='llvm/tools/clang/tools/extra')) @@ -781,7 +1017,7 @@ 'CC_UNDER_TEST=%s/bin/clang' % install_prefix), WithProperties( 'CXX_UNDER_TEST=%s/bin/clang++' % install_prefix)] - f.addStep(SVN(name='svn-clang-gcc-tests', mode='update', + f.addStep(buildbot.steps.source.SVN(name='svn-clang-gcc-tests', mode='update', baseURL='http://llvm.org/svn/llvm-project/clang-tests/', defaultBranch='trunk', workdir='clang-tests')) gcc_dg_ignores = ignores.get('gcc-4_2-testsuite', {}) @@ -800,7 +1036,7 @@ 'CC_UNDER_TEST=%s/bin/clang' % install_prefix), WithProperties( 'CXX_UNDER_TEST=%s/bin/clang++' % install_prefix)] - f.addStep(SVN(name='svn-clang-gdb-tests', mode='update', + f.addStep(buildbot.steps.source.SVN(name='svn-clang-gdb-tests', mode='update', baseURL='http://llvm.org/svn/llvm-project/clang-tests/', defaultBranch='trunk', workdir='clang-tests')) f.addStep(commands.SuppressionDejaGNUCommand.SuppressionDejaGNUCommand( @@ -816,7 +1052,7 @@ 'CXX_FOR_TARGET=\'{0}/bin/clang++\' ' 'CFLAGS_FOR_TARGET=\'-w -fno-limit-debug-info\'' .format(install_prefix))] - f.addStep(SVN(name='svn-clang-modern-gdb-tests', mode='update', + f.addStep(buildbot.steps.source.SVN(name='svn-clang-modern-gdb-tests', mode='update', svnurl='http://llvm.org/svn/llvm-project/clang-tests-external/trunk/gdb/7.5', workdir='clang-tests/src')) f.addStep(Configure(command='../src/configure',