Index: zorg/buildbot/builders/ClangBuilder.py =================================================================== --- zorg/buildbot/builders/ClangBuilder.py +++ zorg/buildbot/builders/ClangBuilder.py @@ -2,11 +2,13 @@ import buildbot.process.factory import os -from buildbot.process.properties import WithProperties +from copy import deepcopy +from buildbot.process.properties import WithProperties, Property from buildbot.steps.shell import Configure, ShellCommand, SetProperty from buildbot.steps.shell import WarningCountingShellCommand -from buildbot.steps.source import SVN +from buildbot.steps.source import SVN, Git from buildbot.steps.transfer import FileDownload +from buildbot.steps.slave import RemoveDirectory import zorg.buildbot.util.artifacts as artifacts import zorg.buildbot.builders.Util as builders_util @@ -15,6 +17,62 @@ import zorg.buildbot.commands.BatchFileDownload as batch_file_download import zorg.buildbot.commands.LitTestCommand as lit_test_command +def slave_env_glob2list(rc, stdout, stderr): + '''Extract function for SetPropertyCommand. Loads Slave Environment + into a dictionary, and returns slave_env property for ShellCommands.''' + if not rc: + slave_env_dict = dict(l.strip().split('=',1) + for l in stdout.split('\n') if len(l.split('=',1))==2) + return {'slave_env': slave_env_dict} + +# SCM, workdir, baseURL, branch, mode +llvmSourceTree = { + 'svn-llvm': ["SVN","%s",'http://llvm.org/svn/llvm-project/llvm/','trunk','update'], + 'svn-clang': ["SVN","%s/tools/clang",'http://llvm.org/svn/llvm-project/cfe/','trunk','update'], + 'svn-clang-tools-extra': ["SVN","%s/tools/clang/tools/extra",'http://llvm.org/svn/llvm-project/clang-tools-extra/','trunk','update'], + 'svn-lldb': ["SVN","%s/tools/lldb",'http://llvm.org/svn/llvm-project/lldb/','trunk','update'], + 'svn-polly': ["SVN","%s/tools/polly",'http://llvm.org/svn/llvm-project/polly/','trunk','update'], + 'svn-libcxx': ["SVN","%s/projects/libcxx",'http://llvm.org/svn/llvm-project/libcxx/','trunk','update'], + 'svn-libcxx-pull': ["SVN","%s/projects/libcxx",'http://llvm.org/svn/llvm-project/libcxx/','trunk','pull'], + 'svn-libcxxabi': ["SVN","%s/projects/libcxxabi",'http://llvm.org/svn/llvm-project/libcxxabi/','trunk','update'], + 'svn-compiler-rt': ["SVN","%s/projects/compiler-rt",'http://llvm.org/svn/llvm-project/compiler-rt/','trunk','update'], + 'svn-dragonegg': ["SVN","dragonegg.src",'http://llvm.org/svn/llvm-project/dragonegg/','trunk','update'], + 'svn-test-suite': ["SVN","test-suite.src",'http://llvm.org/svn/llvm-project/test-suite/','trunk','update'], + 'svn-libiomp5': ["SVN","openmp.src",'http://llvm.org/svn/llvm-project/openmp/','trunk','update'], + 'git-imath': ["GIT","imath.src",'git://codeaurora.org/quic/iml/imath/','isl','update'], + 'git-isl': ["GIT","isl.src",'http://repo.or.cz/r/isl.git/','master','update'], +} +# TODO 'svn-lit': ["SVN","lit.src",'http://llvm.org/svn/llvm-project/llvm/trunk/utils/lit','trunk','incremental'], +# Git mirror not needed 'git-llvm': ["GIT","%s",'http://llvm.org/git/llvm','master','update'], + +def scmSvn(f,llvmsrcdir,project=''): + (scm, workdir, baseURL, branch, mode) = llvmSourceTree[project] + f = f.addStep(SVN(name=project, + mode=mode, + baseURL=baseURL, + defaultBranch=branch, + workdir=workdir % llvmsrcdir)) + return f + +# http://llvm.org/git is a mirror of http://llvm.org/svn +def scmGit(f,llvmsrcdir,project=''): + (scm, workdir, baseURL, branch, mode) = llvmSourceTree[project] + f = f.addStep(Git(name=project, + repourl=baseURL, + branchranch=branch, + alwaysUseLatest=True, + haltOnFailure=True, + workdir=workdir % llvmsrcdir)) + return f + +SCMs = {'SVN':scmSvn,'GIT':scmGit} + +def getLLVMSource(f,llvmsrcdir='.',project_list=[]): + for project in project_list: + if llvmSourceTree[project]: + SCMs[llvmSourceTree[project][0]](f,llvmsrcdir,project) + return f + def getClangBuildFactory( triple=None, clean=True, @@ -444,7 +502,157 @@ return f -# CMake Linux builds + +def getCMakeBuildFactory( + build=[ + {'name': "stage 1", + 'source': ['svn-llvm','svn-clang'], + 'source_dir': 'llvm', + 'config': "Release", + 'slave_envCmd': r"""printenv""", + 'cmakeGenerator': 'Ninja', + 'cmakeProjectfile': 'build.ninja', + 'extra_cmake_args': [], + 'build_cmd': ['ninja'], + 'build_dir': 'stage1.build', + 'install_cmd': ['ninja', 'install'], + 'install_dir': 'stage1.install', + 'check_cmd': ['ninja', 'check-all'], + 'check_dir': '', + 'useTwoStage': True, + 'test': 'ignoreFail', + }, + {'name': "stage 2", + 'useBuiltClang': True, + 'config': "Release", + 'cmakeGenerator': 'Ninja', + 'cmakeProjectfile': 'build.ninja', + 'extra_cmake_args': [], + 'build_cmd': ['ninja'], + 'build_dir': 'stage2.build', + 'install_cmd': ['ninja', 'install'], + 'install_dir': 'stage2.install', + 'built_install_dir': 'stage1.install', + 'check_cmd': ['ninja', 'check-all'], + 'test': 'ignoreFail', + #'check_dir': None, + }, + ], + update=[] # Update each stage [{},...] + ): + + # Apply updates to each stage. [{},{}] + newBuild = deepcopy(build) + for u,b in zip(update,newBuild): + b.update(u) + + # Must have source from one of the stages. + haveSource = False + + f = buildbot.process.factory.BuildFactory() + + # Drive thru build stages. + for stage in newBuild: + # Set the stage name + stageName = stage.get('name','stage') + + # Check for source first. + if 'source' in stage: + stage_source = stage.get('source_dir','llvm') + f = getLLVMSource(f,stage_source,stage['source']) + haveSource = True + + # Set and return slave environment, if slave_envCmd defined. + if 'slave_envCmd' in stage: + f.addStep(SetProperty( + name='get slave env', + command=stage['slave_envCmd'], + extract_fn=slave_env_glob2list)) + env=Property('slave_env') + + # Global configurations + stage_build = stage['build_dir'] or 'stage.build' + stage_install = stage['install_dir'] or 'stage.install' + if 'check_dir' in stage: + stage_check = stage_build+"/"+str(stage['check_dir']) + else: + stage_check = stage_build + + ############# CLEANING + cleanBuildRequested = lambda step: step.build.getProperty("clean") or stage.get('clean',False) + f.addStep(RemoveDirectory(name=stageName+': clean', + dir=stage_build, + warnOnFailure=True, + doStepIf=cleanBuildRequested + )) + cmakeProjectfile = stage.get('cmakeProjectfile','build.ninja') + f.addStep(SetProperty(name=stageName+': check cmake project files', + workdir=stage_build, + command=["sh", "-c", + "test -e %s && echo OK || echo Missing" % cmakeProjectfile], + flunkOnFailure=False, + property="exists_cmake_projectfile")) + + ############# STAGE + cmakeGenerator = stage.get('cmakeGenerator','Ninja') + stage_config = stage.get('config','Release') + lit_args = stage.get('lit_args',"-v") + useBuiltClang = [] + if 'useBuiltClang' in stage: + built_install_dir = stage.get('built_install_dir','stage1.install') + useBuiltClang=[ + WithProperties("-DCMAKE_C_COMPILER=%(workdir)s/"+built_install_dir+"/bin/clang"), + WithProperties("-DCMAKE_CXX_COMPILER=%(workdir)s/"+built_install_dir+"/bin/clang++") + ] + extra_cmake_args = stage.get('extra_cmake_args',[]) + f.addStep(ShellCommand(name=stageName+': cmake', + command=["cmake", "-G", cmakeGenerator, "../"+stage_source, + "-DCMAKE_BUILD_TYPE="+stage_config, + "-DLLVM_ENABLE_ASSERTIONS=True", + "-DLLVM_LIT_ARGS="+lit_args, + "-DCMAKE_INSTALL_PREFIX=../"+stage_install] + + useBuiltClang + + extra_cmake_args, + haltOnFailure=True, + description='cmake stage', + workdir=stage_build, + doStepIf=lambda step: step.build.getProperty("exists_cmake_projectfile") != "OK", + env=Property('slave_env'))) + + build_cmd = stage.get('build_cmd','ninja') + f.addStep(WarningCountingShellCommand(name=stageName+': build', + command=build_cmd, + haltOnFailure=True, + description='build all', + workdir=stage_build, + env=Property('slave_env'))) + + build_check_cmd = stage.get('check_cmd','ninja check-all') + if 'test' in stage: + ignoreFail = (stage['test'] == 'ignoreFail') + f.addStep(lit_test_command.LitTestCommand(name=stageName+': build check', + command=build_check_cmd, + description=["checking stage"], + descriptionDone=["stage checked"], + workdir=stage_check, + flunkOnFailure=ignoreFail, + env=Property('slave_env'))) + + ############# STAGE 2 + if 'useTwoStage' in stage: + build_install_cmd = stage.get('install_cmd','ninja install') + f.addStep(ShellCommand(name=stageName+': install', + command=build_install_cmd, + description='build install', + workdir=stage_build, + env=Property('slave_env'))) + + return f + +def XorY(x,y): + return x or y + +# CMake Linux/Windows builds def getClangCMakeBuildFactory( clean=True, test=True,