Index: buildbot/osuosl/master/config/builders.py =================================================================== --- buildbot/osuosl/master/config/builders.py +++ buildbot/osuosl/master/config/builders.py @@ -374,6 +374,44 @@ extra_configure_args=['--enable-shared', '--target=hexagon-unknown-elf', '--enable-targets=hexagon'])}, + ## Hexagon check-all linux + {'name': "clang-cmake-hexagon-elf", + 'slavenames':["hexagon-build-02"], + 'builddir':"clang-cmake-hexagon-elf", + 'factory' : ClangBuilder.getClangCMakeBuildFactory( + jobs=32, + loadaverage=32, + clean=False, + checkout_compiler_rt=False, + env={'PATH':'/usr/lib/ccache:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'}, + extra_cmake_args=["-DLLVM_TARGETS_TO_BUILD='Hexagon'"])}, + {'name': "clang-msvc-i686", + 'slavenames': ["hexagon-build-01"], + 'builddir': "builddir/clang-msvc-i686", + 'factory': ClangBuilder.getClangCMakeBuildFactory( + clean=False, + checkout_compiler_rt=False, + extra_cmake_args=[ + '-DLLVM_TARGETS_TO_BUILD:=X86', + '-DLLVM_BUILD_RUNTIME:BOOL=OFF', + '-DLIBCLANG_BUILD_STATIC=ON', + '-DLLVM_ENABLE_PIC:BOOL=ON', + '-DLLVM_MINIMAL_INSTALL:BOOL=ON', + '-DLLVM_ENABLE_ASSERTIONS:BOOL=ON', + '-DLLVM_ENABLE_STATS:BOOL=ON', + '-DCMAKE_BUILD_TYPE=RelWithDebInfo', + '-DLLVM_APPEND_VC_REV:BOOL=ON' + ], + # Build and Check (MSVC) + cmakeGenerator='Visual Studio 12', + build_cmd=['MSBuild', 'ALL_BUILD.vcxproj','/maxcpucount', + "/p:Configuration=RelWithDebInfo"], + build_install_cmd=['MSBuild', 'INSTALL.vcxproj','/maxcpucount', + "/p:Configuration=RelWithDebInfo"], + build_check_cmd=['MSBuild', 'check-all.vcxproj','/maxcpucount', + "/p:Configuration=RelWithDebInfo"], + slave_envCmd=r""""%VS120COMNTOOLS%\vsvars32.bat" %PROCESSOR_ARCHITECTURE% & set""", + )}, {'name' : "clang-aarch64-lnt", 'slavenames' :["aarch64-qemu-lnt"], Index: zorg/buildbot/builders/ClangBuilder.py =================================================================== --- zorg/buildbot/builders/ClangBuilder.py +++ zorg/buildbot/builders/ClangBuilder.py @@ -2,11 +2,12 @@ import buildbot.process.factory import os -from buildbot.process.properties import WithProperties +from buildbot.process.properties import WithProperties, Property from buildbot.steps.shell import Configure, ShellCommand, SetProperty from buildbot.steps.shell import WarningCountingShellCommand from buildbot.steps.source import SVN from buildbot.steps.transfer import FileDownload +from buildbot.steps.slave import RemoveDirectory import zorg.buildbot.util.artifacts as artifacts import zorg.buildbot.builders.Util as builders_util @@ -15,6 +16,14 @@ import zorg.buildbot.commands.BatchFileDownload as batch_file_download import zorg.buildbot.commands.LitTestCommand as lit_test_command +def slave_env_glob2list(rc, stdout, stderr): + '''Extract function for SetPropertyCommand. Loads Slave Environment + into a dictionary, and returns slave_env property for ShellCommands.''' + if not rc: + slave_env_dict = dict(l.strip().split('=',1) + for l in stdout.split('\n') if len(l.split('=',1))==2) + return {'slave_env': slave_env_dict} + def getClangBuildFactory( triple=None, clean=True, @@ -450,6 +459,13 @@ test=True, cmake='cmake', jobs=None, + loadaverage=None, + + # Build and Check + cmakeGenerator="Ninja", + build_cmd=['ninja'], + build_install_cmd=['ninja', 'install'], + build_check_cmd=['ninja', 'check-all'], # Multi-stage compilation useTwoStage=False, @@ -460,6 +476,7 @@ # Environmental variables for all steps. env={}, extra_cmake_args=[], + slave_envCmd=None, # Extra repositories checkout_clang_tools_extra=True, @@ -490,17 +507,27 @@ defaultBranch='trunk', workdir='llvm/projects/compiler-rt')) + # Set slave environment, and return to slave_env. + if slave_envCmd is not None: + f.addStep(SetProperty( + command=slave_envCmd, + extract_fn=slave_env_glob2list)) + env=Property('slave_env') + + # If jobs not defined, Ninja will choose a suitable value jobs_cmd=[] lit_args="'-v" if jobs is not None: jobs_cmd=["-j"+str(jobs)] - lit_args+=" -j"+str(jobs)+"'" - else: - lit_args+="'" - ninja_cmd=['ninja'] + jobs_cmd - ninja_install_cmd=['ninja', 'install'] + jobs_cmd - ninja_check_cmd=['ninja', 'check-all'] + jobs_cmd + lit_args+=" -j"+str(jobs) + if loadaverage is not None: + jobs_cmd.extend(["-l"+str(loadaverage)]) + #lit.py:no such option -l + lit_args+="'" + build_cmd.extend(jobs_cmd) + build_install_cmd.extend(jobs_cmd) + build_check_cmd.extend(jobs_cmd) # Global configurations stage1_build='stage1' @@ -507,27 +534,34 @@ stage1_install='stage1.install' stage2_build='stage2' - ############# CLEANING - if clean: - f.addStep(ShellCommand(name='clean stage 1', - command=['rm','-rf',stage1_build], - warnOnFailure=True, - description='cleaning stage 1', - descriptionDone='clean', - workdir='.', - env=env)) + # Project file created by previous cmake run. + if 'Ninja' in cmakeGenerator: + cmakeProjectfile = 'build.ninja' + elif 'Makefiles' in cmakeGenerator: + cmakeProjectfile = 'Makefile' + elif 'Visual' in cmakeGenerator: + cmakeProjectfile = 'ALL_BUILD.vcxproj' else: - f.addStep(SetProperty(name="check ninja files 1", - workdir=stage1_build, - command=["sh", "-c", - "test -e build.ninja && echo OK || echo Missing"], - flunkOnFailure=False, - property="exists_ninja_1")) + cmakeProjectfile = 'Missing' + ############# CLEANING + doCleanIf = lambda step: step.build.getProperty("force_build_clean") or clean + f.addStep(RemoveDirectory(name='clean stage 1', + dir=stage1_build, + warnOnFailure=True, + doStepIf=doCleanIf + )) + f.addStep(SetProperty(name="check cmake project files 1", + workdir=stage1_build, + command=["sh", "-c", + "test -e %s && echo OK || echo Missing" % cmakeProjectfile], + flunkOnFailure=False, + property="exists_cmake_projectfile_1")) + ############# STAGE 1 f.addStep(ShellCommand(name='cmake stage 1', - command=[cmake, "-G", "Ninja", "../llvm", + command=[cmake, "-G", cmakeGenerator, "../llvm", "-DCMAKE_BUILD_TYPE="+stage1_config, "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS="+lit_args, @@ -536,19 +570,19 @@ haltOnFailure=True, description='cmake stage 1', workdir=stage1_build, - doStepIf=lambda step: step.build.getProperty("exists_ninja_1") != "OK", + doStepIf=lambda step: step.build.getProperty("exists_cmake_projectfile_1") != "OK", env=env)) f.addStep(WarningCountingShellCommand(name='build stage 1', - command=ninja_cmd, + command=build_cmd, haltOnFailure=True, - description='ninja all', + description='build all', workdir=stage1_build, env=env)) if test and testStage1: - f.addStep(lit_test_command.LitTestCommand(name='ninja check 1', - command=ninja_check_cmd, + f.addStep(lit_test_command.LitTestCommand(name='build check 1', + command=build_check_cmd, description=["checking stage 1"], descriptionDone=["stage 1 checked"], workdir=stage1_build, @@ -559,30 +593,26 @@ ############# STAGE 2 f.addStep(ShellCommand(name='install stage 1', - command=ninja_install_cmd, - description='ninja install', + command=build_install_cmd, + description='build install', workdir=stage1_build, env=env)) - if clean: - f.addStep(ShellCommand(name='clean stage 2', - command=['rm','-rf',stage2_build], - warnOnFailure=True, - description='cleaning stage 2', - descriptionDone='clean', - workdir='.', - env=env)) - else: - f.addStep(SetProperty(name="check ninja files 2", - workdir=stage2_build, - command=["sh", "-c", - "test -e build.ninja && echo OK || echo Missing"], - flunkOnFailure=False, - property="exists_ninja_2")) + f.addStep(RemoveDirectory(name='clean stage 2', + dir=stage2_build, + warnOnFailure=True, + doStepIf=doCleanIf + )) + f.addStep(SetProperty(name="check cmake project files 2", + workdir=stage2_build, + command=["sh", "-c", + "test -e %s && echo OK || echo Missing" % cmakeProjectfile], + flunkOnFailure=False, + property="exists_cmake_projectfile_2")) f.addStep(ShellCommand(name='cmake stage 2', - command=[cmake, "-G", "Ninja", "../llvm", + command=[cmake, "-G", cmakeGenerator, "../llvm", "-DCMAKE_BUILD_TYPE="+stage2_config, "-DLLVM_ENABLE_ASSERTIONS=True", WithProperties("-DCMAKE_C_COMPILER=%(workdir)s/"+stage1_install+"/bin/clang"), @@ -592,19 +622,19 @@ haltOnFailure=True, description='cmake stage 2', workdir=stage2_build, - doStepIf=lambda step: step.build.getProperty("exists_ninja_2") != "OK", + doStepIf=lambda step: step.build.getProperty("exists_cmake_projectfile_2") != "OK", env=env)) f.addStep(WarningCountingShellCommand(name='build stage 2', - command=ninja_cmd, + command=build_cmd, haltOnFailure=True, - description='ninja all', + description='build all', workdir=stage2_build, env=env)) if test: - f.addStep(lit_test_command.LitTestCommand(name='ninja check 2', - command=ninja_check_cmd, + f.addStep(lit_test_command.LitTestCommand(name='build check 2', + command=build_check_cmd, description=["checking stage 2"], descriptionDone=["stage 2 checked"], workdir=stage2_build,