diff --git a/buildbot/osuosl/master/config/builders.py b/buildbot/osuosl/master/config/builders.py --- a/buildbot/osuosl/master/config/builders.py +++ b/buildbot/osuosl/master/config/builders.py @@ -621,19 +621,17 @@ '-DLLVM_PARALLEL_LINK_JOBS=4', '-DLLVM_TARGETS_TO_BUILD=Mips'])}, - {'name' : "clang-ppc64be-linux-lnt", + {'name' : "clang-ppc64be-linux-test-suite", 'tags' : ["clang", "ppc"], - 'workernames' : ["ppc64be-clang-lnt-test"], - 'builddir': "clang-ppc64be-lnt", - 'factory' : ClangBuilder.getClangCMakeBuildFactory( - clean=False, - checkout_lld=False, - useTwoStage=False, - runTestSuite=True, - stage1_config='Release', - nt_flags=['--threads=16', '--build-threads=16'], - extra_cmake_args=[ + 'workernames' : ["ppc64be-clang-test-suite"], + 'builddir': "clang-ppc64be-test-suite", + 'factory' : TestSuiteBuilder.getTestSuiteBuildFactory( + depends_on_projects=["llvm", "clang", "clang-tools-extra", + "compiler-rt"], + extra_configure_args=[ "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DCMAKE_BUILD_TYPE=Release", + "-DLLVM_LIT_ARGS=-vj 20", "-DLLVM_CCACHE_BUILD=ON"])}, {'name' : "clang-ppc64be-linux-multistage", @@ -650,19 +648,17 @@ "-DLLVM_ENABLE_ASSERTIONS=ON", "-DLLVM_CCACHE_BUILD=ON"])}, - {'name' : "clang-ppc64le-linux-lnt", + {'name' : "clang-ppc64le-linux-test-suite", 'tags' : ["clang", "ppc", "ppc64le"], - 'workernames' : ["ppc64le-clang-lnt-test"], - 'builddir': "clang-ppc64le-lnt", - 'factory' : ClangBuilder.getClangCMakeBuildFactory( - clean=False, - checkout_lld=False, - useTwoStage=False, - runTestSuite=True, - stage1_config='Release', - nt_flags=['--threads=16', '--build-threads=16'], - extra_cmake_args=[ + 'workernames' : ["ppc64le-clang-test-suite"], + 'builddir': "clang-ppc64le-test-suite", + 'factory' : TestSuiteBuilder.getTestSuiteBuildFactory( + depends_on_projects=["llvm", "clang", "clang-tools-extra", + "compiler-rt"], + extra_configure_args=[ "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DCMAKE_BUILD_TYPE=Release", + "-DLLVM_LIT_ARGS=-vj 20", "-DLLVM_CCACHE_BUILD=ON"])}, {'name' : "clang-ppc64le-linux-multistage", @@ -686,7 +682,6 @@ 'builddir': "clang-ppc64le-rhel", 'factory' : TestSuiteBuilder.getTestSuiteBuildFactory( depends_on_projects=["llvm", "clang", "clang-tools-extra", "lld", "compiler-rt"], - checks=["check"], enable_runtimes="auto", extra_configure_args=[ "-DLLVM_ENABLE_ASSERTIONS=On", "-DCMAKE_C_COMPILER=clang", diff --git a/buildbot/osuosl/master/config/workers.py b/buildbot/osuosl/master/config/workers.py --- a/buildbot/osuosl/master/config/workers.py +++ b/buildbot/osuosl/master/config/workers.py @@ -72,12 +72,12 @@ create_worker("suse-gary-m68k-cross", properties={'jobs': 4}, max_builds=1), # POWER7 PowerPC big endian (powerpc64) - create_worker("ppc64be-clang-lnt-test", properties={'jobs': 16, 'vcs_protocol': 'https'}, max_builds=1), + create_worker("ppc64be-clang-test-suite", max_builds=1), create_worker("ppc64be-clang-multistage-test", properties={'jobs': 16}, max_builds=1), create_worker("ppc64be-sanitizer", properties={'jobs': 16}, max_builds=1), # POWER 8 PowerPC little endian (powerpc64le) - create_worker("ppc64le-clang-lnt-test", properties={'jobs': 8}, max_builds=1), + create_worker("ppc64le-clang-test-suite", max_builds=1), create_worker("ppc64le-clang-multistage-test", properties={'jobs': 8}, max_builds=1), create_worker("ppc64le-sanitizer", properties={'jobs': 4}, max_builds=1), create_worker("ppc64le-lld-multistage-test", max_builds=1), diff --git a/zorg/buildbot/builders/TestSuiteBuilder.py b/zorg/buildbot/builders/TestSuiteBuilder.py --- a/zorg/buildbot/builders/TestSuiteBuilder.py +++ b/zorg/buildbot/builders/TestSuiteBuilder.py @@ -1,14 +1,265 @@ -from zorg.buildbot.builders.UnifiedTreeBuilder import getCmakeWithNinjaBuildFactory - -from buildbot.plugins import util +# This builder is similar to the UnifiedTreeBuilders but it adds running +# llvm-test-suite with cmake and ninja step. +from buildbot.plugins import steps, util from zorg.buildbot.commands.CmakeCommand import CmakeCommand from zorg.buildbot.commands.NinjaCommand import NinjaCommand from zorg.buildbot.commands.LitTestCommand import LitTestCommand -# This builder is uses UnifiedTreeBuilders and adds running -# llvm-test-suite with cmake and ninja step. +from zorg.buildbot.process.factory import LLVMBuildFactory + +def getLLVMBuildFactoryAndPrepareForSourcecodeSteps( + depends_on_projects = None, + enable_runtimes = None, + llvm_srcdir = None, + obj_dir = None, + install_dir = None, + cleanBuildRequested = None, + env = None, + **kwargs): + + def cleanBuildRequestedByProperty(step): + return step.build.getProperty("clean") + + if cleanBuildRequested is None: + # We want a clean checkout only if requested by the property. + cleanBuildRequested = cleanBuildRequestedByProperty + + f = LLVMBuildFactory( + depends_on_projects=depends_on_projects, + enable_runtimes=enable_runtimes, + llvm_srcdir=llvm_srcdir, + obj_dir=obj_dir, + install_dir=install_dir, + cleanBuildRequested=cleanBuildRequested, + **kwargs) # Pass through all the extra arguments. + + # Remove the source code for a clean checkout if requested by property. + # TODO: Some Windows workers do not handle RemoveDirectory command well. + # So, consider running "rmdir /S /Q " if the build runs on Windows. + f.addStep(steps.RemoveDirectory(name='Clean source directory', + dir=f.monorepo_dir, + haltOnFailure=False, + flunkOnFailure=False, + doStepIf=cleanBuildRequestedByProperty, + )) + + return f + +def getLLVMBuildFactoryAndSourcecodeSteps( + depends_on_projects = None, + enable_runtimes = None, + llvm_srcdir = None, + src_to_build_dir = None, + obj_dir = None, + install_dir = None, + cleanBuildRequested = None, + **kwargs): + + f = getLLVMBuildFactoryAndPrepareForSourcecodeSteps( + depends_on_projects=depends_on_projects, + enable_runtimes=enable_runtimes, + llvm_srcdir=llvm_srcdir, + src_to_build_dir=src_to_build_dir, + obj_dir=obj_dir, + install_dir=install_dir, + cleanBuildRequested=cleanBuildRequested, + **kwargs) # Pass through all the extra arguments. + + # Get the source code. + f.addGetSourcecodeSteps(**kwargs) + + return f + +def addCmakeSteps( + f, + cleanBuildRequested, + obj_dir, + generator=None, + install_dir = None, + extra_configure_args = None, + env = None, + **kwargs): + + # Make a local copy of the configure args, as we are going to modify that. + if extra_configure_args: + cmake_args = extra_configure_args[:] + else: + cmake_args = list() + + if obj_dir is None: + obj_dir = f.obj_dir + + # This is an incremental build, unless otherwise has been requested. + # Remove obj and install dirs for a clean build. + # TODO: Some Windows workers do not handle RemoveDirectory command well. + # So, consider running "rmdir /S /Q " if the build runs on Windows. + f.addStep(steps.RemoveDirectory(name='Clean %s directory' % obj_dir, + dir=obj_dir, + haltOnFailure=False, + flunkOnFailure=False, + doStepIf=cleanBuildRequested, + )) + + if f.enable_projects: + CmakeCommand.applyDefaultOptions(cmake_args, [ + ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.enable_projects)), + ]) + + if f.enable_runtimes: + CmakeCommand.applyDefaultOptions(cmake_args, [ + ('-DLLVM_ENABLE_RUNTIMES=', ";".join(f.enable_runtimes)), + ]) + + if install_dir: + install_dir_rel = LLVMBuildFactory.pathRelativeTo( + install_dir, + obj_dir) + CmakeCommand.applyRequiredOptions(cmake_args, [ + ('-DCMAKE_INSTALL_PREFIX=', install_dir_rel), + ]) + + f.addStep(steps.RemoveDirectory(name='Clean %s directory' % install_dir, + dir=install_dir, + haltOnFailure=False, + flunkOnFailure=False, + doStepIf=cleanBuildRequested, + )) + + # Reconcile the cmake options for this build. + # Set proper defaults. + CmakeCommand.applyDefaultOptions(cmake_args, [ + ('-DCMAKE_BUILD_TYPE=', 'Release'), + ('-DLLVM_ENABLE_ASSERTIONS=', 'ON'), + ('-DLLVM_LIT_ARGS=', '-v -vv'), + ]) + + # Create configuration files with cmake, unless this has been already done + # for an incremental build. + step_name = "cmake configure llvm" + + src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, obj_dir) + + # Make a local copy of the configure args, as we are going to modify that. + definitions = dict() + options = list() + for d in cmake_args: + if isinstance(d, str) and d.startswith("-D"): + k,v = d[2:].split('=', 1) + definitions[k] = v + else: + options.append(d) + + f.addStep(CmakeCommand(name=step_name, + haltOnFailure=True, + description="Running cmake on llvm dir", + generator=generator, + definitions=definitions, + options=options, + path=src_dir, + env=env or {}, + workdir=obj_dir, + **kwargs # Pass through all the extra arguments. + )) + +def addNinjaSteps( + f, + obj_dir = None, + targets = None, + checks = None, + install_dir = None, + env = None, + stage_name = None, + **kwargs): + + if obj_dir is None: + obj_dir = f.obj_dir + + step_name = "Build llvm with ninja" + step_description=["Build llvm with ninja"] + + if targets: + step_name = "{} for {}".format(step_name, targets) + step_description.extend(targets) + + if isinstance(env, dict): + build_env = env.copy() if env else dict() + build_env['NINJA_STATUS'] = build_env.get('NINJA_STATUS', "%e [%f/%r/%t] ") + else: + build_env = env or {} + + # Build the unified tree. + f.addStep(NinjaCommand(name=step_name, + haltOnFailure=True, + targets=targets, + description=step_description, + env=build_env, + workdir=obj_dir, + **kwargs # Pass through all the extra arguments. + )) + + # Test just built components if requested. + # Note: At this point env could be None, a dictionary, or a Property object. + if isinstance(env, dict): + check_env = env.copy() if env else dict() + check_env['NINJA_STATUS'] = check_env.get('NINJA_STATUS', "%e [%f/%r/%t] ") + else: + check_env = env or {} + + for check in checks: + f.addStep(LitTestCommand(name="Run lit tests %s" % (check), + command=['ninja', check], + description=[ + "Test", "just", "built", "components", "for", + check, + ], + env=check_env, + workdir=obj_dir, + **kwargs # Pass through all the extra arguments. + )) + + # Install just built components + if install_dir: + # TODO: Run this step only if none of the prevous failed. + f.addStep(NinjaCommand(name="Install llvm", + targets=["install"], + description=["Install", "just", "built", "components"], + env=env or {}, + workdir=obj_dir, + **kwargs # Pass through all the extra arguments. + )) + +def getCmakeBuildFactory( + depends_on_projects = None, + enable_runtimes = None, + llvm_srcdir = None, + obj_dir = None, + install_dir = None, + clean = False, + extra_configure_args = None, + env = None, + **kwargs): + + f = getLLVMBuildFactoryAndSourcecodeSteps( + depends_on_projects=depends_on_projects, + enable_runtimes=enable_runtimes, + llvm_srcdir=llvm_srcdir, + obj_dir=obj_dir, + install_dir=install_dir, + **kwargs) # Pass through all the extra arguments. + + cleanBuildRequested = lambda step: step.build.getProperty("clean", default=step.build.getProperty("clean_obj")) or clean + addCmakeSteps( + f, + cleanBuildRequested=cleanBuildRequested, + obj_dir=f.obj_dir, + install_dir=f.install_dir, + extra_configure_args=extra_configure_args, + env=env, + **kwargs) + + return f def addTestSuiteStep( f, compiler_dir = None, @@ -30,22 +281,22 @@ src_dir=test_suite_src_dir, alwaysUseLatest=True) - f.addStep(CmakeCommand(name='cmake Test Suite', + f.addStep(CmakeCommand(name='cmake configure test-suite', haltOnFailure=True, - description='Running cmake on Test Suite dir', + description='Running cmake on the LLVM Test-Suite dir', workdir=test_suite_workdir, options=options, path=test_suite_src_dir, generator='Ninja')) - f.addStep(NinjaCommand(name='ninja Test Suite', - description='Running Ninja on Test Suite dir', + f.addStep(NinjaCommand(name='Build test-suite with ninja', + description='Running Ninja on the LLVM Test-Suite dir', haltOnFailure=True, workdir=test_suite_workdir)) - f.addStep(LitTestCommand(name='Run Test Suite with lit', + f.addStep(LitTestCommand(name='Run test-suite with lit', haltOnFailure=True, - description='Running test suite tests', + description='Running the LLVM Test-Suite tests', workdir=test_suite_workdir, command=[lit] + lit_args + ['.'], env=env, @@ -74,18 +325,45 @@ lit_args = arg.split("=")[1] lit_args = lit_args.split(" ") - f = getCmakeWithNinjaBuildFactory( - depends_on_projects = depends_on_projects, - enable_runtimes = enable_runtimes, - targets = targets, - llvm_srcdir = llvm_srcdir, - obj_dir = obj_dir, - checks = checks, - install_dir = install_dir, - clean = clean, - extra_configure_args = extra_configure_args, - env = env, - **kwargs) + # Make a local copy of the configure args, as we are going to modify that. + if extra_configure_args: + cmake_args = extra_configure_args[:] + else: + cmake_args = list() + + if checks is None: + checks = ['check-all'] + + # Prepare environmental variables. Set here all env we want everywhere. + merged_env = { + 'TERM' : 'dumb' # Be cautious and disable color output from all tools. + } + if env: + # Overwrite pre-set items with the given ones, so user can set anything. + merged_env.update(env) + + # Some options are required for this build no matter what. + CmakeCommand.applyRequiredOptions(cmake_args, [('-G', 'Ninja')]) + + f = getCmakeBuildFactory( + depends_on_projects=depends_on_projects, + enable_runtimes=enable_runtimes, + llvm_srcdir=llvm_srcdir, + obj_dir=obj_dir, + install_dir=install_dir, + clean=clean, + extra_configure_args=cmake_args, + env=merged_env, + **kwargs) # Pass through all the extra arguments. + + addNinjaSteps( + f, + obj_dir=f.obj_dir, + targets=targets, + checks=checks, + install_dir=f.install_dir, + env=merged_env, + **kwargs) addTestSuiteStep(f,