diff --git a/buildbot/osuosl/master/config/builders.py b/buildbot/osuosl/master/config/builders.py --- a/buildbot/osuosl/master/config/builders.py +++ b/buildbot/osuosl/master/config/builders.py @@ -18,6 +18,7 @@ from zorg.buildbot.builders import AnnotatedBuilder from zorg.buildbot.builders import LLDPerformanceTestsuite from zorg.buildbot.builders import FuchsiaBuilder +from zorg.buildbot.builders import LibcBuilder # Plain LLVM builders. def _get_llvm_builders(): @@ -1339,6 +1340,20 @@ ])}, ] +def _get_libc_builders(): + return [ + {'name': "libc-x86_64-debian", + 'slavenames': ["libc-x86_64-debian"], + 'builddir': "libc-x86_64-debian", + 'category' : 'libc', + 'factory': LibcBuilder.getBuildFactory(clean=False)}, + {'name': "libc-x86_64-debian-asan", + 'slavenames': ["libc-x86_64-debian"], + 'builddir': "libc-x86_64-debian-asan", + 'category' : 'libc', + 'factory': LibcBuilder.getBuildFactory(clean=False, asan=True)}, + ] + # Experimental and stopped builders def _get_on_demand_builders(): return [ diff --git a/buildbot/osuosl/master/config/slaves.py b/buildbot/osuosl/master/config/slaves.py --- a/buildbot/osuosl/master/config/slaves.py +++ b/buildbot/osuosl/master/config/slaves.py @@ -188,6 +188,9 @@ # Debian x86_64 Buster Xeon(R) Gold 6154 CPU @ 3.00GHz, 192GB RAM create_slave("lldb-x86_64-debian", properties={'jobs': 72}, max_builds=1), + # Debian x86_64 Intel Broadwell 32 CPUs, 120 GB RAM + create_slave("libc-x86_64-debian", properties={'jobs': 32}, max_builds=2), + # Windows dellfx2-sled3 create_slave("as-builder-3", max_builds=1), diff --git a/buildbot/osuosl/master/config/status.py b/buildbot/osuosl/master/config/status.py --- a/buildbot/osuosl/master/config/status.py +++ b/buildbot/osuosl/master/config/status.py @@ -247,4 +247,12 @@ builders = ["llvm-clang-x86_64-win-fast","lld-x86_64-ubuntu-fast", "llvm-clang-x86_64-expensive-checks-ubuntu"], addLogs=False), + InformativeMailNotifier( + fromaddr = "llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers = True, + extraRecipients = ["sivachandra@google.com"], + subject="Build %(builder)s Failure", + mode = "failing", + builders = ["libc-x86_64-debian", "libc-x86_64_debian-asan"], + addLogs=False), ] diff --git a/buildbot/osuosl/master/master.cfg b/buildbot/osuosl/master/master.cfg --- a/buildbot/osuosl/master/master.cfg +++ b/buildbot/osuosl/master/master.cfg @@ -42,6 +42,7 @@ "clang-tools-extra", "polly", "compiler-rt", + "libc", "libcxx", "libcxxabi", "libunwind", @@ -225,6 +226,12 @@ "llvm", "polly"]))) +c['schedulers'].append(SingleBranchScheduler(name="libc_scheduler", + treeStableTimer=2*60, + builderNames=get_all_for("libc"), + change_filter=depends_on(["llvm", + "libc"]))) + c['schedulers'].extend( config.schedulers.getSingleBranchSchedulers(c['builders'], c['schedulers']) ) diff --git a/zorg/buildbot/builders/LibcBuilder.py b/zorg/buildbot/builders/LibcBuilder.py new file mode 100644 --- /dev/null +++ b/zorg/buildbot/builders/LibcBuilder.py @@ -0,0 +1,37 @@ +from zorg.buildbot.commands.AnnotatedCommand import AnnotatedCommand +from zorg.buildbot.process.factory import LLVMBuildFactory + +LIBC_BUILDER_DIR = "libc_builder" +LLVM_ZORG = "%s/llvm-zorg" % LIBC_BUILDER_DIR +ANNOTATED_STEP_RUNNER = ( + "%s/zorg/buildbot/builders/libc/annotated_step_runner.py" % LLVM_ZORG) + + +def getBuildFactory(clean=False, asan=False, timeout=2400): + f = LLVMBuildFactory(clean=clean, is_legacy_mode=False, + depends_on_projects=["llvm", "libc"]) + + # Get llvm-zorg + f.addGetSourcecodeForProject( + name='checkout-zorg', + project='zorg', + src_dir=LIBC_BUILDER_DIR, + alwaysUseLatest=True) + + additional_env = {} + if clean: + additional_env["BUILDBOT_CLOBBER"] = "1" + + annotated_step_cmd = [ANNOTATED_STEP_RUNNER] + if clean: + annotated_step_cmd.append("--clean") + if asan: + annotated_step_cmd.append("--asan") + + f.addStep(AnnotatedCommand(name='run_annotated_steps', + description=["Run annotated steps"], + command=annotated_step_command, + haltOnFailure=True, + timeout=timeout, + env=additional_env)) + return f diff --git a/zorg/buildbot/builders/libc/__init__.py b/zorg/buildbot/builders/libc/__init__.py new file mode 100644 --- /dev/null +++ b/zorg/buildbot/builders/libc/__init__.py @@ -0,0 +1 @@ + diff --git a/zorg/buildbot/builders/libc/annotated_step_runner.py b/zorg/buildbot/builders/libc/annotated_step_runner.py new file mode 100755 --- /dev/null +++ b/zorg/buildbot/builders/libc/annotated_step_runner.py @@ -0,0 +1,90 @@ +#! /usr/bin/python + +import argparse +import os + +BUILD_DIR = "build" +MONO_REPO_DIR = "llvm-project" + + +def get_options(): + parser = argparse.ArgumentParser() + parser.add_argument("--asan", action="store_true", + help="Build with address sanitizer enabled.") + parser.add_argument("--clean", action="store_true", + help="Do a clobber/clean build.") + return parser.parse_args() + + +def run_cmd(cmd, pwd): + print("Running command: %s" % cmd) + print("Currently all commands are passthrough.") + return 0 + + +def run_step(step_name, cmd, pwd=None): + print("@@@BUILD_STEP %s@@@" % step_name) + retcode = run_cmd(cmd, pwd) + if retcode == 0: + print("@@@BUILD_STEP %s@@@" % step_name) + else: + print("@@@STEP_FAILURE@@@") + + +def create_build_dir(opts): + if not os.path.exists(BUILD_DIR): + run_step("create_build_dir", ["mkdir", "build"]) + + +def clean_build(opts): + should_clobber = (os.environ.get("BUILDBOT_CLOBBER", "0") == "1") + if opts.clean or should_clobber: + run_step("clean_build", ["rm", "-rf", "*"], BUILD_DIR) + + +def update_monorepo(opts): + if not os.path.exists(MONO_REPO_DIR): + run_step("clone_llvm_monorepo", + ["git", "clone", "https://github.com/llvm/llvm-project.git"]) + run_step("update_llvm_monorepo", ["git", "pull", "-r"], MONO_REPO_DIR) + + revision = os.environ.get("BUILDBOT_REVISION") + run_step("checkout_revision", ["git", "checkout", revision], MONO_REPO_DIR) + + +def cmake_configure(opts): + cmd = ["cmake", "-G", "Ninja", "../llvm_project/llvm", + "-DLLVM_ENABLE_PROJECTS=libc"] + if opts.asan: + cmd.append("-DLLVM_ENABLE_SANITIZER=Address") + run_step("cmake_configure", cmd, BUILD_DIR) + + +def build_llvmlibc(opts): + cmd = ["ninja", "llvmlibc"] + run_step("build_llvmlibc", cmd, BUILD_DIR) + + +def run_unittests(opts): + cmd = ["ninja", "llvm_libc_unittests"] + run_step("unittests", cmd, BUILD_DIR) + + +STEPS = [ + create_build_dir, + clean_build, + update_monorepo, + cmake_configure, + build_llvmlibc, + run_unittests, +] + + +def main(): + opts = get_options() + for step in STEPS: + step(opts) + + +if __name__ == "__main__": + main()