[libc builder] Use annotated builder for libc buildbot.

[PaulkaToast]

Summary

This review depends on this one landing: https://reviews.llvm.org/D74193. (landed)

Do I need to specify a scheduler if I'm using the annotated builder?
Also please let me know if I'm missing anything.

Thanks! (:

[PaulkaToast]

This is now tested locally and ready for review (:
@gkistanova

[sivachandra]

I am OK with this change as a first step. But, we should eventually move to a custom script as the libc project will want a lot more customization.

Please wait for Galina to approve.

[gkistanova]

Thanks, Paula!

Do I need to specify a scheduler if I'm using the annotated builder?

No, you do not need to specify a scheduler if you are using the annotated builder. Zorg will take care of that for you.

buildbot/osuosl/master/config/builders.py

def _get_toolchain_builders():
                           "lld",
                           "lldb",
                           "llvm",
                           "openmp",
                           "polly",
                       ])},
         ]
 
+# libc Builders
+def _get_libc_builders():
+    return [
+        {'name': 'libc-x86_64-debian',
+         'slavenames': ['libc-x86_64-debian'],
+         'builddir': 'libc-x86_64-debian',
+         'category': 'libc',
+         'factory' : AnnotatedBuilder.getAnnotatedBuildFactory(
+             script="libc-linux.py",
+             depends_on_projects=['llvm', 'libc'])},
+
+        {'name': "libc-x86_64-debian-asan",
+         'slavenames': ["libc-x86_64-debian"],
+         'builddir': "libc-x86_64-debian-asan",
+         'category' : 'libc',
+         'factory' : AnnotatedBuilder.getAnnotatedBuildFactory(
+             script="libc-linux.py",
+             depends_on_projects=['llvm', 'libc'],
+             extra_args=['--asan'])},
+    ]
 
 # Experimental and stopped builders
 def _get_on_demand_builders():
     return [
         ]
 
 def _get_experimental_scheduled_builders():
     return [
def get_builders():
     for b in _get_libcxx_builders():
         b['category'] = 'libcxx'
         yield b
 
     for b in _get_toolchain_builders():
         b['category'] = 'toolchain'
         yield b
 
+    for b in _get_libc_builders():
+        b['category'] = 'libc'
+        yield b
+
     for b in _get_documentation_builders():
         yield b
 
     for b in _get_experimental_scheduled_builders():
         if not b.get('category', '').endswith('.exp'):
            b['category'] = b.get('category', '') + '.exp'
         yield b
 
     for b in _get_on_demand_builders():
         if not b.get('category', '').endswith('.on-demand'):
            b['category'] = b.get('category', '') + '.on-demand'
         yield b

buildbot/osuosl/master/config/slaves.py

return [
         create_slave("lldb-s390x-fedora", properties={'jobs': 4}, max_builds=1),
 
         # Fedora latest stable ppc64le, POWER8 (architected), 2 cores, 4GB RAM
         create_slave("lldb-ppc64le-fedora", properties={'jobs': 2}, max_builds=1),
 
         # Debian x86_64 Buster Xeon(R) Gold 6154 CPU @ 3.00GHz, 192GB RAM
         create_slave("lldb-x86_64-debian", properties={'jobs': 72}, max_builds=1),
 
+        # Debian x86_64 Intel Broadwell 32 CPUs, 120 GB RAM
+        create_slave("libc-x86_64-debian", properties={'jobs': 32}, max_builds=2),
+
         # Windows Server on Xeon Gold 6130 (2x2.1GHz), 128Gb of RAM
         create_slave("as-builder-1", properties={
                         'remote_test_host': 'jetson6.lab.llvm.org',
                         'remote_test_user': 'ubuntu'
                      },
                      max_builds=1),
 
         # Windows Server on Xeon Gold 6130 (2x2.1GHz), 128Gb of RAM

buildbot/osuosl/master/config/status.py

return [
         InformativeMailNotifier(
             fromaddr = "[e-mail]",
             sendToInterestedUsers = False,
             extraRecipients = ["[e-mail]"],
             subject="Build %(builder)s Failure",
             mode = "failing",
             builders = ["clang-ve-ninja"],
             addLogs=False),
+         InformativeMailNotifier(
+            fromaddr = "[e-mail]",
+            sendToInterestedUsers = True,
+            extraRecipients = ["[e-mail]", "[e-mail]"],
+            subject="Build %(builder)s Failure",
+            mode = "failing",
+            builders = ["libc-x86_64-debian", "libc-x86_64_debian-asan"],
+            addLogs=False),
         ]

buildbot/osuosl/master/master.cfg

 
 llvm_projects = [
     "llvm",
     "cfe",
     "clang-tests-external",
     "clang-tools-extra",
     "polly",
     "compiler-rt",
+    "libc",
     "libcxx",
     "libcxxabi",
     "libunwind",
     "lld",
     "lldb",
     "openmp",
     "lnt",
     "test-suite"]

zorg/buildbot/builders/LibcBuilder.py

-from zorg.buildbot.commands.AnnotatedCommand import AnnotatedCommand
-from zorg.buildbot.process.factory import LLVMBuildFactory
-
-LIBC_BUILDER_DIR = "libc_builder"
-LLVM_ZORG = "%s/llvm-zorg" % LIBC_BUILDER_DIR
-ANNOTATED_STEP_RUNNER = (
-    "%s/zorg/buildbot/builders/libc/annotated_step_runner.py" % LLVM_ZORG)
-
-
-def getBuildFactory(clean=False, asan=False, timeout=2400):
-    f = LLVMBuildFactory(clean=clean, is_legacy_mode=False,
-                         depends_on_projects=["llvm", "libc"])
-
-    # Get llvm-zorg
-    f.addGetSourcecodeForProject(
-        name='checkout-zorg',
-        project='zorg',
-        src_dir=LIBC_BUILDER_DIR,
-        alwaysUseLatest=True)
-
-    additional_env = {}
-    if clean:
-        additional_env["BUILDBOT_CLOBBER"] = "1"
-
-    annotated_step_cmd = [ANNOTATED_STEP_RUNNER]
-    if clean:
-        annotated_step_cmd.append("--clean")
-    if asan:
-        annotated_step_cmd.append("--asan")
-
-    f.addStep(AnnotatedCommand(name='run_annotated_steps',
-                               description=["Run annotated steps"],
-                               command=annotated_step_command,
-                               haltOnFailure=True,
-                               timeout=timeout,
-                               env=additional_env))
-    return f 

zorg/buildbot/builders/annotated/libc-linux.py

+#!/usr/bin/python
+
+import os
+import sys
+import annotated_builder
+
+
+def main(argv):
+    ap = annotated_builder.get_argument_parser()
+    ap.add_argument('--asan', action='store_true', default=False)
+    args = ap.parse_args(argv[1:])
+
+    if args.asan:
+        extra_cmake_args = ['-DLLVM_ENABLE_SANITIZER=Address']
+    else:
+        extra_cmake_args = []
+
+    projects = ['llvm', 'libc']
+    check_targets = ['check-libc']
+
+    builder = annotated_builder.AnnotatedBuilder()
+    builder.run_steps(projects=projects,
+                      check_targets=check_targets,
+                      extra_cmake_args=extra_cmake_args)
+
+
+if __name__ == '__main__':
+    sys.path.append(os.path.dirname(__file__))
+    sys.exit(main(sys.argv))

zorg/buildbot/builders/libc/__init__.py

-

zorg/buildbot/builders/libc/annotated_step_runner.py

-#! /usr/bin/python
-
-import argparse
-import os
-
-BUILD_DIR = "build"
-MONO_REPO_DIR = "llvm-project"
-
-
-def get_options():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--asan", action="store_true",
-                        help="Build with address sanitizer enabled.")
-    parser.add_argument("--clean", action="store_true",
-                        help="Do a clobber/clean build.")
-    return parser.parse_args()
-
-
-def run_cmd(cmd, pwd):
-    print("Running command: %s" % cmd)
-    print("Currently all commands are passthrough.")
-    return 0
-
-
-def run_step(step_name, cmd, pwd=None):
-    print("@@@BUILD_STEP %s@@@" % step_name)
-    retcode = run_cmd(cmd, pwd)
-    if retcode == 0:
-        print("@@@BUILD_STEP %s@@@" % step_name)
-    else:
-        print("@@@STEP_FAILURE@@@")
-
-
-def create_build_dir(opts):
-    if not os.path.exists(BUILD_DIR):
-      run_step("create_build_dir", ["mkdir", "build"])
-
-
-def clean_build(opts):
-    should_clobber = (os.environ.get("BUILDBOT_CLOBBER", "0") == "1")
-    if opts.clean or should_clobber:
-        run_step("clean_build", ["rm", "-rf", "*"], BUILD_DIR)
-
-
-def update_monorepo(opts):
-    if not os.path.exists(MONO_REPO_DIR):
-        run_step("clone_llvm_monorepo",
-                 ["git", "clone", "https://github.com/llvm/llvm-project.git"])
-    run_step("update_llvm_monorepo", ["git", "pull", "-r"], MONO_REPO_DIR)
-
-    revision = os.environ.get("BUILDBOT_REVISION")
-    run_step("checkout_revision", ["git", "checkout", revision], MONO_REPO_DIR)
-
-
-def cmake_configure(opts):
-    cmd = ["cmake", "-G", "Ninja", "../llvm_project/llvm",
-           "-DLLVM_ENABLE_PROJECTS=libc"]
-    if opts.asan:
-        cmd.append("-DLLVM_ENABLE_SANITIZER=Address")
-    run_step("cmake_configure", cmd, BUILD_DIR)
-
-
-def build_llvmlibc(opts):
-    cmd = ["ninja", "llvmlibc"]
-    run_step("build_llvmlibc", cmd, BUILD_DIR)
-
-
-def run_unittests(opts):
-    cmd = ["ninja", "llvm_libc_unittests"]
-    run_step("unittests", cmd, BUILD_DIR)
-
-
-STEPS = [
-    create_build_dir,
-    clean_build,
-    update_monorepo,
-    cmake_configure,
-    build_llvmlibc,
-    run_unittests,
-]
-
-
-def main():
-    opts = get_options()
-    for step in STEPS:
-        step(opts)
-
-
-if __name__ == "__main__":
-    main()