[zorg] Support LLVM_ENABLE_RUNTIMES build in ClangBuilder

[jsji]

Summary

In D90609 and D90645, we support the LLVM_ENABLE_RUNTIMES by passing
down extra cmake args, this is working.

However, there is one *bug* -- the compiler-rt commits are not shown in
buildbot Changes. eg:
asan commit ce193ea9e8430f15d19dd60d73f13a66e4f374ac
is not shown in Changes of
https://lab.llvm.org/buildbot/#/builders/57/builds/5769

but it is actually in between two other commits, like Changes in
https://lab.llvm.org/buildbot/#/builders/76/builds/2104

The problem is the ClangBuilder use the *depends_on* in scheduler,
so when we disable the *checkout_compiler_rt*, we told scheduler to
ignore compiler-rt commits as well.

In this patch, we introduce the flag to support runtime build in
ClangBuilder, update LLVM_ENABLE_PROJECTS and LLVM_ENABLE_RUNTIMES
according to the flag, without changing the depends_on list.

And use it in clang-ppc64le-rhel builder.

[jsji]

Ping ..

[jsji]

Ping ... @gkistanova Appreciate if you can help to review or comment of this? Thanks.

[jsji]

ping ...

[jsji]

RHEL bots has now been switched to UnifiedTreeBuilder, and runtime support was added recently in https://reviews.llvm.org/rZORG90c0b74c2ee54639b417bbda061c45ba8967ed9f.

buildbot/osuosl/master/config/builders.py

'factory' : ClangBuilder.getClangCMakeBuildFactory(
                     extra_cmake_args=["-DLLVM_ENABLE_ASSERTIONS=ON"])},
 
     {'name' : "clang-ppc64le-rhel",
     'tags'  : ["clang", "ppc", "ppc64le"],
     'workernames' : ["ppc64le-clang-rhel-test"],
     'builddir': "clang-ppc64le-rhel",
     'factory' : ClangBuilder.getClangCMakeBuildFactory(clean=False,
                     checkout_clang_tools_extra=True,
-                    checkout_compiler_rt=False,
+                    checkout_compiler_rt=True,
                     checkout_lld=True,
                     checkout_libcxx=False,
                     useTwoStage=False,
                     runTestSuite=True,
                     stage1_config='Release',
+                    runtimebuild_compiler_rt=True,
                     nt_flags=['--threads=16', '--build-threads=16'],
                     extra_cmake_args=[
-                        '-DLLVM_ENABLE_PROJECTS=clang;llvm;clang-tools-extra;lld',
-                        '-DLLVM_ENABLE_RUNTIMES=compiler-rt',
                         "-DLLVM_ENABLE_ASSERTIONS=On", "-DCMAKE_C_COMPILER=clang",
                         "-DCMAKE_CXX_COMPILER=clang++",
                         "-DCLANG_DEFAULT_LINKER=lld",
                         "-DCMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN:PATH=/opt/rh/devtoolset-7/root/usr",
                         "-DCMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN:PATH=/opt/rh/devtoolset-7/root/usr",
                         "-DLLVM_BINUTILS_INCDIR=/usr/include", "-DBUILD_SHARED_LIBS=ON", "-DLLVM_ENABLE_WERROR=ON",
                         '-DLLVM_LIT_ARGS=-vj 20'])},
 

zorg/buildbot/builders/ClangBuilder.py

def getClangCMakeBuildFactory(
             testsuite_flags=None,
             submitURL=None,
             testerName=None,
 
             # Environmental variables for all steps.
             env=None,
             extra_cmake_args=None,
 
+            # Doing runtime build instead of in projects
+            runtimebuild_compiler_rt=False,
+
             # Extra repositories
             checkout_clang_tools_extra=True,
             checkout_compiler_rt=True,
             checkout_lld=True,
             checkout_libcxx=False,
             checkout_flang=False,
             checkout_test_suite=False):
     return _getClangCMakeBuildFactory(
                clean=clean, test=test, cmake=cmake, jobs=jobs, vs=vs,
                vs_target_arch=vs_target_arch, useTwoStage=useTwoStage,
                testStage1=testStage1, stage1_config=stage1_config,
                stage2_config=stage2_config, runTestSuite=runTestSuite,
                nt_flags=nt_flags, testsuite_flags=testsuite_flags,
                submitURL=submitURL, testerName=testerName,
                env=env, extra_cmake_args=extra_cmake_args,
+               runtimebuild_compiler_rt=runtimebuild_compiler_rt,
                checkout_clang_tools_extra=checkout_clang_tools_extra,
                checkout_lld=checkout_lld,
                checkout_compiler_rt=checkout_compiler_rt,
                checkout_libcxx=checkout_libcxx,
                checkout_flang=checkout_flang,
                checkout_test_suite=checkout_test_suite)
 
 def _getClangCMakeBuildFactory(
def _getClangCMakeBuildFactory(
             testsuite_flags=None,
             submitURL=None,
             testerName=None,
 
             # Environmental variables for all steps.
             env=None,
             extra_cmake_args=None,
 
+            # Doing runtime build instead of in projects
+            runtimebuild_compiler_rt=False,
+
             # Extra repositories
             checkout_clang_tools_extra=True,
             checkout_compiler_rt=True,
             checkout_lld=True,
             checkout_libcxx=False,
             checkout_test_suite=False,
             checkout_flang=False,
 
f.addStep(ShellCommand(name='clean stage 1',
                            flunkOnFailure=False,
                            description='cleaning stage 1',
                            descriptionDone='clean',
                            workdir='.',
                            doStepIf=clean_build_requested))
 
 
     ############# STAGE 1
-    CmakeCommand.applyRequiredOptions(extra_cmake_args, [
-        ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)),
-        ])
+    enable_projects = f.depends_on_projects
+    enable_runtimes = 'compiler-rt'
+    if runtimebuild_compiler_rt and checkout_compiler_rt:
+        enable_projects.remove('compiler-rt')
+
+    required =  [('-DLLVM_ENABLE_PROJECTS=', ";".join(enable_projects))]
+    if runtimebuild_compiler_rt:
+        required.append(('-DLLVM_ENABLE_RUNTIMES=', ";".join(enable_runtimes)))
+
+    CmakeCommand.applyRequiredOptions(extra_cmake_args, required)
+
     rel_src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, stage1_build)
 
     f.addStep(ShellCommand(name='cmake stage 1',
                            command=[cmake, "-G", "Ninja", rel_src_dir,
                                     "-DCMAKE_BUILD_TYPE="+stage1_config,
                                     "-DLLVM_ENABLE_ASSERTIONS=True",
                                     "-DLLVM_LIT_ARGS="+lit_args,
                                     "-DCMAKE_INSTALL_PREFIX=../"+stage1_install]