[zorg] Adding two new builders for armv7 and aarch64

[leandron]

Summary

The idea here is to add a couple new LLVM builders with the intention of generating a fast LLVM and clang builds to be part of a armv7 and aarch64 builds.

Successful builds will be uploaded to a Google Cloud account and be integrated with tools such as llvm-bisect. The idea is to keep the builds done and available in less than 20 minutes.

NOTE: This is a provisional code review as I'm still waiting for information about the specific Google Cloud accounts.

[rengolin]

Nice! Great for bisecting! I'll let @maxim-kuvyrkov approve, as he's the one taking care of the Buildbots now.

[leandron]

ping

[sqlbyme]

Though my Buildbot foo is a bit rusty these days, this LGTM.

[gkistanova]

LGTM

zorg/buildbot/builders/ClangBuilder.py

495

We need a better way to abstract the compression command and the compression-related specifics.

[maxim-kuvyrkov]

@leandron , note that you have to commit this patch yourself or (if you don't have commit access) explicitly ask someone here to commit it for you.

[leandron]

committed a long time ago

buildbot/osuosl/master/config/builders.py

 #               clean=True,
 #               env=None,
 #               build_gold=True,
 #               cmake_cache_file="../llvm.src/tools/clang/cmake/caches/3-stage.cmake",
 #               extra_cmake_options=[
 #                   '-GNinja',
 #                   '-DLLVM_TARGETS_TO_BUILD=all',
 #                   '-DLLVM_BINUTILS_INCDIR=/opt/binutils/include'])},
+
+        ## Armv7 build cache
+        {'name': "clang-armv7-linux-build-cache",
+         'slavenames':["packet-linux-armv7-slave-1"],
+         'builddir':"clang-armv7-linux-build-cache",
+         'factory' : ClangBuilder.getClangCMakeGCSBuildFactory(
+                      stage1_config='Release',
+                      clean=True,
+                      checkout_compiler_rt=False,
+                      test=False,
+                      useTwoStage=False,
+                      runTestSuite=False,
+                      checkout_lld=True,
+                      checkout_libcxx=True,
+                      checkout_clang_tools_extra=False,
+                      use_pixz_compression=False,
+                      xz_compression_factor=0,
+                      stage1_upload_directory='clang-cmake-armv7',
+                      extra_cmake_args=[
+                          "-DLLVM_TARGETS_TO_BUILD='ARM;AArch64;X86'",
+                          "-DCMAKE_C_FLAGS='-mthumb'",
+                          "-DCMAKE_CXX_FLAGS='-mthumb'",
+                          ],
+                      env={'BUCKET': 'llvm-build-cache'}
+               ),
+         'category': 'clang'},
+
+        ## AArch64 build cache
+        {'name': "clang-aarch64-linux-build-cache",
+         'slavenames':["packet-linux-aarch64-slave-1"],
+         'builddir':"clang-aarch64-linux-build-cache",
+         'factory' : ClangBuilder.getClangCMakeGCSBuildFactory(
+                      stage1_config='Release',
+                      clean=True,
+                      checkout_compiler_rt=False,
+                      test=False,
+                      useTwoStage=False,
+                      runTestSuite=False,
+                      checkout_lld=True,
+                      checkout_libcxx=True,
+                      checkout_clang_tools_extra=False,
+                      stage1_upload_directory='clang-cmake-aarch64',
+                      use_pixz_compression=True,
+                      extra_cmake_args=[
+                          "-DLLVM_TARGETS_TO_BUILD='ARM;AArch64;X86'",
+                          #"-DCMAKE_C_FLAGS=''",
+                          #"-DCMAKE_CXX_FLAGS=''",
+                          ],
+                      env={'BUCKET': 'llvm-build-cache'}
+               ),
+         'category': 'clang'},
+ 
     ]
 
 # Polly builders.
 def _get_polly_builders():
     return [
         {'name': "polly-amd64-linux",
          'slavenames':["grosser1"],
          'builddir':"polly-amd64-linux",

buildbot/osuosl/master/config/slaves.py

return [
         create_slave("linaro-apm-06", properties={'jobs' : 8}, max_builds=1),
         create_slave("linaro-armv8-01-aarch64-quick", properties={'jobs' : 64}, max_builds=1),
         create_slave("linaro-armv8-01-aarch64-full", properties={'jobs' : 64}, max_builds=1),
         create_slave("linaro-armv8-01-aarch64-global-isel", properties={'jobs' : 64}, max_builds=1),
         create_slave("linaro-armv8-01-aarch64-lld", properties={'jobs' : 64}, max_builds=1),
         create_slave("linaro-armv8-01-aarch64-libcxx", properties={'jobs' : 64}, max_builds=1),
         create_slave("linaro-armv8-01-aarch64-libcxx-noeh", properties={'jobs' : 64}, max_builds=1),
 
+        # ARMv7 build cache slave
+        create_slave("packet-linux-armv7-slave-1", properties={'jobs' : 64}, max_builds=1),
+
+        # AArch64 build cache slave
+        create_slave("packet-linux-aarch64-slave-1", properties={'jobs' : 64}, max_builds=1),
+
+
         # AMD Athlon(tm) 64 X2 Dual Core 3800+, Ubuntu x86_64
         create_slave("grosser1", properties={'jobs': 2}, max_builds=1),
 
         # Intel(R) Core(TM)2 Quad CPU Q6600  @ 2.40GHz, Debian x86_64 GNU/Linux
         #create_slave("grosser2", properties={'jobs': 4}, max_builds=1),
 
         # Polly perf servers
         # Each is a:

buildbot/osuosl/master/config/status.py

return [
             subject="Build %(builder)s Failure",
             mode = "failing",
             builders = ["clang-x86_64-darwin10-gdb", "clang-x86_64-ubuntu-gdb-75"],
             addLogs=False,
             num_lines = 15),
         InformativeMailNotifier(
             fromaddr = "[e-mail]",
             sendToInterestedUsers= False,
+            extraRecipients = ["[e-mail]"],
+            subject="Build %(builder)s Failure",
+            mode = "failing",
+            builders = ["clang-aarch64-linux-build-cache", "clang-armv7-linux-build-cache"],
+            addLogs=False,
+            num_lines = 15),
+        InformativeMailNotifier(
+            fromaddr = "[e-mail]",
+            sendToInterestedUsers= False,
             extraRecipients = ["[e-mail]"],
             subject="Build %(builder)s Failure",
             mode = "failing",
             builders = ["clang-x86_64-debian-fast"],
             addLogs=False,
             num_lines = 15),
         InformativeMailNotifier(
             fromaddr = "[e-mail]",

zorg/buildbot/builders/ClangBuilder.py

if checkout_libcxx:
                       workdir='llvm/projects/libcxxabi'))
         f.addStep(SVN(name='svn-libunwind',
                       mode='update',
                       baseURL='http://llvm.org/svn/llvm-project/libunwind/',
                       defaultBranch='trunk',
                       workdir='llvm/projects/libunwind'))
 
 def addGCSUploadSteps(f, package_name, install_prefix, gcs_directory, env,
-                      gcs_url_property=None):
+                      gcs_url_property=None, use_pixz_compression=False,
+                      xz_compression_factor=6):
     """
     Add steps to upload to the Google Cloud Storage bucket.
 
     f - The BuildFactory to modify.
     package_name - The name of this package for the descriptions (e.g.
                    'stage 1')
     install_prefix - The directory the build has been installed to.
     gcs_directory - The subdirectory of the bucket root to upload to. This
                     should match the builder name.
     env - The environment to use. Set BOTO_CONFIG to use a configuration file
           in a non-standard location, and BUCKET to use a different GCS bucket.
     gcs_url_property - Property to assign the GCS url to.
     """
 
     gcs_url_fmt = ('gs://%(gcs_bucket)s/%(gcs_directory)s/'
                    'clang-r%(got_revision)s-t%(now)s-b%(buildnumber)s.tar.xz')
     time_fmt = '%Y-%m-%d_%H-%M-%S'
+    output_file_name = '../install.tar.xz'
+
     gcs_url = \
         WithProperties(
             gcs_url_fmt,
             gcs_bucket=lambda _: env.get('BUCKET', 'llvm-build-artifacts'),
             gcs_directory=lambda _: gcs_directory,
             now=lambda _: datetime.utcnow().strftime(time_fmt))
 
     if gcs_url_property:
         f.addStep(SetProperty(
                       name="record GCS url for " + package_name,
                       command=['echo', gcs_url],
                       property=gcs_url_property))
 
+    if use_pixz_compression:
+        # tweak the xz compression level to generate packages faster
+        tar_command = ['tar', '-Ipixz', '-cvf', output_file_name, '.']
+    else:
+        xz_command = 'xz -{0}'.format(xz_compression_factor)
+        tar_command = ['tar', '-I', xz_command, '-cvf', output_file_name, '.']
+
     f.addStep(ShellCommand(name='package ' + package_name,
-                           command=['tar', 'cvfJ', '../install.tar.xz', '.'],
+                           command=tar_command,
                            description='packaging ' + package_name + '...',
                            workdir=install_prefix,
                            env=env))
 
     f.addStep(ShellCommand(
                   name='upload ' + package_name + ' to storage bucket',
                   command=['gsutil', 'cp', '../install.tar.xz', gcs_url],
                   description=('uploading ' + package_name +
def getClangCMakeGCSBuildFactory(
             # Environmental variables for all steps.
             env={},
             extra_cmake_args=[],
 
             # Extra repositories
             checkout_clang_tools_extra=True,
             checkout_compiler_rt=True,
             checkout_lld=True,
+            checkout_libcxx=False,
 
             # Upload artifacts to Google Cloud Storage (for the llvmbisect tool)
             stage1_upload_directory=None,
 
+            # Use a lower compression level to generate the build-cache package faster.
+            # defuault is 6 according to xz documentation
+            xz_compression_factor=6,
+            use_pixz_compression=False,
+
             # Triggers
             trigger_after_stage1=[]):
     return _getClangCMakeBuildFactory(
                clean=clean, test=test, cmake=cmake, jobs=jobs, vs=vs,
                vs_target_arch=vs_target_arch, useTwoStage=useTwoStage,
                testStage1=testStage1, stage1_config=stage1_config,
                stage2_config=stage2_config, runTestSuite=runTestSuite,
                nt_flags=nt_flags, testsuite_flags=testsuite_flags,
                submitURL=submitURL, testerName=testerName,
                env=env, extra_cmake_args=extra_cmake_args,
                checkout_clang_tools_extra=checkout_clang_tools_extra,
                checkout_compiler_rt=checkout_compiler_rt,
                checkout_lld=checkout_lld,
+               checkout_libcxx=checkout_libcxx,
                stage1_upload_directory=stage1_upload_directory,
+               xz_compression_factor=xz_compression_factor,
+               use_pixz_compression=use_pixz_compression,
                trigger_after_stage1=trigger_after_stage1)
 
 def getClangCMakeBuildFactory(
             clean=True,
             test=True,
             cmake='cmake',
             jobs=None,
 
def _getClangCMakeBuildFactory(
             checkout_compiler_rt=True,
             checkout_lld=True,
             checkout_libcxx=False,
             checkout_test_suite=False,
 
             # Upload artifacts to Google Cloud Storage (for the llvmbisect tool)
             stage1_upload_directory=None,
 
+            # Use a lower compression level to generate the build-cache package faster
+            # default is 6 according to documentation
+            xz_compression_factor=6,
+            use_pixz_compression=False,
+
             # Triggers
             trigger_after_stage1=[]):
 
     ############# PREPARING
     f = buildbot.process.factory.BuildFactory()
 
     clean_build_requested = lambda step: clean or step.build.getProperty("clean")
 
if useTwoStage or runTestSuite or stage1_upload_directory:
         f.addStep(ShellCommand(name='install stage 1',
                                command=ninja_install_cmd,
                                description='ninja install',
                                workdir=stage1_build,
                                env=env))
 
     if stage1_upload_directory:
         addGCSUploadSteps(f, 'stage 1', stage1_install, stage1_upload_directory,
-                          env, gcs_url_property='stage1_package_gcs_url')
+                          env, gcs_url_property='stage1_package_gcs_url',
+                          use_pixz_compression=use_pixz_compression,
+                          xz_compression_factor=xz_compression_factor)
 
     # Compute the cmake define flag to set the C and C++ compiler to clang. Use
     # clang-cl if we used MSVC for stage1.
     if not vs:
         cc = 'clang'
         cxx = 'clang++'
     else:
         cc = 'clang-cl.exe'