[llvm] Add build bots for ml-driven optimization policies.

[mtrofin]

Summary

RFC: http://lists.llvm.org/pipermail/llvm-dev/2020-April/140763.html

[bcain]

Where can I learn more about the model itself and how it gets feedback? Does this mostly impact target dependent or target independent optimizations? (Sorry for hijacking the review)

[mtrofin]

In D81044#2071337, @bcain wrote:

Where can I learn more about the model itself and how it gets feedback? Does this mostly impact target dependent or target independent optimizations? (Sorry for hijacking the review)

Sorry, forgot to link the RFC this time. Updated the review description.

[gkistanova]

LGTM

buildbot/osuosl/master/config/builders.py

return [
                'name':"publish-sphinx-docs",
                'slavenames':["as-bldslv5"],
                'builddir':"publish-sphinx-docs",
                'factory': SphinxDocsBuilder.getLLVMDocsBuildFactory(clean=True),
                'category' : 'doc'
              }
            ]
 
+# Builders for ML-driven compiler optimizations.
+def _get_ml_compiler_opt_builders():
+    common_extra_args = [
+        "-DCMAKE_BUILD_TYPE=Release",
+        "-DLLVM_CACHE_BUILD=ON",
+        "-DLLVM_ENABLE_ASSERTIONS=ON", ]
+
+    return [
+        # Development mode build bot: tensorflow C APIs are present, and 
+        # we can dynamically load models, and produce training logs.
+        {'name': "ml-opt-dev-x86-64",
+         'mergeRequests': False,
+         'slavenames':["ml-opt-dev-x86-64-b1"],
+         'builddir':"ml-opt-dev-x86-64-b1",
+         'factory': UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory(
+                        clean=True,
+                        depends_on_projects=['llvm'],
+                        extra_configure_args=common_extra_args + [
+                            "-DTENSORFLOW_API_PATH=${TENSORFLOW_API_PATH}"
+                        ])},
+        # Release mode build bot: the model is pre-built and linked in the
+        # compiler. Only the tensorflow pip package is needed, and out of it,
+        # only saved_model_cli (the model compiler) and the thin C++ wrappers
+        # in xla_aot_runtime_src (and include files)
+        {'name': "ml-opt-rel-x86-64",
+         'mergeRequests': False,
+         'slavenames':["ml-opt-rel-x86-64-b1"],
+         'builddir':"ml-opt-rel-x86-64-b1",
+         'factory': UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory(
+                        clean=True,
+                        depends_on_projects=['llvm'],
+                        extra_configure_args=common_extra_args + [
+                            "-DTENSORFLOW_AOT_PATH=${TENSORFLOW_AOT_PATH}"
+                        ])},
+        # Both tensorflow C library, and the pip package, are present.
+        {'name': "ml-opt-devrel-x86-64",
+         'mergeRequests': False,
+         'slavenames':["ml-opt-devrel-x86-64-b1"],
+         'builddir':"ml-opt-devrel-x86-64-b1",
+         'factory': UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory(
+                        clean=True,
+                        depends_on_projects=['llvm'],
+                        extra_configure_args=common_extra_args + [
+                            "-DTENSORFLOW_API_PATH=${TENSORFLOW_API_PATH}",
+                            "-DTENSORFLOW_AOT_PATH=${TENSORFLOW_AOT_PATH}"
+                        ])},
+    ]
+
 def get_builders():
     for b in _get_llvm_builders():
         b['category'] = 'llvm'
         yield b
 
     for b in _get_clang_fast_builders():
         b['category'] = 'clang_fast'
         yield b
def get_builders():
 
     for b in _get_lldb_builders():
         b['category'] = 'lldb'
         yield b
 
     for b in _get_mlir_builders():
         b['category'] = 'mlir'
         yield b
     
+    for b in _get_ml_compiler_opt_builders():
+        b['category'] = 'ml_opt'
+        yield b
+
     for b in _get_sanitizer_builders():
         b['category'] = 'sanitizer'
         yield b
 
     for b in _get_openmp_builders():
         b['category'] = 'openmp'
         yield b
 

buildbot/osuosl/master/config/slaves.py

return [
         create_slave("polly-x86_64-gce2", properties={'jobs': 2}, max_builds=1),
 
         # Ubuntu 18.04.LTS x86_64, Intel(R) Xeon(R) CPU X3460 @ 2.80GHz, 32 GiB RAM
         create_slave("polly-x86_64-fdcserver", properties={'jobs': 8, 'loadaverage': 8}, max_builds=1),
 
         create_slave("flang-aarch64-ubuntu-build"),
         create_slave("flang-aarch64-ubuntu-clang-build"),
         create_slave("nersc-flang"),
+
+        # ML-Driven Compiler Optimizations build slave (Ubuntu x86_64)
+        create_slave("ml-opt-dev-x86-64-b1",
+                     properties={'jobs': 64}, max_builds=1),
+        create_slave("ml-opt-rel-x86-64-b1",
+                     properties={'jobs': 64}, max_builds=1),
+        create_slave("ml-opt-devrel-x86-64-b1",
+                     properties={'jobs': 64}, max_builds=1),
         ]

buildbot/osuosl/master/config/status.py

return [
          InformativeMailNotifier(
             fromaddr = "[e-mail]",
             sendToInterestedUsers = True,
             extraRecipients = ["[e-mail]"],
             subject="Sphinx build %(builder)s Failure",
             mode = "failing",
             builders = ["publish-sphinx-docs"],
             addLogs=False),
+        InformativeMailNotifier(fromaddr="[e-mail]",
+            sendToInterestedUsers=True,
+            extraRecipients=[
+                "[e-mail]"],
+            subject="ML Compiler Opt Failure: %(builder)s",
+            mode="failing",
+            builders=[
+                "ml-opt-dev-x86-64", "ml-opt-rel-x86-64", "ml-opt-devrel-x86-64"],
+            addLogs=False),
         ]