Simplify setting up LLVM as bazel external repo

[csigg]

Summary

Only require one intermediate repository instead of two.
Fewer parameters in llvm_config.

Second attempt of https://reviews.llvm.org/D107714, this time also updating third_party_build and deps_impl paths.

[csigg]

Fix.

[GMNGeoffrey]

Huh I guess this works. To me the labels having //utils/bazel in them is pretty confusing because the WORKSPACE file is under utils/bazel already... Then again, making the way this loads the llvm-project repository consistent with how a dependency would load it is nice I guess. Let me make sure that this works for dependent projects (I missed some subtleties on the last one)

[GMNGeoffrey]

Although as a counterpoint, maybe this means that all labels here and in the llvm-project-overlay will reference things relative to the LLVM root, so maybe that's less confusing. I probably just got used to the way things were. I tried this out with the IREE build and it seems to work fine.

utils/bazel/WORKSPACE

http_archive(
     name = "bazel_skylib",
     sha256 = "1c531376ac7e5a180e0237938a2536de0c54d93f5c278634818e0efc952dd56c",
     urls = [
         "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/{version}/bazel-skylib-{version}.tar.gz".format(version=SKYLIB_VERSION),
         "https://github.com/bazelbuild/bazel-skylib/releases/download/{version}/bazel-skylib-{version}.tar.gz".format(version=SKYLIB_VERSION),
     ],
 )
 
-load(":configure.bzl", "llvm_configure")
-
-llvm_configure(
-    name = "llvm-project",
-    overlay_path = "llvm-project-overlay",
-    src_path = "../..",
+new_local_repository(
+    name = "llvm-raw",
+    path = "../../",
+    build_file_content = "# empty",
 )
 
-load(":terminfo.bzl", "llvm_terminfo_from_env")
+load("@llvm-raw//utils/bazel:configure.bzl", "llvm_configure")
+
+llvm_configure(name = "llvm-project")
+
+load("@llvm-raw//utils/bazel:terminfo.bzl", "llvm_terminfo_from_env")
 
 maybe(
     llvm_terminfo_from_env,
     name = "llvm_terminfo",
 )
 
 maybe(
     http_archive,
     name = "zlib",
-    build_file = "//third_party_build:zlib.BUILD",
+    build_file = "@llvm-raw//utils/bazel/third_party_build:zlib.BUILD",
     sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1",
     strip_prefix = "zlib-1.2.11",
     urls = [
         "https://storage.googleapis.com/mirror.tensorflow.org/zlib.net/zlib-1.2.11.tar.gz",
         "https://zlib.net/zlib-1.2.11.tar.gz",
     ],
 )
 
-load(":zlib.bzl", "llvm_zlib_from_env")
+load("@llvm-raw//utils/bazel:zlib.bzl", "llvm_zlib_from_env")
 
 maybe(
     llvm_zlib_from_env,
     name = "llvm_zlib",
     external_zlib = "@zlib",
 )
 
 maybe(
     http_archive,
     name = "vulkan_headers",
-    build_file = "//third_party_build:vulkan_headers.BUILD",
+    build_file = "@llvm-raw//utils/bazel/third_party_build:vulkan_headers.BUILD",
     sha256 = "19f491784ef0bc73caff877d11c96a48b946b5a1c805079d9006e3fbaa5c1895",
     strip_prefix = "Vulkan-Headers-9bd3f561bcee3f01d22912de10bb07ce4e23d378",
     urls = [
         "https://github.com/KhronosGroup/Vulkan-Headers/archive/9bd3f561bcee3f01d22912de10bb07ce4e23d378.tar.gz",
     ],
 )
 
-load(":vulkan_sdk.bzl", "vulkan_sdk_setup")
+load("@llvm-raw//utils/bazel:vulkan_sdk.bzl", "vulkan_sdk_setup")
 
 maybe(
     vulkan_sdk_setup,
     name = "vulkan_sdk",
 )
 
 http_archive(
     name = "bazel_toolchains",

utils/bazel/configure.bzl

 # This file is licensed under the Apache License v2.0 with LLVM Exceptions.
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 """Helper macros to configure the LLVM overlay project."""
 
 load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
-load("@bazel_skylib//lib:paths.bzl", "paths")
 load(":zlib.bzl", "llvm_zlib_disable", "llvm_zlib_system")
 load(":terminfo.bzl", "llvm_terminfo_disable", "llvm_terminfo_system")
 
 # Directory of overlay files relative to WORKSPACE
 DEFAULT_OVERLAY_PATH = "llvm-project-overlay"
 
 DEFAULT_TARGETS = [
     "AArch64",
DEFAULT_TARGETS = [
     "Sparc",
     "SystemZ",
     "WebAssembly",
     "X86",
     "XCore",
 ]
 
 def _overlay_directories(repository_ctx):
-    src_workspace_path = str(repository_ctx.path(
-        repository_ctx.attr.src_workspace,
-    ).dirname)
-
-    src_path = paths.join(src_workspace_path, repository_ctx.attr.src_path)
-
-    overlay_workspace_path = str(repository_ctx.path(
-        repository_ctx.attr.overlay_workspace,
-    ).dirname)
-    overlay_path = paths.join(
-        overlay_workspace_path,
-        repository_ctx.attr.overlay_path,
-    )
+    src_path = repository_ctx.path(Label("//:WORKSPACE")).dirname
+    bazel_path = src_path.get_child("utils").get_child("bazel")
+    overlay_path = bazel_path.get_child("llvm-project-overlay")
+    script_path = bazel_path.get_child("overlay_directories.py")
 
-    overlay_script = repository_ctx.path(
-        repository_ctx.attr._overlay_script,
-    )
     python_bin = repository_ctx.which("python3")
     if not python_bin:
         # Windows typically just defines "python" as python3. The script itself
         # contains a check to ensure python3.
         python_bin = repository_ctx.which("python")
 
     if not python_bin:
         fail("Failed to find python3 binary")
 
     cmd = [
         python_bin,
-        overlay_script,
+        script_path,
         "--src",
         src_path,
         "--overlay",
         overlay_path,
         "--target",
         ".",
     ]
     exec_result = repository_ctx.execute(cmd, timeout = 20)
repository_ctx.file(
         executable = False,
     )
 
 llvm_configure = repository_rule(
     implementation = _llvm_configure_impl,
     local = True,
     configure = True,
     attrs = {
-        "_overlay_script": attr.label(
-            default = Label("//:overlay_directories.py"),
-            allow_single_file = True,
-        ),
-        "overlay_workspace": attr.label(default = Label("//:WORKSPACE")),
-        "overlay_path": attr.string(default = DEFAULT_OVERLAY_PATH),
-        "src_workspace": attr.label(default = Label("//:WORKSPACE")),
-        "src_path": attr.string(mandatory = True),
         "targets": attr.string_list(default = DEFAULT_TARGETS),
     },
 )
 
 def llvm_disable_optional_support_deps():
     maybe(
         llvm_zlib_disable,
         name = "llvm_zlib",

utils/bazel/examples/submodule/WORKSPACE

http_archive(
     name = "bazel_skylib",
     sha256 = "97e70364e9249702246c0e9444bccdc4b847bed1eb03c5a3ece4f83dfe6abc44",
     urls = [
         "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/{version}/bazel-skylib-{version}.tar.gz".format(version=SKYLIB_VERSION),
         "https://github.com/bazelbuild/bazel-skylib/releases/download/{version}/bazel-skylib-{version}.tar.gz".format(version=SKYLIB_VERSION),
     ],
 )
 
+new_local_repository(
+    name = "llvm-raw",
+    build_file_content = "# empty",
     # Or wherever your submodule is located.
-SUBMODULE_PATH = "third_party/llvm-project"
-
-local_repository(
-    name = "llvm-bazel",
-    path = SUBMODULE_PATH + "/utils/bazel",
+    path = "third_party/llvm-project",
 )
 
-load("@llvm-bazel//:configure.bzl", "llvm_configure", "llvm_disable_optional_support_deps")
+load("@llvm-raw//utils/bazel:configure.bzl", "llvm_configure", "llvm_disable_optional_support_deps")
 
-llvm_configure(
-    name = "llvm-project",
-    src_path = SUBMODULE_PATH,
-    src_workspace = "@submodule_example//:WORKSPACE",
-)
+llvm_configure(name = "llvm-project")
 
 # Disables optional dependencies for Support like zlib and terminfo. You may
 # instead want to configure them using the macros in the corresponding bzl
 # files.
 llvm_disable_optional_support_deps()

utils/bazel/terminfo.bzl

def _llvm_terminfo_disable_impl(repository_ctx):
     repository_ctx.template(
         "BUILD",
         repository_ctx.attr._disable_build_template,
         executable = False,
     )
 
 _terminfo_disable_attrs = {
     "_disable_build_template": attr.label(
-        default = Label("//deps_impl:terminfo_disable.BUILD"),
+        default = Label("//utils/bazel/deps_impl:terminfo_disable.BUILD"),
         allow_single_file = True,
     ),
 }
 
 llvm_terminfo_disable = repository_rule(
     implementation = _llvm_terminfo_disable_impl,
     attrs = _terminfo_disable_attrs,
 )

 def _merge_attrs(attrs_list):
     attrs = {}
     for input_attrs in attrs_list:
         attrs.update(input_attrs)
     return attrs
 
 _terminfo_system_attrs = _merge_attrs([_terminfo_disable_attrs, {
     "_system_build_template": attr.label(
-        default = Label("//deps_impl:terminfo_system.BUILD"),
+        default = Label("//utils/bazel/deps_impl:terminfo_system.BUILD"),
         allow_single_file = True,
     ),
     "_terminfo_test_source": attr.label(
-        default = Label("//deps_impl:terminfo_test.c"),
+        default = Label("//utils/bazel/deps_impl:terminfo_test.c"),
         allow_single_file = True,
     ),
     "candidate_system_linkopts": attr.string_list(
         default = [
             "-lterminfo",
             "-ltinfo",
             "-lcurses",
             "-lncurses",

utils/bazel/zlib.bzl

repository_ctx.template(
         },
         executable = False,
     )
 
 llvm_zlib_external = repository_rule(
     implementation = _llvm_zlib_external_impl,
     attrs = {
         "_external_build_template": attr.label(
-            default = Label("//deps_impl:zlib_external.BUILD"),
+            default = Label("//utils/bazel/deps_impl:zlib_external.BUILD"),
             allow_single_file = True,
         ),
         "external_zlib": attr.string(
             doc = "The dependency that should be used for the external zlib library.",
             mandatory = True,
         ),
     },
 )
 
 def _llvm_zlib_system_impl(repository_ctx):
     repository_ctx.template(
         "BUILD",
         repository_ctx.attr._system_build_template,
         executable = False,
     )
 
 # While it may seem like this needs to be local, it doesn't actually inspect
 # any local state, it just configures to build against that local state.
 llvm_zlib_system = repository_rule(
     implementation = _llvm_zlib_system_impl,
     attrs = {
         "_system_build_template": attr.label(
-            default = Label("//deps_impl:zlib_system.BUILD"),
+            default = Label("//utils/bazel/deps_impl:zlib_system.BUILD"),
             allow_single_file = True,
         ),
     },
 )
 
 def _llvm_zlib_disable_impl(repository_ctx):
     repository_ctx.template(
         "BUILD",
         repository_ctx.attr._disable_build_template,
         executable = False,
     )
 
 llvm_zlib_disable = repository_rule(
     implementation = _llvm_zlib_disable_impl,
     attrs = {
         "_disable_build_template": attr.label(
-            default = Label("//deps_impl:zlib_disable.BUILD"),
+            default = Label("//utils/bazel/deps_impl:zlib_disable.BUILD"),
             allow_single_file = True,
         ),
     },
 )
 
 def _llvm_zlib_from_env_impl(repository_ctx):
     zlib_strategy = repository_ctx.os.environ.get("BAZEL_LLVM_ZLIB_STRATEGY")
     if zlib_strategy == "external":
         _llvm_zlib_external_impl(repository_ctx)
     elif zlib_strategy == "system":
         _llvm_zlib_system_impl(repository_ctx)
     else:
         _llvm_zlib_disable_impl(repository_ctx)
 
 llvm_zlib_from_env = repository_rule(
     implementation = _llvm_zlib_from_env_impl,
     attrs = {
         "_disable_build_template": attr.label(
-            default = Label("//deps_impl:zlib_disable.BUILD"),
+            default = Label("//utils/bazel/deps_impl:zlib_disable.BUILD"),
             allow_single_file = True,
         ),
         "_external_build_template": attr.label(
-            default = Label("//deps_impl:zlib_external.BUILD"),
+            default = Label("//utils/bazel/deps_impl:zlib_external.BUILD"),
             allow_single_file = True,
         ),
         "_system_build_template": attr.label(
-            default = Label("//deps_impl:zlib_system.BUILD"),
+            default = Label("//utils/bazel/deps_impl:zlib_system.BUILD"),
             allow_single_file = True,
         ),
         "external_zlib": attr.label(
             doc = "The dependency that should be used for the external zlib library.",
             mandatory = True,
         ),
     },
     environ = ["BAZEL_LLVM_ZLIB_STRATEGY"],
 )