[clang] Move the clang formatting job to run-buildbot to fix the CI

[philnik]

Next try

[philnik]

Try moving the command to run-buildbot

[ldionne]

clang/utils/ci/buildkite-pipeline.yml

22

Can you try with this instead?

'! grep -rnI ''[[:blank:]]$'' clang/lib clang/include clang/docs || false'

[philnik]

Try Louis' suggestion

[philnik]

Fix diff

[philnik]

clang/utils/ci/buildkite-pipeline.yml

22

Nope, doesn't work.

[ldionne]

libcxx/utils/ci/run-buildbot

206 ↗

(On Diff #536612)

We shouldn't be adding a clang-specific job to this file, since this is for libc++ jobs. Please either find the right way to escape the command in the buildkite-pipeline file or create a script like run-buildbot within Clang, and then move that job there. The latter might make more sense if we want to move more and more stuff to this way of doing CI in the future, but we'll need the Clang folks to be on board.

[philnik]

Address comments

[philnik]

@aaron.ballman @erichkeane @cor3ntin (anybody else?) are you fine with moving the clang build kite-pipeline to clang/utils/ci and adding a run-buildbot?

[ldionne]

I really like where this is going, this will create a framework where Clang can add more pre-commit CI checks if they desire.

clang/utils/ci/run-buildbot

32–35

This should go away.

37–43

Let's remove those since they are not enforced.

66–69

This too.

81–102

Let's get rid of all this.

105–106

108

Let's remove this line entirely, CMake prints the version of the compiler in use.

libcxx/utils/ci/run-buildbot

206 ↗

(On Diff #538208)

This can be removed now.

[aaron.ballman]

In D153920#4481410, @philnik wrote:

@aaron.ballman @erichkeane @cor3ntin (anybody else?) are you fine with moving the clang build kite-pipeline to clang/utils/ci and adding a run-buildbot?

I think that seems reasonable, thank you!

[philnik]

Address comments

[ldionne]

This LGTM but the CI has to pass! Also don't forget to undo the changes to buildkite-pipeline.yml

[philnik]

Try to fix CI

[philnik]

Try to fix CI

clang/lib/Analysis/FlowSensitive/DataflowAnalysisContext.cpp

 
 Atom DataflowAnalysisContext::forkFlowCondition(Atom Token) {
   Atom ForkToken = arena().makeFlowConditionToken();
   FlowConditionDeps[ForkToken].insert(Token);
   addFlowConditionConstraint(ForkToken, arena().makeAtomRef(Token));
   return ForkToken;
 }
 
-Atom 
+Atom
 DataflowAnalysisContext::joinFlowConditions(Atom FirstToken,
                                             Atom SecondToken) {
   Atom Token = arena().makeFlowConditionToken();
   FlowConditionDeps[Token].insert(FirstToken);
   FlowConditionDeps[Token].insert(SecondToken);
   addFlowConditionConstraint(Token,
                              arena().makeOr(arena().makeAtomRef(FirstToken),
                                             arena().makeAtomRef(SecondToken)));

clang/utils/ci/buildkite-pipeline.yml

 # in various configurations.
 #
 env:
     # LLVM RELEASE bump version
     LLVM_HEAD_VERSION: "17"
 steps:
   - label: "Format"
     commands:
-      - "! grep -rnI '[[:blank:]]$' clang/lib clang/include clang/docs || false"
+      - "clang/utils/ci/run-buildbot check-format"
 
     agents:
       queue: "libcxx-builders"
       os: "linux"
     retry:
       automatic:
         - exit_status: -1  # Agent was lost
           limit: 2
     timeout_in_minutes: 120
 
   - wait
 
   - label: "Building clang"
     commands:
-      - "mkdir install"
-      # We use Release here to avoid including debug information. Otherwise, the clang binary is very large, which
-      # is problematic because we need to upload the artifacts for other jobs to use. This may seem like nothing,
-      # but with the number of jobs we run daily, this can result in thousands of GB of network I/O.
-      - "cmake -S llvm -B build -G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=\"ccache\" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=install -DLLVM_ENABLE_PROJECTS=\"clang;compiler-rt\""
-      - "ninja -C build install-clang install-clang-resource-headers"
-      - "ccache -s"
-      - "tar -cJvf install.tar.xz install/"
-      - "buildkite-agent artifact upload --debug install.tar.xz"
+      - "clang/utils/ci/run-buildbot build-clang"
     env:
         CC: "clang-${LLVM_HEAD_VERSION}"
         CXX: "clang++-${LLVM_HEAD_VERSION}"
     agents:
       queue: "libcxx-builders"
       os: "linux"
     retry:
       automatic:
         - exit_status: -1  # Agent was lost
           limit: 2
     timeout_in_minutes: 120
 
   - wait
 
   - label: "C++03"
     commands:
-      - "buildkite-agent artifact download install.tar.xz ."
-      - "tar -xvf install.tar.xz"
-      - "export CC=$(pwd)/install/bin/clang"
-      - "export CXX=$(pwd)/install/bin/clang++"
-      - "chmod +x install/bin/clang install/bin/clang++"
-      - "libcxx/utils/ci/run-buildbot generic-cxx03"
+      - "clang/utils/ci/run-buildbot generic-cxx03"
     artifact_paths:
       - "**/test-results.xml"
       - "**/crash_diagnostics/*"
     env:
         LLVM_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer-${LLVM_HEAD_VERSION}" # TODO: Should we build that from scratch?
         CLANG_CRASH_DIAGNOSTICS_DIR: "crash_diagnostics"
     agents:
       queue: "libcxx-builders"
       os: "linux"
     retry:
       automatic:
         - exit_status: -1  # Agent was lost
           limit: 2
     timeout_in_minutes: 120
 
   - label: "C++26"
     commands:
-      - "buildkite-agent artifact download install.tar.xz ."
-      - "tar -xvf install.tar.xz"
-      - "export CC=$(pwd)/install/bin/clang"
-      - "export CXX=$(pwd)/install/bin/clang++"
-      - "chmod +x install/bin/clang install/bin/clang++"
-      - "libcxx/utils/ci/run-buildbot generic-cxx26"
+      - "clang/utils/ci/run-buildbot generic-cxx26"
     artifact_paths:
       - "**/test-results.xml"
       - "**/crash_diagnostics/*"
     env:
         LLVM_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer-${LLVM_HEAD_VERSION}" # TODO: Should we build that from scratch?
         CLANG_CRASH_DIAGNOSTICS_DIR: "crash_diagnostics"
     agents:
       queue: "libcxx-builders"
       os: "linux"
     retry:
       automatic:
         - exit_status: -1  # Agent was lost
           limit: 2
     timeout_in_minutes: 120
 
   - label: "Modules"
     commands:
-      - "buildkite-agent artifact download install.tar.xz ."
-      - "tar -xvf install.tar.xz"
-      - "export CC=$(pwd)/install/bin/clang"
-      - "export CXX=$(pwd)/install/bin/clang++"
-      - "chmod +x install/bin/clang install/bin/clang++"
-      - "libcxx/utils/ci/run-buildbot generic-modules"
+      - "clang/utils/ci/run-buildbot generic-modules"
     artifact_paths:
       - "**/test-results.xml"
       - "**/crash_diagnostics/*"
     env:
         LLVM_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer-${LLVM_HEAD_VERSION}" # TODO: Should we build that from scratch?
         CLANG_CRASH_DIAGNOSTICS_DIR: "crash_diagnostics"
     agents:
       queue: "libcxx-builders"
       os: "linux"
     retry:
       automatic:
         - exit_status: -1  # Agent was lost
           limit: 2
     timeout_in_minutes: 120

clang/utils/ci/run-buildbot

+#!/usr/bin/env bash
+#===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+#===----------------------------------------------------------------------===##
+
+set -ex
+set -o pipefail
+unset LANG
+unset LC_ALL
+unset LC_COLLATE
+
+PROGNAME="$(basename "${0}")"
+
+function usage() {
+cat <<EOF
+Usage:
+${PROGNAME} [options] <BUILDER>
+
+[-h|--help]         Display this help and exit.
+
+--llvm-root <DIR>   Path to the root of the LLVM monorepo. By default, we try
+                    to figure it out based on the current working directory.
+
+--build-dir <DIR>   The directory to use for building the library. By default,
+                    this is '<llvm-root>/build/<builder>'.
+EOF
+}
+
+if [[ $# == 0 ]]; then
+   usage
+   exit 0
+fi
+
+while [[ $# -gt 0 ]]; do
+    case ${1} in
+        -h|--help)
+            usage
+            exit 0
+            ;;
+        --llvm-root)
+            MONOREPO_ROOT="${2}"
+            shift; shift
+            ;;
+        --build-dir)
+            BUILD_DIR="${2}"
+            shift; shift
+            ;;
+        *)
+            BUILDER="${1}"
+            shift
+            ;;
+    esac
+done
+
+MONOREPO_ROOT="${MONOREPO_ROOT:="$(git rev-parse --show-toplevel)"}"
+BUILD_DIR="${BUILD_DIR:=${MONOREPO_ROOT}/build/${BUILDER}}"
+INSTALL_DIR="${BUILD_DIR}/install"
+
+# Print the version of a few tools to aid diagnostics in some cases
+cmake --version
+ninja --version
+
+case "${BUILDER}" in
+check-format)
+    ! grep -rnI '[[:blank:]]$' clang/lib clang/include clang/docs
+;;
+build-clang)
+    mkdir install
+    # We use Release here to avoid including debug information. Otherwise, the
+    # clang binary is very large, which is problematic because we need to upload
+    # the artifacts for other jobs to use. This may seem like nothing, but with
+    # the number of jobs we run daily, this can result in thousands of GB of
+    # network I/O.
+    cmake                                                                      \
+        -S llvm                                                                \
+        -B build                                                               \
+        -G Ninja                                                               \
+        -DCMAKE_CXX_COMPILER_LAUNCHER="ccache"                                 \
+        -DCMAKE_BUILD_TYPE=Release                                             \
+        -DCMAKE_INSTALL_PREFIX=install                                         \
+        -DLLVM_ENABLE_PROJECTS="clang;compiler-rt"                             \
+
+    ninja -C build install-clang install-clang-resource-headers
+    ccache -s
+    tar -cJvf install.tar.xz install/
+    buildkite-agent artifact upload --debug install.tar.xz
+;;
+generic-cxx03)
+    buildkite-agent artifact download install.tar.xz .
+    tar -xvf install.tar.xz
+    export CC=$(pwd)/install/bin/clang
+    export CXX=$(pwd)/install/bin/clang++
+    chmod +x install/bin/clang install/bin/clang++
+    libcxx/utils/ci/run-buildbot generic-cxx03
+;;
+generic-cxx26)
+    buildkite-agent artifact download install.tar.xz .
+    tar -xvf install.tar.xz
+    export CC=$(pwd)/install/bin/clang
+    export CXX=$(pwd)/install/bin/clang++
+    chmod +x install/bin/clang install/bin/clang++
+    libcxx/utils/ci/run-buildbot generic-cxx26
+;;
+generic-modules)
+    buildkite-agent artifact download install.tar.xz .
+    tar -xvf install.tar.xz
+    export CC=$(pwd)/install/bin/clang
+    export CXX=$(pwd)/install/bin/clang++
+    chmod +x install/bin/clang install/bin/clang++
+    libcxx/utils/ci/run-buildbot generic-modules
+;;
+#################################################################
+# Insert vendor-specific internal configurations below.
+#
+# This allows vendors to extend this file with their own internal
+# configurations without running into merge conflicts with upstream.
+#################################################################
+
+#################################################################
+*)
+    echo "${BUILDER} is not a known configuration"
+    exit 1
+;;
+esac

libcxx/utils/ci/generate-buildkite-pipeline

if git diff --name-only HEAD~1 | grep -q -E "^libcxx/|^libcxxabi/|^libunwind/|^runtimes/|^cmake/"; then
   LIBCXX_CHANGED=true
 fi
 
 if git diff --name-only HEAD~1 | grep -q -E "^clang/"; then
   CLANG_CHANGED=true
 fi
 
 if [[ "${CLANG_CHANGED}" == "true" && "${LIBCXX_CHANGED}" != "true" ]]; then
-  cat libcxx/utils/ci/buildkite-pipeline-clang.yml
+  cat clang/utils/ci/buildkite-pipeline.yml
 else
   cat libcxx/utils/ci/buildkite-pipeline.yml
 fi