update pipeline generation script

[goncharov]

Summary

• Add control over "modified files" set and agent targeting rules. Mostly for testing purposes.

• Moved sections that outputs "steps" after functions

• Added step that report back to Phabricator

That will allow to replace step generation from invocation of script https://github.com/google/llvm-premerge-checks/blob/main/scripts/pipeline_premerge.py to plain call to ".ci/generate-buildkite-pipeline-premerge".

[ldionne]

LGTM with a typo.

.ci/generate-buildkite-pipeline-premerge

219

.ci/generate-buildkite-pipeline-premerge

 #!/usr/bin/env bash
 #===----------------------------------------------------------------------===##
 #
 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
 #===----------------------------------------------------------------------===##
 
 #
 # This file generates a Buildkite pipeline that triggers the various CI jobs for
 # the LLVM project during pre-commit CI (each time a Phabricator diff is uploaded).
 #
 # See https://buildkite.com/docs/agent/v3/cli-pipeline#pipeline-format.
 #
+# As this outputs a yaml file, it's possible to log messages to stderr or
+# prefix with "#".
+
+
+set -eu
+set -o pipefail
+
+# Environment variables script works with:
+# List of files affected by this commit
+: ${MODIFIED_FILES:=$(git diff --name-only HEAD~1)}
+# Filter rules for generic windows tests
+: ${WINDOWS_AGENTS:='{"queue": "windows"}'}
+# Filter rules for generic linux tests
+: ${LINUX_AGENTS:='{"queue": "linux"}'}
+# Service agents, for interacting with Phabricator.
+: ${SERVICE_AGENTS:='{"queue": "service"}'}
+# Set by buildkite
+: ${BUILDKITE_COMMIT:=}
+: ${BUILDKITE_BRANCH:=}
 
 reviewID="$(git log --format=%B -n 1 | sed -nE 's/^Review-ID:[[:space:]]*(.+)$/\1/p')"
 if [[ "${reviewID}" != "" ]]; then
   buildMessage="https://llvm.org/${reviewID}"
 else
   buildMessage="Push to branch ${BUILDKITE_BRANCH}"
 fi
 
 cat <<EOF
 steps:
 EOF
 
-# If libc++ or one of the runtimes directories changed
-if git diff --name-only HEAD~1 | grep -q -E "^libcxx/|^libcxxabi/|^libunwind/|^runtimes/|^cmake/"; then
-  cat <<EOF
-  - trigger: "libcxx-ci"
-    build:
-      message: "${buildMessage}"
-      commit: "${BUILDKITE_COMMIT}"
-      branch: "${BUILDKITE_BRANCH}"
-EOF
-fi
+echo "Files modified:" >&2
+echo "$MODIFIED_FILES" >&2
+modified_dirs=$(echo "$MODIFIED_FILES" | cut -d'/' -f1 | sort -u)
+echo "Directories modified:" >&2
+echo "$modified_dirs" >&2
 
-# If clang changed
-if git diff --name-only HEAD~1 | grep -q -E "^clang/"; then
-  cat <<EOF
-  - trigger: "clang-ci"
-    build:
-      message: "${buildMessage}"
-      commit: "${BUILDKITE_COMMIT}"
-      branch: "${BUILDKITE_BRANCH}"
-EOF
-fi
-
-#
-# If we're not running a more specific pipeline, generate a legacy monolithic pipeline
-# based on which subdirectories have been modified. We have some heuristics for this
-# to be reasonable.
-#
-# Individual projects should instead define the pre-commit CI tests that suits their
-# needs while letting them run on the infrastructure provided by LLVM.
-#
 function compute-projects-to-test() {
   projects=${@}
   for project in ${projects}; do
     echo "${project}"
     case ${project} in
     lld)
       for p in bolt cross-project-tests; do
         echo $p
for project in ${projects}; do
     bolt)                ;; # tests are not supported yet
     *)
       echo "${project}"
     ;;
     esac
   done
 }
 
+# Prints only projects that are both present in $modified_dirs and the passed
+# list.
 function keep-modified-projects() {
   projects=${@}
-  git_diff="$(git diff --name-only HEAD~1)"
   for project in ${projects}; do
-    if echo "${git_diff}" | grep -q -E "^${project}/"; then
+    if echo "$modified_dirs" | grep -q -E "^${project}$"; then
       echo "${project}"
     fi
   done
 }
 
 function check-targets() {
   projects=${@}
   for project in ${projects}; do
for project in ${projects}; do
     ;;
     *)
       echo "check-${project}"
     ;;
     esac
   done
 }
 
+# Project specific pipelines.
+
+# If libc++ or one of the runtimes directories changed.
+if echo "$modified_dirs" | grep -q -E "^(libcxx|libcxxabi|libunwind|runtimes|cmake)$"; then
+  cat <<EOF
+- trigger: "libcxx-ci"
+  build:
+    message: "${buildMessage}"
+    commit: "${BUILDKITE_COMMIT}"
+    branch: "${BUILDKITE_BRANCH}"
+EOF
+fi
+
+# If clang changed.
+if echo "$modified_dirs" | grep -q -E "^(clang)$"; then
+  cat <<EOF
+- trigger: "clang-ci"
+  build:
+    message: "${buildMessage}"
+    commit: "${BUILDKITE_COMMIT}"
+    branch: "${BUILDKITE_BRANCH}"
+EOF
+fi
+
+# Generic pipeline for projects that have not defined custom steps.
+#
+# Individual projects should instead define the pre-commit CI tests that suits their
+# needs while letting them run on the infrastructure provided by LLVM.
+
 # Figure out which projects need to be built on each platform
 all_projects="bolt clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl"
 modified_projects="$(keep-modified-projects ${all_projects})"
 
 linux_projects_to_test=$(exclude-linux $(compute-projects-to-test ${modified_projects}))
 linux_check_targets=$(check-targets ${linux_projects_to_test} | sort | uniq)
 linux_projects=$(add-dependencies ${linux_projects_to_test} | sort | uniq)
 
 windows_projects_to_test=$(exclude-windows $(compute-projects-to-test ${modified_projects}))
 windows_check_targets=$(check-targets ${windows_projects_to_test} | sort | uniq)
 windows_projects=$(add-dependencies ${windows_projects_to_test} | sort | uniq)
 
 # Generate the appropriate pipeline
 if [[ "${linux_projects}" != "" ]]; then
   cat <<EOF
-- label: ':linux: x64 Debian'
+- label: ':linux: Linux x64'
   artifact_paths:
-    - '*_result.json'
-    - 'build/monolithic-linux/test-results.xml'
-  agents:
-    queue: 'linux'
+  - artifacts/**/*
+  agents: ${LINUX_AGENTS}
   retry:
     automatic:
       - exit_status: -1  # Agent was lost
         limit: 2
-      - exit_status: 255
-        limit: 2 # Forced agent shutdown
+      - exit_status: 255 # Forced agent shutdown
+        limit: 2
   timeout_in_minutes: 120
   env:
     CC: 'clang'
     CXX: 'clang++'
   commands:
-    - './.ci/monolithic-linux.sh "$(echo ${linux_projects} | tr ' ' ';')" "$(echo ${linux_check_targets})"'
 EOF
 fi
 
 if [[ "${windows_projects}" != "" ]]; then
   cat <<EOF
-- label: ':windows: x64 Windows'
+- label: ':windows: Windows x64'
   artifact_paths:
-    - '*_result.json'
-    - 'build/monolithic-windows/test-results.xml'
-  agents:
-    queue: 'windows'
+  - artifacts/**/*
+  agents: ${WINDOWS_AGENTS}
   retry:
     automatic:
       - exit_status: -1  # Agent was lost
         limit: 2
-      - exit_status: 255
-        limit: 2 # Forced agent shutdown
+      - exit_status: 255 # Forced agent shutdown
+        limit: 2
   timeout_in_minutes: 150
   env:
     CC: 'cl'
     CXX: 'cl'
     LD: 'link'
   commands:
-    - 'C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64'
-    - 'bash .ci/monolithic-windows.sh "$(echo ${windows_projects} | tr ' ' ';')" "$(echo ${windows_check_targets})"'
 EOF
 fi
+
+# If build was triggered from a Phabricator review - send an update back.
+if [[ -n "${ph_target_phid:-}" ]]; then
+  cat << EOF
+- continue_on_failure: true
+  wait: '~'
+- label: ':phabricator: update build status on Phabricator'
+  agents: ${SERVICE_AGENTS}
+  artifact_paths:
+  - artifacts/**/*
+  env:
+    # Temporary disabled before switching pipelines.
+    ph_dry_run_report: '1'
+  commands:
+  - export SRC=\$\${BUILDKITE_BUILD_PATH}/llvm-premerge-checks
+  - rm -rf \$\${SRC}
+  - git clone --depth 1 https://github.com/google/llvm-premerge-checks.git "\$\${SRC}"
+  - cd \$\${SRC}
+  - git fetch origin "main":x
+  - git checkout x
+  - echo "llvm-premerge-checks commit"
+  - git rev-parse HEAD
+  - pip install -q -r \$\${SRC}/scripts/requirements.txt
+  - cd "\$\$BUILDKITE_BUILD_CHECKOUT_PATH"
+  - \$\${SRC}/scripts/summary.py
+  timeout_in_minutes: 10
+EOF
+fi