diff --git a/.pylintrc b/.pylintrc new file mode 100644 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,7 @@ +# add root folder to Pylint PYTHONPATH, so builders can be checked with zorg.* +# packages +[MASTER] +init-hook='import sys; sys.path.append(".")' + +[FORMAT] +max-line-length=120 \ No newline at end of file diff --git a/buildbot/google/docker/buildbot-clangd-ubuntu-clang/Dockerfile b/buildbot/google/docker/buildbot-clangd-ubuntu-clang/Dockerfile --- a/buildbot/google/docker/buildbot-clangd-ubuntu-clang/Dockerfile +++ b/buildbot/google/docker/buildbot-clangd-ubuntu-clang/Dockerfile @@ -4,7 +4,7 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # #===----------------------------------------------------------------------===// -# Docker image used for the mlir-nvidia builder +# Docker image used for the clangd builder # # Environment variables configurable at runtime: # BUILDBOT_PORT - server port to connect to @@ -14,34 +14,40 @@ FROM ubuntu:18.04 # Install build tools. -RUN apt-get update; \ +RUN apt-get update && \ apt-get install -y software-properties-common apt-transport-https ca-certificates \ ninja-build git wget gnupg \ # Install ccache for local caching of builds. ccache \ # Install python for buildbot and LIT. - python3 python3-pip python3-psutil ; \ + python3 python3-pip python3-psutil && \ # Clean apt cache to reduce image size. apt-get clean # install latest LLVM release -RUN bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)";\ +RUN wget https://apt.llvm.org/llvm.sh && \ + chmod +x llvm.sh && \ + ./llvm.sh 11 && \ + rm -rf llvm.sh && \ # clean apt cache to reduce image size apt-get clean # configure default versions of LLVM tools RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-11 100 ;\ update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-11 100 ;\ - update-alternatives --install /usr/bin/lld lld /usr/bin/lld-11 100 - + update-alternatives --install /usr/bin/ld.lld ld.lld /usr/bin/lld-11 100 +# Configure LLVM tools. +ENV CC=clang +ENV CXX=clang++ +ENV LD=LLD # Ubuntu ships with old cmake version, install the latest one # from https://apt.kitware.com/ RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | \ gpg --dearmor - | \ - tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null ;\ - apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' ;\ - apt-get update ;\ - apt-get install -y cmake ;\ + tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null && \ + apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' && \ + apt-get update && \ + apt-get install -y cmake && \ # Clean apt cache to reduce image size. apt-get clean @@ -53,13 +59,12 @@ # The grpc version shipped with Ubuntu 18.04 crashes on TSan checks. ENV GRPC_INSTALL_PATH=/usr/local/lib/grpc RUN cd /tmp ; \ - git clone -b v1.33.2 --depth 1 --recursive --shallow-submodules https://github.com/grpc/grpc ; \ - cd grpc ; \ - mkdir build; cd build ; \ - cmake -G Ninja -DgRPC_INSTALL=ON -DCMAKE_INSTALL_PREFIX=${GRPC_INSTALL_PATH} -DCMAKE_BUILD_TYPE=Release .. ; \ - cmake --build . --target install ; \ + git clone -b v1.36.3 --depth 1 --recursive --shallow-submodules https://github.com/grpc/grpc && \ + cd grpc && mkdir build && cd build && \ + cmake -G Ninja -DgRPC_INSTALL=ON -DCMAKE_INSTALL_PREFIX=${GRPC_INSTALL_PATH} -DCMAKE_BUILD_TYPE=Release .. && \ + cmake --build . --target install && \ # Clean up sources to reduce image size. - cd / ; rm -rf /tmp/grpc + cd / && rm -rf /tmp/grpc # Workaround permissions issues when writing to named volumes # https://github.com/docker/compose/issues/3270#issuecomment-206214034 @@ -94,10 +99,5 @@ # 9994 - integration ENV BUILDBOT_PORT="9994" -# Configure LLVM tools. -ENV CC=clang -ENV CXX=clang++ -ENV LD=LLD - # Run startup script. CMD /home/buildbot/run.sh diff --git a/buildbot/google/docker/buildbot-clangd-ubuntu-clang/VERSION b/buildbot/google/docker/buildbot-clangd-ubuntu-clang/VERSION --- a/buildbot/google/docker/buildbot-clangd-ubuntu-clang/VERSION +++ b/buildbot/google/docker/buildbot-clangd-ubuntu-clang/VERSION @@ -1 +1 @@ -3 +7 diff --git a/buildbot/google/docker/buildbot-clangd-ubuntu-clang/run.sh b/buildbot/google/docker/buildbot-clangd-ubuntu-clang/run.sh --- a/buildbot/google/docker/buildbot-clangd-ubuntu-clang/run.sh +++ b/buildbot/google/docker/buildbot-clangd-ubuntu-clang/run.sh @@ -6,7 +6,6 @@ # #===----------------------------------------------------------------------===// # This script will start the buildbot worker -# #===----------------------------------------------------------------------===// set -eu @@ -15,20 +14,20 @@ WORKER_PASSWORD=$(cat /vol/secrets/token) # Set up buildbot host and maintainer info. -mkdir -p "${WORKER_NAME}/info/" +mkdir -p "${WORKER_NAME}/info/" echo "Christian Kuehnel " > "${WORKER_NAME}/info/admin" # generate the host information of this worker -( +( uname -a ; \ cat /proc/cpuinfo | grep "model name" | head -n1 | cut -d " " -f 3- ;\ echo "number of cores: $(nproc)" ;\ cat /proc/meminfo | grep MemTo ;\ lsb_release -d | cut -f 2- ; \ clang --version | head -n1 ; \ - ld.lld-11 --version ; \ + ld.lld --version ; \ cmake --version | head -n1 ; \ -) > ${WORKER_NAME}/info/host +) > ${WORKER_NAME}/info/host echo "Host information:" cat ${WORKER_NAME}/info/host @@ -38,9 +37,14 @@ buildbot-worker create-worker --keepalive=200 "${WORKER_NAME}" \ lab.llvm.org:${BUILDBOT_PORT} "${WORKER_NAME}" "${WORKER_PASSWORD}" -# start the daemon, this command return immetiately +# start the daemon, this command returns immediately echo "starting worker..." -buildbot-worker start "${WORKER_NAME}" +set +e +if ! buildbot-worker start "${WORKER_NAME}"; then + echo ERROR: starting worker failed. contents of twistd.log: + cat "${WORKER_NAME}/twistd.log" + exit 1 +fi # To keep the container running and produce log outputs: dump the worker # log to stdout diff --git a/buildbot/google/docker/buildbot-cpp20/Dockerfile b/buildbot/google/docker/buildbot-cpp20/Dockerfile new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-cpp20/Dockerfile @@ -0,0 +1,70 @@ +#===-- Dockerfile --------------------------------------------------------===// +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +#===----------------------------------------------------------------------===// +# Environment variables configurable at runtime: +# BUILDBOT_PORT - server port to connect to +#===----------------------------------------------------------------------===// + +FROM debian:sid + +# Install build tools. +RUN apt-get update && \ + apt-get install -y \ + software-properties-common apt-transport-https ca-certificates \ + git wget gnupg \ + cmake ninja-build \ + ccache \ + python3 python3-pip python3-psutil && \ + # Install clang-15 to make the minimal C++20 support in the toolchain. + echo "deb http://apt.llvm.org/unstable/ llvm-toolchain-15 main" >> /etc/apt/sources.list.d/llvm-15.list && \ + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key >> /etc/apt/trusted.gpg.d/llvm.asc && \ + apt-get update && \ + apt-get install -y clang-15 lld-15 && \ + apt-get clean + +# Install build bot (server was at 2.8.5-dev at time of writing). +RUN pip3 install buildbot-worker==2.8.4 + +# Workaround permissions issues when writing to named volumes +# https://github.com/docker/compose/issues/3270#issuecomment-206214034 +RUN mkdir -p /vol/test /vol/ccache /vol/worker ; \ + chmod -R 777 /vol + +# Volume to mount secrets into the container. +VOLUME /vol/secrets +# Volume to store data for local, manual testing of the container. +VOLUME /vol/test +# Volume to store ccache. +VOLUME /vol/ccache +ENV CCACHE_DIR=/vol/ccache +# Volume for worker working directory. +VOLUME /vol/worker + +# Create user account, some tests fail if run as root. +RUN useradd buildbot --create-home +WORKDIR /vol/worker + +# Copy startup script. +COPY run.sh /home/buildbot/ +RUN chmod a+rx /home/buildbot/run.sh + +USER buildbot +ENV WORKER_NAME="clang-debian-cpp20" + +# Allow the server port of this agent to be configurable during deployment. +# This way we can connect the same image to production and integration. +# Ports: +# 9990 - production +# 9994 - integration +ENV BUILDBOT_PORT="9994" + +# Configure LLVM tools. +ENV CC=clang-15 +ENV CXX=clang++-15 +ENV LD=ld.lld-15 + +# Run startup script. +CMD /home/buildbot/run.sh diff --git a/buildbot/google/docker/buildbot-cpp20/VERSION b/buildbot/google/docker/buildbot-cpp20/VERSION new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-cpp20/VERSION @@ -0,0 +1 @@ +4 diff --git a/buildbot/google/docker/buildbot-clangd-ubuntu-clang/run.sh b/buildbot/google/docker/buildbot-cpp20/run.sh copy from buildbot/google/docker/buildbot-clangd-ubuntu-clang/run.sh copy to buildbot/google/docker/buildbot-cpp20/run.sh --- a/buildbot/google/docker/buildbot-clangd-ubuntu-clang/run.sh +++ b/buildbot/google/docker/buildbot-cpp20/run.sh @@ -6,7 +6,7 @@ # #===----------------------------------------------------------------------===// # This script will start the buildbot worker -# +# #===----------------------------------------------------------------------===// set -eu @@ -15,20 +15,20 @@ WORKER_PASSWORD=$(cat /vol/secrets/token) # Set up buildbot host and maintainer info. -mkdir -p "${WORKER_NAME}/info/" -echo "Christian Kuehnel " > "${WORKER_NAME}/info/admin" +mkdir -p "${WORKER_NAME}/info/" +echo "Ilya Biryukov " > "${WORKER_NAME}/info/admin" # generate the host information of this worker -( +( uname -a ; \ cat /proc/cpuinfo | grep "model name" | head -n1 | cut -d " " -f 3- ;\ echo "number of cores: $(nproc)" ;\ cat /proc/meminfo | grep MemTo ;\ lsb_release -d | cut -f 2- ; \ - clang --version | head -n1 ; \ - ld.lld-11 --version ; \ + $CC --version | head -n1 ; \ + $LD --version ; \ cmake --version | head -n1 ; \ -) > ${WORKER_NAME}/info/host +) > ${WORKER_NAME}/info/host echo "Host information:" cat ${WORKER_NAME}/info/host @@ -38,9 +38,14 @@ buildbot-worker create-worker --keepalive=200 "${WORKER_NAME}" \ lab.llvm.org:${BUILDBOT_PORT} "${WORKER_NAME}" "${WORKER_PASSWORD}" -# start the daemon, this command return immetiately +# start the daemon, this command returns immediately echo "starting worker..." -buildbot-worker start "${WORKER_NAME}" +set +e +if ! buildbot-worker start "${WORKER_NAME}"; then + echo ERROR: starting worker failed. contents of twistd.log: + cat "${WORKER_NAME}/twistd.log" + exit 1 +fi # To keep the container running and produce log outputs: dump the worker # log to stdout diff --git a/buildbot/google/docker/buildbot-mlir-nvidia/Dockerfile b/buildbot/google/docker/buildbot-mlir-nvidia/Dockerfile deleted file mode 100644 --- a/buildbot/google/docker/buildbot-mlir-nvidia/Dockerfile +++ /dev/null @@ -1,87 +0,0 @@ -#===-- Dockerfile --------------------------------------------------------===// -# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -# -#===----------------------------------------------------------------------===// -# Docker image used for the mlir-nvidia builder -# -# Environment variables configurable at runtime: -# BUILDBOT_PORT - server port to connect to -#===----------------------------------------------------------------------===// - -# Use the image from NVIDIA as base -FROM nvidia/cuda:10.2-base-ubuntu18.04 - - -# install build tools -RUN apt-get update; \ - apt-get install -y software-properties-common apt-transport-https ca-certificates \ - clang-8 lld-8 ninja-build git wget gnupg ccache \ - python3 python3-pip python3-psutil pybind11-dev \ - # dumb-init recommended in - # https://hub.docker.com/r/buildbot/buildbot-worker/dockerfile - dumb-init;\ - update-alternatives --install /usr/bin/clang clang /usr/bin/clang-8 100 ;\ - update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-8 100 ;\ - update-alternatives --install /usr/bin/lld lld /usr/bin/lld-8 100 - -# install cuda -# avoid popups for keyboard configurations -RUN DEBIAN_FRONTEND=noninteractive apt-get install -y cuda - -# Ubuntu ships with old cmake version, install the latest one -# from https://apt.kitware.com/ -RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | \ - gpg --dearmor - | \ - tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null ;\ - apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' ;\ - apt-get update ;\ - apt-get install -y cmake - -# Get the Vulkan SDK from LunarG. The CUDA meta package already pulls in NVIDIA -# Vulkan ICD. -RUN wget -qO - http://packages.lunarg.com/lunarg-signing-key-pub.asc 2>/dev/null | \ - apt-key add - ;\ - wget -qO /etc/apt/sources.list.d/lunarg-vulkan-1.2.141-bionic.list \ - http://packages.lunarg.com/vulkan/1.2.141/lunarg-vulkan-1.2.141-bionic.list;\ - apt-get update ;\ - apt-get install -y vulkan-sdk - -# Install build bot (server was at 2.8.5-dev at time of writing). -RUN pip3 install buildbot-worker==2.8.4 - -# Workaround permissions issues when writing to named volumes -# https://github.com/docker/compose/issues/3270#issuecomment-206214034 -RUN mkdir -p /vol/test /vol/ccache /vol/worker ; \ - chmod -R 777 /vol - -# Volume to mount secrets into the container. -VOLUME /vol/secrets -# Volume to store data for local, manual testing of the container. -VOLUME /vol/test -# Volume to store ccache. -VOLUME /vol/ccache -ENV CCACHE_DIR=/vol/ccache -# Volume for worker working directory. -VOLUME /vol/worker - -# Create user account, some tests fail if run as root. -RUN useradd buildbot --create-home -WORKDIR /vol/worker - -# copy startup script -COPY run.sh /home/buildbot/ -RUN chmod a+rx /home/buildbot/run.sh - -USER buildbot -ENV WORKER_NAME="mlir-nvidia" - -# Allow the server port of this agent to be configurable during deployment. -# This way we can connect the same image to production and integration. -# Ports: -# 9990 - production -# 9994 - integration -ENV BUILDBOT_PORT="9994" - -CMD /home/buildbot/run.sh diff --git a/buildbot/google/docker/buildbot-mlir-nvidia/VERSION b/buildbot/google/docker/buildbot-mlir-nvidia/VERSION deleted file mode 100644 --- a/buildbot/google/docker/buildbot-mlir-nvidia/VERSION +++ /dev/null @@ -1 +0,0 @@ -15 diff --git a/buildbot/google/docker/buildbot-sanitizer-windows/Dockerfile b/buildbot/google/docker/buildbot-sanitizer-windows/Dockerfile new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-sanitizer-windows/Dockerfile @@ -0,0 +1,21 @@ +# escape=` +FROM gcr.io/sanitizer-bots/windows-base-vscode2019:8 + +ENV WORKER_NAME=sanitizer-windows + +# copy script to start the agent +COPY run.ps1 . +COPY fix_buildbot_tac_paths.py . + +# Set location for sccache cache +ENV SCCACHE_DIR="C:\volumes\sccache" + +# Set the maximum sccache local cache size +ENV SCCACHE_CACHE_SIZE="10G" + +# Move buildbot work dir to a volume to avoid of disc issues +VOLUME C:\volumes\buildbot + +# Configure 32bit tools ("x86") instead of 64bit ("amd64") +CMD ["powershell.exe", "-NoLogo", "-ExecutionPolicy", "Bypass", ` + "c:\\buildbot\\run.ps1"] diff --git a/buildbot/google/docker/buildbot-sanitizer-windows/VERSION b/buildbot/google/docker/buildbot-sanitizer-windows/VERSION new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-sanitizer-windows/VERSION @@ -0,0 +1 @@ +1 diff --git a/buildbot/google/docker/buildbot-sanitizer-windows/admin b/buildbot/google/docker/buildbot-sanitizer-windows/admin new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-sanitizer-windows/admin @@ -0,0 +1 @@ +Reid Kleckner diff --git a/buildbot/google/docker/buildbot-sanitizer-windows/fix_buildbot_tac_paths.py b/buildbot/google/docker/buildbot-sanitizer-windows/fix_buildbot_tac_paths.py new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-sanitizer-windows/fix_buildbot_tac_paths.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +# Replace backward slashes in the paths in a "buildbot.tac" file + +import argparse +import re +import shutil + + +def fix_slashes(buildbot_tac_path : str): + shutil.copy(buildbot_tac_path, buildbot_tac_path + ".bak") + result = [] + with open(buildbot_tac_path) as buildbot_tac_file: + buildbot_tac = buildbot_tac_file.readlines() + for line in buildbot_tac: + if line.lstrip().startswith('basedir'): + line = line.replace(r'\\','/') + result.append(line) + with open(buildbot_tac_path, 'w') as buildbot_tac_file: + buildbot_tac_file.writelines(result) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('buildbot_tac_path', help="path of 'buildbot.tac' file") + args = parser.parse_args() + fix_slashes(args.buildbot_tac_path) diff --git a/buildbot/google/docker/buildbot-sanitizer-windows/run.ps1 b/buildbot/google/docker/buildbot-sanitizer-windows/run.ps1 new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-sanitizer-windows/run.ps1 @@ -0,0 +1,51 @@ +# Stop the script on the first error +$ErrorActionPreference = "Stop" + +# Read password from file +$WORKER_PASSWORD=Get-Content "C:\volumes\secrets\token" + +#Create short drive alias "Q:" for worker dir, to keep path lengths <260 +SUBST Q: C:\volumes\buildbot +Get-PSDrive + +# chdir to the volume storing the temporary data +# This is a mounted volume +Set-Location Q:\ + +# Write host and admin information +# delete old folder if it exists +# configure powershell output to use UTF-8, otherwiese buildbot can't read the file +if ( Test-Path -Path 'info\' -PathType Container ) { + Remove-Item -Recurse -Force info +} +mkdir "info\" +$HOST_FILE="info\host" +$PSDefaultParameterValues['Out-File:Encoding'] = 'utf8' +Write-Output "Windows version: $(cmd /c ver)" > $HOST_FILE +Write-Output "VCTools version: $(ls C:\BuildTools\VC\Tools\MSVC | Select -ExpandProperty Name)" >> $HOST_FILE +Write-Output "Cores : $((Get-CimInstance Win32_ComputerSystem).NumberOfLogicalProcessors)" >> $HOST_FILE +Write-Output "RAM : $([int][Math]::Round((Get-CimInstance Win32_ComputerSystem).TotalPhysicalMemory / 1073741824)) GB" >> $HOST_FILE +Write-Output "Reid Kleckner " > "info\admin" + +# create the worker +Write-Output "creating worker..." +buildslave create-slave --keepalive=200 Q: ` + "lab.llvm.org:9994" "$env:WORKER_NAME" "$WORKER_PASSWORD" +# Note: Powershell does NOT exit on non-zero return codes, so we need to check manually. +if ($LASTEXITCODE -ne 0) { throw "Exit code of 'buildslave create' is $LASTEXITCODE" } + +# Replace backward slashes with forward slashes on buildbot.tac +# Otherwise Cmake will go into infinite re-configuration loops +python C:\buildbot\fix_buildbot_tac_paths.py buildbot.tac +if ($LASTEXITCODE -ne 0) { throw "Exit code of 'fix_buildbot_tac_paths' is $LASTEXITCODE" } + +# start the daemon, as this does not print and logs to std out, run it in the background +Write-Output "starting worker..." +cmd /c Start /B buildslave start Q: + +# Wait a bit until the logfile exists +Start-Sleep -s 5 + +# To keep the container running and produce log outputs: dump the worker +# log to stdout, this is the windows equivalent of `tail -f` +Get-Content -Path "Q:\twistd.log" -Wait diff --git a/buildbot/google/terraform/README.md b/buildbot/google/terraform/README.md --- a/buildbot/google/terraform/README.md +++ b/buildbot/google/terraform/README.md @@ -52,7 +52,7 @@ place on kubernetes is a "secret". To create a kubernetes secret for the agent token: ```bash -kubectl create secret generic buildbot-token-mlir-nvidia --from-file=token= +kubectl create secret generic buildbot-token- --from-file=token= ``` The file in `` then must contain the password of the buildbot worker in plain text. In the `Deployment` of a container, the secret is defined as a @@ -60,11 +60,11 @@ secret can then be read from that file. An example: -The secret `buildbot-token-mlir-nvidia` is defined (as above) in Kubernetes. -In the [deployment](buildbot/google/terraform/main.tf) `mlir-nvidia` it is +The secret `buildbot-token-` is defined (as above) in Kubernetes. +In the [deployment](buildbot/google/terraform/main.tf) `` it is used as a volume of type `secret` and then mounted at `/secrets`. During the runtime of the docker container, the script -[run.sh](../docker/buildbot-mlir-nvidia/run.sh) reads the secret from the file +[run.sh](../docker/buildbot-/run.sh) reads the secret from the file `/secrets/token` and uses it to create the worker configuration. @@ -74,7 +74,7 @@ So we need to deploy such cases using plain Kubernetes configuration files. See this [issue](https://github.com/terraform-providers/terraform-provider-kubernetes/issues/149) for more details. -The buildbot mlir-nvidia is configured in `deployment-mlir-nvidia.yaml` in this +The buildbot is configured in `deployment-.yaml` in this folder. For all non-GPU cases add a `"kubernetes_deployment"` to `main.tf`. diff --git a/buildbot/google/terraform/deployment-clang-debian-cpp20.yaml b/buildbot/google/terraform/deployment-clang-debian-cpp20.yaml new file mode 100644 --- /dev/null +++ b/buildbot/google/terraform/deployment-clang-debian-cpp20.yaml @@ -0,0 +1,58 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: clang-debian-cpp20 + name: clang-debian-cpp20 + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: clang-debian-cpp20 + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + app: clang-debian-cpp20 + spec: + automountServiceAccountToken: false + containers: + - env: + - name: BUILDBOT_PORT + value: "9990" # 9990 is production, 9994 is staging. + image: gcr.io/sanitizer-bots/buildbot-cpp20:4 + name: buildbot-clang-debian-cpp20 + resources: + limits: + cpu: "15" + memory: 45G + requests: + cpu: "15" + memory: 45G + volumeMounts: + - mountPath: /vol/secrets + mountPropagation: None + name: buildbot-token + - mountPath: /vol/cccache + mountPropagation: None + name: ccache-vol + - mountPath: /vol/worker + mountPropagation: None + name: worker-vol + nodeSelector: + pool: linux-16-core-pool + restartPolicy: Always + volumes: + - name: buildbot-token + secret: + optional: false + secretName: password-clangd-ubuntu-clang + - emptyDir: {} + name: ccache-vol + - emptyDir: {} + name: worker-vol diff --git a/buildbot/google/terraform/deployment-clangd-ubuntu-clang.yaml b/buildbot/google/terraform/deployment-clangd-ubuntu-clang.yaml new file mode 100644 --- /dev/null +++ b/buildbot/google/terraform/deployment-clangd-ubuntu-clang.yaml @@ -0,0 +1,59 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: clangd-ubuntu-clang + name: clangd-ubuntu-clang + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: clangd-ubuntu-clang + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + app: clangd-ubuntu-clang + spec: + automountServiceAccountToken: false + containers: + - env: + - name: BUILDBOT_PORT + value: "9990" # 9990 is production, 9994 is staging. + image: gcr.io/sanitizer-bots/buildbot-clangd-ubuntu-clang:7 + name: buildbot-clangd-ubuntu-clang + resources: + limits: + cpu: "15" + memory: 45G + requests: + cpu: "15" + memory: 45G + volumeMounts: + - mountPath: /vol/secrets + mountPropagation: None + name: buildbot-token + - mountPath: /vol/ccache + mountPropagation: None + name: ccache-vol + - mountPath: /vol/worker + mountPropagation: None + name: worker-vol + nodeSelector: + pool: linux-16-core-pool + restartPolicy: Always + volumes: + - name: buildbot-token + secret: + optional: false + secretName: password-clangd-ubuntu-clang + - emptyDir: {} + name: ccache-vol + - emptyDir: {} + name: worker-vol + diff --git a/buildbot/google/terraform/deployment-mlir-nvidia-production.yaml b/buildbot/google/terraform/deployment-mlir-nvidia-production.yaml deleted file mode 100644 --- a/buildbot/google/terraform/deployment-mlir-nvidia-production.yaml +++ /dev/null @@ -1,76 +0,0 @@ ---- -# deployment for production at http://lab.llvm.org:8011/ -# The deployment for staging is in a separate file! -apiVersion: apps/v1 -kind: Deployment -metadata: - name: mlir-nvidia-production -spec: - # number of instances we want to run - replicas: 1 - selector: - matchLabels: - app: mlir-nvidia-production - # define strategy for updating the images - strategy: - rollingUpdate: - # do not deploy more replicas, as the buildbot server - # can't handle multiple workers with the same credentials - maxSurge: 0 - # Allow to have 0 replicas during updates. - maxUnavailable: 1 - type: RollingUpdate - template: - metadata: - labels: - app: mlir-nvidia-production - spec: - containers: - # the image and version we want to run - - image: gcr.io/sanitizer-bots/buildbot-mlir-nvidia:15 - name: mlir-nvidia-production - env: - # connect to production environment, running at port 9990 - - name: BUILDBOT_PORT - value: "9990" - # reserve "-1" for this image, kubernetes also - # needs <1 core for management tools - resources: - limits: - cpu: "15" - memory: 10Gi - # also request to use the GPU - # Note: this does not work in terraform at the moment - nvidia.com/gpu: "1" - requests: - cpu: "15" - memory: 10Gi - nvidia.com/gpu: "1" - volumeMounts: - # mount the secrets into a folder - - mountPath: /vol/secrets - mountPropagation: None - name: buildbot-token - # storage for ccache - - mountPath: /vol/ccache - mountPropagation: None - name: ccache-vol - # work directory of buildbot - - mountPath: /vol/buildbot - mountPropagation: None - name: buildbot-vol - # specify the nood pool on which to deploy - nodeSelector: - pool: nvidia-16core-pool - restartPolicy: Always - volumes: - # declare the secret as a volume so we can mount it - - name: buildbot-token - secret: - optional: false - secretName: password-mlir-nvidia - - name: ccache-vol - emptyDir: {} - - name: buildbot-vol - emptyDir: {} - diff --git a/buildbot/google/terraform/deployment-mlir-nvidia-staging.yaml b/buildbot/google/terraform/deployment-mlir-nvidia-staging.yaml deleted file mode 100644 --- a/buildbot/google/terraform/deployment-mlir-nvidia-staging.yaml +++ /dev/null @@ -1,91 +0,0 @@ ---- -# deployment for staging, this connects the agent to the staging -# environment at http://lab.llvm.org:8014/ -# the deployment for production is in a separate file -apiVersion: apps/v1 -kind: Deployment -metadata: - name: mlir-nvidia-staging -spec: - # number of instances we want to run - replicas: 1 - selector: - matchLabels: - app: mlir-nvidia-staging - # define strategy for updating the images - strategy: - rollingUpdate: - # do not deploy more replicas, as the buildbot server - # can't handle multiple workers with the same credentials - maxSurge: 0 - # Allow to have 0 replicas during updates. - maxUnavailable: 1 - type: RollingUpdate - template: - metadata: - labels: - app: mlir-nvidia-staging - spec: - containers: - # the image and version we want to run - - image: gcr.io/sanitizer-bots/buildbot-mlir-nvidia:15 - name: mlir-nvidia-staging - # set environment variables - env: - # connect to staging environment, running at port 9994 (instead of 9990) - - name: BUILDBOT_PORT - value: "9994" - # reserve "-1" for this image, kubernetes also - # needs <1 core for management tools - resources: - limits: - cpu: "15" - memory: 10Gi - # also request to use the GPU - # Note: this does not work in terraform at the moment - nvidia.com/gpu: "1" - requests: - cpu: "15" - memory: 10Gi - nvidia.com/gpu: "1" - volumeMounts: - # mount the secrets into a folder - - mountPath: /secrets - mountPropagation: None - name: buildbot-token - # specify the nood pool on which to deploy - nodeSelector: - pool: nvidia-16core-pool - restartPolicy: Always - volumes: - # declare the secret as a volume so we can mount it - - name: buildbot-token - secret: - optional: false - secretName: password-mlir-nvidia - # mount the secrets into a folder - - mountPath: /vol/secrets - mountPropagation: None - name: buildbot-token - # storage for ccache - - mountPath: /vol/ccache - mountPropagation: None - name: ccache-vol - # work directory of buildbot - - mountPath: /vol/buildbot - mountPropagation: None - name: buildbot-vol - # specify the nood pool on which to deploy - nodeSelector: - pool: nvidia-16core-pool - restartPolicy: Always - volumes: - # declare the secret as a volume so we can mount it - - name: buildbot-token - secret: - optional: false - secretName: password-mlir-nvidia - - name: ccache-vol - emptyDir: {} - - name: buildbot-vol - emptyDir: {} diff --git a/buildbot/google/terraform/main.tf b/buildbot/google/terraform/main.tf --- a/buildbot/google/terraform/main.tf +++ b/buildbot/google/terraform/main.tf @@ -50,231 +50,6 @@ } -resource "null_resource" "update_cluster" { - # Add NVIDIA driver daemonset. - depends_on = [google_container_cluster.primary] - # Update kubectl context for the cluster and apply nvidia's daemonset. - provisioner "local-exec" { - command = <-1" for this image, kubernetes also - # needs <1 core for management tools - resources { - limits { - cpu = "31" - memory = "20Gi" - } - requests { - cpu = "31" - memory = "20Gi" - } - } - - # mount the secrets into a folder - volume_mount { - mount_path = "c:\\volumes\\secrets" - name = "buildbot-token" - } - volume_mount { - mount_path = "c:\\volumes\\sccache" - name = "sccache-vol" - } - volume_mount { - mount_path = "c:\\volumes\\buildbot" - name = "buildbot-vol" - } - - } - # select which node pool to deploy to - node_selector = { - pool = "win-32core-pool" - } - # restart in case of any crashes - restart_policy = "Always" - - # select the secret to be mounted - volume { - name = "buildbot-token" - secret { - optional = false - secret_name = "password-windows10-vs2019" - } - } - volume { - name = "sccache-vol" - empty_dir {} - } - volume { - name = "buildbot-vol" - empty_dir {} - } - - # Windows nodes from the node pool are marked with the taint - # "node.kubernetes.io/os=windows". So we need to "tolerate" this to - # deploy to such nodes. - toleration { - effect = "NoSchedule" - key = "node.kubernetes.io/os" - operator = "Equal" - value = "windows" - } - } - } - } -} - - resource "google_container_node_pool" "linux_16_core_pool" { name = "linux-16-core-pool" # specify a zone here (e.g. "-a") to avoid a redundant deployment @@ -290,8 +65,8 @@ node_config { # use preemptible, as this saves costs preemptible = true - #custom machine type: 16 core, 32 GB as tsan needs more RAM - machine_type = "n2d-custom-16-32768" + #custom machine type: 16 core, 48 GB as tsan needs more RAM + machine_type = "n2d-custom-16-49152" disk_size_gb = 100 disk_type = "pd-ssd" @@ -309,106 +84,3 @@ } } } - - -resource "kubernetes_deployment" "clangd-ubuntu-clang" { - metadata { - name = "clangd-ubuntu-clang" - labels = { - app = "clangd-ubuntu-clang" - } - } - - spec { - # create one instance of this container - replicas = 1 - - selector { - match_labels = { - app = "clangd-ubuntu-clang" - } - } - strategy{ - rolling_update{ - # do not deploy more replicas, as the buildbot server - # can't handle multiple workers with the same credentials - max_surge = 0 - # Allow to have 0 replicas during updates. - max_unavailable = 1 - } - type = "RollingUpdate" - } - template { - metadata { - labels = { - app = "clangd-ubuntu-clang" - } - } - - spec { - container { - image = "${var.gcp_config.gcr_prefix}/buildbot-clangd-ubuntu-clang:3" - name = "buildbot-clangd-ubuntu-clang" - - # reserve "-1" for this image, kubernetes also - # needs <1 core for management tools - resources { - limits { - cpu = "15" - memory = "28G" - } - requests { - cpu = "15" - memory = "28G" - } - } - - # mount the secrets into a folder - volume_mount { - mount_path = "/vol/secrets" - name = "buildbot-token" - } - volume_mount { - mount_path = "/vol/cccache" - name = "ccache-vol" - } - volume_mount { - mount_path = "/vol/worker" - name = "worker-vol" - } - - env { - # connect to production environment, running at port 9990 - # staging would be at 9994 - name = "BUILDBOT_PORT" - value = "9990" - } - } - # select which node pool to deploy to - node_selector = { - pool = "linux-16-core-pool" - } - # restart in case of any crashes - restart_policy = "Always" - - # select the secret to be mounted - volume { - name = "buildbot-token" - secret { - optional = false - secret_name = "password-clangd-ubuntu-clang" - } - } - volume { - name = "ccache-vol" - empty_dir {} - } - volume { - name = "worker-vol" - empty_dir {} - } - - } - } - } -} diff --git a/buildbot/mlir-nvidia/Dockerfile b/buildbot/mlir-nvidia/Dockerfile new file mode 100644 --- /dev/null +++ b/buildbot/mlir-nvidia/Dockerfile @@ -0,0 +1,109 @@ +#===-- Dockerfile --------------------------------------------------------===// +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +#===----------------------------------------------------------------------===// +# Docker image used for the mlir-nvidia builder +# +# Environment variables configurable at runtime: +# BUILDBOT_PORT - server port to connect to +#===----------------------------------------------------------------------===// + +# Use the image from NVIDIA +FROM nvidia/cuda:12.1.1-devel-ubuntu20.04 + +# install extra build tools +ENV TZ=Etc/UTC +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + apt-transport-https \ + ca-certificates \ + ccache \ + clang-10 \ + # dumb-init recommended in + # https://hub.docker.com/r/buildbot/buildbot-worker/dockerfile + dumb-init \ + git \ + gnupg \ + libpython3-dev \ + lld-10 \ + ninja-build \ + python3 \ + python3-pip \ + python3-psutil \ + software-properties-common \ + wget \ + && update-alternatives --install /usr/bin/clang clang /usr/bin/clang-10 100 \ + && update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-10 100 \ + && update-alternatives --install /usr/bin/ld.lld lld /usr/bin/lld-10 100 \ + && rm -rf /var/lib/apt/lists/* + +# LTS releases often bundle obsolete pip versions that cannot access newest +# Linux binary wheels. This pinned version is not special: it was just current +# at the time this was added. Refer to compatibility table: +# https://github.com/pypa/manylinux +RUN python3 -m pip install --upgrade pip==23.1.2 + +# Refer to mlir/lib/Bindings/Python/requirements.txt. Listed explicitly here +# and version pinned for consistency as this is a bot. +RUN python3 -m pip install numpy==1.24.3 pybind11==2.10.4 PyYAML==5.3.1 + +# Ubuntu ships with old cmake version, install the latest one +# from https://apt.kitware.com/ +RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null \ + | gpg --dearmor - \ + | \ + tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null \ + && apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' \ + && apt-get update \ + && apt-get install -y --no-install-recommends cmake \ + && rm -rf /var/lib/apt/lists/* + +# Get the Vulkan SDK from LunarG. +RUN wget -qO - http://packages.lunarg.com/lunarg-signing-key-pub.asc 2>/dev/null \ + | apt-key add - \ + && wget -qO /etc/apt/sources.list.d/lunarg-vulkan-1.2.141-bionic.list \ + https://packages.lunarg.com/vulkan/1.3.243/lunarg-vulkan-1.3.243-focal.list \ + && apt-get update \ + && apt-get install -y --no-install-recommends vulkan-sdk \ + && rm -rf /var/lib/apt/lists/* + + +# Right now the container-optimized OS on Google Cloud does not have the Vulkan +# components in the installed driver, install them in the container. This creates +# a coupling with the actual driver running on the VM unfortunately. +RUN apt-get update \ + && apt-get install -y --no-install-recommends libnvidia-gl-470-server \ + && rm -rf /var/lib/apt/lists/* + +# Install build bot (server was at 2.8.5-dev at time of writing). +RUN pip3 install buildbot-worker==2.8.4 + +# Create user account, some tests fail if run as root. +RUN useradd buildbot --create-home +WORKDIR /vol/worker +RUN mkdir -p /vol/worker \ + && chown buildbot /vol/worker + +# Configure ccache (default is 5GB caching) +RUN mkdir /home/buildbot/.ccache/ \ + && echo "max_size = 30.0G" > /home/buildbot/.ccache/ccache.conf \ + && chown -R buildbot /home/buildbot/.ccache + +# copy startup script +COPY run.sh /home/buildbot/ +RUN chmod a+rx /home/buildbot/run.sh + +USER buildbot +ENV WORKER_NAME="mlir-nvidia" + +# Allow the server port of this agent to be configurable during deployment. +# This way we can connect the same image to production and integration. +# Ports: +# 9990 - production +# 9994 - integration +ENV BUILDBOT_PORT="9994" + +CMD /home/buildbot/run.sh diff --git a/buildbot/google/docker/buildbot-mlir-nvidia/run.sh b/buildbot/mlir-nvidia/run.sh rename from buildbot/google/docker/buildbot-mlir-nvidia/run.sh rename to buildbot/mlir-nvidia/run.sh --- a/buildbot/google/docker/buildbot-mlir-nvidia/run.sh +++ b/buildbot/mlir-nvidia/run.sh @@ -12,25 +12,28 @@ set -eu # Read the worker password from a mounted file. -WORKER_PASSWORD=$(cat /vol/secrets/token) +if [[ -z "${WORKER_PASSWORD}" ]]; then + echo "Missing WORKER_PASSWORD environment variable" + exit 1 +fi # Set up buildbot host and maintainer info. -mkdir -p "${WORKER_NAME}/info/" -echo "Christian Kuehnel " > "${WORKER_NAME}/info/admin" +mkdir -p "${WORKER_NAME}/info/" +echo "Mehdi Amini " > "${WORKER_NAME}/info/admin" # generate the host information of this worker -( +( uname -a ; \ cat /proc/cpuinfo | grep "model name" | head -n1 | cut -d " " -f 3- ;\ echo "number of cores: $(nproc)" ;\ nvidia-smi -L | cut -d "(" -f 1 ;\ lsb_release -d | cut -f 2- ; \ clang --version | head -n1 ; \ - ld.lld-8 --version ; \ + ld.lld --version ; \ cmake --version | head -n1 ; \ vulkaninfo 2>/dev/null | grep "Vulkan Instance" ; \ vulkaninfo 2>/dev/null | grep "apiVersion" | cut -d= -f2 | awk '{printf "NVIDIA Vulkan ICD Version: " $2 "\n"}' -) > ${WORKER_NAME}/info/host +) > ${WORKER_NAME}/info/host echo "Full nvidia-smi output:" nvidia-smi @@ -41,9 +44,9 @@ echo "Host information:" cat ${WORKER_NAME}/info/host -# It looks like GKE sometimes deploys the container before the NVIDIA drivers -# are loaded on the host. In this case the GPU is not available during the -# entire lifecycle of the container. Not sure how to fix this properly. +# It looks like GKE sometimes deploys the container before the NVIDIA drivers +# are loaded on the host. In this case the GPU is not available during the +# entire lifecycle of the container. Not sure how to fix this properly. RETURN_CODE=$(nvidia-smi > /dev/null ; echo $?) if [[ "$RETURN_CODE" != "0" ]] ; then @@ -61,4 +64,4 @@ # start the worker, based on # https://hub.docker.com/r/buildbot/buildbot-worker/dockerfile echo "starting worker..." -/usr/bin/dumb-init twistd --pidfile= --nodaemon -l - --python="${WORKER_NAME}/buildbot.tac" \ No newline at end of file +/usr/bin/dumb-init twistd --pidfile= --nodaemon -l - --python="${WORKER_NAME}/buildbot.tac" diff --git a/buildbot/osuosl/master/config/__init__.py b/buildbot/osuosl/master/config/__init__.py --- a/buildbot/osuosl/master/config/__init__.py +++ b/buildbot/osuosl/master/config/__init__.py @@ -12,6 +12,7 @@ import config.auth import config.builders +import config.release_builders import config.schedulers import config.workers import config.status @@ -22,4 +23,5 @@ reload(config.workers) reload(config.schedulers) reload(config.builders) +reload(release_builders) reload(config.status) diff --git a/buildbot/osuosl/master/config/auth.py b/buildbot/osuosl/master/config/auth.py --- a/buildbot/osuosl/master/config/auth.py +++ b/buildbot/osuosl/master/config/auth.py @@ -1,6 +1,10 @@ from buildbot.plugins import util #from twisted.python import log +from zorg.buildbot.util.workerowner import WorkerOwnerAuthz +from zorg.buildbot.util.workerowner import WorkerEndpointMatcher +from zorg.buildbot.util.workerowner import RolesFromWorkerOwner + import config @@ -18,13 +22,15 @@ def getAuthz(): - authz = util.Authz( + authz = WorkerOwnerAuthz( allowRules=[ # Admins can do anything. # defaultDeny=False: if user does not have the admin role, # we continue parsing rules. util.AnyEndpointMatcher(role="LLVM Lab team", defaultDeny=False), + WorkerEndpointMatcher(role="worker-owner", defaultDeny=False), + # Allow authors to stop, force or rebuild their own builds, util.StopBuildEndpointMatcher(role="owner", defaultDeny=False), # Allow bot owners to stop, force or rebuild on their own bots, @@ -45,10 +51,10 @@ util.AnyControlEndpointMatcher(role="LLVM Lab team"), ], roleMatchers=[ - util.RolesFromGroups(groupPrefix="llvm/"), util.RolesFromGroups(groupPrefix="llvm/"), # role owner is granted when property owner matches the email of the user util.RolesFromOwner(role="owner"), + RolesFromWorkerOwner(role="worker-owner"), ], ) diff --git a/buildbot/osuosl/master/config/builders.py b/buildbot/osuosl/master/config/builders.py --- a/buildbot/osuosl/master/config/builders.py +++ b/buildbot/osuosl/master/config/builders.py @@ -1,4 +1,6 @@ -# TODO: Rename workers with "slave" as a part of the name. +from importlib import reload + +from buildbot.plugins import util from buildbot.process.properties import WithProperties @@ -8,7 +10,6 @@ from zorg.buildbot.builders import LLDBBuilder from zorg.buildbot.builders import SanitizerBuilder from zorg.buildbot.builders import OpenMPBuilder -from zorg.buildbot.builders import LibcxxAndAbiBuilder from zorg.buildbot.builders import SphinxDocsBuilder from zorg.buildbot.builders import ABITestsuitBuilder from zorg.buildbot.builders import ClangLTOBuilder @@ -18,8 +19,32 @@ from zorg.buildbot.builders import LLDPerformanceTestsuite from zorg.buildbot.builders import FuchsiaBuilder from zorg.buildbot.builders import XToolchainBuilder - -from buildbot.plugins import util +from zorg.buildbot.builders import TestSuiteBuilder +from zorg.buildbot.builders import BOLTBuilder + +from zorg.buildbot.builders import HtmlDocsBuilder +from zorg.buildbot.builders import DoxygenDocsBuilder + +reload(ClangBuilder) +reload(FlangBuilder) +reload(PollyBuilder) +reload(LLDBBuilder) +reload(SanitizerBuilder) +reload(OpenMPBuilder) +reload(SphinxDocsBuilder) +reload(ABITestsuitBuilder) +reload(ClangLTOBuilder) +reload(UnifiedTreeBuilder) +reload(AOSPBuilder) +reload(AnnotatedBuilder) +reload(LLDPerformanceTestsuite) +reload(FuchsiaBuilder) +reload(XToolchainBuilder) +reload(TestSuiteBuilder) +reload(BOLTBuilder) + +reload(HtmlDocsBuilder) +reload(DoxygenDocsBuilder) all = [ @@ -37,39 +62,19 @@ clean=True, depends_on_projects=['llvm','clang','clang-tools-extra','compiler-rt'], extra_configure_args=[ + "-DLLVM_CCACHE_BUILD=ON", "-DCOMPILER_RT_BUILD_BUILTINS:BOOL=OFF", + "-DCOMPILER_RT_BUILD_ORC:BOOL=OFF", "-DCOMPILER_RT_BUILD_SANITIZERS:BOOL=OFF", "-DCOMPILER_RT_BUILD_XRAY:BOOL=OFF", "-DCOMPILER_RT_INCLUDE_TESTS:BOOL=OFF", + "-DCOMPILER_RT_BUILD_LIBFUZZER:BOOL=OFF", "-DCMAKE_C_FLAGS=-Wdocumentation -Wno-documentation-deprecated-sync", "-DCMAKE_CXX_FLAGS=-std=c++11 -Wdocumentation -Wno-documentation-deprecated-sync", ], env={ 'PATH':'/home/llvmbb/bin/clang-latest/bin:/home/llvmbb/bin:/usr/local/bin:/usr/local/bin:/usr/bin:/bin', - 'CC': 'ccache clang', 'CXX': 'ccache clang++', 'CCACHE_CPP2': 'yes', - })}, - - {'name' : "clang-x86_64-debian-new-pass-manager-fast", - 'tags' : ["clang", "fast"], - 'workernames': ["gribozavr4"], - 'builddir': "clang-x86_64-debian-new-pass-manager-fast", - 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( - llvm_srcdir="llvm.src", - obj_dir="llvm.obj", - clean=True, - depends_on_projects=['llvm','clang','clang-tools-extra','compiler-rt'], - extra_configure_args=[ - "-DCOMPILER_RT_BUILD_BUILTINS:BOOL=OFF", - "-DCOMPILER_RT_BUILD_SANITIZERS:BOOL=OFF", - "-DCOMPILER_RT_BUILD_XRAY:BOOL=OFF", - "-DCOMPILER_RT_INCLUDE_TESTS:BOOL=OFF", - "-DENABLE_EXPERIMENTAL_NEW_PASS_MANAGER=ON", - "-DCMAKE_C_FLAGS=-Wdocumentation -Wno-documentation-deprecated-sync", - "-DCMAKE_CXX_FLAGS=-std=c++11 -Wdocumentation -Wno-documentation-deprecated-sync", - ], - env={ - 'PATH':'/home/llvmbb/bin/clang-latest/bin:/home/llvmbb/bin:/usr/local/bin:/usr/local/bin:/usr/bin:/bin', - 'CC': 'ccache clang', 'CXX': 'ccache clang++', 'CCACHE_CPP2': 'yes', + 'CC': 'clang', 'CXX': 'clang++', })}, {'name' : "llvm-clang-x86_64-win-fast", @@ -92,6 +97,28 @@ "-DLLVM_OPTIMIZED_TABLEGEN=OFF", "-DLLVM_LIT_ARGS=-v --threads=32"])}, + {'name': "llvm-clang-x86_64-sie-ubuntu-fast", + 'tags' : ["clang", "llvm", "clang-tools-extra", "lld", "cross-project-tests"], + 'collapseRequests': False, + 'workernames': ["sie-linux-worker"], + 'builddir': "llvm-clang-x86_64-sie-ubuntu-fast", + 'factory': UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + depends_on_projects=['llvm','clang','clang-tools-extra','lld','cross-project-tests'], + extra_configure_args=[ + "-DCMAKE_C_COMPILER=gcc", + "-DCMAKE_CXX_COMPILER=g++", + "-DCMAKE_BUILD_TYPE=Release", + "-DCLANG_ENABLE_ARCMT=OFF", + "-DCLANG_ENABLE_CLANGD=OFF", + "-DLLVM_BUILD_RUNTIME=OFF", + "-DLLVM_CCACHE_BUILD=ON", + "-DLLVM_INCLUDE_EXAMPLES=OFF", + "-DLLVM_DEFAULT_TARGET_TRIPLE=x86_64-scei-ps4", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_LIT_ARGS=--verbose -j100", + "-DLLVM_TARGETS_TO_BUILD=X86", + "-DLLVM_USE_LINKER=gold"])}, + # Expensive checks builders. {'name' : "llvm-clang-x86_64-expensive-checks-ubuntu", @@ -99,27 +126,29 @@ 'workernames' : ["as-builder-4"], 'builddir': "llvm-clang-x86_64-expensive-checks-ubuntu", 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( - depends_on_projects=["llvm"], + depends_on_projects=["llvm", "lld"], clean=True, extra_configure_args=[ "-DLLVM_ENABLE_EXPENSIVE_CHECKS=ON", "-DLLVM_ENABLE_WERROR=OFF", + "-DLLVM_USE_SPLIT_DWARF=ON", + "-DLLVM_USE_LINKER=gold", "-DCMAKE_BUILD_TYPE=Debug", - "-DCMAKE_CXX_FLAGS=-U_GLIBCXX_DEBUG", + "-DCMAKE_CXX_FLAGS=-U_GLIBCXX_DEBUG -Wno-misleading-indentation", "-DLLVM_LIT_ARGS=-vv -j32"])}, {'name' : "llvm-clang-x86_64-expensive-checks-win", - 'tags' : ["llvm", "clang", "expensive-checks"], + 'tags' : ["llvm", "expensive-checks"], 'workernames' : ["as-worker-93"], 'builddir': "llvm-clang-x86_64-expensive-checks-win", 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaWithMSVCBuildFactory( vs="autodetect", + depends_on_projects=["llvm", "lld"], clean=True, extra_configure_args=[ "-DLLVM_ENABLE_EXPENSIVE_CHECKS=ON", "-DLLVM_ENABLE_WERROR=OFF", - "-DCMAKE_BUILD_TYPE=Debug", - "-DLLVM_LIT_ARGS=-vv"])}, + "-DCMAKE_BUILD_TYPE=Debug"])}, {'name' : "llvm-clang-x86_64-expensive-checks-debian", 'tags' : ["llvm", "expensive-checks"], @@ -127,9 +156,10 @@ 'workernames' : ["gribozavr4"], 'builddir': "llvm-clang-x86_64-expensive-checks-debian", 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( - depends_on_projects=["llvm"], + depends_on_projects=["llvm", "lld"], clean=True, extra_configure_args=[ + "-DLLVM_CCACHE_BUILD=ON", "-DLLVM_ENABLE_EXPENSIVE_CHECKS=ON", "-DLLVM_ENABLE_WERROR=OFF", "-DCMAKE_BUILD_TYPE=Release", @@ -137,13 +167,13 @@ "-DLLVM_LIT_ARGS=-v -vv -j96"], env={ 'PATH':'/home/llvmbb/bin/clang-latest/bin:/home/llvmbb/bin:/usr/local/bin:/usr/local/bin:/usr/bin:/bin', - 'CC': 'ccache clang', 'CXX': 'ccache clang++', 'CCACHE_CPP2': 'yes', + 'CC': 'clang', 'CXX': 'clang++', })}, # Cross builders. {'name' : "llvm-clang-win-x-armv7l", - 'tags' : ["clang", "llvm", "compiler-rt", "cross"," armv7l"], + 'tags' : ["clang", "llvm", "compiler-rt", "cross", "armv7"], 'workernames' : ["as-builder-1"], 'builddir': "x-armv7l", 'factory' : XToolchainBuilder.getCmakeWithMSVCBuildFactory( @@ -153,25 +183,28 @@ "check-llvm", "check-clang", "check-lld", - "check-compiler-rt" + "check-compiler-rt-armv7-unknown-linux-gnueabihf" ], checks_on_target = [ ("libunwind", ["python", "bin/llvm-lit.py", "-v", "-vv", "--threads=32", - "runtimes/runtimes-bins/libunwind/test"]), + "runtimes/runtimes-armv7-unknown-linux-gnueabihf-bins/libunwind/test"]), ("libc++abi", ["python", "bin/llvm-lit.py", "-v", "-vv", "--threads=32", - "runtimes/runtimes-bins/libcxxabi/test"]), + "runtimes/runtimes-armv7-unknown-linux-gnueabihf-bins/libcxxabi/test"]), ("libc++", ['python', 'bin/llvm-lit.py', '-v', '-vv', '--threads=32', - 'runtimes/runtimes-bins/libcxx/test', + 'runtimes/runtimes-armv7-unknown-linux-gnueabihf-bins/libcxx/test', ]) ], extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=ARM", + "-DTOOLCHAIN_TARGET_TRIPLE=armv7-unknown-linux-gnueabihf", "-DDEFAULT_SYSROOT=C:/buildbot/.arm-ubuntu", + "-DZLIB_ROOT=C:/buildbot/.zlib-win32", "-DLLVM_LIT_ARGS=-v -vv --threads=32", WithProperties("%(remote_test_host:+-DREMOTE_TEST_HOST=)s%(remote_test_host:-)s"), WithProperties("%(remote_test_user:+-DREMOTE_TEST_USER=)s%(remote_test_user:-)s"), @@ -179,7 +212,7 @@ cmake_cache="../llvm-project/clang/cmake/caches/CrossWinToARMLinux.cmake")}, {'name' : "llvm-clang-win-x-aarch64", - 'tags' : ["clang", "llvm", "compiler-rt", "cross"," aarch64"], + 'tags' : ["clang", "llvm", "compiler-rt", "cross", "aarch64"], 'workernames' : ["as-builder-2"], 'builddir': "x-aarch64", 'factory' : XToolchainBuilder.getCmakeWithMSVCBuildFactory( @@ -189,27 +222,28 @@ "check-llvm", "check-clang", "check-lld", - "check-compiler-rt" + "check-compiler-rt-aarch64-unknown-linux-gnu" ], checks_on_target = [ ("libunwind", ["python", "bin/llvm-lit.py", "-v", "-vv", "--threads=32", - "runtimes/runtimes-bins/libunwind/test"]), + "runtimes/runtimes-aarch64-unknown-linux-gnu-bins/libunwind/test"]), ("libc++abi", ["python", "bin/llvm-lit.py", "-v", "-vv", "--threads=32", - "runtimes/runtimes-bins/libcxxabi/test"]), + "runtimes/runtimes-aarch64-unknown-linux-gnu-bins/libcxxabi/test"]), ("libc++", ['python', 'bin/llvm-lit.py', '-v', '-vv', '--threads=32', - 'runtimes/runtimes-bins/libcxx/test', + 'runtimes/runtimes-aarch64-unknown-linux-gnu-bins/libcxx/test', ]) ], extra_configure_args=[ "-DLLVM_TARGETS_TO_BUILD=AArch64", - "-DCMAKE_C_COMPILER_TARGET=aarch64-linux-gnu", + "-DTOOLCHAIN_TARGET_TRIPLE=aarch64-unknown-linux-gnu", "-DDEFAULT_SYSROOT=C:/buildbot/.aarch64-ubuntu", + "-DZLIB_ROOT=C:/buildbot/.zlib-win32", "-DLLVM_LIT_ARGS=-v -vv --threads=32", WithProperties("%(remote_test_host:+-DREMOTE_TEST_HOST=)s%(remote_test_host:-)s"), WithProperties("%(remote_test_user:+-DREMOTE_TEST_USER=)s%(remote_test_user:-)s"), @@ -219,18 +253,31 @@ # Clang builders. {'name': "clang-arm64-windows-msvc", - 'tags' : ["clang"], - 'workernames' : ["linaro-armv8-windows-msvc-01", "linaro-armv8-windows-msvc-02"], + 'tags' : ["llvm", "clang", "lld", "flang"], + 'workernames' : ["linaro-armv8-windows-msvc-04"], 'builddir': "clang-arm64-windows-msvc", 'factory' : ClangBuilder.getClangCMakeBuildFactory( vs="manual", - test=False, # Disable testing until MCJIT failures are fixed + clean=False, + checkout_flang=True, + checkout_lld=True, + checkout_compiler_rt=False, extra_cmake_args=[ - "-DLLVM_DEFAULT_TARGET_TRIPLE=aarch64-windows-msvc", - "-DLLVM_HOST_TRIPLE=aarch64-windows-msvc", - "-DLLVM_TARGET_ARCH=AArch64", - "-DCOMPILER_RT_BUILD_SANITIZERS=OFF", - "-DCOMPILER_RT_BUILD_XRAY=OFF"])}, + "-DCMAKE_TRY_COMPILE_CONFIGURATION=Release", + "-DLLVM_TARGETS_TO_BUILD='AArch64'", + "-DCMAKE_C_COMPILER_LAUNCHER=sccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=sccache"])}, + + ## ARMv8 check-all + {'name' : "clang-armv8-quick", + 'tags' : ["clang"], + 'workernames':["linaro-clang-armv8-quick"], + 'builddir':"clang-armv8-quick", + 'factory' : ClangBuilder.getClangCMakeBuildFactory( + clean=False, + checkout_compiler_rt=False, + checkout_lld=False, + extra_cmake_args=["-DLLVM_TARGETS_TO_BUILD='ARM'"])}, # Cortex-A15 LNT test-suite in Benchmark mode {'name' : "clang-native-arm-lnt-perf", @@ -241,8 +288,7 @@ clean=False, checkout_compiler_rt=False, checkout_lld=False, - test=False, - useTwoStage=False, + checks=[], runTestSuite=True, testsuite_flags=[ '--cppflags', '-O3 -mcpu=cortex-a15 -mthumb', @@ -257,27 +303,25 @@ testerName='LNT-Thumb2v7-A15-O3')}, # ARMv7 LNT test-suite in test-only mode - {'name' : "clang-cmake-armv7-lnt", + {'name' : "clang-armv7-lnt", 'tags' : ["clang"], - 'workernames' : ["linaro-armv7-lnt"], - 'builddir': "clang-cmake-armv7-lnt", + 'workernames' : ["linaro-clang-armv7-lnt"], + 'builddir': "clang-armv7-lnt", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, checkout_compiler_rt=False, checkout_lld=False, - test=False, - useTwoStage=False, + checks=[], runTestSuite=True, testsuite_flags=[ '--cppflags', '-mcpu=cortex-a15 -marm', - '--threads=32', '--build-threads=32'], - extra_cmake_args=["-DLLVM_TARGETS_TO_BUILD='ARM'"])}, + '--threads=32', '--build-threads=32'])}, - ## ARMv7 check-all self-host NEON with CMake builder - {'name' : "clang-cmake-armv7-selfhost-neon", + ## ARMv7 check-all 2-stage + {'name' : "clang-armv7-2stage", 'tags' : ["clang"], - 'workernames': ["linaro-armv7-selfhost"], - 'builddir':"clang-cmake-armv7-selfhost-neon", + 'workernames': ["linaro-clang-armv7-2stage"], + 'builddir':"clang-armv7-2stage", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, checkout_compiler_rt=False, @@ -286,42 +330,27 @@ testStage1=False, extra_cmake_args=[ "-DCMAKE_C_FLAGS='-mcpu=cortex-a15 -marm'", - "-DCMAKE_CXX_FLAGS='-mcpu=cortex-a15 -marm'", - "-DLLVM_TARGETS_TO_BUILD='ARM;AArch64'"])}, - - ## ARMv7 check-all with CMake builder - {'name' : "clang-cmake-armv7-quick", - 'tags' : ["clang"], - 'workernames':["linaro-armv7-quick"], - 'builddir':"clang-cmake-armv7-quick", - 'factory' : ClangBuilder.getClangCMakeBuildFactory( - clean=False, - checkout_compiler_rt=False, - checkout_lld=False, - extra_cmake_args=["-DLLVM_TARGETS_TO_BUILD='ARM;AArch64'"])}, + "-DCMAKE_CXX_FLAGS='-mcpu=cortex-a15 -marm'"])}, - ## ARMv7 Clang + LLVM run test-suite with GlobalISel enabled - {'name' : "clang-cmake-armv7-global-isel", + ## ARMv7 run test-suite with GlobalISel enabled + {'name' : "clang-armv7-global-isel", 'tags' : ["clang"], - 'workernames':["linaro-armv7-global-isel"], - 'builddir':"clang-cmake-armv7-global-isel", + 'workernames':["linaro-clang-armv7-global-isel"], + 'builddir':"clang-armv7-global-isel", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, checkout_compiler_rt=False, checkout_lld=False, - test=True, - useTwoStage=False, runTestSuite=True, testsuite_flags=[ '--cppflags', '-mcpu=cortex-a15 -marm -O0 -mllvm -global-isel -mllvm -global-isel-abort=0', - '--threads=32', '--build-threads=32'], - extra_cmake_args=["-DLLVM_TARGETS_TO_BUILD='ARM;AArch64'"])}, + '--threads=32', '--build-threads=32'])}, - ## ARMv7 check-all self-host with CMake builder - {'name' : "clang-cmake-armv7-selfhost", + ## ARMv7 VFPv3 check-all 2-stage + {'name' : "clang-armv7-vfpv3-2stage", 'tags' : ["clang"], - 'workernames' : ["linaro-armv7-selfhost"], - 'builddir': "clang-cmake-armv7-selfhost", + 'workernames' : ["linaro-clang-armv7-vfpv3-2stage"], + 'builddir': "clang-armv7-vfpv3-2stage", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, checkout_compiler_rt=False, @@ -330,36 +359,26 @@ testStage1=False, extra_cmake_args=[ "-DCMAKE_C_FLAGS='-mcpu=cortex-a15 -mfpu=vfpv3 -marm'", - "-DCMAKE_CXX_FLAGS='-mcpu=cortex-a15 -mfpu=vfpv3 -marm'", - "-DLLVM_TARGETS_TO_BUILD='ARM;AArch64'"])}, + "-DCMAKE_CXX_FLAGS='-mcpu=cortex-a15 -mfpu=vfpv3 -marm'"])}, - ## AArch64 Clang+LLVM check-all + test-suite - {'name' : "clang-cmake-aarch64-quick", + ## AArch64 check-all + {'name' : "clang-aarch64-quick", 'tags' : ["clang"], - 'workernames' : ["linaro-aarch64-quick"], - 'builddir': "clang-cmake-aarch64-quick", + 'workernames' : ["linaro-clang-aarch64-quick"], + 'builddir': "clang-aarch64-quick", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, checkout_compiler_rt=False, checkout_lld=False, - test=True, - useTwoStage=False, - runTestSuite=True, - testsuite_flags=[ - '--cppflags', '-mcpu=cortex-a57', - '--threads=32', '--build-threads=32'], - extra_cmake_args=["-DLLVM_TARGETS_TO_BUILD='ARM;AArch64'"])}, + extra_cmake_args=["-DLLVM_TARGETS_TO_BUILD='AArch64'"])}, - ## AArch64 Self-hosting Clang+LLVM check-all + LLD + test-suite - {'name' : "clang-cmake-aarch64-lld", + ## AArch64 check-all + LLD + test-suite 2-stage + {'name' : "clang-aarch64-lld-2stage", 'tags' : ["lld"], - 'workernames' : ["linaro-aarch64-lld"], - 'builddir':"clang-cmake-aarch64-lld", + 'workernames' : ["linaro-clang-aarch64-lld-2stage"], + 'builddir':"clang-aarch64-lld-2stage", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, - checkout_compiler_rt=True, - checkout_lld=True, - test=True, useTwoStage=True, runTestSuite=True, testsuite_flags=[ @@ -368,128 +387,186 @@ extra_cmake_args=[ "-DCMAKE_C_FLAGS='-mcpu=cortex-a57'", "-DCMAKE_CXX_FLAGS='-mcpu=cortex-a57'", - "-DLLVM_TARGETS_TO_BUILD='ARM;AArch64'", - "-DLLVM_ENABLE_LLD=True"])}, + "-DLLVM_ENABLE_LLD=True", + "-DLLVM_LIT_ARGS='-v'"])}, - ## AArch64 Clang+LLVM run test-suite at -O0 (GlobalISel is now default). - {'name' : "clang-cmake-aarch64-global-isel", + ## AArch64 run test-suite at -O0 (GlobalISel is now default). + {'name' : "clang-aarch64-global-isel", 'tags' : ["clang"], - 'workernames' : ["linaro-aarch64-global-isel"], - 'builddir': "clang-cmake-aarch64-global-isel", + 'workernames' : ["linaro-clang-aarch64-global-isel"], + 'builddir': "clang-aarch64-global-isel", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, checkout_compiler_rt=False, checkout_lld=False, - test=True, - useTwoStage=False, runTestSuite=True, testsuite_flags=[ '--cppflags', '-O0', - '--threads=32', '--build-threads=32'], - extra_cmake_args=["-DLLVM_TARGETS_TO_BUILD='ARM;AArch64'"])}, + '--threads=32', '--build-threads=32'])}, - ## ARMv7 without neon; check-all 2-stage full compiler-rt + testsuite - {'name' : "clang-cmake-armv7-full", - 'tags' : ["clang"], - 'workernames' : ["linaro-tk1-06", "linaro-tk1-07", "linaro-tk1-08", "linaro-tk1-09"], - 'builddir': "clang-cmake-armv7-full", + ## AArch32 Self-hosting Clang+LLVM check-all + LLD + test-suite + # Sanitizers build disabled due to PR38690 + {'name' : "clang-armv8-lld-2stage", + 'tags' : ["lld"], + 'workernames' : ["linaro-clang-armv8-lld-2stage"], + 'builddir': "clang-armv8-lld-2stage", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, - checkout_lld=False, - checkout_compiler_rt=True, - testStage1=False, useTwoStage=True, runTestSuite=True, testsuite_flags=[ - '--cppflags', '-mcpu=cortex-a15 -mfpu=vfpv3 -marm', - '--threads=4', '--build-threads=4'], + '--cppflags', '-mcpu=cortex-a57 -fuse-ld=lld', + '--threads=32', '--build-threads=32'], extra_cmake_args=[ - "-DCOMPILER_RT_TEST_COMPILER_CFLAGS='-mcpu=cortex-a15 -mfpu=vfpv3 -marm'", - "-DLLVM_TARGETS_TO_BUILD='ARM'", - "-DLLVM_PARALLEL_LINK_JOBS=2"])}, + "-DCMAKE_C_FLAGS='-mcpu=cortex-a57'", + "-DCMAKE_CXX_FLAGS='-mcpu=cortex-a57'", + "-DCOMPILER_RT_BUILD_SANITIZERS=OFF", + "-DLLVM_ENABLE_LLD=True", + # lld tests cause us to hit thread limits + "-DLLVM_ENABLE_THREADS=OFF"])}, - ## ARMv7 Thumb2 with neon; check-all 2-stage full compiler-rt + testsuite - {'name' : "clang-cmake-thumbv7-full-sh", + # AArch64 check-all + flang + compiler-rt + test-suite + SVE/SME + # mlir-integration-tests 2-stage + {'name' : "clang-aarch64-full-2stage", 'tags' : ["clang"], - 'workernames' : ["linaro-tk1-01", "linaro-tk1-03", "linaro-tk1-04", "linaro-tk1-05"], - 'builddir': "clang-cmake-thumbv7-full-sh", + 'workernames' : ["linaro-clang-aarch64-full-2stage"], + 'builddir': "clang-aarch64-full-2stage", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, + checkout_flang=True, checkout_lld=False, - checkout_compiler_rt=True, - testStage1=False, useTwoStage=True, + testStage1=False, runTestSuite=True, + env={ + 'NO_STOP_MESSAGE':'1', # For Fortran test-suite + }, testsuite_flags=[ - '--cppflags', '-mcpu=cortex-a15 -mthumb', - '--threads=4', '--build-threads=4'], + '--cppflags', '-mcpu=cortex-a57', + '--threads=32', '--build-threads=32'], extra_cmake_args=[ - "-DCMAKE_C_FLAGS='-mcpu=cortex-a15 -mthumb'", - "-DCMAKE_CXX_FLAGS='-mcpu=cortex-a15 -mthumb'", - "-DCOMPILER_RT_TEST_COMPILER_CFLAGS='-mcpu=cortex-a15 -mthumb'", - "-DLLVM_TARGETS_TO_BUILD='ARM'", - "-DLLVM_PARALLEL_LINK_JOBS=2"])}, - - ## AArch32 Self-hosting Clang+LLVM check-all + LLD + test-suite - # Sanitizers build disabled due to PR38690 - {'name' : "clang-cmake-armv8-lld", - 'tags' : ["lld"], - 'workernames' : ["linaro-armv8-lld"], - 'builddir': "clang-cmake-armv8-lld", + "-DCMAKE_C_FLAGS='-mcpu=cortex-a57'", + "-DCMAKE_CXX_FLAGS='-mcpu=cortex-a57'", + "-DLLVM_LIT_ARGS='-v'", + "-DMLIR_INCLUDE_INTEGRATION_TESTS=True", + "-DMLIR_RUN_ARM_SVE_TESTS=True", + "-DMLIR_RUN_ARM_SME_TESTS=True", + "-DARM_EMULATOR_EXECUTABLE=qemu-aarch64"])}, + + + # AArch64 Clang+LLVM+RT+LLD check-all + flang + test-suite + + # mlir-integration-tests w/SVE-Vector-Length-Agnostic Note that in this and + # other clang-aarch64-sve-* builders we set -mllvm + # -treat-scalable-fixed-error-as-warning=false to make compiler fail on + # non-critical SVE codegen issues. This helps us notice and fix SVE + # problems sooner rather than later. + {'name' : "clang-aarch64-sve-vla", + 'tags' : ["clang"], + 'workernames' : ["linaro-g3-01", "linaro-g3-02", "linaro-g3-03", "linaro-g3-04"], + 'builddir': "clang-aarch64-sve-vla", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, - checkout_compiler_rt=True, - checkout_lld=True, - test=True, + checkout_flang=True, + runTestSuite=True, + env={ + 'NO_STOP_MESSAGE':'1', # For Fortran test-suite + }, + testsuite_flags=[ + '--cppflags', '-mcpu=neoverse-512tvb -mllvm -scalable-vectorization=preferred -mllvm -treat-scalable-fixed-error-as-warning=false -O3', + '--threads=32', '--build-threads=32'], + extra_cmake_args=[ + "-DCMAKE_C_FLAGS='-mcpu=neoverse-512tvb'", + "-DCMAKE_CXX_FLAGS='-mcpu=neoverse-512tvb'", + "-DLLVM_ENABLE_LLD=True", + "-DMLIR_INCLUDE_INTEGRATION_TESTS=True", + "-DMLIR_RUN_ARM_SVE_TESTS=True", + "-DLLVM_LIT_ARGS='-v'"])}, + + # AArch64 Clang+LLVM+RT+LLD check-all + flang + test-suite 2-stage w/SVE-Vector-Length-Agnostic + {'name' : "clang-aarch64-sve-vla-2stage", + 'tags' : ["clang"], + 'workernames' : ["linaro-g3-01", "linaro-g3-02", "linaro-g3-03", "linaro-g3-04"], + 'builddir': "clang-aarch64-sve-vla-2stage", + 'factory' : ClangBuilder.getClangCMakeBuildFactory( + clean=False, + checkout_flang=True, useTwoStage=True, + testStage1=False, runTestSuite=True, + env={ + 'NO_STOP_MESSAGE':'1', # For Fortran test-suite + }, testsuite_flags=[ - '--cppflags', '-mcpu=cortex-a57 -fuse-ld=lld', + '--cppflags', '-mcpu=neoverse-512tvb -mllvm -scalable-vectorization=preferred -mllvm -treat-scalable-fixed-error-as-warning=false -O3', '--threads=32', '--build-threads=32'], extra_cmake_args=[ - "-DCMAKE_C_FLAGS='-mcpu=cortex-a57'", - "-DCMAKE_CXX_FLAGS='-mcpu=cortex-a57'", - "-DLLVM_TARGETS_TO_BUILD='ARM;AArch64'", - "-DCOMPILER_RT_BUILD_SANITIZERS=OFF", - "-DLLVM_ENABLE_LLD=True"])}, + "-DCMAKE_C_FLAGS='-mcpu=neoverse-512tvb -mllvm -scalable-vectorization=preferred -mllvm -treat-scalable-fixed-error-as-warning=false'", + "-DCMAKE_CXX_FLAGS='-mcpu=neoverse-512tvb -mllvm -scalable-vectorization=preferred -mllvm -treat-scalable-fixed-error-as-warning=false'", + "-DLLVM_ENABLE_LLD=True", + "-DLLVM_LIT_ARGS='-v'"])}, - # AArch64 Clang+LLVM+RT check-all + flang + test-suite + self-hosting - {'name' : "clang-cmake-aarch64-full", + # AArch64 Clang+LLVM+RT+LLD check-all + flang + test-suite w/SVE-Vector-Length-Specific + {'name' : "clang-aarch64-sve-vls", 'tags' : ["clang"], - 'workernames' : ["linaro-aarch64-full"], - 'builddir': "clang-cmake-aarch64-full", + 'workernames' : ["linaro-g3-01", "linaro-g3-02", "linaro-g3-03", "linaro-g3-04"], + 'builddir': "clang-aarch64-sve-vls", + 'factory' : ClangBuilder.getClangCMakeBuildFactory( + clean=False, + checkout_flang=True, + runTestSuite=True, + env={ + 'NO_STOP_MESSAGE':'1', # For Fortran test-suite + }, + testsuite_flags=[ + '--cppflags', '-mcpu=neoverse-512tvb -msve-vector-bits=256 -mllvm -treat-scalable-fixed-error-as-warning=false -O3', + '--threads=32', '--build-threads=32'], + extra_cmake_args=[ + "-DCMAKE_C_FLAGS='-mcpu=neoverse-512tvb'", + "-DCMAKE_CXX_FLAGS='-mcpu=neoverse-512tvb'", + "-DLLVM_ENABLE_LLD=True", + "-DMLIR_INCLUDE_INTEGRATION_TESTS=True", + "-DMLIR_RUN_ARM_SVE_TESTS=True", + "-DLLVM_LIT_ARGS='-v'"])}, + + # AArch64 Clang+LLVM+RT+LLD check-all + flang + test-suite 2-stage w/SVE-Vector-Length-Specific + {'name' : "clang-aarch64-sve-vls-2stage", + 'tags' : ["clang"], + 'workernames' : ["linaro-g3-01", "linaro-g3-02", "linaro-g3-03", "linaro-g3-04"], + 'builddir': "clang-aarch64-sve-vls-2stage", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, - checkout_compiler_rt=True, checkout_flang=True, - checkout_lld=False, - test=True, useTwoStage=True, testStage1=False, runTestSuite=True, + env={ + 'NO_STOP_MESSAGE':'1', # For Fortran test-suite + }, testsuite_flags=[ - '--cppflags', '-mcpu=cortex-a57', + '--cppflags', '-mcpu=neoverse-512tvb -msve-vector-bits=256 -mllvm -treat-scalable-fixed-error-as-warning=false -O3', '--threads=32', '--build-threads=32'], extra_cmake_args=[ - "-DCMAKE_C_FLAGS='-mcpu=cortex-a57'", - "-DCMAKE_CXX_FLAGS='-mcpu=cortex-a57'", - "-DLLVM_TARGETS_TO_BUILD='ARM;AArch64'"])}, + "-DCMAKE_C_FLAGS='-mcpu=neoverse-512tvb -msve-vector-bits=256 -mllvm -treat-scalable-fixed-error-as-warning=false'", + "-DCMAKE_CXX_FLAGS='-mcpu=neoverse-512tvb -msve-vector-bits=256 -mllvm -treat-scalable-fixed-error-as-warning=false'", + "-DLLVM_ENABLE_LLD=True", + "-DLLVM_LIT_ARGS='-v'"])}, {'name' : "clang-arm64-windows-msvc-2stage", 'tags' : ["clang"], - 'workernames' : ["linaro-armv8-windows-msvc-01", "linaro-armv8-windows-msvc-02"], + 'workernames' : ["linaro-armv8-windows-msvc-01", "linaro-armv8-windows-msvc-02", "linaro-armv8-windows-msvc-03"], 'builddir': "clang-arm64-windows-msvc-2stage", 'factory' : ClangBuilder.getClangCMakeBuildFactory( vs="manual", - test=False, # Disable testing until MCJIT failures are fixed useTwoStage=True, - testStage1=False, + checkout_flang=True, extra_cmake_args=[ - "-DLLVM_DEFAULT_TARGET_TRIPLE=aarch64-windows-msvc", - "-DLLVM_HOST_TRIPLE=aarch64-windows-msvc", - "-DLLVM_TARGET_ARCH=AArch64", + "-DCMAKE_TRY_COMPILE_CONFIGURATION=Release", + "-DCMAKE_C_COMPILER_LAUNCHER=sccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=sccache", + # FIXME: compiler-rt\lib\sanitizer_common\sanitizer_unwind_win.cpp assumes WIN64 is x86_64, + # so, before that's fixed, disable everything that triggers its build. "-DCOMPILER_RT_BUILD_SANITIZERS=OFF", - "-DCOMPILER_RT_BUILD_XRAY=OFF"])}, + "-DCOMPILER_RT_BUILD_PROFILE=OFF"])}, {'name' : 'clang-x64-windows-msvc', 'tags' : ["clang"], @@ -499,101 +576,147 @@ script="clang-windows.py", depends_on_projects=['llvm', 'clang', 'lld', 'debuginfo-tests'])}, - {'name' : "clang-ppc64be-linux-lnt", - 'tags' : ["clang", "ppc"], - 'workernames' : ["ppc64be-clang-lnt-test"], - 'builddir': "clang-ppc64be-lnt", + {'name' : "clang-m68k-linux", + 'tags' : ["clang"], + 'workernames' : ["debian-akiko-m68k"], + 'builddir': "clang-m68k-linux", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, checkout_lld=False, useTwoStage=False, - runTestSuite=True, + enable_runtimes=None, stage1_config='Release', - nt_flags=['--threads=16', '--build-threads=16'], - extra_cmake_args=["-DLLVM_ENABLE_ASSERTIONS=ON"])}, + extra_cmake_args=[ + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=M68k"])}, - {'name' : "clang-ppc64be-linux-multistage", - 'tags' : ["clang", "ppc"], - 'workernames' : ["ppc64be-clang-multistage-test"], - 'builddir': "clang-ppc64be-multistage", + {'name' : "clang-m68k-linux-cross", + 'tags' : ["clang"], + 'workernames' : ["suse-gary-m68k-cross"], + 'builddir': "clang-m68k-linux-cross", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, checkout_lld=False, - useTwoStage=True, + checkout_compiler_rt=False, + useTwoStage=False, stage1_config='Release', - stage2_config='Release', - extra_cmake_args=['-DLLVM_ENABLE_ASSERTIONS=ON'])}, + extra_cmake_args=[ + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_TARGETS_TO_BUILD=X86", + "-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=M68k"])}, - {'name' : "clang-ppc64le-linux-lnt", - 'tags' : ["clang", "ppc", "ppc64le"], - 'workernames' : ["ppc64le-clang-lnt-test"], - 'builddir': "clang-ppc64le-lnt", + {'name' : "clang-mips64el-linux", + 'tags' : ["clang"], + 'workernames' : ["debian-tritium-mips64el"], + 'builddir': "clang-mips64el-linux", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, checkout_lld=False, + enable_runtimes=None, useTwoStage=False, - runTestSuite=True, stage1_config='Release', - nt_flags=['--threads=16', '--build-threads=16'], - extra_cmake_args=["-DLLVM_ENABLE_ASSERTIONS=ON"])}, + extra_cmake_args=['-DLLVM_ENABLE_ASSERTIONS=ON', + '-DLLVM_PARALLEL_LINK_JOBS=4', + '-DCOMPILER_RT_DEFAULT_TARGET_ONLY=ON', + '-DCMAKE_C_COMPILER_TARGET="mips64el-unknown-linux-gnu"', + '-DLLVM_TARGETS_TO_BUILD=Mips'])}, - {'name' : "clang-ppc64le-linux-multistage", - 'tags' : ["clang", "ppc", "ppc64le"], - 'workernames' : ["ppc64le-clang-multistage-test"], - 'builddir': "clang-ppc64le-multistage", + {'name' : "clang-ppc64be-linux-test-suite", + 'tags' : ["clang", "ppc"], + 'workernames' : ["ppc64be-clang-test-suite"], + 'builddir': "clang-ppc64be-test-suite", + 'factory' : TestSuiteBuilder.getTestSuiteBuildFactory( + depends_on_projects=["llvm", "clang", "clang-tools-extra", + "compiler-rt"], + checks=['check-all', 'check-runtimes'], + extra_configure_args=[ + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DCMAKE_BUILD_TYPE=Release", + "-DLLVM_LIT_ARGS=-v", + "-DLLVM_CCACHE_BUILD=ON"])}, + + {'name' : "clang-ppc64be-linux-multistage", + 'tags' : ["clang", "ppc"], + 'workernames' : ["ppc64be-clang-multistage-test"], + 'builddir': "clang-ppc64be-multistage", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, + checks=['check-all', 'check-runtimes'], checkout_lld=False, useTwoStage=True, stage1_config='Release', stage2_config='Release', - extra_cmake_args=['-DLLVM_ENABLE_ASSERTIONS=ON', '-DBUILD_SHARED_LIBS=ON'])}, + extra_cmake_args=[ + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_CCACHE_BUILD=ON"])}, - {'name' : "clang-ppc64be-linux", - 'tags' : ["clang", "ppc"], - 'workernames' : ["ppc64be-clang-test"], - 'builddir': "clang-ppc64be", - 'factory' : ClangBuilder.getClangCMakeBuildFactory( - clean=False, - checkout_lld=False, - useTwoStage=False, - stage1_config='Release', - extra_cmake_args=["-DLLVM_ENABLE_ASSERTIONS=ON"])}, + {'name' : "clang-ppc64le-linux-test-suite", + 'tags' : ["clang", "ppc", "ppc64le"], + 'workernames' : ["ppc64le-clang-test-suite"], + 'builddir': "clang-ppc64le-test-suite", + 'factory' : TestSuiteBuilder.getTestSuiteBuildFactory( + depends_on_projects=["llvm", "clang", "clang-tools-extra", + "compiler-rt"], + checks=['check-all', 'check-runtimes'], + extra_configure_args=[ + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DCMAKE_BUILD_TYPE=Release", + "-DLLVM_LIT_ARGS=-v", + "-DLLVM_CCACHE_BUILD=ON"])}, - {'name' : "clang-ppc64le-linux", + {'name' : "clang-ppc64le-linux-multistage", 'tags' : ["clang", "ppc", "ppc64le"], - 'workernames' : ["ppc64le-clang-test"], - 'builddir': "clang-ppc64le", + 'workernames' : ["ppc64le-clang-multistage-test"], + 'builddir': "clang-ppc64le-multistage", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, + checks=['check-all', 'check-runtimes'], checkout_lld=False, - useTwoStage=False, + useTwoStage=True, stage1_config='Release', - extra_cmake_args=["-DLLVM_ENABLE_ASSERTIONS=ON"])}, + stage2_config='Release', + extra_cmake_args=[ + '-DLLVM_ENABLE_ASSERTIONS=ON', + '-DBUILD_SHARED_LIBS=ON', + '-DLLVM_CCACHE_BUILD=ON'])}, {'name' : "clang-ppc64le-rhel", 'tags' : ["clang", "ppc", "ppc64le"], 'workernames' : ["ppc64le-clang-rhel-test"], 'builddir': "clang-ppc64le-rhel", - 'factory' : ClangBuilder.getClangCMakeBuildFactory(clean=False, - checkout_clang_tools_extra=True, - checkout_compiler_rt=False, - checkout_lld=True, - checkout_libcxx=False, - useTwoStage=False, - runTestSuite=True, - stage1_config='Release', - nt_flags=['--threads=16', '--build-threads=16'], - extra_cmake_args=[ - '-DLLVM_ENABLE_PROJECTS=clang;llvm;clang-tools-extra;lld', - '-DLLVM_ENABLE_RUNTIMES=compiler-rt', - "-DLLVM_ENABLE_ASSERTIONS=On", "-DCMAKE_C_COMPILER=clang", + 'factory' : TestSuiteBuilder.getTestSuiteBuildFactory( + depends_on_projects=["llvm", "clang", "clang-tools-extra", + "lld", "compiler-rt"], + checks=['check-all', 'check-runtimes'], + extra_configure_args=[ + "-DLLVM_ENABLE_ASSERTIONS=On", + "-DCMAKE_C_COMPILER=clang", "-DCMAKE_CXX_COMPILER=clang++", "-DCLANG_DEFAULT_LINKER=lld", - "-DCMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN:PATH=/opt/rh/devtoolset-7/root/usr", - "-DCMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN:PATH=/opt/rh/devtoolset-7/root/usr", - "-DLLVM_BINUTILS_INCDIR=/usr/include", "-DBUILD_SHARED_LIBS=ON", "-DLLVM_ENABLE_WERROR=ON", - '-DLLVM_LIT_ARGS=-vj 20'])}, + "-DLLVM_TOOL_GOLD_BUILD=0", + "-DCMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN:PATH=/usr", + "-DCMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN:PATH=/usr", + "-DLLVM_BINUTILS_INCDIR=/usr/include", + "-DBUILD_SHARED_LIBS=ON", "-DLLVM_ENABLE_WERROR=ON", + "-DCMAKE_BUILD_TYPE=Release", + "-DLLVM_LIT_ARGS=-vj 20"])}, + + {'name' : "clang-ppc64-aix", + 'tags' : ["clang", "aix", "ppc"], + 'workernames' : ["aix-ppc64"], + 'builddir': "clang-ppc64-aix", + 'factory' : TestSuiteBuilder.getTestSuiteBuildFactory( + depends_on_projects=["llvm", "clang", "compiler-rt"], + clean=False, + extra_configure_args=[ + "-DLLVM_ENABLE_ASSERTIONS=On", + "-DCMAKE_C_COMPILER=/usr/local/clang-15.0.0/bin/clang", + "-DCMAKE_CXX_COMPILER=/usr/local/clang-15.0.0/bin/clang++", + "-DPython3_EXECUTABLE:FILEPATH=/opt/freeware/bin/python3_64", + "-DLLVM_ENABLE_ZLIB=OFF", "-DLLVM_APPEND_VC_REV=OFF", + "-DLLVM_PARALLEL_LINK_JOBS=2", + "-DLLVM_ENABLE_WERROR=ON"]), + 'env' : {'OBJECT_MODE': '64'}}, {'name' : "clang-s390x-linux", 'tags' : ["clang"], @@ -606,6 +729,7 @@ useTwoStage=False, stage1_config='Release', extra_cmake_args=[ + "-DLLVM_CCACHE_BUILD=ON", "-DLLVM_ENABLE_ASSERTIONS=ON", "-DLLVM_LIT_ARGS=-v -j4 --param run_long_tests=true"])}, @@ -620,7 +744,9 @@ useTwoStage=True, stage1_config='Release', stage2_config='Release', - extra_cmake_args=['-DLLVM_ENABLE_ASSERTIONS=ON'])}, + extra_cmake_args=[ + "-DLLVM_CCACHE_BUILD=ON", + "-DLLVM_ENABLE_ASSERTIONS=ON"])}, {'name' : "clang-s390x-linux-lnt", 'tags' : ["clang"], @@ -634,7 +760,9 @@ runTestSuite=True, stage1_config='Release', testsuite_flags=['--threads=4', '--build-threads=4'], - extra_cmake_args=["-DLLVM_ENABLE_ASSERTIONS=ON"])}, + extra_cmake_args=[ + "-DLLVM_CCACHE_BUILD=ON", + "-DLLVM_ENABLE_ASSERTIONS=ON"])}, {'name' : "clang-sparc64-linux-multistage", 'tags' : ["clang"], @@ -643,6 +771,7 @@ 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, checkout_lld=False, + enable_runtimes=None, useTwoStage=True, stage1_config='Release', stage2_config='Release', @@ -650,6 +779,22 @@ '-DLLVM_PARALLEL_LINK_JOBS=4', '-DLLVM_TARGETS_TO_BUILD=Sparc'])}, + ## LoongArch64 Clang+LLVM build check-all + test-suite + {'name' : 'clang-loongarch64-linux', + 'tags' : ['clang'], + 'workernames' : ['loongson-loongarch64-clfs-clang-01'], + 'builddir': 'clang-loongarch64-linux', + 'factory' : ClangBuilder.getClangCMakeBuildFactory( + clean=False, + runTestSuite=True, + checkout_clang_tools_extra=False, + checkout_compiler_rt=False, + checkout_lld=False, + testsuite_flags=['--threads=32', '--build-threads=32'], + extra_cmake_args=['-DLLVM_TARGETS_TO_BUILD=', + '-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=LoongArch', + '-DLLVM_ENABLE_PROJECTS=clang'])}, + {'name' : "clang-hexagon-elf", 'tags' : ["clang"], 'workernames' : ["hexagon-build-02", "hexagon-build-03"], @@ -659,7 +804,7 @@ checkout_clang_tools_extra=False, checkout_compiler_rt=False, checkout_lld=False, - env={'LD_LIBRARY_PATH': '/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/lib', 'PATH': ['/local/cmake-3.17.0/bin', '${PATH}']}, + env={'LD_LIBRARY_PATH': '/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/lib'}, extra_cmake_args=[ "-DCMAKE_BUILD_TYPE:STRING=Release", "-DLLVM_TARGETS_TO_BUILD:STRING=Hexagon", @@ -676,156 +821,152 @@ "-DWITH_POLLY:BOOL=OFF", "-DLINK_POLLY_INTO_TOOLS:BOOL=OFF", "-DPOLLY_BUILD_SHARED_LIB:BOOL=OFF", - "-DCMAKE_C_COMPILER:FILEPATH=/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/bin/clang", - "-DCMAKE_CXX_COMPILER:FILEPATH=/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/bin/clang++"])}, + "-DCMAKE_C_COMPILER:FILEPATH=/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/bin/clang", + "-DCMAKE_CXX_COMPILER:FILEPATH=/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/bin/clang++"])}, - ## X86_64 AVX2 Clang+LLVM check-all + test-suite - {'name' : "clang-cmake-x86_64-avx2-linux", + ## X86_64 AVX512 Clang+LLVM check-all + test-suite + {'name' : "clang-cmake-x86_64-avx512-linux", 'tags' : ["clang"], - 'workernames' : ["avx2-intel64"], - 'builddir': "clang-cmake-x86_64-avx2-linux", + 'workernames' : ["avx512-intel64"], + 'builddir': "clang-cmake-x86_64-avx512-linux", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, checkout_clang_tools_extra=False, checkout_compiler_rt=False, checkout_lld=False, - test=True, useTwoStage=False, runTestSuite=True, - nt_flags=['--cflag', '-march=broadwell', '--threads=80', '--build-threads=80'], + testsuite_flags=['--cflag', '-march=cascadelake', '--threads=32', '--build-threads=32'], env={'PATH':'/usr/bin/ccache:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'}, extra_cmake_args=[ "-DLLVM_ENABLE_ASSERTIONS=ON", - "-DCMAKE_C_FLAGS='-march=broadwell'", - "-DCMAKE_CXX_FLAGS='-march=broadwell'", + "-DCMAKE_C_FLAGS='-march=cascadelake'", + "-DCMAKE_CXX_FLAGS='-march=cascadelake'", + "-DLLVM_ENABLE_RUNTIMES=compiler-rt", + "-DCOMPILER_RT_BUILD_SANITIZERS=OFF", "-DLLVM_TARGETS_TO_BUILD='X86'"])}, - ## X86_64 AVX2 LNT test-suite in Benchmark mode - {'name' : "clang-cmake-x86_64-avx2-linux-perf", + ## Windows X86_64 AVX512 Clang+LLVM check-all + test-suite + {'name' : "clang-cmake-x86_64-avx512-win", 'tags' : ["clang"], - 'workernames' : ["avx2-intel64"], - 'builddir': "clang-cmake-x86_64-avx2-linux-perf", + 'workernames' : ["avx512-intel64-win"], + 'builddir': "clang-cmake-x86_64-avx512-win", 'factory' : ClangBuilder.getClangCMakeBuildFactory( + vs="autodetect", + vs_target_arch='x64', + stage1_config='Debug', clean=False, - checkout_clang_tools_extra=False, + checkout_clang_tools_extra=True, checkout_compiler_rt=False, checkout_lld=False, - test=False, useTwoStage=False, runTestSuite=True, - nt_flags=['--cflag', '-march=broadwell', '--threads=1', '--build-threads=80', '--use-perf', - '--benchmarking-only', '--multisample=4', '--exclude-stat-from-submission=compile'], - env={'PATH':'/usr/bin/ccache:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'}, + testsuite_flags=['--cflag', '-march=cascadelake', '--threads=32', '--build-threads=32'], extra_cmake_args=[ - "-DCMAKE_C_FLAGS='-march=broadwell'", - "-DCMAKE_CXX_FLAGS='-march=broadwell'", - "-DLLVM_TARGETS_TO_BUILD='X86'"], - submitURL='http://lnt.llvm.org/submitRun', - testerName='LNT-Broadwell-AVX2-O3')}, + "-DCMAKE_C_FLAGS='-march=cascadelake'", + "-DCMAKE_CXX_FLAGS='-march=cascadelake'", + "-DLLVM_ENABLE_RUNTIMES=compiler-rt", + "-DCOMPILER_RT_BUILD_SANITIZERS=OFF", + "-DCOMPILER_RT_BUILD_ORC=OFF", + "-DLLVM_TARGETS_TO_BUILD=X86"])}, - ## X86_64 Clang+LLVM Run test-suite targeting AVX512 on SDE (Emulator) - {'name' : "clang-cmake-x86_64-sde-avx512-linux", + {'name' : "clang-xcore-ubuntu-20-x64", 'tags' : ["clang"], - 'workernames' : ["sde-avx512-intel64"], - 'builddir': "clang-cmake-x86_64-sde-avx512-linux", + 'workernames' : ["xcore-ubuntu20-x64"], + 'builddir': "clang-xcore-ubuntu-20-x64", 'factory' : ClangBuilder.getClangCMakeBuildFactory( - clean=False, + jobs=4, checkout_clang_tools_extra=False, checkout_compiler_rt=False, checkout_lld=False, - test=True, + testStage1=True, useTwoStage=False, - runTestSuite=True, - nt_flags=['--cflag', '-march=skylake-avx512', '--threads=80', - '--build-threads=80', '--make-param', "RUNUNDER=sde64 -skx --", '--make-param', 'USER_MODE_EMULATION=1', - '--make-param', 'RUNTIMELIMIT=1200'], - env={'PATH':'/home/ssglocal/tools/sde/latest:/usr/bin/ccache:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'}, - extra_cmake_args=[ - "-DLLVM_ENABLE_ASSERTIONS=ON", - "-DCMAKE_C_FLAGS='-march=broadwell'", - "-DCMAKE_CXX_FLAGS='-march=broadwell'", - "-DLLVM_TARGETS_TO_BUILD='X86'"])}, - - ## Armv7 build cache - {'name' : "clang-armv7-linux-build-cache", - 'tags' : ["clang"], - 'workernames' : ["packet-linux-armv7-slave-1"], - 'builddir': "clang-armv7-linux-build-cache", - 'factory' : ClangBuilder.getClangCMakeGCSBuildFactory( stage1_config='Release', - clean=True, - checkout_compiler_rt=False, - test=False, - useTwoStage=False, - runTestSuite=False, - checkout_lld=True, - checkout_libcxx=True, - checkout_clang_tools_extra=False, - use_pixz_compression=False, - xz_compression_factor=0, - stage1_upload_directory='clang-armv7-linux', extra_cmake_args=[ - "-DLLVM_TARGETS_TO_BUILD='ARM;AArch64;X86'", - "-DCMAKE_C_FLAGS='-mthumb'", - "-DCMAKE_CXX_FLAGS='-mthumb'", - ], - env={'BUCKET': 'llvm-build-artifacts'})}, - - ## AArch64 build cache - {'name' : "clang-aarch64-linux-build-cache", - 'tags' : ["clang"], - 'workernames' : ["packet-linux-aarch64-slave-1"], - 'builddir': "clang-aarch64-linux-build-cache", - 'factory' : ClangBuilder.getClangCMakeGCSBuildFactory( - stage1_config='Release', + "-DLLVM_TARGETS_TO_BUILD:STRING=XCore", + "-DLLVM_DEFAULT_TARGET_TRIPLE:STRING=xcore-unknown-unknown-elf", + "-DLLVM_ENABLE_THREADS:BOOL=OFF"])}, + + {'name' : "llvm-clang-x86_64-sie-win", + 'tags' : ["llvm", "clang", "clang-tools-extra", "lld", "cross-project-tests"], + 'workernames' : ["sie-win-worker"], + 'builddir': "llvm-clang-x86_64-sie-win", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaWithMSVCBuildFactory( + vs="autodetect", + target_arch='x64', + depends_on_projects=['llvm','clang','clang-tools-extra','lld','cross-project-tests'], clean=True, - checkout_compiler_rt=False, - test=False, - useTwoStage=False, - runTestSuite=False, - checkout_lld=True, - checkout_libcxx=True, - checkout_clang_tools_extra=False, - stage1_upload_directory='clang-aarch64-linux', - use_pixz_compression=True, - extra_cmake_args=[ - "-DLLVM_TARGETS_TO_BUILD='ARM;AArch64;X86'", - #"-DCMAKE_C_FLAGS=''", - #"-DCMAKE_CXX_FLAGS=''", - ], - env={'BUCKET': 'llvm-build-artifacts'})}, - - {'name' : "llvm-avr-linux", - 'tags' : ["clang"], - 'workernames' : ["avr-build-01"], - 'builddir': "llvm-avr-linux", - 'factory' : ClangBuilder.getClangCMakeBuildFactory( - jobs=8, - clean=False, - checkout_lld=False, - extra_cmake_args=[ - '-DLLVM_ENABLE_ASSERTIONS=ON', - # We need to compile the X86 backend due to a few generic CodeGen tests. - '-DLLVM_TARGETS_TO_BUILD=X86', - '-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=AVR', - '-DBUILD_SHARED_LIBS=ON'])}, + extra_configure_args=[ + "-DCMAKE_BUILD_TYPE=Release", + "-DCLANG_ENABLE_ARCMT=OFF", + "-DCLANG_ENABLE_CLANGD=OFF", + "-DLLVM_CCACHE_BUILD=ON", + "-DLLVM_DEFAULT_TARGET_TRIPLE=x86_64-sie-ps5", + "-DLLVM_INCLUDE_EXAMPLES=OFF", + "-DLLVM_TARGETS_TO_BUILD=X86", + "-DLLVM_VERSION_SUFFIX=", + "-DLLVM_BUILD_RUNTIME=OFF", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_LIT_ARGS=--verbose"])}, - {'name' : "clang-x64-ninja-win7", - 'tags' : ["clang"], - 'workernames' : ["windows7-buildbot"], - 'builddir': "clang-x64-ninja-win7", - 'factory' : ClangBuilder.getClangCMakeBuildFactory( - clean=False, - checkout_lld=False, - vs="autodetect", - vs_target_arch='x64', - testStage1=True, - useTwoStage=True, - stage1_config='Release', - stage2_config='Release', - extra_cmake_args=[ - '-DLLVM_ENABLE_ASSERTIONS=ON', - '-DLLVM_TARGETS_TO_BUILD=X86'])}, + {'name': "cross-project-tests-sie-ubuntu", + 'tags' : ["clang", "llvm", "lldb", "cross-project-tests"], + 'workernames': ["doug-worker-1a"], + 'builddir': "cross-project-tests-sie-ubuntu", + 'factory': UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + depends_on_projects=['llvm','clang','lldb','cross-project-tests'], + checks = ['check-cross-project'], + extra_configure_args=[ + "-DCMAKE_C_COMPILER=gcc", + "-DCMAKE_CXX_COMPILER=g++", + "-DCMAKE_BUILD_TYPE=Release", + "-DCLANG_ENABLE_ARCMT=OFF", + "-DLLDB_ENABLE_PYTHON=TRUE", + "-DLLVM_INCLUDE_EXAMPLES=OFF", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_LIT_ARGS=--verbose", + "-DLLVM_PARALLEL_LINK_JOBS=8", + "-DLLVM_TARGETS_TO_BUILD=X86", + "-DLLVM_USE_LINKER=gold"])}, + + {'name': "cross-project-tests-sie-ubuntu-dwarf5", + 'tags' : ["clang", "llvm", "lldb", "cross-project-tests"], + 'workernames': ["doug-worker-1b"], + 'builddir': "cross-project-tests-sie-ubuntu-dwarf5", + 'factory': UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + depends_on_projects=['llvm','clang','lldb','cross-project-tests'], + checks = ['check-cross-project'], + extra_configure_args=[ + "-DCMAKE_C_COMPILER=gcc", + "-DCMAKE_CXX_COMPILER=g++", + "-DCMAKE_BUILD_TYPE=Release", + "-DCLANG_ENABLE_ARCMT=OFF", + "-DLLDB_ENABLE_PYTHON=TRUE", + "-DLLVM_INCLUDE_EXAMPLES=OFF", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_LIT_ARGS=--verbose", + "-DLLVM_PARALLEL_LINK_JOBS=8", + "-DLLVM_TARGETS_TO_BUILD=X86", + "-DLLVM_USE_LINKER=gold"])}, + + {'name': "llvm-clang-x86_64-gcc-ubuntu", + 'tags' : ["llvm", "clang", "clang-tools-extra", "compiler-rt", "lld", "cross-project-tests"], + 'workernames': ["doug-worker-2a"], + 'builddir': "llvm-clang-x86_64-gcc-ubuntu", + 'factory': UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + depends_on_projects=['llvm','clang','clang-tools-extra','compiler-rt','lld','cross-project-tests'], + extra_configure_args=[ + "-DCMAKE_C_COMPILER=gcc", + "-DCMAKE_CXX_COMPILER=g++", + "-DCMAKE_BUILD_TYPE=Release", + "-DCLANG_ENABLE_CLANGD=OFF", + "-DLLVM_BUILD_RUNTIME=ON", + "-DLLVM_BUILD_TESTS=ON", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_INCLUDE_EXAMPLES=OFF", + "-DLLVM_LIT_ARGS=--verbose -j48", + "-DLLVM_PARALLEL_LINK_JOBS=16", + "-DLLVM_USE_LINKER=gold"])}, # Polly builders. @@ -838,7 +979,7 @@ install=True, make='ninja', jobs=16, - env={'LD_LIBRARY_PATH': '/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/lib', 'PATH': ['/local/cmake-3.17.0/bin', '${PATH}']}, + env={'LD_LIBRARY_PATH': '/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/lib'}, extraCmakeArgs=[ "-G", "Ninja", "-DLLVM_TARGETS_TO_BUILD='ARM;AArch64'", @@ -846,12 +987,13 @@ "-DLLVM_TARGET_ARCH=arm-linux-gnueabi", "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_ENABLE_LIBCXX:BOOL=ON", - "-DCMAKE_C_COMPILER:FILEPATH=/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/bin/clang", - "-DCMAKE_CXX_COMPILER:FILEPATH=/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/bin/clang++"])}, + "-DPOLLY_ENABLE_GPGPU_CODEGEN=OFF", + "-DCMAKE_C_COMPILER:FILEPATH=/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/bin/clang", + "-DCMAKE_CXX_COMPILER:FILEPATH=/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/bin/clang++"])}, {'name' : "polly-x86_64-linux", 'tags' : ["polly"], - 'workernames' : ["polly-x86_64-fdcserver", "polly-x86_64-gce1"], + 'workernames' : ["polly-x86_64-gce1"], 'builddir': "polly-x86_64-linux", 'factory' : PollyBuilder.getPollyBuildFactory( clean=False, @@ -859,15 +1001,155 @@ make='ninja', extraCmakeArgs=[ "-G", "Ninja", + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", + "-DLLVM_ENABLE_ASSERTIONS=True", + "-DLLVM_TARGETS_TO_BUILD='X86;NVPTX'", + "-DCLANG_ENABLE_ARCMT=OFF", + "-DCLANG_ENABLE_STATIC_ANALYZER=OFF", + "-DCLANG_ENABLE_OBJC_REWRITER=OFF", + "-DLLVM_ENABLE_LLD=ON", + "-DPOLLY_ENABLE_GPGPU_CODEGEN=ON" + ])}, + + {'name' : "polly-x86_64-linux-plugin", + 'tags' : ["polly"], + 'workernames' : ["polly-x86_64-gce1"], + 'builddir': "polly-x86_64-linux-plugin", + 'factory' : PollyBuilder.getPollyBuildFactory( + clean=False, + install=False, + make='ninja', + extraCmakeArgs=[ + "-G", "Ninja", + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", + "-DLLVM_ENABLE_ASSERTIONS=True", + "-DLLVM_TARGETS_TO_BUILD='X86;NVPTX'", + "-DCLANG_ENABLE_ARCMT=OFF", + "-DCLANG_ENABLE_STATIC_ANALYZER=OFF", + "-DCLANG_ENABLE_OBJC_REWRITER=OFF", + "-DLLVM_ENABLE_LLD=ON", + "-DLLVM_POLLY_LINK_INTO_TOOLS=OFF", + "-DPOLLY_ENABLE_GPGPU_CODEGEN=OFF" # Not all required symbols available in opt executable + ])}, + + {'name' : "polly-x86_64-linux-noassert", + 'tags' : ["polly"], + 'workernames' : ["polly-x86_64-gce1"], + 'builddir': "polly-x86_64-linux-noassert", + 'factory' : PollyBuilder.getPollyBuildFactory( + clean=False, + install=False, + make='ninja', + extraCmakeArgs=[ + "-G", "Ninja", + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", + "-DLLVM_ENABLE_ASSERTIONS=False", + "-DLLVM_TARGETS_TO_BUILD='X86;NVPTX'", + "-DCLANG_ENABLE_ARCMT=OFF", + "-DCLANG_ENABLE_STATIC_ANALYZER=OFF", + "-DCLANG_ENABLE_OBJC_REWRITER=OFF", + "-DLLVM_ENABLE_LLD=ON", + "-DPOLLY_ENABLE_GPGPU_CODEGEN=ON" + ])}, + + {'name' : "polly-x86_64-linux-shared", + 'tags' : ["polly"], + 'workernames' : ["polly-x86_64-gce2"], + 'builddir': "polly-x86_64-linux-shared", + 'factory' : PollyBuilder.getPollyBuildFactory( + clean=False, + install=False, + make='ninja', + extraCmakeArgs=[ + "-G", "Ninja", + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", + "-DLLVM_ENABLE_ASSERTIONS=True", + "-DLLVM_TARGETS_TO_BUILD='X86;NVPTX'", + "-DCLANG_ENABLE_ARCMT=OFF", + "-DCLANG_ENABLE_STATIC_ANALYZER=OFF", + "-DCLANG_ENABLE_OBJC_REWRITER=OFF", + "-DLLVM_ENABLE_LLD=ON", + "-DBUILD_SHARED_LIBS=ON", + "-DPOLLY_ENABLE_GPGPU_CODEGEN=ON" + ])}, + + {'name' : "polly-x86_64-linux-shared-plugin", + 'tags' : ["polly"], + 'workernames' : ["polly-x86_64-gce2"], + 'builddir': "polly-x86_64-linux-shared-plugin", + 'factory' : PollyBuilder.getPollyBuildFactory( + clean=False, + install=False, + make='ninja', + extraCmakeArgs=[ + "-G", "Ninja", + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", + "-DLLVM_ENABLE_ASSERTIONS=True", + "-DLLVM_TARGETS_TO_BUILD='X86;NVPTX'", + "-DCLANG_ENABLE_ARCMT=OFF", + "-DCLANG_ENABLE_STATIC_ANALYZER=OFF", + "-DCLANG_ENABLE_OBJC_REWRITER=OFF", + "-DLLVM_ENABLE_LLD=ON", + "-DBUILD_SHARED_LIBS=ON", + "-DLLVM_POLLY_LINK_INTO_TOOLS=OFF", + "-DPOLLY_ENABLE_GPGPU_CODEGEN=ON" + ])}, + + {'name' : "polly-x86_64-linux-shlib", + 'tags' : ["polly"], + 'workernames' : ["polly-x86_64-gce2"], + 'builddir': "polly-x86_64-linux-shlib", + 'factory' : PollyBuilder.getPollyBuildFactory( + clean=False, + install=False, + make='ninja', + extraCmakeArgs=[ + "-G", "Ninja", + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", + "-DLLVM_ENABLE_ASSERTIONS=True", + "-DLLVM_TARGETS_TO_BUILD='X86;NVPTX'", + "-DCLANG_ENABLE_ARCMT=OFF", + "-DCLANG_ENABLE_STATIC_ANALYZER=OFF", + "-DCLANG_ENABLE_OBJC_REWRITER=OFF", + "-DLLVM_ENABLE_LLD=ON", + "-DLLVM_BUILD_LLVM_DYLIB=ON", + "-DLLVM_LINK_LLVM_DYLIB=ON", + "-DPOLLY_ENABLE_GPGPU_CODEGEN=ON" + ])}, + + {'name' : "polly-x86_64-linux-shlib-plugin", + 'tags' : ["polly"], + 'workernames' : ["polly-x86_64-gce2"], + 'builddir': "polly-x86_64-linux-shlib-plugin", + 'factory' : PollyBuilder.getPollyBuildFactory( + clean=False, + install=False, + make='ninja', + extraCmakeArgs=[ + "-G", "Ninja", + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_TARGETS_TO_BUILD='X86;NVPTX'", "-DCLANG_ENABLE_ARCMT=OFF", "-DCLANG_ENABLE_STATIC_ANALYZER=OFF", + "-DCLANG_ENABLE_OBJC_REWRITER=OFF", + "-DLLVM_ENABLE_LLD=ON", + "-DLLVM_BUILD_LLVM_DYLIB=ON", + "-DLLVM_LINK_LLVM_DYLIB=ON", + "-DLLVM_POLLY_LINK_INTO_TOOLS=OFF", + "-DPOLLY_ENABLE_GPGPU_CODEGEN=ON" ])}, {'name' : "polly-x86_64-linux-test-suite", 'tags' : ["polly"], - 'workernames' : ["polly-x86_64-fdcserver", "polly-x86_64-gce2"], + 'workernames' : ["polly-x86_64-fdcserver", "minipc-1050ti-linux"], 'builddir': "polly-x86_64-linux-test-suite", 'factory' : PollyBuilder.getPollyBuildFactory( clean=False, @@ -875,13 +1157,22 @@ make='ninja', extraCmakeArgs=[ "-G", "Ninja", + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_TARGETS_TO_BUILD='X86;NVPTX'", "-DCLANG_ENABLE_ARCMT=OFF", "-DCLANG_ENABLE_STATIC_ANALYZER=OFF", + "-DCLANG_ENABLE_OBJC_REWRITER=OFF" ], testsuite=True, - extraTestsuiteCmakeArgs=["-G", "Ninja"] + extraTestsuiteCmakeArgs=[ + "-G", "Ninja", + "-DTEST_SUITE_COLLECT_COMPILE_TIME=OFF", + "-DTEST_SUITE_COLLECT_STATS=OFF", + "-DTEST_SUITE_COLLECT_CODE_SIZE=OFF", + WithProperties("-DTEST_SUITE_EXTERNALS_DIR=%(builddir)s/../../test-suite-externals"), + ] )}, # AOSP builders. @@ -899,14 +1190,15 @@ "-DLLVM_TARGET_ARCH=arm-linux-androideabi", "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_ENABLE_LIBCXX:BOOL=ON", - "-DCMAKE_C_COMPILER:FILEPATH=/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/bin/clang", - "-DCMAKE_CXX_COMPILER:FILEPATH=/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/bin/clang++"], + "-DPOLLY_ENABLE_GPGPU_CODEGEN=OFF", + "-DCMAKE_C_COMPILER:FILEPATH=/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/bin/clang", + "-DCMAKE_CXX_COMPILER:FILEPATH=/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/bin/clang++"], timeout=240, target_clang=None, target_flags="-Wno-error -O3 -mllvm -polly -mllvm -polly-position=before-vectorizer -mllvm -polly-process-unprofitable -fcommon", - jobs=8, + jobs=32, extra_make_args=None, - env={'LD_LIBRARY_PATH': '/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/lib', 'PATH': ['/local/cmake-3.17.0/bin', '${PATH}']}, + env={'LD_LIBRARY_PATH': '/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/lib'}, clean=False, sync=False, patch=None)}, @@ -922,14 +1214,15 @@ make='ninja', jobs=16, checkAll=True, - env={'LD_LIBRARY_PATH': '/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/lib', 'PATH': ['/local/cmake-3.17.0/bin', '${PATH}']}, + env={'LD_LIBRARY_PATH': '/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/lib'}, extraCmakeArgs=[ "-G", "Ninja", "-DLLVM_REVERSE_ITERATION:BOOL=ON", "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_ENABLE_LIBCXX:BOOL=ON", - "-DCMAKE_C_COMPILER:FILEPATH=/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/bin/clang", - "-DCMAKE_CXX_COMPILER:FILEPATH=/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/bin/clang++"])}, + "-DPOLLY_ENABLE_GPGPU_CODEGEN=ON", + "-DCMAKE_C_COMPILER:FILEPATH=/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/bin/clang", + "-DCMAKE_CXX_COMPILER:FILEPATH=/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/bin/clang++"])}, # LLDB builders. @@ -943,59 +1236,50 @@ '-DLLVM_ENABLE_ASSERTIONS=True', '-DLLVM_USE_LINKER=gold', '-DLLDB_ENABLE_PYTHON=True', + '-DLLDB_TEST_USER_ARGS=-t', '-DPYTHON_EXECUTABLE=/usr/bin/python3', '-DCMAKE_C_COMPILER=clang', '-DCMAKE_CXX_COMPILER=clang++'])}, {'name' : "lldb-aarch64-ubuntu", 'tags' : ["lldb"], - 'workernames' : ["linaro-aarch64-lldb"], - 'builddir': "lldb-cmake-aarch64", + 'workernames' : ["linaro-lldb-aarch64-ubuntu"], + 'builddir': "lldb-aarch64-ubuntu", 'factory' : LLDBBuilder.getLLDBCMakeBuildFactory( test=True, + clean=True, extra_cmake_args=[ '-DLLVM_ENABLE_ASSERTIONS=True', - '-DLLVM_LIT_ARGS=-svj 8', + '-DLLVM_LIT_ARGS=-vj 4', '-DLLVM_USE_LINKER=gold'])}, {'name' : "lldb-arm-ubuntu", 'tags' : ["lldb"], - 'workernames' : ["linaro-arm-lldb"], - 'builddir': "lldb-cmake-arm", + 'workernames' : ["linaro-lldb-arm-ubuntu"], + 'builddir': "lldb-arm-ubuntu", 'factory' : LLDBBuilder.getLLDBCMakeBuildFactory( test=True, + clean=True, extra_cmake_args=[ '-DLLVM_ENABLE_ASSERTIONS=True', - '-DLLVM_LIT_ARGS=-svj 8', + '-DLLVM_LIT_ARGS=-vj 4', '-DLLVM_USE_LINKER=gold'])}, - {'name' : "lldb-x64-windows-ninja", + {'name' : "lldb-aarch64-windows", 'tags' : ["lldb"], - 'workernames' : ["win-py3-buildbot"], - 'builddir': "lldb-x64-windows-ninja", + 'workernames' : ["linaro-armv8-windows-msvc-05"], + 'builddir': "lldb-aarch64-windows", 'factory' : LLDBBuilder.getLLDBCMakeBuildFactory( clean=True, - python_source_dir=r'"C:\Program Files (x86)\Microsoft Visual Studio\Shared\Python36_64"', - target_arch='x64', - vs="autodetect", test=True, extra_cmake_args=[ - '-DLLVM_ENABLE_ASSERTIONS=OFF', - '-DLLVM_ENABLE_ZLIB=FALSE', - '-DLLDB_ENABLE_PYTHON=TRUE'])}, + "-DCMAKE_C_COMPILER_LAUNCHER=sccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=sccache", + '-DLLVM_LIT_ARGS=-v', + '-DLLDB_TEST_USER_ARGS=--skip-category=watchpoint'])}, # LLD builders. - {'name' : "lld-x86_64-darwin", - 'tags' : ["lld"], - 'workernames' : ["as-worker-3"], - 'builddir': "lld-x86_64-darwin", - 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( - clean=True, - depends_on_projects=['llvm', 'lld'], - extra_configure_args=[ - '-DLLVM_ENABLE_WERROR=OFF'])}, - {'name' : "lld-x86_64-win", 'tags' : ["lld"], 'workernames' : ["as-worker-93"], @@ -1006,17 +1290,6 @@ extra_configure_args = [ '-DLLVM_ENABLE_WERROR=OFF'])}, - {'name' : "lld-x86_64-freebsd", - 'tags' : ["lld"], - 'workernames' : ["as-worker-4"], - 'builddir': "lld-x86_64-freebsd", - 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( - depends_on_projects=['llvm', 'lld'], - extra_configure_args=[ - '-DCMAKE_EXE_LINKER_FLAGS=-lcxxrt', - '-DLLVM_ENABLE_WERROR=OFF'], - env={'CXXFLAGS' : "-std=c++11 -stdlib=libc++"})}, - {'name' : "ppc64le-lld-multistage-test", 'tags' : ["lld", "ppc", "ppc64le"], 'workernames' : ["ppc64le-lld-multistage-test"], @@ -1024,7 +1297,8 @@ 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaMultistageBuildFactory( extra_configure_args=[ '-DLLVM_ENABLE_ASSERTIONS=ON', - '-DLLVM_LIT_ARGS=-svj 256'], + '-DLLVM_LIT_ARGS=-svj 256', + '-DLLVM_CCACHE_BUILD=ON'], depends_on_projects=['llvm', 'clang', 'lld'])}, {'name' : "lld-x86_64-ubuntu-fast", @@ -1050,28 +1324,120 @@ 'tags' : ["clang","lld","LTO"], 'workernames' : ["as-worker-92"], 'builddir': "clang-with-thin-lto-ubuntu", - 'factory' : ClangLTOBuilder.getClangWithLTOBuildFactory(jobs=72, lto='thin')}, + 'factory' : ClangLTOBuilder.getClangWithLTOBuildFactory( + jobs=72, + lto='thin', + )}, {'name' : "clang-with-thin-lto-wpd-ubuntu", 'tags' : ["clang","lld","LTO"], - 'workernames' : ["thinlto-x86-64-bot1"], + 'workernames' : ["thinlto-x86-64-bot1", "thinlto-x86-64-bot2"], 'builddir': "clang-with-thin-lto-wpd-ubuntu", 'factory' : ClangLTOBuilder.getClangWithLTOBuildFactory( jobs=72, lto='thin', extra_configure_args=[ '-DLLVM_CCACHE_BUILD=ON', - '-DCMAKE_CXX_FLAGS="-O3 -Xclang -fwhole-program-vtables -fno-split-lto-unit"', - '-DCMAKE_C_FLAGS="-O3 -Xclang -fwhole-program-vtables -fno-split-lto-unit"', - '-DCMAKE_EXE_LINKER_FLAGS="-Wl,--lto-whole-program-visibility"'])}, + ], + extra_configure_args_lto_stage=[ + '-DCMAKE_CXX_FLAGS=-O3 -Xclang -fwhole-program-vtables -fno-split-lto-unit', + '-DCMAKE_C_FLAGS=-O3 -Xclang -fwhole-program-vtables -fno-split-lto-unit', + '-DCMAKE_EXE_LINKER_FLAGS=-Wl,--lto-whole-program-visibility -fuse-ld=lld'])}, {'name' : "clang-with-lto-ubuntu", 'tags' : ["clang","lld","LTO"], 'workernames' : ["as-worker-91"], 'builddir': "clang-with-lto-ubuntu", - 'factory' : ClangLTOBuilder.getClangWithLTOBuildFactory(jobs=72)}, + 'factory' : ClangLTOBuilder.getClangWithLTOBuildFactory( + jobs=72, + extra_configure_args_lto_stage=[ + '-DLLVM_PARALLEL_LINK_JOBS=14', + ])}, +] -# Builders for MLIR. +# Common builders options for MLIR. +mlir_ubuntu_workers = [f"mlir-ubuntu-worker{i}" for i in range(5)] +mlir_default_cmake_options = [ + '-DLLVM_CCACHE_BUILD=ON', + '-DLLVM_ENABLE_PROJECTS=mlir', + '-DLLVM_TARGETS_TO_BUILD=host;NVPTX;AMDGPU', + '-DLLVM_BUILD_EXAMPLES=ON', + '-DMLIR_INCLUDE_INTEGRATION_TESTS=ON', + '-DMLIR_ENABLE_BINDINGS_PYTHON=ON', +] + +all += [ + + {'name' : "mlir-ubuntu-asan-ubsan-clang-12.0", + 'tags' : ["mlir"], + 'workernames' : mlir_ubuntu_workers, + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + llvm_srcdir="llvm.src", + obj_dir="llvm.obj", + clean=True, + targets = ['check-mlir-build-only'], + checks = ['check-mlir'], + depends_on_projects=['llvm','mlir'], + extra_configure_args= mlir_default_cmake_options + [ + '-DLLVM_ENABLE_LLD=ON', + '-DBUILD_SHARED_LIBS=ON', + '-DLLVM_USE_SANITIZER=Address;Undefined', + '-DCMAKE_C_COMPILER=/compilers/clang-12.0/bin/clang', + '-DCMAKE_CXX_COMPILER=/compilers/clang-12.0/bin/clang++', + ])}, + + {'name' : "mlir-ubuntu-gcc5.5-shared-libs", + 'tags' : ["mlir"], + 'workernames' : mlir_ubuntu_workers, + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + llvm_srcdir="llvm.src", + obj_dir="llvm.obj", + clean=True, + targets = ['check-mlir-build-only'], + checks = ['check-mlir'], + depends_on_projects=['llvm','mlir'], + extra_configure_args= mlir_default_cmake_options + [ + '-DBUILD_SHARED_LIBS=ON', + '-DCMAKE_C_COMPILER=/usr/bin/gcc-5', + '-DCMAKE_CXX_COMPILER=/usr/bin/g++-5', + ])}, + + {'name' : "mlir-ubuntu-clang-5-link-llvm-dylib", + 'tags' : ["mlir"], + 'workernames' : mlir_ubuntu_workers, + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + llvm_srcdir="llvm.src", + obj_dir="llvm.obj", + clean=True, + targets = ['check-mlir-build-only'], + checks = ['check-mlir'], + depends_on_projects=['llvm','mlir'], + extra_configure_args= mlir_default_cmake_options + [ + '-DLLVM_ENABLE_LLD=OFF', # lld-5 is subject to http://llvm.org/pr49915 + '-DLLVM_LINK_LLVM_DYLIB=ON', + '-DCMAKE_C_COMPILER=/compilers/clang-5.0/bin/clang', + '-DCMAKE_CXX_COMPILER=/compilers/clang-5.0/bin/clang++', + ])}, + + {'name' : "mlir-ubuntu-gcc11-release-intel-sde", + 'tags' : ["mlir"], + 'workernames' : mlir_ubuntu_workers, + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + llvm_srcdir="llvm.src", + obj_dir="llvm.obj", + clean=True, + targets = ['check-mlir-build-only'], + checks = ['check-mlir'], + depends_on_projects=['llvm','mlir'], + extra_configure_args= mlir_default_cmake_options + [ + '-DLLVM_ENABLE_LLD=ON', + '-DLLVM_ENABLE_ASSERTIONS=OFF', + '-DCMAKE_C_COMPILER=/usr/bin/gcc-11', + '-DCMAKE_CXX_COMPILER=/usr/bin/g++-11', + '-DMLIR_RUN_X86VECTOR_TESTS=ON', + '-DMLIR_RUN_AMX_TESTS=ON', + '-DINTEL_SDE_EXECUTABLE=/intel_sde/sde64' + ])}, {'name' : "mlir-nvidia", 'tags' : ["mlir"], @@ -1081,16 +1447,21 @@ llvm_srcdir="llvm.src", obj_dir="llvm.obj", clean=True, + targets = ['check-mlir-build-only'], + checks = ['check-mlir'], depends_on_projects=['llvm','mlir'], extra_configure_args=[ '-DLLVM_BUILD_EXAMPLES=ON', '-DLLVM_TARGETS_TO_BUILD=host;NVPTX', '-DLLVM_ENABLE_PROJECTS=mlir', - '-DMLIR_CUDA_RUNNER_ENABLED=1', + '-DMLIR_ENABLE_CUDA_RUNNER=1', + '-DMLIR_INCLUDE_INTEGRATION_TESTS=ON', '-DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc', - '-DMLIR_VULKAN_RUNNER_ENABLED=1', + '-DMLIR_ENABLE_VULKAN_RUNNER=1', '-DBUILD_SHARED_LIBS=ON', '-DLLVM_CCACHE_BUILD=ON', + '-DMLIR_ENABLE_BINDINGS_PYTHON=ON', + '-DMLIR_RUN_CUDA_TENSOR_CORE_TESTS=ON', ], env={ 'CC':'clang', @@ -1104,87 +1475,227 @@ 'builddir': "mlir-x64-windows-ninja", 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaWithMSVCBuildFactory( clean=True, + targets = ['check-mlir-build-only'], + checks = ['check-mlir'], depends_on_projects=['llvm','mlir'], vs="autodetect", - checks=['check-mlir'], extra_configure_args=[ "-DLLVM_BUILD_EXAMPLES=ON", "-DLLVM_ENABLE_PROJECTS=mlir", + "-DMLIR_ENABLE_BINDINGS_PYTHON=ON", + "-DLLVM_ENABLE_WERROR=ON", "-DLLVM_TARGETS_TO_BUILD='host;NVPTX;AMDGPU'", ])}, {'name' : 'ppc64le-mlir-rhel-clang', 'tags' : ["mlir", "ppc", "ppc64le"], 'collapseRequests' : False, - 'workernames' : ['ppc64le-flang-mlir-rhel-test'], + 'workernames' : ['ppc64le-mlir-rhel-test'], 'builddir': 'ppc64le-mlir-rhel-clang-build', 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( clean=True, depends_on_projects=['llvm', 'mlir'], - checks=['check-mlir'], + targets = ['check-mlir-build-only'], + checks = ['check-mlir'], extra_configure_args=[ '-DLLVM_TARGETS_TO_BUILD=PowerPC', '-DLLVM_INSTALL_UTILS=ON', '-DCMAKE_CXX_STANDARD=17', '-DLLVM_ENABLE_PROJECTS=mlir', '-DLLVM_LIT_ARGS=-vj 256', + '-DLLVM_CCACHE_BUILD=ON', ], env={ 'CC': 'clang', 'CXX': 'clang++', 'LD': 'lld', + 'LD_LIBRARY_PATH': '/usr/lib64', })}, + {'name' : 'mlir-s390x-linux', + 'tags' : ["mlir", "s390x"], + 'workernames' : ["systemz-1"], + 'builddir': 'mlir-s390x-linux', + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + clean=True, + depends_on_projects=['llvm', 'mlir'], + checks=['check-mlir'], + extra_configure_args=[ + "-DLLVM_CCACHE_BUILD=ON", + '-DLLVM_TARGETS_TO_BUILD=SystemZ', + '-DLLVM_ENABLE_PROJECTS=mlir', + '-DLLVM_LIT_ARGS=-vj 4', + ])}, + + {'name' : "mlir-s390x-linux-werror", + 'tags' : ["mlir", "s390x"], + 'workernames' : ["onnx-mlir-nowarn-linux-s390x"], + 'builddir': "onnx-mlir-nowarn-linux-s390x", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + clean=True, + checks = ['check-mlir'], + targets = ['check-mlir-build-only'], + depends_on_projects=['llvm','mlir'], + extra_configure_args=[ + "-DCMAKE_BUILD_TYPE=Release", + "-DLLVM_ENABLE_PROJECTS=mlir", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_ENABLE_RTTI=ON", + "-DLLVM_ENABLE_WERROR=ON", + "-DLLVM_TARGETS_TO_BUILD=host", + ])}, + # Sanitizer builders. +# +# bootstrap-asan, bootstrap-msan, and sanitizer-x86_64-linux-fast have steps +# with large memory usage, so assign them to different workers. {'name' : "sanitizer-x86_64-linux", 'tags' : ["sanitizer"], - 'workernames' : ["sanitizer-buildbot1", "sanitizer-buildbot2"], + 'workernames' : [ + "sanitizer-buildbot1", + "sanitizer-buildbot2", + ], 'builddir': "sanitizer-x86_64-linux", 'factory' : SanitizerBuilder.getSanitizerBuildFactory()}, {'name' : "sanitizer-x86_64-linux-fast", 'tags' : ["sanitizer"], - 'workernames' : ["sanitizer-buildbot1", "sanitizer-buildbot2"], + 'workernames' : [ + "sanitizer-buildbot3", + "sanitizer-buildbot4", + ], 'builddir': "sanitizer-x86_64-linux-fast", - 'factory' : SanitizerBuilder.getSanitizerBuildFactory()}, + 'factory' : SanitizerBuilder.getSanitizerBuildFactory( + extra_depends_on_projects=["mlir", "clang-tools-extra"] + )}, - {'name' : "sanitizer-x86_64-linux-bootstrap", + {'name' : "sanitizer-x86_64-linux-bootstrap-asan", 'tags' : ["sanitizer"], - 'workernames' : ["sanitizer-buildbot3", "sanitizer-buildbot4"], - 'builddir': "sanitizer-x86_64-linux-bootstrap", - 'factory' : SanitizerBuilder.getSanitizerBuildFactory()}, + 'workernames' : [ + "sanitizer-buildbot1", + "sanitizer-buildbot2", + ], + 'builddir': "sanitizer-x86_64-linux-bootstrap-asan", + 'factory' : SanitizerBuilder.getSanitizerBuildFactory( + extra_depends_on_projects=["mlir", "clang-tools-extra"] + )}, + + {'name' : "sanitizer-x86_64-linux-bootstrap-msan", + 'tags' : ["sanitizer"], + 'workernames' : [ + "sanitizer-buildbot5", + "sanitizer-buildbot6", + ], + 'builddir': "sanitizer-x86_64-linux-bootstrap-msan", + 'factory' : SanitizerBuilder.getSanitizerBuildFactory( + extra_depends_on_projects=["mlir", "clang-tools-extra"] + )}, {'name' : "sanitizer-x86_64-linux-bootstrap-ubsan", 'tags' : ["sanitizer"], - 'workernames' : ["sanitizer-buildbot3", "sanitizer-buildbot4"], + 'workernames' : [ + "sanitizer-buildbot3", + "sanitizer-buildbot4", + ], 'builddir': "sanitizer-x86_64-linux-bootstrap-ubsan", - 'factory' : SanitizerBuilder.getSanitizerBuildFactory()}, + 'factory' : SanitizerBuilder.getSanitizerBuildFactory( + extra_depends_on_projects=["mlir", "clang-tools-extra"] + )}, - {'name' : "sanitizer-x86_64-linux-bootstrap-msan", + {'name' : "sanitizer-x86_64-linux-autoconf", 'tags' : ["sanitizer"], - 'workernames' : ["sanitizer-buildbot7", "sanitizer-buildbot8"], - 'builddir': "sanitizer-x86_64-linux-bootstrap-msan", + 'workernames' : [ + "sanitizer-buildbot5", + "sanitizer-buildbot6", + ], + 'builddir': "sanitizer-x86_64-linux-autoconf", 'factory' : SanitizerBuilder.getSanitizerBuildFactory()}, - {'name' : "sanitizer-x86_64-linux-autoconf", + {'name' : "sanitizer-x86_64-linux-qemu", 'tags' : ["sanitizer"], - 'workernames' : ["sanitizer-buildbot7", "sanitizer-buildbot8"], - 'builddir': "sanitizer-x86_64-linux-autoconf", + 'workernames' : [ + "sanitizer-buildbot3", + "sanitizer-buildbot4", + ], + 'builddir': "sanitizer-x86_64-linux-qemu", 'factory' : SanitizerBuilder.getSanitizerBuildFactory()}, {'name' : "sanitizer-x86_64-linux-fuzzer", 'tags' : ["sanitizer"], - 'workernames' : ["sanitizer-buildbot7", "sanitizer-buildbot8"], + 'workernames' : [ + "sanitizer-buildbot1", + "sanitizer-buildbot2", + ], 'builddir': "sanitizer-x86_64-linux-fuzzer", 'factory' : SanitizerBuilder.getSanitizerBuildFactory()}, {'name' : "sanitizer-x86_64-linux-android", 'tags' : ["sanitizer"], - 'workernames' : ["sanitizer-buildbot6"], + 'workernames' : [ + "sanitizer-buildbot-android", + ], 'builddir': "sanitizer-x86_64-linux-android", 'factory' : SanitizerBuilder.getSanitizerBuildFactory()}, + {'name' : "sanitizer-aarch64-linux-bootstrap-asan", + 'tags' : ["sanitizer"], + 'workernames' : [ + "sanitizer-buildbot7", + "sanitizer-buildbot9", + "sanitizer-buildbot11", + ], + 'builddir': "sanitizer-aarch64-linux-bootstrap-asan", + 'factory' : SanitizerBuilder.getSanitizerBuildFactory( + extra_depends_on_projects=["mlir", "clang-tools-extra"] + )}, + + {'name' : "sanitizer-aarch64-linux-bootstrap-hwasan", + 'tags' : ["sanitizer"], + 'workernames' : [ + "sanitizer-buildbot7", + "sanitizer-buildbot9", + "sanitizer-buildbot11", + ], + 'builddir': "sanitizer-aarch64-linux-bootstrap-hwasan", + 'factory' : SanitizerBuilder.getSanitizerBuildFactory( + extra_depends_on_projects=["mlir", "clang-tools-extra"] + )}, + + {'name' : "sanitizer-aarch64-linux-bootstrap-msan", + 'tags' : ["sanitizer"], + 'workernames' : [ + "sanitizer-buildbot8", + "sanitizer-buildbot10", + "sanitizer-buildbot12", + ], + 'builddir': "sanitizer-aarch64-linux-bootstrap-msan", + 'factory' : SanitizerBuilder.getSanitizerBuildFactory( + extra_depends_on_projects=["mlir", "clang-tools-extra"] + )}, + + {'name' : "sanitizer-aarch64-linux-bootstrap-ubsan", + 'tags' : ["sanitizer"], + 'workernames' : [ + "sanitizer-buildbot8", + "sanitizer-buildbot10", + "sanitizer-buildbot12", + ], + 'builddir': "sanitizer-aarch64-linux-bootstrap-ubsan", + 'factory' : SanitizerBuilder.getSanitizerBuildFactory( + extra_depends_on_projects=["mlir", "clang-tools-extra"] + )}, + + {'name' : "sanitizer-aarch64-linux-fuzzer", + 'tags' : ["sanitizer"], + 'workernames' : [ + "sanitizer-buildbot8", + "sanitizer-buildbot10", + "sanitizer-buildbot12", + ], + 'builddir': "sanitizer-aarch64-linux-fuzzer", + 'factory' : SanitizerBuilder.getSanitizerBuildFactory()}, + {'name' : "sanitizer-ppc64be-linux", 'tags' : ["sanitizer", "ppc"], 'workernames' : ["ppc64be-sanitizer"], @@ -1205,14 +1716,19 @@ script="sanitizer-windows.py", depends_on_projects=["llvm", "clang", "lld", "compiler-rt"])}, +# OpenMP builders. + {'name' : "openmp-gcc-x86_64-linux-debian", 'tags' : ["openmp"], 'workernames' : ["gribozavr4"], 'builddir': "openmp-gcc-x86_64-linux-debian", 'factory' : OpenMPBuilder.getOpenMPCMakeBuildFactory( + extraCmakeArgs=[ + '-DLLVM_CCACHE_BUILD=ON', + ], env={ 'PATH':'/home/llvmbb/bin/clang-latest/bin:/home/llvmbb/bin:/usr/local/bin:/usr/local/bin:/usr/bin:/bin', - 'CC': 'ccache clang', 'CXX': 'ccache clang++', 'CCACHE_CPP2': 'yes', + 'CC': 'clang', 'CXX': 'clang++', })}, {'name' : "openmp-clang-x86_64-linux-debian", @@ -1220,129 +1736,173 @@ 'workernames' : ["gribozavr4"], 'builddir': "openmp-clang-x86_64-linux-debian", 'factory' : OpenMPBuilder.getOpenMPCMakeBuildFactory( + extraCmakeArgs=[ + '-DLLVM_CCACHE_BUILD=ON', + ], env={ 'PATH':'/home/llvmbb/bin/clang-latest/bin:/home/llvmbb/bin:/usr/local/bin:/usr/local/bin:/usr/bin:/bin', - 'CC': 'ccache clang', 'CXX': 'ccache clang++', 'CCACHE_CPP2': 'yes', + 'CC': 'clang', 'CXX': 'clang++', })}, -# Libc++ builders. - - {'name': 'libcxx-libcxxabi-x86_64-linux-debian', - 'tags' : ["libcxx"], - 'workernames': ['gribozavr4'], - 'builddir': 'libcxx-libcxxabi-x86_64-linux-debian', - 'factory': LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( - env={'CC': 'clang', 'CXX': 'clang++'}, - lit_extra_args=['--shuffle'], - check_libcxx_abilist=True)}, - - {'name' : 'libcxx-libcxxabi-x86_64-linux-debian-noexceptions', - 'tags' : ["libcxx"], - 'workernames' : ['gribozavr4'], - 'builddir': 'libcxx-libcxxabi-x86_64-linux-debian-noexceptions', - 'factory' : LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( - env={'CC': 'clang', 'CXX': 'clang++'}, - use_cache='Generic-noexceptions.cmake', - lit_extra_args=['--shuffle'])}, - - {'name' : 'libcxx-libcxxabi-libunwind-x86_64-linux-debian', - 'tags' : ["libcxx"], - 'workernames' : ['gribozavr4'], - 'builddir': 'libcxx-libcxxabi-libunwind-x86_64-linux-debian', - 'factory' : LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( - env={'CC': 'clang', 'CXX': 'clang++'}, - cmake_extra_opts={'LIBCXXABI_USE_LLVM_UNWINDER': 'ON'}, - lit_extra_args=['--shuffle'])}, - - {'name' : 'libcxx-libcxxabi-singlethreaded-x86_64-linux-debian', - 'tags' : ["libcxx"], - 'workernames' : ['gribozavr4'], - 'builddir': 'libcxx-libcxxabi-singlethreaded-x86_64-linux-debian', - 'factory' : LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( - env={'CC': 'clang', 'CXX': 'clang++'}, - use_cache='Generic-singlethreaded.cmake')}, - - # ARMv7 LibC++ and LibC++abi tests (require Clang+RT) - {'name' : 'libcxx-libcxxabi-libunwind-armv7-linux', - 'tags' : ["libcxx"], - 'workernames': ['linaro-tk1-02'], - 'builddir': 'libcxx-libcxxabi-libunwind-armv7-linux', - 'factory' : LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( - cmake_extra_opts={ - 'LIBCXXABI_USE_LLVM_UNWINDER': 'ON', - 'CMAKE_C_FLAGS': '-mcpu=cortex-a15 -marm', - 'CMAKE_CXX_FLAGS': '-mcpu=cortex-a15 -marm', - 'LLVM_PARALLEL_LINK_JOBS': '2'})}, - - # ARMv8 LibC++ and LibC++abi tests (require Clang+RT) - {'name' : 'libcxx-libcxxabi-libunwind-armv8-linux', - 'tags' : ["libcxx"], - 'workernames' : ['linaro-armv8-libcxx'], - 'builddir': 'libcxx-libcxxabi-libunwind-armv8-linux', - 'factory' : LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( - cmake_extra_opts={ - 'LIBCXXABI_USE_LLVM_UNWINDER': 'ON', - 'CMAKE_C_FLAGS': '-mcpu=cortex-a57 -marm', - 'CMAKE_CXX_FLAGS': '-mcpu=cortex-a57 -marm'})}, - - # ARMv7 LibC++ and LibC++abi tests w/o EH (require Clang+RT) - {'name' : 'libcxx-libcxxabi-libunwind-armv7-linux-noexceptions', - 'tags' : ["libcxx"], - 'workernames' : ['linaro-tk1-02'], - 'builddir': 'libcxx-libcxxabi-libunwind-armv7-linux-noexceptions', - 'factory' : LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( - cmake_extra_opts={ - 'LIBCXXABI_USE_LLVM_UNWINDER': 'ON', - 'LIBCXX_ENABLE_EXCEPTIONS': 'OFF', - 'LIBCXXABI_ENABLE_EXCEPTIONS': 'OFF', - 'CMAKE_C_FLAGS': '-mcpu=cortex-a15 -mthumb', - 'CMAKE_CXX_FLAGS': '-mcpu=cortex-a15 -mthumb', - 'LLVM_PARALLEL_LINK_JOBS': '2'})}, - - # ARMv8 LibC++ and LibC++abi tests w/o EH (require Clang+RT) - {'name' : 'libcxx-libcxxabi-libunwind-armv8-linux-noexceptions', - 'tags' : ["libcxx"], - 'workernames' : ['linaro-armv8-libcxx'], - 'builddir': 'libcxx-libcxxabi-libunwind-armv8-linux-noexceptions', - 'factory' : LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( - cmake_extra_opts={ - 'LIBCXXABI_USE_LLVM_UNWINDER': 'ON', - 'LIBCXX_ENABLE_EXCEPTIONS': 'OFF', - 'LIBCXXABI_ENABLE_EXCEPTIONS': 'OFF', - 'CMAKE_C_FLAGS': '-mcpu=cortex-a57 -mthumb', - 'CMAKE_CXX_FLAGS': '-mcpu=cortex-a57 -mthumb'})}, - - # AArch64 LibC++ and LibC++abi tests (require Clang+RT) - {'name' : 'libcxx-libcxxabi-libunwind-aarch64-linux', - 'tags' : ["libcxx"], - 'workernames' : ['linaro-aarch64-libcxx'], - 'builddir': 'libcxx-libcxxabi-libunwind-aarch64-linux', - 'factory' : LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( - cmake_extra_opts={ - 'LIBCXXABI_USE_LLVM_UNWINDER': 'ON', - 'CMAKE_C_FLAGS': '-mcpu=cortex-a57', - 'CMAKE_CXX_FLAGS': '-mcpu=cortex-a57'})}, - - {'name' : 'libcxx-libcxxabi-libunwind-aarch64-linux-noexceptions', - 'tags' : ["libcxx"], - 'workernames' : ['linaro-aarch64-libcxx'], - 'builddir': 'libcxx-libcxxabi-libunwind-aarch64-linux-noexceptions', - 'factory' : LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( - cmake_extra_opts={ - 'LIBCXXABI_USE_LLVM_UNWINDER': 'ON', - 'LIBCXX_ENABLE_EXCEPTIONS': 'OFF', - 'LIBCXXABI_ENABLE_EXCEPTIONS': 'OFF', - 'CMAKE_C_FLAGS': '-mcpu=cortex-a57', - 'CMAKE_CXX_FLAGS': '-mcpu=cortex-a57'})}, + {'name' : "openmp-offload-cuda-project", + 'tags' : ["openmp"], + 'workernames' : ["minipc-1050ti-linux"], + 'builddir': "openmp-offload-cuda-project", + 'factory' : OpenMPBuilder.getOpenMPCMakeBuildFactory( + clean=False, + enable_runtimes=[], + extraCmakeArgs=[ + "-DCUDA_TOOLKIT_ROOT_DIR=/opt/cuda", + "-DLIBOMPTARGET_BUILD_NVPTX_BCLIB=ON", + "-DCLANG_ENABLE_STATIC_ANALYZER=OFF", + "-DCLANG_ENABLE_ARCMT=OFF", + "-DCLANG_ENABLE_OBJC_REWRITER=OFF", + "-DLLVM_TARGETS_TO_BUILD=X86;NVPTX", + "-DLLVM_ENABLE_LLD=ON", + '-DLLVM_PARALLEL_LINK_JOBS=2', + ], + install=True, + testsuite=True, + testsuite_sollvevv=True, + extraTestsuiteCmakeArgs=[ + "-DTEST_SUITE_SOLLVEVV_OFFLOADING_CFLAGS=-fopenmp-targets=nvptx64-nvidia-cuda;--cuda-path=/opt/cuda", + "-DTEST_SUITE_SOLLVEVV_OFFLOADING_LDFLAGS=-fopenmp-targets=nvptx64-nvidia-cuda;--cuda-path=/opt/cuda", + ], + )}, - {'name': "fuchsia-x86_64-linux", - 'tags' : ["toolchain"], + {'name' : "openmp-offload-cuda-runtime", + 'tags' : ["openmp"], + 'workernames' : ["minipc-1050ti-linux"], + 'builddir': "openmp-offload-cuda-runtime", + 'factory' : OpenMPBuilder.getOpenMPCMakeBuildFactory( + clean=True, + enable_runtimes=['openmp'], + extraCmakeArgs=[ + "-DCUDA_TOOLKIT_ROOT_DIR=/opt/cuda", + "-DLIBOMPTARGET_BUILD_NVPTX_BCLIB=ON", + "-DCLANG_ENABLE_STATIC_ANALYZER=OFF", + "-DCLANG_ENABLE_ARCMT=OFF", + "-DCLANG_ENABLE_OBJC_REWRITER=OFF", + "-DLLVM_TARGETS_TO_BUILD=X86;NVPTX", + "-DLLVM_ENABLE_LLD=ON", + '-DLLVM_PARALLEL_LINK_JOBS=2', + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", + ], + install=True, + testsuite=True, + testsuite_sollvevv=True, + extraTestsuiteCmakeArgs=[ + "-DTEST_SUITE_SOLLVEVV_OFFLOADING_CFLAGS=-fopenmp-targets=nvptx64-nvidia-cuda;--cuda-path=/opt/cuda", + "-DTEST_SUITE_SOLLVEVV_OFFLOADING_LDFLAGS=-fopenmp-targets=nvptx64-nvidia-cuda;--cuda-path=/opt/cuda", + ], + )}, + +# OpenMP AMDGPU Builders + {'name' : "openmp-offload-amdgpu-runtime", + 'tags' : ["openmp"], + 'workernames' : ["omp-vega20-0"], + 'builddir': "openmp-offload-amdgpu-runtime", + 'factory' : OpenMPBuilder.getOpenMPCMakeBuildFactory( + clean=True, + enable_runtimes=['openmp'], + depends_on_projects=['llvm','clang','lld','openmp'], + extraCmakeArgs=[ + "-DCMAKE_BUILD_TYPE=Release", + "-DCLANG_DEFAULT_LINKER=lld", + "-DLLVM_TARGETS_TO_BUILD=X86;AMDGPU", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_ENABLE_RUNTIMES=openmp", + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", + ], + install=True, + testsuite=False, + testsuite_sollvevv=False, + extraTestsuiteCmakeArgs=[ + "-DTEST_SUITE_SOLLVEVV_OFFLOADING_CFLAGS=-fopenmp-targets=amdgcn-amd-amdhsa;-Xopenmp-target=amdgcn-amd-amdhsa", + "-DTEST_SUITE_SOLLVEVV_OFFLOADING_LDLAGS=-fopenmp-targets=amdgcn-amd-amdhsa;-Xopenmp-target=amdgcn-amd-amdhsa", + ], + )}, + + {'name' : "openmp-offload-libc-amdgpu-runtime", + 'tags' : ["openmp"], + 'workernames' : ["omp-vega20-1"], + 'builddir': "openmp-offload-libc-amdgpu-runtime", + 'factory' : OpenMPBuilder.getOpenMPCMakeBuildFactory( + clean=True, + depends_on_projects=['llvm', 'clang', 'compiler-rt', 'libc', 'lld', 'openmp'], + extraCmakeArgs=[ + "-DCMAKE_BUILD_TYPE=Release", + "-DCLANG_DEFAULT_LINKER=lld", + "-DLLVM_TARGETS_TO_BUILD=X86;AMDGPU", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", + "-DLIBOMPTARGET_FOUND_AMDGPU_GPU=ON", + "-DLIBOMP_ARCHER_SUPPORT=OFF", + "-DLIBC_GPU_BUILD=ON", + "-DLIBC_GPU_ARCHITECTURES=gfx906", + "-DLIBC_GPU_TEST_ARCHITECTURE=gfx906", + "-DLIBC_GPU_TEST_JOBS=1", + ], + install=True, + testsuite=False, + testsuite_sollvevv=False, + extraTestsuiteCmakeArgs=[ + "-DTEST_SUITE_SOLLVEVV_OFFLOADING_CFLAGS=-fopenmp;-fopenmp-targets=amdgcn-amd-amdhsa;-Xopenmp-target=amdgcn-amd-amdhsa;-march=gfx906", + "-DTEST_SUITE_SOLLVEVV_OFFLOADING_LDLAGS=-fopenmp;-fopenmp-targets=amdgcn-amd-amdhsa;-Xopenmp-target=amdgcn-amd-amdhsa;-march=gfx906", + ], + add_lit_checks=["check-clang", "check-llvm", "check-lld", "check-libc"] + )}, + + +# Whole-toolchain builders. + + {'name': "fuchsia-x86_64-linux", + 'tags' : ["toolchain"], 'workernames' :["fuchsia-debian-64-us-central1-a-1", "fuchsia-debian-64-us-central1-b-1"], 'builddir': "fuchsia-x86_64-linux", 'factory': FuchsiaBuilder.getFuchsiaToolchainBuildFactory()}, # libc Builders. + {'name' : 'libc-x86_64-windows-dbg', + 'tags' : ["libc"], + 'workernames' : ['libc-x86_64-windows'], + 'builddir': 'libc-x86_64-windows', + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="libc-windows.py", + depends_on_projects=['llvm', 'libc', 'clang', 'clang-tools-extra'], + extra_args=['--debug'])}, + + {'name' : 'libc-arm32-debian-dbg', + 'tags' : ["libc"], + 'workernames' : ['libc-arm32-debian'], + 'builddir': 'libc-arm32-debian-dbg', + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="libc-linux.py", + depends_on_projects=['llvm', 'libc', 'clang', 'clang-tools-extra'], + extra_args=['--debug'])}, + + {'name' : 'libc-aarch64-ubuntu-dbg', + 'tags' : ["libc"], + 'workernames' : ['libc-aarch64-ubuntu'], + 'builddir': 'libc-aarch64-ubuntu', + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="libc-linux.py", + depends_on_projects=['llvm', 'libc', 'clang', 'clang-tools-extra'], + extra_args=['--debug'])}, + + {'name' : "libc-aarch64-ubuntu-fullbuild-dbg", + 'tags' : ["libc"], + 'workernames' : ["libc-aarch64-ubuntu"], + 'builddir': "libc-aarch64-ubuntu-fullbuild-dbg", + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="libc-linux.py", + depends_on_projects=['llvm', 'libc', 'clang', 'clang-tools-extra'], + extra_args=['--debug'])}, + {'name' : 'libc-x86_64-debian', 'tags' : ["libc"], 'workernames' : ['libc-x86_64-debian'], @@ -1369,59 +1929,107 @@ depends_on_projects=['llvm', 'libc', 'clang', 'clang-tools-extra'], extra_args=['--debug', '--asan'])}, -# Flang builders. + {'name' : "libc-x86_64-debian-dbg-runtimes-build", + 'tags' : ["libc"], + 'workernames' : ["libc-x86_64-debian"], + 'builddir': "libc-x86_64-debian-dbg-runtimes-build", + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="libc-linux.py", + depends_on_projects=['llvm', 'libc', 'clang', 'clang-tools-extra'], + extra_args=['--debug'])}, - {'name' : "flang-aarch64-ubuntu", - 'tags' : ["flang"], - 'workernames' : ["flang-aarch64-ubuntu-build"], - 'builddir': "flang-aarch64-ubuntu-build", - 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( - clean=True, - depends_on_projects=['llvm','mlir','clang','flang'], - extra_configure_args=[ - "-DLLVM_TARGETS_TO_BUILD=AArch64", - "-DCMAKE_C_COMPILER=/usr/bin/gcc-9", - "-DCMAKE_CXX_COMPILER=/usr/bin/g++-9", - "-DLLVM_INSTALL_UTILS=ON", - "-DFLANG_BUILD_NEW_DRIVER=ON", - "-DCMAKE_CXX_STANDARD=17", - ])}, + {'name' : "libc-x86_64-debian-fullbuild-dbg", + 'tags' : ["libc"], + 'workernames' : ["libc-x86_64-debian-fullbuild"], + 'builddir': "libc-x86_64-debian-fullbuild-dbg", + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="libc-linux.py", + depends_on_projects=['llvm', 'libc', 'clang', 'clang-tools-extra'], + extra_args=['--debug'])}, + + {'name' : "libc-x86_64-debian-gcc-fullbuild-dbg", + 'tags' : ["libc"], + 'workernames' : ["libc-x86_64-debian-fullbuild"], + 'builddir': "libc-x86_64-debian-gcc-fullbuild-dbg", + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="libc-linux.py", + depends_on_projects=['llvm', 'libc', 'clang', 'clang-tools-extra'], + extra_args=['--debug'])}, - {'name' : "flang-aarch64-ubuntu-dylib", + {'name' : "libc-x86_64-debian-fullbuild-dbg-asan", + 'tags' : ["libc"], + 'workernames' : ["libc-x86_64-debian-fullbuild"], + 'builddir': "libc-x86_64-debian-fullbuild-dbg-asan", + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="libc-linux.py", + depends_on_projects=['llvm', 'libc', 'clang', 'clang-tools-extra'], + extra_args=['--debug', '--asan'])}, + + {'name' : "libc-x86_64-debian-dbg-lint", + 'tags' : ["libc"], + 'workernames' : ["libc-lint-worker"], + 'builddir': "libc-x86_64-debian-dbg-lint", + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="libc-linux.py", + depends_on_projects=['llvm', 'libc'], + extra_args=['--debug'])}, + + {'name' : 'libc-riscv64-debian-dbg', + 'tags' : ["libc"], + 'workernames' : ['libc-riscv64-debian'], + 'builddir': 'libc-riscv64-debian-dbg', + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="libc-linux.py", + depends_on_projects=['llvm', 'libc'], + extra_args=['--debug'])}, + + {'name' : "libc-riscv64-debian-fullbuild-dbg", + 'tags' : ["libc"], + 'workernames' : ["libc-riscv64-debian"], + 'builddir': "libc-riscv64-debian-fullbuild-dbg", + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="libc-linux.py", + depends_on_projects=['llvm', 'libc'], + extra_args=['--debug'])}, + +# Flang builders. + + {'name' : "flang-aarch64-dylib", 'tags' : ["flang"], - 'workernames' : ["linaro-aarch64-flang-dylib"], - 'builddir': "flang-aarch64-ubuntu-dylib", + 'workernames' : ["linaro-flang-aarch64-dylib"], + 'builddir': "flang-aarch64-dylib", 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( clean=True, + checks=['check-flang'], depends_on_projects=['llvm','mlir','clang','flang'], extra_configure_args=[ "-DLLVM_TARGETS_TO_BUILD=AArch64", - "-DFLANG_BUILD_NEW_DRIVER=ON", "-DLLVM_BUILD_LLVM_DYLIB=ON", "-DLLVM_LINK_LLVM_DYLIB=ON", "-DCMAKE_CXX_STANDARD=17", ])}, - {'name' : "flang-aarch64-ubuntu-sharedlibs", + {'name' : "flang-aarch64-sharedlibs", 'tags' : ["flang"], - 'workernames' : ["linaro-aarch64-flang-sharedlibs"], - 'builddir': "flang-aarch64-ubuntu-sharedlibs", + 'workernames' : ["linaro-flang-aarch64-sharedlibs"], + 'builddir': "flang-aarch64-sharedlibs", 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( clean=True, + checks=['check-flang'], depends_on_projects=['llvm','mlir','clang','flang'], extra_configure_args=[ "-DLLVM_TARGETS_TO_BUILD=AArch64", - "-DFLANG_BUILD_NEW_DRIVER=ON", "-DBUILD_SHARED_LIBS=ON", "-DLLVM_BUILD_EXAMPLES=ON", "-DCMAKE_CXX_STANDARD=17", ])}, - {'name' : "flang-aarch64-ubuntu-out-of-tree", + {'name' : "flang-aarch64-out-of-tree", 'tags' : ["flang"], - 'workernames' : ["linaro-aarch64-flang-oot"], + 'workernames' : ["linaro-flang-aarch64-out-of-tree"], 'builddir': "flang-aarch64-out-of-tree", 'factory' : FlangBuilder.getFlangOutOfTreeBuildFactory( + checks=['check-flang'], llvm_extra_configure_args=[ "-DLLVM_TARGETS_TO_BUILD=AArch64", "-DCMAKE_CXX_STANDARD=17", @@ -1434,16 +2042,89 @@ "-DCMAKE_BUILD_TYPE=Release", ])}, - {'name' : "flang-x86_64-linux", + {'name' : "flang-aarch64-debug", 'tags' : ["flang"], - 'workernames' : ["nersc-flang"], - 'builddir': "flang-x86_64-linux", + 'workernames' : ["linaro-flang-aarch64-debug"], + 'builddir': "flang-aarch64-debug", 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + clean=True, + checks=['check-flang'], depends_on_projects=['llvm','mlir','clang','flang'], extra_configure_args=[ - "-DLLVM_TARGETS_TO_BUILD=X86", + "-DLLVM_TARGETS_TO_BUILD=AArch64", + "-DCMAKE_BUILD_TYPE=Debug", + "-DCMAKE_CXX_STANDARD=17", + "-DLLVM_USE_LINKER=lld", + ])}, + + {'name' : "flang-aarch64-latest-clang", + 'tags' : ['flang'], + 'workernames' : ["linaro-flang-aarch64-latest-clang"], + 'builddir': "flang-aarch64-latest-clang", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + clean=True, + checks=['check-flang'], + depends_on_projects=['llvm','mlir','clang','flang'], + extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=AArch64", "-DLLVM_INSTALL_UTILS=ON", "-DCMAKE_CXX_STANDARD=17", + "-DLLVM_ENABLE_WERROR=OFF", + "-DFLANG_ENABLE_WERROR=ON", + "-DBUILD_SHARED_LIBS=ON", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_ENABLE_LIBCXX=On", + "-DCMAKE_BUILD_TYPE=Release", + ])}, + + {'name' : "flang-aarch64-release", + 'tags' : ["flang"], + 'workernames' : ["linaro-flang-aarch64-release"], + 'builddir': "flang-aarch64-release", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + clean=True, + checks=['check-flang'], + depends_on_projects=['llvm','mlir','clang','flang'], + extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=AArch64", + "-DCMAKE_BUILD_TYPE=Release", + "-DLLVM_ENABLE_ASSERTIONS=OFF", + "-DCMAKE_CXX_STANDARD=17", + ])}, + + {'name' : "flang-aarch64-rel-assert", + 'tags' : ["flang"], + 'workernames' : ["linaro-flang-aarch64-rel-assert"], + 'builddir': "flang-aarch64-rel-assert", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + clean=True, + checks=['check-flang'], + depends_on_projects=['llvm','mlir','clang','flang'], + extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=AArch64", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_BUILD_EXAMPLES=ON", + "-DCMAKE_BUILD_TYPE=Release", + "-DCMAKE_CXX_STANDARD=17", + ])}, + + {'name' : "flang-aarch64-latest-gcc", + 'tags' : ['flang'], + 'workernames' : ["linaro-flang-aarch64-latest-gcc"], + 'builddir': "flang-aarch64-latest-gcc", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + clean=True, + checks=['check-flang'], + depends_on_projects=['llvm','mlir','clang','flang'], + extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=AArch64", + "-DLLVM_INSTALL_UTILS=ON", + "-DCMAKE_CXX_STANDARD=17", + "-DLLVM_ENABLE_WERROR=OFF", + "-DFLANG_ENABLE_WERROR=ON", + "-DBUILD_SHARED_LIBS=ON", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DCMAKE_BUILD_TYPE=Release", ])}, {'name' : "flang-x86_64-knl-linux", @@ -1457,14 +2138,13 @@ "-DCMAKE_C_COMPILER=gcc", "-DCMAKE_CXX_COMPILER=g++", "-DLLVM_INSTALL_UTILS=ON", - "-DFLANG_BUILD_NEW_DRIVER=ON", "-DCMAKE_CXX_STANDARD=17", ])}, {'name' : 'ppc64le-flang-rhel-clang', 'tags' : ["flang", "ppc", "ppc64le"], 'collapseRequests' : False, - 'workernames' : ['ppc64le-flang-mlir-rhel-test'], + 'workernames' : ['ppc64le-flang-rhel-test'], 'builddir': 'ppc64le-flang-rhel-clang-build', 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( clean=True, @@ -1474,8 +2154,8 @@ '-DLLVM_TARGETS_TO_BUILD=PowerPC', '-DLLVM_INSTALL_UTILS=ON', '-DCMAKE_CXX_STANDARD=17', - '-DLLVM_ENABLE_PROJECTS=flang', - '-DLLVM_LIT_ARGS=-vj 256' + '-DLLVM_LIT_ARGS=-vj 256', + '-DLLVM_CCACHE_BUILD=ON' ], env={ 'CC': 'clang', @@ -1483,7 +2163,27 @@ 'LD': 'lld' })}, -# Builders responsible building Sphinix documentation. + {'name' : "flang-x86_64-windows", + 'tags' : ["flang"], + 'workernames' : ["minipc-ryzen-win"], + 'builddir': "flang-x86_64-windows", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + depends_on_projects=['llvm','mlir','clang','flang'], + checks=['check-flang'], + install_dir="flang.install", + extra_configure_args=[ + "-DCLANG_ENABLE_STATIC_ANALYZER=OFF", + "-DCLANG_ENABLE_ARCMT=OFF", + "-DCLANG_ENABLE_OBJC_REWRITER=OFF", + "-DLLVM_TARGETS_TO_BUILD=X86", + "-DLLVM_INSTALL_UTILS=ON", + "-DCMAKE_C_COMPILER=cl", + "-DCMAKE_CXX_COMPILER=cl", + "-DCMAKE_CXX_STANDARD=17", + '-DLLVM_PARALLEL_COMPILE_JOBS=4', + ])}, + +# Builders responsible building Sphinx documentation. {'name' : "llvm-sphinx-docs", 'tags' : ["llvm", "doc"], @@ -1495,7 +2195,7 @@ 'tags' : ["clang", "doc"], 'workernames' : ["gribozavr3"], 'builddir': "clang-sphinx-docs", - 'factory' : SphinxDocsBuilder.getSphinxDocsBuildFactory(clang_html=True)}, + 'factory' : SphinxDocsBuilder.getSphinxDocsBuildFactory(clang_html=True, clang_man=True)}, {'name' : "clang-tools-sphinx-docs", 'tags' : ["clang-tools", "doc"], @@ -1509,17 +2209,11 @@ 'builddir': "lld-sphinx-docs", 'factory' : SphinxDocsBuilder.getSphinxDocsBuildFactory(lld_html=True)}, - {'name' : "lldb-sphinx-docs", - 'tags' : ["lldb", "doc"], - 'workernames' : ["gribozavr3"], - 'builddir': "lldb-sphinx-docs", - 'factory' : SphinxDocsBuilder.getSphinxDocsBuildFactory(lldb_html=True)}, - {'name':"libunwind-sphinx-docs", 'tags' : ["libunwind", "doc"], 'workernames':["gribozavr3"], 'builddir':"libunwind-sphinx-docs", - 'factory': SphinxDocsBuilder.getSphinxDocsBuildFactory(libunwind_html=True)}, + 'factory': SphinxDocsBuilder.getSphinxRuntimesDocsBuildFactory(libunwind_html=True)}, # Sphinx doc Publisher {'name' : "publish-sphinx-docs", @@ -1528,6 +2222,34 @@ 'builddir': "publish-sphinx-docs", 'factory' : SphinxDocsBuilder.getLLVMDocsBuildFactory(clean=True)}, + {'name' : "publish-runtimes-sphinx-docs", + 'tags' : ["doc"], + 'workernames' : ["as-worker-4"], + 'builddir': "publish-runtimes-sphinx-docs", + 'factory' : SphinxDocsBuilder.getLLVMRuntimesDocsBuildFactory(clean=True)}, + + {'name' : "publish-lnt-sphinx-docs", + 'tags' : ["doc"], + 'workernames' : ["as-worker-4"], + 'builddir': "publish-lnt-sphinx-docs", + 'factory' : HtmlDocsBuilder.getHtmlDocsBuildFactory()}, + + {'name' : "publish-doxygen-docs", + 'tags' : ["doc"], + 'workernames' : ["as-worker-4"], #FIXME: Temporarily disabled failing doxygen build - as-builder-8. + 'builddir': "publish-doxygen-docs", + 'factory' : DoxygenDocsBuilder.getLLVMDocsBuildFactory( + # Doxygen builds the final result for really + # long time without any output. + # We have to have a long timeout here. + timeout=172800)}, + + {'name' : "polly-sphinx-docs", + 'tags' : ["llvm", "doc"], + 'workernames' : ["polly-x86_64-gce1"], + 'builddir': "polly-sphinx-docs", + 'factory': SphinxDocsBuilder.getSphinxDocsBuildFactory(polly_html=True)}, + # CUDA builders. {'name' : "clang-cuda-k80", @@ -1553,48 +2275,61 @@ 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( script="/buildbot/cuda-build", checkout_llvm_sources=False)}, +# HIP builders. + + {'name' : "clang-hip-vega20", + 'tags' : ["clang"], + 'workernames' : ["hip-vega20-0"], + 'builddir': "clang-hip-vega20", + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="hip-build.sh", + checkout_llvm_sources=False, + script_interpreter=None)}, +# VE builders. {'name' : "clang-ve-ninja", 'tags' : ["clang"], - 'workernames':["nec-arrproto41"], + 'workernames':["hpce-ve-main"], 'builddir':"clang-ve-ninja", - 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( - clean=True, - depends_on_projects=['llvm','clang','openmp'], - extra_configure_args=[ - "-DLLVM_TARGETS_TO_BUILD=X86", - "-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=VE", - ])}, - -# Latest stable fedora running on Red Hat internal OpenShift cluster (PSI). + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="ve-linux.py", + depends_on_projects=['llvm', 'clang', 'compiler-rt', 'libcxx'])}, + {'name' : "clang-ve-staging", + 'tags' : ["clang"], + 'workernames':["hpce-ve-staging"], + 'builddir':"clang-ve-staging", + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="ve-linux.py", + depends_on_projects=['llvm', 'clang', 'compiler-rt', 'libcxx'])}, - {'name' : 'x86_64-fedora-clang', - 'tags' : ['mlir'], + # Build the LLVM dylib .so with all backends and link tools to it + {'name' : 'llvm-x86_64-debian-dylib', + 'tags' : ['llvm'], 'collapseRequests': False, - 'workernames': ['fedora-llvm-x86_64'], - 'builddir': 'x86_64-fedora-clang', + 'workernames': ['gribozavr4'], + 'builddir': 'llvm-x86_64-debian-dylib', 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( clean=True, - depends_on_projects=['llvm', 'clang', 'clang-tools-extra', 'compiler-rt', 'lld', 'mlir'], + depends_on_projects=['llvm'], checks=['check-all'], extra_configure_args=[ '-DCMAKE_BUILD_TYPE=Release', - '-DCMAKE_C_COMPILER=/usr/bin/gcc', - '-DCMAKE_CXX_COMPILER=/usr/bin/g++', '-DLLVM_ENABLE_ASSERTIONS=On', - '-DLLVM_BUILD_EXAMPLES=On', + '-DLLVM_BUILD_EXAMPLES=Off', "-DLLVM_LIT_ARGS=-v --xunit-xml-output test-results.xml", - '-DLLVM_CCACHE_BUILD=On', - '-DLLVM_CCACHE_DIR=/ccache', - '-DLLVM_CCACHE_MAXSIZE=20G', - '-DLLVM_TARGETS_TO_BUILD=X86', + '-DLLVM_TARGETS_TO_BUILD=all', '-DCMAKE_EXPORT_COMPILE_COMMANDS=1', '-DLLVM_BUILD_LLVM_DYLIB=On', '-DLLVM_LINK_LLVM_DYLIB=On', - '-DCLANG_LINK_CLANG_DYLIB=On', '-DBUILD_SHARED_LIBS=Off', - '-DLLVM_ENABLE_LLD=ON', - ])}, + '-DLLVM_ENABLE_LLD=Off', + '-DLLVM_ENABLE_BINDINGS=Off', + '-DLLVM_CCACHE_BUILD=ON', + ], + env={ + 'PATH':'/home/llvmbb/bin/clang-latest/bin:/home/llvmbb/bin:/usr/local/bin:/usr/local/bin:/usr/bin:/bin', + 'CC': 'clang', 'CXX': 'clang++', + })}, {'name' : "clang-solaris11-amd64", 'tags' : ["clang"], @@ -1603,7 +2338,9 @@ 'factory' : ClangBuilder.getClangCMakeBuildFactory( jobs=8, clean=False, + timeout=1800, checkout_lld=False, + enable_runtimes=None, extra_cmake_args=['-DLLVM_ENABLE_ASSERTIONS=ON', '-DLLVM_TARGETS_TO_BUILD=X86', '-DLLVM_HOST_TRIPLE=amd64-pc-solaris2.11', @@ -1616,96 +2353,14 @@ 'factory' : ClangBuilder.getClangCMakeBuildFactory( jobs=8, clean=False, + timeout=1800, checkout_lld=False, + enable_runtimes=None, extra_cmake_args=['-DLLVM_ENABLE_ASSERTIONS=ON', '-DLLVM_TARGETS_TO_BUILD=Sparc', '-DLLVM_HOST_TRIPLE=sparcv9-sun-solaris2.11', '-DLLVM_PARALLEL_LINK_JOBS=4'])}, - {'name' : "clang-x86-ninja-win10", - 'tags' : ["clang"], - 'workernames' : ["windows10-vs2019"], - 'builddir': "clang-x86-ninja-win10", - 'factory' : ClangBuilder.getClangCMakeBuildFactory( - clean=True, - vs="autodetect", - vs_target_arch='x86', - testStage1=True, - useTwoStage=True, - stage1_config='Release', - stage2_config='Release', - # reduce scope of builds to get stable results - checkout_clang_tools_extra=False, - checkout_compiler_rt=False, - checkout_flang=False, - checkout_libcxx=False, - extra_cmake_args=['-DLLVM_ENABLE_ASSERTIONS=ON', - '-DLLVM_TARGETS_TO_BUILD=X86', - '-DCMAKE_C_COMPILER_LAUNCHER=sccache', - '-DCMAKE_CXX_COMPILER_LAUNCHER=sccache', - '-DLLVM_ENABLE_ZLIB=OFF', - '-DLLVM_LIT_TOOLS_DIR=C:\\Program Files\\GnuWin32\\usr\\bin', - ])}, - -# Flang builders. - - {'name' : "flang-aarch64-ubuntu-clang", - 'tags' : ['flang'], - 'workernames' : ["flang-aarch64-ubuntu-clang-build"], - 'builddir': "flang-aarch64-ubuntu-clang", - 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( - clean=True, - depends_on_projects=['llvm','mlir','clang','flang'], - checks=['check-flang'], - extra_configure_args=[ - "-DLLVM_TARGETS_TO_BUILD=AArch64", - "-DCMAKE_C_COMPILER=/usr/bin/clang-10", - "-DCMAKE_CXX_COMPILER=/usr/bin/clang++-10", - "-DLLVM_INSTALL_UTILS=ON", - "-DCMAKE_CXX_STANDARD=17", - "-DLLVM_ENABLE_WERROR=OFF", - "-DFLANG_ENABLE_WERROR=ON", - "-DBUILD_SHARED_LIBS=ON", - "-DLLVM_ENABLE_ASSERTIONS=ON", - "-DLLVM_ENABLE_LIBCXX=On", - "-DFLANG_BUILD_NEW_DRIVER=ON", - "-DCMAKE_BUILD_TYPE=Release", - ])}, - - {'name' : "flang-aarch64-ubuntu-gcc10", - 'tags' : ['flang'], - 'workernames' : ["flang-aarch64-ubuntu-gcc10-build"], - 'builddir': "flang-aarch64-ubuntu-gcc10", - 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( - clean=True, - depends_on_projects=['llvm','mlir','clang','flang'], - extra_configure_args=[ - "-DLLVM_TARGETS_TO_BUILD=AArch64", - "-DCMAKE_C_COMPILER=/usr/bin/gcc-10", - "-DCMAKE_CXX_COMPILER=/usr/bin/g++-10", - "-DLLVM_INSTALL_UTILS=ON", - "-DCMAKE_CXX_STANDARD=17", - "-DLLVM_ENABLE_WERROR=OFF", - "-DFLANG_ENABLE_WERROR=ON", - "-DBUILD_SHARED_LIBS=ON", - "-DLLVM_ENABLE_ASSERTIONS=ON", - "-DFLANG_BUILD_NEW_DRIVER=ON", - "-DCMAKE_BUILD_TYPE=Release", - ])}, - -# LLDB builders. - - {'name' : "lldb-x86_64-fedora", - 'tags' : ["lldb"], - 'workernames' : ["lldb-x86_64-fedora"], - 'builddir': "lldb-x86_64-fedora", - 'factory' : LLDBBuilder.getLLDBCMakeBuildFactory( - clean=True, - test=True, - extra_cmake_args=['-DLLVM_ENABLE_ASSERTIONS=True', - '-DLLVM_USE_LINKER=gold', - '-DLLVM_LIT_ARGS=-v'])}, - # Builders for ML-driven compiler optimizations. # Development mode build bot: tensorflow C APIs are present, and @@ -1713,7 +2368,7 @@ {'name' : "ml-opt-dev-x86-64", 'tags' : ['ml_opt'], 'collapseRequests': False, - 'workernames' : ["ml-opt-dev-x86-64-b1"], + 'workernames' : ["ml-opt-dev-x86-64-b1", "ml-opt-dev-x86-64-b2"], 'builddir': "ml-opt-dev-x86-64-b1", 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( clean=True, @@ -1722,7 +2377,8 @@ "-DCMAKE_BUILD_TYPE=Release", "-DLLVM_CCACHE_BUILD=ON", "-DLLVM_ENABLE_ASSERTIONS=ON", - "-DTENSORFLOW_API_PATH=/tmp/tensorflow", + "-DTENSORFLOW_C_LIB_PATH=/tmp/tensorflow", + "-C", "/tmp/tflitebuild/tflite.cmake", "-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON" ])}, @@ -1730,7 +2386,7 @@ {'name' : "ml-opt-devrel-x86-64", 'tags' : ["ml_opt"], 'collapseRequests': False, - 'workernames' : ["ml-opt-devrel-x86-64-b1"], + 'workernames' : ["ml-opt-devrel-x86-64-b1", "ml-opt-devrel-x86-64-b2"], 'builddir': "ml-opt-devrel-x86-64-b1", 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( clean=True, @@ -1739,7 +2395,8 @@ "-DCMAKE_BUILD_TYPE=Release", "-DLLVM_CCACHE_BUILD=ON", "-DLLVM_ENABLE_ASSERTIONS=ON", - "-DTENSORFLOW_API_PATH=/tmp/tensorflow", + "-DTENSORFLOW_C_LIB_PATH=/tmp/tensorflow", + "-C", "/tmp/tflitebuild/tflite.cmake", "-DTENSORFLOW_AOT_PATH=/var/lib/buildbot/.local/lib/python3.7/site-packages/tensorflow", "-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON" ])}, @@ -1751,7 +2408,7 @@ {'name' : "ml-opt-rel-x86-64", 'tags' : ["ml_opt"], 'collapseRequests': False, - 'workernames' : ["ml-opt-rel-x86-64-b1"], + 'workernames' : ["ml-opt-rel-x86-64-b1", "ml-opt-rel-x86-64-b2"], 'builddir': "ml-opt-rel-x86-64-b1", 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( clean=True, @@ -1774,13 +2431,382 @@ checks=["check-clangd"], targets=["clangd", "clangd-index-server", "clangd-indexer"], extra_configure_args=[ - "-DLLVM_CCACHE_BUILD=ON", + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", "-DLLVM_USE_SANITIZER=Thread", "-DCMAKE_BUILD_TYPE=Release", "-DCLANGD_ENABLE_REMOTE=ON", "-DLLVM_ENABLE_ASSERTIONS=ON", - "-DGRPC_INSTALL_PATH=/usr/local/lib/grpc" + "-DGRPC_INSTALL_PATH=/usr/local/lib/grpc", + "-DLLVM_OPTIMIZED_TABLEGEN=ON" ])}, + # Build in C++20 configuration. + {'name': "clang-debian-cpp20", + 'tags': ["clang", "c++20"], + 'workernames': ["clang-debian-cpp20"], + 'builddir': "clang-debian-cpp20", + 'factory': UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + clean=True, + depends_on_projects=["llvm", "clang", "clang-tools-extra"], + extra_configure_args=[ + "-DCMAKE_CXX_STANDARD=20", + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", + "-DCMAKE_BUILD_TYPE=Release", + "-DLLVM_ENABLE_ASSERTIONS=ON", + # FIXME: Re-enable after cleaning up LLVM. + # https://github.com/llvm/llvm-project/issues/60101 + "-DCMAKE_CXX_FLAGS=-Wno-deprecated-enum-enum-conversion -Wno-deprecated-declarations -Wno-deprecated-anon-enum-enum-conversion -Wno-ambiguous-reversed-operator", + ])}, + + # Target ARC from Synopsys + {'name': "arc-builder", + 'tags': ["clang", "lld"], + 'workernames' : ["arc-worker"], + 'builddir': "arc-folder", + 'factory': UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + depends_on_projects=["llvm", "clang", "lld"], + extra_configure_args=[ + "-DCMAKE_EXPORT_COMPILE_COMMANDS:BOOL=ON", + "-DLLVM_ENABLE_ASSERTIONS:BOOL=ON", + "-DLLVM_TOOL_CLANG_TOOLS_EXTRA_BUILD=0", + "-DLLVM_ENABLE_LIBPFM=OFF", + "-DLLVM_TARGETS_TO_BUILD=X86", + "-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=ARC", + ])}, + + + # BOLT builders managed by Meta + {'name' : 'bolt-x86_64-ubuntu-nfc', + 'tags' : ["bolt"], + 'collapseRequests': False, + 'workernames' : ['bolt-worker'], + 'builddir': "bolt-x86_64-ubuntu-nfc", + 'factory' : BOLTBuilder.getBOLTCmakeBuildFactory( + bolttests=True, + depends_on_projects=['bolt', 'llvm'], + extra_configure_args=[ + "-DLLVM_APPEND_VC_REV=OFF", + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", + "-DLLVM_ENABLE_PROJECTS=clang;lld;bolt", + "-DLLVM_TARGETS_TO_BUILD=X86;AArch64", + ], + is_nfc=True, + )}, + + {'name': "bolt-x86_64-ubuntu-clang", + 'tags': ["bolt"], + 'workernames':["bolt-worker"], + 'builddir': "bolt-x86_64-ubuntu-clang", + 'factory' : BOLTBuilder.getBOLTCmakeBuildFactory( + bolttests=False, + clean=True, + depends_on_projects=['bolt', 'clang', 'lld', 'llvm'], + caches=[ + 'clang/cmake/caches/BOLT.cmake', + 'clang/cmake/caches/BOLT-PGO.cmake', + ], + targets=['clang-bolt'], + checks=['stage2-clang-bolt'], + extra_configure_args=[ + "-DCMAKE_C_COMPILER=gcc", + "-DCMAKE_CXX_COMPILER=g++", + "-DLLVM_APPEND_VC_REV=OFF", + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", + "-DLLVM_ENABLE_LLD=ON", + "-DBOOTSTRAP_LLVM_ENABLE_LLD=ON", + "-DBOOTSTRAP_BOOTSTRAP_LLVM_ENABLE_LLD=ON", + "-DPGO_INSTRUMENT_LTO=Thin", + ], + )}, + + {'name': "bolt-x86_64-ubuntu-dylib", + 'tags': ["bolt"], + 'workernames':["bolt-worker"], + 'builddir': "bolt-x86_64-ubuntu-dylib", + 'factory' : BOLTBuilder.getBOLTCmakeBuildFactory( + bolttests=False, + depends_on_projects=['bolt', 'llvm'], + extra_configure_args=[ + "-DLLVM_APPEND_VC_REV=OFF", + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", + "-DLLVM_ENABLE_PROJECTS=bolt", + "-DLLVM_TARGETS_TO_BUILD=X86;AArch64", + "-DLLVM_LINK_LLVM_DYLIB=ON", + "-DLLVM_ENABLE_LLD=ON", + "-DBOLT_CLANG_EXE=/usr/bin/clang", + "-DBOLT_LLD_EXE=/usr/bin/ld.lld", + ], + )}, + + {'name': "bolt-x86_64-ubuntu-shared", + 'tags': ["bolt"], + 'workernames':["bolt-worker"], + 'builddir': "bolt-x86_64-ubuntu-shared", + 'factory' : BOLTBuilder.getBOLTCmakeBuildFactory( + bolttests=False, + depends_on_projects=['bolt', 'llvm'], + extra_configure_args=[ + "-DLLVM_APPEND_VC_REV=OFF", + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", + "-DLLVM_ENABLE_PROJECTS=bolt", + "-DLLVM_TARGETS_TO_BUILD=X86;AArch64", + "-DBUILD_SHARED_LIBS=ON", + "-DLLVM_ENABLE_LLD=ON", + "-DBOLT_CLANG_EXE=/usr/bin/clang", + "-DBOLT_LLD_EXE=/usr/bin/ld.lld", + ], + )}, + + {'name': "bolt-aarch64-ubuntu-clang-shared", + 'tags': ["bolt"], + 'workernames':["bolt-worker-aarch64"], + 'builddir': "bolt-aarch64-ubuntu-clang-shared", + 'factory' : BOLTBuilder.getBOLTCmakeBuildFactory( + bolttests=True, + depends_on_projects=['bolt', 'llvm'], + extra_configure_args=[ + "-DCMAKE_C_COMPILER=clang", + "-DCMAKE_CXX_COMPILER=clang++", + "-DLLVM_APPEND_VC_REV=OFF", + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", + "-DLLVM_ENABLE_PROJECTS=bolt", + "-DLLVM_TARGETS_TO_BUILD=X86;AArch64", + "-DBUILD_SHARED_LIBS=ON", + "-DLLVM_USE_LINKER=mold", + "-DBOLT_CLANG_EXE=/usr/bin/clang", + "-DBOLT_LLD_EXE=/usr/bin/ld.lld", + ], + )}, + + # AMD ROCm support. + {'name' : 'mlir-rocm-mi200', + 'tags' : ["mlir"], + 'workernames' : ['mi200-buildbot'], + 'builddir': 'mlir-rocm-mi200', + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + clean=True, + depends_on_projects=['llvm', 'mlir'], + targets = ['check-mlir-build-only'], + checks = ['check-mlir'], + extra_configure_args= mlir_default_cmake_options + [ + '-DLLVM_CCACHE_BUILD=ON', + '-DLLVM_ENABLE_ASSERTIONS=ON', + '-DLLVM_ENABLE_LLD=ON', + '-DMLIR_ENABLE_ROCM_RUNNER=ON', + '-DMLIR_ENABLE_ROCM_CONVERSIONS=ON', + '-DMLIR_INCLUDE_INTEGRATION_TESTS=ON', + ], + env={ + 'CC': 'clang', + 'CXX': 'clang++', + 'LD': 'lld', + })}, + + # Standalone builder + {'name' : "standalone-build-x86_64", + 'tags' : ["clang"], + 'workernames':["standalone-build-x86_64"], + 'builddir':"standalone-build-x86_64", + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="standalone-build.sh", + checkout_llvm_sources=False, + script_interpreter=None)}, + + ## CSKY check-all + test-suite in soft-float + {'name' : "clang-csky-soft", + 'tags' : ["clang"], + 'workernames' : ["thead-clang-csky"], + 'builddir':"clang-csky-softfp", + 'factory' : ClangBuilder.getClangCMakeBuildFactory( + clean=False, + checkout_clang_tools_extra=False, + checkout_compiler_rt=False, + checkout_lld=False, + testStage1=True, + useTwoStage=False, + stage1_config='Release', + runTestSuite=True, + testsuite_flags=[ + '--cflags', '-mcpu=c860 -latomic -DSMALL_PROBLEM_SIZE', + '--cppflags', '-mcpu=c860 -latomic -DSMALL_PROBLEM_SIZE', + '--run-under=/mnt/qemu/bin/qemu-cskyv2 -cpu c860 -csky-extend denormal=on -L /mnt/gcc-csky/csky-linux-gnuabiv2/libc/ck860 -E LD_LIBRARY_PATH=/mnt/gcc-csky/csky-linux-gnuabiv2/lib/ck860', + '--cmake-define=SMALL_PROBLEM_SIZE=On', + '--cmake-define=TEST_SUITE_USER_MODE_EMULATION=True', + '--threads=32', '--build-threads=32'], + extra_cmake_args=[ + "-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD='CSKY'", + "-DLLVM_DEFAULT_TARGET_TRIPLE='csky-unknown-linux'", + "-DGCC_INSTALL_PREFIX=/mnt/gcc-csky/"])}, + + ## CSKY check-all + test-suite in hard-float + {'name' : "clang-csky-hardfp", + 'tags' : ["clang"], + 'workernames' : ["thead-clang-csky"], + 'builddir':"clang-csky-hardfp", + 'factory' : ClangBuilder.getClangCMakeBuildFactory( + clean=False, + checkout_clang_tools_extra=False, + checkout_compiler_rt=False, + checkout_lld=False, + testStage1=True, + useTwoStage=False, + stage1_config='Release', + runTestSuite=True, + testsuite_flags=[ + '--cflags', '-mcpu=c860 -latomic -mhard-float -DSMALL_PROBLEM_SIZE', + '--cppflags', '-mcpu=c860 -latomic -mhard-float -DSMALL_PROBLEM_SIZE', + '--run-under=/mnt/qemu/bin/qemu-cskyv2 -cpu c860 -csky-extend denormal=on -L /mnt/gcc-csky/csky-linux-gnuabiv2/libc/ck860/hard-fp -E LD_LIBRARY_PATH=/mnt/gcc-csky/csky-linux-gnuabiv2/lib/ck860/hard-fp', + '--cmake-define=SMALL_PROBLEM_SIZE=On', + '--cmake-define=TEST_SUITE_USER_MODE_EMULATION=True', + '--threads=32', '--build-threads=32'], + extra_cmake_args=[ + "-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD='CSKY'", + "-DLLVM_DEFAULT_TARGET_TRIPLE='csky-unknown-linux'", + "-DGCC_INSTALL_PREFIX=/mnt/gcc-csky/"])}, + + # NVPTX builders + {'name' : "llvm-nvptx-nvidia-ubuntu", + 'tags' : ["llvm", "nvptx"], + 'collapseRequests': False, + 'workernames' : ["as-builder-7"], + 'builddir': "llvm-nvptx-nvidia-ubuntu", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + depends_on_projects=["llvm"], + clean=True, + checks=["check-llvm"], + extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=X86;NVPTX", + "-DLLVM_DEFAULT_TARGET_TRIPLE=nvptx-nvidia-cuda", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_LIT_ARGS=-vv --threads=32", + "-DLLVM_USE_LINKER=gold", + "-DBUILD_SHARED_LIBS=ON", + "-DLLVM_OPTIMIZED_TABLEGEN=ON"])}, + + {'name' : "llvm-nvptx64-nvidia-ubuntu", + 'tags' : ["llvm", "nvptx"], + 'collapseRequests': False, + 'workernames' : ["as-builder-7"], + 'builddir': "llvm-nvptx64-nvidia-ubuntu", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + depends_on_projects=["llvm"], + clean=True, + checks=["check-llvm"], + extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=X86;NVPTX", + "-DLLVM_DEFAULT_TARGET_TRIPLE=nvptx64-nvidia-cuda", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_LIT_ARGS=-vv --threads=32", + "-DLLVM_USE_LINKER=gold", + "-DBUILD_SHARED_LIBS=ON", + "-DLLVM_OPTIMIZED_TABLEGEN=ON"])}, + {'name' : "llvm-nvptx-nvidia-win", + 'tags' : ["llvm", "nvptx"], + 'workernames' : ["as-builder-8"], + 'builddir': "llvm-nvptx-nvidia-win", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaWithMSVCBuildFactory( + vs="autodetect", + depends_on_projects=["llvm"], + clean=True, + checks=["check-llvm"], + extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=X86;NVPTX", + "-DLLVM_DEFAULT_TARGET_TRIPLE=nvptx-nvidia-cuda", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_LIT_ARGS=-vv --threads=32", + "-DLLVM_OPTIMIZED_TABLEGEN=ON"])}, + + {'name' : "llvm-nvptx64-nvidia-win", + 'tags' : ["llvm", "nvptx"], + 'workernames' : ["as-builder-8"], + 'builddir': "llvm-nvptx64-nvidia-win", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaWithMSVCBuildFactory( + vs="autodetect", + depends_on_projects=["llvm"], + clean=True, + checks=["check-llvm"], + extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=X86;NVPTX", + "-DLLVM_DEFAULT_TARGET_TRIPLE=nvptx64-nvidia-cuda", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_LIT_ARGS=-vv --threads=32", + "-DLLVM_OPTIMIZED_TABLEGEN=ON"])}, + + ## RISC-V RV64GC check-all running under qemu-user. + {'name' : "clang-rv64gc-qemu-user-single-stage", + 'tags' : ["llvm", "clang"], + 'workernames' : ["rv64gc-qemu-user"], + 'builddir': "clang-rv64gc", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + depends_on_projects=["llvm", "clang", "clang-tools-extra", "lld"], + checks=['check-all'], + extra_configure_args=[ + "-DCMAKE_BUILD_TYPE=Release", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", + "-DLLVM_TARGETS_TO_BUILD=all"], + env={ + 'CC':'clang', + 'CXX': 'clang++', + })}, + + # GoogleCloud Buildguards (investigation/stat) + {'name' : "llvm-clang-buildguard5-ubuntu", + 'tags' : ["llvm", "clang", "lld", "lldb", "clang-tools-extra"], + 'collapseRequests' : False, + 'workernames' : ["gc-builder-5"], + 'builddir': "buildguard5", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + depends_on_projects=["llvm", "clang", "clang-tools-extra", "lld", "lldb", "compiler-rt", "libunwind", "libcxxabi", "libcxx"], + enable_runtimes="auto", # get runtimes from depends_on_projects. + checks=['check-all'], + clean=True, + extra_configure_args=[ + "-DLLVM_CCACHE_BUILD=ON", + "-DLLVM_ENABLE_WERROR=OFF", + "-DCMAKE_BUILD_TYPE=Release", + "-DLLVM_USE_SPLIT_DWARF=ON", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=ON", + "-DCLANG_DEFAULT_LINKER=lld", + "-DLLVM_USE_LINKER=gold", + "-DLLVM_OPTIMIZED_TABLEGEN=ON", + "-DCMAKE_CXX_FLAGS=-D__OPTIMIZE__ -Wno-misleading-indentation", + "-DBUILD_SHARED_LIBS=ON", + "-DLLVM_LIT_ARGS=-v -vv"], + env={ + 'CCACHE_DIR' : WithProperties("%(builddir)s/ccache-db"), + })}, + {'name' : "llvm-clang-buildguard6-win", + 'tags' : ["llvm", "clang", "lld", "lldb", "clang-tools-extra"], + 'collapseRequests' : False, + 'workernames' : ["gc-builder-6-win", "gc-builder-7-win"], + 'builddir': "buildguard6", + 'factory' : XToolchainBuilder.getCmakeWithMSVCBuildFactory( + vs="autodetect", + checks=['check-all'], + clean=True, + extra_configure_args=[ + "-DLLVM_CCACHE_BUILD=ON", + "-DLLVM_ENABLE_WERROR=OFF", + "-DCMAKE_BUILD_TYPE=Release", + "-DLLVM_USE_SPLIT_DWARF=ON", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=ON", + "-DCLANG_DEFAULT_LINKER=lld", + "-DLLVM_OPTIMIZED_TABLEGEN=ON", + "-DCMAKE_CXX_FLAGS=-D__OPTIMIZE__", + "-DLLVM_LIT_ARGS=-v -vv"], + env={ + 'CCACHE_DIR' : WithProperties("%(builddir)s/ccache-db"), + })}, ] diff --git a/buildbot/osuosl/master/config/release_builders.py b/buildbot/osuosl/master/config/release_builders.py new file mode 100644 --- /dev/null +++ b/buildbot/osuosl/master/config/release_builders.py @@ -0,0 +1,276 @@ +from importlib import reload + +from buildbot.process.properties import WithProperties + +from zorg.buildbot.builders import ClangBuilder +from zorg.buildbot.builders import FlangBuilder +from zorg.buildbot.builders import PollyBuilder +from zorg.buildbot.builders import LLDBBuilder +from zorg.buildbot.builders import SanitizerBuilder +from zorg.buildbot.builders import OpenMPBuilder +from zorg.buildbot.builders import SphinxDocsBuilder +from zorg.buildbot.builders import ABITestsuitBuilder +from zorg.buildbot.builders import ClangLTOBuilder +from zorg.buildbot.builders import UnifiedTreeBuilder +from zorg.buildbot.builders import AOSPBuilder +from zorg.buildbot.builders import AnnotatedBuilder +from zorg.buildbot.builders import LLDPerformanceTestsuite +from zorg.buildbot.builders import FuchsiaBuilder +from zorg.buildbot.builders import XToolchainBuilder + + +# Release builders. + +all = [ + +# Clang builders. + + {'name' : "llvm-clang-x86_64-win-release", + 'tags' : ["clang"], + 'workernames' : ["as-builder-3"], + 'builddir': "llvm-clang-x86_64-win-rel", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaWithMSVCBuildFactory( + vs="autodetect", + depends_on_projects=['llvm', 'clang'], + clean=True, + checks=[ + "check-llvm-unit", + "check-clang-unit"], + extra_configure_args=[ + "-DLLVM_ENABLE_WERROR=OFF", + "-DLLVM_TARGETS_TO_BUILD=ARM", + "-DLLVM_DEFAULT_TARGET_TRIPLE=armv7-unknown-linux-eabihf", + "-DLLVM_ENABLE_ASSERTIONS=OFF", + "-DLLVM_OPTIMIZED_TABLEGEN=OFF", + "-DLLVM_LIT_ARGS=-v --threads=32"])}, + +# Expensive checks builders. + + + {'name' : "llvm-clang-x86_64-expensive-checks-ubuntu-release", + 'tags' : ["llvm", "expensive-checks"], + 'workernames' : ["as-builder-4"], + 'builddir': "llvm-clang-x86_64-expensive-checks-ubuntu-rel", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + depends_on_projects=["llvm", "lld"], + clean=True, + extra_configure_args=[ + "-DLLVM_ENABLE_EXPENSIVE_CHECKS=ON", + "-DLLVM_ENABLE_WERROR=OFF", + "-DLLVM_USE_SPLIT_DWARF=ON", + "-DLLVM_USE_LINKER=gold", + "-DCMAKE_BUILD_TYPE=Debug", + "-DCMAKE_CXX_FLAGS=-U_GLIBCXX_DEBUG -Wno-misleading-indentation", + "-DLLVM_LIT_ARGS=-vv -j32"])}, + + {'name' : "llvm-clang-x86_64-expensive-checks-win-release", + 'tags' : ["llvm", "expensive-checks"], + 'workernames' : ["as-worker-93"], + 'builddir': "llvm-clang-x86_64-expensive-checks-win-rel", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaWithMSVCBuildFactory( + vs="autodetect", + depends_on_projects=["llvm", "lld"], + clean=True, + extra_configure_args=[ + "-DLLVM_ENABLE_EXPENSIVE_CHECKS=ON", + "-DLLVM_ENABLE_WERROR=OFF", + "-DCMAKE_BUILD_TYPE=Debug"])}, + + {'name' : "llvm-clang-x86_64-expensive-checks-debian-release", + 'tags' : ["llvm", "expensive-checks"], + 'workernames' : ["gribozavr4"], + 'builddir': "llvm-clang-x86_64-expensive-checks-debian-rel", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + depends_on_projects=["llvm", "lld"], + clean=True, + extra_configure_args=[ + "-DLLVM_CCACHE_BUILD=ON", + "-DLLVM_ENABLE_EXPENSIVE_CHECKS=ON", + "-DLLVM_ENABLE_WERROR=OFF", + "-DCMAKE_BUILD_TYPE=Release", + "-DCMAKE_CXX_FLAGS=-U_GLIBCXX_DEBUG", + "-DLLVM_LIT_ARGS=-v -vv -j96"], + env={ + 'PATH':'/home/llvmbb/bin/clang-latest/bin:/home/llvmbb/bin:/usr/local/bin:/usr/local/bin:/usr/bin:/bin', + 'CC': 'clang', 'CXX': 'clang++', + })}, + +# Cross builders. + + {'name' : "llvm-clang-win-x-armv7l-release", + 'tags' : ["clang", "llvm", "compiler-rt", "cross", "armv7"], + 'workernames' : ["as-builder-5"], + 'builddir': "x-armv7l-rel", + 'factory' : XToolchainBuilder.getCmakeWithMSVCBuildFactory( + vs="autodetect", + clean=True, + checks=[ + "check-llvm", + "check-clang", + "check-lld", + "check-compiler-rt-armv7-unknown-linux-gnueabihf" + ], + checks_on_target = [ + ("libunwind", + ["python", "bin/llvm-lit.py", + "-v", "-vv", "--threads=32", + "runtimes/runtimes-armv7-unknown-linux-gnueabihf-bins/libunwind/test"]), + ("libc++abi", + ["python", "bin/llvm-lit.py", + "-v", "-vv", "--threads=32", + "runtimes/runtimes-armv7-unknown-linux-gnueabihf-bins/libcxxabi/test"]), + ("libc++", + ['python', 'bin/llvm-lit.py', + '-v', '-vv', '--threads=32', + 'runtimes/runtimes-armv7-unknown-linux-gnueabihf-bins/libcxx/test', + ]) + ], + extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=ARM", + "-DTOOLCHAIN_TARGET_TRIPLE=armv7-unknown-linux-gnueabihf", + "-DDEFAULT_SYSROOT=C:/buildbot/.arm-ubuntu", + "-DZLIB_ROOT=C:/buildbot/.zlib-win32", + "-DLLVM_LIT_ARGS=-v -vv --threads=32", + WithProperties("%(remote_test_host:+-DREMOTE_TEST_HOST=)s%(remote_test_host:-)s"), + WithProperties("%(remote_test_user:+-DREMOTE_TEST_USER=)s%(remote_test_user:-)s"), + ], + cmake_cache="../llvm-project/clang/cmake/caches/CrossWinToARMLinux.cmake")}, + + {'name' : "llvm-clang-win-x-aarch64-release", + 'tags' : ["clang", "llvm", "compiler-rt", "cross", "aarch64"], + 'workernames' : ["as-builder-6"], + 'builddir': "x-aarch64-rel", + 'factory' : XToolchainBuilder.getCmakeWithMSVCBuildFactory( + vs="autodetect", + clean=True, + checks=[ + "check-llvm", + "check-clang", + "check-lld", + "check-compiler-rt-aarch64-unknown-linux-gnu" + ], + checks_on_target = [ + ("libunwind", + ["python", "bin/llvm-lit.py", + "-v", "-vv", "--threads=32", + "runtimes/runtimes-aarch64-unknown-linux-gnu-bins/libunwind/test"]), + ("libc++abi", + ["python", "bin/llvm-lit.py", + "-v", "-vv", "--threads=32", + "runtimes/runtimes-aarch64-unknown-linux-gnu-bins/libcxxabi/test"]), + ("libc++", + ['python', 'bin/llvm-lit.py', + '-v', '-vv', '--threads=32', + 'runtimes/runtimes-aarch64-unknown-linux-gnu-bins/libcxx/test', + ]) + ], + extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=AArch64", + "-DTOOLCHAIN_TARGET_TRIPLE=aarch64-unknown-linux-gnu", + "-DDEFAULT_SYSROOT=C:/buildbot/.aarch64-ubuntu", + "-DZLIB_ROOT=C:/buildbot/.zlib-win32", + "-DLLVM_LIT_ARGS=-v -vv --threads=32", + WithProperties("%(remote_test_host:+-DREMOTE_TEST_HOST=)s%(remote_test_host:-)s"), + WithProperties("%(remote_test_user:+-DREMOTE_TEST_USER=)s%(remote_test_user:-)s"), + ], + cmake_cache="../llvm-project/clang/cmake/caches/CrossWinToARMLinux.cmake")}, + +# LLD builders. + + {'name' : "lld-x86_64-win-release", + 'tags' : ["lld"], + 'workernames' : ["as-worker-93"], + 'builddir': "lld-x86_64-win-rel", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaWithMSVCBuildFactory( + depends_on_projects=['llvm', 'lld'], + vs="autodetect", + extra_configure_args = [ + '-DLLVM_ENABLE_WERROR=OFF'])}, + + {'name' : "lld-x86_64-ubuntu-release", + 'tags' : ["lld"], + 'workernames' : ["as-builder-4"], + 'builddir' : "lld-x86_64-ubuntu-rel", + 'factory': UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + clean=True, + extra_configure_args=[ + '-DLLVM_ENABLE_WERROR=OFF'], + depends_on_projects=['llvm', 'lld'])}, + +# LTO and ThinLTO builders. + + {'name' : "clang-with-thin-lto-ubuntu-release", + 'tags' : ["clang", "lld", "LTO"], + 'workernames' : ["as-worker-92"], + 'builddir': "clang-with-thin-lto-ubuntu-rel", + 'factory' : ClangLTOBuilder.getClangWithLTOBuildFactory(jobs=72, lto='thin')}, + + {'name' : "clang-with-lto-ubuntu-release", + 'tags' : ["clang", "lld", "LTO"], + 'workernames' : ["as-worker-91"], + 'builddir': "clang-with-lto-ubuntu-rel", + 'factory' : ClangLTOBuilder.getClangWithLTOBuildFactory( + jobs=72, + extra_configure_args_lto_stage=[ + '-DLLVM_PARALLEL_LINK_JOBS=14', + ])}, + +# OpenMP builders. + + {'name' : "openmp-clang-x86_64-linux-debian-release", + 'tags' : ["openmp"], + 'workernames' : ["gribozavr4"], + 'builddir': "openmp-clang-x86_64-linux-debian-rel", + 'factory' : OpenMPBuilder.getOpenMPCMakeBuildFactory( + extraCmakeArgs=[ + '-DLLVM_CCACHE_BUILD=ON', + ], + env={ + 'PATH':'/home/llvmbb/bin/clang-latest/bin:/home/llvmbb/bin:/usr/local/bin:/usr/local/bin:/usr/bin:/bin', + 'CC': 'clang', 'CXX': 'clang++', + })}, + +# Sony builders. + + {'name' : "llvm-clang-x86_64-sie-win-release", + 'tags' : ["llvm", "clang", "clang-tools-extra", "lld", "cross-project-tests"], + 'workernames' : ["sie-win-worker"], + 'builddir': "x86_64-sie-win-rel", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaWithMSVCBuildFactory( + vs="autodetect", + target_arch='x64', + depends_on_projects=['llvm','clang','clang-tools-extra','lld','cross-project-tests'], + clean=True, + extra_configure_args=[ + "-DCMAKE_BUILD_TYPE=Release", + "-DCLANG_ENABLE_ARCMT=OFF", + "-DCLANG_ENABLE_CLANGD=OFF", + "-DLLVM_CCACHE_BUILD=ON", + "-DLLVM_DEFAULT_TARGET_TRIPLE=x86_64-sie-ps5", + "-DLLVM_INCLUDE_EXAMPLES=OFF", + "-DLLVM_TARGETS_TO_BUILD=X86", + "-DLLVM_VERSION_SUFFIX=", + "-DLLVM_BUILD_RUNTIME=OFF", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_LIT_ARGS=--verbose"])}, + + {'name': "llvm-clang-x86_64-gcc-ubuntu-release", + 'tags' : ["llvm", "clang", "clang-tools-extra", "compiler-rt", "lld", "cross-project-tests"], + 'workernames': ["doug-worker-2a"], + 'builddir': "x86_64-gcc-rel", + 'factory': UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + depends_on_projects=['llvm','clang','clang-tools-extra','compiler-rt','lld','cross-project-tests'], + extra_configure_args=[ + "-DCMAKE_C_COMPILER=gcc", + "-DCMAKE_CXX_COMPILER=g++", + "-DCMAKE_BUILD_TYPE=Release", + "-DCLANG_ENABLE_CLANGD=OFF", + "-DLLVM_BUILD_RUNTIME=ON", + "-DLLVM_BUILD_TESTS=ON", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_INCLUDE_EXAMPLES=OFF", + "-DLLVM_LIT_ARGS=--verbose -j48", + "-DLLVM_PARALLEL_LINK_JOBS=16", + "-DLLVM_USE_LINKER=gold"])}, + +] diff --git a/buildbot/osuosl/master/config/schedulers.py b/buildbot/osuosl/master/config/schedulers.py --- a/buildbot/osuosl/master/config/schedulers.py +++ b/buildbot/osuosl/master/config/schedulers.py @@ -19,7 +19,7 @@ # Since we have many parametric builders, we dynamically build the minimum set # of schedulers, which covers all actually used combinations of dependencies. -def getSingleBranchSchedulers( +def getMainBranchSchedulers( builders, explicitly_set_schedulers = None, **kwargs): @@ -43,9 +43,10 @@ builder for builder in builders if builder.name not in builders_with_explicit_schedulers if getattr(builder.factory, 'depends_on_projects', None) + if 'release' not in getattr(builder, 'tags', []) ] - filter_branch = 'master' + filter_branch = 'main' treeStableTimer = kwargs.get('treeStableTimer', None) automatic_schedulers = [] @@ -60,7 +61,7 @@ ]) for projects in set_of_dependencies: - sch_builders = [ + main_builders = [ b.name for b in builders_with_automatic_schedulers if frozenset(getattr(b.factory, 'depends_on_projects')) == projects @@ -71,9 +72,9 @@ automatic_schedulers.append( schedulers.SingleBranchScheduler( name=automatic_scheduler_name, - treeStableTimer=kwargs.get('treeStableTimer', None), - reason="Merge to github %s branch" % filter_branch, - builderNames=sch_builders, + treeStableTimer=treeStableTimer, + reason="Merge to github {} branch".format(filter_branch), + builderNames=main_builders, change_filter=util.ChangeFilter( project_fn= \ lambda c, projects_of_interest=frozenset(projects): @@ -83,18 +84,90 @@ ) log.msg( - "Generated SingleBranchScheduler: { name='%s'" % automatic_scheduler_name, - ", builderNames=", sch_builders, - ", change_filter=", projects, " (branch: %s)" % filter_branch, - ", treeStableTimer=%s" % treeStableTimer, + "Generated SingleBranchScheduler: {{ name='{}'".format(automatic_scheduler_name), + ", builderNames=", main_builders, + ", change_filter=", projects, " (branch: {})".format(filter_branch), + ", treeStableTimer={}".format(treeStableTimer), "}") return automatic_schedulers -# TODO: Move these settings to the configuration file. -_repourl = "https://github.com/llvm/llvm-project" -_branch = "master" +def getReleaseBranchSchedulers( + builders, + explicitly_set_schedulers = None, + **kwargs): + """ + I'm taking over all of not yet assigned builders with the + declared source code dependencies and the 'release' tag, + and automatically generate a minimum set of SingleBranchSchedulers + to handle all the declared source code dependency combinations. + """ + + builders_with_explicit_schedulers = set() + if explicitly_set_schedulers: + # TODO: Make a list of builder names with already set schedulers. + # builders_with_explicit_schedulers.add(builder) + pass + + # For the builders created with LLVMBuildFactory or similar, + # we always use automatic schedulers, + # unless schedulers already explicitly set. + builders_with_automatic_schedulers = [ + builder for builder in builders + if builder.name not in builders_with_explicit_schedulers + if getattr(builder.factory, 'depends_on_projects', None) + if 'release' in getattr(builder, 'tags', []) + ] + + treeStableTimer = kwargs.get('treeStableTimer', None) + + automatic_schedulers = [] + + # Do we have any to take care of? + if builders_with_automatic_schedulers: + # Let's reconsile first to get a unique set of dependencies. + # We need a set of unique sets of dependent projects. + set_of_dependencies = set([ + frozenset(getattr(b.factory, 'depends_on_projects')) + for b in builders_with_automatic_schedulers + ]) + + for projects in set_of_dependencies: + release_builders = [ + b.name + for b in builders_with_automatic_schedulers + if frozenset(getattr(b.factory, 'depends_on_projects')) == projects + ] + + automatic_scheduler_name = "release:" + ",".join(sorted(projects)) + + automatic_schedulers.append( + schedulers.SingleBranchScheduler( + name=automatic_scheduler_name, + treeStableTimer=treeStableTimer, + reason="Merge to github release branch", + builderNames=release_builders, + change_filter=util.ChangeFilter( + project_fn= \ + lambda c, projects_of_interest=frozenset(projects): + isProjectOfInterest(c, projects_of_interest), + branch_fn= \ + lambda branch: branch.startswith('release/')) + ) + ) + + log.msg( + "Generated release SingleBranchScheduler: {{ name='{}'".format(automatic_scheduler_name), + ", builderNames=", release_builders, + ", change_filter=", projects, " (branch: {release/*})", + ", treeStableTimer={}".format(treeStableTimer), + "}") + return automatic_schedulers def getForceSchedulers(builders): + # TODO: Move these settings to the configuration file. + _repourl = "https://github.com/llvm/llvm-project" + _branch = "main" + # Walk over all builders and collect their names. scheduler_builders = [ builder.name for builder in builders @@ -120,8 +193,10 @@ codebases = [ util.CodebaseParameter( codebase = "", - branch = util.FixedParameter( + branch = util.StringParameter( name = "branch", + label = "branch:", + size = 64, default = _branch ), revision = util.StringParameter( @@ -157,22 +232,21 @@ # TODO: Abstract this kind of scheduler better. def getLntSchedulers(): - project = "lnt" - lnt_builders = [ + _project = "lnt" + _branch = 'master' + _repourl = "https://github.com/llvm/llvm-lnt" + _lnt_builders = [ "publish-lnt-sphinx-docs", ] return [ - schedulers.SingleBranchScheduler( + schedulers.Nightly( name="lnt-scheduler", - treeStableTimer=kwargs.get('treeStableTimer', None), - reason="Merge to LNT github {} branch".format(filter_branch), - builderNames=lnt_builders, - change_filter=util.ChangeFilter( - project_fn=project, - branch=filter_branch)), + builderNames=_lnt_builders, + reason='Periodic LNT build', + hour=1), schedulers.ForceScheduler( - name = "force-build-scheduler", + name = "lnt-force-build-scheduler", label = "Force Build", buttonName = "Force Build", reason = util.ChoiceStringParameter( @@ -186,7 +260,7 @@ ], default = "Build a particular revision" ), - builderNames = lnt_builders, + builderNames = _lnt_builders, codebases = [ util.CodebaseParameter( codebase = "", @@ -206,7 +280,7 @@ ), project = util.FixedParameter( name = "project", - default = project + default = _project ) ) ], diff --git a/buildbot/osuosl/master/config/status.py b/buildbot/osuosl/master/config/status.py --- a/buildbot/osuosl/master/config/status.py +++ b/buildbot/osuosl/master/config/status.py @@ -9,18 +9,23 @@ all = [ - # Note: reporters.GitHubStatusPush requires txrequests package to allow - # interaction with GitHub REST API. + # Report github status for all the release builders, + # i.e. those with the "release" tag. reporters.GitHubStatusPush( str(config.options.get('GitHub Status', 'token')), context = Interpolate("%(prop:buildername)s"), verbose = True, # TODO: Turn off the verbosity once this is working reliably. builders = [ - "llvm-clang-x86_64-expensive-checks-ubuntu", - "llvm-clang-x86_64-win-fast", - "clang-x86_64-debian-fast", - "llvm-clang-x86_64-expensive-checks-debian", - ]), + "llvm-clang-x86_64-expensive-checks-ubuntu", + "llvm-clang-x86_64-win-fast", + "clang-x86_64-debian-fast", + "llvm-clang-x86_64-expensive-checks-debian", + "llvm-clang-x86_64-sie-ubuntu-fast", + ] + [ + b.get('name') for b in config.release_builders.all + if 'release' in b.get('tags', []) + ] + ), reporters.IRC( useColors = False, @@ -42,17 +47,12 @@ messageFormatter = LLVMInformativeMailNotifier, # TODO: For debug purposes only. Remove later. dumpMailsToLog = True, + builders = [ + b.get('name') for b in config.builders.all + if 'silent' not in b.get('tags', []) + ] ), - # In addition to that the following notifiers are defined for special - # cases. - reporters.MailNotifier( - fromaddr = "llvm.buildmaster@lab.llvm.org", - sendToInterestedUsers = False, - extraRecipients = ["leandro.nunes@arm.com"], - subject = "Build %(builder)s Failure", - mode = "failing", - builders = ["clang-aarch64-linux-build-cache", "clang-armv7-linux-build-cache"]), reporters.MailNotifier( fromaddr = "llvm.buildmaster@lab.llvm.org", sendToInterestedUsers = False, @@ -62,11 +62,11 @@ builders = ["clang-x86_64-debian-fast"]), reporters.MailNotifier( fromaddr = "llvm.buildmaster@lab.llvm.org", - sendToInterestedUsers = False, - extraRecipients = ["mstester.llvm@gmail.com"], + sendToInterestedUsers = False, + extraRecipients = ["tbaeder@redhat.com", "tstellar@redhat.com"], subject = "Build %(builder)s Failure", mode = "failing", - builders = ["clang-x64-ninja-win7"]), + builders = ["llvm-x86_64-debian-dylib"]), reporters.MailNotifier( fromaddr = "llvm.buildmaster@lab.llvm.org", sendToInterestedUsers = False, @@ -81,7 +81,7 @@ subject = "Build %(builder)s Failure", mode = "failing", builders = ["clang-s390x-linux", "clang-s390x-linux-multistage", - "clang-s390x-linux-lnt"]), + "clang-s390x-linux-lnt", "mlir-s390x-linux"]), reporters.MailNotifier( fromaddr = "llvm.buildmaster@lab.llvm.org", sendToInterestedUsers = False, @@ -90,13 +90,6 @@ subject = "Build %(builder)s Failure", mode = "failing", builders = ["clang-x86_64-linux-abi-test"]), - reporters.MailNotifier( - fromaddr = "llvm.buildmaster@lab.llvm.org", - sendToInterestedUsers = False, - extraRecipients = ["me@dylanmckay.io"], - subject = "Build %(builder)s Failure", - mode = "failing", - builders = ["llvm-avr-linux"]), reporters.MailNotifier( fromaddr = "llvm.buildmaster@lab.llvm.org", sendToInterestedUsers = False, @@ -104,7 +97,7 @@ subject = "Build %(builder)s Failure", mode = "failing", builders = ["lld-x86_64-win", - "lld-x86_64-freebsd", "lld-x86_64-darwin", + "lld-x86_64-freebsd", "clang-x86_64-linux-abi-test", "clang-with-lto-ubuntu", "clang-with-thin-lto-ubuntu", "llvm-clang-x86_64-expensive-checks-win", @@ -141,18 +134,17 @@ reporters.MailNotifier( fromaddr = "llvm.buildmaster@lab.llvm.org", sendToInterestedUsers = False, - extraRecipients = ["stilis@microsoft.com", "jonas@devlieghere.com", - "diprou@microsoft.com", "makudrya@microsoft.com"], + extraRecipients = ["aifxbuildbotdri@microsoft.com", "namcvica@microsoft.com"], subject = "Build %(builder)s Failure", mode = "failing", - builders = ["lldb-x64-windows-ninja"]), + builders = ["mlir-windows"]), reporters.MailNotifier( fromaddr = "llvm.buildmaster@lab.llvm.org", sendToInterestedUsers = False, - extraRecipients = ["stilis@microsoft.com", "namcvica@microsoft.com"], + extraRecipients = ["gongsu@us.ibm.com", "alexe@us.ibm.com"], subject = "Build %(builder)s Failure", mode = "failing", - builders = ["mlir-windows"]), + builders = ["mlir-s390x-linux-werror"]), reporters.MailNotifier( fromaddr = "llvm.buildmaster@lab.llvm.org", sendToInterestedUsers = False, @@ -160,20 +152,6 @@ subject = "Build %(builder)s Failure", mode = "failing", builders = ["fuchsia-x86_64-linux"]), - reporters.MailNotifier( - fromaddr = "llvm.buildmaster@lab.llvm.org", - sendToInterestedUsers = False, - extraRecipients = ["jan.kratochvil@redhat.com"], - subject = "Build %(builder)s Failure", - mode = "failing", - builders = ["lldb-x86_64-fedora"]), - reporters.MailNotifier( - fromaddr = "llvm.buildmaster@lab.llvm.org", - sendToInterestedUsers = False, - extraRecipients = ["kkleine@redhat.com"], - subject = "Build %(builder)s Failure", - mode = "failing", - builders = ["fedora-llvm-x86_64", "x86_64-fedora-clang"]), reporters.MailNotifier( fromaddr = "llvm.buildmaster@lab.llvm.org", sendToInterestedUsers = False, @@ -181,13 +159,6 @@ subject = "Build %(builder)s Failure", mode = "failing", builders = ["lldb-x86_64-debian"]), - reporters.MailNotifier( - fromaddr = "llvm.buildmaster@lab.llvm.org", - sendToInterestedUsers = False, - extraRecipients = ["omair.javaid@linaro.org"], - subject = "Build %(builder)s Failure", - mode = "failing", - builders = ["lldb-arm-ubuntu","lldb-aarch64-ubuntu"]), reporters.MailNotifier( fromaddr = "llvm.buildmaster@lab.llvm.org", sendToInterestedUsers = False, @@ -196,7 +167,9 @@ mode = "failing", builders = ["llvm-clang-x86_64-win-fast","lld-x86_64-ubuntu-fast", "llvm-clang-x86_64-expensive-checks-ubuntu", - "llvm-clang-win-x-armv7l", "llvm-clang-win-x-aarch64"]), + "llvm-clang-win-x-armv7l", "llvm-clang-win-x-aarch64", + "llvm-nvptx-nvidia-ubuntu", "llvm-nvptx64-nvidia-ubuntu", + "llvm-nvptx-nvidia-win", "llvm-nvptx64-nvidia-win"]), reporters.MailNotifier( fromaddr = "llvm.buildmaster@lab.llvm.org", sendToInterestedUsers = False, @@ -207,11 +180,21 @@ reporters.MailNotifier( fromaddr = "llvm.buildmaster@lab.llvm.org", sendToInterestedUsers = False, - extraRecipients = ["sivachandra@google.com", "paulatoth@google.com"], + extraRecipients = ["lntue@google.com", "michaelrj@google.com", + "sivachandra@google.com"], subject = "Build %(builder)s Failure", mode = "failing", builders = ["libc-x86_64-debian", "libc-x86_64_debian-dbg", - "libc-x86_64-debian-dbg-asan"]), + "libc-x86_64-debian-dbg-runtimes-build", + "libc-x86_64-debian-dbg-asan", "libc-aarch64-ubuntu-dbg", + "libc-x86_64-windows-dbg", "libc-arm32-debian-dbg", + "libc-aarch64-ubuntu-fullbuild-dbg", + "libc-x86_64-debian-fullbuild-dbg", + "libc-x86_64-debian-gcc-fullbuild-dbg", + "libc-x86_64-debian-fullbuild-dbg-asan", + "libc-riscv64-debian-dbg", + "libc-riscv64-debian-fullbuild-dbg", + "libc-x86_64-debian-dbg-lint"]), reporters.MailNotifier( fromaddr = "llvm.buildmaster@lab.llvm.org", sendToInterestedUsers = False, @@ -228,33 +211,175 @@ mode = "failing", builders = [ "ml-opt-dev-x86-64", "ml-opt-rel-x86-64", "ml-opt-devrel-x86-64"]), + reporters.MailNotifier( + fromaddr="llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers = False, + extraRecipients=[ + "tejohnson@google.com"], + subject = "ThinLTO WPD Failure: %(builder)s", + mode = "failing", + builders = ["clang-with-thin-lto-wpd-ubuntu"]), reporters.MailNotifier( fromaddr = "llvm.buildmaster@lab.llvm.org", sendToInterestedUsers = False, - extraRecipients = [ - "caroline.concatto@arm.com", "flang_llvm_buildbots@arm.com"], + extraRecipients = ["douglas.yung@sony.com"], subject = "Build %(builder)s Failure", mode = "failing", - builders = [ - "flang-aarch64-ubuntu", "flang-aarch64-ubuntu-clang", - "flang-aarch64-ubuntu-gcc10"]), + builders = ["llvm-clang-x86_64-sie-ubuntu-fast", + "llvm-clang-x86_64-sie-win", + "llvm-clang-x86_64-sie-win-release", + "llvm-clang-x86_64-gcc-ubuntu", + "llvm-clang-x86_64-gcc-ubuntu-release", + "cross-project-tests-sie-ubuntu", + "cross-project-tests-sie-ubuntu-dwarf5"]), + reporters.MailNotifier( + fromaddr = "llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers = False, + extraRecipients = ["tom.weaver@sony.com"], + subject = "Build %(builder)s Failure", + mode = "failing", + builders = ["llvm-clang-x86_64-sie-ubuntu-fast", + "llvm-clang-x86_64-sie-win", + "llvm-clang-x86_64-gcc-ubuntu"]), reporters.MailNotifier( fromaddr="llvm.buildmaster@lab.llvm.org", sendToInterestedUsers = False, extraRecipients=[ - "tejohnson@google.com"], - subject = "ThinLTO WPD Failure: %(builder)s", + "joker.eph@gmail.com"], + subject = "MLIR Build Failure: %(builder)s", mode = "failing", - builders = ["thinlto-x86-64-bot1"]), - + builders = ["mlir-nvidia"]), + reporters.MailNotifier( + fromaddr="llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers = False, + extraRecipients=[ + "mlir-bugs-external+buildbot@googlegroups.com"], + subject = "MLIR Build Failure: %(builder)s", + mode = "failing", + builders = ["mlir-nvidia", + "mlir-windows", + "ppc64le-mlir-rhel-clang"]), + reporters.MailNotifier( + fromaddr="llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers = False, + extraRecipients=["dl.gcr.lightning.buildbot@amd.com"], + subject = "Build Failure: %(builder)s", + mode = "failing", + builders = ["clang-hip-vega20"]), + reporters.MailNotifier( + fromaddr="llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers = False, + extraRecipients=["llvm_arc_buildbot@synopsys.com", "heli@synopsys.com"], + subject = "Build Failure: %(builder)s", + mode = "failing", + builders = ["arc-builder"]), + reporters.MailNotifier( + fromaddr="llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers = False, + extraRecipients=["dl.gcr.lightning.buildbot@amd.com"], + subject = "Build Failure: %(builder)s", + mode = "failing", + builders = ["openmp-offload-amdgpu-runtime"]), + reporters.MailNotifier( + fromaddr="llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers = False, + extraRecipients=["dl.mlse.buildbot@amd.com"], + subject = "Build Failure: %(builder)s", + mode = "failing", + builders = ["mlir-rocm-mi200"]), + reporters.MailNotifier( + fromaddr="llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers = False, + extraRecipients=["flangbuilder@meinersbur.de"], + subject = "Build Failure (flang): %(builder)s", + mode = "failing", + builders = ["flang-x86_64-windows"]), + reporters.MailNotifier( + fromaddr="llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers = False, + extraRecipients=["offloadbuilder@meinersbur.de"], + subject = "Build Failure (offload): %(builder)s", + mode = "failing", + builders = ["openmp-offload-cuda-project","openmp-offload-cuda-runtime"]), + reporters.MailNotifier( + fromaddr="llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers = False, + extraRecipients=["pollybuilder@meinersbur.de"], + subject = "Build Failure (polly): %(builder)s", + mode = "failing", + builders = [ + "polly-x86_64-linux", + "polly-x86_64-linux-noassert", + "polly-x86_64-linux-plugin", + "polly-x86_64-linux-shared", + "polly-x86_64-linux-shared-plugin", + "polly-x86_64-linux-shlib", + "polly-x86_64-linux-shlib-plugin", + "polly-sphinx-docs", + ]), + reporters.MailNotifier( + fromaddr = "llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers = False, + extraRecipients = ["orlando.hyams@sony.com"], + subject = "Build %(builder)s Failure", + mode = "failing", + builders = ["cross-project-tests-sie-ubuntu", + "llvm-clang-x86_64-sie-win"]), + reporters.MailNotifier( + fromaddr = "llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers = False, + extraRecipients = ["kkleine@redhat.com"], + subject = "Build %(builder)s Failure", + mode = "failing", + builders = ["standalone-build-x86_64"]), + reporters.MailNotifier( + fromaddr = "llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers = False, + extraRecipients = ["llvm-bolt@meta.com"], + subject = "BOLT NFC checks mismatch", + mode = ("warnings",), + builders = ["bolt-x86_64-ubuntu-nfc"]), + reporters.MailNotifier( + fromaddr = "llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers = False, + extraRecipients = ["luweining@loongson.cn", "chenli@loongson.cn"], + subject = "Build %(builder)s Failure", + mode = "failing", + builders = ["clang-loongarch64-linux"]), + reporters.MailNotifier( + fromaddr = "llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers = False, + extraRecipients = ["kadircet@google.com", "sammccall@google.com"], + subject = "Build %(builder)s Failure", + mode = "failing", + builders = ["clangd-ubuntu-tsan"]), + reporters.MailNotifier( + fromaddr = "llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers = False, + extraRecipients = ["kadircet@google.com", "ibiryukov@google.com"], + subject = "Build %(builder)s Failure", + mode = "failing", + builders = ["clang-debian-cpp20"]), + reporters.MailNotifier( + fromaddr = "llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers = False, + extraRecipients = ["llvm.buildbot.notification@intel.com"], + subject = "Build %(builder)s Failure", + mode = "failing", + builders = ["clang-cmake-x86_64-avx512-linux"]), ] + +from twisted.python import log + # Returns a list of Status Targets. The results of each build will be # pushed to these targets. buildbot.plugins reporters has a variety # to choose from, including email senders, and IRC bots. def getReporters(): if config.options.getboolean('Master Options', 'is_production'): + log.msg(">>> getReporters: Production mode. All reporters are registered.") return all else: # Staging buildbot does not report issues. + log.msg(">>> getReporters: Staging buildbot does not report issues.") return [] diff --git a/buildbot/osuosl/master/config/workers.py b/buildbot/osuosl/master/config/workers.py --- a/buildbot/osuosl/master/config/workers.py +++ b/buildbot/osuosl/master/config/workers.py @@ -16,72 +16,72 @@ # Linux Ubuntu create_worker("as-worker-5", properties={'jobs' : 16}), - # Mac Pro 2.7 GHz 12-Core Intel Xeon E5, Maverick 10.9.2 - create_worker("as-worker-3", properties={'jobs' : 8}, max_builds=4), - # ARMv7/ARMv8 Linaro workers - create_worker("linaro-tk1-01", properties={'jobs' : 4}, max_builds=1), create_worker("linaro-tk1-02", properties={'jobs' : 4}, max_builds=1), - create_worker("linaro-tk1-03", properties={'jobs' : 4}, max_builds=1), - create_worker("linaro-tk1-04", properties={'jobs' : 4}, max_builds=1), - create_worker("linaro-tk1-05", properties={'jobs' : 4}, max_builds=1), - create_worker("linaro-tk1-06", properties={'jobs' : 4}, max_builds=1), - create_worker("linaro-tk1-07", properties={'jobs' : 4}, max_builds=1), - create_worker("linaro-tk1-08", properties={'jobs' : 4}, max_builds=1), - create_worker("linaro-tk1-09", properties={'jobs' : 4}, max_builds=1), - create_worker("linaro-armv7-lnt", properties={'jobs' : 32}, max_builds=1), - create_worker("linaro-armv7-selfhost", properties={'jobs' : 32}, max_builds=1), - create_worker("linaro-armv7-quick", properties={'jobs' : 32}, max_builds=1), - create_worker("linaro-armv7-global-isel", properties={'jobs' : 32}, max_builds=1), - create_worker("linaro-armv8-lld", properties={'jobs' : 32}, max_builds=1), - # Libcxx testsuite has tests with timing assumptions. Run single-threaded to make - # sure we have plenty CPU cycle to satisfy timing assumptions. - create_worker("linaro-armv8-libcxx", properties={'jobs' : 1}, max_builds=1), - create_worker("linaro-arm-lldb", properties={'jobs' : 16}, max_builds=1), + create_worker("linaro-clang-armv7-lnt", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-clang-armv7-2stage", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-clang-armv7-global-isel", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-clang-armv7-vfpv3-2stage", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-clang-armv8-lld-2stage", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-clang-armv8-quick", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-lldb-arm-ubuntu", properties={'jobs' : 16}, max_builds=1), # AArch64 Linaro workers - create_worker("linaro-aarch64-quick", properties={'jobs' : 32}, max_builds=1), - create_worker("linaro-aarch64-full", properties={'jobs' : 32}, max_builds=1), - create_worker("linaro-aarch64-global-isel", properties={'jobs' : 32}, max_builds=1), - create_worker("linaro-aarch64-lld", properties={'jobs' : 32}, max_builds=1), - create_worker("linaro-aarch64-flang-oot", properties={'jobs' : 32}, max_builds=1), - create_worker("linaro-aarch64-flang-dylib", properties={'jobs' : 32}, max_builds=1), - create_worker("linaro-aarch64-flang-sharedlibs", properties={'jobs' : 32}, max_builds=1), - # Libcxx testsuite has tests with timing assumptions. Run single-threaded to make - # sure we have plenty CPU cycle to satisfy timing assumptions. - create_worker("linaro-aarch64-libcxx", properties={'jobs' : 1}, max_builds=1), - create_worker("linaro-aarch64-lldb", properties={'jobs': 16}, max_builds=1), - - # AArch64 + create_worker("linaro-clang-aarch64-quick", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-clang-aarch64-lld-2stage", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-clang-aarch64-global-isel", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-clang-aarch64-full-2stage", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-lldb-aarch64-ubuntu", properties={'jobs': 16}, max_builds=1), + create_worker("linaro-flang-aarch64-dylib", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-flang-aarch64-sharedlibs", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-flang-aarch64-out-of-tree", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-flang-aarch64-debug", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-flang-aarch64-latest-clang", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-flang-aarch64-release", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-flang-aarch64-rel-assert", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-flang-aarch64-latest-gcc", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-g3-01", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-g3-02", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-g3-03", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-g3-04", properties={'jobs' : 32}, max_builds=1), + + # AArch64 Windows Microsoft Surface X Pro create_worker("linaro-armv8-windows-msvc-01", properties={'jobs' : 8}, max_builds=1), create_worker("linaro-armv8-windows-msvc-02", properties={'jobs' : 8}, max_builds=1), + create_worker("linaro-armv8-windows-msvc-03", properties={'jobs' : 8}, max_builds=1), + create_worker("linaro-armv8-windows-msvc-04", properties={'jobs' : 8}, max_builds=1), + create_worker("linaro-armv8-windows-msvc-05", properties={'jobs' : 8}, max_builds=1), + + # Windows Server 2016 Intel(R) Xeon(R) CPU @ 2.60GHz, 16 Core(s), 128 GB of RAM + create_worker("win-mlir-buildbot", properties={'jobs' : 64}, max_builds=1), - # ARMv7 build cache workers - create_worker("packet-linux-armv7-slave-1", properties={'jobs' : 64}, max_builds=1), + # Linux s390x Ubuntu Focal, IBM z13 (5GHz), 64GB of RAM + create_worker("onnx-mlir-nowarn-linux-s390x", properties={'jobs' : 4}, max_builds=1), - # AArch64 build cache worker - create_worker("packet-linux-aarch64-slave-1", properties={'jobs' : 64}, max_builds=1), + # LoongArch64 Loongson-3C5000L-LL 2GHz, 128GB of RAM + # CLFS repo: https://github.com/sunhaiyong1978/CLFS-for-LoongArch + create_worker("loongson-loongarch64-clfs-clang-01", properties={'jobs': 32}, max_builds=1), - # Windows Server 2016 Intel Xeon(R) Quad 2.30 GHz, 56GB of RAM - create_worker("win-py3-buildbot", properties={'jobs' : 64}, max_builds=1), + # MIPS Loongson-3A R4 (Loongson-3A4000) 64-bit little endian (mips64el) + create_worker("debian-tritium-mips64el", properties={'jobs': 1}, max_builds=1), - # Windows Server 2016 Intel(R) Xeon(R) CPU @ 2.60GHz, 16 Core(s), 128 GB of RAM - create_worker("win-mlir-buildbot", properties={'jobs' : 64}, max_builds=1), + # Motorola 68k 32-bit big endian (m68k) + create_worker("debian-akiko-m68k", properties={'jobs': 1}, max_builds=1), + create_worker("suse-gary-m68k-cross", properties={'jobs': 4}, max_builds=1), # POWER7 PowerPC big endian (powerpc64) - create_worker("ppc64be-clang-test", properties={'jobs': 16}, max_builds=1), - create_worker("ppc64be-clang-lnt-test", properties={'jobs': 16, 'vcs_protocol': 'https'}, max_builds=1), + create_worker("ppc64be-clang-test-suite", max_builds=1), create_worker("ppc64be-clang-multistage-test", properties={'jobs': 16}, max_builds=1), create_worker("ppc64be-sanitizer", properties={'jobs': 16}, max_builds=1), # POWER 8 PowerPC little endian (powerpc64le) - create_worker("ppc64le-clang-test", properties={'jobs': 4}, max_builds=1), - create_worker("ppc64le-clang-lnt-test", properties={'jobs': 8}, max_builds=1), + create_worker("ppc64le-clang-test-suite", max_builds=1), create_worker("ppc64le-clang-multistage-test", properties={'jobs': 8}, max_builds=1), create_worker("ppc64le-sanitizer", properties={'jobs': 4}, max_builds=1), create_worker("ppc64le-lld-multistage-test", max_builds=1), - create_worker("ppc64le-clang-rhel-test", properties={'jobs': 4}, max_builds=1), - create_worker("ppc64le-flang-mlir-rhel-test", max_builds=1), + create_worker("ppc64le-clang-rhel-test", properties={'jobs': 192}, max_builds=1), + create_worker("ppc64le-flang-rhel-test", max_builds=1), + create_worker("ppc64le-mlir-rhel-test", max_builds=1), # SPARC 64-bit big endian (sparc64) create_worker("debian-stadler-sparc64", properties={'jobs': 4}, max_builds=1), @@ -94,9 +94,6 @@ create_worker("hexagon-build-03", properties={'jobs': 16, 'loadaverage':32}, max_builds=1), - # Ubuntu x86-64 - create_worker("avr-build-01", properties={'jobs': 10}, max_builds=1), - # Debian Jessie x86-64 GCE instance. create_worker("gribozavr3", properties={'jobs': 1}, max_builds=2), @@ -108,20 +105,25 @@ "gribozavr@gmail.com", "gkistanova@gmail.com"]), - # Debian 7.7 x86_64 GCE instance + # Ubuntu x86_64 6-core z440 workstation + create_worker("sanitizer-buildbot-android", properties={'jobs': 6}, max_builds=1), + # x86_64 GCE instances create_worker("sanitizer-buildbot1", properties={'jobs': 64}, max_builds=3), - # Debian 7.7 x86_64 GCE instance create_worker("sanitizer-buildbot2", properties={'jobs': 64}, max_builds=3), - # Debian 7.7 x86_64 GCE instance - create_worker("sanitizer-buildbot3", properties={'jobs': 64}, max_builds=2), - # Debian 7.7 x86_64 GCE instance - create_worker("sanitizer-buildbot4", properties={'jobs': 64}, max_builds=2), - # Ubuntu 14.04 x86_64 6-core z440 workstation - create_worker("sanitizer-buildbot6", properties={'jobs': 6}, max_builds=1), - # Debian 7.7 x86_64 GCE instance - create_worker("sanitizer-buildbot7", properties={'jobs': 64}, max_builds=3), - # Debian 7.7 x86_64 GCE instance - create_worker("sanitizer-buildbot8", properties={'jobs': 64}, max_builds=3), + create_worker("sanitizer-buildbot3", properties={'jobs': 64}, max_builds=3), + create_worker("sanitizer-buildbot4", properties={'jobs': 64}, max_builds=3), + create_worker("sanitizer-buildbot5", properties={'jobs': 64}, max_builds=3), + create_worker("sanitizer-buildbot6", properties={'jobs': 64}, max_builds=3), + # aarch64 GCE instances + create_worker("sanitizer-buildbot7", properties={'jobs': 48}, max_builds=2), + create_worker("sanitizer-buildbot8", properties={'jobs': 48}, max_builds=3), + create_worker("sanitizer-buildbot9", properties={'jobs': 48}, max_builds=2), + create_worker("sanitizer-buildbot10", properties={'jobs': 48}, max_builds=3), + create_worker("sanitizer-buildbot11", properties={'jobs': 48}, max_builds=2), + create_worker("sanitizer-buildbot12", properties={'jobs': 48}, max_builds=3), + + # POWER 8 PowerPC AIX 7.2 + create_worker("aix-ppc64", properties={'jobs': 10}, max_builds=1), # IBM z13 (s390x), Ubuntu 16.04.2 create_worker("systemz-1", properties={'jobs': 4, 'vcs_protocol': 'https'}, max_builds=4), @@ -132,8 +134,8 @@ create_worker("windows-gcebot2", properties={'jobs': 32}, max_builds=1), # Ubuntu 14.04 x86_64-scei-ps4, 2 x Intel(R) Xeon(R) CPU E5-2699 v3 @ 2.30GHz - create_worker("as-worker-91"), - create_worker("as-worker-92"), + create_worker("as-worker-91", max_builds=1), + create_worker("as-worker-92", max_builds=1), # Windows 10 Pro x86_64-scei-ps4, 2 x Intel(R) Xeon(R) CPU E5-2699 v3 @ 2.30GHz create_worker("as-worker-93", properties={'jobs': 36}, max_builds=1), @@ -143,28 +145,43 @@ create_worker("cuda-p4-0", max_builds=1), create_worker("cuda-t4-0", max_builds=1), - # X86_64 AVX2, Ubuntu 16.04.2, Intel(R) Xeon(R) CPU E5-2699 v4 @ 2.20GHz - create_worker("avx2-intel64", properties={'jobs': 80}, max_builds=1), + # HIP on Ubuntu 18.04.5, Intel(R) Xeon(R) Gold 5218 @ 2.30GHz, Vega20 GPU + create_worker("hip-vega20-0", max_builds=1), + + # X86_64 AVX512, Ubuntu 22.04.1 LTS, Intel(R) Xeon(R) Silver 4216 CPU @ 2.10GHz + create_worker("avx512-intel64", properties={'jobs': 80}, max_builds=1), - # X86_64 with SDE, Ubuntu 16.04.2, Intel(R) Xeon(R) CPU E5-2699 v4 @ 2.20GHz - create_worker("sde-avx512-intel64", properties={'jobs': 80}, max_builds=1), + # X86_64 AVX512, Windows Server 2022 Standard, Intel(R) Xeon(R) Silver 4216 CPU @ 2.10GHz + create_worker("avx512-intel64-win", properties={'jobs': 80}, max_builds=1), # Debian 9.0 x86_64 64-core GCE instances create_worker("fuchsia-debian-64-us-central1-a-1", properties={'jobs': 64}, max_builds=1), create_worker("fuchsia-debian-64-us-central1-b-1", properties={'jobs': 64}, max_builds=1), - # Fedora latest stable x86_64, Intel i5-2500, 4 cores, 12GB RAM - create_worker("lldb-x86_64-fedora", properties={'jobs': 4}, max_builds=1), - - # Fedora latest stable, arch=x86_64, running on RedHat internal OpenShift PSI cluster - create_worker("fedora-llvm-x86_64", properties={'jobs': 64}, max_builds=1), - # Debian x86_64 Buster Xeon(R) Gold 6154 CPU @ 3.00GHz, 192GB RAM create_worker("lldb-x86_64-debian", properties={'jobs': 72}, max_builds=1), + # Windows x86_64 32 CPUs, 125 GB RAM + create_worker("libc-x86_64-windows", properties={'jobs': 32}, max_builds=2), + + # Debian arm32 single core, 512 MB RAM backed by 32 GB swap memory + create_worker("libc-arm32-debian", properties={'jobs': 1}, max_builds=1), + + # Ubuntu aarch64 128 CPUs, 125 GB RAM + create_worker("libc-aarch64-ubuntu", properties={'jobs': 32}, max_builds=2), + # Debian x86_64 Intel Broadwell 32 CPUs, 120 GB RAM create_worker("libc-x86_64-debian", properties={'jobs': 32}, max_builds=2), + # Debian x86_64 Intel Skylake 32 CPUs, 128 GB RAM + create_worker("libc-x86_64-debian-fullbuild", properties={'jobs': 32}, max_builds=2), + + # Debian x86_64 Intel Cascade Lake 8 CPU, 32 GB RAM + create_worker("libc-lint-worker", properties={'jobs': 16}, max_builds=2), + + # Debian riscv64 4 CPU, 4 GB RAM + create_worker("libc-riscv64-debian", properties={'jobs': 4}, max_builds=1), + # Windows Server on Xeon Gold 6130 (2x2.1GHz), 128Gb of RAM create_worker("as-builder-1", properties={ 'remote_test_host': 'jetson6.lab.llvm.org', @@ -185,26 +202,51 @@ # Ubuntu 18.04.2 LTS x86_64 Intel(R) Xeon(R) Gold CPU @ 2.1GHz, 128GB RAM create_worker("as-builder-4", properties={'jobs': 64}, max_builds=1), + # Windows Server on Xeon Gold 6230 (2x2.1GHz), 256Gb of RAM + create_worker("as-builder-5", properties={ # arm + 'remote_test_host': 'jetson4.lab.llvm.org', + 'remote_test_user': 'ubuntu' + }, + max_builds=1), + create_worker("as-builder-6", properties={ # aarch64 + 'remote_test_host': 'jetson8.lab.llvm.org', + 'remote_test_user': 'ubuntu' + }, + max_builds=1), + + # NVPTX builders. + # Ubuntu 22.04 LTS x86_64 Intel Xeon 6330 CPU 2Sx64Cx128LP @ 2.0GHz, 256GB RAM + create_worker("as-builder-7", properties={'jobs': 64}, max_builds=1), + # Windows Server on Intel Xeon 6330 CPU 2Sx64Cx128LP @ 2.0GHz, 256GB RAM + create_worker("as-builder-8", max_builds=1), + # Solaris 11 create_worker("solaris11-amd64", properties={'jobs' : 8}, max_builds=1), create_worker("solaris11-sparcv9", properties={'jobs' : 8}, max_builds=1), - # Windows 7 Intel(R) Core(TM) CPU i7-4790K (4.00GHz), 16GB of RAM - create_worker("windows7-buildbot", properties={'jobs': 2}, max_builds=1), - - # Windows 10, Visual Studio 2019, Google Cloud 16 cores - create_worker("windows10-vs2019", properties={'jobs': 32}, max_builds=1), - - # CentOS 7.5.1804 on Intel(R) Xeon(R) Gold 6126 CPU @ 2.60GHz, 96GB RAM - create_worker("nec-arrproto41", properties={'jobs': 12}, max_builds=1), + # CentOS 7.8.1127 on Intel(R) Xeon(R) Gold 6126 CPU @ 2.60GHz, 96GB RAM + create_worker("hpce-aurora2", properties={'jobs': 8}, max_builds=1), + create_worker("hpce-ve-main", properties={'jobs': 8}, max_builds=1), + create_worker("hpce-ve-staging", properties={'jobs': 8}, max_builds=1), # Ubuntu 18.04 amd64 on Google Cloud, 16 core, Nvidia Tesla T4 create_worker("mlir-nvidia", properties={'jobs': 16}, max_builds=1), + # Ubuntu 18.04 pool of VMs on GCP + create_worker("mlir-ubuntu-worker0", max_builds=1), + create_worker("mlir-ubuntu-worker1", max_builds=1), + create_worker("mlir-ubuntu-worker2", max_builds=1), + create_worker("mlir-ubuntu-worker3", max_builds=1), + create_worker("mlir-ubuntu-worker4", max_builds=1), + # Ubuntu 18.04 on Google Cloud, for machine configuration check # buildbot/google/terraform/main.tf create_worker("clangd-ubuntu-clang", max_builds=1), + # Debian bullseye, build in C++20 configuration, for worker + # configuration check buildbot/google/terraform/main.tf + create_worker("clang-debian-cpp20", max_builds=1), + # Ubuntu 18.04.LTS x86_64, GCE instance create_worker("polly-x86_64-gce1", properties={'jobs': 2}, max_builds=1), create_worker("polly-x86_64-gce2", properties={'jobs': 2}, max_builds=1), @@ -212,20 +254,79 @@ # Ubuntu 18.04.LTS x86_64, Intel(R) Xeon(R) CPU X3460 @ 2.80GHz, 32 GiB RAM create_worker("polly-x86_64-fdcserver", properties={'jobs': 8, 'loadaverage': 8}, max_builds=1), - create_worker("flang-aarch64-ubuntu-build"), - create_worker("flang-aarch64-ubuntu-clang-build", properties={'jobs': 14}), - create_worker("flang-aarch64-ubuntu-gcc10-build", properties={'jobs': 14}), - create_worker("nersc-flang"), + # Windows 10, AMD Ryzen 5 PRO 4650G, 16 GiB RAM + create_worker("minipc-ryzen-win", properties={'jobs': 12}, max_builds=1), + + # Ubuntu 20.04.LTS x86_64, Intel(R) Core(TM) i5-9400F CPU @ 2.90Ghz, 16 GiB RAM, NVIDIA GeForce GTX 1050 Ti (Pascal, sm_61, 4GiB) + create_worker("minipc-1050ti-linux", properties={'jobs': 6}, max_builds=1), + create_worker("alcf-theta-flang", properties={'jobs': 12}, max_builds=1), # ML-Driven Compiler Optimizations build workers (Ubuntu x86_64) create_worker("ml-opt-dev-x86-64-b1", - properties={'jobs': 64}, max_builds=1), + properties={'jobs': 64}, max_builds=1, + notify_on_missing='mlcompileropt-buildbot@google.com'), create_worker("ml-opt-rel-x86-64-b1", - properties={'jobs': 64}, max_builds=1), + properties={'jobs': 64}, max_builds=1, + notify_on_missing='mlcompileropt-buildbot@google.com'), create_worker("ml-opt-devrel-x86-64-b1", - properties={'jobs': 64}, max_builds=1), + properties={'jobs': 64}, max_builds=1, + notify_on_missing='mlcompileropt-buildbot@google.com'), + create_worker("ml-opt-dev-x86-64-b2", + properties={'jobs': 64}, max_builds=1, + notify_on_missing='mlcompileropt-buildbot@google.com'), + create_worker("ml-opt-rel-x86-64-b2", + properties={'jobs': 64}, max_builds=1, + notify_on_missing='mlcompileropt-buildbot@google.com'), + create_worker("ml-opt-devrel-x86-64-b2", + properties={'jobs': 64}, max_builds=1, + notify_on_missing='mlcompileropt-buildbot@google.com'), # Ubuntu x86_64 create_worker("thinlto-x86-64-bot1", properties={'jobs': 64}, max_builds=1), + create_worker("thinlto-x86-64-bot2", properties={'jobs': 64}, max_builds=1), + + # Ubuntu 20.04 on AWS, x86_64 PS4 target + create_worker("sie-linux-worker", properties={'jobs': 40}, max_builds=1), + # 2012 Mac Mini host, 16GB memory: + # - Ubuntu 18.04 in docker container + create_worker("doug-worker-1a", properties={'jobs': 10}, max_builds=1), + # - Ubuntu 22.04 in docker container + create_worker("doug-worker-1b", properties={'jobs': 10}, max_builds=1), + # Ubuntu 18.04 in docker container on Ryzen 4800U + create_worker("doug-worker-2a", properties={'jobs': 22}, max_builds=1), + + # Windows Server 2019 on AWS, x86_64 PS4 target + create_worker("sie-win-worker", properties={'jobs': 64}, max_builds=1), + + # XCore target, Ubuntu 20.04 x64 host + create_worker("xcore-ubuntu20-x64", properties={'jobs': 4}, max_builds=1), + + # ARC Worker, CentOS 7.9 x86_64 Intel Xeon Platinum 8000 @ 3.6GHz, 32GB RAM + create_worker("arc-worker", properties={'jobs': 16}, max_builds=1), + + # OpenMP on AMDGPU, Ubuntu 18.04.5, Intel(R) Xeon(R) Gold 5218 @ 2.30GHz with 64GB Memory, 1 Vega20 GPU with 16GB Memory + create_worker("omp-vega20-0", properties={'jobs': 32}, max_builds=1), + create_worker("omp-vega20-1", properties={'jobs': 32}, max_builds=1), + + # AMD ROCm support, Ubuntu 18.04.6, AMD Ryzen @ 1.5 GHz, MI200 GPU + create_worker("mi200-buildbot", max_builds=1), + + # BOLT workers + create_worker("bolt-worker", max_builds=1), + create_worker("bolt-worker-aarch64", max_builds=1), + + # Fedora worker + create_worker("standalone-build-x86_64", max_builds=1), + + # CSKY T-HEAD workers + create_worker("thead-clang-csky", properties={'jobs' : 32}, max_builds=1), + + # RISC-V workers + create_worker("rv64gc-qemu-user", properties={'jobs' : 32}, max_builds=1), + + # Google Cloud workers. + create_worker("gc-builder-5", properties={'jobs' : 112}, max_builds=1), + create_worker("gc-builder-6-win", properties={'jobs' : 112}, max_builds=1), + create_worker("gc-builder-7-win", properties={'jobs' : 112}, max_builds=1), ] diff --git a/buildbot/osuosl/master/master.cfg b/buildbot/osuosl/master/master.cfg --- a/buildbot/osuosl/master/master.cfg +++ b/buildbot/osuosl/master/master.cfg @@ -65,12 +65,23 @@ util.BuilderConfig(**b) for b in config.builders.all ] +for rb in config.release_builders.all: + # Make sure a release byuilder has the "release" tag. + tags = rb.get('tags', []) + if 'release' not in tags: + rb['tags'] = tags + ['release'] + builders.append(util.BuilderConfig(**rb)) + ####### SCHEDULERS -c['schedulers'] = config.schedulers.getSingleBranchSchedulers( - c['builders']) +c['schedulers'] = config.schedulers.getMainBranchSchedulers( + builders) +c['schedulers'].extend(config.schedulers.getLntSchedulers()) +c['schedulers'].extend(config.schedulers.getReleaseBranchSchedulers( + builders, + treeStableTimer=5*60)) c['schedulers'].extend(config.schedulers.getForceSchedulers( - c['builders'])) + builders)) ####### BUILDBOT SERVICES diff --git a/llvmbisect/llvmlab/llvmlab.py b/llvmbisect/llvmlab/llvmlab.py --- a/llvmbisect/llvmlab/llvmlab.py +++ b/llvmbisect/llvmlab/llvmlab.py @@ -46,7 +46,7 @@ def frombasename(str, url=None): str = os.path.basename(str) - revision = timestamp = build = None + revision = revision_prefix = sha = timestamp = build = None # Check if this is a BNI style build. m = BUILD_NAME_REGEX.match(str) @@ -58,7 +58,7 @@ revision.append(int(minor_str)) if micro_str: revision.append(int(micro_str)) - return Build(name, tuple(revision), None, build, suffix) + return Build(name, None, tuple(revision), None, None, build, suffix) if '.' in str: str, suffix = str.split('.', 1) @@ -74,12 +74,20 @@ if m: str, timestamp = m.groups() + m = re.match(r'(.*)-d([0-9]+)-(.*)', str) + if m: + str, revision, sha = m.groups() + revision = int(revision) + revision_prefix = 'd' + m = re.match(r'(.*)-r([0-9]+)', str) if m: str, revision = m.groups() revision = int(revision) + revision_prefix = 'r' - return Build(str, revision, timestamp, build, suffix, url) + return Build(str, revision_prefix, revision, sha, timestamp, + build, suffix, url) @staticmethod def fromdata(data): @@ -89,13 +97,17 @@ def todata(self): return {'name': self.name, 'revision': self.revision, + 'sha': self.sha, 'timestamp': self.timestamp, 'build': self.build, 'suffix': self.suffix} - def __init__(self, name, revision, timestamp, build, suffix, url): + def __init__(self, name, revision_prefix, revision, sha, timestamp, + build, suffix, url): self.name = name + self.revision_prefix = revision_prefix self.revision = revision + self.sha = sha self.timestamp = timestamp self.build = build self.suffix = suffix @@ -108,7 +120,9 @@ basename += '-' + '.'.join(str(r) for r in self.revision) else: assert isinstance(self.revision, int) - basename += '-r%d' % self.revision + basename += '-%s%d' % (self.revision_prefix, self.revision) + if self.sha is not None: + basename += '-%s' % self.sha if self.timestamp is not None: basename += '-t%s' % self.timestamp if self.build is not None: @@ -122,8 +136,8 @@ def __repr__(self): return "%s%r" % (self.__class__.__name__, - (self.name, self.revision, self.timestamp, self.build, - self.suffix)) + (self.name, self.revision, self.sha, self.timestamp, + self.build, self.suffix)) def __cmp__(self, other): return cmp((self.revision, self.timestamp, diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -1,29 +1,28 @@ -attrs==20.2.0 +attrs==20.3.0 six==1.15.0 Automat==20.2.0 constantly==15.1.0 idna==2.10 -hyperlink==20.0.1 -incremental==17.5.0 -Jinja2==2.11.2 -PyHamcrest==2.0.2 +hyperlink==21.0.0 +incremental==21.3.0 +Jinja2==2.11.3 Tempita==0.5.2 decorator==4.4.2 pbr==5.5.1 -sqlparse==0.4.1 -SQLAlchemy==1.3.20 +sqlparse==0.4.2 +SQLAlchemy==1.3.23 sqlalchemy-migrate==0.13.0 -certifi==2020.6.20 -chardet==3.0.4 -urllib3==1.25.11 -requests==2.24.0 -zope.interface==5.1.2 +certifi==2022.12.7 +chardet==4.0.0 +urllib3==1.26.8 +requests==2.25.1 +zope.interface==5.2.0 pycparser==2.20 -pyOpenSSL==19.1.0 +pyOpenSSL==20.0.1 pyasn1==0.4.8 pyasn1-modules==0.2.8 service-identity==18.1.0 -cryptography==3.2.1 -cffi==1.14.3 -Twisted==20.3.0 -treq==20.9.0 +cryptography==3.4.5 +cffi==1.14.5 +Twisted==22.4.0 +treq==22.1.0 diff --git a/tasks/cmake/caches/util/xcode_sdk.cmake b/tasks/cmake/caches/util/xcode_sdk.cmake --- a/tasks/cmake/caches/util/xcode_sdk.cmake +++ b/tasks/cmake/caches/util/xcode_sdk.cmake @@ -51,6 +51,12 @@ # because of "CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY" explained just # above. We work around that by forcing it to NO. set(HAVE_RE_COMP CACHE BOOL NO) + + # Likewise, some benchmarks look for librt, which does not exist on darwin + # systems, but the check returns YES because we don't actually link, given + # the CMAKE_TRY_COMPILE_TARGET_TYPE setting above. We work around that by + # forcing it to NO. + set(HAVE_LIB_RT CACHE BOOL NO) else() # Search and use compiler coming with the SDK. # Note that we do not search CMAKE_CXX_COMPILER here. cmake will try diff --git a/tasks/tasktool/README.md b/tasks/tasktool/README.md --- a/tasks/tasktool/README.md +++ b/tasks/tasktool/README.md @@ -92,7 +92,7 @@ { "url": "https://github.com/my-project.git", "type": "git", - "default_rev": "refs/heads/master" + "default_rev": "refs/heads/main" } ``` @@ -166,7 +166,7 @@ : ${VARIABLE:='DefaultValue'} ``` -- Abstract artifacts like `refs/heads/master` in a git repository or +- Abstract artifacts like `refs/heads/main` in a git repository or `clang-stage2-cmake` on an A2 server are resolved to a concrete git revision or a specific URL when the task is started. - A task with a set of artifact and parameter inputs is called a build. diff --git a/tasks/tasktool/tasktool/repos/git.py b/tasks/tasktool/tasktool/repos/git.py --- a/tasks/tasktool/tasktool/repos/git.py +++ b/tasks/tasktool/tasktool/repos/git.py @@ -25,7 +25,7 @@ rev = config.get('rev') default_rev = config.pop('default_rev', None) if default_rev is None: - default_rev = "refs/heads/master" + default_rev = "refs/heads/main" if rev is not None: return url = config['url'] diff --git a/tasks/utils/venv.sh b/tasks/utils/venv.sh --- a/tasks/utils/venv.sh +++ b/tasks/utils/venv.sh @@ -1,5 +1,5 @@ echo "@@@ LNT VirtualEnv @@@" -/usr/local/bin/virtualenv venv +/usr/local/bin/virtualenv --python=python3 venv set +u . venv/bin/activate set -u diff --git a/test/jenkins/test_monorepo_build.py b/test/jenkins/test_monorepo_build.py --- a/test/jenkins/test_monorepo_build.py +++ b/test/jenkins/test_monorepo_build.py @@ -3,9 +3,9 @@ # RUN: export TESTING=1 # RUN: export JOB_NAME="FOO" # RUN: export BUILD_NUMBER=321 -# RUN: export BRANCH=master +# RUN: export BRANCH=main # Tell monorepo_build.py to just print commands instead of running. -# RUN: mkdir -p %t.SANDBOX/host-compiler/lib %t.SANDBOX/host-compiler/bin %t.SANDBOX/llvm-project/llvm %t.SANDBOX/llvm-project/clang %t.SANDBOX/llvm-project/libcxx %t.SANDBOX/llvm-project/compiler-rt %t.SANDBOX/llvm-project/debuginfo-tests %t.SANDBOX/llvm-project/clang-tools-extra %t.SANDBOX/llvm-project/lldb +# RUN: mkdir -p %t.SANDBOX/host-compiler/lib %t.SANDBOX/host-compiler/bin %t.SANDBOX/llvm-project/llvm %t.SANDBOX/llvm-project/clang %t.SANDBOX/llvm-project/compiler-rt %t.SANDBOX/llvm-project/debuginfo-tests %t.SANDBOX/llvm-project/clang-tools-extra %t.SANDBOX/llvm-project/lldb # RUN: touch %t.SANDBOX/host-compiler/bin/clang # RUN: python %{src_root}/zorg/jenkins/monorepo_build.py clang all > %t.log # RUN: FileCheck --check-prefix CHECK-SIMPLE < %t.log %s @@ -20,7 +20,7 @@ # CHECK-SIMPLE: '/usr/local/bin/cmake' '-G' 'Ninja' '-C' # CHECK-SIMPLE: '-DLLVM_ENABLE_ASSERTIONS:BOOL=FALSE' # CHECK-SIMPLE: '-DCMAKE_BUILD_TYPE=RelWithDebInfo' -# CHECK-SIMPLE: '-DLLVM_ENABLE_PROJECTS=clang;clang-tools-extra;compiler-rt;libcxx' +# CHECK-SIMPLE: '-DLLVM_ENABLE_PROJECTS=clang;clang-tools-extra;compiler-rt' # CHECK-SIMPLE: '-DCMAKE_MAKE_PROGRAM=/usr/local/bin/ninja' # CHECK-SIMPLE: '-DLLVM_VERSION_PATCH=99' # CHECK-SIMPLE: '-DLLVM_VERSION_SUFFIX=""' @@ -146,4 +146,14 @@ # Test to check if default timeout is being set to 600. # RUN: python %{src_root}/zorg/jenkins/monorepo_build.py cmake all > %t-timeout-default.log # RUN: FileCheck --check-prefix CHECK-TIMEOUT-DEFAULT < %t-timeout-default.log %s -# CHECK-TIMEOUT-DEFAULT: --timeout=600 \ No newline at end of file +# CHECK-TIMEOUT-DEFAULT: --timeout=600 + +# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py lldb-cmake-matrix configure \ +# RUN: --lldb-test-compiler="MY_LLDB_TEST_COMPILER" \ +# RUN: --cmake-flag=-DLLVM_TARGETS_TO_BUILD=X86 > %t-lldb-configure.log +# RUN: FileCheck --check-prefix CHECK-LLDB-CONFIG < %t-lldb-configure.log %s + +# CHECK-LLDB-CONFIG: -DLLDB_TEST_COMPILER=MY_LLDB_TEST_COMPILER +# CHECK-LLDB-CONFIG: -DLLVM_TARGETS_TO_BUILD=X86 +# there may be other "target to build", but the one above should be the last: +# CHECK-LLDB-CONFIG-NOT: -DLLVM_TARGETS_TO_BUILD diff --git a/zorg/buildbot/builders/AOSPBuilder.py b/zorg/buildbot/builders/AOSPBuilder.py --- a/zorg/buildbot/builders/AOSPBuilder.py +++ b/zorg/buildbot/builders/AOSPBuilder.py @@ -25,8 +25,9 @@ target_flags=None, # Extra C/CXX flags for AOSP target build jobs=None, # Number of concurrent jobs extra_make_args=None): # Extra args for the make command - command = "source build/envsetup.sh" + command = "/bin/bash -c \"source build/envsetup.sh" command += " && lunch aosp_%s-userdebug" % device + command += " && export LC_ALL=C" command += " && make -k" if timeout: command += " TIMEOUT=%s" % timeout @@ -38,6 +39,7 @@ command += " " + extra_make_args if jobs: command += " -j" + str(jobs) + command += "\"" return command diff --git a/zorg/buildbot/builders/AnnotatedBuilder.py b/zorg/buildbot/builders/AnnotatedBuilder.py --- a/zorg/buildbot/builders/AnnotatedBuilder.py +++ b/zorg/buildbot/builders/AnnotatedBuilder.py @@ -11,7 +11,9 @@ env=None, extra_args=None, timeout=1200, - checkout_llvm_sources=True): + checkout_llvm_sources=True, + script_interpreter="python", + warnOnWarnings=False): """ Returns a new build factory that uses AnnotatedCommand, which allows the build to be run by version-controlled scripts that do @@ -26,6 +28,7 @@ env: environment overrides (map; default is no overrides) extra_args: extra arguments to pass to the script (default: []) timeout: specifies the builder's timeout in seconds (default: 1200) + script_interpreter: specifies the interpreter to run scripts (default: "python") """ if depends_on_projects is None: @@ -38,7 +41,10 @@ "libunwind", "lld"] if extra_args is None: - extra_args = [] + # We used to add --jobs to all script invocations. Perserve this + # for cases when the user did not specify extra_args, but allow + # overriding it if the user did specify extra_args. + extra_args = ["--jobs=%(jobs:-)s"] f = LLVMBuildFactory( clean=clean, @@ -84,13 +90,19 @@ command = [script] else: script_path = "../llvm-zorg/zorg/buildbot/builders/annotated/%s" % (script) - command = ["python", script_path, WithProperties("--jobs=%(jobs:-)s")] + # Handle scripts with script_interpreter, otherwise execute the script directly. + if script_interpreter: + command = [script_interpreter, script_path] + else: + command = [script_path] + command += extra_args_with_props f.addStep(AnnotatedCommand(name="annotate", description="annotate", timeout=timeout, haltOnFailure=True, + warnOnWarnings=warnOnWarnings, command=command, env=merged_env)) return f diff --git a/zorg/buildbot/builders/BOLTBuilder.py b/zorg/buildbot/builders/BOLTBuilder.py new file mode 100644 --- /dev/null +++ b/zorg/buildbot/builders/BOLTBuilder.py @@ -0,0 +1,128 @@ +from buildbot.plugins import steps +from buildbot.steps.shell import ShellCommand +from zorg.buildbot.builders.UnifiedTreeBuilder import getLLVMBuildFactoryAndSourcecodeSteps, addCmakeSteps, addNinjaSteps +from zorg.buildbot.commands.LitTestCommand import LitTestCommand +from zorg.buildbot.commands.CmakeCommand import CmakeCommand +from zorg.buildbot.conditions.FileConditions import FileExists +from zorg.buildbot.process.factory import LLVMBuildFactory + +def getBOLTCmakeBuildFactory( + clean = False, + bolttests = False, + is_nfc = False, + targets = None, + checks = None, + caches = None, + extra_configure_args = None, + env = None, + depends_on_projects = None, + **kwargs): + + if env is None: + env = dict() + + bolttests_dir = "bolt-tests" + + cleanBuildRequested = lambda step: clean or step.build.getProperty("clean", default=step.build.getProperty("clean_obj")) + cleanBuildRequestedByProperty = lambda step: step.build.getProperty("clean") + + if not targets: + targets = ['bolt'] + if not checks: + checks = ['check-bolt'] + + f = getLLVMBuildFactoryAndSourcecodeSteps( + depends_on_projects=depends_on_projects, + **kwargs) # Pass through all the extra arguments. + + if bolttests: + checks += ['check-large-bolt'] + extra_configure_args += [ + '-DLLVM_EXTERNAL_PROJECTS=bolttests', + '-DLLVM_EXTERNAL_BOLTTESTS_SOURCE_DIR=' + LLVMBuildFactory.pathRelativeTo(bolttests_dir, f.monorepo_dir), + ] + # Clean checkout of bolt-tests if cleanBuildRequested + f.addSteps([ + steps.RemoveDirectory(name="BOLT tests: clean", + dir=bolttests_dir, + haltOnFailure=True, + warnOnFailure=True, + doStepIf=cleanBuildRequestedByProperty), + + steps.Git(name="BOLT tests: checkout", + description="fetching", + descriptionDone="fetch", + descriptionSuffix="BOLT Tests", + repourl='https://github.com/rafaelauler/bolt-tests.git', + workdir=bolttests_dir, + alwaysUseLatest=True), + ]) + + # Some options are required for this build no matter what. + CmakeCommand.applyRequiredOptions(extra_configure_args, [ + ('-G', 'Ninja'), + ]) + + if caches: + for cache in caches: + extra_configure_args += [f"-C../{f.monorepo_dir}/{cache}"] + + addCmakeSteps( + f, + cleanBuildRequested=cleanBuildRequested, + extra_configure_args=extra_configure_args, + obj_dir=None, + env=env, + **kwargs) + + addNinjaSteps( + f, + targets=targets, + checks=checks, + env=env, + **kwargs) + + if is_nfc: + f.addSteps([ + ShellCommand( + name='nfc-check-setup', + command=[f"../{f.monorepo_dir}/bolt/utils/nfc-check-setup.py"], + description=('Setup NFC testing'), + warnOnFailure=True, + haltOnFailure=False, + flunkOnFailure=False, + env=env), + ShellCommand( + name='check-bolt-different', + command='rm -f .llvm-bolt.diff; cmp -s bin/llvm-bolt.old bin/llvm-bolt.new || touch .llvm-bolt.diff', + description=('Check if llvm-bolt binaries are different and ' + 'skip the following nfc-check steps'), + haltOnFailure=False, + env=env), + LitTestCommand( + name='nfc-check-bolt', + command=['bin/llvm-lit', '-sv', '-j4', + # bolt-info will always mismatch in NFC mode + '--xfail=bolt-info.test', + 'tools/bolt/test'], + description=["running", "NFC", "check-bolt"], + descriptionDone=["NFC", "check-bolt", "completed"], + warnOnFailure=True, + haltOnFailure=False, + flunkOnFailure=False, + doStepIf=FileExists('build/.llvm-bolt.diff'), + env=env), + LitTestCommand( + name='nfc-check-large-bolt', + command=['bin/llvm-lit', '-sv', '-j2', + 'tools/bolttests'], + description=["running", "NFC", "check-large-bolt"], + descriptionDone=["NFC", "check-large-bolt", "completed"], + warnOnFailure=True, + haltOnFailure=False, + flunkOnFailure=False, + doStepIf=FileExists('build/.llvm-bolt.diff'), + env=env), + ]) + + return f diff --git a/zorg/buildbot/builders/ClangBuilder.py b/zorg/buildbot/builders/ClangBuilder.py --- a/zorg/buildbot/builders/ClangBuilder.py +++ b/zorg/buildbot/builders/ClangBuilder.py @@ -1,6 +1,7 @@ import copy from datetime import datetime +from buildbot.plugins import util from buildbot.process.properties import WithProperties, Property from buildbot.steps.shell import ShellCommand, SetProperty from buildbot.steps.shell import WarningCountingShellCommand @@ -71,7 +72,7 @@ def getClangCMakeGCSBuildFactory( clean=True, - test=True, + checks=None, cmake='cmake', jobs=None, @@ -114,7 +115,7 @@ # Triggers trigger_after_stage1=None): return _getClangCMakeBuildFactory( - clean=clean, test=test, cmake=cmake, jobs=jobs, vs=vs, + clean=clean, checks=checks, cmake=cmake, jobs=jobs, vs=vs, vs_target_arch=vs_target_arch, useTwoStage=useTwoStage, testStage1=testStage1, stage1_config=stage1_config, stage2_config=stage2_config, runTestSuite=runTestSuite, @@ -132,9 +133,10 @@ def getClangCMakeBuildFactory( clean=True, - test=True, + checks=None, cmake='cmake', jobs=None, + timeout=None, # TODO: Implement support for timeout # VS tools environment variable if using MSVC. For example, # %VS120COMNTOOLS% selects the 2013 toolchain. @@ -164,9 +166,11 @@ checkout_lld=True, checkout_libcxx=False, checkout_flang=False, - checkout_test_suite=False): + checkout_test_suite=False, + + enable_runtimes="auto"): return _getClangCMakeBuildFactory( - clean=clean, test=test, cmake=cmake, jobs=jobs, vs=vs, + clean=clean, checks=checks, cmake=cmake, jobs=jobs, vs=vs, vs_target_arch=vs_target_arch, useTwoStage=useTwoStage, testStage1=testStage1, stage1_config=stage1_config, stage2_config=stage2_config, runTestSuite=runTestSuite, @@ -178,11 +182,12 @@ checkout_compiler_rt=checkout_compiler_rt, checkout_libcxx=checkout_libcxx, checkout_flang=checkout_flang, - checkout_test_suite=checkout_test_suite) + checkout_test_suite=checkout_test_suite, + enable_runtimes=enable_runtimes) def _getClangCMakeBuildFactory( clean=True, - test=True, + checks=None, cmake='cmake', jobs=None, @@ -216,6 +221,8 @@ checkout_test_suite=False, checkout_flang=False, + enable_runtimes="auto", + # Upload artifacts to Google Cloud Storage (for the llvmbisect tool) stage1_upload_directory=None, @@ -228,6 +235,8 @@ trigger_after_stage1=None): ############# PREPARING + if checks is None: + checks = ['check-all'] if nt_flags is None: nt_flags = [] if testsuite_flags is None: @@ -263,7 +272,8 @@ f = LLVMBuildFactory( depends_on_projects=depends_on_projects, - llvm_srcdir='llvm') + llvm_srcdir='llvm', + enable_runtimes=enable_runtimes) # Checkout the latest code for LNT # and the test-suite separately. Le's do this first, @@ -291,7 +301,7 @@ lit_args += "'" ninja_cmd = ['ninja'] + jobs_cmd ninja_install_cmd = ['ninja', 'install'] + jobs_cmd - ninja_check_cmd = ['ninja', 'check-all'] + jobs_cmd + ninja_check_cmd = ['ninja'] + checks + jobs_cmd # Global configurations stage1_build = 'stage1' @@ -321,9 +331,15 @@ ############# STAGE 1 - CmakeCommand.applyRequiredOptions(extra_cmake_args, [ - ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)), - ]) + if f.enable_projects: + CmakeCommand.applyRequiredOptions(extra_cmake_args, [ + ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.enable_projects)), + ]) + if f.enable_runtimes: + CmakeCommand.applyRequiredOptions(extra_cmake_args, [ + ('-DLLVM_ENABLE_RUNTIMES=', ";".join(f.enable_runtimes)), + ]) + rel_src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, stage1_build) f.addStep(ShellCommand(name='cmake stage 1', @@ -346,7 +362,7 @@ workdir=stage1_build, env=env)) - if test and testStage1: + if checks and testStage1: haltOnStage1Check = not useTwoStage and not runTestSuite f.addStep(LitTestCommand(name='ninja check 1', command=ninja_check_cmd, @@ -382,9 +398,11 @@ if not vs: cc = 'clang' cxx = 'clang++' + fc = 'flang-new' else: cc = 'clang-cl.exe' cxx = 'clang-cl.exe' + fc = 'flang-new.exe' ############# STAGE 2 @@ -435,7 +453,7 @@ workdir=stage2_build, env=env)) - if test: + if checks: f.addStep(LitTestCommand(name='ninja check 2', command=ninja_check_cmd, haltOnFailure=not runTestSuite, @@ -496,6 +514,16 @@ '--cc', cc, '--cxx', cxx, '--use-lit', lit] + # Enable fortran if flang is checked out + if checkout_flang: + fortran_flags = [ + '--cmake-define=TEST_SUITE_FORTRAN:STRING=ON', + '--cmake-define=CMAKE_Fortran_FLAGS:STRING=' + + '-lpgmath', + util.Interpolate( + '--cmake-define=CMAKE_Fortran_COMPILER=' + + '%(prop:builddir)s/'+compiler_path+'/bin/'+fc)] + test_suite_cmd.extend(fortran_flags) # Append any option provided by the user test_suite_cmd.extend(testsuite_flags) @@ -521,7 +549,7 @@ workdir='test', env=env)) f.addStep(ShellCommand(name='recreate sandbox', - command=['virtualenv', 'sandbox'], + command=['virtualenv', '--python=python3', 'sandbox'], haltOnFailure=True, description='recreating sandbox', workdir='test', diff --git a/zorg/buildbot/builders/ClangLTOBuilder.py b/zorg/buildbot/builders/ClangLTOBuilder.py --- a/zorg/buildbot/builders/ClangLTOBuilder.py +++ b/zorg/buildbot/builders/ClangLTOBuilder.py @@ -65,9 +65,14 @@ ('-DCMAKE_INSTALL_PREFIX=', install_dir), ]) - CmakeCommand.applyRequiredOptions(cmake_args, [ - ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)), - ]) + if f.enable_projects: + CmakeCommand.applyRequiredOptions(cmake_args, [ + ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.enable_projects)), + ]) + if f.enable_runtimes: + CmakeCommand.applyRequiredOptions(cmake_args, [ + ('-DLLVM_ENABLE_RUNTIMES=', ";".join(f.enable_runtimes)), + ]) # Note: On this stage we do not care of warnings, as we build with # a system toolchain and cannot control the environment. @@ -181,9 +186,14 @@ "-DCMAKE_C_COMPILER=%(builddir)s/" + staged_install + "/bin/clang" )) - CmakeCommand.applyRequiredOptions(cmake_args, [ - ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)), - ]) + if f.enable_projects: + CmakeCommand.applyRequiredOptions(cmake_args, [ + ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.enable_projects)), + ]) + if f.enable_runtimes: + CmakeCommand.applyRequiredOptions(cmake_args, [ + ('-DLLVM_ENABLE_RUNTIMES=', ";".join(f.enable_runtimes)), + ]) # Create configuration files with cmake f.addStep(CmakeCommand(name="cmake-configure-stage%s" % stage_num, @@ -233,6 +243,7 @@ clean = False, jobs = None, extra_configure_args = None, + extra_configure_args_lto_stage = None, compare_last_2_stages = True, lto = None, # The string gets passed to -flto flag as is. Like -flto=thin. env = None, @@ -253,10 +264,16 @@ else: extra_configure_args = list(extra_configure_args) + if extra_configure_args_lto_stage is None: + extra_configure_args_lto_stage = [] + else: + extra_configure_args_lto_stage = list(extra_configure_args_lto_stage) + # Make sure CMAKE_INSTALL_PREFIX and -G are not specified # in the extra_configure_args. We set them internally as needed. # TODO: assert extra_configure_args. - install_prefix_specified = any(a.startswith('-DCMAKE_INSTALL_PREFIX=') for a in extra_configure_args) + install_prefix_specified = (any(a.startswith('-DCMAKE_INSTALL_PREFIX=') for a in extra_configure_args) or + any(a.startswith('-DCMAKE_INSTALL_PREFIX=') for a in extra_configure_args_lto_stage)) assert not install_prefix_specified, "Please do not explicitly specify the install prefix for multi-stage build." # Prepare environmental variables. Set here all env we want everywhere. @@ -308,6 +325,10 @@ CmakeCommand.applyRequiredOptions(extra_configure_args, [ ('-DLLVM_ENABLE_LTO=', lto), + # NOTE: Diagnostic messages could contain __FILE__, thus final executable would depend on build path. + # We do not want that since we expect executables built from the same source code in different + # build directories be exactly the same. + ('-DLLVM_ENABLE_ASSERTIONS=', 'OFF'), ]) # If we build LLD, we would link with LLD. @@ -321,7 +342,7 @@ s = f.staged_compiler_idx + 1 staged_install = f.stage_installdirs[f.staged_compiler_idx] for i in range(s, len(f.stage_objdirs[s:]) + s): - configure_args = extra_configure_args[:] + configure_args = extra_configure_args[:] + extra_configure_args_lto_stage[:] configure_args.append( WithProperties( diff --git a/zorg/buildbot/builders/DoxygenDocsBuilder.py b/zorg/buildbot/builders/DoxygenDocsBuilder.py new file mode 100644 --- /dev/null +++ b/zorg/buildbot/builders/DoxygenDocsBuilder.py @@ -0,0 +1,122 @@ +from collections import OrderedDict +from importlib import reload + +from buildbot.plugins import steps, util +from buildbot.steps.shell import ShellCommand + +from zorg.buildbot.builders import UnifiedTreeBuilder +from zorg.buildbot.commands import CmakeCommand + +from zorg.buildbot.process import factory +reload(factory) +reload(UnifiedTreeBuilder) +reload(CmakeCommand) + +llvm_docs = OrderedDict([ + # Project Build target Local path Remote path + ("llvm", ("doxygen", "docs/doxygen/html/", "llvm")), + ("clang", (None, "tools/clang/docs/doxygen/html/", "cfe")), + ("clang-tools-extra", (None, "tools/clang/tools/extra/docs/doxygen/html/", "cfe-extra")), + ("flang", (None, "tools/flang/docs/doxygen/html/", "flang")), + ("polly", (None, "tools/polly/docs/doxygen/html/", "polly")), + ("openmp", (None, "projects/openmp/docs/doxygen/html/", "openmp")), + ("lldb", ("lldb-cpp-doc", "tools/lldb/docs/cpp_reference/", "lldb/cpp_reference")), + # NOTE: 5/9/2020 lldb-python-doc fails to build. Disabled till be fixed. + #(None, ("lldb-python-doc", "tools/lldb/docs/python_reference/", "lldb")), +]) + +def getLLVMDocsBuildFactory( + clean = True, + depends_on_projects = None, + extra_configure_args = None, + timeout=10800, + env = None, + **kwargs): + + if depends_on_projects is None: + # All the projects from llvm_docs, and remove all duplicates. + _depends_on_projects=list(set( + [project for project in llvm_docs if project])) + else: + # Make a local copy of depends_on_projects, as we are going to modify + # that. + _depends_on_projects=depends_on_projects[:] + # Some projects are interdependent for the purpose of documentation. + # Enforce the dependencies. + # TODO: Check later the dependencies for doxygen docs and enforce them + # here if needed. + + # Make a local copy of the configure args, as we are going to modify that. + if extra_configure_args: + cmake_args = extra_configure_args[:] + else: + cmake_args = list() + + # Prepare environmental variables. Set here all env we want everywhere. + merged_env = { + 'TERM' : 'dumb' # Be cautious and disable color output from all tools. + } + if env is not None: + # Overwrite pre-set items with the given ones, so user can set anything. + merged_env.update(env) + + CmakeCommand.CmakeCommand.applyDefaultOptions(cmake_args, [ + ("-G", "Ninja"), + ("-DLLVM_ENABLE_DOXYGEN=", "ON"), + ("-DLLVM_BUILD_DOCS=", "ON"), + ("-DCLANG_TOOLS_EXTRA_INCLUDE_DOCS=", "ON"), + ("-DLLVM_ENABLE_ASSERTIONS=", "OFF"), + ("-DCMAKE_BUILD_TYPE=", "Release"), + ]) + + f = UnifiedTreeBuilder.getCmakeBuildFactory( + clean=clean, + depends_on_projects=_depends_on_projects, + enable_runtimes=[], # Docs don't support runtimes build yet. + extra_configure_args=cmake_args, + env=merged_env, + **kwargs) # Pass through all the extra arguments. + + # Build the documentation for all the projects. + for project in llvm_docs: + target = llvm_docs[project][0] + + # Build only those with specifies targets. + if target: + UnifiedTreeBuilder.addNinjaSteps( + f, + # Doxygen builds the final result for really + # long time without any output. + # We have to have a long timeout at this step. + timeout=timeout, + targets=[target], + checks=[], + env=merged_env, + **kwargs) + + # Publish just built documentation + for project in llvm_docs: + target, local_path, remote_path = llvm_docs[project] + + f.addStep( + ShellCommand( + name="Publish {}".format(project or target), + description=[ + "Publish", "just", "built", "documentation", "for", + "{}".format(project or target) + ], + command=[ + 'rsync', + '-vrl', + '--delete', '--force', '--delay-updates', '--delete-delay', + '--ignore-times', + '--checksum', + '-p', '--chmod=Du=rwx,Dg=rwx,Do=rx,Fu=rw,Fg=rw,Fo=r', + "{}".format(local_path), + "lists.llvm.org:web/doxygen/{}".format(remote_path), + ], + env=merged_env, + ) + ) + + return f diff --git a/zorg/buildbot/builders/FlangBuilder.py b/zorg/buildbot/builders/FlangBuilder.py --- a/zorg/buildbot/builders/FlangBuilder.py +++ b/zorg/buildbot/builders/FlangBuilder.py @@ -14,7 +14,7 @@ env = dict() f = getCmakeWithNinjaBuildFactory( - depends_on_projects=['llvm','mlir'], + depends_on_projects=['llvm','clang','mlir'], obj_dir="build_llvm", checks=[], clean=clean, @@ -42,6 +42,7 @@ # Add LLVM_DIR and MLIR_DIR to the CMake invocation. llvm_dir = "{}/lib/cmake/llvm".format(f.obj_dir) mlir_dir = "{}/lib/cmake/mlir".format(f.obj_dir) + clang_dir = "{}/lib/cmake/clang".format(f.obj_dir) CmakeCommand.applyRequiredOptions(flang_cmake_args, [ # We actually need the paths to be relative to the source directory, # otherwise find_package can't locate the config files. @@ -49,6 +50,8 @@ LLVMBuildFactory.pathRelativeTo(llvm_dir, flang_src_dir)), ('-DMLIR_DIR:PATH=', LLVMBuildFactory.pathRelativeTo(mlir_dir, flang_src_dir)), + ('-DCLANG_DIR:PATH=', + LLVMBuildFactory.pathRelativeTo(clang_dir, flang_src_dir)), ]) # We can't use addCmakeSteps as that would use the path in f.llvm_srcdir. diff --git a/zorg/buildbot/builders/FuchsiaBuilder.py b/zorg/buildbot/builders/FuchsiaBuilder.py --- a/zorg/buildbot/builders/FuchsiaBuilder.py +++ b/zorg/buildbot/builders/FuchsiaBuilder.py @@ -28,14 +28,16 @@ f = UnifiedTreeBuilder.getLLVMBuildFactoryAndSourcecodeSteps( depends_on_projects=[ - "llvm", + "bolt", "clang", "clang-tools-extra", "compiler-rt", "libcxx", "libcxxabi", "libunwind", - "lld" + "lld", + "llvm", + "polly", ], llvm_srcdir=src_dir, obj_dir=obj_dir, @@ -92,8 +94,6 @@ # Some options are required for this stage no matter what. CmakeCommand.applyRequiredOptions(cmake_options, [ ("-G", "Ninja"), - ("-DLLVM_ENABLE_PROJECTS=", "clang;clang-tools-extra;lld"), - ("-DLLVM_ENABLE_RUNTIMES=", "compiler-rt;libcxx;libcxxabi;libunwind"), ]) # Set proper defaults. diff --git a/zorg/buildbot/builders/HtmlDocsBuilder.py b/zorg/buildbot/builders/HtmlDocsBuilder.py new file mode 100644 --- /dev/null +++ b/zorg/buildbot/builders/HtmlDocsBuilder.py @@ -0,0 +1,105 @@ +from collections import OrderedDict +from importlib import reload + +from buildbot.plugins import steps, util +from buildbot.steps.shell import ShellCommand +from buildbot.steps.shell import WarningCountingShellCommand + +from zorg.buildbot.process import factory +reload(factory) + +llvm_docs = OrderedDict([ + # Project Build target Build path Local path Remote path + ("lnt", ("html", "docs", "_build/html/", "lnt")), +]) + +# We build with make for now. Change later if needed. +build_cmd = 'make' + +def getHtmlDocsBuildFactory( + depends_on_projects = None, + clean = False, + env = None, + **kwargs): + + if depends_on_projects is None: + # All the projects by default. + _depends_on_projects=llvm_docs.keys() + else: + # Make a local copy of depends_on_projects, as we are going to modify + # that. + _depends_on_projects=depends_on_projects[:] + + # Prepare environmental variables. Set here all env we want everywhere. + merged_env = { + 'TERM' : 'dumb' # Be cautious and disable color output from all tools. + } + if env is not None: + # Overwrite pre-set items with the given ones, so user can set anything. + merged_env.update(env) + + # HTML Sphinx documentation builds in tree, each in its own directory. + # For that, make sure the obj_dir is the same as llvm_srcdir. + src_dir = kwargs.pop('llvm_srcdir', '.') + f = factory.LLVMBuildFactory( + clean=clean, + depends_on_projects=_depends_on_projects, + llvm_srcdir=src_dir, + obj_dir=src_dir, + **kwargs) # Pass through all the extra arguments. + + # Build the documentation + for project in llvm_docs: + + # Checkout the source code and remove all the untracked files, so + # we would build a fresh new documentation. + f.addStep( + steps.Git( + name='Checkout the {} source code'.format(project), + repourl=f.repourl_prefix + "llvm-{}.git".format(project), + mode='full', + method='fresh', + progress=True, + workdir=util.WithProperties(project), + env=merged_env, + **kwargs)) + + target, build_path, local_path, remote_path = llvm_docs[project] + + build_dir = util.WithProperties( + "{}".format("/".join([ + project, + build_path]))) + f.addStep( + steps.WarningCountingShellCommand( + name="Build {} documentation".format(project), + command=[build_cmd, target], + haltOnFailure=True, + workdir=build_dir, + env=merged_env, + **kwargs)) + + # Publish just built documentation + f.addStep( + ShellCommand( + name="Publish {}".format(project), + description=[ + "Publish", "just", "built", "documentation", "for", + "{}".format(project) + ], + command=[ + 'rsync', + '-vrl', + '--delete', '--force', '--delay-updates', '--delete-delay', + '--ignore-times', + '--checksum', + '-p', '--chmod=Du=rwx,Dg=rwx,Do=rx,Fu=rw,Fg=rw,Fo=r', + "{}".format(local_path), + "lists.llvm.org:web/{}".format(remote_path), + ], + workdir=build_dir, + env=merged_env, + ) + ) + + return f diff --git a/zorg/buildbot/builders/LLDBBuilder.py b/zorg/buildbot/builders/LLDBBuilder.py --- a/zorg/buildbot/builders/LLDBBuilder.py +++ b/zorg/buildbot/builders/LLDBBuilder.py @@ -48,6 +48,7 @@ build_cmd=['ninja'] install_cmd = ['ninja','install'] test_cmd = ['ninja','check-lldb'] + lit_args = '-v' if jobs: build_cmd.append(WithProperties("-j%s" % jobs)) @@ -67,6 +68,7 @@ cmake_options = [ "-G", "Ninja", "-DCMAKE_BUILD_TYPE=" + config, + "-DLLVM_LIT_ARGS='%s'" % lit_args, "-DCMAKE_INSTALL_PREFIX=../install", "-DLLVM_ENABLE_PROJECTS=%s" % ";".join(f.depends_on_projects), ] diff --git a/zorg/buildbot/builders/LibcxxAndAbiBuilder.py b/zorg/buildbot/builders/LibcxxAndAbiBuilder.py deleted file mode 100644 --- a/zorg/buildbot/builders/LibcxxAndAbiBuilder.py +++ /dev/null @@ -1,132 +0,0 @@ -import buildbot.steps.shell -import buildbot.process.properties as properties - -from zorg.buildbot.commands.LitTestCommand import LitTestCommand -from zorg.buildbot.commands.CmakeCommand import CmakeCommand -from zorg.buildbot.process.factory import LLVMBuildFactory -from zorg.buildbot.builders import UnifiedTreeBuilder - -def getLibcxxAndAbiBuilder(f=None, env=None, - cmake_extra_opts=None, lit_extra_opts=None, - lit_extra_args=None, check_libcxx_abilist=False, - check_libcxx_benchmarks=None, - depends_on_projects=None, - use_cache=None, - **kwargs): - - if env is None: - env = {} - if cmake_extra_opts is None: - cmake_extra_opts = {} - if lit_extra_opts is None: - lit_extra_opts = {} - if lit_extra_args is None: - lit_extra_args = [] - - if depends_on_projects is None: - depends_on_projects = ['libcxx','libcxxabi','libunwind'] - - src_root = 'llvm' - build_path = 'build' - - if f is None: - f = UnifiedTreeBuilder.getLLVMBuildFactoryAndSourcecodeSteps( - depends_on_projects=depends_on_projects, - llvm_srcdir=src_root, - obj_dir=build_path, - **kwargs) # Pass through all the extra arguments. - - rel_src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, build_path) - - # Specify the max number of threads using properties so LIT doesn't use - # all the threads on the system. - litTestArgs = '-vv --show-unsupported --show-xfail --threads=%(jobs)s' - if lit_extra_args: - litTestArgs += ' ' + ' '.join(lit_extra_args) - - for key in lit_extra_opts: - litTestArgs += (' --param=' + key + '=' + lit_extra_opts[key]) - - cmake_opts = [properties.WithProperties('-DLLVM_LIT_ARGS='+litTestArgs)] - for key in cmake_extra_opts: - cmake_opts.append('-D' + key + '=' + cmake_extra_opts[key]) - - if use_cache: - libcxx_cache_dir = '%s/../libcxx/cmake/caches' % rel_src_dir - cache = '%s/%s' % (libcxx_cache_dir, use_cache) - cmake_opts.append('-C' + cache) - - # FIXME: The libc++ abilist's are generated in release mode with debug - # symbols Other configurations may contain additional non-inlined symbols. - if check_libcxx_abilist and not 'CMAKE_BUILD_TYPE' in cmake_extra_opts: - cmake_opts.append('-DCMAKE_BUILD_TYPE=RELWITHDEBINFO') - - # Force libc++ to use the in-tree libc++abi unless otherwise specified. - if 'LIBCXX_CXX_ABI' not in cmake_extra_opts: - cmake_opts.append('-DLIBCXX_CXX_ABI=libcxxabi') - - # Nuke/remake build directory and run CMake - f.addStep(buildbot.steps.shell.ShellCommand( - name='rm.builddir', command=['rm', '-rf', build_path], - workdir=".", - haltOnFailure=False)) - - CmakeCommand.applyRequiredOptions(cmake_opts, [ - ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)), - ]) - - f.addStep(buildbot.steps.shell.ShellCommand( - name='cmake', command=['cmake', rel_src_dir] + cmake_opts, - haltOnFailure=True, workdir=build_path, env=env)) - - # Build libcxxabi - jobs_flag = properties.WithProperties('-j%(jobs)s') - f.addStep(buildbot.steps.shell.ShellCommand( - name='build.libcxxabi', command=['make', jobs_flag, 'cxxabi'], - haltOnFailure=True, workdir=build_path)) - - # Build libcxx - f.addStep(buildbot.steps.shell.ShellCommand( - name='build.libcxx', command=['make', jobs_flag, 'cxx'], - haltOnFailure=True, workdir=build_path)) - - # Test libc++abi - f.addStep(LitTestCommand( - name = 'test.libcxxabi', - command = ['make', jobs_flag, 'check-cxxabi'], - description = ['testing', 'libcxxabi'], - descriptionDone = ['test', 'libcxxabi'], - workdir = build_path)) - - # Test libc++ - f.addStep(LitTestCommand( - name = 'test.libcxx', - command = ['make', jobs_flag, 'check-cxx'], - description = ['testing', 'libcxx'], - descriptionDone = ['test', 'libcxx'], - workdir = build_path)) - - if check_libcxx_abilist: - f.addStep(buildbot.steps.shell.ShellCommand( - name = 'test.libcxx.abilist', - command = ['make', 'check-cxx-abilist'], - description = ['testing', 'libcxx', 'abi'], - descriptionDone = ['test', 'libcxx', 'abi'], - workdir = build_path)) - - if check_libcxx_benchmarks: - # Build the libc++ benchmarks - f.addStep(buildbot.steps.shell.ShellCommand( - name='build.libcxx.benchmarks', - command=['make', jobs_flag, 'cxx-benchmarks'], - haltOnFailure=True, workdir=build_path)) - - # Run the benchmarks - f.addStep(LitTestCommand( - name = 'test.libcxx.benchmarks', - command = ['make', jobs_flag, 'check-cxx-benchmarks'], - description = ['testing', 'libcxx', 'benchmarks'], - descriptionDone = ['test', 'libcxx', 'benchmarks'], - workdir = build_path)) - - return f diff --git a/zorg/buildbot/builders/OpenMPBuilder.py b/zorg/buildbot/builders/OpenMPBuilder.py --- a/zorg/buildbot/builders/OpenMPBuilder.py +++ b/zorg/buildbot/builders/OpenMPBuilder.py @@ -1,5 +1,6 @@ from buildbot.steps.shell import ShellCommand from buildbot.process.properties import WithProperties +from buildbot.plugins import steps from zorg.buildbot.commands.LitTestCommand import LitTestCommand from zorg.buildbot.commands.CmakeCommand import CmakeCommand @@ -11,11 +12,22 @@ jobs = '%(jobs)s', # Number of concurrent jobs. clean = True, # "clean" step is requested if true env = None, # Environmental variables for all steps. - ompt = False, # Whether to enable the OpenMP Tools Interface. test = True, # Test the built libraries. depends_on_projects = None, + enable_runtimes = "auto", + extraCmakeArgs = None, + install = False, + testsuite = False, + testsuite_sollvevv = False, + extraTestsuiteCmakeArgs = None, + add_lit_checks = None, **kwargs): + if extraCmakeArgs is None: + extraCmakeArgs = [] + if extraTestsuiteCmakeArgs is None: + extraTestsuiteCmakeArgs = [] + # Prepare environmental variables. Set here all env we want everywhere. merged_env = { 'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences. @@ -26,8 +38,17 @@ llvm_srcdir = 'llvm.src' llvm_builddir = 'llvm.build' + llvm_instdir = 'llvm.inst' + testsuite_srcdir = "test-suite.src" + testsuite_builddir = "test-suite.build" + sollvevv_srcdir = "sollvevv.src" - cleanBuildRequested = lambda step: clean or step.build.getProperty("clean", default=step.build.getProperty("clean_obj")) + # If true, clean everything, including source dirs + def cleanBuildRequested(step): + return step.build.getProperty("clean") + # If true, clean build products; implied if cleanBuildRequested is true + def cleanObjRequested(step): + return cleanBuildRequested(step) or clean or step.build.getProperty("clean_obj") if depends_on_projects is None: # Monorepo configuration requires llvm and clang to get cmake work. @@ -35,33 +56,36 @@ f = UnifiedTreeBuilder.getLLVMBuildFactoryAndSourcecodeSteps( depends_on_projects=depends_on_projects, + enable_runtimes=enable_runtimes, llvm_srcdir=llvm_srcdir, obj_dir=llvm_builddir, cleanBuildRequested=cleanBuildRequested, env=merged_env, **kwargs) # Pass through all the extra arguments. - f.addStep( - ShellCommand( - name = 'clean', - command = ['rm', '-rf', f.obj_dir], - warnOnFailure = True, - description = ['clean'], - doStepIf = cleanBuildRequested, - workdir = '.', - env = merged_env)) + f.addStep(steps.RemoveDirectory(name='clean', + dir=f.obj_dir, + warnOnFailure=True, + doStepIf=cleanObjRequested)) # Configure LLVM and OpenMP (and Clang, if requested). cmake_args = ['-DCMAKE_BUILD_TYPE=Release', '-DLLVM_ENABLE_ASSERTIONS=ON'] - if ompt: - cmake_args += ['-DLIBOMP_OMPT_SUPPORT=ON'] if test: lit_args = '-vv --show-unsupported --show-xfail -j %s' % jobs cmake_args += [WithProperties('-DLLVM_LIT_ARGS=%s' % lit_args)] + if install: + cmake_args += [WithProperties('-DCMAKE_INSTALL_PREFIX=%(builddir)s/' + llvm_instdir)] + cmake_args += extraCmakeArgs - CmakeCommand.applyRequiredOptions(cmake_args, [ - ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)), - ]) + if f.enable_projects: + CmakeCommand.applyDefaultOptions(cmake_args, [ + ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.enable_projects)), + ]) + + if f.enable_runtimes: + CmakeCommand.applyDefaultOptions(cmake_args, [ + ('-DLLVM_ENABLE_RUNTIMES=', ";".join(f.enable_runtimes)), + ]) # Add llvm-lit and clang (if built) to PATH merged_env.update({ @@ -103,6 +127,134 @@ description = 'test openmp', workdir = f.obj_dir, env = merged_env, - haltOnFailure=True)) + haltOnFailure=False, + flunkOnFailure=True)) + # When requested run additional lit tests + if add_lit_checks != None: + for add_check in add_lit_checks: + ninja_test_args = ['ninja', WithProperties('-j %s' % jobs)] + + f.addStep(LitTestCommand( + name = 'Add check ' + add_check, + command = [ninja_test_args, add_check], + description = ["Additional check in OpenMP for", add_check,], + env = merged_env, + workdir = f.obj_dir, + haltOnFailure = False, + flunkOnFailure=True)) + + clangexe = "%(builddir)s/" + llvm_builddir + "/bin/clang" + clangxxexe = "%(builddir)s/" + llvm_builddir + "/bin/clang++" + litexe = "%(builddir)s/" + llvm_builddir + "/bin/llvm-lit" + libdir = "%(builddir)s/" + llvm_builddir + "/lib" + if install: + f.addStep(steps.RemoveDirectory(name="LLVM: Clean Install Dir", + dir=llvm_instdir, + haltOnFailure=False)) + + f.addStep(NinjaCommand(name="LLVM: Install", + description="installing", + descriptionDone="install", + descriptionSuffix="LLVM", + workdir=f.obj_dir, + targets=['install'], + env=merged_env, + haltOnFailure=True)) + # If installing, use the installed version of clang. + clangexe = "%(builddir)s/" + llvm_instdir + "/bin/clang" + clangxxexe = "%(builddir)s/" + llvm_instdir + "/bin/clang++" + libdir = "%(builddir)s/" + llvm_instdir + "/lib" + + + if testsuite: + f.addStep(steps.RemoveDirectory(name="Test-Suite: Clean Source", + dir=testsuite_srcdir, + haltOnFailure=False, + warnOnFailure=True, + doStepIf=cleanBuildRequested)) + + f.addGetSourcecodeForProject(name="Test-Suite: Checkout", + description="fetching", + descriptionDone="fetch", + descriptionSuffix="Test-Suite", + project='test-suite', + src_dir=testsuite_srcdir, + alwaysUseLatest=True) + + if testsuite_sollvevv: + f.addStep(steps.RemoveDirectory(name="SOLLVE V&V: Clean Source", + dir=sollvevv_srcdir, + haltOnFailure=False, + warnOnFailure=True, + doStepIf=cleanBuildRequested)) + + f.addStep(steps.Git(name="SOLLVE V&V: Checkout", + description="fetching", + descriptionDone="fetch", + descriptionSuffix="SOLLVE V&V", + repourl='https://github.com/SOLLVE/sollve_vv.git', + workdir=sollvevv_srcdir, + alwaysUseLatest=True)) + + f.addStep(steps.RemoveDirectory(name="Test-Suite: Clean Build", + dir=testsuite_builddir, + haltOnFailure=False, + warnOnFailure=True)) + + testsuite_options = [ + "-DCMAKE_BUILD_TYPE=Release", + "-DTEST_SUITE_LIT_FLAGS=-vv;-s;-j6;-o;report.json", + "-DTEST_SUITE_EXTRA_C_FLAGS=-gline-tables-only", + "-DTEST_SUITE_EXTRA_CXX_FLAGS=-gline-tables-only", + WithProperties("-DCMAKE_C_COMPILER=" + clangexe), + WithProperties("-DCMAKE_CXX_COMPILER=" + clangxxexe), + WithProperties("-DTEST_SUITE_LIT=" + litexe), + ] + if testsuite_sollvevv: + testsuite_options += [ + "-DTEST_SUITE_SUBDIRS=External/sollve_vv", + "-DTEST_SUITE_SOLLVEVV_ROOT=../" + sollvevv_srcdir, + ] + testsuite_options += extraTestsuiteCmakeArgs + f.addStep(CmakeCommand(name="Test-Suite: Configure", + description="configuring", + descriptionDone="configure", + descriptionSuffix="Test-Suite", + generator='Ninja', + path='../' + testsuite_srcdir, + workdir=testsuite_builddir, + options=testsuite_options, + haltOnFailure=True, + logfiles={ + 'CMakeCache.txt' : 'CMakeCache.txt', + })) + + f.addStep(NinjaCommand(name="Test-Suite: Compile", + description="compiling", + descriptionDone="compile", + descriptionSuffix="Test-Suite", + options=['-k0'], # Continue building; programs that don't compile will fail with NOEXE. + haltOnFailure=False, + flunkOnFailure=False, # SOLLVE V&V contains tests that clang have not been implemented yet. + warnOnFailure=True, + workdir=testsuite_builddir)) + + merged_env.update({ + 'LD_LIBRARY_PATH': WithProperties(libdir + ':${LD_LIBRARY_PATH}') + }) + f.addStep(LitTestCommand(name="Test-Suite: Run", + description="running", + descriptionDone="run", + descriptionSuffix="Test-Suite", + command=[WithProperties(litexe), '-vv', '-s', '-j6', '-o','report.json', '.'], + haltOnFailure=False, + flunkOnFailure=False, # SOLLVE V&V contains tests that clang have not been implemented yet. + warnOnFailure=True, + workdir=testsuite_builddir, + env=merged_env, + logfiles={ + 'test.log' : 'test.log', + 'report.json': 'report.json' + })) return f diff --git a/zorg/buildbot/builders/PollyBuilder.py b/zorg/buildbot/builders/PollyBuilder.py --- a/zorg/buildbot/builders/PollyBuilder.py +++ b/zorg/buildbot/builders/PollyBuilder.py @@ -44,12 +44,13 @@ merged_env.update(env) # Overwrite pre-set items with the given ones, so user can set anything. depends_on_projects = ['llvm','clang','polly'] - if testsuite: - # XRay tests in test-suite require compiler-rt - depends_on_projects += ['compiler-rt'] - cleanBuildRequestedByProperty = lambda step: step.build.getProperty("clean", False) - cleanBuildRequested = lambda step: clean or step.build.getProperty("clean", default=step.build.getProperty("clean_obj")) + # If true, clean everything, including source dirs + def cleanBuildRequested(step): + return step.build.getProperty("clean") + # If true, clean build products; implied if cleanBuildRequested is true + def cleanObjRequested(step): + return cleanBuildRequested(step) or clean or step.build.getProperty("clean_obj") f = LLVMBuildFactory( depends_on_projects=depends_on_projects, @@ -62,7 +63,7 @@ f.addStep(steps.RemoveDirectory(name='clean-src-dir', dir=f.monorepo_dir, warnOnFailure=True, - doStepIf=cleanBuildRequestedByProperty)) + doStepIf=cleanBuildRequested)) # Get the source code. f.addGetSourcecodeSteps(**kwargs) @@ -71,13 +72,12 @@ f.addStep(steps.RemoveDirectory(name='clean-build-dir', dir=llvm_objdir, warnOnFailure=True, - doStepIf=cleanBuildRequested)) + doStepIf=cleanObjRequested)) # Create configuration files with cmake cmakeCommand = ["cmake", "../%s/llvm" % llvm_srcdir, "-DCMAKE_COLOR_MAKEFILE=OFF", "-DPOLLY_TEST_DISABLE_BAR=ON", - "-DPOLLY_ENABLE_GPGPU_CODEGEN=ON", "-DCMAKE_BUILD_TYPE=Release", "-DLLVM_POLLY_LINK_INTO_TOOLS=ON", "-DLLVM_ENABLE_PROJECTS=%s" % ";".join(f.depends_on_projects), @@ -87,7 +87,10 @@ haltOnFailure=False, description=["cmake configure"], workdir=llvm_objdir, - env=merged_env)) + env=merged_env, + logfiles={ + 'CMakeCache.txt' : llvm_objdir + '/CMakeCache.txt', + })) # Build f.addStep(WarningCountingShellCommand(name="build", @@ -107,7 +110,7 @@ f.addStep(steps.RemoveDirectory(name='clean-install-dir', dir=llvm_instdir, haltOnFailure=False, - doStepIf=cleanBuildRequested)) + doStepIf=cleanObjRequested)) f.addStep(ShellCommand(name="install", command=install_cmd, @@ -142,7 +145,7 @@ dir=testsuite_srcdir, haltOnFailure=False, warnOnFailure=True, - doStepIf=cleanBuildRequestedByProperty)) + doStepIf=cleanBuildRequested)) f.addGetSourcecodeForProject( project='test-suite', @@ -154,14 +157,13 @@ haltOnFailure=False, warnOnFailure=True)) - # -Wno-unused-command-line-argument is needed because linking will not uses the "-mllvm -polly" argument. + # -Wno-unused-command-line-argument is needed because linking will not use the "-mllvm -polly" argument. f.addStep(ShellCommand(name='test-suite_cmake-configure', description=["Test-Suite: cmake"], command=["cmake", '-B', testsuite_builddir, '-S', testsuite_srcdir, "-DCMAKE_BUILD_TYPE=Release", - "-DTEST_SUITE_COLLECT_STATS=ON", "-DTEST_SUITE_EXTRA_C_FLAGS=-Wno-unused-command-line-argument -mllvm -polly", - "-DTEST_SUITE_EXTRA_CXX_FLAGS=-mllvm -polly", + "-DTEST_SUITE_EXTRA_CXX_FLAGS=-Wno-unused-command-line-argument -mllvm -polly", "-DTEST_SUITE_LIT_FLAGS=-vv;-o;report.json", WithProperties("-DCMAKE_C_COMPILER=" + clangexe), WithProperties("-DCMAKE_CXX_COMPILER=" + clangxxexe), @@ -170,7 +172,10 @@ ] + extraTestsuiteCmakeArgs, haltOnFailure=True, workdir='.', - env=merged_env)) + env=merged_env, + logfiles={ + 'CMakeCache.txt' : testsuite_builddir + '/CMakeCache.txt', + })) f.addStep(WarningCountingShellCommand(name='test-suite_build', description=["Test-Suite: build"], @@ -183,7 +188,7 @@ f.addStep(LitTestCommand(name='test-suite_run', description=['Test-Suite: run'], - command=[WithProperties(litexe), '-vv', '-o', 'report.json', '.'], + command=[WithProperties(litexe), '-vv', '-s', '-o', 'report.json', '.'], haltOnFailure=True, workdir=testsuite_builddir, logfiles={ diff --git a/zorg/buildbot/builders/SanitizerBuilder.py b/zorg/buildbot/builders/SanitizerBuilder.py --- a/zorg/buildbot/builders/SanitizerBuilder.py +++ b/zorg/buildbot/builders/SanitizerBuilder.py @@ -5,7 +5,8 @@ def getSanitizerBuildFactory( clean=False, - depends_on_projects=None, + extra_depends_on_projects=[], + extra_configure_args=None, env=None, timeout=1200): @@ -20,15 +21,20 @@ if env is not None: merged_env.update(env) - if depends_on_projects is None: - depends_on_projects = [ - "llvm", - "clang", - "compiler-rt", - "libcxx", - "libcxxabi", - "libunwind", - "lld"] + depends_on_projects = [ + "llvm", + "clang", + "compiler-rt", + "libcxx", + "libcxxabi", + "libunwind", + "lld", + ] + extra_depends_on_projects + + if extra_configure_args: + extra_configure_args = ['--'] + list(extra_configure_args) + else: + extra_configure_args = [] # Explicitly use '/' as separator, because it works on *nix and Windows. sanitizer_script_dir = "sanitizer_buildbot/sanitizers" @@ -47,10 +53,11 @@ alwaysUseLatest=True) # Run annotated command for sanitizer. + command = ['python', sanitizer_script] + extra_configure_args f.addStep(AnnotatedCommand(name="annotate", description="annotate", timeout=timeout, haltOnFailure=True, - command="python " + sanitizer_script, + command=command, env=merged_env)) return f diff --git a/zorg/buildbot/builders/SphinxDocsBuilder.py b/zorg/buildbot/builders/SphinxDocsBuilder.py --- a/zorg/buildbot/builders/SphinxDocsBuilder.py +++ b/zorg/buildbot/builders/SphinxDocsBuilder.py @@ -11,8 +11,9 @@ ("llvm", ("docs-llvm-html", "docs/html/", "llvm")), ("clang", ("docs-clang-html", "tools/clang/docs/html/", "cfe")), ("clang-tools-extra", ("docs-clang-tools-html", "tools/clang/tools/extra/docs/html/", "clang-tools-extra")), - ("libcxx", ("docs-libcxx-html", "projects/libcxx/docs/html/", "libcxx")), - ("libunwind", ("docs-libunwind-html", "projects/libunwind/docs/html/", "libunwind")), + ("libc", ("docs-libc-html", "libc/docs/html/", "libc")), + ("libcxx", ("docs-libcxx-html", "libcxx/docs/html/", "libcxx")), + ("libunwind", ("docs-libunwind-html", "libunwind/docs/html/", "libunwind")), ("lld", ("docs-lld-html", "tools/lld/docs/html/", "lld")), ("lldb", ("docs-lldb-html", "tools/lldb/docs/html/", "lldb")), ('flang', ("docs-flang-html", "tools/flang/docs/html/", "flang")), @@ -25,11 +26,11 @@ llvm_html = False, # Build LLVM HTML documentation llvm_man = False, # Build LLVM man pages clang_html = False, # Build Clang HTML documentation + clang_man = False, # Build Clang man pages clang_tools_html = False, # Build Clang Extra Tools HTML documentation lld_html = False, # Build LLD HTML documentation - libcxx_html = False, # Build Libc++ HTML documentation - libunwind_html = False, # Build libunwind HTML documentation lldb_html = False, # Build LLDB HTML documentation + polly_html = False, # Build Polly HTML documentation extra_configure_args = None, **kwargs): @@ -53,7 +54,7 @@ llvm_objdir = 'llvm/build' depends_on_projects = ['llvm'] - if clang_html or clang_tools_html or lldb_html: + if clang_html or clang_man or clang_tools_html or lldb_html: depends_on_projects.append('clang') if clang_tools_html: depends_on_projects.append('clang-tools-extra') @@ -61,11 +62,8 @@ depends_on_projects.append('lld') if lldb_html: depends_on_projects.append('lldb') - if libcxx_html: - depends_on_projects.append('libcxx') - depends_on_projects.append('libcxxabi') - if libunwind_html: - depends_on_projects.append('libunwind') + if polly_html: + depends_on_projects.append('polly') f = UnifiedTreeBuilder.getCmakeBuildFactory( depends_on_projects=depends_on_projects, @@ -98,6 +96,14 @@ targets=['docs-clang-html'] )) + if clang_man: + f.addStep(NinjaCommand(name="docs-clang-man", + haltOnFailure=True, + description=["Build Clang Sphinx man pages"], + workdir=llvm_objdir, + targets=['docs-clang-man'] + )) + if clang_tools_html: f.addStep(NinjaCommand(name="docs-clang-tools-html", haltOnFailure=True, @@ -122,11 +128,67 @@ targets=['docs-lldb-html'] )) + if polly_html: + f.addStep(NinjaCommand(name="docs-polly-html", + haltOnFailure=True, + description=["Build Polly Sphinx HTML documentation"], + workdir=llvm_objdir, + targets=['docs-polly-html'] + )) + + return f + +def getSphinxRuntimesDocsBuildFactory( + libcxx_html = False, # Build Libc++ HTML documentation + libunwind_html = False, # Build libunwind HTML documentation + libc_html = False, # Build Libc HTML documentation + extra_configure_args = None, + **kwargs): + + if extra_configure_args: + cmake_args = extra_configure_args[:] + else: + cmake_args = list() + + # Set proper defaults for the config flags. + CmakeCommand.applyDefaultOptions(cmake_args, [ + ('-G', 'Ninja'), + ('-DLLVM_ENABLE_SPHINX=', 'ON'), + ('-DSPHINX_OUTPUT_HTML=', 'ON'), + ('-DSPHINX_OUTPUT_MAN=', 'ON'), + ('-DLLDB_INCLUDE_TESTS=', 'OFF'), + ('-DLLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN=', 'ON'), + ('-DLLVM_ENABLE_ASSERTIONS=', 'OFF'), + ]) + + depends_on_runtimes = [] + if libcxx_html: + depends_on_runtimes.append('libcxx') + depends_on_runtimes.append('libcxxabi') + if libunwind_html: + depends_on_runtimes.append('libunwind') + if libc_html: + depends_on_runtimes.append('libc') + + f = UnifiedTreeBuilder.getLLVMBuildFactoryAndSourcecodeSteps( + depends_on_projects=depends_on_runtimes, + enable_runtimes=depends_on_runtimes, + src_to_build_dir='runtimes', + **kwargs) # Pass through all the extra arguments. + + UnifiedTreeBuilder.addCmakeSteps( + f, + cleanBuildRequested=f.cleanBuildRequested, + obj_dir=f.obj_dir, + install_dir=f.install_dir, + extra_configure_args=cmake_args, + **kwargs) + if libcxx_html: f.addStep(NinjaCommand(name="docs-libcxx-html", haltOnFailure=True, description=["Build Libc++ Sphinx HTML documentation"], - workdir=llvm_objdir, + workdir=f.obj_dir, targets=['docs-libcxx-html'] )) @@ -134,10 +196,18 @@ f.addStep(NinjaCommand(name="docs-libunwind-html", haltOnFailure=True, description=["Build libunwind Sphinx HTML documentation"], - workdir=llvm_objdir, + workdir=f.obj_dir, targets=['docs-libunwind-html'] )) + if libc_html: + f.addStep(NinjaCommand(name="docs-libc-html", + haltOnFailure=True, + description=["Build libc Sphinx HTML documentation"], + workdir=f.obj_dir, + targets=['docs-libc-html'] + )) + return f @@ -154,9 +224,6 @@ "llvm", "clang", "clang-tools-extra", - "libcxx", - "libcxxabi", - "libunwind", "lld", "lldb", "flang", @@ -173,12 +240,6 @@ "lldb" in _depends_on_projects ) and "clang" not in _depends_on_projects: _depends_on_projects.append("clang") - if "libcxx" in _depends_on_projects and \ - "libcxxabi" not in _depends_on_projects: - _depends_on_projects.append("libcxxabi") - if "libcxxabi" in _depends_on_projects and \ - "libcxx" not in _depends_on_projects: - _depends_on_projects.append("libcxx") # Make a local copy of the configure args, as we are going to modify that. if extra_configure_args: @@ -248,3 +309,111 @@ ) return f + +def getLLVMRuntimesDocsBuildFactory( + clean = False, + depends_on_runtimes = None, + extra_configure_args = None, + env = None, + **kwargs): + + if depends_on_runtimes is None: + # All the projects by default. + _depends_on_runtimes=[ + "libcxx", + "libcxxabi", + "libunwind", + "libc", + ] + else: + # Make a local copy of depends_on_runtimes, as we are going to modify + # that. + _depends_on_runtimes=depends_on_runtimes[:] + # Some runtimes are interdependent for the purpose of documentation. + # Enforce the dependencies. + if "libcxx" in _depends_on_runtimes and \ + "libcxxabi" not in _depends_on_runtimes: + _depends_on_runtimes.append("libcxxabi") + if "libcxxabi" in _depends_on_runtimes and \ + "libcxx" not in _depends_on_runtimes: + _depends_on_runtimes.append("libcxx") + + # Make a local copy of the configure args, as we are going to modify that. + if extra_configure_args: + cmake_args = extra_configure_args[:] + else: + cmake_args = list() + + # Prepare environmental variables. Set here all env we want everywhere. + merged_env = { + 'TERM' : 'dumb' # Be cautious and disable color output from all tools. + } + if env is not None: + # Overwrite pre-set items with the given ones, so user can set anything. + merged_env.update(env) + + CmakeCommand.applyDefaultOptions(cmake_args, [ + ("-G", "Ninja"), + ("-DLLVM_ENABLE_SPHINX=", "ON"), + ("-DSPHINX_OUTPUT_HTML=", "ON"), + ("-DSPHINX_OUTPUT_MAN=", "OFF"), + ("-DSPHINX_WARNINGS_AS_ERRORS=", "OFF"), + ("-DLLVM_ENABLE_ASSERTIONS=", "OFF"), + ("-DCMAKE_BUILD_TYPE=", "Release"), + ]) + + # Build docs for each of the runtimes this builder depends on + docs = [ + llvm_docs[project] for project in llvm_docs.keys() + if project in _depends_on_runtimes + ] + + cleanBuildRequested = lambda step: step.build.getProperty("clean", default=step.build.getProperty("clean_obj")) or clean + + f = UnifiedTreeBuilder.getLLVMBuildFactoryAndSourcecodeSteps( + depends_on_projects=_depends_on_runtimes, + enable_runtimes=_depends_on_runtimes, + src_to_build_dir='runtimes', + cleanBuildRequested=cleanBuildRequested, + **kwargs) # Pass through all the extra arguments. + + UnifiedTreeBuilder.addCmakeSteps( + f, + cleanBuildRequested=cleanBuildRequested, + obj_dir=f.obj_dir, + install_dir=f.install_dir, + extra_configure_args=cmake_args, + env=merged_env, + **kwargs) + + UnifiedTreeBuilder.addNinjaSteps( + f, + targets=[d[0] for d in docs], + checks=[], + env=merged_env, + **kwargs) + + # Publish just built documentation + for target, local_path, remote_path in docs: + f.addStep( + ShellCommand( + name="Publish {}".format(target), + description=[ + "Publish", "just", "built", "documentation", "for", + "{}".format(target), + ], + command=[ + 'rsync', + '-vrl', + '--delete', '--force', '--delay-updates', '--delete-delay', + '--ignore-times', + '--checksum', + '-p', '--chmod=Du=rwx,Dg=rwx,Do=rx,Fu=rw,Fg=rw,Fo=r', + "{}".format(local_path), + "lists.llvm.org:web/{}".format(remote_path), + ], + env=merged_env, + ) + ) + + return f diff --git a/zorg/buildbot/builders/TestSuiteBuilder.py b/zorg/buildbot/builders/TestSuiteBuilder.py new file mode 100644 --- /dev/null +++ b/zorg/buildbot/builders/TestSuiteBuilder.py @@ -0,0 +1,108 @@ +from zorg.buildbot.builders.UnifiedTreeBuilder import getCmakeWithNinjaBuildFactory + +from buildbot.plugins import util + +from buildbot.steps.shell import ShellCommand + +from zorg.buildbot.commands.CmakeCommand import CmakeCommand +from zorg.buildbot.commands.NinjaCommand import NinjaCommand +from zorg.buildbot.commands.LitTestCommand import LitTestCommand + +# This builder is uses UnifiedTreeBuilders and adds running +# llvm-test-suite with cmake and ninja step. + +def addTestSuiteStep( + f, + compiler_dir = None, + env = None, + lit_args = [], + **kwargs): + + cc = util.Interpolate('-DCMAKE_C_COMPILER=' + '%(prop:builddir)s/'+compiler_dir+'/bin/clang') + cxx = util.Interpolate('-DCMAKE_CXX_COMPILER=' + '%(prop:builddir)s/'+compiler_dir+'/bin/clang++') + lit = util.Interpolate('%(prop:builddir)s/' + compiler_dir + '/bin/llvm-lit') + test_suite_base_dir = util.Interpolate('%(prop:builddir)s/' + 'test') + test_suite_src_dir = util.Interpolate('%(prop:builddir)s/' + 'test/test-suite') + test_suite_workdir = util.Interpolate('%(prop:builddir)s/' + 'test/build-test-suite') + cmake_lit_arg = util.Interpolate('-DTEST_SUITE_LIT:FILEPATH=' + '%(prop:builddir)s/' + compiler_dir + '/bin/llvm-lit') + # used for cmake building test-suite step + options = [cc, cxx, cmake_lit_arg] + + # always clobber the build directory to test each new compiler + f.addStep(ShellCommand(name='Clean Test Suite Build dir', + command=['rm', '-rf', test_suite_workdir], + haltOnFailure=True, + description='Removing the Test Suite build directory', + workdir=test_suite_base_dir, + env=env)) + + f.addGetSourcecodeForProject( + project='test-suite', + src_dir=test_suite_src_dir, + alwaysUseLatest=True) + + f.addStep(CmakeCommand(name='cmake Test Suite', + haltOnFailure=True, + description='Running cmake on Test Suite dir', + workdir=test_suite_workdir, + options=options, + path=test_suite_src_dir, + generator='Ninja')) + + f.addStep(NinjaCommand(name='ninja Test Suite', + description='Running Ninja on Test Suite dir', + haltOnFailure=True, + workdir=test_suite_workdir)) + + f.addStep(LitTestCommand(name='Run Test Suite with lit', + haltOnFailure=True, + description='Running test suite tests', + workdir=test_suite_workdir, + command=[lit] + lit_args + ['.'], + env=env, + **kwargs)) + + return f + +def getTestSuiteBuildFactory( + depends_on_projects = None, + enable_runtimes = "auto", + targets = None, + llvm_srcdir = None, + obj_dir = None, + checks = None, + install_dir = None, + clean = False, + extra_configure_args = None, + env = None, + **kwargs): + + # handle the -DCMAKE args for lit + lit_args = list() + if any("DLLVM_LIT_ARGS" in arg for arg in extra_configure_args): + arg = [arg for arg in extra_configure_args + if "DLLVM_LIT_ARGS" in arg][0] + lit_args = arg.split("=")[1] + lit_args = lit_args.split(" ") + + f = getCmakeWithNinjaBuildFactory( + depends_on_projects = depends_on_projects, + enable_runtimes = enable_runtimes, + targets = targets, + llvm_srcdir = llvm_srcdir, + obj_dir = obj_dir, + checks = checks, + install_dir = install_dir, + clean = clean, + extra_configure_args = extra_configure_args, + env = env, + **kwargs) + + + addTestSuiteStep(f, + compiler_dir=f.obj_dir, + env=env, + lit_args=lit_args, + **kwargs) + + return f diff --git a/zorg/buildbot/builders/UnifiedTreeBuilder.py b/zorg/buildbot/builders/UnifiedTreeBuilder.py --- a/zorg/buildbot/builders/UnifiedTreeBuilder.py +++ b/zorg/buildbot/builders/UnifiedTreeBuilder.py @@ -13,6 +13,7 @@ def getLLVMBuildFactoryAndPrepareForSourcecodeSteps( depends_on_projects = None, + enable_runtimes = "auto", llvm_srcdir = None, obj_dir = None, install_dir = None, @@ -29,6 +30,7 @@ f = LLVMBuildFactory( depends_on_projects=depends_on_projects, + enable_runtimes=enable_runtimes, llvm_srcdir=llvm_srcdir, obj_dir=obj_dir, install_dir=install_dir, @@ -49,7 +51,9 @@ def getLLVMBuildFactoryAndSourcecodeSteps( depends_on_projects = None, + enable_runtimes = "auto", llvm_srcdir = None, + src_to_build_dir = None, obj_dir = None, install_dir = None, cleanBuildRequested = None, @@ -57,7 +61,9 @@ f = getLLVMBuildFactoryAndPrepareForSourcecodeSteps( depends_on_projects=depends_on_projects, + enable_runtimes=enable_runtimes, llvm_srcdir=llvm_srcdir, + src_to_build_dir=src_to_build_dir, obj_dir=obj_dir, install_dir=install_dir, cleanBuildRequested=cleanBuildRequested, @@ -99,9 +105,15 @@ doStepIf=cleanBuildRequested, )) - CmakeCommand.applyDefaultOptions(cmake_args, [ - ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)), - ]) + if f.enable_projects: + CmakeCommand.applyDefaultOptions(cmake_args, [ + ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.enable_projects)), + ]) + + if f.enable_runtimes: + CmakeCommand.applyDefaultOptions(cmake_args, [ + ('-DLLVM_ENABLE_RUNTIMES=', ";".join(f.enable_runtimes)), + ]) if install_dir: install_dir_rel = LLVMBuildFactory.pathRelativeTo( @@ -206,7 +218,7 @@ check_env = env or {} for check in checks: - f.addStep(LitTestCommand(name="test-%s%s" % (step_name, check), + f.addStep(LitTestCommand(name="test-%s-%s" % (step_name, check), command=['ninja', check], description=[ "Test", "just", "built", "components", "for", @@ -230,6 +242,7 @@ def getCmakeBuildFactory( depends_on_projects = None, + enable_runtimes = "auto", llvm_srcdir = None, obj_dir = None, install_dir = None, @@ -240,6 +253,7 @@ f = getLLVMBuildFactoryAndSourcecodeSteps( depends_on_projects=depends_on_projects, + enable_runtimes=enable_runtimes, llvm_srcdir=llvm_srcdir, obj_dir=obj_dir, install_dir=install_dir, @@ -260,6 +274,7 @@ def getCmakeWithNinjaBuildFactory( depends_on_projects = None, + enable_runtimes = "auto", targets = None, llvm_srcdir = None, obj_dir = None, @@ -294,6 +309,7 @@ f = getCmakeBuildFactory( depends_on_projects=depends_on_projects, + enable_runtimes=enable_runtimes, llvm_srcdir=llvm_srcdir, obj_dir=obj_dir, install_dir=install_dir, @@ -315,6 +331,7 @@ def getCmakeWithNinjaWithMSVCBuildFactory( depends_on_projects = None, + enable_runtimes = "auto", targets = None, llvm_srcdir = None, obj_dir = None, @@ -329,8 +346,6 @@ env = None, **kwargs): - assert not env, "Can't have custom builder env vars with MSVC build" - # Make a local copy of the configure args, as we are going to modify that. if extra_configure_args: cmake_args = extra_configure_args[:] @@ -342,6 +357,7 @@ f = getLLVMBuildFactoryAndSourcecodeSteps( depends_on_projects=depends_on_projects, + enable_runtimes=enable_runtimes, llvm_srcdir=llvm_srcdir, obj_dir=obj_dir, install_dir=install_dir, @@ -349,7 +365,8 @@ f.addStep(SetPropertyFromCommand( command=builders_util.getVisualStudioEnvironment(vs, target_arch), - extract_fn=builders_util.extractVSEnvironment)) + extract_fn=builders_util.extractVSEnvironment, + env=env)) env = util.Property('vs_env') cleanBuildRequested = lambda step: step.build.getProperty("clean", default=step.build.getProperty("clean_obj")) or clean @@ -377,7 +394,9 @@ def getCmakeWithNinjaMultistageBuildFactory( depends_on_projects = None, + enable_runtimes = "auto", llvm_srcdir = None, + src_to_build_dir = None, obj_dir = None, checks = None, install_dir = None, @@ -426,7 +445,9 @@ f = getLLVMBuildFactoryAndPrepareForSourcecodeSteps( depends_on_projects=depends_on_projects, + enable_runtimes=enable_runtimes, llvm_srcdir=llvm_srcdir, + src_to_build_dir=src_to_build_dir, obj_dir=obj_dir, install_dir=install_dir, env=merged_env, diff --git a/zorg/buildbot/builders/XToolchainBuilder.py b/zorg/buildbot/builders/XToolchainBuilder.py --- a/zorg/buildbot/builders/XToolchainBuilder.py +++ b/zorg/buildbot/builders/XToolchainBuilder.py @@ -40,7 +40,6 @@ 'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences. } if env is not None: - assert not vs, "Cannot have custom builder env vars with VS setup." # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) @@ -117,14 +116,10 @@ # Configure MSVC environment if requested. f.addStep(SetPropertyFromCommand( command=builders_util.getVisualStudioEnvironment(vs, None), - extract_fn=builders_util.extractVSEnvironment)) + extract_fn=builders_util.extractVSEnvironment, + env=merged_env)) merged_env = Property('vs_env') - # Since this is a build of a cross toolchain, we build only the host-side - # tools first by the host system compiler. Libraries will be cross-compiled. - cmake_args.append(InterpolateToPosixPath( - '-DLLVM_AR=%(builddir)s/' + f.obj_dir + '/bin/llvm-ar.exe')), - CmakeCommand.applyDefaultOptions(cmake_args, [ ('-G', 'Ninja'), ('-DLLVM_ENABLE_PROJECTS=', 'llvm;clang;clang-tools-extra;lld'), diff --git a/zorg/buildbot/builders/annotated/annotated_builder.py b/zorg/buildbot/builders/annotated/annotated_builder.py --- a/zorg/buildbot/builders/annotated/annotated_builder.py +++ b/zorg/buildbot/builders/annotated/annotated_builder.py @@ -13,6 +13,7 @@ from os.path import join as pjoin VSWHERE_PATH = "C:/Program Files (x86)/Microsoft Visual Studio/Installer/vswhere.exe" +BUILDTOOLS_VSDEVCMD = "C:/BuildTools/Common7/Tools/VsDevCmd.bat" def get_argument_parser(*args, **kwargs): ap = argparse.ArgumentParser(*args, **kwargs) @@ -211,10 +212,13 @@ cxx_compiler, linker, previous_stage_bin): + archiver = 'llvm-ar' + if os.name == 'nt': + archiver = 'llvm-lib' return ( cmake_args + [ '-DCMAKE_AR=%s' % ( - util.cmake_pjoin(previous_stage_bin, 'llvm-ar'),), + util.cmake_pjoin(previous_stage_bin, archiver),), '-DCMAKE_RANLIB=%s' % ( util.cmake_pjoin(previous_stage_bin, 'llvm-ranlib'),), ] + self.cmake_compiler_flags( @@ -323,9 +327,17 @@ def get_vcvars(vs_tools, arch): - """Get the VC tools environment using vswhere.exe from VS 2017 + """Get the VC tools environment using vswhere.exe or buildtools docker + + This is intended to work either when VS is in its standard installation + location, or when the docker instructions have been followed, and we can + find Visual C++ in C:/BuildTools. + + Visual Studio provides a docker image with instructions here: + https://docs.microsoft.com/en-us/visualstudio/install/build-tools-container?view=vs-2019 - This code is following the guidelines from strategy 1 in this blog post: + This vswhere code is following the guidelines from strategy 1 in this blog + post: https://blogs.msdn.microsoft.com/vcblog/2017/03/06/finding-the-visual-c-compiler-tools-in-visual-studio-2017/ It doesn't work when VS is not installed at the default location. @@ -342,13 +354,16 @@ # Use vswhere.exe if it exists. if os.path.exists(VSWHERE_PATH): cmd = [VSWHERE_PATH, "-latest", "-property", "installationPath"] - vs_path = subprocess.check_output(cmd).strip() + vs_path = subprocess.check_output(cmd).decode(sys.stdout.encoding) + vs_path = vs_path.strip() util.report("Running vswhere to find VS: " + repr(cmd)) util.report("vswhere output: " + vs_path) if not os.path.isdir(vs_path): raise ValueError("VS install path does not exist: " + vs_path) vcvars_path = pjoin(vs_path, 'VC', 'Auxiliary', 'Build', 'vcvarsall.bat') + elif os.path.exists(BUILDTOOLS_VSDEVCMD): + vcvars_path = BUILDTOOLS_VSDEVCMD elif vs_tools is None: vs_tools = os.path.expandvars('%VS140COMNTOOLS%') vcvars_path = pjoin(vs_tools, '..', '..', 'VC', 'vcvarsall.bat') @@ -357,7 +372,8 @@ # Windows /dev/null. cmd = util.shquote_cmd([vcvars_path, arch]) + ' > NUL && set' util.report("Running vcvars: " + cmd) - output = subprocess.check_output(cmd, shell=True) + output = \ + subprocess.check_output(cmd, shell=True).decode(sys.stdout.encoding) new_env = {} for line in output.splitlines(): var, val = line.split('=', 1) diff --git a/zorg/buildbot/builders/annotated/buildbot-helper.sh b/zorg/buildbot/builders/annotated/buildbot-helper.sh new file mode 100755 --- /dev/null +++ b/zorg/buildbot/builders/annotated/buildbot-helper.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# Buildbot Helper functions + +# Stop if we've encountered an error. +halt_on_failure() { + echo "@@@HALT_ON_FAILURE@@@" +} + +build_step() { + echo "@@@BUILD_STEP ${@}@@@" +} diff --git a/zorg/buildbot/builders/annotated/hip-build.sh b/zorg/buildbot/builders/annotated/hip-build.sh new file mode 100755 --- /dev/null +++ b/zorg/buildbot/builders/annotated/hip-build.sh @@ -0,0 +1,114 @@ +#!/bin/bash + +# Enable Error tracing +set -o errtrace + +# Print trace for all commands ran before execution +set -x + +# Include the Buildbot helper functions +HERE="$(dirname $0)" +. ${HERE}/buildbot-helper.sh + +# Ensure all commands pass, and not dereferencing unset variables. +set -eu +halt_on_failure + +BUILDBOT_ROOT=/buildbot +LLVM_ROOT="${BUILDBOT_ROOT}/llvm-project" +REVISION="${BUILDBOT_REVISION:-origin/main}" +AMDGPU_ARCHS=${AMDGPU_ARCHS:="gfx900;gfx906;gfx908;gfx1030"} + +# Set-up llvm-project +if [ ! -d "${LLVM_ROOT}" ]; then + build_step "Cloning llvm-project repo" + git clone --progress https://github.com/llvm/llvm-project.git ${LLVM_ROOT} +fi + +build_step "Updating llvm-project repo" +git -C "${LLVM_ROOT}" fetch origin +git -C "${LLVM_ROOT}" reset --hard "${REVISION}" + +# Set-up llvm-test-suite +TESTSUITE_ROOT="${BUILDBOT_ROOT}/llvm-test-suite" +if [ ! -d "${TESTSUITE_ROOT}" ]; then + build_step "Cloning llvm-test-suite repo" + git clone --progress https://github.com/llvm/llvm-test-suite.git ${TESTSUITE_ROOT} +fi + +build_step "Updating llvm-test-suite repo" +git -C "${TESTSUITE_ROOT}" fetch origin +git -C "${TESTSUITE_ROOT}" reset --hard origin/main + +# Set-up variables +BUILDBOT_SLAVENAME=$(whoami) +BUILD_DIR="${BUILDBOT_ROOT}/${BUILDBOT_SLAVENAME}/${BUILDBOT_BUILDERNAME}" +DESTDIR=${BUILD_DIR}/install +EXTERNAL_DIR=/buildbot/Externals + +build_step "Setting up the buildbot" +echo "BUILDBOT_ROOT=${BUILDBOT_ROOT}" +echo "LLVM_ROOT=${LLVM_ROOT}" +echo "BUILD_DIR=${BUILD_DIR}" +echo "DESTDIR=${DESTDIR}" +echo "EXTERNAL_DIR=${EXTERNAL_DIR}" + +# Start building LLVM, Clang, Lld, clang-tools-extra, compiler-rt +build_step "Configure LLVM Build" +LLVM_BUILD_DIR="${BUILD_DIR}/llvm" +mkdir -p "${LLVM_BUILD_DIR}" +cd "${LLVM_BUILD_DIR}" +cmake -G Ninja \ + -DCMAKE_BUILD_TYPE="Release" \ + -DCMAKE_VERBOSE_MAKEFILE=1 \ + -DLLVM_TARGETS_TO_BUILD="AMDGPU;X86" \ + -DLLVM_ENABLE_PROJECTS="clang;lld;clang-tools-extra;compiler-rt" \ + -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi" \ + -DLIBCXX_ENABLE_SHARED=OFF \ + -DLIBCXX_ENABLE_STATIC=ON \ + -DLIBCXX_INSTALL_LIBRARY=OFF \ + -DLIBCXX_INSTALL_HEADERS=OFF \ + -DLIBCXXABI_ENABLE_SHARED=OFF \ + -DLIBCXXABI_ENABLE_STATIC=ON \ + -DLIBCXXABI_INSTALL_STATIC_LIBRARY=OFF \ + -DCMAKE_INSTALL_PREFIX="${DESTDIR}" \ + -DLLVM_ENABLE_ASSERTIONS=ON \ + -DLLVM_ENABLE_Z3_SOLVER=OFF \ + -DLLVM_ENABLE_ZLIB=ON \ + -DLLVM_LIT_ARGS="-v -vv" \ + ${LLVM_ROOT}/llvm + +build_step "Building LLVM" +ninja runtimes +ninja + +build_step "Install LLVM" +rm -rf "${DESTDIR}" +ninja install-runtimes +ninja install + +# Start building llvm-test-suite's hip tests +build_step "Configuring HIP test-suite" +TEST_BUILD_DIR=${BUILD_DIR}/test-suite-build +rm -rf ${TEST_BUILD_DIR} +mkdir -p ${TEST_BUILD_DIR} +cd ${TEST_BUILD_DIR} +PATH="${LLVM_BUILD_DIR}/bin:$PATH" cmake -G Ninja \ + -DTEST_SUITE_SUBDIRS=External \ + -DTEST_SUITE_EXTERNALS_DIR=${EXTERNAL_DIR}/ \ + -DTEST_SUITE_COLLECT_CODE_SIZE=OFF \ + -DTEST_SUITE_COLLECT_COMPILE_TIME=OFF \ + -DAMDGPU_ARCHS="${AMDGPU_ARCHS}" \ + -DCMAKE_CXX_COMPILER="${LLVM_BUILD_DIR}/bin/clang++" \ + -DCMAKE_C_COMPILER="${LLVM_BUILD_DIR}/bin/clang" \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + ${TESTSUITE_ROOT} + +build_step "Building HIP test-suite" +ninja hip-tests-simple + +build_step "Testing HIP test-suite" +ninja check-hip-simple + +exit 0 + diff --git a/zorg/buildbot/builders/annotated/libc-linux.py b/zorg/buildbot/builders/annotated/libc-linux.py --- a/zorg/buildbot/builders/annotated/libc-linux.py +++ b/zorg/buildbot/builders/annotated/libc-linux.py @@ -9,6 +9,25 @@ from contextlib import contextmanager +def is_fullbuild_builder(builder_name): + return ('fullbuild' in builder_name.split('-')) + +def is_runtimes_builder(builder_name): + return ('runtimes' in builder_name.split('-')) + +def is_gcc_builder(builder_name): + return ('gcc' in builder_name.split('-')) + +def is_lint_builder(builder_name): + return ('lint' in builder_name.split('-')) + +def is_riscv_builder(builder_name): + return 'riscv' in builder_name + +def is_x86_64_builder(builder_name): + return 'x86_64' in builder_name + + def main(argv): ap = argparse.ArgumentParser() ap.add_argument('--asan', action='store_true', default=False, @@ -18,11 +37,44 @@ args, _ = ap.parse_known_args() source_dir = os.path.join('..', 'llvm-project') + builder_name = os.environ.get('BUILDBOT_BUILDERNAME') + fullbuild = is_fullbuild_builder(builder_name) + runtimes_build = is_runtimes_builder(builder_name) + gcc_build = is_gcc_builder(builder_name) + lint_build = is_lint_builder(builder_name) + riscv_build = is_riscv_builder(builder_name) + x86_64_build = is_x86_64_builder(builder_name) + + if gcc_build: + cc = 'gcc' + cxx = 'g++' + else: + if lint_build: + full_build = True + cc = '/home/libc-lint-tools/bin/clang' + cxx = '/home/libc-lint-tools/bin/clang++' + clang_tidy = '/home/libc-lint-tools/bin/clang-tidy' + else: + cc = 'clang' + cxx = 'clang++' with step('cmake', halt_on_fail=True): - projects = ['llvm', 'libc', 'clang', 'clang-tools-extra'] + # On most systems the default generator is make, and the default + # compilers are gcc and g++. We make the compiler and the generator + # explicit here, which reduces one step of setting environment + # variables when setting up workers. + cmake_args = ['-GNinja', + '-DCMAKE_C_COMPILER=%s' % cc, + '-DCMAKE_CXX_COMPILER=%s' % cxx] + if lint_build: + cmake_args.append('-DLLVM_LIBC_CLANG_TIDY=%s' % clang_tidy) + + if runtimes_build: + projects = ['llvm', 'clang'] + cmake_args.append('-DLLVM_ENABLE_RUNTIMES=libc') + else: + projects = ['llvm', 'libc'] - cmake_args = ['-GNinja'] if args.debug: cmake_args.append('-DCMAKE_BUILD_TYPE=Debug') else: @@ -31,30 +83,61 @@ if args.asan: cmake_args.append('-DLLVM_USE_SANITIZER=Address') + if fullbuild and not args.asan and not lint_build and not riscv_build: + projects.extend(['clang', 'compiler-rt']) + cmake_args.append('-DLLVM_ENABLE_PROJECTS={}'.format(';'.join(projects))) - run_command(['cmake', os.path.join(source_dir, 'llvm')] + cmake_args) + if fullbuild and not args.asan and not lint_build and not riscv_build: + cmake_args.append('-DLLVM_LIBC_INCLUDE_SCUDO=ON') + cmake_args.append('-DLIBC_INCLUDE_BENCHMARKS=ON') - with step('build llvmlibc', halt_on_fail=True): - run_command(['ninja', 'llvmlibc']) + if fullbuild: + cmake_args.extend(['-DLLVM_LIBC_FULL_BUILD=ON']), - with step('check-libc'): - run_command(['ninja', 'check-libc']) + run_command(['cmake', os.path.join(source_dir, 'llvm')] + cmake_args) - if not args.asan: - with step('Loader Tests'): - run_command(['ninja', 'libc_loader_tests']) - with step('Integration Tests'): - run_command(['ninja', 'libc-integration-test']) + if lint_build: + with step('lint libc'): + run_command(['ninja', 'libc-lint']) + return + + with step('build libc'): + run_command(['ninja', 'libc']) + + if fullbuild: + with step('build libc-startup'): + run_command(['ninja', 'libc-startup']) + + if runtimes_build: + with step('check-libc'): + run_command(['ninja', 'check-libc']) + else: + with step('libc-unit-tests'): + run_command(['ninja', 'libc-unit-tests']) + + if fullbuild and not args.asan: + with step('libc-integration-tests'): + run_command(['ninja', 'libc-integration-tests']) + with step('libc-api-test'): + run_command(['ninja', 'libc-api-test']) + if gcc_build or ('riscv' in builder_name): + # The rest of the targets are either not yet gcc-clean or + # not yet availabe on riscv. + return + with step('libc-scudo-integration-test'): + run_command(['ninja', 'libc-scudo-integration-test']) with step('AOR Tests'): aor_dir = os.path.join(source_dir, 'libc', 'AOR_v20.02') # Remove the AOR build dir. util.clean_dir(os.path.join(aor_dir, 'build')) run_command(['make', 'check'], directory=aor_dir) - - if not args.debug: with step('Benchmark Utils Tests'): run_command(['ninja', 'libc-benchmark-util-tests']) + + if not (fullbuild or runtimes_build) and x86_64_build: + with step('libc-fuzzer'): + run_command(['ninja', 'libc-fuzzer']) @contextmanager diff --git a/zorg/buildbot/builders/annotated/libc-linux.py b/zorg/buildbot/builders/annotated/libc-windows.py copy from zorg/buildbot/builders/annotated/libc-linux.py copy to zorg/buildbot/builders/annotated/libc-windows.py --- a/zorg/buildbot/builders/annotated/libc-linux.py +++ b/zorg/buildbot/builders/annotated/libc-windows.py @@ -1,14 +1,14 @@ -#!/usr/bin/python + +import annotated_builder +import util import argparse import os import subprocess import sys import traceback -import util from contextlib import contextmanager - def main(argv): ap = argparse.ArgumentParser() ap.add_argument('--asan', action='store_true', default=False, @@ -18,11 +18,19 @@ args, _ = ap.parse_known_args() source_dir = os.path.join('..', 'llvm-project') + vc_vars = annotated_builder.get_vcvars(vs_tools=None, arch='amd64') + for (var, val) in vc_vars.items(): + os.environ[var] = val with step('cmake', halt_on_fail=True): - projects = ['llvm', 'libc', 'clang', 'clang-tools-extra'] - - cmake_args = ['-GNinja'] + # On most systems the default generator is make and the default + # compilers are gcc and g++. We make it explicit here that we want + # clang and ninja which reduces one step of setting environment + # variables when setting up workers. + ## adjust compiler location definitions based on VM config + cmake_args = ['-GNinja', + '-DCMAKE_C_COMPILER=clang-cl', + '-DCMAKE_CXX_COMPILER=clang-cl'] if args.debug: cmake_args.append('-DCMAKE_BUILD_TYPE=Debug') else: @@ -31,7 +39,12 @@ if args.asan: cmake_args.append('-DLLVM_USE_SANITIZER=Address') - cmake_args.append('-DLLVM_ENABLE_PROJECTS={}'.format(';'.join(projects))) + cmake_args.append('-DLLVM_ENABLE_PROJECTS=libc') + cmake_args.append('-DLLVM_TARGETS_TO_BUILD=X86') + cmake_args.append('-DLLVM_FORCE_BUILD_RUNTIME=libc') + cmake_args.append('-DLLVM_NATIVE_ARCH=x86_64') + cmake_args.append('-DLLVM_HOST_TRIPLE=x86_64-window-x86-gnu') + cmake_args.append('-DLLVM_LIBC_MPFR_INSTALL_PATH=C:/src/install') run_command(['cmake', os.path.join(source_dir, 'llvm')] + cmake_args) @@ -41,21 +54,6 @@ with step('check-libc'): run_command(['ninja', 'check-libc']) - if not args.asan: - with step('Loader Tests'): - run_command(['ninja', 'libc_loader_tests']) - with step('Integration Tests'): - run_command(['ninja', 'libc-integration-test']) - with step('AOR Tests'): - aor_dir = os.path.join(source_dir, 'libc', 'AOR_v20.02') - # Remove the AOR build dir. - util.clean_dir(os.path.join(aor_dir, 'build')) - run_command(['make', 'check'], directory=aor_dir) - - if not args.debug: - with step('Benchmark Utils Tests'): - run_command(['ninja', 'libc-benchmark-util-tests']) - @contextmanager def step(step_name, halt_on_fail=False): diff --git a/zorg/buildbot/builders/annotated/standalone-build.sh b/zorg/buildbot/builders/annotated/standalone-build.sh new file mode 100755 --- /dev/null +++ b/zorg/buildbot/builders/annotated/standalone-build.sh @@ -0,0 +1,250 @@ +#!/bin/bash + +# This script builds LLVM and Clang in standalone mode that means it first +# builds LLVM and installs it into a specific directory. That directory is then +# used when building Clang which depends on it. + +# Enable Error tracing +set -o errtrace + +# Print trace for all commands ran before execution +set -x + +# Include the Buildbot helper functions +HERE="$(realpath $(dirname $0))" +. ${HERE}/buildbot-helper.sh + +# Ensure all commands pass, and not dereferencing unset variables. +set -eu +halt_on_failure + +BUILDBOT_ROOT=${BUILDBOT_ROOT:-${HERE}} +REVISION="${BUILDBOT_REVISION:-origin/main}" +LLVM_ROOT="${BUILDBOT_ROOT}/llvm-project" +BUILD_TYPE=Release +INSTALL_ROOT_DIR=${BUILDBOT_ROOT}/install +BUILD_ROOT_DIR=${BUILDBOT_ROOT}/build + +# Enable CMake tracing during configure per component +CMAKE_TRACE_CONFIGURE_LLVM=${CMAKE_TRACE_CONFIGURE_LLVM:-} +CMAKE_TRACE_CONFIGURE_CLANG=${CMAKE_TRACE_CONFIGURE_CLANG:-} +CMAKE_TRACE_CONFIGURE_LLD=${CMAKE_TRACE_CONFIGURE_LLD:-} + +# Turn CMake's verbose mode for build, check and install steps per component +CMAKE_VERBOSE_LLVM=${CMAKE_VERBOSE_LLVM:-} +CMAKE_VERBOSE_CLANG=${CMAKE_VERBOSE_CLANG:-} +CMAKE_VERBOSE_LLD=${CMAKE_VERBOSE_LLD:-} + +# Enable CMake verbose-mode during building +CMAKE_VERBOSE_BUILD_LLVM=${CMAKE_VERBOSE_BUILD_LLVM:-${CMAKE_VERBOSE_LLVM}} +CMAKE_VERBOSE_BUILD_CLANG=${CMAKE_VERBOSE_BUILD_CLANG:-${CMAKE_VERBOSE_CLANG}} +CMAKE_VERBOSE_BUILD_LLD=${CMAKE_VERBOSE_BUILD_LLD:-${CMAKE_VERBOSE_LLD}} + +# Enable CMake verbose-mode during checking +CMAKE_VERBOSE_CHECK_LLVM=${CMAKE_VERBOSE_CHECK_LLVM:-${CMAKE_VERBOSE_LLVM}} +CMAKE_VERBOSE_CHECK_CLANG=${CMAKE_VERBOSE_CHECK_CLANG:-${CMAKE_VERBOSE_CLANG}} +CMAKE_VERBOSE_CHECK_LLD=${CMAKE_VERBOSE_CHECK_LLD:-${CMAKE_VERBOSE_LLD}} + +# Enable CMake verbose-mode during installation +CMAKE_VERBOSE_INSTALL_LLVM=${CMAKE_VERBOSE_INSTALL_LLVM:-${CMAKE_VERBOSE_LLVM}} +CMAKE_VERBOSE_INSTALL_CLANG=${CMAKE_VERBOSE_INSTALL_CLANG:-${CMAKE_VERBOSE_CLANG}} +CMAKE_VERBOSE_INSTALL_LLD=${CMAKE_VERBOSE_INSTALL_LLD:-${CMAKE_VERBOSE_LLD}} + +install_dir() { + echo ${INSTALL_ROOT_DIR}/$1 +} + +build_dir() { + echo ${BUILD_ROOT_DIR}/$1 +} + +setup_llvm_project() { + build_step "Setup llvm-project" + + if [ ! -d "${LLVM_ROOT}" ]; then + build_step "Cloning llvm-project repo" + git clone --progress https://github.com/llvm/llvm-project.git ${LLVM_ROOT} + fi + + build_step "Updating llvm-project repo" + git -C "${LLVM_ROOT}" fetch origin + git -C "${LLVM_ROOT}" reset --hard "${REVISION}" + git -C "${LLVM_ROOT}" sparse-checkout init --cone +} + +pre_build_cleanup() { + build_step "Pre-build cleanup" + rm -rf ${INSTALL_ROOT_DIR} + rm -rf ${BUILD_ROOT_DIR} +} + +build_python_lit() { + local PYTHON_LIT_BUILD_DIR=$(build_dir python-lit) + local PYTHON_LIT_INSTALL_DIR=$(install_dir python-lit) + + build_step "Sparse checkout out python-lit" + git -C "${LLVM_ROOT}" sparse-checkout set llvm/utils/lit + + build_step "Building python-lit" + cd "${LLVM_ROOT}"/llvm/utils/lit + python3 setup.py build '--executable=/usr/bin/python3 -s' + + # TODO(kkleine): Fix failing test: FAIL: lit :: shtest-not.py (43 of 50) + #build_step "Testing python-lit" + #python3 lit.py tests + + build_step "Installing python-lit" + rm -rf ${PYTHON_LIT_INSTALL_DIR} + python3 setup.py install -O1 --skip-build --root ${PYTHON_LIT_INSTALL_DIR} + + build_step "Removing python-lit build dir" + rm -rf ${PYTHON_LIT_BUILD_DIR} + + cd - +} + +# We don't install python to the system so we need help python to find it +discover_python_lit() { + build_step "Help discover python-lit module" + local PYTHON_LIT_INSTALL_DIR=$(install_dir python-lit) + local pyver=$(echo 'import sys; print("{}.{}".format(sys.version_info.major, sys.version_info.minor))' | python3) + export PYTHONPATH="${PYTHONPATH:-}:${PYTHON_LIT_INSTALL_DIR}/usr/local/lib/python${pyver}/site-packages" +} + +build_llvm() { + local PYTHON_LIT_INSTALL_DIR=$(install_dir python-lit) + local LLVM_BUILD_DIR=$(build_dir llvm) + local LLVM_INSTALL_DIR=$(install_dir llvm) + + build_step "Sparse checkout out llvm" + git -C "${LLVM_ROOT}" sparse-checkout set llvm cmake third-party + + build_step "Configuring llvm" + + cmake ${CMAKE_TRACE_CONFIGURE_LLVM} \ + -S ${LLVM_ROOT}/llvm \ + -B ${LLVM_BUILD_DIR} \ + -G Ninja \ + -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ + -DLLVM_BUILD_LLVM_DYLIB=ON \ + -DLLVM_LINK_LLVM_DYLIB=ON \ + -DLLVM_INCLUDE_BENCHMARKS=OFF \ + -DLLVM_INSTALL_UTILS=ON \ + -DCMAKE_INSTALL_PREFIX=${LLVM_INSTALL_DIR} \ + -DLLVM_EXTERNAL_LIT=${PYTHON_LIT_INSTALL_DIR}/usr/local/bin/lit \ + -DLLVM_INCLUDE_UTILS:BOOL=ON \ + -DLLVM_INSTALL_UTILS:BOOL=ON \ + -DLLVM_UTILS_INSTALL_DIR:PATH=${LLVM_INSTALL_DIR}/bin \ + -DLLVM_CCACHE_BUILD=ON \ + -DLLVM_INSTALL_GTEST=ON + + build_step "Building llvm" + cmake --build ${LLVM_BUILD_DIR} ${CMAKE_VERBOSE_BUILD_LLVM} + + build_step "Testing llvm" + LD_LIBRARY_PATH="${LLVM_INSTALL_DIR}/lib64" cmake --build ${LLVM_BUILD_DIR} --target check-all ${CMAKE_VERBOSE_CHECK_LLVM} + + build_step "Installing llvm" + rm -rf ${LLVM_INSTALL_DIR} + cmake --install ${LLVM_BUILD_DIR} ${CMAKE_VERBOSE_INSTALL_LLVM} + + # This is meant to extinguish any dependency on files being taken + # from the llvm build dir when building clang. + build_step "Removing llvm build dir" + rm -rf "${LLVM_BUILD_DIR}" +} + +build_clang() { + local PYTHON_LIT_INSTALL_DIR=$(install_dir python-lit) + local LLVM_INSTALL_DIR=$(install_dir llvm) + local CLANG_BUILD_DIR=$(build_dir clang) + local CLANG_INSTALL_DIR=$(install_dir clang) + + build_step "Sparse checkout out clang" + git -C "${LLVM_ROOT}" sparse-checkout set clang cmake + + build_step "Configuring clang" + cmake ${CMAKE_TRACE_CONFIGURE_CLANG} \ + -S ${LLVM_ROOT}/clang \ + -B ${CLANG_BUILD_DIR} \ + -G Ninja \ + -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ + -DCLANG_LINK_CLANG_DYLIB=ON \ + -DCLANG_INCLUDE_TESTS=ON \ + -DLLVM_EXTERNAL_LIT=${PYTHON_LIT_INSTALL_DIR}/usr/local/bin/lit \ + -DCMAKE_INSTALL_PREFIX=${CLANG_INSTALL_DIR} \ + -DLLVM_ROOT=${LLVM_INSTALL_DIR} \ + -DLLVM_ENABLE_ASSERTIONS=ON \ + -DLLVM_TABLEGEN_EXE=${LLVM_INSTALL_DIR}/bin/llvm-tblgen \ + -DLLVM_CCACHE_BUILD=ON + + build_step "Building clang" + LD_LIBRARY_PATH="${LLVM_INSTALL_DIR}/lib64" cmake --build ${CLANG_BUILD_DIR} ${CMAKE_VERBOSE_BUILD_CLANG} + + # TODO(kwk): Add Clang check + + build_step "Installing clang" + rm -rf ${CLANG_INSTALL_DIR} + cmake --install ${CLANG_BUILD_DIR} ${CMAKE_VERBOSE_INSTALL_CLANG} + + build_step "Removing clang build dir" + rm -rf ${CLANG_BUILD_DIR} +} + +build_lld() { + local LLVM_INSTALL_DIR=$(install_dir llvm) + local PYTHON_LIT_INSTALL_DIR=$(install_dir python-lit) + local LLD_BUILD_DIR=$(build_dir lld) + local LLD_INSTALL_DIR=$(install_dir lld) + + build_step "Sparse checkout out lld" + git -C "${LLVM_ROOT}" sparse-checkout set lld cmake libunwind + + # We don't want to checkout the llvm source tree but sadly there are paths + # like ${LLVM_MAIN_SRC_DIR}/../libunwind/include in lld source code. They + # resolve to //../llvm/../libunwind/include which makes absolutely + # no sense when the llvm dir doesn't exist. Let's fix this by just providing + # the empty llvm dir so that paths are resolved without errors. + # TODO: I don't know how to fix this easily + rm -rf "${LLVM_ROOT}"/llvm && mkdir "${LLVM_ROOT}"/llvm + rm -rf ${LLD_INSTALL_DIR} + rm -rf ${LLD_BUILD_DIR} + + build_step "Configuring lld" + cmake ${CMAKE_TRACE_CONFIGURE_LLD} \ + -S ${LLVM_ROOT}/lld \ + -B ${LLD_BUILD_DIR} \ + -G Ninja \ + -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ + -DLLVM_LINK_LLVM_DYLIB=ON \ + -DCMAKE_INSTALL_PREFIX=${LLD_INSTALL_DIR} \ + -DLLVM_ROOT=${LLVM_INSTALL_DIR} \ + -DLLVM_CCACHE_BUILD=ON \ + -DLLVM_INCLUDE_TESTS=ON \ + -DLLVM_EXTERNAL_LIT=${PYTHON_LIT_INSTALL_DIR}/usr/local/bin/lit + + build_step "Building lld" + LD_LIBRARY_PATH="${LLVM_INSTALL_DIR}/lib64" cmake --build ${LLD_BUILD_DIR} ${CMAKE_VERBOSE_BUILD_LLD} + + build_step "Testing lld" + LD_LIBRARY_PATH="${LLVM_INSTALL_DIR}/lib64" cmake --build ${LLD_BUILD_DIR} --target check-lld ${CMAKE_VERBOSE_CHECK_LLD} + + build_step "Installing lld" + rm -rf ${LLD_INSTALL_DIR} + cmake --install ${LLD_BUILD_DIR} ${CMAKE_VERBOSE_INSTALL_LLD} + + build_step "Removing lld build dir" + rm -rf ${LLD_BUILD_DIR} +} + +setup_llvm_project +pre_build_cleanup + +build_python_lit +discover_python_lit +build_llvm +build_clang +build_lld + +exit 0 + diff --git a/zorg/buildbot/builders/annotated/ve-linux-steps.make b/zorg/buildbot/builders/annotated/ve-linux-steps.make new file mode 100644 --- /dev/null +++ b/zorg/buildbot/builders/annotated/ve-linux-steps.make @@ -0,0 +1,194 @@ +##### Interface Variables & Targets ##### +BUILDROOT?=$(error "BUILDROOT has to be the path to the worker's build/ directory.") + +# Renders one target per line in make order. Each target will made with a +# build step with the 'get-steps' annotated builder (ve-linux.py). +get-steps: + @echo "prepare" + @echo "build-llvm" + @echo "check-llvm" + @echo "install-llvm" + @echo "build-crt-ve" + @echo "check-crt-ve" + @echo "install-crt-ve" + @echo "build-runtimes-ve" + @echo "install-runtimes-ve" +# @echo "check-runtimes-ve" + +##### Tools ##### +CMAKE?=cmake +NINJA?=ninja +TOOL_CONFIG_CACHE?=${HOME}/tools-config.cmake + +##### Derived Configuration ##### + +# Path + +MONOREPO=${BUILDROOT}/../llvm-project + +# Build foders +LLVM_BUILD=${BUILDROOT}/build_llvm +CRT_BUILD_VE=${BUILDROOT}/build_crt_ve +RUNTIMES_BUILD_VE=${BUILDROOT}/build_runtimes_ve + +# 'Install' into the LLVM build tree. +LLVM_PREFIX=${BUILDROOT}/install + +# Install prefix structure +BUILT_CLANG=${LLVM_PREFIX}/bin/clang +BUILT_CLANGXX=${LLVM_PREFIX}/bin/clang++ +X86_TARGET=x86_64-unknown-linux-gnu +VE_TARGET=ve-unknown-linux-gnu + +### LLVM +LLVM_BUILD_TYPE=RelWithDebInfo + +### Compiler-RT +CRT_BUILD_TYPE=Release +CRT_OPTFLAGS=-O2 +CRT_TEST_OPTFLAGS=-O2 + +## Runtimes +RUNTIMES_BUILD_TYPE=Release +RUNTIMES_OPTFLAGS=-O2 + +##### Build Steps ##### + +# Standalone build has been prohibited. However, runtime build is not +# possible for VE because crt-ve is needed to be compiled by just compiled +# llvm. Such bootstrap build CMakefile is not merged yet. Check +# https://reviews.llvm.org/D89492 for details. +# +# As a result, we compile llvm for ve using following three steps. +# 1. Build llvm for X86 and VE with only X86 runtimes. +# 1.1 check-llvm +# 2. Build llvm for X86 and VE with x86 and VE compiler-rt runtimes. +# 2.1 check-compiler-rt for VE +# 3. Build x86 and VE all possible runtimes using 2. + +### Vanilla LLVM stage ### +build-llvm: + touch "${TOOL_CONFIG_CACHE}" + mkdir -p "${LLVM_BUILD}" + cd "${LLVM_BUILD}" && ${CMAKE} "${MONOREPO}/llvm" -G Ninja \ + -C "${TOOL_CONFIG_CACHE}" \ + -DCMAKE_BUILD_TYPE="${LLVM_BUILD_TYPE}" \ + -DCMAKE_INSTALL_PREFIX="${LLVM_PREFIX}" \ + -DCLANG_LINK_CLANG_DYLIB=Off \ + -DLLVM_BUILD_LLVM_DYLIB=Off \ + -DLLVM_LINK_LLVM_DYLIB=Off \ + -DLLVM_TARGETS_TO_BUILD="X86;VE" \ + -DLLVM_ENABLE_PROJECTS="clang" \ + -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=On + cd "${LLVM_BUILD}" && ${NINJA} + +check-llvm: + cd "${LLVM_BUILD}" && ${NINJA} check-all + +install-llvm: + cd "${LLVM_BUILD}" && ${NINJA} install + +### Compiler-RT ### + +build-crt-ve: + mkdir -p "${CRT_BUILD_VE}" + cd "${CRT_BUILD_VE}" && ${CMAKE} "${MONOREPO}/llvm" -G Ninja \ + -C "${TOOL_CONFIG_CACHE}" \ + -DCMAKE_BUILD_TYPE="${CRT_BUILD_TYPE}" \ + -DCMAKE_INSTALL_PREFIX="${LLVM_PREFIX}" \ + -DCMAKE_C_COMPILER="${BUILT_CLANG}" \ + -DCMAKE_CXX_COMPILER="${BUILT_CLANGXX}" \ + -DCMAKE_CXX_FLAGS_RELEASE="${CRT_OPTFLAGS}" \ + -DCMAKE_C_FLAGS_RELEASE="${CRT_OPTFLAGS}" \ + -DCLANG_LINK_CLANG_DYLIB=Off \ + -DLLVM_BUILD_LLVM_DYLIB=Off \ + -DLLVM_LINK_LLVM_DYLIB=Off \ + -DLLVM_TARGETS_TO_BUILD="X86;VE" \ + -DLLVM_ENABLE_PROJECTS="clang" \ + -DLLVM_ENABLE_RUNTIMES="compiler-rt" \ + -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=On \ + -DLLVM_INSTALL_UTILS=On \ + -DLLVM_RUNTIME_TARGETS="${X86_TARGET};${VE_TARGET}" \ + -DRUNTIMES_${VE_TARGET}_COMPILER_RT_BUILD_BUILTINS=On \ + -DRUNTIMES_${VE_TARGET}_COMPILER_RT_BUILD_CRT=On \ + -DRUNTIMES_${VE_TARGET}_COMPILER_RT_BUILD_SANITIZERS=Off \ + -DRUNTIMES_${VE_TARGET}_COMPILER_RT_BUILD_XRAY=Off \ + -DRUNTIMES_${VE_TARGET}_COMPILER_RT_BUILD_LIBFUZZER=Off \ + -DRUNTIMES_${VE_TARGET}_COMPILER_RT_BUILD_PROFILE=On \ + -DRUNTIMES_${VE_TARGET}_COMPILER_RT_BUILD_MEMPROF=Off \ + -DRUNTIMES_${VE_TARGET}_COMPILER_RT_BUILD_ORC=Off \ + -DRUNTIMES_${VE_TARGET}_COMPILER_RT_BUILD_GWP_ASAN=Off \ + -DRUNTIMES_${VE_TARGET}_COMPILER_RT_USE_BUILTINS_LIBRARY=On \ + -DRUNTIMES_${VE_TARGET}_COMPILER_RT_TEST_COMPILER_CFLAGS="-target ${VE_TARGET} ${CRT_TEST_OPTFLAGS}" + cd "${CRT_BUILD_VE}" && ${NINJA} + +check-crt-ve: + cd "${CRT_BUILD_VE}" && ${NINJA} check-compiler-rt-ve-unknown-linux-gnu + +install-crt-ve: + cd "${CRT_BUILD_VE}" && ${NINJA} install + +### Runtimes ### + +# It is not possible to compile clang nor compiler-rt with other runtimes +# at once for VE. Because the crtbegin bootstrap mechanism is merged yet. +# See https://reviews.llvm.org/D89492 for details. Without this patch, +# running regression tests for runtimes is also not possible for VE. +# +# Once this patch is merged, it will be possible to compile runtimes +# with following defines using single build step. +# -DLLVM_ENABLE_PROJECTS="clang" \ +# -DLLVM_ENABLE_RUNTIMES="compiler-rt;libunwind" \ +# -DRUNTIMES_${VE_TARGET}_COMPILER_RT_BOOTSTRAP=On \ + +build-runtimes-ve: + mkdir -p "${RUNTIMES_BUILD_VE}" + cd "${RUNTIMES_BUILD_VE}" && ${CMAKE} "${MONOREPO}/llvm" -G Ninja \ + -C "${TOOL_CONFIG_CACHE}" \ + -DCMAKE_BUILD_TYPE="${RUNTIMES_BUILD_TYPE}" \ + -DCMAKE_INSTALL_PREFIX="${LLVM_PREFIX}" \ + -DCMAKE_C_COMPILER="${BUILT_CLANG}" \ + -DCMAKE_CXX_COMPILER="${BUILT_CLANGXX}" \ + -DCMAKE_CXX_FLAGS_RELEASE="${RUNTIMES_OPTFLAGS}" \ + -DCMAKE_C_FLAGS_RELEASE="${RUNTIMES_OPTFLAGS}" \ + -DCLANG_LINK_CLANG_DYLIB=Off \ + -DLLVM_BUILD_LLVM_DYLIB=Off \ + -DLLVM_LINK_LLVM_DYLIB=Off \ + -DLLVM_TARGETS_TO_BUILD="X86;VE" \ + -DLLVM_ENABLE_PROJECTS="" \ + -DLLVM_ENABLE_RUNTIMES="libunwind" \ + -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=On \ + -DLLVM_RUNTIME_TARGETS="${X86_TARGET};${VE_TARGET}" \ + -DRUNTIMES_${VE_TARGET}_COMPILER_RT_BUILD_BUILTINS=On \ + -DRUNTIMES_${VE_TARGET}_COMPILER_RT_BUILD_CRT=On \ + -DRUNTIMES_${VE_TARGET}_COMPILER_RT_BUILD_SANITIZERS=Off \ + -DRUNTIMES_${VE_TARGET}_COMPILER_RT_BUILD_XRAY=Off \ + -DRUNTIMES_${VE_TARGET}_COMPILER_RT_BUILD_LIBFUZZER=Off \ + -DRUNTIMES_${VE_TARGET}_COMPILER_RT_BUILD_PROFILE=On \ + -DRUNTIMES_${VE_TARGET}_COMPILER_RT_BUILD_MEMPROF=Off \ + -DRUNTIMES_${VE_TARGET}_COMPILER_RT_BUILD_ORC=Off \ + -DRUNTIMES_${VE_TARGET}_COMPILER_RT_BUILD_GWP_ASAN=Off \ + -DRUNTIMES_${VE_TARGET}_COMPILER_RT_USE_BUILTINS_LIBRARY=On \ + -DRUNTIMES_${VE_TARGET}_LIBCXXABI_USE_LLVM_UNWINDER=On \ + -DRUNTIMES_${VE_TARGET}_LIBCXXABI_USE_COMPILER_RT=On \ + -DRUNTIMES_${VE_TARGET}_LIBCXX_USE_COMPILER_RT=On \ + -DRUNTIMES_${VE_TARGET}_CMAKE_C_COMPILER="${BUILT_CLANG}" \ + -DRUNTIMES_${VE_TARGET}_COMPILER_RT_TEST_COMPILER_CFLAGS="-target ${VE_TARGET} ${RUNTIMES_TEST_OPTFLAGS}" \ + -DRUNTIMES_${VE_TARGET}_LIBCXXABI_TEST_COMPILER_CFLAGS="-target ${VE_TARGET} ${RUNTIMES_TEST_OPTFLAGS}" \ + -DRUNTIMES_${VE_TARGET}_LIBCXX_TEST_COMPILER_CFLAGS="-target ${VE_TARGET} ${RUNTIMES_TEST_OPTFLAGS}" \ + -DRUNTIMES_${VE_TARGET}_LIBUNWIND_TEST_COMPILER_CFLAGS="-target ${VE_TARGET} ${RUNTIMES_TEST_OPTFLAGS}" + cd "${RUNTIMES_BUILD_VE}" && ${NINJA} + +check-runtimes-ve: + cd "${RUNTIMES_BUILD_VE}" && ${NINJA} check-runtimes-ve-unknown-linux-gnu + +install-runtimes-ve: + cd "${RUNTIMES_BUILD_VE}" && ${NINJA} install + +# Clearout the temporary install prefix. +prepare: + # Build everything from scratch - TODO: incrementalize later. + rm -rf "${LLVM_PREFIX}" + rm -rf "${LLVM_BUILD}" + rm -rf "${CRT_BUILD_VE}" + rm -rf "${RUNTIMES_BUILD_VE}" diff --git a/zorg/buildbot/builders/annotated/ve-linux.py b/zorg/buildbot/builders/annotated/ve-linux.py new file mode 100644 --- /dev/null +++ b/zorg/buildbot/builders/annotated/ve-linux.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 + +import argparse +import os +import subprocess +import sys +import traceback +import util +from contextlib import contextmanager + + +def main(argv): + # Avoid a buildmaster update by hard-coding this for now. + makefile = 've-linux-steps.make' + + # TODO: Make Makefile configurable and update buildmaster. + # ap = argparse.ArgumentParser() + # ap.add_argument('makefile', help='The Makefile to use. (relative to annoted/ folder of llvm-zorg).') + # args, _ = ap.parse_known_args() + # makefile = args.makefile + + worker_dir = os.path.abspath(os.path.join('..')) + annotated_dir = os.path.join(worker_dir, 'llvm-zorg', 'zorg', 'buildbot', 'builders', 'annotated') + makefile_path = os.path.join(annotated_dir, makefile) + + # Query step list from makefile. + build_targets=[] + with step('prepare', halt_on_fail=True): + build_targets = get_steps(makefile_path) + + make_vars = { + 'BUILDROOT' : os.path.join(worker_dir, 'build') + } + + for target in build_targets: + halt_on_fail = not ("check-" in target) + with step(target, halt_on_fail=halt_on_fail): + make_cmd = build_make_cmd(makefile_path, target, make_vars) + run_command(make_cmd, cwd='.') + +@contextmanager +def step(step_name, halt_on_fail=False): + util.report('@@@BUILD_STEP {}@@@'.format(step_name)) + if halt_on_fail: + util.report('@@@HALT_ON_FAILURE@@@') + try: + yield + except Exception as e: + if isinstance(e, subprocess.CalledProcessError): + util.report( + '{} exited with return code {}.'.format(e.cmd, e.returncode) + ) + util.report('The build step threw an exception...') + traceback.print_exc() + + util.report('@@@STEP_FAILURE@@@') + finally: + sys.stdout.flush() + + +def get_steps(makefile): + try: + make_cmd = build_make_cmd(makefile, 'get-steps') + raw_steps = capture_cmd_stdout(make_cmd) + return raw_steps.decode('utf-8').split('\n')[:-1] + except: + return [] + +def build_make_cmd(makefile, target, make_vars={}): + make_cmd = ['make', '-f', makefile] + if not target is None: + make_cmd.append(target) + for k,v in make_vars.items(): + make_cmd += ["{}={}".format(k, v)] + return make_cmd + +def capture_cmd_stdout(cmd, **kwargs): + return subprocess.run(cmd, shell=False, check=True, stdout=subprocess.PIPE, **kwargs).stdout + +def run_command(cmd, **kwargs): + util.report_run_cmd(cmd, **kwargs) + +if __name__ == '__main__': + sys.path.append(os.path.dirname(__file__)) + sys.exit(main(sys.argv)) diff --git a/zorg/buildbot/builders/sanitizers/buildbot_android.sh b/zorg/buildbot/builders/sanitizers/buildbot_android.sh --- a/zorg/buildbot/builders/sanitizers/buildbot_android.sh +++ b/zorg/buildbot/builders/sanitizers/buildbot_android.sh @@ -8,18 +8,7 @@ . ${HERE}/buildbot_functions.sh . ${HERE}/buildbot_android_functions.sh -ROOT=`pwd` -PLATFORM=`uname` LOCAL_IPS=`hostname -I` -export PATH="/usr/local/bin:$PATH" - -LLVM=$ROOT/llvm -CMAKE_COMMON_OPTIONS="${CMAKE_COMMON_OPTIONS:-}" -CMAKE_COMMON_OPTIONS="${CMAKE_COMMON_OPTIONS:-} -GNinja -DCMAKE_BUILD_TYPE=Release" - -if [ -e /usr/include/plugin-api.h ]; then - CMAKE_COMMON_OPTIONS="${CMAKE_COMMON_OPTIONS}" -fi clobber @@ -32,7 +21,7 @@ buildbot_update -CMAKE_COMMON_OPTIONS="$CMAKE_COMMON_OPTIONS -DLLVM_ENABLE_ASSERTIONS=ON" +CMAKE_COMMON_OPTIONS+=" -DLLVM_ENABLE_ASSERTIONS=ON" build_stage2_android @@ -47,5 +36,5 @@ build_android arm build_android i686 -# Arm hardware is temporarily offline -test_android i686:x86 arm:armeabi-v7a aarch64:arm64-v8a +# i686:x86 hardware is offline. +test_android arm:armeabi-v7a aarch64:arm64-v8a diff --git a/zorg/buildbot/builders/sanitizers/buildbot_android_functions.sh b/zorg/buildbot/builders/sanitizers/buildbot_android_functions.sh --- a/zorg/buildbot/builders/sanitizers/buildbot_android_functions.sh +++ b/zorg/buildbot/builders/sanitizers/buildbot_android_functions.sh @@ -1,9 +1,5 @@ #!/usr/bin/env bash -CLOBBER="android_ndk android-ndk-* platform-tools *.zip shards_* test_android_* tested_arch_*" -STAGE2_CLOBBER="compiler_rt_build_android_* llvm_build_android_*" -STAGE1_CLOBBER="llvm_build64 ${STAGE2_CLOBBER}" - ANDROID_NDK_VERSION=21 ANDROID_API=24 NDK_DIR=android_ndk @@ -17,7 +13,7 @@ [[ -d ${NDK_DIR} ]] && rm -rf ${NDK_DIR} [[ -d ${VERSION} ]] && rm -rf ${VERSION} [[ -f ${FILE_NAME} ]] && rm -f ${FILE_NAME} - wget ${NDK_URL} + wget -e dotbytes=10m ${NDK_URL} unzip ${FILE_NAME} > /dev/null mv ${VERSION} ${NDK_DIR} echo ${NDK_URL} > ${NDK_DIR}/android_ndk_url @@ -27,7 +23,7 @@ echo @@@BUILD_STEP downloading Android Platform Tools@@@ local FILE_NAME=platform-tools-latest-linux.zip [[ -f ${FILE_NAME} ]] && rm -f ${FILE_NAME} - wget https://dl.google.com/android/repository/${FILE_NAME} + wget -e dotbytes=10m https://dl.google.com/android/repository/${FILE_NAME} unzip ${FILE_NAME} > /dev/null fi export PATH=$ROOT/platform-tools/:$PATH @@ -36,22 +32,15 @@ function build_stage2_android() { # Build self-hosted tree with fresh Clang and -Werror. local CMAKE_OPTIONS="${CMAKE_COMMON_OPTIONS} -DLLVM_ENABLE_WERROR=ON ${STAGE1_AS_COMPILER} -DCMAKE_C_FLAGS=-gmlt -DCMAKE_CXX_FLAGS=-gmlt" - CMAKE_OPTIONS="${CMAKE_OPTIONS} -DLLVM_ENABLE_PROJECTS='clang;lld' -DLLVM_USE_LINKER=lld" - - if ccache -s ; then - CMAKE_OPTIONS="${CMAKE_OPTIONS} -DLLVM_CCACHE_BUILD=ON" - rm_dirs llvm_build64 - fi + CMAKE_OPTIONS="${CMAKE_OPTIONS} -DLLVM_ENABLE_PROJECTS='clang;lld' -DCLANG_DEFAULT_RTLIB=libgcc" echo @@@BUILD_STEP bootstrap clang@@@ - rm_dirs ${STAGE2_CLOBBER} - mkdir -p llvm_build64 if [[ "$(cat llvm_build64/CMAKE_OPTIONS)" != "${CMAKE_OPTIONS}" ]] ; then (cd llvm_build64 && cmake ${CMAKE_OPTIONS} -DLLVM_BUILD_EXTERNAL_COMPILER_RT=ON $LLVM && \ - echo ${CMAKE_OPTIONS} > CMAKE_OPTIONS) || echo @@@STEP_FAILURE@@@ + echo ${CMAKE_OPTIONS} > CMAKE_OPTIONS) || build_failure fi - ninja -C llvm_build64 || (echo @@@STEP_FAILURE@@@ && exit 2) + ninja -C llvm_build64 || (build_failure && exit 2) } function configure_android { # ARCH triple @@ -63,7 +52,7 @@ local ANDROID_LIBRARY_OUTPUT_DIR=$(ls -d $ROOT/llvm_build64/lib/clang/* | tail -1) local ANDROID_EXEC_OUTPUT_DIR=$ROOT/llvm_build64/bin - local ANDROID_FLAGS="--target=${_triple}${ANDROID_API} --sysroot=$ANDROID_TOOLCHAIN/sysroot -gcc-toolchain $ANDROID_TOOLCHAIN -B$ANDROID_TOOLCHAIN" + local ANDROID_FLAGS="--target=${_triple}${ANDROID_API} --sysroot=$ANDROID_TOOLCHAIN/sysroot --gcc-toolchain=$ANDROID_TOOLCHAIN -B$ANDROID_TOOLCHAIN" local ANDROID_CXX_FLAGS="$ANDROID_FLAGS -stdlib=libc++ -I/$ANDROID_TOOLCHAIN/sysroot/usr/include/c++/v1" local CLANG_PATH=$ROOT/llvm_build64/bin/clang @@ -77,6 +66,7 @@ mkdir -p llvm_build_android_$_arch (cd llvm_build_android_$_arch && cmake \ + ${CMAKE_COMMON_OPTIONS} \ -DLLVM_ENABLE_WERROR=OFF \ -DCMAKE_C_COMPILER=$CLANG_PATH \ -DCMAKE_CXX_COMPILER=$CLANGXX_PATH \ @@ -88,15 +78,15 @@ -DCMAKE_SKIP_RPATH=ON \ -DLLVM_BUILD_RUNTIME=OFF \ -DLLVM_TABLEGEN=$ROOT/llvm_build64/bin/llvm-tblgen \ - ${CMAKE_COMMON_OPTIONS} \ - $LLVM || echo @@@STEP_FAILURE@@@) & + $LLVM || build_failure) & local COMPILER_RT_OPTIONS="$(readlink -f $LLVM/../compiler-rt)" (cd compiler_rt_build_android_$_arch && cmake \ + ${CMAKE_COMMON_OPTIONS} \ -DCMAKE_C_COMPILER=$CLANG_PATH \ -DCMAKE_CXX_COMPILER=$CLANGXX_PATH \ - -DLLVM_CONFIG_PATH=$ROOT/llvm_build64/bin/llvm-config \ + -DLLVM_CMAKE_DIR=$ROOT/llvm_build64 \ -DCOMPILER_RT_BUILD_BUILTINS=OFF \ -DCOMPILER_RT_INCLUDE_TESTS=ON \ -DCOMPILER_RT_ENABLE_WERROR=ON \ @@ -105,12 +95,11 @@ -DCMAKE_CXX_FLAGS="$ANDROID_CXX_FLAGS" \ -DSANITIZER_CXX_ABI="libcxxabi" \ -DCOMPILER_RT_TEST_COMPILER_CFLAGS="$ANDROID_FLAGS" \ - -DCOMPILER_RT_TEST_TARGET_TRIPLE=$_triple \ + -DCOMPILER_RT_DEFAULT_TARGET_TRIPLE=$_triple \ -DCOMPILER_RT_OUTPUT_DIR="$ANDROID_LIBRARY_OUTPUT_DIR" \ -DCOMPILER_RT_EXEC_OUTPUT_DIR="$ANDROID_EXEC_OUTPUT_DIR" \ -DLLVM_LIT_ARGS="-vv --show-unsupported --show-xfail" \ - ${CMAKE_COMMON_OPTIONS} \ - ${COMPILER_RT_OPTIONS} || echo @@@STEP_FAILURE@@@) & + ${COMPILER_RT_OPTIONS} || build_failure) & } BUILD_RT_ERR="" @@ -120,22 +109,26 @@ echo @@@BUILD_STEP build android/$_arch@@@ if ! ninja -C llvm_build_android_$_arch llvm-symbolizer ; then BUILD_RT_ERR="${BUILD_RT_ERR}|${_arch}|" - echo @@@STEP_FAILURE@@@ + build_failure fi if ! ninja -C compiler_rt_build_android_$_arch ; then BUILD_RT_ERR="${BUILD_RT_ERR}|${_arch}|" - echo @@@STEP_FAILURE@@@ + build_failure fi } -# If a multiarch device has x86 as the first arch, remove everything else from -# the list. This captures cases like [x86,armeabi-v7a], where the arm part is -# software emulation and incompatible with ASan. +# If a multiarch device's first arch starts with x86, remove archs that don't +# start with x86. Handle cases like: +# - x86,armeabi-v7a +# - x86_64,x86,arm64-v8a,armeabi-v7a,armeabi +# - x86_64,arm64-v8a,x86,armeabi-v7a,armeabi +# The arm part is software emulation and incompatible with ASan. function patch_abilist { # IN OUT local _abilist=$1 local _out=$2 - if [[ "$_abilist" == "x86,"* ]]; then - _abilist="x86" + if [[ "$_abilist" == "x86"* ]]; then + _abilist=$(echo $_abilist | tr , '\n' | grep '^x86') + _abilist=$(echo $_abilist | tr ' ' ,) fi eval $_out="'$_abilist'" } @@ -154,19 +147,25 @@ shift ABILIST=$(${ADB} -s $_serial shell getprop ro.product.cpu.abilist) - patch_abilist $ABILIST ABILIST + patch_abilist "$ABILIST" ABILIST for _arg in "$@"; do local _arch=${_arg%:*} local _abi=${_arg#*:} if [[ $BUILD_RT_ERR == *"|${_arch}|"* ]]; then - echo "@@@STEP_FAILURE@@ skipping tests on ${_arch}" + echo "skipping tests on ${_arch}" + build_failure continue fi if [[ $ABILIST == *"$_abi"* ]]; then echo "$_serial" >> tested_arch_$_arch BUILD_ID=$(${ADB} -s $_serial shell getprop ro.build.id | tr -d '\r') BUILD_FLAVOR=$(${ADB} -s $_serial shell getprop ro.build.flavor | tr -d '\r') - test_arch_on_device "$_arch" "$_serial" "$BUILD_ID" "$BUILD_FLAVOR" + ( + # Test only one arch at a time to avoid simultaneously writes into the + # same compiler-rt build dir. + flock -x $lock_fd + test_arch_on_device "$_arch" "$_serial" "$BUILD_ID" "$BUILD_FLAVOR" + ) {lock_fd}>$ROOT/${_arch}.lock fi done } @@ -181,7 +180,7 @@ } function test_android { - if [[ "${BUILDBOT_SLAVENAME:-}" != "" ]]; then + if [[ -v BUILDBOT_BUILDERNAME ]]; then restart_adb_server fi @@ -192,13 +191,12 @@ rm -rf test_android_log_* rm -rf tested_arch_* rm -rf shards_* - LOGS= for SERIAL in $ANDROID_DEVICES; do - LOG="$(mktemp test_android_log_XXXX)" - (test_on_device "$SERIAL" $@ 2>&1 >"$LOG") & - LOGS="$LOGS $LOG,$!" + test_on_device "$SERIAL" $@ done - tail_pids "$LOGS" + + # Return to avoid exception if we already have error. + [[ $BUILD_RT_ERR == "" ]] || return for _arg in "$@"; do local _arch=${_arg%:*} @@ -227,8 +225,9 @@ LOGS= for ((SHARD=0; SHARD < $NUM_SHARDS; SHARD++)); do LOG=${_log_prefix}_$SHARD - local ENV="$_env GTEST_TOTAL_SHARDS=$NUM_SHARDS GTEST_SHARD_INDEX=$SHARD LD_LIBRARY_PATH=$DEVICE_ROOT" - ( (run_command_on_device "$ENV $DEVICE_ROOT/$_test" || echo @@@STEP_FAILURE@@@) 2>&1 >${_log_prefix}_$SHARD ) & + # 'adb shell ' on Fugu is missing TMPDIR. + local ENV="$_env GTEST_TOTAL_SHARDS=$NUM_SHARDS GTEST_SHARD_INDEX=$SHARD LD_LIBRARY_PATH=$DEVICE_ROOT TMPDIR=$DEVICE_TMPDIR" + ( (run_command_on_device "$ENV $DEVICE_ROOT/$_test" || build_failure) 2>&1 >${_log_prefix}_$SHARD ) & LOGS="$LOGS $LOG,$!" done tail_pids "$LOGS" || true @@ -250,6 +249,7 @@ local RT_DIR=$($ROOT/llvm_build64/bin/clang -print-resource-dir)/lib/linux local COMPILER_RT_BUILD_DIR=$ROOT/compiler_rt_build_android_$_arch export ADB=adb + export DEVICE_TMPDIR=/data/local/tmp export DEVICE_ROOT=/data/local/tmp/Output export ANDROID_SERIAL=$_serial echo "Serial $_serial" @@ -257,6 +257,7 @@ echo @@@BUILD_STEP device setup [$DEVICE_DESCRIPTION]@@@ $ADB wait-for-device $ADB devices + $ADB shell getprop ro.build.version.release # Kill leftover symbolizers. TODO: figure out what's going on. $ADB shell pkill llvm-symbolizer || true @@ -272,12 +273,12 @@ $COMPILER_RT_BUILD_DIR/lib/asan/tests/AsanNoinstTest" for F in $FILES ; do - ( $ADB push $F $DEVICE_ROOT/ >/dev/null || echo @@@STEP_FAILURE@@@ )& + ( $ADB push $F $DEVICE_ROOT/ >/dev/null || build_failure )& done wait echo @@@BUILD_STEP run lit tests [$DEVICE_DESCRIPTION]@@@ - (cd $COMPILER_RT_BUILD_DIR && ninja check-all) || echo @@@STEP_FAILURE@@@ + (cd $COMPILER_RT_BUILD_DIR && ninja check-all) || build_failure run_tests_sharded sanitizer_common SanitizerTest "" run_tests_sharded asan AsanNoinstTest "" diff --git a/zorg/buildbot/builders/sanitizers/buildbot_bisect_run.sh b/zorg/buildbot/builders/sanitizers/buildbot_bisect_run.sh new file mode 100755 --- /dev/null +++ b/zorg/buildbot/builders/sanitizers/buildbot_bisect_run.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# This script uses BUILDBOT_REVISION to read bad and good patch for bisect. +# The format is: BUILDBOT_REVISION=good:bad + +set -x +set -e +set -u + +HERE="$(cd $(dirname $0) && pwd)" +. ${HERE}/buildbot_functions.sh + +BUILDBOT_REVISION=origin/main buildbot_update + +echo @@@BUILD_STEP bisecting ${BUILDBOT_REVISION}@@@ + +# Try to get them out from the bisect string in BUILDBOT_REVISION first. +GOOD="${BUILDBOT_REVISION/:*/}" +BAD="${BUILDBOT_REVISION/*:/}" + +# If not provided through BUILDBOT_REVISION, use some defaults. +if [[ "$BAD" == "" ]]; then + BAD="origin/main" +fi +if [[ "$GOOD" == "" ]]; then + GOOD="origin/main~100" +fi + +cd "${LLVM}/.." + +( + export BUILDBOT_BISECT_MODE=1 + + if git bisect start $BAD $GOOD; then + git bisect run bash -c "cd $ROOT && $*" + fi + + echo @@@BUILD_STEP bisect result@@@ + git bisect log +) || true + +git bisect reset +build_exception # Bisect is neither FAIL nor PASS. diff --git a/zorg/buildbot/builders/sanitizers/buildbot_bisect_test.sh b/zorg/buildbot/builders/sanitizers/buildbot_bisect_test.sh new file mode 100755 --- /dev/null +++ b/zorg/buildbot/builders/sanitizers/buildbot_bisect_test.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -x +set -e +set -u + +HERE="$(cd $(dirname $0) && pwd)" +. ${HERE}/buildbot_functions.sh + +buildbot_update + +if git -c $LLVM merge-base HEAD "${1}" | grep "${1}"; then + build_failure +fi diff --git a/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_asan.sh b/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_asan.sh --- a/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_asan.sh +++ b/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_asan.sh @@ -7,12 +7,7 @@ HERE="$(cd $(dirname $0) && pwd)" . ${HERE}/buildbot_functions.sh -ROOT=`pwd` -PLATFORM=`uname` -export PATH="/usr/local/bin:$PATH" - -LLVM=$ROOT/llvm -CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON" +CMAKE_COMMON_OPTIONS+=" -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF" clobber @@ -22,6 +17,9 @@ build_stage1_clang +check_stage1_asan + + # Stage 2 / AddressSanitizer build_stage2_asan @@ -35,3 +33,4 @@ check_stage3_asan +cleanup diff --git a/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_hwasan.sh b/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_hwasan.sh new file mode 100755 --- /dev/null +++ b/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_hwasan.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +set -x +set -e +set -u + +HERE="$(cd $(dirname $0) && pwd)" +. ${HERE}/buildbot_functions.sh + +CMAKE_COMMON_OPTIONS+=" -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF" + +clobber + +buildbot_update + +# Stage 1 + +build_stage1_clang + +check_stage1_hwasan + +# Stage 2 / HWAddressSanitizer + +build_stage2_hwasan + +check_stage2_hwasan + +# Stage 3 / HWAddressSanitizer + +build_stage3_hwasan + +check_stage3_hwasan + +cleanup diff --git a/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_msan.sh b/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_msan.sh --- a/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_msan.sh +++ b/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_msan.sh @@ -7,12 +7,7 @@ HERE="$(cd $(dirname $0) && pwd)" . ${HERE}/buildbot_functions.sh -ROOT=`pwd` -PLATFORM=`uname` -export PATH="/usr/local/bin:$PATH" - -LLVM=$ROOT/llvm -CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON" +CMAKE_COMMON_OPTIONS+=" -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF" clobber @@ -22,14 +17,58 @@ build_stage1_clang -# Stage 2 / Memory Sanitizer +check_stage1_msan + +for I in 1 2 ; do + # Stage 2 / Memory Sanitizer + + build_stage2_msan + + ( + check_stage2_msan |& tee stage2_msan.log + exit ${PIPESTATUS[0]} + ) + + if grep "WARNING: MemorySanitizer" stage2_msan.log ; then + # Stage 2 / MemoryWithOriginsSanitizer + ( + build_stage2_msan_track_origins + + check_stage2_msan_track_origins + ) + fi + + # Stage 3 / MemorySanitizer + ( + { + build_stage3_msan + + check_stage3_msan + } |& tee stage3_msan.log + exit ${PIPESTATUS[0]} + ) + + if grep "WARNING: MemorySanitizer" stage3_msan.log ; then + # Stage 3 / MemoryWithOriginsSanitizer + ( + build_stage2_msan_track_origins + + build_stage3_msan_track_origins -build_stage2_msan + check_stage3_msan_track_origins + ) + fi -check_stage2_msan + # FIXME: stage3/msan check_1 crashes. + break -# Stage 3 / MemorySanitizer + cleanup -build_stage3_msan + # Repeat with strict settings. + CMAKE_COMMON_OPTIONS+=" -DCMAKE_C_FLAGS=-fno-inline" + CMAKE_COMMON_OPTIONS+=" -DCMAKE_CXX_FLAGS=-fno-inline" + CMAKE_COMMON_OPTIONS+=" -DCMAKE_C_FLAGS_RELEASE=-Oz" + CMAKE_COMMON_OPTIONS+=" -DCMAKE_CXX_FLAGS_RELEASE=-Oz" +done -check_stage3_msan +cleanup \ No newline at end of file diff --git a/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_ubsan.sh b/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_ubsan.sh --- a/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_ubsan.sh +++ b/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_ubsan.sh @@ -7,12 +7,7 @@ HERE="$(cd $(dirname $0) && pwd)" . ${HERE}/buildbot_functions.sh -ROOT=`pwd` -PLATFORM=`uname` -export PATH="/usr/local/bin:$PATH" - -LLVM=$ROOT/llvm -CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON" +CMAKE_COMMON_OPTIONS+=" -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF" clobber @@ -22,6 +17,8 @@ build_stage1_clang +check_stage1_ubsan + # Stage 2 / UndefinedBehaviorSanitizer build_stage2_ubsan @@ -33,3 +30,5 @@ build_stage3_ubsan check_stage3_ubsan + +cleanup diff --git a/zorg/buildbot/builders/sanitizers/buildbot_cmake.sh b/zorg/buildbot/builders/sanitizers/buildbot_cmake.sh --- a/zorg/buildbot/builders/sanitizers/buildbot_cmake.sh +++ b/zorg/buildbot/builders/sanitizers/buildbot_cmake.sh @@ -7,38 +7,38 @@ HERE="$(dirname $0)" . ${HERE}/buildbot_functions.sh -ROOT=`pwd` PLATFORM=`uname` ARCH=`uname -m` -export PATH="/usr/local/bin:$PATH" +ninja --version || export PATH="/usr/local/bin:$PATH" export ANDROID_SDK_HOME=$ROOT/../../.. -# Always clobber bootstrap build trees. -rm -rf compiler_rt_build llvm_build* symbolizer_build* +CMAKE_ARGS="" +for arg in "$@" +do + case $arg in + --CMAKE_ARGS=*) + CMAKE_ARGS="${arg#*=}" + esac +done -USE_CCACHE= -if ccache -s ; then - USE_CCACHE="-DLLVM_CCACHE_BUILD=ON" - rm -rf clang_build -fi +# Always clobber bootstrap build trees. +clobber SUPPORTS_32_BITS=${SUPPORTS_32_BITS:-1} -MAKE_JOBS=${MAX_MAKE_JOBS:-$(nproc)} -LLVM=$ROOT/llvm ZLIB=$ROOT/zlib -CMAKE_COMMON_OPTIONS="${CMAKE_COMMON_OPTIONS:-} -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_PARALLEL_LINK_JOBS=10" +CMAKE_COMMON_OPTIONS+=" -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_PARALLEL_LINK_JOBS=10 -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF ${CMAKE_ARGS}" ENABLE_LIBCXX_FLAG= if [ "$PLATFORM" == "Darwin" ]; then - CMAKE_COMMON_OPTIONS="${CMAKE_COMMON_OPTIONS} -DPYTHON_EXECUTABLE=/usr/bin/python" + CMAKE_COMMON_OPTIONS+=" -DPYTHON_EXECUTABLE=/usr/bin/python" ENABLE_LIBCXX_FLAG="-DLLVM_ENABLE_LIBCXX=ON" fi if [ -e /usr/include/plugin-api.h ]; then - CMAKE_COMMON_OPTIONS="${CMAKE_COMMON_OPTIONS} -DLLVM_BINUTILS_INCDIR=/usr/include" + CMAKE_COMMON_OPTIONS+=" -DLLVM_BINUTILS_INCDIR=/usr/include" fi -CMAKE_COMMON_OPTIONS="${CMAKE_COMMON_OPTIONS} -DLLVM_ENABLE_PROJECTS='clang;compiler-rt'" +CMAKE_COMMON_OPTIONS+=" -DLLVM_ENABLE_PROJECTS='clang;compiler-rt'" CHECK_LIBCXX=${CHECK_LIBCXX:-1} CHECK_SYMBOLIZER=${CHECK_SYMBOLIZER:-$CHECK_LIBCXX} @@ -50,7 +50,6 @@ CHECK_TSAN=0 CHECK_LSAN=0 CHECK_DFSAN=0 -CHECK_SCUDO=0 CHECK_SCUDO_STANDALONE=0 CHECK_CFI=0 case "$ARCH" in @@ -62,7 +61,6 @@ CHECK_TSAN=1 CHECK_LSAN=1 CHECK_DFSAN=1 - CHECK_SCUDO=1 CHECK_SCUDO_STANDALONE=1 CHECK_CFI=1 ;; @@ -73,7 +71,6 @@ CHECK_TSAN=1 CHECK_LSAN=1 CHECK_DFSAN=1 - CHECK_SCUDO=1 ;; mips64*) CHECK_ASAN=1 @@ -88,7 +85,12 @@ CHECK_UBSAN=1 CHECK_MSAN=1 CHECK_TSAN=1 - CHECK_SCUDO=1 + CMAKE_COMMON_OPTIONS+=" -DLLVM_TARGETS_TO_BUILD=PowerPC" + if [[ "$ARCH" == "ppc64le" ]]; then + CMAKE_COMMON_OPTIONS+=" -DLLVM_LIT_ARGS=-vj256" + else + CMAKE_COMMON_OPTIONS+=" -DLLVM_LIT_ARGS=-vj80" + fi ;; i*86) CHECK_UBSAN=1 @@ -107,50 +109,42 @@ ;; esac -PROJECTS="clang;compiler-rt;libcxx;libcxxabi" +PROJECTS="clang;compiler-rt" if [[ "$CHECK_LLD" != "0" ]]; then - PROJECTS="${PROJECTS};lld" + PROJECTS+=";lld" fi -CMAKE_COMMON_OPTIONS="${CMAKE_COMMON_OPTIONS} -DLLVM_ENABLE_PROJECTS='${PROJECTS}'" +CMAKE_COMMON_OPTIONS+=" -DLLVM_ENABLE_PROJECTS='${PROJECTS}' -DLLVM_ENABLE_RUNTIMES=libcxx;libcxxabi" if [[ "$CHECK_CFI" == "1" ]]; then # We need this after https://reviews.llvm.org/D62050 - CMAKE_COMMON_OPTIONS="$CMAKE_COMMON_OPTIONS -DLLVM_BUILD_LLVM_DYLIB=ON" + CMAKE_COMMON_OPTIONS+=" -DLLVM_BUILD_LLVM_DYLIB=ON" fi -CLOBBER="zlib clang_build" -clobber buildbot_update COMPILER_RT=$LLVM/../compiler-rt LIBCXX=$LLVM/../libcxx -echo @@@BUILD_STEP lint@@@ -CHECK_LINT=${COMPILER_RT}/lib/sanitizer_common/scripts/check_lint.sh -${CHECK_LINT} || echo @@@STEP_WARNINGS@@@ - # Use both gcc and just-built Clang/LLD as a host compiler/linker for sanitizer # tests. Assume that self-hosted build tree should compile with -Werror. echo @@@BUILD_STEP build fresh toolchain@@@ -if [ ! -d clang_build ]; then - mkdir clang_build -fi -(cd clang_build && cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo ${CMAKE_COMMON_OPTIONS} ${USE_CCACHE} $LLVM ) || (rm -rf clang_build ; echo @@@STEP_FAILURE@@@) +mkdir -p clang_build +(cd clang_build && cmake ${CMAKE_COMMON_OPTIONS} $LLVM ) || (rm -rf clang_build ; build_failure) BOOTSTRAP_BUILD_TARGETS="clang" if [[ "$CHECK_LLD" != "0" ]]; then BOOTSTRAP_BUILD_TARGETS="$BOOTSTRAP_BUILD_TARGETS lld" fi -(cd clang_build && make $BOOTSTRAP_BUILD_TARGETS -j$MAKE_JOBS) || (echo @@@STEP_FAILURE@@@ ; exit 1) || echo @@@STEP_FAILURE@@@ +(cd clang_build && ninja $BOOTSTRAP_BUILD_TARGETS) || build_failure # If we're building with libcxx, install the headers to clang_build/include. if [ ! -z ${ENABLE_LIBCXX_FLAG} ]; then -(cd clang_build && make -C ${LIBCXX} installheaders \ - HEADER_DIR=${PWD}/include) || echo @@@STEP_FAILURE@@@ +(cd clang_build && ninja -C ${LIBCXX} installheaders \ + HEADER_DIR=${PWD}/include) || build_failure fi -# Do a sanity check on Linux: build and test sanitizers using gcc as a host +# Check on Linux: build and test sanitizers using gcc as a host # compiler. if [ "$PLATFORM" == "Linux" ]; then check_in_gcc() { @@ -158,7 +152,7 @@ SANITIZER=$2 if [ "$CONDITION" == "1" ]; then echo @@@BUILD_STEP check-$SANITIZER in gcc build@@@ - (cd clang_build && make -j$MAKE_JOBS check-$SANITIZER) || echo @@@STEP_FAILURE@@@ + (cd clang_build && ninja check-$SANITIZER) || build_failure fi } check_in_gcc 1 sanitizer @@ -168,7 +162,6 @@ check_in_gcc $CHECK_DFSAN dfsan check_in_gcc $CHECK_LSAN lsan check_in_gcc $CHECK_MSAN msan - check_in_gcc $CHECK_SCUDO scudo check_in_gcc $CHECK_SCUDO_STANDALONE scudo_standalone LDFLAGS=-no-pie check_in_gcc $CHECK_TSAN tsan check_in_gcc $CHECK_UBSAN ubsan @@ -179,35 +172,33 @@ CLANG_PATH=${ROOT}/clang_build/bin # Build self-hosted tree with fresh Clang and -Werror. CMAKE_CLANG_OPTIONS="${CMAKE_COMMON_OPTIONS} -DLLVM_ENABLE_WERROR=ON -DCMAKE_C_COMPILER=${CLANG_PATH}/clang -DCMAKE_CXX_COMPILER=${CLANG_PATH}/clang++ -DCMAKE_C_FLAGS=-gmlt -DCMAKE_CXX_FLAGS=-gmlt" -BUILD_TYPE=Release echo @@@BUILD_STEP bootstrap clang@@@ -if [ ! -d llvm_build64 ]; then - mkdir llvm_build64 -fi -(cd llvm_build64 && cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE \ - ${CMAKE_CLANG_OPTIONS} -DLLVM_BUILD_EXTERNAL_COMPILER_RT=ON \ - ${ENABLE_LIBCXX_FLAG} $LLVM) || echo @@@STEP_FAILURE@@@ +mkdir -p llvm_build64 +(cd llvm_build64 && cmake ${CMAKE_CLANG_OPTIONS} \ + -DLLVM_BUILD_EXTERNAL_COMPILER_RT=ON \ + ${ENABLE_LIBCXX_FLAG} \ + $LLVM) || build_failure # First, build only Clang. -(cd llvm_build64 && make -j$MAKE_JOBS clang) || echo @@@STEP_FAILURE@@@ +(cd llvm_build64 && ninja clang) || build_failure # If needed, install the headers to clang_build/include. if [ ! -z ${ENABLE_LIBCXX_FLAG} ]; then - (cd llvm_build64 && make -C ${LIBCXX} installheaders \ - HEADER_DIR=${PWD}/include) || echo @@@STEP_FAILURE@@@ + (cd llvm_build64 && ninja -C ${LIBCXX} installheaders \ + HEADER_DIR=${PWD}/include) || build_failure fi # Now build everything else. -(cd llvm_build64 && make -j$MAKE_JOBS) || echo @@@STEP_FAILURE@@@ +(cd llvm_build64 && ninja) || build_failure # Symbolizer dependencies. -(cd llvm_build64 && make -j$MAKE_JOBS llvm-ar llvm-link llvm-tblgen opt) || echo @@@STEP_FAILURE@@@ +(cd llvm_build64 && ninja llvm-ar llvm-link llvm-tblgen opt) || build_failure if [ "$CHECK_CFI" == "1" ]; then # FIXME: Make these true dependencies of check-cfi-and-supported when # compiler-rt is configured as an external project. - (cd llvm_build64 && make -j$MAKE_JOBS LLVMgold opt sanstats) || echo @@@STEP_FAILURE@@@ + (cd llvm_build64 && ninja LLVMgold opt sanstats) || build_failure fi check_64bit() { @@ -215,7 +206,7 @@ SANITIZER=$2 if [ "$CONDITION" == "1" ]; then echo @@@BUILD_STEP 64-bit check-$SANITIZER@@@ - (cd llvm_build64 && make -j$MAKE_JOBS check-$SANITIZER) || echo @@@STEP_FAILURE@@@ + (cd llvm_build64 && ninja check-$SANITIZER) || build_failure fi } @@ -228,7 +219,6 @@ check_64bit $CHECK_DFSAN dfsan check_64bit $CHECK_LSAN lsan check_64bit $CHECK_MSAN msan - # check_64bit $CHECK_SCUDO scudo, No check-scudo target for this config check_64bit $CHECK_TSAN tsan check_64bit $CHECK_UBSAN ubsan check_64bit $CHECK_UBSAN ubsan-minimal @@ -240,40 +230,17 @@ if [ ! -d compiler_rt_build ]; then mkdir compiler_rt_build fi -(cd compiler_rt_build && cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE \ +(cd compiler_rt_build && cmake -GNinja \ -DCMAKE_C_COMPILER=${FRESH_CLANG_PATH}/clang \ -DCMAKE_CXX_COMPILER=${FRESH_CLANG_PATH}/clang++ \ -DCOMPILER_RT_INCLUDE_TESTS=ON \ -DCOMPILER_RT_ENABLE_WERROR=ON \ - -DLLVM_CONFIG_PATH=${FRESH_CLANG_PATH}/llvm-config \ - $COMPILER_RT) || echo @@@STEP_FAILURE@@@ -(cd compiler_rt_build && make -j$MAKE_JOBS) || echo @@@STEP_FAILURE@@@ + -DLLVM_CMAKE_DIR=${ROOT}/llvm_build64 \ + $COMPILER_RT) || build_failure +(cd compiler_rt_build && ninja) || build_failure echo @@@BUILD_STEP test standalone compiler-rt@@@ -(cd compiler_rt_build && make -j$MAKE_JOBS check-all) || echo @@@STEP_FAILURE@@@ - -build_symbolizer() { - echo @@@BUILD_STEP build $1-bit symbolizer for $2@@@ - if [ ! -d symbolizer_build$1 ]; then - mkdir symbolizer_build$1 - fi - (cd symbolizer_build$1 && ZLIB_SRC=$ZLIB FLAGS=-m$1 \ - CLANG=${FRESH_CLANG_PATH}/clang \ - bash -eux $COMPILER_RT/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh \ - $(dirname $(find ../$2/ -name libclang_rt.*.a | head -n1)) || echo @@@STEP_FAILURE@@@) -} - -if [ "$CHECK_SYMBOLIZER" == "1" ]; then - echo @@@BUILD_STEP update zlib@@@ - (cd $ZLIB && git pull --rebase) || \ - git clone https://github.com/madler/zlib.git $ZLIB || echo @@@STEP_FAILURE@@@ - - build_symbolizer 32 compiler_rt_build - build_symbolizer 64 compiler_rt_build - - echo @@@BUILD_STEP test standalone compiler-rt with symbolizer@@@ - (cd compiler_rt_build && make -j$MAKE_JOBS check-all) || echo @@@STEP_FAILURE@@@ -fi +(cd compiler_rt_build && ninja check-all) || build_failure HAVE_NINJA=${HAVE_NINJA:-1} if [ "$PLATFORM" == "Linux" -a $HAVE_NINJA == 1 ]; then @@ -281,18 +248,18 @@ if [ ! -d llvm_build_ninja ]; then mkdir llvm_build_ninja fi - CMAKE_NINJA_OPTIONS="${CMAKE_CLANG_OPTIONS} -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -G Ninja" - (cd llvm_build_ninja && cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE \ - ${CMAKE_NINJA_OPTIONS} $LLVM) || echo @@@STEP_FAILURE@@@ + CMAKE_NINJA_OPTIONS="${CMAKE_CLANG_OPTIONS} -GNinja" + (cd llvm_build_ninja && cmake \ + ${CMAKE_NINJA_OPTIONS} $LLVM) || build_failure ln -sf llvm_build_ninja/compile_commands.json $LLVM - (cd llvm_build_ninja && ninja) || echo @@@STEP_FAILURE@@@ + (cd llvm_build_ninja && ninja) || build_failure check_ninja() { CONDITION=$1 SANITIZER=$2 if [ "$CONDITION" == "1" ]; then echo @@@BUILD_STEP ninja check-$SANITIZER@@@ - (cd llvm_build_ninja && ninja check-$SANITIZER) || echo @@@STEP_FAILURE@@@ + (cd llvm_build_ninja && ninja check-$SANITIZER) || build_failure fi } @@ -303,36 +270,51 @@ check_ninja $CHECK_DFSAN dfsan check_ninja $CHECK_LSAN lsan check_ninja $CHECK_MSAN msan - check_ninja $CHECK_SCUDO scudo check_ninja $CHECK_SCUDO_STANDALONE scudo_standalone check_ninja $CHECK_TSAN tsan check_ninja $CHECK_UBSAN ubsan check_ninja $CHECK_UBSAN ubsan-minimal if [ "$CHECK_SYMBOLIZER" == "1" ]; then + build_symbolizer() { + echo @@@BUILD_STEP build $1-bit symbolizer for $2@@@ + if [ ! -d symbolizer_build$1 ]; then + mkdir symbolizer_build$1 + fi + (cd symbolizer_build$1 && ZLIB_SRC=$ZLIB FLAGS=-m$1 \ + CLANG=${FRESH_CLANG_PATH}/clang \ + bash -eux $COMPILER_RT/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh \ + $(dirname $(find ../$2/ -name libclang_rt.*.a | head -n1)) || build_failure) + } + + echo @@@BUILD_STEP update zlib@@@ + (cd $ZLIB && git pull --rebase) || \ + git clone https://github.com/madler/zlib.git $ZLIB || build_failure + build_symbolizer 32 llvm_build_ninja build_symbolizer 64 llvm_build_ninja check_ninja_with_symbolizer() { CONDITION=$1 SANITIZER=$2 - # Disabled, tests are not working yet. - if [ "$CONDITION" == "-1" ]; then - echo @@@BUILD_STEP ninja check-$SANITIZER with symbolizer@@@ - (cd llvm_build_ninja && ninja check-$SANITIZER) || echo @@@STEP_FAILURE@@@ + if [ "$CONDITION" == "1" ]; then + echo @@@BUILD_STEP ninja check-$SANITIZER with internal symbolizer@@@ + (cd llvm_build_ninja && ninja check-$SANITIZER) || build_failure fi } - check_ninja_with_symbolizer 1 sanitizer - check_ninja_with_symbolizer $CHECK_ASAN asan - check_ninja_with_symbolizer $CHECK_HWASAN hwasan - check_ninja_with_symbolizer $CHECK_CFI cfi-and-supported + # TODO: Replace LIT_FILTER_OUT with lit features. + LIT_FILTER_OUT=":: (max_allocation_size.cpp|Linux/soft_rss_limit_mb_test.cpp)$" check_ninja_with_symbolizer 1 sanitizer + LIT_FILTER_OUT=":: TestCases/Linux/check_memcpy.c$" check_ninja_with_symbolizer $CHECK_ASAN asan + # check_ninja_with_symbolizer $CHECK_HWASAN hwasan + # check_ninja_with_symbolizer $CHECK_CFI cfi-and-supported check_ninja_with_symbolizer $CHECK_DFSAN dfsan - check_ninja_with_symbolizer $CHECK_LSAN lsan - check_ninja_with_symbolizer $CHECK_MSAN msan - check_ninja_with_symbolizer $CHECK_SCUDO scudo + LIT_FILTER_OUT=":: TestCases/(realloc_too_big.c|recoverable_leak_check.cpp|suppressions_file.cpp)$" check_ninja_with_symbolizer $CHECK_LSAN lsan + LIT_FILTER_OUT=":: Linux/check_memcpy.c$" check_ninja_with_symbolizer $CHECK_MSAN msan check_ninja_with_symbolizer $CHECK_SCUDO_STANDALONE scudo_standalone - check_ninja_with_symbolizer $CHECK_TSAN tsan - check_ninja_with_symbolizer $CHECK_UBSAN ubsan + LIT_FILTER_OUT=":: Linux/check_memcpy.c$" check_ninja_with_symbolizer $CHECK_TSAN tsan + LIT_FILTER_OUT=":: TestCases/TypeCheck/(vptr-virtual-base.cpp|vptr.cpp)$" check_ninja_with_symbolizer $CHECK_UBSAN ubsan fi fi + +cleanup diff --git a/zorg/buildbot/builders/sanitizers/buildbot_fast.sh b/zorg/buildbot/builders/sanitizers/buildbot_fast.sh --- a/zorg/buildbot/builders/sanitizers/buildbot_fast.sh +++ b/zorg/buildbot/builders/sanitizers/buildbot_fast.sh @@ -7,39 +7,33 @@ HERE="$(cd $(dirname $0) && pwd)" . ${HERE}/buildbot_functions.sh -ROOT=`pwd` -PLATFORM=`uname` -export PATH="/usr/local/bin:$PATH" - -LLVM=$ROOT/llvm -CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON" - clobber # Stage 1 build_stage1_clang_at_revison -if ccache -s ; then - CMAKE_COMMON_OPTIONS="${CMAKE_COMMON_OPTIONS} -DLLVM_CCACHE_BUILD=ON" -fi - -buildbot_update +CMAKE_COMMON_OPTIONS+=" -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF" -# Stage 2 / Memory Sanitizer +# Some of them are slow. +STAGE2_SKIP_TEST_CXX=1 -build_stage2_msan +buildbot_update -check_stage2_msan +# Stage 2 / Address and Undefined Sanitizer -# Stage 2 / AddressSanitizer +build_stage2_asan_ubsan -build_stage2_asan +check_stage2_asan_ubsan -check_stage2_asan +# Stage 2 / Memory Sanitizer -# Stage 2 / UndefinedBehaviorSanitizer +CMAKE_COMMON_OPTIONS+=" -DCMAKE_C_FLAGS=-fno-inline" +CMAKE_COMMON_OPTIONS+=" -DCMAKE_CXX_FLAGS=-fno-inline" +CMAKE_COMMON_OPTIONS+=" -DCMAKE_C_FLAGS_RELEASE=-Oz" +CMAKE_COMMON_OPTIONS+=" -DCMAKE_CXX_FLAGS_RELEASE=-Oz" +build_stage2_msan -build_stage2_ubsan +check_stage2_msan -check_stage2_ubsan +cleanup diff --git a/zorg/buildbot/builders/sanitizers/buildbot_functions.sh b/zorg/buildbot/builders/sanitizers/buildbot_functions.sh --- a/zorg/buildbot/builders/sanitizers/buildbot_functions.sh +++ b/zorg/buildbot/builders/sanitizers/buildbot_functions.sh @@ -1,49 +1,101 @@ #!/usr/bin/env bash +ROOT=`pwd` +LLVM=$ROOT/llvm + +BUILDBOT_CLOBBER="${BUILDBOT_CLOBBER:-}" +BUILDBOT_REVISION="${BUILDBOT_REVISION:-origin/main}" + +HOST_CLANG_REVISION=llvmorg-$(curl https://api.github.com/repos/llvm/llvm-project/releases/latest -s | jq .name -r | cut -f2 -d' ') +CMAKE_COMMON_OPTIONS+=" -DLLVM_APPEND_VC_REV=OFF -GNinja -DCMAKE_BUILD_TYPE=Release" + +export LC_ALL=C + +if ccache -s ; then + CMAKE_COMMON_OPTIONS+=" -DLLVM_CCACHE_BUILD=ON" +fi + +if lld --version ; then + CMAKE_COMMON_OPTIONS+=" -DLLVM_USE_LINKER=lld" +fi + +function include_config() { + local P=. + while true ; do + local F=${P}/sanitizer_buildbot_config + if [[ -f ${F} ]] ; then + . ${F} + break + fi + [[ "${P}" -ef '/' ]] && break + P="${P}/.." + done +} + +include_config + echo @@@BUILD_STEP Info@@@ ( set +e - env + env | sort echo uptime echo - ulimit -n 100000 + ulimit -n 1000000 ulimit -a echo df -h echo - ccache -s - echo - echo "How to reproduce locally: https://github.com/google/sanitizers/wiki/SanitizerBotReproduceBuild" - echo + ccache -ps exit 0 ) + echo @@@BUILD_STEP Prepare@@@ -BUILDBOT_CLOBBER="${BUILDBOT_CLOBBER:-}" -BUILDBOT_REVISION="${BUILDBOT_REVISION:-origin/master}" +export LIT_OPTS="--time-tests" function rm_dirs { while ! rm -rf $@ ; do sleep 1; done } -function stage1_clobber { - rm_dirs llvm_build2_* llvm_build_* libcxx_build_* ${STAGE1_CLOBBER:-} +function cleanup() { + [[ -v BUILDBOT_BUILDERNAME ]] || return 0 + echo @@@BUILD_STEP cleanup@@@ + rm_dirs llvm_build2_* llvm_build_* libcxx_build_* compiler_rt_build* symbolizer_build* $@ + if ccache -s >/dev/null ; then + rm_dirs llvm_build64 clang_build + fi + ls } function clobber { - if [ "$BUILDBOT_CLOBBER" != "" ]; then + if [[ "$BUILDBOT_CLOBBER" != "" ]]; then echo @@@BUILD_STEP clobber@@@ - rm_dirs svn_checkout llvm llvm-project llvm_build0 ${CLOBBER:-} - stage1_clobber - ! test "$(ls -A .)" || echo @@@STEP_EXCEPTION@@@ + if [[ ! -v BUILDBOT_BUILDERNAME ]]; then + echo "Clobbering is supported only on buildbot only!" + exit 1 + fi + rm_dirs * + else + BUILDBOT_BUILDERNAME=1 cleanup $@ fi } BUILDBOT_MONO_REPO_PATH=${BUILDBOT_MONO_REPO_PATH:-} function buildbot_update { - echo @@@BUILD_STEP update $BUILDBOT_REVISION@@@ + if [[ "${BUILDBOT_BISECT_MODE:-}" == "1" ]]; then + echo "@@@BUILD_STEP bisect status@@@" + ( + cd llvm-project + git bisect visualize --oneline + git bisect log + git status + ) + LLVM=$ROOT/llvm-project/llvm + return 0 + fi + echo "@@@BUILD_STEP update $BUILDBOT_REVISION@@@" if [[ -d "$BUILDBOT_MONO_REPO_PATH" ]]; then LLVM=$BUILDBOT_MONO_REPO_PATH/llvm else @@ -54,27 +106,16 @@ cd llvm-project git init git remote add origin https://github.com/llvm/llvm-project.git + git config --local advice.detachedHead false ) cd llvm-project - git fetch --depth $DEPTH origin master + git fetch origin git clean -fd - local REV=${BUILDBOT_REVISION} - if [[ "$REV" != "origin/master" ]] ; then - # "git fetch --depth 1 origin $REV" does not work with 2.11 on bots - while true ; do - git checkout $REV && break - git rev-list --pretty --max-count=1 origin/master - git rev-list --pretty --max-parents=0 origin/master - echo "DEPTH=$DEPTH is too small" - [[ "$DEPTH" -le "1000000" ]] || exit 1 - DEPTH=$(( $DEPTH * 10 )) - git fetch --depth $DEPTH origin - done - fi - git checkout $REV + git reset --hard + git checkout -f "${BUILDBOT_REVISION}" git status git rev-list --pretty --max-count=1 HEAD - ) || { echo @@@STEP_EXCEPTION@@@ ; exit 1 ; } + ) || { build_exception ; exit 1 ; } LLVM=$ROOT/llvm-project/llvm fi } @@ -83,198 +124,398 @@ STAGE1_DIR=llvm_build0 stage1_clang_path=$ROOT/${STAGE1_DIR}/bin llvm_symbolizer_path=${stage1_clang_path}/llvm-symbolizer + export LLVM_SYMBOLIZER_PATH="${llvm_symbolizer_path}" STAGE1_AS_COMPILER="-DCMAKE_C_COMPILER=${stage1_clang_path}/clang -DCMAKE_CXX_COMPILER=${stage1_clang_path}/clang++" } function build_stage1_clang_impl { - rm -rf ${STAGE1_DIR} + [[ -f "${STAGE1_DIR}/delete_next_time" ]] && rm -rf "${STAGE1_DIR}" mkdir -p ${STAGE1_DIR} - local cmake_stage1_options="${CMAKE_COMMON_OPTIONS}" - cmake_stage1_options="${cmake_stage1_options} -DLLVM_ENABLE_PROJECTS='clang;compiler-rt;lld'" - if ccache -s ; then - cmake_stage1_options="${cmake_stage1_options} -DLLVM_CCACHE_BUILD=ON" + local cmake_stage1_options="${CMAKE_COMMON_OPTIONS} -DLLVM_ENABLE_PROJECTS='clang;compiler-rt;lld'" + if clang -v ; then + cmake_stage1_options+=" -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++" fi - (cd ${STAGE1_DIR} && cmake ${cmake_stage1_options} $LLVM && ninja) + (cd ${STAGE1_DIR} && cmake ${cmake_stage1_options} $LLVM && ninja) || { + touch "${STAGE1_DIR}/delete_next_time" + return 1 + } + md5sum ${STAGE1_DIR}/bin/clang || true } function build_stage1_clang { echo @@@BUILD_STEP stage1 build all@@@ - export STAGE1_DIR=llvm_build0 common_stage1_variables build_stage1_clang_impl - - echo @@@BUILD_STEP Clobber stage1 users - stage1_clobber } -function build_stage1_clang_at_revison { +function download_clang_from_chromium { common_stage1_variables - curl -s https://raw.githubusercontent.com/chromium/chromium/master/tools/clang/scripts/update.py \ - | python - --output-dir=${STAGE1_DIR} + curl -s https://raw.githubusercontent.com/chromium/chromium/main/tools/clang/scripts/update.py \ + | python3 - --output-dir=${STAGE1_DIR} echo @@@BUILD_STEP using pre-built stage1 clang at $(cat ${STAGE1_DIR}/cr_build_revision)@@@ } +function build_clang_at_release_tag { + common_stage1_variables + + if [ -r ${STAGE1_DIR}/host_clang_revision ] && \ + [ "$(cat ${STAGE1_DIR}/host_clang_revision)" == $HOST_CLANG_REVISION ] + then + echo "@@@BUILD_STEP using pre-built stage1 clang at r${HOST_CLANG_REVISION}@@@" + else + BUILDBOT_MONO_REPO_PATH= BUILDBOT_REVISION="${HOST_CLANG_REVISION}" buildbot_update + + rm -rf ${STAGE1_DIR} + echo @@@BUILD_STEP build stage1 clang at $HOST_CLANG_REVISION@@@ + build_stage1_clang_impl && \ + ( echo $HOST_CLANG_REVISION > ${STAGE1_DIR}/host_clang_revision ) + fi +} + +function build_stage1_clang_at_revison { + build_clang_at_release_tag +} + function common_stage2_variables { - cmake_stage2_common_options="\ - ${CMAKE_COMMON_OPTIONS} ${STAGE1_AS_COMPILER} -DLLVM_USE_LINKER=lld" + cmake_stage2_common_options="${CMAKE_COMMON_OPTIONS} ${STAGE1_AS_COMPILER}" } function build_stage2 { local sanitizer_name=$1 - local step_result=$2 + echo @@@BUILD_STEP stage2/$sanitizer_name build libcxx@@@ + local libcxx_build_dir=libcxx_build_${sanitizer_name} local build_dir=llvm_build_${sanitizer_name} export STAGE2_DIR=${build_dir} + local cmake_libcxx_cflags= common_stage2_variables + local fno_sanitize_flag= + if [ "$sanitizer_name" == "msan" ]; then export MSAN_SYMBOLIZER_PATH="${llvm_symbolizer_path}" - local llvm_use_sanitizer="Memory" - local fsanitize_flag="-fsanitize=memory" - BUILDBOT_MSAN_ORIGINS=${BUILDBOT_MSAN_ORIGINS:-} - if [ "$BUILDBOT_MSAN_ORIGINS" != "" ]; then - llvm_use_sanitizer="MemoryWithOrigins" - fi - local build_type="Release" + llvm_use_sanitizer="Memory" + fsanitize_flag="-fsanitize=memory" + elif [ "$sanitizer_name" == "msan_track_origins" ]; then + export MSAN_SYMBOLIZER_PATH="${llvm_symbolizer_path}" + llvm_use_sanitizer="MemoryWithOrigins" + fsanitize_flag="-fsanitize=memory -fsanitize-memory-track-origins" elif [ "$sanitizer_name" == "asan" ]; then export ASAN_SYMBOLIZER_PATH="${llvm_symbolizer_path}" - export ASAN_OPTIONS="check_initialization_order=true:detect_stack_use_after_return=1:detect_leaks=1" - local llvm_use_sanitizer="Address" - local fsanitize_flag="-fsanitize=address" - local build_type="Release" + export ASAN_OPTIONS="check_initialization_order=true" + llvm_use_sanitizer="Address" + fsanitize_flag="-fsanitize=address" + # FIXME: False ODR violations in libcxx tests. + # https://github.com/google/sanitizers/issues/1017 + cmake_libcxx_cflags="-mllvm -asan-use-private-alias=1" + elif [ "$sanitizer_name" == "hwasan" ]; then + export HWASAN_SYMBOLIZER_PATH="${llvm_symbolizer_path}" + export HWASAN_OPTIONS="abort_on_error=1" + llvm_use_sanitizer="HWAddress" + fsanitize_flag="-fsanitize=hwaddress -mllvm -hwasan-use-after-scope=1" + # FIXME: Support globals with DSO https://github.com/llvm/llvm-project/issues/57206 + cmake_stage2_common_options+=" -DLLVM_ENABLE_PLUGINS=OFF" elif [ "$sanitizer_name" == "ubsan" ]; then export UBSAN_OPTIONS="external_symbolizer_path=${llvm_symbolizer_path}:print_stacktrace=1" - local llvm_use_sanitizer="Undefined" - local fsanitize_flag="-fsanitize=undefined" - local build_type="Release" + llvm_use_sanitizer="Undefined" + fsanitize_flag="-fsanitize=undefined" + # FIXME: After switching to LLVM_ENABLE_RUNTIMES, vptr has infitine + # recursion. + fno_sanitize_flag+=" -fno-sanitize=vptr" + elif [ "$sanitizer_name" == "asan_ubsan" ]; then + export ASAN_SYMBOLIZER_PATH="${llvm_symbolizer_path}" + export ASAN_OPTIONS="check_initialization_order=true" + llvm_use_sanitizer="Address;Undefined" + fsanitize_flag="-fsanitize=address,undefined" + # FIXME: After switching to LLVM_ENABLE_RUNTIMES, vptr has infitine + # recursion. + fno_sanitize_flag+=" -fno-sanitize=vptr" else echo "Unknown sanitizer!" exit 1 fi - # Don't use libc++/libc++abi in UBSan builds (due to known bugs). - echo @@@BUILD_STEP stage2/$sanitizer_name build libcxx@@@ - rm -rf ${libcxx_build_dir} mkdir -p ${libcxx_build_dir} (cd ${libcxx_build_dir} && \ cmake \ ${cmake_stage2_common_options} \ - -DLLVM_ENABLE_PROJECTS='libcxx;libcxxabi' \ - -DCMAKE_BUILD_TYPE=${build_type} \ + -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' \ -DLLVM_USE_SANITIZER=${llvm_use_sanitizer} \ - $LLVM && \ - ninja cxx cxxabi) || echo $step_result - local sanitizer_ldflags="-lc++abi -Wl,--rpath=${ROOT}/${libcxx_build_dir}/lib -L${ROOT}/${libcxx_build_dir}/lib" - local sanitizer_cflags="-nostdinc++ -isystem ${ROOT}/${libcxx_build_dir}/include -isystem ${ROOT}/${libcxx_build_dir}/include/c++/v1" + -DCMAKE_C_FLAGS="${fsanitize_flag} ${cmake_libcxx_cflags} ${fno_sanitize_flag}" \ + -DCMAKE_CXX_FLAGS="${fsanitize_flag} ${cmake_libcxx_cflags} ${fno_sanitize_flag}" \ + $LLVM/../runtimes && \ + ninja cxx cxxabi) || build_failure + + local libcxx_runtime_path=$(dirname $(find ${ROOT}/${libcxx_build_dir} -name libc++.so)) + local sanitizer_ldflags="-Wl,--rpath=${libcxx_runtime_path} -L${libcxx_runtime_path}" + local sanitizer_cflags="-nostdinc++ -isystem ${ROOT}/${libcxx_build_dir}/include -isystem ${ROOT}/${libcxx_build_dir}/include/c++/v1 $fsanitize_flag" echo @@@BUILD_STEP stage2/$sanitizer_name build@@@ # See http://llvm.org/bugs/show_bug.cgi?id=19071, http://www.cmake.org/Bug/view.php?id=15264 - local cmake_bug_workaround_cflags="$sanitizer_ldflags $fsanitize_flag -w" - sanitizer_cflags="$sanitizer_cflags $cmake_bug_workaround_cflags" + sanitizer_cflags+=" $sanitizer_ldflags -w" - rm -rf ${build_dir} mkdir -p ${build_dir} - local cmake_stage2_clang_options="-DLLVM_ENABLE_PROJECTS='clang;lld;clang-tools-extra'" + local cmake_stage2_clang_options="-DLLVM_ENABLE_PROJECTS='clang;lld;clang-tools-extra;mlir'" + if [[ "$(arch)" == "aarch64" ]] ; then + # FIXME: clangd tests fail. + cmake_stage2_clang_options="-DLLVM_ENABLE_PROJECTS='clang;lld;mlir'" + fi (cd ${build_dir} && \ cmake \ ${cmake_stage2_common_options} \ ${cmake_stage2_clang_options} \ - -DCMAKE_BUILD_TYPE=${build_type} \ -DLLVM_USE_SANITIZER=${llvm_use_sanitizer} \ -DLLVM_ENABLE_LIBCXX=ON \ -DCMAKE_C_FLAGS="${sanitizer_cflags}" \ -DCMAKE_CXX_FLAGS="${sanitizer_cflags}" \ -DCMAKE_EXE_LINKER_FLAGS="${sanitizer_ldflags}" \ $LLVM && \ - ninja) || echo $step_result + time ninja) || build_failure + md5sum ${build_dir}/bin/clang || true } function build_stage2_msan { - build_stage2 msan @@@STEP_FAILURE@@@ + build_stage2 msan +} + +function build_stage2_msan_track_origins { + build_stage2 msan_track_origins } function build_stage2_asan { - build_stage2 asan @@@STEP_FAILURE@@@ + build_stage2 asan +} + +function build_stage2_hwasan { + build_stage2 hwasan } function build_stage2_ubsan { - build_stage2 ubsan @@@STEP_FAILURE@@@ + build_stage2 ubsan +} + +function build_stage2_asan_ubsan { + build_stage2 asan_ubsan +} + +function check_stage1 { + local sanitizer_name=$1 + + echo @@@BUILD_STEP stage1/$sanitizer_name check-sanitizer@@@ + ninja -C ${STAGE1_DIR} check-sanitizer || build_failure + + # Uses by asan and hwasan. + echo @@@BUILD_STEP stage1/$sanitizer_name check-lsan@@@ + ninja -C ${STAGE1_DIR} check-lsan || build_failure + + echo @@@BUILD_STEP stage1/$sanitizer_name check-${sanitizer_name}@@@ + ninja -C ${STAGE1_DIR} check-${sanitizer_name} || build_failure +} + +function check_stage1_msan { + check_stage1 msan +} + +function check_stage1_asan { + check_stage1 asan +} + +function check_stage1_hwasan { + check_stage1 hwasan +} + +function check_stage1_ubsan { + check_stage1 ubsan +} + +function check_stage1_asan_ubsan { + check_stage1 asan_ubsan } function check_stage2 { local sanitizer_name=$1 - local step_result=$2 - local build_dir=${STAGE2_DIR} - + + if [[ "${STAGE2_SKIP_TEST_CXX:-}" != "1" ]] ; then + ( + # Very slow, run in background. + LIT_OPTS+=" --timeout=1500" + ( + # Very slow. + export LIT_FILTER_OUT="modules_include.sh.cpp" + LIT_FILTER_OUT+="|std/algorithms/alg.modifying.operations/alg.transform/ranges.transform.pass.cpp" + LIT_FILTER_OUT+="|std/containers/sequences/deque/deque.modifiers/insert_iter_iter.pass.cpp" + LIT_FILTER_OUT+="|std/numerics/rand/rand.dist/rand.dist.bern/rand.dist.bern.negbin/eval.pass.cpp" + LIT_FILTER_OUT+="|std/numerics/rand/rand.dist/rand.dist.samp/rand.dist.samp.discrete/eval.pass.cpp" + LIT_FILTER_OUT+="|std/utilities/charconv/charconv.msvc/test.pass.cpp" + LIT_FILTER_OUT+="|std/utilities/format/format.functions/format_to_n.locale.pass.cpp" + LIT_FILTER_OUT+="|std/utilities/format/format.functions/format_to_n.pass.cpp" + LIT_FILTER_OUT+="|std/utilities/format/format.functions/format_to.locale.pass.cpp" + LIT_FILTER_OUT+="|std/utilities/format/format.functions/format_to.pass.cpp" + LIT_FILTER_OUT+="|std/utilities/format/format.functions/format.locale.pass.cpp" + LIT_FILTER_OUT+="|std/utilities/format/format.functions/format.pass.cpp" + LIT_FILTER_OUT+="|std/utilities/format/format.functions/formatted_size.locale.pass.cpp" + LIT_FILTER_OUT+="|std/utilities/format/format.functions/formatted_size.pass.cpp" + LIT_FILTER_OUT+="|std/utilities/format/format.functions/vformat" + LIT_FILTER_OUT+="|std/utilities/variant/variant.visit/visit_return_type.pass.cpp" + LIT_FILTER_OUT+="|std/utilities/variant/variant.visit/visit.pass.cpp" + + if [[ "$(arch)" == "aarch64" && "$sanitizer_name" == "asan" ]] ; then + # TODO: Investigate one leak and two slowest tests. + LIT_FILTER_OUT+="|test_vector2.pass.cpp|catch_multi_level_pointer.pass.cpp" + LIT_FILTER_OUT+="|guard_threaded_test.pass.cpp" + fi + if [[ "$(arch)" == "aarch64" && "$sanitizer_name" == "msan" ]] ; then + # TODO: Investigate one slow tests. + LIT_FILTER_OUT+="|catch_multi_level_pointer.pass.cpp" + LIT_FILTER_OUT+="|guard_threaded_test.pass.cpp" + LIT_FILTER_OUT+="|test_demangle.pass.cpp" + fi + if [[ "$(arch)" == "aarch64" && "$sanitizer_name" == "hwasan" ]] ; then + # TODO: Investigate one slow tests. + LIT_FILTER_OUT+="|catch_multi_level_pointer.pass.cpp" + LIT_FILTER_OUT+="|guard_threaded_test.pass.cpp" + LIT_FILTER_OUT+="|test_demangle.pass.cpp" + LIT_FILTER_OUT+="|test_vector2.pass.cpp" + LIT_FILTER_OUT+="|forced_unwind2.pass.cpp" + fi + ninja -C libcxx_build_${sanitizer_name} check-cxx check-cxxabi + ) || build_failure + ) &>check_cxx.log & + fi + echo @@@BUILD_STEP stage2/$sanitizer_name check@@@ - ninja -C ${build_dir} check-all || echo $step_result + ninja -C ${STAGE2_DIR} check-all || build_failure + + if [[ "${STAGE2_SKIP_TEST_CXX:-}" != "1" ]] ; then + echo @@@BUILD_STEP stage2/$sanitizer_name check-cxx@@@ + wait + sleep 5 + cat check_cxx.log + fi } function check_stage2_msan { - check_stage2 msan @@@STEP_FAILURE@@@ + check_stage2 msan +} + +function check_stage2_msan_track_origins { + check_stage2 msan_track_origins } function check_stage2_asan { - check_stage2 asan @@@STEP_FAILURE@@@ + check_stage2 asan +} + +function check_stage2_hwasan { + check_stage2 hwasan } function check_stage2_ubsan { - check_stage2 ubsan @@@STEP_FAILURE@@@ + check_stage2 ubsan +} + +function check_stage2_asan_ubsan { + check_stage2 asan_ubsan } function build_stage3 { local sanitizer_name=$1 - local step_result=$2 + echo @@@BUILD_STEP build stage3/$sanitizer_name build@@@ + local build_dir=llvm_build2_${sanitizer_name} local clang_path=$ROOT/${STAGE2_DIR}/bin - echo @@@BUILD_STEP build stage3/$sanitizer_name build@@@ - rm -rf ${build_dir} + local sanitizer_cflags= mkdir -p ${build_dir} + local stage3_projects='clang;lld;clang-tools-extra' + if [[ "$(arch)" == "aarch64" ]] ; then + # FIXME: clangd tests fail. + stage3_projects='clang;lld' + fi (cd ${build_dir} && \ cmake \ ${CMAKE_COMMON_OPTIONS} \ - -DLLVM_ENABLE_PROJECTS='clang;lld,clang-tools-extra' \ + -DLLVM_ENABLE_PROJECTS="${stage3_projects}" \ -DCMAKE_C_COMPILER=${clang_path}/clang \ -DCMAKE_CXX_COMPILER=${clang_path}/clang++ \ - -DLLVM_USE_LINKER=lld \ + -DCMAKE_CXX_FLAGS="${sanitizer_cflags}" \ + -DLLVM_CCACHE_BUILD=OFF \ $LLVM && \ - ninja clang) || echo $step_result + time ninja) || build_failure + md5sum ${build_dir}/bin/clang* || true } function build_stage3_msan { - build_stage3 msan @@@STEP_FAILURE@@@ + build_stage3 msan +} + +function build_stage3_msan_track_origins { + build_stage3 msan_track_origins } function build_stage3_asan { - build_stage3 asan @@@STEP_FAILURE@@@ + build_stage3 asan +} + +function build_stage3_hwasan { + build_stage3 hwasan } function build_stage3_ubsan { - build_stage3 ubsan @@@STEP_FAILURE@@@ + build_stage3 ubsan } function check_stage3 { local sanitizer_name=$1 - local step_result=$2 + echo @@@BUILD_STEP stage3/$sanitizer_name check@@@ + local build_dir=llvm_build2_${sanitizer_name} - echo @@@BUILD_STEP stage3/$sanitizer_name check@@@ - (cd ${build_dir} && ninja check-all) || echo $step_result + (cd ${build_dir} && env && ninja check-all) || build_failure } function check_stage3_msan { - check_stage3 msan @@@STEP_FAILURE@@@ + check_stage3 msan +} + +function check_stage3_msan_track_origins { + check_stage3 msan_track_origins } function check_stage3_asan { - check_stage3 asan @@@STEP_FAILURE@@@ + check_stage3 asan +} + +function check_stage3_hwasan { + check_stage3 hwasan } function check_stage3_ubsan { - check_stage3 ubsan @@@STEP_FAILURE@@@ + check_stage3 ubsan +} + +function build_failure() { + # In bisect mode exit early. + echo + echo "How to reproduce locally: https://github.com/google/sanitizers/wiki/SanitizerBotReproduceBuild" + echo + + sleep 5 + echo "@@@STEP_FAILURE@@@" + if [[ "${BUILDBOT_BISECT_MODE:-}" == "1" ]] ; then + exit 1 + fi +} + +function build_exception() { + echo + echo "How to reproduce locally: https://github.com/google/sanitizers/wiki/SanitizerBotReproduceBuild" + echo + + sleep 5 + echo "@@@STEP_EXCEPTION@@@" } diff --git a/zorg/buildbot/builders/sanitizers/buildbot_fuzzer.sh b/zorg/buildbot/builders/sanitizers/buildbot_fuzzer.sh --- a/zorg/buildbot/builders/sanitizers/buildbot_fuzzer.sh +++ b/zorg/buildbot/builders/sanitizers/buildbot_fuzzer.sh @@ -7,20 +7,13 @@ HERE="$(cd $(dirname $0) && pwd)" . ${HERE}/buildbot_functions.sh -ROOT=`pwd` -PLATFORM=`uname` -export PATH="/usr/local/bin:$PATH" - MAKE_JOBS=${MAX_MAKE_JOBS:-$(nproc)} -LLVM=$ROOT/llvm LIBFUZZER=$LLVM/lib/Fuzzer # No assertions. Need to clean up the existing assertion failures first. # Also, the Fuzzer does not provide reproducers on assertion failures yet. -CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=OFF -DLLVM_PARALLEL_LINK_JOBS=8 -DLIBFUZZER_ENABLE_TESTS=ON" +CMAKE_COMMON_OPTIONS+=" -DLLVM_ENABLE_ASSERTIONS=OFF" -CLOBBER=fuzzer-test-suite -STAGE1_CLOBBER="RUNDIR-* $LIBFUZZER" -clobber +clobber RUNDIR-* # Make sure asan intercepts SIGABRT so that the fuzzer can print the test cases # for assertion failures. @@ -37,7 +30,7 @@ echo @@@BUILD_STEP check-fuzzer@@@ -(cd ${STAGE1_DIR} && ninja check-fuzzer) || echo @@@STEP_FAILURE@@@ +(cd ${STAGE1_DIR} && ninja check-fuzzer) || build_failure echo @@@BUILD_STEP get fuzzer-test-suite @@@ [ ! -e fuzzer-test-suite ] && git clone https://github.com/google/fuzzer-test-suite.git @@ -53,14 +46,18 @@ ulimit -t 3600 +# FIXME: Some aarch64 tests are extremly slow, maybe related to + export JOBS=${MAKE_JOBS} -RunFuzzerTest libxml2-v2.9.2 || echo @@@STEP_FAILURE@@@ -RunFuzzerTest libpng-1.2.56 || echo @@@STEP_FAILURE@@@ -RunFuzzerTest libssh-2017-1272 || echo @@@STEP_FAILURE@@@ -RunFuzzerTest re2-2014-12-09 || echo @@@STEP_FAILURE@@@ -RunFuzzerTest c-ares-CVE-2016-5180 || echo @@@STEP_FAILURE@@@ -RunFuzzerTest openssl-1.0.1f || echo @@@STEP_FAILURE@@@ -RunFuzzerTest openssl-1.0.2d || echo @@@STEP_FAILURE@@@ -RunFuzzerTest proj4-2017-08-14 || echo @@@STEP_FAILURE@@@ -#RunFuzzerTest woff2-2016-05-06 || echo @@@STEP_WARNINGS@@@ # Often can't find the bug in the given time. +RunFuzzerTest libxml2-v2.9.2 || build_failure +[[ "$(arch)" == "aarch64" ]] || RunFuzzerTest libpng-1.2.56 || build_failure +RunFuzzerTest libssh-2017-1272 || build_failure +RunFuzzerTest re2-2014-12-09 || build_failure +RunFuzzerTest c-ares-CVE-2016-5180 || build_failure +RunFuzzerTest openssl-1.0.1f || build_failure +[[ "$(arch)" == "aarch64" ]] || RunFuzzerTest openssl-1.0.2d || build_failure +[[ "$(arch)" == "aarch64" ]] || RunFuzzerTest proj4-2017-08-14 || build_failure +#RunFuzzerTest woff2-2016-05-06 || build_failure # Often can't find the bug in the given time. + +cleanup $STAGE1_DIR RUNDIR-* diff --git a/zorg/buildbot/builders/sanitizers/buildbot_qemu.sh b/zorg/buildbot/builders/sanitizers/buildbot_qemu.sh new file mode 100755 --- /dev/null +++ b/zorg/buildbot/builders/sanitizers/buildbot_qemu.sh @@ -0,0 +1,393 @@ +#!/usr/bin/env bash + +set -x +set -e +set -u + +HERE="$(cd $(dirname $0) && pwd)" +. ${HERE}/buildbot_functions.sh + +clobber + +build_stage1_clang_at_revison +CMAKE_COMMON_OPTIONS+=" -DLLVM_ENABLE_ASSERTIONS=ON" + +buildbot_update + +readonly STAGE2_DIR=llvm_build2_host +( + echo @@@BUILD_STEP build host clang@@@ + COMPILER_BIN_DIR="$(readlink -f ${STAGE1_DIR})/bin" + + rm -rf ${STAGE2_DIR} + mkdir -p ${STAGE2_DIR} + cd ${STAGE2_DIR} + + cmake \ + ${CMAKE_COMMON_OPTIONS} \ + -DLLVM_ENABLE_PROJECTS="clang;compiler-rt;lld" \ + -DCMAKE_C_COMPILER=${COMPILER_BIN_DIR}/clang \ + -DCMAKE_CXX_COMPILER=${COMPILER_BIN_DIR}/clang++ \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + $LLVM && ninja +) || build_failure + +set +x # Avoid echoing STEP_FAILURE because of the command trace. +MISSING_QEMU_IMAGE_MESSAGE=$(cat << 'EOF' +===================================================================== +Looks like you're missing the QEMU system images for x86_64 LAM +HWASan testing. These system images aren't automatically generated +as part of the buildbot script because they require root (sorry!). + +If you have the system images already built, you can run +buildbot_qemu.sh with QEMU_IMAGE_DIR=path/to/qemu/images. Otherwise, +you can build the images (only necessary once-per-clobber) by: + 1. WARNING: Read the script at https://github.com/google/sanitizers/blob/master/hwaddress-sanitizer/create_qemu_image.sh + You're running this as ROOT, so make sure you're comfortable with + that. + 2. In your terminal, run: + $ sudo su -c "bash <(wget -qO- https://raw.githubusercontent.com/google/sanitizers/master/hwaddress-sanitizer/create_qemu_image.sh)" root && \ + sudo chown $(whoami) debian.* + +You can also choose to skip the x86_64 HWASan LAM testing by supplying +SKIP_HWASAN_LAM=true in your invocation of this script. +===================================================================== +EOF +) +set -x + +SKIP_HWASAN_LAM=${SKIP_HWASAN_LAM:-} + +function setup_lam_qemu_image { + # Full system emulation is required for x86_64 testing with LAM, as some + # sanitizers aren't friendly with usermode emulation under x86_64 LAM. + echo @@@BUILD_STEP Check x86_64 LAM Prerequisites@@@ + + # Allow specifying the QEMU image dir, otherwise assume in the local dir. + QEMU_IMAGE_DIR=${QEMU_IMAGE_DIR:=${ROOT}} + + # Ensure the buildbot start script created a QEMU image for us. + { + QEMU_IMAGE="${QEMU_IMAGE_DIR}/debian.img" + QEMU_SSH_KEY="${QEMU_IMAGE_DIR}/debian.id_rsa" + ls ${QEMU_IMAGE} + ls ${QEMU_SSH_KEY} + } || { + # Make the "missing file" error clearer by not effectively echoing twice. + set +x + echo "$MISSING_QEMU_IMAGE_MESSAGE" + build_exception + exit 2 + } +} + +[[ -z "$SKIP_HWASAN_LAM" ]] && setup_lam_qemu_image || SKIP_HWASAN_LAM=1 + +function git_clone_at_revision { + local src_dir_name="${1}" + local git_url="${2}" + local revision="${3}" + local build_dir="${4}" + ( + cd ${ROOT} + [[ -d ${src_dir_name} ]] || git clone ${git_url} ${src_dir_name} || exit 1 + cd ${ROOT}/${src_dir_name} + git remote set-url origin ${git_url} + git fetch origin + + # Short circuit if we already have a build at the correct revision. + [[ "$(git rev-parse HEAD)" == "$(git rev-parse ${revision})" ]] && exit 0 + + # Delete the build dir to ensure a rebuild happens at the new revision. + rm -rf ${build_dir} + + git reset --hard ${revision} + git submodule update --init --recursive + ) +} + +function build_qemu { + local qemu_name="${1}" + local qemu_url="${2}" + local qemu_revision="${3}" + + local build_dir="${ROOT}/${qemu_name}_build" + + echo "@@@BUILD_STEP build ${qemu_name}@@@" + ( + git_clone_at_revision ${qemu_name} ${qemu_url} ${qemu_revision} \ + ${build_dir} || exit 1 + + ${build_dir}/qemu-x86_64 --version && + ${build_dir}/qemu-system-x86_64 --version && + ${build_dir}/qemu-img --version && + exit 0 + + rm -rf ${build_dir} && + mkdir -p ${build_dir} && + cd ${build_dir} && + ${ROOT}/${qemu_name}/configure --enable-linux-user && + ninja && + ${build_dir}/qemu-x86_64 --version && + ${build_dir}/qemu-system-x86_64 --version && + ${build_dir}/qemu-img --version + ) || ( + build_exception + exit 2 + ) +} + +function build_lam_linux { + echo "@@@BUILD_STEP build lam linux@@@" + local build_dir="${ROOT}/lam_linux_build" + LAM_KERNEL="${build_dir}/arch/x86_64/boot/bzImage" + ( + git_clone_at_revision lam_linux https://github.com/vitalybuka/linux.git \ + origin/lam ${build_dir} || exit 1 + + ls "${LAM_KERNEL}" && exit 0 + + rm -rf ${build_dir} && + mkdir -p ${build_dir} && + cd ${ROOT}/lam_linux && + make O=${build_dir} LD=ld.bfd defconfig && + make O=${build_dir} LD=ld.bfd -j $(nproc) && + ls "${LAM_KERNEL}" + ) || ( + build_exception + exit 2 + ) +} + +build_qemu qemu https://github.com/vitalybuka/qemu.git origin/sanitizer_bot +[[ -z "$SKIP_HWASAN_LAM" ]] && build_lam_linux + +SCUDO_BUILDS= + +function configure_scudo_compiler_rt { + local arch=$1 + local target="${arch}-linux-gnu${2:-}" + + local name="" + if [[ "$DBG" == "ON" ]] ; then + name=debug_ + fi + name+="${arch}" + + local qemu_cmd="" + if [[ "${QEMU:-}" != "0" ]] ; then + name+="_qemu" + local qemu_arch=${arch} + [[ "${arch}" == "powerpc64le" ]] && qemu_arch="ppc64le" + qemu_cmd="$ROOT/qemu_build/qemu-${qemu_arch} -L /usr/${target}" + if [[ ! -z "${QEMU_CPU:-}" ]] ; then + qemu_cmd+=" -cpu ${QEMU_CPU}" + name+="_${QEMU_CPU}" + fi + fi + + local linker_flags= + [[ "${arch}" =~ "mips*" ]] && linker_flags="-latomic -Wl,-z,notext -Wno-unused-command-line-argument" + + local out_dir=llvm_build2_${name} + rm -rf ${out_dir} + mkdir -p ${out_dir} + + SCUDO_BUILDS+=" ${name}" + + ( + # We need a fresh base compiler to test standalone build, as this config + # does not build own clang. + local COMPILER_BIN_DIR="$(readlink -f ${STAGE2_DIR})/bin" + cd ${out_dir} + + ( + cmake \ + ${CMAKE_COMMON_OPTIONS} \ + -DCOMPILER_RT_DEBUG=$DBG \ + -DLLVM_CMAKE_DIR="${COMPILER_BIN_DIR}/.." \ + -DCMAKE_C_COMPILER=${COMPILER_BIN_DIR}/clang \ + -DCMAKE_CXX_COMPILER=${COMPILER_BIN_DIR}/clang++ \ + -DCOMPILER_RT_HAS_LLD=ON \ + -DCOMPILER_RT_TEST_USE_LLD=ON \ + -DCMAKE_INSTALL_PREFIX=$(${COMPILER_BIN_DIR}/clang -print-resource-dir) \ + -DCOMPILER_RT_BUILD_BUILTINS=OFF \ + -DCOMPILER_RT_DEFAULT_TARGET_ONLY=ON \ + -DCMAKE_CROSSCOMPILING=True \ + -DCOMPILER_RT_INCLUDE_TESTS=ON \ + -DCOMPILER_RT_BUILD_LIBFUZZER=OFF \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DCMAKE_CXX_FLAGS=-fPIC \ + -DCMAKE_C_FLAGS=-fPIC \ + -DCMAKE_SHARED_LINKER_FLAGS="-fuse-ld=lld ${linker_flags}" \ + -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld ${linker_flags}" \ + -DCOMPILER_RT_TEST_COMPILER_CFLAGS="--target=${target} ${linker_flags}" \ + -DCMAKE_C_COMPILER_TARGET=${target} \ + -DCMAKE_CXX_COMPILER_TARGET=${target} \ + -DCOMPILER_RT_EMULATOR="${qemu_cmd:-}" \ + $LLVM/../compiler-rt + ) >& configure.log + ) & +} + +function configure_hwasan_lam { + local out_dir=llvm_build2_x86_64_lam_qemu + rm -rf ${out_dir} + mkdir -p ${out_dir} + + ( + # We don't need fresh base compiler, as this config will build own clang. + local COMPILER_BIN_DIR="$(readlink -f ${STAGE1_DIR})/bin" + cd ${out_dir} + + ( + # STAGE1_DIR is build once, so we can use CCACHE. + cmake \ + ${CMAKE_COMMON_OPTIONS} \ + -DLLVM_ENABLE_PROJECTS="clang;compiler-rt;lld" \ + -DCMAKE_C_COMPILER=${COMPILER_BIN_DIR}/clang \ + -DCMAKE_CXX_COMPILER=${COMPILER_BIN_DIR}/clang++ \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DCOMPILER_RT_EMULATOR="env SSH_CONTROL_SOCKET=${SSH_CONTROL_SOCKET} ${HERE}/ssh_run.sh" \ + $LLVM + ) >& configure.log + ) & +} + +function run_scudo_tests { + local name="${1}" + local out_dir=llvm_build2_${name} + + echo "@@@BUILD_STEP scudo $name@@@" + + ( + cd ${out_dir} + + cat configure.log + + # Copy into clang resource dir to make -fsanitize= work in lit tests. + ninja install-scudo_standalone + + ninja check-scudo_standalone || exit 3 + ) || build_failure +} + +QEMU_PID="" +readonly QEMU_TMPDIR="$ROOT/qemu_tmp" +readonly SSH_CONTROL_SOCKET="${QEMU_TMPDIR}/ssh-control-socket" + +function kill_qemu { + if kill "${QEMU_PID}"; then + echo "Waiting for QEMU to shutdown..." >&2 + { + timeout -k 5s 5s wait "${QEMU_PID}" && return + kill -9 "${QEMU_PID}" && wait "${QEMU_PID}" || true + } &>/dev/null + fi +} + +function boot_qemu { + trap kill_qemu EXIT + + # Path to a qemu-system-x86_64 binary built with LAM support. + local QEMU="${ROOT}/qemu_build/qemu-system-x86_64" + # Path to a qemu-img binary. + local QEMU_IMG="${ROOT}/qemu_build/qemu-img" + + echo "Booting QEMU..." >&2 + + # Try up to 10 random port numbers until one succeeds. + for i in {0..10}; do + rm -rf ${QEMU_TMPDIR} + mkdir -p ${QEMU_TMPDIR} + # Create a delta image to boot from. + local DELTA_IMAGE="${QEMU_TMPDIR}/delta.qcow2" + "${QEMU_IMG}" create -F raw -b "${QEMU_IMAGE}" -f qcow2 "${DELTA_IMAGE}" + + local SSH_PORT="$(shuf -i 1000-65535 -n 1)" + + "${QEMU}" -hda "${DELTA_IMAGE}" -nographic \ + -net "user,host=10.0.2.10,hostfwd=tcp:127.0.0.1:${SSH_PORT}-:22" \ + -net "nic,model=e1000" -machine "type=q35,accel=tcg" \ + -smp $(($(nproc) / 2)) -cpu "qemu64,+la57,+lam" -kernel "${LAM_KERNEL}" \ + -append "root=/dev/sda net.ifnames=0 console=ttyS0" -m "16G" & + QEMU_PID=$! + + # If QEMU is running, the port number worked. + sleep 1 + ps -p "${QEMU_PID}" &>/dev/null || continue + + echo "Waiting for QEMU ssh daemon..." >&2 + for i in {0..10}; do + echo "SSH into VM, try ${i}" >&2 + sleep 15 + + # Set up persistent SSH connection for faster command execution inside QEMU. + ssh -p "${SSH_PORT}" -o "StrictHostKeyChecking=no" \ + -o "UserKnownHostsFile=/dev/null" -o "ControlPersist=30m" \ + -M -S "${SSH_CONTROL_SOCKET}" -i "${QEMU_SSH_KEY}" root@localhost "uname -a" 1>&2 && + return + done + + kill_qemu + done + + # Fail fast if QEMU is not running. + ps -p "${QEMU_PID}" &>/dev/null +} + +function run_hwasan_lam_tests { + local name="x86_64_lam_qemu" + local out_dir=llvm_build2_${name} + + echo "@@@BUILD_STEP configure hwasan ${name}@@@" + + ( + cd ${out_dir} + cat configure.log + + # Build most stuff before starting VM. + echo "@@@BUILD_STEP build tools ${name}@@@" + ninja clang lld llvm-symbolizer || exit 3 + + echo "@@@BUILD_STEP start LAM QEMU@@@" + boot_qemu || build_exception + + ssh -S "${SSH_CONTROL_SOCKET}" root@localhost \ + "mkdir -p /b && mount -t nfs 10.0.2.10:/b /b" + + echo + echo "@@@BUILD_STEP test hwasan ${name}@@@" + # Fixme: figure out why is the test failing with detect_leaks tunred on. + export LIT_FILTER_OUT="print-memory-usage.c" + ninja check-hwasan-lam || exit 3 + ) || build_failure +} + +echo "@@@BUILD_STEP configure@@@" + +for DBG in OFF ON ; do + QEMU=0 configure_scudo_compiler_rt x86_64 + configure_scudo_compiler_rt x86_64 + # FIXME: investigate and reenable configure_scudo_compiler_rt arm eabihf + configure_scudo_compiler_rt aarch64 + QEMU_CPU="cortex-a72" configure_scudo_compiler_rt aarch64 + #configure_scudo_compiler_rt mips + #configure_scudo_compiler_rt mipsel + configure_scudo_compiler_rt mips64 abi64 + configure_scudo_compiler_rt mips64el abi64 + configure_scudo_compiler_rt powerpc64le +done +[[ -z "$SKIP_HWASAN_LAM" ]] && configure_hwasan_lam + +wait + +for B in $SCUDO_BUILDS ; do + run_scudo_tests $B +done + +[[ -z "$SKIP_HWASAN_LAM" ]] && ( + run_hwasan_lam_tests +) + +cleanup $STAGE2_DIR diff --git a/zorg/buildbot/builders/sanitizers/buildbot_selector.py b/zorg/buildbot/builders/sanitizers/buildbot_selector.py --- a/zorg/buildbot/builders/sanitizers/buildbot_selector.py +++ b/zorg/buildbot/builders/sanitizers/buildbot_selector.py @@ -5,63 +5,73 @@ import sys THIS_DIR=os.path.dirname(sys.argv[0]) +extra_args = sys.argv[1:] +BOT_DIR = '/b' - -def bash(path): - return 'bash ' + os.path.join(THIS_DIR, path) - -def cmd_call(path): - return 'call ' + os.path.join(THIS_DIR, path) +def in_script_dir(path): + return os.path.join(THIS_DIR, path) BOT_ASSIGNMENT = { - 'sanitizer-ppc64le-linux': bash('buildbot_cmake.sh'), - 'sanitizer-ppc64be-linux': bash('buildbot_cmake.sh'), - 'sanitizer-x86_64-linux': bash('buildbot_cmake.sh'), - 'sanitizer-x86_64-linux-fast': bash('buildbot_fast.sh'), - 'sanitizer-x86_64-linux-autoconf': bash('buildbot_standard.sh'), - 'sanitizer-x86_64-linux-fuzzer': bash('buildbot_fuzzer.sh'), - 'sanitizer-x86_64-linux-android': bash('buildbot_android.sh'), - 'sanitizer-x86_64-linux-bootstrap': bash('buildbot_bootstrap_asan.sh'), - 'sanitizer-x86_64-linux-bootstrap-msan': bash('buildbot_bootstrap_msan.sh'), - 'sanitizer-x86_64-linux-bootstrap-ubsan': bash('buildbot_bootstrap_ubsan.sh'), - 'sanitizer-x86_64-linux-gn': 'true', + 'sanitizer-ppc64le-linux': 'buildbot_cmake.sh', + 'sanitizer-ppc64be-linux': 'buildbot_cmake.sh', + 'sanitizer-x86_64-linux': 'buildbot_cmake.sh', + 'sanitizer-x86_64-linux-fast': 'buildbot_fast.sh', + 'sanitizer-x86_64-linux-autoconf': 'buildbot_standard.sh', + 'sanitizer-x86_64-linux-fuzzer': 'buildbot_fuzzer.sh', + 'sanitizer-x86_64-linux-android': 'buildbot_android.sh', + 'sanitizer-x86_64-linux-bootstrap-asan': 'buildbot_bootstrap_asan.sh', + 'sanitizer-x86_64-linux-bootstrap-msan': 'buildbot_bootstrap_msan.sh', + 'sanitizer-x86_64-linux-bootstrap-ubsan': 'buildbot_bootstrap_ubsan.sh', + 'sanitizer-x86_64-linux-qemu': 'buildbot_qemu.sh', + 'sanitizer-aarch64-linux-fuzzer': 'buildbot_fuzzer.sh', + 'sanitizer-aarch64-linux-bootstrap-asan': 'buildbot_bootstrap_asan.sh', + 'sanitizer-aarch64-linux-bootstrap-hwasan': 'buildbot_bootstrap_hwasan.sh', + 'sanitizer-aarch64-linux-bootstrap-msan': 'buildbot_bootstrap_msan.sh', + 'sanitizer-aarch64-linux-bootstrap-ubsan': 'buildbot_bootstrap_ubsan.sh', } BOT_ADDITIONAL_ENV = { - 'sanitizer-ppc64le-linux': { 'HAVE_NINJA': '1', 'CHECK_LIBCXX': '0', 'CHECK_LLD': '0' }, - 'sanitizer-ppc64be-linux': { 'HAVE_NINJA': '1', 'CHECK_LIBCXX': '0', 'CHECK_LLD': '0' }, - 'sanitizer-x86_64-linux': { 'HAVE_NINJA' : '1' }, - 'sanitizer-x86_64-linux-bootstrap': {}, + 'sanitizer-ppc64le-linux': { 'CHECK_LIBCXX': '0' }, + 'sanitizer-ppc64be-linux': { 'CHECK_LIBCXX': '0' }, + 'sanitizer-x86_64-linux': {}, 'sanitizer-x86_64-linux-fast': {}, 'sanitizer-x86_64-linux-autoconf': {}, 'sanitizer-x86_64-linux-fuzzer': {}, 'sanitizer-x86_64-linux-android': {}, + 'sanitizer-x86_64-linux-bootstrap-asan': {}, 'sanitizer-x86_64-linux-bootstrap-msan': {}, 'sanitizer-x86_64-linux-bootstrap-ubsan': {}, - 'sanitizer-x86_64-linux-gn': {}, + 'sanitizer-x86_64-linux-qemu': { 'QEMU_IMAGE_DIR': BOT_DIR + '/qemu_image' }, + 'sanitizer-aarch64-linux-fuzzer': {}, + 'sanitizer-aarch64-linux-bootstrap-asan': {}, + 'sanitizer-aarch64-linux-bootstrap-hwasan': {}, + 'sanitizer-aarch64-linux-bootstrap-msan': {}, + 'sanitizer-aarch64-linux-bootstrap-ubsan': {}, } def Main(): builder = os.environ.get('BUILDBOT_BUILDERNAME') - print "builder name: %s" % (builder) - cmd = BOT_ASSIGNMENT.get(builder) + revision = os.environ.get('BUILDBOT_REVISION') + print("builder name: %s" % (builder)) + cmd = [in_script_dir(BOT_ASSIGNMENT.get(builder))] + extra_args if not cmd: sys.stderr.write('ERROR - unset/invalid builder name\n') sys.exit(1) - print "%s runs: %s\n" % (builder, cmd) + print("%s runs: %s\n" % (builder, ' '.join(cmd))) sys.stdout.flush() bot_env = os.environ + bot_env['BOT_DIR'] = BOT_DIR add_env = BOT_ADDITIONAL_ENV.get(builder) for var in add_env: bot_env[var] = add_env[var] if 'TMPDIR' in bot_env: del bot_env['TMPDIR'] - retcode = subprocess.call(cmd, env=bot_env, shell=True) - sys.exit(retcode) - + if ':' in revision: + cmd = [in_script_dir('buildbot_bisect_run.sh')] + cmd + sys.exit(subprocess.call(cmd, env=bot_env)) if __name__ == '__main__': Main() diff --git a/zorg/buildbot/builders/sanitizers/buildbot_standard.sh b/zorg/buildbot/builders/sanitizers/buildbot_standard.sh --- a/zorg/buildbot/builders/sanitizers/buildbot_standard.sh +++ b/zorg/buildbot/builders/sanitizers/buildbot_standard.sh @@ -11,15 +11,10 @@ TSAN_FULL_DEBUG_BUILD_DIR=tsan_full_debug_build TSAN_RELEASE_BUILD_DIR=tsan_release_build -rm -rf $TSAN_DEBUG_BUILD_DIR $TSAN_FULL_DEBUG_BUILD_DIR $TSAN_RELEASE_BUILD_DIR -clobber +CLEANUP="$TSAN_DEBUG_BUILD_DIR $TSAN_FULL_DEBUG_BUILD_DIR $TSAN_RELEASE_BUILD_DIR" +clobber $CLEANUP -ROOT=`pwd` -PLATFORM=`uname` -MAKE_JOBS=${MAX_MAKE_JOBS:-$(nproc)} - -LLVM=${ROOT}/llvm -CMAKE_COMMON_OPTIONS="-GNinja -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_BUILD_EXTERNAL_COMPILER_RT=ON -DLLVM_USE_LINKER=gold" +CMAKE_COMMON_OPTIONS+=" -DLLVM_BUILD_EXTERNAL_COMPILER_RT=ON" function build_tsan { local build_dir=$1 @@ -28,28 +23,22 @@ if [ ! -d $build_dir ]; then mkdir $build_dir fi - (cd $build_dir && CC="$3" CXX="$4" cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + (cd $build_dir && CC="$3" CXX="$4" cmake \ ${CMAKE_COMMON_OPTIONS} ${extra_cmake_args} \ ${LLVM}) - (cd $build_dir && ninja ${targets}) || echo @@@STEP_FAILURE@@@ - (cd $build_dir && ninja compiler-rt-clear) || echo @@@STEP_FAILURE@@@ - (cd $build_dir && ninja tsan) || echo @@@STEP_FAILURE@@@ + (cd $build_dir && ninja ${targets}) || build_failure + (cd $build_dir && ninja tsan) || build_failure } buildbot_update -USE_CCACHE= -if ccache -s ; then - USE_CCACHE="-DLLVM_CCACHE_BUILD=ON" -fi - echo @@@BUILD_STEP build fresh clang + debug compiler-rt@@@ -build_tsan "${TSAN_DEBUG_BUILD_DIR}" "-DCOMPILER_RT_DEBUG=ON $USE_CCACHE" gcc g++ +build_tsan "${TSAN_DEBUG_BUILD_DIR}" "-DCOMPILER_RT_DEBUG=ON" gcc g++ echo @@@BUILD_STEP test tsan in debug compiler-rt build@@@ -(cd $TSAN_DEBUG_BUILD_DIR && ninja check-tsan) || echo @@@STEP_FAILURE@@@ +(cd $TSAN_DEBUG_BUILD_DIR && ninja check-tsan) || build_failure echo @@@BUILD_STEP build tsan with stats and debug output@@@ -build_tsan "${TSAN_FULL_DEBUG_BUILD_DIR}" "-DCOMPILER_RT_DEBUG=ON -DCOMPILER_RT_TSAN_DEBUG_OUTPUT=ON -DLLVM_INCLUDE_TESTS=OFF $USE_CCACHE" gcc g++ +build_tsan "${TSAN_FULL_DEBUG_BUILD_DIR}" "-DCOMPILER_RT_DEBUG=ON -DCOMPILER_RT_TSAN_DEBUG_OUTPUT=ON -DLLVM_INCLUDE_TESTS=OFF" gcc g++ echo @@@BUILD_STEP build release tsan with clang@@@ build_tsan "${TSAN_RELEASE_BUILD_DIR}" "-DCOMPILER_RT_DEBUG=OFF" "$ROOT/$TSAN_DEBUG_BUILD_DIR/bin/clang" "$ROOT/$TSAN_DEBUG_BUILD_DIR/bin/clang++" @@ -58,4 +47,6 @@ BIN=$(mktemp -t tsan_exe.XXXXXXXX) echo "int main() {return 0;}" | $TSAN_RELEASE_BUILD_DIR/bin/clang -x c++ - -fsanitize=thread -O2 -o ${BIN} COMPILER_RT=$LLVM/../compiler-rt -$COMPILER_RT/lib/tsan/check_analyze.sh ${BIN} || echo @@@STEP_FAILURE@@@ +$COMPILER_RT/lib/tsan/check_analyze.sh ${BIN} || build_failure + +cleanup $CLEANUP diff --git a/zorg/buildbot/builders/sanitizers/ssh_run.sh b/zorg/buildbot/builders/sanitizers/ssh_run.sh new file mode 100755 --- /dev/null +++ b/zorg/buildbot/builders/sanitizers/ssh_run.sh @@ -0,0 +1,12 @@ +#!/bin/bash -eu +# +# Runs the given command over ssh using SSH_CONTROL_SOCKET. +# + +function run_in_qemu { + ssh -S "${SSH_CONTROL_SOCKET}" root@localhost "${@}" +} + +# Run binary in QEMU. +ENV="HWASAN_OPTIONS=\"${HWASAN_OPTIONS:-}\" PATH=\"${PATH:-}\"" +run_in_qemu "cd $(pwd) && ${ENV} $@" diff --git a/zorg/buildbot/changes/llvmgitpoller.py b/zorg/buildbot/changes/llvmgitpoller.py --- a/zorg/buildbot/changes/llvmgitpoller.py +++ b/zorg/buildbot/changes/llvmgitpoller.py @@ -11,31 +11,39 @@ class LLVMPoller(changes.GitPoller): """ - Poll LLVM repository for changes and submit them to the change master. + Poll LLVM repository for changes and submit them for builds scheduling. Following Multiple LLVM Projects. This source will poll a remote LLVM git _monorepo_ for changes and submit - them to the change master.""" + them for builds scheduling.""" _repourl = "https://github.com/llvm/llvm-project" - _branch = "master" - compare_attrs = ["repourl", "branch", "workdir", + compare_attrs = ["repourl", "branches", "workdir", "pollInterval", "gitbin", "usetimestamps", "category", "project", "projects"] + def _check_branches(branch): + if branch == "refs/heads/main": + # Always listen for changes in the main branch. + return True + else: + # We are also interested in the release branches. + # Some builders will be building changes from there as well. + return re.search(r"refs\/heads\/release\/\d\d+.*", branch) + def __init__(self, - repourl=_repourl, branch=_branch, + repourl=_repourl, branches=_check_branches, **kwargs): self.cleanRe = re.compile(r"Require(?:s?)\s*.*\s*clean build", re.IGNORECASE + re.MULTILINE) self.cleanCfg = re.compile(r"(CMakeLists\.txt$|\.cmake$|\.cmake\.in$)") - # TODO: Add support for an optional list of projects. - # For now we always watch all the projects. + # Note: We always watch all the projects, then schedulers decide + # to build or not to build. - super().__init__(repourl=repourl, branch=branch, **kwargs) + super().__init__(repourl=repourl, branches=branches, **kwargs) def _transform_path(self, fileList): """ @@ -111,7 +119,7 @@ newRev, branch)) else: # This branch is known, but it now points to a different - # commit than last time we saw it, rebuild. + # commit than the last time we saw it, rebuild. log.msg('LLVMPoller: rebuilding {} for updated branch "{}"'.format( newRev, branch)) @@ -163,8 +171,9 @@ properties['clean_obj'] = (True, "change") log.msg("LLVMPoller: creating a change rev=%s" % rev) - log.msg(" >>> revision=%s, timestamp=%s, author=%s, committer=%s, project=%s, files=%s, comments=\"%s\", properties=%s" % \ - (bytes2unicode(rev, encoding=self.encoding), datetime.fromtimestamp(timestamp), author, committer, + log.msg(" >>> branch=%s, revision=%s, timestamp=%s, author=%s, committer=%s, project=%s, files=%s, comments=\"%s\", properties=%s" % \ + (bytes2unicode(self._removeHeads(branch)), + bytes2unicode(rev, encoding=self.encoding), datetime.fromtimestamp(timestamp), author, committer, projects, files, comments, properties)) yield self.master.data.updates.addChange( diff --git a/zorg/buildbot/commands/AnnotatedCommand.py b/zorg/buildbot/commands/AnnotatedCommand.py --- a/zorg/buildbot/commands/AnnotatedCommand.py +++ b/zorg/buildbot/commands/AnnotatedCommand.py @@ -97,7 +97,7 @@ @defer.inlineCallbacks def addStep(self): - debuglog("AnnotatedBuildStep: adding step '%s': buildid=%s" % ( + debuglog("AnnotatedBuildStep: adding step '{}': buildid={}".format( self.name, self.build.buildid)) yield buildstep.BuildStep.addStep(self) # Put ourselves into a list of processed steps of the build object. @@ -107,14 +107,14 @@ self.description = [self.name] self.stdio_log = yield self.addLog('stdio') - debuglog("AnnotatedBuildStep: added step '%s': stepid=%s, buildid=%s" % ( + debuglog("AnnotatedBuildStep: added step '{}': stepid={}, buildid={}".format( self.name, self.stepid, self.build.buildid)) @defer.inlineCallbacks def finishStep(self): hidden = False - debuglog("AnnotatedBuildStep: finish step '%s': stepid=%s, buildid=%s, results=%s" % ( + debuglog("AnnotatedBuildStep: finish step '{}': stepid={}, buildid={}, results={}".format( self.name, self.stepid, self.build.buildid, self.results)) yield self.master.data.updates.finishStep(self.stepid, self.results, hidden) @@ -123,8 +123,8 @@ @defer.inlineCallbacks def startStep(self, remote, done=False): - debuglog("AnnotatedBuildStep::startStep() starting '%s': " - "buildid=%s, done=%s" % ( + debuglog("AnnotatedBuildStep::startStep() starting '{}': " + "buildid={}, done={}".format( self.name, self.build.buildid, done)) try: @@ -165,13 +165,13 @@ # Update step status in the database. yield self.finishStep() - debuglog("AnnotatedBuildStep::startStep() completed '%s': " - "stepid=%s, buildid=%s, results=%s" % ( + debuglog("AnnotatedBuildStep::startStep() completed '{}': " + "stepid={}, buildid={}, results={}".format( self.name, self.stepid, self.build.buildid, self.results)) def requestFinish(self, status=None): - debuglog("AnnotatedBuildStep::requestFinish(%r): '%s': " - "stepid=%s, buildid=%s, results=%r" % ( + debuglog("AnnotatedBuildStep::requestFinish({}): '{}': " + "stepid={}, buildid={}, results={}".format( status, self.name, self.stepid, self.build.buildid, self.results)) # Update the current step status with the worst result. if status is not None: @@ -212,8 +212,8 @@ @defer.inlineCallbacks def run(self): - debuglog("AnnotatedBuildStep::run() starting '%s' step: " - "stepid=%s, buildid=%s" % ( + debuglog("AnnotatedBuildStep::run() starting '{}' step: " + "stepid={}, buildid={}".format( self.name, self.stepid, self.build.buildid)) # Save previously collected log lines. @@ -229,8 +229,8 @@ if self._loglines: yield self._flushLogs() - debuglog("AnnotatedBuildStep::run() exiting '%s' step: " - "stepid=%s, buildid=%s, results=%s" % ( + debuglog("AnnotatedBuildStep::run() exiting '{}' step: " + "stepid={}, buildid={}, results={}".format( self.name, self.stepid, self.build.buildid, self.results)) return self.results @@ -321,8 +321,6 @@ 'BUILDBOT_GOT_REVISION': util.Interpolate('%(prop:got_revision:-None)s'), 'BUILDBOT_REVISION': util.Interpolate('%(prop:revision:-None)s'), 'BUILDBOT_SCHEDULER': util.Interpolate('%(prop:scheduler:-None)s'), - 'BUILDBOT_SLAVENAME': util.Interpolate('%(prop:slavename:-None)s'), - 'BUILDBOT_MSAN_ORIGINS': util.Interpolate('%(prop:msan_origins:-)s'), } # Apply the passed in environment on top. old_env = kwargs.get('env') or {} @@ -350,7 +348,7 @@ return self.annotated_steps[-1] if self.annotated_steps else None def _updateLastAnnotatedStepStatus(self, status): - debuglog(">>> AnnotatedCommand::_updateLastAnnotatedStepStatus(%r)" % status) + debuglog(">>> AnnotatedCommand::_updateLastAnnotatedStepStatus({})".format(status)) # Alway update the common annotate command status. self.annotate_status = results.worst_status(self.annotate_status, status) @@ -365,7 +363,8 @@ def _updateLastAnnotatedStepText(self, text = None): """Updating a step text. None to clean up.""" - debuglog(">>> AnnotatedCommand::_updateLastAnnotatedStepText('%s')" % (text if text else '')) + debuglog(">>> AnnotatedCommand::_updateLastAnnotatedStepText('{}')".format( + text if text else '')) s = self._getLastAnnotatedStep() if s is not None: @@ -373,7 +372,8 @@ def _updateLastAnnotatedStepSummaryInfo(self, text = None): """Updating a step summary info. None to clean up.""" - debuglog(">>> AnnotatedCommand::_updateLastAnnotatedStepSummaryInfo('%s')" % (text if text else '')) + debuglog(">>> AnnotatedCommand::_updateLastAnnotatedStepSummaryInfo('{}')".format( + text if text else '')) s = self._getLastAnnotatedStep() if s is not None: @@ -388,7 +388,7 @@ s.requestFinish() def _scheduleNewAnnotatedStep(self, name, logline): - debuglog(">>> AnnotatedCommand::_scheduleNewAnnotatedStep('%s')" % name) + debuglog(">>> AnnotatedCommand::_scheduleNewAnnotatedStep('{}')".format(name)) self._fixupActiveAnnotatedStep() @@ -517,7 +517,7 @@ ancmd, args = parse_annotate_cmd(line) if ancmd: - debuglog(">>> AnnotatedCommand::processAnnotatedCommand(): %s, %r" % (ancmd, args)) + debuglog(">>> AnnotatedCommand::processAnnotatedCommand(): {}, {}".format(ancmd, args)) try: if ancmd == STEP_LINK: diff --git a/zorg/buildbot/commands/LitTestCommand.py b/zorg/buildbot/commands/LitTestCommand.py --- a/zorg/buildbot/commands/LitTestCommand.py +++ b/zorg/buildbot/commands/LitTestCommand.py @@ -19,7 +19,7 @@ # These are the codes for which we will include the log output in the buildbot # step results. - failingCodes = set(['FAIL', 'XPASS', 'KPASS', 'UNRESOLVED', 'TIMEOUT', 'ERROR']) + failingCodes = set(['FAIL', 'XPASS', 'KPASS', 'UNRESOLVED', 'TIMEOUT', 'ERROR', 'NOEXE']) # Regular expressions for start of summary marker. kStartSummaryRE = re.compile(r'^Failing Tests \(\d*\)$') @@ -142,9 +142,9 @@ class LitTestCommand(Test): resultNames = {'FAIL':'unexpected failures', + 'XPASS':'unexpected passes', 'PASS':'expected passes', 'XFAIL':'expected failures', - 'XPASS':'unexpected passes', 'KFAIL':'known failures', 'KPASS':'unknown passes', 'UNRESOLVED':'unresolved testcases', @@ -152,7 +152,9 @@ 'REGRESSED':'runtime performance regression', 'IMPROVED':'runtime performance improvement', 'UNSUPPORTED':'unsupported tests', - 'TIMEOUT':'timeout waiting for results'} + 'SKIPPED':'skipped tests', + 'TIMEOUT':'timeout waiting for results', + 'NOEXE':'test executable is missing'} def __init__(self, ignore=[], flaky=[], max_logs=20, parseSummaryOnly=False, *args, **kwargs): @@ -176,9 +178,17 @@ def describe(self, done=False): description = Test.describe(self, done) or list() - for name, count in self.logObserver.resultCounts.items(): - if name in self.resultNames: - description.append('{0} {1}'.format(count, self.resultNames[name])) - else: - description.append('Unexpected test result output ' + name) + + # We use resultNames here so that the printed order is always the same. + # resultCounts is a dictionary that goes by insertion order instead. + for resultName, resultDescription in self.resultNames.items(): + count = self.logObserver.resultCounts.get(resultName, 0) + if count: + print(">>>> LitLogObserver.describe: description={}".format(description)) + description.append('{0} {1}'.format(count, resultDescription)) + + unknown = set(self.logObserver.resultCounts.keys()) - set(self.resultNames.keys()) + for name in unknown: + description.append('Unexpected test result output ' + name) + return description diff --git a/zorg/buildbot/process/factory.py b/zorg/buildbot/process/factory.py --- a/zorg/buildbot/process/factory.py +++ b/zorg/buildbot/process/factory.py @@ -5,22 +5,70 @@ from buildbot.process.factory import BuildFactory from buildbot.plugins import util, steps +_all_runtimes = frozenset([ + "compiler-rt", + "libc", + "libcxx", + "libcxxabi", + "libunwind", + "openmp", +]) + class LLVMBuildFactory(BuildFactory): """ TODO: Document + + depends_on_projects is a list of LLVM projects a produced builder + depends on. If None, it gets discovered depending on other params. + + enable_runtimes is a list of enabled runtimes. If None, + it gets discovered based on the depends_on_projects list. """ def __init__(self, steps=None, depends_on_projects=None, **kwargs): # Cannot use "super" here as BuildFactory is an old style class. BuildFactory.__init__(self, steps) + # Handle the dependencies. if depends_on_projects is None: - self.depends_on_projects = frozenset(['llvm']) + # llvm project is always included. + self.depends_on_projects = set(['llvm']) else: self.depends_on_projects = frozenset(depends_on_projects) + enable_runtimes = kwargs.pop('enable_runtimes', None) + # If specified, we either got a givem list of + # enabled runtimes, or "auto", or "all". + + if enable_runtimes is None: + # For the backward compatibility, we do not use + # enable_runtimes unless it is requested explicitly. + self.enable_runtimes = frozenset([]) + elif enable_runtimes == "auto": + # Let's build the list of runtimes based on the given + # depends_on_projects list. + self.enable_runtimes = \ + self.depends_on_projects.intersection(_all_runtimes) + else: + if enable_runtimes == "all": + # Let's replace the "all" placeholder by + # the actual list of all runtimes. + self.enable_runtimes = frozenset(_all_runtimes) + else: + # Let's just use the given list, no need to discover. + self.enable_runtimes = frozenset(enable_runtimes) + + # Update the list of dependencies. + if depends_on_projects is None: + self.depends_on_projects.update(self.enable_runtimes) + + # Build the list of projects to enable. + self.enable_projects = \ + self.depends_on_projects.difference(self.enable_runtimes) + # Directories. - self.llvm_srcdir = kwargs.pop('llvm_srcdir', None) + self.monorepo_dir = kwargs.pop('llvm_srcdir', None) + self.src_to_build_dir = kwargs.pop('src_to_build_dir', None) self.obj_dir = kwargs.pop('obj_dir', None) self.install_dir = kwargs.pop('install_dir', None) @@ -29,9 +77,10 @@ for k,v in kwargs.items(): setattr(self, k, v) - self.monorepo_dir = self.llvm_srcdir or "llvm-project" + self.monorepo_dir = self.monorepo_dir or "llvm-project" + self.src_to_build_dir = self.src_to_build_dir or 'llvm' self.llvm_srcdir = \ - "%(monorepo_dir)s/llvm" % {'monorepo_dir' : self.monorepo_dir} + "{}/{}".format(self.monorepo_dir, self.src_to_build_dir) self.obj_dir = \ self.obj_dir or "build" @@ -71,11 +120,14 @@ def addGetSourcecodeSteps(self, **kwargs): # Checkout the monorepo. + # Documentation: http://docs.buildbot.net/current/manual/configuration/buildsteps.html#git self.addStep(steps.Git( name='Checkout the source code', repourl=self.repourl_prefix + "llvm-project.git", progress=True, workdir=util.WithProperties(self.monorepo_dir), + retryFetch=True, + clobberOnFailure=True, **kwargs)) @@ -104,4 +156,6 @@ repourl=_repourl, progress=True, workdir=util.WithProperties(src_dir), + retryFetch=True, + clobberOnFailure=True, **kwargs)) diff --git a/zorg/buildbot/util/InformativeMailNotifier.py b/zorg/buildbot/util/InformativeMailNotifier.py --- a/zorg/buildbot/util/InformativeMailNotifier.py +++ b/zorg/buildbot/util/InformativeMailNotifier.py @@ -1,38 +1,104 @@ -# TODO: Add support for clang crash dumps. -# TODO: Better handle unit/regression tests failures - -# TODO: For debug purposes. Remove this later. -from twisted.python import log +import traceback from buildbot.plugins import reporters +from buildbot.process.results import CANCELLED +from buildbot.process.results import EXCEPTION +from buildbot.process.results import FAILURE +from buildbot.process.results import RETRY +from buildbot.process.results import SKIPPED +from buildbot.process.results import SUCCESS +from buildbot.process.results import WARNINGS +from buildbot.process.results import statusToString + +def get_log_details(build): + text = "" + try: + for step in build['steps']: + results = step['results'] + if results == SUCCESS or results == SKIPPED: + continue + + text += "Step {} ({}) {}: {}\n".format(step['number'], step['name'], statusToString(results), step['state_string']) + + logs = step['logs'] + if logs: + log_index = -1 + log_type = 0 + for i, _log in enumerate(logs): + if _log['name'].startswith("FAIL: "): # Use only first logchunk FAIL: + log_type = 3 + log_index = i + elif log_type < 2 and _log['name'].startswith("warnings "): + log_type = 2 + log_index = i + elif log_type < 1 and _log['type'] == "s": # stdio + log_type = 1 + log_index = i + + if log_index < 0: + continue + + log_text = logs[log_index]['content']['content'] + if log_type == 1: + # Parse stdio + lines = log_text.splitlines() + for line in lines[:]: + if line.startswith("h"): + lines.remove(line) + for j, line in enumerate(lines): + if line.startswith("o") or line.startswith("e"): + lines[j] = line[1:] + for j, line in enumerate(lines): + if line.find("FAIL:") != -1 or line.find("FAILED") != -1: + if j > 10: + del lines[:j-10] # Start 10 lines before FAIL + lines = ["..."] + lines + del lines[50:] # Keep up to 50 lines around FAIL + break + if len(lines) > 50: + del lines[:len(lines)-50] # Otherwise keep last 50 lines + lines = ["..."] + lines + + log_text = "\n".join(lines) + + elif logs[log_index]['num_lines'] > 50: + # Keep first 50 lines + lines = log_text.splitlines() + del lines[50:] + log_text = "\n".join(lines + ["..."]) + + text += log_text + "\n" -def _get_logs_and_tracebacks_from_build(build): - # TODO: Implement interesting parts of the logs and tracebacks extraction. - return dict() + except Exception as err: + print("Exception in LLVMMessageFormatter.get_log_details(): {}\n{}".format(err, traceback.format_exc())) + # TODO: We should send something useful in this case. + + return dict(details=text) + +# TODO: Add build reason if we have that valid and available +#Build Reason: {{ build['properties'].get('reason', [""])[0] }} MAIL_TEMPLATE = """\ The Buildbot has detected a {{ status_detected }} on builder {{ buildername }} while building {{ projects }}. + Full details are available at: {{ build_url }} -Buildbot URL: {{ buildbot_url }} + Worker for this Build: {{ workername }} -Build Reason: {{ build['properties'].get('reason', [""])[0] }} -Blamelist: {{ ", ".join(blamelist) }} +Blamelist: + {{ ",\n ".join(blamelist) }} + {{ summary }} + +{{ details }} Sincerely, LLVM Buildbot """ class LLVMMessageFormatter(reporters.MessageFormatter): def buildAdditionalContext(self, master, ctx): - #log.msg(">>> LLVMMessageFormatter.buildAdditionalContext got ctx={}".format(ctx)) ctx.update(self.ctx) - - build = ctx["build"] - build_interesting_data = _get_logs_and_tracebacks_from_build(build) - #log.msg(">>> LLVMMessageFormatter.buildAdditionalContext build_interesting_data={}",format(build_interesting_data)) - ctx["build"].update(build_interesting_data) - + ctx.update(get_log_details(ctx["build"])) LLVMInformativeMailNotifier = LLVMMessageFormatter( template=MAIL_TEMPLATE, diff --git a/zorg/buildbot/util/workerowner.py b/zorg/buildbot/util/workerowner.py new file mode 100644 --- /dev/null +++ b/zorg/buildbot/util/workerowner.py @@ -0,0 +1,110 @@ +import re + +from twisted.internet import defer + +from buildbot.util import unicode2bytes + +from buildbot.www.authz.endpointmatchers import Match +from buildbot.www.authz.endpointmatchers import EndpointMatcherBase +from buildbot.www.authz.roles import RolesFromBase +from buildbot.www.authz.authz import Forbidden +from buildbot.www.authz.authz import Authz + + +class WorkerOwnerMatch(Match): + + def __init__(self, master, workers, **kwargs): + super().__init__(master, **kwargs) + self.workerOwners = [] + for worker in workers: + if 'workerinfo' in worker: + workerinfo = worker['workerinfo'] + if 'admin' in workerinfo: + email = re.search('.*<(.*)>.*', workerinfo['admin']) + if email: + self.workerOwners.append(email.group(1)) + + def getWorkerOwners(self): + return self.workerOwners + + +class WorkerEndpointMatcher(EndpointMatcherBase): + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + @defer.inlineCallbacks + def match_WorkerEndpoint(self, epobject, epdict, options): + worker = yield epobject.get({}, epdict) + return WorkerOwnerMatch(self.master, [worker]) + + @defer.inlineCallbacks + def match_BuildRequestEndpoint_cancel(self, epobject, epdict, options): + buildrequest = yield epobject.get({}, epdict) + if 'builderid' in buildrequest: + workers = yield self.master.db.workers.getWorkers(builderid=buildrequest['builderid']) + if workers: + return WorkerOwnerMatch(self.master, workers) + return None + + +class RolesFromWorkerOwner(RolesFromBase): + + def __init__(self, role): + super().__init__() + self.role = role + + def getRolesFromUser(self, userDetails, workerOwners): + if 'email' in userDetails: + if workerOwners and userDetails['email'] in workerOwners: + return [self.role] + return [] + + +class WorkerOwnerAuthz(Authz): + + def __init__(self, roleMatchers=None, **kwargs): + super().__init__(roleMatchers=roleMatchers, **kwargs) + self.workerOwnerRoleMatchers = [ + r for r in roleMatchers if isinstance(r, RolesFromWorkerOwner)] + self.roleMatchers = [ # Remove workerOwnerRoleMatchers from roleMatchers + r for r in self.roleMatchers if r not in self.workerOwnerRoleMatchers] + + def setMaster(self, master): + super().setMaster(master) + for r in self.workerOwnerRoleMatchers: + r.setAuthz(self) + + @defer.inlineCallbacks + def assertUserAllowed(self, ep, action, options, userDetails): + IsWorkerOwnerMatch = False + roles = self.getRolesFromUser(userDetails) + for rule in self.allowRules: + match = yield rule.match(ep, action, options) + if match is not None: + # only try to get owner if there are owner Matchers + if self.ownerRoleMatchers: + owner = yield match.getOwner() + if owner: + for r in self.ownerRoleMatchers: + roles.update(set(r.getRolesFromUser(userDetails, owner))) + + if self.workerOwnerRoleMatchers and isinstance(match, WorkerOwnerMatch): + IsWorkerOwnerMatch = True + workerOwners = match.getWorkerOwners() + if workerOwners: + for r in self.workerOwnerRoleMatchers: + roles.update(set(r.getRolesFromUser(userDetails, workerOwners))) + + for role in roles: + if self.match(role, rule.role): # fnmatch or re.match + return None + + if not rule.defaultDeny: + continue # check next suitable rule if not denied + + # f" You need to have role '{rule.role}'." + error_msg = unicode2bytes(" You must be the worker owner." if IsWorkerOwnerMatch else + " You must be an author of the commit triggered the build.") + raise Forbidden(error_msg) + return None diff --git a/zorg/jenkins/build.py b/zorg/jenkins/build.py --- a/zorg/jenkins/build.py +++ b/zorg/jenkins/build.py @@ -15,7 +15,7 @@ from contextlib import contextmanager from urllib2 import urlopen, URLError, HTTPError -SERVER = "labmaster2.lab.llvm.org" +SERVER = "green-dragon-21.local" NINJA = "/usr/local/bin/ninja" @@ -71,7 +71,7 @@ self.max_parallel_tests = os.environ.get('MAX_PARALLEL_TESTS', None) self.max_parallel_links = os.environ.get('MAX_PARALLEL_LINKS', None) self.host_compiler_url = os.environ.get('HOST_URL', - 'http://labmaster2.local/artifacts/') + 'http://green-dragon-21.local/artifacts/') self.artifact_url = os.environ.get('ARTIFACT', 'NONE') self.job_name = os.environ.get('JOB_NAME', 'NONE') self.build_id = os.environ.get('BUILD_ID', 'NONE') @@ -395,9 +395,6 @@ '-DLLVM_INCLUDE_TESTS=On', '-DCLANG_INCLUDE_TESTS=On', '-DLLVM_INCLUDE_UTILS=On', - '-DLIBCXX_INSTALL_HEADERS=On', - '-DLIBCXX_OVERRIDE_DARWIN_INSTALL=On', - '-DLIBCXX_INSTALL_LIBRARY=Off', '-DCMAKE_MACOSX_RPATH=On', ] @@ -615,9 +612,8 @@ footer() header("Run Tests") - run_cmd(conf.lldbbuilddir(), [NINJA, '-v', 'check-debuginfo']) run_cmd(conf.lldbbuilddir(), ['/usr/bin/env', 'TERM=vt100', NINJA, '-v', - 'check-lldb']) + 'check-debuginfo', 'check-lldb', '-k2']) footer() diff --git a/zorg/jenkins/clang_build_dependencies.dep b/zorg/jenkins/clang_build_dependencies.dep --- a/zorg/jenkins/clang_build_dependencies.dep +++ b/zorg/jenkins/clang_build_dependencies.dep @@ -2,5 +2,4 @@ brew cmake >= 3.17.2 # All nodes have 3.17.2 brew ninja >= 1.8.2 # Stage2 verbose output pip psutil >= 5.4.5 # Required for timeouts in the GTest Suite. -pip six >= 1.11.0 # Required for urllib -xcode == 11.5 +xcode >= 11.5 # Node configs are not currently all on the same Xcode. diff --git a/zorg/jenkins/common.groovy b/zorg/jenkins/common.groovy --- a/zorg/jenkins/common.groovy +++ b/zorg/jenkins/common.groovy @@ -30,7 +30,11 @@ checkout poll: false, changelog: true, scm: [ $class: 'GitSCM', branches: [[name: sha ]], - userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-project.git']] + extensions: [[ + $class: 'CloneOption', + reference: '/Users/Shared/llvm-project.git' + ]], + userRemoteConfigs: [[url: 'https://github.com/llvm/llvm-project.git']] ] } } @@ -81,7 +85,7 @@ try { stage('main') { dir('config') { - git url: 'https://github.com/llvm/llvm-zorg.git', branch: 'master', poll: false + git url: 'https://github.com/llvm/llvm-zorg.git', branch: 'main', poll: false } body() } @@ -102,6 +106,7 @@ def benchmark_pipeline(label, body) { properties([ + disableResume(), parameters([ string(name: 'ARTIFACT'), string(name: 'GIT_DISTANCE'), @@ -119,10 +124,10 @@ def testsuite_pipeline(label, body) { benchmark_pipeline(label) { dir('lnt') { - git url: 'https://github.com/llvm/llvm-lnt.git', branch: 'master', poll: false + git url: 'https://github.com/llvm/llvm-lnt.git', branch: 'main', poll: false } dir('test-suite') { - git url: 'https://github.com/llvm/llvm-test-suite.git', branch: 'master', poll: false + git url: 'https://github.com/llvm/llvm-test-suite.git', branch: 'main', poll: false } body() } diff --git a/zorg/jenkins/inspect_log.py b/zorg/jenkins/inspect_log.py --- a/zorg/jenkins/inspect_log.py +++ b/zorg/jenkins/inspect_log.py @@ -26,7 +26,7 @@ '''Matching engine. Keeps data structures to match a single line against a fixed list of patterns.''' def __init__(self, patterns): - # Create a combined master regex combining all patterns. + # Create a combined main regex combining all patterns. combined = '' merge = '' for pattern in patterns: diff --git a/zorg/jenkins/jobs/jobs/clang-san-iossim b/zorg/jenkins/jobs/jobs/clang-san-iossim new file mode 100644 --- /dev/null +++ b/zorg/jenkins/jobs/jobs/clang-san-iossim @@ -0,0 +1,103 @@ +pipeline { + agent { label 'green-dragon-08' } + + parameters { + string(name: 'GIT_REVISION', defaultValue: '*/main', description: 'Git revision to build') + } + + stages { + stage('Checkout') { + steps { + dir('llvm-project') { + checkout([$class: 'GitSCM', branches: [[name: params.GIT_REVISION]], userRemoteConfigs: [[url: 'https://github.com/llvm/llvm-project.git']]]) + } + dir('llvm-zorg') { + checkout([$class: 'GitSCM', branches: [[name: '*/main']], userRemoteConfigs: [[url: 'https://github.com/llvm/llvm-zorg.git']]]) + } + } + } + stage('Build') { + steps { + timeout(90) { + sh ''' + set -u + + cd llvm-project + git tag -a -m "First Commit" first_commit 97724f18c79c7cc81ced24239eb5e883bf1398ef || true + + git_desc=$(git describe --match "first_commit") + export GIT_DISTANCE=$(echo ${git_desc} | cut -f 2 -d "-") + + sha=$(echo ${git_desc} | cut -f 3 -d "-") + export GIT_SHA=${sha:1} + + cd - + + export PATH=$PATH:/usr/bin:/usr/local/bin + + # Set a MacOS minimum deployment target + export MACOSX_DEPLOYMENT_TARGET=10.14 + + rm -rf clang-install *.tar.gz + python llvm-zorg/zorg/jenkins/monorepo_build.py cmake build \ + --assertions --cmake-type=RelWithDebInfo \ + --projects="clang;clang-tools-extra;compiler-rt" + ''' + } + } + } + stage('Test') { + steps { + timeout(90) { + sh ''' + set -u + export PATH=$PATH:/usr/bin:/usr/local/bin + + export SANITIZER_IOSSIM_TEST_DEVICE_IDENTIFIER='iPhone 12' + + COMPILER_RT_TEST_DIR="${WORKSPACE}/clang-build/tools/clang/runtime/compiler-rt-bins/test" + + cd $COMPILER_RT_TEST_DIR/asan && python3 ${WORKSPACE}/clang-build/./bin/llvm-lit \ + --xunit-xml-output=testresults-asan-IOSSimX86_64Config.xunit.xml -v -vv --timeout=600 \ + $COMPILER_RT_TEST_DIR/asan/IOSSimX86_64Config/ + + cd $COMPILER_RT_TEST_DIR/tsan && python3 ${WORKSPACE}/clang-build/./bin/llvm-lit \ + --xunit-xml-output=testresults-tsan-IOSSimX86_64Config.xunit.xml -v -vv --timeout=600 \ + $COMPILER_RT_TEST_DIR/tsan/IOSSimX86_64Config/ + + cd $COMPILER_RT_TEST_DIR/ubsan && python3 ${WORKSPACE}/clang-build/./bin/llvm-lit \ + --xunit-xml-output=testresults-ubsan-AddressSanitizer-iossim-x86_64.xunit.xml -v -vv --timeout=600 \ + $COMPILER_RT_TEST_DIR/ubsan/AddressSanitizer-iossim-x86_64/ + + cd $COMPILER_RT_TEST_DIR/ubsan && python3 ${WORKSPACE}/clang-build/./bin/llvm-lit \ + --xunit-xml-output=testresults-ubsan-Standalone-iossim-x86_64.xunit.xml -v -vv --timeout=600 \ + $COMPILER_RT_TEST_DIR/ubsan/Standalone-iossim-x86_64/ + + cd $COMPILER_RT_TEST_DIR/ubsan && python3 ${WORKSPACE}/clang-build/./bin/llvm-lit \ + --xunit-xml-output=testresults-ubsan-ThreadSanitizer-iossim-x86_64.xunit.xml -v -vv --timeout=600 \ + $COMPILER_RT_TEST_DIR/ubsan/ThreadSanitizer-iossim-x86_64/ + ''' + } + junit 'clang-build/**/testresults-*.xunit.xml' + } + } + } + post { + success { + scanForIssues tool: clang() + } + regression { + emailext subject: '$DEFAULT_SUBJECT', + presendScript: '$DEFAULT_PRESEND_SCRIPT', + postsendScript: '$DEFAULT_POSTSEND_SCRIPT', + recipientProviders: [ + [$class: 'CulpritsRecipientProvider'], + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'], + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS', + body:'$DEFAULT_CONTENT' + } + } +} diff --git a/zorg/jenkins/jobs/jobs/clang-stage1-RA b/zorg/jenkins/jobs/jobs/clang-stage1-RA new file mode 100644 --- /dev/null +++ b/zorg/jenkins/jobs/jobs/clang-stage1-RA @@ -0,0 +1,119 @@ +pipeline { + agent { label 'green-dragon-02' } + + parameters { + string(name: 'GIT_REVISION', defaultValue: '*/main', description: 'Git revision to build') + } + + stages { + stage('Checkout') { + steps { + dir('llvm-project') { + checkout([$class: 'GitSCM', branches: [ + [name: params.GIT_REVISION] + ], extensions: [ + [$class: 'CloneOption', + reference: '/Users/Shared/llvm-project.git'] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-project.git'] + ]]) + } + dir('llvm-zorg') { + checkout([$class: 'GitSCM', branches: [ + [name: '*/main'] + ], extensions: [ + [$class: 'CloneOption', + reference: '/Users/Shared/llvm-zorg.git'] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-zorg.git'] + ]]) + } + } + } + stage('Build') { + steps { + timeout(90) { + sh ''' + set -u + rm -rf build.properties + + cd llvm-project + git tag -a -m "First Commit" first_commit 97724f18c79c7cc81ced24239eb5e883bf1398ef || true + + git_desc=$(git describe --match "first_commit") + export GIT_DISTANCE=$(echo ${git_desc} | cut -f 2 -d "-") + + sha=$(echo ${git_desc} | cut -f 3 -d "-") + export GIT_SHA=${sha:1} + + # Also save the LLVM_REV until LNT server is taught about GIT + export LLVM_REV=$(git show -q | grep "llvm-svn:" | cut -f2 -d":" | tr -d " ") + + cd - + + echo "GIT_DISTANCE=$GIT_DISTANCE" > build.properties + echo "GIT_SHA=$GIT_SHA" >> build.properties + echo "ARTIFACT=$JOB_NAME/clang-d$GIT_DISTANCE-g$GIT_SHA-t$BUILD_ID-b$BUILD_NUMBER.tar.gz" >> build.properties + + export PATH=$PATH:/usr/bin:/usr/local/bin + + # Set a MacOS minimum deployment target + export MACOSX_DEPLOYMENT_TARGET=10.14 + + rm -rf clang-build clang-install *.tar.gz + python llvm-zorg/zorg/jenkins/monorepo_build.py cmake build \ + --assertions --cmake-type=RelWithDebInfo \ + --projects="clang;clang-tools-extra;compiler-rt" + ''' + } + } + } + stage('Test') { + steps { + timeout(90) { + sh ''' + set -u + export PATH=$PATH:/usr/bin:/usr/local/bin + python llvm-zorg/zorg/jenkins/monorepo_build.py cmake testlong + ''' + } + junit 'clang-build/**/testresults.xunit.xml' + } + } + } + post { + always { + scanForIssues tool: clang() + } + regression { + emailext subject: '$DEFAULT_SUBJECT', + presendScript: '$DEFAULT_PRESEND_SCRIPT', + postsendScript: '$DEFAULT_POSTSEND_SCRIPT', + recipientProviders: [ + [$class: 'CulpritsRecipientProvider'], + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'], + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS', + body:'$DEFAULT_CONTENT' + } + success { + build job: 'clang-stage2-Rthinlto_relay', wait: false + build job: 'clang-stage2-cmake-RgSan_relay', wait: false + //build job: 'phase2_modules_relay', wait: false + build job: 'relay-lnt-test-suite', wait: false + build job: 'relay-lnt-ctmark', wait: false + build job: 'relay-test-suite-verify-machineinstrs', wait: false + } + unstable { + build job: 'clang-stage2-Rthinlto_relay', wait: false + build job: 'clang-stage2-cmake-RgSan_relay', wait: false + //build job: 'phase2_modules_relay', wait: false + build job: 'relay-lnt-test-suite', wait: false + build job: 'relay-lnt-ctmark', wait: false + build job: 'relay-test-suite-verify-machineinstrs', wait: false + } + } +} + diff --git a/zorg/jenkins/jobs/jobs/clang-stage1-cmake-RA-incremental b/zorg/jenkins/jobs/jobs/clang-stage1-cmake-RA-incremental new file mode 100644 --- /dev/null +++ b/zorg/jenkins/jobs/jobs/clang-stage1-cmake-RA-incremental @@ -0,0 +1,104 @@ +pipeline { + agent { label 'green-dragon-04' } + + parameters { + string(name: 'GIT_REVISION', defaultValue: '*/main', description: 'Git revision to build') + } + + stages { + stage('Checkout') { + steps { + dir('llvm-project') { + checkout([$class: 'GitSCM', branches: [ + [name: params.GIT_REVISION] + ], extensions: [ + [$class: 'CloneOption', + reference: '/Users/Shared/llvm-project.git'] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-project.git'] + ]]) + } + dir('llvm-zorg') { + checkout([$class: 'GitSCM', branches: [ + [name: '*/main'] + ], extensions: [ + [$class: 'CloneOption', + reference: '/Users/Shared/llvm-zorg.git'] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-zorg.git'] + ]]) + } + } + } + stage('Build') { + steps { + timeout(120) { + sh ''' + set -u + rm -rf build.properties + + cd llvm-project + git tag -a -m "First Commit" first_commit 97724f18c79c7cc81ced24239eb5e883bf1398ef || true + + git_desc=$(git describe --match "first_commit") + + export GIT_DISTANCE=$(echo ${git_desc} | cut -f 2 -d "-") + + sha=$(echo ${git_desc} | cut -f 3 -d "-") + export GIT_SHA=${sha:1} + + # Also save the LLVM_REV until LNT server is taught about GIT + export LLVM_REV=$(git show -q | grep "llvm-svn:" | cut -f2 -d":" | tr -d " ") + + cd - + + export PATH=$PATH:/usr/bin:/usr/local/bin + + # Set a MacOS minimum deployment target + export MACOSX_DEPLOYMENT_TARGET=10.14 + + python llvm-zorg/zorg/jenkins/monorepo_build.py cmake build --assertions --projects="clang" + + # Removing the local artifact + rm -rf clang-*.tar.gz + ''' + } + } + } + stage('Test') { + steps { + timeout(120) { + sh ''' + set -u + export PATH=$PATH:/usr/bin:/usr/local/bin + + rm -rf clang-build/testresults.xunit.xml + + python llvm-zorg/zorg/jenkins/monorepo_build.py cmake testlong + ''' + } + junit 'clang-build/**/testresults.xunit.xml' + } + } + } + post { + always { + scanForIssues tool: clang() + } + /* Disabled email notification. + regression { + emailext subject: '$DEFAULT_SUBJECT', + presendScript: '$DEFAULT_PRESEND_SCRIPT', + postsendScript: '$DEFAULT_POSTSEND_SCRIPT', + recipientProviders: [ + [$class: 'CulpritsRecipientProvider'], + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'], + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS', + body:'$DEFAULT_CONTENT' + } + */ + } +} diff --git a/zorg/jenkins/jobs/jobs/clang-stage2-Rthinlto b/zorg/jenkins/jobs/jobs/clang-stage2-Rthinlto new file mode 100644 --- /dev/null +++ b/zorg/jenkins/jobs/jobs/clang-stage2-Rthinlto @@ -0,0 +1,103 @@ +pipeline { + agent { label 'green-dragon-22' } + + options { disableResume() } + + parameters { + string(name: 'GIT_SHA', defaultValue: '*/main', description: 'Git revision to build') + string(name: 'ARTIFACT', defaultValue: 'clang-stage1-RA/latest', description: 'description') + } + + stages { + stage('Checkout') { + steps { + dir('llvm-project') { + checkout([$class: 'GitSCM', branches: [ + [name: params.GIT_SHA] + ], extensions: [ + [$class: 'CloneOption', + reference: '/Users/Shared/llvm-project.git'] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-project.git'] + ]]) + } + dir('llvm-zorg') { + checkout([$class: 'GitSCM', branches: [ + [name: '*/main'] + ], extensions: [ + [$class: 'CloneOption', + reference: '/Users/Shared/llvm-zorg.git'] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-zorg.git'] + ]]) + } + } + } + stage('Build') { + steps { + timeout(600) { + sh ''' + set -u + rm -rf build.properties + + cd llvm-project + git tag -a -m "First Commit" first_commit 97724f18c79c7cc81ced24239eb5e883bf1398ef || true + + git_desc=$(git describe --match "first_commit") + export GIT_DISTANCE=$(echo ${git_desc} | cut -f 2 -d "-") + + sha=$(echo ${git_desc} | cut -f 3 -d "-") + export GIT_SHA=${sha:1} + + cd - + + echo "ARTIFACT=$JOB_NAME/clang-d$GIT_DISTANCE-g$GIT_SHA-t$BUILD_ID-b$BUILD_NUMBER.tar.gz" > build.properties + + export PATH=$PATH:/usr/bin:/usr/local/bin + + rm -rf clang-build clang-install *.tar.gz + + python llvm-zorg/zorg/jenkins/monorepo_build.py fetch + python llvm-zorg/zorg/jenkins/monorepo_build.py clang build \ + --thinlto --projects="clang;compiler-rt" \ + --cmake-flag="-DCMAKE_DSYMUTIL=$WORKSPACE/host-compiler/bin/dsymutil" + ''' + } + } + } + + stage('Test') { + steps { + timeout(240) { + sh ''' + set -u + export PATH=$PATH:/usr/bin:/usr/local/bin + + rm -rf clang-build/testresults.xunit.xml + + python llvm-zorg/zorg/jenkins/monorepo_build.py clang test + ''' + } + junit 'clang-build/**/testresults.xunit.xml' + } + } + } + post { + always { + scanForIssues tool: clang() + } + regression { + emailext subject: '$DEFAULT_SUBJECT', + presendScript: '$DEFAULT_PRESEND_SCRIPT', + postsendScript: '$DEFAULT_POSTSEND_SCRIPT', + recipientProviders: [ + [$class: 'CulpritsRecipientProvider'], + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'], + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS', + body:'$DEFAULT_CONTENT' + } + } +} diff --git a/zorg/jenkins/jobs/jobs/lldb-cmake b/zorg/jenkins/jobs/jobs/lldb-cmake deleted file mode 100644 --- a/zorg/jenkins/jobs/jobs/lldb-cmake +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env groovy -pipeline { - agent { label 'green-dragon-10' } - parameters { - string(name: 'GIT_REVISION', defaultValue: '*/master', description: 'Git revision to build') - string(name: 'ARTIFACT', defaultValue: 'clang-stage1-RA/latest', description: 'Compiler artifact to use for building the project') - string(name: 'BUILD_TYPE', defaultValue: 'Release', description: 'Default CMake build type; one of: Release, Debug, ...') - string(name: 'CLEAN', defaultValue: "false", description: 'Whether or not to clean the build directory before building') - } - stages { - stage('Checkout') { - steps { - timeout(30) { - dir('llvm-project') { - checkout([$class: 'GitSCM', branches: [[name: params.GIT_REVISION]], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-project.git']]]) - } - dir('llvm-zorg') { - checkout([$class: 'GitSCM', branches: [[name: '*/master']], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-zorg.git']]]) - } - } - } - } - stage('Fetch Host Compiler') { - steps { - timeout(10) { - sh ''' - python llvm-zorg/zorg/jenkins/monorepo_build.py fetch - ''' - } - } - } - stage('Build') { - steps { - timeout(120) { - sh ''' - set -u - rm -rf build.properties - - cd llvm-project - git tag -a -m "First Commit" first_commit 97724f18c79c7cc81ced24239eb5e883bf1398ef || true - - git_desc=$(git describe --match "first_commit") - - export GIT_DISTANCE=$(echo ${git_desc} | cut -f 2 -d "-") - - sha=$(echo ${git_desc} | cut -f 3 -d "-") - export GIT_SHA=${sha:1} - - cd - - - export MAX_PARALLEL_TESTS=$(sysctl hw.physicalcpu |awk '{print $2}') - python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake build \ - --assertions \ - --projects="clang;libcxx;libcxxabi;compiler-rt;lld;lldb;debuginfo-tests" \ - --compiler-flag="-Wdocumentation" \ - --cmake-type=Release || (echo "Retrying. debuginfo-tests may have failed due to high load."; sleep 30; python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake build \ - --assertions \ - --projects="clang;libcxx;libcxxabi;compiler-rt;lld;lldb;debuginfo-tests" \ - --compiler-flag="-Wdocumentation" \ - --cmake-type=Release) - ''' - } - } - } - stage('Test') { - steps { - timeout(90) { - sh ''' - set -u - export PATH=$PATH:/usr/bin:/usr/local/bin - export LLDB_CAPTURE_REPRODUCER=1 - - rm -rf test/results.xml - - export MAX_PARALLEL_TESTS=$(sysctl hw.physicalcpu |awk '{print $2}') - python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake testlong - ''' - } - } - } - } - post { - always { - scanForIssues tool: clang() - junit 'test/results.xml' - } - changed { - emailext subject: '$DEFAULT_SUBJECT', - presendScript: '$DEFAULT_PRESEND_SCRIPT', - postsendScript: '$DEFAULT_POSTSEND_SCRIPT', - recipientProviders: [ - [$class: 'CulpritsRecipientProvider'], - [$class: 'DevelopersRecipientProvider'], - [$class: 'RequesterRecipientProvider'], - ], - replyTo: '$DEFAULT_REPLYTO', - to: '$DEFAULT_RECIPIENTS', - body:'$DEFAULT_CONTENT' - } - } -} diff --git a/zorg/jenkins/jobs/jobs/lldb-cmake-as b/zorg/jenkins/jobs/jobs/lldb-cmake-as new file mode 100644 --- /dev/null +++ b/zorg/jenkins/jobs/jobs/lldb-cmake-as @@ -0,0 +1,162 @@ +#!/usr/bin/env groovy +pipeline { + agent { label 'green-dragon-13' } + parameters { + string(name: 'GIT_SHA', defaultValue: '*/main', description: 'Git revision to build') + string(name: 'ARTIFACT', defaultValue: 'clang-stage2-Rthinlto/latest', description: 'Compiler artifact to use for building the project') + string(name: 'BUILD_TYPE', defaultValue: 'Release', description: 'Default CMake build type; one of: Release, Debug, ...') + string(name: 'CLEAN', defaultValue: "false", description: 'Whether or not to clean the build directory before building') + } + stages { + stage('Checkout') { + steps { + timeout(30) { + dir('llvm-project') { + checkout([$class: 'GitSCM', branches: [ + [name: params.GIT_SHA] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-project.git'] + ], extensions: [ + [$class: 'CloneOption', + reference: '/Users/Shared/llvm-project.git', + noTags: true, timeout: 30] + ]]) + } + dir('llvm-zorg') { + checkout([$class: 'GitSCM', branches: [ + [name: '*/main'] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-zorg.git'] + ], extensions: [ + [$class: 'CloneOption', + reference: '/Users/Shared/llvm-zorg.git'] + ]]) + } + } + } + } + stage('Build') { + steps { + timeout(120) { + sh ''' + set -u + rm -rf build.properties + + cd llvm-project + git tag -a -m "First Commit" first_commit 97724f18c79c7cc81ced24239eb5e883bf1398ef || true + + git_desc=$(git describe --match "first_commit") + + export GIT_DISTANCE=$(echo ${git_desc} | cut -f 2 -d "-") + + sha=$(echo ${git_desc} | cut -f 3 -d "-") + export GIT_SHA=${sha:1} + + cd - + + if test -f /opt/codesign-workaround.sh; then sh /opt/codesign-workaround.sh init lldb-build/bin/debugserver; fi + export MAX_PARALLEL_TESTS=$(sysctl hw.physicalcpu |awk '{print $2}') + python3 llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake build \ + --assertions \ + --projects="clang;lld;lldb;cross-project-tests" \ + --runtimes="libcxx;libcxxabi;compiler-rt" \ + --compiler-flag="-Wdocumentation" \ + --cmake-type=Release || (echo "Retrying. debuginfo-tests may have failed due to high load."; sleep 30; python3 llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake build \ + --assertions \ + --projects="clang;lld;lldb;cross-project-tests" \ + --runtimes="libcxx;libcxxabi;compiler-rt" \ + --compiler-flag="-Wdocumentation" \ + --cmake-type=Release) + + if test -f /opt/codesign-workaround.sh; then sh /opt/codesign-workaround.sh sign lldb-build/bin/debugserver; fi + ''' + } + } + } + stage('Clean Test Results') { + steps { + timeout(10) { + sh ''' + set -u + export PATH=$PATH:/usr/bin:/usr/local/bin + + rm -rf test/results.xml + ''' + } + } + } + stage('Test (Debug Info)') { + steps { + timeout(30) { + // We want to continue with the rest of the pipeline even + // if the "Debug Info" tests fail, so catch the error but + // label this stage and the whole pipeline as FAILURE. + catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { + sh ''' + set -u + export PATH=$PATH:/usr/bin:/usr/local/bin + + export MAX_PARALLEL_TESTS=$(sysctl hw.physicalcpu |awk '{print $2}') + + python3 llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-debuginfo test + ''' + } + } + } + } + stage('Test (LLDB)') { + steps { + timeout(90) { + sh ''' + set -u + export PATH=$PATH:/usr/bin:/usr/local/bin + + export MAX_PARALLEL_TESTS=$(sysctl hw.physicalcpu |awk '{print $2}') + python3 llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake test + ''' + } + } + } + stage('Upload artifacts (LLDB)') { + steps { + timeout(90) { + sh ''' + set -u + cd llvm-project + git_desc=$(git describe --match "first_commit") + + export GIT_DISTANCE=$(echo ${git_desc} | cut -f 2 -d "-") + + sha=$(echo ${git_desc} | cut -f 3 -d "-") + export GIT_SHA=${sha:1} + + cd - + python3 llvm-zorg/zorg/jenkins/monorepo_build.py properties + ''' + } + } + } + } + post { + always { + scanForIssues tool: clang() + junit 'test/results.xml' + } + changed { + emailext subject: '$DEFAULT_SUBJECT', + presendScript: '$DEFAULT_PRESEND_SCRIPT', + postsendScript: '$DEFAULT_POSTSEND_SCRIPT', + recipientProviders: [ + [$class: 'CulpritsRecipientProvider'], + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'], + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS', + body:'$DEFAULT_CONTENT' + } + success { + build job: 'relay-lldb', wait: false + } + } +} diff --git a/zorg/jenkins/jobs/jobs/lldb-cmake-intel b/zorg/jenkins/jobs/jobs/lldb-cmake-intel new file mode 100644 --- /dev/null +++ b/zorg/jenkins/jobs/jobs/lldb-cmake-intel @@ -0,0 +1,173 @@ +#!/usr/bin/env groovy +pipeline { + agent { label 'green-dragon-03' } + parameters { + string(name: 'GIT_SHA', defaultValue: '*/main', description: 'Git revision to build') + string(name: 'ARTIFACT', defaultValue: 'clang-stage2-Rthinlto/latest', description: 'Compiler artifact to use for building the project') + string(name: 'BUILD_TYPE', defaultValue: 'Release', description: 'Default CMake build type; one of: Release, Debug, ...') + string(name: 'CLEAN', defaultValue: "false", description: 'Whether or not to clean the build directory before building') + } + stages { + stage('Checkout') { + steps { + timeout(30) { + dir('llvm-project') { + checkout([$class: 'GitSCM', branches: [ + [name: params.GIT_SHA] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-project.git'] + ], extensions: [ + [$class: 'CloneOption', + reference: '/Users/Shared/llvm-project.git', + noTags: true, timeout: 30] + ]]) + } + dir('llvm-zorg') { + checkout([$class: 'GitSCM', branches: [ + [name: '*/main'] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-zorg.git'] + ], extensions: [ + [$class: 'CloneOption', + reference: '/Users/Shared/llvm-zorg.git'] + ]]) + } + } + } + } + stage('Fetch Host Compiler') { + steps { + timeout(10) { + sh ''' + echo "Skipping geting the just-built compiler to work around an issue with libcxx." + echo python3 llvm-zorg/zorg/jenkins/monorepo_build.py fetch + rm -rf host-compiler + ''' + } + } + } + stage('Build') { + steps { + timeout(120) { + sh ''' + set -u + rm -rf build.properties + + cd llvm-project + git tag -a -m "First Commit" first_commit 97724f18c79c7cc81ced24239eb5e883bf1398ef || true + + git_desc=$(git describe --match "first_commit") + + export GIT_DISTANCE=$(echo ${git_desc} | cut -f 2 -d "-") + + sha=$(echo ${git_desc} | cut -f 3 -d "-") + export GIT_SHA=${sha:1} + + cd - + + if test -f /opt/codesign-workaround.sh; then sh /opt/codesign-workaround.sh init lldb-build/bin/debugserver; fi + export MAX_PARALLEL_TESTS=$(sysctl hw.physicalcpu |awk '{print $2}') + python3 llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake build \ + --assertions \ + --projects="clang;lld;lldb;cross-project-tests" \ + --runtimes="libcxx;libcxxabi;compiler-rt" \ + --compiler-flag="-Wdocumentation" \ + --cmake-type=Release || (echo "Retrying. debuginfo-tests may have failed due to high load."; sleep 30; python3 llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake build \ + --assertions \ + --projects="clang;lld;lldb;cross-project-tests" \ + --runtimes="libcxx;libcxxabi;compiler-rt" \ + --compiler-flag="-Wdocumentation" \ + --cmake-type=Release) + + if test -f /opt/codesign-workaround.sh; then sh /opt/codesign-workaround.sh sign lldb-build/bin/debugserver; fi + ''' + } + } + } + stage('Clean Test Results') { + steps { + timeout(10) { + sh ''' + set -u + export PATH=$PATH:/usr/bin:/usr/local/bin + + rm -rf test/results.xml + ''' + } + } + } + stage('Test (Debug Info)') { + steps { + timeout(30) { + // We want to continue with the rest of the pipeline even + // if the "Debug Info" tests fail, so catch the error but + // label this stage and the whole pipeline as FAILURE. + catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { + sh ''' + set -u + export PATH=$PATH:/usr/bin:/usr/local/bin + + export MAX_PARALLEL_TESTS=$(sysctl hw.physicalcpu |awk '{print $2}') + + python3 llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-debuginfo test + ''' + } + } + } + } + stage('Test (LLDB)') { + steps { + timeout(90) { + sh ''' + set -u + export PATH=$PATH:/usr/bin:/usr/local/bin + + export MAX_PARALLEL_TESTS=$(sysctl hw.physicalcpu |awk '{print $2}') + python3 llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake test + ''' + } + } + } + stage('Upload artifacts (LLDB)') { + steps { + timeout(90) { + sh ''' + set -u + cd llvm-project + git_desc=$(git describe --match "first_commit") + + export GIT_DISTANCE=$(echo ${git_desc} | cut -f 2 -d "-") + + sha=$(echo ${git_desc} | cut -f 3 -d "-") + export GIT_SHA=${sha:1} + + cd - + python3 llvm-zorg/zorg/jenkins/monorepo_build.py properties + ''' + } + } + } + } + post { + always { + scanForIssues tool: clang() + junit 'test/results.xml' + } + changed { + emailext subject: '$DEFAULT_SUBJECT', + presendScript: '$DEFAULT_PRESEND_SCRIPT', + postsendScript: '$DEFAULT_POSTSEND_SCRIPT', + recipientProviders: [ + [$class: 'CulpritsRecipientProvider'], + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'], + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS', + body:'$DEFAULT_CONTENT' + } + success { + build job: 'relay-lldb', wait: false + } + } +} diff --git a/zorg/jenkins/jobs/jobs/lldb-cmake-matrix b/zorg/jenkins/jobs/jobs/lldb-cmake-matrix --- a/zorg/jenkins/jobs/jobs/lldb-cmake-matrix +++ b/zorg/jenkins/jobs/jobs/lldb-cmake-matrix @@ -1,22 +1,40 @@ #!/usr/bin/env groovy pipeline { - agent { label 'green-dragon-23' } + agent { label 'green-dragon-09' } parameters { - string(name: 'GIT_REVISION', defaultValue: '*/master', description: 'Git revision to build') - string(name: 'ARTIFACT', defaultValue: 'clang-stage1-RA/latest', description: 'Compiler artifact to use for building the project') + string(name: 'GIT_SHA', defaultValue: '*/main', description: 'Git revision to build') + string(name: 'ARTIFACT', defaultValue: 'clang-stage2-Rthinlto/latest', description: 'Compiler artifact to use for building the project') string(name: 'BUILD_TYPE', defaultValue: 'Release', description: 'Default CMake build type; one of: Release, Debug, ...') string(name: 'CLEAN', defaultValue: "false", description: 'Whether or not to clean the build directory before building') } + environment { + MATRIX_COMMON_BUILD_PARAMETERS = '--cmake-flag=-DLLVM_TARGETS_TO_BUILD=X86' + } stages { stage('Checkout') { steps { timeout(30) { dir('llvm-project') { - checkout([$class: 'GitSCM', branches: [[name: params.GIT_REVISION]], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-project.git']]]) + checkout([$class: 'GitSCM', branches: [ + [name: params.GIT_SHA] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-project.git'] + ], extensions: [ + [$class: 'CloneOption', + reference: '/Users/Shared/llvm-project.git', + noTags: true, timeout: 30] + ]]) } dir('llvm-zorg') { - checkout([$class: 'GitSCM', branches: [[name: '*/master']], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-zorg.git']]]) + checkout([$class: 'GitSCM', branches: [ + [name: '*/main'] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-zorg.git'] + ], extensions: [ + [$class: 'CloneOption', + reference: '/Users/Shared/llvm-zorg.git'] + ]]) } } } @@ -25,7 +43,13 @@ steps { timeout(10) { sh ''' - python llvm-zorg/zorg/jenkins/monorepo_build.py fetch + # The relay job sets ARTIFACT to the last good LLDB build. + # Here we use it to download the last good Clang. + export ARTIFACT=clang-stage2-Rthinlto/latest + python3 llvm-zorg/zorg/jenkins/monorepo_build.py fetch + + echo "Skipping geting the just-built compiler to work around an issue with libcxx." + rm -rf host-compiler ''' } } @@ -50,14 +74,18 @@ cd - export PATH=$PATH:/usr/bin:/usr/local/bin + if test -f /opt/codesign-workaround.sh; then sh /opt/codesign-workaround.sh init lldb-build/bin/debugserver; fi - python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake build \ + python3 llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake build \ + $MATRIX_COMMON_BUILD_PARAMETERS \ --assertions \ - --projects="clang;libcxx;libcxxabi;lldb" \ + --projects="clang;lldb" \ + --runtimes="libcxx;libcxxabi" \ --cmake-type=Release \ --dotest-flag="--skip-category gmodules" \ - --dotest-flag="--arch=x86_64" \ - --cmake-flag="-DLLVM_TARGETS_TO_BUILD=X86" + --dotest-flag="--arch=x86_64" + + if test -f /opt/codesign-workaround.sh; then sh /opt/codesign-workaround.sh sign lldb-build/bin/debugserver; fi ''' } } @@ -67,19 +95,28 @@ timeout(60) { sh ''' export PATH=$PATH:/usr/bin:/usr/local/bin - python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix configure \ + python3 llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix configure \ + $MATRIX_COMMON_BUILD_PARAMETERS \ --assertions \ - --projects="clang;libcxx;libcxxabi;lldb" \ + --lldb-test-compiler="$WORKSPACE/lldb-build/bin/clang" \ + --projects="clang;lldb" \ + --runtimes="libcxx;libcxxabi" \ --cmake-type=Release \ --dotest-flag="--skip-category" \ --dotest-flag="gmodules" \ --dotest-flag="--skip-category" \ --dotest-flag="watchpoint" \ + --dotest-flag="--skip-category" \ + --dotest-flag="llgs" \ + --dotest-flag="--skip-category" \ + --dotest-flag="debugserver" \ --dotest-flag="--arch=x86_64" \ --dotest-flag="--dwarf-version=2" set +e - python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix test + # Give the system some time to recover. + sleep 120 + python3 llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix test set -e ''' } @@ -91,19 +128,28 @@ timeout(60) { sh ''' export PATH=$PATH:/usr/bin:/usr/local/bin - python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix configure \ + python3 llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix configure \ + $MATRIX_COMMON_BUILD_PARAMETERS \ + --lldb-test-compiler="$WORKSPACE/lldb-build/bin/clang" \ --assertions \ - --projects="clang;libcxx;libcxxabi;lldb" \ + --projects="clang;lldb" \ + --runtimes="libcxx;libcxxabi" \ --cmake-type=Release \ --dotest-flag="--skip-category" \ --dotest-flag="gmodules" \ --dotest-flag="--skip-category" \ --dotest-flag="watchpoint" \ + --dotest-flag="--skip-category" \ + --dotest-flag="llgs" \ + --dotest-flag="--skip-category" \ + --dotest-flag="debugserver" \ --dotest-flag="--arch=x86_64" \ --dotest-flag="--dwarf-version=4" set +e - python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix test + # Give the system some time to recover. + sleep 120 + python3 llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix test set -e ''' } @@ -115,38 +161,189 @@ timeout(60) { sh ''' export PATH=$PATH:/usr/bin:/usr/local/bin - python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix configure \ + python3 llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix configure \ + $MATRIX_COMMON_BUILD_PARAMETERS \ + --lldb-test-compiler="$WORKSPACE/lldb-build/bin/clang" \ --assertions \ - --projects="clang;libcxx;libcxxabi;lldb" \ + --projects="clang;lldb" \ + --runtimes="libcxx;libcxxabi" \ --cmake-type=Release \ --dotest-flag="--skip-category" \ --dotest-flag="gmodules" \ --dotest-flag="--skip-category" \ --dotest-flag="watchpoint" \ + --dotest-flag="--skip-category" \ + --dotest-flag="llgs" \ + --dotest-flag="--skip-category" \ + --dotest-flag="debugserver" \ --dotest-flag="--arch=x86_64" \ --dotest-flag="--dwarf-version=5" set +e - python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix test + # Give the system some time to recover. + sleep 120 + echo "****************************************************************" + echo "** DWARF 5 tests are temporarily being skipped because the many test failures result in extremely large log output." + echo "****************************************************************" + echo python3 llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix test + set -e + ''' + } + junit 'test/results.xml' + } + } + stage('Build Clang 7.0.1') { + steps { + timeout(30) { + dir('clang_701') { + checkout([$class: 'GitSCM', branches: [ + [name: "llvmorg-7.0.1"] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-project.git'] + ], extensions: [ + [$class: 'CloneOption', timeout: 30, + reference: '/Users/Shared/llvm-project.git'] + ]]) + } + } + timeout(90) { + sh ''' + export PATH=$PATH:/usr/bin:/usr/local/bin + export SRC_DIR='clang_701' + export BUILD_DIR='clang_701_build' + + python3 llvm-zorg/zorg/jenkins/monorepo_build.py cmake build \ + $MATRIX_COMMON_BUILD_PARAMETERS \ + --assertions \ + --noupload \ + --noinstall \ + --projects="clang;libcxx;libcxxabi" + ''' + } + } + } + stage('Test Clang 7.0.1') { + steps { + timeout(60) { + sh ''' + build_dir=$WORKSPACE/clang_701_build + export PATH=$PATH:/usr/bin:/usr/local/bin + python3 llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix configure \ + $MATRIX_COMMON_BUILD_PARAMETERS \ + --assertions \ + --lldb-test-compiler="$build_dir/bin/clang" \ + --projects="clang;lldb" \ + --runtimes="libcxx;libcxxabi" \ + --cmake-type=Release \ + --dotest-flag="--libcxx-include-dir=$build_dir/include/c++/v1/" \ + --dotest-flag="--libcxx-library-dir=$build_dir/lib/" \ + --dotest-flag="--skip-category" \ + --dotest-flag="gmodules" \ + --dotest-flag="--skip-category" \ + --dotest-flag="watchpoint" \ + --dotest-flag="--skip-category" \ + --dotest-flag="llgs" \ + --dotest-flag="--skip-category" \ + --dotest-flag="debugserver" \ + --dotest-flag="--arch=x86_64" + + set +e + # Give the system some time to recover. + sleep 120 + python3 llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix test + set -e + ''' + } + junit 'test/results.xml' + } + } + stage('Build Clang 9.0.0') { + steps { + timeout(30) { + dir('clang_900') { + checkout([$class: 'GitSCM', branches: [ + [name: "llvmorg-9.0.0"] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-project.git'] + ], extensions: [ + [$class: 'CloneOption', timeout: 30, + reference: '/Users/Shared/llvm-project.git'] + ]]) + } + } + timeout(90) { + sh ''' + export PATH=$PATH:/usr/bin:/usr/local/bin + export SRC_DIR='clang_900' + export BUILD_DIR='clang_900_build' + + python3 llvm-zorg/zorg/jenkins/monorepo_build.py cmake build \ + $MATRIX_COMMON_BUILD_PARAMETERS \ + --assertions \ + --noupload \ + --noinstall \ + --projects="clang;libcxx;libcxxabi" + ''' + } + } + } + stage('Test Clang 9.0.0') { + steps { + timeout(60) { + sh ''' + build_dir=$WORKSPACE/clang_900_build + export PATH=$PATH:/usr/bin:/usr/local/bin + python3 llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix configure \ + $MATRIX_COMMON_BUILD_PARAMETERS \ + --assertions \ + --lldb-test-compiler="$build_dir/bin/clang" \ + --projects="clang;lldb" \ + --runtimes="libcxx;libcxxabi" \ + --cmake-type=Release \ + --dotest-flag="--libcxx-include-dir=$build_dir/include/c++/v1/" \ + --dotest-flag="--libcxx-library-dir=$build_dir/lib/" \ + --dotest-flag="--skip-category" \ + --dotest-flag="gmodules" \ + --dotest-flag="--skip-category" \ + --dotest-flag="watchpoint" \ + --dotest-flag="--skip-category" \ + --dotest-flag="llgs" \ + --dotest-flag="--skip-category" \ + --dotest-flag="debugserver" \ + --dotest-flag="--arch=x86_64" + + set +e + # Give the system some time to recover. + sleep 120 + python3 llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix test set -e ''' } + junit 'test/results.xml' } } - stage('Build Clang 5.0.2') { + stage('Build Clang 11.0.1') { steps { timeout(30) { - dir('clang_502') { - checkout([$class: 'GitSCM', branches: [[name: "llvmorg-5.0.2"]], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-project.git']]]) + dir('clang_1101') { + checkout([$class: 'GitSCM', branches: [ + [name: "llvmorg-11.0.1"] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-project.git'] + ], extensions: [ + [$class: 'CloneOption', timeout: 30, + reference: '/Users/Shared/llvm-project.git'] + ]]) } } timeout(90) { sh ''' export PATH=$PATH:/usr/bin:/usr/local/bin - export SRC_DIR='clang_502' - export BUILD_DIR='clang_502_build' + export SRC_DIR='clang_1101' + export BUILD_DIR='clang_1101_build' - python llvm-zorg/zorg/jenkins/monorepo_build.py cmake build \ + python3 llvm-zorg/zorg/jenkins/monorepo_build.py cmake build \ + $MATRIX_COMMON_BUILD_PARAMETERS \ --assertions \ --noupload \ --noinstall \ @@ -155,44 +352,63 @@ } } } - stage('Test Clang 5.0.2') { + stage('Test Clang 11.0.1') { steps { timeout(60) { sh ''' + build_dir=$WORKSPACE/clang_1101_build export PATH=$PATH:/usr/bin:/usr/local/bin - export LLDB_TEST_COMPILER="$WORKSPACE/clang_502_build/bin/clang" - python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix configure \ + python3 llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix configure \ + $MATRIX_COMMON_BUILD_PARAMETERS \ --assertions \ - --projects="clang;libcxx;libcxxabi;lldb" \ + --lldb-test-compiler="$build_dir/bin/clang" \ + --projects="clang;lldb" \ + --runtimes="libcxx;libcxxabi" \ --cmake-type=Release \ + --dotest-flag="--libcxx-include-dir=$build_dir/include/c++/v1/" \ + --dotest-flag="--libcxx-library-dir=$build_dir/lib/" \ --dotest-flag="--skip-category" \ --dotest-flag="gmodules" \ --dotest-flag="--skip-category" \ --dotest-flag="watchpoint" \ + --dotest-flag="--skip-category" \ + --dotest-flag="llgs" \ + --dotest-flag="--skip-category" \ + --dotest-flag="debugserver" \ --dotest-flag="--arch=x86_64" set +e - python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix test + # Give the system some time to recover. + sleep 120 + python3 llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix test set -e ''' } junit 'test/results.xml' } } - stage('Build Clang 7.0.1') { + stage('Build Clang 13.0.0') { steps { timeout(30) { - dir('clang_701') { - checkout([$class: 'GitSCM', branches: [[name: "llvmorg-7.0.1"]], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-project.git']]]) + dir('clang_1300') { + checkout([$class: 'GitSCM', branches: [ + [name: "llvmorg-13.0.0"] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-project.git'] + ], extensions: [ + [$class: 'CloneOption', timeout: 30, + reference: '/Users/Shared/llvm-project.git'] + ]]) } } timeout(90) { sh ''' export PATH=$PATH:/usr/bin:/usr/local/bin - export SRC_DIR='clang_701' - export BUILD_DIR='clang_701_build' + export SRC_DIR='clang_1300' + export BUILD_DIR='clang_1300_build' - python llvm-zorg/zorg/jenkins/monorepo_build.py cmake build \ + python3 llvm-zorg/zorg/jenkins/monorepo_build.py cmake build \ + $MATRIX_COMMON_BUILD_PARAMETERS \ --assertions \ --noupload \ --noinstall \ @@ -201,44 +417,63 @@ } } } - stage('Test Clang 7.0.1') { + stage('Test Clang 13.0.0') { steps { timeout(60) { sh ''' + build_dir=$WORKSPACE/clang_1300_build export PATH=$PATH:/usr/bin:/usr/local/bin - export LLDB_TEST_COMPILER="$WORKSPACE/clang_701_build/bin/clang" - python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix configure \ + python3 llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix configure \ + $MATRIX_COMMON_BUILD_PARAMETERS \ --assertions \ - --projects="clang;libcxx;libcxxabi;lldb" \ + --lldb-test-compiler="$build_dir/bin/clang" \ + --projects="clang;lldb" \ + --runtimes="libcxx;libcxxabi" \ --cmake-type=Release \ + --dotest-flag="--libcxx-include-dir=$build_dir/include/c++/v1/" \ + --dotest-flag="--libcxx-library-dir=$build_dir/lib/" \ --dotest-flag="--skip-category" \ --dotest-flag="gmodules" \ --dotest-flag="--skip-category" \ --dotest-flag="watchpoint" \ + --dotest-flag="--skip-category" \ + --dotest-flag="llgs" \ + --dotest-flag="--skip-category" \ + --dotest-flag="debugserver" \ --dotest-flag="--arch=x86_64" set +e - python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix test + # Give the system some time to recover. + sleep 120 + python3 llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix test set -e ''' } junit 'test/results.xml' } } - stage('Build Clang 9.0.0') { + stage('Build Clang 15.0.1') { steps { timeout(30) { - dir('clang_900') { - checkout([$class: 'GitSCM', branches: [[name: "llvmorg-9.0.0"]], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-project.git']]]) + dir('clang_1501') { + checkout([$class: 'GitSCM', branches: [ + [name: "llvmorg-15.0.1"] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-project.git'] + ], extensions: [ + [$class: 'CloneOption', timeout: 30, + reference: '/Users/Shared/llvm-project.git'] + ]]) } } timeout(90) { sh ''' export PATH=$PATH:/usr/bin:/usr/local/bin - export SRC_DIR='clang_900' - export BUILD_DIR='clang_900_build' + export SRC_DIR='clang_1501' + export BUILD_DIR='clang_1501_build' - python llvm-zorg/zorg/jenkins/monorepo_build.py cmake build \ + python3 llvm-zorg/zorg/jenkins/monorepo_build.py cmake build \ + $MATRIX_COMMON_BUILD_PARAMETERS \ --assertions \ --noupload \ --noinstall \ @@ -247,24 +482,35 @@ } } } - stage('Test Clang 9.0.0') { + stage('Test Clang 15.0.1') { steps { timeout(60) { sh ''' + build_dir=$WORKSPACE/clang_1501_build export PATH=$PATH:/usr/bin:/usr/local/bin - export LLDB_TEST_COMPILER="$WORKSPACE/clang_900_build/bin/clang" - python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix configure \ + python3 llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix configure \ + $MATRIX_COMMON_BUILD_PARAMETERS \ --assertions \ - --projects="clang;libcxx;libcxxabi;lldb" \ + --lldb-test-compiler="$build_dir/bin/clang" \ + --projects="clang;lldb" \ + --runtimes="libcxx;libcxxabi" \ --cmake-type=Release \ + --dotest-flag="--libcxx-include-dir=$build_dir/include/c++/v1/" \ + --dotest-flag="--libcxx-library-dir=$build_dir/lib/" \ --dotest-flag="--skip-category" \ --dotest-flag="gmodules" \ --dotest-flag="--skip-category" \ --dotest-flag="watchpoint" \ + --dotest-flag="--skip-category" \ + --dotest-flag="llgs" \ + --dotest-flag="--skip-category" \ + --dotest-flag="debugserver" \ --dotest-flag="--arch=x86_64" set +e - python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix test + # Give the system some time to recover. + sleep 120 + python3 llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix test set -e ''' } diff --git a/zorg/jenkins/jobs/jobs/lldb-cmake-reproducers b/zorg/jenkins/jobs/jobs/lldb-cmake-reproducers deleted file mode 100644 --- a/zorg/jenkins/jobs/jobs/lldb-cmake-reproducers +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env groovy -pipeline { - agent { label 'green-dragon-23' } - parameters { - string(name: 'GIT_REVISION', defaultValue: '*/master', description: 'Git revision to build') - string(name: 'ARTIFACT', defaultValue: 'clang-stage1-RA/latest', description: 'Compiler artifact to use for building the project') - string(name: 'BUILD_TYPE', defaultValue: 'Release', description: 'Default CMake build type; one of: Release, Debug, ...') - string(name: 'CLEAN', defaultValue: "false", description: 'Whether or not to clean the build directory before building') - } - stages { - stage('Checkout') { - steps { - timeout(30) { - dir('llvm-project') { - checkout([$class: 'GitSCM', branches: [[name: params.GIT_REVISION]], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-project.git']]]) - } - dir('llvm-zorg') { - checkout([$class: 'GitSCM', branches: [[name: '*/master']], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-zorg.git']]]) - } - } - } - } - stage('Fetch Host Compiler') { - steps { - timeout(10) { - sh ''' - python llvm-zorg/zorg/jenkins/monorepo_build.py fetch - ''' - } - } - } - stage('Build') { - steps { - timeout(90) { - sh ''' - set -u - rm -rf build.properties - - cd llvm-project - git tag -a -m "First Commit" first_commit 97724f18c79c7cc81ced24239eb5e883bf1398ef || true - - git_desc=$(git describe --match "first_commit") - - export GIT_DISTANCE=$(echo ${git_desc} | cut -f 2 -d "-") - - sha=$(echo ${git_desc} | cut -f 3 -d "-") - export GIT_SHA=${sha:1} - - cd - - - python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake build \ - --assertions \ - --projects="clang;libcxx;libcxxabi;compiler-rt;lld;lldb;debuginfo-tests" \ - --compiler-flag="-Wdocumentation" \ - --cmake-type=Release - ''' - } - } - } - stage('Test Reproducers') { - steps { - timeout(120) { - sh ''' - set -u - export PATH=$PATH:/usr/bin:/usr/local/bin - rm -rf test/results.xml - python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake \ - --cmake-test-target check-lldb-reproducers - ''' - } - } - } - } - post { - always { - scanForIssues tool: clang() - junit 'test/results.xml' - } - } -} diff --git a/zorg/jenkins/jobs/jobs/lldb-cmake-sanitized b/zorg/jenkins/jobs/jobs/lldb-cmake-sanitized --- a/zorg/jenkins/jobs/jobs/lldb-cmake-sanitized +++ b/zorg/jenkins/jobs/jobs/lldb-cmake-sanitized @@ -1,9 +1,9 @@ #!/usr/bin/env groovy pipeline { - agent { label 'green-dragon-23' } + agent { label 'green-dragon-09' } parameters { - string(name: 'GIT_REVISION', defaultValue: '*/master', description: 'Git revision to build') - string(name: 'ARTIFACT', defaultValue: 'clang-stage1-RA/latest', description: 'Compiler artifact to use for building the project') + string(name: 'GIT_SHA', defaultValue: '*/main', description: 'Git revision to build') + string(name: 'ARTIFACT', defaultValue: 'clang-stage2-Rthinlto/latest', description: 'Compiler artifact to use for building the project') string(name: 'BUILD_TYPE', defaultValue: 'Release', description: 'Default CMake build type; one of: Release, Debug, ...') string(name: 'CLEAN', defaultValue: "false", description: 'Whether or not to clean the build directory before building') } @@ -12,10 +12,25 @@ steps { timeout(30) { dir('llvm-project') { - checkout([$class: 'GitSCM', branches: [[name: params.GIT_REVISION]], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-project.git']]]) + checkout([$class: 'GitSCM', branches: [ + [name: params.GIT_SHA] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-project.git'] + ], extensions: [ + [$class: 'CloneOption', + reference: '/Users/Shared/llvm-project.git', + noTags: true, timeout: 30] + ]]) } dir('llvm-zorg') { - checkout([$class: 'GitSCM', branches: [[name: '*/master']], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-zorg.git']]]) + checkout([$class: 'GitSCM', branches: [ + [name: '*/main'] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-zorg.git'] + ], extensions: [ + [$class: 'CloneOption', + reference: '/Users/Shared/llvm-zorg.git'] + ]]) } } } @@ -24,14 +39,20 @@ steps { timeout(10) { sh ''' - python llvm-zorg/zorg/jenkins/monorepo_build.py fetch + # The relay job sets ARTIFACT to the last good LLDB build. + # Here we use it to download the last good Clang. + export ARTIFACT=clang-stage2-Rthinlto/latest + python3 llvm-zorg/zorg/jenkins/monorepo_build.py fetch + + echo "Skipping geting the just-built compiler to work around an issue with libcxx." + rm -rf host-compiler ''' } } } stage('Build') { steps { - timeout(120) { + timeout(240) { sh ''' set -u rm -rf build.properties @@ -50,27 +71,31 @@ export PATH=$PATH:/usr/bin:/usr/local/bin + if test -f /opt/codesign-workaround.sh; then sh /opt/codesign-workaround.sh init lldb-build/bin/debugserver; fi # Running too many asanified threads is too stressful for the kernel and we get >90% system time. export MAX_PARALLEL_TESTS=$(sysctl hw.physicalcpu |awk '{print int(($2+1)/2)}') - python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-sanitized build \ + python3 llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-sanitized build \ --assertions \ - --projects="clang;libcxx;libcxxabi;lld;lldb" \ + --projects="clang;lld;lldb" \ + --runtimes="libcxx;libcxxabi" \ --cmake-type=Release + + if test -f /opt/codesign-workaround.sh; then sh /opt/codesign-workaround.sh sign lldb-build/bin/debugserver; fi ''' } } } stage('Test') { steps { - timeout(120) { + timeout(240) { sh ''' set -u export PATH=$PATH:/usr/bin:/usr/local/bin rm -rf test/results.xml - python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-sanitized test + python3 llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-sanitized test ''' } } diff --git a/zorg/jenkins/jobs/jobs/lldb-cmake-standalone b/zorg/jenkins/jobs/jobs/lldb-cmake-standalone --- a/zorg/jenkins/jobs/jobs/lldb-cmake-standalone +++ b/zorg/jenkins/jobs/jobs/lldb-cmake-standalone @@ -2,8 +2,8 @@ pipeline { agent {label 'green-dragon-23'} parameters { - string(name: 'GIT_REVISION', defaultValue: '*/master', description: 'Git revision to build') - string(name: 'ARTIFACT', defaultValue: 'clang-stage1-RA/latest', description: 'Compiler artifact to use for building the project') + string(name: 'GIT_SHA', defaultValue: '*/main', description: 'Git revision to build') + string(name: 'ARTIFACT', defaultValue: 'clang-stage2-Rthinlto/latest', description: 'Compiler artifact to use for building the project') string(name: 'BUILD_TYPE', defaultValue: 'Release', description: 'Default CMake build type; one of: Release, Debug, ...') string(name: 'CLEAN', defaultValue: "false", description: 'Whether or not to clean the build directory before building') } @@ -12,10 +12,25 @@ steps { timeout(30) { dir('llvm-project') { - checkout([$class: 'GitSCM', branches: [[name: params.GIT_REVISION]], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-project.git']]]) + checkout([$class: 'GitSCM', branches: [ + [name: params.GIT_SHA] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-project.git'] + ], extensions: [ + [$class: 'CloneOption', + reference: '/Users/Shared/llvm-project.git', + noTags: true, timeout: 30] + ]]) } dir('llvm-zorg') { - checkout([$class: 'GitSCM', branches: [[name: '*/master']], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-zorg.git']]]) + checkout([$class: 'GitSCM', branches: [ + [name: '*/main'] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-zorg.git'] + ], extensions: [ + [$class: 'CloneOption', + reference: '/Users/Shared/llvm-zorg.git'] + ]]) } } } @@ -24,7 +39,13 @@ steps { timeout(90) { sh ''' - python llvm-zorg/zorg/jenkins/monorepo_build.py fetch + # The relay job sets ARTIFACT to the last good LLDB build. + # Here we use it to download the last good Clang. + export ARTIFACT=clang-stage2-Rthinlto/latest + python3 llvm-zorg/zorg/jenkins/monorepo_build.py fetch + + echo "Skipping geting the just-built compiler to work around an issue with libcxx." + rm -rf host-compiler ''' } } @@ -50,10 +71,11 @@ export PATH=$PATH:/usr/bin:/usr/local/bin - python llvm-zorg/zorg/jenkins/monorepo_build.py cmake build \ + python3 llvm-zorg/zorg/jenkins/monorepo_build.py cmake build \ --assertions \ --noupload \ - --projects="clang;libcxx;libcxxabi" \ + --projects="clang" \ + --runtimes="libcxx;libcxxabi" \ --cmake-flag="-DLLVM_INSTALL_UTILS=On" \ --cmake-flag="-DLLVM_INSTALL_TOOLCHAIN_ONLY=Off" \ --cmake-flag="-DLLVM_TARGETS_TO_BUILD=X86" @@ -83,7 +105,7 @@ export PATH=$PATH:/usr/bin:/usr/local/bin export LLDB_STANDALONE_TYPE="build-tree" - python $WORKSPACE/llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-standalone build \ + python3 $WORKSPACE/llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-standalone build \ --assertions \ --cmake-type=Release ''' @@ -112,7 +134,7 @@ export PATH=$PATH:/usr/bin:/usr/local/bin export LLDB_STANDALONE_TYPE="install-tree" - python $WORKSPACE/llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-standalone build \ + python3 $WORKSPACE/llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-standalone build \ --assertions \ --cmake-type=Release ''' @@ -140,7 +162,7 @@ export PATH=$PATH:/usr/bin:/usr/local/bin - python $WORKSPACE/llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-xcode build \ + python3 $WORKSPACE/llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-xcode build \ --assertions \ --cmake-type=Release ''' @@ -154,7 +176,7 @@ set -u export PATH=$PATH:/usr/bin:/usr/local/bin - python $WORKSPACE/llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-xcode test \ + echo "TEMPORARILY DISABLED (runs with too many jobs in parallel)" python3 $WORKSPACE/llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-xcode test \ --assertions \ --cmake-type=Release ''' diff --git a/zorg/jenkins/jobs/jobs/llvm-coverage b/zorg/jenkins/jobs/jobs/llvm-coverage new file mode 100755 --- /dev/null +++ b/zorg/jenkins/jobs/jobs/llvm-coverage @@ -0,0 +1,155 @@ +#!/usr/bin/env groovy +pipeline { + agent { label 'green-dragon-23' } + + parameters { + string(name: 'GIT_REVISION', defaultValue: '*/main', description: 'Git revision to build') + string(name: 'ARTIFACT', defaultValue: 'clang-stage1-RA/latest', description: 'Compiler artifact to use for building the project') + string(name: 'BUILD_TYPE', defaultValue: 'Release', description: 'Default CMake build type; one of: Release, Debug, ...') + string(name: 'CLEAN', defaultValue: "false", description: 'Whether or not to clean the build directory before building') + } + + stages { + stage('Checkout') { + steps { + dir('llvm-project') { + checkout([$class: 'GitSCM', branches: [ + [name: params.GIT_REVISION] + ], extensions: [ + [$class: 'CloneOption', + reference: '/Users/Shared/llvm-project.git'] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-project.git'] + ]]) + } + dir('llvm-zorg') { + checkout([$class: 'GitSCM', branches: [ + [name: '*/main'] + ], extensions: [ + [$class: 'CloneOption', + reference: '/Users/Shared/llvm-zorg.git'] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-zorg.git'] + ]]) + } + } + } + stage('Fetch') { + steps { + timeout(10) { + sh ''' + python llvm-zorg/zorg/jenkins/monorepo_build.py fetch + ''' + } + } + } + stage('Build') { + steps { + timeout(180) { + sh ''' + set -u + rm -rf build.properties + + cd llvm-project + git tag -a -m "First Commit" first_commit 97724f18c79c7cc81ced24239eb5e883bf1398ef || true + + git_desc=$(git describe --match "first_commit") + + export GIT_DISTANCE=$(echo ${git_desc} | cut -f 2 -d "-") + + sha=$(echo ${git_desc} | cut -f 3 -d "-") + export GIT_SHA=${sha:1} + + cd - + + export PATH=$PATH:/usr/bin:/usr/local/bin + + python llvm-zorg/zorg/jenkins/monorepo_build.py cmake build \ + --cmake-build-target FileCheck \ + --cmake-build-target count \ + --cmake-build-target not \ + --cmake-flag="-DLLVM_TARGETS_TO_BUILD=host" \ + --cmake-type=Release \ + --projects "" \ + --noinstall \ + --noupload + + python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake build \ + --assertions \ + --projects="clang;lldb;cross-project-tests" \ + --runtimes="libcxx;libcxxabi" \ + --cmake-flag="-DPYTHON_LIBRARY=/usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/lib/libpython3.7.dylib" \ + --cmake-flag="-DPYTHON_INCLUDE_DIR=/usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/include/python3.7m" \ + --cmake-flag="-DPYTHON_LIBRARY_DEBUG=/usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/lib/libpython3.7.dylib" \ + --cmake-flag="-DPYTHON_EXECUTABLE=/usr/local/Cellar/python/3.7.0/bin/python3.7" \ + --cmake-flag="-DLLDB_USE_SYSTEM_DEBUGSERVER=ON" \ + --cmake-flag="-DLLVM_BUILD_INSTRUMENTED_COVERAGE=ON" \ + --cmake-flag="-DLLVM_PROFILE_MERGE_POOL_SIZE=1" \ + --cmake-flag="-DLLVM_ENABLE_MODULES=Off" \ + --cmake-flag="-DLLVM_TARGETS_TO_BUILD=X86;ARM;AArch64;AMDGPU" \ + --cmake-type=Release \ + --noupload + ''' + } + } + } + stage('Test') { + steps { + timeout(480) { + sh ''' + set -u + export PATH=$PATH:/usr/bin:/usr/local/bin + + rm -rf test/results.xml + + BUILD_DIR=$(pwd)/lldb-build + FAST_BUILD_DIR=$(pwd)/clang-build + REPORT_DIR=$(pwd)/coverage-reports + HOST=$(pwd)/host-compiler/bin + LLVM_PROFDATA=$HOST/llvm-profdata + LLVM_COV=$HOST/llvm-cov + ARTIFACT_PREP_SCRIPT=$WORKSPACE/llvm-project/llvm/utils/prepare-code-coverage-artifact.py + + FAST_TOOLS=(FileCheck count not) + for TOOL in ${FAST_TOOLS[@]}; do + cp $FAST_BUILD_DIR/bin/$TOOL $BUILD_DIR/bin/$TOOL + done + + # Clear out any stale profiles. + rm -rf $BUILD_DIR/profiles + + # Run the tests. + IGNORE_ERRORS_OVERRIDE=1 python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake testlong || echo "Some tests may have failed." + + cd $BUILD_DIR + ninja -k 0 check-llvm check-clang || echo "Some tests may have failed." + cd - + + # Temporarily exclude debugserver from the reports. + # It is a universal binary and the coverage report script + # doesn't yet know how to handle it. + rm -rf $BUILD_DIR/bin/debugserver + + COV_BINARIES=$(find $BUILD_DIR/bin $BUILD_DIR/lib -depth 1 -type f -exec file {} \\; | grep Mach-O | cut -d':' -f1 | grep -vE '/(FileCheck|count|not)$' | xargs) + + rm -rf $REPORT_DIR + mkdir -p $REPORT_DIR + python $ARTIFACT_PREP_SCRIPT $LLVM_PROFDATA $LLVM_COV $BUILD_DIR/profiles $REPORT_DIR $COV_BINARIES --unified-report --restrict $WORKSPACE/llvm-project + + scp -r $REPORT_DIR buildslave@labmaster2.local:/Library/WebServer/Documents/coverage + ssh buildslave@labmaster2.local "chmod -R 777 /Library/WebServer/Documents/coverage" + + rm -rf lldb-build/profiles + ''' + } + } + } + } + + post { + always { + scanForIssues tool: clang() + junit 'test/results.xml' + } + } +} diff --git a/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-O0-g b/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-O0-g --- a/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-O0-g +++ b/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-O0-g @@ -1,9 +1,11 @@ #!/usr/bin/env groovy def common = evaluate readTrusted('zorg/jenkins/common.groovy') -common.testsuite_pipeline(label: 'green-dragon-12') { - sh """ +common.testsuite_pipeline(label: 'green-dragon-11') { + timeout(30) { + sh """ LNT_FLAGS+=" -C config/tasks/cmake/caches/target-arm64-iphoneos.cmake" LNT_FLAGS+=" -C config/tasks/cmake/caches/opt-O0-g.cmake" config/tasks/task jenkinsrun config/tasks/lnt-ctmark.sh -a compiler="${params.ARTIFACT}" -D LNT_FLAGS="\${LNT_FLAGS}" - """ + """ + } } diff --git a/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-O3-flto b/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-O3-flto --- a/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-O3-flto +++ b/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-O3-flto @@ -1,9 +1,11 @@ #!/usr/bin/env groovy def common = evaluate readTrusted('zorg/jenkins/common.groovy') -common.testsuite_pipeline(label: 'green-dragon-12') { - sh """ +common.testsuite_pipeline(label: 'green-dragon-11') { + timeout(30) { + sh """ LNT_FLAGS+=" -C config/tasks/cmake/caches/target-arm64-iphoneos.cmake" LNT_FLAGS+=" -C config/tasks/cmake/caches/opt-O3-flto.cmake" config/tasks/task jenkinsrun config/tasks/lnt-ctmark.sh -a compiler="${params.ARTIFACT}" -D LNT_FLAGS="\${LNT_FLAGS}" - """ + """ + } } diff --git a/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-Os b/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-Os --- a/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-Os +++ b/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-Os @@ -1,9 +1,11 @@ #!/usr/bin/env groovy def common = evaluate readTrusted('zorg/jenkins/common.groovy') -common.testsuite_pipeline(label: 'green-dragon-12') { - sh """ +common.testsuite_pipeline(label: 'green-dragon-11') { + timeout(30) { + sh """ LNT_FLAGS+=" -C config/tasks/cmake/caches/target-arm64-iphoneos.cmake" LNT_FLAGS+=" -C config/tasks/cmake/caches/opt-Os.cmake" config/tasks/task jenkinsrun config/tasks/lnt-ctmark.sh -a compiler="${params.ARTIFACT}" -D LNT_FLAGS="\${LNT_FLAGS}" - """ + """ + } } diff --git a/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-Oz b/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-Oz --- a/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-Oz +++ b/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-Oz @@ -1,9 +1,11 @@ #!/usr/bin/env groovy def common = evaluate readTrusted('zorg/jenkins/common.groovy') -common.testsuite_pipeline(label: 'green-dragon-12') { - sh """ +common.testsuite_pipeline(label: 'green-dragon-11') { + timeout(30) { + sh """ LNT_FLAGS+=" -C config/tasks/cmake/caches/target-arm64-iphoneos.cmake" LNT_FLAGS+=" -C config/tasks/cmake/caches/opt-Oz.cmake" config/tasks/task jenkinsrun config/tasks/lnt-ctmark.sh -a compiler="${params.ARTIFACT}" -D LNT_FLAGS="\${LNT_FLAGS}" - """ + """ + } } diff --git a/zorg/jenkins/jobs/jobs/relay-lldb b/zorg/jenkins/jobs/jobs/relay-lldb new file mode 100644 --- /dev/null +++ b/zorg/jenkins/jobs/jobs/relay-lldb @@ -0,0 +1,3 @@ +#!/usr/bin/env groovy +def relay = evaluate readTrusted('zorg/jenkins/relay.groovy') +relay.lldb_pipeline "lldb-cmake-(.+)" diff --git a/zorg/jenkins/jobs/jobs/test-suite-verify-machineinstrs-aarch64-O0-g b/zorg/jenkins/jobs/jobs/test-suite-verify-machineinstrs-aarch64-O0-g --- a/zorg/jenkins/jobs/jobs/test-suite-verify-machineinstrs-aarch64-O0-g +++ b/zorg/jenkins/jobs/jobs/test-suite-verify-machineinstrs-aarch64-O0-g @@ -1,6 +1,6 @@ #!/usr/bin/env groovy def common = evaluate readTrusted('zorg/jenkins/common.groovy') -common.testsuite_pipeline(label: 'green-dragon-12') { +common.testsuite_pipeline(label: 'green-dragon-11') { sh """ CMAKE_FLAGS+=" -C ../config/tasks/cmake/caches/target-arm64-iphoneos.cmake" CMAKE_FLAGS+=" -C ../config/tasks/cmake/caches/opt-O0-g.cmake" diff --git a/zorg/jenkins/jobs/jobs/test-suite-verify-machineinstrs-aarch64-O3 b/zorg/jenkins/jobs/jobs/test-suite-verify-machineinstrs-aarch64-O3 --- a/zorg/jenkins/jobs/jobs/test-suite-verify-machineinstrs-aarch64-O3 +++ b/zorg/jenkins/jobs/jobs/test-suite-verify-machineinstrs-aarch64-O3 @@ -1,6 +1,6 @@ #!/usr/bin/env groovy def common = evaluate readTrusted('zorg/jenkins/common.groovy') -common.testsuite_pipeline(label: 'green-dragon-12') { +common.testsuite_pipeline(label: 'green-dragon-11') { sh """ CMAKE_FLAGS+=" -C ../config/tasks/cmake/caches/target-arm64-iphoneos.cmake" CMAKE_FLAGS+=" -C ../config/tasks/cmake/caches/opt-O3.cmake" diff --git a/zorg/jenkins/jobs/jobs/test-suite-verify-machineinstrs-aarch64-globalisel-O0-g b/zorg/jenkins/jobs/jobs/test-suite-verify-machineinstrs-aarch64-globalisel-O0-g --- a/zorg/jenkins/jobs/jobs/test-suite-verify-machineinstrs-aarch64-globalisel-O0-g +++ b/zorg/jenkins/jobs/jobs/test-suite-verify-machineinstrs-aarch64-globalisel-O0-g @@ -1,6 +1,6 @@ #!/usr/bin/env groovy def common = evaluate readTrusted('zorg/jenkins/common.groovy') -common.testsuite_pipeline(label: 'green-dragon-12') { +common.testsuite_pipeline(label: 'green-dragon-11') { sh """ CMAKE_FLAGS+=" -C ../config/tasks/cmake/caches/target-arm64-iphoneos.cmake" CMAKE_FLAGS+=" -C ../config/tasks/cmake/caches/opt-O0-g.cmake" diff --git a/zorg/jenkins/jobs/jobs/test-suite-verify-machineinstrs-x86_64-O0-g b/zorg/jenkins/jobs/jobs/test-suite-verify-machineinstrs-x86_64-O0-g --- a/zorg/jenkins/jobs/jobs/test-suite-verify-machineinstrs-x86_64-O0-g +++ b/zorg/jenkins/jobs/jobs/test-suite-verify-machineinstrs-x86_64-O0-g @@ -1,6 +1,6 @@ #!/usr/bin/env groovy def common = evaluate readTrusted('zorg/jenkins/common.groovy') -common.testsuite_pipeline(label: 'green-dragon-12') { +common.testsuite_pipeline(label: 'green-dragon-11') { sh """ CMAKE_FLAGS+=" -C ../config/tasks/cmake/caches/target-x86_64-macos.cmake" CMAKE_FLAGS+=" -C ../config/tasks/cmake/caches/opt-O0-g.cmake" diff --git a/zorg/jenkins/jobs/jobs/test-suite-verify-machineinstrs-x86_64-O3 b/zorg/jenkins/jobs/jobs/test-suite-verify-machineinstrs-x86_64-O3 --- a/zorg/jenkins/jobs/jobs/test-suite-verify-machineinstrs-x86_64-O3 +++ b/zorg/jenkins/jobs/jobs/test-suite-verify-machineinstrs-x86_64-O3 @@ -1,6 +1,6 @@ #!/usr/bin/env groovy def common = evaluate readTrusted('zorg/jenkins/common.groovy') -common.testsuite_pipeline(label: 'green-dragon-12') { +common.testsuite_pipeline(label: 'green-dragon-11') { sh """ CMAKE_FLAGS+=" -C ../config/tasks/cmake/caches/target-x86_64-macos.cmake" CMAKE_FLAGS+=" -C ../config/tasks/cmake/caches/opt-O3.cmake" diff --git a/zorg/jenkins/jobs/jobs/test-suite-verify-machineinstrs-x86_64h-O3 b/zorg/jenkins/jobs/jobs/test-suite-verify-machineinstrs-x86_64h-O3 --- a/zorg/jenkins/jobs/jobs/test-suite-verify-machineinstrs-x86_64h-O3 +++ b/zorg/jenkins/jobs/jobs/test-suite-verify-machineinstrs-x86_64h-O3 @@ -1,6 +1,6 @@ #!/usr/bin/env groovy def common = evaluate readTrusted('zorg/jenkins/common.groovy') -common.testsuite_pipeline(label: 'green-dragon-12') { +common.testsuite_pipeline(label: 'green-dragon-11') { sh """ CMAKE_FLAGS+=" -C ../config/tasks/cmake/caches/target-x86_64h-macos.cmake" CMAKE_FLAGS+=" -C ../config/tasks/cmake/caches/opt-O3.cmake" diff --git a/zorg/jenkins/jobs/util/make_pipeline.py b/zorg/jenkins/jobs/util/make_pipeline.py --- a/zorg/jenkins/jobs/util/make_pipeline.py +++ b/zorg/jenkins/jobs/util/make_pipeline.py @@ -32,7 +32,7 @@ - */master + */main false diff --git a/zorg/jenkins/monorepo_build.py b/zorg/jenkins/monorepo_build.py --- a/zorg/jenkins/monorepo_build.py +++ b/zorg/jenkins/monorepo_build.py @@ -11,11 +11,11 @@ import shutil import math import re +import requests import xml.etree.ElementTree as ET from contextlib import contextmanager -from six.moves import urllib -SERVER = "labmaster2.lab.llvm.org" +SERVER = "green-dragon-21.local" NINJA = "/usr/local/bin/ninja" @@ -44,6 +44,7 @@ def header(name): print("@@@", name, "@@@") + print("Starting at: " + time.strftime("%FT%T")) def footer(): @@ -112,13 +113,12 @@ self._lldb_standalone_type = os.environ.get('LLDB_STANDALONE_TYPE', 'build-tree') self._lldb_xcode_build_dir = os.environ.get('LLDB_XCODE_BUILD_DIR', 'lldb-xcode-build') self._lldb_install_dir = os.environ.get('LLDB_INSTALL_DIR', 'lldb-install') - self._lldb_test_compiler = os.environ.get('LLDB_TEST_COMPILER', '') self._install_dir = os.environ.get('INSTALL_DIR', 'clang-install') self.j_level = os.environ.get('J_LEVEL', None) self.max_parallel_tests = os.environ.get('MAX_PARALLEL_TESTS', None) self.max_parallel_links = os.environ.get('MAX_PARALLEL_LINKS', None) self.host_compiler_url = os.environ.get('HOST_URL', - 'http://labmaster2.local/artifacts/') + 'http://green-dragon-21.local/artifacts/') self.artifact_url = os.environ.get('ARTIFACT', 'NONE') self.job_name = os.environ.get('JOB_NAME', 'NONE') self.build_id = os.environ.get('BUILD_ID', 'NONE') @@ -129,7 +129,6 @@ self.nobootstrap = True self.device = None self.node_name = os.environ.get('NODE_NAME', None) - self.lldb_test_archs = os.environ.get('LLDB_TEST_ARCHS', 'x86_64').split() # Import all of the command line arguments into the config object self.__dict__.update(vars(args)) @@ -166,10 +165,6 @@ """The install directory for the lldb build.""" return os.path.join(self.workspace, self._lldb_install_dir) - def lldbtestcompiler(self): - """The compiler used to build LLDB tests.""" - return self._lldb_test_compiler - def installdir(self): """The install directory for the compile.""" return os.path.join(self.workspace, self._install_dir) @@ -202,7 +197,7 @@ def branch(self): """Figure out the source branch name. - Not using GIT_BRANCH env var from Jenkins as that includes the + Not using GIT_BRANCH env var from Jenkins as that includes the remote name too. """ if not os.environ.get('TESTING', False): @@ -213,16 +208,16 @@ cmd = ['git', '-C', conf.srcdir(), 'rev-parse', 'HEAD'] out = run_collect_output(cmd, stderr=subprocess.STDOUT).strip() return out - return 'master' + return 'main' def link_memory_usage(self): """Guesstimate the maximum link memory usage for this build. - We are only building master here so we will just use that value + We are only building main here so we will just use that value """ # Determinited experimentally. - usages = {'master': 3.5} - if self.branch() == 'master': - return usages['master'] + usages = {'main': 3.5} + if self.branch() == 'main': + return usages['main'] else: raise NotImplementedError( "Unknown link memory usage." + self.branch()) @@ -247,6 +242,9 @@ "-DLLVM_ENABLE_PROJECTS=" + conf.llvm_enable_projects, conf.llvmsrcdir()] + if conf.llvm_enable_runtimes: + cmake_cmd += ["-DLLVM_ENABLE_RUNTIMES=" + conf.llvm_enable_runtimes] + compiler_flags = conf.compiler_flags max_parallel_links = conf.max_parallel_links @@ -350,7 +348,7 @@ # unless the user asked for something else. targets = ['check', 'check-clang'] - run_cmd(conf.builddir(), ninja_cmd + targets) + run_cmd(conf.builddir(), ninja_cmd + targets, err_okay=True) footer() if conf.sccache: @@ -428,12 +426,12 @@ '-DLLVM_INCLUDE_TESTS=On', '-DCLANG_INCLUDE_TESTS=On', '-DLLVM_INCLUDE_UTILS=On', - '-DLIBCXX_INSTALL_HEADERS=On', - '-DLIBCXX_OVERRIDE_DARWIN_INSTALL=On', - '-DLIBCXX_INSTALL_LIBRARY=Off', '-DCMAKE_MACOSX_RPATH=On', ] + if conf.llvm_enable_runtimes: + cmake_command.append('-DLLVM_ENABLE_RUNTIMES={}'.format(conf.llvm_enable_runtimes)) + if conf.sccache: cmake_command.append('-DCMAKE_C_COMPILER_LAUNCHER=' + conf.sccache_path) cmake_command.append('-DCMAKE_CXX_COMPILER_LAUNCHER=' + conf.sccache_path) @@ -497,7 +495,7 @@ cmd[-1] += ' --param use_gmalloc=1 ' \ '--param gmalloc_path=$(xcodebuild -find-library' \ ' libgmalloc.dylib)' - run_cmd(obj_dir, cmd, env={'MALLOC_LOG_FILE': '/dev/null'}) + run_cmd(obj_dir, cmd, env={'MALLOC_LOG_FILE': '/dev/null'}, err_okay=True) if conf.sccache: run_ws([conf.sccache_path, "--stop-server"]) @@ -531,14 +529,13 @@ cmake_build_type = conf.cmake_build_type if conf.cmake_build_type else 'RelWithDebInfo' # Construct dotest.py arguments. - dotest_args=['--arch', 'x86_64', '--build-dir', - conf.lldbbuilddir()+'/lldb-test-build.noindex', + dotest_args=['--build-dir', conf.lldbbuilddir()+'/lldb-test-build.noindex', '-t', '--env', 'TERM=vt100'] dotest_args.extend(conf.dotest_flags) # Construct lit arguments. lit_args = ['-v', '--time-tests', '--shuffle', - '--xunit-xml-output={}'.format(results_file), '-v'] + '--xunit-xml-output={}'.format(results_file), '-v'] if conf.max_parallel_tests: lit_args.extend(['-j', conf.max_parallel_tests]) if variant == 'sanitized': @@ -549,9 +546,11 @@ conf.llvmsrcdir(), '-DCMAKE_BUILD_TYPE={}'.format(cmake_build_type), '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON', + '-DLLVM_TARGETS_TO_BUILD=X86;ARM;AArch64', '-DCMAKE_INSTALL_PREFIX={}'.format(conf.lldbinstalldir()), '-DCMAKE_MAKE_PROGRAM={}'.format(NINJA), '-DLLDB_TEST_USER_ARGS='+';'.join(dotest_args), + '-DLLDB_ENFORCE_STRICT_TEST_REQUIREMENTS=On', '-DLLDB_ENABLE_PYTHON=On', '-DLLVM_ENABLE_ASSERTIONS:BOOL={}'.format("TRUE" if conf.assertions else "FALSE"), '-DLLVM_ENABLE_MODULES=On', @@ -559,6 +558,12 @@ '-DLLVM_LIT_ARGS={}'.format(' '.join(lit_args)), '-DLLVM_VERSION_PATCH=99'] + if conf.llvm_enable_runtimes: + cmake_cmd.append('-DLLVM_ENABLE_RUNTIMES={}'.format(conf.llvm_enable_runtimes)) + + if conf.lua: + cmake_cmd.append('-DLLDB_ENABLE_LUA=On') + if variant == 'sanitized': cmake_cmd.append('-DLLVM_TARGETS_TO_BUILD=X86') cmake_cmd.append('-DLLVM_USE_SANITIZER=Address;Undefined') @@ -566,8 +571,7 @@ # if we have a host compiler available. if conf.CC(): cmake_cmd.extend([ - '-DLLDB_TEST_C_COMPILER=' + conf.CC(), - '-DLLDB_TEST_CXX_COMPILER=' + conf.CC() + "++" + '-DLLDB_TEST_COMPILER=' + conf.CC(), ]) if conf.compiler_flags: @@ -576,10 +580,9 @@ "-DCMAKE_CXX_FLAGS={}".format(' '.join(conf.compiler_flags)) ]) - if conf.lldbtestcompiler(): + if conf.lldb_test_compiler is not None: cmake_cmd.extend([ - '-DLLDB_TEST_C_COMPILER=' + conf.lldbtestcompiler(), - '-DLLDB_TEST_CXX_COMPILER=' + conf.lldbtestcompiler() + "++" + '-DLLDB_TEST_COMPILER=' + conf.lldb_test_compiler, ]) cmake_cmd.extend(conf.cmake_flags) @@ -609,15 +612,15 @@ run_cmd(conf.lldbbuilddir(), [NINJA, '-v', 'install']) footer() - if target == 'all' or target == 'testlong': - header("Run Debug Info Tests") - run_cmd(conf.lldbbuilddir(), [NINJA, '-v', 'check-debuginfo']) - footer() - if target == 'all' or target == 'test' or target == 'testlong': header("Run Tests") - run_cmd(conf.lldbbuilddir(), - ['/usr/bin/env', 'TERM=vt100', NINJA, '-v', 'check-lldb']) + if variant == 'debuginfo': + test_command = ['/usr/bin/env', 'TERM=vt100', NINJA, + '-v', 'check-debuginfo'] + else: + test_command = ['/usr/bin/env', 'TERM=vt100', NINJA, + '-v', 'check-lldb'] + run_cmd(conf.lldbbuilddir(), test_command) footer() for test_target in conf.cmake_test_targets: @@ -660,6 +663,7 @@ '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON', '-DCMAKE_MAKE_PROGRAM={}'.format(NINJA), '-DLLDB_TEST_USER_ARGS='+';'.join(dotest_args), + '-DLLDB_ENFORCE_STRICT_TEST_REQUIREMENTS=On', '-DLLVM_ENABLE_ASSERTIONS:BOOL={}'.format( "TRUE" if conf.assertions else "FALSE"), '-DLLVM_ENABLE_MODULES=Off', @@ -705,6 +709,7 @@ '-L', conf.lldbsrcdir(), '-DLLVM_ENABLE_ASSERTIONS:BOOL={}'.format("TRUE" if conf.assertions else "FALSE"), '-DLLVM_ENABLE_MODULES=Off', + '-DLLDB_ENFORCE_STRICT_TEST_REQUIREMENTS=On', '-DLLVM_DIR={}'.format(llvm_dir), '-DClang_DIR={}'.format(clang_dir), '-DLLVM_VERSION_PATCH=99'] @@ -771,20 +776,21 @@ Print error and exit if download fails. """ + chunk_size = 100 * 1024 * 1024 try: print("GETting", url, "to", dest, "...") - f = urllib.request.urlopen(url) + session = requests.Session() + r = session.get(url) + r.raise_for_status() # Open our local file for writing with open(dest, "wb") as local_file: - local_file.write(f.read()) + for chunk in r.iter_content(chunk_size=chunk_size): + local_file.write(chunk) - except urllib.HTTPError as e: - print("HTTP Error:", e.code, url) + except requests.exceptions.HTTPError as e: + print("HTTP Error:", e.response.status_code, url) sys.exit(1) - except urllib.URLError as e: - print("URL Error:", e.reason, url) - sys.exit(1) print("done.") @@ -864,6 +870,29 @@ conf.job_name + "/g" + conf.git_sha] run_cmd(conf.workspace, lng_cmd) +def build_upload_properties(): + """Create artifact for this build, and upload to server.""" + if conf.noupload: + print('Not uploading artificats') + return + header("Uploading Artifact") + prop_file = "last_good_build.properties" + + with open(prop_file, 'w') as prop_fd: + prop_fd.write("LLVM_REV={}\n".format(conf.svn_rev)) + prop_fd.write("GIT_DISTANCE={}\n".format(conf.git_distance)) + prop_fd.write("GIT_SHA={}\n".format(conf.git_sha)) + + mkdir_cmd = ["ssh", "buildslave@" + SERVER, "mkdir", "-p", "/Library/WebServer/Documents/artifacts/" + conf.job_name] + + run_cmd(conf.workspace, mkdir_cmd) + + upload_cmd = ["scp", prop_file, + "buildslave@" + SERVER + ":/Library/WebServer/Documents/artifacts/" + + conf.job_name + "/"] + + run_cmd(conf.workspace, upload_cmd) + def run_cmd(working_dir, cmd, env=None, sudo=False, err_okay=False): """Run a command in a working directory, and make sure it returns zero.""" @@ -903,9 +932,18 @@ KNOWN_TARGETS = ['all', 'configure', 'build', 'test', 'testlong', 'install'] KNOWN_BUILDS = [ - 'clang', 'cmake', 'lldb-cmake', 'lldb-cmake-standalone', - 'lldb-cmake-xcode', 'lldb-cmake-sanitized', 'lldb-cmake-matrix', 'fetch', - 'artifact', 'static-analyzer-benchmarks' + 'clang', + 'cmake', + 'lldb-cmake', + 'lldb-cmake-standalone', + 'lldb-cmake-debuginfo', + 'lldb-cmake-xcode', + 'lldb-cmake-sanitized', + 'lldb-cmake-matrix', + 'fetch', + 'artifact', + 'properties', + 'static-analyzer-benchmarks', ] @@ -1005,6 +1043,7 @@ parser.add_argument('--lto', dest='lto', action='store_true') parser.add_argument('--thinlto', dest='thinlto', action='store_true') parser.add_argument('--debug', dest='debug', action='store_true') + parser.add_argument('--lua', dest='lua', action='store_true') parser.add_argument('--cmake-type', dest='cmake_build_type', help="Override cmake type Release, Debug, " "RelWithDebInfo and MinSizeRel") @@ -1033,10 +1072,15 @@ action='store_true', help="Turn on the experimental" " GlobalISel CMake flag.") parser.add_argument('--projects', dest='llvm_enable_projects', - default="clang;clang-tools-extra;compiler-rt;libcxx", - help="Semicolon seperated list of projects to build.") + default="clang;clang-tools-extra;compiler-rt", + help="Semicolon seperated list of projects to enable.") + parser.add_argument('--runtimes', dest='llvm_enable_runtimes', + default="", + help="Semicolon seperated list of runtimes to enable.") parser.add_argument('--timeout', dest='timeout', type=int, default='600', help='Individual test timeout in seconds.') + parser.add_argument('--lldb-test-compiler', + help='The compiler used to build LLDB tests.') args = parser.parse_args() if args.thinlto: args.lto = True @@ -1055,6 +1099,8 @@ clang_builder(args.build_target) elif args.build_type == 'lldb-cmake': lldb_cmake_builder(args.build_target) + elif args.build_type == 'lldb-cmake-debuginfo': + lldb_cmake_builder(args.build_target, 'debuginfo') elif args.build_type == 'lldb-cmake-sanitized': lldb_cmake_builder(args.build_target, 'sanitized') elif args.build_type == 'lldb-cmake-matrix': @@ -1069,6 +1115,8 @@ fetch_compiler() elif args.build_type == 'artifact': build_upload_artifact() + elif args.build_type == 'properties': + build_upload_properties() elif args.build_type == 'static-analyzer-benchmarks': static_analyzer_benchmarks_builder() except subprocess.CalledProcessError as exct: diff --git a/zorg/jenkins/relay.groovy b/zorg/jenkins/relay.groovy --- a/zorg/jenkins/relay.groovy +++ b/zorg/jenkins/relay.groovy @@ -28,7 +28,7 @@ curl -fksSO "${last_good_properties_url}" """ def props = readProperties file: propfile - def artifact = "http://labmaster2.local/artifacts/${props.ARTIFACT}" + def artifact = "http://green-dragon-21.local/artifacts/${props.ARTIFACT}" currentBuild.setDisplayName("${props.GIT_DISTANCE}-${props.GIT_SHA}") // Trigger all jobs with names matching the `job_pattern` regex. @@ -56,12 +56,23 @@ } def pipeline(job_pattern, - artifact_url='http://labmaster2.local/artifacts/', - last_good_properties_url='http://labmaster2.local/artifacts/clang-stage1-RA/last_good_build.properties') { + artifact_url='http://green-dragon-21.local/artifacts/', + last_good_properties_url='http://green-dragon-21.local/artifacts/clang-stage1-RA/last_good_build.properties') { node('master') { stage('main') { relay_steps job_pattern, artifact_url, last_good_properties_url } } } + +def lldb_pipeline(job_pattern, + artifact_url='http://green-dragon-21.local/artifacts/', + last_good_properties_url='http://green-dragon-21.local/artifacts/lldb-cmake/last_good_build.properties') { + node('master') { + stage('main') { + relay_steps job_pattern, artifact_url, last_good_properties_url + } + } +} + return this