diff --git a/.pylintrc b/.pylintrc new file mode 100644 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,7 @@ +# add root folder to Pylint PYTHONPATH, so builders can be checked with zorg.* +# packages +[MASTER] +init-hook='import sys; sys.path.append(".")' + +[FORMAT] +max-line-length=120 \ No newline at end of file diff --git a/buildbot/google/docker/buildbot-clangd-ubuntu-clang/Dockerfile b/buildbot/google/docker/buildbot-clangd-ubuntu-clang/Dockerfile --- a/buildbot/google/docker/buildbot-clangd-ubuntu-clang/Dockerfile +++ b/buildbot/google/docker/buildbot-clangd-ubuntu-clang/Dockerfile @@ -53,7 +53,7 @@ # The grpc version shipped with Ubuntu 18.04 crashes on TSan checks. ENV GRPC_INSTALL_PATH=/usr/local/lib/grpc RUN cd /tmp ; \ - git clone -b v1.33.2 --depth 1 --recursive --shallow-submodules https://github.com/grpc/grpc ; \ + git clone -b v1.36.3 --depth 1 --recursive --shallow-submodules https://github.com/grpc/grpc ; \ cd grpc ; \ mkdir build; cd build ; \ cmake -G Ninja -DgRPC_INSTALL=ON -DCMAKE_INSTALL_PREFIX=${GRPC_INSTALL_PATH} -DCMAKE_BUILD_TYPE=Release .. ; \ diff --git a/buildbot/google/docker/buildbot-clangd-ubuntu-clang/VERSION b/buildbot/google/docker/buildbot-clangd-ubuntu-clang/VERSION --- a/buildbot/google/docker/buildbot-clangd-ubuntu-clang/VERSION +++ b/buildbot/google/docker/buildbot-clangd-ubuntu-clang/VERSION @@ -1 +1 @@ -3 +5 diff --git a/buildbot/google/docker/buildbot-clangd-ubuntu-clang/run.sh b/buildbot/google/docker/buildbot-clangd-ubuntu-clang/run.sh --- a/buildbot/google/docker/buildbot-clangd-ubuntu-clang/run.sh +++ b/buildbot/google/docker/buildbot-clangd-ubuntu-clang/run.sh @@ -38,9 +38,14 @@ buildbot-worker create-worker --keepalive=200 "${WORKER_NAME}" \ lab.llvm.org:${BUILDBOT_PORT} "${WORKER_NAME}" "${WORKER_PASSWORD}" -# start the daemon, this command return immetiately +# start the daemon, this command returns immediately echo "starting worker..." -buildbot-worker start "${WORKER_NAME}" +set +e +if ! buildbot-worker start "${WORKER_NAME}"; then + echo ERROR: starting worker failed. contents of twistd.log: + cat "${WORKER_NAME}/twistd.log" + exit 1 +fi # To keep the container running and produce log outputs: dump the worker # log to stdout diff --git a/buildbot/google/docker/buildbot-mlir-nvidia/Dockerfile b/buildbot/google/docker/buildbot-mlir-nvidia/Dockerfile --- a/buildbot/google/docker/buildbot-mlir-nvidia/Dockerfile +++ b/buildbot/google/docker/buildbot-mlir-nvidia/Dockerfile @@ -5,7 +5,7 @@ # #===----------------------------------------------------------------------===// # Docker image used for the mlir-nvidia builder -# +# # Environment variables configurable at runtime: # BUILDBOT_PORT - server port to connect to #===----------------------------------------------------------------------===// @@ -18,7 +18,7 @@ RUN apt-get update; \ apt-get install -y software-properties-common apt-transport-https ca-certificates \ clang-8 lld-8 ninja-build git wget gnupg ccache \ - python3 python3-pip python3-psutil pybind11-dev \ + python3 python3-pip python3-psutil \ # dumb-init recommended in # https://hub.docker.com/r/buildbot/buildbot-worker/dockerfile dumb-init;\ @@ -26,6 +26,16 @@ update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-8 100 ;\ update-alternatives --install /usr/bin/lld lld /usr/bin/lld-8 100 +# LTS releases often bundle obsolete pip versions that cannot access newest +# Linux binary wheels. This pinned version is not special: it was just current +# at the time this was added. Refer to compatibility table: +# https://github.com/pypa/manylinux +RUN python3 -m pip install --upgrade pip==21.0.1 + +# Refer to mlir/lib/Bindings/Python/requirements.txt. Listed explicitly here +# and version pinned for consistency as this is a bot. +RUN python3 -m pip install numpy==1.19.3 pybind11==2.8.0 PyYAML==5.4.1 + # install cuda # avoid popups for keyboard configurations RUN DEBIAN_FRONTEND=noninteractive apt-get install -y cuda @@ -78,7 +88,7 @@ ENV WORKER_NAME="mlir-nvidia" # Allow the server port of this agent to be configurable during deployment. -# This way we can connect the same image to production and integration. +# This way we can connect the same image to production and integration. # Ports: # 9990 - production # 9994 - integration diff --git a/buildbot/google/docker/buildbot-mlir-nvidia/VERSION b/buildbot/google/docker/buildbot-mlir-nvidia/VERSION --- a/buildbot/google/docker/buildbot-mlir-nvidia/VERSION +++ b/buildbot/google/docker/buildbot-mlir-nvidia/VERSION @@ -1 +1 @@ -15 +18 diff --git a/buildbot/google/docker/buildbot-sanitizer-windows/Dockerfile b/buildbot/google/docker/buildbot-sanitizer-windows/Dockerfile new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-sanitizer-windows/Dockerfile @@ -0,0 +1,21 @@ +# escape=` +FROM gcr.io/sanitizer-bots/windows-base-vscode2019:8 + +ENV WORKER_NAME=sanitizer-windows + +# copy script to start the agent +COPY run.ps1 . +COPY fix_buildbot_tac_paths.py . + +# Set location for sccache cache +ENV SCCACHE_DIR="C:\volumes\sccache" + +# Set the maximum sccache local cache size +ENV SCCACHE_CACHE_SIZE="10G" + +# Move buildbot work dir to a volume to avoid of disc issues +VOLUME C:\volumes\buildbot + +# Configure 32bit tools ("x86") instead of 64bit ("amd64") +CMD ["powershell.exe", "-NoLogo", "-ExecutionPolicy", "Bypass", ` + "c:\\buildbot\\run.ps1"] diff --git a/buildbot/google/docker/buildbot-sanitizer-windows/VERSION b/buildbot/google/docker/buildbot-sanitizer-windows/VERSION new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-sanitizer-windows/VERSION @@ -0,0 +1 @@ +1 diff --git a/buildbot/google/docker/buildbot-sanitizer-windows/admin b/buildbot/google/docker/buildbot-sanitizer-windows/admin new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-sanitizer-windows/admin @@ -0,0 +1 @@ +Reid Kleckner diff --git a/buildbot/google/docker/buildbot-sanitizer-windows/fix_buildbot_tac_paths.py b/buildbot/google/docker/buildbot-sanitizer-windows/fix_buildbot_tac_paths.py new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-sanitizer-windows/fix_buildbot_tac_paths.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +# Replace backward slashes in the paths in a "buildbot.tac" file + +import argparse +import re +import shutil + + +def fix_slashes(buildbot_tac_path : str): + shutil.copy(buildbot_tac_path, buildbot_tac_path + ".bak") + result = [] + with open(buildbot_tac_path) as buildbot_tac_file: + buildbot_tac = buildbot_tac_file.readlines() + for line in buildbot_tac: + if line.lstrip().startswith('basedir'): + line = line.replace(r'\\','/') + result.append(line) + with open(buildbot_tac_path, 'w') as buildbot_tac_file: + buildbot_tac_file.writelines(result) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('buildbot_tac_path', help="path of 'buildbot.tac' file") + args = parser.parse_args() + fix_slashes(args.buildbot_tac_path) diff --git a/buildbot/google/docker/buildbot-sanitizer-windows/run.ps1 b/buildbot/google/docker/buildbot-sanitizer-windows/run.ps1 new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-sanitizer-windows/run.ps1 @@ -0,0 +1,51 @@ +# Stop the script on the first error +$ErrorActionPreference = "Stop" + +# Read password from file +$WORKER_PASSWORD=Get-Content "C:\volumes\secrets\token" + +#Create short drive alias "Q:" for worker dir, to keep path lengths <260 +SUBST Q: C:\volumes\buildbot +Get-PSDrive + +# chdir to the volume storing the temporary data +# This is a mounted volume +Set-Location Q:\ + +# Write host and admin information +# delete old folder if it exists +# configure powershell output to use UTF-8, otherwiese buildbot can't read the file +if ( Test-Path -Path 'info\' -PathType Container ) { + Remove-Item -Recurse -Force info +} +mkdir "info\" +$HOST_FILE="info\host" +$PSDefaultParameterValues['Out-File:Encoding'] = 'utf8' +Write-Output "Windows version: $(cmd /c ver)" > $HOST_FILE +Write-Output "VCTools version: $(ls C:\BuildTools\VC\Tools\MSVC | Select -ExpandProperty Name)" >> $HOST_FILE +Write-Output "Cores : $((Get-CimInstance Win32_ComputerSystem).NumberOfLogicalProcessors)" >> $HOST_FILE +Write-Output "RAM : $([int][Math]::Round((Get-CimInstance Win32_ComputerSystem).TotalPhysicalMemory / 1073741824)) GB" >> $HOST_FILE +Write-Output "Reid Kleckner " > "info\admin" + +# create the worker +Write-Output "creating worker..." +buildslave create-slave --keepalive=200 Q: ` + "lab.llvm.org:9994" "$env:WORKER_NAME" "$WORKER_PASSWORD" +# Note: Powershell does NOT exit on non-zero return codes, so we need to check manually. +if ($LASTEXITCODE -ne 0) { throw "Exit code of 'buildslave create' is $LASTEXITCODE" } + +# Replace backward slashes with forward slashes on buildbot.tac +# Otherwise Cmake will go into infinite re-configuration loops +python C:\buildbot\fix_buildbot_tac_paths.py buildbot.tac +if ($LASTEXITCODE -ne 0) { throw "Exit code of 'fix_buildbot_tac_paths' is $LASTEXITCODE" } + +# start the daemon, as this does not print and logs to std out, run it in the background +Write-Output "starting worker..." +cmd /c Start /B buildslave start Q: + +# Wait a bit until the logfile exists +Start-Sleep -s 5 + +# To keep the container running and produce log outputs: dump the worker +# log to stdout, this is the windows equivalent of `tail -f` +Get-Content -Path "Q:\twistd.log" -Wait diff --git a/buildbot/google/terraform/deployment-mlir-nvidia-production.yaml b/buildbot/google/terraform/deployment-mlir-nvidia-production.yaml --- a/buildbot/google/terraform/deployment-mlir-nvidia-production.yaml +++ b/buildbot/google/terraform/deployment-mlir-nvidia-production.yaml @@ -27,7 +27,7 @@ spec: containers: # the image and version we want to run - - image: gcr.io/sanitizer-bots/buildbot-mlir-nvidia:15 + - image: gcr.io/sanitizer-bots/buildbot-mlir-nvidia:18 name: mlir-nvidia-production env: # connect to production environment, running at port 9990 diff --git a/buildbot/google/terraform/main.tf b/buildbot/google/terraform/main.tf --- a/buildbot/google/terraform/main.tf +++ b/buildbot/google/terraform/main.tf @@ -120,161 +120,6 @@ } } -# node pool for windows machines -resource "google_container_node_pool" "windows_32core_pool_nodes" { - name = "windows-32core-pool" - # specify a zone here (e.g. "-a") to avoid a redundant deployment - location = var.gcp_config.zone_a - cluster = google_container_cluster.primary.name - - # use autoscaling to only create a machine when there is a deployment - autoscaling { - min_node_count = 0 - max_node_count = 1 - } - - node_config { - # use preemptible, as this saves costs - preemptible = true - machine_type = "e2-highcpu-32" - # Windows deployments tend to require more disk space, so using 300GB here. - disk_size_gb = 300 - # FIXME: test if SSDs are actually faster than HDDs for our use case - disk_type = "pd-ssd" - - # Configure Windows image. As Windows is picky about the combination of - # host and container OS versions, this must be compatible with the version - # in your container. Recommondation: Use LTSC for long-term stability. - # For details see - # https://docs.microsoft.com/en-us/virtualization/windowscontainers/deploy-containers/version-compatibility - # https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#choose_your_windows_server_node_image - image_type = "WINDOWS_LTSC" - - # set the premissions required for the deployment later - oauth_scopes = [ - "https://www.googleapis.com/auth/logging.write", - "https://www.googleapis.com/auth/monitoring", - "https://www.googleapis.com/auth/devstorage.read_only", - ] - - # add a label to all machines of this type, so we can select them - # during deployment - labels = { - pool = "win-32core-pool" - } - } -} - -# Deployment for the buildbot windows10_vs2019 running on Windows rather than -# Linux. -# Note: Deploying this takes significantly longer (~35 min) than on Linux -# as the images tend to be larger (~18GB) and IO performance is lower. -resource "kubernetes_deployment" "windows10_vs2019" { - metadata { - name = "windows10-vs2019" - labels = { - app = "windows10_vs2019" - } - } - - spec { - # create one instance of this container - replicas = 1 - - selector { - match_labels = { - app = "windows10_vs2019" - } - } - strategy{ - rolling_update{ - # do not deploy more replicas, as the buildbot server - # can't handle multiple workers with the same credentials - max_surge = 0 - # Allow to have 0 replicas during updates. - max_unavailable = 1 - } - type = "RollingUpdate" - } - template { - metadata { - labels = { - app = "windows10_vs2019" - } - } - - spec { - container { - image = "${var.gcp_config.gcr_prefix}/buildbot-windows10-vs2019:17" - name = "windows10-vs2019" - - # reserve "-1" for this image, kubernetes also - # needs <1 core for management tools - resources { - limits { - cpu = "31" - memory = "20Gi" - } - requests { - cpu = "31" - memory = "20Gi" - } - } - - # mount the secrets into a folder - volume_mount { - mount_path = "c:\\volumes\\secrets" - name = "buildbot-token" - } - volume_mount { - mount_path = "c:\\volumes\\sccache" - name = "sccache-vol" - } - volume_mount { - mount_path = "c:\\volumes\\buildbot" - name = "buildbot-vol" - } - - } - # select which node pool to deploy to - node_selector = { - pool = "win-32core-pool" - } - # restart in case of any crashes - restart_policy = "Always" - - # select the secret to be mounted - volume { - name = "buildbot-token" - secret { - optional = false - secret_name = "password-windows10-vs2019" - } - } - volume { - name = "sccache-vol" - empty_dir {} - } - volume { - name = "buildbot-vol" - empty_dir {} - } - - # Windows nodes from the node pool are marked with the taint - # "node.kubernetes.io/os=windows". So we need to "tolerate" this to - # deploy to such nodes. - toleration { - effect = "NoSchedule" - key = "node.kubernetes.io/os" - operator = "Equal" - value = "windows" - } - } - } - } -} - - resource "google_container_node_pool" "linux_16_core_pool" { name = "linux-16-core-pool" # specify a zone here (e.g. "-a") to avoid a redundant deployment @@ -290,8 +135,8 @@ node_config { # use preemptible, as this saves costs preemptible = true - #custom machine type: 16 core, 32 GB as tsan needs more RAM - machine_type = "n2d-custom-16-32768" + #custom machine type: 16 core, 48 GB as tsan needs more RAM + machine_type = "n2d-custom-16-49152" disk_size_gb = 100 disk_type = "pd-ssd" @@ -347,7 +192,7 @@ spec { container { - image = "${var.gcp_config.gcr_prefix}/buildbot-clangd-ubuntu-clang:3" + image = "${var.gcp_config.gcr_prefix}/buildbot-clangd-ubuntu-clang:5" name = "buildbot-clangd-ubuntu-clang" # reserve "-1" for this image, kubernetes also @@ -355,11 +200,11 @@ resources { limits { cpu = "15" - memory = "28G" + memory = "45G" } requests { cpu = "15" - memory = "28G" + memory = "45G" } } diff --git a/buildbot/osuosl/master/config/__init__.py b/buildbot/osuosl/master/config/__init__.py --- a/buildbot/osuosl/master/config/__init__.py +++ b/buildbot/osuosl/master/config/__init__.py @@ -12,6 +12,7 @@ import config.auth import config.builders +import config.release_builders import config.schedulers import config.workers import config.status @@ -22,4 +23,5 @@ reload(config.workers) reload(config.schedulers) reload(config.builders) +reload(release_builders) reload(config.status) diff --git a/buildbot/osuosl/master/config/builders.py b/buildbot/osuosl/master/config/builders.py --- a/buildbot/osuosl/master/config/builders.py +++ b/buildbot/osuosl/master/config/builders.py @@ -1,4 +1,6 @@ -# TODO: Rename workers with "slave" as a part of the name. +from importlib import reload + +from buildbot.plugins import util from buildbot.process.properties import WithProperties @@ -8,7 +10,6 @@ from zorg.buildbot.builders import LLDBBuilder from zorg.buildbot.builders import SanitizerBuilder from zorg.buildbot.builders import OpenMPBuilder -from zorg.buildbot.builders import LibcxxAndAbiBuilder from zorg.buildbot.builders import SphinxDocsBuilder from zorg.buildbot.builders import ABITestsuitBuilder from zorg.buildbot.builders import ClangLTOBuilder @@ -18,8 +19,14 @@ from zorg.buildbot.builders import LLDPerformanceTestsuite from zorg.buildbot.builders import FuchsiaBuilder from zorg.buildbot.builders import XToolchainBuilder +from zorg.buildbot.builders import TestSuiteBuilder +from zorg.buildbot.builders import BOLTBuilder -from buildbot.plugins import util +from zorg.buildbot.builders import HtmlDocsBuilder +from zorg.buildbot.builders import DoxygenDocsBuilder + +reload(HtmlDocsBuilder) +reload(DoxygenDocsBuilder) all = [ @@ -37,7 +44,9 @@ clean=True, depends_on_projects=['llvm','clang','clang-tools-extra','compiler-rt'], extra_configure_args=[ + "-DLLVM_CCACHE_BUILD=ON", "-DCOMPILER_RT_BUILD_BUILTINS:BOOL=OFF", + "-DCOMPILER_RT_BUILD_ORC:BOOL=OFF", "-DCOMPILER_RT_BUILD_SANITIZERS:BOOL=OFF", "-DCOMPILER_RT_BUILD_XRAY:BOOL=OFF", "-DCOMPILER_RT_INCLUDE_TESTS:BOOL=OFF", @@ -46,30 +55,7 @@ ], env={ 'PATH':'/home/llvmbb/bin/clang-latest/bin:/home/llvmbb/bin:/usr/local/bin:/usr/local/bin:/usr/bin:/bin', - 'CC': 'ccache clang', 'CXX': 'ccache clang++', 'CCACHE_CPP2': 'yes', - })}, - - {'name' : "clang-x86_64-debian-new-pass-manager-fast", - 'tags' : ["clang", "fast"], - 'workernames': ["gribozavr4"], - 'builddir': "clang-x86_64-debian-new-pass-manager-fast", - 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( - llvm_srcdir="llvm.src", - obj_dir="llvm.obj", - clean=True, - depends_on_projects=['llvm','clang','clang-tools-extra','compiler-rt'], - extra_configure_args=[ - "-DCOMPILER_RT_BUILD_BUILTINS:BOOL=OFF", - "-DCOMPILER_RT_BUILD_SANITIZERS:BOOL=OFF", - "-DCOMPILER_RT_BUILD_XRAY:BOOL=OFF", - "-DCOMPILER_RT_INCLUDE_TESTS:BOOL=OFF", - "-DENABLE_EXPERIMENTAL_NEW_PASS_MANAGER=ON", - "-DCMAKE_C_FLAGS=-Wdocumentation -Wno-documentation-deprecated-sync", - "-DCMAKE_CXX_FLAGS=-std=c++11 -Wdocumentation -Wno-documentation-deprecated-sync", - ], - env={ - 'PATH':'/home/llvmbb/bin/clang-latest/bin:/home/llvmbb/bin:/usr/local/bin:/usr/local/bin:/usr/bin:/bin', - 'CC': 'ccache clang', 'CXX': 'ccache clang++', 'CCACHE_CPP2': 'yes', + 'CC': 'clang', 'CXX': 'clang++', })}, {'name' : "llvm-clang-x86_64-win-fast", @@ -92,6 +78,27 @@ "-DLLVM_OPTIMIZED_TABLEGEN=OFF", "-DLLVM_LIT_ARGS=-v --threads=32"])}, + {'name': "llvm-clang-x86_64-sie-ubuntu-fast", + 'tags' : ["clang", "llvm", "clang-tools-extra", "lld", "cross-project-tests"], + 'workernames': ["sie-linux-worker"], + 'builddir': "llvm-clang-x86_64-sie-ubuntu-fast", + 'factory': UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + depends_on_projects=['llvm','clang','clang-tools-extra','lld','cross-project-tests'], + extra_configure_args=[ + "-DCMAKE_C_COMPILER=gcc", + "-DCMAKE_CXX_COMPILER=g++", + "-DCMAKE_BUILD_TYPE=Release", + "-DCLANG_ENABLE_ARCMT=OFF", + "-DCLANG_ENABLE_CLANGD=OFF", + "-DLLVM_BUILD_RUNTIME=OFF", + "-DLLVM_CCACHE_BUILD=ON", + "-DLLVM_INCLUDE_EXAMPLES=OFF", + "-DLLVM_DEFAULT_TARGET_TRIPLE=x86_64-scei-ps4", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_LIT_ARGS=--verbose -j100", + "-DLLVM_TARGETS_TO_BUILD=X86", + "-DLLVM_USE_LINKER=gold"])}, + # Expensive checks builders. {'name' : "llvm-clang-x86_64-expensive-checks-ubuntu", @@ -99,27 +106,29 @@ 'workernames' : ["as-builder-4"], 'builddir': "llvm-clang-x86_64-expensive-checks-ubuntu", 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( - depends_on_projects=["llvm"], + depends_on_projects=["llvm", "lld"], clean=True, extra_configure_args=[ "-DLLVM_ENABLE_EXPENSIVE_CHECKS=ON", "-DLLVM_ENABLE_WERROR=OFF", + "-DLLVM_USE_SPLIT_DWARF=ON", + "-DLLVM_USE_LINKER=gold", "-DCMAKE_BUILD_TYPE=Debug", - "-DCMAKE_CXX_FLAGS=-U_GLIBCXX_DEBUG", + "-DCMAKE_CXX_FLAGS=-U_GLIBCXX_DEBUG -Wno-misleading-indentation", "-DLLVM_LIT_ARGS=-vv -j32"])}, {'name' : "llvm-clang-x86_64-expensive-checks-win", - 'tags' : ["llvm", "clang", "expensive-checks"], + 'tags' : ["llvm", "expensive-checks"], 'workernames' : ["as-worker-93"], 'builddir': "llvm-clang-x86_64-expensive-checks-win", 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaWithMSVCBuildFactory( vs="autodetect", + depends_on_projects=["llvm", "lld"], clean=True, extra_configure_args=[ "-DLLVM_ENABLE_EXPENSIVE_CHECKS=ON", "-DLLVM_ENABLE_WERROR=OFF", - "-DCMAKE_BUILD_TYPE=Debug", - "-DLLVM_LIT_ARGS=-vv"])}, + "-DCMAKE_BUILD_TYPE=Debug"])}, {'name' : "llvm-clang-x86_64-expensive-checks-debian", 'tags' : ["llvm", "expensive-checks"], @@ -127,9 +136,10 @@ 'workernames' : ["gribozavr4"], 'builddir': "llvm-clang-x86_64-expensive-checks-debian", 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( - depends_on_projects=["llvm"], + depends_on_projects=["llvm", "lld"], clean=True, extra_configure_args=[ + "-DLLVM_CCACHE_BUILD=ON", "-DLLVM_ENABLE_EXPENSIVE_CHECKS=ON", "-DLLVM_ENABLE_WERROR=OFF", "-DCMAKE_BUILD_TYPE=Release", @@ -137,13 +147,13 @@ "-DLLVM_LIT_ARGS=-v -vv -j96"], env={ 'PATH':'/home/llvmbb/bin/clang-latest/bin:/home/llvmbb/bin:/usr/local/bin:/usr/local/bin:/usr/bin:/bin', - 'CC': 'ccache clang', 'CXX': 'ccache clang++', 'CCACHE_CPP2': 'yes', + 'CC': 'clang', 'CXX': 'clang++', })}, # Cross builders. {'name' : "llvm-clang-win-x-armv7l", - 'tags' : ["clang", "llvm", "compiler-rt", "cross"," armv7l"], + 'tags' : ["clang", "llvm", "compiler-rt", "cross", "armv7"], 'workernames' : ["as-builder-1"], 'builddir': "x-armv7l", 'factory' : XToolchainBuilder.getCmakeWithMSVCBuildFactory( @@ -153,25 +163,28 @@ "check-llvm", "check-clang", "check-lld", - "check-compiler-rt" + "check-compiler-rt-armv7-unknown-linux-gnueabihf" ], checks_on_target = [ ("libunwind", ["python", "bin/llvm-lit.py", "-v", "-vv", "--threads=32", - "runtimes/runtimes-bins/libunwind/test"]), + "runtimes/runtimes-armv7-unknown-linux-gnueabihf-bins/libunwind/test"]), ("libc++abi", ["python", "bin/llvm-lit.py", "-v", "-vv", "--threads=32", - "runtimes/runtimes-bins/libcxxabi/test"]), + "runtimes/runtimes-armv7-unknown-linux-gnueabihf-bins/libcxxabi/test"]), ("libc++", ['python', 'bin/llvm-lit.py', '-v', '-vv', '--threads=32', - 'runtimes/runtimes-bins/libcxx/test', + 'runtimes/runtimes-armv7-unknown-linux-gnueabihf-bins/libcxx/test', ]) ], extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=ARM", + "-DTOOLCHAIN_TARGET_TRIPLE=armv7-unknown-linux-gnueabihf", "-DDEFAULT_SYSROOT=C:/buildbot/.arm-ubuntu", + "-DZLIB_ROOT=C:/buildbot/.zlib-win32", "-DLLVM_LIT_ARGS=-v -vv --threads=32", WithProperties("%(remote_test_host:+-DREMOTE_TEST_HOST=)s%(remote_test_host:-)s"), WithProperties("%(remote_test_user:+-DREMOTE_TEST_USER=)s%(remote_test_user:-)s"), @@ -179,7 +192,7 @@ cmake_cache="../llvm-project/clang/cmake/caches/CrossWinToARMLinux.cmake")}, {'name' : "llvm-clang-win-x-aarch64", - 'tags' : ["clang", "llvm", "compiler-rt", "cross"," aarch64"], + 'tags' : ["clang", "llvm", "compiler-rt", "cross", "aarch64"], 'workernames' : ["as-builder-2"], 'builddir': "x-aarch64", 'factory' : XToolchainBuilder.getCmakeWithMSVCBuildFactory( @@ -189,27 +202,28 @@ "check-llvm", "check-clang", "check-lld", - "check-compiler-rt" + "check-compiler-rt-aarch64-unknown-linux-gnu" ], checks_on_target = [ ("libunwind", ["python", "bin/llvm-lit.py", "-v", "-vv", "--threads=32", - "runtimes/runtimes-bins/libunwind/test"]), + "runtimes/runtimes-aarch64-unknown-linux-gnu-bins/libunwind/test"]), ("libc++abi", ["python", "bin/llvm-lit.py", "-v", "-vv", "--threads=32", - "runtimes/runtimes-bins/libcxxabi/test"]), + "runtimes/runtimes-aarch64-unknown-linux-gnu-bins/libcxxabi/test"]), ("libc++", ['python', 'bin/llvm-lit.py', '-v', '-vv', '--threads=32', - 'runtimes/runtimes-bins/libcxx/test', + 'runtimes/runtimes-aarch64-unknown-linux-gnu-bins/libcxx/test', ]) ], extra_configure_args=[ "-DLLVM_TARGETS_TO_BUILD=AArch64", - "-DCMAKE_C_COMPILER_TARGET=aarch64-linux-gnu", + "-DTOOLCHAIN_TARGET_TRIPLE=aarch64-unknown-linux-gnu", "-DDEFAULT_SYSROOT=C:/buildbot/.aarch64-ubuntu", + "-DZLIB_ROOT=C:/buildbot/.zlib-win32", "-DLLVM_LIT_ARGS=-v -vv --threads=32", WithProperties("%(remote_test_host:+-DREMOTE_TEST_HOST=)s%(remote_test_host:-)s"), WithProperties("%(remote_test_user:+-DREMOTE_TEST_USER=)s%(remote_test_user:-)s"), @@ -224,12 +238,26 @@ 'builddir': "clang-arm64-windows-msvc", 'factory' : ClangBuilder.getClangCMakeBuildFactory( vs="manual", - test=False, # Disable testing until MCJIT failures are fixed + checkout_flang=True, + # Only enabling checks we expect to work. + checks=['check-flang'], extra_cmake_args=[ - "-DLLVM_DEFAULT_TARGET_TRIPLE=aarch64-windows-msvc", - "-DLLVM_HOST_TRIPLE=aarch64-windows-msvc", + "-DCMAKE_TRY_COMPILE_CONFIGURATION=Release", + "-DLLVM_DEFAULT_TARGET_TRIPLE=aarch64-pc-windows-msvc", + "-DLLVM_HOST_TRIPLE=aarch64-pc-windows-msvc", "-DLLVM_TARGET_ARCH=AArch64", + "-DCMAKE_C_FLAGS=-fms-compatibility-version=19.27", + "-DCMAKE_CXX_FLAGS=-fms-compatibility-version=19.27", + # The BUILTINS environment variable is expected to already exist + # on the worker when it is launched. + "-DCMAKE_EXE_LINKER_FLAGS=%BUILTINS%", + "-DCMAKE_SHARED_LINKER_FLAGS=%BUILTINS%", + "-DCMAKE_STATIC_LINKER_FLAGS=%BUILTINS%", + "-DCMAKE_MODULE_LINKER_FLAGS=%BUILTINS%", + # FIXME: compiler-rt\lib\sanitizer_common\sanitizer_unwind_win.cpp assumes WIN64 is x86_64, + # so, before that's fixed, disable everything that triggers its build. "-DCOMPILER_RT_BUILD_SANITIZERS=OFF", + "-DCOMPILER_RT_BUILD_MEMPROF=OFF", "-DCOMPILER_RT_BUILD_XRAY=OFF"])}, # Cortex-A15 LNT test-suite in Benchmark mode @@ -241,8 +269,7 @@ clean=False, checkout_compiler_rt=False, checkout_lld=False, - test=False, - useTwoStage=False, + checks=[], runTestSuite=True, testsuite_flags=[ '--cppflags', '-O3 -mcpu=cortex-a15 -mthumb', @@ -257,27 +284,25 @@ testerName='LNT-Thumb2v7-A15-O3')}, # ARMv7 LNT test-suite in test-only mode - {'name' : "clang-cmake-armv7-lnt", + {'name' : "clang-armv7-lnt", 'tags' : ["clang"], - 'workernames' : ["linaro-armv7-lnt"], - 'builddir': "clang-cmake-armv7-lnt", + 'workernames' : ["linaro-clang-armv7-lnt"], + 'builddir': "clang-armv7-lnt", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, checkout_compiler_rt=False, checkout_lld=False, - test=False, - useTwoStage=False, + checks=[], runTestSuite=True, testsuite_flags=[ '--cppflags', '-mcpu=cortex-a15 -marm', - '--threads=32', '--build-threads=32'], - extra_cmake_args=["-DLLVM_TARGETS_TO_BUILD='ARM'"])}, + '--threads=32', '--build-threads=32'])}, - ## ARMv7 check-all self-host NEON with CMake builder - {'name' : "clang-cmake-armv7-selfhost-neon", + ## ARMv7 check-all 2-stage + {'name' : "clang-armv7-2stage", 'tags' : ["clang"], - 'workernames': ["linaro-armv7-selfhost"], - 'builddir':"clang-cmake-armv7-selfhost-neon", + 'workernames': ["linaro-clang-armv7-2stage"], + 'builddir':"clang-armv7-2stage", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, checkout_compiler_rt=False, @@ -286,42 +311,38 @@ testStage1=False, extra_cmake_args=[ "-DCMAKE_C_FLAGS='-mcpu=cortex-a15 -marm'", - "-DCMAKE_CXX_FLAGS='-mcpu=cortex-a15 -marm'", - "-DLLVM_TARGETS_TO_BUILD='ARM;AArch64'"])}, + "-DCMAKE_CXX_FLAGS='-mcpu=cortex-a15 -marm'"])}, - ## ARMv7 check-all with CMake builder - {'name' : "clang-cmake-armv7-quick", + ## ARMv7 check-all + {'name' : "clang-armv7-quick", 'tags' : ["clang"], - 'workernames':["linaro-armv7-quick"], - 'builddir':"clang-cmake-armv7-quick", + 'workernames':["linaro-clang-armv7-quick"], + 'builddir':"clang-armv7-quick", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, checkout_compiler_rt=False, checkout_lld=False, - extra_cmake_args=["-DLLVM_TARGETS_TO_BUILD='ARM;AArch64'"])}, + extra_cmake_args=["-DLLVM_TARGETS_TO_BUILD='ARM'"])}, - ## ARMv7 Clang + LLVM run test-suite with GlobalISel enabled - {'name' : "clang-cmake-armv7-global-isel", + ## ARMv7 run test-suite with GlobalISel enabled + {'name' : "clang-armv7-global-isel", 'tags' : ["clang"], - 'workernames':["linaro-armv7-global-isel"], - 'builddir':"clang-cmake-armv7-global-isel", + 'workernames':["linaro-clang-armv7-global-isel"], + 'builddir':"clang-armv7-global-isel", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, checkout_compiler_rt=False, checkout_lld=False, - test=True, - useTwoStage=False, runTestSuite=True, testsuite_flags=[ '--cppflags', '-mcpu=cortex-a15 -marm -O0 -mllvm -global-isel -mllvm -global-isel-abort=0', - '--threads=32', '--build-threads=32'], - extra_cmake_args=["-DLLVM_TARGETS_TO_BUILD='ARM;AArch64'"])}, + '--threads=32', '--build-threads=32'])}, - ## ARMv7 check-all self-host with CMake builder - {'name' : "clang-cmake-armv7-selfhost", + ## ARMv7 VFPv3 check-all 2-stage + {'name' : "clang-armv7-vfpv3-2stage", 'tags' : ["clang"], - 'workernames' : ["linaro-armv7-selfhost"], - 'builddir': "clang-cmake-armv7-selfhost", + 'workernames' : ["linaro-clang-armv7-vfpv3-2stage"], + 'builddir': "clang-armv7-vfpv3-2stage", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, checkout_compiler_rt=False, @@ -330,36 +351,26 @@ testStage1=False, extra_cmake_args=[ "-DCMAKE_C_FLAGS='-mcpu=cortex-a15 -mfpu=vfpv3 -marm'", - "-DCMAKE_CXX_FLAGS='-mcpu=cortex-a15 -mfpu=vfpv3 -marm'", - "-DLLVM_TARGETS_TO_BUILD='ARM;AArch64'"])}, + "-DCMAKE_CXX_FLAGS='-mcpu=cortex-a15 -mfpu=vfpv3 -marm'"])}, - ## AArch64 Clang+LLVM check-all + test-suite - {'name' : "clang-cmake-aarch64-quick", + ## AArch64 check-all + {'name' : "clang-aarch64-quick", 'tags' : ["clang"], - 'workernames' : ["linaro-aarch64-quick"], - 'builddir': "clang-cmake-aarch64-quick", + 'workernames' : ["linaro-clang-aarch64-quick"], + 'builddir': "clang-aarch64-quick", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, checkout_compiler_rt=False, checkout_lld=False, - test=True, - useTwoStage=False, - runTestSuite=True, - testsuite_flags=[ - '--cppflags', '-mcpu=cortex-a57', - '--threads=32', '--build-threads=32'], - extra_cmake_args=["-DLLVM_TARGETS_TO_BUILD='ARM;AArch64'"])}, + extra_cmake_args=["-DLLVM_TARGETS_TO_BUILD='AArch64'"])}, - ## AArch64 Self-hosting Clang+LLVM check-all + LLD + test-suite - {'name' : "clang-cmake-aarch64-lld", + ## AArch64 check-all + LLD + test-suite 2-stage + {'name' : "clang-aarch64-lld-2stage", 'tags' : ["lld"], - 'workernames' : ["linaro-aarch64-lld"], - 'builddir':"clang-cmake-aarch64-lld", + 'workernames' : ["linaro-clang-aarch64-lld-2stage"], + 'builddir':"clang-aarch64-lld-2stage", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, - checkout_compiler_rt=True, - checkout_lld=True, - test=True, useTwoStage=True, runTestSuite=True, testsuite_flags=[ @@ -368,55 +379,50 @@ extra_cmake_args=[ "-DCMAKE_C_FLAGS='-mcpu=cortex-a57'", "-DCMAKE_CXX_FLAGS='-mcpu=cortex-a57'", - "-DLLVM_TARGETS_TO_BUILD='ARM;AArch64'", - "-DLLVM_ENABLE_LLD=True"])}, + "-DLLVM_ENABLE_LLD=True", + "-DLLVM_LIT_ARGS='-v -j4'"])}, - ## AArch64 Clang+LLVM run test-suite at -O0 (GlobalISel is now default). - {'name' : "clang-cmake-aarch64-global-isel", + ## AArch64 run test-suite at -O0 (GlobalISel is now default). + {'name' : "clang-aarch64-global-isel", 'tags' : ["clang"], - 'workernames' : ["linaro-aarch64-global-isel"], - 'builddir': "clang-cmake-aarch64-global-isel", + 'workernames' : ["linaro-clang-aarch64-global-isel"], + 'builddir': "clang-aarch64-global-isel", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, checkout_compiler_rt=False, checkout_lld=False, - test=True, - useTwoStage=False, runTestSuite=True, testsuite_flags=[ '--cppflags', '-O0', - '--threads=32', '--build-threads=32'], - extra_cmake_args=["-DLLVM_TARGETS_TO_BUILD='ARM;AArch64'"])}, + '--threads=32', '--build-threads=32'])}, - ## ARMv7 without neon; check-all 2-stage full compiler-rt + testsuite - {'name' : "clang-cmake-armv7-full", + ## ARMv7 VFPv3 check-all + compiler-rt + testsuite 2-stage + {'name' : "clang-armv7-vfpv3-full-2stage", 'tags' : ["clang"], 'workernames' : ["linaro-tk1-06", "linaro-tk1-07", "linaro-tk1-08", "linaro-tk1-09"], - 'builddir': "clang-cmake-armv7-full", + 'builddir': "clang-armv7-vfpv3-full-2stage", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, checkout_lld=False, - checkout_compiler_rt=True, - testStage1=False, useTwoStage=True, + testStage1=False, runTestSuite=True, testsuite_flags=[ '--cppflags', '-mcpu=cortex-a15 -mfpu=vfpv3 -marm', '--threads=4', '--build-threads=4'], extra_cmake_args=[ "-DCOMPILER_RT_TEST_COMPILER_CFLAGS='-mcpu=cortex-a15 -mfpu=vfpv3 -marm'", - "-DLLVM_TARGETS_TO_BUILD='ARM'", - "-DLLVM_PARALLEL_LINK_JOBS=2"])}, + "-DLLVM_PARALLEL_LINK_JOBS=2", + "-DCOMPILER_RT_BUILD_GWP_ASAN=OFF"])}, - ## ARMv7 Thumb2 with neon; check-all 2-stage full compiler-rt + testsuite - {'name' : "clang-cmake-thumbv7-full-sh", + ## ARMv7 Thumb2 check-all + compiler-rt + testsuite 2-stage + {'name' : "clang-thumbv7-full-2stage", 'tags' : ["clang"], 'workernames' : ["linaro-tk1-01", "linaro-tk1-03", "linaro-tk1-04", "linaro-tk1-05"], - 'builddir': "clang-cmake-thumbv7-full-sh", + 'builddir': "clang-thumbv7-full-2stage", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, checkout_lld=False, - checkout_compiler_rt=True, testStage1=False, useTwoStage=True, runTestSuite=True, @@ -427,20 +433,17 @@ "-DCMAKE_C_FLAGS='-mcpu=cortex-a15 -mthumb'", "-DCMAKE_CXX_FLAGS='-mcpu=cortex-a15 -mthumb'", "-DCOMPILER_RT_TEST_COMPILER_CFLAGS='-mcpu=cortex-a15 -mthumb'", - "-DLLVM_TARGETS_TO_BUILD='ARM'", - "-DLLVM_PARALLEL_LINK_JOBS=2"])}, + "-DLLVM_PARALLEL_LINK_JOBS=2", + "-DCOMPILER_RT_BUILD_GWP_ASAN=OFF"])}, ## AArch32 Self-hosting Clang+LLVM check-all + LLD + test-suite # Sanitizers build disabled due to PR38690 - {'name' : "clang-cmake-armv8-lld", + {'name' : "clang-armv8-lld-2stage", 'tags' : ["lld"], - 'workernames' : ["linaro-armv8-lld"], - 'builddir': "clang-cmake-armv8-lld", + 'workernames' : ["linaro-clang-armv8-lld-2stage"], + 'builddir': "clang-armv8-lld-2stage", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, - checkout_compiler_rt=True, - checkout_lld=True, - test=True, useTwoStage=True, runTestSuite=True, testsuite_flags=[ @@ -449,31 +452,126 @@ extra_cmake_args=[ "-DCMAKE_C_FLAGS='-mcpu=cortex-a57'", "-DCMAKE_CXX_FLAGS='-mcpu=cortex-a57'", - "-DLLVM_TARGETS_TO_BUILD='ARM;AArch64'", "-DCOMPILER_RT_BUILD_SANITIZERS=OFF", - "-DLLVM_ENABLE_LLD=True"])}, + "-DLLVM_ENABLE_LLD=True", + # lld tests cause us to hit thread limits + "-DLLVM_ENABLE_THREADS=OFF"])}, - # AArch64 Clang+LLVM+RT check-all + flang + test-suite + self-hosting - {'name' : "clang-cmake-aarch64-full", + # AArch64 check-all + flang + compiler-rt + test-suite 2-stage + {'name' : "clang-aarch64-full-2stage", 'tags' : ["clang"], - 'workernames' : ["linaro-aarch64-full"], - 'builddir': "clang-cmake-aarch64-full", + 'workernames' : ["linaro-clang-aarch64-full-2stage"], + 'builddir': "clang-aarch64-full-2stage", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, - checkout_compiler_rt=True, checkout_flang=True, checkout_lld=False, - test=True, useTwoStage=True, testStage1=False, runTestSuite=True, + env={ + 'NO_STOP_MESSAGE':'1', # For Fortran test-suite + }, testsuite_flags=[ '--cppflags', '-mcpu=cortex-a57', '--threads=32', '--build-threads=32'], extra_cmake_args=[ "-DCMAKE_C_FLAGS='-mcpu=cortex-a57'", "-DCMAKE_CXX_FLAGS='-mcpu=cortex-a57'", - "-DLLVM_TARGETS_TO_BUILD='ARM;AArch64'"])}, + "-DLLVM_LIT_ARGS='-v -j4'"])}, + + + # AArch64 Clang+LLVM+RT+LLD check-all + flang + test-suite w/SVE-Vector-Length-Agnostic + # Note that in this and other clang-aarch64-sve-* builders we set + # -mllvm -treat-scalable-fixed-error-as-warning=false to make compiler + # fail on non-critical SVE codegen issues. This helps us notice and fix + # SVE problems sooner rather than later. + {'name' : "clang-aarch64-sve-vla", + 'tags' : ["clang"], + 'workernames' : ["linaro-clang-aarch64-sve-vla"], + 'builddir': "clang-aarch64-sve-vla", + 'factory' : ClangBuilder.getClangCMakeBuildFactory( + clean=False, + checkout_flang=True, + runTestSuite=True, + env={ + 'NO_STOP_MESSAGE':'1', # For Fortran test-suite + }, + testsuite_flags=[ + '--cppflags', '-mcpu=a64fx -mllvm -scalable-vectorization=preferred -mllvm -treat-scalable-fixed-error-as-warning=false -O3', + '--threads=48', '--build-threads=48'], + extra_cmake_args=[ + "-DCMAKE_C_FLAGS='-mcpu=a64fx'", + "-DCMAKE_CXX_FLAGS='-mcpu=a64fx'", + "-DLLVM_ENABLE_LLD=True", + "-DLLVM_LIT_ARGS='-v -j12'"])}, + + # AArch64 Clang+LLVM+RT+LLD check-all + flang + test-suite 2-stage w/SVE-Vector-Length-Agnostic + {'name' : "clang-aarch64-sve-vla-2stage", + 'tags' : ["clang"], + 'workernames' : ["linaro-clang-aarch64-sve-vla-2stage"], + 'builddir': "clang-aarch64-sve-vla-2stage", + 'factory' : ClangBuilder.getClangCMakeBuildFactory( + clean=False, + checkout_flang=True, + useTwoStage=True, + testStage1=False, + runTestSuite=True, + env={ + 'NO_STOP_MESSAGE':'1', # For Fortran test-suite + }, + testsuite_flags=[ + '--cppflags', '-mcpu=a64fx -mllvm -scalable-vectorization=preferred -mllvm -treat-scalable-fixed-error-as-warning=false -O3', + '--threads=48', '--build-threads=48'], + extra_cmake_args=[ + "-DCMAKE_C_FLAGS='-mcpu=a64fx -mllvm -scalable-vectorization=preferred -mllvm -treat-scalable-fixed-error-as-warning=false'", + "-DCMAKE_CXX_FLAGS='-mcpu=a64fx -mllvm -scalable-vectorization=preferred -mllvm -treat-scalable-fixed-error-as-warning=false'", + "-DLLVM_ENABLE_LLD=True", + "-DLLVM_LIT_ARGS='-v -j12'"])}, + + # AArch64 Clang+LLVM+RT+LLD check-all + flang + test-suite w/SVE-Vector-Length-Specific + {'name' : "clang-aarch64-sve-vls", + 'tags' : ["clang"], + 'workernames' : ["linaro-clang-aarch64-sve-vls"], + 'builddir': "clang-aarch64-sve-vls", + 'factory' : ClangBuilder.getClangCMakeBuildFactory( + clean=False, + checkout_flang=True, + runTestSuite=True, + env={ + 'NO_STOP_MESSAGE':'1', # For Fortran test-suite + }, + testsuite_flags=[ + '--cppflags', '-mcpu=a64fx -msve-vector-bits=512 -mllvm -treat-scalable-fixed-error-as-warning=false -O3', + '--threads=48', '--build-threads=48'], + extra_cmake_args=[ + "-DCMAKE_C_FLAGS='-mcpu=a64fx'", + "-DCMAKE_CXX_FLAGS='-mcpu=a64fx'", + "-DLLVM_ENABLE_LLD=True", + "-DLLVM_LIT_ARGS='-v -j12'"])}, + + # AArch64 Clang+LLVM+RT+LLD check-all + flang + test-suite 2-stage w/SVE-Vector-Length-Specific + {'name' : "clang-aarch64-sve-vls-2stage", + 'tags' : ["clang"], + 'workernames' : ["linaro-clang-aarch64-sve-vls-2stage"], + 'builddir': "clang-aarch64-sve-vls-2stage", + 'factory' : ClangBuilder.getClangCMakeBuildFactory( + clean=False, + checkout_flang=True, + useTwoStage=True, + testStage1=False, + runTestSuite=True, + env={ + 'NO_STOP_MESSAGE':'1', # For Fortran test-suite + }, + testsuite_flags=[ + '--cppflags', '-mcpu=a64fx -msve-vector-bits=512 -mllvm -treat-scalable-fixed-error-as-warning=false -O3', + '--threads=48', '--build-threads=48'], + extra_cmake_args=[ + "-DCMAKE_C_FLAGS='-mcpu=a64fx -msve-vector-bits=512 -mllvm -treat-scalable-fixed-error-as-warning=false'", + "-DCMAKE_CXX_FLAGS='-mcpu=a64fx -msve-vector-bits=512 -mllvm -treat-scalable-fixed-error-as-warning=false'", + "-DLLVM_ENABLE_LLD=True", + "-DLLVM_LIT_ARGS='-v -j12'"])}, {'name' : "clang-arm64-windows-msvc-2stage", 'tags' : ["clang"], @@ -481,14 +579,20 @@ 'builddir': "clang-arm64-windows-msvc-2stage", 'factory' : ClangBuilder.getClangCMakeBuildFactory( vs="manual", - test=False, # Disable testing until MCJIT failures are fixed + checks=[], # Disable testing until MCJIT failures are fixed useTwoStage=True, testStage1=False, extra_cmake_args=[ - "-DLLVM_DEFAULT_TARGET_TRIPLE=aarch64-windows-msvc", - "-DLLVM_HOST_TRIPLE=aarch64-windows-msvc", + "-DCMAKE_TRY_COMPILE_CONFIGURATION=Release", + "-DLLVM_DEFAULT_TARGET_TRIPLE=aarch64-pc-windows-msvc", + "-DLLVM_HOST_TRIPLE=aarch64-pc-windows-msvc", "-DLLVM_TARGET_ARCH=AArch64", + "-DCMAKE_C_FLAGS=-fms-compatibility-version=19.27", + "-DCMAKE_CXX_FLAGS=-fms-compatibility-version=19.27", + # FIXME: compiler-rt\lib\sanitizer_common\sanitizer_unwind_win.cpp assumes WIN64 is x86_64, + # so, before that's fixed, disable everything that triggers its build. "-DCOMPILER_RT_BUILD_SANITIZERS=OFF", + "-DCOMPILER_RT_BUILD_MEMPROF=OFF", "-DCOMPILER_RT_BUILD_XRAY=OFF"])}, {'name' : 'clang-x64-windows-msvc', @@ -499,101 +603,135 @@ script="clang-windows.py", depends_on_projects=['llvm', 'clang', 'lld', 'debuginfo-tests'])}, - {'name' : "clang-ppc64be-linux-lnt", - 'tags' : ["clang", "ppc"], - 'workernames' : ["ppc64be-clang-lnt-test"], - 'builddir': "clang-ppc64be-lnt", + {'name' : "clang-m68k-linux", + 'tags' : ["clang"], + 'workernames' : ["debian-akiko-m68k"], + 'builddir': "clang-m68k-linux", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, checkout_lld=False, useTwoStage=False, - runTestSuite=True, stage1_config='Release', - nt_flags=['--threads=16', '--build-threads=16'], - extra_cmake_args=["-DLLVM_ENABLE_ASSERTIONS=ON"])}, + extra_cmake_args=[ + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=M68k"])}, - {'name' : "clang-ppc64be-linux-multistage", - 'tags' : ["clang", "ppc"], - 'workernames' : ["ppc64be-clang-multistage-test"], - 'builddir': "clang-ppc64be-multistage", + {'name' : "clang-m68k-linux-cross", + 'tags' : ["clang"], + 'workernames' : ["suse-gary-m68k-cross"], + 'builddir': "clang-m68k-linux-cross", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, checkout_lld=False, - useTwoStage=True, + useTwoStage=False, stage1_config='Release', - stage2_config='Release', - extra_cmake_args=['-DLLVM_ENABLE_ASSERTIONS=ON'])}, + extra_cmake_args=[ + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_TARGETS_TO_BUILD=X86", + "-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=M68k"])}, - {'name' : "clang-ppc64le-linux-lnt", - 'tags' : ["clang", "ppc", "ppc64le"], - 'workernames' : ["ppc64le-clang-lnt-test"], - 'builddir': "clang-ppc64le-lnt", + {'name' : "clang-mips64el-linux", + 'tags' : ["clang"], + 'workernames' : ["debian-tritium-mips64el"], + 'builddir': "clang-mips64el-linux", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, checkout_lld=False, useTwoStage=False, - runTestSuite=True, stage1_config='Release', - nt_flags=['--threads=16', '--build-threads=16'], - extra_cmake_args=["-DLLVM_ENABLE_ASSERTIONS=ON"])}, + extra_cmake_args=['-DLLVM_ENABLE_ASSERTIONS=ON', + '-DLLVM_PARALLEL_LINK_JOBS=4', + '-DLLVM_TARGETS_TO_BUILD=Mips'])}, - {'name' : "clang-ppc64le-linux-multistage", - 'tags' : ["clang", "ppc", "ppc64le"], - 'workernames' : ["ppc64le-clang-multistage-test"], - 'builddir': "clang-ppc64le-multistage", + {'name' : "clang-ppc64be-linux-test-suite", + 'tags' : ["clang", "ppc"], + 'workernames' : ["ppc64be-clang-test-suite"], + 'builddir': "clang-ppc64be-test-suite", + 'factory' : TestSuiteBuilder.getTestSuiteBuildFactory( + depends_on_projects=["llvm", "clang", "clang-tools-extra", + "compiler-rt"], + extra_configure_args=[ + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DCMAKE_BUILD_TYPE=Release", + "-DLLVM_LIT_ARGS=-v", + "-DLLVM_CCACHE_BUILD=ON"])}, + + {'name' : "clang-ppc64be-linux-multistage", + 'tags' : ["clang", "ppc"], + 'workernames' : ["ppc64be-clang-multistage-test"], + 'builddir': "clang-ppc64be-multistage", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, checkout_lld=False, useTwoStage=True, stage1_config='Release', stage2_config='Release', - extra_cmake_args=['-DLLVM_ENABLE_ASSERTIONS=ON', '-DBUILD_SHARED_LIBS=ON'])}, + extra_cmake_args=[ + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_CCACHE_BUILD=ON"])}, - {'name' : "clang-ppc64be-linux", - 'tags' : ["clang", "ppc"], - 'workernames' : ["ppc64be-clang-test"], - 'builddir': "clang-ppc64be", - 'factory' : ClangBuilder.getClangCMakeBuildFactory( - clean=False, - checkout_lld=False, - useTwoStage=False, - stage1_config='Release', - extra_cmake_args=["-DLLVM_ENABLE_ASSERTIONS=ON"])}, + {'name' : "clang-ppc64le-linux-test-suite", + 'tags' : ["clang", "ppc", "ppc64le"], + 'workernames' : ["ppc64le-clang-test-suite"], + 'builddir': "clang-ppc64le-test-suite", + 'factory' : TestSuiteBuilder.getTestSuiteBuildFactory( + depends_on_projects=["llvm", "clang", "clang-tools-extra", + "compiler-rt"], + extra_configure_args=[ + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DCMAKE_BUILD_TYPE=Release", + "-DLLVM_LIT_ARGS=-v", + "-DLLVM_CCACHE_BUILD=ON"])}, - {'name' : "clang-ppc64le-linux", + {'name' : "clang-ppc64le-linux-multistage", 'tags' : ["clang", "ppc", "ppc64le"], - 'workernames' : ["ppc64le-clang-test"], - 'builddir': "clang-ppc64le", + 'workernames' : ["ppc64le-clang-multistage-test"], + 'builddir': "clang-ppc64le-multistage", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, checkout_lld=False, - useTwoStage=False, + useTwoStage=True, stage1_config='Release', - extra_cmake_args=["-DLLVM_ENABLE_ASSERTIONS=ON"])}, + stage2_config='Release', + extra_cmake_args=[ + '-DLLVM_ENABLE_ASSERTIONS=ON', + '-DBUILD_SHARED_LIBS=ON', + '-DLLVM_CCACHE_BUILD=ON'])}, {'name' : "clang-ppc64le-rhel", 'tags' : ["clang", "ppc", "ppc64le"], 'workernames' : ["ppc64le-clang-rhel-test"], 'builddir': "clang-ppc64le-rhel", - 'factory' : ClangBuilder.getClangCMakeBuildFactory(clean=False, - checkout_clang_tools_extra=True, - checkout_compiler_rt=False, - checkout_lld=True, - checkout_libcxx=False, - useTwoStage=False, - runTestSuite=True, - stage1_config='Release', - nt_flags=['--threads=16', '--build-threads=16'], - extra_cmake_args=[ - '-DLLVM_ENABLE_PROJECTS=clang;llvm;clang-tools-extra;lld', - '-DLLVM_ENABLE_RUNTIMES=compiler-rt', + 'factory' : TestSuiteBuilder.getTestSuiteBuildFactory( + depends_on_projects=["llvm", "clang", "clang-tools-extra", "lld", "compiler-rt"], + enable_runtimes="auto", + extra_configure_args=[ "-DLLVM_ENABLE_ASSERTIONS=On", "-DCMAKE_C_COMPILER=clang", "-DCMAKE_CXX_COMPILER=clang++", "-DCLANG_DEFAULT_LINKER=lld", "-DCMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN:PATH=/opt/rh/devtoolset-7/root/usr", "-DCMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN:PATH=/opt/rh/devtoolset-7/root/usr", "-DLLVM_BINUTILS_INCDIR=/usr/include", "-DBUILD_SHARED_LIBS=ON", "-DLLVM_ENABLE_WERROR=ON", - '-DLLVM_LIT_ARGS=-vj 20'])}, + "-DCMAKE_BUILD_TYPE=Release", + "-DLLVM_LIT_ARGS=-vj 20"])}, + + {'name' : "clang-ppc64-aix", + 'tags' : ["clang", "aix", "ppc"], + 'workernames' : ["aix-ppc64"], + 'builddir': "clang-ppc64-aix", + 'factory' : TestSuiteBuilder.getTestSuiteBuildFactory( + depends_on_projects=["llvm", "clang", "compiler-rt"], + enable_runtimes="auto", + clean=False, + extra_configure_args=[ + "-DLLVM_ENABLE_ASSERTIONS=On", + "-DCMAKE_C_COMPILER=/usr/local/clang-15.0.0/bin/clang", + "-DCMAKE_CXX_COMPILER=/usr/local/clang-15.0.0/bin/clang++", + "-DPython3_EXECUTABLE:FILEPATH=/opt/freeware/bin/python3_64", + "-DLLVM_ENABLE_ZLIB=OFF", "-DLLVM_APPEND_VC_REV=OFF", + "-DLLVM_PARALLEL_LINK_JOBS=2", + "-DLLVM_ENABLE_WERROR=ON"]), + 'env' : {'OBJECT_MODE': '64'}}, {'name' : "clang-s390x-linux", 'tags' : ["clang"], @@ -606,6 +744,7 @@ useTwoStage=False, stage1_config='Release', extra_cmake_args=[ + "-DLLVM_CCACHE_BUILD=ON", "-DLLVM_ENABLE_ASSERTIONS=ON", "-DLLVM_LIT_ARGS=-v -j4 --param run_long_tests=true"])}, @@ -620,7 +759,9 @@ useTwoStage=True, stage1_config='Release', stage2_config='Release', - extra_cmake_args=['-DLLVM_ENABLE_ASSERTIONS=ON'])}, + extra_cmake_args=[ + "-DLLVM_CCACHE_BUILD=ON", + "-DLLVM_ENABLE_ASSERTIONS=ON"])}, {'name' : "clang-s390x-linux-lnt", 'tags' : ["clang"], @@ -634,7 +775,9 @@ runTestSuite=True, stage1_config='Release', testsuite_flags=['--threads=4', '--build-threads=4'], - extra_cmake_args=["-DLLVM_ENABLE_ASSERTIONS=ON"])}, + extra_cmake_args=[ + "-DLLVM_CCACHE_BUILD=ON", + "-DLLVM_ENABLE_ASSERTIONS=ON"])}, {'name' : "clang-sparc64-linux-multistage", 'tags' : ["clang"], @@ -659,7 +802,7 @@ checkout_clang_tools_extra=False, checkout_compiler_rt=False, checkout_lld=False, - env={'LD_LIBRARY_PATH': '/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/lib', 'PATH': ['/local/cmake-3.17.0/bin', '${PATH}']}, + env={'LD_LIBRARY_PATH': '/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/lib'}, extra_cmake_args=[ "-DCMAKE_BUILD_TYPE:STRING=Release", "-DLLVM_TARGETS_TO_BUILD:STRING=Hexagon", @@ -676,8 +819,8 @@ "-DWITH_POLLY:BOOL=OFF", "-DLINK_POLLY_INTO_TOOLS:BOOL=OFF", "-DPOLLY_BUILD_SHARED_LIB:BOOL=OFF", - "-DCMAKE_C_COMPILER:FILEPATH=/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/bin/clang", - "-DCMAKE_CXX_COMPILER:FILEPATH=/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/bin/clang++"])}, + "-DCMAKE_C_COMPILER:FILEPATH=/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/bin/clang", + "-DCMAKE_CXX_COMPILER:FILEPATH=/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/bin/clang++"])}, ## X86_64 AVX2 Clang+LLVM check-all + test-suite {'name' : "clang-cmake-x86_64-avx2-linux", @@ -689,7 +832,6 @@ checkout_clang_tools_extra=False, checkout_compiler_rt=False, checkout_lld=False, - test=True, useTwoStage=False, runTestSuite=True, nt_flags=['--cflag', '-march=broadwell', '--threads=80', '--build-threads=80'], @@ -710,7 +852,7 @@ checkout_clang_tools_extra=False, checkout_compiler_rt=False, checkout_lld=False, - test=False, + checks=[], useTwoStage=False, runTestSuite=True, nt_flags=['--cflag', '-march=broadwell', '--threads=1', '--build-threads=80', '--use-perf', @@ -721,7 +863,7 @@ "-DCMAKE_CXX_FLAGS='-march=broadwell'", "-DLLVM_TARGETS_TO_BUILD='X86'"], submitURL='http://lnt.llvm.org/submitRun', - testerName='LNT-Broadwell-AVX2-O3')}, + testerName='LNT-Broadwell-AVX2-O5')}, ## X86_64 Clang+LLVM Run test-suite targeting AVX512 on SDE (Emulator) {'name' : "clang-cmake-x86_64-sde-avx512-linux", @@ -733,7 +875,6 @@ checkout_clang_tools_extra=False, checkout_compiler_rt=False, checkout_lld=False, - test=True, useTwoStage=False, runTestSuite=True, nt_flags=['--cflag', '-march=skylake-avx512', '--threads=80', @@ -746,55 +887,6 @@ "-DCMAKE_CXX_FLAGS='-march=broadwell'", "-DLLVM_TARGETS_TO_BUILD='X86'"])}, - ## Armv7 build cache - {'name' : "clang-armv7-linux-build-cache", - 'tags' : ["clang"], - 'workernames' : ["packet-linux-armv7-slave-1"], - 'builddir': "clang-armv7-linux-build-cache", - 'factory' : ClangBuilder.getClangCMakeGCSBuildFactory( - stage1_config='Release', - clean=True, - checkout_compiler_rt=False, - test=False, - useTwoStage=False, - runTestSuite=False, - checkout_lld=True, - checkout_libcxx=True, - checkout_clang_tools_extra=False, - use_pixz_compression=False, - xz_compression_factor=0, - stage1_upload_directory='clang-armv7-linux', - extra_cmake_args=[ - "-DLLVM_TARGETS_TO_BUILD='ARM;AArch64;X86'", - "-DCMAKE_C_FLAGS='-mthumb'", - "-DCMAKE_CXX_FLAGS='-mthumb'", - ], - env={'BUCKET': 'llvm-build-artifacts'})}, - - ## AArch64 build cache - {'name' : "clang-aarch64-linux-build-cache", - 'tags' : ["clang"], - 'workernames' : ["packet-linux-aarch64-slave-1"], - 'builddir': "clang-aarch64-linux-build-cache", - 'factory' : ClangBuilder.getClangCMakeGCSBuildFactory( - stage1_config='Release', - clean=True, - checkout_compiler_rt=False, - test=False, - useTwoStage=False, - runTestSuite=False, - checkout_lld=True, - checkout_libcxx=True, - checkout_clang_tools_extra=False, - stage1_upload_directory='clang-aarch64-linux', - use_pixz_compression=True, - extra_cmake_args=[ - "-DLLVM_TARGETS_TO_BUILD='ARM;AArch64;X86'", - #"-DCMAKE_C_FLAGS=''", - #"-DCMAKE_CXX_FLAGS=''", - ], - env={'BUCKET': 'llvm-build-artifacts'})}, - {'name' : "llvm-avr-linux", 'tags' : ["clang"], 'workernames' : ["avr-build-01"], @@ -810,22 +902,64 @@ '-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=AVR', '-DBUILD_SHARED_LIBS=ON'])}, - {'name' : "clang-x64-ninja-win7", + {'name' : "clang-xcore-ubuntu-20-x64", 'tags' : ["clang"], - 'workernames' : ["windows7-buildbot"], - 'builddir': "clang-x64-ninja-win7", + 'workernames' : ["xcore-ubuntu20-x64"], + 'builddir': "clang-xcore-ubuntu-20-x64", 'factory' : ClangBuilder.getClangCMakeBuildFactory( - clean=False, + jobs=4, + checkout_clang_tools_extra=False, + checkout_compiler_rt=False, checkout_lld=False, - vs="autodetect", - vs_target_arch='x64', testStage1=True, - useTwoStage=True, + useTwoStage=False, stage1_config='Release', - stage2_config='Release', extra_cmake_args=[ - '-DLLVM_ENABLE_ASSERTIONS=ON', - '-DLLVM_TARGETS_TO_BUILD=X86'])}, + "-DLLVM_TARGETS_TO_BUILD:STRING=XCore", + "-DLLVM_DEFAULT_TARGET_TRIPLE:STRING=xcore-unknown-unknown-elf", + "-DLLVM_ENABLE_THREADS:BOOL=OFF"])}, + + {'name' : "llvm-clang-x86_64-sie-win", + 'tags' : ["llvm", "clang", "clang-tools-extra", "lld", "cross-project-tests"], + 'workernames' : ["sie-win-worker"], + 'builddir': "llvm-clang-x86_64-sie-win", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaWithMSVCBuildFactory( + vs="autodetect", + target_arch='x64', + depends_on_projects=['llvm','clang','clang-tools-extra','lld','cross-project-tests'], + clean=False, + extra_configure_args=[ + "-DCMAKE_BUILD_TYPE=Release", + "-DCLANG_ENABLE_ARCMT=OFF", + "-DCLANG_ENABLE_CLANGD=OFF", + "-DLLVM_DEFAULT_TARGET_TRIPLE=x86_64-sie-ps5", + "-DLLVM_INCLUDE_EXAMPLES=OFF", + "-DLLVM_TARGETS_TO_BUILD=X86", + "-DLLVM_VERSION_SUFFIX=", + "-DLLVM_BUILD_RUNTIME=OFF", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_LIT_ARGS=--verbose -j61", + "-DPYTHON_EXECUTABLE=C:\Python310\python.exe"])}, + + {'name': "cross-project-tests-sie-ubuntu", + 'tags' : ["clang", "llvm", "lldb", "cross-project-tests"], + 'workernames': ["sie-linux-worker"], + 'builddir': "cross-project-tests-sie-ubuntu", + 'factory': UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + depends_on_projects=['llvm','clang','lldb','cross-project-tests'], + checks = ['check-cross-project'], + extra_configure_args=[ + "-DCMAKE_C_COMPILER=gcc", + "-DCMAKE_CXX_COMPILER=g++", + "-DCMAKE_BUILD_TYPE=Release", + "-DCLANG_ENABLE_ARCMT=OFF", + "-DLLDB_ENABLE_PYTHON=TRUE", + "-DLLVM_CCACHE_BUILD=ON", + "-DLLVM_INCLUDE_EXAMPLES=OFF", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_LIT_ARGS=--verbose -j100", + "-DLLVM_TARGETS_TO_BUILD=X86", + "-DLLVM_USE_LINKER=gold"])}, # Polly builders. @@ -838,7 +972,7 @@ install=True, make='ninja', jobs=16, - env={'LD_LIBRARY_PATH': '/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/lib', 'PATH': ['/local/cmake-3.17.0/bin', '${PATH}']}, + env={'LD_LIBRARY_PATH': '/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/lib'}, extraCmakeArgs=[ "-G", "Ninja", "-DLLVM_TARGETS_TO_BUILD='ARM;AArch64'", @@ -846,12 +980,13 @@ "-DLLVM_TARGET_ARCH=arm-linux-gnueabi", "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_ENABLE_LIBCXX:BOOL=ON", - "-DCMAKE_C_COMPILER:FILEPATH=/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/bin/clang", - "-DCMAKE_CXX_COMPILER:FILEPATH=/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/bin/clang++"])}, + "-DPOLLY_ENABLE_GPGPU_CODEGEN=OFF", + "-DCMAKE_C_COMPILER:FILEPATH=/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/bin/clang", + "-DCMAKE_CXX_COMPILER:FILEPATH=/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/bin/clang++"])}, {'name' : "polly-x86_64-linux", 'tags' : ["polly"], - 'workernames' : ["polly-x86_64-fdcserver", "polly-x86_64-gce1"], + 'workernames' : ["polly-x86_64-gce1"], 'builddir': "polly-x86_64-linux", 'factory' : PollyBuilder.getPollyBuildFactory( clean=False, @@ -859,54 +994,204 @@ make='ninja', extraCmakeArgs=[ "-G", "Ninja", + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_TARGETS_TO_BUILD='X86;NVPTX'", "-DCLANG_ENABLE_ARCMT=OFF", "-DCLANG_ENABLE_STATIC_ANALYZER=OFF", + "-DCLANG_ENABLE_OBJC_REWRITER=OFF", + "-DLLVM_ENABLE_LLD=ON", + "-DPOLLY_ENABLE_GPGPU_CODEGEN=ON" ])}, - {'name' : "polly-x86_64-linux-test-suite", + {'name' : "polly-x86_64-linux-plugin", 'tags' : ["polly"], - 'workernames' : ["polly-x86_64-fdcserver", "polly-x86_64-gce2"], - 'builddir': "polly-x86_64-linux-test-suite", + 'workernames' : ["polly-x86_64-gce1"], + 'builddir': "polly-x86_64-linux-plugin", 'factory' : PollyBuilder.getPollyBuildFactory( clean=False, install=False, make='ninja', extraCmakeArgs=[ "-G", "Ninja", + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_TARGETS_TO_BUILD='X86;NVPTX'", "-DCLANG_ENABLE_ARCMT=OFF", "-DCLANG_ENABLE_STATIC_ANALYZER=OFF", - ], - testsuite=True, - extraTestsuiteCmakeArgs=["-G", "Ninja"] - )}, + "-DCLANG_ENABLE_OBJC_REWRITER=OFF", + "-DLLVM_ENABLE_LLD=ON", + "-DLLVM_POLLY_LINK_INTO_TOOLS=OFF", + "-DPOLLY_ENABLE_GPGPU_CODEGEN=OFF" # Not all required symbols available in opt executable + ])}, -# AOSP builders. + {'name' : "polly-x86_64-linux-noassert", + 'tags' : ["polly"], + 'workernames' : ["polly-x86_64-gce1"], + 'builddir': "polly-x86_64-linux-noassert", + 'factory' : PollyBuilder.getPollyBuildFactory( + clean=False, + install=False, + make='ninja', + extraCmakeArgs=[ + "-G", "Ninja", + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", + "-DLLVM_ENABLE_ASSERTIONS=False", + "-DLLVM_TARGETS_TO_BUILD='X86;NVPTX'", + "-DCLANG_ENABLE_ARCMT=OFF", + "-DCLANG_ENABLE_STATIC_ANALYZER=OFF", + "-DCLANG_ENABLE_OBJC_REWRITER=OFF", + "-DLLVM_ENABLE_LLD=ON", + "-DPOLLY_ENABLE_GPGPU_CODEGEN=ON" + ])}, - {'name' : "aosp-O3-polly-before-vectorizer-unprofitable", - 'tags' : ["polly", "aosp"], - 'workernames' : ["hexagon-build-03"], - 'builddir': "aosp", - 'factory' : AOSPBuilder.getAOSPBuildFactory( - device="angler", - extra_cmake_args=[ + {'name' : "polly-x86_64-linux-shared", + 'tags' : ["polly"], + 'workernames' : ["polly-x86_64-gce2"], + 'builddir': "polly-x86_64-linux-shared", + 'factory' : PollyBuilder.getPollyBuildFactory( + clean=False, + install=False, + make='ninja', + extraCmakeArgs=[ + "-G", "Ninja", + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", + "-DLLVM_ENABLE_ASSERTIONS=True", + "-DLLVM_TARGETS_TO_BUILD='X86;NVPTX'", + "-DCLANG_ENABLE_ARCMT=OFF", + "-DCLANG_ENABLE_STATIC_ANALYZER=OFF", + "-DCLANG_ENABLE_OBJC_REWRITER=OFF", + "-DLLVM_ENABLE_LLD=ON", + "-DBUILD_SHARED_LIBS=ON", + "-DPOLLY_ENABLE_GPGPU_CODEGEN=ON" + ])}, + + {'name' : "polly-x86_64-linux-shared-plugin", + 'tags' : ["polly"], + 'workernames' : ["polly-x86_64-gce2"], + 'builddir': "polly-x86_64-linux-shared-plugin", + 'factory' : PollyBuilder.getPollyBuildFactory( + clean=False, + install=False, + make='ninja', + extraCmakeArgs=[ + "-G", "Ninja", + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", + "-DLLVM_ENABLE_ASSERTIONS=True", + "-DLLVM_TARGETS_TO_BUILD='X86;NVPTX'", + "-DCLANG_ENABLE_ARCMT=OFF", + "-DCLANG_ENABLE_STATIC_ANALYZER=OFF", + "-DCLANG_ENABLE_OBJC_REWRITER=OFF", + "-DLLVM_ENABLE_LLD=ON", + "-DBUILD_SHARED_LIBS=ON", + "-DLLVM_POLLY_LINK_INTO_TOOLS=OFF", + "-DPOLLY_ENABLE_GPGPU_CODEGEN=ON" + ])}, + + {'name' : "polly-x86_64-linux-shlib", + 'tags' : ["polly"], + 'workernames' : ["polly-x86_64-gce2"], + 'builddir': "polly-x86_64-linux-shlib", + 'factory' : PollyBuilder.getPollyBuildFactory( + clean=False, + install=False, + make='ninja', + extraCmakeArgs=[ + "-G", "Ninja", + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", + "-DLLVM_ENABLE_ASSERTIONS=True", + "-DLLVM_TARGETS_TO_BUILD='X86;NVPTX'", + "-DCLANG_ENABLE_ARCMT=OFF", + "-DCLANG_ENABLE_STATIC_ANALYZER=OFF", + "-DCLANG_ENABLE_OBJC_REWRITER=OFF", + "-DLLVM_ENABLE_LLD=ON", + "-DLLVM_BUILD_LLVM_DYLIB=ON", + "-DLLVM_LINK_LLVM_DYLIB=ON", + "-DPOLLY_ENABLE_GPGPU_CODEGEN=ON" + ])}, + + {'name' : "polly-x86_64-linux-shlib-plugin", + 'tags' : ["polly"], + 'workernames' : ["polly-x86_64-gce2"], + 'builddir': "polly-x86_64-linux-shlib-plugin", + 'factory' : PollyBuilder.getPollyBuildFactory( + clean=False, + install=False, + make='ninja', + extraCmakeArgs=[ + "-G", "Ninja", + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", + "-DLLVM_ENABLE_ASSERTIONS=True", + "-DLLVM_TARGETS_TO_BUILD='X86;NVPTX'", + "-DCLANG_ENABLE_ARCMT=OFF", + "-DCLANG_ENABLE_STATIC_ANALYZER=OFF", + "-DCLANG_ENABLE_OBJC_REWRITER=OFF", + "-DLLVM_ENABLE_LLD=ON", + "-DLLVM_BUILD_LLVM_DYLIB=ON", + "-DLLVM_LINK_LLVM_DYLIB=ON", + "-DLLVM_POLLY_LINK_INTO_TOOLS=OFF", + "-DPOLLY_ENABLE_GPGPU_CODEGEN=ON" + ])}, + + {'name' : "polly-x86_64-linux-test-suite", + 'tags' : ["polly"], + 'workernames' : ["polly-x86_64-fdcserver", "minipc-1050ti-linux"], + 'builddir': "polly-x86_64-linux-test-suite", + 'factory' : PollyBuilder.getPollyBuildFactory( + clean=False, + install=False, + make='ninja', + extraCmakeArgs=[ + "-G", "Ninja", + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", + "-DLLVM_ENABLE_ASSERTIONS=True", + "-DLLVM_TARGETS_TO_BUILD='X86;NVPTX'", + "-DCLANG_ENABLE_ARCMT=OFF", + "-DCLANG_ENABLE_STATIC_ANALYZER=OFF", + "-DCLANG_ENABLE_OBJC_REWRITER=OFF" + ], + testsuite=True, + extraTestsuiteCmakeArgs=[ + "-G", "Ninja", + "-DTEST_SUITE_COLLECT_COMPILE_TIME=OFF", + "-DTEST_SUITE_COLLECT_STATS=OFF", + "-DTEST_SUITE_COLLECT_CODE_SIZE=OFF", + WithProperties("-DTEST_SUITE_EXTERNALS_DIR=%(builddir)s/../../test-suite-externals"), + ] + )}, + +# AOSP builders. + + {'name' : "aosp-O3-polly-before-vectorizer-unprofitable", + 'tags' : ["polly", "aosp"], + 'workernames' : ["hexagon-build-03"], + 'builddir': "aosp", + 'factory' : AOSPBuilder.getAOSPBuildFactory( + device="angler", + extra_cmake_args=[ "-G", "Ninja", "-DLLVM_TARGETS_TO_BUILD='ARM;AArch64'", "-DLLVM_DEFAULT_TARGET_TRIPLE=arm-linux-androideabi", "-DLLVM_TARGET_ARCH=arm-linux-androideabi", "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_ENABLE_LIBCXX:BOOL=ON", - "-DCMAKE_C_COMPILER:FILEPATH=/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/bin/clang", - "-DCMAKE_CXX_COMPILER:FILEPATH=/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/bin/clang++"], + "-DPOLLY_ENABLE_GPGPU_CODEGEN=OFF", + "-DCMAKE_C_COMPILER:FILEPATH=/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/bin/clang", + "-DCMAKE_CXX_COMPILER:FILEPATH=/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/bin/clang++"], timeout=240, target_clang=None, target_flags="-Wno-error -O3 -mllvm -polly -mllvm -polly-position=before-vectorizer -mllvm -polly-process-unprofitable -fcommon", - jobs=8, + jobs=32, extra_make_args=None, - env={'LD_LIBRARY_PATH': '/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/lib', 'PATH': ['/local/cmake-3.17.0/bin', '${PATH}']}, + env={'LD_LIBRARY_PATH': '/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/lib'}, clean=False, sync=False, patch=None)}, @@ -922,14 +1207,15 @@ make='ninja', jobs=16, checkAll=True, - env={'LD_LIBRARY_PATH': '/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/lib', 'PATH': ['/local/cmake-3.17.0/bin', '${PATH}']}, + env={'LD_LIBRARY_PATH': '/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/lib'}, extraCmakeArgs=[ "-G", "Ninja", "-DLLVM_REVERSE_ITERATION:BOOL=ON", "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_ENABLE_LIBCXX:BOOL=ON", - "-DCMAKE_C_COMPILER:FILEPATH=/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/bin/clang", - "-DCMAKE_CXX_COMPILER:FILEPATH=/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/bin/clang++"])}, + "-DPOLLY_ENABLE_GPGPU_CODEGEN=ON", + "-DCMAKE_C_COMPILER:FILEPATH=/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/bin/clang", + "-DCMAKE_CXX_COMPILER:FILEPATH=/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/bin/clang++"])}, # LLDB builders. @@ -943,30 +1229,33 @@ '-DLLVM_ENABLE_ASSERTIONS=True', '-DLLVM_USE_LINKER=gold', '-DLLDB_ENABLE_PYTHON=True', + '-DLLDB_TEST_USER_ARGS=-t', '-DPYTHON_EXECUTABLE=/usr/bin/python3', '-DCMAKE_C_COMPILER=clang', '-DCMAKE_CXX_COMPILER=clang++'])}, {'name' : "lldb-aarch64-ubuntu", 'tags' : ["lldb"], - 'workernames' : ["linaro-aarch64-lldb"], - 'builddir': "lldb-cmake-aarch64", + 'workernames' : ["linaro-lldb-aarch64-ubuntu"], + 'builddir': "lldb-aarch64-ubuntu", 'factory' : LLDBBuilder.getLLDBCMakeBuildFactory( test=True, + clean=True, extra_cmake_args=[ '-DLLVM_ENABLE_ASSERTIONS=True', - '-DLLVM_LIT_ARGS=-svj 8', + '-DLLVM_LIT_ARGS=-svj 4', '-DLLVM_USE_LINKER=gold'])}, {'name' : "lldb-arm-ubuntu", 'tags' : ["lldb"], - 'workernames' : ["linaro-arm-lldb"], - 'builddir': "lldb-cmake-arm", + 'workernames' : ["linaro-lldb-arm-ubuntu"], + 'builddir': "lldb-arm-ubuntu", 'factory' : LLDBBuilder.getLLDBCMakeBuildFactory( test=True, + clean=True, extra_cmake_args=[ '-DLLVM_ENABLE_ASSERTIONS=True', - '-DLLVM_LIT_ARGS=-svj 8', + '-DLLVM_LIT_ARGS=-svj 4', '-DLLVM_USE_LINKER=gold'])}, {'name' : "lldb-x64-windows-ninja", @@ -975,26 +1264,36 @@ 'builddir': "lldb-x64-windows-ninja", 'factory' : LLDBBuilder.getLLDBCMakeBuildFactory( clean=True, - python_source_dir=r'"C:\Program Files (x86)\Microsoft Visual Studio\Shared\Python36_64"', target_arch='x64', vs="autodetect", test=True, extra_cmake_args=[ + '-DLLDB_ENABLE_PYTHON=TRUE', + '-DLLDB_TEST_USER_ARGS=--skip-category=watchpoint', '-DLLVM_ENABLE_ASSERTIONS=OFF', '-DLLVM_ENABLE_ZLIB=FALSE', - '-DLLDB_ENABLE_PYTHON=TRUE'])}, - -# LLD builders. + '-DLLVM_LIT_ARGS=-vj 8'])}, - {'name' : "lld-x86_64-darwin", - 'tags' : ["lld"], - 'workernames' : ["as-worker-3"], - 'builddir': "lld-x86_64-darwin", - 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + {'name' : "lldb-aarch64-windows", + 'tags' : ["lldb"], + 'workernames' : ["linaro-lldb-aarch64-windows"], + 'builddir': "lldb-aarch64-windows", + 'factory' : LLDBBuilder.getLLDBCMakeBuildFactory( clean=True, - depends_on_projects=['llvm', 'lld'], - extra_configure_args=[ - '-DLLVM_ENABLE_WERROR=OFF'])}, + test=True, + extra_cmake_args=[ + '-DCLANG_DEFAULT_LINKER=lld', + '-DLLDB_ENABLE_PYTHON=TRUE', + '-DLLDB_TEST_USER_ARGS=--skip-category=watchpoint', + '-DLLVM_NATIVE_ARCH=AArch64', + '-DLLVM_HOST_TRIPLE=aarch64-windows-msvc', + '-DLLVM_DEFAULT_TARGET_TRIPLE=aarch64-windows-msvc', + '-DLLVM_FORCE_USE_OLD_TOOLCHAIN=True', + '-DLLVM_ENABLE_ASSERTIONS=OFF', + '-DLLVM_ENABLE_ZLIB=FALSE', + '-DLLVM_LIT_ARGS=-vj 4'])}, + +# LLD builders. {'name' : "lld-x86_64-win", 'tags' : ["lld"], @@ -1006,17 +1305,6 @@ extra_configure_args = [ '-DLLVM_ENABLE_WERROR=OFF'])}, - {'name' : "lld-x86_64-freebsd", - 'tags' : ["lld"], - 'workernames' : ["as-worker-4"], - 'builddir': "lld-x86_64-freebsd", - 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( - depends_on_projects=['llvm', 'lld'], - extra_configure_args=[ - '-DCMAKE_EXE_LINKER_FLAGS=-lcxxrt', - '-DLLVM_ENABLE_WERROR=OFF'], - env={'CXXFLAGS' : "-std=c++11 -stdlib=libc++"})}, - {'name' : "ppc64le-lld-multistage-test", 'tags' : ["lld", "ppc", "ppc64le"], 'workernames' : ["ppc64le-lld-multistage-test"], @@ -1024,7 +1312,8 @@ 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaMultistageBuildFactory( extra_configure_args=[ '-DLLVM_ENABLE_ASSERTIONS=ON', - '-DLLVM_LIT_ARGS=-svj 256'], + '-DLLVM_LIT_ARGS=-svj 256', + '-DLLVM_CCACHE_BUILD=ON'], depends_on_projects=['llvm', 'clang', 'lld'])}, {'name' : "lld-x86_64-ubuntu-fast", @@ -1050,28 +1339,122 @@ 'tags' : ["clang","lld","LTO"], 'workernames' : ["as-worker-92"], 'builddir': "clang-with-thin-lto-ubuntu", - 'factory' : ClangLTOBuilder.getClangWithLTOBuildFactory(jobs=72, lto='thin')}, + 'factory' : ClangLTOBuilder.getClangWithLTOBuildFactory( + jobs=72, + lto='thin', + extra_configure_args_lto_stage=[ + '-DLLVM_ENABLE_ASSERTIONS=OFF', + ])}, {'name' : "clang-with-thin-lto-wpd-ubuntu", 'tags' : ["clang","lld","LTO"], - 'workernames' : ["thinlto-x86-64-bot1"], + 'workernames' : ["thinlto-x86-64-bot1", "thinlto-x86-64-bot2"], 'builddir': "clang-with-thin-lto-wpd-ubuntu", 'factory' : ClangLTOBuilder.getClangWithLTOBuildFactory( jobs=72, lto='thin', extra_configure_args=[ '-DLLVM_CCACHE_BUILD=ON', - '-DCMAKE_CXX_FLAGS="-O3 -Xclang -fwhole-program-vtables -fno-split-lto-unit"', - '-DCMAKE_C_FLAGS="-O3 -Xclang -fwhole-program-vtables -fno-split-lto-unit"', - '-DCMAKE_EXE_LINKER_FLAGS="-Wl,--lto-whole-program-visibility"'])}, + ], + extra_configure_args_lto_stage=[ + '-DCMAKE_CXX_FLAGS=-O3 -Xclang -fwhole-program-vtables -fno-split-lto-unit', + '-DCMAKE_C_FLAGS=-O3 -Xclang -fwhole-program-vtables -fno-split-lto-unit', + '-DCMAKE_EXE_LINKER_FLAGS=-Wl,--lto-whole-program-visibility -fuse-ld=lld'])}, {'name' : "clang-with-lto-ubuntu", 'tags' : ["clang","lld","LTO"], 'workernames' : ["as-worker-91"], 'builddir': "clang-with-lto-ubuntu", - 'factory' : ClangLTOBuilder.getClangWithLTOBuildFactory(jobs=72)}, + 'factory' : ClangLTOBuilder.getClangWithLTOBuildFactory( + jobs=72, + extra_configure_args_lto_stage=[ + '-DLLVM_PARALLEL_LINK_JOBS=14', + ])}, +] + +# Common builders options for MLIR. +mlir_ubuntu_workers = [f"mlir-ubuntu-worker{i}" for i in range(5)] +mlir_default_cmake_options = [ + '-DLLVM_CCACHE_BUILD=ON', + '-DLLVM_ENABLE_PROJECTS=mlir', + '-DLLVM_TARGETS_TO_BUILD=host;NVPTX;AMDGPU', + '-DLLVM_BUILD_EXAMPLES=ON', + '-DMLIR_INCLUDE_INTEGRATION_TESTS=ON', + '-DMLIR_ENABLE_BINDINGS_PYTHON=ON', +] + +all += [ + + {'name' : "mlir-ubuntu-asan-ubsan-clang-12.0", + 'tags' : ["mlir"], + 'workernames' : mlir_ubuntu_workers, + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + llvm_srcdir="llvm.src", + obj_dir="llvm.obj", + clean=True, + targets = ['check-mlir-build-only'], + checks = ['check-mlir'], + depends_on_projects=['llvm','mlir'], + extra_configure_args= mlir_default_cmake_options + [ + '-DLLVM_ENABLE_LLD=ON', + '-DBUILD_SHARED_LIBS=ON', + '-DLLVM_USE_SANITIZER=Address;Undefined', + '-DCMAKE_C_COMPILER=/compilers/clang-12.0/bin/clang', + '-DCMAKE_CXX_COMPILER=/compilers/clang-12.0/bin/clang++', + ])}, + + {'name' : "mlir-ubuntu-gcc5.5-shared-libs", + 'tags' : ["mlir"], + 'workernames' : mlir_ubuntu_workers, + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + llvm_srcdir="llvm.src", + obj_dir="llvm.obj", + clean=True, + targets = ['check-mlir-build-only'], + checks = ['check-mlir'], + depends_on_projects=['llvm','mlir'], + extra_configure_args= mlir_default_cmake_options + [ + '-DBUILD_SHARED_LIBS=ON', + '-DCMAKE_C_COMPILER=/usr/bin/gcc-5', + '-DCMAKE_CXX_COMPILER=/usr/bin/g++-5', + ])}, + + {'name' : "mlir-ubuntu-clang-5-link-llvm-dylib", + 'tags' : ["mlir"], + 'workernames' : mlir_ubuntu_workers, + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + llvm_srcdir="llvm.src", + obj_dir="llvm.obj", + clean=True, + targets = ['check-mlir-build-only'], + checks = ['check-mlir'], + depends_on_projects=['llvm','mlir'], + extra_configure_args= mlir_default_cmake_options + [ + '-DLLVM_ENABLE_LLD=OFF', # lld-5 is subject to http://llvm.org/pr49915 + '-DLLVM_LINK_LLVM_DYLIB=ON', + '-DCMAKE_C_COMPILER=/compilers/clang-5.0/bin/clang', + '-DCMAKE_CXX_COMPILER=/compilers/clang-5.0/bin/clang++', + ])}, -# Builders for MLIR. + {'name' : "mlir-ubuntu-gcc11-release-intel-sde", + 'tags' : ["mlir"], + 'workernames' : mlir_ubuntu_workers, + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + llvm_srcdir="llvm.src", + obj_dir="llvm.obj", + clean=True, + targets = ['check-mlir-build-only'], + checks = ['check-mlir'], + depends_on_projects=['llvm','mlir'], + extra_configure_args= mlir_default_cmake_options + [ + '-DLLVM_ENABLE_LLD=ON', + '-DLLVM_ENABLE_ASSERTIONS=OFF', + '-DCMAKE_C_COMPILER=/usr/bin/gcc-11', + '-DCMAKE_CXX_COMPILER=/usr/bin/g++-11', + '-DMLIR_RUN_X86VECTOR_TESTS=ON', + '-DMLIR_RUN_AMX_TESTS=ON', + '-DINTEL_SDE_EXECUTABLE=/intel_sde/sde64' + ])}, {'name' : "mlir-nvidia", 'tags' : ["mlir"], @@ -1081,16 +1464,21 @@ llvm_srcdir="llvm.src", obj_dir="llvm.obj", clean=True, + targets = ['check-mlir-build-only'], + checks = ['check-mlir'], depends_on_projects=['llvm','mlir'], extra_configure_args=[ '-DLLVM_BUILD_EXAMPLES=ON', '-DLLVM_TARGETS_TO_BUILD=host;NVPTX', '-DLLVM_ENABLE_PROJECTS=mlir', - '-DMLIR_CUDA_RUNNER_ENABLED=1', + '-DMLIR_ENABLE_CUDA_RUNNER=1', + '-DMLIR_INCLUDE_INTEGRATION_TESTS=ON', '-DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc', - '-DMLIR_VULKAN_RUNNER_ENABLED=1', + '-DMLIR_ENABLE_VULKAN_RUNNER=1', '-DBUILD_SHARED_LIBS=ON', '-DLLVM_CCACHE_BUILD=ON', + '-DMLIR_ENABLE_BINDINGS_PYTHON=ON', + '-DMLIR_RUN_CUDA_TENSOR_CORE_TESTS=ON', ], env={ 'CC':'clang', @@ -1104,98 +1492,217 @@ 'builddir': "mlir-x64-windows-ninja", 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaWithMSVCBuildFactory( clean=True, + targets = ['check-mlir-build-only'], + checks = ['check-mlir'], depends_on_projects=['llvm','mlir'], vs="autodetect", - checks=['check-mlir'], extra_configure_args=[ "-DLLVM_BUILD_EXAMPLES=ON", "-DLLVM_ENABLE_PROJECTS=mlir", + "-DMLIR_ENABLE_BINDINGS_PYTHON=ON", + "-DLLVM_ENABLE_WERROR=ON", "-DLLVM_TARGETS_TO_BUILD='host;NVPTX;AMDGPU'", ])}, {'name' : 'ppc64le-mlir-rhel-clang', 'tags' : ["mlir", "ppc", "ppc64le"], 'collapseRequests' : False, - 'workernames' : ['ppc64le-flang-mlir-rhel-test'], + 'workernames' : ['ppc64le-mlir-rhel-test'], 'builddir': 'ppc64le-mlir-rhel-clang-build', 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( clean=True, depends_on_projects=['llvm', 'mlir'], - checks=['check-mlir'], + targets = ['check-mlir-build-only'], + checks = ['check-mlir'], extra_configure_args=[ '-DLLVM_TARGETS_TO_BUILD=PowerPC', '-DLLVM_INSTALL_UTILS=ON', '-DCMAKE_CXX_STANDARD=17', '-DLLVM_ENABLE_PROJECTS=mlir', '-DLLVM_LIT_ARGS=-vj 256', + '-DLLVM_CCACHE_BUILD=ON', ], env={ 'CC': 'clang', 'CXX': 'clang++', 'LD': 'lld', + 'LD_LIBRARY_PATH': '/usr/lib64', })}, + {'name' : 'mlir-s390x-linux', + 'tags' : ["mlir", "s390x"], + 'workernames' : ["systemz-1"], + 'builddir': 'mlir-s390x-linux', + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + clean=True, + depends_on_projects=['llvm', 'mlir'], + checks=['check-mlir'], + extra_configure_args=[ + "-DLLVM_CCACHE_BUILD=ON", + '-DLLVM_TARGETS_TO_BUILD=SystemZ', + '-DLLVM_ENABLE_PROJECTS=mlir', + '-DLLVM_LIT_ARGS=-vj 4', + ])}, + # Sanitizer builders. +# +# bootstrap-asan, bootstrap-msan, and sanitizer-x86_64-linux-fast have steps +# with large memory usage, so assign them to different workers. {'name' : "sanitizer-x86_64-linux", 'tags' : ["sanitizer"], - 'workernames' : ["sanitizer-buildbot1", "sanitizer-buildbot2"], + 'workernames' : [ + "sanitizer-buildbot1", + "sanitizer-buildbot2", + ], 'builddir': "sanitizer-x86_64-linux", 'factory' : SanitizerBuilder.getSanitizerBuildFactory()}, {'name' : "sanitizer-x86_64-linux-fast", 'tags' : ["sanitizer"], - 'workernames' : ["sanitizer-buildbot1", "sanitizer-buildbot2"], + 'workernames' : [ + "sanitizer-buildbot3", + "sanitizer-buildbot4", + ], 'builddir': "sanitizer-x86_64-linux-fast", - 'factory' : SanitizerBuilder.getSanitizerBuildFactory()}, + 'factory' : SanitizerBuilder.getSanitizerBuildFactory( + extra_depends_on_projects=["mlir", "clang-tools-extra"] + )}, - {'name' : "sanitizer-x86_64-linux-bootstrap", + {'name' : "sanitizer-x86_64-linux-bootstrap-asan", 'tags' : ["sanitizer"], - 'workernames' : ["sanitizer-buildbot3", "sanitizer-buildbot4"], - 'builddir': "sanitizer-x86_64-linux-bootstrap", - 'factory' : SanitizerBuilder.getSanitizerBuildFactory()}, + 'workernames' : [ + "sanitizer-buildbot1", + "sanitizer-buildbot2", + ], + 'builddir': "sanitizer-x86_64-linux-bootstrap-asan", + 'factory' : SanitizerBuilder.getSanitizerBuildFactory( + extra_depends_on_projects=["mlir", "clang-tools-extra"] + )}, + + {'name' : "sanitizer-x86_64-linux-bootstrap-msan", + 'tags' : ["sanitizer"], + 'workernames' : [ + "sanitizer-buildbot5", + "sanitizer-buildbot6", + ], + 'builddir': "sanitizer-x86_64-linux-bootstrap-msan", + 'factory' : SanitizerBuilder.getSanitizerBuildFactory( + extra_depends_on_projects=["mlir", "clang-tools-extra"] + )}, {'name' : "sanitizer-x86_64-linux-bootstrap-ubsan", 'tags' : ["sanitizer"], - 'workernames' : ["sanitizer-buildbot3", "sanitizer-buildbot4"], + 'workernames' : [ + "sanitizer-buildbot3", + "sanitizer-buildbot4", + ], 'builddir': "sanitizer-x86_64-linux-bootstrap-ubsan", - 'factory' : SanitizerBuilder.getSanitizerBuildFactory()}, + 'factory' : SanitizerBuilder.getSanitizerBuildFactory( + extra_depends_on_projects=["mlir", "clang-tools-extra"] + )}, - {'name' : "sanitizer-x86_64-linux-bootstrap-msan", + {'name' : "sanitizer-x86_64-linux-autoconf", 'tags' : ["sanitizer"], - 'workernames' : ["sanitizer-buildbot7", "sanitizer-buildbot8"], - 'builddir': "sanitizer-x86_64-linux-bootstrap-msan", + 'workernames' : [ + "sanitizer-buildbot5", + "sanitizer-buildbot6", + ], + 'builddir': "sanitizer-x86_64-linux-autoconf", 'factory' : SanitizerBuilder.getSanitizerBuildFactory()}, - {'name' : "sanitizer-x86_64-linux-autoconf", + {'name' : "sanitizer-x86_64-linux-qemu", 'tags' : ["sanitizer"], - 'workernames' : ["sanitizer-buildbot7", "sanitizer-buildbot8"], - 'builddir': "sanitizer-x86_64-linux-autoconf", + 'workernames' : [ + "sanitizer-buildbot3", + "sanitizer-buildbot4", + ], + 'builddir': "sanitizer-x86_64-linux-qemu", 'factory' : SanitizerBuilder.getSanitizerBuildFactory()}, {'name' : "sanitizer-x86_64-linux-fuzzer", 'tags' : ["sanitizer"], - 'workernames' : ["sanitizer-buildbot7", "sanitizer-buildbot8"], + 'workernames' : [ + "sanitizer-buildbot1", + "sanitizer-buildbot2", + ], 'builddir': "sanitizer-x86_64-linux-fuzzer", 'factory' : SanitizerBuilder.getSanitizerBuildFactory()}, {'name' : "sanitizer-x86_64-linux-android", 'tags' : ["sanitizer"], - 'workernames' : ["sanitizer-buildbot6"], + 'workernames' : [ + "sanitizer-buildbot-android", + ], 'builddir': "sanitizer-x86_64-linux-android", 'factory' : SanitizerBuilder.getSanitizerBuildFactory()}, + {'name' : "sanitizer-aarch64-linux-bootstrap-asan", + 'tags' : ["sanitizer"], + 'workernames' : [ + "sanitizer-buildbot7", + ], + 'builddir': "sanitizer-aarch64-linux-bootstrap-asan", + 'factory' : SanitizerBuilder.getSanitizerBuildFactory( + extra_depends_on_projects=["mlir", "clang-tools-extra"] + )}, + + {'name' : "sanitizer-aarch64-linux-bootstrap-hwasan", + 'tags' : ["sanitizer"], + 'workernames' : [ + "sanitizer-buildbot7", + "sanitizer-buildbot8", + ], + 'builddir': "sanitizer-aarch64-linux-bootstrap-hwasan", + 'factory' : SanitizerBuilder.getSanitizerBuildFactory( + extra_depends_on_projects=["mlir", "clang-tools-extra"] + )}, + + {'name' : "sanitizer-aarch64-linux-bootstrap-msan", + 'tags' : ["sanitizer"], + 'workernames' : [ + "sanitizer-buildbot8", + ], + 'builddir': "sanitizer-aarch64-linux-bootstrap-msan", + 'factory' : SanitizerBuilder.getSanitizerBuildFactory( + extra_depends_on_projects=["mlir", "clang-tools-extra"] + )}, + + {'name' : "sanitizer-aarch64-linux-bootstrap-ubsan", + 'tags' : ["sanitizer"], + 'workernames' : [ + "sanitizer-buildbot8", + ], + 'builddir': "sanitizer-aarch64-linux-bootstrap-ubsan", + 'factory' : SanitizerBuilder.getSanitizerBuildFactory( + extra_depends_on_projects=["mlir", "clang-tools-extra"] + )}, + + {'name' : "sanitizer-aarch64-linux-fuzzer", + 'tags' : ["sanitizer"], + 'workernames' : [ + "sanitizer-buildbot7", + ], + 'builddir': "sanitizer-aarch64-linux-fuzzer", + 'factory' : SanitizerBuilder.getSanitizerBuildFactory()}, + {'name' : "sanitizer-ppc64be-linux", 'tags' : ["sanitizer", "ppc"], 'workernames' : ["ppc64be-sanitizer"], 'builddir': "sanitizer-ppc64be", - 'factory' : SanitizerBuilder.getSanitizerBuildFactory(timeout=1800)}, + 'factory' : SanitizerBuilder.getSanitizerBuildFactory( + timeout=1800, + extra_configure_args=["-DLLVM_CCACHE_BUILD=ON"])}, {'name' : "sanitizer-ppc64le-linux", 'tags' : ["sanitizer", "ppc", "ppc64le"], 'workernames' : ["ppc64le-sanitizer"], 'builddir': "sanitizer-ppc64le", - 'factory' : SanitizerBuilder.getSanitizerBuildFactory(timeout=1800)}, + 'factory' : SanitizerBuilder.getSanitizerBuildFactory( + timeout=1800, + extra_configure_args=[ + "-CMAKE_ARGS='-DLLVM_LIT_ARGS=-v -j256'", + "-DLLVM_CCACHE_BUILD=ON"])}, {'name' : "sanitizer-windows", 'tags' : ["sanitizer"], @@ -1205,14 +1712,19 @@ script="sanitizer-windows.py", depends_on_projects=["llvm", "clang", "lld", "compiler-rt"])}, +# OpenMP builders. + {'name' : "openmp-gcc-x86_64-linux-debian", 'tags' : ["openmp"], 'workernames' : ["gribozavr4"], 'builddir': "openmp-gcc-x86_64-linux-debian", 'factory' : OpenMPBuilder.getOpenMPCMakeBuildFactory( + extraCmakeArgs=[ + '-DLLVM_CCACHE_BUILD=ON', + ], env={ 'PATH':'/home/llvmbb/bin/clang-latest/bin:/home/llvmbb/bin:/usr/local/bin:/usr/local/bin:/usr/bin:/bin', - 'CC': 'ccache clang', 'CXX': 'ccache clang++', 'CCACHE_CPP2': 'yes', + 'CC': 'clang', 'CXX': 'clang++', })}, {'name' : "openmp-clang-x86_64-linux-debian", @@ -1220,120 +1732,123 @@ 'workernames' : ["gribozavr4"], 'builddir': "openmp-clang-x86_64-linux-debian", 'factory' : OpenMPBuilder.getOpenMPCMakeBuildFactory( + extraCmakeArgs=[ + '-DLLVM_CCACHE_BUILD=ON', + ], env={ 'PATH':'/home/llvmbb/bin/clang-latest/bin:/home/llvmbb/bin:/usr/local/bin:/usr/local/bin:/usr/bin:/bin', - 'CC': 'ccache clang', 'CXX': 'ccache clang++', 'CCACHE_CPP2': 'yes', + 'CC': 'clang', 'CXX': 'clang++', })}, -# Libc++ builders. + {'name' : "openmp-offload-cuda-project", + 'tags' : ["openmp"], + 'workernames' : ["minipc-1050ti-linux"], + 'builddir': "openmp-offload-cuda-project", + 'factory' : OpenMPBuilder.getOpenMPCMakeBuildFactory( + clean=False, + enable_runtimes=[], + extraCmakeArgs=[ + "-DCUDA_TOOLKIT_ROOT_DIR=/opt/cuda", + "-DLIBOMPTARGET_BUILD_NVPTX_BCLIB=ON", + "-DCLANG_ENABLE_STATIC_ANALYZER=OFF", + "-DCLANG_ENABLE_ARCMT=OFF", + "-DCLANG_ENABLE_OBJC_REWRITER=OFF", + "-DLLVM_TARGETS_TO_BUILD=X86;NVPTX", + "-DLLVM_ENABLE_LLD=ON", + '-DLLVM_PARALLEL_LINK_JOBS=2', + ], + install=True, + testsuite=True, + testsuite_sollvevv=True, + extraTestsuiteCmakeArgs=[ + "-DTEST_SUITE_SOLLVEVV_OFFLOADING_CFLAGS=-fopenmp-targets=nvptx64-nvidia-cuda;--cuda-path=/opt/cuda", + "-DTEST_SUITE_SOLLVEVV_OFFLOADING_LDFLAGS=-fopenmp-targets=nvptx64-nvidia-cuda;--cuda-path=/opt/cuda", + ], + )}, - {'name': 'libcxx-libcxxabi-x86_64-linux-debian', - 'tags' : ["libcxx"], - 'workernames': ['gribozavr4'], - 'builddir': 'libcxx-libcxxabi-x86_64-linux-debian', - 'factory': LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( - env={'CC': 'clang', 'CXX': 'clang++'}, - lit_extra_args=['--shuffle'], - check_libcxx_abilist=True)}, - - {'name' : 'libcxx-libcxxabi-x86_64-linux-debian-noexceptions', - 'tags' : ["libcxx"], - 'workernames' : ['gribozavr4'], - 'builddir': 'libcxx-libcxxabi-x86_64-linux-debian-noexceptions', - 'factory' : LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( - env={'CC': 'clang', 'CXX': 'clang++'}, - use_cache='Generic-noexceptions.cmake', - lit_extra_args=['--shuffle'])}, - - {'name' : 'libcxx-libcxxabi-libunwind-x86_64-linux-debian', - 'tags' : ["libcxx"], - 'workernames' : ['gribozavr4'], - 'builddir': 'libcxx-libcxxabi-libunwind-x86_64-linux-debian', - 'factory' : LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( - env={'CC': 'clang', 'CXX': 'clang++'}, - cmake_extra_opts={'LIBCXXABI_USE_LLVM_UNWINDER': 'ON'}, - lit_extra_args=['--shuffle'])}, - - {'name' : 'libcxx-libcxxabi-singlethreaded-x86_64-linux-debian', - 'tags' : ["libcxx"], - 'workernames' : ['gribozavr4'], - 'builddir': 'libcxx-libcxxabi-singlethreaded-x86_64-linux-debian', - 'factory' : LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( - env={'CC': 'clang', 'CXX': 'clang++'}, - use_cache='Generic-singlethreaded.cmake')}, - - # ARMv7 LibC++ and LibC++abi tests (require Clang+RT) - {'name' : 'libcxx-libcxxabi-libunwind-armv7-linux', - 'tags' : ["libcxx"], - 'workernames': ['linaro-tk1-02'], - 'builddir': 'libcxx-libcxxabi-libunwind-armv7-linux', - 'factory' : LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( - cmake_extra_opts={ - 'LIBCXXABI_USE_LLVM_UNWINDER': 'ON', - 'CMAKE_C_FLAGS': '-mcpu=cortex-a15 -marm', - 'CMAKE_CXX_FLAGS': '-mcpu=cortex-a15 -marm', - 'LLVM_PARALLEL_LINK_JOBS': '2'})}, - - # ARMv8 LibC++ and LibC++abi tests (require Clang+RT) - {'name' : 'libcxx-libcxxabi-libunwind-armv8-linux', - 'tags' : ["libcxx"], - 'workernames' : ['linaro-armv8-libcxx'], - 'builddir': 'libcxx-libcxxabi-libunwind-armv8-linux', - 'factory' : LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( - cmake_extra_opts={ - 'LIBCXXABI_USE_LLVM_UNWINDER': 'ON', - 'CMAKE_C_FLAGS': '-mcpu=cortex-a57 -marm', - 'CMAKE_CXX_FLAGS': '-mcpu=cortex-a57 -marm'})}, - - # ARMv7 LibC++ and LibC++abi tests w/o EH (require Clang+RT) - {'name' : 'libcxx-libcxxabi-libunwind-armv7-linux-noexceptions', - 'tags' : ["libcxx"], - 'workernames' : ['linaro-tk1-02'], - 'builddir': 'libcxx-libcxxabi-libunwind-armv7-linux-noexceptions', - 'factory' : LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( - cmake_extra_opts={ - 'LIBCXXABI_USE_LLVM_UNWINDER': 'ON', - 'LIBCXX_ENABLE_EXCEPTIONS': 'OFF', - 'LIBCXXABI_ENABLE_EXCEPTIONS': 'OFF', - 'CMAKE_C_FLAGS': '-mcpu=cortex-a15 -mthumb', - 'CMAKE_CXX_FLAGS': '-mcpu=cortex-a15 -mthumb', - 'LLVM_PARALLEL_LINK_JOBS': '2'})}, - - # ARMv8 LibC++ and LibC++abi tests w/o EH (require Clang+RT) - {'name' : 'libcxx-libcxxabi-libunwind-armv8-linux-noexceptions', - 'tags' : ["libcxx"], - 'workernames' : ['linaro-armv8-libcxx'], - 'builddir': 'libcxx-libcxxabi-libunwind-armv8-linux-noexceptions', - 'factory' : LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( - cmake_extra_opts={ - 'LIBCXXABI_USE_LLVM_UNWINDER': 'ON', - 'LIBCXX_ENABLE_EXCEPTIONS': 'OFF', - 'LIBCXXABI_ENABLE_EXCEPTIONS': 'OFF', - 'CMAKE_C_FLAGS': '-mcpu=cortex-a57 -mthumb', - 'CMAKE_CXX_FLAGS': '-mcpu=cortex-a57 -mthumb'})}, - - # AArch64 LibC++ and LibC++abi tests (require Clang+RT) - {'name' : 'libcxx-libcxxabi-libunwind-aarch64-linux', - 'tags' : ["libcxx"], - 'workernames' : ['linaro-aarch64-libcxx'], - 'builddir': 'libcxx-libcxxabi-libunwind-aarch64-linux', - 'factory' : LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( - cmake_extra_opts={ - 'LIBCXXABI_USE_LLVM_UNWINDER': 'ON', - 'CMAKE_C_FLAGS': '-mcpu=cortex-a57', - 'CMAKE_CXX_FLAGS': '-mcpu=cortex-a57'})}, - - {'name' : 'libcxx-libcxxabi-libunwind-aarch64-linux-noexceptions', - 'tags' : ["libcxx"], - 'workernames' : ['linaro-aarch64-libcxx'], - 'builddir': 'libcxx-libcxxabi-libunwind-aarch64-linux-noexceptions', - 'factory' : LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( - cmake_extra_opts={ - 'LIBCXXABI_USE_LLVM_UNWINDER': 'ON', - 'LIBCXX_ENABLE_EXCEPTIONS': 'OFF', - 'LIBCXXABI_ENABLE_EXCEPTIONS': 'OFF', - 'CMAKE_C_FLAGS': '-mcpu=cortex-a57', - 'CMAKE_CXX_FLAGS': '-mcpu=cortex-a57'})}, + {'name' : "openmp-offload-cuda-runtime", + 'tags' : ["openmp"], + 'workernames' : ["minipc-1050ti-linux"], + 'builddir': "openmp-offload-cuda-runtime", + 'factory' : OpenMPBuilder.getOpenMPCMakeBuildFactory( + clean=True, + enable_runtimes=['openmp'], + extraCmakeArgs=[ + "-DCUDA_TOOLKIT_ROOT_DIR=/opt/cuda", + "-DLIBOMPTARGET_BUILD_NVPTX_BCLIB=ON", + "-DCLANG_ENABLE_STATIC_ANALYZER=OFF", + "-DCLANG_ENABLE_ARCMT=OFF", + "-DCLANG_ENABLE_OBJC_REWRITER=OFF", + "-DLLVM_TARGETS_TO_BUILD=X86;NVPTX", + "-DLLVM_ENABLE_LLD=ON", + '-DLLVM_PARALLEL_LINK_JOBS=2', + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", + ], + install=True, + testsuite=True, + testsuite_sollvevv=True, + extraTestsuiteCmakeArgs=[ + "-DTEST_SUITE_SOLLVEVV_OFFLOADING_CFLAGS=-fopenmp-targets=nvptx64-nvidia-cuda;--cuda-path=/opt/cuda", + "-DTEST_SUITE_SOLLVEVV_OFFLOADING_LDFLAGS=-fopenmp-targets=nvptx64-nvidia-cuda;--cuda-path=/opt/cuda", + ], + )}, + +# OpenMP AMDGPU Builders + {'name' : "openmp-offload-amdgpu-runtime", + 'tags' : ["openmp"], + 'workernames' : ["omp-vega20-0"], + 'builddir': "openmp-offload-amdgpu-runtime", + 'factory' : OpenMPBuilder.getOpenMPCMakeBuildFactory( + clean=True, + enable_runtimes=['openmp'], + depends_on_projects=['llvm','clang','lld','openmp'], + extraCmakeArgs=[ + "-DCMAKE_BUILD_TYPE=Release", + "-DCLANG_DEFAULT_LINKER=lld", + "-DLLVM_TARGETS_TO_BUILD=X86;AMDGPU", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_ENABLE_RUNTIMES=openmp", + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", + ], + install=True, + testsuite=False, + testsuite_sollvevv=False, + extraTestsuiteCmakeArgs=[ + "-DTEST_SUITE_SOLLVEVV_OFFLOADING_CFLAGS=-fopenmp-targets=amdgcn-amd-amdhsa;-Xopenmp-target=amdgcn-amd-amdhsa", + "-DTEST_SUITE_SOLLVEVV_OFFLOADING_LDLAGS=-fopenmp-targets=amdgcn-amd-amdhsa;-Xopenmp-target=amdgcn-amd-amdhsa", + ], + )}, + + {'name' : "openmp-offload-amdgpu-runtime-experimental", + 'tags' : ["openmp"], + 'workernames' : ["omp-vega20-1"], + 'builddir': "openmp-offload-amdgpu-runtime-experimental", + 'factory' : OpenMPBuilder.getOpenMPCMakeBuildFactory( + clean=True, + enable_runtimes=['openmp'], + depends_on_projects=['llvm','clang','lld','openmp'], + extraCmakeArgs=[ + "-DCMAKE_BUILD_TYPE=Release", + "-DCLANG_DEFAULT_LINKER=lld", + "-DLLVM_TARGETS_TO_BUILD=X86;AMDGPU", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_ENABLE_RUNTIMES=openmp", + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", + ], + install=True, + testsuite=True, + testsuite_sollvevv=True, + extraTestsuiteCmakeArgs=[ + "-DTEST_SUITE_SOLLVEVV_OFFLOADING_CFLAGS=-fopenmp;-fopenmp-targets=amdgcn-amd-amdhsa;-Xopenmp-target=amdgcn-amd-amdhsa;-march=gfx906", + "-DTEST_SUITE_SOLLVEVV_OFFLOADING_LDLAGS=-fopenmp;-fopenmp-targets=amdgcn-amd-amdhsa;-Xopenmp-target=amdgcn-amd-amdhsa;-march=gfx906", + ], + )}, + + +# Whole-toolchain builders. {'name': "fuchsia-x86_64-linux", 'tags' : ["toolchain"], @@ -1343,6 +1858,42 @@ # libc Builders. + {'name' : 'libc-x86_64-windows-dbg', + 'tags' : ["libc"], + 'workernames' : ['libc-x86_64-windows'], + 'builddir': 'libc-x86_64-windows', + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="libc-windows.py", + depends_on_projects=['llvm', 'libc', 'clang', 'clang-tools-extra'], + extra_args=['--debug'])}, + + {'name' : 'libc-arm32-debian-dbg', + 'tags' : ["libc"], + 'workernames' : ['libc-arm32-debian'], + 'builddir': 'libc-arm32-debian-dbg', + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="libc-linux.py", + depends_on_projects=['llvm', 'libc', 'clang', 'clang-tools-extra'], + extra_args=['--debug'])}, + + {'name' : 'libc-aarch64-ubuntu-dbg', + 'tags' : ["libc"], + 'workernames' : ['libc-aarch64-ubuntu'], + 'builddir': 'libc-aarch64-ubuntu', + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="libc-linux.py", + depends_on_projects=['llvm', 'libc', 'clang', 'clang-tools-extra'], + extra_args=['--debug'])}, + + {'name' : "libc-aarch64-ubuntu-fullbuild-dbg", + 'tags' : ["libc"], + 'workernames' : ["libc-aarch64-ubuntu"], + 'builddir': "libc-aarch64-ubuntu-fullbuild-dbg", + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="libc-linux.py", + depends_on_projects=['llvm', 'libc', 'clang', 'clang-tools-extra'], + extra_args=['--debug'])}, + {'name' : 'libc-x86_64-debian', 'tags' : ["libc"], 'workernames' : ['libc-x86_64-debian'], @@ -1369,59 +1920,72 @@ depends_on_projects=['llvm', 'libc', 'clang', 'clang-tools-extra'], extra_args=['--debug', '--asan'])}, + {'name' : "libc-x86_64-debian-dbg-runtimes-build", + 'tags' : ["libc"], + 'workernames' : ["libc-x86_64-debian"], + 'builddir': "libc-x86_64-debian-dbg-runtimes-build", + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="libc-linux.py", + depends_on_projects=['llvm', 'libc', 'clang', 'clang-tools-extra'], + extra_args=['--debug'])}, + + {'name' : "libc-x86_64-debian-fullbuild-dbg", + 'tags' : ["libc"], + 'workernames' : ["libc-x86_64-debian-fullbuild"], + 'builddir': "libc-x86_64-debian-fullbuild-dbg", + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="libc-linux.py", + depends_on_projects=['llvm', 'libc', 'clang', 'clang-tools-extra'], + extra_args=['--debug'])}, + + {'name' : "libc-x86_64-debian-fullbuild-dbg-asan", + 'tags' : ["libc"], + 'workernames' : ["libc-x86_64-debian-fullbuild"], + 'builddir': "libc-x86_64-debian-fullbuild-dbg-asan", + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="libc-linux.py", + depends_on_projects=['llvm', 'libc', 'clang', 'clang-tools-extra'], + extra_args=['--debug', '--asan'])}, + # Flang builders. - {'name' : "flang-aarch64-ubuntu", + {'name' : "flang-aarch64-dylib", 'tags' : ["flang"], - 'workernames' : ["flang-aarch64-ubuntu-build"], - 'builddir': "flang-aarch64-ubuntu-build", - 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( - clean=True, - depends_on_projects=['llvm','mlir','clang','flang'], - extra_configure_args=[ - "-DLLVM_TARGETS_TO_BUILD=AArch64", - "-DCMAKE_C_COMPILER=/usr/bin/gcc-9", - "-DCMAKE_CXX_COMPILER=/usr/bin/g++-9", - "-DLLVM_INSTALL_UTILS=ON", - "-DFLANG_BUILD_NEW_DRIVER=ON", - "-DCMAKE_CXX_STANDARD=17", - ])}, - - {'name' : "flang-aarch64-ubuntu-dylib", - 'tags' : ["flang"], - 'workernames' : ["linaro-aarch64-flang-dylib"], - 'builddir': "flang-aarch64-ubuntu-dylib", + 'workernames' : ["linaro-flang-aarch64-dylib"], + 'builddir': "flang-aarch64-dylib", 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( clean=True, + checks=['check-flang'], depends_on_projects=['llvm','mlir','clang','flang'], extra_configure_args=[ "-DLLVM_TARGETS_TO_BUILD=AArch64", - "-DFLANG_BUILD_NEW_DRIVER=ON", "-DLLVM_BUILD_LLVM_DYLIB=ON", "-DLLVM_LINK_LLVM_DYLIB=ON", "-DCMAKE_CXX_STANDARD=17", ])}, - {'name' : "flang-aarch64-ubuntu-sharedlibs", + {'name' : "flang-aarch64-sharedlibs", 'tags' : ["flang"], - 'workernames' : ["linaro-aarch64-flang-sharedlibs"], - 'builddir': "flang-aarch64-ubuntu-sharedlibs", + 'workernames' : ["linaro-flang-aarch64-sharedlibs"], + 'builddir': "flang-aarch64-sharedlibs", 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( clean=True, + checks=['check-flang'], depends_on_projects=['llvm','mlir','clang','flang'], extra_configure_args=[ "-DLLVM_TARGETS_TO_BUILD=AArch64", - "-DFLANG_BUILD_NEW_DRIVER=ON", "-DBUILD_SHARED_LIBS=ON", "-DLLVM_BUILD_EXAMPLES=ON", + "-DFLANG_BUILD_EXAMPLES=ON", "-DCMAKE_CXX_STANDARD=17", ])}, - {'name' : "flang-aarch64-ubuntu-out-of-tree", + {'name' : "flang-aarch64-out-of-tree", 'tags' : ["flang"], - 'workernames' : ["linaro-aarch64-flang-oot"], + 'workernames' : ["linaro-flang-aarch64-out-of-tree"], 'builddir': "flang-aarch64-out-of-tree", 'factory' : FlangBuilder.getFlangOutOfTreeBuildFactory( + checks=['check-flang'], llvm_extra_configure_args=[ "-DLLVM_TARGETS_TO_BUILD=AArch64", "-DCMAKE_CXX_STANDARD=17", @@ -1434,16 +1998,88 @@ "-DCMAKE_BUILD_TYPE=Release", ])}, - {'name' : "flang-x86_64-linux", + {'name' : "flang-aarch64-debug", 'tags' : ["flang"], - 'workernames' : ["nersc-flang"], - 'builddir': "flang-x86_64-linux", + 'workernames' : ["linaro-flang-aarch64-debug"], + 'builddir': "flang-aarch64-debug", 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + clean=True, + checks=['check-flang'], depends_on_projects=['llvm','mlir','clang','flang'], extra_configure_args=[ - "-DLLVM_TARGETS_TO_BUILD=X86", + "-DLLVM_TARGETS_TO_BUILD=AArch64", + "-DCMAKE_BUILD_TYPE=Debug", + "-DCMAKE_CXX_STANDARD=17", + "-DLLVM_USE_LINKER=lld", + ])}, + + {'name' : "flang-aarch64-latest-clang", + 'tags' : ['flang'], + 'workernames' : ["linaro-flang-aarch64-latest-clang"], + 'builddir': "flang-aarch64-latest-clang", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + clean=True, + checks=['check-flang'], + depends_on_projects=['llvm','mlir','clang','flang'], + extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=AArch64", "-DLLVM_INSTALL_UTILS=ON", "-DCMAKE_CXX_STANDARD=17", + "-DLLVM_ENABLE_WERROR=OFF", + "-DFLANG_ENABLE_WERROR=ON", + "-DBUILD_SHARED_LIBS=ON", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_ENABLE_LIBCXX=On", + "-DCMAKE_BUILD_TYPE=Release", + ])}, + + {'name' : "flang-aarch64-release", + 'tags' : ["flang"], + 'workernames' : ["linaro-flang-aarch64-release"], + 'builddir': "flang-aarch64-release", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + clean=True, + checks=['check-flang'], + depends_on_projects=['llvm','mlir','clang','flang'], + extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=AArch64", + "-DCMAKE_BUILD_TYPE=Release", + "-DCMAKE_CXX_STANDARD=17", + ])}, + + {'name' : "flang-aarch64-rel-assert", + 'tags' : ["flang"], + 'workernames' : ["linaro-flang-aarch64-rel-assert"], + 'builddir': "flang-aarch64-rel-assert", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + clean=True, + checks=['check-flang'], + depends_on_projects=['llvm','mlir','clang','flang'], + extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=AArch64", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DFLANG_BUILD_EXAMPLES=ON", + "-DCMAKE_BUILD_TYPE=Release", + "-DCMAKE_CXX_STANDARD=17", + ])}, + + {'name' : "flang-aarch64-latest-gcc", + 'tags' : ['flang'], + 'workernames' : ["linaro-flang-aarch64-latest-gcc"], + 'builddir': "flang-aarch64-latest-gcc", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + clean=True, + checks=['check-flang'], + depends_on_projects=['llvm','mlir','clang','flang'], + extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=AArch64", + "-DLLVM_INSTALL_UTILS=ON", + "-DCMAKE_CXX_STANDARD=17", + "-DLLVM_ENABLE_WERROR=OFF", + "-DFLANG_ENABLE_WERROR=ON", + "-DBUILD_SHARED_LIBS=ON", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DCMAKE_BUILD_TYPE=Release", ])}, {'name' : "flang-x86_64-knl-linux", @@ -1457,14 +2093,13 @@ "-DCMAKE_C_COMPILER=gcc", "-DCMAKE_CXX_COMPILER=g++", "-DLLVM_INSTALL_UTILS=ON", - "-DFLANG_BUILD_NEW_DRIVER=ON", "-DCMAKE_CXX_STANDARD=17", ])}, {'name' : 'ppc64le-flang-rhel-clang', 'tags' : ["flang", "ppc", "ppc64le"], 'collapseRequests' : False, - 'workernames' : ['ppc64le-flang-mlir-rhel-test'], + 'workernames' : ['ppc64le-flang-rhel-test'], 'builddir': 'ppc64le-flang-rhel-clang-build', 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( clean=True, @@ -1474,8 +2109,8 @@ '-DLLVM_TARGETS_TO_BUILD=PowerPC', '-DLLVM_INSTALL_UTILS=ON', '-DCMAKE_CXX_STANDARD=17', - '-DLLVM_ENABLE_PROJECTS=flang', - '-DLLVM_LIT_ARGS=-vj 256' + '-DLLVM_LIT_ARGS=-vj 256', + '-DLLVM_CCACHE_BUILD=ON' ], env={ 'CC': 'clang', @@ -1483,7 +2118,27 @@ 'LD': 'lld' })}, -# Builders responsible building Sphinix documentation. + {'name' : "flang-x86_64-windows", + 'tags' : ["flang"], + 'workernames' : ["minipc-ryzen-win"], + 'builddir': "flang-x86_64-windows", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + depends_on_projects=['llvm','mlir','clang','flang'], + checks=['check-flang'], + install_dir="flang.install", + extra_configure_args=[ + "-DCLANG_ENABLE_STATIC_ANALYZER=OFF", + "-DCLANG_ENABLE_ARCMT=OFF", + "-DCLANG_ENABLE_OBJC_REWRITER=OFF", + "-DLLVM_TARGETS_TO_BUILD=X86", + "-DLLVM_INSTALL_UTILS=ON", + "-DCMAKE_C_COMPILER=cl", + "-DCMAKE_CXX_COMPILER=cl", + "-DCMAKE_CXX_STANDARD=17", + '-DLLVM_PARALLEL_COMPILE_JOBS=4', + ])}, + +# Builders responsible building Sphinx documentation. {'name' : "llvm-sphinx-docs", 'tags' : ["llvm", "doc"], @@ -1509,12 +2164,6 @@ 'builddir': "lld-sphinx-docs", 'factory' : SphinxDocsBuilder.getSphinxDocsBuildFactory(lld_html=True)}, - {'name' : "lldb-sphinx-docs", - 'tags' : ["lldb", "doc"], - 'workernames' : ["gribozavr3"], - 'builddir': "lldb-sphinx-docs", - 'factory' : SphinxDocsBuilder.getSphinxDocsBuildFactory(lldb_html=True)}, - {'name':"libunwind-sphinx-docs", 'tags' : ["libunwind", "doc"], 'workernames':["gribozavr3"], @@ -1528,6 +2177,28 @@ 'builddir': "publish-sphinx-docs", 'factory' : SphinxDocsBuilder.getLLVMDocsBuildFactory(clean=True)}, + {'name' : "publish-lnt-sphinx-docs", + 'tags' : ["doc"], + 'workernames' : ["as-worker-4"], + 'builddir': "publish-lnt-sphinx-docs", + 'factory' : HtmlDocsBuilder.getHtmlDocsBuildFactory()}, + + {'name' : "publish-doxygen-docs", + 'tags' : ["doc"], + 'workernames' : ["as-worker-4"], + 'builddir': "publish-doxygen-docs", + 'factory' : DoxygenDocsBuilder.getLLVMDocsBuildFactory( + # Doxygen builds the final result for really + # long time without any output. + # We have to have a long timeout here. + timeout=32400)}, + + {'name' : "polly-sphinx-docs", + 'tags' : ["llvm", "doc"], + 'workernames' : ["polly-x86_64-gce1"], + 'builddir': "polly-sphinx-docs", + 'factory': SphinxDocsBuilder.getSphinxDocsBuildFactory(polly_html=True)}, + # CUDA builders. {'name' : "clang-cuda-k80", @@ -1553,48 +2224,61 @@ 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( script="/buildbot/cuda-build", checkout_llvm_sources=False)}, +# HIP builders. + + {'name' : "clang-hip-vega20", + 'tags' : ["clang"], + 'workernames' : ["hip-vega20-0"], + 'builddir': "clang-hip-vega20", + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="hip-build.sh", + checkout_llvm_sources=False, + script_interpreter=None)}, +# VE builders. {'name' : "clang-ve-ninja", 'tags' : ["clang"], - 'workernames':["nec-arrproto41"], + 'workernames':["hpce-ve-main"], 'builddir':"clang-ve-ninja", - 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( - clean=True, - depends_on_projects=['llvm','clang','openmp'], - extra_configure_args=[ - "-DLLVM_TARGETS_TO_BUILD=X86", - "-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=VE", - ])}, - -# Latest stable fedora running on Red Hat internal OpenShift cluster (PSI). + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="ve-linux.py", + depends_on_projects=['llvm', 'clang', 'compiler-rt', 'libcxx'])}, + {'name' : "clang-ve-staging", + 'tags' : ["clang"], + 'workernames':["hpce-ve-staging"], + 'builddir':"clang-ve-staging", + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="ve-linux.py", + depends_on_projects=['llvm', 'clang', 'compiler-rt', 'libcxx'])}, - {'name' : 'x86_64-fedora-clang', - 'tags' : ['mlir'], + # Build the LLVM dylib .so with all backends and link tools to it + {'name' : 'llvm-x86_64-debian-dylib', + 'tags' : ['llvm'], 'collapseRequests': False, - 'workernames': ['fedora-llvm-x86_64'], - 'builddir': 'x86_64-fedora-clang', + 'workernames': ['gribozavr4'], + 'builddir': 'llvm-x86_64-debian-dylib', 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( clean=True, - depends_on_projects=['llvm', 'clang', 'clang-tools-extra', 'compiler-rt', 'lld', 'mlir'], + depends_on_projects=['llvm'], checks=['check-all'], extra_configure_args=[ '-DCMAKE_BUILD_TYPE=Release', - '-DCMAKE_C_COMPILER=/usr/bin/gcc', - '-DCMAKE_CXX_COMPILER=/usr/bin/g++', '-DLLVM_ENABLE_ASSERTIONS=On', - '-DLLVM_BUILD_EXAMPLES=On', + '-DLLVM_BUILD_EXAMPLES=Off', "-DLLVM_LIT_ARGS=-v --xunit-xml-output test-results.xml", - '-DLLVM_CCACHE_BUILD=On', - '-DLLVM_CCACHE_DIR=/ccache', - '-DLLVM_CCACHE_MAXSIZE=20G', - '-DLLVM_TARGETS_TO_BUILD=X86', + '-DLLVM_TARGETS_TO_BUILD=all', '-DCMAKE_EXPORT_COMPILE_COMMANDS=1', '-DLLVM_BUILD_LLVM_DYLIB=On', '-DLLVM_LINK_LLVM_DYLIB=On', - '-DCLANG_LINK_CLANG_DYLIB=On', '-DBUILD_SHARED_LIBS=Off', - '-DLLVM_ENABLE_LLD=ON', - ])}, + '-DLLVM_ENABLE_LLD=Off', + '-DLLVM_ENABLE_BINDINGS=Off', + '-DLLVM_CCACHE_BUILD=ON', + ], + env={ + 'PATH':'/home/llvmbb/bin/clang-latest/bin:/home/llvmbb/bin:/usr/local/bin:/usr/local/bin:/usr/bin:/bin', + 'CC': 'clang', 'CXX': 'clang++', + })}, {'name' : "clang-solaris11-amd64", 'tags' : ["clang"], @@ -1603,6 +2287,7 @@ 'factory' : ClangBuilder.getClangCMakeBuildFactory( jobs=8, clean=False, + timeout=1800, checkout_lld=False, extra_cmake_args=['-DLLVM_ENABLE_ASSERTIONS=ON', '-DLLVM_TARGETS_TO_BUILD=X86', @@ -1616,96 +2301,13 @@ 'factory' : ClangBuilder.getClangCMakeBuildFactory( jobs=8, clean=False, + timeout=1800, checkout_lld=False, extra_cmake_args=['-DLLVM_ENABLE_ASSERTIONS=ON', '-DLLVM_TARGETS_TO_BUILD=Sparc', '-DLLVM_HOST_TRIPLE=sparcv9-sun-solaris2.11', '-DLLVM_PARALLEL_LINK_JOBS=4'])}, - {'name' : "clang-x86-ninja-win10", - 'tags' : ["clang"], - 'workernames' : ["windows10-vs2019"], - 'builddir': "clang-x86-ninja-win10", - 'factory' : ClangBuilder.getClangCMakeBuildFactory( - clean=True, - vs="autodetect", - vs_target_arch='x86', - testStage1=True, - useTwoStage=True, - stage1_config='Release', - stage2_config='Release', - # reduce scope of builds to get stable results - checkout_clang_tools_extra=False, - checkout_compiler_rt=False, - checkout_flang=False, - checkout_libcxx=False, - extra_cmake_args=['-DLLVM_ENABLE_ASSERTIONS=ON', - '-DLLVM_TARGETS_TO_BUILD=X86', - '-DCMAKE_C_COMPILER_LAUNCHER=sccache', - '-DCMAKE_CXX_COMPILER_LAUNCHER=sccache', - '-DLLVM_ENABLE_ZLIB=OFF', - '-DLLVM_LIT_TOOLS_DIR=C:\\Program Files\\GnuWin32\\usr\\bin', - ])}, - -# Flang builders. - - {'name' : "flang-aarch64-ubuntu-clang", - 'tags' : ['flang'], - 'workernames' : ["flang-aarch64-ubuntu-clang-build"], - 'builddir': "flang-aarch64-ubuntu-clang", - 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( - clean=True, - depends_on_projects=['llvm','mlir','clang','flang'], - checks=['check-flang'], - extra_configure_args=[ - "-DLLVM_TARGETS_TO_BUILD=AArch64", - "-DCMAKE_C_COMPILER=/usr/bin/clang-10", - "-DCMAKE_CXX_COMPILER=/usr/bin/clang++-10", - "-DLLVM_INSTALL_UTILS=ON", - "-DCMAKE_CXX_STANDARD=17", - "-DLLVM_ENABLE_WERROR=OFF", - "-DFLANG_ENABLE_WERROR=ON", - "-DBUILD_SHARED_LIBS=ON", - "-DLLVM_ENABLE_ASSERTIONS=ON", - "-DLLVM_ENABLE_LIBCXX=On", - "-DFLANG_BUILD_NEW_DRIVER=ON", - "-DCMAKE_BUILD_TYPE=Release", - ])}, - - {'name' : "flang-aarch64-ubuntu-gcc10", - 'tags' : ['flang'], - 'workernames' : ["flang-aarch64-ubuntu-gcc10-build"], - 'builddir': "flang-aarch64-ubuntu-gcc10", - 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( - clean=True, - depends_on_projects=['llvm','mlir','clang','flang'], - extra_configure_args=[ - "-DLLVM_TARGETS_TO_BUILD=AArch64", - "-DCMAKE_C_COMPILER=/usr/bin/gcc-10", - "-DCMAKE_CXX_COMPILER=/usr/bin/g++-10", - "-DLLVM_INSTALL_UTILS=ON", - "-DCMAKE_CXX_STANDARD=17", - "-DLLVM_ENABLE_WERROR=OFF", - "-DFLANG_ENABLE_WERROR=ON", - "-DBUILD_SHARED_LIBS=ON", - "-DLLVM_ENABLE_ASSERTIONS=ON", - "-DFLANG_BUILD_NEW_DRIVER=ON", - "-DCMAKE_BUILD_TYPE=Release", - ])}, - -# LLDB builders. - - {'name' : "lldb-x86_64-fedora", - 'tags' : ["lldb"], - 'workernames' : ["lldb-x86_64-fedora"], - 'builddir': "lldb-x86_64-fedora", - 'factory' : LLDBBuilder.getLLDBCMakeBuildFactory( - clean=True, - test=True, - extra_cmake_args=['-DLLVM_ENABLE_ASSERTIONS=True', - '-DLLVM_USE_LINKER=gold', - '-DLLVM_LIT_ARGS=-v'])}, - # Builders for ML-driven compiler optimizations. # Development mode build bot: tensorflow C APIs are present, and @@ -1713,7 +2315,7 @@ {'name' : "ml-opt-dev-x86-64", 'tags' : ['ml_opt'], 'collapseRequests': False, - 'workernames' : ["ml-opt-dev-x86-64-b1"], + 'workernames' : ["ml-opt-dev-x86-64-b1", "ml-opt-dev-x86-64-b2"], 'builddir': "ml-opt-dev-x86-64-b1", 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( clean=True, @@ -1722,7 +2324,8 @@ "-DCMAKE_BUILD_TYPE=Release", "-DLLVM_CCACHE_BUILD=ON", "-DLLVM_ENABLE_ASSERTIONS=ON", - "-DTENSORFLOW_API_PATH=/tmp/tensorflow", + "-DTENSORFLOW_C_LIB_PATH=/tmp/tensorflow", + "-C", "/tmp/tflitebuild/tflite.cmake", "-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON" ])}, @@ -1730,7 +2333,7 @@ {'name' : "ml-opt-devrel-x86-64", 'tags' : ["ml_opt"], 'collapseRequests': False, - 'workernames' : ["ml-opt-devrel-x86-64-b1"], + 'workernames' : ["ml-opt-devrel-x86-64-b1", "ml-opt-devrel-x86-64-b2"], 'builddir': "ml-opt-devrel-x86-64-b1", 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( clean=True, @@ -1739,7 +2342,8 @@ "-DCMAKE_BUILD_TYPE=Release", "-DLLVM_CCACHE_BUILD=ON", "-DLLVM_ENABLE_ASSERTIONS=ON", - "-DTENSORFLOW_API_PATH=/tmp/tensorflow", + "-DTENSORFLOW_C_LIB_PATH=/tmp/tensorflow", + "-C", "/tmp/tflitebuild/tflite.cmake", "-DTENSORFLOW_AOT_PATH=/var/lib/buildbot/.local/lib/python3.7/site-packages/tensorflow", "-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON" ])}, @@ -1751,7 +2355,7 @@ {'name' : "ml-opt-rel-x86-64", 'tags' : ["ml_opt"], 'collapseRequests': False, - 'workernames' : ["ml-opt-rel-x86-64-b1"], + 'workernames' : ["ml-opt-rel-x86-64-b1", "ml-opt-rel-x86-64-b2"], 'builddir': "ml-opt-rel-x86-64-b1", 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( clean=True, @@ -1782,5 +2386,303 @@ "-DGRPC_INSTALL_PATH=/usr/local/lib/grpc" ])}, + # Target ARC from Synopsys + {'name': "arc-builder", + 'tags': ["clang", "lld"], + 'collapseRequests': False, + 'workernames' : ["arc-worker"], + 'builddir': "arc-folder", + 'factory': UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + depends_on_projects=["llvm", "clang", "lld"], + extra_configure_args=[ + "-DCMAKE_EXPORT_COMPILE_COMMANDS:BOOL=ON", + "-DLLVM_ENABLE_ASSERTIONS:BOOL=ON", + "-DLLVM_TOOL_CLANG_TOOLS_EXTRA_BUILD=0", + "-DLLVM_ENABLE_LIBPFM=OFF", + "-DLLVM_TARGETS_TO_BUILD=X86", + "-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=ARC", + ])}, + + + # BOLT builders managed by Meta + {'name': "bolt-x86_64-ubuntu", + 'tags': ["bolt"], + 'collapseRequests': False, + 'workernames':["bolt-worker"], + 'builddir': "bolt-x86_64-ubuntu-bolttests", + 'factory' : BOLTBuilder.getBOLTCmakeBuildFactory( + bolttests=True, + extra_configure_args=[ + "-DLLVM_APPEND_VC_REV=OFF", + "-DLLVM_CCACHE_BUILD=ON", + "-DLLVM_ENABLE_PROJECTS=clang;lld;bolt", + "-DLLVM_TARGETS_TO_BUILD=X86;AArch64", + ], + )}, + + {'name' : 'bolt-x86_64-ubuntu-nfc', + 'tags' : ["bolt"], + 'collapseRequests': False, + 'workernames' : ['bolt-worker'], + 'builddir': "bolt-x86_64-ubuntu-nfc", + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="bolt-nfc.py", + depends_on_projects=['llvm', 'bolt'], + )}, + + {'name': "bolt-x86_64-ubuntu-clang-bolt", + 'tags': ["bolt"], + 'collapseRequests': False, + 'workernames':["bolt-worker"], + 'builddir': "bolt-x86_64-ubuntu-clang-bolt", + 'factory' : BOLTBuilder.getBOLTCmakeBuildFactory( + bolttests=False, + clangbolt=True, + extra_configure_args=[ + "-DLLVM_APPEND_VC_REV=OFF", + "-DLLVM_CCACHE_BUILD=ON", + ], + )}, + + {'name': "bolt-x86_64-ubuntu-dylib", + 'tags': ["bolt"], + 'collapseRequests': False, + 'workernames':["bolt-worker"], + 'builddir': "bolt-x86_64-ubuntu-dylib", + 'factory' : BOLTBuilder.getBOLTCmakeBuildFactory( + bolttests=False, + extra_configure_args=[ + "-DLLVM_APPEND_VC_REV=OFF", + "-DLLVM_CCACHE_BUILD=ON", + "-DLLVM_ENABLE_PROJECTS=bolt", + "-DLLVM_TARGETS_TO_BUILD=X86;AArch64", + "-DLLVM_LINK_LLVM_DYLIB=ON", + "-DLLVM_ENABLE_LLD=ON", + "-DBOLT_CLANG_EXE=/usr/bin/clang", + "-DBOLT_LLD_EXE=/usr/bin/ld.lld", + ], + )}, + + {'name': "bolt-x86_64-ubuntu-shared", + 'tags': ["bolt"], + 'collapseRequests': False, + 'workernames':["bolt-worker"], + 'builddir': "bolt-x86_64-ubuntu-shared", + 'factory' : BOLTBuilder.getBOLTCmakeBuildFactory( + bolttests=False, + extra_configure_args=[ + "-DLLVM_APPEND_VC_REV=OFF", + "-DLLVM_CCACHE_BUILD=ON", + "-DLLVM_ENABLE_PROJECTS=bolt", + "-DLLVM_TARGETS_TO_BUILD=X86;AArch64", + "-DBUILD_SHARED_LIBS=ON", + "-DLLVM_ENABLE_LLD=ON", + "-DBOLT_CLANG_EXE=/usr/bin/clang", + "-DBOLT_LLD_EXE=/usr/bin/ld.lld", + ], + )}, + + {'name': "bolt-aarch64-ubuntu-clang-shared", + 'tags': ["bolt"], + 'collapseRequests': True, + 'workernames':["bolt-worker-aarch64"], + 'builddir': "bolt-aarch64-ubuntu-clang-shared", + 'factory' : BOLTBuilder.getBOLTCmakeBuildFactory( + bolttests=True, + extra_configure_args=[ + "-DCMAKE_C_COMPILER=clang", + "-DCMAKE_CXX_COMPILER=clang++", + "-DLLVM_APPEND_VC_REV=OFF", + "-DLLVM_CCACHE_BUILD=ON", + "-DLLVM_ENABLE_PROJECTS=bolt", + "-DLLVM_TARGETS_TO_BUILD=X86;AArch64", + "-DBUILD_SHARED_LIBS=ON", + "-DLLVM_ENABLE_LLD=ON", + "-DBOLT_CLANG_EXE=/usr/bin/clang", + "-DBOLT_LLD_EXE=/usr/bin/ld.lld", + ], + )}, + + {'name' : "bolt-x86_64-wine-msvc", + 'tags' : ["bolt"], + 'collapseRequests': True, + 'workernames' : ["bolt-worker"], + 'builddir': "bolt-x86_64-wine-msvc", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + targets = ['bolt'], + checks = [], + depends_on_projects=['llvm','bolt'], + extra_configure_args=[ + "-DCMAKE_C_COMPILER=cl", + "-DCMAKE_CXX_COMPILER=cl", + "-DCMAKE_SYSTEM_NAME=Windows", + "-DLLVM_APPEND_VC_REV=OFF", + "-DLLVM_CCACHE_BUILD=ON", + "-DLLVM_ENABLE_PROJECTS=bolt", + "-DLLVM_TARGETS_TO_BUILD=X86;AArch64", + "-DLLVM_OPTIMIZED_TABLEGEN=ON", + ], + env={ + 'PATH':'/home/worker/msvc-wine/cmake/bin:/home/worker/msvc-wine/msvc/bin/x64:/usr/bin:/bin', + })}, + + # AMD ROCm support. + {'name' : 'mlir-rocm-mi200', + 'tags' : ["mlir"], + 'collapseRequests' : False, + 'workernames' : ['mi200-buildbot'], + 'builddir': 'mlir-rocm-mi200', + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + clean=True, + depends_on_projects=['llvm', 'mlir'], + targets = ['check-mlir-build-only'], + checks = ['check-mlir'], + extra_configure_args= mlir_default_cmake_options + [ + '-DLLVM_CCACHE_BUILD=ON', + '-DLLVM_ENABLE_ASSERTIONS=ON', + '-DLLVM_ENABLE_LLD=ON', + '-DMLIR_ENABLE_ROCM_RUNNER=ON', + '-DMLIR_ENABLE_ROCM_CONVERSIONS=ON', + '-DMLIR_INCLUDE_INTEGRATION_TESTS=ON', + ], + env={ + 'CC': 'clang', + 'CXX': 'clang++', + 'LD': 'lld', + })}, + + # Standalone builder + {'name' : "standalone-build-x86_64", + 'tags' : ["clang"], + 'collapseRequests': False, + 'workernames':["standalone-build-x86_64"], + 'builddir':"standalone-build-x86_64", + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="standalone-build.sh", + checkout_llvm_sources=False, + script_interpreter=None)}, + + ## CSKY check-all + test-suite in soft-float + {'name' : "clang-csky-soft", + 'tags' : ["clang"], + 'workernames' : ["thead-clang-csky"], + 'builddir':"clang-csky-softfp", + 'factory' : ClangBuilder.getClangCMakeBuildFactory( + clean=False, + checkout_clang_tools_extra=False, + checkout_compiler_rt=False, + checkout_lld=False, + testStage1=True, + useTwoStage=False, + stage1_config='Release', + runTestSuite=True, + testsuite_flags=[ + '--cflags', '-mcpu=c860 -latomic -DSMALL_PROBLEM_SIZE', + '--cppflags', '-mcpu=c860 -latomic -DSMALL_PROBLEM_SIZE', + '--run-under=/mnt/qemu/bin/qemu-cskyv2 -cpu c860 -csky-extend denormal=on -L /mnt/gcc-csky/csky-linux-gnuabiv2/libc/ck860 -E LD_LIBRARY_PATH=/mnt/gcc-csky/csky-linux-gnuabiv2/lib/ck860', + '--cmake-define=SMALL_PROBLEM_SIZE=On', + '--cmake-define=TEST_SUITE_USER_MODE_EMULATION=True', + '--threads=32', '--build-threads=32'], + extra_cmake_args=[ + "-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD='CSKY'", + "-DLLVM_DEFAULT_TARGET_TRIPLE='csky-unknown-linux'", + "-DGCC_INSTALL_PREFIX=/mnt/gcc-csky/"])}, + + ## CSKY check-all + test-suite in hard-float + {'name' : "clang-csky-hardfp", + 'tags' : ["clang"], + 'workernames' : ["thead-clang-csky"], + 'builddir':"clang-csky-hardfp", + 'factory' : ClangBuilder.getClangCMakeBuildFactory( + clean=False, + checkout_clang_tools_extra=False, + checkout_compiler_rt=False, + checkout_lld=False, + testStage1=True, + useTwoStage=False, + stage1_config='Release', + runTestSuite=True, + testsuite_flags=[ + '--cflags', '-mcpu=c860 -latomic -mhard-float -DSMALL_PROBLEM_SIZE', + '--cppflags', '-mcpu=c860 -latomic -mhard-float -DSMALL_PROBLEM_SIZE', + '--run-under=/mnt/qemu/bin/qemu-cskyv2 -cpu c860 -csky-extend denormal=on -L /mnt/gcc-csky/csky-linux-gnuabiv2/libc/ck860/hard-fp -E LD_LIBRARY_PATH=/mnt/gcc-csky/csky-linux-gnuabiv2/lib/ck860/hard-fp', + '--cmake-define=SMALL_PROBLEM_SIZE=On', + '--cmake-define=TEST_SUITE_USER_MODE_EMULATION=True', + '--threads=32', '--build-threads=32'], + extra_cmake_args=[ + "-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD='CSKY'", + "-DLLVM_DEFAULT_TARGET_TRIPLE='csky-unknown-linux'", + "-DGCC_INSTALL_PREFIX=/mnt/gcc-csky/"])}, + + # NVPTX builders + {'name' : "llvm-nvptx-nvidia-ubuntu", + 'tags' : ["llvm", "nvptx"], + 'collapseRequests': False, + 'workernames' : ["as-builder-7"], + 'builddir': "llvm-nvptx-nvidia-ubuntu", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + depends_on_projects=["llvm"], + clean=True, + checks=["check-llvm"], + extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=X86;NVPTX", + "-DLLVM_DEFAULT_TARGET_TRIPLE=nvptx-nvidia-cuda", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_LIT_ARGS=-vv --threads=32", + "-DLLVM_USE_LINKER=gold", + "-DBUILD_SHARED_LIBS=ON", + "-DLLVM_OPTIMIZED_TABLEGEN=ON"])}, + + {'name' : "llvm-nvptx64-nvidia-ubuntu", + 'tags' : ["llvm", "nvptx"], + 'collapseRequests': False, + 'workernames' : ["as-builder-7"], + 'builddir': "llvm-nvptx64-nvidia-ubuntu", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + depends_on_projects=["llvm"], + clean=True, + checks=["check-llvm"], + extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=X86;NVPTX", + "-DLLVM_DEFAULT_TARGET_TRIPLE=nvptx64-nvidia-cuda", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_LIT_ARGS=-vv --threads=32", + "-DLLVM_USE_LINKER=gold", + "-DBUILD_SHARED_LIBS=ON", + "-DLLVM_OPTIMIZED_TABLEGEN=ON"])}, + + {'name' : "llvm-nvptx-nvidia-win", + 'tags' : ["llvm", "nvptx"], + 'collapseRequests': False, + 'workernames' : ["as-builder-8"], + 'builddir': "llvm-nvptx-nvidia-win", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaWithMSVCBuildFactory( + vs="autodetect", + depends_on_projects=["llvm"], + clean=True, + checks=["check-llvm"], + extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=X86;NVPTX", + "-DLLVM_DEFAULT_TARGET_TRIPLE=nvptx-nvidia-cuda", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_LIT_ARGS=-vv --threads=32", + "-DLLVM_OPTIMIZED_TABLEGEN=ON"])}, + + {'name' : "llvm-nvptx64-nvidia-win", + 'tags' : ["llvm", "nvptx"], + 'collapseRequests': False, + 'workernames' : ["as-builder-8"], + 'builddir': "llvm-nvptx64-nvidia-win", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaWithMSVCBuildFactory( + vs="autodetect", + depends_on_projects=["llvm"], + clean=True, + checks=["check-llvm"], + extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=X86;NVPTX", + "-DLLVM_DEFAULT_TARGET_TRIPLE=nvptx64-nvidia-cuda", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_LIT_ARGS=-vv --threads=32", + "-DLLVM_OPTIMIZED_TABLEGEN=ON"])}, ] diff --git a/buildbot/osuosl/master/config/release_builders.py b/buildbot/osuosl/master/config/release_builders.py new file mode 100644 --- /dev/null +++ b/buildbot/osuosl/master/config/release_builders.py @@ -0,0 +1,233 @@ +from importlib import reload + +from buildbot.process.properties import WithProperties + +from zorg.buildbot.builders import ClangBuilder +from zorg.buildbot.builders import FlangBuilder +from zorg.buildbot.builders import PollyBuilder +from zorg.buildbot.builders import LLDBBuilder +from zorg.buildbot.builders import SanitizerBuilder +from zorg.buildbot.builders import OpenMPBuilder +from zorg.buildbot.builders import SphinxDocsBuilder +from zorg.buildbot.builders import ABITestsuitBuilder +from zorg.buildbot.builders import ClangLTOBuilder +from zorg.buildbot.builders import UnifiedTreeBuilder +from zorg.buildbot.builders import AOSPBuilder +from zorg.buildbot.builders import AnnotatedBuilder +from zorg.buildbot.builders import LLDPerformanceTestsuite +from zorg.buildbot.builders import FuchsiaBuilder +from zorg.buildbot.builders import XToolchainBuilder + + +# Release builders. + +all = [ + +# Clang builders. + + {'name' : "llvm-clang-x86_64-win-release", + 'tags' : ["clang", "release"], + 'workernames' : ["as-builder-3"], + 'builddir': "llvm-clang-x86_64-win-rel", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaWithMSVCBuildFactory( + vs="autodetect", + depends_on_projects=['llvm', 'clang'], + clean=True, + checks=[ + "check-llvm-unit", + "check-clang-unit"], + extra_configure_args=[ + "-DLLVM_ENABLE_WERROR=OFF", + "-DLLVM_TARGETS_TO_BUILD=ARM", + "-DLLVM_DEFAULT_TARGET_TRIPLE=armv7-unknown-linux-eabihf", + "-DLLVM_ENABLE_ASSERTIONS=OFF", + "-DLLVM_OPTIMIZED_TABLEGEN=OFF", + "-DLLVM_LIT_ARGS=-v --threads=32"])}, + +# Expensive checks builders. + + + {'name' : "llvm-clang-x86_64-expensive-checks-ubuntu-release", + 'tags' : ["llvm", "expensive-checks", "release"], + 'workernames' : ["as-builder-4"], + 'builddir': "llvm-clang-x86_64-expensive-checks-ubuntu-rel", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + depends_on_projects=["llvm", "lld"], + clean=True, + extra_configure_args=[ + "-DLLVM_ENABLE_EXPENSIVE_CHECKS=ON", + "-DLLVM_ENABLE_WERROR=OFF", + "-DLLVM_USE_SPLIT_DWARF=ON", + "-DLLVM_USE_LINKER=gold", + "-DCMAKE_BUILD_TYPE=Debug", + "-DCMAKE_CXX_FLAGS=-U_GLIBCXX_DEBUG -Wno-misleading-indentation", + "-DLLVM_LIT_ARGS=-vv -j32"])}, + + {'name' : "llvm-clang-x86_64-expensive-checks-win-release", + 'tags' : ["llvm", "expensive-checks", "release"], + 'workernames' : ["as-worker-93"], + 'builddir': "llvm-clang-x86_64-expensive-checks-win-rel", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaWithMSVCBuildFactory( + vs="autodetect", + depends_on_projects=["llvm", "lld"], + clean=True, + extra_configure_args=[ + "-DLLVM_ENABLE_EXPENSIVE_CHECKS=ON", + "-DLLVM_ENABLE_WERROR=OFF", + "-DCMAKE_BUILD_TYPE=Debug"])}, + + {'name' : "llvm-clang-x86_64-expensive-checks-debian-release", + 'tags' : ["llvm", "expensive-checks", "release"], + 'workernames' : ["gribozavr4"], + 'builddir': "llvm-clang-x86_64-expensive-checks-debian-rel", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + depends_on_projects=["llvm", "lld"], + clean=True, + extra_configure_args=[ + "-DLLVM_CCACHE_BUILD=ON", + "-DLLVM_ENABLE_EXPENSIVE_CHECKS=ON", + "-DLLVM_ENABLE_WERROR=OFF", + "-DCMAKE_BUILD_TYPE=Release", + "-DCMAKE_CXX_FLAGS=-U_GLIBCXX_DEBUG", + "-DLLVM_LIT_ARGS=-v -vv -j96"], + env={ + 'PATH':'/home/llvmbb/bin/clang-latest/bin:/home/llvmbb/bin:/usr/local/bin:/usr/local/bin:/usr/bin:/bin', + 'CC': 'clang', 'CXX': 'clang++', + })}, + +# Cross builders. + + {'name' : "llvm-clang-win-x-armv7l-release", + 'tags' : ["clang", "llvm", "compiler-rt", "cross", "armv7", "release"], + 'workernames' : ["as-builder-5"], + 'builddir': "x-armv7l-rel", + 'factory' : XToolchainBuilder.getCmakeWithMSVCBuildFactory( + vs="autodetect", + clean=True, + checks=[ + "check-llvm", + "check-clang", + "check-lld", + "check-compiler-rt-armv7-unknown-linux-gnueabihf" + ], + checks_on_target = [ + ("libunwind", + ["python", "bin/llvm-lit.py", + "-v", "-vv", "--threads=32", + "runtimes/runtimes-armv7-unknown-linux-gnueabihf-bins/libunwind/test"]), + ("libc++abi", + ["python", "bin/llvm-lit.py", + "-v", "-vv", "--threads=32", + "runtimes/runtimes-armv7-unknown-linux-gnueabihf-bins/libcxxabi/test"]), + ("libc++", + ['python', 'bin/llvm-lit.py', + '-v', '-vv', '--threads=32', + 'runtimes/runtimes-armv7-unknown-linux-gnueabihf-bins/libcxx/test', + ]) + ], + extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=ARM", + "-DTOOLCHAIN_TARGET_TRIPLE=armv7-unknown-linux-gnueabihf", + "-DDEFAULT_SYSROOT=C:/buildbot/.arm-ubuntu", + "-DZLIB_ROOT=C:/buildbot/.zlib-win32", + "-DLLVM_LIT_ARGS=-v -vv --threads=32", + WithProperties("%(remote_test_host:+-DREMOTE_TEST_HOST=)s%(remote_test_host:-)s"), + WithProperties("%(remote_test_user:+-DREMOTE_TEST_USER=)s%(remote_test_user:-)s"), + ], + cmake_cache="../llvm-project/clang/cmake/caches/CrossWinToARMLinux.cmake")}, + + {'name' : "llvm-clang-win-x-aarch64-release", + 'tags' : ["clang", "llvm", "compiler-rt", "cross", "aarch64", "release"], + 'workernames' : ["as-builder-6"], + 'builddir': "x-aarch64-rel", + 'factory' : XToolchainBuilder.getCmakeWithMSVCBuildFactory( + vs="autodetect", + clean=True, + checks=[ + "check-llvm", + "check-clang", + "check-lld", + "check-compiler-rt-aarch64-unknown-linux-gnu" + ], + checks_on_target = [ + ("libunwind", + ["python", "bin/llvm-lit.py", + "-v", "-vv", "--threads=32", + "runtimes/runtimes-aarch64-unknown-linux-gnu-bins/libunwind/test"]), + ("libc++abi", + ["python", "bin/llvm-lit.py", + "-v", "-vv", "--threads=32", + "runtimes/runtimes-aarch64-unknown-linux-gnu-bins/libcxxabi/test"]), + ("libc++", + ['python', 'bin/llvm-lit.py', + '-v', '-vv', '--threads=32', + 'runtimes/runtimes-aarch64-unknown-linux-gnu-bins/libcxx/test', + ]) + ], + extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=AArch64", + "-DTOOLCHAIN_TARGET_TRIPLE=aarch64-unknown-linux-gnu", + "-DDEFAULT_SYSROOT=C:/buildbot/.aarch64-ubuntu", + "-DZLIB_ROOT=C:/buildbot/.zlib-win32", + "-DLLVM_LIT_ARGS=-v -vv --threads=32", + WithProperties("%(remote_test_host:+-DREMOTE_TEST_HOST=)s%(remote_test_host:-)s"), + WithProperties("%(remote_test_user:+-DREMOTE_TEST_USER=)s%(remote_test_user:-)s"), + ], + cmake_cache="../llvm-project/clang/cmake/caches/CrossWinToARMLinux.cmake")}, + +# LLD builders. + + {'name' : "lld-x86_64-win-release", + 'tags' : ["lld", "release"], + 'workernames' : ["as-worker-93"], + 'builddir': "lld-x86_64-win-rel", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaWithMSVCBuildFactory( + depends_on_projects=['llvm', 'lld'], + vs="autodetect", + extra_configure_args = [ + '-DLLVM_ENABLE_WERROR=OFF'])}, + + {'name' : "lld-x86_64-ubuntu-release", + 'tags' : ["lld", "release"], + 'workernames' : ["as-builder-4"], + 'builddir' : "lld-x86_64-ubuntu-rel", + 'factory': UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + clean=True, + extra_configure_args=[ + '-DLLVM_ENABLE_WERROR=OFF'], + depends_on_projects=['llvm', 'lld'])}, + +# LTO and ThinLTO builders. + + {'name' : "clang-with-thin-lto-ubuntu-release", + 'tags' : ["clang", "lld", "LTO", "release"], + 'workernames' : ["as-worker-92"], + 'builddir': "clang-with-thin-lto-ubuntu-rel", + 'factory' : ClangLTOBuilder.getClangWithLTOBuildFactory(jobs=72, lto='thin')}, + + {'name' : "clang-with-lto-ubuntu-release", + 'tags' : ["clang", "lld", "LTO", "release"], + 'workernames' : ["as-worker-91"], + 'builddir': "clang-with-lto-ubuntu-rel", + 'factory' : ClangLTOBuilder.getClangWithLTOBuildFactory( + jobs=72, + extra_configure_args_lto_stage=[ + '-DLLVM_PARALLEL_LINK_JOBS=14', + ])}, + +# OpenMP builders. + + {'name' : "openmp-clang-x86_64-linux-debian-release", + 'tags' : ["openmp", "release"], + 'workernames' : ["gribozavr4"], + 'builddir': "openmp-clang-x86_64-linux-debian-rel", + 'factory' : OpenMPBuilder.getOpenMPCMakeBuildFactory( + extraCmakeArgs=[ + '-DLLVM_CCACHE_BUILD=ON', + ], + env={ + 'PATH':'/home/llvmbb/bin/clang-latest/bin:/home/llvmbb/bin:/usr/local/bin:/usr/local/bin:/usr/bin:/bin', + 'CC': 'clang', 'CXX': 'clang++', + })}, + +] diff --git a/buildbot/osuosl/master/config/schedulers.py b/buildbot/osuosl/master/config/schedulers.py --- a/buildbot/osuosl/master/config/schedulers.py +++ b/buildbot/osuosl/master/config/schedulers.py @@ -19,7 +19,7 @@ # Since we have many parametric builders, we dynamically build the minimum set # of schedulers, which covers all actually used combinations of dependencies. -def getSingleBranchSchedulers( +def getMainBranchSchedulers( builders, explicitly_set_schedulers = None, **kwargs): @@ -43,9 +43,10 @@ builder for builder in builders if builder.name not in builders_with_explicit_schedulers if getattr(builder.factory, 'depends_on_projects', None) + if 'release' not in getattr(builder, 'tags', []) ] - filter_branch = 'master' + filter_branch = 'main' treeStableTimer = kwargs.get('treeStableTimer', None) automatic_schedulers = [] @@ -60,7 +61,7 @@ ]) for projects in set_of_dependencies: - sch_builders = [ + main_builders = [ b.name for b in builders_with_automatic_schedulers if frozenset(getattr(b.factory, 'depends_on_projects')) == projects @@ -71,9 +72,9 @@ automatic_schedulers.append( schedulers.SingleBranchScheduler( name=automatic_scheduler_name, - treeStableTimer=kwargs.get('treeStableTimer', None), - reason="Merge to github %s branch" % filter_branch, - builderNames=sch_builders, + treeStableTimer=treeStableTimer, + reason="Merge to github {} branch".format(filter_branch), + builderNames=main_builders, change_filter=util.ChangeFilter( project_fn= \ lambda c, projects_of_interest=frozenset(projects): @@ -83,18 +84,90 @@ ) log.msg( - "Generated SingleBranchScheduler: { name='%s'" % automatic_scheduler_name, - ", builderNames=", sch_builders, - ", change_filter=", projects, " (branch: %s)" % filter_branch, - ", treeStableTimer=%s" % treeStableTimer, + "Generated SingleBranchScheduler: {{ name='{}'".format(automatic_scheduler_name), + ", builderNames=", main_builders, + ", change_filter=", projects, " (branch: {})".format(filter_branch), + ", treeStableTimer={}".format(treeStableTimer), "}") return automatic_schedulers -# TODO: Move these settings to the configuration file. -_repourl = "https://github.com/llvm/llvm-project" -_branch = "master" +def getReleaseBranchSchedulers( + builders, + explicitly_set_schedulers = None, + **kwargs): + """ + I'm taking over all of not yet assigned builders with the + declared source code dependencies and the 'release' tag, + and automatically generate a minimum set of SingleBranchSchedulers + to handle all the declared source code dependency combinations. + """ + + builders_with_explicit_schedulers = set() + if explicitly_set_schedulers: + # TODO: Make a list of builder names with already set schedulers. + # builders_with_explicit_schedulers.add(builder) + pass + + # For the builders created with LLVMBuildFactory or similar, + # we always use automatic schedulers, + # unless schedulers already explicitly set. + builders_with_automatic_schedulers = [ + builder for builder in builders + if builder.name not in builders_with_explicit_schedulers + if getattr(builder.factory, 'depends_on_projects', None) + if 'release' in getattr(builder, 'tags', []) + ] + + treeStableTimer = kwargs.get('treeStableTimer', None) + + automatic_schedulers = [] + + # Do we have any to take care of? + if builders_with_automatic_schedulers: + # Let's reconsile first to get a unique set of dependencies. + # We need a set of unique sets of dependent projects. + set_of_dependencies = set([ + frozenset(getattr(b.factory, 'depends_on_projects')) + for b in builders_with_automatic_schedulers + ]) + + for projects in set_of_dependencies: + release_builders = [ + b.name + for b in builders_with_automatic_schedulers + if frozenset(getattr(b.factory, 'depends_on_projects')) == projects + ] + + automatic_scheduler_name = "release:" + ",".join(sorted(projects)) + + automatic_schedulers.append( + schedulers.SingleBranchScheduler( + name=automatic_scheduler_name, + treeStableTimer=treeStableTimer, + reason="Merge to github release branch", + builderNames=release_builders, + change_filter=util.ChangeFilter( + project_fn= \ + lambda c, projects_of_interest=frozenset(projects): + isProjectOfInterest(c, projects_of_interest), + branch_fn= \ + lambda branch: branch.startswith('release/')) + ) + ) + + log.msg( + "Generated release SingleBranchScheduler: {{ name='{}'".format(automatic_scheduler_name), + ", builderNames=", release_builders, + ", change_filter=", projects, " (branch: {release/*})", + ", treeStableTimer={}".format(treeStableTimer), + "}") + return automatic_schedulers def getForceSchedulers(builders): + # TODO: Move these settings to the configuration file. + _repourl = "https://github.com/llvm/llvm-project" + _branch = "main" + # Walk over all builders and collect their names. scheduler_builders = [ builder.name for builder in builders @@ -120,8 +193,10 @@ codebases = [ util.CodebaseParameter( codebase = "", - branch = util.FixedParameter( + branch = util.StringParameter( name = "branch", + label = "branch:", + size = 64, default = _branch ), revision = util.StringParameter( @@ -157,19 +232,21 @@ # TODO: Abstract this kind of scheduler better. def getLntSchedulers(): - project = "lnt" - lnt_builders = [ + _project = "lnt" + _branch = 'master' + _repourl = "https://github.com/llvm/llvm-lnt" + _lnt_builders = [ "publish-lnt-sphinx-docs", ] return [ schedulers.SingleBranchScheduler( name="lnt-scheduler", treeStableTimer=kwargs.get('treeStableTimer', None), - reason="Merge to LNT github {} branch".format(filter_branch), - builderNames=lnt_builders, + reason="Merge to LNT github {} branch".format(_branch), + builderNames=_lnt_builders, change_filter=util.ChangeFilter( - project_fn=project, - branch=filter_branch)), + project_fn=_project, + branch=_branch)), schedulers.ForceScheduler( name = "force-build-scheduler", @@ -186,7 +263,7 @@ ], default = "Build a particular revision" ), - builderNames = lnt_builders, + builderNames = _lnt_builders, codebases = [ util.CodebaseParameter( codebase = "", @@ -206,7 +283,7 @@ ), project = util.FixedParameter( name = "project", - default = project + default = _project ) ) ], diff --git a/buildbot/osuosl/master/config/status.py b/buildbot/osuosl/master/config/status.py --- a/buildbot/osuosl/master/config/status.py +++ b/buildbot/osuosl/master/config/status.py @@ -9,18 +9,22 @@ all = [ - # Note: reporters.GitHubStatusPush requires txrequests package to allow - # interaction with GitHub REST API. + # Report github status for all the release builders, + # i.e. those with the "release" tag. reporters.GitHubStatusPush( str(config.options.get('GitHub Status', 'token')), context = Interpolate("%(prop:buildername)s"), verbose = True, # TODO: Turn off the verbosity once this is working reliably. builders = [ - "llvm-clang-x86_64-expensive-checks-ubuntu", - "llvm-clang-x86_64-win-fast", - "clang-x86_64-debian-fast", - "llvm-clang-x86_64-expensive-checks-debian", - ]), + "llvm-clang-x86_64-expensive-checks-ubuntu", + "llvm-clang-x86_64-win-fast", + "clang-x86_64-debian-fast", + "llvm-clang-x86_64-expensive-checks-debian", + ] + [ + b.get('name') for b in config.release_builders.all + if 'release' in b.get('tags', []) + ] + ), reporters.IRC( useColors = False, @@ -42,17 +46,12 @@ messageFormatter = LLVMInformativeMailNotifier, # TODO: For debug purposes only. Remove later. dumpMailsToLog = True, + builders = [ + b.get('name') for b in config.builders.all + if 'silent' not in b.get('tags', []) + ] ), - # In addition to that the following notifiers are defined for special - # cases. - reporters.MailNotifier( - fromaddr = "llvm.buildmaster@lab.llvm.org", - sendToInterestedUsers = False, - extraRecipients = ["leandro.nunes@arm.com"], - subject = "Build %(builder)s Failure", - mode = "failing", - builders = ["clang-aarch64-linux-build-cache", "clang-armv7-linux-build-cache"]), reporters.MailNotifier( fromaddr = "llvm.buildmaster@lab.llvm.org", sendToInterestedUsers = False, @@ -62,11 +61,11 @@ builders = ["clang-x86_64-debian-fast"]), reporters.MailNotifier( fromaddr = "llvm.buildmaster@lab.llvm.org", - sendToInterestedUsers = False, - extraRecipients = ["mstester.llvm@gmail.com"], + sendToInterestedUsers = False, + extraRecipients = ["tbaeder@redhat.com", "tstellar@redhat.com"], subject = "Build %(builder)s Failure", mode = "failing", - builders = ["clang-x64-ninja-win7"]), + builders = ["llvm-x86_64-debian-dylib"]), reporters.MailNotifier( fromaddr = "llvm.buildmaster@lab.llvm.org", sendToInterestedUsers = False, @@ -81,7 +80,7 @@ subject = "Build %(builder)s Failure", mode = "failing", builders = ["clang-s390x-linux", "clang-s390x-linux-multistage", - "clang-s390x-linux-lnt"]), + "clang-s390x-linux-lnt", "mlir-s390x-linux"]), reporters.MailNotifier( fromaddr = "llvm.buildmaster@lab.llvm.org", sendToInterestedUsers = False, @@ -104,7 +103,7 @@ subject = "Build %(builder)s Failure", mode = "failing", builders = ["lld-x86_64-win", - "lld-x86_64-freebsd", "lld-x86_64-darwin", + "lld-x86_64-freebsd", "clang-x86_64-linux-abi-test", "clang-with-lto-ubuntu", "clang-with-thin-lto-ubuntu", "llvm-clang-x86_64-expensive-checks-win", @@ -141,8 +140,9 @@ reporters.MailNotifier( fromaddr = "llvm.buildmaster@lab.llvm.org", sendToInterestedUsers = False, - extraRecipients = ["stilis@microsoft.com", "jonas@devlieghere.com", - "diprou@microsoft.com", "makudrya@microsoft.com"], + extraRecipients = ["stilis@microsoft.com", + "diprou@microsoft.com", + "makudrya@microsoft.com"], subject = "Build %(builder)s Failure", mode = "failing", builders = ["lldb-x64-windows-ninja"]), @@ -160,20 +160,6 @@ subject = "Build %(builder)s Failure", mode = "failing", builders = ["fuchsia-x86_64-linux"]), - reporters.MailNotifier( - fromaddr = "llvm.buildmaster@lab.llvm.org", - sendToInterestedUsers = False, - extraRecipients = ["jan.kratochvil@redhat.com"], - subject = "Build %(builder)s Failure", - mode = "failing", - builders = ["lldb-x86_64-fedora"]), - reporters.MailNotifier( - fromaddr = "llvm.buildmaster@lab.llvm.org", - sendToInterestedUsers = False, - extraRecipients = ["kkleine@redhat.com"], - subject = "Build %(builder)s Failure", - mode = "failing", - builders = ["fedora-llvm-x86_64", "x86_64-fedora-clang"]), reporters.MailNotifier( fromaddr = "llvm.buildmaster@lab.llvm.org", sendToInterestedUsers = False, @@ -196,7 +182,9 @@ mode = "failing", builders = ["llvm-clang-x86_64-win-fast","lld-x86_64-ubuntu-fast", "llvm-clang-x86_64-expensive-checks-ubuntu", - "llvm-clang-win-x-armv7l", "llvm-clang-win-x-aarch64"]), + "llvm-clang-win-x-armv7l", "llvm-clang-win-x-aarch64", + "llvm-nvptx-nvidia-ubuntu", "llvm-nvptx64-nvidia-ubuntu", + "llvm-nvptx-nvidia-win", "llvm-nvptx64-nvidia-win"]), reporters.MailNotifier( fromaddr = "llvm.buildmaster@lab.llvm.org", sendToInterestedUsers = False, @@ -207,11 +195,17 @@ reporters.MailNotifier( fromaddr = "llvm.buildmaster@lab.llvm.org", sendToInterestedUsers = False, - extraRecipients = ["sivachandra@google.com", "paulatoth@google.com"], + extraRecipients = ["lntue@google.com", "michaelrj@google.com", + "sivachandra@google.com"], subject = "Build %(builder)s Failure", mode = "failing", builders = ["libc-x86_64-debian", "libc-x86_64_debian-dbg", - "libc-x86_64-debian-dbg-asan"]), + "libc-x86_64-debian-dbg-runtimes-build", + "libc-x86_64-debian-dbg-asan", "libc-aarch64-ubuntu-dbg", + "libc-x86_64-windows-dbg", "libc-arm32-debian-dbg", + "libc-aarch64-ubuntu-fullbuild-dbg", + "libc-x86_64-debian-fullbuild-dbg", + "libc-x86_64-debian-fullbuild-dbg-asan"]), reporters.MailNotifier( fromaddr = "llvm.buildmaster@lab.llvm.org", sendToInterestedUsers = False, @@ -228,33 +222,119 @@ mode = "failing", builders = [ "ml-opt-dev-x86-64", "ml-opt-rel-x86-64", "ml-opt-devrel-x86-64"]), + reporters.MailNotifier( + fromaddr="llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers = False, + extraRecipients=[ + "tejohnson@google.com"], + subject = "ThinLTO WPD Failure: %(builder)s", + mode = "failing", + builders = ["clang-with-thin-lto-wpd-ubuntu"]), reporters.MailNotifier( fromaddr = "llvm.buildmaster@lab.llvm.org", sendToInterestedUsers = False, - extraRecipients = [ - "caroline.concatto@arm.com", "flang_llvm_buildbots@arm.com"], + extraRecipients = ["douglas.yung@sony.com"], subject = "Build %(builder)s Failure", mode = "failing", - builders = [ - "flang-aarch64-ubuntu", "flang-aarch64-ubuntu-clang", - "flang-aarch64-ubuntu-gcc10"]), + builders = ["llvm-clang-x86_64-sie-ubuntu-fast", + "llvm-clang-x86_64-sie-win"]), reporters.MailNotifier( fromaddr="llvm.buildmaster@lab.llvm.org", sendToInterestedUsers = False, extraRecipients=[ - "tejohnson@google.com"], - subject = "ThinLTO WPD Failure: %(builder)s", + "mlir-bugs-external+buildbot@googlegroups.com"], + subject = "MLIR Build Failure: %(builder)s", mode = "failing", - builders = ["thinlto-x86-64-bot1"]), + builders = ["mlir-nvidia", + "mlir-windows", + "ppc64le-mlir-rhel-clang"]), + reporters.MailNotifier( + fromaddr="llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers = False, + extraRecipients=["dl.gcr.lightning.buildbot@amd.com"], + subject = "Build Failure: %(builder)s", + mode = "failing", + builders = ["clang-hip-vega20"]), + reporters.MailNotifier( + fromaddr="llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers = False, + extraRecipients=["llvm_arc_buildbot@synopsys.com", "heli@synopsys.com"], + subject = "Build Failure: %(builder)s", + mode = "failing", + builders = ["arc-builder"]), + reporters.MailNotifier( + fromaddr="llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers = False, + extraRecipients=["dl.gcr.lightning.buildbot@amd.com"], + subject = "Build Failure: %(builder)s", + mode = "failing", + builders = ["openmp-offload-amdgpu-runtime"]), + reporters.MailNotifier( + fromaddr="llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers = False, + extraRecipients=["dl.mlse.buildbot@amd.com"], + subject = "Build Failure: %(builder)s", + mode = "failing", + builders = ["mlir-rocm-mi200"]), + reporters.MailNotifier( + fromaddr="llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers = False, + extraRecipients=["flangbuilder@meinersbur.de"], + subject = "Build Failure (flang): %(builder)s", + mode = "failing", + builders = ["flang-x86_64-windows"]), + reporters.MailNotifier( + fromaddr="llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers = False, + extraRecipients=["offloadbuilder@meinersbur.de"], + subject = "Build Failure (offload): %(builder)s", + mode = "failing", + builders = ["openmp-offload-cuda-project","openmp-offload-cuda-runtime"]), + reporters.MailNotifier( + fromaddr="llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers = False, + extraRecipients=["pollybuilder@meinersbur.de"], + subject = "Build Failure (polly): %(builder)s", + mode = "failing", + builders = [ + "polly-x86_64-linux", + "polly-x86_64-linux-noassert", + "polly-x86_64-linux-plugin", + "polly-x86_64-linux-shared", + "polly-x86_64-linux-shared-plugin", + "polly-x86_64-linux-shlib", + "polly-x86_64-linux-shlib-plugin", + "polly-sphinx-docs", + ]), + reporters.MailNotifier( + fromaddr = "llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers = False, + extraRecipients = ["orlando.hyams@sony.com"], + subject = "Build %(builder)s Failure", + mode = "failing", + builders = ["cross-project-tests-sie-ubuntu", + "llvm-clang-x86_64-sie-win"]), + reporters.MailNotifier( + fromaddr = "llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers = False, + extraRecipients = ["kkleine@redhat.com"], + subject = "Build %(builder)s Failure", + mode = "failing", + builders = ["standalone-build-x86_64"]), ] + +from twisted.python import log + # Returns a list of Status Targets. The results of each build will be # pushed to these targets. buildbot.plugins reporters has a variety # to choose from, including email senders, and IRC bots. def getReporters(): if config.options.getboolean('Master Options', 'is_production'): + log.msg(">>> getReporters: Production mode. All reporters are registered.") return all else: # Staging buildbot does not report issues. + log.msg(">>> getReporters: Staging buildbot does not report issues.") return [] diff --git a/buildbot/osuosl/master/config/workers.py b/buildbot/osuosl/master/config/workers.py --- a/buildbot/osuosl/master/config/workers.py +++ b/buildbot/osuosl/master/config/workers.py @@ -16,9 +16,6 @@ # Linux Ubuntu create_worker("as-worker-5", properties={'jobs' : 16}), - # Mac Pro 2.7 GHz 12-Core Intel Xeon E5, Maverick 10.9.2 - create_worker("as-worker-3", properties={'jobs' : 8}, max_builds=4), - # ARMv7/ARMv8 Linaro workers create_worker("linaro-tk1-01", properties={'jobs' : 4}, max_builds=1), create_worker("linaro-tk1-02", properties={'jobs' : 4}, max_builds=1), @@ -29,38 +26,37 @@ create_worker("linaro-tk1-07", properties={'jobs' : 4}, max_builds=1), create_worker("linaro-tk1-08", properties={'jobs' : 4}, max_builds=1), create_worker("linaro-tk1-09", properties={'jobs' : 4}, max_builds=1), - create_worker("linaro-armv7-lnt", properties={'jobs' : 32}, max_builds=1), - create_worker("linaro-armv7-selfhost", properties={'jobs' : 32}, max_builds=1), - create_worker("linaro-armv7-quick", properties={'jobs' : 32}, max_builds=1), - create_worker("linaro-armv7-global-isel", properties={'jobs' : 32}, max_builds=1), - create_worker("linaro-armv8-lld", properties={'jobs' : 32}, max_builds=1), - # Libcxx testsuite has tests with timing assumptions. Run single-threaded to make - # sure we have plenty CPU cycle to satisfy timing assumptions. - create_worker("linaro-armv8-libcxx", properties={'jobs' : 1}, max_builds=1), - create_worker("linaro-arm-lldb", properties={'jobs' : 16}, max_builds=1), + create_worker("linaro-clang-armv7-lnt", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-clang-armv7-2stage", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-clang-armv7-quick", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-clang-armv7-global-isel", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-clang-armv7-vfpv3-2stage", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-clang-armv8-lld-2stage", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-lldb-arm-ubuntu", properties={'jobs' : 16}, max_builds=1), # AArch64 Linaro workers - create_worker("linaro-aarch64-quick", properties={'jobs' : 32}, max_builds=1), - create_worker("linaro-aarch64-full", properties={'jobs' : 32}, max_builds=1), - create_worker("linaro-aarch64-global-isel", properties={'jobs' : 32}, max_builds=1), - create_worker("linaro-aarch64-lld", properties={'jobs' : 32}, max_builds=1), - create_worker("linaro-aarch64-flang-oot", properties={'jobs' : 32}, max_builds=1), - create_worker("linaro-aarch64-flang-dylib", properties={'jobs' : 32}, max_builds=1), - create_worker("linaro-aarch64-flang-sharedlibs", properties={'jobs' : 32}, max_builds=1), - # Libcxx testsuite has tests with timing assumptions. Run single-threaded to make - # sure we have plenty CPU cycle to satisfy timing assumptions. - create_worker("linaro-aarch64-libcxx", properties={'jobs' : 1}, max_builds=1), - create_worker("linaro-aarch64-lldb", properties={'jobs': 16}, max_builds=1), - - # AArch64 + create_worker("linaro-clang-aarch64-quick", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-clang-aarch64-lld-2stage", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-clang-aarch64-global-isel", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-clang-aarch64-full-2stage", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-clang-aarch64-sve-vla", properties={'jobs' : 48}, max_builds=1), + create_worker("linaro-clang-aarch64-sve-vla-2stage", properties={'jobs' : 48}, max_builds=1), + create_worker("linaro-clang-aarch64-sve-vls", properties={'jobs' : 48}, max_builds=1), + create_worker("linaro-clang-aarch64-sve-vls-2stage", properties={'jobs' : 48}, max_builds=1), + create_worker("linaro-lldb-aarch64-ubuntu", properties={'jobs': 16}, max_builds=1), + create_worker("linaro-flang-aarch64-dylib", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-flang-aarch64-sharedlibs", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-flang-aarch64-out-of-tree", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-flang-aarch64-debug", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-flang-aarch64-latest-clang", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-flang-aarch64-release", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-flang-aarch64-rel-assert", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-flang-aarch64-latest-gcc", properties={'jobs' : 32}, max_builds=1), + + # AArch64 Windows Microsoft Surface X Pro create_worker("linaro-armv8-windows-msvc-01", properties={'jobs' : 8}, max_builds=1), create_worker("linaro-armv8-windows-msvc-02", properties={'jobs' : 8}, max_builds=1), - - # ARMv7 build cache workers - create_worker("packet-linux-armv7-slave-1", properties={'jobs' : 64}, max_builds=1), - - # AArch64 build cache worker - create_worker("packet-linux-aarch64-slave-1", properties={'jobs' : 64}, max_builds=1), + create_worker("linaro-lldb-aarch64-windows", properties={'jobs' : 8}, max_builds=1), # Windows Server 2016 Intel Xeon(R) Quad 2.30 GHz, 56GB of RAM create_worker("win-py3-buildbot", properties={'jobs' : 64}, max_builds=1), @@ -68,20 +64,26 @@ # Windows Server 2016 Intel(R) Xeon(R) CPU @ 2.60GHz, 16 Core(s), 128 GB of RAM create_worker("win-mlir-buildbot", properties={'jobs' : 64}, max_builds=1), + # MIPS Loongson-3A R4 (Loongson-3A4000) 64-bit little endian (mips64el) + create_worker("debian-tritium-mips64el", properties={'jobs': 1}, max_builds=1), + + # Motorola 68k 32-bit big endian (m68k) + create_worker("debian-akiko-m68k", properties={'jobs': 1}, max_builds=1), + create_worker("suse-gary-m68k-cross", properties={'jobs': 4}, max_builds=1), + # POWER7 PowerPC big endian (powerpc64) - create_worker("ppc64be-clang-test", properties={'jobs': 16}, max_builds=1), - create_worker("ppc64be-clang-lnt-test", properties={'jobs': 16, 'vcs_protocol': 'https'}, max_builds=1), + create_worker("ppc64be-clang-test-suite", max_builds=1), create_worker("ppc64be-clang-multistage-test", properties={'jobs': 16}, max_builds=1), create_worker("ppc64be-sanitizer", properties={'jobs': 16}, max_builds=1), # POWER 8 PowerPC little endian (powerpc64le) - create_worker("ppc64le-clang-test", properties={'jobs': 4}, max_builds=1), - create_worker("ppc64le-clang-lnt-test", properties={'jobs': 8}, max_builds=1), + create_worker("ppc64le-clang-test-suite", max_builds=1), create_worker("ppc64le-clang-multistage-test", properties={'jobs': 8}, max_builds=1), create_worker("ppc64le-sanitizer", properties={'jobs': 4}, max_builds=1), create_worker("ppc64le-lld-multistage-test", max_builds=1), - create_worker("ppc64le-clang-rhel-test", properties={'jobs': 4}, max_builds=1), - create_worker("ppc64le-flang-mlir-rhel-test", max_builds=1), + create_worker("ppc64le-clang-rhel-test", properties={'jobs': 192}, max_builds=1), + create_worker("ppc64le-flang-rhel-test", max_builds=1), + create_worker("ppc64le-mlir-rhel-test", max_builds=1), # SPARC 64-bit big endian (sparc64) create_worker("debian-stadler-sparc64", properties={'jobs': 4}, max_builds=1), @@ -108,20 +110,21 @@ "gribozavr@gmail.com", "gkistanova@gmail.com"]), - # Debian 7.7 x86_64 GCE instance + # Ubuntu x86_64 6-core z440 workstation + create_worker("sanitizer-buildbot-android", properties={'jobs': 6}, max_builds=1), + # x86_64 GCE instances create_worker("sanitizer-buildbot1", properties={'jobs': 64}, max_builds=3), - # Debian 7.7 x86_64 GCE instance create_worker("sanitizer-buildbot2", properties={'jobs': 64}, max_builds=3), - # Debian 7.7 x86_64 GCE instance - create_worker("sanitizer-buildbot3", properties={'jobs': 64}, max_builds=2), - # Debian 7.7 x86_64 GCE instance - create_worker("sanitizer-buildbot4", properties={'jobs': 64}, max_builds=2), - # Ubuntu 14.04 x86_64 6-core z440 workstation - create_worker("sanitizer-buildbot6", properties={'jobs': 6}, max_builds=1), - # Debian 7.7 x86_64 GCE instance - create_worker("sanitizer-buildbot7", properties={'jobs': 64}, max_builds=3), - # Debian 7.7 x86_64 GCE instance - create_worker("sanitizer-buildbot8", properties={'jobs': 64}, max_builds=3), + create_worker("sanitizer-buildbot3", properties={'jobs': 64}, max_builds=3), + create_worker("sanitizer-buildbot4", properties={'jobs': 64}, max_builds=3), + create_worker("sanitizer-buildbot5", properties={'jobs': 64}, max_builds=3), + create_worker("sanitizer-buildbot6", properties={'jobs': 64}, max_builds=3), + # aarch64 GCE instances + create_worker("sanitizer-buildbot7", properties={'jobs': 48}, max_builds=3), + create_worker("sanitizer-buildbot8", properties={'jobs': 48}, max_builds=3), + + # POWER 8 PowerPC AIX 7.2 + create_worker("aix-ppc64", properties={'jobs': 10}, max_builds=1), # IBM z13 (s390x), Ubuntu 16.04.2 create_worker("systemz-1", properties={'jobs': 4, 'vcs_protocol': 'https'}, max_builds=4), @@ -132,8 +135,8 @@ create_worker("windows-gcebot2", properties={'jobs': 32}, max_builds=1), # Ubuntu 14.04 x86_64-scei-ps4, 2 x Intel(R) Xeon(R) CPU E5-2699 v3 @ 2.30GHz - create_worker("as-worker-91"), - create_worker("as-worker-92"), + create_worker("as-worker-91", max_builds=1), + create_worker("as-worker-92", max_builds=1), # Windows 10 Pro x86_64-scei-ps4, 2 x Intel(R) Xeon(R) CPU E5-2699 v3 @ 2.30GHz create_worker("as-worker-93", properties={'jobs': 36}, max_builds=1), @@ -143,6 +146,9 @@ create_worker("cuda-p4-0", max_builds=1), create_worker("cuda-t4-0", max_builds=1), + # HIP on Ubuntu 18.04.5, Intel(R) Xeon(R) Gold 5218 @ 2.30GHz, Vega20 GPU + create_worker("hip-vega20-0", max_builds=1), + # X86_64 AVX2, Ubuntu 16.04.2, Intel(R) Xeon(R) CPU E5-2699 v4 @ 2.20GHz create_worker("avx2-intel64", properties={'jobs': 80}, max_builds=1), @@ -153,18 +159,24 @@ create_worker("fuchsia-debian-64-us-central1-a-1", properties={'jobs': 64}, max_builds=1), create_worker("fuchsia-debian-64-us-central1-b-1", properties={'jobs': 64}, max_builds=1), - # Fedora latest stable x86_64, Intel i5-2500, 4 cores, 12GB RAM - create_worker("lldb-x86_64-fedora", properties={'jobs': 4}, max_builds=1), - - # Fedora latest stable, arch=x86_64, running on RedHat internal OpenShift PSI cluster - create_worker("fedora-llvm-x86_64", properties={'jobs': 64}, max_builds=1), - # Debian x86_64 Buster Xeon(R) Gold 6154 CPU @ 3.00GHz, 192GB RAM create_worker("lldb-x86_64-debian", properties={'jobs': 72}, max_builds=1), + # Windows x86_64 32 CPUs, 125 GB RAM + create_worker("libc-x86_64-windows", properties={'jobs': 32}, max_builds=2), + + # Debian arm32 single core, 512 MB RAM backed by 32 GB swap memory + create_worker("libc-arm32-debian", properties={'jobs': 1}, max_builds=1), + + # Ubuntu aarch64 128 CPUs, 125 GB RAM + create_worker("libc-aarch64-ubuntu", properties={'jobs': 32}, max_builds=2), + # Debian x86_64 Intel Broadwell 32 CPUs, 120 GB RAM create_worker("libc-x86_64-debian", properties={'jobs': 32}, max_builds=2), + # Debian x86_64 Intel Skylake 32 CPUs, 128 GB RAM + create_worker("libc-x86_64-debian-fullbuild", properties={'jobs': 32}, max_builds=2), + # Windows Server on Xeon Gold 6130 (2x2.1GHz), 128Gb of RAM create_worker("as-builder-1", properties={ 'remote_test_host': 'jetson6.lab.llvm.org', @@ -185,22 +197,43 @@ # Ubuntu 18.04.2 LTS x86_64 Intel(R) Xeon(R) Gold CPU @ 2.1GHz, 128GB RAM create_worker("as-builder-4", properties={'jobs': 64}, max_builds=1), + # Windows Server on Xeon Gold 6230 (2x2.1GHz), 256Gb of RAM + create_worker("as-builder-5", properties={ # arm + 'remote_test_host': 'jetson4.lab.llvm.org', + 'remote_test_user': 'ubuntu' + }, + max_builds=1), + create_worker("as-builder-6", properties={ # aarch64 + 'remote_test_host': 'jetson8.lab.llvm.org', + 'remote_test_user': 'ubuntu' + }, + max_builds=1), + + # NVPTX builders. + # Ubuntu 22.04 LTS x86_64 Intel Xeon 6330 CPU 2Sx64Cx128LP @ 2.0GHz, 256GB RAM + create_worker("as-builder-7", properties={'jobs': 64}, max_builds=2), + # Windows Server on Intel Xeon 6330 CPU 2Sx64Cx128LP @ 2.0GHz, 256GB RAM + create_worker("as-builder-8", max_builds=2), + # Solaris 11 create_worker("solaris11-amd64", properties={'jobs' : 8}, max_builds=1), create_worker("solaris11-sparcv9", properties={'jobs' : 8}, max_builds=1), - # Windows 7 Intel(R) Core(TM) CPU i7-4790K (4.00GHz), 16GB of RAM - create_worker("windows7-buildbot", properties={'jobs': 2}, max_builds=1), - - # Windows 10, Visual Studio 2019, Google Cloud 16 cores - create_worker("windows10-vs2019", properties={'jobs': 32}, max_builds=1), - - # CentOS 7.5.1804 on Intel(R) Xeon(R) Gold 6126 CPU @ 2.60GHz, 96GB RAM - create_worker("nec-arrproto41", properties={'jobs': 12}, max_builds=1), + # CentOS 7.8.1127 on Intel(R) Xeon(R) Gold 6126 CPU @ 2.60GHz, 96GB RAM + create_worker("hpce-aurora2", properties={'jobs': 8}, max_builds=1), + create_worker("hpce-ve-main", properties={'jobs': 8}, max_builds=1), + create_worker("hpce-ve-staging", properties={'jobs': 8}, max_builds=1), # Ubuntu 18.04 amd64 on Google Cloud, 16 core, Nvidia Tesla T4 create_worker("mlir-nvidia", properties={'jobs': 16}, max_builds=1), + # Ubuntu 18.04 pool of VMs on GCP + create_worker("mlir-ubuntu-worker0", max_builds=1), + create_worker("mlir-ubuntu-worker1", max_builds=1), + create_worker("mlir-ubuntu-worker2", max_builds=1), + create_worker("mlir-ubuntu-worker3", max_builds=1), + create_worker("mlir-ubuntu-worker4", max_builds=1), + # Ubuntu 18.04 on Google Cloud, for machine configuration check # buildbot/google/terraform/main.tf create_worker("clangd-ubuntu-clang", max_builds=1), @@ -212,20 +245,64 @@ # Ubuntu 18.04.LTS x86_64, Intel(R) Xeon(R) CPU X3460 @ 2.80GHz, 32 GiB RAM create_worker("polly-x86_64-fdcserver", properties={'jobs': 8, 'loadaverage': 8}, max_builds=1), - create_worker("flang-aarch64-ubuntu-build"), - create_worker("flang-aarch64-ubuntu-clang-build", properties={'jobs': 14}), - create_worker("flang-aarch64-ubuntu-gcc10-build", properties={'jobs': 14}), - create_worker("nersc-flang"), + # Windows 10, AMD Ryzen 5 PRO 4650G, 16 GiB RAM + create_worker("minipc-ryzen-win", properties={'jobs': 12}, max_builds=1), + + # Ubuntu 20.04.LTS x86_64, Intel(R) Core(TM) i5-9400F CPU @ 2.90Ghz, 16 GiB RAM, NVIDIA GeForce GTX 1050 Ti (Pascal, sm_61, 4GiB) + create_worker("minipc-1050ti-linux", properties={'jobs': 6}, max_builds=1), + create_worker("alcf-theta-flang", properties={'jobs': 12}, max_builds=1), # ML-Driven Compiler Optimizations build workers (Ubuntu x86_64) create_worker("ml-opt-dev-x86-64-b1", - properties={'jobs': 64}, max_builds=1), + properties={'jobs': 64}, max_builds=1, + notify_on_missing='mlcompileropt-buildbot@google.com'), create_worker("ml-opt-rel-x86-64-b1", - properties={'jobs': 64}, max_builds=1), + properties={'jobs': 64}, max_builds=1, + notify_on_missing='mlcompileropt-buildbot@google.com'), create_worker("ml-opt-devrel-x86-64-b1", - properties={'jobs': 64}, max_builds=1), + properties={'jobs': 64}, max_builds=1, + notify_on_missing='mlcompileropt-buildbot@google.com'), + create_worker("ml-opt-dev-x86-64-b2", + properties={'jobs': 64}, max_builds=1, + notify_on_missing='mlcompileropt-buildbot@google.com'), + create_worker("ml-opt-rel-x86-64-b2", + properties={'jobs': 64}, max_builds=1, + notify_on_missing='mlcompileropt-buildbot@google.com'), + create_worker("ml-opt-devrel-x86-64-b2", + properties={'jobs': 64}, max_builds=1, + notify_on_missing='mlcompileropt-buildbot@google.com'), # Ubuntu x86_64 create_worker("thinlto-x86-64-bot1", properties={'jobs': 64}, max_builds=1), + create_worker("thinlto-x86-64-bot2", properties={'jobs': 64}, max_builds=1), + + # Ubuntu 20.04 on AWS, x86_64 PS4 target + create_worker("sie-linux-worker", properties={'jobs': 40}, max_builds=1), + + # Windows Server 2019 on AWS, x86_64 PS4 target + create_worker("sie-win-worker", properties={'jobs': 64}, max_builds=1), + + # XCore target, Ubuntu 20.04 x64 host + create_worker("xcore-ubuntu20-x64", properties={'jobs': 4}, max_builds=1), + + # ARC Worker, CentOS 7.9 x86_64 Intel Xeon Platinum 8000 @ 3.6GHz, 32GB RAM + create_worker("arc-worker", properties={'jobs': 16}, max_builds=1), + + # OpenMP on AMDGPU, Ubuntu 18.04.5, Intel(R) Xeon(R) Gold 5218 @ 2.30GHz with 64GB Memory, 1 Vega20 GPU with 16GB Memory + create_worker("omp-vega20-0", properties={'jobs': 32}, max_builds=1), + create_worker("omp-vega20-1", properties={'jobs': 32}, max_builds=1), + + # AMD ROCm support, Ubuntu 18.04.6, AMD Ryzen @ 1.5 GHz, MI200 GPU + create_worker("mi200-buildbot", max_builds=1), + + # BOLT workers + create_worker("bolt-worker", properties={'jobs' : 16}, max_builds=1), + create_worker("bolt-worker-aarch64", properties={'jobs' : 2}, max_builds=1), + + # Fedora worker + create_worker("standalone-build-x86_64", max_builds=1), + + # CSKY T-HEAD workers + create_worker("thead-clang-csky", properties={'jobs' : 32}, max_builds=1), ] diff --git a/buildbot/osuosl/master/master.cfg b/buildbot/osuosl/master/master.cfg --- a/buildbot/osuosl/master/master.cfg +++ b/buildbot/osuosl/master/master.cfg @@ -64,13 +64,19 @@ c['builders'] = builders = [ util.BuilderConfig(**b) for b in config.builders.all ] +builders.extend([ + util.BuilderConfig(**b) for b in config.release_builders.all +]) ####### SCHEDULERS -c['schedulers'] = config.schedulers.getSingleBranchSchedulers( - c['builders']) +c['schedulers'] = config.schedulers.getMainBranchSchedulers( + builders) +c['schedulers'].extend(config.schedulers.getReleaseBranchSchedulers( + builders, + treeStableTimer=5*60)) c['schedulers'].extend(config.schedulers.getForceSchedulers( - c['builders'])) + builders)) ####### BUILDBOT SERVICES diff --git a/llvmbisect/llvmlab/llvmlab.py b/llvmbisect/llvmlab/llvmlab.py --- a/llvmbisect/llvmlab/llvmlab.py +++ b/llvmbisect/llvmlab/llvmlab.py @@ -46,7 +46,7 @@ def frombasename(str, url=None): str = os.path.basename(str) - revision = timestamp = build = None + revision = revision_prefix = sha = timestamp = build = None # Check if this is a BNI style build. m = BUILD_NAME_REGEX.match(str) @@ -58,7 +58,7 @@ revision.append(int(minor_str)) if micro_str: revision.append(int(micro_str)) - return Build(name, tuple(revision), None, build, suffix) + return Build(name, None, tuple(revision), None, None, build, suffix) if '.' in str: str, suffix = str.split('.', 1) @@ -74,12 +74,20 @@ if m: str, timestamp = m.groups() + m = re.match(r'(.*)-d([0-9]+)-(.*)', str) + if m: + str, revision, sha = m.groups() + revision = int(revision) + revision_prefix = 'd' + m = re.match(r'(.*)-r([0-9]+)', str) if m: str, revision = m.groups() revision = int(revision) + revision_prefix = 'r' - return Build(str, revision, timestamp, build, suffix, url) + return Build(str, revision_prefix, revision, sha, timestamp, + build, suffix, url) @staticmethod def fromdata(data): @@ -89,13 +97,17 @@ def todata(self): return {'name': self.name, 'revision': self.revision, + 'sha': self.sha, 'timestamp': self.timestamp, 'build': self.build, 'suffix': self.suffix} - def __init__(self, name, revision, timestamp, build, suffix, url): + def __init__(self, name, revision_prefix, revision, sha, timestamp, + build, suffix, url): self.name = name + self.revision_prefix = revision_prefix self.revision = revision + self.sha = sha self.timestamp = timestamp self.build = build self.suffix = suffix @@ -108,7 +120,9 @@ basename += '-' + '.'.join(str(r) for r in self.revision) else: assert isinstance(self.revision, int) - basename += '-r%d' % self.revision + basename += '-%s%d' % (self.revision_prefix, self.revision) + if self.sha is not None: + basename += '-%s' % self.sha if self.timestamp is not None: basename += '-t%s' % self.timestamp if self.build is not None: @@ -122,8 +136,8 @@ def __repr__(self): return "%s%r" % (self.__class__.__name__, - (self.name, self.revision, self.timestamp, self.build, - self.suffix)) + (self.name, self.revision, self.sha, self.timestamp, + self.build, self.suffix)) def __cmp__(self, other): return cmp((self.revision, self.timestamp, diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -1,29 +1,28 @@ -attrs==20.2.0 +attrs==20.3.0 six==1.15.0 Automat==20.2.0 constantly==15.1.0 idna==2.10 -hyperlink==20.0.1 -incremental==17.5.0 -Jinja2==2.11.2 -PyHamcrest==2.0.2 +hyperlink==21.0.0 +incremental==21.3.0 +Jinja2==2.11.3 Tempita==0.5.2 decorator==4.4.2 pbr==5.5.1 -sqlparse==0.4.1 -SQLAlchemy==1.3.20 +sqlparse==0.4.2 +SQLAlchemy==1.3.23 sqlalchemy-migrate==0.13.0 -certifi==2020.6.20 -chardet==3.0.4 -urllib3==1.25.11 -requests==2.24.0 -zope.interface==5.1.2 +certifi==2020.12.5 +chardet==4.0.0 +urllib3==1.26.8 +requests==2.25.1 +zope.interface==5.2.0 pycparser==2.20 -pyOpenSSL==19.1.0 +pyOpenSSL==20.0.1 pyasn1==0.4.8 pyasn1-modules==0.2.8 service-identity==18.1.0 -cryptography==3.2.1 -cffi==1.14.3 -Twisted==20.3.0 -treq==20.9.0 +cryptography==3.4.5 +cffi==1.14.5 +Twisted==22.4.0 +treq==22.1.0 diff --git a/tasks/cmake/caches/util/xcode_sdk.cmake b/tasks/cmake/caches/util/xcode_sdk.cmake --- a/tasks/cmake/caches/util/xcode_sdk.cmake +++ b/tasks/cmake/caches/util/xcode_sdk.cmake @@ -51,6 +51,12 @@ # because of "CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY" explained just # above. We work around that by forcing it to NO. set(HAVE_RE_COMP CACHE BOOL NO) + + # Likewise, some benchmarks look for librt, which does not exist on darwin + # systems, but the check returns YES because we don't actually link, given + # the CMAKE_TRY_COMPILE_TARGET_TYPE setting above. We work around that by + # forcing it to NO. + set(HAVE_LIB_RT CACHE BOOL NO) else() # Search and use compiler coming with the SDK. # Note that we do not search CMAKE_CXX_COMPILER here. cmake will try diff --git a/tasks/tasktool/README.md b/tasks/tasktool/README.md --- a/tasks/tasktool/README.md +++ b/tasks/tasktool/README.md @@ -92,7 +92,7 @@ { "url": "https://github.com/my-project.git", "type": "git", - "default_rev": "refs/heads/master" + "default_rev": "refs/heads/main" } ``` @@ -166,7 +166,7 @@ : ${VARIABLE:='DefaultValue'} ``` -- Abstract artifacts like `refs/heads/master` in a git repository or +- Abstract artifacts like `refs/heads/main` in a git repository or `clang-stage2-cmake` on an A2 server are resolved to a concrete git revision or a specific URL when the task is started. - A task with a set of artifact and parameter inputs is called a build. diff --git a/tasks/tasktool/tasktool/repos/git.py b/tasks/tasktool/tasktool/repos/git.py --- a/tasks/tasktool/tasktool/repos/git.py +++ b/tasks/tasktool/tasktool/repos/git.py @@ -25,7 +25,7 @@ rev = config.get('rev') default_rev = config.pop('default_rev', None) if default_rev is None: - default_rev = "refs/heads/master" + default_rev = "refs/heads/main" if rev is not None: return url = config['url'] diff --git a/tasks/utils/venv.sh b/tasks/utils/venv.sh --- a/tasks/utils/venv.sh +++ b/tasks/utils/venv.sh @@ -1,5 +1,5 @@ echo "@@@ LNT VirtualEnv @@@" -/usr/local/bin/virtualenv venv +/usr/local/bin/virtualenv --python=python3 venv set +u . venv/bin/activate set -u diff --git a/test/jenkins/test_monorepo_build.py b/test/jenkins/test_monorepo_build.py --- a/test/jenkins/test_monorepo_build.py +++ b/test/jenkins/test_monorepo_build.py @@ -3,9 +3,9 @@ # RUN: export TESTING=1 # RUN: export JOB_NAME="FOO" # RUN: export BUILD_NUMBER=321 -# RUN: export BRANCH=master +# RUN: export BRANCH=main # Tell monorepo_build.py to just print commands instead of running. -# RUN: mkdir -p %t.SANDBOX/host-compiler/lib %t.SANDBOX/host-compiler/bin %t.SANDBOX/llvm-project/llvm %t.SANDBOX/llvm-project/clang %t.SANDBOX/llvm-project/libcxx %t.SANDBOX/llvm-project/compiler-rt %t.SANDBOX/llvm-project/debuginfo-tests %t.SANDBOX/llvm-project/clang-tools-extra %t.SANDBOX/llvm-project/lldb +# RUN: mkdir -p %t.SANDBOX/host-compiler/lib %t.SANDBOX/host-compiler/bin %t.SANDBOX/llvm-project/llvm %t.SANDBOX/llvm-project/clang %t.SANDBOX/llvm-project/compiler-rt %t.SANDBOX/llvm-project/debuginfo-tests %t.SANDBOX/llvm-project/clang-tools-extra %t.SANDBOX/llvm-project/lldb # RUN: touch %t.SANDBOX/host-compiler/bin/clang # RUN: python %{src_root}/zorg/jenkins/monorepo_build.py clang all > %t.log # RUN: FileCheck --check-prefix CHECK-SIMPLE < %t.log %s @@ -20,7 +20,7 @@ # CHECK-SIMPLE: '/usr/local/bin/cmake' '-G' 'Ninja' '-C' # CHECK-SIMPLE: '-DLLVM_ENABLE_ASSERTIONS:BOOL=FALSE' # CHECK-SIMPLE: '-DCMAKE_BUILD_TYPE=RelWithDebInfo' -# CHECK-SIMPLE: '-DLLVM_ENABLE_PROJECTS=clang;clang-tools-extra;compiler-rt;libcxx' +# CHECK-SIMPLE: '-DLLVM_ENABLE_PROJECTS=clang;clang-tools-extra;compiler-rt' # CHECK-SIMPLE: '-DCMAKE_MAKE_PROGRAM=/usr/local/bin/ninja' # CHECK-SIMPLE: '-DLLVM_VERSION_PATCH=99' # CHECK-SIMPLE: '-DLLVM_VERSION_SUFFIX=""' @@ -146,4 +146,14 @@ # Test to check if default timeout is being set to 600. # RUN: python %{src_root}/zorg/jenkins/monorepo_build.py cmake all > %t-timeout-default.log # RUN: FileCheck --check-prefix CHECK-TIMEOUT-DEFAULT < %t-timeout-default.log %s -# CHECK-TIMEOUT-DEFAULT: --timeout=600 \ No newline at end of file +# CHECK-TIMEOUT-DEFAULT: --timeout=600 + +# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py lldb-cmake-matrix configure \ +# RUN: --lldb-test-compiler="MY_LLDB_TEST_COMPILER" \ +# RUN: --cmake-flag=-DLLVM_TARGETS_TO_BUILD=X86 > %t-lldb-configure.log +# RUN: FileCheck --check-prefix CHECK-LLDB-CONFIG < %t-lldb-configure.log %s + +# CHECK-LLDB-CONFIG: -DLLDB_TEST_COMPILER=MY_LLDB_TEST_COMPILER +# CHECK-LLDB-CONFIG: -DLLVM_TARGETS_TO_BUILD=X86 +# there may be other "target to build", but the one above should be the last: +# CHECK-LLDB-CONFIG-NOT: -DLLVM_TARGETS_TO_BUILD diff --git a/zorg/buildbot/builders/AOSPBuilder.py b/zorg/buildbot/builders/AOSPBuilder.py --- a/zorg/buildbot/builders/AOSPBuilder.py +++ b/zorg/buildbot/builders/AOSPBuilder.py @@ -25,8 +25,9 @@ target_flags=None, # Extra C/CXX flags for AOSP target build jobs=None, # Number of concurrent jobs extra_make_args=None): # Extra args for the make command - command = "source build/envsetup.sh" + command = "/bin/bash -c \"source build/envsetup.sh" command += " && lunch aosp_%s-userdebug" % device + command += " && export LC_ALL=C" command += " && make -k" if timeout: command += " TIMEOUT=%s" % timeout @@ -38,6 +39,7 @@ command += " " + extra_make_args if jobs: command += " -j" + str(jobs) + command += "\"" return command diff --git a/zorg/buildbot/builders/AnnotatedBuilder.py b/zorg/buildbot/builders/AnnotatedBuilder.py --- a/zorg/buildbot/builders/AnnotatedBuilder.py +++ b/zorg/buildbot/builders/AnnotatedBuilder.py @@ -11,7 +11,8 @@ env=None, extra_args=None, timeout=1200, - checkout_llvm_sources=True): + checkout_llvm_sources=True, + script_interpreter="python"): """ Returns a new build factory that uses AnnotatedCommand, which allows the build to be run by version-controlled scripts that do @@ -26,6 +27,7 @@ env: environment overrides (map; default is no overrides) extra_args: extra arguments to pass to the script (default: []) timeout: specifies the builder's timeout in seconds (default: 1200) + script_interpreter: specifies the interpreter to run scripts (default: "python") """ if depends_on_projects is None: @@ -38,7 +40,10 @@ "libunwind", "lld"] if extra_args is None: - extra_args = [] + # We used to add --jobs to all script invocations. Perserve this + # for cases when the user did not specify extra_args, but allow + # overriding it if the user did specify extra_args. + extra_args = ["--jobs=%(jobs:-)s"] f = LLVMBuildFactory( clean=clean, @@ -84,7 +89,12 @@ command = [script] else: script_path = "../llvm-zorg/zorg/buildbot/builders/annotated/%s" % (script) - command = ["python", script_path, WithProperties("--jobs=%(jobs:-)s")] + # Handle scripts with script_interpreter, otherwise execute the script directly. + if script_interpreter: + command = [script_interpreter, script_path] + else: + command = [script_path] + command += extra_args_with_props f.addStep(AnnotatedCommand(name="annotate", diff --git a/zorg/buildbot/builders/BOLTBuilder.py b/zorg/buildbot/builders/BOLTBuilder.py new file mode 100644 --- /dev/null +++ b/zorg/buildbot/builders/BOLTBuilder.py @@ -0,0 +1,79 @@ +from buildbot.plugins import steps +from zorg.buildbot.commands.CmakeCommand import CmakeCommand +from zorg.buildbot.builders.UnifiedTreeBuilder import getLLVMBuildFactoryAndSourcecodeSteps, addCmakeSteps, addNinjaSteps +from zorg.buildbot.process.factory import LLVMBuildFactory + +def getBOLTCmakeBuildFactory( + clean = False, + bolttests = False, + clangbolt = False, + extra_configure_args = None, + env = None, + **kwargs): + + if env is None: + env = dict() + + bolttests_dir = "bolt-tests" + + cleanBuildRequested = lambda step: clean or step.build.getProperty("clean", default=step.build.getProperty("clean_obj")) + cleanBuildRequestedByProperty = lambda step: step.build.getProperty("clean") + + targets = ['bolt'] + checks = ['check-bolt'] + + f = getLLVMBuildFactoryAndSourcecodeSteps( + depends_on_projects=['bolt', 'llvm'], + **kwargs) # Pass through all the extra arguments. + + if clangbolt: + targets += ['clang++-bolt'] + checks = [] + CmakeCommand.applyRequiredOptions(cmake_options, [ + ("-C", "../" + src_dir + "/clang/cmake/caches/BOLT.cmake"), + ]) + + if bolttests: + checks += ['check-large-bolt'] + extra_configure_args += [ + '-DLLVM_EXTERNAL_PROJECTS=bolttests', + '-DLLVM_EXTERNAL_BOLTTESTS_SOURCE_DIR=' + LLVMBuildFactory.pathRelativeTo(bolttests_dir, f.monorepo_dir), + ] + # Clean checkout of bolt-tests if cleanBuildRequested + f.addSteps([ + steps.RemoveDirectory(name="BOLT tests: clean", + dir=bolttests_dir, + haltOnFailure=True, + warnOnFailure=True, + doStepIf=cleanBuildRequestedByProperty), + + steps.Git(name="BOLT tests: checkout", + description="fetching", + descriptionDone="fetch", + descriptionSuffix="BOLT Tests", + repourl='https://github.com/rafaelauler/bolt-tests.git', + workdir=bolttests_dir, + alwaysUseLatest=True), + ]) + + # Some options are required for this build no matter what. + CmakeCommand.applyRequiredOptions(extra_configure_args, [ + ('-G', 'Ninja'), + ]) + + addCmakeSteps( + f, + cleanBuildRequested=cleanBuildRequested, + extra_configure_args=extra_configure_args, + obj_dir=None, + env=env, + **kwargs) + + addNinjaSteps( + f, + targets=targets, + checks=checks, + env=env, + **kwargs) + + return f diff --git a/zorg/buildbot/builders/ClangBuilder.py b/zorg/buildbot/builders/ClangBuilder.py --- a/zorg/buildbot/builders/ClangBuilder.py +++ b/zorg/buildbot/builders/ClangBuilder.py @@ -1,6 +1,7 @@ import copy from datetime import datetime +from buildbot.plugins import util from buildbot.process.properties import WithProperties, Property from buildbot.steps.shell import ShellCommand, SetProperty from buildbot.steps.shell import WarningCountingShellCommand @@ -71,7 +72,7 @@ def getClangCMakeGCSBuildFactory( clean=True, - test=True, + checks=None, cmake='cmake', jobs=None, @@ -114,7 +115,7 @@ # Triggers trigger_after_stage1=None): return _getClangCMakeBuildFactory( - clean=clean, test=test, cmake=cmake, jobs=jobs, vs=vs, + clean=clean, checks=checks, cmake=cmake, jobs=jobs, vs=vs, vs_target_arch=vs_target_arch, useTwoStage=useTwoStage, testStage1=testStage1, stage1_config=stage1_config, stage2_config=stage2_config, runTestSuite=runTestSuite, @@ -132,9 +133,10 @@ def getClangCMakeBuildFactory( clean=True, - test=True, + checks=None, cmake='cmake', jobs=None, + timeout=None, # TODO: Implement support for timeout # VS tools environment variable if using MSVC. For example, # %VS120COMNTOOLS% selects the 2013 toolchain. @@ -166,7 +168,7 @@ checkout_flang=False, checkout_test_suite=False): return _getClangCMakeBuildFactory( - clean=clean, test=test, cmake=cmake, jobs=jobs, vs=vs, + clean=clean, checks=checks, cmake=cmake, jobs=jobs, vs=vs, vs_target_arch=vs_target_arch, useTwoStage=useTwoStage, testStage1=testStage1, stage1_config=stage1_config, stage2_config=stage2_config, runTestSuite=runTestSuite, @@ -182,7 +184,7 @@ def _getClangCMakeBuildFactory( clean=True, - test=True, + checks=None, cmake='cmake', jobs=None, @@ -228,6 +230,8 @@ trigger_after_stage1=None): ############# PREPARING + if checks is None: + checks = ['check-all'] if nt_flags is None: nt_flags = [] if testsuite_flags is None: @@ -291,7 +295,7 @@ lit_args += "'" ninja_cmd = ['ninja'] + jobs_cmd ninja_install_cmd = ['ninja', 'install'] + jobs_cmd - ninja_check_cmd = ['ninja', 'check-all'] + jobs_cmd + ninja_check_cmd = ['ninja'] + checks + jobs_cmd # Global configurations stage1_build = 'stage1' @@ -346,7 +350,7 @@ workdir=stage1_build, env=env)) - if test and testStage1: + if checks and testStage1: haltOnStage1Check = not useTwoStage and not runTestSuite f.addStep(LitTestCommand(name='ninja check 1', command=ninja_check_cmd, @@ -382,9 +386,11 @@ if not vs: cc = 'clang' cxx = 'clang++' + fc = 'flang-new' else: cc = 'clang-cl.exe' cxx = 'clang-cl.exe' + fc = 'flang-new.exe' ############# STAGE 2 @@ -435,7 +441,7 @@ workdir=stage2_build, env=env)) - if test: + if checks: f.addStep(LitTestCommand(name='ninja check 2', command=ninja_check_cmd, haltOnFailure=not runTestSuite, @@ -496,6 +502,16 @@ '--cc', cc, '--cxx', cxx, '--use-lit', lit] + # Enable fortran if flang is checked out + if checkout_flang: + fortran_flags = [ + '--cmake-define=TEST_SUITE_FORTRAN:STRING=ON', + '--cmake-define=CMAKE_Fortran_FLAGS:STRING=' + + '-flang-experimental-exec -lpgmath', + util.Interpolate( + '--cmake-define=CMAKE_Fortran_COMPILER=' + + '%(prop:builddir)s/'+compiler_path+'/bin/'+fc)] + test_suite_cmd.extend(fortran_flags) # Append any option provided by the user test_suite_cmd.extend(testsuite_flags) @@ -521,7 +537,7 @@ workdir='test', env=env)) f.addStep(ShellCommand(name='recreate sandbox', - command=['virtualenv', 'sandbox'], + command=['virtualenv', '--python=python3', 'sandbox'], haltOnFailure=True, description='recreating sandbox', workdir='test', diff --git a/zorg/buildbot/builders/ClangLTOBuilder.py b/zorg/buildbot/builders/ClangLTOBuilder.py --- a/zorg/buildbot/builders/ClangLTOBuilder.py +++ b/zorg/buildbot/builders/ClangLTOBuilder.py @@ -65,9 +65,14 @@ ('-DCMAKE_INSTALL_PREFIX=', install_dir), ]) - CmakeCommand.applyRequiredOptions(cmake_args, [ - ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)), - ]) + if f.enable_projects: + CmakeCommand.applyRequiredOptions(cmake_args, [ + ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.enable_projects)), + ]) + if f.enable_runtimes: + CmakeCommand.applyRequiredOptions(cmake_args, [ + ('-DLLVM_ENABLE_RUNTIMES=', ";".join(f.enable_runtimes)), + ]) # Note: On this stage we do not care of warnings, as we build with # a system toolchain and cannot control the environment. @@ -181,9 +186,14 @@ "-DCMAKE_C_COMPILER=%(builddir)s/" + staged_install + "/bin/clang" )) - CmakeCommand.applyRequiredOptions(cmake_args, [ - ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)), - ]) + if f.enable_projects: + CmakeCommand.applyRequiredOptions(cmake_args, [ + ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.enable_projects)), + ]) + if f.enable_runtimes: + CmakeCommand.applyRequiredOptions(cmake_args, [ + ('-DLLVM_ENABLE_RUNTIMES=', ";".join(f.enable_runtimes)), + ]) # Create configuration files with cmake f.addStep(CmakeCommand(name="cmake-configure-stage%s" % stage_num, @@ -233,6 +243,7 @@ clean = False, jobs = None, extra_configure_args = None, + extra_configure_args_lto_stage = None, compare_last_2_stages = True, lto = None, # The string gets passed to -flto flag as is. Like -flto=thin. env = None, @@ -253,10 +264,16 @@ else: extra_configure_args = list(extra_configure_args) + if extra_configure_args_lto_stage is None: + extra_configure_args_lto_stage = [] + else: + extra_configure_args_lto_stage = list(extra_configure_args_lto_stage) + # Make sure CMAKE_INSTALL_PREFIX and -G are not specified # in the extra_configure_args. We set them internally as needed. # TODO: assert extra_configure_args. - install_prefix_specified = any(a.startswith('-DCMAKE_INSTALL_PREFIX=') for a in extra_configure_args) + install_prefix_specified = (any(a.startswith('-DCMAKE_INSTALL_PREFIX=') for a in extra_configure_args) or + any(a.startswith('-DCMAKE_INSTALL_PREFIX=') for a in extra_configure_args_lto_stage)) assert not install_prefix_specified, "Please do not explicitly specify the install prefix for multi-stage build." # Prepare environmental variables. Set here all env we want everywhere. @@ -321,7 +338,7 @@ s = f.staged_compiler_idx + 1 staged_install = f.stage_installdirs[f.staged_compiler_idx] for i in range(s, len(f.stage_objdirs[s:]) + s): - configure_args = extra_configure_args[:] + configure_args = extra_configure_args[:] + extra_configure_args_lto_stage[:] configure_args.append( WithProperties( diff --git a/zorg/buildbot/builders/DoxygenDocsBuilder.py b/zorg/buildbot/builders/DoxygenDocsBuilder.py new file mode 100644 --- /dev/null +++ b/zorg/buildbot/builders/DoxygenDocsBuilder.py @@ -0,0 +1,122 @@ +from collections import OrderedDict +from importlib import reload + +from buildbot.plugins import steps, util +from buildbot.steps.shell import ShellCommand + +from zorg.buildbot.builders import UnifiedTreeBuilder +from zorg.buildbot.commands import CmakeCommand + +from zorg.buildbot.process import factory +reload(factory) +reload(UnifiedTreeBuilder) +reload(CmakeCommand) + +llvm_docs = OrderedDict([ + # Project Build target Local path Remote path + ("llvm", ("doxygen", "docs/doxygen/html/", "llvm")), + ("clang", (None, "tools/clang/docs/doxygen/html/", "cfe")), + ("clang-tools-extra", (None, "tools/clang/tools/extra/docs/doxygen/html/", "cfe-extra")), + ("flang", (None, "tools/flang/docs/doxygen/html/", "flang")), + ("polly", (None, "tools/polly/docs/doxygen/html/", "polly")), + ("openmp", (None, "projects/openmp/docs/doxygen/html/", "openmp")), + ("lldb", ("lldb-cpp-doc", "tools/lldb/docs/cpp_reference/", "lldb/cpp_reference")), + # NOTE: 5/9/2020 lldb-python-doc fails to build. Disabled till be fixed. + #(None, ("lldb-python-doc", "tools/lldb/docs/python_reference/", "lldb")), +]) + +def getLLVMDocsBuildFactory( + clean = True, + depends_on_projects = None, + extra_configure_args = None, + timeout=10800, + env = None, + **kwargs): + + if depends_on_projects is None: + # All the projects from llvm_docs, and remove all duplicates. + _depends_on_projects=list(set( + [project for project in llvm_docs if project])) + else: + # Make a local copy of depends_on_projects, as we are going to modify + # that. + _depends_on_projects=depends_on_projects[:] + # Some projects are interdependent for the purpose of documentation. + # Enforce the dependencies. + # TODO: Check later the dependencies for doxygen docs and enforce them + # here if needed. + + # Make a local copy of the configure args, as we are going to modify that. + if extra_configure_args: + cmake_args = extra_configure_args[:] + else: + cmake_args = list() + + # Prepare environmental variables. Set here all env we want everywhere. + merged_env = { + 'TERM' : 'dumb' # Be cautious and disable color output from all tools. + } + if env is not None: + # Overwrite pre-set items with the given ones, so user can set anything. + merged_env.update(env) + + CmakeCommand.CmakeCommand.applyDefaultOptions(cmake_args, [ + ("-G", "Ninja"), + ("-DLLVM_ENABLE_DOXYGEN=", "ON"), + ("-DLLVM_BUILD_DOCS=", "ON"), + ("-DCLANG_TOOLS_EXTRA_INCLUDE_DOCS=", "ON"), + ("-DLLVM_ENABLE_ASSERTIONS=", "OFF"), + ("-DCMAKE_BUILD_TYPE=", "Release"), + ]) + + f = UnifiedTreeBuilder.getCmakeBuildFactory( + clean=clean, + depends_on_projects=_depends_on_projects, + enable_runtimes=[], # Docs don't support runtimes build yet. + extra_configure_args=cmake_args, + env=merged_env, + **kwargs) # Pass through all the extra arguments. + + # Build the documentation for all the projects. + for project in llvm_docs: + target = llvm_docs[project][0] + + # Build only those with specifies targets. + if target: + UnifiedTreeBuilder.addNinjaSteps( + f, + # Doxygen builds the final result for really + # long time without any output. + # We have to have a long timeout at this step. + timeout=timeout, + targets=[target], + checks=[], + env=merged_env, + **kwargs) + + # Publish just built documentation + for project in llvm_docs: + target, local_path, remote_path = llvm_docs[project] + + f.addStep( + ShellCommand( + name="Publish {}".format(project or target), + description=[ + "Publish", "just", "built", "documentation", "for", + "{}".format(project or target) + ], + command=[ + 'rsync', + '-vrl', + '--delete', '--force', '--delay-updates', '--delete-delay', + '--ignore-times', + '--checksum', + '-p', '--chmod=Du=rwx,Dg=rwx,Do=rx,Fu=rw,Fg=rw,Fo=r', + "{}".format(local_path), + "lists.llvm.org:web/doxygen/{}".format(remote_path), + ], + env=merged_env, + ) + ) + + return f diff --git a/zorg/buildbot/builders/FlangBuilder.py b/zorg/buildbot/builders/FlangBuilder.py --- a/zorg/buildbot/builders/FlangBuilder.py +++ b/zorg/buildbot/builders/FlangBuilder.py @@ -14,7 +14,7 @@ env = dict() f = getCmakeWithNinjaBuildFactory( - depends_on_projects=['llvm','mlir'], + depends_on_projects=['llvm','clang','mlir'], obj_dir="build_llvm", checks=[], clean=clean, @@ -42,6 +42,7 @@ # Add LLVM_DIR and MLIR_DIR to the CMake invocation. llvm_dir = "{}/lib/cmake/llvm".format(f.obj_dir) mlir_dir = "{}/lib/cmake/mlir".format(f.obj_dir) + clang_dir = "{}/lib/cmake/clang".format(f.obj_dir) CmakeCommand.applyRequiredOptions(flang_cmake_args, [ # We actually need the paths to be relative to the source directory, # otherwise find_package can't locate the config files. @@ -49,6 +50,8 @@ LLVMBuildFactory.pathRelativeTo(llvm_dir, flang_src_dir)), ('-DMLIR_DIR:PATH=', LLVMBuildFactory.pathRelativeTo(mlir_dir, flang_src_dir)), + ('-DCLANG_DIR:PATH=', + LLVMBuildFactory.pathRelativeTo(clang_dir, flang_src_dir)), ]) # We can't use addCmakeSteps as that would use the path in f.llvm_srcdir. diff --git a/zorg/buildbot/builders/FuchsiaBuilder.py b/zorg/buildbot/builders/FuchsiaBuilder.py --- a/zorg/buildbot/builders/FuchsiaBuilder.py +++ b/zorg/buildbot/builders/FuchsiaBuilder.py @@ -28,14 +28,16 @@ f = UnifiedTreeBuilder.getLLVMBuildFactoryAndSourcecodeSteps( depends_on_projects=[ - "llvm", + "bolt", "clang", "clang-tools-extra", "compiler-rt", "libcxx", "libcxxabi", "libunwind", - "lld" + "lld", + "llvm", + "polly", ], llvm_srcdir=src_dir, obj_dir=obj_dir, @@ -92,8 +94,6 @@ # Some options are required for this stage no matter what. CmakeCommand.applyRequiredOptions(cmake_options, [ ("-G", "Ninja"), - ("-DLLVM_ENABLE_PROJECTS=", "clang;clang-tools-extra;lld"), - ("-DLLVM_ENABLE_RUNTIMES=", "compiler-rt;libcxx;libcxxabi;libunwind"), ]) # Set proper defaults. diff --git a/zorg/buildbot/builders/HtmlDocsBuilder.py b/zorg/buildbot/builders/HtmlDocsBuilder.py new file mode 100644 --- /dev/null +++ b/zorg/buildbot/builders/HtmlDocsBuilder.py @@ -0,0 +1,105 @@ +from collections import OrderedDict +from importlib import reload + +from buildbot.plugins import steps, util +from buildbot.steps.shell import ShellCommand +from buildbot.steps.shell import WarningCountingShellCommand + +from zorg.buildbot.process import factory +reload(factory) + +llvm_docs = OrderedDict([ + # Project Build target Build path Local path Remote path + ("lnt", ("html", "docs", "_build/html/", "lnt")), +]) + +# We build with make for now. Change later if needed. +build_cmd = 'make' + +def getHtmlDocsBuildFactory( + depends_on_projects = None, + clean = False, + env = None, + **kwargs): + + if depends_on_projects is None: + # All the projects by default. + _depends_on_projects=llvm_docs.keys() + else: + # Make a local copy of depends_on_projects, as we are going to modify + # that. + _depends_on_projects=depends_on_projects[:] + + # Prepare environmental variables. Set here all env we want everywhere. + merged_env = { + 'TERM' : 'dumb' # Be cautious and disable color output from all tools. + } + if env is not None: + # Overwrite pre-set items with the given ones, so user can set anything. + merged_env.update(env) + + # HTML Sphinx documentation builds in tree, each in its own directory. + # For that, make sure the obj_dir is the same as llvm_srcdir. + src_dir = kwargs.pop('llvm_srcdir', '.') + f = factory.LLVMBuildFactory( + clean=clean, + depends_on_projects=_depends_on_projects, + llvm_srcdir=src_dir, + obj_dir=src_dir, + **kwargs) # Pass through all the extra arguments. + + # Build the documentation + for project in llvm_docs: + + # Checkout the source code and remove all the untracked files, so + # we would build a fresh new documentation. + f.addStep( + steps.Git( + name='Checkout the {} source code'.format(project), + repourl=f.repourl_prefix + "llvm-{}.git".format(project), + mode='full', + method='fresh', + progress=True, + workdir=util.WithProperties(project), + env=merged_env, + **kwargs)) + + target, build_path, local_path, remote_path = llvm_docs[project] + + build_dir = util.WithProperties( + "{}".format("/".join([ + project, + build_path]))) + f.addStep( + steps.WarningCountingShellCommand( + name="Build {} documentation".format(project), + command=[build_cmd, target], + haltOnFailure=True, + workdir=build_dir, + env=merged_env, + **kwargs)) + + # Publish just built documentation + f.addStep( + ShellCommand( + name="Publish {}".format(project), + description=[ + "Publish", "just", "built", "documentation", "for", + "{}".format(project) + ], + command=[ + 'rsync', + '-vrl', + '--delete', '--force', '--delay-updates', '--delete-delay', + '--ignore-times', + '--checksum', + '-p', '--chmod=Du=rwx,Dg=rwx,Do=rx,Fu=rw,Fg=rw,Fo=r', + "{}".format(local_path), + "lists.llvm.org:web/{}".format(remote_path), + ], + workdir=build_dir, + env=merged_env, + ) + ) + + return f diff --git a/zorg/buildbot/builders/LLDBBuilder.py b/zorg/buildbot/builders/LLDBBuilder.py --- a/zorg/buildbot/builders/LLDBBuilder.py +++ b/zorg/buildbot/builders/LLDBBuilder.py @@ -48,6 +48,7 @@ build_cmd=['ninja'] install_cmd = ['ninja','install'] test_cmd = ['ninja','check-lldb'] + lit_args = '-v' if jobs: build_cmd.append(WithProperties("-j%s" % jobs)) @@ -67,6 +68,7 @@ cmake_options = [ "-G", "Ninja", "-DCMAKE_BUILD_TYPE=" + config, + "-DLLVM_LIT_ARGS='%s'" % lit_args, "-DCMAKE_INSTALL_PREFIX=../install", "-DLLVM_ENABLE_PROJECTS=%s" % ";".join(f.depends_on_projects), ] diff --git a/zorg/buildbot/builders/LibcxxAndAbiBuilder.py b/zorg/buildbot/builders/LibcxxAndAbiBuilder.py deleted file mode 100644 --- a/zorg/buildbot/builders/LibcxxAndAbiBuilder.py +++ /dev/null @@ -1,132 +0,0 @@ -import buildbot.steps.shell -import buildbot.process.properties as properties - -from zorg.buildbot.commands.LitTestCommand import LitTestCommand -from zorg.buildbot.commands.CmakeCommand import CmakeCommand -from zorg.buildbot.process.factory import LLVMBuildFactory -from zorg.buildbot.builders import UnifiedTreeBuilder - -def getLibcxxAndAbiBuilder(f=None, env=None, - cmake_extra_opts=None, lit_extra_opts=None, - lit_extra_args=None, check_libcxx_abilist=False, - check_libcxx_benchmarks=None, - depends_on_projects=None, - use_cache=None, - **kwargs): - - if env is None: - env = {} - if cmake_extra_opts is None: - cmake_extra_opts = {} - if lit_extra_opts is None: - lit_extra_opts = {} - if lit_extra_args is None: - lit_extra_args = [] - - if depends_on_projects is None: - depends_on_projects = ['libcxx','libcxxabi','libunwind'] - - src_root = 'llvm' - build_path = 'build' - - if f is None: - f = UnifiedTreeBuilder.getLLVMBuildFactoryAndSourcecodeSteps( - depends_on_projects=depends_on_projects, - llvm_srcdir=src_root, - obj_dir=build_path, - **kwargs) # Pass through all the extra arguments. - - rel_src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, build_path) - - # Specify the max number of threads using properties so LIT doesn't use - # all the threads on the system. - litTestArgs = '-vv --show-unsupported --show-xfail --threads=%(jobs)s' - if lit_extra_args: - litTestArgs += ' ' + ' '.join(lit_extra_args) - - for key in lit_extra_opts: - litTestArgs += (' --param=' + key + '=' + lit_extra_opts[key]) - - cmake_opts = [properties.WithProperties('-DLLVM_LIT_ARGS='+litTestArgs)] - for key in cmake_extra_opts: - cmake_opts.append('-D' + key + '=' + cmake_extra_opts[key]) - - if use_cache: - libcxx_cache_dir = '%s/../libcxx/cmake/caches' % rel_src_dir - cache = '%s/%s' % (libcxx_cache_dir, use_cache) - cmake_opts.append('-C' + cache) - - # FIXME: The libc++ abilist's are generated in release mode with debug - # symbols Other configurations may contain additional non-inlined symbols. - if check_libcxx_abilist and not 'CMAKE_BUILD_TYPE' in cmake_extra_opts: - cmake_opts.append('-DCMAKE_BUILD_TYPE=RELWITHDEBINFO') - - # Force libc++ to use the in-tree libc++abi unless otherwise specified. - if 'LIBCXX_CXX_ABI' not in cmake_extra_opts: - cmake_opts.append('-DLIBCXX_CXX_ABI=libcxxabi') - - # Nuke/remake build directory and run CMake - f.addStep(buildbot.steps.shell.ShellCommand( - name='rm.builddir', command=['rm', '-rf', build_path], - workdir=".", - haltOnFailure=False)) - - CmakeCommand.applyRequiredOptions(cmake_opts, [ - ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)), - ]) - - f.addStep(buildbot.steps.shell.ShellCommand( - name='cmake', command=['cmake', rel_src_dir] + cmake_opts, - haltOnFailure=True, workdir=build_path, env=env)) - - # Build libcxxabi - jobs_flag = properties.WithProperties('-j%(jobs)s') - f.addStep(buildbot.steps.shell.ShellCommand( - name='build.libcxxabi', command=['make', jobs_flag, 'cxxabi'], - haltOnFailure=True, workdir=build_path)) - - # Build libcxx - f.addStep(buildbot.steps.shell.ShellCommand( - name='build.libcxx', command=['make', jobs_flag, 'cxx'], - haltOnFailure=True, workdir=build_path)) - - # Test libc++abi - f.addStep(LitTestCommand( - name = 'test.libcxxabi', - command = ['make', jobs_flag, 'check-cxxabi'], - description = ['testing', 'libcxxabi'], - descriptionDone = ['test', 'libcxxabi'], - workdir = build_path)) - - # Test libc++ - f.addStep(LitTestCommand( - name = 'test.libcxx', - command = ['make', jobs_flag, 'check-cxx'], - description = ['testing', 'libcxx'], - descriptionDone = ['test', 'libcxx'], - workdir = build_path)) - - if check_libcxx_abilist: - f.addStep(buildbot.steps.shell.ShellCommand( - name = 'test.libcxx.abilist', - command = ['make', 'check-cxx-abilist'], - description = ['testing', 'libcxx', 'abi'], - descriptionDone = ['test', 'libcxx', 'abi'], - workdir = build_path)) - - if check_libcxx_benchmarks: - # Build the libc++ benchmarks - f.addStep(buildbot.steps.shell.ShellCommand( - name='build.libcxx.benchmarks', - command=['make', jobs_flag, 'cxx-benchmarks'], - haltOnFailure=True, workdir=build_path)) - - # Run the benchmarks - f.addStep(LitTestCommand( - name = 'test.libcxx.benchmarks', - command = ['make', jobs_flag, 'check-cxx-benchmarks'], - description = ['testing', 'libcxx', 'benchmarks'], - descriptionDone = ['test', 'libcxx', 'benchmarks'], - workdir = build_path)) - - return f diff --git a/zorg/buildbot/builders/OpenMPBuilder.py b/zorg/buildbot/builders/OpenMPBuilder.py --- a/zorg/buildbot/builders/OpenMPBuilder.py +++ b/zorg/buildbot/builders/OpenMPBuilder.py @@ -1,5 +1,6 @@ from buildbot.steps.shell import ShellCommand from buildbot.process.properties import WithProperties +from buildbot.plugins import steps from zorg.buildbot.commands.LitTestCommand import LitTestCommand from zorg.buildbot.commands.CmakeCommand import CmakeCommand @@ -14,6 +15,12 @@ ompt = False, # Whether to enable the OpenMP Tools Interface. test = True, # Test the built libraries. depends_on_projects = None, + enable_runtimes = None, + extraCmakeArgs = [], + install = False, + testsuite = False, + testsuite_sollvevv = False, + extraTestsuiteCmakeArgs = [], **kwargs): # Prepare environmental variables. Set here all env we want everywhere. @@ -26,8 +33,17 @@ llvm_srcdir = 'llvm.src' llvm_builddir = 'llvm.build' - - cleanBuildRequested = lambda step: clean or step.build.getProperty("clean", default=step.build.getProperty("clean_obj")) + llvm_instdir = 'llvm.inst' + testsuite_srcdir = "test-suite.src" + testsuite_builddir = "test-suite.build" + sollvevv_srcdir = "sollvevv.src" + + # If true, clean everything, including source dirs + def cleanBuildRequested(step): + return step.build.getProperty("clean") + # If true, clean build products; implied if cleanBuildRequested is true + def cleanObjRequested(step): + return cleanBuildRequested(step) or clean or step.build.getProperty("clean_obj") if depends_on_projects is None: # Monorepo configuration requires llvm and clang to get cmake work. @@ -35,21 +51,17 @@ f = UnifiedTreeBuilder.getLLVMBuildFactoryAndSourcecodeSteps( depends_on_projects=depends_on_projects, + enable_runtimes=enable_runtimes, llvm_srcdir=llvm_srcdir, obj_dir=llvm_builddir, cleanBuildRequested=cleanBuildRequested, env=merged_env, **kwargs) # Pass through all the extra arguments. - f.addStep( - ShellCommand( - name = 'clean', - command = ['rm', '-rf', f.obj_dir], - warnOnFailure = True, - description = ['clean'], - doStepIf = cleanBuildRequested, - workdir = '.', - env = merged_env)) + f.addStep(steps.RemoveDirectory(name='clean', + dir=f.obj_dir, + warnOnFailure=True, + doStepIf=cleanObjRequested)) # Configure LLVM and OpenMP (and Clang, if requested). cmake_args = ['-DCMAKE_BUILD_TYPE=Release', '-DLLVM_ENABLE_ASSERTIONS=ON'] @@ -58,10 +70,19 @@ if test: lit_args = '-vv --show-unsupported --show-xfail -j %s' % jobs cmake_args += [WithProperties('-DLLVM_LIT_ARGS=%s' % lit_args)] + if install: + cmake_args += [WithProperties('-DCMAKE_INSTALL_PREFIX=%(builddir)s/' + llvm_instdir)] + cmake_args += extraCmakeArgs + + if f.enable_projects: + CmakeCommand.applyDefaultOptions(cmake_args, [ + ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.enable_projects)), + ]) - CmakeCommand.applyRequiredOptions(cmake_args, [ - ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)), - ]) + if f.enable_runtimes: + CmakeCommand.applyDefaultOptions(cmake_args, [ + ('-DLLVM_ENABLE_RUNTIMES=', ";".join(f.enable_runtimes)), + ]) # Add llvm-lit and clang (if built) to PATH merged_env.update({ @@ -103,6 +124,121 @@ description = 'test openmp', workdir = f.obj_dir, env = merged_env, - haltOnFailure=True)) + haltOnFailure=False, + flunkOnFailure=True)) + + clangexe = "%(builddir)s/" + llvm_builddir + "/bin/clang" + clangxxexe = "%(builddir)s/" + llvm_builddir + "/bin/clang++" + litexe = "%(builddir)s/" + llvm_builddir + "/bin/llvm-lit" + libdir = "%(builddir)s/" + llvm_builddir + "/lib" + if install: + f.addStep(steps.RemoveDirectory(name="LLVM: Clean Install Dir", + dir=llvm_instdir, + haltOnFailure=False)) + + f.addStep(NinjaCommand(name="LLVM: Install", + description="installing", + descriptionDone="install", + descriptionSuffix="LLVM", + workdir=f.obj_dir, + targets=['install'], + env=merged_env, + haltOnFailure=True)) + # If installing, use the installed version of clang. + clangexe = "%(builddir)s/" + llvm_instdir + "/bin/clang" + clangxxexe = "%(builddir)s/" + llvm_instdir + "/bin/clang++" + libdir = "%(builddir)s/" + llvm_instdir + "/lib" + + + if testsuite: + f.addStep(steps.RemoveDirectory(name="Test-Suite: Clean Source", + dir=testsuite_srcdir, + haltOnFailure=False, + warnOnFailure=True, + doStepIf=cleanBuildRequested)) + + f.addGetSourcecodeForProject(name="Test-Suite: Checkout", + description="fetching", + descriptionDone="fetch", + descriptionSuffix="Test-Suite", + project='test-suite', + src_dir=testsuite_srcdir, + alwaysUseLatest=True) + + if testsuite_sollvevv: + f.addStep(steps.RemoveDirectory(name="SOLLVE V&V: Clean Source", + dir=sollvevv_srcdir, + haltOnFailure=False, + warnOnFailure=True, + doStepIf=cleanBuildRequested)) + + f.addStep(steps.Git(name="SOLLVE V&V: Checkout", + description="fetching", + descriptionDone="fetch", + descriptionSuffix="SOLLVE V&V", + repourl='https://github.com/SOLLVE/sollve_vv.git', + workdir=sollvevv_srcdir, + alwaysUseLatest=True)) + + f.addStep(steps.RemoveDirectory(name="Test-Suite: Clean Build", + dir=testsuite_builddir, + haltOnFailure=False, + warnOnFailure=True)) + + testsuite_options = [ + "-DCMAKE_BUILD_TYPE=Release", + "-DTEST_SUITE_LIT_FLAGS=-vv;-s;-j6;-o;report.json", + "-DTEST_SUITE_EXTRA_C_FLAGS=-gline-tables-only", + "-DTEST_SUITE_EXTRA_CXX_FLAGS=-gline-tables-only", + WithProperties("-DCMAKE_C_COMPILER=" + clangexe), + WithProperties("-DCMAKE_CXX_COMPILER=" + clangxxexe), + WithProperties("-DTEST_SUITE_LIT=" + litexe), + ] + if testsuite_sollvevv: + testsuite_options += [ + "-DTEST_SUITE_SUBDIRS=External/sollve_vv", + "-DTEST_SUITE_SOLLVEVV_ROOT=../" + sollvevv_srcdir, + ] + testsuite_options += extraTestsuiteCmakeArgs + f.addStep(CmakeCommand(name="Test-Suite: Configure", + description="configuring", + descriptionDone="configure", + descriptionSuffix="Test-Suite", + generator='Ninja', + path='../' + testsuite_srcdir, + workdir=testsuite_builddir, + options=testsuite_options, + haltOnFailure=True, + logfiles={ + 'CMakeCache.txt' : 'CMakeCache.txt', + })) + + f.addStep(NinjaCommand(name="Test-Suite: Compile", + description="compiling", + descriptionDone="compile", + descriptionSuffix="Test-Suite", + options=['-k0'], # Continue building; programs that don't compile will fail with NOEXE. + haltOnFailure=False, + flunkOnFailure=False, # SOLLVE V&V contains tests that clang have not been implemented yet. + warnOnFailure=True, + workdir=testsuite_builddir)) + + merged_env.update({ + 'LD_LIBRARY_PATH': WithProperties(libdir + ':${LD_LIBRARY_PATH}') + }) + f.addStep(LitTestCommand(name="Test-Suite: Run", + description="running", + descriptionDone="run", + descriptionSuffix="Test-Suite", + command=[WithProperties(litexe), '-vv', '-s', '-j6', '-o','report.json', '.'], + haltOnFailure=False, + flunkOnFailure=False, # SOLLVE V&V contains tests that clang have not been implemented yet. + warnOnFailure=True, + workdir=testsuite_builddir, + env=merged_env, + logfiles={ + 'test.log' : 'test.log', + 'report.json': 'report.json' + })) return f diff --git a/zorg/buildbot/builders/PollyBuilder.py b/zorg/buildbot/builders/PollyBuilder.py --- a/zorg/buildbot/builders/PollyBuilder.py +++ b/zorg/buildbot/builders/PollyBuilder.py @@ -44,12 +44,13 @@ merged_env.update(env) # Overwrite pre-set items with the given ones, so user can set anything. depends_on_projects = ['llvm','clang','polly'] - if testsuite: - # XRay tests in test-suite require compiler-rt - depends_on_projects += ['compiler-rt'] - cleanBuildRequestedByProperty = lambda step: step.build.getProperty("clean", False) - cleanBuildRequested = lambda step: clean or step.build.getProperty("clean", default=step.build.getProperty("clean_obj")) + # If true, clean everything, including source dirs + def cleanBuildRequested(step): + return step.build.getProperty("clean") + # If true, clean build products; implied if cleanBuildRequested is true + def cleanObjRequested(step): + return cleanBuildRequested(step) or clean or step.build.getProperty("clean_obj") f = LLVMBuildFactory( depends_on_projects=depends_on_projects, @@ -62,7 +63,7 @@ f.addStep(steps.RemoveDirectory(name='clean-src-dir', dir=f.monorepo_dir, warnOnFailure=True, - doStepIf=cleanBuildRequestedByProperty)) + doStepIf=cleanBuildRequested)) # Get the source code. f.addGetSourcecodeSteps(**kwargs) @@ -71,13 +72,12 @@ f.addStep(steps.RemoveDirectory(name='clean-build-dir', dir=llvm_objdir, warnOnFailure=True, - doStepIf=cleanBuildRequested)) + doStepIf=cleanObjRequested)) # Create configuration files with cmake cmakeCommand = ["cmake", "../%s/llvm" % llvm_srcdir, "-DCMAKE_COLOR_MAKEFILE=OFF", "-DPOLLY_TEST_DISABLE_BAR=ON", - "-DPOLLY_ENABLE_GPGPU_CODEGEN=ON", "-DCMAKE_BUILD_TYPE=Release", "-DLLVM_POLLY_LINK_INTO_TOOLS=ON", "-DLLVM_ENABLE_PROJECTS=%s" % ";".join(f.depends_on_projects), @@ -87,7 +87,10 @@ haltOnFailure=False, description=["cmake configure"], workdir=llvm_objdir, - env=merged_env)) + env=merged_env, + logfiles={ + 'CMakeCache.txt' : llvm_objdir + '/CMakeCache.txt', + })) # Build f.addStep(WarningCountingShellCommand(name="build", @@ -107,7 +110,7 @@ f.addStep(steps.RemoveDirectory(name='clean-install-dir', dir=llvm_instdir, haltOnFailure=False, - doStepIf=cleanBuildRequested)) + doStepIf=cleanObjRequested)) f.addStep(ShellCommand(name="install", command=install_cmd, @@ -142,7 +145,7 @@ dir=testsuite_srcdir, haltOnFailure=False, warnOnFailure=True, - doStepIf=cleanBuildRequestedByProperty)) + doStepIf=cleanBuildRequested)) f.addGetSourcecodeForProject( project='test-suite', @@ -154,14 +157,13 @@ haltOnFailure=False, warnOnFailure=True)) - # -Wno-unused-command-line-argument is needed because linking will not uses the "-mllvm -polly" argument. + # -Wno-unused-command-line-argument is needed because linking will not use the "-mllvm -polly" argument. f.addStep(ShellCommand(name='test-suite_cmake-configure', description=["Test-Suite: cmake"], command=["cmake", '-B', testsuite_builddir, '-S', testsuite_srcdir, "-DCMAKE_BUILD_TYPE=Release", - "-DTEST_SUITE_COLLECT_STATS=ON", "-DTEST_SUITE_EXTRA_C_FLAGS=-Wno-unused-command-line-argument -mllvm -polly", - "-DTEST_SUITE_EXTRA_CXX_FLAGS=-mllvm -polly", + "-DTEST_SUITE_EXTRA_CXX_FLAGS=-Wno-unused-command-line-argument -mllvm -polly", "-DTEST_SUITE_LIT_FLAGS=-vv;-o;report.json", WithProperties("-DCMAKE_C_COMPILER=" + clangexe), WithProperties("-DCMAKE_CXX_COMPILER=" + clangxxexe), @@ -170,7 +172,10 @@ ] + extraTestsuiteCmakeArgs, haltOnFailure=True, workdir='.', - env=merged_env)) + env=merged_env, + logfiles={ + 'CMakeCache.txt' : testsuite_builddir + '/CMakeCache.txt', + })) f.addStep(WarningCountingShellCommand(name='test-suite_build', description=["Test-Suite: build"], @@ -183,7 +188,7 @@ f.addStep(LitTestCommand(name='test-suite_run', description=['Test-Suite: run'], - command=[WithProperties(litexe), '-vv', '-o', 'report.json', '.'], + command=[WithProperties(litexe), '-vv', '-s', '-o', 'report.json', '.'], haltOnFailure=True, workdir=testsuite_builddir, logfiles={ diff --git a/zorg/buildbot/builders/SanitizerBuilder.py b/zorg/buildbot/builders/SanitizerBuilder.py --- a/zorg/buildbot/builders/SanitizerBuilder.py +++ b/zorg/buildbot/builders/SanitizerBuilder.py @@ -5,7 +5,8 @@ def getSanitizerBuildFactory( clean=False, - depends_on_projects=None, + extra_depends_on_projects=[], + extra_configure_args=None, env=None, timeout=1200): @@ -20,15 +21,20 @@ if env is not None: merged_env.update(env) - if depends_on_projects is None: - depends_on_projects = [ - "llvm", - "clang", - "compiler-rt", - "libcxx", - "libcxxabi", - "libunwind", - "lld"] + depends_on_projects = [ + "llvm", + "clang", + "compiler-rt", + "libcxx", + "libcxxabi", + "libunwind", + "lld", + ] + extra_depends_on_projects + + if extra_configure_args: + extra_configure_args = ['--'] + list(extra_configure_args) + else: + extra_configure_args = [] # Explicitly use '/' as separator, because it works on *nix and Windows. sanitizer_script_dir = "sanitizer_buildbot/sanitizers" @@ -47,10 +53,11 @@ alwaysUseLatest=True) # Run annotated command for sanitizer. + command = ['python', sanitizer_script] + extra_configure_args f.addStep(AnnotatedCommand(name="annotate", description="annotate", timeout=timeout, haltOnFailure=True, - command="python " + sanitizer_script, + command=command, env=merged_env)) return f diff --git a/zorg/buildbot/builders/SphinxDocsBuilder.py b/zorg/buildbot/builders/SphinxDocsBuilder.py --- a/zorg/buildbot/builders/SphinxDocsBuilder.py +++ b/zorg/buildbot/builders/SphinxDocsBuilder.py @@ -11,6 +11,7 @@ ("llvm", ("docs-llvm-html", "docs/html/", "llvm")), ("clang", ("docs-clang-html", "tools/clang/docs/html/", "cfe")), ("clang-tools-extra", ("docs-clang-tools-html", "tools/clang/tools/extra/docs/html/", "clang-tools-extra")), + ("libc", ("docs-libc-html", "projects/libc/docs/html/", "libc")), ("libcxx", ("docs-libcxx-html", "projects/libcxx/docs/html/", "libcxx")), ("libunwind", ("docs-libunwind-html", "projects/libunwind/docs/html/", "libunwind")), ("lld", ("docs-lld-html", "tools/lld/docs/html/", "lld")), @@ -30,6 +31,7 @@ libcxx_html = False, # Build Libc++ HTML documentation libunwind_html = False, # Build libunwind HTML documentation lldb_html = False, # Build LLDB HTML documentation + polly_html = False, # Build Polly HTML documentation extra_configure_args = None, **kwargs): @@ -66,9 +68,12 @@ depends_on_projects.append('libcxxabi') if libunwind_html: depends_on_projects.append('libunwind') + if polly_html: + depends_on_projects.append('polly') f = UnifiedTreeBuilder.getCmakeBuildFactory( depends_on_projects=depends_on_projects, + enable_runtimes=[], # Docs don't support runtimes build yet. llvm_srcdir=llvm_srcdir, obj_dir=llvm_objdir, extra_configure_args=cmake_args, @@ -138,6 +143,14 @@ targets=['docs-libunwind-html'] )) + if polly_html: + f.addStep(NinjaCommand(name="docs-polly-html", + haltOnFailure=True, + description=["Build Polly Sphinx HTML documentation"], + workdir=llvm_objdir, + targets=['docs-polly-html'] + )) + return f @@ -154,6 +167,7 @@ "llvm", "clang", "clang-tools-extra", + "libc", "libcxx", "libcxxabi", "libunwind", @@ -213,6 +227,7 @@ f = UnifiedTreeBuilder.getCmakeBuildFactory( clean=clean, depends_on_projects=_depends_on_projects, + enable_runtimes=[], # Docs don't support runtimes build yet. extra_configure_args=cmake_args, env=merged_env, **kwargs) # Pass through all the extra arguments. diff --git a/zorg/buildbot/builders/TestSuiteBuilder.py b/zorg/buildbot/builders/TestSuiteBuilder.py new file mode 100644 --- /dev/null +++ b/zorg/buildbot/builders/TestSuiteBuilder.py @@ -0,0 +1,97 @@ +from zorg.buildbot.builders.UnifiedTreeBuilder import getCmakeWithNinjaBuildFactory + +from buildbot.plugins import util + +from zorg.buildbot.commands.CmakeCommand import CmakeCommand +from zorg.buildbot.commands.NinjaCommand import NinjaCommand +from zorg.buildbot.commands.LitTestCommand import LitTestCommand + +# This builder is uses UnifiedTreeBuilders and adds running +# llvm-test-suite with cmake and ninja step. + +def addTestSuiteStep( + f, + compiler_dir = None, + env = None, + lit_args = [], + **kwargs): + + cc = util.Interpolate('-DCMAKE_C_COMPILER=' + '%(prop:builddir)s/'+compiler_dir+'/bin/clang') + cxx = util.Interpolate('-DCMAKE_CXX_COMPILER=' + '%(prop:builddir)s/'+compiler_dir+'/bin/clang++') + lit = util.Interpolate('%(prop:builddir)s/' + compiler_dir + '/bin/llvm-lit') + test_suite_src_dir = util.Interpolate('%(prop:builddir)s/' + 'test/test-suite') + test_suite_workdir='test/sandbox/test-suite' + cmake_lit_arg = util.Interpolate('-DTEST_SUITE_LIT:FILEPATH=' + '%(prop:builddir)s/' + compiler_dir + '/bin/llvm-lit') + # used for cmake building test-suite step + options = [cc, cxx, cmake_lit_arg] + + f.addGetSourcecodeForProject( + project='test-suite', + src_dir=test_suite_src_dir, + alwaysUseLatest=True) + + f.addStep(CmakeCommand(name='cmake Test Suite', + haltOnFailure=True, + description='Running cmake on Test Suite dir', + workdir=test_suite_workdir, + options=options, + path=test_suite_src_dir, + generator='Ninja')) + + f.addStep(NinjaCommand(name='ninja Test Suite', + description='Running Ninja on Test Suite dir', + haltOnFailure=True, + workdir=test_suite_workdir)) + + f.addStep(LitTestCommand(name='Run Test Suite with lit', + haltOnFailure=True, + description='Running test suite tests', + workdir=test_suite_workdir, + command=[lit] + lit_args + ['.'], + env=env, + **kwargs)) + + return f + +def getTestSuiteBuildFactory( + depends_on_projects = None, + enable_runtimes = None, + targets = None, + llvm_srcdir = None, + obj_dir = None, + checks = None, + install_dir = None, + clean = False, + extra_configure_args = None, + env = None, + **kwargs): + + # handle the -DCMAKE args for lit + lit_args = list() + if any("DLLVM_LIT_ARGS" in arg for arg in extra_configure_args): + arg = [arg for arg in extra_configure_args + if "DLLVM_LIT_ARGS" in arg][0] + lit_args = arg.split("=")[1] + lit_args = lit_args.split(" ") + + f = getCmakeWithNinjaBuildFactory( + depends_on_projects = depends_on_projects, + enable_runtimes = enable_runtimes, + targets = targets, + llvm_srcdir = llvm_srcdir, + obj_dir = obj_dir, + checks = checks, + install_dir = install_dir, + clean = clean, + extra_configure_args = extra_configure_args, + env = env, + **kwargs) + + + addTestSuiteStep(f, + compiler_dir=f.obj_dir, + env=env, + lit_args=lit_args, + **kwargs) + + return f diff --git a/zorg/buildbot/builders/UnifiedTreeBuilder.py b/zorg/buildbot/builders/UnifiedTreeBuilder.py --- a/zorg/buildbot/builders/UnifiedTreeBuilder.py +++ b/zorg/buildbot/builders/UnifiedTreeBuilder.py @@ -13,6 +13,7 @@ def getLLVMBuildFactoryAndPrepareForSourcecodeSteps( depends_on_projects = None, + enable_runtimes = None, llvm_srcdir = None, obj_dir = None, install_dir = None, @@ -29,6 +30,7 @@ f = LLVMBuildFactory( depends_on_projects=depends_on_projects, + enable_runtimes=enable_runtimes, llvm_srcdir=llvm_srcdir, obj_dir=obj_dir, install_dir=install_dir, @@ -49,7 +51,9 @@ def getLLVMBuildFactoryAndSourcecodeSteps( depends_on_projects = None, + enable_runtimes = None, llvm_srcdir = None, + src_to_build_dir = None, obj_dir = None, install_dir = None, cleanBuildRequested = None, @@ -57,7 +61,9 @@ f = getLLVMBuildFactoryAndPrepareForSourcecodeSteps( depends_on_projects=depends_on_projects, + enable_runtimes=enable_runtimes, llvm_srcdir=llvm_srcdir, + src_to_build_dir=src_to_build_dir, obj_dir=obj_dir, install_dir=install_dir, cleanBuildRequested=cleanBuildRequested, @@ -99,9 +105,15 @@ doStepIf=cleanBuildRequested, )) - CmakeCommand.applyDefaultOptions(cmake_args, [ - ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)), - ]) + if f.enable_projects: + CmakeCommand.applyDefaultOptions(cmake_args, [ + ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.enable_projects)), + ]) + + if f.enable_runtimes: + CmakeCommand.applyDefaultOptions(cmake_args, [ + ('-DLLVM_ENABLE_RUNTIMES=', ";".join(f.enable_runtimes)), + ]) if install_dir: install_dir_rel = LLVMBuildFactory.pathRelativeTo( @@ -230,6 +242,7 @@ def getCmakeBuildFactory( depends_on_projects = None, + enable_runtimes = None, llvm_srcdir = None, obj_dir = None, install_dir = None, @@ -240,6 +253,7 @@ f = getLLVMBuildFactoryAndSourcecodeSteps( depends_on_projects=depends_on_projects, + enable_runtimes=enable_runtimes, llvm_srcdir=llvm_srcdir, obj_dir=obj_dir, install_dir=install_dir, @@ -260,6 +274,7 @@ def getCmakeWithNinjaBuildFactory( depends_on_projects = None, + enable_runtimes = None, targets = None, llvm_srcdir = None, obj_dir = None, @@ -294,6 +309,7 @@ f = getCmakeBuildFactory( depends_on_projects=depends_on_projects, + enable_runtimes=enable_runtimes, llvm_srcdir=llvm_srcdir, obj_dir=obj_dir, install_dir=install_dir, @@ -315,6 +331,7 @@ def getCmakeWithNinjaWithMSVCBuildFactory( depends_on_projects = None, + enable_runtimes = None, targets = None, llvm_srcdir = None, obj_dir = None, @@ -342,6 +359,7 @@ f = getLLVMBuildFactoryAndSourcecodeSteps( depends_on_projects=depends_on_projects, + enable_runtimes=enable_runtimes, llvm_srcdir=llvm_srcdir, obj_dir=obj_dir, install_dir=install_dir, @@ -377,7 +395,9 @@ def getCmakeWithNinjaMultistageBuildFactory( depends_on_projects = None, + enable_runtimes = None, llvm_srcdir = None, + src_to_build_dir = None, obj_dir = None, checks = None, install_dir = None, @@ -426,7 +446,9 @@ f = getLLVMBuildFactoryAndPrepareForSourcecodeSteps( depends_on_projects=depends_on_projects, + enable_runtimes=enable_runtimes, llvm_srcdir=llvm_srcdir, + src_to_build_dir=src_to_build_dir, obj_dir=obj_dir, install_dir=install_dir, env=merged_env, diff --git a/zorg/buildbot/builders/XToolchainBuilder.py b/zorg/buildbot/builders/XToolchainBuilder.py --- a/zorg/buildbot/builders/XToolchainBuilder.py +++ b/zorg/buildbot/builders/XToolchainBuilder.py @@ -120,11 +120,6 @@ extract_fn=builders_util.extractVSEnvironment)) merged_env = Property('vs_env') - # Since this is a build of a cross toolchain, we build only the host-side - # tools first by the host system compiler. Libraries will be cross-compiled. - cmake_args.append(InterpolateToPosixPath( - '-DLLVM_AR=%(builddir)s/' + f.obj_dir + '/bin/llvm-ar.exe')), - CmakeCommand.applyDefaultOptions(cmake_args, [ ('-G', 'Ninja'), ('-DLLVM_ENABLE_PROJECTS=', 'llvm;clang;clang-tools-extra;lld'), diff --git a/zorg/buildbot/builders/annotated/annotated_builder.py b/zorg/buildbot/builders/annotated/annotated_builder.py --- a/zorg/buildbot/builders/annotated/annotated_builder.py +++ b/zorg/buildbot/builders/annotated/annotated_builder.py @@ -13,6 +13,7 @@ from os.path import join as pjoin VSWHERE_PATH = "C:/Program Files (x86)/Microsoft Visual Studio/Installer/vswhere.exe" +BUILDTOOLS_VSDEVCMD = "C:/BuildTools/Common7/Tools/VsDevCmd.bat" def get_argument_parser(*args, **kwargs): ap = argparse.ArgumentParser(*args, **kwargs) @@ -211,10 +212,13 @@ cxx_compiler, linker, previous_stage_bin): + archiver = 'llvm-ar' + if os.name == 'nt': + archiver = 'llvm-lib' return ( cmake_args + [ '-DCMAKE_AR=%s' % ( - util.cmake_pjoin(previous_stage_bin, 'llvm-ar'),), + util.cmake_pjoin(previous_stage_bin, archiver),), '-DCMAKE_RANLIB=%s' % ( util.cmake_pjoin(previous_stage_bin, 'llvm-ranlib'),), ] + self.cmake_compiler_flags( @@ -323,9 +327,17 @@ def get_vcvars(vs_tools, arch): - """Get the VC tools environment using vswhere.exe from VS 2017 + """Get the VC tools environment using vswhere.exe or buildtools docker + + This is intended to work either when VS is in its standard installation + location, or when the docker instructions have been followed, and we can + find Visual C++ in C:/BuildTools. + + Visual Studio provides a docker image with instructions here: + https://docs.microsoft.com/en-us/visualstudio/install/build-tools-container?view=vs-2019 - This code is following the guidelines from strategy 1 in this blog post: + This vswhere code is following the guidelines from strategy 1 in this blog + post: https://blogs.msdn.microsoft.com/vcblog/2017/03/06/finding-the-visual-c-compiler-tools-in-visual-studio-2017/ It doesn't work when VS is not installed at the default location. @@ -342,13 +354,16 @@ # Use vswhere.exe if it exists. if os.path.exists(VSWHERE_PATH): cmd = [VSWHERE_PATH, "-latest", "-property", "installationPath"] - vs_path = subprocess.check_output(cmd).strip() + vs_path = subprocess.check_output(cmd).decode(sys.stdout.encoding) + vs_path = vs_path.strip() util.report("Running vswhere to find VS: " + repr(cmd)) util.report("vswhere output: " + vs_path) if not os.path.isdir(vs_path): raise ValueError("VS install path does not exist: " + vs_path) vcvars_path = pjoin(vs_path, 'VC', 'Auxiliary', 'Build', 'vcvarsall.bat') + elif os.path.exists(BUILDTOOLS_VSDEVCMD): + vcvars_path = BUILDTOOLS_VSDEVCMD elif vs_tools is None: vs_tools = os.path.expandvars('%VS140COMNTOOLS%') vcvars_path = pjoin(vs_tools, '..', '..', 'VC', 'vcvarsall.bat') @@ -357,7 +372,8 @@ # Windows /dev/null. cmd = util.shquote_cmd([vcvars_path, arch]) + ' > NUL && set' util.report("Running vcvars: " + cmd) - output = subprocess.check_output(cmd, shell=True) + output = \ + subprocess.check_output(cmd, shell=True).decode(sys.stdout.encoding) new_env = {} for line in output.splitlines(): var, val = line.split('=', 1) diff --git a/zorg/buildbot/builders/annotated/bolt-nfc.py b/zorg/buildbot/builders/annotated/bolt-nfc.py new file mode 100644 --- /dev/null +++ b/zorg/buildbot/builders/annotated/bolt-nfc.py @@ -0,0 +1,77 @@ +#!/usr/bin/python3 + +import argparse +import os +import subprocess +import sys +import traceback +import util +from contextlib import contextmanager +from enum import Enum + +class FailAction(Enum): + HALT = 1 + WARN = 2 + +def main(): + source_dir = os.path.join('..', 'llvm-project') + + with step('cmake'): + # On most systems the default generator is make and the default + # compilers are gcc and g++. We make it explicit here that we want + # clang and ninja which reduces one step of setting environment + # variables when setting up workers. + cmake_args = ['-GNinja', + '-DCMAKE_C_COMPILER=clang', + '-DCMAKE_CXX_COMPILER=clang++'] + + cmake_args.append('-DCMAKE_BUILD_TYPE=Release') + cmake_args.append('-DLLVM_ENABLE_ASSERTIONS=ON') + cmake_args.append('-DLLVM_ENABLE_PROJECTS=bolt;clang;lld') + + run_command(['cmake', os.path.join(source_dir, 'llvm')] + cmake_args) + + with step('build bolt'): + run_command(['ninja', 'bolt']) + + with step('check-bolt'): + run_command(['ninja', 'check-bolt']) + + with step('nfc-check-setup'): + run_command([os.path.join(source_dir, 'bolt', 'utils', + 'nfc-check-setup.py')]) + + with step('nfc-check-bolt'): + run_command([os.path.join('bin', 'llvm-lit'), '-sv', '-j2', + 'tools/bolt/test'], warn_on_fail=True) + + +@contextmanager +def step(step_name, warn_on_fail=False): + util.report('@@@BUILD_STEP {}@@@'.format(step_name)) + if warn_on_fail: + util.report('@@@STEP_WARNINGS@@@') + else: + util.report('@@@HALT_ON_FAILURE@@@') + try: + yield + except Exception as e: + if isinstance(e, subprocess.CalledProcessError): + util.report( + '{} exited with return code {}.'.format(e.cmd, e.returncode) + ) + util.report('The build step threw an exception...') + traceback.print_exc() + + util.report('@@@STEP_FAILURE@@@') + finally: + sys.stdout.flush() + + +def run_command(cmd, directory='.'): + util.report_run_cmd(cmd, cwd=directory) + + +if __name__ == '__main__': + sys.path.append(os.path.dirname(__file__)) + sys.exit(main()) diff --git a/zorg/buildbot/builders/annotated/buildbot-helper.sh b/zorg/buildbot/builders/annotated/buildbot-helper.sh new file mode 100755 --- /dev/null +++ b/zorg/buildbot/builders/annotated/buildbot-helper.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# Buildbot Helper functions + +# Stop if we've encountered an error. +halt_on_failure() { + echo "@@@HALT_ON_FAILURE@@@" +} + +build_step() { + echo "@@@BUILD_STEP ${@}@@@" +} diff --git a/zorg/buildbot/builders/annotated/hip-build.sh b/zorg/buildbot/builders/annotated/hip-build.sh new file mode 100755 --- /dev/null +++ b/zorg/buildbot/builders/annotated/hip-build.sh @@ -0,0 +1,114 @@ +#!/bin/bash + +# Enable Error tracing +set -o errtrace + +# Print trace for all commands ran before execution +set -x + +# Include the Buildbot helper functions +HERE="$(dirname $0)" +. ${HERE}/buildbot-helper.sh + +# Ensure all commands pass, and not dereferencing unset variables. +set -eu +halt_on_failure + +BUILDBOT_ROOT=/buildbot +LLVM_ROOT="${BUILDBOT_ROOT}/llvm-project" +REVISION=${BUILDBOT_REVISION:-origin/main} +AMDGPU_ARCHS=${AMDGPU_ARCHS:="gfx900;gfx906;gfx908;gfx1030"} + +# Set-up llvm-project +if [ ! -d "${LLVM_ROOT}" ]; then + build_step "Cloning llvm-project repo" + git clone --progress https://github.com/llvm/llvm-project.git ${LLVM_ROOT} +fi + +build_step "Updating llvm-project repo" +git -C "${LLVM_ROOT}" fetch origin +git -C "${LLVM_ROOT}" reset --hard ${REVISION} + +# Set-up llvm-test-suite +TESTSUITE_ROOT="${BUILDBOT_ROOT}/llvm-test-suite" +if [ ! -d "${TESTSUITE_ROOT}" ]; then + build_step "Cloning llvm-test-suite repo" + git clone --progress https://github.com/llvm/llvm-test-suite.git ${TESTSUITE_ROOT} +fi + +build_step "Updating llvm-test-suite repo" +git -C "${TESTSUITE_ROOT}" fetch origin +git -C "${TESTSUITE_ROOT}" reset --hard origin/main + +# Set-up variables +BUILDBOT_SLAVENAME=$(whoami) +BUILD_DIR="${BUILDBOT_ROOT}/${BUILDBOT_SLAVENAME}/${BUILDBOT_BUILDERNAME}" +DESTDIR=${BUILD_DIR}/install +EXTERNAL_DIR=/buildbot/Externals + +build_step "Setting up the buildbot" +echo "BUILDBOT_ROOT=${BUILDBOT_ROOT}" +echo "LLVM_ROOT=${LLVM_ROOT}" +echo "BUILD_DIR=${BUILD_DIR}" +echo "DESTDIR=${DESTDIR}" +echo "EXTERNAL_DIR=${EXTERNAL_DIR}" + +# Start building LLVM, Clang, Lld, clang-tools-extra, compiler-rt +build_step "Configure LLVM Build" +LLVM_BUILD_DIR="${BUILD_DIR}/llvm" +mkdir -p "${LLVM_BUILD_DIR}" +cd "${LLVM_BUILD_DIR}" +cmake -G Ninja \ + -DCMAKE_BUILD_TYPE="Release" \ + -DCMAKE_VERBOSE_MAKEFILE=1 \ + -DLLVM_TARGETS_TO_BUILD="AMDGPU;X86" \ + -DLLVM_ENABLE_PROJECTS="clang;lld;clang-tools-extra;compiler-rt" \ + -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi" \ + -DLIBCXX_ENABLE_SHARED=OFF \ + -DLIBCXX_ENABLE_STATIC=ON \ + -DLIBCXX_INSTALL_LIBRARY=OFF \ + -DLIBCXX_INSTALL_HEADERS=OFF \ + -DLIBCXXABI_ENABLE_SHARED=OFF \ + -DLIBCXXABI_ENABLE_STATIC=ON \ + -DLIBCXXABI_INSTALL_STATIC_LIBRARY=OFF \ + -DCMAKE_INSTALL_PREFIX="${DESTDIR}" \ + -DLLVM_ENABLE_ASSERTIONS=ON \ + -DLLVM_ENABLE_Z3_SOLVER=OFF \ + -DLLVM_ENABLE_ZLIB=ON \ + -DLLVM_LIT_ARGS="-v -vv" \ + ${LLVM_ROOT}/llvm + +build_step "Building LLVM" +ninja runtimes +ninja + +build_step "Install LLVM" +rm -rf "${DESTDIR}" +ninja install-runtimes +ninja install + +# Start building llvm-test-suite's hip tests +build_step "Configuring HIP test-suite" +TEST_BUILD_DIR=${BUILD_DIR}/test-suite-build +rm -rf ${TEST_BUILD_DIR} +mkdir -p ${TEST_BUILD_DIR} +cd ${TEST_BUILD_DIR} +PATH="${LLVM_BUILD_DIR}/bin:$PATH" cmake -G Ninja \ + -DTEST_SUITE_SUBDIRS=External \ + -DTEST_SUITE_EXTERNALS_DIR=${EXTERNAL_DIR}/ \ + -DTEST_SUITE_COLLECT_CODE_SIZE=OFF \ + -DTEST_SUITE_COLLECT_COMPILE_TIME=OFF \ + -DAMDGPU_ARCHS="${AMDGPU_ARCHS}" \ + -DCMAKE_CXX_COMPILER="${LLVM_BUILD_DIR}/bin/clang++" \ + -DCMAKE_C_COMPILER="${LLVM_BUILD_DIR}/bin/clang" \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + ${TESTSUITE_ROOT} + +build_step "Building HIP test-suite" +ninja hip-tests-simple + +build_step "Testing HIP test-suite" +ninja check-hip-simple + +exit 0 + diff --git a/zorg/buildbot/builders/annotated/libc-linux.py b/zorg/buildbot/builders/annotated/libc-linux.py --- a/zorg/buildbot/builders/annotated/libc-linux.py +++ b/zorg/buildbot/builders/annotated/libc-linux.py @@ -9,6 +9,13 @@ from contextlib import contextmanager +def is_fullbuild_builder(builder_name): + return ('fullbuild' in builder_name.split('-')) + +def is_runtimes_builder(builder_name): + return ('runtimes' in builder_name.split('-')) + + def main(argv): ap = argparse.ArgumentParser() ap.add_argument('--asan', action='store_true', default=False, @@ -18,11 +25,25 @@ args, _ = ap.parse_known_args() source_dir = os.path.join('..', 'llvm-project') + builder_name = os.environ.get('BUILDBOT_BUILDERNAME') + fullbuild = is_fullbuild_builder(builder_name) + runtimes_build = is_runtimes_builder(builder_name) with step('cmake', halt_on_fail=True): - projects = ['llvm', 'libc', 'clang', 'clang-tools-extra'] + # On most systems the default generator is make and the default + # compilers are gcc and g++. We make it explicit here that we want + # clang and ninja which reduces one step of setting environment + # variables when setting up workers. + cmake_args = ['-GNinja', + '-DCMAKE_C_COMPILER=clang', + '-DCMAKE_CXX_COMPILER=clang++'] + + if runtimes_build: + projects = ['llvm', 'clang'] + cmake_args.append('-DLLVM_ENABLE_RUNTIMES=libc') + else: + projects = ['llvm', 'libc'] - cmake_args = ['-GNinja'] if args.debug: cmake_args.append('-DCMAKE_BUILD_TYPE=Debug') else: @@ -31,28 +52,46 @@ if args.asan: cmake_args.append('-DLLVM_USE_SANITIZER=Address') + if fullbuild and not args.asan: + projects.extend(['clang', 'compiler-rt']) + cmake_args.append('-DLLVM_ENABLE_PROJECTS={}'.format(';'.join(projects))) + if fullbuild and not args.asan: + cmake_args.append('-DLLVM_LIBC_INCLUDE_SCUDO=ON') + cmake_args.append('-DLIBC_INCLUDE_BENCHMARKS=ON') + + if fullbuild: + cmake_args.extend(['-DLLVM_LIBC_FULL_BUILD=ON']), + run_command(['cmake', os.path.join(source_dir, 'llvm')] + cmake_args) - with step('build llvmlibc', halt_on_fail=True): - run_command(['ninja', 'llvmlibc']) + with step('build libc'): + if fullbuild: + run_command(['ninja', 'libc']) + else: + run_command(['ninja', 'llvmlibc']) with step('check-libc'): - run_command(['ninja', 'check-libc']) - - if not args.asan: - with step('Loader Tests'): - run_command(['ninja', 'libc_loader_tests']) - with step('Integration Tests'): - run_command(['ninja', 'libc-integration-test']) + if runtimes_build: + run_command(['ninja', 'check-llvmlibc']) + else: + run_command(['ninja', 'check-libc']) + + if fullbuild and not args.asan: + with step('libc-integration-tests'): + run_command(['ninja', 'libc-integration-tests']) + with step('libc-api-test'): + run_command(['ninja', 'libc-api-test']) + with step('libc-fuzzer'): + run_command(['ninja', 'libc-fuzzer']) + with step('libc-scudo-integration-test'): + run_command(['ninja', 'libc-scudo-integration-test']) with step('AOR Tests'): aor_dir = os.path.join(source_dir, 'libc', 'AOR_v20.02') # Remove the AOR build dir. util.clean_dir(os.path.join(aor_dir, 'build')) run_command(['make', 'check'], directory=aor_dir) - - if not args.debug: with step('Benchmark Utils Tests'): run_command(['ninja', 'libc-benchmark-util-tests']) diff --git a/zorg/buildbot/builders/annotated/libc-linux.py b/zorg/buildbot/builders/annotated/libc-windows.py copy from zorg/buildbot/builders/annotated/libc-linux.py copy to zorg/buildbot/builders/annotated/libc-windows.py --- a/zorg/buildbot/builders/annotated/libc-linux.py +++ b/zorg/buildbot/builders/annotated/libc-windows.py @@ -1,14 +1,14 @@ -#!/usr/bin/python + +import annotated_builder +import util import argparse import os import subprocess import sys import traceback -import util from contextlib import contextmanager - def main(argv): ap = argparse.ArgumentParser() ap.add_argument('--asan', action='store_true', default=False, @@ -18,11 +18,19 @@ args, _ = ap.parse_known_args() source_dir = os.path.join('..', 'llvm-project') + vc_vars = annotated_builder.get_vcvars(vs_tools=None, arch='amd64') + for (var, val) in vc_vars.items(): + os.environ[var] = val with step('cmake', halt_on_fail=True): - projects = ['llvm', 'libc', 'clang', 'clang-tools-extra'] - - cmake_args = ['-GNinja'] + # On most systems the default generator is make and the default + # compilers are gcc and g++. We make it explicit here that we want + # clang and ninja which reduces one step of setting environment + # variables when setting up workers. + ## adjust compiler location definitions based on VM config + cmake_args = ['-GNinja', + '-DCMAKE_C_COMPILER=clang-cl', + '-DCMAKE_CXX_COMPILER=clang-cl'] if args.debug: cmake_args.append('-DCMAKE_BUILD_TYPE=Debug') else: @@ -31,7 +39,12 @@ if args.asan: cmake_args.append('-DLLVM_USE_SANITIZER=Address') - cmake_args.append('-DLLVM_ENABLE_PROJECTS={}'.format(';'.join(projects))) + cmake_args.append('-DLLVM_ENABLE_PROJECTS=libc') + cmake_args.append('-DLLVM_TARGETS_TO_BUILD=X86') + cmake_args.append('-DLLVM_FORCE_BUILD_RUNTIME=libc') + cmake_args.append('-DLLVM_NATIVE_ARCH=x86_64') + cmake_args.append('-DLLVM_HOST_TRIPLE=x86_64-window-x86-gnu') + cmake_args.append('-DLLVM_LIBC_MPFR_INSTALL_PATH=C:/src/install') run_command(['cmake', os.path.join(source_dir, 'llvm')] + cmake_args) @@ -41,21 +54,6 @@ with step('check-libc'): run_command(['ninja', 'check-libc']) - if not args.asan: - with step('Loader Tests'): - run_command(['ninja', 'libc_loader_tests']) - with step('Integration Tests'): - run_command(['ninja', 'libc-integration-test']) - with step('AOR Tests'): - aor_dir = os.path.join(source_dir, 'libc', 'AOR_v20.02') - # Remove the AOR build dir. - util.clean_dir(os.path.join(aor_dir, 'build')) - run_command(['make', 'check'], directory=aor_dir) - - if not args.debug: - with step('Benchmark Utils Tests'): - run_command(['ninja', 'libc-benchmark-util-tests']) - @contextmanager def step(step_name, halt_on_fail=False): diff --git a/zorg/buildbot/builders/annotated/standalone-build.sh b/zorg/buildbot/builders/annotated/standalone-build.sh new file mode 100755 --- /dev/null +++ b/zorg/buildbot/builders/annotated/standalone-build.sh @@ -0,0 +1,214 @@ +#!/bin/bash + +# This script builds LLVM and Clang in standalone mode that means it first +# builds LLVM and installs it into a specific directory. That directory is then +# used when building Clang which depends on it. + +# Enable Error tracing +set -o errtrace + +# Print trace for all commands ran before execution +set -x + +# Include the Buildbot helper functions +HERE="$(realpath $(dirname $0))" +. ${HERE}/buildbot-helper.sh + +# Ensure all commands pass, and not dereferencing unset variables. +set -eu +halt_on_failure + +BUILDBOT_ROOT=${BUILDBOT_ROOT:-${HERE}} +REVISION=${BUILDBOT_REVISION:-origin/main} +LLVM_ROOT="${BUILDBOT_ROOT}/llvm-project" +BUILD_TYPE=Release +INSTALL_ROOT_DIR=${BUILDBOT_ROOT}/install +BUILD_ROOT_DIR=${BUILDBOT_ROOT}/build + +install_dir() { + echo ${INSTALL_ROOT_DIR}/$1 +} + +build_dir() { + echo ${BUILD_ROOT_DIR}/$1 +} + +setup_llvm_project() { + build_step "Setup llvm-project" + + if [ ! -d "${LLVM_ROOT}" ]; then + build_step "Cloning llvm-project repo" + git clone --progress https://github.com/llvm/llvm-project.git ${LLVM_ROOT} + fi + + build_step "Updating llvm-project repo" + git -C "${LLVM_ROOT}" fetch origin + git -C "${LLVM_ROOT}" reset --hard ${REVISION} + git -C "${LLVM_ROOT}" sparse-checkout init --cone +} + +pre_build_cleanup() { + build_step "Pre-build cleanup" + rm -rf ${INSTALL_ROOT_DIR} + rm -rf ${BUILD_ROOT_DIR} +} + +build_python_lit() { + local PYTHON_LIT_BUILD_DIR=$(build_dir python-lit) + local PYTHON_LIT_INSTALL_DIR=$(install_dir python-lit) + + build_step "Sparse checkout out python-lit" + git -C "${LLVM_ROOT}" sparse-checkout set llvm/utils/lit + + build_step "Building python-lit" + cd "${LLVM_ROOT}"/llvm/utils/lit + python3 setup.py build '--executable=/usr/bin/python3 -s' + + # TODO(kkleine): Fix failing test: FAIL: lit :: shtest-not.py (43 of 50) + #build_step "Testing python-lit" + #python3 lit.py tests + + build_step "Installing python-lit" + rm -rf ${PYTHON_LIT_INSTALL_DIR} + python3 setup.py install -O1 --skip-build --root ${PYTHON_LIT_INSTALL_DIR} + + build_step "Removing python-lit build dir" + rm -rf ${PYTHON_LIT_BUILD_DIR} + + cd - +} + +# We don't install python to the system so we need help python to find it +discover_python_lit() { + build_step "Help discover python-lit module" + local PYTHON_LIT_INSTALL_DIR=$(install_dir python-lit) + local pyver=$(echo 'import sys; print("{}.{}".format(sys.version_info.major, sys.version_info.minor))' | python3) + export PYTHONPATH="${PYTHONPATH:-}:${PYTHON_LIT_INSTALL_DIR}/usr/local/lib/python${pyver}/site-packages" +} + +build_llvm() { + local PYTHON_LIT_INSTALL_DIR=$(install_dir python-lit) + local LLVM_BUILD_DIR=$(build_dir llvm) + local LLVM_INSTALL_DIR=$(install_dir llvm) + + build_step "Sparse checkout out llvm" + git -C "${LLVM_ROOT}" sparse-checkout set llvm cmake third-party + + build_step "Configuring llvm" + + cmake \ + -S ${LLVM_ROOT}/llvm \ + -B ${LLVM_BUILD_DIR} \ + -G Ninja \ + -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ + -DLLVM_BUILD_LLVM_DYLIB=ON \ + -DLLVM_LINK_LLVM_DYLIB=ON \ + -DLLVM_INCLUDE_BENCHMARKS=OFF \ + -DLLVM_INSTALL_UTILS=ON \ + -DCMAKE_INSTALL_PREFIX=${LLVM_INSTALL_DIR} \ + -DLLVM_EXTERNAL_LIT=${PYTHON_LIT_INSTALL_DIR}/usr/local/bin/lit \ + -DLLVM_INCLUDE_UTILS:BOOL=ON \ + -DLLVM_INSTALL_UTILS:BOOL=ON \ + -DLLVM_UTILS_INSTALL_DIR:PATH=${LLVM_INSTALL_DIR}/bin + + build_step "Building llvm" + cmake --build ${LLVM_BUILD_DIR} + + build_step "Testing llvm" + LD_LIBRARY_PATH="${LLVM_INSTALL_DIR}/lib64" cmake --build ${LLVM_BUILD_DIR} --target check-all + + build_step "Installing llvm" + rm -rf ${LLVM_INSTALL_DIR} + cmake --install ${LLVM_BUILD_DIR} + + # This is meant to extinguish any dependency on files being taken + # from the llvm build dir when building clang. + build_step "Removing llvm build dir" + rm -rf "${LLVM_BUILD_DIR}" +} + +build_clang() { + local PYTHON_LIT_INSTALL_DIR=$(install_dir python-lit) + local LLVM_INSTALL_DIR=$(install_dir llvm) + local CLANG_BUILD_DIR=$(build_dir clang) + local CLANG_INSTALL_DIR=$(install_dir clang) + + build_step "Sparse checkout out clang" + git -C "${LLVM_ROOT}" sparse-checkout set clang cmake + + build_step "Configuring clang" + cmake \ + -S ${LLVM_ROOT}/clang \ + -B ${CLANG_BUILD_DIR} \ + -G Ninja \ + -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ + -DCLANG_LINK_CLANG_DYLIB=ON \ + -DCLANG_INCLUDE_TESTS=ON \ + -DLLVM_EXTERNAL_LIT=${PYTHON_LIT_INSTALL_DIR}/usr/local/bin/lit \ + -DCMAKE_INSTALL_PREFIX=${CLANG_INSTALL_DIR} \ + -DLLVM_ROOT=${LLVM_INSTALL_DIR} \ + -DLLVM_ENABLE_ASSERTIONS=ON \ + -DLLVM_TABLEGEN_EXE=${LLVM_INSTALL_DIR}/bin/llvm-tblgen + + build_step "Building clang" + LD_LIBRARY_PATH="${LLVM_INSTALL_DIR}/lib64" cmake --build ${CLANG_BUILD_DIR} + + build_step "Installing clang" + rm -rf ${CLANG_INSTALL_DIR} + cmake --install ${CLANG_BUILD_DIR} + + build_step "Removing clang build dir" + rm -rf ${CLANG_BUILD_DIR} +} + +build_lld() { + local LLVM_INSTALL_DIR=$(install_dir llvm) + local PYTHON_LIT_INSTALL_DIR=$(install_dir python-lit) + local LLD_BUILD_DIR=$(build_dir lld) + local LLD_INSTALL_DIR=$(install_dir lld) + + build_step "Sparse checkout out lld" + git -C "${LLVM_ROOT}" sparse-checkout set lld cmake libunwind + + # We don't want to checkout the llvm source tree but sadly there are paths + # like ${LLVM_MAIN_SRC_DIR}/../libunwind/include in lld source code. They + # resolve to //../llvm/../libunwind/include which makes absolutely + # no sense when the llvm dir doesn't exist. Let's fix this by just providing + # the empty llvm dir so that paths are resolved without errors. + # TODO: I don't know how to fix this easily + rm -rf "${LLVM_ROOT}"/llvm && mkdir "${LLVM_ROOT}"/llvm + rm -rf ${LLD_INSTALL_DIR} + rm -rf ${LLD_BUILD_DIR} + + build_step "Configuring lld" + cmake \ + -S ${LLVM_ROOT}/lld \ + -B ${LLD_BUILD_DIR} \ + -G Ninja \ + -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ + -DLLVM_LINK_LLVM_DYLIB=ON \ + -DCMAKE_INSTALL_PREFIX=${LLD_INSTALL_DIR} \ + -DLLVM_ROOT=${LLVM_INSTALL_DIR} + + build_step "Building lld" + LD_LIBRARY_PATH="${LLVM_INSTALL_DIR}/lib64" cmake --build ${LLD_BUILD_DIR} + + build_step "Installing lld" + rm -rf ${LLD_INSTALL_DIR} + cmake --install ${LLD_BUILD_DIR} + + build_step "Removing lld build dir" + rm -rf ${LLD_BUILD_DIR} +} + +setup_llvm_project +pre_build_cleanup + +build_python_lit +discover_python_lit +build_llvm +build_clang +build_lld + +exit 0 + diff --git a/zorg/buildbot/builders/annotated/ve-linux-steps.make b/zorg/buildbot/builders/annotated/ve-linux-steps.make new file mode 100644 --- /dev/null +++ b/zorg/buildbot/builders/annotated/ve-linux-steps.make @@ -0,0 +1,244 @@ +##### Interface Variables & Targets ##### +BUILDROOT?=$(error "BUILDROOT has to be the path to the worker's build/ directory.") + +# Renders one target per line in make order. Each target will made with a +# build step with the 'get-steps' annotated builder (ve-linux.py). +get-steps: + @echo "prepare" + @echo "build-llvm" + @echo "check-llvm" + @echo "build-crt-ve" + @echo "install-crt-ve" + @echo "check-crt-ve" + @echo "build-libunwind-ve" + @echo "install-libunwind-ve" +# @echo "build-libcxx-ve" +# @echo "install-libcxx-ve" +# @echo "build-libcxxabi-ve" +# @echo "install-libcxxabi-ve" + + +##### Tools ##### +CMAKE?=cmake +NINJA?=ninja +TOOL_CONFIG_CACHE?=${HOME}/tools-config.cmake + +##### Derived Configuration ##### + +# Path + +MONOREPO=${BUILDROOT}/../llvm-project + +# Build foders +LLVM_BUILD="${BUILDROOT}/build_llvm" +CRT_BUILD_VE="${BUILDROOT}/build_crt_ve" +LIBUNWIND_BUILD_VE="${BUILDROOT}/build_libunwind_ve" +LIBCXXABI_BUILD_VE="${BUILDROOT}/build_libcxxabi_ve" +LIBCXX_BUILD_VE="${BUILDROOT}/build_libcxx_ve" + +# 'Install' into the LLVM build tree. +INTREE_PREFIX="${LLVM_BUILD}" +LLVM_PREFIX="${BUILDROOT}/install" + +# Install prefix structure +BUILT_CLANG="${INTREE_PREFIX}/bin/clang" +BUILT_CLANGXX="${INTREE_PREFIX}/bin/clang++" +VE_TARGET="ve-unknown-linux-gnu" +LINUX_VE_LIBSUFFIX=/linux/ve + +# Resource dir (Requires clang to be installed before this variable gets expanded) +RES_VERSION=$(shell ${INTREE_PREFIX}/bin/llvm-config --version | sed -n 's/git//p') +CLANG_RESDIR="${INTREE_PREFIX}/lib/clang/${RES_VERSION}" + +### LLVM +LLVM_BUILD_TYPE=RelWithDebInfo + +### Compiler-RT +CRT_BUILD_TYPE=Release +CRT_OPTFLAGS=-O2 +CRT_TEST_OPTFLAGS=-O2 + +## libunwind +LIBUNWIND_BUILD_TYPE=Release +LIBUNWIND_OPTFLAGS=-O2 + +## libcxxabi +LIBCXXABI_BUILD_TYPE=Release +LIBCXXABI_OPTFLAGS=-O2 + +## libcxxabi +LIBCXX_BUILD_TYPE=Release +LIBCXX_OPTFLAGS=-O2 + + + + +##### Build Steps ##### + +### Vanilla LLVM stage ### +build-llvm: + touch ${TOOL_CONFIG_CACHE} + mkdir -p ${LLVM_BUILD} + cd ${LLVM_BUILD} && ${CMAKE} ${MONOREPO}/llvm -G Ninja \ + -C ${TOOL_CONFIG_CACHE} \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DLLVM_BUILD_LLVM_DYLIB=Off \ + -DLLVM_LINK_LLVM_DYLIB=Off \ + -DCLANG_LINK_CLANG_DYLIB=Off \ + -DLLVM_TARGETS_TO_BUILD="X86;VE" \ + -DLLVM_ENABLE_PROJECTS="clang" \ + -DCMAKE_INSTALL_PREFIX="${LLVM_PREFIX}" \ + -DLLVM_INSTALL_UTILS=On + cd ${LLVM_BUILD} && ${NINJA} + +# install-llvm: +# # build-llvm +# cd ${LLVM_BUILD} && ${NINJA} install + +check-llvm: + # build-llvm + cd ${LLVM_BUILD} && ${NINJA} check-all + + +### Compiler-RT standalone ### + +build-crt-ve: + mkdir -p ${CRT_BUILD_VE} + cd ${CRT_BUILD_VE} && ${CMAKE} ${MONOREPO}/runtimes -G Ninja \ + -DLLVM_ENABLE_RUNTIMES="compiler-rt" \ + -DCOMPILER_RT_BUILD_BUILTINS=ON \ + -DCOMPILER_RT_BUILD_SANITIZERS=OFF \ + -DCOMPILER_RT_BUILD_XRAY=OFF \ + -DCOMPILER_RT_BUILD_LIBFUZZER=OFF \ + -DCOMPILER_RT_BUILD_PROFILE=ON \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_C_COMPILER=${BUILT_CLANG} \ + -DCMAKE_C_COMPILER_TARGET="${VE_TARGET}" \ + -DCMAKE_CXX_COMPILER=${BUILT_CLANGXX} \ + -DCMAKE_CXX_COMPILER_TARGET="${VE_TARGET}" \ + -DCMAKE_ASM_COMPILER_TARGET="${VE_TARGET}" \ + -DCMAKE_AR=${INTREE_PREFIX}/bin/llvm-ar \ + -DCMAKE_RANLIB=${INTREE_PREFIX}/bin/llvm-ranlib \ + -DCOMPILER_RT_DEFAULT_TARGET_ONLY=ON \ + -DLLVM_CONFIG_PATH=${INTREE_PREFIX}/bin/llvm-config \ + -DCMAKE_BUILD_TYPE="${CRT_BUILD_TYPE}" \ + -DCMAKE_INSTALL_PREFIX="${CLANG_RESDIR}" \ + -DCMAKE_CXX_FLAGS="-nostdlib" \ + -DCMAKE_CXX_FLAGS_RELEASE="${CRT_OPTFLAGS}" \ + -DCMAKE_C_FLAGS="-nostdlib" \ + -DCMAKE_C_FLAGS_RELEASE="${CRT_OPTFLAGS}" \ + -DCOMPILER_RT_INCLUDE_TESTS=ON \ + -DCOMPILER_RT_TEST_COMPILER=${BUILT_CLANG} \ + -DCOMPILER_RT_TEST_COMPILER_CFLAGS="-target ${VE_TARGET} ${CRT_TEST_OPTFLAGS}" + cd ${CRT_BUILD_VE} && ${NINJA} + +check-crt-ve: build-crt-ve + cd ${CRT_BUILD_VE} && env PATH=${INTREE_PREFIX}/bin:${PATH} ${NINJA} check-compiler-rt + +install-crt-ve: build-crt-ve + cd ${CRT_BUILD_VE} && ${NINJA} install + + +### libunwind standalone ### +build-libunwind-ve: + mkdir -p ${LIBUNWIND_BUILD_VE} + cd ${LIBUNWIND_BUILD_VE} && ${CMAKE} ${MONOREPO}/runtimes -G Ninja \ + -DLLVM_ENABLE_RUNTIMES="libunwind" \ + -DCMAKE_C_COMPILER=${BUILT_CLANG} \ + -DCMAKE_CXX_COMPILER=${BUILT_CLANGXX} \ + -DCMAKE_AR=${INTREE_PREFIX}/bin/llvm-ar \ + -DCMAKE_RANLIB=${INTREE_PREFIX}/bin/llvm-ranlib \ + -DCMAKE_ASM_COMPILER_TARGET="${VE_TARGET}" \ + -DCMAKE_C_COMPILER_TARGET="${VE_TARGET}" \ + -DCMAKE_CXX_COMPILER_TARGET="${VE_TARGET}" \ + -DCMAKE_BUILD_TYPE="${LIBUNWIND_BUILD_TYPE}" \ + -DCMAKE_INSTALL_PREFIX="${CLANG_RESDIR}" \ + -DLIBUNWIND_LIBDIR_SUFFIX="${LINUX_VE_LIBSUFFIX}" \ + -DCMAKE_CXX_FLAGS="-nostdlib" \ + -DCMAKE_CXX_FLAGS_RELEASE="${LIBUNWIND_OPTFLAGS}" \ + -DCMAKE_C_FLAGS="-nostdlib" \ + -DCMAKE_C_FLAGS_RELEASE="${LIBUNWIND_OPTFLAGS}" \ + -DLIBUNWIND_LIBCXX_PATH=${MONOREPO}/libcxx \ + -DLLVM_PATH=${MONOREPO}/llvm + cd ${LIBUNWIND_BUILD_VE} && ${NINJA} + +install-libunwind-ve: + cd ${LIBUNWIND_BUILD_VE} && ${NINJA} install + + +### libcxx standalone ### + +build-libcxx-ve: + mkdir -p ${LIBCXX_BUILD_VE} + cd ${LIBCXX_BUILD_VE} && ${CMAKE} ${MONOREPO}/runtimes -G Ninja \ + -DLLVM_ENABLE_RUNTIMES="libcxx" \ + -DLIBCXX_USE_COMPILER_RT=True \ + -DCMAKE_C_COMPILER=${BUILT_CLANG} \ + -DCMAKE_CXX_COMPILER=${BUILT_CLANGXX} \ + -DCMAKE_AR=${INTREE_PREFIX}/bin/llvm-ar \ + -DCMAKE_RANLIB=${INTREE_PREFIX}/bin/llvm-ranlib \ + -DCMAKE_C_COMPILER_TARGET="${VE_TARGET}" \ + -DCMAKE_CXX_COMPILER_TARGET="${VE_TARGET}" \ + -DCMAKE_BUILD_TYPE="${LIBCXX_BUILD_TYPE}" \ + -DCMAKE_INSTALL_PREFIX="${CLANG_RESDIR}" \ + -DLIBCXX_LIBDIR_SUFFIX="${LINUX_VE_LIBSUFFIX}" \ + -DLIBCXXABI_USE_LLVM_UNWINDER=True \ + -DLIBCXX_CXX_ABI=libcxxabi \ + -DLIBCXX_CXX_ABI_INCLUDE_PATHS=${MONOREPO}/libcxxabi/include \ + -DCMAKE_C_FLAGS_RELEASE="${LIBCXX_OPTFLAGS}" \ + -DCMAKE_CXX_FLAGS="-nostdlib++" \ + -DCMAKE_CXX_FLAGS_RELEASE="${LIBCXX_OPTFLAGS}" \ + -DLIBCXX_USE_COMPILER_RT=True + cd ${LIBCXX_BUILD_VE} && ${NINJA} + +check-libcxx-ve: + cd ${LIBCXX_BUILD_VE} && ${NINJA} check-cxx + +install-libcxx-ve: + cd ${LIBCXX_BUILD_VE} && ${NINJA} install + + + + +### libcxxabi standalone ### + +build-libcxxabi-ve: + mkdir -p ${LIBCXXABI_BUILD_VE} + cd ${LIBCXXABI_BUILD_VE} && ${CMAKE} ${MONOREPO}/runtimes -G Ninja \ + -DLLVM_ENABLE_RUNTIMES="libcxxabi" \ + -DCMAKE_C_COMPILER=${BUILT_CLANG} \ + -DCMAKE_CXX_COMPILER=${BUILT_CLANGXX} \ + -DCMAKE_AR=${INTREE_PREFIX}/bin/llvm-ar \ + -DCMAKE_RANLIB=${INTREE_PREFIX}/bin/llvm-ranlib \ + -DCMAKE_C_COMPILER_TARGET="${VE_TARGET}" \ + -DCMAKE_CXX_COMPILER_TARGET="${VE_TARGET}" \ + -DLLVM_CONFIG_PATH=${INTREE_PREFIX}/bin/llvm-config \ + -DCMAKE_BUILD_TYPE="${LIBCXXABI_BUILD_TYPE}" \ + -DCMAKE_INSTALL_PREFIX="${CLANG_RESDIR}" \ + -DLIBCXXABI_LIBDIR_SUFFIX="${LINUX_VE_LIBSUFFIX}" \ + -DLIBCXXABI_USE_LLVM_UNWINDER=YES \ + -DCMAKE_CXX_FLAGS="-nostdlib++" \ + -DCMAKE_CXX_FLAGS_RELEASE="${LIBCXX_OPTFLAGS}" \ + -DCMAKE_C_FLAGS_RELEASE="${LIBCXX_OPTFLAGS}" \ + -DLLVM_PATH=${MONOREPO}/llvm \ + -DLLVM_MAIN_SRC_DIR=${MONOREPO}/llvm \ + -DLIBCXXABI_USE_COMPILER_RT=True \ + -DLIBCXXABI_HAS_NOSTDINCXX_FLAG=True \ + -DLIBCXXABI_LIBCXX_INCLUDES="${CLANG_RESDIR}/include/c++/v1/" + cd ${LIBCXXABI_BUILD_VE} && ${NINJA} + +check-libcxxabi-ve: + cd ${LIBCXXABI_BUILD_VE} && ${NINJA} check-cxxabi + +install-libcxxabi-ve: + cd ${LIBCXXABI_BUILD_VE} && ${NINJA} install + +# Clearout the temporary install prefix. +prepare: + # Build everything from scratch - TODO: incrementalize later. + rm -rf ${LLVM_PREFIX} + rm -rf ${LLVM_BUILD} + rm -rf ${CRT_BUILD_VE} + rm -rf ${LIBUNWIND_BUILD_VE} + rm -rf ${LIBCXXABI_BUILD_VE} + rm -rf ${LIBCXX_BUILD_VE} diff --git a/zorg/buildbot/builders/annotated/ve-linux.py b/zorg/buildbot/builders/annotated/ve-linux.py new file mode 100644 --- /dev/null +++ b/zorg/buildbot/builders/annotated/ve-linux.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 + +import argparse +import os +import subprocess +import sys +import traceback +import util +from contextlib import contextmanager + + +def main(argv): + # Avoid a buildmaster update by hard-coding this for now. + makefile = 've-linux-steps.make' + + # TODO: Make Makefile configurable and update buildmaster. + # ap = argparse.ArgumentParser() + # ap.add_argument('makefile', help='The Makefile to use. (relative to annoted/ folder of llvm-zorg).') + # args, _ = ap.parse_known_args() + # makefile = args.makefile + + worker_dir = os.path.abspath(os.path.join('..')) + annotated_dir = os.path.join(worker_dir, 'llvm-zorg', 'zorg', 'buildbot', 'builders', 'annotated') + makefile_path = os.path.join(annotated_dir, makefile) + + # Query step list from makefile. + build_targets=[] + with step('prepare', halt_on_fail=True): + build_targets = get_steps(makefile_path) + + make_vars = { + 'BUILDROOT' : os.path.join(worker_dir, 'build') + } + + for target in build_targets: + halt_on_fail = not ("check-" in target) + with step(target, halt_on_fail=halt_on_fail): + make_cmd = build_make_cmd(makefile_path, target, make_vars) + run_command(make_cmd, cwd='.') + +@contextmanager +def step(step_name, halt_on_fail=False): + util.report('@@@BUILD_STEP {}@@@'.format(step_name)) + if halt_on_fail: + util.report('@@@HALT_ON_FAILURE@@@') + try: + yield + except Exception as e: + if isinstance(e, subprocess.CalledProcessError): + util.report( + '{} exited with return code {}.'.format(e.cmd, e.returncode) + ) + util.report('The build step threw an exception...') + traceback.print_exc() + + util.report('@@@STEP_FAILURE@@@') + finally: + sys.stdout.flush() + + +def get_steps(makefile): + try: + make_cmd = build_make_cmd(makefile, 'get-steps') + raw_steps = capture_cmd_stdout(make_cmd) + return raw_steps.decode('utf-8').split('\n')[:-1] + except: + return [] + +def build_make_cmd(makefile, target, make_vars={}): + make_cmd = ['make', '-f', makefile] + if not target is None: + make_cmd.append(target) + for k,v in make_vars.items(): + make_cmd += ["{}={}".format(k, v)] + return make_cmd + +def capture_cmd_stdout(cmd, **kwargs): + return subprocess.run(cmd, shell=False, check=True, stdout=subprocess.PIPE, **kwargs).stdout + +def run_command(cmd, **kwargs): + util.report_run_cmd(cmd, **kwargs) + +if __name__ == '__main__': + sys.path.append(os.path.dirname(__file__)) + sys.exit(main(sys.argv)) diff --git a/zorg/buildbot/builders/sanitizers/buildbot_android.sh b/zorg/buildbot/builders/sanitizers/buildbot_android.sh --- a/zorg/buildbot/builders/sanitizers/buildbot_android.sh +++ b/zorg/buildbot/builders/sanitizers/buildbot_android.sh @@ -14,12 +14,7 @@ export PATH="/usr/local/bin:$PATH" LLVM=$ROOT/llvm -CMAKE_COMMON_OPTIONS="${CMAKE_COMMON_OPTIONS:-}" -CMAKE_COMMON_OPTIONS="${CMAKE_COMMON_OPTIONS:-} -GNinja -DCMAKE_BUILD_TYPE=Release" - -if [ -e /usr/include/plugin-api.h ]; then - CMAKE_COMMON_OPTIONS="${CMAKE_COMMON_OPTIONS}" -fi +CMAKE_COMMON_OPTIONS+=" -GNinja -DCMAKE_BUILD_TYPE=Release" clobber @@ -32,7 +27,7 @@ buildbot_update -CMAKE_COMMON_OPTIONS="$CMAKE_COMMON_OPTIONS -DLLVM_ENABLE_ASSERTIONS=ON" +CMAKE_COMMON_OPTIONS+=" -DLLVM_ENABLE_ASSERTIONS=ON" build_stage2_android @@ -47,5 +42,5 @@ build_android arm build_android i686 -# Arm hardware is temporarily offline -test_android i686:x86 arm:armeabi-v7a aarch64:arm64-v8a +# i686:x86 hardware is offline. +test_android arm:armeabi-v7a aarch64:arm64-v8a diff --git a/zorg/buildbot/builders/sanitizers/buildbot_android_functions.sh b/zorg/buildbot/builders/sanitizers/buildbot_android_functions.sh --- a/zorg/buildbot/builders/sanitizers/buildbot_android_functions.sh +++ b/zorg/buildbot/builders/sanitizers/buildbot_android_functions.sh @@ -1,9 +1,5 @@ #!/usr/bin/env bash -CLOBBER="android_ndk android-ndk-* platform-tools *.zip shards_* test_android_* tested_arch_*" -STAGE2_CLOBBER="compiler_rt_build_android_* llvm_build_android_*" -STAGE1_CLOBBER="llvm_build64 ${STAGE2_CLOBBER}" - ANDROID_NDK_VERSION=21 ANDROID_API=24 NDK_DIR=android_ndk @@ -17,7 +13,7 @@ [[ -d ${NDK_DIR} ]] && rm -rf ${NDK_DIR} [[ -d ${VERSION} ]] && rm -rf ${VERSION} [[ -f ${FILE_NAME} ]] && rm -f ${FILE_NAME} - wget ${NDK_URL} + wget -e dotbytes=10m ${NDK_URL} unzip ${FILE_NAME} > /dev/null mv ${VERSION} ${NDK_DIR} echo ${NDK_URL} > ${NDK_DIR}/android_ndk_url @@ -27,7 +23,7 @@ echo @@@BUILD_STEP downloading Android Platform Tools@@@ local FILE_NAME=platform-tools-latest-linux.zip [[ -f ${FILE_NAME} ]] && rm -f ${FILE_NAME} - wget https://dl.google.com/android/repository/${FILE_NAME} + wget -e dotbytes=10m https://dl.google.com/android/repository/${FILE_NAME} unzip ${FILE_NAME} > /dev/null fi export PATH=$ROOT/platform-tools/:$PATH @@ -36,22 +32,19 @@ function build_stage2_android() { # Build self-hosted tree with fresh Clang and -Werror. local CMAKE_OPTIONS="${CMAKE_COMMON_OPTIONS} -DLLVM_ENABLE_WERROR=ON ${STAGE1_AS_COMPILER} -DCMAKE_C_FLAGS=-gmlt -DCMAKE_CXX_FLAGS=-gmlt" - CMAKE_OPTIONS="${CMAKE_OPTIONS} -DLLVM_ENABLE_PROJECTS='clang;lld' -DLLVM_USE_LINKER=lld" + CMAKE_OPTIONS="${CMAKE_OPTIONS} -DLLVM_ENABLE_PROJECTS='clang;lld' -DLLVM_USE_LINKER=lld -DCLANG_DEFAULT_RTLIB=libgcc" if ccache -s ; then CMAKE_OPTIONS="${CMAKE_OPTIONS} -DLLVM_CCACHE_BUILD=ON" - rm_dirs llvm_build64 fi echo @@@BUILD_STEP bootstrap clang@@@ - rm_dirs ${STAGE2_CLOBBER} - mkdir -p llvm_build64 if [[ "$(cat llvm_build64/CMAKE_OPTIONS)" != "${CMAKE_OPTIONS}" ]] ; then (cd llvm_build64 && cmake ${CMAKE_OPTIONS} -DLLVM_BUILD_EXTERNAL_COMPILER_RT=ON $LLVM && \ - echo ${CMAKE_OPTIONS} > CMAKE_OPTIONS) || echo @@@STEP_FAILURE@@@ + echo ${CMAKE_OPTIONS} > CMAKE_OPTIONS) || build_failure fi - ninja -C llvm_build64 || (echo @@@STEP_FAILURE@@@ && exit 2) + ninja -C llvm_build64 || (build_failure && exit 2) } function configure_android { # ARCH triple @@ -63,7 +56,7 @@ local ANDROID_LIBRARY_OUTPUT_DIR=$(ls -d $ROOT/llvm_build64/lib/clang/* | tail -1) local ANDROID_EXEC_OUTPUT_DIR=$ROOT/llvm_build64/bin - local ANDROID_FLAGS="--target=${_triple}${ANDROID_API} --sysroot=$ANDROID_TOOLCHAIN/sysroot -gcc-toolchain $ANDROID_TOOLCHAIN -B$ANDROID_TOOLCHAIN" + local ANDROID_FLAGS="--target=${_triple}${ANDROID_API} --sysroot=$ANDROID_TOOLCHAIN/sysroot --gcc-toolchain=$ANDROID_TOOLCHAIN -B$ANDROID_TOOLCHAIN" local ANDROID_CXX_FLAGS="$ANDROID_FLAGS -stdlib=libc++ -I/$ANDROID_TOOLCHAIN/sysroot/usr/include/c++/v1" local CLANG_PATH=$ROOT/llvm_build64/bin/clang @@ -77,6 +70,7 @@ mkdir -p llvm_build_android_$_arch (cd llvm_build_android_$_arch && cmake \ + ${CMAKE_COMMON_OPTIONS} \ -DLLVM_ENABLE_WERROR=OFF \ -DCMAKE_C_COMPILER=$CLANG_PATH \ -DCMAKE_CXX_COMPILER=$CLANGXX_PATH \ @@ -88,12 +82,12 @@ -DCMAKE_SKIP_RPATH=ON \ -DLLVM_BUILD_RUNTIME=OFF \ -DLLVM_TABLEGEN=$ROOT/llvm_build64/bin/llvm-tblgen \ - ${CMAKE_COMMON_OPTIONS} \ - $LLVM || echo @@@STEP_FAILURE@@@) & + $LLVM || build_failure) & local COMPILER_RT_OPTIONS="$(readlink -f $LLVM/../compiler-rt)" (cd compiler_rt_build_android_$_arch && cmake \ + ${CMAKE_COMMON_OPTIONS} \ -DCMAKE_C_COMPILER=$CLANG_PATH \ -DCMAKE_CXX_COMPILER=$CLANGXX_PATH \ -DLLVM_CONFIG_PATH=$ROOT/llvm_build64/bin/llvm-config \ @@ -105,12 +99,11 @@ -DCMAKE_CXX_FLAGS="$ANDROID_CXX_FLAGS" \ -DSANITIZER_CXX_ABI="libcxxabi" \ -DCOMPILER_RT_TEST_COMPILER_CFLAGS="$ANDROID_FLAGS" \ - -DCOMPILER_RT_TEST_TARGET_TRIPLE=$_triple \ + -DCOMPILER_RT_DEFAULT_TARGET_TRIPLE=$_triple \ -DCOMPILER_RT_OUTPUT_DIR="$ANDROID_LIBRARY_OUTPUT_DIR" \ -DCOMPILER_RT_EXEC_OUTPUT_DIR="$ANDROID_EXEC_OUTPUT_DIR" \ -DLLVM_LIT_ARGS="-vv --show-unsupported --show-xfail" \ - ${CMAKE_COMMON_OPTIONS} \ - ${COMPILER_RT_OPTIONS} || echo @@@STEP_FAILURE@@@) & + ${COMPILER_RT_OPTIONS} || build_failure) & } BUILD_RT_ERR="" @@ -120,22 +113,26 @@ echo @@@BUILD_STEP build android/$_arch@@@ if ! ninja -C llvm_build_android_$_arch llvm-symbolizer ; then BUILD_RT_ERR="${BUILD_RT_ERR}|${_arch}|" - echo @@@STEP_FAILURE@@@ + build_failure fi if ! ninja -C compiler_rt_build_android_$_arch ; then BUILD_RT_ERR="${BUILD_RT_ERR}|${_arch}|" - echo @@@STEP_FAILURE@@@ + build_failure fi } -# If a multiarch device has x86 as the first arch, remove everything else from -# the list. This captures cases like [x86,armeabi-v7a], where the arm part is -# software emulation and incompatible with ASan. +# If a multiarch device's first arch starts with x86, remove archs that don't +# start with x86. Handle cases like: +# - x86,armeabi-v7a +# - x86_64,x86,arm64-v8a,armeabi-v7a,armeabi +# - x86_64,arm64-v8a,x86,armeabi-v7a,armeabi +# The arm part is software emulation and incompatible with ASan. function patch_abilist { # IN OUT local _abilist=$1 local _out=$2 - if [[ "$_abilist" == "x86,"* ]]; then - _abilist="x86" + if [[ "$_abilist" == "x86"* ]]; then + _abilist=$(echo $_abilist | tr , '\n' | grep '^x86') + _abilist=$(echo $_abilist | tr ' ' ,) fi eval $_out="'$_abilist'" } @@ -154,19 +151,25 @@ shift ABILIST=$(${ADB} -s $_serial shell getprop ro.product.cpu.abilist) - patch_abilist $ABILIST ABILIST + patch_abilist "$ABILIST" ABILIST for _arg in "$@"; do local _arch=${_arg%:*} local _abi=${_arg#*:} if [[ $BUILD_RT_ERR == *"|${_arch}|"* ]]; then - echo "@@@STEP_FAILURE@@ skipping tests on ${_arch}" + echo "skipping tests on ${_arch}" + build_failure continue fi if [[ $ABILIST == *"$_abi"* ]]; then echo "$_serial" >> tested_arch_$_arch BUILD_ID=$(${ADB} -s $_serial shell getprop ro.build.id | tr -d '\r') BUILD_FLAVOR=$(${ADB} -s $_serial shell getprop ro.build.flavor | tr -d '\r') - test_arch_on_device "$_arch" "$_serial" "$BUILD_ID" "$BUILD_FLAVOR" + ( + # Test only one arch at a time to avoid simultaneously writes into the + # same compiler-rt build dir. + flock -x $lock_fd + test_arch_on_device "$_arch" "$_serial" "$BUILD_ID" "$BUILD_FLAVOR" + ) {lock_fd}>$ROOT/${_arch}.lock fi done } @@ -181,7 +184,7 @@ } function test_android { - if [[ "${BUILDBOT_SLAVENAME:-}" != "" ]]; then + if [[ -v BUILDBOT_BUILDERNAME ]]; then restart_adb_server fi @@ -192,13 +195,12 @@ rm -rf test_android_log_* rm -rf tested_arch_* rm -rf shards_* - LOGS= for SERIAL in $ANDROID_DEVICES; do - LOG="$(mktemp test_android_log_XXXX)" - (test_on_device "$SERIAL" $@ 2>&1 >"$LOG") & - LOGS="$LOGS $LOG,$!" + test_on_device "$SERIAL" $@ done - tail_pids "$LOGS" + + # Return to avoid exception if we already have error. + [[ $BUILD_RT_ERR == "" ]] || return for _arg in "$@"; do local _arch=${_arg%:*} @@ -227,8 +229,9 @@ LOGS= for ((SHARD=0; SHARD < $NUM_SHARDS; SHARD++)); do LOG=${_log_prefix}_$SHARD - local ENV="$_env GTEST_TOTAL_SHARDS=$NUM_SHARDS GTEST_SHARD_INDEX=$SHARD LD_LIBRARY_PATH=$DEVICE_ROOT" - ( (run_command_on_device "$ENV $DEVICE_ROOT/$_test" || echo @@@STEP_FAILURE@@@) 2>&1 >${_log_prefix}_$SHARD ) & + # 'adb shell ' on Fugu is missing TMPDIR. + local ENV="$_env GTEST_TOTAL_SHARDS=$NUM_SHARDS GTEST_SHARD_INDEX=$SHARD LD_LIBRARY_PATH=$DEVICE_ROOT TMPDIR=$DEVICE_TMPDIR" + ( (run_command_on_device "$ENV $DEVICE_ROOT/$_test" || build_failure) 2>&1 >${_log_prefix}_$SHARD ) & LOGS="$LOGS $LOG,$!" done tail_pids "$LOGS" || true @@ -250,6 +253,7 @@ local RT_DIR=$($ROOT/llvm_build64/bin/clang -print-resource-dir)/lib/linux local COMPILER_RT_BUILD_DIR=$ROOT/compiler_rt_build_android_$_arch export ADB=adb + export DEVICE_TMPDIR=/data/local/tmp export DEVICE_ROOT=/data/local/tmp/Output export ANDROID_SERIAL=$_serial echo "Serial $_serial" @@ -272,12 +276,12 @@ $COMPILER_RT_BUILD_DIR/lib/asan/tests/AsanNoinstTest" for F in $FILES ; do - ( $ADB push $F $DEVICE_ROOT/ >/dev/null || echo @@@STEP_FAILURE@@@ )& + ( $ADB push $F $DEVICE_ROOT/ >/dev/null || build_failure )& done wait echo @@@BUILD_STEP run lit tests [$DEVICE_DESCRIPTION]@@@ - (cd $COMPILER_RT_BUILD_DIR && ninja check-all) || echo @@@STEP_FAILURE@@@ + (cd $COMPILER_RT_BUILD_DIR && ninja check-all) || build_failure run_tests_sharded sanitizer_common SanitizerTest "" run_tests_sharded asan AsanNoinstTest "" diff --git a/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_asan.sh b/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_asan.sh --- a/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_asan.sh +++ b/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_asan.sh @@ -12,7 +12,7 @@ export PATH="/usr/local/bin:$PATH" LLVM=$ROOT/llvm -CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON" +CMAKE_COMMON_OPTIONS+=" -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF" clobber @@ -22,6 +22,9 @@ build_stage1_clang +check_stage1_asan + + # Stage 2 / AddressSanitizer build_stage2_asan @@ -35,3 +38,4 @@ check_stage3_asan +cleanup $STAGE1_DIR diff --git a/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_hwasan.sh b/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_hwasan.sh new file mode 100755 --- /dev/null +++ b/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_hwasan.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +set -x +set -e +set -u + +HERE="$(cd $(dirname $0) && pwd)" +. ${HERE}/buildbot_functions.sh + +ROOT=`pwd` +PLATFORM=`uname` +export PATH="/usr/local/bin:$PATH" + +LLVM=$ROOT/llvm +CMAKE_COMMON_OPTIONS+=" -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF" + +clobber + +buildbot_update + +# Stage 1 + +build_stage1_clang + +check_stage1_hwasan + +# Stage 2 / HWAddressSanitizer + +build_stage2_hwasan + +check_stage2_hwasan + +# Stage 3 / HWAddressSanitizer + +build_stage3_hwasan + +check_stage3_hwasan + +cleanup $STAGE1_DIR diff --git a/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_msan.sh b/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_msan.sh --- a/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_msan.sh +++ b/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_msan.sh @@ -12,7 +12,7 @@ export PATH="/usr/local/bin:$PATH" LLVM=$ROOT/llvm -CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON" +CMAKE_COMMON_OPTIONS+=" -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF" clobber @@ -22,14 +22,39 @@ build_stage1_clang +check_stage1_msan + # Stage 2 / Memory Sanitizer build_stage2_msan -check_stage2_msan +check_stage2_msan |& tee stage2_msan.log + +if grep "WARNING: MemorySanitizer" stage2_msan.log ; then + # Stage 2 / MemoryWithOriginsSanitizer + ( + build_stage2_msan_track_origins + + check_stage2_msan_track_origins + ) +fi # Stage 3 / MemorySanitizer +{ + build_stage3_msan + + check_stage3_msan +} |& tee stage3_msan.log + +if grep "WARNING: MemorySanitizer" stage3_msan.log ; then + # Stage 3 / MemoryWithOriginsSanitizer + ( + build_stage2_msan_track_origins + + build_stage3_msan_track_origins -build_stage3_msan + check_stage3_msan_track_origins + ) +fi -check_stage3_msan +cleanup $STAGE1_DIR diff --git a/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_ubsan.sh b/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_ubsan.sh --- a/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_ubsan.sh +++ b/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_ubsan.sh @@ -12,7 +12,7 @@ export PATH="/usr/local/bin:$PATH" LLVM=$ROOT/llvm -CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON" +CMAKE_COMMON_OPTIONS+=" -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF" clobber @@ -22,6 +22,8 @@ build_stage1_clang +check_stage1_ubsan + # Stage 2 / UndefinedBehaviorSanitizer build_stage2_ubsan @@ -33,3 +35,5 @@ build_stage3_ubsan check_stage3_ubsan + +cleanup $STAGE1_DIR diff --git a/zorg/buildbot/builders/sanitizers/buildbot_cmake.sh b/zorg/buildbot/builders/sanitizers/buildbot_cmake.sh --- a/zorg/buildbot/builders/sanitizers/buildbot_cmake.sh +++ b/zorg/buildbot/builders/sanitizers/buildbot_cmake.sh @@ -13,32 +13,40 @@ export PATH="/usr/local/bin:$PATH" export ANDROID_SDK_HOME=$ROOT/../../.. -# Always clobber bootstrap build trees. -rm -rf compiler_rt_build llvm_build* symbolizer_build* +CMAKE_ARGS="" +for arg in "$@" +do + case $arg in + --CMAKE_ARGS=*) + CMAKE_ARGS="${arg#*=}" + esac +done USE_CCACHE= if ccache -s ; then USE_CCACHE="-DLLVM_CCACHE_BUILD=ON" - rm -rf clang_build fi +# Always clobber bootstrap build trees. +clobber + SUPPORTS_32_BITS=${SUPPORTS_32_BITS:-1} MAKE_JOBS=${MAX_MAKE_JOBS:-$(nproc)} LLVM=$ROOT/llvm ZLIB=$ROOT/zlib -CMAKE_COMMON_OPTIONS="${CMAKE_COMMON_OPTIONS:-} -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_PARALLEL_LINK_JOBS=10" +CMAKE_COMMON_OPTIONS+=" -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_PARALLEL_LINK_JOBS=10 -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF ${CMAKE_ARGS}" ENABLE_LIBCXX_FLAG= if [ "$PLATFORM" == "Darwin" ]; then - CMAKE_COMMON_OPTIONS="${CMAKE_COMMON_OPTIONS} -DPYTHON_EXECUTABLE=/usr/bin/python" + CMAKE_COMMON_OPTIONS+=" -DPYTHON_EXECUTABLE=/usr/bin/python" ENABLE_LIBCXX_FLAG="-DLLVM_ENABLE_LIBCXX=ON" fi if [ -e /usr/include/plugin-api.h ]; then - CMAKE_COMMON_OPTIONS="${CMAKE_COMMON_OPTIONS} -DLLVM_BINUTILS_INCDIR=/usr/include" + CMAKE_COMMON_OPTIONS+=" -DLLVM_BINUTILS_INCDIR=/usr/include" fi -CMAKE_COMMON_OPTIONS="${CMAKE_COMMON_OPTIONS} -DLLVM_ENABLE_PROJECTS='clang;compiler-rt'" +CMAKE_COMMON_OPTIONS+=" -DLLVM_ENABLE_PROJECTS='clang;compiler-rt'" CHECK_LIBCXX=${CHECK_LIBCXX:-1} CHECK_SYMBOLIZER=${CHECK_SYMBOLIZER:-$CHECK_LIBCXX} @@ -107,50 +115,42 @@ ;; esac -PROJECTS="clang;compiler-rt;libcxx;libcxxabi" +PROJECTS="clang;compiler-rt" if [[ "$CHECK_LLD" != "0" ]]; then - PROJECTS="${PROJECTS};lld" + PROJECTS+=";lld" fi -CMAKE_COMMON_OPTIONS="${CMAKE_COMMON_OPTIONS} -DLLVM_ENABLE_PROJECTS='${PROJECTS}'" +CMAKE_COMMON_OPTIONS+=" -DLLVM_ENABLE_PROJECTS='${PROJECTS}' -DLLVM_ENABLE_RUNTIMES=libcxx;libcxxabi" if [[ "$CHECK_CFI" == "1" ]]; then # We need this after https://reviews.llvm.org/D62050 - CMAKE_COMMON_OPTIONS="$CMAKE_COMMON_OPTIONS -DLLVM_BUILD_LLVM_DYLIB=ON" + CMAKE_COMMON_OPTIONS+=" -DLLVM_BUILD_LLVM_DYLIB=ON" fi -CLOBBER="zlib clang_build" -clobber buildbot_update COMPILER_RT=$LLVM/../compiler-rt LIBCXX=$LLVM/../libcxx -echo @@@BUILD_STEP lint@@@ -CHECK_LINT=${COMPILER_RT}/lib/sanitizer_common/scripts/check_lint.sh -${CHECK_LINT} || echo @@@STEP_WARNINGS@@@ - # Use both gcc and just-built Clang/LLD as a host compiler/linker for sanitizer # tests. Assume that self-hosted build tree should compile with -Werror. echo @@@BUILD_STEP build fresh toolchain@@@ -if [ ! -d clang_build ]; then - mkdir clang_build -fi -(cd clang_build && cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo ${CMAKE_COMMON_OPTIONS} ${USE_CCACHE} $LLVM ) || (rm -rf clang_build ; echo @@@STEP_FAILURE@@@) +mkdir -p clang_build +(cd clang_build && cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo ${CMAKE_COMMON_OPTIONS} ${USE_CCACHE} $LLVM ) || (rm -rf clang_build ; build_failure) BOOTSTRAP_BUILD_TARGETS="clang" if [[ "$CHECK_LLD" != "0" ]]; then BOOTSTRAP_BUILD_TARGETS="$BOOTSTRAP_BUILD_TARGETS lld" fi -(cd clang_build && make $BOOTSTRAP_BUILD_TARGETS -j$MAKE_JOBS) || (echo @@@STEP_FAILURE@@@ ; exit 1) || echo @@@STEP_FAILURE@@@ +(cd clang_build && make $BOOTSTRAP_BUILD_TARGETS -j$MAKE_JOBS) || build_failure # If we're building with libcxx, install the headers to clang_build/include. if [ ! -z ${ENABLE_LIBCXX_FLAG} ]; then (cd clang_build && make -C ${LIBCXX} installheaders \ - HEADER_DIR=${PWD}/include) || echo @@@STEP_FAILURE@@@ + HEADER_DIR=${PWD}/include) || build_failure fi -# Do a sanity check on Linux: build and test sanitizers using gcc as a host +# Check on Linux: build and test sanitizers using gcc as a host # compiler. if [ "$PLATFORM" == "Linux" ]; then check_in_gcc() { @@ -158,7 +158,7 @@ SANITIZER=$2 if [ "$CONDITION" == "1" ]; then echo @@@BUILD_STEP check-$SANITIZER in gcc build@@@ - (cd clang_build && make -j$MAKE_JOBS check-$SANITIZER) || echo @@@STEP_FAILURE@@@ + (cd clang_build && make -j$MAKE_JOBS check-$SANITIZER) || build_failure fi } check_in_gcc 1 sanitizer @@ -182,32 +182,30 @@ BUILD_TYPE=Release echo @@@BUILD_STEP bootstrap clang@@@ -if [ ! -d llvm_build64 ]; then - mkdir llvm_build64 -fi +mkdir -p llvm_build64 (cd llvm_build64 && cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE \ ${CMAKE_CLANG_OPTIONS} -DLLVM_BUILD_EXTERNAL_COMPILER_RT=ON \ - ${ENABLE_LIBCXX_FLAG} $LLVM) || echo @@@STEP_FAILURE@@@ + ${ENABLE_LIBCXX_FLAG} $LLVM) || build_failure # First, build only Clang. -(cd llvm_build64 && make -j$MAKE_JOBS clang) || echo @@@STEP_FAILURE@@@ +(cd llvm_build64 && make -j$MAKE_JOBS clang) || build_failure # If needed, install the headers to clang_build/include. if [ ! -z ${ENABLE_LIBCXX_FLAG} ]; then (cd llvm_build64 && make -C ${LIBCXX} installheaders \ - HEADER_DIR=${PWD}/include) || echo @@@STEP_FAILURE@@@ + HEADER_DIR=${PWD}/include) || build_failure fi # Now build everything else. -(cd llvm_build64 && make -j$MAKE_JOBS) || echo @@@STEP_FAILURE@@@ +(cd llvm_build64 && make -j$MAKE_JOBS) || build_failure # Symbolizer dependencies. -(cd llvm_build64 && make -j$MAKE_JOBS llvm-ar llvm-link llvm-tblgen opt) || echo @@@STEP_FAILURE@@@ +(cd llvm_build64 && make -j$MAKE_JOBS llvm-ar llvm-link llvm-tblgen opt) || build_failure if [ "$CHECK_CFI" == "1" ]; then # FIXME: Make these true dependencies of check-cfi-and-supported when # compiler-rt is configured as an external project. - (cd llvm_build64 && make -j$MAKE_JOBS LLVMgold opt sanstats) || echo @@@STEP_FAILURE@@@ + (cd llvm_build64 && make -j$MAKE_JOBS LLVMgold opt sanstats) || build_failure fi check_64bit() { @@ -215,7 +213,7 @@ SANITIZER=$2 if [ "$CONDITION" == "1" ]; then echo @@@BUILD_STEP 64-bit check-$SANITIZER@@@ - (cd llvm_build64 && make -j$MAKE_JOBS check-$SANITIZER) || echo @@@STEP_FAILURE@@@ + (cd llvm_build64 && make -j$MAKE_JOBS check-$SANITIZER) || build_failure fi } @@ -246,34 +244,11 @@ -DCOMPILER_RT_INCLUDE_TESTS=ON \ -DCOMPILER_RT_ENABLE_WERROR=ON \ -DLLVM_CONFIG_PATH=${FRESH_CLANG_PATH}/llvm-config \ - $COMPILER_RT) || echo @@@STEP_FAILURE@@@ -(cd compiler_rt_build && make -j$MAKE_JOBS) || echo @@@STEP_FAILURE@@@ + $COMPILER_RT) || build_failure +(cd compiler_rt_build && make -j$MAKE_JOBS) || build_failure echo @@@BUILD_STEP test standalone compiler-rt@@@ -(cd compiler_rt_build && make -j$MAKE_JOBS check-all) || echo @@@STEP_FAILURE@@@ - -build_symbolizer() { - echo @@@BUILD_STEP build $1-bit symbolizer for $2@@@ - if [ ! -d symbolizer_build$1 ]; then - mkdir symbolizer_build$1 - fi - (cd symbolizer_build$1 && ZLIB_SRC=$ZLIB FLAGS=-m$1 \ - CLANG=${FRESH_CLANG_PATH}/clang \ - bash -eux $COMPILER_RT/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh \ - $(dirname $(find ../$2/ -name libclang_rt.*.a | head -n1)) || echo @@@STEP_FAILURE@@@) -} - -if [ "$CHECK_SYMBOLIZER" == "1" ]; then - echo @@@BUILD_STEP update zlib@@@ - (cd $ZLIB && git pull --rebase) || \ - git clone https://github.com/madler/zlib.git $ZLIB || echo @@@STEP_FAILURE@@@ - - build_symbolizer 32 compiler_rt_build - build_symbolizer 64 compiler_rt_build - - echo @@@BUILD_STEP test standalone compiler-rt with symbolizer@@@ - (cd compiler_rt_build && make -j$MAKE_JOBS check-all) || echo @@@STEP_FAILURE@@@ -fi +(cd compiler_rt_build && make -j$MAKE_JOBS check-all) || build_failure HAVE_NINJA=${HAVE_NINJA:-1} if [ "$PLATFORM" == "Linux" -a $HAVE_NINJA == 1 ]; then @@ -283,16 +258,16 @@ fi CMAKE_NINJA_OPTIONS="${CMAKE_CLANG_OPTIONS} -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -G Ninja" (cd llvm_build_ninja && cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE \ - ${CMAKE_NINJA_OPTIONS} $LLVM) || echo @@@STEP_FAILURE@@@ + ${CMAKE_NINJA_OPTIONS} $LLVM) || build_failure ln -sf llvm_build_ninja/compile_commands.json $LLVM - (cd llvm_build_ninja && ninja) || echo @@@STEP_FAILURE@@@ + (cd llvm_build_ninja && ninja) || build_failure check_ninja() { CONDITION=$1 SANITIZER=$2 if [ "$CONDITION" == "1" ]; then echo @@@BUILD_STEP ninja check-$SANITIZER@@@ - (cd llvm_build_ninja && ninja check-$SANITIZER) || echo @@@STEP_FAILURE@@@ + (cd llvm_build_ninja && ninja check-$SANITIZER) || build_failure fi } @@ -310,29 +285,46 @@ check_ninja $CHECK_UBSAN ubsan-minimal if [ "$CHECK_SYMBOLIZER" == "1" ]; then + build_symbolizer() { + echo @@@BUILD_STEP build $1-bit symbolizer for $2@@@ + if [ ! -d symbolizer_build$1 ]; then + mkdir symbolizer_build$1 + fi + (cd symbolizer_build$1 && ZLIB_SRC=$ZLIB FLAGS=-m$1 \ + CLANG=${FRESH_CLANG_PATH}/clang \ + bash -eux $COMPILER_RT/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh \ + $(dirname $(find ../$2/ -name libclang_rt.*.a | head -n1)) || build_failure) + } + + echo @@@BUILD_STEP update zlib@@@ + (cd $ZLIB && git pull --rebase) || \ + git clone https://github.com/madler/zlib.git $ZLIB || build_failure + build_symbolizer 32 llvm_build_ninja build_symbolizer 64 llvm_build_ninja check_ninja_with_symbolizer() { CONDITION=$1 SANITIZER=$2 - # Disabled, tests are not working yet. - if [ "$CONDITION" == "-1" ]; then - echo @@@BUILD_STEP ninja check-$SANITIZER with symbolizer@@@ - (cd llvm_build_ninja && ninja check-$SANITIZER) || echo @@@STEP_FAILURE@@@ + if [ "$CONDITION" == "1" ]; then + echo @@@BUILD_STEP ninja check-$SANITIZER with internal symbolizer@@@ + (cd llvm_build_ninja && ninja check-$SANITIZER) || build_failure fi } - check_ninja_with_symbolizer 1 sanitizer + # TODO: Replace LIT_FILTER_OUT with lit features. + LIT_FILTER_OUT=":: (max_allocation_size.cpp|Linux/soft_rss_limit_mb_test.cpp)$" check_ninja_with_symbolizer 1 sanitizer check_ninja_with_symbolizer $CHECK_ASAN asan - check_ninja_with_symbolizer $CHECK_HWASAN hwasan - check_ninja_with_symbolizer $CHECK_CFI cfi-and-supported + # check_ninja_with_symbolizer $CHECK_HWASAN hwasan + # check_ninja_with_symbolizer $CHECK_CFI cfi-and-supported check_ninja_with_symbolizer $CHECK_DFSAN dfsan - check_ninja_with_symbolizer $CHECK_LSAN lsan + LIT_FILTER_OUT=":: TestCases/(realloc_too_big.c|recoverable_leak_check.cpp|suppressions_file.cpp)$" check_ninja_with_symbolizer $CHECK_LSAN lsan check_ninja_with_symbolizer $CHECK_MSAN msan check_ninja_with_symbolizer $CHECK_SCUDO scudo check_ninja_with_symbolizer $CHECK_SCUDO_STANDALONE scudo_standalone - check_ninja_with_symbolizer $CHECK_TSAN tsan - check_ninja_with_symbolizer $CHECK_UBSAN ubsan + LIT_FILTER_OUT=":: Linux/check_memcpy.c$" check_ninja_with_symbolizer $CHECK_TSAN tsan + LIT_FILTER_OUT=":: TestCases/TypeCheck/(vptr-virtual-base.cpp|vptr.cpp)$" check_ninja_with_symbolizer $CHECK_UBSAN ubsan fi fi + +cleanup diff --git a/zorg/buildbot/builders/sanitizers/buildbot_fast.sh b/zorg/buildbot/builders/sanitizers/buildbot_fast.sh --- a/zorg/buildbot/builders/sanitizers/buildbot_fast.sh +++ b/zorg/buildbot/builders/sanitizers/buildbot_fast.sh @@ -12,7 +12,7 @@ export PATH="/usr/local/bin:$PATH" LLVM=$ROOT/llvm -CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON" +CMAKE_COMMON_OPTIONS+=" -GNinja -DCMAKE_BUILD_TYPE=Release" clobber @@ -20,10 +20,16 @@ build_stage1_clang_at_revison +CMAKE_COMMON_OPTIONS+=" -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF" + if ccache -s ; then - CMAKE_COMMON_OPTIONS="${CMAKE_COMMON_OPTIONS} -DLLVM_CCACHE_BUILD=ON" + CMAKE_COMMON_OPTIONS+=" -DLLVM_CCACHE_BUILD=ON" fi +# This bot uses older compiler. +STAGE2_USE_LATE_EP=1 +STAGE2_SKIP_TEST_CXX=1 + buildbot_update # Stage 2 / Memory Sanitizer @@ -43,3 +49,5 @@ build_stage2_ubsan check_stage2_ubsan + +cleanup diff --git a/zorg/buildbot/builders/sanitizers/buildbot_functions.sh b/zorg/buildbot/builders/sanitizers/buildbot_functions.sh --- a/zorg/buildbot/builders/sanitizers/buildbot_functions.sh +++ b/zorg/buildbot/builders/sanitizers/buildbot_functions.sh @@ -1,5 +1,20 @@ #!/usr/bin/env bash +function include_config() { + local P=. + while true ; do + local F=${P}/sanitizer_buildbot_config + if [[ -f ${F} ]] ; then + . ${F} + break + fi + [[ "${P}" -ef '/' ]] && break + P="${P}/.." + done +} + +include_config + echo @@@BUILD_STEP Info@@@ ( set +e @@ -7,36 +22,45 @@ echo uptime echo - ulimit -n 100000 + ulimit -n 1000000 ulimit -a echo df -h echo ccache -s - echo - echo "How to reproduce locally: https://github.com/google/sanitizers/wiki/SanitizerBotReproduceBuild" - echo exit 0 ) echo @@@BUILD_STEP Prepare@@@ BUILDBOT_CLOBBER="${BUILDBOT_CLOBBER:-}" -BUILDBOT_REVISION="${BUILDBOT_REVISION:-origin/master}" +BUILDBOT_REVISION="${BUILDBOT_REVISION:-origin/main}" + +export LIT_OPTS="--time-tests" function rm_dirs { while ! rm -rf $@ ; do sleep 1; done } -function stage1_clobber { - rm_dirs llvm_build2_* llvm_build_* libcxx_build_* ${STAGE1_CLOBBER:-} +function cleanup() { + [[ -v BUILDBOT_BUILDERNAME ]] || return 0 + echo @@@BUILD_STEP cleanup@@@ + rm_dirs llvm_build2_* llvm_build_* libcxx_build_* compiler_rt_build* symbolizer_build* $@ + if ccache -s >/dev/null ; then + rm_dirs llvm_build64 clang_build + fi + ls } function clobber { - if [ "$BUILDBOT_CLOBBER" != "" ]; then + if [[ "$BUILDBOT_CLOBBER" != "" ]]; then echo @@@BUILD_STEP clobber@@@ - rm_dirs svn_checkout llvm llvm-project llvm_build0 ${CLOBBER:-} - stage1_clobber - ! test "$(ls -A .)" || echo @@@STEP_EXCEPTION@@@ + if [[ ! -v BUILDBOT_BUILDERNAME ]]; then + echo "Clobbering is supported only on buildbot only!" + exit 1 + fi + rm_dirs * + else + BUILDBOT_BUILDERNAME=1 cleanup $@ fi } @@ -54,27 +78,16 @@ cd llvm-project git init git remote add origin https://github.com/llvm/llvm-project.git + git config --local advice.detachedHead false ) cd llvm-project - git fetch --depth $DEPTH origin master + git fetch origin git clean -fd local REV=${BUILDBOT_REVISION} - if [[ "$REV" != "origin/master" ]] ; then - # "git fetch --depth 1 origin $REV" does not work with 2.11 on bots - while true ; do - git checkout $REV && break - git rev-list --pretty --max-count=1 origin/master - git rev-list --pretty --max-parents=0 origin/master - echo "DEPTH=$DEPTH is too small" - [[ "$DEPTH" -le "1000000" ]] || exit 1 - DEPTH=$(( $DEPTH * 10 )) - git fetch --depth $DEPTH origin - done - fi - git checkout $REV + git checkout -f ${BUILDBOT_REVISION} git status git rev-list --pretty --max-count=1 HEAD - ) || { echo @@@STEP_EXCEPTION@@@ ; exit 1 ; } + ) || { build_exception ; exit 1 ; } LLVM=$ROOT/llvm-project/llvm fi } @@ -83,39 +96,59 @@ STAGE1_DIR=llvm_build0 stage1_clang_path=$ROOT/${STAGE1_DIR}/bin llvm_symbolizer_path=${stage1_clang_path}/llvm-symbolizer + export LLVM_SYMBOLIZER_PATH="${llvm_symbolizer_path}" STAGE1_AS_COMPILER="-DCMAKE_C_COMPILER=${stage1_clang_path}/clang -DCMAKE_CXX_COMPILER=${stage1_clang_path}/clang++" } function build_stage1_clang_impl { - rm -rf ${STAGE1_DIR} mkdir -p ${STAGE1_DIR} - local cmake_stage1_options="${CMAKE_COMMON_OPTIONS}" - cmake_stage1_options="${cmake_stage1_options} -DLLVM_ENABLE_PROJECTS='clang;compiler-rt;lld'" + local cmake_stage1_options="${CMAKE_COMMON_OPTIONS} -DLLVM_ENABLE_PROJECTS='clang;compiler-rt;lld'" + if clang -v ; then + cmake_stage1_options+=" -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++" + fi if ccache -s ; then - cmake_stage1_options="${cmake_stage1_options} -DLLVM_CCACHE_BUILD=ON" + cmake_stage1_options+=" -DLLVM_CCACHE_BUILD=ON" fi (cd ${STAGE1_DIR} && cmake ${cmake_stage1_options} $LLVM && ninja) } function build_stage1_clang { echo @@@BUILD_STEP stage1 build all@@@ - export STAGE1_DIR=llvm_build0 common_stage1_variables build_stage1_clang_impl - - echo @@@BUILD_STEP Clobber stage1 users - stage1_clobber } -function build_stage1_clang_at_revison { +function download_clang_from_chromium { common_stage1_variables - curl -s https://raw.githubusercontent.com/chromium/chromium/master/tools/clang/scripts/update.py \ - | python - --output-dir=${STAGE1_DIR} + curl -s https://raw.githubusercontent.com/chromium/chromium/main/tools/clang/scripts/update.py \ + | python3 - --output-dir=${STAGE1_DIR} echo @@@BUILD_STEP using pre-built stage1 clang at $(cat ${STAGE1_DIR}/cr_build_revision)@@@ } +function build_clang_at_release_tag { + local HOST_CLANG_REVISION=llvmorg-$(curl https://api.github.com/repos/llvm/llvm-project/releases/latest -s | jq .name -r | cut -f2 -d' ') + common_stage1_variables + + if [ -r ${STAGE1_DIR}/host_clang_revision ] && \ + [ "$(cat ${STAGE1_DIR}/host_clang_revision)" == $HOST_CLANG_REVISION ] + then + echo @@@BUILD_STEP using pre-built stage1 clang at r$HOST_CLANG_REVISION@@@ + else + BUILDBOT_MONO_REPO_PATH= BUILDBOT_REVISION=$HOST_CLANG_REVISION buildbot_update + + rm -rf ${STAGE1_DIR} + echo @@@BUILD_STEP build stage1 clang at $HOST_CLANG_REVISION@@@ + build_stage1_clang_impl && \ + ( echo $HOST_CLANG_REVISION > ${STAGE1_DIR}/host_clang_revision ) + fi +} + +function build_stage1_clang_at_revison { + build_clang_at_release_tag +} + function common_stage2_variables { cmake_stage2_common_options="\ ${CMAKE_COMMON_OPTIONS} ${STAGE1_AS_COMPILER} -DLLVM_USE_LINKER=lld" @@ -123,62 +156,90 @@ function build_stage2 { local sanitizer_name=$1 - local step_result=$2 + echo @@@BUILD_STEP stage2/$sanitizer_name build libcxx@@@ + local libcxx_build_dir=libcxx_build_${sanitizer_name} local build_dir=llvm_build_${sanitizer_name} export STAGE2_DIR=${build_dir} + local build_type="Release" + local cmake_libcxx_cflags= common_stage2_variables + local fno_sanitize_flag= + if [ "$sanitizer_name" == "msan" ]; then export MSAN_SYMBOLIZER_PATH="${llvm_symbolizer_path}" - local llvm_use_sanitizer="Memory" - local fsanitize_flag="-fsanitize=memory" - BUILDBOT_MSAN_ORIGINS=${BUILDBOT_MSAN_ORIGINS:-} - if [ "$BUILDBOT_MSAN_ORIGINS" != "" ]; then - llvm_use_sanitizer="MemoryWithOrigins" - fi - local build_type="Release" + llvm_use_sanitizer="Memory" + fsanitize_flag="-fsanitize=memory -fsanitize-memory-use-after-dtor -fsanitize-memory-param-retval" + elif [ "$sanitizer_name" == "msan_track_origins" ]; then + export MSAN_SYMBOLIZER_PATH="${llvm_symbolizer_path}" + llvm_use_sanitizer="MemoryWithOrigins" + fsanitize_flag="-fsanitize=memory -fsanitize-memory-track-origins -fsanitize-memory-use-after-dtor -fsanitize-memory-param-retval" elif [ "$sanitizer_name" == "asan" ]; then export ASAN_SYMBOLIZER_PATH="${llvm_symbolizer_path}" - export ASAN_OPTIONS="check_initialization_order=true:detect_stack_use_after_return=1:detect_leaks=1" - local llvm_use_sanitizer="Address" - local fsanitize_flag="-fsanitize=address" - local build_type="Release" + export ASAN_OPTIONS="check_initialization_order=true:detect_stack_use_after_return=1" + if [[ "$(arch)" == "aarch64" ]] ; then + # FIXME: https://github.com/google/sanitizers/issues/703 + ASAN_OPTIONS+=":detect_leaks=0" + else + ASAN_OPTIONS+=":detect_leaks=1" + fi + llvm_use_sanitizer="Address" + fsanitize_flag="-fsanitize=address" + # FIXME: False ODR violations in libcxx tests. + # https://github.com/google/sanitizers/issues/1017 + cmake_libcxx_cflags="-mllvm -asan-use-private-alias=1" + elif [ "$sanitizer_name" == "hwasan" ]; then + export HWASAN_SYMBOLIZER_PATH="${llvm_symbolizer_path}" + export HWASAN_OPTIONS="abort_on_error=1" + llvm_use_sanitizer="HWAddress" + fsanitize_flag="-fsanitize=hwaddress -mllvm -hwasan-use-after-scope=1" + # FIXME: Support globals with DSO https://github.com/llvm/llvm-project/issues/57206 + cmake_stage2_common_options+=" -DLLVM_ENABLE_PLUGINS=OFF" elif [ "$sanitizer_name" == "ubsan" ]; then export UBSAN_OPTIONS="external_symbolizer_path=${llvm_symbolizer_path}:print_stacktrace=1" - local llvm_use_sanitizer="Undefined" - local fsanitize_flag="-fsanitize=undefined" - local build_type="Release" + llvm_use_sanitizer="Undefined" + fsanitize_flag="-fsanitize=undefined" + # FIXME: After switching to LLVM_ENABLE_RUNTIMES, vptr has infitine + # recursion. + fno_sanitize_flag+=" -fno-sanitize=vptr" else echo "Unknown sanitizer!" exit 1 fi - # Don't use libc++/libc++abi in UBSan builds (due to known bugs). - echo @@@BUILD_STEP stage2/$sanitizer_name build libcxx@@@ - rm -rf ${libcxx_build_dir} + if [[ "${STAGE2_USE_LATE_EP:-}" != "1" ]] ; then + fsanitize_flag+=" -mllvm -sanitizer-early-opt-ep" + fi + mkdir -p ${libcxx_build_dir} (cd ${libcxx_build_dir} && \ cmake \ ${cmake_stage2_common_options} \ - -DLLVM_ENABLE_PROJECTS='libcxx;libcxxabi' \ + -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' \ -DCMAKE_BUILD_TYPE=${build_type} \ -DLLVM_USE_SANITIZER=${llvm_use_sanitizer} \ - $LLVM && \ - ninja cxx cxxabi) || echo $step_result - local sanitizer_ldflags="-lc++abi -Wl,--rpath=${ROOT}/${libcxx_build_dir}/lib -L${ROOT}/${libcxx_build_dir}/lib" - local sanitizer_cflags="-nostdinc++ -isystem ${ROOT}/${libcxx_build_dir}/include -isystem ${ROOT}/${libcxx_build_dir}/include/c++/v1" + -DCMAKE_C_FLAGS="${fsanitize_flag} ${cmake_libcxx_cflags} ${fno_sanitize_flag}" \ + -DCMAKE_CXX_FLAGS="${fsanitize_flag} ${cmake_libcxx_cflags} ${fno_sanitize_flag}" \ + $LLVM/../runtimes && \ + ninja cxx cxxabi) || build_failure + + local libcxx_runtime_path=$(dirname $(find ${ROOT}/${libcxx_build_dir} -name libc++.so)) + local sanitizer_ldflags="-Wl,--rpath=${libcxx_runtime_path} -L${libcxx_runtime_path}" + local sanitizer_cflags="-nostdinc++ -isystem ${ROOT}/${libcxx_build_dir}/include -isystem ${ROOT}/${libcxx_build_dir}/include/c++/v1 $fsanitize_flag" echo @@@BUILD_STEP stage2/$sanitizer_name build@@@ # See http://llvm.org/bugs/show_bug.cgi?id=19071, http://www.cmake.org/Bug/view.php?id=15264 - local cmake_bug_workaround_cflags="$sanitizer_ldflags $fsanitize_flag -w" - sanitizer_cflags="$sanitizer_cflags $cmake_bug_workaround_cflags" + sanitizer_cflags+=" $sanitizer_ldflags -w" - rm -rf ${build_dir} mkdir -p ${build_dir} - local cmake_stage2_clang_options="-DLLVM_ENABLE_PROJECTS='clang;lld;clang-tools-extra'" + local cmake_stage2_clang_options="-DLLVM_ENABLE_PROJECTS='clang;lld;clang-tools-extra;mlir'" + if [[ "$(arch)" == "aarch64" ]] ; then + # FIXME: clangd tests fail. + cmake_stage2_clang_options="-DLLVM_ENABLE_PROJECTS='clang;lld;mlir'" + fi (cd ${build_dir} && \ cmake \ ${cmake_stage2_common_options} \ @@ -190,91 +251,224 @@ -DCMAKE_CXX_FLAGS="${sanitizer_cflags}" \ -DCMAKE_EXE_LINKER_FLAGS="${sanitizer_ldflags}" \ $LLVM && \ - ninja) || echo $step_result + ninja) || build_failure } function build_stage2_msan { - build_stage2 msan @@@STEP_FAILURE@@@ + build_stage2 msan +} + +function build_stage2_msan_track_origins { + build_stage2 msan_track_origins } function build_stage2_asan { - build_stage2 asan @@@STEP_FAILURE@@@ + build_stage2 asan +} + +function build_stage2_hwasan { + build_stage2 hwasan } function build_stage2_ubsan { - build_stage2 ubsan @@@STEP_FAILURE@@@ + build_stage2 ubsan +} + +function check_stage1 { + local sanitizer_name=$1 + + echo @@@BUILD_STEP stage1/$sanitizer_name check-sanitizer@@@ + ninja -C ${STAGE1_DIR} check-sanitizer || build_failure + + echo @@@BUILD_STEP stage1/$sanitizer_name check-${sanitizer_name}@@@ + ninja -C ${STAGE1_DIR} check-${sanitizer_name} || build_failure +} + +function check_stage1_msan { + check_stage1 msan +} + +function check_stage1_asan { + check_stage1 asan +} + +function check_stage1_hwasan { + check_stage1 hwasan +} + +function check_stage1_ubsan { + check_stage1 ubsan } function check_stage2 { local sanitizer_name=$1 - local step_result=$2 - local build_dir=${STAGE2_DIR} - + + if [[ "${STAGE2_SKIP_TEST_CXX:-}" != "1" ]] ; then + ( + # Very slow, run in background. + LIT_OPTS+=" --timeout=1500" + ( + # Very slow. + export LIT_FILTER_OUT="modules_include.sh.cpp" + + LIT_FILTER_OUT+="|std/algorithms/alg.modifying.operations/alg.transform/ranges.transform.pass.cpp" + LIT_FILTER_OUT+="|std/containers/sequences/deque/deque.modifiers/insert_iter_iter.pass.cpp" + LIT_FILTER_OUT+="|std/numerics/rand/rand.dist/rand.dist.bern/rand.dist.bern.negbin/eval.pass.cpp" + LIT_FILTER_OUT+="|std/numerics/rand/rand.dist/rand.dist.samp/rand.dist.samp.discrete/eval.pass.cpp" + LIT_FILTER_OUT+="|std/utilities/charconv/charconv.msvc/test.pass.cpp" + LIT_FILTER_OUT+="|std/utilities/format/format.functions/format_to_n.locale.pass.cpp" + LIT_FILTER_OUT+="|std/utilities/format/format.functions/format_to_n.pass.cpp" + LIT_FILTER_OUT+="|std/utilities/format/format.functions/format_to.locale.pass.cpp" + LIT_FILTER_OUT+="|std/utilities/format/format.functions/format_to.pass.cpp" + LIT_FILTER_OUT+="|std/utilities/format/format.functions/format.locale.pass.cpp" + LIT_FILTER_OUT+="|std/utilities/format/format.functions/format.pass.cpp" + LIT_FILTER_OUT+="|std/utilities/format/format.functions/formatted_size.locale.pass.cpp" + LIT_FILTER_OUT+="|std/utilities/format/format.functions/formatted_size.pass.cpp" + LIT_FILTER_OUT+="|std/utilities/format/format.functions/vformat" + LIT_FILTER_OUT+="|std/utilities/variant/variant.visit/visit_return_type.pass.cpp" + LIT_FILTER_OUT+="|std/utilities/variant/variant.visit/visit.pass.cpp" + + ninja -C libcxx_build_${sanitizer_name} check-cxx + ) || build_failure + ) &>check_cxx.log & + + echo @@@BUILD_STEP stage2/$sanitizer_name check-cxxabi@@@ + ( + LIT_OPTS+=" --timeout=1100" + if [[ "$(arch)" == "aarch64" && "$sanitizer_name" == "asan" ]] ; then + # TODO: Investigate one leak and two slowest tests. + export LIT_FILTER_OUT="test_vector2.pass.cpp|catch_multi_level_pointer.pass.cpp|guard_threaded_test.pass.cpp" + fi + if [[ "$(arch)" == "aarch64" && "$sanitizer_name" == "msan" ]] ; then + # TODO: Investigate one slow tests. + export LIT_FILTER_OUT="catch_multi_level_pointer.pass.cpp|guard_threaded_test.pass.cpp|test_demangle.pass.cpp" + fi + if [[ "$(arch)" == "aarch64" && "$sanitizer_name" == "hwasan" ]] ; then + # TODO: Investigate one slow tests. + export LIT_FILTER_OUT="catch_multi_level_pointer.pass.cpp|guard_threaded_test.pass.cpp|test_demangle.pass.cpp" + fi + ninja -C libcxx_build_${sanitizer_name} check-cxxabi + ) || build_failure + fi + echo @@@BUILD_STEP stage2/$sanitizer_name check@@@ - ninja -C ${build_dir} check-all || echo $step_result + ninja -C ${STAGE2_DIR} check-all || build_failure + + if [[ "${STAGE2_SKIP_TEST_CXX:-}" != "1" ]] ; then + echo @@@BUILD_STEP stage2/$sanitizer_name check-cxx@@@ + wait + sleep 5 + cat check_cxx.log + fi } function check_stage2_msan { - check_stage2 msan @@@STEP_FAILURE@@@ + check_stage2 msan +} + +function check_stage2_msan_track_origins { + check_stage2 msan_track_origins } function check_stage2_asan { - check_stage2 asan @@@STEP_FAILURE@@@ + check_stage2 asan +} + +function check_stage2_hwasan { + check_stage2 hwasan } function check_stage2_ubsan { - check_stage2 ubsan @@@STEP_FAILURE@@@ + check_stage2 ubsan } function build_stage3 { local sanitizer_name=$1 - local step_result=$2 + echo @@@BUILD_STEP build stage3/$sanitizer_name build@@@ + local build_dir=llvm_build2_${sanitizer_name} local clang_path=$ROOT/${STAGE2_DIR}/bin - echo @@@BUILD_STEP build stage3/$sanitizer_name build@@@ - rm -rf ${build_dir} mkdir -p ${build_dir} + local stage3_projects='clang;lld;clang-tools-extra' + if [[ "$(arch)" == "aarch64" ]] ; then + # FIXME: clangd tests fail. + stage3_projects='clang;lld' + fi (cd ${build_dir} && \ cmake \ ${CMAKE_COMMON_OPTIONS} \ - -DLLVM_ENABLE_PROJECTS='clang;lld,clang-tools-extra' \ + -DLLVM_ENABLE_PROJECTS="${stage3_projects}" \ -DCMAKE_C_COMPILER=${clang_path}/clang \ -DCMAKE_CXX_COMPILER=${clang_path}/clang++ \ -DLLVM_USE_LINKER=lld \ $LLVM && \ - ninja clang) || echo $step_result + ninja clang) || build_failure } function build_stage3_msan { - build_stage3 msan @@@STEP_FAILURE@@@ + build_stage3 msan +} + +function build_stage3_msan_track_origins { + build_stage3 msan_track_origins } function build_stage3_asan { - build_stage3 asan @@@STEP_FAILURE@@@ + build_stage3 asan +} + +function build_stage3_hwasan { + build_stage3 hwasan } function build_stage3_ubsan { - build_stage3 ubsan @@@STEP_FAILURE@@@ + build_stage3 ubsan } function check_stage3 { local sanitizer_name=$1 - local step_result=$2 + echo @@@BUILD_STEP stage3/$sanitizer_name check@@@ + local build_dir=llvm_build2_${sanitizer_name} - echo @@@BUILD_STEP stage3/$sanitizer_name check@@@ - (cd ${build_dir} && ninja check-all) || echo $step_result + (cd ${build_dir} && env && ninja check-all) || build_failure } function check_stage3_msan { - check_stage3 msan @@@STEP_FAILURE@@@ + check_stage3 msan +} + +function check_stage3_msan_track_origins { + check_stage3 msan_track_origins } function check_stage3_asan { - check_stage3 asan @@@STEP_FAILURE@@@ + check_stage3 asan +} + +function check_stage3_hwasan { + check_stage3 hwasan } function check_stage3_ubsan { - check_stage3 ubsan @@@STEP_FAILURE@@@ + check_stage3 ubsan +} + +function build_failure() { + echo + echo "How to reproduce locally: https://github.com/google/sanitizers/wiki/SanitizerBotReproduceBuild" + echo + + sleep 5 + echo "@@@STEP_FAILURE@@@" +} + +function build_exception() { + echo + echo "How to reproduce locally: https://github.com/google/sanitizers/wiki/SanitizerBotReproduceBuild" + echo + + sleep 5 + echo "@@@STEP_EXCEPTION@@@" } diff --git a/zorg/buildbot/builders/sanitizers/buildbot_fuzzer.sh b/zorg/buildbot/builders/sanitizers/buildbot_fuzzer.sh --- a/zorg/buildbot/builders/sanitizers/buildbot_fuzzer.sh +++ b/zorg/buildbot/builders/sanitizers/buildbot_fuzzer.sh @@ -16,11 +16,9 @@ LIBFUZZER=$LLVM/lib/Fuzzer # No assertions. Need to clean up the existing assertion failures first. # Also, the Fuzzer does not provide reproducers on assertion failures yet. -CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=OFF -DLLVM_PARALLEL_LINK_JOBS=8 -DLIBFUZZER_ENABLE_TESTS=ON" +CMAKE_COMMON_OPTIONS+=" -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=OFF" -CLOBBER=fuzzer-test-suite -STAGE1_CLOBBER="RUNDIR-* $LIBFUZZER" -clobber +clobber RUNDIR-* # Make sure asan intercepts SIGABRT so that the fuzzer can print the test cases # for assertion failures. @@ -37,7 +35,7 @@ echo @@@BUILD_STEP check-fuzzer@@@ -(cd ${STAGE1_DIR} && ninja check-fuzzer) || echo @@@STEP_FAILURE@@@ +(cd ${STAGE1_DIR} && ninja check-fuzzer) || build_failure echo @@@BUILD_STEP get fuzzer-test-suite @@@ [ ! -e fuzzer-test-suite ] && git clone https://github.com/google/fuzzer-test-suite.git @@ -55,12 +53,14 @@ export JOBS=${MAKE_JOBS} -RunFuzzerTest libxml2-v2.9.2 || echo @@@STEP_FAILURE@@@ -RunFuzzerTest libpng-1.2.56 || echo @@@STEP_FAILURE@@@ -RunFuzzerTest libssh-2017-1272 || echo @@@STEP_FAILURE@@@ -RunFuzzerTest re2-2014-12-09 || echo @@@STEP_FAILURE@@@ -RunFuzzerTest c-ares-CVE-2016-5180 || echo @@@STEP_FAILURE@@@ -RunFuzzerTest openssl-1.0.1f || echo @@@STEP_FAILURE@@@ -RunFuzzerTest openssl-1.0.2d || echo @@@STEP_FAILURE@@@ -RunFuzzerTest proj4-2017-08-14 || echo @@@STEP_FAILURE@@@ -#RunFuzzerTest woff2-2016-05-06 || echo @@@STEP_WARNINGS@@@ # Often can't find the bug in the given time. +RunFuzzerTest libxml2-v2.9.2 || build_failure +RunFuzzerTest libpng-1.2.56 || build_failure +RunFuzzerTest libssh-2017-1272 || build_failure +RunFuzzerTest re2-2014-12-09 || build_failure +RunFuzzerTest c-ares-CVE-2016-5180 || build_failure +RunFuzzerTest openssl-1.0.1f || build_failure +RunFuzzerTest openssl-1.0.2d || build_failure +RunFuzzerTest proj4-2017-08-14 || build_failure +#RunFuzzerTest woff2-2016-05-06 || build_failure # Often can't find the bug in the given time. + +cleanup $STAGE1_DIR RUNDIR-* diff --git a/zorg/buildbot/builders/sanitizers/buildbot_qemu.sh b/zorg/buildbot/builders/sanitizers/buildbot_qemu.sh new file mode 100755 --- /dev/null +++ b/zorg/buildbot/builders/sanitizers/buildbot_qemu.sh @@ -0,0 +1,406 @@ +#!/usr/bin/env bash + +set -x +set -e +set -u + +HERE="$(cd $(dirname $0) && pwd)" +. ${HERE}/buildbot_functions.sh + +ROOT=`pwd` +PLATFORM=`uname` +export PATH="/usr/local/bin:$PATH" + +LLVM=$ROOT/llvm +CMAKE_COMMON_OPTIONS+=" -GNinja -DCMAKE_BUILD_TYPE=Release" + +clobber + +build_stage1_clang_at_revison +CMAKE_COMMON_OPTIONS+=" -DLLVM_ENABLE_ASSERTIONS=ON" + +buildbot_update + +readonly STAGE2_DIR=llvm_build2_host +( + echo @@@BUILD_STEP build host clang@@@ + COMPILER_BIN_DIR="$(readlink -f ${STAGE1_DIR})/bin" + if ccache -s >/dev/null ; then + CMAKE_COMMON_OPTIONS+=" -DLLVM_CCACHE_BUILD=ON" + fi + + rm -rf ${STAGE2_DIR} + mkdir -p ${STAGE2_DIR} + cd ${STAGE2_DIR} + + cmake \ + ${CMAKE_COMMON_OPTIONS} \ + -DLLVM_ENABLE_PROJECTS="clang;compiler-rt;lld" \ + -DCMAKE_C_COMPILER=${COMPILER_BIN_DIR}/clang \ + -DCMAKE_CXX_COMPILER=${COMPILER_BIN_DIR}/clang++ \ + -DLLVM_ENABLE_LLD=ON \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + $LLVM && ninja +) || build_failure + +set +x # Avoid echoing STEP_FAILURE because of the command trace. +MISSING_QEMU_IMAGE_MESSAGE=$(cat << 'EOF' +===================================================================== +Looks like you're missing the QEMU system images for x86_64 LAM +HWASan testing. These system images aren't automatically generated +as part of the buildbot script because they require root (sorry!). + +If you have the system images already built, you can run +buildbot_qemu.sh with QEMU_IMAGE_DIR=path/to/qemu/images. Otherwise, +you can build the images (only necessary once-per-clobber) by: + 1. WARNING: Read the script at https://github.com/google/sanitizers/blob/master/hwaddress-sanitizer/create_qemu_image.sh + You're running this as ROOT, so make sure you're comfortable with + that. + 2. In your terminal, run: + $ sudo su -c "bash <(wget -qO- https://raw.githubusercontent.com/google/sanitizers/master/hwaddress-sanitizer/create_qemu_image.sh)" root && \ + sudo chown $(whoami) debian.* + +You can also choose to skip the x86_64 HWASan LAM testing by supplying +SKIP_HWASAN_LAM=true in your invocation of this script. +===================================================================== +EOF +) +set -x + +SKIP_HWASAN_LAM=${SKIP_HWASAN_LAM:-} + +function setup_lam_qemu_image { + # Full system emulation is required for x86_64 testing with LAM, as some + # sanitizers aren't friendly with usermode emulation under x86_64 LAM. + echo @@@BUILD_STEP Check x86_64 LAM Prerequisites@@@ + + # Allow specifying the QEMU image dir, otherwise assume in the local dir. + QEMU_IMAGE_DIR=${QEMU_IMAGE_DIR:=${ROOT}} + + # Ensure the buildbot start script created a QEMU image for us. + { + QEMU_IMAGE="${QEMU_IMAGE_DIR}/debian.img" + QEMU_SSH_KEY="${QEMU_IMAGE_DIR}/debian.id_rsa" + ls ${QEMU_IMAGE} + ls ${QEMU_SSH_KEY} + } || { + # Make the "missing file" error clearer by not effectively echoing twice. + set +x + echo "$MISSING_QEMU_IMAGE_MESSAGE" + build_exception + exit 2 + } +} + +[[ -z "$SKIP_HWASAN_LAM" ]] && setup_lam_qemu_image || SKIP_HWASAN_LAM=1 + +function git_clone_at_revision { + local src_dir_name="${1}" + local git_url="${2}" + local revision="${3}" + local build_dir="${4}" + ( + cd ${ROOT} + [[ -d ${src_dir_name} ]] || git clone ${git_url} ${src_dir_name} || exit 1 + cd ${ROOT}/${src_dir_name} + git remote set-url origin ${git_url} + git fetch origin + + # Short circuit if we already have a build at the correct revision. + [[ "$(git rev-parse HEAD)" == "$(git rev-parse ${revision})" ]] && exit 0 + + # Delete the build dir to ensure a rebuild happens at the new revision. + rm -rf ${build_dir} + + git reset --hard ${revision} + git submodule update --init --recursive + ) +} + +function build_qemu { + local qemu_name="${1}" + local qemu_url="${2}" + local qemu_revision="${3}" + + local build_dir="${ROOT}/${qemu_name}_build" + + echo "@@@BUILD_STEP build ${qemu_name}@@@" + ( + git_clone_at_revision ${qemu_name} ${qemu_url} ${qemu_revision} \ + ${build_dir} || exit 1 + + ${build_dir}/qemu-x86_64 --version && + ${build_dir}/qemu-system-x86_64 --version && + ${build_dir}/qemu-img --version && + exit 0 + + rm -rf ${build_dir} && + mkdir -p ${build_dir} && + cd ${build_dir} && + ${ROOT}/${qemu_name}/configure --enable-linux-user && + ninja && + ${build_dir}/qemu-x86_64 --version && + ${build_dir}/qemu-system-x86_64 --version && + ${build_dir}/qemu-img --version + ) || ( + build_exception + exit 2 + ) +} + +function build_lam_linux { + echo "@@@BUILD_STEP build lam linux@@@" + local build_dir="${ROOT}/lam_linux_build" + LAM_KERNEL="${build_dir}/arch/x86_64/boot/bzImage" + ( + git_clone_at_revision lam_linux https://github.com/vitalybuka/linux.git \ + origin/lam ${build_dir} || exit 1 + + ls "${LAM_KERNEL}" && exit 0 + + rm -rf ${build_dir} && + mkdir -p ${build_dir} && + cd ${ROOT}/lam_linux && + make O=${build_dir} LD=ld.bfd defconfig && + make O=${build_dir} LD=ld.bfd -j $(nproc) && + ls "${LAM_KERNEL}" + ) || ( + build_exception + exit 2 + ) +} + +build_qemu qemu https://github.com/vitalybuka/qemu.git origin/sanitizer_bot +[[ -z "$SKIP_HWASAN_LAM" ]] && build_lam_linux + +SCUDO_BUILDS= + +function configure_scudo_compiler_rt { + local arch=$1 + local target="${arch}-linux-gnu${2:-}" + + local name="" + if [[ "$DBG" == "ON" ]] ; then + name=debug_ + fi + name+="${arch}" + + local qemu_cmd="" + if [[ "${QEMU:-}" != "0" ]] ; then + name+="_qemu" + local qemu_arch=${arch} + [[ "${arch}" == "powerpc64le" ]] && qemu_arch="ppc64le" + qemu_cmd="$ROOT/qemu_build/qemu-${qemu_arch} -L /usr/${target}" + if [[ ! -z "${QEMU_CPU:-}" ]] ; then + qemu_cmd+=" -cpu ${QEMU_CPU}" + name+="_${QEMU_CPU}" + fi + fi + + local linker_flags= + [[ "${arch}" =~ "mips*" ]] && linker_flags="-latomic -Wl,-z,notext -Wno-unused-command-line-argument" + + local out_dir=llvm_build2_${name} + rm -rf ${out_dir} + mkdir -p ${out_dir} + + SCUDO_BUILDS+=" ${name}" + + ( + # We need a fresh base compiler to test standalone build, as this config + # does not build own clang. + local COMPILER_BIN_DIR="$(readlink -f ${STAGE2_DIR})/bin" + cd ${out_dir} + + ( + cmake \ + ${CMAKE_COMMON_OPTIONS} \ + -DCOMPILER_RT_DEBUG=$DBG \ + -DLLVM_CONFIG_PATH=${COMPILER_BIN_DIR}/llvm-config \ + -DCMAKE_C_COMPILER=${COMPILER_BIN_DIR}/clang \ + -DCMAKE_CXX_COMPILER=${COMPILER_BIN_DIR}/clang++ \ + -DCOMPILER_RT_HAS_LLD=ON \ + -DCOMPILER_RT_TEST_USE_LLD=ON \ + -DCMAKE_INSTALL_PREFIX=$(${COMPILER_BIN_DIR}/clang -print-resource-dir) \ + -DCOMPILER_RT_BUILD_BUILTINS=OFF \ + -DCOMPILER_RT_DEFAULT_TARGET_ONLY=ON \ + -DCMAKE_CROSSCOMPILING=True \ + -DCOMPILER_RT_INCLUDE_TESTS=ON \ + -DCOMPILER_RT_BUILD_LIBFUZZER=OFF \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DCMAKE_CXX_FLAGS=-fPIC \ + -DCMAKE_C_FLAGS=-fPIC \ + -DCMAKE_SHARED_LINKER_FLAGS="-fuse-ld=lld ${linker_flags}" \ + -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld ${linker_flags}" \ + -DCOMPILER_RT_TEST_COMPILER_CFLAGS="--target=${target} ${linker_flags}" \ + -DCMAKE_C_COMPILER_TARGET=${target} \ + -DCMAKE_CXX_COMPILER_TARGET=${target} \ + -DCOMPILER_RT_EMULATOR="${qemu_cmd:-}" \ + $LLVM/../compiler-rt + ) >& configure.log + ) & +} + +function configure_hwasan_lam { + local out_dir=llvm_build2_x86_64_lam_qemu + rm -rf ${out_dir} + mkdir -p ${out_dir} + + ( + # We don't need fresh base compiler, as this config will build own clang. + local COMPILER_BIN_DIR="$(readlink -f ${STAGE1_DIR})/bin" + cd ${out_dir} + + ( + # STAGE1_DIR is build once, so we can use CCACHE. + if ccache -s >/dev/null ; then + CMAKE_COMMON_OPTIONS+=" -DLLVM_CCACHE_BUILD=ON" + fi + cmake \ + ${CMAKE_COMMON_OPTIONS} \ + -DLLVM_ENABLE_PROJECTS="clang;compiler-rt;lld" \ + -DCMAKE_C_COMPILER=${COMPILER_BIN_DIR}/clang \ + -DCMAKE_CXX_COMPILER=${COMPILER_BIN_DIR}/clang++ \ + -DLLVM_ENABLE_LLD=ON \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DCOMPILER_RT_EMULATOR="env SSH_CONTROL_SOCKET=${SSH_CONTROL_SOCKET} ${HERE}/ssh_run.sh" \ + $LLVM + ) >& configure.log + ) & +} + +function run_scudo_tests { + local name="${1}" + local out_dir=llvm_build2_${name} + + echo "@@@BUILD_STEP scudo $name@@@" + + ( + cd ${out_dir} + + cat configure.log + + # Copy into clang resource dir to make -fsanitize= work in lit tests. + ninja install-scudo_standalone + + ninja check-scudo_standalone || exit 3 + ) || build_failure +} + +QEMU_PID="" +readonly QEMU_TMPDIR="$ROOT/qemu_tmp" +readonly SSH_CONTROL_SOCKET="${QEMU_TMPDIR}/ssh-control-socket" + +function kill_qemu { + if kill "${QEMU_PID}"; then + echo "Waiting for QEMU to shutdown..." >&2 + { + timeout -k 5s 5s wait "${QEMU_PID}" && return + kill -9 "${QEMU_PID}" && wait "${QEMU_PID}" || true + } &>/dev/null + fi +} + +function boot_qemu { + trap kill_qemu EXIT + + # Path to a qemu-system-x86_64 binary built with LAM support. + local QEMU="${ROOT}/qemu_build/qemu-system-x86_64" + # Path to a qemu-img binary. + local QEMU_IMG="${ROOT}/qemu_build/qemu-img" + + echo "Booting QEMU..." >&2 + + # Try up to 10 random port numbers until one succeeds. + for i in {0..10}; do + rm -rf ${QEMU_TMPDIR} + mkdir -p ${QEMU_TMPDIR} + # Create a delta image to boot from. + local DELTA_IMAGE="${QEMU_TMPDIR}/delta.qcow2" + "${QEMU_IMG}" create -F raw -b "${QEMU_IMAGE}" -f qcow2 "${DELTA_IMAGE}" + + local SSH_PORT="$(shuf -i 1000-65535 -n 1)" + + "${QEMU}" -hda "${DELTA_IMAGE}" -nographic \ + -net "user,host=10.0.2.10,hostfwd=tcp:127.0.0.1:${SSH_PORT}-:22" \ + -net "nic,model=e1000" -machine "type=q35,accel=tcg" \ + -smp $(($(nproc) / 2)) -cpu "qemu64,+la57,+lam" -kernel "${LAM_KERNEL}" \ + -append "root=/dev/sda net.ifnames=0 console=ttyS0" -m "16G" & + QEMU_PID=$! + + # If QEMU is running, the port number worked. + sleep 1 + ps -p "${QEMU_PID}" &>/dev/null || continue + + echo "Waiting for QEMU ssh daemon..." >&2 + for i in {0..10}; do + echo "SSH into VM, try ${i}" >&2 + sleep 15 + + # Set up persistent SSH connection for faster command execution inside QEMU. + ssh -p "${SSH_PORT}" -o "StrictHostKeyChecking=no" \ + -o "UserKnownHostsFile=/dev/null" -o "ControlPersist=30m" \ + -M -S "${SSH_CONTROL_SOCKET}" -i "${QEMU_SSH_KEY}" root@localhost "uname -a" 1>&2 && + return + done + + kill_qemu + done + + # Fail fast if QEMU is not running. + ps -p "${QEMU_PID}" &>/dev/null +} + +function run_hwasan_lam_tests { + local name="x86_64_lam_qemu" + local out_dir=llvm_build2_${name} + + echo "@@@BUILD_STEP configure hwasan ${name}@@@" + + ( + cd ${out_dir} + cat configure.log + + # Build most stuff before starting VM. + echo "@@@BUILD_STEP build tools ${name}@@@" + ninja clang lld llvm-symbolizer || exit 3 + + echo "@@@BUILD_STEP start LAM QEMU@@@" + boot_qemu || build_exception + + ssh -S "${SSH_CONTROL_SOCKET}" root@localhost \ + "mkdir -p /b && mount -t nfs 10.0.2.10:/b /b" + + echo + echo "@@@BUILD_STEP test hwasan ${name}@@@" + ninja check-hwasan-lam || exit 3 + ) || build_failure +} + +echo "@@@BUILD_STEP configure@@@" + +for DBG in OFF ON ; do + QEMU=0 configure_scudo_compiler_rt x86_64 + configure_scudo_compiler_rt x86_64 + configure_scudo_compiler_rt arm eabihf + configure_scudo_compiler_rt aarch64 + QEMU_CPU="cortex-a72" configure_scudo_compiler_rt aarch64 + #configure_scudo_compiler_rt mips + #configure_scudo_compiler_rt mipsel + configure_scudo_compiler_rt mips64 abi64 + configure_scudo_compiler_rt mips64el abi64 + configure_scudo_compiler_rt powerpc64le +done +[[ -z "$SKIP_HWASAN_LAM" ]] && configure_hwasan_lam + +wait + +for B in $SCUDO_BUILDS ; do + run_scudo_tests $B +done + +[[ -z "$SKIP_HWASAN_LAM" ]] && ( + run_hwasan_lam_tests +) + +cleanup $STAGE2_DIR diff --git a/zorg/buildbot/builders/sanitizers/buildbot_selector.py b/zorg/buildbot/builders/sanitizers/buildbot_selector.py --- a/zorg/buildbot/builders/sanitizers/buildbot_selector.py +++ b/zorg/buildbot/builders/sanitizers/buildbot_selector.py @@ -5,7 +5,8 @@ import sys THIS_DIR=os.path.dirname(sys.argv[0]) - +extra_args = sys.argv[1:] +BOT_DIR = '/b' def bash(path): return 'bash ' + os.path.join(THIS_DIR, path) @@ -21,38 +22,49 @@ 'sanitizer-x86_64-linux-autoconf': bash('buildbot_standard.sh'), 'sanitizer-x86_64-linux-fuzzer': bash('buildbot_fuzzer.sh'), 'sanitizer-x86_64-linux-android': bash('buildbot_android.sh'), - 'sanitizer-x86_64-linux-bootstrap': bash('buildbot_bootstrap_asan.sh'), + 'sanitizer-x86_64-linux-bootstrap-asan': bash('buildbot_bootstrap_asan.sh'), 'sanitizer-x86_64-linux-bootstrap-msan': bash('buildbot_bootstrap_msan.sh'), 'sanitizer-x86_64-linux-bootstrap-ubsan': bash('buildbot_bootstrap_ubsan.sh'), - 'sanitizer-x86_64-linux-gn': 'true', + 'sanitizer-x86_64-linux-qemu': bash('buildbot_qemu.sh'), + 'sanitizer-aarch64-linux-fuzzer': bash('buildbot_fuzzer.sh'), + 'sanitizer-aarch64-linux-bootstrap-asan': bash('buildbot_bootstrap_asan.sh'), + 'sanitizer-aarch64-linux-bootstrap-hwasan': bash('buildbot_bootstrap_hwasan.sh'), + 'sanitizer-aarch64-linux-bootstrap-msan': bash('buildbot_bootstrap_msan.sh'), + 'sanitizer-aarch64-linux-bootstrap-ubsan': bash('buildbot_bootstrap_ubsan.sh'), } BOT_ADDITIONAL_ENV = { 'sanitizer-ppc64le-linux': { 'HAVE_NINJA': '1', 'CHECK_LIBCXX': '0', 'CHECK_LLD': '0' }, 'sanitizer-ppc64be-linux': { 'HAVE_NINJA': '1', 'CHECK_LIBCXX': '0', 'CHECK_LLD': '0' }, 'sanitizer-x86_64-linux': { 'HAVE_NINJA' : '1' }, - 'sanitizer-x86_64-linux-bootstrap': {}, 'sanitizer-x86_64-linux-fast': {}, 'sanitizer-x86_64-linux-autoconf': {}, 'sanitizer-x86_64-linux-fuzzer': {}, 'sanitizer-x86_64-linux-android': {}, + 'sanitizer-x86_64-linux-bootstrap-asan': {}, 'sanitizer-x86_64-linux-bootstrap-msan': {}, 'sanitizer-x86_64-linux-bootstrap-ubsan': {}, - 'sanitizer-x86_64-linux-gn': {}, + 'sanitizer-x86_64-linux-qemu': { 'QEMU_IMAGE_DIR': BOT_DIR + '/qemu_image' }, + 'sanitizer-aarch64-linux-fuzzer': {}, + 'sanitizer-aarch64-linux-bootstrap-asan': {}, + 'sanitizer-aarch64-linux-bootstrap-hwasan': {}, + 'sanitizer-aarch64-linux-bootstrap-msan': {}, + 'sanitizer-aarch64-linux-bootstrap-ubsan': {}, } def Main(): builder = os.environ.get('BUILDBOT_BUILDERNAME') - print "builder name: %s" % (builder) - cmd = BOT_ASSIGNMENT.get(builder) + print("builder name: %s" % (builder)) + cmd = BOT_ASSIGNMENT.get(builder) + ' ' + ' '.join(extra_args) if not cmd: sys.stderr.write('ERROR - unset/invalid builder name\n') sys.exit(1) - print "%s runs: %s\n" % (builder, cmd) + print("%s runs: %s\n" % (builder, cmd)) sys.stdout.flush() bot_env = os.environ + bot_env['BOT_DIR'] = BOT_DIR add_env = BOT_ADDITIONAL_ENV.get(builder) for var in add_env: bot_env[var] = add_env[var] diff --git a/zorg/buildbot/builders/sanitizers/buildbot_standard.sh b/zorg/buildbot/builders/sanitizers/buildbot_standard.sh --- a/zorg/buildbot/builders/sanitizers/buildbot_standard.sh +++ b/zorg/buildbot/builders/sanitizers/buildbot_standard.sh @@ -11,15 +11,15 @@ TSAN_FULL_DEBUG_BUILD_DIR=tsan_full_debug_build TSAN_RELEASE_BUILD_DIR=tsan_release_build -rm -rf $TSAN_DEBUG_BUILD_DIR $TSAN_FULL_DEBUG_BUILD_DIR $TSAN_RELEASE_BUILD_DIR -clobber +CLEANUP="$TSAN_DEBUG_BUILD_DIR $TSAN_FULL_DEBUG_BUILD_DIR $TSAN_RELEASE_BUILD_DIR" +clobber $CLEANUP ROOT=`pwd` PLATFORM=`uname` MAKE_JOBS=${MAX_MAKE_JOBS:-$(nproc)} LLVM=${ROOT}/llvm -CMAKE_COMMON_OPTIONS="-GNinja -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_BUILD_EXTERNAL_COMPILER_RT=ON -DLLVM_USE_LINKER=gold" +CMAKE_COMMON_OPTIONS+=" -GNinja -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_BUILD_EXTERNAL_COMPILER_RT=ON -DLLVM_USE_LINKER=gold" function build_tsan { local build_dir=$1 @@ -31,9 +31,9 @@ (cd $build_dir && CC="$3" CXX="$4" cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo \ ${CMAKE_COMMON_OPTIONS} ${extra_cmake_args} \ ${LLVM}) - (cd $build_dir && ninja ${targets}) || echo @@@STEP_FAILURE@@@ - (cd $build_dir && ninja compiler-rt-clear) || echo @@@STEP_FAILURE@@@ - (cd $build_dir && ninja tsan) || echo @@@STEP_FAILURE@@@ + (cd $build_dir && ninja ${targets}) || build_failure + (cd $build_dir && ninja compiler-rt-clear) || build_failure + (cd $build_dir && ninja tsan) || build_failure } buildbot_update @@ -46,7 +46,7 @@ echo @@@BUILD_STEP build fresh clang + debug compiler-rt@@@ build_tsan "${TSAN_DEBUG_BUILD_DIR}" "-DCOMPILER_RT_DEBUG=ON $USE_CCACHE" gcc g++ echo @@@BUILD_STEP test tsan in debug compiler-rt build@@@ -(cd $TSAN_DEBUG_BUILD_DIR && ninja check-tsan) || echo @@@STEP_FAILURE@@@ +(cd $TSAN_DEBUG_BUILD_DIR && ninja check-tsan) || build_failure echo @@@BUILD_STEP build tsan with stats and debug output@@@ build_tsan "${TSAN_FULL_DEBUG_BUILD_DIR}" "-DCOMPILER_RT_DEBUG=ON -DCOMPILER_RT_TSAN_DEBUG_OUTPUT=ON -DLLVM_INCLUDE_TESTS=OFF $USE_CCACHE" gcc g++ @@ -58,4 +58,6 @@ BIN=$(mktemp -t tsan_exe.XXXXXXXX) echo "int main() {return 0;}" | $TSAN_RELEASE_BUILD_DIR/bin/clang -x c++ - -fsanitize=thread -O2 -o ${BIN} COMPILER_RT=$LLVM/../compiler-rt -$COMPILER_RT/lib/tsan/check_analyze.sh ${BIN} || echo @@@STEP_FAILURE@@@ +$COMPILER_RT/lib/tsan/check_analyze.sh ${BIN} || build_failure + +cleanup $CLEANUP diff --git a/zorg/buildbot/builders/sanitizers/ssh_run.sh b/zorg/buildbot/builders/sanitizers/ssh_run.sh new file mode 100755 --- /dev/null +++ b/zorg/buildbot/builders/sanitizers/ssh_run.sh @@ -0,0 +1,12 @@ +#!/bin/bash -eu +# +# Runs the given command over ssh using SSH_CONTROL_SOCKET. +# + +function run_in_qemu { + ssh -S "${SSH_CONTROL_SOCKET}" root@localhost "${@}" +} + +# Run binary in QEMU. +ENV="HWASAN_OPTIONS=\"${HWASAN_OPTIONS:-}\" PATH=\"${PATH:-}\"" +run_in_qemu "cd $(pwd) && ${ENV} $@" diff --git a/zorg/buildbot/changes/llvmgitpoller.py b/zorg/buildbot/changes/llvmgitpoller.py --- a/zorg/buildbot/changes/llvmgitpoller.py +++ b/zorg/buildbot/changes/llvmgitpoller.py @@ -11,31 +11,39 @@ class LLVMPoller(changes.GitPoller): """ - Poll LLVM repository for changes and submit them to the change master. + Poll LLVM repository for changes and submit them for builds scheduling. Following Multiple LLVM Projects. This source will poll a remote LLVM git _monorepo_ for changes and submit - them to the change master.""" + them for builds scheduling.""" _repourl = "https://github.com/llvm/llvm-project" - _branch = "master" - compare_attrs = ["repourl", "branch", "workdir", + compare_attrs = ["repourl", "branches", "workdir", "pollInterval", "gitbin", "usetimestamps", "category", "project", "projects"] + def _check_branches(branch): + if branch == "refs/heads/main": + # Always listen for changes in the main branch. + return True + else: + # We are also interested in the release branches. + # Some builders will be building changes from there as well. + return re.search(r"refs\/heads\/release\/\d\d+.*", branch) + def __init__(self, - repourl=_repourl, branch=_branch, + repourl=_repourl, branches=_check_branches, **kwargs): self.cleanRe = re.compile(r"Require(?:s?)\s*.*\s*clean build", re.IGNORECASE + re.MULTILINE) self.cleanCfg = re.compile(r"(CMakeLists\.txt$|\.cmake$|\.cmake\.in$)") - # TODO: Add support for an optional list of projects. - # For now we always watch all the projects. + # Note: We always watch all the projects, then schedulers decide + # to build or not to build. - super().__init__(repourl=repourl, branch=branch, **kwargs) + super().__init__(repourl=repourl, branches=branches, **kwargs) def _transform_path(self, fileList): """ @@ -111,7 +119,7 @@ newRev, branch)) else: # This branch is known, but it now points to a different - # commit than last time we saw it, rebuild. + # commit than the last time we saw it, rebuild. log.msg('LLVMPoller: rebuilding {} for updated branch "{}"'.format( newRev, branch)) @@ -163,8 +171,9 @@ properties['clean_obj'] = (True, "change") log.msg("LLVMPoller: creating a change rev=%s" % rev) - log.msg(" >>> revision=%s, timestamp=%s, author=%s, committer=%s, project=%s, files=%s, comments=\"%s\", properties=%s" % \ - (bytes2unicode(rev, encoding=self.encoding), datetime.fromtimestamp(timestamp), author, committer, + log.msg(" >>> branch=%s, revision=%s, timestamp=%s, author=%s, committer=%s, project=%s, files=%s, comments=\"%s\", properties=%s" % \ + (bytes2unicode(self._removeHeads(branch)), + bytes2unicode(rev, encoding=self.encoding), datetime.fromtimestamp(timestamp), author, committer, projects, files, comments, properties)) yield self.master.data.updates.addChange( diff --git a/zorg/buildbot/commands/AnnotatedCommand.py b/zorg/buildbot/commands/AnnotatedCommand.py --- a/zorg/buildbot/commands/AnnotatedCommand.py +++ b/zorg/buildbot/commands/AnnotatedCommand.py @@ -97,7 +97,7 @@ @defer.inlineCallbacks def addStep(self): - debuglog("AnnotatedBuildStep: adding step '%s': buildid=%s" % ( + debuglog("AnnotatedBuildStep: adding step '{}': buildid={}".format( self.name, self.build.buildid)) yield buildstep.BuildStep.addStep(self) # Put ourselves into a list of processed steps of the build object. @@ -107,14 +107,14 @@ self.description = [self.name] self.stdio_log = yield self.addLog('stdio') - debuglog("AnnotatedBuildStep: added step '%s': stepid=%s, buildid=%s" % ( + debuglog("AnnotatedBuildStep: added step '{}': stepid={}, buildid={}".format( self.name, self.stepid, self.build.buildid)) @defer.inlineCallbacks def finishStep(self): hidden = False - debuglog("AnnotatedBuildStep: finish step '%s': stepid=%s, buildid=%s, results=%s" % ( + debuglog("AnnotatedBuildStep: finish step '{}': stepid={}, buildid={}, results={}".format( self.name, self.stepid, self.build.buildid, self.results)) yield self.master.data.updates.finishStep(self.stepid, self.results, hidden) @@ -123,8 +123,8 @@ @defer.inlineCallbacks def startStep(self, remote, done=False): - debuglog("AnnotatedBuildStep::startStep() starting '%s': " - "buildid=%s, done=%s" % ( + debuglog("AnnotatedBuildStep::startStep() starting '{}': " + "buildid={}, done={}".format( self.name, self.build.buildid, done)) try: @@ -165,13 +165,13 @@ # Update step status in the database. yield self.finishStep() - debuglog("AnnotatedBuildStep::startStep() completed '%s': " - "stepid=%s, buildid=%s, results=%s" % ( + debuglog("AnnotatedBuildStep::startStep() completed '{}': " + "stepid={}, buildid={}, results={}".format( self.name, self.stepid, self.build.buildid, self.results)) def requestFinish(self, status=None): - debuglog("AnnotatedBuildStep::requestFinish(%r): '%s': " - "stepid=%s, buildid=%s, results=%r" % ( + debuglog("AnnotatedBuildStep::requestFinish({}): '{}': " + "stepid={}, buildid={}, results={}".format( status, self.name, self.stepid, self.build.buildid, self.results)) # Update the current step status with the worst result. if status is not None: @@ -212,8 +212,8 @@ @defer.inlineCallbacks def run(self): - debuglog("AnnotatedBuildStep::run() starting '%s' step: " - "stepid=%s, buildid=%s" % ( + debuglog("AnnotatedBuildStep::run() starting '{}' step: " + "stepid={}, buildid={}".format( self.name, self.stepid, self.build.buildid)) # Save previously collected log lines. @@ -229,8 +229,8 @@ if self._loglines: yield self._flushLogs() - debuglog("AnnotatedBuildStep::run() exiting '%s' step: " - "stepid=%s, buildid=%s, results=%s" % ( + debuglog("AnnotatedBuildStep::run() exiting '{}' step: " + "stepid={}, buildid={}, results={}".format( self.name, self.stepid, self.build.buildid, self.results)) return self.results @@ -321,8 +321,6 @@ 'BUILDBOT_GOT_REVISION': util.Interpolate('%(prop:got_revision:-None)s'), 'BUILDBOT_REVISION': util.Interpolate('%(prop:revision:-None)s'), 'BUILDBOT_SCHEDULER': util.Interpolate('%(prop:scheduler:-None)s'), - 'BUILDBOT_SLAVENAME': util.Interpolate('%(prop:slavename:-None)s'), - 'BUILDBOT_MSAN_ORIGINS': util.Interpolate('%(prop:msan_origins:-)s'), } # Apply the passed in environment on top. old_env = kwargs.get('env') or {} @@ -350,7 +348,7 @@ return self.annotated_steps[-1] if self.annotated_steps else None def _updateLastAnnotatedStepStatus(self, status): - debuglog(">>> AnnotatedCommand::_updateLastAnnotatedStepStatus(%r)" % status) + debuglog(">>> AnnotatedCommand::_updateLastAnnotatedStepStatus({})".format(status)) # Alway update the common annotate command status. self.annotate_status = results.worst_status(self.annotate_status, status) @@ -365,7 +363,8 @@ def _updateLastAnnotatedStepText(self, text = None): """Updating a step text. None to clean up.""" - debuglog(">>> AnnotatedCommand::_updateLastAnnotatedStepText('%s')" % (text if text else '')) + debuglog(">>> AnnotatedCommand::_updateLastAnnotatedStepText('{}')".format( + text if text else '')) s = self._getLastAnnotatedStep() if s is not None: @@ -373,7 +372,8 @@ def _updateLastAnnotatedStepSummaryInfo(self, text = None): """Updating a step summary info. None to clean up.""" - debuglog(">>> AnnotatedCommand::_updateLastAnnotatedStepSummaryInfo('%s')" % (text if text else '')) + debuglog(">>> AnnotatedCommand::_updateLastAnnotatedStepSummaryInfo('{}')".format( + text if text else '')) s = self._getLastAnnotatedStep() if s is not None: @@ -388,7 +388,7 @@ s.requestFinish() def _scheduleNewAnnotatedStep(self, name, logline): - debuglog(">>> AnnotatedCommand::_scheduleNewAnnotatedStep('%s')" % name) + debuglog(">>> AnnotatedCommand::_scheduleNewAnnotatedStep('{}')".format(name)) self._fixupActiveAnnotatedStep() @@ -517,7 +517,7 @@ ancmd, args = parse_annotate_cmd(line) if ancmd: - debuglog(">>> AnnotatedCommand::processAnnotatedCommand(): %s, %r" % (ancmd, args)) + debuglog(">>> AnnotatedCommand::processAnnotatedCommand(): {}, {}".format(ancmd, args)) try: if ancmd == STEP_LINK: diff --git a/zorg/buildbot/commands/LitTestCommand.py b/zorg/buildbot/commands/LitTestCommand.py --- a/zorg/buildbot/commands/LitTestCommand.py +++ b/zorg/buildbot/commands/LitTestCommand.py @@ -19,7 +19,7 @@ # These are the codes for which we will include the log output in the buildbot # step results. - failingCodes = set(['FAIL', 'XPASS', 'KPASS', 'UNRESOLVED', 'TIMEOUT', 'ERROR']) + failingCodes = set(['FAIL', 'XPASS', 'KPASS', 'UNRESOLVED', 'TIMEOUT', 'ERROR', 'NOEXE']) # Regular expressions for start of summary marker. kStartSummaryRE = re.compile(r'^Failing Tests \(\d*\)$') @@ -152,7 +152,9 @@ 'REGRESSED':'runtime performance regression', 'IMPROVED':'runtime performance improvement', 'UNSUPPORTED':'unsupported tests', - 'TIMEOUT':'timeout waiting for results'} + 'SKIPPED':'skipped tests', + 'TIMEOUT':'timeout waiting for results', + 'NOEXE':'test executable is missing'} def __init__(self, ignore=[], flaky=[], max_logs=20, parseSummaryOnly=False, *args, **kwargs): diff --git a/zorg/buildbot/process/factory.py b/zorg/buildbot/process/factory.py --- a/zorg/buildbot/process/factory.py +++ b/zorg/buildbot/process/factory.py @@ -5,22 +5,70 @@ from buildbot.process.factory import BuildFactory from buildbot.plugins import util, steps +_all_runtimes = frozenset([ + "compiler-rt", + "libc", + "libcxx", + "libcxxabi", + "libunwind", + "openmp", +]) + class LLVMBuildFactory(BuildFactory): """ TODO: Document + + depends_on_projects is a list of LLVM projects a produced builder + depends on. If None, it gets discovered depending on other params. + + enable_runtimes is a list of enabled runtimes. If None, + it gets discovered based on the depends_on_projects list. """ def __init__(self, steps=None, depends_on_projects=None, **kwargs): # Cannot use "super" here as BuildFactory is an old style class. BuildFactory.__init__(self, steps) + # Handle the dependencies. if depends_on_projects is None: - self.depends_on_projects = frozenset(['llvm']) + # llvm project is always included. + self.depends_on_projects = set(['llvm']) else: self.depends_on_projects = frozenset(depends_on_projects) + enable_runtimes = kwargs.pop('enable_runtimes', None) + # If specified, we either got a givem list of + # enabled runtimes, or "auto", or "all". + + if enable_runtimes is None: + # For the backward compatibility, we do not use + # enable_runtimes unless it is requested explicitly. + self.enable_runtimes = frozenset([]) + elif enable_runtimes == "auto": + # Let's build the list of runtimes based on the given + # depends_on_projects list. + self.enable_runtimes = \ + self.depends_on_projects.intersection(_all_runtimes) + else: + if enable_runtimes == "all": + # Let's replace the "all" placeholder by + # the actual list of all runtimes. + self.enable_runtimes = frozenset(_all_runtimes) + else: + # Let's just use the given list, no need to discover. + self.enable_runtimes = frozenset(enable_runtimes) + + # Update the list of dependencies. + if depends_on_projects is None: + self.depends_on_projects.update(self.enable_runtimes) + + # Build the list of projects to enable. + self.enable_projects = \ + self.depends_on_projects.difference(self.enable_runtimes) + # Directories. - self.llvm_srcdir = kwargs.pop('llvm_srcdir', None) + self.monorepo_dir = kwargs.pop('llvm_srcdir', None) + self.src_to_build_dir = kwargs.pop('src_to_build_dir', None) self.obj_dir = kwargs.pop('obj_dir', None) self.install_dir = kwargs.pop('install_dir', None) @@ -29,9 +77,10 @@ for k,v in kwargs.items(): setattr(self, k, v) - self.monorepo_dir = self.llvm_srcdir or "llvm-project" + self.monorepo_dir = self.monorepo_dir or "llvm-project" + self.src_to_build_dir = self.src_to_build_dir or 'llvm' self.llvm_srcdir = \ - "%(monorepo_dir)s/llvm" % {'monorepo_dir' : self.monorepo_dir} + "{}/{}".format(self.monorepo_dir, self.src_to_build_dir) self.obj_dir = \ self.obj_dir or "build" @@ -71,11 +120,14 @@ def addGetSourcecodeSteps(self, **kwargs): # Checkout the monorepo. + # Documentation: http://docs.buildbot.net/current/manual/configuration/buildsteps.html#git self.addStep(steps.Git( name='Checkout the source code', repourl=self.repourl_prefix + "llvm-project.git", progress=True, workdir=util.WithProperties(self.monorepo_dir), + retryFetch=True, + clobberOnFailure=True, **kwargs)) @@ -104,4 +156,6 @@ repourl=_repourl, progress=True, workdir=util.WithProperties(src_dir), + retryFetch=True, + clobberOnFailure=True, **kwargs)) diff --git a/zorg/buildbot/util/InformativeMailNotifier.py b/zorg/buildbot/util/InformativeMailNotifier.py --- a/zorg/buildbot/util/InformativeMailNotifier.py +++ b/zorg/buildbot/util/InformativeMailNotifier.py @@ -1,38 +1,104 @@ -# TODO: Add support for clang crash dumps. -# TODO: Better handle unit/regression tests failures - -# TODO: For debug purposes. Remove this later. -from twisted.python import log +import traceback from buildbot.plugins import reporters +from buildbot.process.results import CANCELLED +from buildbot.process.results import EXCEPTION +from buildbot.process.results import FAILURE +from buildbot.process.results import RETRY +from buildbot.process.results import SKIPPED +from buildbot.process.results import SUCCESS +from buildbot.process.results import WARNINGS +from buildbot.process.results import statusToString + +def get_log_details(build): + text = "" + try: + for step in build['steps']: + results = step['results'] + if results == SUCCESS or results == SKIPPED: + continue + + text += "Step {} ({}) {}: {}\n".format(step['number'], step['name'], statusToString(results), step['state_string']) + + logs = step['logs'] + if logs: + log_index = -1 + log_type = 0 + for i, _log in enumerate(logs): + if _log['name'].startswith("FAIL: "): # Use only first logchunk FAIL: + log_type = 3 + log_index = i + elif log_type < 2 and _log['name'].startswith("warnings "): + log_type = 2 + log_index = i + elif log_type < 1 and _log['type'] == "s": # stdio + log_type = 1 + log_index = i + + if log_index < 0: + continue + + log_text = logs[log_index]['content']['content'] + if log_type == 1: + # Parse stdio + lines = log_text.splitlines() + for line in lines[:]: + if line.startswith("h"): + lines.remove(line) + for j, line in enumerate(lines): + if line.startswith("o") or line.startswith("e"): + lines[j] = line[1:] + for j, line in enumerate(lines): + if line.find("FAIL:") != -1 or line.find("FAILED") != -1: + if j > 10: + del lines[:j-10] # Start 10 lines before FAIL + lines = ["..."] + lines + del lines[50:] # Keep up to 50 lines around FAIL + break + if len(lines) > 50: + del lines[:len(lines)-50] # Otherwise keep last 50 lines + lines = ["..."] + lines + + log_text = "\n".join(lines) + + elif logs[log_index]['num_lines'] > 50: + # Keep first 50 lines + lines = log_text.splitlines() + del lines[50:] + log_text = "\n".join(lines + ["..."]) + + text += log_text + "\n" -def _get_logs_and_tracebacks_from_build(build): - # TODO: Implement interesting parts of the logs and tracebacks extraction. - return dict() + except Exception as err: + print("Exception in LLVMMessageFormatter.get_log_details(): {}\n{}".format(err, traceback.format_exc())) + # TODO: We should send something useful in this case. + + return dict(details=text) + +# TODO: Add build reason if we have that valid and available +#Build Reason: {{ build['properties'].get('reason', [""])[0] }} MAIL_TEMPLATE = """\ The Buildbot has detected a {{ status_detected }} on builder {{ buildername }} while building {{ projects }}. + Full details are available at: {{ build_url }} -Buildbot URL: {{ buildbot_url }} + Worker for this Build: {{ workername }} -Build Reason: {{ build['properties'].get('reason', [""])[0] }} -Blamelist: {{ ", ".join(blamelist) }} +Blamelist: + {{ ",\n ".join(blamelist) }} + {{ summary }} + +{{ details }} Sincerely, LLVM Buildbot """ class LLVMMessageFormatter(reporters.MessageFormatter): def buildAdditionalContext(self, master, ctx): - #log.msg(">>> LLVMMessageFormatter.buildAdditionalContext got ctx={}".format(ctx)) ctx.update(self.ctx) - - build = ctx["build"] - build_interesting_data = _get_logs_and_tracebacks_from_build(build) - #log.msg(">>> LLVMMessageFormatter.buildAdditionalContext build_interesting_data={}",format(build_interesting_data)) - ctx["build"].update(build_interesting_data) - + ctx.update(get_log_details(ctx["build"])) LLVMInformativeMailNotifier = LLVMMessageFormatter( template=MAIL_TEMPLATE, diff --git a/zorg/jenkins/build.py b/zorg/jenkins/build.py --- a/zorg/jenkins/build.py +++ b/zorg/jenkins/build.py @@ -15,7 +15,7 @@ from contextlib import contextmanager from urllib2 import urlopen, URLError, HTTPError -SERVER = "labmaster2.lab.llvm.org" +SERVER = "green-dragon-21.local" NINJA = "/usr/local/bin/ninja" @@ -71,7 +71,7 @@ self.max_parallel_tests = os.environ.get('MAX_PARALLEL_TESTS', None) self.max_parallel_links = os.environ.get('MAX_PARALLEL_LINKS', None) self.host_compiler_url = os.environ.get('HOST_URL', - 'http://labmaster2.local/artifacts/') + 'http://green-dragon-21.local/artifacts/') self.artifact_url = os.environ.get('ARTIFACT', 'NONE') self.job_name = os.environ.get('JOB_NAME', 'NONE') self.build_id = os.environ.get('BUILD_ID', 'NONE') @@ -395,9 +395,6 @@ '-DLLVM_INCLUDE_TESTS=On', '-DCLANG_INCLUDE_TESTS=On', '-DLLVM_INCLUDE_UTILS=On', - '-DLIBCXX_INSTALL_HEADERS=On', - '-DLIBCXX_OVERRIDE_DARWIN_INSTALL=On', - '-DLIBCXX_INSTALL_LIBRARY=Off', '-DCMAKE_MACOSX_RPATH=On', ] @@ -615,9 +612,8 @@ footer() header("Run Tests") - run_cmd(conf.lldbbuilddir(), [NINJA, '-v', 'check-debuginfo']) run_cmd(conf.lldbbuilddir(), ['/usr/bin/env', 'TERM=vt100', NINJA, '-v', - 'check-lldb']) + 'check-debuginfo', 'check-lldb', '-k2']) footer() diff --git a/zorg/jenkins/clang_build_dependencies.dep b/zorg/jenkins/clang_build_dependencies.dep --- a/zorg/jenkins/clang_build_dependencies.dep +++ b/zorg/jenkins/clang_build_dependencies.dep @@ -3,4 +3,4 @@ brew ninja >= 1.8.2 # Stage2 verbose output pip psutil >= 5.4.5 # Required for timeouts in the GTest Suite. pip six >= 1.11.0 # Required for urllib -xcode == 11.5 +xcode >= 11.5 # Node configs are not currently all on the same Xcode. diff --git a/zorg/jenkins/common.groovy b/zorg/jenkins/common.groovy --- a/zorg/jenkins/common.groovy +++ b/zorg/jenkins/common.groovy @@ -30,7 +30,11 @@ checkout poll: false, changelog: true, scm: [ $class: 'GitSCM', branches: [[name: sha ]], - userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-project.git']] + extensions: [[ + $class: 'CloneOption', + reference: '/Users/Shared/llvm-project.git' + ]], + userRemoteConfigs: [[url: 'https://github.com/llvm/llvm-project.git']] ] } } @@ -81,7 +85,7 @@ try { stage('main') { dir('config') { - git url: 'https://github.com/llvm/llvm-zorg.git', branch: 'master', poll: false + git url: 'https://github.com/llvm/llvm-zorg.git', branch: 'main', poll: false } body() } @@ -102,6 +106,7 @@ def benchmark_pipeline(label, body) { properties([ + disableResume(), parameters([ string(name: 'ARTIFACT'), string(name: 'GIT_DISTANCE'), @@ -119,10 +124,10 @@ def testsuite_pipeline(label, body) { benchmark_pipeline(label) { dir('lnt') { - git url: 'https://github.com/llvm/llvm-lnt.git', branch: 'master', poll: false + git url: 'https://github.com/llvm/llvm-lnt.git', branch: 'main', poll: false } dir('test-suite') { - git url: 'https://github.com/llvm/llvm-test-suite.git', branch: 'master', poll: false + git url: 'https://github.com/llvm/llvm-test-suite.git', branch: 'main', poll: false } body() } diff --git a/zorg/jenkins/inspect_log.py b/zorg/jenkins/inspect_log.py --- a/zorg/jenkins/inspect_log.py +++ b/zorg/jenkins/inspect_log.py @@ -26,7 +26,7 @@ '''Matching engine. Keeps data structures to match a single line against a fixed list of patterns.''' def __init__(self, patterns): - # Create a combined master regex combining all patterns. + # Create a combined main regex combining all patterns. combined = '' merge = '' for pattern in patterns: diff --git a/zorg/jenkins/jobs/jobs/clang-san-iossim b/zorg/jenkins/jobs/jobs/clang-san-iossim new file mode 100644 --- /dev/null +++ b/zorg/jenkins/jobs/jobs/clang-san-iossim @@ -0,0 +1,103 @@ +pipeline { + agent { label 'green-dragon-08' } + + parameters { + string(name: 'GIT_REVISION', defaultValue: '*/main', description: 'Git revision to build') + } + + stages { + stage('Checkout') { + steps { + dir('llvm-project') { + checkout([$class: 'GitSCM', branches: [[name: params.GIT_REVISION]], userRemoteConfigs: [[url: 'https://github.com/llvm/llvm-project.git']]]) + } + dir('llvm-zorg') { + checkout([$class: 'GitSCM', branches: [[name: '*/main']], userRemoteConfigs: [[url: 'https://github.com/llvm/llvm-zorg.git']]]) + } + } + } + stage('Build') { + steps { + timeout(90) { + sh ''' + set -u + + cd llvm-project + git tag -a -m "First Commit" first_commit 97724f18c79c7cc81ced24239eb5e883bf1398ef || true + + git_desc=$(git describe --match "first_commit") + export GIT_DISTANCE=$(echo ${git_desc} | cut -f 2 -d "-") + + sha=$(echo ${git_desc} | cut -f 3 -d "-") + export GIT_SHA=${sha:1} + + cd - + + export PATH=$PATH:/usr/bin:/usr/local/bin + + # Set a MacOS minimum deployment target + export MACOSX_DEPLOYMENT_TARGET=10.14 + + rm -rf clang-install *.tar.gz + python llvm-zorg/zorg/jenkins/monorepo_build.py cmake build \ + --assertions --cmake-type=RelWithDebInfo \ + --projects="clang;clang-tools-extra;compiler-rt" + ''' + } + } + } + stage('Test') { + steps { + timeout(90) { + sh ''' + set -u + export PATH=$PATH:/usr/bin:/usr/local/bin + + export SANITIZER_IOSSIM_TEST_DEVICE_IDENTIFIER='iPhone 12' + + COMPILER_RT_TEST_DIR="${WORKSPACE}/clang-build/tools/clang/runtime/compiler-rt-bins/test" + + cd $COMPILER_RT_TEST_DIR/asan && python3 ${WORKSPACE}/clang-build/./bin/llvm-lit \ + --xunit-xml-output=testresults-asan-IOSSimX86_64Config.xunit.xml -v -vv --timeout=600 \ + $COMPILER_RT_TEST_DIR/asan/IOSSimX86_64Config/ + + cd $COMPILER_RT_TEST_DIR/tsan && python3 ${WORKSPACE}/clang-build/./bin/llvm-lit \ + --xunit-xml-output=testresults-tsan-IOSSimX86_64Config.xunit.xml -v -vv --timeout=600 \ + $COMPILER_RT_TEST_DIR/tsan/IOSSimX86_64Config/ + + cd $COMPILER_RT_TEST_DIR/ubsan && python3 ${WORKSPACE}/clang-build/./bin/llvm-lit \ + --xunit-xml-output=testresults-ubsan-AddressSanitizer-iossim-x86_64.xunit.xml -v -vv --timeout=600 \ + $COMPILER_RT_TEST_DIR/ubsan/AddressSanitizer-iossim-x86_64/ + + cd $COMPILER_RT_TEST_DIR/ubsan && python3 ${WORKSPACE}/clang-build/./bin/llvm-lit \ + --xunit-xml-output=testresults-ubsan-Standalone-iossim-x86_64.xunit.xml -v -vv --timeout=600 \ + $COMPILER_RT_TEST_DIR/ubsan/Standalone-iossim-x86_64/ + + cd $COMPILER_RT_TEST_DIR/ubsan && python3 ${WORKSPACE}/clang-build/./bin/llvm-lit \ + --xunit-xml-output=testresults-ubsan-ThreadSanitizer-iossim-x86_64.xunit.xml -v -vv --timeout=600 \ + $COMPILER_RT_TEST_DIR/ubsan/ThreadSanitizer-iossim-x86_64/ + ''' + } + junit 'clang-build/**/testresults-*.xunit.xml' + } + } + } + post { + success { + scanForIssues tool: clang() + } + regression { + emailext subject: '$DEFAULT_SUBJECT', + presendScript: '$DEFAULT_PRESEND_SCRIPT', + postsendScript: '$DEFAULT_POSTSEND_SCRIPT', + recipientProviders: [ + [$class: 'CulpritsRecipientProvider'], + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'], + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS', + body:'$DEFAULT_CONTENT' + } + } +} diff --git a/zorg/jenkins/jobs/jobs/clang-stage1-RA b/zorg/jenkins/jobs/jobs/clang-stage1-RA new file mode 100644 --- /dev/null +++ b/zorg/jenkins/jobs/jobs/clang-stage1-RA @@ -0,0 +1,119 @@ +pipeline { + agent { label 'green-dragon-20' } + + parameters { + string(name: 'GIT_REVISION', defaultValue: '*/main', description: 'Git revision to build') + } + + stages { + stage('Checkout') { + steps { + dir('llvm-project') { + checkout([$class: 'GitSCM', branches: [ + [name: params.GIT_REVISION] + ], extensions: [ + [$class: 'CloneOption', + reference: '/Users/Shared/llvm-project.git'] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-project.git'] + ]]) + } + dir('llvm-zorg') { + checkout([$class: 'GitSCM', branches: [ + [name: '*/main'] + ], extensions: [ + [$class: 'CloneOption', + reference: '/Users/Shared/llvm-zorg.git'] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-zorg.git'] + ]]) + } + } + } + stage('Build') { + steps { + timeout(90) { + sh ''' + set -u + rm -rf build.properties + + cd llvm-project + git tag -a -m "First Commit" first_commit 97724f18c79c7cc81ced24239eb5e883bf1398ef || true + + git_desc=$(git describe --match "first_commit") + export GIT_DISTANCE=$(echo ${git_desc} | cut -f 2 -d "-") + + sha=$(echo ${git_desc} | cut -f 3 -d "-") + export GIT_SHA=${sha:1} + + # Also save the LLVM_REV until LNT server is taught about GIT + export LLVM_REV=$(git show -q | grep "llvm-svn:" | cut -f2 -d":" | tr -d " ") + + cd - + + echo "GIT_DISTANCE=$GIT_DISTANCE" > build.properties + echo "GIT_SHA=$GIT_SHA" >> build.properties + echo "ARTIFACT=$JOB_NAME/clang-d$GIT_DISTANCE-g$GIT_SHA-t$BUILD_ID-b$BUILD_NUMBER.tar.gz" >> build.properties + + export PATH=$PATH:/usr/bin:/usr/local/bin + + # Set a MacOS minimum deployment target + export MACOSX_DEPLOYMENT_TARGET=10.14 + + rm -rf clang-build clang-install *.tar.gz + python llvm-zorg/zorg/jenkins/monorepo_build.py cmake build \ + --assertions --cmake-type=RelWithDebInfo \ + --projects="clang;clang-tools-extra;compiler-rt" + ''' + } + } + } + stage('Test') { + steps { + timeout(90) { + sh ''' + set -u + export PATH=$PATH:/usr/bin:/usr/local/bin + python llvm-zorg/zorg/jenkins/monorepo_build.py cmake testlong + ''' + } + junit 'clang-build/**/testresults.xunit.xml' + } + } + } + post { + always { + scanForIssues tool: clang() + } + regression { + emailext subject: '$DEFAULT_SUBJECT', + presendScript: '$DEFAULT_PRESEND_SCRIPT', + postsendScript: '$DEFAULT_POSTSEND_SCRIPT', + recipientProviders: [ + [$class: 'CulpritsRecipientProvider'], + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'], + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS', + body:'$DEFAULT_CONTENT' + } + success { + build job: 'clang-stage2-Rthinlto_relay', wait: false + build job: 'clang-stage2-cmake-RgSan_relay', wait: false + //build job: 'phase2_modules_relay', wait: false + build job: 'relay-lnt-test-suite', wait: false + build job: 'relay-lnt-ctmark', wait: false + build job: 'relay-test-suite-verify-machineinstrs', wait: false + } + unstable { + build job: 'clang-stage2-Rthinlto_relay', wait: false + build job: 'clang-stage2-cmake-RgSan_relay', wait: false + //build job: 'phase2_modules_relay', wait: false + build job: 'relay-lnt-test-suite', wait: false + build job: 'relay-lnt-ctmark', wait: false + build job: 'relay-test-suite-verify-machineinstrs', wait: false + } + } +} + diff --git a/zorg/jenkins/jobs/jobs/clang-stage1-cmake-RA-incremental b/zorg/jenkins/jobs/jobs/clang-stage1-cmake-RA-incremental new file mode 100644 --- /dev/null +++ b/zorg/jenkins/jobs/jobs/clang-stage1-cmake-RA-incremental @@ -0,0 +1,104 @@ +pipeline { + agent { label 'green-dragon-07' } + + parameters { + string(name: 'GIT_REVISION', defaultValue: '*/main', description: 'Git revision to build') + } + + stages { + stage('Checkout') { + steps { + dir('llvm-project') { + checkout([$class: 'GitSCM', branches: [ + [name: params.GIT_REVISION] + ], extensions: [ + [$class: 'CloneOption', + reference: '/Users/Shared/llvm-project.git'] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-project.git'] + ]]) + } + dir('llvm-zorg') { + checkout([$class: 'GitSCM', branches: [ + [name: '*/main'] + ], extensions: [ + [$class: 'CloneOption', + reference: '/Users/Shared/llvm-zorg.git'] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-zorg.git'] + ]]) + } + } + } + stage('Build') { + steps { + timeout(90) { + sh ''' + set -u + rm -rf build.properties + + cd llvm-project + git tag -a -m "First Commit" first_commit 97724f18c79c7cc81ced24239eb5e883bf1398ef || true + + git_desc=$(git describe --match "first_commit") + + export GIT_DISTANCE=$(echo ${git_desc} | cut -f 2 -d "-") + + sha=$(echo ${git_desc} | cut -f 3 -d "-") + export GIT_SHA=${sha:1} + + # Also save the LLVM_REV until LNT server is taught about GIT + export LLVM_REV=$(git show -q | grep "llvm-svn:" | cut -f2 -d":" | tr -d " ") + + cd - + + export PATH=$PATH:/usr/bin:/usr/local/bin + + # Set a MacOS minimum deployment target + export MACOSX_DEPLOYMENT_TARGET=10.14 + + python llvm-zorg/zorg/jenkins/monorepo_build.py cmake build --assertions --projects="clang" + + # Removing the local artifact + rm -rf clang-*.tar.gz + ''' + } + } + } + stage('Test') { + steps { + timeout(90) { + sh ''' + set -u + export PATH=$PATH:/usr/bin:/usr/local/bin + + rm -rf clang-build/testresults.xunit.xml + + python llvm-zorg/zorg/jenkins/monorepo_build.py cmake testlong + ''' + } + junit 'clang-build/**/testresults.xunit.xml' + } + } + } + post { + always { + scanForIssues tool: clang() + } + /* Disabled email notification. + regression { + emailext subject: '$DEFAULT_SUBJECT', + presendScript: '$DEFAULT_PRESEND_SCRIPT', + postsendScript: '$DEFAULT_POSTSEND_SCRIPT', + recipientProviders: [ + [$class: 'CulpritsRecipientProvider'], + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'], + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS', + body:'$DEFAULT_CONTENT' + } + */ + } +} diff --git a/zorg/jenkins/jobs/jobs/clang-stage2-Rthinlto b/zorg/jenkins/jobs/jobs/clang-stage2-Rthinlto new file mode 100644 --- /dev/null +++ b/zorg/jenkins/jobs/jobs/clang-stage2-Rthinlto @@ -0,0 +1,103 @@ +pipeline { + agent { label 'green-dragon-22' } + + options { disableResume() } + + parameters { + string(name: 'GIT_SHA', defaultValue: '*/main', description: 'Git revision to build') + string(name: 'ARTIFACT', defaultValue: 'clang-stage1-RA/latest', description: 'description') + } + + stages { + stage('Checkout') { + steps { + dir('llvm-project') { + checkout([$class: 'GitSCM', branches: [ + [name: params.GIT_SHA] + ], extensions: [ + [$class: 'CloneOption', + reference: '/Users/Shared/llvm-project.git'] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-project.git'] + ]]) + } + dir('llvm-zorg') { + checkout([$class: 'GitSCM', branches: [ + [name: '*/main'] + ], extensions: [ + [$class: 'CloneOption', + reference: '/Users/Shared/llvm-zorg.git'] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-zorg.git'] + ]]) + } + } + } + stage('Build') { + steps { + timeout(600) { + sh ''' + set -u + rm -rf build.properties + + cd llvm-project + git tag -a -m "First Commit" first_commit 97724f18c79c7cc81ced24239eb5e883bf1398ef || true + + git_desc=$(git describe --match "first_commit") + export GIT_DISTANCE=$(echo ${git_desc} | cut -f 2 -d "-") + + sha=$(echo ${git_desc} | cut -f 3 -d "-") + export GIT_SHA=${sha:1} + + cd - + + echo "ARTIFACT=$JOB_NAME/clang-d$GIT_DISTANCE-g$GIT_SHA-t$BUILD_ID-b$BUILD_NUMBER.tar.gz" > build.properties + + export PATH=$PATH:/usr/bin:/usr/local/bin + + rm -rf clang-build clang-install *.tar.gz + + python llvm-zorg/zorg/jenkins/monorepo_build.py fetch + python llvm-zorg/zorg/jenkins/monorepo_build.py clang build \ + --thinlto --projects="clang;compiler-rt" \ + --cmake-flag="-DCMAKE_DSYMUTIL=$WORKSPACE/host-compiler/bin/dsymutil" + ''' + } + } + } + + stage('Test') { + steps { + timeout(240) { + sh ''' + set -u + export PATH=$PATH:/usr/bin:/usr/local/bin + + rm -rf clang-build/testresults.xunit.xml + + python llvm-zorg/zorg/jenkins/monorepo_build.py clang test + ''' + } + junit 'clang-build/**/testresults.xunit.xml' + } + } + } + post { + always { + scanForIssues tool: clang() + } + regression { + emailext subject: '$DEFAULT_SUBJECT', + presendScript: '$DEFAULT_PRESEND_SCRIPT', + postsendScript: '$DEFAULT_POSTSEND_SCRIPT', + recipientProviders: [ + [$class: 'CulpritsRecipientProvider'], + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'], + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS', + body:'$DEFAULT_CONTENT' + } + } +} diff --git a/zorg/jenkins/jobs/jobs/lldb-cmake b/zorg/jenkins/jobs/jobs/lldb-cmake --- a/zorg/jenkins/jobs/jobs/lldb-cmake +++ b/zorg/jenkins/jobs/jobs/lldb-cmake @@ -1,9 +1,9 @@ #!/usr/bin/env groovy pipeline { - agent { label 'green-dragon-10' } + agent { label 'green-dragon-04' } parameters { - string(name: 'GIT_REVISION', defaultValue: '*/master', description: 'Git revision to build') - string(name: 'ARTIFACT', defaultValue: 'clang-stage1-RA/latest', description: 'Compiler artifact to use for building the project') + string(name: 'GIT_SHA', defaultValue: '*/main', description: 'Git revision to build') + string(name: 'ARTIFACT', defaultValue: 'clang-stage2-Rthinlto/latest', description: 'Compiler artifact to use for building the project') string(name: 'BUILD_TYPE', defaultValue: 'Release', description: 'Default CMake build type; one of: Release, Debug, ...') string(name: 'CLEAN', defaultValue: "false", description: 'Whether or not to clean the build directory before building') } @@ -12,10 +12,25 @@ steps { timeout(30) { dir('llvm-project') { - checkout([$class: 'GitSCM', branches: [[name: params.GIT_REVISION]], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-project.git']]]) + checkout([$class: 'GitSCM', branches: [ + [name: params.GIT_SHA] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-project.git'] + ], extensions: [ + [$class: 'CloneOption', + reference: '/Users/Shared/llvm-project.git', + noTags: true, timeout: 30] + ]]) } dir('llvm-zorg') { - checkout([$class: 'GitSCM', branches: [[name: '*/master']], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-zorg.git']]]) + checkout([$class: 'GitSCM', branches: [ + [name: '*/main'] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-zorg.git'] + ], extensions: [ + [$class: 'CloneOption', + reference: '/Users/Shared/llvm-zorg.git'] + ]]) } } } @@ -48,36 +63,88 @@ cd - + if test $(hostname) == "green-dragon-04"; then sh /opt/codesign-workaround.sh init lldb-build/bin/debugserver; fi export MAX_PARALLEL_TESTS=$(sysctl hw.physicalcpu |awk '{print $2}') python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake build \ --assertions \ - --projects="clang;libcxx;libcxxabi;compiler-rt;lld;lldb;debuginfo-tests" \ + --projects="clang;lld;lldb;cross-project-tests" \ + --runtimes="libcxx;libcxxabi;compiler-rt" \ --compiler-flag="-Wdocumentation" \ --cmake-type=Release || (echo "Retrying. debuginfo-tests may have failed due to high load."; sleep 30; python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake build \ --assertions \ - --projects="clang;libcxx;libcxxabi;compiler-rt;lld;lldb;debuginfo-tests" \ + --projects="clang;lld;lldb;cross-project-tests" \ + --runtimes="libcxx;libcxxabi;compiler-rt" \ --compiler-flag="-Wdocumentation" \ --cmake-type=Release) + + if test $(hostname) == "green-dragon-04"; then sh /opt/codesign-workaround.sh sign lldb-build/bin/debugserver; fi ''' } } } - stage('Test') { + stage('Clean Test Results') { steps { - timeout(90) { + timeout(10) { sh ''' set -u export PATH=$PATH:/usr/bin:/usr/local/bin - export LLDB_CAPTURE_REPRODUCER=1 rm -rf test/results.xml + ''' + } + } + } + stage('Test (Debug Info)') { + steps { + timeout(30) { + // We want to continue with the rest of the pipeline even + // if the "Debug Info" tests fail, so catch the error but + // label this stage and the whole pipeline as FAILURE. + catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { + sh ''' + set -u + export PATH=$PATH:/usr/bin:/usr/local/bin + + export MAX_PARALLEL_TESTS=$(sysctl hw.physicalcpu |awk '{print $2}') + + python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-debuginfo test + ''' + } + } + } + } + stage('Test (LLDB)') { + steps { + timeout(90) { + sh ''' + set -u + export PATH=$PATH:/usr/bin:/usr/local/bin export MAX_PARALLEL_TESTS=$(sysctl hw.physicalcpu |awk '{print $2}') - python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake testlong + python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake test ''' } } } + stage('Upload artifacts (LLDB)') { + steps { + timeout(90) { + sh ''' + set -u + cd llvm-project + git_desc=$(git describe --match "first_commit") + + export GIT_DISTANCE=$(echo ${git_desc} | cut -f 2 -d "-") + + sha=$(echo ${git_desc} | cut -f 3 -d "-") + export GIT_SHA=${sha:1} + + cd - + python llvm-zorg/zorg/jenkins/monorepo_build.py properties + ''' + } + } + } } post { always { @@ -97,5 +164,8 @@ to: '$DEFAULT_RECIPIENTS', body:'$DEFAULT_CONTENT' } + success { + build job: 'relay-lldb', wait: false + } } } diff --git a/zorg/jenkins/jobs/jobs/lldb-cmake-matrix b/zorg/jenkins/jobs/jobs/lldb-cmake-matrix --- a/zorg/jenkins/jobs/jobs/lldb-cmake-matrix +++ b/zorg/jenkins/jobs/jobs/lldb-cmake-matrix @@ -3,20 +3,38 @@ pipeline { agent { label 'green-dragon-23' } parameters { - string(name: 'GIT_REVISION', defaultValue: '*/master', description: 'Git revision to build') - string(name: 'ARTIFACT', defaultValue: 'clang-stage1-RA/latest', description: 'Compiler artifact to use for building the project') + string(name: 'GIT_SHA', defaultValue: '*/main', description: 'Git revision to build') + string(name: 'ARTIFACT', defaultValue: 'clang-stage2-Rthinlto/latest', description: 'Compiler artifact to use for building the project') string(name: 'BUILD_TYPE', defaultValue: 'Release', description: 'Default CMake build type; one of: Release, Debug, ...') string(name: 'CLEAN', defaultValue: "false", description: 'Whether or not to clean the build directory before building') } + environment { + MATRIX_COMMON_BUILD_PARAMETERS = '--cmake-flag=-DLLVM_TARGETS_TO_BUILD=X86' + } stages { stage('Checkout') { steps { timeout(30) { dir('llvm-project') { - checkout([$class: 'GitSCM', branches: [[name: params.GIT_REVISION]], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-project.git']]]) + checkout([$class: 'GitSCM', branches: [ + [name: params.GIT_SHA] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-project.git'] + ], extensions: [ + [$class: 'CloneOption', + reference: '/Users/Shared/llvm-project.git', + noTags: true, timeout: 30] + ]]) } dir('llvm-zorg') { - checkout([$class: 'GitSCM', branches: [[name: '*/master']], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-zorg.git']]]) + checkout([$class: 'GitSCM', branches: [ + [name: '*/main'] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-zorg.git'] + ], extensions: [ + [$class: 'CloneOption', + reference: '/Users/Shared/llvm-zorg.git'] + ]]) } } } @@ -25,7 +43,10 @@ steps { timeout(10) { sh ''' - python llvm-zorg/zorg/jenkins/monorepo_build.py fetch + # The relay job sets ARTIFACT to the last good LLDB build. + # Here we use it to download the last good Clang. + export ARTIFACT=clang-stage2-Rthinlto/latest + python llvm-zorg/zorg/jenkins/monorepo_build.py fetch ''' } } @@ -52,12 +73,13 @@ export PATH=$PATH:/usr/bin:/usr/local/bin python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake build \ + $MATRIX_COMMON_BUILD_PARAMETERS \ --assertions \ - --projects="clang;libcxx;libcxxabi;lldb" \ + --projects="clang;lldb" \ + --runtimes="libcxx;libcxxabi" \ --cmake-type=Release \ --dotest-flag="--skip-category gmodules" \ - --dotest-flag="--arch=x86_64" \ - --cmake-flag="-DLLVM_TARGETS_TO_BUILD=X86" + --dotest-flag="--arch=x86_64" ''' } } @@ -68,17 +90,26 @@ sh ''' export PATH=$PATH:/usr/bin:/usr/local/bin python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix configure \ + $MATRIX_COMMON_BUILD_PARAMETERS \ --assertions \ - --projects="clang;libcxx;libcxxabi;lldb" \ + --lldb-test-compiler="$WORKSPACE/lldb-build/bin/clang" \ + --projects="clang;lldb" \ + --runtimes="libcxx;libcxxabi" \ --cmake-type=Release \ --dotest-flag="--skip-category" \ --dotest-flag="gmodules" \ --dotest-flag="--skip-category" \ --dotest-flag="watchpoint" \ + --dotest-flag="--skip-category" \ + --dotest-flag="llgs" \ + --dotest-flag="--skip-category" \ + --dotest-flag="debugserver" \ --dotest-flag="--arch=x86_64" \ --dotest-flag="--dwarf-version=2" set +e + # Give the system some time to recover. + sleep 120 python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix test set -e ''' @@ -92,17 +123,26 @@ sh ''' export PATH=$PATH:/usr/bin:/usr/local/bin python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix configure \ + $MATRIX_COMMON_BUILD_PARAMETERS \ + --lldb-test-compiler="$WORKSPACE/lldb-build/bin/clang" \ --assertions \ - --projects="clang;libcxx;libcxxabi;lldb" \ + --projects="clang;lldb" \ + --runtimes="libcxx;libcxxabi" \ --cmake-type=Release \ --dotest-flag="--skip-category" \ --dotest-flag="gmodules" \ --dotest-flag="--skip-category" \ --dotest-flag="watchpoint" \ + --dotest-flag="--skip-category" \ + --dotest-flag="llgs" \ + --dotest-flag="--skip-category" \ + --dotest-flag="debugserver" \ --dotest-flag="--arch=x86_64" \ --dotest-flag="--dwarf-version=4" set +e + # Give the system some time to recover. + sleep 120 python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix test set -e ''' @@ -116,37 +156,185 @@ sh ''' export PATH=$PATH:/usr/bin:/usr/local/bin python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix configure \ + $MATRIX_COMMON_BUILD_PARAMETERS \ + --lldb-test-compiler="$WORKSPACE/lldb-build/bin/clang" \ --assertions \ - --projects="clang;libcxx;libcxxabi;lldb" \ + --projects="clang;lldb" \ + --runtimes="libcxx;libcxxabi" \ --cmake-type=Release \ --dotest-flag="--skip-category" \ --dotest-flag="gmodules" \ --dotest-flag="--skip-category" \ --dotest-flag="watchpoint" \ + --dotest-flag="--skip-category" \ + --dotest-flag="llgs" \ + --dotest-flag="--skip-category" \ + --dotest-flag="debugserver" \ --dotest-flag="--arch=x86_64" \ --dotest-flag="--dwarf-version=5" set +e + # Give the system some time to recover. + sleep 120 + python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix test + set -e + ''' + } + junit 'test/results.xml' + } + } + stage('Build Clang 7.0.1') { + steps { + timeout(30) { + dir('clang_701') { + checkout([$class: 'GitSCM', branches: [ + [name: "llvmorg-7.0.1"] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-project.git'] + ], extensions: [ + [$class: 'CloneOption', timeout: 30, + reference: '/Users/Shared/llvm-project.git'] + ]]) + } + } + timeout(90) { + sh ''' + export PATH=$PATH:/usr/bin:/usr/local/bin + export SRC_DIR='clang_701' + export BUILD_DIR='clang_701_build' + + python llvm-zorg/zorg/jenkins/monorepo_build.py cmake build \ + $MATRIX_COMMON_BUILD_PARAMETERS \ + --assertions \ + --noupload \ + --noinstall \ + --projects="clang;libcxx;libcxxabi" + ''' + } + } + } + stage('Test Clang 7.0.1') { + steps { + timeout(60) { + sh ''' + build_dir=$WORKSPACE/clang_701_build + export PATH=$PATH:/usr/bin:/usr/local/bin + python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix configure \ + $MATRIX_COMMON_BUILD_PARAMETERS \ + --assertions \ + --lldb-test-compiler="$build_dir/bin/clang" \ + --projects="clang;lldb" \ + --runtimes="libcxx;libcxxabi" \ + --cmake-type=Release \ + --dotest-flag="--libcxx-include-dir=$build_dir/include/c++/v1/" \ + --dotest-flag="--libcxx-library-dir=$build_dir/lib/" \ + --dotest-flag="--skip-category" \ + --dotest-flag="gmodules" \ + --dotest-flag="--skip-category" \ + --dotest-flag="watchpoint" \ + --dotest-flag="--skip-category" \ + --dotest-flag="llgs" \ + --dotest-flag="--skip-category" \ + --dotest-flag="debugserver" \ + --dotest-flag="--arch=x86_64" + + set +e + # Give the system some time to recover. + sleep 120 + python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix test + set -e + ''' + } + junit 'test/results.xml' + } + } + stage('Build Clang 9.0.0') { + steps { + timeout(30) { + dir('clang_900') { + checkout([$class: 'GitSCM', branches: [ + [name: "llvmorg-9.0.0"] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-project.git'] + ], extensions: [ + [$class: 'CloneOption', timeout: 30, + reference: '/Users/Shared/llvm-project.git'] + ]]) + } + } + timeout(90) { + sh ''' + export PATH=$PATH:/usr/bin:/usr/local/bin + export SRC_DIR='clang_900' + export BUILD_DIR='clang_900_build' + + python llvm-zorg/zorg/jenkins/monorepo_build.py cmake build \ + $MATRIX_COMMON_BUILD_PARAMETERS \ + --assertions \ + --noupload \ + --noinstall \ + --projects="clang;libcxx;libcxxabi" + ''' + } + } + } + stage('Test Clang 9.0.0') { + steps { + timeout(60) { + sh ''' + build_dir=$WORKSPACE/clang_900_build + export PATH=$PATH:/usr/bin:/usr/local/bin + python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix configure \ + $MATRIX_COMMON_BUILD_PARAMETERS \ + --assertions \ + --lldb-test-compiler="$build_dir/bin/clang" \ + --projects="clang;lldb" \ + --runtimes="libcxx;libcxxabi" \ + --cmake-type=Release \ + --dotest-flag="--libcxx-include-dir=$build_dir/include/c++/v1/" \ + --dotest-flag="--libcxx-library-dir=$build_dir/lib/" \ + --dotest-flag="--skip-category" \ + --dotest-flag="gmodules" \ + --dotest-flag="--skip-category" \ + --dotest-flag="watchpoint" \ + --dotest-flag="--skip-category" \ + --dotest-flag="llgs" \ + --dotest-flag="--skip-category" \ + --dotest-flag="debugserver" \ + --dotest-flag="--arch=x86_64" + + set +e + # Give the system some time to recover. + sleep 120 python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix test set -e ''' } + junit 'test/results.xml' } } - stage('Build Clang 5.0.2') { + stage('Build Clang 11.0.1') { steps { timeout(30) { - dir('clang_502') { - checkout([$class: 'GitSCM', branches: [[name: "llvmorg-5.0.2"]], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-project.git']]]) + dir('clang_1101') { + checkout([$class: 'GitSCM', branches: [ + [name: "llvmorg-11.0.1"] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-project.git'] + ], extensions: [ + [$class: 'CloneOption', timeout: 30, + reference: '/Users/Shared/llvm-project.git'] + ]]) } } timeout(90) { sh ''' export PATH=$PATH:/usr/bin:/usr/local/bin - export SRC_DIR='clang_502' - export BUILD_DIR='clang_502_build' + export SRC_DIR='clang_1101' + export BUILD_DIR='clang_1101_build' python llvm-zorg/zorg/jenkins/monorepo_build.py cmake build \ + $MATRIX_COMMON_BUILD_PARAMETERS \ --assertions \ --noupload \ --noinstall \ @@ -155,23 +343,34 @@ } } } - stage('Test Clang 5.0.2') { + stage('Test Clang 11.0.1') { steps { timeout(60) { sh ''' + build_dir=$WORKSPACE/clang_1101_build export PATH=$PATH:/usr/bin:/usr/local/bin - export LLDB_TEST_COMPILER="$WORKSPACE/clang_502_build/bin/clang" python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix configure \ + $MATRIX_COMMON_BUILD_PARAMETERS \ --assertions \ - --projects="clang;libcxx;libcxxabi;lldb" \ + --lldb-test-compiler="$build_dir/bin/clang" \ + --projects="clang;lldb" \ + --runtimes="libcxx;libcxxabi" \ --cmake-type=Release \ + --dotest-flag="--libcxx-include-dir=$build_dir/include/c++/v1/" \ + --dotest-flag="--libcxx-library-dir=$build_dir/lib/" \ --dotest-flag="--skip-category" \ --dotest-flag="gmodules" \ --dotest-flag="--skip-category" \ --dotest-flag="watchpoint" \ + --dotest-flag="--skip-category" \ + --dotest-flag="llgs" \ + --dotest-flag="--skip-category" \ + --dotest-flag="debugserver" \ --dotest-flag="--arch=x86_64" set +e + # Give the system some time to recover. + sleep 120 python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix test set -e ''' @@ -179,20 +378,28 @@ junit 'test/results.xml' } } - stage('Build Clang 7.0.1') { + stage('Build Clang 13.0.0') { steps { timeout(30) { - dir('clang_701') { - checkout([$class: 'GitSCM', branches: [[name: "llvmorg-7.0.1"]], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-project.git']]]) + dir('clang_1300') { + checkout([$class: 'GitSCM', branches: [ + [name: "llvmorg-13.0.0"] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-project.git'] + ], extensions: [ + [$class: 'CloneOption', timeout: 30, + reference: '/Users/Shared/llvm-project.git'] + ]]) } } timeout(90) { sh ''' export PATH=$PATH:/usr/bin:/usr/local/bin - export SRC_DIR='clang_701' - export BUILD_DIR='clang_701_build' + export SRC_DIR='clang_1300' + export BUILD_DIR='clang_1300_build' python llvm-zorg/zorg/jenkins/monorepo_build.py cmake build \ + $MATRIX_COMMON_BUILD_PARAMETERS \ --assertions \ --noupload \ --noinstall \ @@ -201,23 +408,34 @@ } } } - stage('Test Clang 7.0.1') { + stage('Test Clang 13.0.0') { steps { timeout(60) { sh ''' + build_dir=$WORKSPACE/clang_1300_build export PATH=$PATH:/usr/bin:/usr/local/bin - export LLDB_TEST_COMPILER="$WORKSPACE/clang_701_build/bin/clang" python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix configure \ + $MATRIX_COMMON_BUILD_PARAMETERS \ --assertions \ - --projects="clang;libcxx;libcxxabi;lldb" \ + --lldb-test-compiler="$build_dir/bin/clang" \ + --projects="clang;lldb" \ + --runtimes="libcxx;libcxxabi" \ --cmake-type=Release \ + --dotest-flag="--libcxx-include-dir=$build_dir/include/c++/v1/" \ + --dotest-flag="--libcxx-library-dir=$build_dir/lib/" \ --dotest-flag="--skip-category" \ --dotest-flag="gmodules" \ --dotest-flag="--skip-category" \ --dotest-flag="watchpoint" \ + --dotest-flag="--skip-category" \ + --dotest-flag="llgs" \ + --dotest-flag="--skip-category" \ + --dotest-flag="debugserver" \ --dotest-flag="--arch=x86_64" set +e + # Give the system some time to recover. + sleep 120 python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix test set -e ''' @@ -225,20 +443,28 @@ junit 'test/results.xml' } } - stage('Build Clang 9.0.0') { + stage('Build Clang 15.0.1') { steps { timeout(30) { - dir('clang_900') { - checkout([$class: 'GitSCM', branches: [[name: "llvmorg-9.0.0"]], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-project.git']]]) + dir('clang_1501') { + checkout([$class: 'GitSCM', branches: [ + [name: "llvmorg-15.0.1"] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-project.git'] + ], extensions: [ + [$class: 'CloneOption', timeout: 30, + reference: '/Users/Shared/llvm-project.git'] + ]]) } } timeout(90) { sh ''' export PATH=$PATH:/usr/bin:/usr/local/bin - export SRC_DIR='clang_900' - export BUILD_DIR='clang_900_build' + export SRC_DIR='clang_1501' + export BUILD_DIR='clang_1501_build' python llvm-zorg/zorg/jenkins/monorepo_build.py cmake build \ + $MATRIX_COMMON_BUILD_PARAMETERS \ --assertions \ --noupload \ --noinstall \ @@ -247,23 +473,34 @@ } } } - stage('Test Clang 9.0.0') { + stage('Test Clang 15.0.1') { steps { timeout(60) { sh ''' + build_dir=$WORKSPACE/clang_1501_build export PATH=$PATH:/usr/bin:/usr/local/bin - export LLDB_TEST_COMPILER="$WORKSPACE/clang_900_build/bin/clang" python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix configure \ + $MATRIX_COMMON_BUILD_PARAMETERS \ --assertions \ - --projects="clang;libcxx;libcxxabi;lldb" \ + --lldb-test-compiler="$build_dir/bin/clang" \ + --projects="clang;lldb" \ + --runtimes="libcxx;libcxxabi" \ --cmake-type=Release \ + --dotest-flag="--libcxx-include-dir=$build_dir/include/c++/v1/" \ + --dotest-flag="--libcxx-library-dir=$build_dir/lib/" \ --dotest-flag="--skip-category" \ --dotest-flag="gmodules" \ --dotest-flag="--skip-category" \ --dotest-flag="watchpoint" \ + --dotest-flag="--skip-category" \ + --dotest-flag="llgs" \ + --dotest-flag="--skip-category" \ + --dotest-flag="debugserver" \ --dotest-flag="--arch=x86_64" set +e + # Give the system some time to recover. + sleep 120 python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-matrix test set -e ''' diff --git a/zorg/jenkins/jobs/jobs/lldb-cmake-reproducers b/zorg/jenkins/jobs/jobs/lldb-cmake-reproducers deleted file mode 100644 --- a/zorg/jenkins/jobs/jobs/lldb-cmake-reproducers +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env groovy -pipeline { - agent { label 'green-dragon-23' } - parameters { - string(name: 'GIT_REVISION', defaultValue: '*/master', description: 'Git revision to build') - string(name: 'ARTIFACT', defaultValue: 'clang-stage1-RA/latest', description: 'Compiler artifact to use for building the project') - string(name: 'BUILD_TYPE', defaultValue: 'Release', description: 'Default CMake build type; one of: Release, Debug, ...') - string(name: 'CLEAN', defaultValue: "false", description: 'Whether or not to clean the build directory before building') - } - stages { - stage('Checkout') { - steps { - timeout(30) { - dir('llvm-project') { - checkout([$class: 'GitSCM', branches: [[name: params.GIT_REVISION]], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-project.git']]]) - } - dir('llvm-zorg') { - checkout([$class: 'GitSCM', branches: [[name: '*/master']], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-zorg.git']]]) - } - } - } - } - stage('Fetch Host Compiler') { - steps { - timeout(10) { - sh ''' - python llvm-zorg/zorg/jenkins/monorepo_build.py fetch - ''' - } - } - } - stage('Build') { - steps { - timeout(90) { - sh ''' - set -u - rm -rf build.properties - - cd llvm-project - git tag -a -m "First Commit" first_commit 97724f18c79c7cc81ced24239eb5e883bf1398ef || true - - git_desc=$(git describe --match "first_commit") - - export GIT_DISTANCE=$(echo ${git_desc} | cut -f 2 -d "-") - - sha=$(echo ${git_desc} | cut -f 3 -d "-") - export GIT_SHA=${sha:1} - - cd - - - python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake build \ - --assertions \ - --projects="clang;libcxx;libcxxabi;compiler-rt;lld;lldb;debuginfo-tests" \ - --compiler-flag="-Wdocumentation" \ - --cmake-type=Release - ''' - } - } - } - stage('Test Reproducers') { - steps { - timeout(120) { - sh ''' - set -u - export PATH=$PATH:/usr/bin:/usr/local/bin - rm -rf test/results.xml - python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake \ - --cmake-test-target check-lldb-reproducers - ''' - } - } - } - } - post { - always { - scanForIssues tool: clang() - junit 'test/results.xml' - } - } -} diff --git a/zorg/jenkins/jobs/jobs/lldb-cmake-sanitized b/zorg/jenkins/jobs/jobs/lldb-cmake-sanitized --- a/zorg/jenkins/jobs/jobs/lldb-cmake-sanitized +++ b/zorg/jenkins/jobs/jobs/lldb-cmake-sanitized @@ -2,8 +2,8 @@ pipeline { agent { label 'green-dragon-23' } parameters { - string(name: 'GIT_REVISION', defaultValue: '*/master', description: 'Git revision to build') - string(name: 'ARTIFACT', defaultValue: 'clang-stage1-RA/latest', description: 'Compiler artifact to use for building the project') + string(name: 'GIT_SHA', defaultValue: '*/main', description: 'Git revision to build') + string(name: 'ARTIFACT', defaultValue: 'clang-stage2-Rthinlto/latest', description: 'Compiler artifact to use for building the project') string(name: 'BUILD_TYPE', defaultValue: 'Release', description: 'Default CMake build type; one of: Release, Debug, ...') string(name: 'CLEAN', defaultValue: "false", description: 'Whether or not to clean the build directory before building') } @@ -12,10 +12,25 @@ steps { timeout(30) { dir('llvm-project') { - checkout([$class: 'GitSCM', branches: [[name: params.GIT_REVISION]], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-project.git']]]) + checkout([$class: 'GitSCM', branches: [ + [name: params.GIT_SHA] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-project.git'] + ], extensions: [ + [$class: 'CloneOption', + reference: '/Users/Shared/llvm-project.git', + noTags: true, timeout: 30] + ]]) } dir('llvm-zorg') { - checkout([$class: 'GitSCM', branches: [[name: '*/master']], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-zorg.git']]]) + checkout([$class: 'GitSCM', branches: [ + [name: '*/main'] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-zorg.git'] + ], extensions: [ + [$class: 'CloneOption', + reference: '/Users/Shared/llvm-zorg.git'] + ]]) } } } @@ -24,6 +39,9 @@ steps { timeout(10) { sh ''' + # The relay job sets ARTIFACT to the last good LLDB build. + # Here we use it to download the last good Clang. + export ARTIFACT=clang-stage2-Rthinlto/latest python llvm-zorg/zorg/jenkins/monorepo_build.py fetch ''' } @@ -55,7 +73,8 @@ python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-sanitized build \ --assertions \ - --projects="clang;libcxx;libcxxabi;lld;lldb" \ + --projects="clang;lld;lldb" \ + --runtimes="libcxx;libcxxabi" \ --cmake-type=Release ''' } diff --git a/zorg/jenkins/jobs/jobs/lldb-cmake-standalone b/zorg/jenkins/jobs/jobs/lldb-cmake-standalone --- a/zorg/jenkins/jobs/jobs/lldb-cmake-standalone +++ b/zorg/jenkins/jobs/jobs/lldb-cmake-standalone @@ -2,8 +2,8 @@ pipeline { agent {label 'green-dragon-23'} parameters { - string(name: 'GIT_REVISION', defaultValue: '*/master', description: 'Git revision to build') - string(name: 'ARTIFACT', defaultValue: 'clang-stage1-RA/latest', description: 'Compiler artifact to use for building the project') + string(name: 'GIT_SHA', defaultValue: '*/main', description: 'Git revision to build') + string(name: 'ARTIFACT', defaultValue: 'clang-stage2-Rthinlto/latest', description: 'Compiler artifact to use for building the project') string(name: 'BUILD_TYPE', defaultValue: 'Release', description: 'Default CMake build type; one of: Release, Debug, ...') string(name: 'CLEAN', defaultValue: "false", description: 'Whether or not to clean the build directory before building') } @@ -12,10 +12,25 @@ steps { timeout(30) { dir('llvm-project') { - checkout([$class: 'GitSCM', branches: [[name: params.GIT_REVISION]], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-project.git']]]) + checkout([$class: 'GitSCM', branches: [ + [name: params.GIT_SHA] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-project.git'] + ], extensions: [ + [$class: 'CloneOption', + reference: '/Users/Shared/llvm-project.git', + noTags: true, timeout: 30] + ]]) } dir('llvm-zorg') { - checkout([$class: 'GitSCM', branches: [[name: '*/master']], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-zorg.git']]]) + checkout([$class: 'GitSCM', branches: [ + [name: '*/main'] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-zorg.git'] + ], extensions: [ + [$class: 'CloneOption', + reference: '/Users/Shared/llvm-zorg.git'] + ]]) } } } @@ -24,6 +39,9 @@ steps { timeout(90) { sh ''' + # The relay job sets ARTIFACT to the last good LLDB build. + # Here we use it to download the last good Clang. + export ARTIFACT=clang-stage2-Rthinlto/latest python llvm-zorg/zorg/jenkins/monorepo_build.py fetch ''' } @@ -53,7 +71,8 @@ python llvm-zorg/zorg/jenkins/monorepo_build.py cmake build \ --assertions \ --noupload \ - --projects="clang;libcxx;libcxxabi" \ + --projects="clang" \ + --runtimes="libcxx;libcxxabi" \ --cmake-flag="-DLLVM_INSTALL_UTILS=On" \ --cmake-flag="-DLLVM_INSTALL_TOOLCHAIN_ONLY=Off" \ --cmake-flag="-DLLVM_TARGETS_TO_BUILD=X86" @@ -154,7 +173,7 @@ set -u export PATH=$PATH:/usr/bin:/usr/local/bin - python $WORKSPACE/llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-xcode test \ + echo "TEMPORARILY DISABLED (runs with too many jobs in parallel)" python $WORKSPACE/llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-xcode test \ --assertions \ --cmake-type=Release ''' diff --git a/zorg/jenkins/jobs/jobs/llvm-coverage b/zorg/jenkins/jobs/jobs/llvm-coverage new file mode 100755 --- /dev/null +++ b/zorg/jenkins/jobs/jobs/llvm-coverage @@ -0,0 +1,150 @@ +#!/usr/bin/env groovy +pipeline { + agent { label 'green-dragon-23' } + + parameters { + string(name: 'GIT_REVISION', defaultValue: '*/main', description: 'Git revision to build') + string(name: 'ARTIFACT', defaultValue: 'clang-stage1-RA/latest', description: 'Compiler artifact to use for building the project') + string(name: 'BUILD_TYPE', defaultValue: 'Release', description: 'Default CMake build type; one of: Release, Debug, ...') + string(name: 'CLEAN', defaultValue: "false", description: 'Whether or not to clean the build directory before building') + } + + stages { + stage('Checkout') { + steps { + dir('llvm-project') { + checkout([$class: 'GitSCM', branches: [ + [name: params.GIT_REVISION] + ], extensions: [ + [$class: 'CloneOption', + reference: '/Users/Shared/llvm-project.git'] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-project.git'] + ]]) + } + dir('llvm-zorg') { + checkout([$class: 'GitSCM', branches: [ + [name: '*/main'] + ], extensions: [ + [$class: 'CloneOption', + reference: '/Users/Shared/llvm-zorg.git'] + ], userRemoteConfigs: [ + [url: 'https://github.com/llvm/llvm-zorg.git'] + ]]) + } + } + } + stage('Fetch') { + steps { + timeout(60) { + sh ''' + python llvm-zorg/zorg/jenkins/monorepo_build.py fetch + ''' + } + } + } + stage('Build') { + steps { + timeout(300) { + sh ''' + set -u + rm -rf build.properties + + cd llvm-project + git tag -a -m "First Commit" first_commit 97724f18c79c7cc81ced24239eb5e883bf1398ef || true + + git_desc=$(git describe --match "first_commit") + + export GIT_DISTANCE=$(echo ${git_desc} | cut -f 2 -d "-") + + sha=$(echo ${git_desc} | cut -f 3 -d "-") + export GIT_SHA=${sha:1} + + cd - + + export PATH=$PATH:/usr/bin:/usr/local/bin + + python llvm-zorg/zorg/jenkins/monorepo_build.py cmake build \ + --cmake-build-target FileCheck \ + --cmake-build-target count \ + --cmake-build-target not \ + --cmake-flag="-DLLVM_TARGETS_TO_BUILD=host" \ + --cmake-type=Release \ + --projects "" \ + --noinstall \ + --noupload + + python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake build \ + --assertions \ + --projects="clang;lldb;cross-project-tests" \ + --runtimes="libcxx;libcxxabi" \ + --cmake-flag="-DPYTHON_LIBRARY=/usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/lib/libpython3.7.dylib" \ + --cmake-flag="-DPYTHON_INCLUDE_DIR=/usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/include/python3.7m" \ + --cmake-flag="-DPYTHON_LIBRARY_DEBUG=/usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/lib/libpython3.7.dylib" \ + --cmake-flag="-DPYTHON_EXECUTABLE=/usr/local/Cellar/python/3.7.0/bin/python3.7" \ + --cmake-flag="-DLLDB_USE_SYSTEM_DEBUGSERVER=ON" \ + --cmake-flag="-DLLVM_BUILD_INSTRUMENTED_COVERAGE=ON" \ + --cmake-flag="-DLLVM_PROFILE_MERGE_POOL_SIZE=1" \ + --cmake-flag="-DLLVM_ENABLE_MODULES=Off" \ + --cmake-flag="-DLLVM_TARGETS_TO_BUILD=X86;ARM;AArch64;AMDGPU" \ + --cmake-type=Release \ + --noupload + ''' + } + } + } + stage('Test') { + steps { + timeout(300) { + sh ''' + set -u + export PATH=$PATH:/usr/bin:/usr/local/bin + + rm -rf test/results.xml + + BUILD_DIR=$(pwd)/lldb-build + FAST_BUILD_DIR=$(pwd)/clang-build + REPORT_DIR=$(pwd)/coverage-reports + HOST=$(pwd)/host-compiler/bin + LLVM_PROFDATA=$HOST/llvm-profdata + LLVM_COV=$HOST/llvm-cov + ARTIFACT_PREP_SCRIPT=$WORKSPACE/llvm-project/llvm/utils/prepare-code-coverage-artifact.py + + FAST_TOOLS=(FileCheck count not) + for TOOL in ${FAST_TOOLS[@]}; do + cp $FAST_BUILD_DIR/bin/$TOOL $BUILD_DIR/bin/$TOOL + done + + # Clear out any stale profiles. + rm -rf $BUILD_DIR/profiles + + # Run the tests. + IGNORE_ERRORS_OVERRIDE=1 python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake testlong || echo "Some tests may have failed." + + cd $BUILD_DIR + ninja -k 0 check-llvm check-clang || echo "Some tests may have failed." + cd - + + COV_BINARIES=$(find $BUILD_DIR/bin $BUILD_DIR/lib -depth 1 -type f -exec file {} \\; | grep Mach-O | cut -d':' -f1 | grep -vE '/(FileCheck|count|not)$' | xargs) + + rm -rf $REPORT_DIR + mkdir -p $REPORT_DIR + python $ARTIFACT_PREP_SCRIPT $LLVM_PROFDATA $LLVM_COV $BUILD_DIR/profiles $REPORT_DIR $COV_BINARIES --unified-report --restrict $WORKSPACE/llvm-project + + scp -r $REPORT_DIR buildslave@labmaster2.local:/Library/WebServer/Documents/coverage + ssh buildslave@labmaster2.local "chmod -R 777 /Library/WebServer/Documents/coverage" + + rm -rf lldb-build/profiles + ''' + } + } + } + } + + post { + always { + scanForIssues tool: clang() + junit 'test/results.xml' + } + } +} diff --git a/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-O0-g b/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-O0-g --- a/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-O0-g +++ b/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-O0-g @@ -1,9 +1,11 @@ #!/usr/bin/env groovy def common = evaluate readTrusted('zorg/jenkins/common.groovy') common.testsuite_pipeline(label: 'green-dragon-12') { - sh """ + timeout(30) { + sh """ LNT_FLAGS+=" -C config/tasks/cmake/caches/target-arm64-iphoneos.cmake" LNT_FLAGS+=" -C config/tasks/cmake/caches/opt-O0-g.cmake" config/tasks/task jenkinsrun config/tasks/lnt-ctmark.sh -a compiler="${params.ARTIFACT}" -D LNT_FLAGS="\${LNT_FLAGS}" - """ + """ + } } diff --git a/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-O3-flto b/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-O3-flto --- a/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-O3-flto +++ b/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-O3-flto @@ -1,9 +1,11 @@ #!/usr/bin/env groovy def common = evaluate readTrusted('zorg/jenkins/common.groovy') common.testsuite_pipeline(label: 'green-dragon-12') { - sh """ + timeout(30) { + sh """ LNT_FLAGS+=" -C config/tasks/cmake/caches/target-arm64-iphoneos.cmake" LNT_FLAGS+=" -C config/tasks/cmake/caches/opt-O3-flto.cmake" config/tasks/task jenkinsrun config/tasks/lnt-ctmark.sh -a compiler="${params.ARTIFACT}" -D LNT_FLAGS="\${LNT_FLAGS}" - """ + """ + } } diff --git a/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-Os b/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-Os --- a/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-Os +++ b/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-Os @@ -1,9 +1,11 @@ #!/usr/bin/env groovy def common = evaluate readTrusted('zorg/jenkins/common.groovy') common.testsuite_pipeline(label: 'green-dragon-12') { - sh """ + timeout(30) { + sh """ LNT_FLAGS+=" -C config/tasks/cmake/caches/target-arm64-iphoneos.cmake" LNT_FLAGS+=" -C config/tasks/cmake/caches/opt-Os.cmake" config/tasks/task jenkinsrun config/tasks/lnt-ctmark.sh -a compiler="${params.ARTIFACT}" -D LNT_FLAGS="\${LNT_FLAGS}" - """ + """ + } } diff --git a/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-Oz b/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-Oz --- a/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-Oz +++ b/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-Oz @@ -1,9 +1,11 @@ #!/usr/bin/env groovy def common = evaluate readTrusted('zorg/jenkins/common.groovy') common.testsuite_pipeline(label: 'green-dragon-12') { - sh """ + timeout(30) { + sh """ LNT_FLAGS+=" -C config/tasks/cmake/caches/target-arm64-iphoneos.cmake" LNT_FLAGS+=" -C config/tasks/cmake/caches/opt-Oz.cmake" config/tasks/task jenkinsrun config/tasks/lnt-ctmark.sh -a compiler="${params.ARTIFACT}" -D LNT_FLAGS="\${LNT_FLAGS}" - """ + """ + } } diff --git a/zorg/jenkins/jobs/jobs/relay-lldb b/zorg/jenkins/jobs/jobs/relay-lldb new file mode 100644 --- /dev/null +++ b/zorg/jenkins/jobs/jobs/relay-lldb @@ -0,0 +1,3 @@ +#!/usr/bin/env groovy +def relay = evaluate readTrusted('zorg/jenkins/relay.groovy') +relay.lldb_pipeline "lldb-cmake-(.+)" diff --git a/zorg/jenkins/jobs/util/make_pipeline.py b/zorg/jenkins/jobs/util/make_pipeline.py --- a/zorg/jenkins/jobs/util/make_pipeline.py +++ b/zorg/jenkins/jobs/util/make_pipeline.py @@ -32,7 +32,7 @@ - */master + */main false diff --git a/zorg/jenkins/monorepo_build.py b/zorg/jenkins/monorepo_build.py --- a/zorg/jenkins/monorepo_build.py +++ b/zorg/jenkins/monorepo_build.py @@ -11,11 +11,11 @@ import shutil import math import re +import requests import xml.etree.ElementTree as ET from contextlib import contextmanager -from six.moves import urllib -SERVER = "labmaster2.lab.llvm.org" +SERVER = "green-dragon-21.local" NINJA = "/usr/local/bin/ninja" @@ -44,6 +44,7 @@ def header(name): print("@@@", name, "@@@") + print("Starting at: " + time.strftime("%FT%T")) def footer(): @@ -112,13 +113,12 @@ self._lldb_standalone_type = os.environ.get('LLDB_STANDALONE_TYPE', 'build-tree') self._lldb_xcode_build_dir = os.environ.get('LLDB_XCODE_BUILD_DIR', 'lldb-xcode-build') self._lldb_install_dir = os.environ.get('LLDB_INSTALL_DIR', 'lldb-install') - self._lldb_test_compiler = os.environ.get('LLDB_TEST_COMPILER', '') self._install_dir = os.environ.get('INSTALL_DIR', 'clang-install') self.j_level = os.environ.get('J_LEVEL', None) self.max_parallel_tests = os.environ.get('MAX_PARALLEL_TESTS', None) self.max_parallel_links = os.environ.get('MAX_PARALLEL_LINKS', None) self.host_compiler_url = os.environ.get('HOST_URL', - 'http://labmaster2.local/artifacts/') + 'http://green-dragon-21.local/artifacts/') self.artifact_url = os.environ.get('ARTIFACT', 'NONE') self.job_name = os.environ.get('JOB_NAME', 'NONE') self.build_id = os.environ.get('BUILD_ID', 'NONE') @@ -166,10 +166,6 @@ """The install directory for the lldb build.""" return os.path.join(self.workspace, self._lldb_install_dir) - def lldbtestcompiler(self): - """The compiler used to build LLDB tests.""" - return self._lldb_test_compiler - def installdir(self): """The install directory for the compile.""" return os.path.join(self.workspace, self._install_dir) @@ -202,7 +198,7 @@ def branch(self): """Figure out the source branch name. - Not using GIT_BRANCH env var from Jenkins as that includes the + Not using GIT_BRANCH env var from Jenkins as that includes the remote name too. """ if not os.environ.get('TESTING', False): @@ -213,16 +209,16 @@ cmd = ['git', '-C', conf.srcdir(), 'rev-parse', 'HEAD'] out = run_collect_output(cmd, stderr=subprocess.STDOUT).strip() return out - return 'master' + return 'main' def link_memory_usage(self): """Guesstimate the maximum link memory usage for this build. - We are only building master here so we will just use that value + We are only building main here so we will just use that value """ # Determinited experimentally. - usages = {'master': 3.5} - if self.branch() == 'master': - return usages['master'] + usages = {'main': 3.5} + if self.branch() == 'main': + return usages['main'] else: raise NotImplementedError( "Unknown link memory usage." + self.branch()) @@ -247,6 +243,9 @@ "-DLLVM_ENABLE_PROJECTS=" + conf.llvm_enable_projects, conf.llvmsrcdir()] + if conf.llvm_enable_runtimes: + cmake_cmd += ["-DLLVM_ENABLE_RUNTIMES=" + conf.llvm_enable_runtimes] + compiler_flags = conf.compiler_flags max_parallel_links = conf.max_parallel_links @@ -350,7 +349,7 @@ # unless the user asked for something else. targets = ['check', 'check-clang'] - run_cmd(conf.builddir(), ninja_cmd + targets) + run_cmd(conf.builddir(), ninja_cmd + targets, err_okay=True) footer() if conf.sccache: @@ -428,12 +427,12 @@ '-DLLVM_INCLUDE_TESTS=On', '-DCLANG_INCLUDE_TESTS=On', '-DLLVM_INCLUDE_UTILS=On', - '-DLIBCXX_INSTALL_HEADERS=On', - '-DLIBCXX_OVERRIDE_DARWIN_INSTALL=On', - '-DLIBCXX_INSTALL_LIBRARY=Off', '-DCMAKE_MACOSX_RPATH=On', ] + if conf.llvm_enable_runtimes: + cmake_command.append('-DLLVM_ENABLE_RUNTIMES={}'.format(conf.llvm_enable_runtimes)) + if conf.sccache: cmake_command.append('-DCMAKE_C_COMPILER_LAUNCHER=' + conf.sccache_path) cmake_command.append('-DCMAKE_CXX_COMPILER_LAUNCHER=' + conf.sccache_path) @@ -497,7 +496,7 @@ cmd[-1] += ' --param use_gmalloc=1 ' \ '--param gmalloc_path=$(xcodebuild -find-library' \ ' libgmalloc.dylib)' - run_cmd(obj_dir, cmd, env={'MALLOC_LOG_FILE': '/dev/null'}) + run_cmd(obj_dir, cmd, env={'MALLOC_LOG_FILE': '/dev/null'}, err_okay=True) if conf.sccache: run_ws([conf.sccache_path, "--stop-server"]) @@ -538,7 +537,7 @@ # Construct lit arguments. lit_args = ['-v', '--time-tests', '--shuffle', - '--xunit-xml-output={}'.format(results_file), '-v'] + '--xunit-xml-output={}'.format(results_file), '-v'] if conf.max_parallel_tests: lit_args.extend(['-j', conf.max_parallel_tests]) if variant == 'sanitized': @@ -549,6 +548,7 @@ conf.llvmsrcdir(), '-DCMAKE_BUILD_TYPE={}'.format(cmake_build_type), '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON', + '-DLLVM_TARGETS_TO_BUILD=X86;ARM;AArch64', '-DCMAKE_INSTALL_PREFIX={}'.format(conf.lldbinstalldir()), '-DCMAKE_MAKE_PROGRAM={}'.format(NINJA), '-DLLDB_TEST_USER_ARGS='+';'.join(dotest_args), @@ -559,6 +559,9 @@ '-DLLVM_LIT_ARGS={}'.format(' '.join(lit_args)), '-DLLVM_VERSION_PATCH=99'] + if conf.llvm_enable_runtimes: + cmake_cmd.append('-DLLVM_ENABLE_RUNTIMES={}'.format(conf.llvm_enable_runtimes)) + if variant == 'sanitized': cmake_cmd.append('-DLLVM_TARGETS_TO_BUILD=X86') cmake_cmd.append('-DLLVM_USE_SANITIZER=Address;Undefined') @@ -566,8 +569,7 @@ # if we have a host compiler available. if conf.CC(): cmake_cmd.extend([ - '-DLLDB_TEST_C_COMPILER=' + conf.CC(), - '-DLLDB_TEST_CXX_COMPILER=' + conf.CC() + "++" + '-DLLDB_TEST_COMPILER=' + conf.CC(), ]) if conf.compiler_flags: @@ -576,10 +578,9 @@ "-DCMAKE_CXX_FLAGS={}".format(' '.join(conf.compiler_flags)) ]) - if conf.lldbtestcompiler(): + if conf.lldb_test_compiler is not None: cmake_cmd.extend([ - '-DLLDB_TEST_C_COMPILER=' + conf.lldbtestcompiler(), - '-DLLDB_TEST_CXX_COMPILER=' + conf.lldbtestcompiler() + "++" + '-DLLDB_TEST_COMPILER=' + conf.lldb_test_compiler, ]) cmake_cmd.extend(conf.cmake_flags) @@ -609,15 +610,15 @@ run_cmd(conf.lldbbuilddir(), [NINJA, '-v', 'install']) footer() - if target == 'all' or target == 'testlong': - header("Run Debug Info Tests") - run_cmd(conf.lldbbuilddir(), [NINJA, '-v', 'check-debuginfo']) - footer() - if target == 'all' or target == 'test' or target == 'testlong': header("Run Tests") - run_cmd(conf.lldbbuilddir(), - ['/usr/bin/env', 'TERM=vt100', NINJA, '-v', 'check-lldb']) + if variant == 'debuginfo': + test_command = ['/usr/bin/env', 'TERM=vt100', NINJA, + '-v', 'check-debuginfo'] + else: + test_command = ['/usr/bin/env', 'TERM=vt100', NINJA, + '-v', 'check-lldb'] + run_cmd(conf.lldbbuilddir(), test_command) footer() for test_target in conf.cmake_test_targets: @@ -771,20 +772,21 @@ Print error and exit if download fails. """ + chunk_size = 100 * 1024 * 1024 try: print("GETting", url, "to", dest, "...") - f = urllib.request.urlopen(url) + session = requests.Session() + r = session.get(url) + r.raise_for_status() # Open our local file for writing with open(dest, "wb") as local_file: - local_file.write(f.read()) + for chunk in r.iter_content(chunk_size=chunk_size): + local_file.write(chunk) - except urllib.HTTPError as e: - print("HTTP Error:", e.code, url) + except requests.exceptions.HTTPError as e: + print("HTTP Error:", e.response.status_code, url) sys.exit(1) - except urllib.URLError as e: - print("URL Error:", e.reason, url) - sys.exit(1) print("done.") @@ -864,6 +866,29 @@ conf.job_name + "/g" + conf.git_sha] run_cmd(conf.workspace, lng_cmd) +def build_upload_properties(): + """Create artifact for this build, and upload to server.""" + if conf.noupload: + print('Not uploading artificats') + return + header("Uploading Artifact") + prop_file = "last_good_build.properties" + + with open(prop_file, 'w') as prop_fd: + prop_fd.write("LLVM_REV={}\n".format(conf.svn_rev)) + prop_fd.write("GIT_DISTANCE={}\n".format(conf.git_distance)) + prop_fd.write("GIT_SHA={}\n".format(conf.git_sha)) + + mkdir_cmd = ["ssh", "buildslave@" + SERVER, "mkdir", "-p", "/Library/WebServer/Documents/artifacts/" + conf.job_name] + + run_cmd(conf.workspace, mkdir_cmd) + + upload_cmd = ["scp", prop_file, + "buildslave@" + SERVER + ":/Library/WebServer/Documents/artifacts/" + + conf.job_name + "/"] + + run_cmd(conf.workspace, upload_cmd) + def run_cmd(working_dir, cmd, env=None, sudo=False, err_okay=False): """Run a command in a working directory, and make sure it returns zero.""" @@ -903,9 +928,18 @@ KNOWN_TARGETS = ['all', 'configure', 'build', 'test', 'testlong', 'install'] KNOWN_BUILDS = [ - 'clang', 'cmake', 'lldb-cmake', 'lldb-cmake-standalone', - 'lldb-cmake-xcode', 'lldb-cmake-sanitized', 'lldb-cmake-matrix', 'fetch', - 'artifact', 'static-analyzer-benchmarks' + 'clang', + 'cmake', + 'lldb-cmake', + 'lldb-cmake-standalone', + 'lldb-cmake-debuginfo', + 'lldb-cmake-xcode', + 'lldb-cmake-sanitized', + 'lldb-cmake-matrix', + 'fetch', + 'artifact', + 'properties', + 'static-analyzer-benchmarks', ] @@ -1033,10 +1067,15 @@ action='store_true', help="Turn on the experimental" " GlobalISel CMake flag.") parser.add_argument('--projects', dest='llvm_enable_projects', - default="clang;clang-tools-extra;compiler-rt;libcxx", - help="Semicolon seperated list of projects to build.") + default="clang;clang-tools-extra;compiler-rt", + help="Semicolon seperated list of projects to enable.") + parser.add_argument('--runtimes', dest='llvm_enable_runtimes', + default="", + help="Semicolon seperated list of runtimes to enable.") parser.add_argument('--timeout', dest='timeout', type=int, default='600', help='Individual test timeout in seconds.') + parser.add_argument('--lldb-test-compiler', + help='The compiler used to build LLDB tests.') args = parser.parse_args() if args.thinlto: args.lto = True @@ -1055,6 +1094,8 @@ clang_builder(args.build_target) elif args.build_type == 'lldb-cmake': lldb_cmake_builder(args.build_target) + elif args.build_type == 'lldb-cmake-debuginfo': + lldb_cmake_builder(args.build_target, 'debuginfo') elif args.build_type == 'lldb-cmake-sanitized': lldb_cmake_builder(args.build_target, 'sanitized') elif args.build_type == 'lldb-cmake-matrix': @@ -1069,6 +1110,8 @@ fetch_compiler() elif args.build_type == 'artifact': build_upload_artifact() + elif args.build_type == 'properties': + build_upload_properties() elif args.build_type == 'static-analyzer-benchmarks': static_analyzer_benchmarks_builder() except subprocess.CalledProcessError as exct: diff --git a/zorg/jenkins/relay.groovy b/zorg/jenkins/relay.groovy --- a/zorg/jenkins/relay.groovy +++ b/zorg/jenkins/relay.groovy @@ -28,7 +28,7 @@ curl -fksSO "${last_good_properties_url}" """ def props = readProperties file: propfile - def artifact = "http://labmaster2.local/artifacts/${props.ARTIFACT}" + def artifact = "http://green-dragon-21.local/artifacts/${props.ARTIFACT}" currentBuild.setDisplayName("${props.GIT_DISTANCE}-${props.GIT_SHA}") // Trigger all jobs with names matching the `job_pattern` regex. @@ -56,12 +56,23 @@ } def pipeline(job_pattern, - artifact_url='http://labmaster2.local/artifacts/', - last_good_properties_url='http://labmaster2.local/artifacts/clang-stage1-RA/last_good_build.properties') { + artifact_url='http://green-dragon-21.local/artifacts/', + last_good_properties_url='http://green-dragon-21.local/artifacts/clang-stage1-RA/last_good_build.properties') { node('master') { stage('main') { relay_steps job_pattern, artifact_url, last_good_properties_url } } } + +def lldb_pipeline(job_pattern, + artifact_url='http://green-dragon-21.local/artifacts/', + last_good_properties_url='http://green-dragon-21.local/artifacts/lldb-cmake/last_good_build.properties') { + node('master') { + stage('main') { + relay_steps job_pattern, artifact_url, last_good_properties_url + } + } +} + return this