diff --git a/.pylintrc b/.pylintrc new file mode 100644 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,7 @@ +# add root folder to Pylint PYTHONPATH, so builders can be checked with zorg.* +# packages +[MASTER] +init-hook='import sys; sys.path.append(".")' + +[FORMAT] +max-line-length=120 \ No newline at end of file diff --git a/buildbot/google/docker/buildbot-clangd-ubuntu-clang/Dockerfile b/buildbot/google/docker/buildbot-clangd-ubuntu-clang/Dockerfile --- a/buildbot/google/docker/buildbot-clangd-ubuntu-clang/Dockerfile +++ b/buildbot/google/docker/buildbot-clangd-ubuntu-clang/Dockerfile @@ -53,7 +53,7 @@ # The grpc version shipped with Ubuntu 18.04 crashes on TSan checks. ENV GRPC_INSTALL_PATH=/usr/local/lib/grpc RUN cd /tmp ; \ - git clone -b v1.33.2 --depth 1 --recursive --shallow-submodules https://github.com/grpc/grpc ; \ + git clone -b v1.36.3 --depth 1 --recursive --shallow-submodules https://github.com/grpc/grpc ; \ cd grpc ; \ mkdir build; cd build ; \ cmake -G Ninja -DgRPC_INSTALL=ON -DCMAKE_INSTALL_PREFIX=${GRPC_INSTALL_PATH} -DCMAKE_BUILD_TYPE=Release .. ; \ diff --git a/buildbot/google/docker/buildbot-clangd-ubuntu-clang/VERSION b/buildbot/google/docker/buildbot-clangd-ubuntu-clang/VERSION --- a/buildbot/google/docker/buildbot-clangd-ubuntu-clang/VERSION +++ b/buildbot/google/docker/buildbot-clangd-ubuntu-clang/VERSION @@ -1 +1 @@ -3 +5 diff --git a/buildbot/google/docker/buildbot-clangd-ubuntu-clang/run.sh b/buildbot/google/docker/buildbot-clangd-ubuntu-clang/run.sh --- a/buildbot/google/docker/buildbot-clangd-ubuntu-clang/run.sh +++ b/buildbot/google/docker/buildbot-clangd-ubuntu-clang/run.sh @@ -38,9 +38,14 @@ buildbot-worker create-worker --keepalive=200 "${WORKER_NAME}" \ lab.llvm.org:${BUILDBOT_PORT} "${WORKER_NAME}" "${WORKER_PASSWORD}" -# start the daemon, this command return immetiately +# start the daemon, this command returns immediately echo "starting worker..." -buildbot-worker start "${WORKER_NAME}" +set +e +if ! buildbot-worker start "${WORKER_NAME}"; then + echo ERROR: starting worker failed. contents of twistd.log: + cat "${WORKER_NAME}/twistd.log" + exit 1 +fi # To keep the container running and produce log outputs: dump the worker # log to stdout diff --git a/buildbot/google/docker/buildbot-mlir-nvidia/Dockerfile b/buildbot/google/docker/buildbot-mlir-nvidia/Dockerfile --- a/buildbot/google/docker/buildbot-mlir-nvidia/Dockerfile +++ b/buildbot/google/docker/buildbot-mlir-nvidia/Dockerfile @@ -5,7 +5,7 @@ # #===----------------------------------------------------------------------===// # Docker image used for the mlir-nvidia builder -# +# # Environment variables configurable at runtime: # BUILDBOT_PORT - server port to connect to #===----------------------------------------------------------------------===// @@ -18,7 +18,7 @@ RUN apt-get update; \ apt-get install -y software-properties-common apt-transport-https ca-certificates \ clang-8 lld-8 ninja-build git wget gnupg ccache \ - python3 python3-pip python3-psutil pybind11-dev \ + python3 python3-pip python3-psutil \ # dumb-init recommended in # https://hub.docker.com/r/buildbot/buildbot-worker/dockerfile dumb-init;\ @@ -26,6 +26,16 @@ update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-8 100 ;\ update-alternatives --install /usr/bin/lld lld /usr/bin/lld-8 100 +# LTS releases often bundle obsolete pip versions that cannot access newest +# Linux binary wheels. This pinned version is not special: it was just current +# at the time this was added. Refer to compatibility table: +# https://github.com/pypa/manylinux +RUN python3 -m pip install --upgrade pip==21.0.1 + +# Refer to mlir/lib/Bindings/Python/requirements.txt. Listed explicitly here +# and version pinned for consistency as this is a bot. +RUN python3 -m pip install numpy==1.19.3 pybind11==2.6.0 PyYAML==5.4.1 + # install cuda # avoid popups for keyboard configurations RUN DEBIAN_FRONTEND=noninteractive apt-get install -y cuda @@ -78,7 +88,7 @@ ENV WORKER_NAME="mlir-nvidia" # Allow the server port of this agent to be configurable during deployment. -# This way we can connect the same image to production and integration. +# This way we can connect the same image to production and integration. # Ports: # 9990 - production # 9994 - integration diff --git a/buildbot/google/docker/buildbot-mlir-nvidia/VERSION b/buildbot/google/docker/buildbot-mlir-nvidia/VERSION --- a/buildbot/google/docker/buildbot-mlir-nvidia/VERSION +++ b/buildbot/google/docker/buildbot-mlir-nvidia/VERSION @@ -1 +1 @@ -15 +17 diff --git a/buildbot/google/docker/buildbot-sanitizer-windows/Dockerfile b/buildbot/google/docker/buildbot-sanitizer-windows/Dockerfile new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-sanitizer-windows/Dockerfile @@ -0,0 +1,21 @@ +# escape=` +FROM gcr.io/sanitizer-bots/windows-base-vscode2019:8 + +ENV WORKER_NAME=sanitizer-windows + +# copy script to start the agent +COPY run.ps1 . +COPY fix_buildbot_tac_paths.py . + +# Set location for sccache cache +ENV SCCACHE_DIR="C:\volumes\sccache" + +# Set the maximum sccache local cache size +ENV SCCACHE_CACHE_SIZE="10G" + +# Move buildbot work dir to a volume to avoid of disc issues +VOLUME C:\volumes\buildbot + +# Configure 32bit tools ("x86") instead of 64bit ("amd64") +CMD ["powershell.exe", "-NoLogo", "-ExecutionPolicy", "Bypass", ` + "c:\\buildbot\\run.ps1"] diff --git a/buildbot/google/docker/buildbot-sanitizer-windows/VERSION b/buildbot/google/docker/buildbot-sanitizer-windows/VERSION new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-sanitizer-windows/VERSION @@ -0,0 +1 @@ +1 diff --git a/buildbot/google/docker/buildbot-sanitizer-windows/admin b/buildbot/google/docker/buildbot-sanitizer-windows/admin new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-sanitizer-windows/admin @@ -0,0 +1 @@ +Reid Kleckner diff --git a/buildbot/google/docker/buildbot-sanitizer-windows/fix_buildbot_tac_paths.py b/buildbot/google/docker/buildbot-sanitizer-windows/fix_buildbot_tac_paths.py new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-sanitizer-windows/fix_buildbot_tac_paths.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +# Replace backward slashes in the paths in a "buildbot.tac" file + +import argparse +import re +import shutil + + +def fix_slashes(buildbot_tac_path : str): + shutil.copy(buildbot_tac_path, buildbot_tac_path + ".bak") + result = [] + with open(buildbot_tac_path) as buildbot_tac_file: + buildbot_tac = buildbot_tac_file.readlines() + for line in buildbot_tac: + if line.lstrip().startswith('basedir'): + line = line.replace(r'\\','/') + result.append(line) + with open(buildbot_tac_path, 'w') as buildbot_tac_file: + buildbot_tac_file.writelines(result) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('buildbot_tac_path', help="path of 'buildbot.tac' file") + args = parser.parse_args() + fix_slashes(args.buildbot_tac_path) diff --git a/buildbot/google/docker/buildbot-sanitizer-windows/run.ps1 b/buildbot/google/docker/buildbot-sanitizer-windows/run.ps1 new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-sanitizer-windows/run.ps1 @@ -0,0 +1,51 @@ +# Stop the script on the first error +$ErrorActionPreference = "Stop" + +# Read password from file +$WORKER_PASSWORD=Get-Content "C:\volumes\secrets\token" + +#Create short drive alias "Q:" for worker dir, to keep path lengths <260 +SUBST Q: C:\volumes\buildbot +Get-PSDrive + +# chdir to the volume storing the temporary data +# This is a mounted volume +Set-Location Q:\ + +# Write host and admin information +# delete old folder if it exists +# configure powershell output to use UTF-8, otherwiese buildbot can't read the file +if ( Test-Path -Path 'info\' -PathType Container ) { + Remove-Item -Recurse -Force info +} +mkdir "info\" +$HOST_FILE="info\host" +$PSDefaultParameterValues['Out-File:Encoding'] = 'utf8' +Write-Output "Windows version: $(cmd /c ver)" > $HOST_FILE +Write-Output "VCTools version: $(ls C:\BuildTools\VC\Tools\MSVC | Select -ExpandProperty Name)" >> $HOST_FILE +Write-Output "Cores : $((Get-CimInstance Win32_ComputerSystem).NumberOfLogicalProcessors)" >> $HOST_FILE +Write-Output "RAM : $([int][Math]::Round((Get-CimInstance Win32_ComputerSystem).TotalPhysicalMemory / 1073741824)) GB" >> $HOST_FILE +Write-Output "Reid Kleckner " > "info\admin" + +# create the worker +Write-Output "creating worker..." +buildslave create-slave --keepalive=200 Q: ` + "lab.llvm.org:9994" "$env:WORKER_NAME" "$WORKER_PASSWORD" +# Note: Powershell does NOT exit on non-zero return codes, so we need to check manually. +if ($LASTEXITCODE -ne 0) { throw "Exit code of 'buildslave create' is $LASTEXITCODE" } + +# Replace backward slashes with forward slashes on buildbot.tac +# Otherwise Cmake will go into infinite re-configuration loops +python C:\buildbot\fix_buildbot_tac_paths.py buildbot.tac +if ($LASTEXITCODE -ne 0) { throw "Exit code of 'fix_buildbot_tac_paths' is $LASTEXITCODE" } + +# start the daemon, as this does not print and logs to std out, run it in the background +Write-Output "starting worker..." +cmd /c Start /B buildslave start Q: + +# Wait a bit until the logfile exists +Start-Sleep -s 5 + +# To keep the container running and produce log outputs: dump the worker +# log to stdout, this is the windows equivalent of `tail -f` +Get-Content -Path "Q:\twistd.log" -Wait diff --git a/buildbot/google/terraform/deployment-mlir-nvidia-production.yaml b/buildbot/google/terraform/deployment-mlir-nvidia-production.yaml --- a/buildbot/google/terraform/deployment-mlir-nvidia-production.yaml +++ b/buildbot/google/terraform/deployment-mlir-nvidia-production.yaml @@ -27,7 +27,7 @@ spec: containers: # the image and version we want to run - - image: gcr.io/sanitizer-bots/buildbot-mlir-nvidia:15 + - image: gcr.io/sanitizer-bots/buildbot-mlir-nvidia:17 name: mlir-nvidia-production env: # connect to production environment, running at port 9990 diff --git a/buildbot/google/terraform/main.tf b/buildbot/google/terraform/main.tf --- a/buildbot/google/terraform/main.tf +++ b/buildbot/google/terraform/main.tf @@ -120,161 +120,6 @@ } } -# node pool for windows machines -resource "google_container_node_pool" "windows_32core_pool_nodes" { - name = "windows-32core-pool" - # specify a zone here (e.g. "-a") to avoid a redundant deployment - location = var.gcp_config.zone_a - cluster = google_container_cluster.primary.name - - # use autoscaling to only create a machine when there is a deployment - autoscaling { - min_node_count = 0 - max_node_count = 1 - } - - node_config { - # use preemptible, as this saves costs - preemptible = true - machine_type = "e2-highcpu-32" - # Windows deployments tend to require more disk space, so using 300GB here. - disk_size_gb = 300 - # FIXME: test if SSDs are actually faster than HDDs for our use case - disk_type = "pd-ssd" - - # Configure Windows image. As Windows is picky about the combination of - # host and container OS versions, this must be compatible with the version - # in your container. Recommondation: Use LTSC for long-term stability. - # For details see - # https://docs.microsoft.com/en-us/virtualization/windowscontainers/deploy-containers/version-compatibility - # https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#choose_your_windows_server_node_image - image_type = "WINDOWS_LTSC" - - # set the premissions required for the deployment later - oauth_scopes = [ - "https://www.googleapis.com/auth/logging.write", - "https://www.googleapis.com/auth/monitoring", - "https://www.googleapis.com/auth/devstorage.read_only", - ] - - # add a label to all machines of this type, so we can select them - # during deployment - labels = { - pool = "win-32core-pool" - } - } -} - -# Deployment for the buildbot windows10_vs2019 running on Windows rather than -# Linux. -# Note: Deploying this takes significantly longer (~35 min) than on Linux -# as the images tend to be larger (~18GB) and IO performance is lower. -resource "kubernetes_deployment" "windows10_vs2019" { - metadata { - name = "windows10-vs2019" - labels = { - app = "windows10_vs2019" - } - } - - spec { - # create one instance of this container - replicas = 1 - - selector { - match_labels = { - app = "windows10_vs2019" - } - } - strategy{ - rolling_update{ - # do not deploy more replicas, as the buildbot server - # can't handle multiple workers with the same credentials - max_surge = 0 - # Allow to have 0 replicas during updates. - max_unavailable = 1 - } - type = "RollingUpdate" - } - template { - metadata { - labels = { - app = "windows10_vs2019" - } - } - - spec { - container { - image = "${var.gcp_config.gcr_prefix}/buildbot-windows10-vs2019:17" - name = "windows10-vs2019" - - # reserve "-1" for this image, kubernetes also - # needs <1 core for management tools - resources { - limits { - cpu = "31" - memory = "20Gi" - } - requests { - cpu = "31" - memory = "20Gi" - } - } - - # mount the secrets into a folder - volume_mount { - mount_path = "c:\\volumes\\secrets" - name = "buildbot-token" - } - volume_mount { - mount_path = "c:\\volumes\\sccache" - name = "sccache-vol" - } - volume_mount { - mount_path = "c:\\volumes\\buildbot" - name = "buildbot-vol" - } - - } - # select which node pool to deploy to - node_selector = { - pool = "win-32core-pool" - } - # restart in case of any crashes - restart_policy = "Always" - - # select the secret to be mounted - volume { - name = "buildbot-token" - secret { - optional = false - secret_name = "password-windows10-vs2019" - } - } - volume { - name = "sccache-vol" - empty_dir {} - } - volume { - name = "buildbot-vol" - empty_dir {} - } - - # Windows nodes from the node pool are marked with the taint - # "node.kubernetes.io/os=windows". So we need to "tolerate" this to - # deploy to such nodes. - toleration { - effect = "NoSchedule" - key = "node.kubernetes.io/os" - operator = "Equal" - value = "windows" - } - } - } - } -} - - resource "google_container_node_pool" "linux_16_core_pool" { name = "linux-16-core-pool" # specify a zone here (e.g. "-a") to avoid a redundant deployment @@ -290,8 +135,8 @@ node_config { # use preemptible, as this saves costs preemptible = true - #custom machine type: 16 core, 32 GB as tsan needs more RAM - machine_type = "n2d-custom-16-32768" + #custom machine type: 16 core, 48 GB as tsan needs more RAM + machine_type = "n2d-custom-16-49152" disk_size_gb = 100 disk_type = "pd-ssd" @@ -347,7 +192,7 @@ spec { container { - image = "${var.gcp_config.gcr_prefix}/buildbot-clangd-ubuntu-clang:3" + image = "${var.gcp_config.gcr_prefix}/buildbot-clangd-ubuntu-clang:5" name = "buildbot-clangd-ubuntu-clang" # reserve "-1" for this image, kubernetes also @@ -355,11 +200,11 @@ resources { limits { cpu = "15" - memory = "28G" + memory = "45G" } requests { cpu = "15" - memory = "28G" + memory = "45G" } } diff --git a/buildbot/osuosl/master/config/__init__.py b/buildbot/osuosl/master/config/__init__.py --- a/buildbot/osuosl/master/config/__init__.py +++ b/buildbot/osuosl/master/config/__init__.py @@ -12,6 +12,7 @@ import config.auth import config.builders +import config.release_builders import config.schedulers import config.workers import config.status @@ -22,4 +23,5 @@ reload(config.workers) reload(config.schedulers) reload(config.builders) +reload(release_builders) reload(config.status) diff --git a/buildbot/osuosl/master/config/builders.py b/buildbot/osuosl/master/config/builders.py --- a/buildbot/osuosl/master/config/builders.py +++ b/buildbot/osuosl/master/config/builders.py @@ -1,4 +1,6 @@ -# TODO: Rename workers with "slave" as a part of the name. +from importlib import reload + +from buildbot.plugins import util from buildbot.process.properties import WithProperties @@ -19,7 +21,11 @@ from zorg.buildbot.builders import FuchsiaBuilder from zorg.buildbot.builders import XToolchainBuilder -from buildbot.plugins import util +from zorg.buildbot.builders import HtmlDocsBuilder +from zorg.buildbot.builders import DoxygenDocsBuilder + +reload(HtmlDocsBuilder) +reload(DoxygenDocsBuilder) all = [ @@ -38,6 +44,7 @@ depends_on_projects=['llvm','clang','clang-tools-extra','compiler-rt'], extra_configure_args=[ "-DCOMPILER_RT_BUILD_BUILTINS:BOOL=OFF", + "-DCOMPILER_RT_BUILD_ORC:BOOL=OFF", "-DCOMPILER_RT_BUILD_SANITIZERS:BOOL=OFF", "-DCOMPILER_RT_BUILD_XRAY:BOOL=OFF", "-DCOMPILER_RT_INCLUDE_TESTS:BOOL=OFF", @@ -49,29 +56,6 @@ 'CC': 'ccache clang', 'CXX': 'ccache clang++', 'CCACHE_CPP2': 'yes', })}, - {'name' : "clang-x86_64-debian-new-pass-manager-fast", - 'tags' : ["clang", "fast"], - 'workernames': ["gribozavr4"], - 'builddir': "clang-x86_64-debian-new-pass-manager-fast", - 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( - llvm_srcdir="llvm.src", - obj_dir="llvm.obj", - clean=True, - depends_on_projects=['llvm','clang','clang-tools-extra','compiler-rt'], - extra_configure_args=[ - "-DCOMPILER_RT_BUILD_BUILTINS:BOOL=OFF", - "-DCOMPILER_RT_BUILD_SANITIZERS:BOOL=OFF", - "-DCOMPILER_RT_BUILD_XRAY:BOOL=OFF", - "-DCOMPILER_RT_INCLUDE_TESTS:BOOL=OFF", - "-DENABLE_EXPERIMENTAL_NEW_PASS_MANAGER=ON", - "-DCMAKE_C_FLAGS=-Wdocumentation -Wno-documentation-deprecated-sync", - "-DCMAKE_CXX_FLAGS=-std=c++11 -Wdocumentation -Wno-documentation-deprecated-sync", - ], - env={ - 'PATH':'/home/llvmbb/bin/clang-latest/bin:/home/llvmbb/bin:/usr/local/bin:/usr/local/bin:/usr/bin:/bin', - 'CC': 'ccache clang', 'CXX': 'ccache clang++', 'CCACHE_CPP2': 'yes', - })}, - {'name' : "llvm-clang-x86_64-win-fast", 'tags' : ["clang", "fast"], 'collapseRequests': False, @@ -92,6 +76,27 @@ "-DLLVM_OPTIMIZED_TABLEGEN=OFF", "-DLLVM_LIT_ARGS=-v --threads=32"])}, + {'name': "llvm-clang-x86_64-sie-ubuntu-fast", + 'tags' : ["clang", "llvm", "clang-tools-extra", "lld", "cross-project-tests"], + 'workernames': ["sie-linux-builder"], + 'builddir': "llvm-clang-x86_64-sie-ubuntu-fast", + 'factory': UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + depends_on_projects=['llvm','clang','clang-tools-extra','lld','cross-project-tests'], + extra_configure_args=[ + "-DCMAKE_C_COMPILER=gcc", + "-DCMAKE_CXX_COMPILER=g++", + "-DCMAKE_BUILD_TYPE=Release", + "-DCLANG_ENABLE_ARCMT=OFF", + "-DCLANG_ENABLE_CLANGD=OFF", + "-DLLVM_BUILD_RUNTIME=OFF", + "-DLLVM_CCACHE_BUILD=ON", + "-DLLVM_INCLUDE_EXAMPLES=OFF", + "-DLLVM_DEFAULT_TARGET_TRIPLE=x86_64-scei-ps4", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_LIT_ARGS=--verbose -j100", + "-DLLVM_TARGETS_TO_BUILD=X86", + "-DLLVM_USE_LINKER=gold"])}, + # Expensive checks builders. {'name' : "llvm-clang-x86_64-expensive-checks-ubuntu", @@ -99,7 +104,7 @@ 'workernames' : ["as-builder-4"], 'builddir': "llvm-clang-x86_64-expensive-checks-ubuntu", 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( - depends_on_projects=["llvm"], + depends_on_projects=["llvm", "lld"], clean=True, extra_configure_args=[ "-DLLVM_ENABLE_EXPENSIVE_CHECKS=ON", @@ -109,17 +114,17 @@ "-DLLVM_LIT_ARGS=-vv -j32"])}, {'name' : "llvm-clang-x86_64-expensive-checks-win", - 'tags' : ["llvm", "clang", "expensive-checks"], + 'tags' : ["llvm", "expensive-checks"], 'workernames' : ["as-worker-93"], 'builddir': "llvm-clang-x86_64-expensive-checks-win", 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaWithMSVCBuildFactory( vs="autodetect", + depends_on_projects=["llvm", "lld"], clean=True, extra_configure_args=[ "-DLLVM_ENABLE_EXPENSIVE_CHECKS=ON", "-DLLVM_ENABLE_WERROR=OFF", - "-DCMAKE_BUILD_TYPE=Debug", - "-DLLVM_LIT_ARGS=-vv"])}, + "-DCMAKE_BUILD_TYPE=Debug"])}, {'name' : "llvm-clang-x86_64-expensive-checks-debian", 'tags' : ["llvm", "expensive-checks"], @@ -127,7 +132,7 @@ 'workernames' : ["gribozavr4"], 'builddir': "llvm-clang-x86_64-expensive-checks-debian", 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( - depends_on_projects=["llvm"], + depends_on_projects=["llvm", "lld"], clean=True, extra_configure_args=[ "-DLLVM_ENABLE_EXPENSIVE_CHECKS=ON", @@ -171,6 +176,8 @@ ]) ], extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=ARM", + "-DTARGET_TRIPLE=armv7-linux-gnueabihf", "-DDEFAULT_SYSROOT=C:/buildbot/.arm-ubuntu", "-DLLVM_LIT_ARGS=-v -vv --threads=32", WithProperties("%(remote_test_host:+-DREMOTE_TEST_HOST=)s%(remote_test_host:-)s"), @@ -208,7 +215,7 @@ ], extra_configure_args=[ "-DLLVM_TARGETS_TO_BUILD=AArch64", - "-DCMAKE_C_COMPILER_TARGET=aarch64-linux-gnu", + "-DTARGET_TRIPLE=aarch64-linux-gnu", "-DDEFAULT_SYSROOT=C:/buildbot/.aarch64-ubuntu", "-DLLVM_LIT_ARGS=-v -vv --threads=32", WithProperties("%(remote_test_host:+-DREMOTE_TEST_HOST=)s%(remote_test_host:-)s"), @@ -229,7 +236,10 @@ "-DLLVM_DEFAULT_TARGET_TRIPLE=aarch64-windows-msvc", "-DLLVM_HOST_TRIPLE=aarch64-windows-msvc", "-DLLVM_TARGET_ARCH=AArch64", + # FIXME: compiler-rt\lib\sanitizer_common\sanitizer_unwind_win.cpp assumes WIN64 is x86_64, + # so, before that's fixed, disable everything that triggers its build. "-DCOMPILER_RT_BUILD_SANITIZERS=OFF", + "-DCOMPILER_RT_BUILD_MEMPROF=OFF", "-DCOMPILER_RT_BUILD_XRAY=OFF"])}, # Cortex-A15 LNT test-suite in Benchmark mode @@ -242,7 +252,6 @@ checkout_compiler_rt=False, checkout_lld=False, test=False, - useTwoStage=False, runTestSuite=True, testsuite_flags=[ '--cppflags', '-O3 -mcpu=cortex-a15 -mthumb', @@ -257,27 +266,25 @@ testerName='LNT-Thumb2v7-A15-O3')}, # ARMv7 LNT test-suite in test-only mode - {'name' : "clang-cmake-armv7-lnt", + {'name' : "clang-armv7-lnt", 'tags' : ["clang"], - 'workernames' : ["linaro-armv7-lnt"], - 'builddir': "clang-cmake-armv7-lnt", + 'workernames' : ["linaro-clang-armv7-lnt"], + 'builddir': "clang-armv7-lnt", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, checkout_compiler_rt=False, checkout_lld=False, test=False, - useTwoStage=False, runTestSuite=True, testsuite_flags=[ '--cppflags', '-mcpu=cortex-a15 -marm', - '--threads=32', '--build-threads=32'], - extra_cmake_args=["-DLLVM_TARGETS_TO_BUILD='ARM'"])}, + '--threads=32', '--build-threads=32'])}, - ## ARMv7 check-all self-host NEON with CMake builder - {'name' : "clang-cmake-armv7-selfhost-neon", + ## ARMv7 check-all 2-stage + {'name' : "clang-armv7-2stage", 'tags' : ["clang"], - 'workernames': ["linaro-armv7-selfhost"], - 'builddir':"clang-cmake-armv7-selfhost-neon", + 'workernames': ["linaro-clang-armv7-2stage"], + 'builddir':"clang-armv7-2stage", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, checkout_compiler_rt=False, @@ -286,42 +293,38 @@ testStage1=False, extra_cmake_args=[ "-DCMAKE_C_FLAGS='-mcpu=cortex-a15 -marm'", - "-DCMAKE_CXX_FLAGS='-mcpu=cortex-a15 -marm'", - "-DLLVM_TARGETS_TO_BUILD='ARM;AArch64'"])}, + "-DCMAKE_CXX_FLAGS='-mcpu=cortex-a15 -marm'"])}, - ## ARMv7 check-all with CMake builder - {'name' : "clang-cmake-armv7-quick", + ## ARMv7 check-all + {'name' : "clang-armv7-quick", 'tags' : ["clang"], - 'workernames':["linaro-armv7-quick"], - 'builddir':"clang-cmake-armv7-quick", + 'workernames':["linaro-clang-armv7-quick"], + 'builddir':"clang-armv7-quick", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, checkout_compiler_rt=False, checkout_lld=False, - extra_cmake_args=["-DLLVM_TARGETS_TO_BUILD='ARM;AArch64'"])}, + extra_cmake_args=["-DLLVM_TARGETS_TO_BUILD='ARM'"])}, - ## ARMv7 Clang + LLVM run test-suite with GlobalISel enabled - {'name' : "clang-cmake-armv7-global-isel", + ## ARMv7 run test-suite with GlobalISel enabled + {'name' : "clang-armv7-global-isel", 'tags' : ["clang"], - 'workernames':["linaro-armv7-global-isel"], - 'builddir':"clang-cmake-armv7-global-isel", + 'workernames':["linaro-clang-armv7-global-isel"], + 'builddir':"clang-armv7-global-isel", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, checkout_compiler_rt=False, checkout_lld=False, - test=True, - useTwoStage=False, runTestSuite=True, testsuite_flags=[ '--cppflags', '-mcpu=cortex-a15 -marm -O0 -mllvm -global-isel -mllvm -global-isel-abort=0', - '--threads=32', '--build-threads=32'], - extra_cmake_args=["-DLLVM_TARGETS_TO_BUILD='ARM;AArch64'"])}, + '--threads=32', '--build-threads=32'])}, - ## ARMv7 check-all self-host with CMake builder - {'name' : "clang-cmake-armv7-selfhost", + ## ARMv7 VFPv3 check-all 2-stage + {'name' : "clang-armv7-vfpv3-2stage", 'tags' : ["clang"], - 'workernames' : ["linaro-armv7-selfhost"], - 'builddir': "clang-cmake-armv7-selfhost", + 'workernames' : ["linaro-clang-armv7-vfpv3-2stage"], + 'builddir': "clang-armv7-vfpv3-2stage", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, checkout_compiler_rt=False, @@ -330,36 +333,26 @@ testStage1=False, extra_cmake_args=[ "-DCMAKE_C_FLAGS='-mcpu=cortex-a15 -mfpu=vfpv3 -marm'", - "-DCMAKE_CXX_FLAGS='-mcpu=cortex-a15 -mfpu=vfpv3 -marm'", - "-DLLVM_TARGETS_TO_BUILD='ARM;AArch64'"])}, + "-DCMAKE_CXX_FLAGS='-mcpu=cortex-a15 -mfpu=vfpv3 -marm'"])}, - ## AArch64 Clang+LLVM check-all + test-suite - {'name' : "clang-cmake-aarch64-quick", + ## AArch64 check-all + {'name' : "clang-aarch64-quick", 'tags' : ["clang"], - 'workernames' : ["linaro-aarch64-quick"], - 'builddir': "clang-cmake-aarch64-quick", + 'workernames' : ["linaro-clang-aarch64-quick"], + 'builddir': "clang-aarch64-quick", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, checkout_compiler_rt=False, checkout_lld=False, - test=True, - useTwoStage=False, - runTestSuite=True, - testsuite_flags=[ - '--cppflags', '-mcpu=cortex-a57', - '--threads=32', '--build-threads=32'], - extra_cmake_args=["-DLLVM_TARGETS_TO_BUILD='ARM;AArch64'"])}, + extra_cmake_args=["-DLLVM_TARGETS_TO_BUILD='AArch64'"])}, - ## AArch64 Self-hosting Clang+LLVM check-all + LLD + test-suite - {'name' : "clang-cmake-aarch64-lld", + ## AArch64 check-all + LLD + test-suite 2-stage + {'name' : "clang-aarch64-lld-2stage", 'tags' : ["lld"], - 'workernames' : ["linaro-aarch64-lld"], - 'builddir':"clang-cmake-aarch64-lld", + 'workernames' : ["linaro-clang-aarch64-lld-2stage"], + 'builddir':"clang-aarch64-lld-2stage", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, - checkout_compiler_rt=True, - checkout_lld=True, - test=True, useTwoStage=True, runTestSuite=True, testsuite_flags=[ @@ -368,55 +361,48 @@ extra_cmake_args=[ "-DCMAKE_C_FLAGS='-mcpu=cortex-a57'", "-DCMAKE_CXX_FLAGS='-mcpu=cortex-a57'", - "-DLLVM_TARGETS_TO_BUILD='ARM;AArch64'", "-DLLVM_ENABLE_LLD=True"])}, - ## AArch64 Clang+LLVM run test-suite at -O0 (GlobalISel is now default). - {'name' : "clang-cmake-aarch64-global-isel", + ## AArch64 run test-suite at -O0 (GlobalISel is now default). + {'name' : "clang-aarch64-global-isel", 'tags' : ["clang"], - 'workernames' : ["linaro-aarch64-global-isel"], - 'builddir': "clang-cmake-aarch64-global-isel", + 'workernames' : ["linaro-clang-aarch64-global-isel"], + 'builddir': "clang-aarch64-global-isel", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, checkout_compiler_rt=False, checkout_lld=False, - test=True, - useTwoStage=False, runTestSuite=True, testsuite_flags=[ '--cppflags', '-O0', - '--threads=32', '--build-threads=32'], - extra_cmake_args=["-DLLVM_TARGETS_TO_BUILD='ARM;AArch64'"])}, + '--threads=32', '--build-threads=32'])}, - ## ARMv7 without neon; check-all 2-stage full compiler-rt + testsuite - {'name' : "clang-cmake-armv7-full", + ## ARMv7 VFPv3 check-all + compiler-rt + testsuite 2-stage + {'name' : "clang-armv7-vfpv3-full-2stage", 'tags' : ["clang"], 'workernames' : ["linaro-tk1-06", "linaro-tk1-07", "linaro-tk1-08", "linaro-tk1-09"], - 'builddir': "clang-cmake-armv7-full", + 'builddir': "clang-armv7-vfpv3-full-2stage", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, checkout_lld=False, - checkout_compiler_rt=True, - testStage1=False, useTwoStage=True, + testStage1=False, runTestSuite=True, testsuite_flags=[ '--cppflags', '-mcpu=cortex-a15 -mfpu=vfpv3 -marm', '--threads=4', '--build-threads=4'], extra_cmake_args=[ "-DCOMPILER_RT_TEST_COMPILER_CFLAGS='-mcpu=cortex-a15 -mfpu=vfpv3 -marm'", - "-DLLVM_TARGETS_TO_BUILD='ARM'", "-DLLVM_PARALLEL_LINK_JOBS=2"])}, - ## ARMv7 Thumb2 with neon; check-all 2-stage full compiler-rt + testsuite - {'name' : "clang-cmake-thumbv7-full-sh", + ## ARMv7 Thumb2 check-all + compiler-rt + testsuite 2-stage + {'name' : "clang-thumbv7-full-2stage", 'tags' : ["clang"], 'workernames' : ["linaro-tk1-01", "linaro-tk1-03", "linaro-tk1-04", "linaro-tk1-05"], - 'builddir': "clang-cmake-thumbv7-full-sh", + 'builddir': "clang-thumbv7-full-2stage", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, checkout_lld=False, - checkout_compiler_rt=True, testStage1=False, useTwoStage=True, runTestSuite=True, @@ -427,20 +413,16 @@ "-DCMAKE_C_FLAGS='-mcpu=cortex-a15 -mthumb'", "-DCMAKE_CXX_FLAGS='-mcpu=cortex-a15 -mthumb'", "-DCOMPILER_RT_TEST_COMPILER_CFLAGS='-mcpu=cortex-a15 -mthumb'", - "-DLLVM_TARGETS_TO_BUILD='ARM'", "-DLLVM_PARALLEL_LINK_JOBS=2"])}, ## AArch32 Self-hosting Clang+LLVM check-all + LLD + test-suite # Sanitizers build disabled due to PR38690 - {'name' : "clang-cmake-armv8-lld", + {'name' : "clang-armv8-lld-2stage", 'tags' : ["lld"], - 'workernames' : ["linaro-armv8-lld"], - 'builddir': "clang-cmake-armv8-lld", + 'workernames' : ["linaro-clang-armv8-lld-2stage"], + 'builddir': "clang-armv8-lld-2stage", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, - checkout_compiler_rt=True, - checkout_lld=True, - test=True, useTwoStage=True, runTestSuite=True, testsuite_flags=[ @@ -449,21 +431,18 @@ extra_cmake_args=[ "-DCMAKE_C_FLAGS='-mcpu=cortex-a57'", "-DCMAKE_CXX_FLAGS='-mcpu=cortex-a57'", - "-DLLVM_TARGETS_TO_BUILD='ARM;AArch64'", "-DCOMPILER_RT_BUILD_SANITIZERS=OFF", "-DLLVM_ENABLE_LLD=True"])}, - # AArch64 Clang+LLVM+RT check-all + flang + test-suite + self-hosting - {'name' : "clang-cmake-aarch64-full", + # AArch64 check-all + flang + compiler-rt + test-suite 2-stage + {'name' : "clang-aarch64-full-2stage", 'tags' : ["clang"], - 'workernames' : ["linaro-aarch64-full"], - 'builddir': "clang-cmake-aarch64-full", + 'workernames' : ["linaro-clang-aarch64-full-2stage"], + 'builddir': "clang-aarch64-full-2stage", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, - checkout_compiler_rt=True, checkout_flang=True, checkout_lld=False, - test=True, useTwoStage=True, testStage1=False, runTestSuite=True, @@ -472,8 +451,43 @@ '--threads=32', '--build-threads=32'], extra_cmake_args=[ "-DCMAKE_C_FLAGS='-mcpu=cortex-a57'", - "-DCMAKE_CXX_FLAGS='-mcpu=cortex-a57'", - "-DLLVM_TARGETS_TO_BUILD='ARM;AArch64'"])}, + "-DCMAKE_CXX_FLAGS='-mcpu=cortex-a57'"])}, + + # AArch64 Clang+LLVM+RT+LLD check-all + flang + test-suite w/SVE + {'name' : "clang-aarch64-sve-vls", + 'tags' : ["clang"], + 'workernames' : ["linaro-clang-aarch64-sve-vls"], + 'builddir': "clang-aarch64-sve-vls", + 'factory' : ClangBuilder.getClangCMakeBuildFactory( + clean=False, + checkout_flang=True, + runTestSuite=True, + testsuite_flags=[ + '--cppflags', '-mcpu=a64fx -mllvm -aarch64-sve-vector-bits-min=512', + '--threads=48', '--build-threads=48'], + extra_cmake_args=[ + "-DCMAKE_C_FLAGS='-mcpu=a64fx'", + "-DCMAKE_CXX_FLAGS='-mcpu=a64fx'", + "-DLLVM_ENABLE_LLD=True"])}, + + # AArch64 Clang+LLVM+RT+LLD check-all + flang + test-suite 2-stage w/SVE + {'name' : "clang-aarch64-sve-vls-2stage", + 'tags' : ["clang"], + 'workernames' : ["linaro-clang-aarch64-sve-vls-2stage"], + 'builddir': "clang-aarch64-sve-vls-2stage", + 'factory' : ClangBuilder.getClangCMakeBuildFactory( + clean=False, + checkout_flang=True, + useTwoStage=True, + testStage1=False, + runTestSuite=True, + testsuite_flags=[ + '--cppflags', '-mcpu=a64fx -mllvm -aarch64-sve-vector-bits-min=512', + '--threads=48', '--build-threads=48'], + extra_cmake_args=[ + "-DCMAKE_C_FLAGS='-mcpu=a64fx -mllvm -aarch64-sve-vector-bits-min=512'", + "-DCMAKE_CXX_FLAGS='-mcpu=a64fx -mllvm -aarch64-sve-vector-bits-min=512'", + "-DLLVM_ENABLE_LLD=True"])}, {'name' : "clang-arm64-windows-msvc-2stage", 'tags' : ["clang"], @@ -488,7 +502,10 @@ "-DLLVM_DEFAULT_TARGET_TRIPLE=aarch64-windows-msvc", "-DLLVM_HOST_TRIPLE=aarch64-windows-msvc", "-DLLVM_TARGET_ARCH=AArch64", + # FIXME: compiler-rt\lib\sanitizer_common\sanitizer_unwind_win.cpp assumes WIN64 is x86_64, + # so, before that's fixed, disable everything that triggers its build. "-DCOMPILER_RT_BUILD_SANITIZERS=OFF", + "-DCOMPILER_RT_BUILD_MEMPROF=OFF", "-DCOMPILER_RT_BUILD_XRAY=OFF"])}, {'name' : 'clang-x64-windows-msvc', @@ -499,6 +516,33 @@ script="clang-windows.py", depends_on_projects=['llvm', 'clang', 'lld', 'debuginfo-tests'])}, + {'name' : "clang-m68k-linux", + 'tags' : ["clang"], + 'workernames' : ["debian-akiko-m68k"], + 'builddir': "clang-m68k-linux", + 'factory' : ClangBuilder.getClangCMakeBuildFactory( + clean=False, + checkout_lld=False, + useTwoStage=False, + stage1_config='Release', + extra_cmake_args=[ + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=M68k"])}, + + {'name' : "clang-m68k-linux-cross", + 'tags' : ["clang"], + 'workernames' : ["suse-gary-m68k-cross"], + 'builddir': "clang-m68k-linux-cross", + 'factory' : ClangBuilder.getClangCMakeBuildFactory( + clean=False, + checkout_lld=False, + useTwoStage=False, + stage1_config='Release', + extra_cmake_args=[ + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_TARGETS_TO_BUILD=X86", + "-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=M68k"])}, + {'name' : "clang-ppc64be-linux-lnt", 'tags' : ["clang", "ppc"], 'workernames' : ["ppc64be-clang-lnt-test"], @@ -569,31 +613,52 @@ checkout_lld=False, useTwoStage=False, stage1_config='Release', - extra_cmake_args=["-DLLVM_ENABLE_ASSERTIONS=ON"])}, + extra_cmake_args=[ + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DPYTHON_EXECUTABLE:FILEPATH=/usr/local/bin/python3"])}, {'name' : "clang-ppc64le-rhel", 'tags' : ["clang", "ppc", "ppc64le"], 'workernames' : ["ppc64le-clang-rhel-test"], 'builddir': "clang-ppc64le-rhel", - 'factory' : ClangBuilder.getClangCMakeBuildFactory(clean=False, - checkout_clang_tools_extra=True, - checkout_compiler_rt=False, - checkout_lld=True, - checkout_libcxx=False, - useTwoStage=False, - runTestSuite=True, - stage1_config='Release', - nt_flags=['--threads=16', '--build-threads=16'], - extra_cmake_args=[ - '-DLLVM_ENABLE_PROJECTS=clang;llvm;clang-tools-extra;lld', - '-DLLVM_ENABLE_RUNTIMES=compiler-rt', + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + depends_on_projects=["llvm", "clang", "clang-tools-extra", "lld", "compiler-rt"], + checks=["check"], + enable_runtimes="auto", + extra_configure_args=[ "-DLLVM_ENABLE_ASSERTIONS=On", "-DCMAKE_C_COMPILER=clang", "-DCMAKE_CXX_COMPILER=clang++", "-DCLANG_DEFAULT_LINKER=lld", "-DCMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN:PATH=/opt/rh/devtoolset-7/root/usr", "-DCMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN:PATH=/opt/rh/devtoolset-7/root/usr", "-DLLVM_BINUTILS_INCDIR=/usr/include", "-DBUILD_SHARED_LIBS=ON", "-DLLVM_ENABLE_WERROR=ON", - '-DLLVM_LIT_ARGS=-vj 20'])}, + "-DCMAKE_BUILD_TYPE=Release", + "-DLLVM_LIT_ARGS=-vj 20"])}, + + {'name' : "clang-ppc64-aix-ppc64le", + 'tags' : ["clang", "aix", "ppc", "ppc64le"], + 'workernames' : ["aix-ppc64-ppc64le"], + 'builddir': "clang-ppc64-aix-ppc64le", + 'factory' : ClangBuilder.getClangCMakeBuildFactory( + clean=False, + checkout_clang_tools_extra=False, + checkout_compiler_rt=False, + checkout_lld=False, + checkout_libcxx=False, + useTwoStage=False, + runTestSuite=False, + stage1_config='Release', + extra_cmake_args=[ + "-DLLVM_ENABLE_ASSERTIONS=On", + "-DCMAKE_C_COMPILER=/opt/IBM/xlC/16.1.0/bin/xlclang", + "-DCMAKE_CXX_COMPILER=/opt/IBM/xlC/16.1.0/bin/xlclang++", + "-DCMAKE_C_FLAGS='-qmaxmem=-1 -qarch=pwr8 -qaltivec -qalias=noansi'", + "-DCMAKE_CXX_FLAGS='-qmaxmem=-1 -qarch=pwr8 -qaltivec -qalias=noansi'", + "-DPython3_EXECUTABLE:FILEPATH=/opt/freeware/bin/python3_64", + "-DLLVM_ENABLE_ZLIB=OFF", "-DLLVM_APPEND_VC_REV=OFF", + "-DLLVM_DEFAULT_TARGET_TRIPLE=powerpc64le-unknown-linux-gnu", + "-DLLVM_PARALLEL_LINK_JOBS=6", + "-DLLVM_ENABLE_WERROR=ON"])}, {'name' : "clang-s390x-linux", 'tags' : ["clang"], @@ -659,7 +724,7 @@ checkout_clang_tools_extra=False, checkout_compiler_rt=False, checkout_lld=False, - env={'LD_LIBRARY_PATH': '/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/lib', 'PATH': ['/local/cmake-3.17.0/bin', '${PATH}']}, + env={'LD_LIBRARY_PATH': '/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/lib'}, extra_cmake_args=[ "-DCMAKE_BUILD_TYPE:STRING=Release", "-DLLVM_TARGETS_TO_BUILD:STRING=Hexagon", @@ -676,8 +741,8 @@ "-DWITH_POLLY:BOOL=OFF", "-DLINK_POLLY_INTO_TOOLS:BOOL=OFF", "-DPOLLY_BUILD_SHARED_LIB:BOOL=OFF", - "-DCMAKE_C_COMPILER:FILEPATH=/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/bin/clang", - "-DCMAKE_CXX_COMPILER:FILEPATH=/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/bin/clang++"])}, + "-DCMAKE_C_COMPILER:FILEPATH=/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/bin/clang", + "-DCMAKE_CXX_COMPILER:FILEPATH=/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/bin/clang++"])}, ## X86_64 AVX2 Clang+LLVM check-all + test-suite {'name' : "clang-cmake-x86_64-avx2-linux", @@ -746,55 +811,6 @@ "-DCMAKE_CXX_FLAGS='-march=broadwell'", "-DLLVM_TARGETS_TO_BUILD='X86'"])}, - ## Armv7 build cache - {'name' : "clang-armv7-linux-build-cache", - 'tags' : ["clang"], - 'workernames' : ["packet-linux-armv7-slave-1"], - 'builddir': "clang-armv7-linux-build-cache", - 'factory' : ClangBuilder.getClangCMakeGCSBuildFactory( - stage1_config='Release', - clean=True, - checkout_compiler_rt=False, - test=False, - useTwoStage=False, - runTestSuite=False, - checkout_lld=True, - checkout_libcxx=True, - checkout_clang_tools_extra=False, - use_pixz_compression=False, - xz_compression_factor=0, - stage1_upload_directory='clang-armv7-linux', - extra_cmake_args=[ - "-DLLVM_TARGETS_TO_BUILD='ARM;AArch64;X86'", - "-DCMAKE_C_FLAGS='-mthumb'", - "-DCMAKE_CXX_FLAGS='-mthumb'", - ], - env={'BUCKET': 'llvm-build-artifacts'})}, - - ## AArch64 build cache - {'name' : "clang-aarch64-linux-build-cache", - 'tags' : ["clang"], - 'workernames' : ["packet-linux-aarch64-slave-1"], - 'builddir': "clang-aarch64-linux-build-cache", - 'factory' : ClangBuilder.getClangCMakeGCSBuildFactory( - stage1_config='Release', - clean=True, - checkout_compiler_rt=False, - test=False, - useTwoStage=False, - runTestSuite=False, - checkout_lld=True, - checkout_libcxx=True, - checkout_clang_tools_extra=False, - stage1_upload_directory='clang-aarch64-linux', - use_pixz_compression=True, - extra_cmake_args=[ - "-DLLVM_TARGETS_TO_BUILD='ARM;AArch64;X86'", - #"-DCMAKE_C_FLAGS=''", - #"-DCMAKE_CXX_FLAGS=''", - ], - env={'BUCKET': 'llvm-build-artifacts'})}, - {'name' : "llvm-avr-linux", 'tags' : ["clang"], 'workernames' : ["avr-build-01"], @@ -827,6 +843,24 @@ '-DLLVM_ENABLE_ASSERTIONS=ON', '-DLLVM_TARGETS_TO_BUILD=X86'])}, + {'name' : "clang-xcore-ubuntu-20-x64", + 'tags' : ["clang"], + 'workernames' : ["xcore-ubuntu20-x64"], + 'builddir': "clang-xcore-ubuntu-20-x64", + 'factory' : ClangBuilder.getClangCMakeBuildFactory( + jobs=4, + checkout_clang_tools_extra=False, + checkout_compiler_rt=False, + checkout_lld=False, + test=True, + testStage1=True, + useTwoStage=False, + stage1_config='Release', + extra_cmake_args=[ + "-DLLVM_TARGETS_TO_BUILD:STRING=XCore", + "-DLLVM_DEFAULT_TARGET_TRIPLE:STRING=xcore-unknown-unknown-elf", + "-DLLVM_ENABLE_THREADS:BOOL=OFF"])}, + # Polly builders. {'name' : "polly-arm-linux", @@ -838,7 +872,7 @@ install=True, make='ninja', jobs=16, - env={'LD_LIBRARY_PATH': '/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/lib', 'PATH': ['/local/cmake-3.17.0/bin', '${PATH}']}, + env={'LD_LIBRARY_PATH': '/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/lib'}, extraCmakeArgs=[ "-G", "Ninja", "-DLLVM_TARGETS_TO_BUILD='ARM;AArch64'", @@ -846,8 +880,9 @@ "-DLLVM_TARGET_ARCH=arm-linux-gnueabi", "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_ENABLE_LIBCXX:BOOL=ON", - "-DCMAKE_C_COMPILER:FILEPATH=/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/bin/clang", - "-DCMAKE_CXX_COMPILER:FILEPATH=/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/bin/clang++"])}, + "-DPOLLY_ENABLE_GPGPU_CODEGEN=OFF", # Auto-enabled in getPollyBuildFactory + "-DCMAKE_C_COMPILER:FILEPATH=/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/bin/clang", + "-DCMAKE_CXX_COMPILER:FILEPATH=/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/bin/clang++"])}, {'name' : "polly-x86_64-linux", 'tags' : ["polly"], @@ -899,14 +934,15 @@ "-DLLVM_TARGET_ARCH=arm-linux-androideabi", "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_ENABLE_LIBCXX:BOOL=ON", - "-DCMAKE_C_COMPILER:FILEPATH=/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/bin/clang", - "-DCMAKE_CXX_COMPILER:FILEPATH=/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/bin/clang++"], + "-DPOLLY_ENABLE_GPGPU_CODEGEN=OFF", # Auto-enabled in getPollyBuildFactory (used indirectly) + "-DCMAKE_C_COMPILER:FILEPATH=/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/bin/clang", + "-DCMAKE_CXX_COMPILER:FILEPATH=/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/bin/clang++"], timeout=240, target_clang=None, target_flags="-Wno-error -O3 -mllvm -polly -mllvm -polly-position=before-vectorizer -mllvm -polly-process-unprofitable -fcommon", - jobs=8, + jobs=32, extra_make_args=None, - env={'LD_LIBRARY_PATH': '/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/lib', 'PATH': ['/local/cmake-3.17.0/bin', '${PATH}']}, + env={'LD_LIBRARY_PATH': '/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/lib'}, clean=False, sync=False, patch=None)}, @@ -922,14 +958,14 @@ make='ninja', jobs=16, checkAll=True, - env={'LD_LIBRARY_PATH': '/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/lib', 'PATH': ['/local/cmake-3.17.0/bin', '${PATH}']}, + env={'LD_LIBRARY_PATH': '/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/lib'}, extraCmakeArgs=[ "-G", "Ninja", "-DLLVM_REVERSE_ITERATION:BOOL=ON", "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_ENABLE_LIBCXX:BOOL=ON", - "-DCMAKE_C_COMPILER:FILEPATH=/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/bin/clang", - "-DCMAKE_CXX_COMPILER:FILEPATH=/local/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/bin/clang++"])}, + "-DCMAKE_C_COMPILER:FILEPATH=/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/bin/clang", + "-DCMAKE_CXX_COMPILER:FILEPATH=/local/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-16.04/bin/clang++"])}, # LLDB builders. @@ -949,24 +985,24 @@ {'name' : "lldb-aarch64-ubuntu", 'tags' : ["lldb"], - 'workernames' : ["linaro-aarch64-lldb"], - 'builddir': "lldb-cmake-aarch64", + 'workernames' : ["linaro-lldb-aarch64-ubuntu"], + 'builddir': "lldb-aarch64-ubuntu", 'factory' : LLDBBuilder.getLLDBCMakeBuildFactory( test=True, extra_cmake_args=[ '-DLLVM_ENABLE_ASSERTIONS=True', - '-DLLVM_LIT_ARGS=-svj 8', + '-DLLVM_LIT_ARGS=-svj 4', '-DLLVM_USE_LINKER=gold'])}, {'name' : "lldb-arm-ubuntu", 'tags' : ["lldb"], - 'workernames' : ["linaro-arm-lldb"], - 'builddir': "lldb-cmake-arm", + 'workernames' : ["linaro-lldb-arm-ubuntu"], + 'builddir': "lldb-arm-ubuntu", 'factory' : LLDBBuilder.getLLDBCMakeBuildFactory( test=True, extra_cmake_args=[ '-DLLVM_ENABLE_ASSERTIONS=True', - '-DLLVM_LIT_ARGS=-svj 8', + '-DLLVM_LIT_ARGS=-svj 4', '-DLLVM_USE_LINKER=gold'])}, {'name' : "lldb-x64-windows-ninja", @@ -1006,17 +1042,6 @@ extra_configure_args = [ '-DLLVM_ENABLE_WERROR=OFF'])}, - {'name' : "lld-x86_64-freebsd", - 'tags' : ["lld"], - 'workernames' : ["as-worker-4"], - 'builddir': "lld-x86_64-freebsd", - 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( - depends_on_projects=['llvm', 'lld'], - extra_configure_args=[ - '-DCMAKE_EXE_LINKER_FLAGS=-lcxxrt', - '-DLLVM_ENABLE_WERROR=OFF'], - env={'CXXFLAGS' : "-std=c++11 -stdlib=libc++"})}, - {'name' : "ppc64le-lld-multistage-test", 'tags' : ["lld", "ppc", "ppc64le"], 'workernames' : ["ppc64le-lld-multistage-test"], @@ -1059,17 +1084,21 @@ 'factory' : ClangLTOBuilder.getClangWithLTOBuildFactory( jobs=72, lto='thin', - extra_configure_args=[ + extra_configure_args_lto_stage=[ '-DLLVM_CCACHE_BUILD=ON', - '-DCMAKE_CXX_FLAGS="-O3 -Xclang -fwhole-program-vtables -fno-split-lto-unit"', - '-DCMAKE_C_FLAGS="-O3 -Xclang -fwhole-program-vtables -fno-split-lto-unit"', - '-DCMAKE_EXE_LINKER_FLAGS="-Wl,--lto-whole-program-visibility"'])}, + '-DCMAKE_CXX_FLAGS=-O3 -Xclang -fwhole-program-vtables -fno-split-lto-unit', + '-DCMAKE_C_FLAGS=-O3 -Xclang -fwhole-program-vtables -fno-split-lto-unit', + '-DCMAKE_EXE_LINKER_FLAGS=-Wl,--lto-whole-program-visibility -fuse-ld=lld'])}, {'name' : "clang-with-lto-ubuntu", 'tags' : ["clang","lld","LTO"], 'workernames' : ["as-worker-91"], 'builddir': "clang-with-lto-ubuntu", - 'factory' : ClangLTOBuilder.getClangWithLTOBuildFactory(jobs=72)}, + 'factory' : ClangLTOBuilder.getClangWithLTOBuildFactory( + jobs=72, + extra_configure_args_lto_stage=[ + '-DLLVM_PARALLEL_LINK_JOBS=14', + ])}, # Builders for MLIR. @@ -1086,11 +1115,14 @@ '-DLLVM_BUILD_EXAMPLES=ON', '-DLLVM_TARGETS_TO_BUILD=host;NVPTX', '-DLLVM_ENABLE_PROJECTS=mlir', - '-DMLIR_CUDA_RUNNER_ENABLED=1', + '-DMLIR_ENABLE_CUDA_RUNNER=1', + '-DMLIR_INCLUDE_INTEGRATION_TESTS=ON', '-DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc', - '-DMLIR_VULKAN_RUNNER_ENABLED=1', + '-DMLIR_ENABLE_VULKAN_RUNNER=1', '-DBUILD_SHARED_LIBS=ON', '-DLLVM_CCACHE_BUILD=ON', + '-DMLIR_ENABLE_BINDINGS_PYTHON=ON', + '-DMLIR_RUN_CUDA_TENSOR_CORE_TESTS=ON', ], env={ 'CC':'clang', @@ -1133,49 +1165,82 @@ 'CC': 'clang', 'CXX': 'clang++', 'LD': 'lld', + 'LD_LIBRARY_PATH': '/usr/lib64', })}, # Sanitizer builders. {'name' : "sanitizer-x86_64-linux", 'tags' : ["sanitizer"], - 'workernames' : ["sanitizer-buildbot1", "sanitizer-buildbot2"], + 'workernames' : [ + "sanitizer-buildbot1", + "sanitizer-buildbot2", + ], 'builddir': "sanitizer-x86_64-linux", 'factory' : SanitizerBuilder.getSanitizerBuildFactory()}, {'name' : "sanitizer-x86_64-linux-fast", 'tags' : ["sanitizer"], - 'workernames' : ["sanitizer-buildbot1", "sanitizer-buildbot2"], + 'workernames' : [ + "sanitizer-buildbot1", + "sanitizer-buildbot2", + "sanitizer-buildbot7", + "sanitizer-buildbot8", + ], 'builddir': "sanitizer-x86_64-linux-fast", 'factory' : SanitizerBuilder.getSanitizerBuildFactory()}, - {'name' : "sanitizer-x86_64-linux-bootstrap", + {'name' : "sanitizer-x86_64-linux-bootstrap-asan", 'tags' : ["sanitizer"], - 'workernames' : ["sanitizer-buildbot3", "sanitizer-buildbot4"], - 'builddir': "sanitizer-x86_64-linux-bootstrap", + 'workernames' : [ + "sanitizer-buildbot3", + "sanitizer-buildbot4", + ], + 'builddir': "sanitizer-x86_64-linux-bootstrap-asan", 'factory' : SanitizerBuilder.getSanitizerBuildFactory()}, {'name' : "sanitizer-x86_64-linux-bootstrap-ubsan", 'tags' : ["sanitizer"], - 'workernames' : ["sanitizer-buildbot3", "sanitizer-buildbot4"], + 'workernames' : [ + "sanitizer-buildbot3", + "sanitizer-buildbot4", + ], 'builddir': "sanitizer-x86_64-linux-bootstrap-ubsan", 'factory' : SanitizerBuilder.getSanitizerBuildFactory()}, {'name' : "sanitizer-x86_64-linux-bootstrap-msan", 'tags' : ["sanitizer"], - 'workernames' : ["sanitizer-buildbot7", "sanitizer-buildbot8"], + 'workernames' : [ + "sanitizer-buildbot3", + "sanitizer-buildbot4", + ], 'builddir': "sanitizer-x86_64-linux-bootstrap-msan", 'factory' : SanitizerBuilder.getSanitizerBuildFactory()}, {'name' : "sanitizer-x86_64-linux-autoconf", 'tags' : ["sanitizer"], - 'workernames' : ["sanitizer-buildbot7", "sanitizer-buildbot8"], + 'workernames' : [ + "sanitizer-buildbot7", + "sanitizer-buildbot8", + ], 'builddir': "sanitizer-x86_64-linux-autoconf", 'factory' : SanitizerBuilder.getSanitizerBuildFactory()}, + {'name' : "sanitizer-x86_64-linux-qemu", + 'tags' : ["sanitizer"], + 'workernames' : [ + "sanitizer-buildbot7", + "sanitizer-buildbot8", + ], + 'builddir': "sanitizer-x86_64-linux-qemu", + 'factory' : SanitizerBuilder.getSanitizerBuildFactory()}, + {'name' : "sanitizer-x86_64-linux-fuzzer", 'tags' : ["sanitizer"], - 'workernames' : ["sanitizer-buildbot7", "sanitizer-buildbot8"], + 'workernames' : [ + "sanitizer-buildbot7", + "sanitizer-buildbot8", + ], 'builddir': "sanitizer-x86_64-linux-fuzzer", 'factory' : SanitizerBuilder.getSanitizerBuildFactory()}, @@ -1195,7 +1260,10 @@ 'tags' : ["sanitizer", "ppc", "ppc64le"], 'workernames' : ["ppc64le-sanitizer"], 'builddir': "sanitizer-ppc64le", - 'factory' : SanitizerBuilder.getSanitizerBuildFactory(timeout=1800)}, + 'factory' : SanitizerBuilder.getSanitizerBuildFactory( + timeout=1800, + extra_configure_args=[ + "-CMAKE_ARGS='-DLLVM_LIT_ARGS=-v -j256'"])}, {'name' : "sanitizer-windows", 'tags' : ["sanitizer"], @@ -1205,6 +1273,8 @@ script="sanitizer-windows.py", depends_on_projects=["llvm", "clang", "lld", "compiler-rt"])}, +# OpenMP builders. + {'name' : "openmp-gcc-x86_64-linux-debian", 'tags' : ["openmp"], 'workernames' : ["gribozavr4"], @@ -1225,115 +1295,131 @@ 'CC': 'ccache clang', 'CXX': 'ccache clang++', 'CCACHE_CPP2': 'yes', })}, -# Libc++ builders. - - {'name': 'libcxx-libcxxabi-x86_64-linux-debian', - 'tags' : ["libcxx"], - 'workernames': ['gribozavr4'], - 'builddir': 'libcxx-libcxxabi-x86_64-linux-debian', - 'factory': LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( - env={'CC': 'clang', 'CXX': 'clang++'}, - lit_extra_args=['--shuffle'], - check_libcxx_abilist=True)}, - - {'name' : 'libcxx-libcxxabi-x86_64-linux-debian-noexceptions', - 'tags' : ["libcxx"], - 'workernames' : ['gribozavr4'], - 'builddir': 'libcxx-libcxxabi-x86_64-linux-debian-noexceptions', - 'factory' : LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( - env={'CC': 'clang', 'CXX': 'clang++'}, - use_cache='Generic-noexceptions.cmake', - lit_extra_args=['--shuffle'])}, + {'name' : "openmp-offload-cuda-project", + 'tags' : ["openmp"], + 'workernames' : ["minipc-1050ti-linux"], + 'builddir': "openmp-offload-cuda-project", + 'factory' : OpenMPBuilder.getOpenMPCMakeBuildFactory( + clean=False, + enable_runtimes=[], + extraCmakeArgs=[ + "-DCUDA_TOOLKIT_ROOT_DIR=/opt/cuda", + "-DLIBOMPTARGET_BUILD_NVPTX_BCLIB=ON", + "-DCLANG_ENABLE_STATIC_ANALYZER=OFF", + "-DCLANG_ENABLE_ARCMT=OFF", + "-DCLANG_ENABLE_OBJC_REWRITER=OFF", + "-DLLVM_TARGETS_TO_BUILD=X86;NVPTX", + "-DLLVM_ENABLE_LLD=ON", + '-DLLVM_PARALLEL_LINK_JOBS=2', + ], + install=True, + testsuite=True, + testsuite_sollvevv=True, + extraTestsuiteCmakeArgs=[ + "-DTEST_SUITE_SOLLVEVV_OFFLOADING_CFLAGS=-fopenmp-targets=nvptx64-nvidia-cuda;--cuda-path=/opt/cuda", + "-DTEST_SUITE_SOLLVEVV_OFFLOADING_LDFLAGS=-fopenmp-targets=nvptx64-nvidia-cuda;--cuda-path=/opt/cuda", + ], + )}, - {'name' : 'libcxx-libcxxabi-libunwind-x86_64-linux-debian', - 'tags' : ["libcxx"], - 'workernames' : ['gribozavr4'], - 'builddir': 'libcxx-libcxxabi-libunwind-x86_64-linux-debian', - 'factory' : LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( - env={'CC': 'clang', 'CXX': 'clang++'}, - cmake_extra_opts={'LIBCXXABI_USE_LLVM_UNWINDER': 'ON'}, - lit_extra_args=['--shuffle'])}, + {'name' : "openmp-offload-cuda-runtime", + 'tags' : ["openmp"], + 'workernames' : ["minipc-1050ti-linux"], + 'builddir': "openmp-offload-cuda-runtime", + 'factory' : OpenMPBuilder.getOpenMPCMakeBuildFactory( + clean=True, + enable_runtimes=['openmp'], + extraCmakeArgs=[ + "-DCUDA_TOOLKIT_ROOT_DIR=/opt/cuda", + "-DLIBOMPTARGET_BUILD_NVPTX_BCLIB=ON", + "-DCLANG_ENABLE_STATIC_ANALYZER=OFF", + "-DCLANG_ENABLE_ARCMT=OFF", + "-DCLANG_ENABLE_OBJC_REWRITER=OFF", + "-DLLVM_TARGETS_TO_BUILD=X86;NVPTX", + "-DLLVM_ENABLE_LLD=ON", + '-DLLVM_PARALLEL_LINK_JOBS=2', + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", + ], + install=True, + testsuite=True, + testsuite_sollvevv=True, + extraTestsuiteCmakeArgs=[ + "-DTEST_SUITE_SOLLVEVV_OFFLOADING_CFLAGS=-fopenmp-targets=nvptx64-nvidia-cuda;--cuda-path=/opt/cuda", + "-DTEST_SUITE_SOLLVEVV_OFFLOADING_LDFLAGS=-fopenmp-targets=nvptx64-nvidia-cuda;--cuda-path=/opt/cuda", + ], + )}, - {'name' : 'libcxx-libcxxabi-singlethreaded-x86_64-linux-debian', - 'tags' : ["libcxx"], - 'workernames' : ['gribozavr4'], - 'builddir': 'libcxx-libcxxabi-singlethreaded-x86_64-linux-debian', - 'factory' : LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( - env={'CC': 'clang', 'CXX': 'clang++'}, - use_cache='Generic-singlethreaded.cmake')}, +# OpenMP AMDGPU Builders + {'name' : "openmp-offload-amdgpu-project", + 'tags' : ["openmp"], + 'workernames' : ["omp-vega20-0"], + 'builddir': "openmp-offload-amdgpu-project", + 'factory' : OpenMPBuilder.getOpenMPCMakeBuildFactory( + clean=False, + enable_runtimes=[], + depends_on_projects=['llvm','clang','lld','openmp'], + extraCmakeArgs=[ + "-DCMAKE_BUILD_TYPE=Release", + "-DCLANG_DEFAULT_LINKER=lld", + "-DLLVM_TARGETS_TO_BUILD=X86;AMDGPU", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_ENABLE_RUNTIMES=openmp", + ], + install=True, + testsuite=False, + testsuite_sollvevv=False, + extraTestsuiteCmakeArgs=[ + "-DTEST_SUITE_SOLLVEVV_OFFLOADING_CFLAGS=-fopenmp-targets=amdgcn-amd-amdhsa;-Xopenmp-target=amdgcn-amd-amdhsa", + "-DTEST_SUITE_SOLLVEVV_OFFLOADING_LDLAGS=-fopenmp-targets=amdgcn-amd-amdhsa;-Xopenmp-target=amdgcn-amd-amdhsa", + ], + )}, - # ARMv7 LibC++ and LibC++abi tests (require Clang+RT) - {'name' : 'libcxx-libcxxabi-libunwind-armv7-linux', - 'tags' : ["libcxx"], - 'workernames': ['linaro-tk1-02'], - 'builddir': 'libcxx-libcxxabi-libunwind-armv7-linux', - 'factory' : LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( - cmake_extra_opts={ - 'LIBCXXABI_USE_LLVM_UNWINDER': 'ON', - 'CMAKE_C_FLAGS': '-mcpu=cortex-a15 -marm', - 'CMAKE_CXX_FLAGS': '-mcpu=cortex-a15 -marm', - 'LLVM_PARALLEL_LINK_JOBS': '2'})}, + {'name' : "openmp-offload-amdgpu-runtime", + 'tags' : ["openmp"], + 'workernames' : ["omp-vega20-0"], + 'builddir': "openmp-offload-amdgpu-runtime", + 'factory' : OpenMPBuilder.getOpenMPCMakeBuildFactory( + clean=True, + enable_runtimes=['openmp'], + depends_on_projects=['llvm','clang','lld','openmp'], + extraCmakeArgs=[ + "-DCMAKE_BUILD_TYPE=Release", + "-DCLANG_DEFAULT_LINKER=lld", + "-DLLVM_TARGETS_TO_BUILD=X86;AMDGPU", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_ENABLE_RUNTIMES=openmp", + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", + ], + install=True, + testsuite=False, + testsuite_sollvevv=False, + extraTestsuiteCmakeArgs=[ + "-DTEST_SUITE_SOLLVEVV_OFFLOADING_CFLAGS=-fopenmp-targets=amdgcn-amd-amdhsa;-Xopenmp-target=amdgcn-amd-amdhsa", + "-DTEST_SUITE_SOLLVEVV_OFFLOADING_LDLAGS=-fopenmp-targets=amdgcn-amd-amdhsa;-Xopenmp-target=amdgcn-amd-amdhsa", + ], + )}, - # ARMv8 LibC++ and LibC++abi tests (require Clang+RT) - {'name' : 'libcxx-libcxxabi-libunwind-armv8-linux', - 'tags' : ["libcxx"], - 'workernames' : ['linaro-armv8-libcxx'], - 'builddir': 'libcxx-libcxxabi-libunwind-armv8-linux', - 'factory' : LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( - cmake_extra_opts={ - 'LIBCXXABI_USE_LLVM_UNWINDER': 'ON', - 'CMAKE_C_FLAGS': '-mcpu=cortex-a57 -marm', - 'CMAKE_CXX_FLAGS': '-mcpu=cortex-a57 -marm'})}, - # ARMv7 LibC++ and LibC++abi tests w/o EH (require Clang+RT) - {'name' : 'libcxx-libcxxabi-libunwind-armv7-linux-noexceptions', - 'tags' : ["libcxx"], - 'workernames' : ['linaro-tk1-02'], - 'builddir': 'libcxx-libcxxabi-libunwind-armv7-linux-noexceptions', - 'factory' : LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( - cmake_extra_opts={ - 'LIBCXXABI_USE_LLVM_UNWINDER': 'ON', - 'LIBCXX_ENABLE_EXCEPTIONS': 'OFF', - 'LIBCXXABI_ENABLE_EXCEPTIONS': 'OFF', - 'CMAKE_C_FLAGS': '-mcpu=cortex-a15 -mthumb', - 'CMAKE_CXX_FLAGS': '-mcpu=cortex-a15 -mthumb', - 'LLVM_PARALLEL_LINK_JOBS': '2'})}, - - # ARMv8 LibC++ and LibC++abi tests w/o EH (require Clang+RT) - {'name' : 'libcxx-libcxxabi-libunwind-armv8-linux-noexceptions', - 'tags' : ["libcxx"], - 'workernames' : ['linaro-armv8-libcxx'], - 'builddir': 'libcxx-libcxxabi-libunwind-armv8-linux-noexceptions', - 'factory' : LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( - cmake_extra_opts={ - 'LIBCXXABI_USE_LLVM_UNWINDER': 'ON', - 'LIBCXX_ENABLE_EXCEPTIONS': 'OFF', - 'LIBCXXABI_ENABLE_EXCEPTIONS': 'OFF', - 'CMAKE_C_FLAGS': '-mcpu=cortex-a57 -mthumb', - 'CMAKE_CXX_FLAGS': '-mcpu=cortex-a57 -mthumb'})}, - - # AArch64 LibC++ and LibC++abi tests (require Clang+RT) - {'name' : 'libcxx-libcxxabi-libunwind-aarch64-linux', - 'tags' : ["libcxx"], - 'workernames' : ['linaro-aarch64-libcxx'], - 'builddir': 'libcxx-libcxxabi-libunwind-aarch64-linux', - 'factory' : LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( - cmake_extra_opts={ - 'LIBCXXABI_USE_LLVM_UNWINDER': 'ON', - 'CMAKE_C_FLAGS': '-mcpu=cortex-a57', - 'CMAKE_CXX_FLAGS': '-mcpu=cortex-a57'})}, +# Libc++ builders. - {'name' : 'libcxx-libcxxabi-libunwind-aarch64-linux-noexceptions', + {'name' : 'libcxx-libcxxabi-libunwind-ppc-aix', 'tags' : ["libcxx"], - 'workernames' : ['linaro-aarch64-libcxx'], - 'builddir': 'libcxx-libcxxabi-libunwind-aarch64-linux-noexceptions', + 'workernames' : ['aix-ppc-libcxx'], + 'builddir': 'libcxx-libcxxabi-libunwind-ppc-aix', 'factory' : LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( + env={'OBJECT_MODE': '32'}, cmake_extra_opts={ - 'LIBCXXABI_USE_LLVM_UNWINDER': 'ON', - 'LIBCXX_ENABLE_EXCEPTIONS': 'OFF', - 'LIBCXXABI_ENABLE_EXCEPTIONS': 'OFF', - 'CMAKE_C_FLAGS': '-mcpu=cortex-a57', - 'CMAKE_CXX_FLAGS': '-mcpu=cortex-a57'})}, + 'CMAKE_BUILD_TYPE': 'Release', + 'CMAKE_C_COMPILER': 'gcc', + 'CMAKE_CXX_COMPILER': 'g++', + 'CMAKE_AR': '/usr/bin/ar', + 'LIBCXX_CXX_ABI': 'libstdc++', + 'LIBCXX_CXX_ABI_INCLUDE_PATHS': '/opt/freeware/lib/gcc/powerpc-ibm-aix7.2.0.0/8/include/c++;/opt/freeware/lib/gcc/powerpc-ibm-aix7.2.0.0/8/include/c++/powerpc-ibm-aix7.2.0.0', + 'CMAKE_THREAD_LIBS_INIT': '-lpthread', + 'LIBCXX_ENABLE_STATIC:BOOL': 'OFF', + 'LIBCXX_ENABLE_ABI_LINKER_SCRIPT': 'OFF'}, + depends_on_projects=['libcxx'])}, {'name': "fuchsia-x86_64-linux", 'tags' : ["toolchain"], @@ -1343,6 +1429,24 @@ # libc Builders. + {'name' : 'libc-x86_64-windows-dbg', + 'tags' : ["libc"], + 'workernames' : ['libc-x86_64-windows'], + 'builddir': 'libc-x86_64-windows', + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="libc-windows.py", + depends_on_projects=['llvm', 'libc', 'clang', 'clang-tools-extra'], + extra_args=['--debug'])}, + + {'name' : 'libc-aarch64-ubuntu-dbg', + 'tags' : ["libc"], + 'workernames' : ['libc-aarch64-ubuntu'], + 'builddir': 'libc-aarch64-ubuntu', + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="libc-linux.py", + depends_on_projects=['llvm', 'libc', 'clang', 'clang-tools-extra'], + extra_args=['--debug'])}, + {'name' : 'libc-x86_64-debian', 'tags' : ["libc"], 'workernames' : ['libc-x86_64-debian'], @@ -1369,59 +1473,63 @@ depends_on_projects=['llvm', 'libc', 'clang', 'clang-tools-extra'], extra_args=['--debug', '--asan'])}, -# Flang builders. + {'name' : "libc-x86_64-debian-fullbuild-dbg", + 'tags' : ["libc"], + 'workernames' : ["libc-x86_64-debian-fullbuild"], + 'builddir': "libc-x86_64-debian-fullbuild-dbg", + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="libc-linux.py", + depends_on_projects=['llvm', 'libc', 'clang', 'clang-tools-extra'], + extra_args=['--debug'])}, - {'name' : "flang-aarch64-ubuntu", - 'tags' : ["flang"], - 'workernames' : ["flang-aarch64-ubuntu-build"], - 'builddir': "flang-aarch64-ubuntu-build", - 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( - clean=True, - depends_on_projects=['llvm','mlir','clang','flang'], - extra_configure_args=[ - "-DLLVM_TARGETS_TO_BUILD=AArch64", - "-DCMAKE_C_COMPILER=/usr/bin/gcc-9", - "-DCMAKE_CXX_COMPILER=/usr/bin/g++-9", - "-DLLVM_INSTALL_UTILS=ON", - "-DFLANG_BUILD_NEW_DRIVER=ON", - "-DCMAKE_CXX_STANDARD=17", - ])}, + {'name' : "libc-x86_64-debian-fullbuild-dbg-asan", + 'tags' : ["libc"], + 'workernames' : ["libc-x86_64-debian-fullbuild"], + 'builddir': "libc-x86_64-debian-fullbuild-dbg-asan", + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="libc-linux.py", + depends_on_projects=['llvm', 'libc', 'clang', 'clang-tools-extra'], + extra_args=['--debug', '--asan'])}, + +# Flang builders. - {'name' : "flang-aarch64-ubuntu-dylib", + {'name' : "flang-aarch64-dylib", 'tags' : ["flang"], - 'workernames' : ["linaro-aarch64-flang-dylib"], - 'builddir': "flang-aarch64-ubuntu-dylib", + 'workernames' : ["linaro-flang-aarch64-dylib"], + 'builddir': "flang-aarch64-dylib", 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( clean=True, + checks=['check-flang'], depends_on_projects=['llvm','mlir','clang','flang'], extra_configure_args=[ "-DLLVM_TARGETS_TO_BUILD=AArch64", - "-DFLANG_BUILD_NEW_DRIVER=ON", "-DLLVM_BUILD_LLVM_DYLIB=ON", "-DLLVM_LINK_LLVM_DYLIB=ON", "-DCMAKE_CXX_STANDARD=17", ])}, - {'name' : "flang-aarch64-ubuntu-sharedlibs", + {'name' : "flang-aarch64-sharedlibs", 'tags' : ["flang"], - 'workernames' : ["linaro-aarch64-flang-sharedlibs"], - 'builddir': "flang-aarch64-ubuntu-sharedlibs", + 'workernames' : ["linaro-flang-aarch64-sharedlibs"], + 'builddir': "flang-aarch64-sharedlibs", 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( clean=True, + checks=['check-flang'], depends_on_projects=['llvm','mlir','clang','flang'], extra_configure_args=[ "-DLLVM_TARGETS_TO_BUILD=AArch64", - "-DFLANG_BUILD_NEW_DRIVER=ON", "-DBUILD_SHARED_LIBS=ON", "-DLLVM_BUILD_EXAMPLES=ON", + "-DFLANG_BUILD_EXAMPLES=ON", "-DCMAKE_CXX_STANDARD=17", ])}, - {'name' : "flang-aarch64-ubuntu-out-of-tree", + {'name' : "flang-aarch64-out-of-tree", 'tags' : ["flang"], - 'workernames' : ["linaro-aarch64-flang-oot"], + 'workernames' : ["linaro-flang-aarch64-out-of-tree"], 'builddir': "flang-aarch64-out-of-tree", 'factory' : FlangBuilder.getFlangOutOfTreeBuildFactory( + checks=['check-flang'], llvm_extra_configure_args=[ "-DLLVM_TARGETS_TO_BUILD=AArch64", "-DCMAKE_CXX_STANDARD=17", @@ -1434,6 +1542,90 @@ "-DCMAKE_BUILD_TYPE=Release", ])}, + {'name' : "flang-aarch64-debug", + 'tags' : ["flang"], + 'workernames' : ["linaro-flang-aarch64-debug"], + 'builddir': "flang-aarch64-debug", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + clean=True, + checks=['check-flang'], + depends_on_projects=['llvm','mlir','clang','flang'], + extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=AArch64", + "-DCMAKE_BUILD_TYPE=Debug", + "-DCMAKE_CXX_STANDARD=17", + "-DLLVM_USE_LINKER=lld", + ])}, + + {'name' : "flang-aarch64-latest-clang", + 'tags' : ['flang'], + 'workernames' : ["linaro-flang-aarch64-latest-clang"], + 'builddir': "flang-aarch64-latest-clang", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + clean=True, + checks=['check-flang'], + depends_on_projects=['llvm','mlir','clang','flang'], + extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=AArch64", + "-DLLVM_INSTALL_UTILS=ON", + "-DCMAKE_CXX_STANDARD=17", + "-DLLVM_ENABLE_WERROR=OFF", + "-DFLANG_ENABLE_WERROR=ON", + "-DBUILD_SHARED_LIBS=ON", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_ENABLE_LIBCXX=On", + "-DCMAKE_BUILD_TYPE=Release", + ])}, + + {'name' : "flang-aarch64-release", + 'tags' : ["flang"], + 'workernames' : ["linaro-flang-aarch64-release"], + 'builddir': "flang-aarch64-release", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + clean=True, + checks=['check-flang'], + depends_on_projects=['llvm','mlir','clang','flang'], + extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=AArch64", + "-DCMAKE_BUILD_TYPE=Release", + "-DCMAKE_CXX_STANDARD=17", + ])}, + + {'name' : "flang-aarch64-rel-assert", + 'tags' : ["flang"], + 'workernames' : ["linaro-flang-aarch64-rel-assert"], + 'builddir': "flang-aarch64-rel-assert", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + clean=True, + checks=['check-flang'], + depends_on_projects=['llvm','mlir','clang','flang'], + extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=AArch64", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DFLANG_BUILD_EXAMPLES=ON", + "-DCMAKE_BUILD_TYPE=Release", + "-DCMAKE_CXX_STANDARD=17", + ])}, + + {'name' : "flang-aarch64-latest-gcc", + 'tags' : ['flang'], + 'workernames' : ["linaro-flang-aarch64-latest-gcc"], + 'builddir': "flang-aarch64-latest-gcc", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + clean=True, + checks=['check-flang'], + depends_on_projects=['llvm','mlir','clang','flang'], + extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=AArch64", + "-DLLVM_INSTALL_UTILS=ON", + "-DCMAKE_CXX_STANDARD=17", + "-DLLVM_ENABLE_WERROR=OFF", + "-DFLANG_ENABLE_WERROR=ON", + "-DBUILD_SHARED_LIBS=ON", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DCMAKE_BUILD_TYPE=Release", + ])}, + {'name' : "flang-x86_64-linux", 'tags' : ["flang"], 'workernames' : ["nersc-flang"], @@ -1457,7 +1649,6 @@ "-DCMAKE_C_COMPILER=gcc", "-DCMAKE_CXX_COMPILER=g++", "-DLLVM_INSTALL_UTILS=ON", - "-DFLANG_BUILD_NEW_DRIVER=ON", "-DCMAKE_CXX_STANDARD=17", ])}, @@ -1474,7 +1665,6 @@ '-DLLVM_TARGETS_TO_BUILD=PowerPC', '-DLLVM_INSTALL_UTILS=ON', '-DCMAKE_CXX_STANDARD=17', - '-DLLVM_ENABLE_PROJECTS=flang', '-DLLVM_LIT_ARGS=-vj 256' ], env={ @@ -1483,7 +1673,27 @@ 'LD': 'lld' })}, -# Builders responsible building Sphinix documentation. + {'name' : "flang-x86_64-windows", + 'tags' : ["flang"], + 'workernames' : ["minipc-ryzen-win"], + 'builddir': "flang-x86_64-windows", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + depends_on_projects=['llvm','mlir','clang','flang'], + checks=['check-flang'], + install_dir="flang.install", + extra_configure_args=[ + "-DCLANG_ENABLE_STATIC_ANALYZER=OFF", + "-DCLANG_ENABLE_ARCMT=OFF", + "-DCLANG_ENABLE_OBJC_REWRITER=OFF", + "-DLLVM_TARGETS_TO_BUILD=X86", + "-DLLVM_INSTALL_UTILS=ON", + "-DCMAKE_C_COMPILER=cl", + "-DCMAKE_CXX_COMPILER=cl", + "-DCMAKE_CXX_STANDARD=17", + '-DLLVM_PARALLEL_COMPILE_JOBS=4', + ])}, + +# Builders responsible building Sphinx documentation. {'name' : "llvm-sphinx-docs", 'tags' : ["llvm", "doc"], @@ -1509,12 +1719,6 @@ 'builddir': "lld-sphinx-docs", 'factory' : SphinxDocsBuilder.getSphinxDocsBuildFactory(lld_html=True)}, - {'name' : "lldb-sphinx-docs", - 'tags' : ["lldb", "doc"], - 'workernames' : ["gribozavr3"], - 'builddir': "lldb-sphinx-docs", - 'factory' : SphinxDocsBuilder.getSphinxDocsBuildFactory(lldb_html=True)}, - {'name':"libunwind-sphinx-docs", 'tags' : ["libunwind", "doc"], 'workernames':["gribozavr3"], @@ -1528,6 +1732,18 @@ 'builddir': "publish-sphinx-docs", 'factory' : SphinxDocsBuilder.getLLVMDocsBuildFactory(clean=True)}, + {'name' : "publish-lnt-sphinx-docs", + 'tags' : ["doc"], + 'workernames' : ["as-worker-4"], + 'builddir': "publish-lnt-sphinx-docs", + 'factory' : HtmlDocsBuilder.getHtmlDocsBuildFactory()}, + + {'name' : "publish-doxygen-docs", + 'tags' : ["doc"], + 'workernames' : ["as-worker-4"], + 'builddir': "publish-doxygen-docs", + 'factory' : DoxygenDocsBuilder.getLLVMDocsBuildFactory()}, + # CUDA builders. {'name' : "clang-cuda-k80", @@ -1556,7 +1772,7 @@ {'name' : "clang-ve-ninja", 'tags' : ["clang"], - 'workernames':["nec-arrproto41"], + 'workernames':["hpce-aurora2"], 'builddir':"clang-ve-ninja", 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( clean=True, @@ -1564,8 +1780,24 @@ extra_configure_args=[ "-DLLVM_TARGETS_TO_BUILD=X86", "-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=VE", + "-DLLVM_ENABLE_RUNTIMES=compiler-rt", + "-DLLVM_BUILD_LLVM_DYLIB=On", + "-DBUILD_SHARED_LIBS=Off", + "-DLLVM_LINK_LLVM_DYLIB=On", + "-DCLANG_LINK_CLANG_DYLIB=On", ])}, +# HIP builders. + + {'name' : "clang-hip-vega20", + 'tags' : ["clang"], + 'workernames' : ["hip-vega20-0"], + 'builddir': "clang-hip-vega20", + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="hip-build.sh", + checkout_llvm_sources=False, + script_interpreter=None)}, + # Latest stable fedora running on Red Hat internal OpenShift cluster (PSI). {'name' : 'x86_64-fedora-clang', @@ -1596,6 +1828,33 @@ '-DLLVM_ENABLE_LLD=ON', ])}, + # Build the LLVM dylib .so with all backends and link tools to it + {'name' : 'llvm-x86_64-debian-dylib', + 'tags' : ['llvm'], + 'collapseRequests': False, + 'workernames': ['gribozavr4'], + 'builddir': 'llvm-x86_64-debian-dylib', + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + clean=True, + depends_on_projects=['llvm'], + checks=['check-all'], + extra_configure_args=[ + '-DCMAKE_BUILD_TYPE=Release', + '-DLLVM_ENABLE_ASSERTIONS=On', + '-DLLVM_BUILD_EXAMPLES=Off', + "-DLLVM_LIT_ARGS=-v --xunit-xml-output test-results.xml", + '-DLLVM_TARGETS_TO_BUILD=All', + '-DCMAKE_EXPORT_COMPILE_COMMANDS=1', + '-DLLVM_BUILD_LLVM_DYLIB=On', + '-DLLVM_LINK_LLVM_DYLIB=On', + '-DBUILD_SHARED_LIBS=Off', + '-DLLVM_ENABLE_LLD=Off', + ], + env={ + 'PATH':'/home/llvmbb/bin/clang-latest/bin:/home/llvmbb/bin:/usr/local/bin:/usr/local/bin:/usr/bin:/bin', + 'CC': 'ccache clang', 'CXX': 'ccache clang++', 'CCACHE_CPP2': 'yes' + })}, + {'name' : "clang-solaris11-amd64", 'tags' : ["clang"], 'workernames' : ["solaris11-amd64"], @@ -1647,52 +1906,6 @@ '-DLLVM_LIT_TOOLS_DIR=C:\\Program Files\\GnuWin32\\usr\\bin', ])}, -# Flang builders. - - {'name' : "flang-aarch64-ubuntu-clang", - 'tags' : ['flang'], - 'workernames' : ["flang-aarch64-ubuntu-clang-build"], - 'builddir': "flang-aarch64-ubuntu-clang", - 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( - clean=True, - depends_on_projects=['llvm','mlir','clang','flang'], - checks=['check-flang'], - extra_configure_args=[ - "-DLLVM_TARGETS_TO_BUILD=AArch64", - "-DCMAKE_C_COMPILER=/usr/bin/clang-10", - "-DCMAKE_CXX_COMPILER=/usr/bin/clang++-10", - "-DLLVM_INSTALL_UTILS=ON", - "-DCMAKE_CXX_STANDARD=17", - "-DLLVM_ENABLE_WERROR=OFF", - "-DFLANG_ENABLE_WERROR=ON", - "-DBUILD_SHARED_LIBS=ON", - "-DLLVM_ENABLE_ASSERTIONS=ON", - "-DLLVM_ENABLE_LIBCXX=On", - "-DFLANG_BUILD_NEW_DRIVER=ON", - "-DCMAKE_BUILD_TYPE=Release", - ])}, - - {'name' : "flang-aarch64-ubuntu-gcc10", - 'tags' : ['flang'], - 'workernames' : ["flang-aarch64-ubuntu-gcc10-build"], - 'builddir': "flang-aarch64-ubuntu-gcc10", - 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( - clean=True, - depends_on_projects=['llvm','mlir','clang','flang'], - extra_configure_args=[ - "-DLLVM_TARGETS_TO_BUILD=AArch64", - "-DCMAKE_C_COMPILER=/usr/bin/gcc-10", - "-DCMAKE_CXX_COMPILER=/usr/bin/g++-10", - "-DLLVM_INSTALL_UTILS=ON", - "-DCMAKE_CXX_STANDARD=17", - "-DLLVM_ENABLE_WERROR=OFF", - "-DFLANG_ENABLE_WERROR=ON", - "-DBUILD_SHARED_LIBS=ON", - "-DLLVM_ENABLE_ASSERTIONS=ON", - "-DFLANG_BUILD_NEW_DRIVER=ON", - "-DCMAKE_BUILD_TYPE=Release", - ])}, - # LLDB builders. {'name' : "lldb-x86_64-fedora", @@ -1722,7 +1935,7 @@ "-DCMAKE_BUILD_TYPE=Release", "-DLLVM_CCACHE_BUILD=ON", "-DLLVM_ENABLE_ASSERTIONS=ON", - "-DTENSORFLOW_API_PATH=/tmp/tensorflow", + "-DTENSORFLOW_C_LIB_PATH=/tmp/tensorflow", "-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON" ])}, @@ -1739,7 +1952,7 @@ "-DCMAKE_BUILD_TYPE=Release", "-DLLVM_CCACHE_BUILD=ON", "-DLLVM_ENABLE_ASSERTIONS=ON", - "-DTENSORFLOW_API_PATH=/tmp/tensorflow", + "-DTENSORFLOW_C_LIB_PATH=/tmp/tensorflow", "-DTENSORFLOW_AOT_PATH=/var/lib/buildbot/.local/lib/python3.7/site-packages/tensorflow", "-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON" ])}, @@ -1782,5 +1995,22 @@ "-DGRPC_INSTALL_PATH=/usr/local/lib/grpc" ])}, + # Target ARC from Synopsys + {'name': "arc-builder", + 'tags': ["clang", "lld"], + 'collapseRequests': False, + 'workernames' : ["arc-worker"], + 'builddir': "arc-folder", + 'factory': UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + depends_on_projects=["llvm", "clang", "lld"], + extra_configure_args=[ + "-DCMAKE_EXPORT_COMPILE_COMMANDS:BOOL=ON", + "-DLLVM_ENABLE_ASSERTIONS:BOOL=ON", + "-DLLVM_TOOL_CLANG_TOOLS_EXTRA_BUILD=0", + "-DLLVM_ENABLE_LIBPFM=OFF", + "-DLLVM_TARGETS_TO_BUILD=X86", + "-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=ARC", + ])}, + ] diff --git a/buildbot/osuosl/master/config/release_builders.py b/buildbot/osuosl/master/config/release_builders.py new file mode 100644 --- /dev/null +++ b/buildbot/osuosl/master/config/release_builders.py @@ -0,0 +1,225 @@ +from importlib import reload + +from buildbot.process.properties import WithProperties + +from zorg.buildbot.builders import ClangBuilder +from zorg.buildbot.builders import FlangBuilder +from zorg.buildbot.builders import PollyBuilder +from zorg.buildbot.builders import LLDBBuilder +from zorg.buildbot.builders import SanitizerBuilder +from zorg.buildbot.builders import OpenMPBuilder +from zorg.buildbot.builders import SphinxDocsBuilder +from zorg.buildbot.builders import ABITestsuitBuilder +from zorg.buildbot.builders import ClangLTOBuilder +from zorg.buildbot.builders import UnifiedTreeBuilder +from zorg.buildbot.builders import AOSPBuilder +from zorg.buildbot.builders import AnnotatedBuilder +from zorg.buildbot.builders import LLDPerformanceTestsuite +from zorg.buildbot.builders import FuchsiaBuilder +from zorg.buildbot.builders import XToolchainBuilder + + +# Release builders. + +all = [ + +# Clang builders. + + {'name' : "llvm-clang-x86_64-win-release", + 'tags' : ["clang", "release"], + 'workernames' : ["as-builder-3"], + 'builddir': "llvm-clang-x86_64-win-rel", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaWithMSVCBuildFactory( + vs="autodetect", + depends_on_projects=['llvm', 'clang'], + clean=True, + checks=[ + "check-llvm-unit", + "check-clang-unit"], + extra_configure_args=[ + "-DLLVM_ENABLE_WERROR=OFF", + "-DLLVM_TARGETS_TO_BUILD=ARM", + "-DLLVM_DEFAULT_TARGET_TRIPLE=armv7-unknown-linux-eabihf", + "-DLLVM_ENABLE_ASSERTIONS=OFF", + "-DLLVM_OPTIMIZED_TABLEGEN=OFF", + "-DLLVM_LIT_ARGS=-v --threads=32"])}, + +# Expensive checks builders. + + + {'name' : "llvm-clang-x86_64-expensive-checks-ubuntu-release", + 'tags' : ["llvm", "expensive-checks", "release"], + 'workernames' : ["as-builder-4"], + 'builddir': "llvm-clang-x86_64-expensive-checks-ubuntu-rel", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + depends_on_projects=["llvm", "lld"], + clean=True, + extra_configure_args=[ + "-DLLVM_ENABLE_EXPENSIVE_CHECKS=ON", + "-DLLVM_ENABLE_WERROR=OFF", + "-DCMAKE_BUILD_TYPE=Debug", + "-DCMAKE_CXX_FLAGS=-U_GLIBCXX_DEBUG", + "-DLLVM_LIT_ARGS=-vv -j32"])}, + + {'name' : "llvm-clang-x86_64-expensive-checks-win-release", + 'tags' : ["llvm", "expensive-checks", "release"], + 'workernames' : ["as-worker-93"], + 'builddir': "llvm-clang-x86_64-expensive-checks-win-rel", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaWithMSVCBuildFactory( + vs="autodetect", + depends_on_projects=["llvm", "lld"], + clean=True, + extra_configure_args=[ + "-DLLVM_ENABLE_EXPENSIVE_CHECKS=ON", + "-DLLVM_ENABLE_WERROR=OFF", + "-DCMAKE_BUILD_TYPE=Debug"])}, + + {'name' : "llvm-clang-x86_64-expensive-checks-debian-release", + 'tags' : ["llvm", "expensive-checks", "release"], + 'workernames' : ["gribozavr4"], + 'builddir': "llvm-clang-x86_64-expensive-checks-debian-rel", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + depends_on_projects=["llvm", "lld"], + clean=True, + extra_configure_args=[ + "-DLLVM_ENABLE_EXPENSIVE_CHECKS=ON", + "-DLLVM_ENABLE_WERROR=OFF", + "-DCMAKE_BUILD_TYPE=Release", + "-DCMAKE_CXX_FLAGS=-U_GLIBCXX_DEBUG", + "-DLLVM_LIT_ARGS=-v -vv -j96"], + env={ + 'PATH':'/home/llvmbb/bin/clang-latest/bin:/home/llvmbb/bin:/usr/local/bin:/usr/local/bin:/usr/bin:/bin', + 'CC': 'ccache clang', 'CXX': 'ccache clang++', 'CCACHE_CPP2': 'yes', + })}, + +# Cross builders. + + {'name' : "llvm-clang-win-x-armv7l-release", + 'tags' : ["clang", "llvm", "compiler-rt", "cross"," armv7l", "release"], + 'workernames' : ["as-builder-1"], + 'builddir': "x-armv7l-rel", + 'factory' : XToolchainBuilder.getCmakeWithMSVCBuildFactory( + vs="autodetect", + clean=True, + checks=[ + "check-llvm", + "check-clang", + "check-lld", + "check-compiler-rt" + ], + checks_on_target = [ + ("libunwind", + ["python", "bin/llvm-lit.py", + "-v", "-vv", "--threads=32", + "runtimes/runtimes-bins/libunwind/test"]), + ("libc++abi", + ["python", "bin/llvm-lit.py", + "-v", "-vv", "--threads=32", + "runtimes/runtimes-bins/libcxxabi/test"]), + ("libc++", + ['python', 'bin/llvm-lit.py', + '-v', '-vv', '--threads=32', + 'runtimes/runtimes-bins/libcxx/test', + ]) + ], + extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=ARM", + "-DTARGET_TRIPLE=armv7-linux-gnueabihf", + "-DDEFAULT_SYSROOT=C:/buildbot/.arm-ubuntu", + "-DLLVM_LIT_ARGS=-v -vv --threads=32", + WithProperties("%(remote_test_host:+-DREMOTE_TEST_HOST=)s%(remote_test_host:-)s"), + WithProperties("%(remote_test_user:+-DREMOTE_TEST_USER=)s%(remote_test_user:-)s"), + ], + cmake_cache="../llvm-project/clang/cmake/caches/CrossWinToARMLinux.cmake")}, + + {'name' : "llvm-clang-win-x-aarch64-release", + 'tags' : ["clang", "llvm", "compiler-rt", "cross", "aarch64", "release"], + 'workernames' : ["as-builder-2"], + 'builddir': "x-aarch64-rel", + 'factory' : XToolchainBuilder.getCmakeWithMSVCBuildFactory( + vs="autodetect", + clean=True, + checks=[ + "check-llvm", + "check-clang", + "check-lld", + "check-compiler-rt" + ], + checks_on_target = [ + ("libunwind", + ["python", "bin/llvm-lit.py", + "-v", "-vv", "--threads=32", + "runtimes/runtimes-bins/libunwind/test"]), + ("libc++abi", + ["python", "bin/llvm-lit.py", + "-v", "-vv", "--threads=32", + "runtimes/runtimes-bins/libcxxabi/test"]), + ("libc++", + ['python', 'bin/llvm-lit.py', + '-v', '-vv', '--threads=32', + 'runtimes/runtimes-bins/libcxx/test', + ]) + ], + extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=AArch64", + "-DTARGET_TRIPLE=aarch64-linux-gnu", + "-DDEFAULT_SYSROOT=C:/buildbot/.aarch64-ubuntu", + "-DLLVM_LIT_ARGS=-v -vv --threads=32", + WithProperties("%(remote_test_host:+-DREMOTE_TEST_HOST=)s%(remote_test_host:-)s"), + WithProperties("%(remote_test_user:+-DREMOTE_TEST_USER=)s%(remote_test_user:-)s"), + ], + cmake_cache="../llvm-project/clang/cmake/caches/CrossWinToARMLinux.cmake")}, + +# LLD builders. + + {'name' : "lld-x86_64-win-release", + 'tags' : ["lld", "release"], + 'workernames' : ["as-worker-93"], + 'builddir': "lld-x86_64-win-rel", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaWithMSVCBuildFactory( + depends_on_projects=['llvm', 'lld'], + vs="autodetect", + extra_configure_args = [ + '-DLLVM_ENABLE_WERROR=OFF'])}, + + {'name' : "lld-x86_64-ubuntu-release", + 'tags' : ["lld", "release"], + 'workernames' : ["as-builder-4"], + 'builddir' : "lld-x86_64-ubuntu-rel", + 'factory': UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + clean=True, + extra_configure_args=[ + '-DLLVM_ENABLE_WERROR=OFF'], + depends_on_projects=['llvm', 'lld'])}, + +# LTO and ThinLTO builders. + + {'name' : "clang-with-thin-lto-ubuntu-release", + 'tags' : ["clang", "lld", "LTO", "release"], + 'workernames' : ["as-worker-92"], + 'builddir': "clang-with-thin-lto-ubuntu-rel", + 'factory' : ClangLTOBuilder.getClangWithLTOBuildFactory(jobs=72, lto='thin')}, + + {'name' : "clang-with-lto-ubuntu-release", + 'tags' : ["clang", "lld", "LTO", "release"], + 'workernames' : ["as-worker-91"], + 'builddir': "clang-with-lto-ubuntu-rel", + 'factory' : ClangLTOBuilder.getClangWithLTOBuildFactory( + jobs=72, + extra_configure_args_lto_stage=[ + '-DLLVM_PARALLEL_LINK_JOBS=14', + ])}, + +# OpenMP builders. + + {'name' : "openmp-clang-x86_64-linux-debian-release", + 'tags' : ["openmp", "release"], + 'workernames' : ["gribozavr4"], + 'builddir': "openmp-clang-x86_64-linux-debian-rel", + 'factory' : OpenMPBuilder.getOpenMPCMakeBuildFactory( + env={ + 'PATH':'/home/llvmbb/bin/clang-latest/bin:/home/llvmbb/bin:/usr/local/bin:/usr/local/bin:/usr/bin:/bin', + 'CC': 'ccache clang', 'CXX': 'ccache clang++', 'CCACHE_CPP2': 'yes', + })}, + +] diff --git a/buildbot/osuosl/master/config/schedulers.py b/buildbot/osuosl/master/config/schedulers.py --- a/buildbot/osuosl/master/config/schedulers.py +++ b/buildbot/osuosl/master/config/schedulers.py @@ -19,7 +19,7 @@ # Since we have many parametric builders, we dynamically build the minimum set # of schedulers, which covers all actually used combinations of dependencies. -def getSingleBranchSchedulers( +def getMainBranchSchedulers( builders, explicitly_set_schedulers = None, **kwargs): @@ -43,9 +43,10 @@ builder for builder in builders if builder.name not in builders_with_explicit_schedulers if getattr(builder.factory, 'depends_on_projects', None) + if 'release' not in getattr(builder, 'tags', []) ] - filter_branch = 'master' + filter_branch = 'main' treeStableTimer = kwargs.get('treeStableTimer', None) automatic_schedulers = [] @@ -60,7 +61,7 @@ ]) for projects in set_of_dependencies: - sch_builders = [ + main_builders = [ b.name for b in builders_with_automatic_schedulers if frozenset(getattr(b.factory, 'depends_on_projects')) == projects @@ -71,9 +72,9 @@ automatic_schedulers.append( schedulers.SingleBranchScheduler( name=automatic_scheduler_name, - treeStableTimer=kwargs.get('treeStableTimer', None), - reason="Merge to github %s branch" % filter_branch, - builderNames=sch_builders, + treeStableTimer=treeStableTimer, + reason="Merge to github {} branch".format(filter_branch), + builderNames=main_builders, change_filter=util.ChangeFilter( project_fn= \ lambda c, projects_of_interest=frozenset(projects): @@ -83,18 +84,90 @@ ) log.msg( - "Generated SingleBranchScheduler: { name='%s'" % automatic_scheduler_name, - ", builderNames=", sch_builders, - ", change_filter=", projects, " (branch: %s)" % filter_branch, - ", treeStableTimer=%s" % treeStableTimer, + "Generated SingleBranchScheduler: {{ name='{}'".format(automatic_scheduler_name), + ", builderNames=", main_builders, + ", change_filter=", projects, " (branch: {})".format(filter_branch), + ", treeStableTimer={}".format(treeStableTimer), "}") return automatic_schedulers -# TODO: Move these settings to the configuration file. -_repourl = "https://github.com/llvm/llvm-project" -_branch = "master" +def getReleaseBranchSchedulers( + builders, + explicitly_set_schedulers = None, + **kwargs): + """ + I'm taking over all of not yet assigned builders with the + declared source code dependencies and the 'release' tag, + and automatically generate a minimum set of SingleBranchSchedulers + to handle all the declared source code dependency combinations. + """ + + builders_with_explicit_schedulers = set() + if explicitly_set_schedulers: + # TODO: Make a list of builder names with already set schedulers. + # builders_with_explicit_schedulers.add(builder) + pass + + # For the builders created with LLVMBuildFactory or similar, + # we always use automatic schedulers, + # unless schedulers already explicitly set. + builders_with_automatic_schedulers = [ + builder for builder in builders + if builder.name not in builders_with_explicit_schedulers + if getattr(builder.factory, 'depends_on_projects', None) + if 'release' in getattr(builder, 'tags', []) + ] + + treeStableTimer = kwargs.get('treeStableTimer', None) + + automatic_schedulers = [] + + # Do we have any to take care of? + if builders_with_automatic_schedulers: + # Let's reconsile first to get a unique set of dependencies. + # We need a set of unique sets of dependent projects. + set_of_dependencies = set([ + frozenset(getattr(b.factory, 'depends_on_projects')) + for b in builders_with_automatic_schedulers + ]) + + for projects in set_of_dependencies: + release_builders = [ + b.name + for b in builders_with_automatic_schedulers + if frozenset(getattr(b.factory, 'depends_on_projects')) == projects + ] + + automatic_scheduler_name = "release:" + ",".join(sorted(projects)) + + automatic_schedulers.append( + schedulers.SingleBranchScheduler( + name=automatic_scheduler_name, + treeStableTimer=treeStableTimer, + reason="Merge to github release branch", + builderNames=release_builders, + change_filter=util.ChangeFilter( + project_fn= \ + lambda c, projects_of_interest=frozenset(projects): + isProjectOfInterest(c, projects_of_interest), + branch_fn= \ + lambda branch: branch.startswith('release/')) + ) + ) + + log.msg( + "Generated release SingleBranchScheduler: {{ name='{}'".format(automatic_scheduler_name), + ", builderNames=", release_builders, + ", change_filter=", projects, " (branch: {release/*})", + ", treeStableTimer={}".format(treeStableTimer), + "}") + return automatic_schedulers def getForceSchedulers(builders): + # TODO: Move these settings to the configuration file. + _repourl = "https://github.com/llvm/llvm-project" + _branch = "main" + # Walk over all builders and collect their names. scheduler_builders = [ builder.name for builder in builders @@ -120,8 +193,10 @@ codebases = [ util.CodebaseParameter( codebase = "", - branch = util.FixedParameter( + branch = util.StringParameter( name = "branch", + label = "branch:", + size = 64, default = _branch ), revision = util.StringParameter( @@ -157,19 +232,21 @@ # TODO: Abstract this kind of scheduler better. def getLntSchedulers(): - project = "lnt" - lnt_builders = [ + _project = "lnt" + _branch = 'master' + _repourl = "https://github.com/llvm/llvm-lnt" + _lnt_builders = [ "publish-lnt-sphinx-docs", ] return [ schedulers.SingleBranchScheduler( name="lnt-scheduler", treeStableTimer=kwargs.get('treeStableTimer', None), - reason="Merge to LNT github {} branch".format(filter_branch), - builderNames=lnt_builders, + reason="Merge to LNT github {} branch".format(_branch), + builderNames=_lnt_builders, change_filter=util.ChangeFilter( - project_fn=project, - branch=filter_branch)), + project_fn=_project, + branch=_branch)), schedulers.ForceScheduler( name = "force-build-scheduler", @@ -186,7 +263,7 @@ ], default = "Build a particular revision" ), - builderNames = lnt_builders, + builderNames = _lnt_builders, codebases = [ util.CodebaseParameter( codebase = "", @@ -206,7 +283,7 @@ ), project = util.FixedParameter( name = "project", - default = project + default = _project ) ) ], diff --git a/buildbot/osuosl/master/config/status.py b/buildbot/osuosl/master/config/status.py --- a/buildbot/osuosl/master/config/status.py +++ b/buildbot/osuosl/master/config/status.py @@ -9,18 +9,22 @@ all = [ - # Note: reporters.GitHubStatusPush requires txrequests package to allow - # interaction with GitHub REST API. + # Report github status for all the release builders, + # i.e. those with the "release" tag. reporters.GitHubStatusPush( str(config.options.get('GitHub Status', 'token')), context = Interpolate("%(prop:buildername)s"), verbose = True, # TODO: Turn off the verbosity once this is working reliably. builders = [ - "llvm-clang-x86_64-expensive-checks-ubuntu", - "llvm-clang-x86_64-win-fast", - "clang-x86_64-debian-fast", - "llvm-clang-x86_64-expensive-checks-debian", - ]), + "llvm-clang-x86_64-expensive-checks-ubuntu", + "llvm-clang-x86_64-win-fast", + "clang-x86_64-debian-fast", + "llvm-clang-x86_64-expensive-checks-debian", + ] + [ + b.get('name') for b in config.release_builders.all + if 'release' in b.get('tags', []) + ] + ), reporters.IRC( useColors = False, @@ -44,15 +48,6 @@ dumpMailsToLog = True, ), - # In addition to that the following notifiers are defined for special - # cases. - reporters.MailNotifier( - fromaddr = "llvm.buildmaster@lab.llvm.org", - sendToInterestedUsers = False, - extraRecipients = ["leandro.nunes@arm.com"], - subject = "Build %(builder)s Failure", - mode = "failing", - builders = ["clang-aarch64-linux-build-cache", "clang-armv7-linux-build-cache"]), reporters.MailNotifier( fromaddr = "llvm.buildmaster@lab.llvm.org", sendToInterestedUsers = False, @@ -141,8 +136,9 @@ reporters.MailNotifier( fromaddr = "llvm.buildmaster@lab.llvm.org", sendToInterestedUsers = False, - extraRecipients = ["stilis@microsoft.com", "jonas@devlieghere.com", - "diprou@microsoft.com", "makudrya@microsoft.com"], + extraRecipients = ["stilis@microsoft.com", + "diprou@microsoft.com", + "makudrya@microsoft.com"], subject = "Build %(builder)s Failure", mode = "failing", builders = ["lldb-x64-windows-ninja"]), @@ -211,7 +207,10 @@ subject = "Build %(builder)s Failure", mode = "failing", builders = ["libc-x86_64-debian", "libc-x86_64_debian-dbg", - "libc-x86_64-debian-dbg-asan"]), + "libc-x86_64-debian-dbg-asan", "libc-aarch64-ubuntu-dbg", + "libc-x86_64-windows-dbg", + "libc-x86_64-debian-fullbuild-dbg", + "libc-x86_64-debian-fullbuild-dbg-asan"]), reporters.MailNotifier( fromaddr = "llvm.buildmaster@lab.llvm.org", sendToInterestedUsers = False, @@ -228,33 +227,65 @@ mode = "failing", builders = [ "ml-opt-dev-x86-64", "ml-opt-rel-x86-64", "ml-opt-devrel-x86-64"]), + reporters.MailNotifier( + fromaddr="llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers = False, + extraRecipients=[ + "tejohnson@google.com"], + subject = "ThinLTO WPD Failure: %(builder)s", + mode = "failing", + builders = ["thinlto-x86-64-bot1"]), reporters.MailNotifier( fromaddr = "llvm.buildmaster@lab.llvm.org", sendToInterestedUsers = False, - extraRecipients = [ - "caroline.concatto@arm.com", "flang_llvm_buildbots@arm.com"], + extraRecipients = ["douglas.yung@sony.com"], subject = "Build %(builder)s Failure", mode = "failing", - builders = [ - "flang-aarch64-ubuntu", "flang-aarch64-ubuntu-clang", - "flang-aarch64-ubuntu-gcc10"]), + builders = ["llvm-clang-x86_64-sie-ubuntu-fast"]), reporters.MailNotifier( fromaddr="llvm.buildmaster@lab.llvm.org", sendToInterestedUsers = False, extraRecipients=[ - "tejohnson@google.com"], - subject = "ThinLTO WPD Failure: %(builder)s", + "mlir-bugs-external+buildbot@googlegroups.com"], + subject = "MLIR Build Failure: %(builder)s", mode = "failing", - builders = ["thinlto-x86-64-bot1"]), - + builders = ["mlir-nvidia", + "mlir-windows", + "ppc64le-mlir-rhel-clang"]), + reporters.MailNotifier( + fromaddr="llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers = False, + extraRecipients=["dl.gcr.lightning.buildbot@amd.com"], + subject = "Build Failure: %(builder)s", + mode = "failing", + builders = ["clang-hip-vega20"]), + reporters.MailNotifier( + fromaddr="llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers = False, + extraRecipients=["llvm_arc_buildbot@synopsys.com", "heli@synopsys.com"], + subject = "Build Failure: %(builder)s", + mode = "failing", + builders = ["arc-builder"]), + reporters.MailNotifier( + fromaddr="llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers = False, + extraRecipients=["dl.gcr.lightning.buildbot@amd.com"], + subject = "Build Failure: %(builder)s", + mode = "failing", + builders = ["openmp-offload-amdgpu-project","openmp-offload-amdgpu-runtime"]), ] + +from twisted.python import log + # Returns a list of Status Targets. The results of each build will be # pushed to these targets. buildbot.plugins reporters has a variety # to choose from, including email senders, and IRC bots. def getReporters(): if config.options.getboolean('Master Options', 'is_production'): + log.msg(">>> getReporters: Production mode. All reporters are registered.") return all else: # Staging buildbot does not report issues. + log.msg(">>> getReporters: Staging buildbot does not report issues.") return [] diff --git a/buildbot/osuosl/master/config/workers.py b/buildbot/osuosl/master/config/workers.py --- a/buildbot/osuosl/master/config/workers.py +++ b/buildbot/osuosl/master/config/workers.py @@ -29,45 +29,45 @@ create_worker("linaro-tk1-07", properties={'jobs' : 4}, max_builds=1), create_worker("linaro-tk1-08", properties={'jobs' : 4}, max_builds=1), create_worker("linaro-tk1-09", properties={'jobs' : 4}, max_builds=1), - create_worker("linaro-armv7-lnt", properties={'jobs' : 32}, max_builds=1), - create_worker("linaro-armv7-selfhost", properties={'jobs' : 32}, max_builds=1), - create_worker("linaro-armv7-quick", properties={'jobs' : 32}, max_builds=1), - create_worker("linaro-armv7-global-isel", properties={'jobs' : 32}, max_builds=1), - create_worker("linaro-armv8-lld", properties={'jobs' : 32}, max_builds=1), - # Libcxx testsuite has tests with timing assumptions. Run single-threaded to make - # sure we have plenty CPU cycle to satisfy timing assumptions. - create_worker("linaro-armv8-libcxx", properties={'jobs' : 1}, max_builds=1), - create_worker("linaro-arm-lldb", properties={'jobs' : 16}, max_builds=1), + create_worker("linaro-clang-armv7-lnt", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-clang-armv7-2stage", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-clang-armv7-quick", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-clang-armv7-global-isel", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-clang-armv7-vfpv3-2stage", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-clang-armv8-lld-2stage", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-lldb-arm-ubuntu", properties={'jobs' : 16}, max_builds=1), # AArch64 Linaro workers - create_worker("linaro-aarch64-quick", properties={'jobs' : 32}, max_builds=1), - create_worker("linaro-aarch64-full", properties={'jobs' : 32}, max_builds=1), - create_worker("linaro-aarch64-global-isel", properties={'jobs' : 32}, max_builds=1), - create_worker("linaro-aarch64-lld", properties={'jobs' : 32}, max_builds=1), - create_worker("linaro-aarch64-flang-oot", properties={'jobs' : 32}, max_builds=1), - create_worker("linaro-aarch64-flang-dylib", properties={'jobs' : 32}, max_builds=1), - create_worker("linaro-aarch64-flang-sharedlibs", properties={'jobs' : 32}, max_builds=1), - # Libcxx testsuite has tests with timing assumptions. Run single-threaded to make - # sure we have plenty CPU cycle to satisfy timing assumptions. - create_worker("linaro-aarch64-libcxx", properties={'jobs' : 1}, max_builds=1), - create_worker("linaro-aarch64-lldb", properties={'jobs': 16}, max_builds=1), + create_worker("linaro-clang-aarch64-quick", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-clang-aarch64-lld-2stage", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-clang-aarch64-global-isel", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-clang-aarch64-full-2stage", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-clang-aarch64-sve-vls", properties={'jobs' : 48}, max_builds=1), + create_worker("linaro-clang-aarch64-sve-vls-2stage", properties={'jobs' : 48}, max_builds=1), + create_worker("linaro-lldb-aarch64-ubuntu", properties={'jobs': 16}, max_builds=1), + create_worker("linaro-flang-aarch64-dylib", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-flang-aarch64-sharedlibs", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-flang-aarch64-out-of-tree", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-flang-aarch64-debug", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-flang-aarch64-latest-clang", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-flang-aarch64-release", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-flang-aarch64-rel-assert", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-flang-aarch64-latest-gcc", properties={'jobs' : 32}, max_builds=1), # AArch64 create_worker("linaro-armv8-windows-msvc-01", properties={'jobs' : 8}, max_builds=1), create_worker("linaro-armv8-windows-msvc-02", properties={'jobs' : 8}, max_builds=1), - # ARMv7 build cache workers - create_worker("packet-linux-armv7-slave-1", properties={'jobs' : 64}, max_builds=1), - - # AArch64 build cache worker - create_worker("packet-linux-aarch64-slave-1", properties={'jobs' : 64}, max_builds=1), - # Windows Server 2016 Intel Xeon(R) Quad 2.30 GHz, 56GB of RAM create_worker("win-py3-buildbot", properties={'jobs' : 64}, max_builds=1), # Windows Server 2016 Intel(R) Xeon(R) CPU @ 2.60GHz, 16 Core(s), 128 GB of RAM create_worker("win-mlir-buildbot", properties={'jobs' : 64}, max_builds=1), + # Motorola 68k 32-bit big endian (m68k) + create_worker("debian-akiko-m68k", properties={'jobs': 1}, max_builds=1), + create_worker("suse-gary-m68k-cross", properties={'jobs': 4}, max_builds=1), + # POWER7 PowerPC big endian (powerpc64) create_worker("ppc64be-clang-test", properties={'jobs': 16}, max_builds=1), create_worker("ppc64be-clang-lnt-test", properties={'jobs': 16, 'vcs_protocol': 'https'}, max_builds=1), @@ -80,7 +80,7 @@ create_worker("ppc64le-clang-multistage-test", properties={'jobs': 8}, max_builds=1), create_worker("ppc64le-sanitizer", properties={'jobs': 4}, max_builds=1), create_worker("ppc64le-lld-multistage-test", max_builds=1), - create_worker("ppc64le-clang-rhel-test", properties={'jobs': 4}, max_builds=1), + create_worker("ppc64le-clang-rhel-test", properties={'jobs': 192}, max_builds=1), create_worker("ppc64le-flang-mlir-rhel-test", max_builds=1), # SPARC 64-bit big endian (sparc64) @@ -108,20 +108,24 @@ "gribozavr@gmail.com", "gkistanova@gmail.com"]), - # Debian 7.7 x86_64 GCE instance - create_worker("sanitizer-buildbot1", properties={'jobs': 64}, max_builds=3), - # Debian 7.7 x86_64 GCE instance - create_worker("sanitizer-buildbot2", properties={'jobs': 64}, max_builds=3), - # Debian 7.7 x86_64 GCE instance + # Debian x86_64 GCE instance + create_worker("sanitizer-buildbot1", properties={'jobs': 64}, max_builds=2), + # Debian x86_64 GCE instance + create_worker("sanitizer-buildbot2", properties={'jobs': 64}, max_builds=2), + # Debian x86_64 GCE instance create_worker("sanitizer-buildbot3", properties={'jobs': 64}, max_builds=2), - # Debian 7.7 x86_64 GCE instance + # Debian x86_64 GCE instance create_worker("sanitizer-buildbot4", properties={'jobs': 64}, max_builds=2), - # Ubuntu 14.04 x86_64 6-core z440 workstation + # Ubuntu x86_64 6-core z440 workstation create_worker("sanitizer-buildbot6", properties={'jobs': 6}, max_builds=1), - # Debian 7.7 x86_64 GCE instance - create_worker("sanitizer-buildbot7", properties={'jobs': 64}, max_builds=3), - # Debian 7.7 x86_64 GCE instance - create_worker("sanitizer-buildbot8", properties={'jobs': 64}, max_builds=3), + # Debian x86_64 GCE instance + create_worker("sanitizer-buildbot7", properties={'jobs': 64}, max_builds=2), + # Debian x86_64 GCE instance + create_worker("sanitizer-buildbot8", properties={'jobs': 64}, max_builds=2), + + # POWER 8 PowerPC AIX 7.2 + create_worker("aix-ppc64-ppc64le", properties={'jobs': 10}, max_builds=1), + create_worker("aix-ppc-libcxx", properties={'jobs': 4}, max_builds=1), # IBM z13 (s390x), Ubuntu 16.04.2 create_worker("systemz-1", properties={'jobs': 4, 'vcs_protocol': 'https'}, max_builds=4), @@ -132,8 +136,8 @@ create_worker("windows-gcebot2", properties={'jobs': 32}, max_builds=1), # Ubuntu 14.04 x86_64-scei-ps4, 2 x Intel(R) Xeon(R) CPU E5-2699 v3 @ 2.30GHz - create_worker("as-worker-91"), - create_worker("as-worker-92"), + create_worker("as-worker-91", max_builds=1), + create_worker("as-worker-92", max_builds=1), # Windows 10 Pro x86_64-scei-ps4, 2 x Intel(R) Xeon(R) CPU E5-2699 v3 @ 2.30GHz create_worker("as-worker-93", properties={'jobs': 36}, max_builds=1), @@ -143,6 +147,9 @@ create_worker("cuda-p4-0", max_builds=1), create_worker("cuda-t4-0", max_builds=1), + # HIP on Ubuntu 18.04.5, Intel(R) Xeon(R) Gold 5218 @ 2.30GHz, Vega20 GPU + create_worker("hip-vega20-0", max_builds=1), + # X86_64 AVX2, Ubuntu 16.04.2, Intel(R) Xeon(R) CPU E5-2699 v4 @ 2.20GHz create_worker("avx2-intel64", properties={'jobs': 80}, max_builds=1), @@ -162,9 +169,18 @@ # Debian x86_64 Buster Xeon(R) Gold 6154 CPU @ 3.00GHz, 192GB RAM create_worker("lldb-x86_64-debian", properties={'jobs': 72}, max_builds=1), + # Windows x86_64 32 CPUs, 125 GB RAM + create_worker("libc-x86_64-windows", properties={'jobs': 32}, max_builds=2), + + # Ubuntu aarch64 128 CPUs, 125 GB RAM + create_worker("libc-aarch64-ubuntu", properties={'jobs': 32}, max_builds=2), + # Debian x86_64 Intel Broadwell 32 CPUs, 120 GB RAM create_worker("libc-x86_64-debian", properties={'jobs': 32}, max_builds=2), + # Debian x86_64 Intel Skylake 32 CPUs, 128 GB RAM + create_worker("libc-x86_64-debian-fullbuild", properties={'jobs': 32}, max_builds=2), + # Windows Server on Xeon Gold 6130 (2x2.1GHz), 128Gb of RAM create_worker("as-builder-1", properties={ 'remote_test_host': 'jetson6.lab.llvm.org', @@ -195,8 +211,8 @@ # Windows 10, Visual Studio 2019, Google Cloud 16 cores create_worker("windows10-vs2019", properties={'jobs': 32}, max_builds=1), - # CentOS 7.5.1804 on Intel(R) Xeon(R) Gold 6126 CPU @ 2.60GHz, 96GB RAM - create_worker("nec-arrproto41", properties={'jobs': 12}, max_builds=1), + # CentOS 7.8.1127 on Intel(R) Xeon(R) Gold 6126 CPU @ 2.60GHz, 96GB RAM + create_worker("hpce-aurora2", properties={'jobs': 8}, max_builds=1), # Ubuntu 18.04 amd64 on Google Cloud, 16 core, Nvidia Tesla T4 create_worker("mlir-nvidia", properties={'jobs': 16}, max_builds=1), @@ -212,9 +228,12 @@ # Ubuntu 18.04.LTS x86_64, Intel(R) Xeon(R) CPU X3460 @ 2.80GHz, 32 GiB RAM create_worker("polly-x86_64-fdcserver", properties={'jobs': 8, 'loadaverage': 8}, max_builds=1), - create_worker("flang-aarch64-ubuntu-build"), - create_worker("flang-aarch64-ubuntu-clang-build", properties={'jobs': 14}), - create_worker("flang-aarch64-ubuntu-gcc10-build", properties={'jobs': 14}), + # Windows 10, AMD Ryzen 5 PRO 4650G, 16 GiB RAM + create_worker("minipc-ryzen-win", properties={'jobs': 12}, max_builds=1), + + # Ubuntu 20.04.LTS x86_64, Intel(R) Core(TM) i5-9400F CPU @ 2.90Ghz, 16 GiB RAM, NVIDIA GeForce GTX 1050 Ti (Pascal, sm_61, 4GiB) + create_worker("minipc-1050ti-linux", properties={'jobs': 6}, max_builds=1), + create_worker("nersc-flang"), create_worker("alcf-theta-flang", properties={'jobs': 12}, max_builds=1), @@ -228,4 +247,16 @@ # Ubuntu x86_64 create_worker("thinlto-x86-64-bot1", properties={'jobs': 64}, max_builds=1), + + # Ubuntu 20.04 on AWS, x86_64 PS4 target + create_worker("sie-linux-builder", properties={'jobs': 40}, max_builds=1), + + # XCore target, Ubuntu 20.04 x64 host + create_worker("xcore-ubuntu20-x64", properties={'jobs': 4}, max_builds=1), + + # ARC Worker, CentOS 7.9 x86_64 Intel Xeon Platinum 8000 @ 3.6GHz, 32GB RAM + create_worker("arc-worker", properties={'jobs': 16}, max_builds=1), + + # OpenMP on AMDGPU, Ubuntu 18.04.5, Intel(R) Xeon(R) Gold 5218 @ 2.30GHz with 64GB Memory, 1 Vega20 GPU with 16GB Memory + create_worker("omp-vega20-0", properties={'jobs': 32}, max_builds=1), ] diff --git a/buildbot/osuosl/master/master.cfg b/buildbot/osuosl/master/master.cfg --- a/buildbot/osuosl/master/master.cfg +++ b/buildbot/osuosl/master/master.cfg @@ -64,13 +64,19 @@ c['builders'] = builders = [ util.BuilderConfig(**b) for b in config.builders.all ] +builders.extend([ + util.BuilderConfig(**b) for b in config.release_builders.all +]) ####### SCHEDULERS -c['schedulers'] = config.schedulers.getSingleBranchSchedulers( - c['builders']) +c['schedulers'] = config.schedulers.getMainBranchSchedulers( + builders) +c['schedulers'].extend(config.schedulers.getReleaseBranchSchedulers( + builders, + treeStableTimer=5*60)) c['schedulers'].extend(config.schedulers.getForceSchedulers( - c['builders'])) + builders)) ####### BUILDBOT SERVICES diff --git a/llvmbisect/llvmlab/llvmlab.py b/llvmbisect/llvmlab/llvmlab.py --- a/llvmbisect/llvmlab/llvmlab.py +++ b/llvmbisect/llvmlab/llvmlab.py @@ -46,7 +46,7 @@ def frombasename(str, url=None): str = os.path.basename(str) - revision = timestamp = build = None + revision = revision_prefix = sha = timestamp = build = None # Check if this is a BNI style build. m = BUILD_NAME_REGEX.match(str) @@ -58,7 +58,7 @@ revision.append(int(minor_str)) if micro_str: revision.append(int(micro_str)) - return Build(name, tuple(revision), None, build, suffix) + return Build(name, None, tuple(revision), None, None, build, suffix) if '.' in str: str, suffix = str.split('.', 1) @@ -74,12 +74,20 @@ if m: str, timestamp = m.groups() + m = re.match(r'(.*)-d([0-9]+)-(.*)', str) + if m: + str, revision, sha = m.groups() + revision = int(revision) + revision_prefix = 'd' + m = re.match(r'(.*)-r([0-9]+)', str) if m: str, revision = m.groups() revision = int(revision) + revision_prefix = 'r' - return Build(str, revision, timestamp, build, suffix, url) + return Build(str, revision_prefix, revision, sha, timestamp, + build, suffix, url) @staticmethod def fromdata(data): @@ -89,13 +97,17 @@ def todata(self): return {'name': self.name, 'revision': self.revision, + 'sha': self.sha, 'timestamp': self.timestamp, 'build': self.build, 'suffix': self.suffix} - def __init__(self, name, revision, timestamp, build, suffix, url): + def __init__(self, name, revision_prefix, revision, sha, timestamp, + build, suffix, url): self.name = name + self.revision_prefix = revision_prefix self.revision = revision + self.sha = sha self.timestamp = timestamp self.build = build self.suffix = suffix @@ -108,7 +120,9 @@ basename += '-' + '.'.join(str(r) for r in self.revision) else: assert isinstance(self.revision, int) - basename += '-r%d' % self.revision + basename += '-%s%d' % (self.revision_prefix, self.revision) + if self.sha is not None: + basename += '-%s' % self.sha if self.timestamp is not None: basename += '-t%s' % self.timestamp if self.build is not None: @@ -122,8 +136,8 @@ def __repr__(self): return "%s%r" % (self.__class__.__name__, - (self.name, self.revision, self.timestamp, self.build, - self.suffix)) + (self.name, self.revision, self.sha, self.timestamp, + self.build, self.suffix)) def __cmp__(self, other): return cmp((self.revision, self.timestamp, diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -1,29 +1,29 @@ -attrs==20.2.0 +attrs==20.3.0 six==1.15.0 Automat==20.2.0 constantly==15.1.0 idna==2.10 -hyperlink==20.0.1 +hyperlink==21.0.0 incremental==17.5.0 -Jinja2==2.11.2 +Jinja2==2.11.3 PyHamcrest==2.0.2 Tempita==0.5.2 decorator==4.4.2 pbr==5.5.1 sqlparse==0.4.1 -SQLAlchemy==1.3.20 +SQLAlchemy==1.3.23 sqlalchemy-migrate==0.13.0 -certifi==2020.6.20 -chardet==3.0.4 -urllib3==1.25.11 -requests==2.24.0 -zope.interface==5.1.2 +certifi==2020.12.5 +chardet==4.0.0 +urllib3==1.26.4 +requests==2.25.1 +zope.interface==5.2.0 pycparser==2.20 -pyOpenSSL==19.1.0 +pyOpenSSL==20.0.1 pyasn1==0.4.8 pyasn1-modules==0.2.8 service-identity==18.1.0 -cryptography==3.2.1 -cffi==1.14.3 +cryptography==3.4.5 +cffi==1.14.5 Twisted==20.3.0 -treq==20.9.0 +treq==21.1.0 diff --git a/tasks/tasktool/README.md b/tasks/tasktool/README.md --- a/tasks/tasktool/README.md +++ b/tasks/tasktool/README.md @@ -92,7 +92,7 @@ { "url": "https://github.com/my-project.git", "type": "git", - "default_rev": "refs/heads/master" + "default_rev": "refs/heads/main" } ``` @@ -166,7 +166,7 @@ : ${VARIABLE:='DefaultValue'} ``` -- Abstract artifacts like `refs/heads/master` in a git repository or +- Abstract artifacts like `refs/heads/main` in a git repository or `clang-stage2-cmake` on an A2 server are resolved to a concrete git revision or a specific URL when the task is started. - A task with a set of artifact and parameter inputs is called a build. diff --git a/tasks/tasktool/tasktool/repos/git.py b/tasks/tasktool/tasktool/repos/git.py --- a/tasks/tasktool/tasktool/repos/git.py +++ b/tasks/tasktool/tasktool/repos/git.py @@ -25,7 +25,7 @@ rev = config.get('rev') default_rev = config.pop('default_rev', None) if default_rev is None: - default_rev = "refs/heads/master" + default_rev = "refs/heads/main" if rev is not None: return url = config['url'] diff --git a/tasks/utils/venv.sh b/tasks/utils/venv.sh --- a/tasks/utils/venv.sh +++ b/tasks/utils/venv.sh @@ -1,5 +1,5 @@ echo "@@@ LNT VirtualEnv @@@" -/usr/local/bin/virtualenv venv +/usr/local/bin/virtualenv --python=python3 venv set +u . venv/bin/activate set -u diff --git a/test/jenkins/test_monorepo_build.py b/test/jenkins/test_monorepo_build.py --- a/test/jenkins/test_monorepo_build.py +++ b/test/jenkins/test_monorepo_build.py @@ -3,7 +3,7 @@ # RUN: export TESTING=1 # RUN: export JOB_NAME="FOO" # RUN: export BUILD_NUMBER=321 -# RUN: export BRANCH=master +# RUN: export BRANCH=main # Tell monorepo_build.py to just print commands instead of running. # RUN: mkdir -p %t.SANDBOX/host-compiler/lib %t.SANDBOX/host-compiler/bin %t.SANDBOX/llvm-project/llvm %t.SANDBOX/llvm-project/clang %t.SANDBOX/llvm-project/libcxx %t.SANDBOX/llvm-project/compiler-rt %t.SANDBOX/llvm-project/debuginfo-tests %t.SANDBOX/llvm-project/clang-tools-extra %t.SANDBOX/llvm-project/lldb # RUN: touch %t.SANDBOX/host-compiler/bin/clang @@ -146,4 +146,4 @@ # Test to check if default timeout is being set to 600. # RUN: python %{src_root}/zorg/jenkins/monorepo_build.py cmake all > %t-timeout-default.log # RUN: FileCheck --check-prefix CHECK-TIMEOUT-DEFAULT < %t-timeout-default.log %s -# CHECK-TIMEOUT-DEFAULT: --timeout=600 \ No newline at end of file +# CHECK-TIMEOUT-DEFAULT: --timeout=600 diff --git a/zorg/buildbot/builders/AOSPBuilder.py b/zorg/buildbot/builders/AOSPBuilder.py --- a/zorg/buildbot/builders/AOSPBuilder.py +++ b/zorg/buildbot/builders/AOSPBuilder.py @@ -25,8 +25,9 @@ target_flags=None, # Extra C/CXX flags for AOSP target build jobs=None, # Number of concurrent jobs extra_make_args=None): # Extra args for the make command - command = "source build/envsetup.sh" + command = "/bin/bash -c \"source build/envsetup.sh" command += " && lunch aosp_%s-userdebug" % device + command += " && export LC_ALL=C" command += " && make -k" if timeout: command += " TIMEOUT=%s" % timeout @@ -38,6 +39,7 @@ command += " " + extra_make_args if jobs: command += " -j" + str(jobs) + command += "\"" return command diff --git a/zorg/buildbot/builders/AnnotatedBuilder.py b/zorg/buildbot/builders/AnnotatedBuilder.py --- a/zorg/buildbot/builders/AnnotatedBuilder.py +++ b/zorg/buildbot/builders/AnnotatedBuilder.py @@ -11,7 +11,8 @@ env=None, extra_args=None, timeout=1200, - checkout_llvm_sources=True): + checkout_llvm_sources=True, + script_interpreter="python"): """ Returns a new build factory that uses AnnotatedCommand, which allows the build to be run by version-controlled scripts that do @@ -26,6 +27,7 @@ env: environment overrides (map; default is no overrides) extra_args: extra arguments to pass to the script (default: []) timeout: specifies the builder's timeout in seconds (default: 1200) + script_interpreter: specifies the interpreter to run scripts (default: "python") """ if depends_on_projects is None: @@ -38,7 +40,10 @@ "libunwind", "lld"] if extra_args is None: - extra_args = [] + # We used to add --jobs to all script invocations. Perserve this + # for cases when the user did not specify extra_args, but allow + # overriding it if the user did specify extra_args. + extra_args = ["--jobs=%(jobs:-)s"] f = LLVMBuildFactory( clean=clean, @@ -84,7 +89,12 @@ command = [script] else: script_path = "../llvm-zorg/zorg/buildbot/builders/annotated/%s" % (script) - command = ["python", script_path, WithProperties("--jobs=%(jobs:-)s")] + # Handle scripts with script_interpreter, otherwise execute the script directly. + if script_interpreter: + command = [script_interpreter, script_path] + else: + command = [script_path] + command += extra_args_with_props f.addStep(AnnotatedCommand(name="annotate", diff --git a/zorg/buildbot/builders/ClangBuilder.py b/zorg/buildbot/builders/ClangBuilder.py --- a/zorg/buildbot/builders/ClangBuilder.py +++ b/zorg/buildbot/builders/ClangBuilder.py @@ -1,6 +1,7 @@ import copy from datetime import datetime +from buildbot.plugins import util from buildbot.process.properties import WithProperties, Property from buildbot.steps.shell import ShellCommand, SetProperty from buildbot.steps.shell import WarningCountingShellCommand @@ -382,9 +383,11 @@ if not vs: cc = 'clang' cxx = 'clang++' + fc = 'flang' else: cc = 'clang-cl.exe' cxx = 'clang-cl.exe' + fc = 'flang.exe' ############# STAGE 2 @@ -496,6 +499,14 @@ '--cc', cc, '--cxx', cxx, '--use-lit', lit] + # Enable fortran if flang is checked out + if checkout_flang: + fortran_flags = [ + '--cmake-define=TEST_SUITE_FORTRAN:STRING=ON', + util.Interpolate( + '--cmake-define=CMAKE_Fortran_COMPILER=' + + '%(prop:builddir)s/'+compiler_path+'/bin/'+fc)] + test_suite_cmd.extend(fortran_flags) # Append any option provided by the user test_suite_cmd.extend(testsuite_flags) @@ -521,7 +532,7 @@ workdir='test', env=env)) f.addStep(ShellCommand(name='recreate sandbox', - command=['virtualenv', 'sandbox'], + command=['virtualenv', '--python=python3', 'sandbox'], haltOnFailure=True, description='recreating sandbox', workdir='test', diff --git a/zorg/buildbot/builders/ClangLTOBuilder.py b/zorg/buildbot/builders/ClangLTOBuilder.py --- a/zorg/buildbot/builders/ClangLTOBuilder.py +++ b/zorg/buildbot/builders/ClangLTOBuilder.py @@ -65,9 +65,14 @@ ('-DCMAKE_INSTALL_PREFIX=', install_dir), ]) - CmakeCommand.applyRequiredOptions(cmake_args, [ - ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)), - ]) + if f.enable_projects: + CmakeCommand.applyRequiredOptions(cmake_args, [ + ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.enable_projects)), + ]) + if f.enable_runtimes: + CmakeCommand.applyRequiredOptions(cmake_args, [ + ('-DLLVM_ENABLE_RUNTIMES=', ";".join(f.enable_runtimes)), + ]) # Note: On this stage we do not care of warnings, as we build with # a system toolchain and cannot control the environment. @@ -181,9 +186,14 @@ "-DCMAKE_C_COMPILER=%(builddir)s/" + staged_install + "/bin/clang" )) - CmakeCommand.applyRequiredOptions(cmake_args, [ - ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)), - ]) + if f.enable_projects: + CmakeCommand.applyRequiredOptions(cmake_args, [ + ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.enable_projects)), + ]) + if f.enable_runtimes: + CmakeCommand.applyRequiredOptions(cmake_args, [ + ('-DLLVM_ENABLE_RUNTIMES=', ";".join(f.enable_runtimes)), + ]) # Create configuration files with cmake f.addStep(CmakeCommand(name="cmake-configure-stage%s" % stage_num, @@ -233,6 +243,7 @@ clean = False, jobs = None, extra_configure_args = None, + extra_configure_args_lto_stage = None, compare_last_2_stages = True, lto = None, # The string gets passed to -flto flag as is. Like -flto=thin. env = None, @@ -253,10 +264,16 @@ else: extra_configure_args = list(extra_configure_args) + if extra_configure_args_lto_stage is None: + extra_configure_args_lto_stage = [] + else: + extra_configure_args_lto_stage = list(extra_configure_args_lto_stage) + # Make sure CMAKE_INSTALL_PREFIX and -G are not specified # in the extra_configure_args. We set them internally as needed. # TODO: assert extra_configure_args. - install_prefix_specified = any(a.startswith('-DCMAKE_INSTALL_PREFIX=') for a in extra_configure_args) + install_prefix_specified = (any(a.startswith('-DCMAKE_INSTALL_PREFIX=') for a in extra_configure_args) or + any(a.startswith('-DCMAKE_INSTALL_PREFIX=') for a in extra_configure_args_lto_stage)) assert not install_prefix_specified, "Please do not explicitly specify the install prefix for multi-stage build." # Prepare environmental variables. Set here all env we want everywhere. @@ -321,7 +338,7 @@ s = f.staged_compiler_idx + 1 staged_install = f.stage_installdirs[f.staged_compiler_idx] for i in range(s, len(f.stage_objdirs[s:]) + s): - configure_args = extra_configure_args[:] + configure_args = extra_configure_args[:] + extra_configure_args_lto_stage[:] configure_args.append( WithProperties( diff --git a/zorg/buildbot/builders/DoxygenDocsBuilder.py b/zorg/buildbot/builders/DoxygenDocsBuilder.py new file mode 100644 --- /dev/null +++ b/zorg/buildbot/builders/DoxygenDocsBuilder.py @@ -0,0 +1,121 @@ +from collections import OrderedDict +from importlib import reload + +from buildbot.plugins import steps, util +from buildbot.steps.shell import ShellCommand + +from zorg.buildbot.builders import UnifiedTreeBuilder +from zorg.buildbot.commands import CmakeCommand + +from zorg.buildbot.process import factory +reload(factory) +reload(UnifiedTreeBuilder) +reload(CmakeCommand) + +llvm_docs = OrderedDict([ + # Project Build target Local path Remote path + ("llvm", ("doxygen", "docs/doxygen/html/", "llvm")), + ("clang", (None, "tools/clang/docs/doxygen/html/", "cfe")), + ("clang-tools-extra", (None, "tools/clang/tools/extra/docs/doxygen/html/", "cfe-extra")), + ("flang", (None, "tools/flang/docs/doxygen/html/", "flang")), + ("polly", (None, "tools/polly/docs/doxygen/html/", "polly")), + ("openmp", (None, "projects/openmp/docs/doxygen/html/", "openmp")), + ("lldb", ("lldb-cpp-doc", "tools/lldb/docs/cpp_reference/", "lldb/cpp_reference")), + # NOTE: 5/9/2020 lldb-python-doc fails to build. Disabled till be fixed. + #(None, ("lldb-python-doc", "tools/lldb/docs/python_reference/", "lldb")), +]) + +def getLLVMDocsBuildFactory( + clean = True, + depends_on_projects = None, + extra_configure_args = None, + env = None, + **kwargs): + + if depends_on_projects is None: + # All the projects from llvm_docs, and remove all duplicates. + _depends_on_projects=list(set( + [project for project in llvm_docs if project])) + else: + # Make a local copy of depends_on_projects, as we are going to modify + # that. + _depends_on_projects=depends_on_projects[:] + # Some projects are interdependent for the purpose of documentation. + # Enforce the dependencies. + # TODO: Check later the dependencies for doxygen docs and enforce them + # here if needed. + + # Make a local copy of the configure args, as we are going to modify that. + if extra_configure_args: + cmake_args = extra_configure_args[:] + else: + cmake_args = list() + + # Prepare environmental variables. Set here all env we want everywhere. + merged_env = { + 'TERM' : 'dumb' # Be cautious and disable color output from all tools. + } + if env is not None: + # Overwrite pre-set items with the given ones, so user can set anything. + merged_env.update(env) + + CmakeCommand.CmakeCommand.applyDefaultOptions(cmake_args, [ + ("-G", "Ninja"), + ("-DLLVM_ENABLE_DOXYGEN=", "ON"), + ("-DLLVM_BUILD_DOCS=", "ON"), + ("-DCLANG_TOOLS_EXTRA_INCLUDE_DOCS=", "ON"), + ("-DLLVM_ENABLE_ASSERTIONS=", "OFF"), + ("-DCMAKE_BUILD_TYPE=", "Release"), + ]) + + f = UnifiedTreeBuilder.getCmakeBuildFactory( + clean=clean, + depends_on_projects=_depends_on_projects, + enable_runtimes=[], # Docs don't support runtimes build yet. + extra_configure_args=cmake_args, + env=merged_env, + **kwargs) # Pass through all the extra arguments. + + # Build the documentation for all the projects. + for project in llvm_docs: + target = llvm_docs[project][0] + + # Build only those with specifies targets. + if target: + UnifiedTreeBuilder.addNinjaSteps( + f, + # Doxygen builds the final result for really + # long time without any output. + # We have to have a long timeout at this step. + timeout=21600, + targets=[target], + checks=[], + env=merged_env, + **kwargs) + + # Publish just built documentation + for project in llvm_docs: + target, local_path, remote_path = llvm_docs[project] + + f.addStep( + ShellCommand( + name="Publish {}".format(project or target), + description=[ + "Publish", "just", "built", "documentation", "for", + "{}".format(project or target) + ], + command=[ + 'rsync', + '-vrl', + '--delete', '--force', '--delay-updates', '--delete-delay', + '--ignore-times', + '--checksum', + '-p', '--chmod=Du=rwx,Dg=rwx,Do=rx,Fu=rw,Fg=rw,Fo=r', + "{}".format(local_path), + "lists.llvm.org:web/doxygen/{}".format(remote_path), + ], + env=merged_env, + ) + ) + + return f diff --git a/zorg/buildbot/builders/FlangBuilder.py b/zorg/buildbot/builders/FlangBuilder.py --- a/zorg/buildbot/builders/FlangBuilder.py +++ b/zorg/buildbot/builders/FlangBuilder.py @@ -14,9 +14,10 @@ env = dict() f = getCmakeWithNinjaBuildFactory( - depends_on_projects=['llvm','mlir'], + depends_on_projects=['llvm','clang','mlir'], obj_dir="build_llvm", checks=[], + install_dir="install", clean=clean, extra_configure_args=llvm_extra_configure_args, env=env, @@ -42,6 +43,7 @@ # Add LLVM_DIR and MLIR_DIR to the CMake invocation. llvm_dir = "{}/lib/cmake/llvm".format(f.obj_dir) mlir_dir = "{}/lib/cmake/mlir".format(f.obj_dir) + clang_dir = "{}/lib/cmake/clang".format(f.install_dir) CmakeCommand.applyRequiredOptions(flang_cmake_args, [ # We actually need the paths to be relative to the source directory, # otherwise find_package can't locate the config files. @@ -49,6 +51,8 @@ LLVMBuildFactory.pathRelativeTo(llvm_dir, flang_src_dir)), ('-DMLIR_DIR:PATH=', LLVMBuildFactory.pathRelativeTo(mlir_dir, flang_src_dir)), + ('-DCLANG_DIR:PATH=', + LLVMBuildFactory.pathRelativeTo(clang_dir, flang_src_dir)), ]) # We can't use addCmakeSteps as that would use the path in f.llvm_srcdir. diff --git a/zorg/buildbot/builders/HtmlDocsBuilder.py b/zorg/buildbot/builders/HtmlDocsBuilder.py new file mode 100644 --- /dev/null +++ b/zorg/buildbot/builders/HtmlDocsBuilder.py @@ -0,0 +1,105 @@ +from collections import OrderedDict +from importlib import reload + +from buildbot.plugins import steps, util +from buildbot.steps.shell import ShellCommand +from buildbot.steps.shell import WarningCountingShellCommand + +from zorg.buildbot.process import factory +reload(factory) + +llvm_docs = OrderedDict([ + # Project Build target Build path Local path Remote path + ("lnt", ("html", "docs", "_build/html/", "lnt")), +]) + +# We build with make for now. Change later if needed. +build_cmd = 'make' + +def getHtmlDocsBuildFactory( + depends_on_projects = None, + clean = False, + env = None, + **kwargs): + + if depends_on_projects is None: + # All the projects by default. + _depends_on_projects=llvm_docs.keys() + else: + # Make a local copy of depends_on_projects, as we are going to modify + # that. + _depends_on_projects=depends_on_projects[:] + + # Prepare environmental variables. Set here all env we want everywhere. + merged_env = { + 'TERM' : 'dumb' # Be cautious and disable color output from all tools. + } + if env is not None: + # Overwrite pre-set items with the given ones, so user can set anything. + merged_env.update(env) + + # HTML Sphinx documentation builds in tree, each in its own directory. + # For that, make sure the obj_dir is the same as llvm_srcdir. + src_dir = kwargs.pop('llvm_srcdir', '.') + f = factory.LLVMBuildFactory( + clean=clean, + depends_on_projects=_depends_on_projects, + llvm_srcdir=src_dir, + obj_dir=src_dir, + **kwargs) # Pass through all the extra arguments. + + # Build the documentation + for project in llvm_docs: + + # Checkout the source code and remove all the untracked files, so + # we would build a fresh new documentation. + f.addStep( + steps.Git( + name='Checkout the {} source code'.format(project), + repourl=f.repourl_prefix + "llvm-{}.git".format(project), + mode='full', + method='fresh', + progress=True, + workdir=util.WithProperties(project), + env=merged_env, + **kwargs)) + + target, build_path, local_path, remote_path = llvm_docs[project] + + build_dir = util.WithProperties( + "{}".format("/".join([ + project, + build_path]))) + f.addStep( + steps.WarningCountingShellCommand( + name="Build {} documentation".format(project), + command=[build_cmd, target], + haltOnFailure=True, + workdir=build_dir, + env=merged_env, + **kwargs)) + + # Publish just built documentation + f.addStep( + ShellCommand( + name="Publish {}".format(project), + description=[ + "Publish", "just", "built", "documentation", "for", + "{}".format(project) + ], + command=[ + 'rsync', + '-vrl', + '--delete', '--force', '--delay-updates', '--delete-delay', + '--ignore-times', + '--checksum', + '-p', '--chmod=Du=rwx,Dg=rwx,Do=rx,Fu=rw,Fg=rw,Fo=r', + "{}".format(local_path), + "lists.llvm.org:web/{}".format(remote_path), + ], + workdir=build_dir, + env=merged_env, + ) + ) + + return f diff --git a/zorg/buildbot/builders/LibcxxAndAbiBuilder.py b/zorg/buildbot/builders/LibcxxAndAbiBuilder.py --- a/zorg/buildbot/builders/LibcxxAndAbiBuilder.py +++ b/zorg/buildbot/builders/LibcxxAndAbiBuilder.py @@ -12,6 +12,7 @@ check_libcxx_benchmarks=None, depends_on_projects=None, use_cache=None, + build_standalone=False, **kwargs): if env is None: @@ -27,12 +28,19 @@ depends_on_projects = ['libcxx','libcxxabi','libunwind'] src_root = 'llvm' + build_path = 'build' + if build_standalone: + src_to_build_dir = 'runtimes' + else: + src_to_build_dir = None + if f is None: f = UnifiedTreeBuilder.getLLVMBuildFactoryAndSourcecodeSteps( depends_on_projects=depends_on_projects, llvm_srcdir=src_root, + src_to_build_dir=src_to_build_dir, obj_dir=build_path, **kwargs) # Pass through all the extra arguments. @@ -71,7 +79,17 @@ workdir=".", haltOnFailure=False)) - CmakeCommand.applyRequiredOptions(cmake_opts, [ + if build_standalone: + if f.enable_projects: + CmakeCommand.applyRequiredOptions(cmake_opts, [ + ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.enable_projects)), + ]) + if f.enable_runtimes: + CmakeCommand.applyRequiredOptions(cmake_opts, [ + ('-DLLVM_ENABLE_RUNTIMES=', ";".join(f.enable_runtimes)), + ]) + else: + CmakeCommand.applyRequiredOptions(cmake_opts, [ ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)), ]) @@ -79,32 +97,38 @@ name='cmake', command=['cmake', rel_src_dir] + cmake_opts, haltOnFailure=True, workdir=build_path, env=env)) - # Build libcxxabi jobs_flag = properties.WithProperties('-j%(jobs)s') - f.addStep(buildbot.steps.shell.ShellCommand( - name='build.libcxxabi', command=['make', jobs_flag, 'cxxabi'], - haltOnFailure=True, workdir=build_path)) + # Build libcxxabi + if 'libcxxabi' in depends_on_projects: + f.addStep(buildbot.steps.shell.ShellCommand( + name='build.libcxxabi', command=['make', jobs_flag, 'cxxabi'], + haltOnFailure=True, workdir=build_path, env=env)) # Build libcxx - f.addStep(buildbot.steps.shell.ShellCommand( - name='build.libcxx', command=['make', jobs_flag, 'cxx'], - haltOnFailure=True, workdir=build_path)) + if 'libcxx' in depends_on_projects: + f.addStep(buildbot.steps.shell.ShellCommand( + name='build.libcxx', command=['make', jobs_flag, 'cxx'], + haltOnFailure=True, workdir=build_path, env=env)) # Test libc++abi - f.addStep(LitTestCommand( - name = 'test.libcxxabi', - command = ['make', jobs_flag, 'check-cxxabi'], - description = ['testing', 'libcxxabi'], - descriptionDone = ['test', 'libcxxabi'], - workdir = build_path)) + if 'libcxxabi' in depends_on_projects: + f.addStep(LitTestCommand( + name = 'test.libcxxabi', + command = ['make', jobs_flag, 'check-cxxabi'], + description = ['testing', 'libcxxabi'], + descriptionDone = ['test', 'libcxxabi'], + workdir = build_path, + env = env)) # Test libc++ - f.addStep(LitTestCommand( - name = 'test.libcxx', - command = ['make', jobs_flag, 'check-cxx'], - description = ['testing', 'libcxx'], - descriptionDone = ['test', 'libcxx'], - workdir = build_path)) + if 'libcxx' in depends_on_projects: + f.addStep(LitTestCommand( + name = 'test.libcxx', + command = ['make', jobs_flag, 'check-cxx'], + description = ['testing', 'libcxx'], + descriptionDone = ['test', 'libcxx'], + workdir = build_path, + env = env)) if check_libcxx_abilist: f.addStep(buildbot.steps.shell.ShellCommand( @@ -112,14 +136,15 @@ command = ['make', 'check-cxx-abilist'], description = ['testing', 'libcxx', 'abi'], descriptionDone = ['test', 'libcxx', 'abi'], - workdir = build_path)) + workdir = build_path, + env = env)) if check_libcxx_benchmarks: # Build the libc++ benchmarks f.addStep(buildbot.steps.shell.ShellCommand( name='build.libcxx.benchmarks', command=['make', jobs_flag, 'cxx-benchmarks'], - haltOnFailure=True, workdir=build_path)) + haltOnFailure=True, workdir=build_path, env=env)) # Run the benchmarks f.addStep(LitTestCommand( @@ -127,6 +152,7 @@ command = ['make', jobs_flag, 'check-cxx-benchmarks'], description = ['testing', 'libcxx', 'benchmarks'], descriptionDone = ['test', 'libcxx', 'benchmarks'], - workdir = build_path)) + workdir = build_path, + env = env)) return f diff --git a/zorg/buildbot/builders/OpenMPBuilder.py b/zorg/buildbot/builders/OpenMPBuilder.py --- a/zorg/buildbot/builders/OpenMPBuilder.py +++ b/zorg/buildbot/builders/OpenMPBuilder.py @@ -1,5 +1,6 @@ from buildbot.steps.shell import ShellCommand from buildbot.process.properties import WithProperties +from buildbot.plugins import steps from zorg.buildbot.commands.LitTestCommand import LitTestCommand from zorg.buildbot.commands.CmakeCommand import CmakeCommand @@ -14,6 +15,12 @@ ompt = False, # Whether to enable the OpenMP Tools Interface. test = True, # Test the built libraries. depends_on_projects = None, + enable_runtimes = None, + extraCmakeArgs = [], + install = False, + testsuite = False, + testsuite_sollvevv = False, + extraTestsuiteCmakeArgs = [], **kwargs): # Prepare environmental variables. Set here all env we want everywhere. @@ -26,6 +33,10 @@ llvm_srcdir = 'llvm.src' llvm_builddir = 'llvm.build' + llvm_instdir = 'llvm.inst' + testsuite_srcdir = "test-suite.src" + testsuite_builddir = "test-suite.build" + sollvevv_srcdir = "sollvevv.src" cleanBuildRequested = lambda step: clean or step.build.getProperty("clean", default=step.build.getProperty("clean_obj")) @@ -35,6 +46,7 @@ f = UnifiedTreeBuilder.getLLVMBuildFactoryAndSourcecodeSteps( depends_on_projects=depends_on_projects, + enable_runtimes=enable_runtimes, llvm_srcdir=llvm_srcdir, obj_dir=llvm_builddir, cleanBuildRequested=cleanBuildRequested, @@ -58,10 +70,19 @@ if test: lit_args = '-vv --show-unsupported --show-xfail -j %s' % jobs cmake_args += [WithProperties('-DLLVM_LIT_ARGS=%s' % lit_args)] + if install: + cmake_args += [WithProperties('-DCMAKE_INSTALL_PREFIX=%(builddir)s/' + llvm_instdir)] + cmake_args += extraCmakeArgs - CmakeCommand.applyRequiredOptions(cmake_args, [ - ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)), - ]) + if f.enable_projects: + CmakeCommand.applyDefaultOptions(cmake_args, [ + ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.enable_projects)), + ]) + + if f.enable_runtimes: + CmakeCommand.applyDefaultOptions(cmake_args, [ + ('-DLLVM_ENABLE_RUNTIMES=', ";".join(f.enable_runtimes)), + ]) # Add llvm-lit and clang (if built) to PATH merged_env.update({ @@ -103,6 +124,121 @@ description = 'test openmp', workdir = f.obj_dir, env = merged_env, - haltOnFailure=True)) + haltOnFailure=False, + flunkOnFailure=True)) + + clangexe = "%(builddir)s/" + llvm_builddir + "/bin/clang" + clangxxexe = "%(builddir)s/" + llvm_builddir + "/bin/clang++" + litexe = "%(builddir)s/" + llvm_builddir + "/bin/llvm-lit" + libdir = "%(builddir)s/" + llvm_builddir + "/lib" + if install: + f.addStep(steps.RemoveDirectory(name="LLVM: Clean Install Dir", + dir=llvm_instdir, + haltOnFailure=False)) + + f.addStep(NinjaCommand(name="LLVM: Install", + description="installing", + descriptionDone="install", + descriptionSuffix="LLVM", + workdir=f.obj_dir, + targets=['install'], + env=merged_env, + haltOnFailure=True)) + # If installing, use the installed version of clang. + clangexe = "%(builddir)s/" + llvm_instdir + "/bin/clang" + clangxxexe = "%(builddir)s/" + llvm_instdir + "/bin/clang++" + libdir = "%(builddir)s/" + llvm_instdir + "/lib" + + + if testsuite: + f.addStep(steps.RemoveDirectory(name="Test-Suite: Clean Source", + dir=testsuite_srcdir, + haltOnFailure=False, + warnOnFailure=True, + doStepIf=cleanBuildRequested)) + + f.addGetSourcecodeForProject(name="Test-Suite: Checkout", + description="fetching", + descriptionDone="fetch", + descriptionSuffix="Test-Suite", + project='test-suite', + src_dir=testsuite_srcdir, + alwaysUseLatest=True) + + if testsuite_sollvevv: + f.addStep(steps.RemoveDirectory(name="SOLLVE V&V: Clean Source", + dir=sollvevv_srcdir, + haltOnFailure=False, + warnOnFailure=True, + doStepIf=cleanBuildRequested)) + + f.addStep(steps.Git(name="SOLLVE V&V: Checkout", + description="fetching", + descriptionDone="fetch", + descriptionSuffix="SOLLVE V&V", + repourl='https://github.com/SOLLVE/sollve_vv.git', + workdir=sollvevv_srcdir, + alwaysUseLatest=True)) + + f.addStep(steps.RemoveDirectory(name="Test-Suite: Clean Build", + dir=testsuite_builddir, + haltOnFailure=False, + warnOnFailure=True)) + + testsuite_options = [ + "-DCMAKE_BUILD_TYPE=Release", + "-DTEST_SUITE_LIT_FLAGS=-vv;-s;-j6;-o;report.json", + "-DTEST_SUITE_EXTRA_C_FLAGS=-gline-tables-only", + "-DTEST_SUITE_EXTRA_CXX_FLAGS=-gline-tables-only", + WithProperties("-DCMAKE_C_COMPILER=" + clangexe), + WithProperties("-DCMAKE_CXX_COMPILER=" + clangxxexe), + WithProperties("-DTEST_SUITE_LIT=" + litexe), + ] + if testsuite_sollvevv: + testsuite_options += [ + "-DTEST_SUITE_SUBDIRS=External/sollve_vv", + "-DTEST_SUITE_SOLLVEVV_ROOT=../" + sollvevv_srcdir, + ] + testsuite_options += extraTestsuiteCmakeArgs + f.addStep(CmakeCommand(name="Test-Suite: Configure", + description="configuring", + descriptionDone="configure", + descriptionSuffix="Test-Suite", + generator='Ninja', + path='../' + testsuite_srcdir, + workdir=testsuite_builddir, + options=testsuite_options, + haltOnFailure=True, + logfiles={ + 'CMakeCache.txt' : 'CMakeCache.txt', + })) + + f.addStep(NinjaCommand(name="Test-Suite: Compile", + description="compiling", + descriptionDone="compile", + descriptionSuffix="Test-Suite", + options=['-k0'], # Continue building; programs that don't compile will fail with NOEXE. + haltOnFailure=False, + flunkOnFailure=False, # SOLLVE V&V contains tests that clang have not been implemented yet. + warnOnFailure=True, + workdir=testsuite_builddir)) + + merged_env.update({ + 'LD_LIBRARY_PATH': WithProperties(libdir + ':${LD_LIBRARY_PATH}') + }) + f.addStep(LitTestCommand(name="Test-Suite: Run", + description="running", + descriptionDone="run", + descriptionSuffix="Test-Suite", + command=[WithProperties(litexe), '-vv', '-s', '-j6', '-o','report.json', '.'], + haltOnFailure=False, + flunkOnFailure=False, # SOLLVE V&V contains tests that clang have not been implemented yet. + warnOnFailure=True, + workdir=testsuite_builddir, + env=merged_env, + logfiles={ + 'test.log' : 'test.log', + 'report.json': 'report.json' + })) return f diff --git a/zorg/buildbot/builders/SanitizerBuilder.py b/zorg/buildbot/builders/SanitizerBuilder.py --- a/zorg/buildbot/builders/SanitizerBuilder.py +++ b/zorg/buildbot/builders/SanitizerBuilder.py @@ -6,6 +6,7 @@ def getSanitizerBuildFactory( clean=False, depends_on_projects=None, + extra_configure_args=None, env=None, timeout=1200): @@ -30,6 +31,11 @@ "libunwind", "lld"] + if extra_configure_args: + extra_configure_args = ['--'] + list(extra_configure_args) + else: + extra_configure_args = [] + # Explicitly use '/' as separator, because it works on *nix and Windows. sanitizer_script_dir = "sanitizer_buildbot/sanitizers" sanitizer_script = "../%s/zorg/buildbot/builders/sanitizers/%s" % (sanitizer_script_dir, "buildbot_selector.py") @@ -47,10 +53,11 @@ alwaysUseLatest=True) # Run annotated command for sanitizer. + command = ['python', sanitizer_script] + extra_configure_args f.addStep(AnnotatedCommand(name="annotate", description="annotate", timeout=timeout, haltOnFailure=True, - command="python " + sanitizer_script, + command=command, env=merged_env)) return f diff --git a/zorg/buildbot/builders/SphinxDocsBuilder.py b/zorg/buildbot/builders/SphinxDocsBuilder.py --- a/zorg/buildbot/builders/SphinxDocsBuilder.py +++ b/zorg/buildbot/builders/SphinxDocsBuilder.py @@ -69,6 +69,7 @@ f = UnifiedTreeBuilder.getCmakeBuildFactory( depends_on_projects=depends_on_projects, + enable_runtimes=[], # Docs don't support runtimes build yet. llvm_srcdir=llvm_srcdir, obj_dir=llvm_objdir, extra_configure_args=cmake_args, @@ -213,6 +214,7 @@ f = UnifiedTreeBuilder.getCmakeBuildFactory( clean=clean, depends_on_projects=_depends_on_projects, + enable_runtimes=[], # Docs don't support runtimes build yet. extra_configure_args=cmake_args, env=merged_env, **kwargs) # Pass through all the extra arguments. diff --git a/zorg/buildbot/builders/UnifiedTreeBuilder.py b/zorg/buildbot/builders/UnifiedTreeBuilder.py --- a/zorg/buildbot/builders/UnifiedTreeBuilder.py +++ b/zorg/buildbot/builders/UnifiedTreeBuilder.py @@ -13,6 +13,7 @@ def getLLVMBuildFactoryAndPrepareForSourcecodeSteps( depends_on_projects = None, + enable_runtimes = None, llvm_srcdir = None, obj_dir = None, install_dir = None, @@ -29,6 +30,7 @@ f = LLVMBuildFactory( depends_on_projects=depends_on_projects, + enable_runtimes=enable_runtimes, llvm_srcdir=llvm_srcdir, obj_dir=obj_dir, install_dir=install_dir, @@ -49,7 +51,9 @@ def getLLVMBuildFactoryAndSourcecodeSteps( depends_on_projects = None, + enable_runtimes = None, llvm_srcdir = None, + src_to_build_dir = None, obj_dir = None, install_dir = None, cleanBuildRequested = None, @@ -57,7 +61,9 @@ f = getLLVMBuildFactoryAndPrepareForSourcecodeSteps( depends_on_projects=depends_on_projects, + enable_runtimes=enable_runtimes, llvm_srcdir=llvm_srcdir, + src_to_build_dir=src_to_build_dir, obj_dir=obj_dir, install_dir=install_dir, cleanBuildRequested=cleanBuildRequested, @@ -99,9 +105,15 @@ doStepIf=cleanBuildRequested, )) - CmakeCommand.applyDefaultOptions(cmake_args, [ - ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)), - ]) + if f.enable_projects: + CmakeCommand.applyDefaultOptions(cmake_args, [ + ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.enable_projects)), + ]) + + if f.enable_runtimes: + CmakeCommand.applyDefaultOptions(cmake_args, [ + ('-DLLVM_ENABLE_RUNTIMES=', ";".join(f.enable_runtimes)), + ]) if install_dir: install_dir_rel = LLVMBuildFactory.pathRelativeTo( @@ -230,6 +242,7 @@ def getCmakeBuildFactory( depends_on_projects = None, + enable_runtimes = None, llvm_srcdir = None, obj_dir = None, install_dir = None, @@ -240,6 +253,7 @@ f = getLLVMBuildFactoryAndSourcecodeSteps( depends_on_projects=depends_on_projects, + enable_runtimes=enable_runtimes, llvm_srcdir=llvm_srcdir, obj_dir=obj_dir, install_dir=install_dir, @@ -260,6 +274,7 @@ def getCmakeWithNinjaBuildFactory( depends_on_projects = None, + enable_runtimes = None, targets = None, llvm_srcdir = None, obj_dir = None, @@ -294,6 +309,7 @@ f = getCmakeBuildFactory( depends_on_projects=depends_on_projects, + enable_runtimes=enable_runtimes, llvm_srcdir=llvm_srcdir, obj_dir=obj_dir, install_dir=install_dir, @@ -315,6 +331,7 @@ def getCmakeWithNinjaWithMSVCBuildFactory( depends_on_projects = None, + enable_runtimes = None, targets = None, llvm_srcdir = None, obj_dir = None, @@ -342,6 +359,7 @@ f = getLLVMBuildFactoryAndSourcecodeSteps( depends_on_projects=depends_on_projects, + enable_runtimes=enable_runtimes, llvm_srcdir=llvm_srcdir, obj_dir=obj_dir, install_dir=install_dir, @@ -377,7 +395,9 @@ def getCmakeWithNinjaMultistageBuildFactory( depends_on_projects = None, + enable_runtimes = None, llvm_srcdir = None, + src_to_build_dir = None, obj_dir = None, checks = None, install_dir = None, @@ -426,7 +446,9 @@ f = getLLVMBuildFactoryAndPrepareForSourcecodeSteps( depends_on_projects=depends_on_projects, + enable_runtimes=enable_runtimes, llvm_srcdir=llvm_srcdir, + src_to_build_dir=src_to_build_dir, obj_dir=obj_dir, install_dir=install_dir, env=merged_env, diff --git a/zorg/buildbot/builders/annotated/annotated_builder.py b/zorg/buildbot/builders/annotated/annotated_builder.py --- a/zorg/buildbot/builders/annotated/annotated_builder.py +++ b/zorg/buildbot/builders/annotated/annotated_builder.py @@ -13,6 +13,7 @@ from os.path import join as pjoin VSWHERE_PATH = "C:/Program Files (x86)/Microsoft Visual Studio/Installer/vswhere.exe" +BUILDTOOLS_VSDEVCMD = "C:/BuildTools/Common7/Tools/VsDevCmd.bat" def get_argument_parser(*args, **kwargs): ap = argparse.ArgumentParser(*args, **kwargs) @@ -211,10 +212,13 @@ cxx_compiler, linker, previous_stage_bin): + archiver = 'llvm-ar' + if os.name == 'nt': + archiver = 'llvm-lib' return ( cmake_args + [ '-DCMAKE_AR=%s' % ( - util.cmake_pjoin(previous_stage_bin, 'llvm-ar'),), + util.cmake_pjoin(previous_stage_bin, archiver),), '-DCMAKE_RANLIB=%s' % ( util.cmake_pjoin(previous_stage_bin, 'llvm-ranlib'),), ] + self.cmake_compiler_flags( @@ -323,9 +327,17 @@ def get_vcvars(vs_tools, arch): - """Get the VC tools environment using vswhere.exe from VS 2017 + """Get the VC tools environment using vswhere.exe or buildtools docker + + This is intended to work either when VS is in its standard installation + location, or when the docker instructions have been followed, and we can + find Visual C++ in C:/BuildTools. + + Visual Studio provides a docker image with instructions here: + https://docs.microsoft.com/en-us/visualstudio/install/build-tools-container?view=vs-2019 - This code is following the guidelines from strategy 1 in this blog post: + This vswhere code is following the guidelines from strategy 1 in this blog + post: https://blogs.msdn.microsoft.com/vcblog/2017/03/06/finding-the-visual-c-compiler-tools-in-visual-studio-2017/ It doesn't work when VS is not installed at the default location. @@ -342,13 +354,16 @@ # Use vswhere.exe if it exists. if os.path.exists(VSWHERE_PATH): cmd = [VSWHERE_PATH, "-latest", "-property", "installationPath"] - vs_path = subprocess.check_output(cmd).strip() + vs_path = subprocess.check_output(cmd).decode(sys.stdout.encoding) + vs_path = vs_path.strip() util.report("Running vswhere to find VS: " + repr(cmd)) util.report("vswhere output: " + vs_path) if not os.path.isdir(vs_path): raise ValueError("VS install path does not exist: " + vs_path) vcvars_path = pjoin(vs_path, 'VC', 'Auxiliary', 'Build', 'vcvarsall.bat') + elif os.path.exists(BUILDTOOLS_VSDEVCMD): + vcvars_path = BUILDTOOLS_VSDEVCMD elif vs_tools is None: vs_tools = os.path.expandvars('%VS140COMNTOOLS%') vcvars_path = pjoin(vs_tools, '..', '..', 'VC', 'vcvarsall.bat') @@ -357,7 +372,8 @@ # Windows /dev/null. cmd = util.shquote_cmd([vcvars_path, arch]) + ' > NUL && set' util.report("Running vcvars: " + cmd) - output = subprocess.check_output(cmd, shell=True) + output = \ + subprocess.check_output(cmd, shell=True).decode(sys.stdout.encoding) new_env = {} for line in output.splitlines(): var, val = line.split('=', 1) diff --git a/zorg/buildbot/builders/annotated/buildbot-helper.sh b/zorg/buildbot/builders/annotated/buildbot-helper.sh new file mode 100755 --- /dev/null +++ b/zorg/buildbot/builders/annotated/buildbot-helper.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# Buildbot Helper functions + +# Stop if we've encountered an error. +halt_on_failure() { + echo "@@@HALT_ON_FAILURE@@@" +} + +build_step() { + echo "@@@BUILD_STEP ${@}@@@" +} diff --git a/zorg/buildbot/builders/annotated/hip-build.sh b/zorg/buildbot/builders/annotated/hip-build.sh new file mode 100755 --- /dev/null +++ b/zorg/buildbot/builders/annotated/hip-build.sh @@ -0,0 +1,114 @@ +#!/bin/bash + +# Enable Error tracing +set -o errtrace + +# Print trace for all commands ran before execution +set -x + +# Include the Buildbot helper functions +HERE="$(dirname $0)" +. ${HERE}/buildbot-helper.sh + +# Ensure all commands pass, and not dereferencing unset variables. +set -eu +halt_on_failure + +BUILDBOT_ROOT=/buildbot +LLVM_ROOT="${BUILDBOT_ROOT}/llvm-project" +REVISION=${BUILDBOT_REVISION:-origin/main} +AMDGPU_ARCHS=${AMDGPU_ARCHS:="gfx900;gfx906;gfx908;gfx1030"} + +# Set-up llvm-project +if [ ! -d "${LLVM_ROOT}" ]; then + build_step "Cloning llvm-project repo" + git clone --progress https://github.com/llvm/llvm-project.git ${LLVM_ROOT} +fi + +build_step "Updating llvm-project repo" +git -C "${LLVM_ROOT}" fetch origin +git -C "${LLVM_ROOT}" reset --hard ${REVISION} + +# Set-up llvm-test-suite +TESTSUITE_ROOT="${BUILDBOT_ROOT}/llvm-test-suite" +if [ ! -d "${TESTSUITE_ROOT}" ]; then + build_step "Cloning llvm-test-suite repo" + git clone --progress https://github.com/llvm/llvm-test-suite.git ${TESTSUITE_ROOT} +fi + +build_step "Updating llvm-test-suite repo" +git -C "${TESTSUITE_ROOT}" fetch origin +git -C "${TESTSUITE_ROOT}" reset --hard origin/main + +# Set-up variables +BUILDBOT_SLAVENAME=$(whoami) +BUILD_DIR="${BUILDBOT_ROOT}/${BUILDBOT_SLAVENAME}/${BUILDBOT_BUILDERNAME}" +DESTDIR=${BUILD_DIR}/install +EXTERNAL_DIR=/buildbot/Externals + +build_step "Setting up the buildbot" +echo "BUILDBOT_ROOT=${BUILDBOT_ROOT}" +echo "LLVM_ROOT=${LLVM_ROOT}" +echo "BUILD_DIR=${BUILD_DIR}" +echo "DESTDIR=${DESTDIR}" +echo "EXTERNAL_DIR=${EXTERNAL_DIR}" + +# Start building LLVM, Clang, Lld, clang-tools-extra, compiler-rt +build_step "Configure LLVM Build" +LLVM_BUILD_DIR="${BUILD_DIR}/llvm" +mkdir -p "${LLVM_BUILD_DIR}" +cd "${LLVM_BUILD_DIR}" +cmake -G Ninja \ + -DCMAKE_BUILD_TYPE="Release" \ + -DCMAKE_VERBOSE_MAKEFILE=1 \ + -DLLVM_TARGETS_TO_BUILD="AMDGPU;X86" \ + -DLLVM_ENABLE_PROJECTS="clang;lld;clang-tools-extra;compiler-rt" \ + -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi" \ + -DLIBCXX_ENABLE_SHARED=OFF \ + -DLIBCXX_ENABLE_STATIC=ON \ + -DLIBCXX_INSTALL_LIBRARY=OFF \ + -DLIBCXX_INSTALL_HEADERS=OFF \ + -DLIBCXXABI_ENABLE_SHARED=OFF \ + -DLIBCXXABI_ENABLE_STATIC=ON \ + -DLIBCXXABI_INSTALL_STATIC_LIBRARY=OFF \ + -DCMAKE_INSTALL_PREFIX="${DESTDIR}" \ + -DLLVM_ENABLE_ASSERTIONS=ON \ + -DLLVM_ENABLE_Z3_SOLVER=OFF \ + -DLLVM_ENABLE_ZLIB=ON \ + -DLLVM_LIT_ARGS="-v -vv" \ + ${LLVM_ROOT}/llvm + +build_step "Building LLVM" +ninja runtimes +ninja + +build_step "Install LLVM" +rm -rf "${DESTDIR}" +ninja install-runtimes +ninja install + +# Start building llvm-test-suite's hip tests +build_step "Configuring HIP test-suite" +TEST_BUILD_DIR=${BUILD_DIR}/test-suite-build +rm -rf ${TEST_BUILD_DIR} +mkdir -p ${TEST_BUILD_DIR} +cd ${TEST_BUILD_DIR} +PATH="${LLVM_BUILD_DIR}/bin:$PATH" cmake -G Ninja \ + -DTEST_SUITE_SUBDIRS=External \ + -DTEST_SUITE_EXTERNALS_DIR=${EXTERNAL_DIR}/ \ + -DTEST_SUITE_COLLECT_CODE_SIZE=OFF \ + -DTEST_SUITE_COLLECT_COMPILE_TIME=OFF \ + -DAMDGPU_ARCHS="${AMDGPU_ARCHS}" \ + -DCMAKE_CXX_COMPILER="${LLVM_BUILD_DIR}/bin/clang++" \ + -DCMAKE_C_COMPILER="${LLVM_BUILD_DIR}/bin/clang" \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + ${TESTSUITE_ROOT} + +build_step "Building HIP test-suite" +ninja hip-tests-simple + +build_step "Testing HIP test-suite" +ninja check-hip-simple + +exit 0 + diff --git a/zorg/buildbot/builders/annotated/libc-linux.py b/zorg/buildbot/builders/annotated/libc-linux.py --- a/zorg/buildbot/builders/annotated/libc-linux.py +++ b/zorg/buildbot/builders/annotated/libc-linux.py @@ -9,6 +9,10 @@ from contextlib import contextmanager +def is_fullbuild_builder(builder_name): + return ('fullbuild' in builder_name.split('-')) + + def main(argv): ap = argparse.ArgumentParser() ap.add_argument('--asan', action='store_true', default=False, @@ -18,11 +22,18 @@ args, _ = ap.parse_known_args() source_dir = os.path.join('..', 'llvm-project') + fullbuild = is_fullbuild_builder(os.environ.get('BUILDBOT_BUILDERNAME')) with step('cmake', halt_on_fail=True): projects = ['llvm', 'libc', 'clang', 'clang-tools-extra'] - cmake_args = ['-GNinja'] + # On most systems the default generator is make and the default + # compilers are gcc and g++. We make it explicit here that we want + # clang and ninja which reduces one step of setting environment + # variables when setting up workers. + cmake_args = ['-GNinja', + '-DCMAKE_C_COMPILER=clang', + '-DCMAKE_CXX_COMPILER=clang++'] if args.debug: cmake_args.append('-DCMAKE_BUILD_TYPE=Debug') else: @@ -31,8 +42,18 @@ if args.asan: cmake_args.append('-DLLVM_USE_SANITIZER=Address') + if fullbuild and not args.asan: + projects.append('compiler-rt') + cmake_args.append('-DLLVM_ENABLE_PROJECTS={}'.format(';'.join(projects))) + if fullbuild and not args.asan: + cmake_args.append('-DLLVM_LIBC_INCLUDE_SCUDO=ON') + + if fullbuild: + cmake_args.extend(['-DLLVM_LIBC_FULL_BUILD=ON', + '-DLLVM_LIBC_ENABLE_LINTING=ON']) + run_command(['cmake', os.path.join(source_dir, 'llvm')] + cmake_args) with step('build llvmlibc', halt_on_fail=True): @@ -41,18 +62,20 @@ with step('check-libc'): run_command(['ninja', 'check-libc']) - if not args.asan: - with step('Loader Tests'): + if fullbuild and not args.asan: + with step('libc-loader-tests'): run_command(['ninja', 'libc_loader_tests']) - with step('Integration Tests'): + with step('libc-integration-test'): run_command(['ninja', 'libc-integration-test']) + with step('libc-fuzzer'): + run_command(['ninja', 'libc-fuzzer']) + with step('libc-scudo-integration-test'): + run_command(['ninja', 'libc-scudo-integration-test']) with step('AOR Tests'): aor_dir = os.path.join(source_dir, 'libc', 'AOR_v20.02') # Remove the AOR build dir. util.clean_dir(os.path.join(aor_dir, 'build')) run_command(['make', 'check'], directory=aor_dir) - - if not args.debug: with step('Benchmark Utils Tests'): run_command(['ninja', 'libc-benchmark-util-tests']) diff --git a/zorg/buildbot/builders/annotated/libc-linux.py b/zorg/buildbot/builders/annotated/libc-windows.py copy from zorg/buildbot/builders/annotated/libc-linux.py copy to zorg/buildbot/builders/annotated/libc-windows.py --- a/zorg/buildbot/builders/annotated/libc-linux.py +++ b/zorg/buildbot/builders/annotated/libc-windows.py @@ -1,14 +1,14 @@ -#!/usr/bin/python + +import annotated_builder +import util import argparse import os import subprocess import sys import traceback -import util from contextlib import contextmanager - def main(argv): ap = argparse.ArgumentParser() ap.add_argument('--asan', action='store_true', default=False, @@ -18,11 +18,19 @@ args, _ = ap.parse_known_args() source_dir = os.path.join('..', 'llvm-project') + vc_vars = annotated_builder.get_vcvars(vs_tools=None, arch='amd64') + for (var, val) in vc_vars.items(): + os.environ[var] = val with step('cmake', halt_on_fail=True): - projects = ['llvm', 'libc', 'clang', 'clang-tools-extra'] - - cmake_args = ['-GNinja'] + # On most systems the default generator is make and the default + # compilers are gcc and g++. We make it explicit here that we want + # clang and ninja which reduces one step of setting environment + # variables when setting up workers. + ## adjust compiler location definitions based on VM config + cmake_args = ['-GNinja', + '-DCMAKE_C_COMPILER=clang-cl', + '-DCMAKE_CXX_COMPILER=clang-cl'] if args.debug: cmake_args.append('-DCMAKE_BUILD_TYPE=Debug') else: @@ -31,7 +39,12 @@ if args.asan: cmake_args.append('-DLLVM_USE_SANITIZER=Address') - cmake_args.append('-DLLVM_ENABLE_PROJECTS={}'.format(';'.join(projects))) + cmake_args.append('-DLLVM_ENABLE_PROJECTS=libc') + cmake_args.append('-DLLVM_TARGETS_TO_BUILD=X86') + cmake_args.append('-DLLVM_FORCE_BUILD_RUNTIME=libc') + cmake_args.append('-DLLVM_NATIVE_ARCH=x86_64') + cmake_args.append('-DLLVM_HOST_TRIPLE=x86_64-window-x86-gnu') + cmake_args.append('-DLLVM_LIBC_MPFR_INSTALL_PATH=C:/src/install') run_command(['cmake', os.path.join(source_dir, 'llvm')] + cmake_args) @@ -41,21 +54,6 @@ with step('check-libc'): run_command(['ninja', 'check-libc']) - if not args.asan: - with step('Loader Tests'): - run_command(['ninja', 'libc_loader_tests']) - with step('Integration Tests'): - run_command(['ninja', 'libc-integration-test']) - with step('AOR Tests'): - aor_dir = os.path.join(source_dir, 'libc', 'AOR_v20.02') - # Remove the AOR build dir. - util.clean_dir(os.path.join(aor_dir, 'build')) - run_command(['make', 'check'], directory=aor_dir) - - if not args.debug: - with step('Benchmark Utils Tests'): - run_command(['ninja', 'libc-benchmark-util-tests']) - @contextmanager def step(step_name, halt_on_fail=False): diff --git a/zorg/buildbot/builders/sanitizers/buildbot_android_functions.sh b/zorg/buildbot/builders/sanitizers/buildbot_android_functions.sh --- a/zorg/buildbot/builders/sanitizers/buildbot_android_functions.sh +++ b/zorg/buildbot/builders/sanitizers/buildbot_android_functions.sh @@ -1,9 +1,5 @@ #!/usr/bin/env bash -CLOBBER="android_ndk android-ndk-* platform-tools *.zip shards_* test_android_* tested_arch_*" -STAGE2_CLOBBER="compiler_rt_build_android_* llvm_build_android_*" -STAGE1_CLOBBER="llvm_build64 ${STAGE2_CLOBBER}" - ANDROID_NDK_VERSION=21 ANDROID_API=24 NDK_DIR=android_ndk @@ -17,7 +13,7 @@ [[ -d ${NDK_DIR} ]] && rm -rf ${NDK_DIR} [[ -d ${VERSION} ]] && rm -rf ${VERSION} [[ -f ${FILE_NAME} ]] && rm -f ${FILE_NAME} - wget ${NDK_URL} + wget -e dotbytes=10m ${NDK_URL} unzip ${FILE_NAME} > /dev/null mv ${VERSION} ${NDK_DIR} echo ${NDK_URL} > ${NDK_DIR}/android_ndk_url @@ -27,7 +23,7 @@ echo @@@BUILD_STEP downloading Android Platform Tools@@@ local FILE_NAME=platform-tools-latest-linux.zip [[ -f ${FILE_NAME} ]] && rm -f ${FILE_NAME} - wget https://dl.google.com/android/repository/${FILE_NAME} + wget -e dotbytes=10m https://dl.google.com/android/repository/${FILE_NAME} unzip ${FILE_NAME} > /dev/null fi export PATH=$ROOT/platform-tools/:$PATH @@ -36,16 +32,13 @@ function build_stage2_android() { # Build self-hosted tree with fresh Clang and -Werror. local CMAKE_OPTIONS="${CMAKE_COMMON_OPTIONS} -DLLVM_ENABLE_WERROR=ON ${STAGE1_AS_COMPILER} -DCMAKE_C_FLAGS=-gmlt -DCMAKE_CXX_FLAGS=-gmlt" - CMAKE_OPTIONS="${CMAKE_OPTIONS} -DLLVM_ENABLE_PROJECTS='clang;lld' -DLLVM_USE_LINKER=lld" + CMAKE_OPTIONS="${CMAKE_OPTIONS} -DLLVM_ENABLE_PROJECTS='clang;lld' -DLLVM_USE_LINKER=lld -DCLANG_DEFAULT_RTLIB=libgcc" if ccache -s ; then CMAKE_OPTIONS="${CMAKE_OPTIONS} -DLLVM_CCACHE_BUILD=ON" - rm_dirs llvm_build64 fi echo @@@BUILD_STEP bootstrap clang@@@ - rm_dirs ${STAGE2_CLOBBER} - mkdir -p llvm_build64 if [[ "$(cat llvm_build64/CMAKE_OPTIONS)" != "${CMAKE_OPTIONS}" ]] ; then (cd llvm_build64 && cmake ${CMAKE_OPTIONS} -DLLVM_BUILD_EXTERNAL_COMPILER_RT=ON $LLVM && \ @@ -63,7 +56,7 @@ local ANDROID_LIBRARY_OUTPUT_DIR=$(ls -d $ROOT/llvm_build64/lib/clang/* | tail -1) local ANDROID_EXEC_OUTPUT_DIR=$ROOT/llvm_build64/bin - local ANDROID_FLAGS="--target=${_triple}${ANDROID_API} --sysroot=$ANDROID_TOOLCHAIN/sysroot -gcc-toolchain $ANDROID_TOOLCHAIN -B$ANDROID_TOOLCHAIN" + local ANDROID_FLAGS="--target=${_triple}${ANDROID_API} --sysroot=$ANDROID_TOOLCHAIN/sysroot --gcc-toolchain=$ANDROID_TOOLCHAIN -B$ANDROID_TOOLCHAIN" local ANDROID_CXX_FLAGS="$ANDROID_FLAGS -stdlib=libc++ -I/$ANDROID_TOOLCHAIN/sysroot/usr/include/c++/v1" local CLANG_PATH=$ROOT/llvm_build64/bin/clang @@ -128,14 +121,18 @@ fi } -# If a multiarch device has x86 as the first arch, remove everything else from -# the list. This captures cases like [x86,armeabi-v7a], where the arm part is -# software emulation and incompatible with ASan. +# If a multiarch device's first arch starts with x86, remove archs that don't +# start with x86. Handle cases like: +# - x86,armeabi-v7a +# - x86_64,x86,arm64-v8a,armeabi-v7a,armeabi +# - x86_64,arm64-v8a,x86,armeabi-v7a,armeabi +# The arm part is software emulation and incompatible with ASan. function patch_abilist { # IN OUT local _abilist=$1 local _out=$2 - if [[ "$_abilist" == "x86,"* ]]; then - _abilist="x86" + if [[ "$_abilist" == "x86"* ]]; then + _abilist=$(echo $_abilist | tr , '\n' | grep '^x86') + _abilist=$(echo $_abilist | tr ' ' ,) fi eval $_out="'$_abilist'" } @@ -154,7 +151,7 @@ shift ABILIST=$(${ADB} -s $_serial shell getprop ro.product.cpu.abilist) - patch_abilist $ABILIST ABILIST + patch_abilist "$ABILIST" ABILIST for _arg in "$@"; do local _arch=${_arg%:*} local _abi=${_arg#*:} @@ -166,7 +163,12 @@ echo "$_serial" >> tested_arch_$_arch BUILD_ID=$(${ADB} -s $_serial shell getprop ro.build.id | tr -d '\r') BUILD_FLAVOR=$(${ADB} -s $_serial shell getprop ro.build.flavor | tr -d '\r') - test_arch_on_device "$_arch" "$_serial" "$BUILD_ID" "$BUILD_FLAVOR" + ( + # Test only one arch at a time to avoid simultaneously writes into the + # same compiler-rt build dir. + flock -x $lock_fd + test_arch_on_device "$_arch" "$_serial" "$BUILD_ID" "$BUILD_FLAVOR" + ) {lock_fd}>$ROOT/${_arch}.lock fi done } @@ -181,7 +183,7 @@ } function test_android { - if [[ "${BUILDBOT_SLAVENAME:-}" != "" ]]; then + if [[ -v BUILDBOT_BUILDERNAME ]]; then restart_adb_server fi @@ -200,6 +202,9 @@ done tail_pids "$LOGS" + # Return to avoid exception if we already have error. + [[ $BUILD_RT_ERR == "" ]] || return + for _arg in "$@"; do local _arch=${_arg%:*} if [[ ! -f tested_arch_$_arch ]]; then @@ -227,7 +232,8 @@ LOGS= for ((SHARD=0; SHARD < $NUM_SHARDS; SHARD++)); do LOG=${_log_prefix}_$SHARD - local ENV="$_env GTEST_TOTAL_SHARDS=$NUM_SHARDS GTEST_SHARD_INDEX=$SHARD LD_LIBRARY_PATH=$DEVICE_ROOT" + # 'adb shell ' on Fugu is missing TMPDIR. + local ENV="$_env GTEST_TOTAL_SHARDS=$NUM_SHARDS GTEST_SHARD_INDEX=$SHARD LD_LIBRARY_PATH=$DEVICE_ROOT TMPDIR=$DEVICE_TMPDIR" ( (run_command_on_device "$ENV $DEVICE_ROOT/$_test" || echo @@@STEP_FAILURE@@@) 2>&1 >${_log_prefix}_$SHARD ) & LOGS="$LOGS $LOG,$!" done @@ -250,6 +256,7 @@ local RT_DIR=$($ROOT/llvm_build64/bin/clang -print-resource-dir)/lib/linux local COMPILER_RT_BUILD_DIR=$ROOT/compiler_rt_build_android_$_arch export ADB=adb + export DEVICE_TMPDIR=/data/local/tmp export DEVICE_ROOT=/data/local/tmp/Output export ANDROID_SERIAL=$_serial echo "Serial $_serial" diff --git a/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_asan.sh b/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_asan.sh --- a/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_asan.sh +++ b/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_asan.sh @@ -12,7 +12,7 @@ export PATH="/usr/local/bin:$PATH" LLVM=$ROOT/llvm -CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON" +CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF" clobber @@ -35,3 +35,4 @@ check_stage3_asan +cleanup $STAGE1_DIR diff --git a/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_msan.sh b/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_msan.sh --- a/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_msan.sh +++ b/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_msan.sh @@ -12,7 +12,7 @@ export PATH="/usr/local/bin:$PATH" LLVM=$ROOT/llvm -CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON" +CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF" clobber @@ -33,3 +33,5 @@ build_stage3_msan check_stage3_msan + +cleanup $STAGE1_DIR diff --git a/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_ubsan.sh b/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_ubsan.sh --- a/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_ubsan.sh +++ b/zorg/buildbot/builders/sanitizers/buildbot_bootstrap_ubsan.sh @@ -12,7 +12,7 @@ export PATH="/usr/local/bin:$PATH" LLVM=$ROOT/llvm -CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON" +CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF" clobber @@ -33,3 +33,5 @@ build_stage3_ubsan check_stage3_ubsan + +cleanup $STAGE1_DIR diff --git a/zorg/buildbot/builders/sanitizers/buildbot_cmake.sh b/zorg/buildbot/builders/sanitizers/buildbot_cmake.sh --- a/zorg/buildbot/builders/sanitizers/buildbot_cmake.sh +++ b/zorg/buildbot/builders/sanitizers/buildbot_cmake.sh @@ -13,21 +13,29 @@ export PATH="/usr/local/bin:$PATH" export ANDROID_SDK_HOME=$ROOT/../../.. -# Always clobber bootstrap build trees. -rm -rf compiler_rt_build llvm_build* symbolizer_build* +CMAKE_ARGS="" +for arg in "$@" +do + case $arg in + --CMAKE_ARGS=*) + CMAKE_ARGS="${arg#*=}" + esac +done USE_CCACHE= if ccache -s ; then USE_CCACHE="-DLLVM_CCACHE_BUILD=ON" - rm -rf clang_build fi +# Always clobber bootstrap build trees. +clobber + SUPPORTS_32_BITS=${SUPPORTS_32_BITS:-1} MAKE_JOBS=${MAX_MAKE_JOBS:-$(nproc)} LLVM=$ROOT/llvm ZLIB=$ROOT/zlib -CMAKE_COMMON_OPTIONS="${CMAKE_COMMON_OPTIONS:-} -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_PARALLEL_LINK_JOBS=10" +CMAKE_COMMON_OPTIONS="${CMAKE_COMMON_OPTIONS:-} -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_PARALLEL_LINK_JOBS=10 -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF ${CMAKE_ARGS}" ENABLE_LIBCXX_FLAG= if [ "$PLATFORM" == "Darwin" ]; then CMAKE_COMMON_OPTIONS="${CMAKE_COMMON_OPTIONS} -DPYTHON_EXECUTABLE=/usr/bin/python" @@ -118,24 +126,16 @@ CMAKE_COMMON_OPTIONS="$CMAKE_COMMON_OPTIONS -DLLVM_BUILD_LLVM_DYLIB=ON" fi -CLOBBER="zlib clang_build" -clobber buildbot_update COMPILER_RT=$LLVM/../compiler-rt LIBCXX=$LLVM/../libcxx -echo @@@BUILD_STEP lint@@@ -CHECK_LINT=${COMPILER_RT}/lib/sanitizer_common/scripts/check_lint.sh -${CHECK_LINT} || echo @@@STEP_WARNINGS@@@ - # Use both gcc and just-built Clang/LLD as a host compiler/linker for sanitizer # tests. Assume that self-hosted build tree should compile with -Werror. echo @@@BUILD_STEP build fresh toolchain@@@ -if [ ! -d clang_build ]; then - mkdir clang_build -fi +mkdir -p clang_build (cd clang_build && cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo ${CMAKE_COMMON_OPTIONS} ${USE_CCACHE} $LLVM ) || (rm -rf clang_build ; echo @@@STEP_FAILURE@@@) BOOTSTRAP_BUILD_TARGETS="clang" @@ -171,6 +171,7 @@ check_in_gcc $CHECK_SCUDO scudo check_in_gcc $CHECK_SCUDO_STANDALONE scudo_standalone LDFLAGS=-no-pie check_in_gcc $CHECK_TSAN tsan + LDFLAGS=-no-pie check_in_gcc $CHECK_TSAN tsan-shared check_in_gcc $CHECK_UBSAN ubsan check_in_gcc $CHECK_UBSAN ubsan-minimal fi @@ -182,9 +183,7 @@ BUILD_TYPE=Release echo @@@BUILD_STEP bootstrap clang@@@ -if [ ! -d llvm_build64 ]; then - mkdir llvm_build64 -fi +mkdir -p llvm_build64 (cd llvm_build64 && cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE \ ${CMAKE_CLANG_OPTIONS} -DLLVM_BUILD_EXTERNAL_COMPILER_RT=ON \ ${ENABLE_LIBCXX_FLAG} $LLVM) || echo @@@STEP_FAILURE@@@ @@ -230,6 +229,7 @@ check_64bit $CHECK_MSAN msan # check_64bit $CHECK_SCUDO scudo, No check-scudo target for this config check_64bit $CHECK_TSAN tsan + check_64bit $CHECK_TSAN tsan-shared check_64bit $CHECK_UBSAN ubsan check_64bit $CHECK_UBSAN ubsan-minimal fi @@ -306,6 +306,7 @@ check_ninja $CHECK_SCUDO scudo check_ninja $CHECK_SCUDO_STANDALONE scudo_standalone check_ninja $CHECK_TSAN tsan + check_ninja $CHECK_TSAN tsan-shared check_ninja $CHECK_UBSAN ubsan check_ninja $CHECK_UBSAN ubsan-minimal @@ -333,6 +334,9 @@ check_ninja_with_symbolizer $CHECK_SCUDO scudo check_ninja_with_symbolizer $CHECK_SCUDO_STANDALONE scudo_standalone check_ninja_with_symbolizer $CHECK_TSAN tsan + check_ninja_with_symbolizer $CHECK_TSAN tsan-shared check_ninja_with_symbolizer $CHECK_UBSAN ubsan fi fi + +cleanup diff --git a/zorg/buildbot/builders/sanitizers/buildbot_fast.sh b/zorg/buildbot/builders/sanitizers/buildbot_fast.sh --- a/zorg/buildbot/builders/sanitizers/buildbot_fast.sh +++ b/zorg/buildbot/builders/sanitizers/buildbot_fast.sh @@ -12,7 +12,7 @@ export PATH="/usr/local/bin:$PATH" LLVM=$ROOT/llvm -CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON" +CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF" clobber @@ -43,3 +43,5 @@ build_stage2_ubsan check_stage2_ubsan + +cleanup diff --git a/zorg/buildbot/builders/sanitizers/buildbot_functions.sh b/zorg/buildbot/builders/sanitizers/buildbot_functions.sh --- a/zorg/buildbot/builders/sanitizers/buildbot_functions.sh +++ b/zorg/buildbot/builders/sanitizers/buildbot_functions.sh @@ -21,22 +21,32 @@ echo @@@BUILD_STEP Prepare@@@ BUILDBOT_CLOBBER="${BUILDBOT_CLOBBER:-}" -BUILDBOT_REVISION="${BUILDBOT_REVISION:-origin/master}" +BUILDBOT_REVISION="${BUILDBOT_REVISION:-origin/main}" function rm_dirs { while ! rm -rf $@ ; do sleep 1; done } -function stage1_clobber { - rm_dirs llvm_build2_* llvm_build_* libcxx_build_* ${STAGE1_CLOBBER:-} +function cleanup() { + [[ -v BUILDBOT_BUILDERNAME ]] || return 0 + echo @@@BUILD_STEP cleanup@@@ + rm_dirs llvm_build2_* llvm_build_* libcxx_build_* compiler_rt_build* symbolizer_build* $@ + if ccache -s >/dev/null ; then + rm_dirs llvm_build64 clang_build + fi + ls } function clobber { - if [ "$BUILDBOT_CLOBBER" != "" ]; then + if [[ "$BUILDBOT_CLOBBER" != "" ]]; then echo @@@BUILD_STEP clobber@@@ - rm_dirs svn_checkout llvm llvm-project llvm_build0 ${CLOBBER:-} - stage1_clobber - ! test "$(ls -A .)" || echo @@@STEP_EXCEPTION@@@ + if [[ ! -v BUILDBOT_BUILDERNAME ]]; then + echo "Clobbering is supported only on buildbot only!" + exit 1 + fi + rm_dirs * + else + BUILDBOT_BUILDERNAME=1 cleanup $@ fi } @@ -54,17 +64,18 @@ cd llvm-project git init git remote add origin https://github.com/llvm/llvm-project.git + git config --local advice.detachedHead false ) cd llvm-project - git fetch --depth $DEPTH origin master + git fetch --depth $DEPTH origin main git clean -fd local REV=${BUILDBOT_REVISION} - if [[ "$REV" != "origin/master" ]] ; then + if [[ "$REV" != "origin/main" ]] ; then # "git fetch --depth 1 origin $REV" does not work with 2.11 on bots while true ; do git checkout $REV && break - git rev-list --pretty --max-count=1 origin/master - git rev-list --pretty --max-parents=0 origin/master + git rev-list --pretty --max-count=1 origin/main + git rev-list --pretty --max-parents=0 origin/main echo "DEPTH=$DEPTH is too small" [[ "$DEPTH" -le "1000000" ]] || exit 1 DEPTH=$(( $DEPTH * 10 )) @@ -87,7 +98,6 @@ } function build_stage1_clang_impl { - rm -rf ${STAGE1_DIR} mkdir -p ${STAGE1_DIR} local cmake_stage1_options="${CMAKE_COMMON_OPTIONS}" cmake_stage1_options="${cmake_stage1_options} -DLLVM_ENABLE_PROJECTS='clang;compiler-rt;lld'" @@ -99,12 +109,8 @@ function build_stage1_clang { echo @@@BUILD_STEP stage1 build all@@@ - export STAGE1_DIR=llvm_build0 common_stage1_variables build_stage1_clang_impl - - echo @@@BUILD_STEP Clobber stage1 users - stage1_clobber } function build_stage1_clang_at_revison { @@ -134,10 +140,6 @@ export MSAN_SYMBOLIZER_PATH="${llvm_symbolizer_path}" local llvm_use_sanitizer="Memory" local fsanitize_flag="-fsanitize=memory" - BUILDBOT_MSAN_ORIGINS=${BUILDBOT_MSAN_ORIGINS:-} - if [ "$BUILDBOT_MSAN_ORIGINS" != "" ]; then - llvm_use_sanitizer="MemoryWithOrigins" - fi local build_type="Release" elif [ "$sanitizer_name" == "asan" ]; then export ASAN_SYMBOLIZER_PATH="${llvm_symbolizer_path}" @@ -167,7 +169,9 @@ -DLLVM_USE_SANITIZER=${llvm_use_sanitizer} \ $LLVM && \ ninja cxx cxxabi) || echo $step_result - local sanitizer_ldflags="-lc++abi -Wl,--rpath=${ROOT}/${libcxx_build_dir}/lib -L${ROOT}/${libcxx_build_dir}/lib" + + libcxx_runtime_path=$(dirname $(find ${ROOT}/${libcxx_build_dir} -name libc++.so)) + local sanitizer_ldflags="-lc++abi -Wl,--rpath=${libcxx_runtime_path} -L${libcxx_runtime_path}" local sanitizer_cflags="-nostdinc++ -isystem ${ROOT}/${libcxx_build_dir}/include -isystem ${ROOT}/${libcxx_build_dir}/include/c++/v1" echo @@@BUILD_STEP stage2/$sanitizer_name build@@@ @@ -278,3 +282,4 @@ function check_stage3_ubsan { check_stage3 ubsan @@@STEP_FAILURE@@@ } + diff --git a/zorg/buildbot/builders/sanitizers/buildbot_fuzzer.sh b/zorg/buildbot/builders/sanitizers/buildbot_fuzzer.sh --- a/zorg/buildbot/builders/sanitizers/buildbot_fuzzer.sh +++ b/zorg/buildbot/builders/sanitizers/buildbot_fuzzer.sh @@ -16,11 +16,9 @@ LIBFUZZER=$LLVM/lib/Fuzzer # No assertions. Need to clean up the existing assertion failures first. # Also, the Fuzzer does not provide reproducers on assertion failures yet. -CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=OFF -DLLVM_PARALLEL_LINK_JOBS=8 -DLIBFUZZER_ENABLE_TESTS=ON" +CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=OFF -DLLVM_LIT_ARGS=-v\;--time-tests" -CLOBBER=fuzzer-test-suite -STAGE1_CLOBBER="RUNDIR-* $LIBFUZZER" -clobber +clobber RUNDIR-* # Make sure asan intercepts SIGABRT so that the fuzzer can print the test cases # for assertion failures. @@ -64,3 +62,5 @@ RunFuzzerTest openssl-1.0.2d || echo @@@STEP_FAILURE@@@ RunFuzzerTest proj4-2017-08-14 || echo @@@STEP_FAILURE@@@ #RunFuzzerTest woff2-2016-05-06 || echo @@@STEP_WARNINGS@@@ # Often can't find the bug in the given time. + +cleanup $STAGE1_DIR RUNDIR-* diff --git a/zorg/buildbot/builders/sanitizers/buildbot_qemu.sh b/zorg/buildbot/builders/sanitizers/buildbot_qemu.sh new file mode 100755 --- /dev/null +++ b/zorg/buildbot/builders/sanitizers/buildbot_qemu.sh @@ -0,0 +1,350 @@ +#!/usr/bin/env bash + +set -x +set -e +set -u + +HERE="$(cd $(dirname $0) && pwd)" +. ${HERE}/buildbot_functions.sh + +ROOT=`pwd` +PLATFORM=`uname` +export PATH="/usr/local/bin:$PATH" + +LLVM=$ROOT/llvm +CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=OFF" + +clobber + +buildbot_update + +build_stage1_clang + +set +x # Avoid echoing STEP_FAILURE because of the command trace. +MISSING_QEMU_IMAGE_MESSAGE=$(cat << 'EOF' +===================================================================== +Looks like you're missing the QEMU system images for x86_64 LAM +HWASan testing. These system images aren't automatically generated +as part of the buildbot script because they require root (sorry!). + +If you have the system images already built, you can run +buildbot_qemu.sh with QEMU_IMAGE_DIR=path/to/qemu/images. Otherwise, +you can build the images (only necessary once-per-clobber) by: + 1. WARNING: Read the script at https://github.com/google/sanitizers/blob/master/hwaddress-sanitizer/create_qemu_image.sh + You're running this as ROOT, so make sure you're comfortable with + that. + 2. In your terminal, run: + $ sudo su -c "bash <(wget -qO- https://raw.githubusercontent.com/google/sanitizers/master/hwaddress-sanitizer/create_qemu_image.sh)" root && \ + sudo chown $(whoami) debian.* + +You can also choose to skip the x86_64 HWASan LAM testing by supplying +SKIP_HWASAN_LAM=true in your invocation of this script. +===================================================================== +@@@STEP_EXCEPTION@@@ +EOF +) +set -x + +SKIP_HWASAN_LAM=${SKIP_HWASAN_LAM:-} + +function setup_lam_qemu_image { + # Full system emulation is required for x86_64 testing with LAM, as some + # sanitizers aren't friendly with usermode emulation under x86_64 LAM. + echo @@@BUILD_STEP Check x86_64 LAM Prerequisites@@@ + + # Allow specifying the QEMU image dir, otherwise assume in the local dir. + QEMU_IMAGE_DIR=${QEMU_IMAGE_DIR:=${ROOT}} + + # Ensure the buildbot start script created a QEMU image for us. + ( + ls ${QEMU_IMAGE_DIR}/debian.img + ls ${QEMU_IMAGE_DIR}/debian.id_rsa + ) || + ( + # Make the "missing file" error clearer by not effectively echoing twice. + set +x + echo "$MISSING_QEMU_IMAGE_MESSAGE" + exit 1 + ) +} + +([[ -z "$SKIP_HWASAN_LAM" ]] && setup_lam_qemu_image) || SKIP_HWASAN_LAM=1 + +COMPILER_BIN_DIR=$(readlink -f ${STAGE1_DIR})/bin + +function git_clone_at_revision { + local src_dir_name="${1}" + local git_url="${2}" + local revision="${3}" + local build_dir="${4}" + ( + cd ${ROOT} + [[ -d ${src_dir_name} ]] || git clone ${git_url} ${src_dir_name} || exit 1 + cd ${ROOT}/${src_dir_name} + git remote set-url origin ${git_url} + git fetch origin + + # Short circuit if we already have a build at the correct revision. + [[ "$(git rev-parse HEAD)" == "$(git rev-parse ${revision})" ]] && exit 0 + + # Delete the build dir to ensure a rebuild happens at the new revision. + rm -rf ${build_dir} + + git reset --hard ${revision} + git submodule update --init --recursive + ) +} + +function build_qemu { + local qemu_name="${1}" + local qemu_url="${2}" + local qemu_revision="${3}" + + local build_dir="${ROOT}/${qemu_name}_build" + + echo "@@@BUILD_STEP build ${qemu_name}@@@" + ( + git_clone_at_revision ${qemu_name} ${qemu_url} ${qemu_revision} \ + ${build_dir} || exit 1 + + ${build_dir}/qemu-x86_64 --version && + ${build_dir}/qemu-system-x86_64 --version && + ${build_dir}/qemu-img --version && + exit 0 + + rm -rf ${build_dir} && + mkdir -p ${build_dir} && + cd ${build_dir} && + ${ROOT}/${qemu_name}/configure --enable-linux-user && + ninja && + ${build_dir}/qemu-x86_64 --version && + ${build_dir}/qemu-system-x86_64 --version && + ${build_dir}/qemu-img --version + ) || ( + echo "@@@STEP_EXCEPTION@@@" + exit 2 + ) +} + +function build_lam_linux { + local build_dir="${ROOT}/lam_linux_build" + + echo "@@@BUILD_STEP build lam linux@@@" + ( + git_clone_at_revision lam_linux https://github.com/morehouse/linux.git \ + origin/lam ${build_dir} || exit 1 + + ls ${build_dir}/arch/x86_64/boot/bzImage && exit 0 + + rm -rf ${build_dir} && + mkdir -p ${build_dir} && + cd ${ROOT}/lam_linux && + make O=${build_dir} LD=ld.bfd defconfig && + make O=${build_dir} LD=ld.bfd -j $(nproc) && + ls ${build_dir}/arch/x86_64/boot/bzImage + ) || ( + echo "@@@STEP_EXCEPTION@@@" + exit 2 + ) +} + +build_qemu qemu https://github.com/vitalybuka/qemu.git origin/sanitizer_bot +[[ -z "$SKIP_HWASAN_LAM" ]] && ( + build_qemu lam_qemu https://github.com/morehouse/qemu.git origin/lam + build_lam_linux +) + +SCUDO_BUILDS= + +function configure_scudo_compiler_rt { + local arch=$1 + local target="${arch}-linux-gnu${2:-}" + + local name="" + if [[ "$DBG" == "ON" ]] ; then + name=debug_ + fi + name+="${arch}" + + local qemu_cmd="" + if [[ "${QEMU:-}" != "0" ]] ; then + name+="_qemu" + local qemu_arch=${arch} + [[ "${arch}" == "powerpc64le" ]] && qemu_arch="ppc64le" + qemu_cmd="$ROOT/qemu_build/qemu-${qemu_arch} -L /usr/${target}" + if [[ ! -z "${QEMU_CPU:-}" ]] ; then + qemu_cmd+=" -cpu ${QEMU_CPU}" + name+="_${QEMU_CPU}" + fi + fi + + local linker_flags= + [[ "${arch}" =~ "mips*" ]] && linker_flags="-latomic -Wl,-z,notext -Wno-unused-command-line-argument" + + local out_dir=llvm_build2_${name} + rm -rf ${out_dir} + mkdir -p ${out_dir} + + SCUDO_BUILDS+=" ${name}" + + ( + cd ${out_dir} + + ( + cmake \ + ${CMAKE_COMMON_OPTIONS} \ + -DCOMPILER_RT_DEBUG=$DBG \ + -DLLVM_CONFIG_PATH=${COMPILER_BIN_DIR}/llvm-config \ + -DCMAKE_C_COMPILER=${COMPILER_BIN_DIR}/clang \ + -DCMAKE_CXX_COMPILER=${COMPILER_BIN_DIR}/clang++ \ + -DCOMPILER_RT_HAS_LLD=ON \ + -DCOMPILER_RT_TEST_USE_LLD=ON \ + -DCMAKE_INSTALL_PREFIX=$(${COMPILER_BIN_DIR}/clang -print-resource-dir) \ + -DLLVM_LIT_ARGS="-v --time-tests" \ + -DCOMPILER_RT_BUILD_BUILTINS=OFF \ + -DCOMPILER_RT_DEFAULT_TARGET_ONLY=ON \ + -DCMAKE_CROSSCOMPILING=True \ + -DCOMPILER_RT_INCLUDE_TESTS=ON \ + -DCOMPILER_RT_BUILD_LIBFUZZER=OFF \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DCMAKE_CXX_FLAGS=-fPIC \ + -DCMAKE_C_FLAGS=-fPIC \ + -DCMAKE_SHARED_LINKER_FLAGS="-fuse-ld=lld ${linker_flags}" \ + -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld ${linker_flags}" \ + -DCOMPILER_RT_TEST_COMPILER_CFLAGS="--target=${target} ${linker_flags}" \ + -DCMAKE_C_COMPILER_TARGET=${target} \ + -DCMAKE_CXX_COMPILER_TARGET=${target} \ + -DCOMPILER_RT_EMULATOR="${qemu_cmd:-}" \ + $LLVM/../compiler-rt + ) >& configure.log + ) & +} + +readonly LLVM_SYMBOLIZER_DIR="${ROOT}/llvm_build2_x86_64_symbolizer" + +function configure_llvm_symbolizer { + rm -rf ${LLVM_SYMBOLIZER_DIR} + mkdir -p ${LLVM_SYMBOLIZER_DIR} + + ( + cd ${LLVM_SYMBOLIZER_DIR} + + ( + cmake \ + ${CMAKE_COMMON_OPTIONS} \ + -DCMAKE_C_COMPILER=${COMPILER_BIN_DIR}/clang \ + -DCMAKE_CXX_COMPILER=${COMPILER_BIN_DIR}/clang++ \ + -DLLVM_BUILD_RUNTIME=OFF \ + -DLLVM_STATIC_LINK_CXX_STDLIB=ON \ + $LLVM + ) >& configure.log + ) & +} + +function configure_hwasan_lam { + git_clone_at_revision sanitizers https://github.com/google/sanitizers.git \ + origin/master "" + local script="${ROOT}/sanitizers/hwaddress-sanitizer/run_in_qemu_with_lam.sh" + ls ${script} + + local out_dir=llvm_build2_x86_64_lam_qemu + rm -rf ${out_dir} + mkdir -p ${out_dir} + + ( + cd ${out_dir} + + ( + cmake \ + ${CMAKE_COMMON_OPTIONS} \ + -DLLVM_ENABLE_PROJECTS="clang;compiler-rt;lld;libcxx;libcxxabi" \ + -DCMAKE_C_COMPILER=${COMPILER_BIN_DIR}/clang \ + -DCMAKE_CXX_COMPILER=${COMPILER_BIN_DIR}/clang++ \ + -DLLVM_ENABLE_LLD=ON \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DLLVM_LIT_ARGS="-v --time-tests" \ + -DCOMPILER_RT_EMULATOR="env ROOT=${ROOT} QEMU_IMAGE_DIR=${QEMU_IMAGE_DIR} ${script}" \ + $LLVM + ) >& configure.log + ) & +} + +function run_scudo_tests { + local name="${1}" + local out_dir=llvm_build2_${name} + + echo "@@@BUILD_STEP scudo $name@@@" + + ( + cd ${out_dir} + + cat configure.log + + # Copy into clang resource dir to make -fsanitize= work in lit tests. + ninja install-scudo_standalone + + ninja check-scudo_standalone || exit 3 + ) || echo "@@@STEP_FAILURE@@@" +} + +function build_llvm_symbolizer { + echo "@@@BUILD_STEP llvm-symbolizer x86_64@@@" + + ( + cd ${LLVM_SYMBOLIZER_DIR} + + cat configure.log + + ninja llvm-symbolizer || exit 3 + ) || echo "@@@STEP_FAILURE@@@" +} + +function run_hwasan_lam_tests { + local name="x86_64_lam_qemu" + local out_dir=llvm_build2_${name} + + echo "@@@BUILD_STEP hwasan ${name}@@@" + + ( + cd ${out_dir} + + cat configure.log + + # LLD must be built first since HWASan tests use -fuse-ld=lld and the + # buildbots don't have LLD preinstalled. + ninja lld || exit 3 + + ninja check-hwasan-lam || exit 3 + ) || echo "@@@STEP_FAILURE@@@" +} + +echo "@@@BUILD_STEP configure@@@" + +for DBG in OFF ON ; do + QEMU=0 configure_scudo_compiler_rt x86_64 + configure_scudo_compiler_rt x86_64 + configure_scudo_compiler_rt arm eabihf + configure_scudo_compiler_rt aarch64 + QEMU_CPU="cortex-a72" configure_scudo_compiler_rt aarch64 + #configure_scudo_compiler_rt mips + #configure_scudo_compiler_rt mipsel + configure_scudo_compiler_rt mips64 abi64 + configure_scudo_compiler_rt mips64el abi64 + configure_scudo_compiler_rt powerpc64le +done +configure_llvm_symbolizer +[[ -z "$SKIP_HWASAN_LAM" ]] && configure_hwasan_lam + +wait + +for B in $SCUDO_BUILDS ; do + run_scudo_tests $B +done + +[[ -z "$SKIP_HWASAN_LAM" ]] && ( + # Symbolizer only required for HWASan LAM tests. + build_llvm_symbolizer + run_hwasan_lam_tests +) + +cleanup $STAGE1_DIR diff --git a/zorg/buildbot/builders/sanitizers/buildbot_selector.py b/zorg/buildbot/builders/sanitizers/buildbot_selector.py --- a/zorg/buildbot/builders/sanitizers/buildbot_selector.py +++ b/zorg/buildbot/builders/sanitizers/buildbot_selector.py @@ -5,7 +5,8 @@ import sys THIS_DIR=os.path.dirname(sys.argv[0]) - +extra_args = sys.argv[1:] +BOT_DIR = '/b' def bash(path): return 'bash ' + os.path.join(THIS_DIR, path) @@ -22,29 +23,31 @@ 'sanitizer-x86_64-linux-fuzzer': bash('buildbot_fuzzer.sh'), 'sanitizer-x86_64-linux-android': bash('buildbot_android.sh'), 'sanitizer-x86_64-linux-bootstrap': bash('buildbot_bootstrap_asan.sh'), + 'sanitizer-x86_64-linux-bootstrap-asan': bash('buildbot_bootstrap_asan.sh'), 'sanitizer-x86_64-linux-bootstrap-msan': bash('buildbot_bootstrap_msan.sh'), 'sanitizer-x86_64-linux-bootstrap-ubsan': bash('buildbot_bootstrap_ubsan.sh'), - 'sanitizer-x86_64-linux-gn': 'true', + 'sanitizer-x86_64-linux-qemu': bash('buildbot_qemu.sh'), } BOT_ADDITIONAL_ENV = { 'sanitizer-ppc64le-linux': { 'HAVE_NINJA': '1', 'CHECK_LIBCXX': '0', 'CHECK_LLD': '0' }, 'sanitizer-ppc64be-linux': { 'HAVE_NINJA': '1', 'CHECK_LIBCXX': '0', 'CHECK_LLD': '0' }, 'sanitizer-x86_64-linux': { 'HAVE_NINJA' : '1' }, - 'sanitizer-x86_64-linux-bootstrap': {}, 'sanitizer-x86_64-linux-fast': {}, 'sanitizer-x86_64-linux-autoconf': {}, 'sanitizer-x86_64-linux-fuzzer': {}, 'sanitizer-x86_64-linux-android': {}, + 'sanitizer-x86_64-linux-bootstrap': {}, + 'sanitizer-x86_64-linux-bootstrap-asan': {}, 'sanitizer-x86_64-linux-bootstrap-msan': {}, 'sanitizer-x86_64-linux-bootstrap-ubsan': {}, - 'sanitizer-x86_64-linux-gn': {}, + 'sanitizer-x86_64-linux-qemu': { 'QEMU_IMAGE_DIR': BOT_DIR + '/qemu_image' }, } def Main(): builder = os.environ.get('BUILDBOT_BUILDERNAME') print "builder name: %s" % (builder) - cmd = BOT_ASSIGNMENT.get(builder) + cmd = BOT_ASSIGNMENT.get(builder) + ' ' + ' '.join(extra_args) if not cmd: sys.stderr.write('ERROR - unset/invalid builder name\n') sys.exit(1) @@ -53,6 +56,7 @@ sys.stdout.flush() bot_env = os.environ + bot_env['BOT_DIR'] = BOT_DIR add_env = BOT_ADDITIONAL_ENV.get(builder) for var in add_env: bot_env[var] = add_env[var] diff --git a/zorg/buildbot/builders/sanitizers/buildbot_standard.sh b/zorg/buildbot/builders/sanitizers/buildbot_standard.sh --- a/zorg/buildbot/builders/sanitizers/buildbot_standard.sh +++ b/zorg/buildbot/builders/sanitizers/buildbot_standard.sh @@ -11,8 +11,8 @@ TSAN_FULL_DEBUG_BUILD_DIR=tsan_full_debug_build TSAN_RELEASE_BUILD_DIR=tsan_release_build -rm -rf $TSAN_DEBUG_BUILD_DIR $TSAN_FULL_DEBUG_BUILD_DIR $TSAN_RELEASE_BUILD_DIR -clobber +CLEANUP="$TSAN_DEBUG_BUILD_DIR $TSAN_FULL_DEBUG_BUILD_DIR $TSAN_RELEASE_BUILD_DIR" +clobber $CLEANUP ROOT=`pwd` PLATFORM=`uname` @@ -59,3 +59,5 @@ echo "int main() {return 0;}" | $TSAN_RELEASE_BUILD_DIR/bin/clang -x c++ - -fsanitize=thread -O2 -o ${BIN} COMPILER_RT=$LLVM/../compiler-rt $COMPILER_RT/lib/tsan/check_analyze.sh ${BIN} || echo @@@STEP_FAILURE@@@ + +cleanup $CLEANUP diff --git a/zorg/buildbot/changes/llvmgitpoller.py b/zorg/buildbot/changes/llvmgitpoller.py --- a/zorg/buildbot/changes/llvmgitpoller.py +++ b/zorg/buildbot/changes/llvmgitpoller.py @@ -11,31 +11,41 @@ class LLVMPoller(changes.GitPoller): """ - Poll LLVM repository for changes and submit them to the change master. + Poll LLVM repository for changes and submit them for builds scheduling. Following Multiple LLVM Projects. This source will poll a remote LLVM git _monorepo_ for changes and submit - them to the change master.""" + them for builds scheduling.""" _repourl = "https://github.com/llvm/llvm-project" - _branch = "master" - compare_attrs = ["repourl", "branch", "workdir", + compare_attrs = ["repourl", "branches", "workdir", "pollInterval", "gitbin", "usetimestamps", "category", "project", "projects"] + def _check_branches(branch): + log.msg("LLVMPoller: _check_branches: branch={}", branch) + + if branch == "refs/heads/main": + # Always listen for changes in the main branch. + return True + else: + # We are also interested in the release branches. + # Some builders will be building changes from there as well. + return re.search(r"refs\/heads\/release\/\d\d+.*", branch) + def __init__(self, - repourl=_repourl, branch=_branch, + repourl=_repourl, branches=_check_branches, **kwargs): self.cleanRe = re.compile(r"Require(?:s?)\s*.*\s*clean build", re.IGNORECASE + re.MULTILINE) self.cleanCfg = re.compile(r"(CMakeLists\.txt$|\.cmake$|\.cmake\.in$)") - # TODO: Add support for an optional list of projects. - # For now we always watch all the projects. + # Note: We always watch all the projects, then schedulers decide + # to build or not to build. - super().__init__(repourl=repourl, branch=branch, **kwargs) + super().__init__(repourl=repourl, branches=branches, **kwargs) def _transform_path(self, fileList): """ @@ -111,7 +121,7 @@ newRev, branch)) else: # This branch is known, but it now points to a different - # commit than last time we saw it, rebuild. + # commit than the last time we saw it, rebuild. log.msg('LLVMPoller: rebuilding {} for updated branch "{}"'.format( newRev, branch)) @@ -163,8 +173,9 @@ properties['clean_obj'] = (True, "change") log.msg("LLVMPoller: creating a change rev=%s" % rev) - log.msg(" >>> revision=%s, timestamp=%s, author=%s, committer=%s, project=%s, files=%s, comments=\"%s\", properties=%s" % \ - (bytes2unicode(rev, encoding=self.encoding), datetime.fromtimestamp(timestamp), author, committer, + log.msg(" >>> branch=%s, revision=%s, timestamp=%s, author=%s, committer=%s, project=%s, files=%s, comments=\"%s\", properties=%s" % \ + (bytes2unicode(self._removeHeads(branch)), + bytes2unicode(rev, encoding=self.encoding), datetime.fromtimestamp(timestamp), author, committer, projects, files, comments, properties)) yield self.master.data.updates.addChange( diff --git a/zorg/buildbot/commands/AnnotatedCommand.py b/zorg/buildbot/commands/AnnotatedCommand.py --- a/zorg/buildbot/commands/AnnotatedCommand.py +++ b/zorg/buildbot/commands/AnnotatedCommand.py @@ -321,8 +321,6 @@ 'BUILDBOT_GOT_REVISION': util.Interpolate('%(prop:got_revision:-None)s'), 'BUILDBOT_REVISION': util.Interpolate('%(prop:revision:-None)s'), 'BUILDBOT_SCHEDULER': util.Interpolate('%(prop:scheduler:-None)s'), - 'BUILDBOT_SLAVENAME': util.Interpolate('%(prop:slavename:-None)s'), - 'BUILDBOT_MSAN_ORIGINS': util.Interpolate('%(prop:msan_origins:-)s'), } # Apply the passed in environment on top. old_env = kwargs.get('env') or {} diff --git a/zorg/buildbot/process/factory.py b/zorg/buildbot/process/factory.py --- a/zorg/buildbot/process/factory.py +++ b/zorg/buildbot/process/factory.py @@ -5,22 +5,70 @@ from buildbot.process.factory import BuildFactory from buildbot.plugins import util, steps +_all_runtimes = frozenset([ + "compiler-rt", + "libc", + "libcxx", + "libcxxabi", + "libunwind", + "openmp", +]) + class LLVMBuildFactory(BuildFactory): """ TODO: Document + + depends_on_projects is a list of LLVM projects a produced builder + depends on. If None, it gets discovered depending on other params. + + enable_runtimes is a list of enabled runtimes. If None, + it gets discovered based on the depends_on_projects list. """ def __init__(self, steps=None, depends_on_projects=None, **kwargs): # Cannot use "super" here as BuildFactory is an old style class. BuildFactory.__init__(self, steps) + # Handle the dependencies. if depends_on_projects is None: - self.depends_on_projects = frozenset(['llvm']) + # llvm project is always included. + self.depends_on_projects = set(['llvm']) else: self.depends_on_projects = frozenset(depends_on_projects) + enable_runtimes = kwargs.pop('enable_runtimes', None) + # If specified, we either got a givem list of + # enabled runtimes, or "auto", or "all". + + if enable_runtimes is None: + # For the backward compatibility, we do not use + # enable_runtimes unless it is requested explicitly. + self.enable_runtimes = frozenset([]) + elif enable_runtimes == "auto": + # Let's build the list of runtimes based on the given + # depends_on_projects list. + self.enable_runtimes = \ + self.depends_on_projects.intersection(_all_runtimes) + else: + if enable_runtimes == "all": + # Let's replace the "all" placeholder by + # the actual list of all runtimes. + self.enable_runtimes = frozenset(_all_runtimes) + else: + # Let's just use the given list, no need to discover. + self.enable_runtimes = frozenset(enable_runtimes) + + # Update the list of dependencies. + if depends_on_projects is None: + self.depends_on_projects.update(self.enable_runtimes) + + # Build the list of projects to enable. + self.enable_projects = \ + self.depends_on_projects.difference(self.enable_runtimes) + # Directories. - self.llvm_srcdir = kwargs.pop('llvm_srcdir', None) + self.monorepo_dir = kwargs.pop('llvm_srcdir', None) + self.src_to_build_dir = kwargs.pop('src_to_build_dir', None) self.obj_dir = kwargs.pop('obj_dir', None) self.install_dir = kwargs.pop('install_dir', None) @@ -29,9 +77,10 @@ for k,v in kwargs.items(): setattr(self, k, v) - self.monorepo_dir = self.llvm_srcdir or "llvm-project" + self.monorepo_dir = self.monorepo_dir or "llvm-project" + self.src_to_build_dir = self.src_to_build_dir or 'llvm' self.llvm_srcdir = \ - "%(monorepo_dir)s/llvm" % {'monorepo_dir' : self.monorepo_dir} + "{}/{}".format(self.monorepo_dir, self.src_to_build_dir) self.obj_dir = \ self.obj_dir or "build" @@ -71,11 +120,14 @@ def addGetSourcecodeSteps(self, **kwargs): # Checkout the monorepo. + # Documentation: http://docs.buildbot.net/current/manual/configuration/buildsteps.html#git self.addStep(steps.Git( name='Checkout the source code', repourl=self.repourl_prefix + "llvm-project.git", progress=True, workdir=util.WithProperties(self.monorepo_dir), + retryFetch=True, + clobberOnFailure=True, **kwargs)) @@ -104,4 +156,6 @@ repourl=_repourl, progress=True, workdir=util.WithProperties(src_dir), + retryFetch=True, + clobberOnFailure=True, **kwargs)) diff --git a/zorg/buildbot/util/InformativeMailNotifier.py b/zorg/buildbot/util/InformativeMailNotifier.py --- a/zorg/buildbot/util/InformativeMailNotifier.py +++ b/zorg/buildbot/util/InformativeMailNotifier.py @@ -1,38 +1,104 @@ -# TODO: Add support for clang crash dumps. -# TODO: Better handle unit/regression tests failures - -# TODO: For debug purposes. Remove this later. -from twisted.python import log +import traceback from buildbot.plugins import reporters +from buildbot.process.results import CANCELLED +from buildbot.process.results import EXCEPTION +from buildbot.process.results import FAILURE +from buildbot.process.results import RETRY +from buildbot.process.results import SKIPPED +from buildbot.process.results import SUCCESS +from buildbot.process.results import WARNINGS +from buildbot.process.results import statusToString + +def get_log_details(build): + text = "" + try: + for step in build['steps']: + results = step['results'] + if results == SUCCESS or results == SKIPPED: + continue + + text += "Step {} ({}) {}: {}\n".format(step['number'], step['name'], statusToString(results), step['state_string']) + + logs = step['logs'] + if logs: + log_index = -1 + log_type = 0 + for i, _log in enumerate(logs): + if _log['name'].startswith("FAIL: "): # Use only first logchunk FAIL: + log_type = 3 + log_index = i + elif log_type < 2 and _log['name'].startswith("warnings "): + log_type = 2 + log_index = i + elif log_type < 1 and _log['type'] == "s": # stdio + log_type = 1 + log_index = i + + if log_index < 0: + continue + + log_text = logs[log_index]['content']['content'] + if log_type == 1: + # Parse stdio + lines = log_text.splitlines() + for line in lines[:]: + if line.startswith("h"): + lines.remove(line) + for j, line in enumerate(lines): + if line.startswith("o") or line.startswith("e"): + lines[j] = line[1:] + for j, line in enumerate(lines): + if line.find("FAIL:") != -1 or line.find("FAILED") != -1: + if j > 10: + del lines[:j-10] # Start 10 lines before FAIL + lines = ["..."] + lines + del lines[50:] # Keep up to 50 lines around FAIL + break + if len(lines) > 50: + del lines[:len(lines)-50] # Otherwise keep last 50 lines + lines = ["..."] + lines + + log_text = "\n".join(lines) + + elif logs[log_index]['num_lines'] > 50: + # Keep first 50 lines + lines = log_text.splitlines() + del lines[50:] + log_text = "\n".join(lines + ["..."]) + + text += log_text + "\n" -def _get_logs_and_tracebacks_from_build(build): - # TODO: Implement interesting parts of the logs and tracebacks extraction. - return dict() + except Exception as err: + print("Exception in LLVMMessageFormatter.get_log_details(): {}\n{}".format(err, traceback.format_exc())) + # TODO: We should send something useful in this case. + + return dict(details=text) + +# TODO: Add build reason if we have that valid and available +#Build Reason: {{ build['properties'].get('reason', [""])[0] }} MAIL_TEMPLATE = """\ The Buildbot has detected a {{ status_detected }} on builder {{ buildername }} while building {{ projects }}. + Full details are available at: {{ build_url }} -Buildbot URL: {{ buildbot_url }} + Worker for this Build: {{ workername }} -Build Reason: {{ build['properties'].get('reason', [""])[0] }} -Blamelist: {{ ", ".join(blamelist) }} +Blamelist: + {{ ",\n ".join(blamelist) }} + {{ summary }} + +{{ details }} Sincerely, LLVM Buildbot """ class LLVMMessageFormatter(reporters.MessageFormatter): def buildAdditionalContext(self, master, ctx): - #log.msg(">>> LLVMMessageFormatter.buildAdditionalContext got ctx={}".format(ctx)) ctx.update(self.ctx) - - build = ctx["build"] - build_interesting_data = _get_logs_and_tracebacks_from_build(build) - #log.msg(">>> LLVMMessageFormatter.buildAdditionalContext build_interesting_data={}",format(build_interesting_data)) - ctx["build"].update(build_interesting_data) - + ctx.update(get_log_details(ctx["build"])) LLVMInformativeMailNotifier = LLVMMessageFormatter( template=MAIL_TEMPLATE, diff --git a/zorg/jenkins/build.py b/zorg/jenkins/build.py --- a/zorg/jenkins/build.py +++ b/zorg/jenkins/build.py @@ -615,9 +615,8 @@ footer() header("Run Tests") - run_cmd(conf.lldbbuilddir(), [NINJA, '-v', 'check-debuginfo']) run_cmd(conf.lldbbuilddir(), ['/usr/bin/env', 'TERM=vt100', NINJA, '-v', - 'check-lldb']) + 'check-debuginfo', 'check-lldb', '-k2']) footer() diff --git a/zorg/jenkins/common.groovy b/zorg/jenkins/common.groovy --- a/zorg/jenkins/common.groovy +++ b/zorg/jenkins/common.groovy @@ -81,7 +81,7 @@ try { stage('main') { dir('config') { - git url: 'https://github.com/llvm/llvm-zorg.git', branch: 'master', poll: false + git url: 'https://github.com/llvm/llvm-zorg.git', branch: 'main', poll: false } body() } @@ -102,6 +102,7 @@ def benchmark_pipeline(label, body) { properties([ + disableResume(), parameters([ string(name: 'ARTIFACT'), string(name: 'GIT_DISTANCE'), @@ -119,10 +120,10 @@ def testsuite_pipeline(label, body) { benchmark_pipeline(label) { dir('lnt') { - git url: 'https://github.com/llvm/llvm-lnt.git', branch: 'master', poll: false + git url: 'https://github.com/llvm/llvm-lnt.git', branch: 'main', poll: false } dir('test-suite') { - git url: 'https://github.com/llvm/llvm-test-suite.git', branch: 'master', poll: false + git url: 'https://github.com/llvm/llvm-test-suite.git', branch: 'main', poll: false } body() } diff --git a/zorg/jenkins/inspect_log.py b/zorg/jenkins/inspect_log.py --- a/zorg/jenkins/inspect_log.py +++ b/zorg/jenkins/inspect_log.py @@ -26,7 +26,7 @@ '''Matching engine. Keeps data structures to match a single line against a fixed list of patterns.''' def __init__(self, patterns): - # Create a combined master regex combining all patterns. + # Create a combined main regex combining all patterns. combined = '' merge = '' for pattern in patterns: diff --git a/zorg/jenkins/jobs/jobs/clang-stage1-RA b/zorg/jenkins/jobs/jobs/clang-stage1-RA new file mode 100644 --- /dev/null +++ b/zorg/jenkins/jobs/jobs/clang-stage1-RA @@ -0,0 +1,109 @@ +pipeline { + agent { label 'green-dragon-20' } + + parameters { + string(name: 'GIT_REVISION', defaultValue: '*/main', description: 'Git revision to build') + } + + stages { + stage('Checkout') { + steps { + dir('llvm-project') { + checkout([$class: 'GitSCM', branches: [[name: params.GIT_REVISION]], userRemoteConfigs: [[url: 'https://github.com/llvm/llvm-project.git']]]) + } + dir('llvm-zorg') { + checkout([$class: 'GitSCM', branches: [[name: '*/main']], userRemoteConfigs: [[url: 'https://github.com/llvm/llvm-zorg.git']]]) + } + } + } + stage('Build') { + steps { + timeout(90) { + sh ''' + set -u + rm -rf build.properties + + cd llvm-project + git tag -a -m "First Commit" first_commit 97724f18c79c7cc81ced24239eb5e883bf1398ef || true + + git_desc=$(git describe --match "first_commit") + export GIT_DISTANCE=$(echo ${git_desc} | cut -f 2 -d "-") + + sha=$(echo ${git_desc} | cut -f 3 -d "-") + export GIT_SHA=${sha:1} + + # Also save the LLVM_REV until LNT server is taught about GIT + export LLVM_REV=$(git show -q | grep "llvm-svn:" | cut -f2 -d":" | tr -d " ") + + cd - + + echo "GIT_DISTANCE=$GIT_DISTANCE" > build.properties + echo "GIT_SHA=$GIT_SHA" >> build.properties + echo "ARTIFACT=$JOB_NAME/clang-d$GIT_DISTANCE-g$GIT_SHA-t$BUILD_ID-b$BUILD_NUMBER.tar.gz" >> build.properties + + export PATH=$PATH:/usr/bin:/usr/local/bin + + # Set a MacOS minimum deployment target + export MACOSX_DEPLOYMENT_TARGET=10.9 + + rm -rf clang-build clang-install *.tar.gz + python llvm-zorg/zorg/jenkins/monorepo_build.py cmake build \ + --assertions --cmake-type=RelWithDebInfo \ + --sccache --projects="clang;clang-tools-extra;compiler-rt;libcxx" \ + --cmake-flag='-DLIBCXX_ENABLE_SHARED=OFF'\ + --cmake-flag='-DLIBCXX_ENABLE_STATIC=OFF' \ + --cmake-flag='-DLIBCXX_INCLUDE_TESTS=OFF' \ + --cmake-flag='-DLIBCXX_ENABLE_EXPERIMENTAL_LIBRARY=OFF' + ''' + } + } + } + stage('Test') { + steps { + timeout(90) { + sh ''' + set -u + export PATH=$PATH:/usr/bin:/usr/local/bin + python llvm-zorg/zorg/jenkins/monorepo_build.py cmake testlong + ''' + } + junit 'clang-build/**/testresults.xunit.xml' + } + } + } + post { + always { + scanForIssues tool: clang() + } + regression { + emailext subject: '$DEFAULT_SUBJECT', + presendScript: '$DEFAULT_PRESEND_SCRIPT', + postsendScript: '$DEFAULT_POSTSEND_SCRIPT', + recipientProviders: [ + [$class: 'CulpritsRecipientProvider'], + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'], + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS', + body:'$DEFAULT_CONTENT' + } + success { + build job: 'clang-stage2-Rthinlto_relay', wait: false + build job: 'clang-stage2-cmake-RgSan_relay', wait: false + //build job: 'phase2_modules_relay', wait: false + build job: 'relay-lnt-test-suite', wait: false + build job: 'relay-lnt-ctmark', wait: false + build job: 'relay-test-suite-verify-machineinstrs', wait: false + } + unstable { + build job: 'clang-stage2-Rthinlto_relay', wait: false + build job: 'clang-stage2-cmake-RgSan_relay', wait: false + //build job: 'phase2_modules_relay', wait: false + build job: 'relay-lnt-test-suite', wait: false + build job: 'relay-lnt-ctmark', wait: false + build job: 'relay-test-suite-verify-machineinstrs', wait: false + } + } +} + diff --git a/zorg/jenkins/jobs/jobs/clang-stage1-cmake-RA-incremental b/zorg/jenkins/jobs/jobs/clang-stage1-cmake-RA-incremental new file mode 100644 --- /dev/null +++ b/zorg/jenkins/jobs/jobs/clang-stage1-cmake-RA-incremental @@ -0,0 +1,90 @@ +pipeline { + agent { label 'green-dragon-07' } + + parameters { + string(name: 'GIT_REVISION', defaultValue: '*/main', description: 'Git revision to build') + } + + stages { + stage('Checkout') { + steps { + dir('llvm-project') { + checkout([$class: 'GitSCM', branches: [[name: params.GIT_REVISION]], userRemoteConfigs: [[url: 'https://github.com/llvm/llvm-project.git']]]) + } + dir('llvm-zorg') { + checkout([$class: 'GitSCM', branches: [[name: '*/main']], userRemoteConfigs: [[url: 'https://github.com/llvm/llvm-zorg.git']]]) + } + } + } + stage('Build') { + steps { + timeout(90) { + sh ''' + set -u + rm -rf build.properties + + cd llvm-project + git tag -a -m "First Commit" first_commit 97724f18c79c7cc81ced24239eb5e883bf1398ef || true + + git_desc=$(git describe --match "first_commit") + + export GIT_DISTANCE=$(echo ${git_desc} | cut -f 2 -d "-") + + sha=$(echo ${git_desc} | cut -f 3 -d "-") + export GIT_SHA=${sha:1} + + # Also save the LLVM_REV until LNT server is taught about GIT + export LLVM_REV=$(git show -q | grep "llvm-svn:" | cut -f2 -d":" | tr -d " ") + + cd - + + export PATH=$PATH:/usr/bin:/usr/local/bin + + # Set a MacOS minimum deployment target + export MACOSX_DEPLOYMENT_TARGET=10.9 + + python llvm-zorg/zorg/jenkins/monorepo_build.py cmake build --assertions --projects="clang" + + # Removing the local artifact + rm -rf clang-*.tar.gz + ''' + } + } + } + stage('Test') { + steps { + timeout(90) { + sh ''' + set -u + export PATH=$PATH:/usr/bin:/usr/local/bin + + rm -rf clang-build/testresults.xunit.xml + + python llvm-zorg/zorg/jenkins/monorepo_build.py cmake testlong + ''' + } + junit 'clang-build/**/testresults.xunit.xml' + } + } + } + post { + always { + scanForIssues tool: clang() + } + /* Disabled email notification. + regression { + emailext subject: '$DEFAULT_SUBJECT', + presendScript: '$DEFAULT_PRESEND_SCRIPT', + postsendScript: '$DEFAULT_POSTSEND_SCRIPT', + recipientProviders: [ + [$class: 'CulpritsRecipientProvider'], + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'], + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS', + body:'$DEFAULT_CONTENT' + } + */ + } +} diff --git a/zorg/jenkins/jobs/jobs/clang-stage2-Rthinlto b/zorg/jenkins/jobs/jobs/clang-stage2-Rthinlto new file mode 100644 --- /dev/null +++ b/zorg/jenkins/jobs/jobs/clang-stage2-Rthinlto @@ -0,0 +1,94 @@ +pipeline { + agent { label 'green-dragon-22' } + + options { disableResume() } + + parameters { + string(name: 'GIT_SHA', defaultValue: '*/main', description: 'Git revision to build') + string(name: 'ARTIFACT', defaultValue: 'clang-stage1-RA/latest', description: 'description') + } + + stages { + stage('Checkout') { + steps { + dir('llvm-project') { + checkout([$class: 'GitSCM', branches: [[name: params.GIT_SHA]], userRemoteConfigs: [[url: 'https://github.com/llvm/llvm-project.git']]]) + } + dir('llvm-zorg') { + checkout([$class: 'GitSCM', branches: [[name: '*/main']], userRemoteConfigs: [[url: 'https://github.com/llvm/llvm-zorg.git']]]) + } + } + } + stage('Build') { + steps { + timeout(600) { + sh ''' + set -u + rm -rf build.properties + + cd llvm-project + git tag -a -m "First Commit" first_commit 97724f18c79c7cc81ced24239eb5e883bf1398ef || true + + git_desc=$(git describe --match "first_commit") + export GIT_DISTANCE=$(echo ${git_desc} | cut -f 2 -d "-") + + sha=$(echo ${git_desc} | cut -f 3 -d "-") + export GIT_SHA=${sha:1} + + cd - + + echo "ARTIFACT=$JOB_NAME/clang-d$GIT_DISTANCE-g$GIT_SHA-t$BUILD_ID-b$BUILD_NUMBER.tar.gz" > build.properties + + export PATH=$PATH:/usr/bin:/usr/local/bin + + rm -rf clang-build clang-install *.tar.gz + + python llvm-zorg/zorg/jenkins/monorepo_build.py fetch + python llvm-zorg/zorg/jenkins/monorepo_build.py clang build \ + --thinlto --projects="clang;compiler-rt;libcxx" \ + --cmake-flag="-DLIBCXX_ENABLE_SHARED=OFF" \ + --cmake-flag="-DLIBCXX_ENABLE_STATIC=OFF" \ + --cmake-flag="-DLIBCXX_INCLUDE_TESTS=OFF" \ + --cmake-flag="-DLIBCXX_ENABLE_EXPERIMENTAL_LIBRARY=OFF" \ + --cmake-flag="-DCMAKE_DSYMUTIL=$WORKSPACE/host-compiler/bin/dsymutil" \ + --sccache + ''' + } + } + } + + stage('Test') { + steps { + timeout(240) { + sh ''' + set -u + export PATH=$PATH:/usr/bin:/usr/local/bin + + rm -rf clang-build/testresults.xunit.xml + + python llvm-zorg/zorg/jenkins/monorepo_build.py clang test + ''' + } + junit 'clang-build/**/testresults.xunit.xml' + } + } + } + post { + always { + scanForIssues tool: clang() + } + regression { + emailext subject: '$DEFAULT_SUBJECT', + presendScript: '$DEFAULT_PRESEND_SCRIPT', + postsendScript: '$DEFAULT_POSTSEND_SCRIPT', + recipientProviders: [ + [$class: 'CulpritsRecipientProvider'], + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'], + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS', + body:'$DEFAULT_CONTENT' + } + } +} diff --git a/zorg/jenkins/jobs/jobs/lldb-cmake b/zorg/jenkins/jobs/jobs/lldb-cmake --- a/zorg/jenkins/jobs/jobs/lldb-cmake +++ b/zorg/jenkins/jobs/jobs/lldb-cmake @@ -2,8 +2,8 @@ pipeline { agent { label 'green-dragon-10' } parameters { - string(name: 'GIT_REVISION', defaultValue: '*/master', description: 'Git revision to build') - string(name: 'ARTIFACT', defaultValue: 'clang-stage1-RA/latest', description: 'Compiler artifact to use for building the project') + string(name: 'GIT_REVISION', defaultValue: '*/main', description: 'Git revision to build') + string(name: 'ARTIFACT', defaultValue: 'clang-stage2-Rthinlto/latest', description: 'Compiler artifact to use for building the project') string(name: 'BUILD_TYPE', defaultValue: 'Release', description: 'Default CMake build type; one of: Release, Debug, ...') string(name: 'CLEAN', defaultValue: "false", description: 'Whether or not to clean the build directory before building') } @@ -15,7 +15,7 @@ checkout([$class: 'GitSCM', branches: [[name: params.GIT_REVISION]], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-project.git']]]) } dir('llvm-zorg') { - checkout([$class: 'GitSCM', branches: [[name: '*/master']], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-zorg.git']]]) + checkout([$class: 'GitSCM', branches: [[name: '*/main']], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-zorg.git']]]) } } } @@ -51,11 +51,11 @@ export MAX_PARALLEL_TESTS=$(sysctl hw.physicalcpu |awk '{print $2}') python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake build \ --assertions \ - --projects="clang;libcxx;libcxxabi;compiler-rt;lld;lldb;debuginfo-tests" \ + --projects="clang;libcxx;libcxxabi;compiler-rt;lld;lldb;cross-project-tests" \ --compiler-flag="-Wdocumentation" \ --cmake-type=Release || (echo "Retrying. debuginfo-tests may have failed due to high load."; sleep 30; python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake build \ --assertions \ - --projects="clang;libcxx;libcxxabi;compiler-rt;lld;lldb;debuginfo-tests" \ + --projects="clang;libcxx;libcxxabi;compiler-rt;lld;lldb;cross-project-tests" \ --compiler-flag="-Wdocumentation" \ --cmake-type=Release) ''' @@ -68,7 +68,6 @@ sh ''' set -u export PATH=$PATH:/usr/bin:/usr/local/bin - export LLDB_CAPTURE_REPRODUCER=1 rm -rf test/results.xml diff --git a/zorg/jenkins/jobs/jobs/lldb-cmake-matrix b/zorg/jenkins/jobs/jobs/lldb-cmake-matrix --- a/zorg/jenkins/jobs/jobs/lldb-cmake-matrix +++ b/zorg/jenkins/jobs/jobs/lldb-cmake-matrix @@ -3,8 +3,8 @@ pipeline { agent { label 'green-dragon-23' } parameters { - string(name: 'GIT_REVISION', defaultValue: '*/master', description: 'Git revision to build') - string(name: 'ARTIFACT', defaultValue: 'clang-stage1-RA/latest', description: 'Compiler artifact to use for building the project') + string(name: 'GIT_REVISION', defaultValue: '*/main', description: 'Git revision to build') + string(name: 'ARTIFACT', defaultValue: 'clang-stage2-Rthinlto/latest', description: 'Compiler artifact to use for building the project') string(name: 'BUILD_TYPE', defaultValue: 'Release', description: 'Default CMake build type; one of: Release, Debug, ...') string(name: 'CLEAN', defaultValue: "false", description: 'Whether or not to clean the build directory before building') } @@ -16,7 +16,7 @@ checkout([$class: 'GitSCM', branches: [[name: params.GIT_REVISION]], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-project.git']]]) } dir('llvm-zorg') { - checkout([$class: 'GitSCM', branches: [[name: '*/master']], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-zorg.git']]]) + checkout([$class: 'GitSCM', branches: [[name: '*/main']], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-zorg.git']]]) } } } @@ -271,5 +271,19 @@ junit 'test/results.xml' } } + stage('Test Reproducers') { + steps { + timeout(120) { + sh ''' + export PATH=$PATH:/usr/bin:/usr/local/bin + + set +e + python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake \ + --cmake-test-target check-lldb-reproducers + set -e + ''' + } + } + } } } diff --git a/zorg/jenkins/jobs/jobs/lldb-cmake-reproducers b/zorg/jenkins/jobs/jobs/lldb-cmake-reproducers deleted file mode 100644 --- a/zorg/jenkins/jobs/jobs/lldb-cmake-reproducers +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env groovy -pipeline { - agent { label 'green-dragon-23' } - parameters { - string(name: 'GIT_REVISION', defaultValue: '*/master', description: 'Git revision to build') - string(name: 'ARTIFACT', defaultValue: 'clang-stage1-RA/latest', description: 'Compiler artifact to use for building the project') - string(name: 'BUILD_TYPE', defaultValue: 'Release', description: 'Default CMake build type; one of: Release, Debug, ...') - string(name: 'CLEAN', defaultValue: "false", description: 'Whether or not to clean the build directory before building') - } - stages { - stage('Checkout') { - steps { - timeout(30) { - dir('llvm-project') { - checkout([$class: 'GitSCM', branches: [[name: params.GIT_REVISION]], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-project.git']]]) - } - dir('llvm-zorg') { - checkout([$class: 'GitSCM', branches: [[name: '*/master']], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-zorg.git']]]) - } - } - } - } - stage('Fetch Host Compiler') { - steps { - timeout(10) { - sh ''' - python llvm-zorg/zorg/jenkins/monorepo_build.py fetch - ''' - } - } - } - stage('Build') { - steps { - timeout(90) { - sh ''' - set -u - rm -rf build.properties - - cd llvm-project - git tag -a -m "First Commit" first_commit 97724f18c79c7cc81ced24239eb5e883bf1398ef || true - - git_desc=$(git describe --match "first_commit") - - export GIT_DISTANCE=$(echo ${git_desc} | cut -f 2 -d "-") - - sha=$(echo ${git_desc} | cut -f 3 -d "-") - export GIT_SHA=${sha:1} - - cd - - - python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake build \ - --assertions \ - --projects="clang;libcxx;libcxxabi;compiler-rt;lld;lldb;debuginfo-tests" \ - --compiler-flag="-Wdocumentation" \ - --cmake-type=Release - ''' - } - } - } - stage('Test Reproducers') { - steps { - timeout(120) { - sh ''' - set -u - export PATH=$PATH:/usr/bin:/usr/local/bin - rm -rf test/results.xml - python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake \ - --cmake-test-target check-lldb-reproducers - ''' - } - } - } - } - post { - always { - scanForIssues tool: clang() - junit 'test/results.xml' - } - } -} diff --git a/zorg/jenkins/jobs/jobs/lldb-cmake-sanitized b/zorg/jenkins/jobs/jobs/lldb-cmake-sanitized --- a/zorg/jenkins/jobs/jobs/lldb-cmake-sanitized +++ b/zorg/jenkins/jobs/jobs/lldb-cmake-sanitized @@ -2,8 +2,8 @@ pipeline { agent { label 'green-dragon-23' } parameters { - string(name: 'GIT_REVISION', defaultValue: '*/master', description: 'Git revision to build') - string(name: 'ARTIFACT', defaultValue: 'clang-stage1-RA/latest', description: 'Compiler artifact to use for building the project') + string(name: 'GIT_REVISION', defaultValue: '*/main', description: 'Git revision to build') + string(name: 'ARTIFACT', defaultValue: 'clang-stage2-Rthinlto/latest', description: 'Compiler artifact to use for building the project') string(name: 'BUILD_TYPE', defaultValue: 'Release', description: 'Default CMake build type; one of: Release, Debug, ...') string(name: 'CLEAN', defaultValue: "false", description: 'Whether or not to clean the build directory before building') } @@ -15,7 +15,7 @@ checkout([$class: 'GitSCM', branches: [[name: params.GIT_REVISION]], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-project.git']]]) } dir('llvm-zorg') { - checkout([$class: 'GitSCM', branches: [[name: '*/master']], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-zorg.git']]]) + checkout([$class: 'GitSCM', branches: [[name: '*/main']], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-zorg.git']]]) } } } diff --git a/zorg/jenkins/jobs/jobs/lldb-cmake-standalone b/zorg/jenkins/jobs/jobs/lldb-cmake-standalone --- a/zorg/jenkins/jobs/jobs/lldb-cmake-standalone +++ b/zorg/jenkins/jobs/jobs/lldb-cmake-standalone @@ -2,8 +2,8 @@ pipeline { agent {label 'green-dragon-23'} parameters { - string(name: 'GIT_REVISION', defaultValue: '*/master', description: 'Git revision to build') - string(name: 'ARTIFACT', defaultValue: 'clang-stage1-RA/latest', description: 'Compiler artifact to use for building the project') + string(name: 'GIT_REVISION', defaultValue: '*/main', description: 'Git revision to build') + string(name: 'ARTIFACT', defaultValue: 'clang-stage2-Rthinlto/latest', description: 'Compiler artifact to use for building the project') string(name: 'BUILD_TYPE', defaultValue: 'Release', description: 'Default CMake build type; one of: Release, Debug, ...') string(name: 'CLEAN', defaultValue: "false", description: 'Whether or not to clean the build directory before building') } @@ -15,7 +15,7 @@ checkout([$class: 'GitSCM', branches: [[name: params.GIT_REVISION]], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-project.git']]]) } dir('llvm-zorg') { - checkout([$class: 'GitSCM', branches: [[name: '*/master']], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-zorg.git']]]) + checkout([$class: 'GitSCM', branches: [[name: '*/main']], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-zorg.git']]]) } } } diff --git a/zorg/jenkins/jobs/jobs/llvm-coverage b/zorg/jenkins/jobs/jobs/llvm-coverage new file mode 100755 --- /dev/null +++ b/zorg/jenkins/jobs/jobs/llvm-coverage @@ -0,0 +1,134 @@ +#!/usr/bin/env groovy +pipeline { + agent { label 'green-dragon-23' } + + parameters { + string(name: 'GIT_REVISION', defaultValue: '*/main', description: 'Git revision to build') + string(name: 'ARTIFACT', defaultValue: 'clang-stage1-RA/latest', description: 'Compiler artifact to use for building the project') + string(name: 'BUILD_TYPE', defaultValue: 'Release', description: 'Default CMake build type; one of: Release, Debug, ...') + string(name: 'CLEAN', defaultValue: "false", description: 'Whether or not to clean the build directory before building') + } + + stages { + stage('Checkout') { + steps { + dir('llvm-project') { + checkout([$class: 'GitSCM', branches: [[name: params.GIT_REVISION]], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-project.git']]]) + } + dir('llvm-zorg') { + checkout([$class: 'GitSCM', branches: [[name: '*/main']], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-zorg.git']]]) + } + } + } + stage('Fetch') { + steps { + timeout(10) { + sh ''' + python llvm-zorg/zorg/jenkins/monorepo_build.py fetch + ''' + } + } + } + stage('Build') { + steps { + timeout(300) { + sh ''' + set -u + rm -rf build.properties + + cd llvm-project + git tag -a -m "First Commit" first_commit 97724f18c79c7cc81ced24239eb5e883bf1398ef || true + + git_desc=$(git describe --match "first_commit") + + export GIT_DISTANCE=$(echo ${git_desc} | cut -f 2 -d "-") + + sha=$(echo ${git_desc} | cut -f 3 -d "-") + export GIT_SHA=${sha:1} + + cd - + + export PATH=$PATH:/usr/bin:/usr/local/bin + + python llvm-zorg/zorg/jenkins/monorepo_build.py cmake build \ + --cmake-build-target FileCheck \ + --cmake-build-target count \ + --cmake-build-target not \ + --cmake-flag="-DLLVM_TARGETS_TO_BUILD=host" \ + --cmake-type=Release \ + --projects "" \ + --noinstall \ + --noupload + + python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake build \ + --assertions \ + --projects="clang;libcxx;libcxxabi;lldb;cross-project-tests" \ + --cmake-flag="-DPYTHON_LIBRARY=/usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/lib/libpython3.7.dylib" \ + --cmake-flag="-DPYTHON_INCLUDE_DIR=/usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/include/python3.7m" \ + --cmake-flag="-DPYTHON_LIBRARY_DEBUG=/usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/lib/libpython3.7.dylib" \ + --cmake-flag="-DPYTHON_EXECUTABLE=/usr/local/Cellar/python/3.7.0/bin/python3.7" \ + --cmake-flag="-DLLDB_USE_SYSTEM_DEBUGSERVER=ON" \ + --cmake-flag="-DLLVM_BUILD_INSTRUMENTED_COVERAGE=ON" \ + --cmake-flag="-DLLVM_PROFILE_MERGE_POOL_SIZE=1" \ + --cmake-flag="-DLLVM_ENABLE_MODULES=Off" \ + --cmake-type=Release \ + --noupload + ''' + } + } + } + stage('Test') { + steps { + timeout(300) { + sh ''' + set -u + export PATH=$PATH:/usr/bin:/usr/local/bin + + rm -rf test/results.xml + + BUILD_DIR=$(pwd)/lldb-build + FAST_BUILD_DIR=$(pwd)/clang-build + REPORT_DIR=$(pwd)/coverage-reports + HOST=$(pwd)/host-compiler/bin + LLVM_PROFDATA=$HOST/llvm-profdata + LLVM_COV=$HOST/llvm-cov + ARTIFACT_PREP_SCRIPT=$WORKSPACE/llvm-project/llvm/utils/prepare-code-coverage-artifact.py + + FAST_TOOLS=(FileCheck count not) + for TOOL in ${FAST_TOOLS[@]}; do + cp $FAST_BUILD_DIR/bin/$TOOL $BUILD_DIR/bin/$TOOL + done + + # Clear out any stale profiles. + rm -rf $BUILD_DIR/profiles + + # Run the tests. + IGNORE_ERRORS_OVERRIDE=1 python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake testlong || echo "Some tests may have failed." + + cd $BUILD_DIR + ninja -k 0 check-llvm check-clang || echo "Some tests may have failed." + cd - + + COV_BINARIES=$(find $BUILD_DIR/bin $BUILD_DIR/lib -depth 1 -type f -exec file {} \\; | grep Mach-O | cut -d':' -f1 | grep -vE '/(FileCheck|count|not)$' | xargs) + + rm -rf $REPORT_DIR + mkdir -p $REPORT_DIR + python $ARTIFACT_PREP_SCRIPT $LLVM_PROFDATA $LLVM_COV $BUILD_DIR/profiles $REPORT_DIR $COV_BINARIES --unified-report --restrict $WORKSPACE/llvm-project + + scp -r $REPORT_DIR buildslave@labmaster2.local:/Library/WebServer/Documents/coverage + ssh buildslave@labmaster2.local "chmod -R 777 /Library/WebServer/Documents/coverage" + + rm -rf lldb-build/profiles + ''' + } + } + } + } + + post { + always { + scanForIssues tool: clang() + junit 'test/results.xml' + } + } +} diff --git a/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-O0-g b/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-O0-g --- a/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-O0-g +++ b/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-O0-g @@ -1,9 +1,11 @@ #!/usr/bin/env groovy def common = evaluate readTrusted('zorg/jenkins/common.groovy') common.testsuite_pipeline(label: 'green-dragon-12') { - sh """ + timeout(30) { + sh """ LNT_FLAGS+=" -C config/tasks/cmake/caches/target-arm64-iphoneos.cmake" LNT_FLAGS+=" -C config/tasks/cmake/caches/opt-O0-g.cmake" config/tasks/task jenkinsrun config/tasks/lnt-ctmark.sh -a compiler="${params.ARTIFACT}" -D LNT_FLAGS="\${LNT_FLAGS}" - """ + """ + } } diff --git a/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-O3-flto b/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-O3-flto --- a/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-O3-flto +++ b/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-O3-flto @@ -1,9 +1,11 @@ #!/usr/bin/env groovy def common = evaluate readTrusted('zorg/jenkins/common.groovy') common.testsuite_pipeline(label: 'green-dragon-12') { - sh """ + timeout(30) { + sh """ LNT_FLAGS+=" -C config/tasks/cmake/caches/target-arm64-iphoneos.cmake" LNT_FLAGS+=" -C config/tasks/cmake/caches/opt-O3-flto.cmake" config/tasks/task jenkinsrun config/tasks/lnt-ctmark.sh -a compiler="${params.ARTIFACT}" -D LNT_FLAGS="\${LNT_FLAGS}" - """ + """ + } } diff --git a/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-Os b/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-Os --- a/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-Os +++ b/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-Os @@ -1,9 +1,11 @@ #!/usr/bin/env groovy def common = evaluate readTrusted('zorg/jenkins/common.groovy') common.testsuite_pipeline(label: 'green-dragon-12') { - sh """ + timeout(30) { + sh """ LNT_FLAGS+=" -C config/tasks/cmake/caches/target-arm64-iphoneos.cmake" LNT_FLAGS+=" -C config/tasks/cmake/caches/opt-Os.cmake" config/tasks/task jenkinsrun config/tasks/lnt-ctmark.sh -a compiler="${params.ARTIFACT}" -D LNT_FLAGS="\${LNT_FLAGS}" - """ + """ + } } diff --git a/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-Oz b/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-Oz --- a/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-Oz +++ b/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-Oz @@ -1,9 +1,11 @@ #!/usr/bin/env groovy def common = evaluate readTrusted('zorg/jenkins/common.groovy') common.testsuite_pipeline(label: 'green-dragon-12') { - sh """ + timeout(30) { + sh """ LNT_FLAGS+=" -C config/tasks/cmake/caches/target-arm64-iphoneos.cmake" LNT_FLAGS+=" -C config/tasks/cmake/caches/opt-Oz.cmake" config/tasks/task jenkinsrun config/tasks/lnt-ctmark.sh -a compiler="${params.ARTIFACT}" -D LNT_FLAGS="\${LNT_FLAGS}" - """ + """ + } } diff --git a/zorg/jenkins/jobs/util/make_pipeline.py b/zorg/jenkins/jobs/util/make_pipeline.py --- a/zorg/jenkins/jobs/util/make_pipeline.py +++ b/zorg/jenkins/jobs/util/make_pipeline.py @@ -32,7 +32,7 @@ - */master + */main false diff --git a/zorg/jenkins/monorepo_build.py b/zorg/jenkins/monorepo_build.py --- a/zorg/jenkins/monorepo_build.py +++ b/zorg/jenkins/monorepo_build.py @@ -202,7 +202,7 @@ def branch(self): """Figure out the source branch name. - Not using GIT_BRANCH env var from Jenkins as that includes the + Not using GIT_BRANCH env var from Jenkins as that includes the remote name too. """ if not os.environ.get('TESTING', False): @@ -213,16 +213,16 @@ cmd = ['git', '-C', conf.srcdir(), 'rev-parse', 'HEAD'] out = run_collect_output(cmd, stderr=subprocess.STDOUT).strip() return out - return 'master' + return 'main' def link_memory_usage(self): """Guesstimate the maximum link memory usage for this build. - We are only building master here so we will just use that value + We are only building main here so we will just use that value """ # Determinited experimentally. - usages = {'master': 3.5} - if self.branch() == 'master': - return usages['master'] + usages = {'main': 3.5} + if self.branch() == 'main': + return usages['main'] else: raise NotImplementedError( "Unknown link memory usage." + self.branch()) @@ -350,7 +350,7 @@ # unless the user asked for something else. targets = ['check', 'check-clang'] - run_cmd(conf.builddir(), ninja_cmd + targets) + run_cmd(conf.builddir(), ninja_cmd + targets, err_okay=True) footer() if conf.sccache: @@ -497,7 +497,7 @@ cmd[-1] += ' --param use_gmalloc=1 ' \ '--param gmalloc_path=$(xcodebuild -find-library' \ ' libgmalloc.dylib)' - run_cmd(obj_dir, cmd, env={'MALLOC_LOG_FILE': '/dev/null'}) + run_cmd(obj_dir, cmd, env={'MALLOC_LOG_FILE': '/dev/null'}, err_okay=True) if conf.sccache: run_ws([conf.sccache_path, "--stop-server"]) @@ -538,7 +538,7 @@ # Construct lit arguments. lit_args = ['-v', '--time-tests', '--shuffle', - '--xunit-xml-output={}'.format(results_file), '-v'] + '--xunit-xml-output={}'.format(results_file), '-v'] if conf.max_parallel_tests: lit_args.extend(['-j', conf.max_parallel_tests]) if variant == 'sanitized': @@ -566,8 +566,7 @@ # if we have a host compiler available. if conf.CC(): cmake_cmd.extend([ - '-DLLDB_TEST_C_COMPILER=' + conf.CC(), - '-DLLDB_TEST_CXX_COMPILER=' + conf.CC() + "++" + '-DLLDB_TEST_COMPILER=' + conf.CC(), ]) if conf.compiler_flags: @@ -578,8 +577,7 @@ if conf.lldbtestcompiler(): cmake_cmd.extend([ - '-DLLDB_TEST_C_COMPILER=' + conf.lldbtestcompiler(), - '-DLLDB_TEST_CXX_COMPILER=' + conf.lldbtestcompiler() + "++" + '-DLLDB_TEST_COMPILER=' + conf.lldbtestcompiler(), ]) cmake_cmd.extend(conf.cmake_flags) @@ -609,15 +607,15 @@ run_cmd(conf.lldbbuilddir(), [NINJA, '-v', 'install']) footer() - if target == 'all' or target == 'testlong': - header("Run Debug Info Tests") - run_cmd(conf.lldbbuilddir(), [NINJA, '-v', 'check-debuginfo']) - footer() - if target == 'all' or target == 'test' or target == 'testlong': header("Run Tests") - run_cmd(conf.lldbbuilddir(), - ['/usr/bin/env', 'TERM=vt100', NINJA, '-v', 'check-lldb']) + if variant == 'matrix' or variant == 'sanitized': + test_command = ['/usr/bin/env', 'TERM=vt100', NINJA, + '-v', 'check-lldb'] + else: + test_command = ['/usr/bin/env', 'TERM=vt100', NINJA, + '-v', 'check-debuginfo', 'check-lldb', '-k2'] + run_cmd(conf.lldbbuilddir(), test_command) footer() for test_target in conf.cmake_test_targets: @@ -778,11 +776,11 @@ with open(dest, "wb") as local_file: local_file.write(f.read()) - except urllib.HTTPError as e: + except urllib.error.HTTPError as e: print("HTTP Error:", e.code, url) sys.exit(1) - except urllib.URLError as e: + except urllib.error.URLError as e: print("URL Error:", e.reason, url) sys.exit(1) print("done.")