diff --git a/.pylintrc b/.pylintrc new file mode 100644 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,7 @@ +# add root folder to Pylint PYTHONPATH, so builders can be checked with zorg.* +# packages +[MASTER] +init-hook='import sys; sys.path.append(".")' + +[FORMAT] +max-line-length=120 \ No newline at end of file diff --git a/buildbot/google/docker/buildbot-clangd-ubuntu-clang/VERSION b/buildbot/google/docker/buildbot-clangd-ubuntu-clang/VERSION --- a/buildbot/google/docker/buildbot-clangd-ubuntu-clang/VERSION +++ b/buildbot/google/docker/buildbot-clangd-ubuntu-clang/VERSION @@ -1 +1 @@ -3 +4 diff --git a/buildbot/google/docker/buildbot-clangd-ubuntu-clang/run.sh b/buildbot/google/docker/buildbot-clangd-ubuntu-clang/run.sh --- a/buildbot/google/docker/buildbot-clangd-ubuntu-clang/run.sh +++ b/buildbot/google/docker/buildbot-clangd-ubuntu-clang/run.sh @@ -38,9 +38,14 @@ buildbot-worker create-worker --keepalive=200 "${WORKER_NAME}" \ lab.llvm.org:${BUILDBOT_PORT} "${WORKER_NAME}" "${WORKER_PASSWORD}" -# start the daemon, this command return immetiately +# start the daemon, this command returns immediately echo "starting worker..." -buildbot-worker start "${WORKER_NAME}" +set +e +if ! buildbot-worker start "${WORKER_NAME}"; then + echo ERROR: starting worker failed. contents of twistd.log: + cat "${WORKER_NAME}/twistd.log" + exit 1 +fi # To keep the container running and produce log outputs: dump the worker # log to stdout diff --git a/buildbot/google/docker/buildbot-mlir-nvidia/Dockerfile b/buildbot/google/docker/buildbot-mlir-nvidia/Dockerfile --- a/buildbot/google/docker/buildbot-mlir-nvidia/Dockerfile +++ b/buildbot/google/docker/buildbot-mlir-nvidia/Dockerfile @@ -5,7 +5,7 @@ # #===----------------------------------------------------------------------===// # Docker image used for the mlir-nvidia builder -# +# # Environment variables configurable at runtime: # BUILDBOT_PORT - server port to connect to #===----------------------------------------------------------------------===// @@ -18,7 +18,7 @@ RUN apt-get update; \ apt-get install -y software-properties-common apt-transport-https ca-certificates \ clang-8 lld-8 ninja-build git wget gnupg ccache \ - python3 python3-pip python3-psutil pybind11-dev \ + python3 python3-pip python3-psutil \ # dumb-init recommended in # https://hub.docker.com/r/buildbot/buildbot-worker/dockerfile dumb-init;\ @@ -26,6 +26,16 @@ update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-8 100 ;\ update-alternatives --install /usr/bin/lld lld /usr/bin/lld-8 100 +# LTS releases often bundle obsolete pip versions that cannot access newest +# Linux binary wheels. This pinned version is not special: it was just current +# at the time this was added. Refer to compatibility table: +# https://github.com/pypa/manylinux +RUN python3 -m pip install --upgrade pip==21.0.1 + +# Refer to mlir/lib/Bindings/Python/requirements.txt. Listed explicitly here +# and version pinned for consistency as this is a bot. +RUN python3 -m pip install numpy==1.19.3 pybind11==2.6.0 PyYAML==5.4.1 + # install cuda # avoid popups for keyboard configurations RUN DEBIAN_FRONTEND=noninteractive apt-get install -y cuda @@ -78,7 +88,7 @@ ENV WORKER_NAME="mlir-nvidia" # Allow the server port of this agent to be configurable during deployment. -# This way we can connect the same image to production and integration. +# This way we can connect the same image to production and integration. # Ports: # 9990 - production # 9994 - integration diff --git a/buildbot/google/docker/buildbot-mlir-nvidia/VERSION b/buildbot/google/docker/buildbot-mlir-nvidia/VERSION --- a/buildbot/google/docker/buildbot-mlir-nvidia/VERSION +++ b/buildbot/google/docker/buildbot-mlir-nvidia/VERSION @@ -1 +1 @@ -15 +16 diff --git a/buildbot/google/docker/buildbot-sanitizer-windows/Dockerfile b/buildbot/google/docker/buildbot-sanitizer-windows/Dockerfile new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-sanitizer-windows/Dockerfile @@ -0,0 +1,21 @@ +# escape=` +FROM gcr.io/sanitizer-bots/windows-base-vscode2019:8 + +ENV WORKER_NAME=sanitizer-windows + +# copy script to start the agent +COPY run.ps1 . +COPY fix_buildbot_tac_paths.py . + +# Set location for sccache cache +ENV SCCACHE_DIR="C:\volumes\sccache" + +# Set the maximum sccache local cache size +ENV SCCACHE_CACHE_SIZE="10G" + +# Move buildbot work dir to a volume to avoid of disc issues +VOLUME C:\volumes\buildbot + +# Configure 32bit tools ("x86") instead of 64bit ("amd64") +CMD ["powershell.exe", "-NoLogo", "-ExecutionPolicy", "Bypass", ` + "c:\\buildbot\\run.ps1"] diff --git a/buildbot/google/docker/buildbot-sanitizer-windows/VERSION b/buildbot/google/docker/buildbot-sanitizer-windows/VERSION new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-sanitizer-windows/VERSION @@ -0,0 +1 @@ +1 diff --git a/buildbot/google/docker/buildbot-sanitizer-windows/admin b/buildbot/google/docker/buildbot-sanitizer-windows/admin new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-sanitizer-windows/admin @@ -0,0 +1 @@ +Reid Kleckner diff --git a/buildbot/google/docker/buildbot-sanitizer-windows/fix_buildbot_tac_paths.py b/buildbot/google/docker/buildbot-sanitizer-windows/fix_buildbot_tac_paths.py new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-sanitizer-windows/fix_buildbot_tac_paths.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +# Replace backward slashes in the paths in a "buildbot.tac" file + +import argparse +import re +import shutil + + +def fix_slashes(buildbot_tac_path : str): + shutil.copy(buildbot_tac_path, buildbot_tac_path + ".bak") + result = [] + with open(buildbot_tac_path) as buildbot_tac_file: + buildbot_tac = buildbot_tac_file.readlines() + for line in buildbot_tac: + if line.lstrip().startswith('basedir'): + line = line.replace(r'\\','/') + result.append(line) + with open(buildbot_tac_path, 'w') as buildbot_tac_file: + buildbot_tac_file.writelines(result) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('buildbot_tac_path', help="path of 'buildbot.tac' file") + args = parser.parse_args() + fix_slashes(args.buildbot_tac_path) diff --git a/buildbot/google/docker/buildbot-sanitizer-windows/run.ps1 b/buildbot/google/docker/buildbot-sanitizer-windows/run.ps1 new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-sanitizer-windows/run.ps1 @@ -0,0 +1,51 @@ +# Stop the script on the first error +$ErrorActionPreference = "Stop" + +# Read password from file +$WORKER_PASSWORD=Get-Content "C:\volumes\secrets\token" + +#Create short drive alias "Q:" for worker dir, to keep path lengths <260 +SUBST Q: C:\volumes\buildbot +Get-PSDrive + +# chdir to the volume storing the temporary data +# This is a mounted volume +Set-Location Q:\ + +# Write host and admin information +# delete old folder if it exists +# configure powershell output to use UTF-8, otherwiese buildbot can't read the file +if ( Test-Path -Path 'info\' -PathType Container ) { + Remove-Item -Recurse -Force info +} +mkdir "info\" +$HOST_FILE="info\host" +$PSDefaultParameterValues['Out-File:Encoding'] = 'utf8' +Write-Output "Windows version: $(cmd /c ver)" > $HOST_FILE +Write-Output "VCTools version: $(ls C:\BuildTools\VC\Tools\MSVC | Select -ExpandProperty Name)" >> $HOST_FILE +Write-Output "Cores : $((Get-CimInstance Win32_ComputerSystem).NumberOfLogicalProcessors)" >> $HOST_FILE +Write-Output "RAM : $([int][Math]::Round((Get-CimInstance Win32_ComputerSystem).TotalPhysicalMemory / 1073741824)) GB" >> $HOST_FILE +Write-Output "Reid Kleckner " > "info\admin" + +# create the worker +Write-Output "creating worker..." +buildslave create-slave --keepalive=200 Q: ` + "lab.llvm.org:9994" "$env:WORKER_NAME" "$WORKER_PASSWORD" +# Note: Powershell does NOT exit on non-zero return codes, so we need to check manually. +if ($LASTEXITCODE -ne 0) { throw "Exit code of 'buildslave create' is $LASTEXITCODE" } + +# Replace backward slashes with forward slashes on buildbot.tac +# Otherwise Cmake will go into infinite re-configuration loops +python C:\buildbot\fix_buildbot_tac_paths.py buildbot.tac +if ($LASTEXITCODE -ne 0) { throw "Exit code of 'fix_buildbot_tac_paths' is $LASTEXITCODE" } + +# start the daemon, as this does not print and logs to std out, run it in the background +Write-Output "starting worker..." +cmd /c Start /B buildslave start Q: + +# Wait a bit until the logfile exists +Start-Sleep -s 5 + +# To keep the container running and produce log outputs: dump the worker +# log to stdout, this is the windows equivalent of `tail -f` +Get-Content -Path "Q:\twistd.log" -Wait diff --git a/buildbot/google/terraform/deployment-mlir-nvidia-production.yaml b/buildbot/google/terraform/deployment-mlir-nvidia-production.yaml --- a/buildbot/google/terraform/deployment-mlir-nvidia-production.yaml +++ b/buildbot/google/terraform/deployment-mlir-nvidia-production.yaml @@ -27,7 +27,7 @@ spec: containers: # the image and version we want to run - - image: gcr.io/sanitizer-bots/buildbot-mlir-nvidia:15 + - image: gcr.io/sanitizer-bots/buildbot-mlir-nvidia:16 name: mlir-nvidia-production env: # connect to production environment, running at port 9990 diff --git a/buildbot/google/terraform/main.tf b/buildbot/google/terraform/main.tf --- a/buildbot/google/terraform/main.tf +++ b/buildbot/google/terraform/main.tf @@ -290,8 +290,8 @@ node_config { # use preemptible, as this saves costs preemptible = true - #custom machine type: 16 core, 32 GB as tsan needs more RAM - machine_type = "n2d-custom-16-32768" + #custom machine type: 16 core, 48 GB as tsan needs more RAM + machine_type = "n2d-custom-16-49152" disk_size_gb = 100 disk_type = "pd-ssd" @@ -347,7 +347,7 @@ spec { container { - image = "${var.gcp_config.gcr_prefix}/buildbot-clangd-ubuntu-clang:3" + image = "${var.gcp_config.gcr_prefix}/buildbot-clangd-ubuntu-clang:4" name = "buildbot-clangd-ubuntu-clang" # reserve "-1" for this image, kubernetes also @@ -355,11 +355,11 @@ resources { limits { cpu = "15" - memory = "28G" + memory = "45G" } requests { cpu = "15" - memory = "28G" + memory = "45G" } } diff --git a/buildbot/osuosl/master/config/__init__.py b/buildbot/osuosl/master/config/__init__.py --- a/buildbot/osuosl/master/config/__init__.py +++ b/buildbot/osuosl/master/config/__init__.py @@ -12,6 +12,7 @@ import config.auth import config.builders +import config.release_builders import config.schedulers import config.workers import config.status @@ -22,4 +23,5 @@ reload(config.workers) reload(config.schedulers) reload(config.builders) +reload(release_builders) reload(config.status) diff --git a/buildbot/osuosl/master/config/builders.py b/buildbot/osuosl/master/config/builders.py --- a/buildbot/osuosl/master/config/builders.py +++ b/buildbot/osuosl/master/config/builders.py @@ -1,4 +1,6 @@ -# TODO: Rename workers with "slave" as a part of the name. +from importlib import reload + +from buildbot.plugins import util from buildbot.process.properties import WithProperties @@ -18,8 +20,13 @@ from zorg.buildbot.builders import LLDPerformanceTestsuite from zorg.buildbot.builders import FuchsiaBuilder from zorg.buildbot.builders import XToolchainBuilder +from zorg.buildbot.builders import ZigBuilder -from buildbot.plugins import util +from zorg.buildbot.builders import HtmlDocsBuilder +from zorg.buildbot.builders import DoxygenDocsBuilder + +reload(HtmlDocsBuilder) +reload(DoxygenDocsBuilder) all = [ @@ -92,6 +99,27 @@ "-DLLVM_OPTIMIZED_TABLEGEN=OFF", "-DLLVM_LIT_ARGS=-v --threads=32"])}, + {'name': "llvm-clang-x86_64-sie-ubuntu-fast", + 'tags' : ["clang", "llvm", "clang-tools-extra", "lld"], + 'workernames': ["sie-linux-builder"], + 'builddir': "llvm-clang-x86_64-sie-ubuntu-fast", + 'factory': UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + depends_on_projects=['llvm','clang','clang-tools-extra','lld'], + extra_configure_args=[ + "-DCMAKE_C_COMPILER=gcc", + "-DCMAKE_CXX_COMPILER=g++", + "-DCMAKE_BUILD_TYPE=Release", + "-DCLANG_ENABLE_ARCMT=OFF", + "-DCLANG_ENABLE_CLANGD=OFF", + "-DLLVM_BUILD_RUNTIME=OFF", + "-DLLVM_CCACHE_BUILD=ON", + "-DLLVM_INCLUDE_EXAMPLES=OFF", + "-DLLVM_DEFAULT_TARGET_TRIPLE=x86_64-scei-ps4", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_LIT_ARGS=--verbose -j100", + "-DLLVM_TARGETS_TO_BUILD=X86", + "-DLLVM_USE_LINKER=gold"])}, + # Expensive checks builders. {'name' : "llvm-clang-x86_64-expensive-checks-ubuntu", @@ -109,17 +137,17 @@ "-DLLVM_LIT_ARGS=-vv -j32"])}, {'name' : "llvm-clang-x86_64-expensive-checks-win", - 'tags' : ["llvm", "clang", "expensive-checks"], + 'tags' : ["llvm", "expensive-checks"], 'workernames' : ["as-worker-93"], 'builddir': "llvm-clang-x86_64-expensive-checks-win", 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaWithMSVCBuildFactory( vs="autodetect", + depends_on_projects=["llvm"], clean=True, extra_configure_args=[ "-DLLVM_ENABLE_EXPENSIVE_CHECKS=ON", "-DLLVM_ENABLE_WERROR=OFF", - "-DCMAKE_BUILD_TYPE=Debug", - "-DLLVM_LIT_ARGS=-vv"])}, + "-DCMAKE_BUILD_TYPE=Debug"])}, {'name' : "llvm-clang-x86_64-expensive-checks-debian", 'tags' : ["llvm", "expensive-checks"], @@ -229,7 +257,10 @@ "-DLLVM_DEFAULT_TARGET_TRIPLE=aarch64-windows-msvc", "-DLLVM_HOST_TRIPLE=aarch64-windows-msvc", "-DLLVM_TARGET_ARCH=AArch64", + # FIXME: compiler-rt\lib\sanitizer_common\sanitizer_unwind_win.cpp assumes WIN64 is x86_64, + # so, before that's fixed, disable everything that triggers its build. "-DCOMPILER_RT_BUILD_SANITIZERS=OFF", + "-DCOMPILER_RT_BUILD_MEMPROF=OFF", "-DCOMPILER_RT_BUILD_XRAY=OFF"])}, # Cortex-A15 LNT test-suite in Benchmark mode @@ -344,7 +375,6 @@ checkout_lld=False, test=True, useTwoStage=False, - runTestSuite=True, testsuite_flags=[ '--cppflags', '-mcpu=cortex-a57', '--threads=32', '--build-threads=32'], @@ -488,7 +518,10 @@ "-DLLVM_DEFAULT_TARGET_TRIPLE=aarch64-windows-msvc", "-DLLVM_HOST_TRIPLE=aarch64-windows-msvc", "-DLLVM_TARGET_ARCH=AArch64", + # FIXME: compiler-rt\lib\sanitizer_common\sanitizer_unwind_win.cpp assumes WIN64 is x86_64, + # so, before that's fixed, disable everything that triggers its build. "-DCOMPILER_RT_BUILD_SANITIZERS=OFF", + "-DCOMPILER_RT_BUILD_MEMPROF=OFF", "-DCOMPILER_RT_BUILD_XRAY=OFF"])}, {'name' : 'clang-x64-windows-msvc', @@ -499,6 +532,19 @@ script="clang-windows.py", depends_on_projects=['llvm', 'clang', 'lld', 'debuginfo-tests'])}, + {'name' : "clang-m68k-linux", + 'tags' : ["clang"], + 'workernames' : ["debian-akiko-m68k"], + 'builddir': "clang-m68k-linux", + 'factory' : ClangBuilder.getClangCMakeBuildFactory( + clean=False, + checkout_lld=False, + useTwoStage=False, + stage1_config='Release', + extra_cmake_args=[ + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=M68k"])}, + {'name' : "clang-ppc64be-linux-lnt", 'tags' : ["clang", "ppc"], 'workernames' : ["ppc64be-clang-lnt-test"], @@ -569,7 +615,9 @@ checkout_lld=False, useTwoStage=False, stage1_config='Release', - extra_cmake_args=["-DLLVM_ENABLE_ASSERTIONS=ON"])}, + extra_cmake_args=[ + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DPYTHON_EXECUTABLE:FILEPATH=/usr/local/bin/python3"])}, {'name' : "clang-ppc64le-rhel", 'tags' : ["clang", "ppc", "ppc64le"], @@ -582,6 +630,7 @@ checkout_libcxx=False, useTwoStage=False, runTestSuite=True, + cmake_test_suite=True, stage1_config='Release', nt_flags=['--threads=16', '--build-threads=16'], extra_cmake_args=[ @@ -595,6 +644,31 @@ "-DLLVM_BINUTILS_INCDIR=/usr/include", "-DBUILD_SHARED_LIBS=ON", "-DLLVM_ENABLE_WERROR=ON", '-DLLVM_LIT_ARGS=-vj 20'])}, + {'name' : "clang-ppc64-aix-ppc64le", + 'tags' : ["clang", "aix", "ppc", "ppc64le"], + 'workernames' : ["aix-ppc64-ppc64le"], + 'builddir': "clang-ppc64-aix-ppc64le", + 'factory' : ClangBuilder.getClangCMakeBuildFactory( + jobs=4, + clean=False, + checkout_clang_tools_extra=False, + checkout_compiler_rt=False, + checkout_lld=False, + checkout_libcxx=False, + useTwoStage=False, + runTestSuite=False, + stage1_config='Release', + extra_cmake_args=[ + "-DLLVM_ENABLE_ASSERTIONS=On", + "-DCMAKE_C_COMPILER=/opt/IBM/xlC/16.1.0/bin/xlclang", + "-DCMAKE_CXX_COMPILER=/opt/IBM/xlC/16.1.0/bin/xlclang++", + "-DCMAKE_C_FLAGS='-qmaxmem=-1 -qarch=pwr8 -qaltivec -qalias=noansi'", + "-DCMAKE_CXX_FLAGS='-qmaxmem=-1 -qarch=pwr8 -qaltivec -qalias=noansi'", + "-DPython3_EXECUTABLE:FILEPATH=/opt/freeware/bin/python3_64", + "-DLLVM_ENABLE_ZLIB=OFF", "-DLLVM_APPEND_VC_REV=OFF", + "-DLLVM_DEFAULT_TARGET_TRIPLE=powerpc64le-unknown-linux-gnu", + "-DLLVM_ENABLE_WERROR=ON"])}, + {'name' : "clang-s390x-linux", 'tags' : ["clang"], 'workernames' : ["systemz-1"], @@ -746,55 +820,6 @@ "-DCMAKE_CXX_FLAGS='-march=broadwell'", "-DLLVM_TARGETS_TO_BUILD='X86'"])}, - ## Armv7 build cache - {'name' : "clang-armv7-linux-build-cache", - 'tags' : ["clang"], - 'workernames' : ["packet-linux-armv7-slave-1"], - 'builddir': "clang-armv7-linux-build-cache", - 'factory' : ClangBuilder.getClangCMakeGCSBuildFactory( - stage1_config='Release', - clean=True, - checkout_compiler_rt=False, - test=False, - useTwoStage=False, - runTestSuite=False, - checkout_lld=True, - checkout_libcxx=True, - checkout_clang_tools_extra=False, - use_pixz_compression=False, - xz_compression_factor=0, - stage1_upload_directory='clang-armv7-linux', - extra_cmake_args=[ - "-DLLVM_TARGETS_TO_BUILD='ARM;AArch64;X86'", - "-DCMAKE_C_FLAGS='-mthumb'", - "-DCMAKE_CXX_FLAGS='-mthumb'", - ], - env={'BUCKET': 'llvm-build-artifacts'})}, - - ## AArch64 build cache - {'name' : "clang-aarch64-linux-build-cache", - 'tags' : ["clang"], - 'workernames' : ["packet-linux-aarch64-slave-1"], - 'builddir': "clang-aarch64-linux-build-cache", - 'factory' : ClangBuilder.getClangCMakeGCSBuildFactory( - stage1_config='Release', - clean=True, - checkout_compiler_rt=False, - test=False, - useTwoStage=False, - runTestSuite=False, - checkout_lld=True, - checkout_libcxx=True, - checkout_clang_tools_extra=False, - stage1_upload_directory='clang-aarch64-linux', - use_pixz_compression=True, - extra_cmake_args=[ - "-DLLVM_TARGETS_TO_BUILD='ARM;AArch64;X86'", - #"-DCMAKE_C_FLAGS=''", - #"-DCMAKE_CXX_FLAGS=''", - ], - env={'BUCKET': 'llvm-build-artifacts'})}, - {'name' : "llvm-avr-linux", 'tags' : ["clang"], 'workernames' : ["avr-build-01"], @@ -827,6 +852,24 @@ '-DLLVM_ENABLE_ASSERTIONS=ON', '-DLLVM_TARGETS_TO_BUILD=X86'])}, + {'name' : "clang-xcore-ubuntu-20-x64", + 'tags' : ["clang"], + 'workernames' : ["xcore-ubuntu20-x64"], + 'builddir': "clang-xcore-ubuntu-20-x64", + 'factory' : ClangBuilder.getClangCMakeBuildFactory( + jobs=4, + checkout_clang_tools_extra=False, + checkout_compiler_rt=False, + checkout_lld=False, + test=True, + testStage1=True, + useTwoStage=False, + stage1_config='Release', + extra_cmake_args=[ + "-DLLVM_TARGETS_TO_BUILD:STRING=XCore", + "-DLLVM_DEFAULT_TARGET_TRIPLE:STRING=xcore-unknown-unknown-elf", + "-DLLVM_ENABLE_THREADS:BOOL=OFF"])}, + # Polly builders. {'name' : "polly-arm-linux", @@ -1006,17 +1049,6 @@ extra_configure_args = [ '-DLLVM_ENABLE_WERROR=OFF'])}, - {'name' : "lld-x86_64-freebsd", - 'tags' : ["lld"], - 'workernames' : ["as-worker-4"], - 'builddir': "lld-x86_64-freebsd", - 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( - depends_on_projects=['llvm', 'lld'], - extra_configure_args=[ - '-DCMAKE_EXE_LINKER_FLAGS=-lcxxrt', - '-DLLVM_ENABLE_WERROR=OFF'], - env={'CXXFLAGS' : "-std=c++11 -stdlib=libc++"})}, - {'name' : "ppc64le-lld-multistage-test", 'tags' : ["lld", "ppc", "ppc64le"], 'workernames' : ["ppc64le-lld-multistage-test"], @@ -1059,11 +1091,11 @@ 'factory' : ClangLTOBuilder.getClangWithLTOBuildFactory( jobs=72, lto='thin', - extra_configure_args=[ + extra_configure_args_lto_stage=[ '-DLLVM_CCACHE_BUILD=ON', - '-DCMAKE_CXX_FLAGS="-O3 -Xclang -fwhole-program-vtables -fno-split-lto-unit"', - '-DCMAKE_C_FLAGS="-O3 -Xclang -fwhole-program-vtables -fno-split-lto-unit"', - '-DCMAKE_EXE_LINKER_FLAGS="-Wl,--lto-whole-program-visibility"'])}, + '-DCMAKE_CXX_FLAGS=-O3 -Xclang -fwhole-program-vtables -fno-split-lto-unit', + '-DCMAKE_C_FLAGS=-O3 -Xclang -fwhole-program-vtables -fno-split-lto-unit', + '-DCMAKE_EXE_LINKER_FLAGS=-Wl,--lto-whole-program-visibility -fuse-ld=lld'])}, {'name' : "clang-with-lto-ubuntu", 'tags' : ["clang","lld","LTO"], @@ -1087,10 +1119,12 @@ '-DLLVM_TARGETS_TO_BUILD=host;NVPTX', '-DLLVM_ENABLE_PROJECTS=mlir', '-DMLIR_CUDA_RUNNER_ENABLED=1', + '-DMLIR_INCLUDE_INTEGRATION_TESTS=ON', '-DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc', '-DMLIR_VULKAN_RUNNER_ENABLED=1', '-DBUILD_SHARED_LIBS=ON', '-DLLVM_CCACHE_BUILD=ON', + '-DMLIR_BINDINGS_PYTHON_ENABLED=ON', ], env={ 'CC':'clang', @@ -1195,7 +1229,10 @@ 'tags' : ["sanitizer", "ppc", "ppc64le"], 'workernames' : ["ppc64le-sanitizer"], 'builddir': "sanitizer-ppc64le", - 'factory' : SanitizerBuilder.getSanitizerBuildFactory(timeout=1800)}, + 'factory' : SanitizerBuilder.getSanitizerBuildFactory( + timeout=1800, + extra_configure_args=[ + "-CMAKE_ARGS='-DLLVM_LIT_ARGS=-v -j256'"])}, {'name' : "sanitizer-windows", 'tags' : ["sanitizer"], @@ -1205,6 +1242,8 @@ script="sanitizer-windows.py", depends_on_projects=["llvm", "clang", "lld", "compiler-rt"])}, +# OpenMP builders. + {'name' : "openmp-gcc-x86_64-linux-debian", 'tags' : ["openmp"], 'workernames' : ["gribozavr4"], @@ -1262,78 +1301,23 @@ env={'CC': 'clang', 'CXX': 'clang++'}, use_cache='Generic-singlethreaded.cmake')}, - # ARMv7 LibC++ and LibC++abi tests (require Clang+RT) - {'name' : 'libcxx-libcxxabi-libunwind-armv7-linux', - 'tags' : ["libcxx"], - 'workernames': ['linaro-tk1-02'], - 'builddir': 'libcxx-libcxxabi-libunwind-armv7-linux', - 'factory' : LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( - cmake_extra_opts={ - 'LIBCXXABI_USE_LLVM_UNWINDER': 'ON', - 'CMAKE_C_FLAGS': '-mcpu=cortex-a15 -marm', - 'CMAKE_CXX_FLAGS': '-mcpu=cortex-a15 -marm', - 'LLVM_PARALLEL_LINK_JOBS': '2'})}, - - # ARMv8 LibC++ and LibC++abi tests (require Clang+RT) - {'name' : 'libcxx-libcxxabi-libunwind-armv8-linux', - 'tags' : ["libcxx"], - 'workernames' : ['linaro-armv8-libcxx'], - 'builddir': 'libcxx-libcxxabi-libunwind-armv8-linux', - 'factory' : LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( - cmake_extra_opts={ - 'LIBCXXABI_USE_LLVM_UNWINDER': 'ON', - 'CMAKE_C_FLAGS': '-mcpu=cortex-a57 -marm', - 'CMAKE_CXX_FLAGS': '-mcpu=cortex-a57 -marm'})}, - - # ARMv7 LibC++ and LibC++abi tests w/o EH (require Clang+RT) - {'name' : 'libcxx-libcxxabi-libunwind-armv7-linux-noexceptions', - 'tags' : ["libcxx"], - 'workernames' : ['linaro-tk1-02'], - 'builddir': 'libcxx-libcxxabi-libunwind-armv7-linux-noexceptions', - 'factory' : LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( - cmake_extra_opts={ - 'LIBCXXABI_USE_LLVM_UNWINDER': 'ON', - 'LIBCXX_ENABLE_EXCEPTIONS': 'OFF', - 'LIBCXXABI_ENABLE_EXCEPTIONS': 'OFF', - 'CMAKE_C_FLAGS': '-mcpu=cortex-a15 -mthumb', - 'CMAKE_CXX_FLAGS': '-mcpu=cortex-a15 -mthumb', - 'LLVM_PARALLEL_LINK_JOBS': '2'})}, - - # ARMv8 LibC++ and LibC++abi tests w/o EH (require Clang+RT) - {'name' : 'libcxx-libcxxabi-libunwind-armv8-linux-noexceptions', - 'tags' : ["libcxx"], - 'workernames' : ['linaro-armv8-libcxx'], - 'builddir': 'libcxx-libcxxabi-libunwind-armv8-linux-noexceptions', - 'factory' : LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( - cmake_extra_opts={ - 'LIBCXXABI_USE_LLVM_UNWINDER': 'ON', - 'LIBCXX_ENABLE_EXCEPTIONS': 'OFF', - 'LIBCXXABI_ENABLE_EXCEPTIONS': 'OFF', - 'CMAKE_C_FLAGS': '-mcpu=cortex-a57 -mthumb', - 'CMAKE_CXX_FLAGS': '-mcpu=cortex-a57 -mthumb'})}, - - # AArch64 LibC++ and LibC++abi tests (require Clang+RT) - {'name' : 'libcxx-libcxxabi-libunwind-aarch64-linux', - 'tags' : ["libcxx"], - 'workernames' : ['linaro-aarch64-libcxx'], - 'builddir': 'libcxx-libcxxabi-libunwind-aarch64-linux', - 'factory' : LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( - cmake_extra_opts={ - 'LIBCXXABI_USE_LLVM_UNWINDER': 'ON', - 'CMAKE_C_FLAGS': '-mcpu=cortex-a57', - 'CMAKE_CXX_FLAGS': '-mcpu=cortex-a57'})}, - - {'name' : 'libcxx-libcxxabi-libunwind-aarch64-linux-noexceptions', + {'name' : 'libcxx-libcxxabi-libunwind-ppc-aix', 'tags' : ["libcxx"], - 'workernames' : ['linaro-aarch64-libcxx'], - 'builddir': 'libcxx-libcxxabi-libunwind-aarch64-linux-noexceptions', + 'workernames' : ['aix-ppc-libcxx'], + 'builddir': 'libcxx-libcxxabi-libunwind-ppc-aix', 'factory' : LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( + env={'OBJECT_MODE': '32'}, cmake_extra_opts={ - 'LIBCXXABI_USE_LLVM_UNWINDER': 'ON', - 'LIBCXX_ENABLE_EXCEPTIONS': 'OFF', - 'LIBCXXABI_ENABLE_EXCEPTIONS': 'OFF', - 'CMAKE_C_FLAGS': '-mcpu=cortex-a57', - 'CMAKE_CXX_FLAGS': '-mcpu=cortex-a57'})}, + 'CMAKE_BUILD_TYPE': 'Release', + 'CMAKE_C_COMPILER': 'gcc', + 'CMAKE_CXX_COMPILER': 'g++', + 'CMAKE_AR': '/usr/bin/ar', + 'LIBCXX_CXX_ABI': 'libstdc++', + 'LIBCXX_CXX_ABI_INCLUDE_PATHS': '/opt/freeware/lib/gcc/powerpc-ibm-aix7.2.0.0/8/include/c++;/opt/freeware/lib/gcc/powerpc-ibm-aix7.2.0.0/8/include/c++/powerpc-ibm-aix7.2.0.0', + 'CMAKE_THREAD_LIBS_INIT': '-lpthread', + 'LIBCXX_ENABLE_STATIC:BOOL': 'OFF', + 'LIBCXX_ENABLE_ABI_LINKER_SCRIPT': 'OFF'}, + depends_on_projects=['libcxx'])}, {'name': "fuchsia-x86_64-linux", 'tags' : ["toolchain"], @@ -1343,6 +1327,15 @@ # libc Builders. + {'name' : 'libc-aarch64-ubuntu-dbg', + 'tags' : ["libc"], + 'workernames' : ['libc-aarch64-ubuntu'], + 'builddir': 'libc-aarch64-ubuntu', + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="libc-linux.py", + depends_on_projects=['llvm', 'libc', 'clang', 'clang-tools-extra'], + extra_args=['--debug'])}, + {'name' : 'libc-x86_64-debian', 'tags' : ["libc"], 'workernames' : ['libc-x86_64-debian'], @@ -1369,6 +1362,24 @@ depends_on_projects=['llvm', 'libc', 'clang', 'clang-tools-extra'], extra_args=['--debug', '--asan'])}, + {'name' : "libc-x86_64-debian-fullbuild-dbg", + 'tags' : ["libc"], + 'workernames' : ["libc-x86_64-debian-fullbuild"], + 'builddir': "libc-x86_64-debian-fullbuild-dbg", + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="libc-linux.py", + depends_on_projects=['llvm', 'libc', 'clang', 'clang-tools-extra'], + extra_args=['--debug'])}, + + {'name' : "libc-x86_64-debian-fullbuild-dbg-asan", + 'tags' : ["libc"], + 'workernames' : ["libc-x86_64-debian-fullbuild"], + 'builddir': "libc-x86_64-debian-fullbuild-dbg-asan", + 'factory' : AnnotatedBuilder.getAnnotatedBuildFactory( + script="libc-linux.py", + depends_on_projects=['llvm', 'libc', 'clang', 'clang-tools-extra'], + extra_args=['--debug', '--asan'])}, + # Flang builders. {'name' : "flang-aarch64-ubuntu", @@ -1393,6 +1404,7 @@ 'builddir': "flang-aarch64-ubuntu-dylib", 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( clean=True, + checks=['check-flang'], depends_on_projects=['llvm','mlir','clang','flang'], extra_configure_args=[ "-DLLVM_TARGETS_TO_BUILD=AArch64", @@ -1408,6 +1420,7 @@ 'builddir': "flang-aarch64-ubuntu-sharedlibs", 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( clean=True, + checks=['check-flang'], depends_on_projects=['llvm','mlir','clang','flang'], extra_configure_args=[ "-DLLVM_TARGETS_TO_BUILD=AArch64", @@ -1422,6 +1435,25 @@ 'workernames' : ["linaro-aarch64-flang-oot"], 'builddir': "flang-aarch64-out-of-tree", 'factory' : FlangBuilder.getFlangOutOfTreeBuildFactory( + checks=['check-flang'], + llvm_extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=AArch64", + "-DCMAKE_CXX_STANDARD=17", + "-DLLVM_ENABLE_WERROR=OFF", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DCMAKE_BUILD_TYPE=Release", + ], + flang_extra_configure_args=[ + "-DFLANG_ENABLE_WERROR=ON", + "-DCMAKE_BUILD_TYPE=Release", + ])}, + + {'name' : "flang-aarch64-ubuntu-out-of-tree-new-driver", + 'tags' : ["flang"], + 'workernames' : ["linaro-aarch64-flang-oot-new-driver"], + 'builddir': "flang-aarch64-out-of-tree-new-driver", + 'factory' : FlangBuilder.getFlangOutOfTreeBuildFactory( + checks=['check-flang'], llvm_extra_configure_args=[ "-DLLVM_TARGETS_TO_BUILD=AArch64", "-DCMAKE_CXX_STANDARD=17", @@ -1432,6 +1464,95 @@ flang_extra_configure_args=[ "-DFLANG_ENABLE_WERROR=ON", "-DCMAKE_BUILD_TYPE=Release", + "-DFLANG_BUILD_NEW_DRIVER=ON", + ])}, + + {'name' : "flang-aarch64-ubuntu-debug", + 'tags' : ["flang"], + 'workernames' : ["linaro-aarch64-flang-debug"], + 'builddir': "flang-aarch64-debug", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + clean=True, + checks=['check-flang'], + depends_on_projects=['llvm','mlir','clang','flang'], + extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=AArch64", + "-DFLANG_BUILD_NEW_DRIVER=ON", + "-DCMAKE_BUILD_TYPE=Debug", + "-DCMAKE_CXX_STANDARD=17", + "-DLLVM_USE_LINKER=lld", + ])}, + + {'name' : "flang-aarch64-ubuntu-latest-clang", + 'tags' : ['flang'], + 'workernames' : ["linaro-aarch64-flang-latest-clang"], + 'builddir': "flang-aarch64-ubuntu-latest-clang", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + clean=True, + checks=['check-flang'], + depends_on_projects=['llvm','mlir','clang','flang'], + extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=AArch64", + "-DLLVM_INSTALL_UTILS=ON", + "-DCMAKE_CXX_STANDARD=17", + "-DLLVM_ENABLE_WERROR=OFF", + "-DFLANG_ENABLE_WERROR=ON", + "-DBUILD_SHARED_LIBS=ON", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_ENABLE_LIBCXX=On", + "-DFLANG_BUILD_NEW_DRIVER=ON", + "-DCMAKE_BUILD_TYPE=Release", + ])}, + + {'name' : "flang-aarch64-ubuntu-release", + 'tags' : ["flang"], + 'workernames' : ["linaro-aarch64-flang-release"], + 'builddir': "flang-aarch64-release", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + clean=True, + checks=['check-flang'], + depends_on_projects=['llvm','mlir','clang','flang'], + extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=AArch64", + "-DFLANG_BUILD_NEW_DRIVER=ON", + "-DCMAKE_BUILD_TYPE=Release", + "-DCMAKE_CXX_STANDARD=17", + ])}, + + {'name' : "flang-aarch64-ubuntu-rel-assert", + 'tags' : ["flang"], + 'workernames' : ["linaro-aarch64-flang-rel-assert"], + 'builddir': "flang-aarch64-rel", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + clean=True, + checks=['check-flang'], + depends_on_projects=['llvm','mlir','clang','flang'], + extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=AArch64", + "-DFLANG_BUILD_NEW_DRIVER=ON", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DCMAKE_BUILD_TYPE=Release", + "-DCMAKE_CXX_STANDARD=17", + ])}, + + {'name' : "flang-aarch64-latest-gcc", + 'tags' : ['flang'], + 'workernames' : ["linaro-aarch64-flang-latest-gcc"], + 'builddir': "flang-aarch64-latest-gcc", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + clean=True, + checks=['check-flang'], + depends_on_projects=['llvm','mlir','clang','flang'], + extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=AArch64", + "-DLLVM_INSTALL_UTILS=ON", + "-DCMAKE_CXX_STANDARD=17", + "-DLLVM_ENABLE_WERROR=OFF", + "-DFLANG_ENABLE_WERROR=ON", + "-DBUILD_SHARED_LIBS=ON", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DFLANG_BUILD_NEW_DRIVER=ON", + "-DCMAKE_BUILD_TYPE=Release", ])}, {'name' : "flang-x86_64-linux", @@ -1528,6 +1649,18 @@ 'builddir': "publish-sphinx-docs", 'factory' : SphinxDocsBuilder.getLLVMDocsBuildFactory(clean=True)}, + {'name' : "publish-lnt-sphinx-docs", + 'tags' : ["doc"], + 'workernames' : ["as-worker-4"], + 'builddir': "publish-lnt-sphinx-docs", + 'factory' : HtmlDocsBuilder.getHtmlDocsBuildFactory()}, + + {'name' : "publish-doxygen-docs", + 'tags' : ["doc"], + 'workernames' : ["as-worker-4"], + 'builddir': "publish-doxygen-docs", + 'factory' : DoxygenDocsBuilder.getLLVMDocsBuildFactory()}, + # CUDA builders. {'name' : "clang-cuda-k80", @@ -1556,14 +1689,20 @@ {'name' : "clang-ve-ninja", 'tags' : ["clang"], - 'workernames':["nec-arrproto41"], + 'workernames':["hpce-aurora2"], 'builddir':"clang-ve-ninja", 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( clean=True, depends_on_projects=['llvm','clang','openmp'], extra_configure_args=[ + "-C=~/llvm_ve_caches/llvm_ve_runtimes.cmake", "-DLLVM_TARGETS_TO_BUILD=X86", "-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=VE", + "-DLLVM_ENABLE_RUNTIMES=compiler-rt", + "-DLLVM_BUILD_LLVM_DYLIB=On", + "-DBUILD_SHARED_LIBS=Off", + "-DLLVM_LINK_LLVM_DYLIB=On", + "-DCLANG_LINK_CLANG_DYLIB=On", ])}, # Latest stable fedora running on Red Hat internal OpenShift cluster (PSI). @@ -1722,7 +1861,7 @@ "-DCMAKE_BUILD_TYPE=Release", "-DLLVM_CCACHE_BUILD=ON", "-DLLVM_ENABLE_ASSERTIONS=ON", - "-DTENSORFLOW_API_PATH=/tmp/tensorflow", + "-DTENSORFLOW_C_LIB_PATH=/tmp/tensorflow", "-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON" ])}, @@ -1739,7 +1878,7 @@ "-DCMAKE_BUILD_TYPE=Release", "-DLLVM_CCACHE_BUILD=ON", "-DLLVM_ENABLE_ASSERTIONS=ON", - "-DTENSORFLOW_API_PATH=/tmp/tensorflow", + "-DTENSORFLOW_C_LIB_PATH=/tmp/tensorflow", "-DTENSORFLOW_AOT_PATH=/var/lib/buildbot/.local/lib/python3.7/site-packages/tensorflow", "-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON" ])}, @@ -1782,5 +1921,15 @@ "-DGRPC_INSTALL_PATH=/usr/local/lib/grpc" ])}, - + {'name': "ziglang-carbon", + 'tags': ["ziglang"], + 'workernames': ["ziglang-carbon-linux-x86_64"], + 'builddir': "ziglang-carbon-root", + 'factory': ZigBuilder.getZiglangToolchainBuildFactory( + clean=True, + test=True, + env={ + 'CC': 'clang', 'CXX': 'clang++', 'LD': 'lld', + } + )} ] diff --git a/buildbot/osuosl/master/config/release_builders.py b/buildbot/osuosl/master/config/release_builders.py new file mode 100644 --- /dev/null +++ b/buildbot/osuosl/master/config/release_builders.py @@ -0,0 +1,232 @@ +from importlib import reload + +from buildbot.process.properties import WithProperties + +from zorg.buildbot.builders import ClangBuilder +from zorg.buildbot.builders import FlangBuilder +from zorg.buildbot.builders import PollyBuilder +from zorg.buildbot.builders import LLDBBuilder +from zorg.buildbot.builders import SanitizerBuilder +from zorg.buildbot.builders import OpenMPBuilder +from zorg.buildbot.builders import LibcxxAndAbiBuilder +from zorg.buildbot.builders import SphinxDocsBuilder +from zorg.buildbot.builders import ABITestsuitBuilder +from zorg.buildbot.builders import ClangLTOBuilder +from zorg.buildbot.builders import UnifiedTreeBuilder +from zorg.buildbot.builders import AOSPBuilder +from zorg.buildbot.builders import AnnotatedBuilder +from zorg.buildbot.builders import LLDPerformanceTestsuite +from zorg.buildbot.builders import FuchsiaBuilder +from zorg.buildbot.builders import XToolchainBuilder + + +# Release builders. + +all = [ + +# Clang builders. + + {'name' : "llvm-clang-x86_64-win-release", + 'tags' : ["clang", "release"], + 'workernames' : ["as-builder-3"], + 'builddir': "llvm-clang-x86_64-win-rel", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaWithMSVCBuildFactory( + vs="autodetect", + depends_on_projects=['llvm', 'clang'], + clean=True, + checks=[ + "check-llvm-unit", + "check-clang-unit"], + extra_configure_args=[ + "-DLLVM_ENABLE_WERROR=OFF", + "-DLLVM_TARGETS_TO_BUILD=ARM", + "-DLLVM_DEFAULT_TARGET_TRIPLE=armv7-unknown-linux-eabihf", + "-DLLVM_ENABLE_ASSERTIONS=OFF", + "-DLLVM_OPTIMIZED_TABLEGEN=OFF", + "-DLLVM_LIT_ARGS=-v --threads=32"])}, + +# Expensive checks builders. + + + {'name' : "llvm-clang-x86_64-expensive-checks-ubuntu-release", + 'tags' : ["llvm", "expensive-checks", "release"], + 'workernames' : ["as-builder-4"], + 'builddir': "llvm-clang-x86_64-expensive-checks-ubuntu-rel", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + depends_on_projects=["llvm"], + clean=True, + extra_configure_args=[ + "-DLLVM_ENABLE_EXPENSIVE_CHECKS=ON", + "-DLLVM_ENABLE_WERROR=OFF", + "-DCMAKE_BUILD_TYPE=Debug", + "-DCMAKE_CXX_FLAGS=-U_GLIBCXX_DEBUG", + "-DLLVM_LIT_ARGS=-vv -j32"])}, + + {'name' : "llvm-clang-x86_64-expensive-checks-win-release", + 'tags' : ["llvm", "expensive-checks", "release"], + 'workernames' : ["as-worker-93"], + 'builddir': "llvm-clang-x86_64-expensive-checks-win-rel", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaWithMSVCBuildFactory( + vs="autodetect", + depends_on_projects=["llvm"], + clean=True, + extra_configure_args=[ + "-DLLVM_ENABLE_EXPENSIVE_CHECKS=ON", + "-DLLVM_ENABLE_WERROR=OFF", + "-DCMAKE_BUILD_TYPE=Debug"])}, + + {'name' : "llvm-clang-x86_64-expensive-checks-debian-release", + 'tags' : ["llvm", "expensive-checks", "release"], + 'workernames' : ["gribozavr4"], + 'builddir': "llvm-clang-x86_64-expensive-checks-debian-rel", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + depends_on_projects=["llvm"], + clean=True, + extra_configure_args=[ + "-DLLVM_ENABLE_EXPENSIVE_CHECKS=ON", + "-DLLVM_ENABLE_WERROR=OFF", + "-DCMAKE_BUILD_TYPE=Release", + "-DCMAKE_CXX_FLAGS=-U_GLIBCXX_DEBUG", + "-DLLVM_LIT_ARGS=-v -vv -j96"], + env={ + 'PATH':'/home/llvmbb/bin/clang-latest/bin:/home/llvmbb/bin:/usr/local/bin:/usr/local/bin:/usr/bin:/bin', + 'CC': 'ccache clang', 'CXX': 'ccache clang++', 'CCACHE_CPP2': 'yes', + })}, + +# Cross builders. + + {'name' : "llvm-clang-win-x-armv7l-release", + 'tags' : ["clang", "llvm", "compiler-rt", "cross"," armv7l", "release"], + 'workernames' : ["as-builder-1"], + 'builddir': "x-armv7l-rel", + 'factory' : XToolchainBuilder.getCmakeWithMSVCBuildFactory( + vs="autodetect", + clean=True, + checks=[ + "check-llvm", + "check-clang", + "check-lld", + "check-compiler-rt" + ], + checks_on_target = [ + ("libunwind", + ["python", "bin/llvm-lit.py", + "-v", "-vv", "--threads=32", + "runtimes/runtimes-bins/libunwind/test"]), + ("libc++abi", + ["python", "bin/llvm-lit.py", + "-v", "-vv", "--threads=32", + "runtimes/runtimes-bins/libcxxabi/test"]), + ("libc++", + ['python', 'bin/llvm-lit.py', + '-v', '-vv', '--threads=32', + 'runtimes/runtimes-bins/libcxx/test', + ]) + ], + extra_configure_args=[ + "-DDEFAULT_SYSROOT=C:/buildbot/.arm-ubuntu", + "-DLLVM_LIT_ARGS=-v -vv --threads=32", + WithProperties("%(remote_test_host:+-DREMOTE_TEST_HOST=)s%(remote_test_host:-)s"), + WithProperties("%(remote_test_user:+-DREMOTE_TEST_USER=)s%(remote_test_user:-)s"), + ], + cmake_cache="../llvm-project/clang/cmake/caches/CrossWinToARMLinux.cmake")}, + + {'name' : "llvm-clang-win-x-aarch64-release", + 'tags' : ["clang", "llvm", "compiler-rt", "cross", "aarch64", "release"], + 'workernames' : ["as-builder-2"], + 'builddir': "x-aarch64-rel", + 'factory' : XToolchainBuilder.getCmakeWithMSVCBuildFactory( + vs="autodetect", + clean=True, + checks=[ + "check-llvm", + "check-clang", + "check-lld", + "check-compiler-rt" + ], + checks_on_target = [ + ("libunwind", + ["python", "bin/llvm-lit.py", + "-v", "-vv", "--threads=32", + "runtimes/runtimes-bins/libunwind/test"]), + ("libc++abi", + ["python", "bin/llvm-lit.py", + "-v", "-vv", "--threads=32", + "runtimes/runtimes-bins/libcxxabi/test"]), + ("libc++", + ['python', 'bin/llvm-lit.py', + '-v', '-vv', '--threads=32', + 'runtimes/runtimes-bins/libcxx/test', + ]) + ], + extra_configure_args=[ + "-DLLVM_TARGETS_TO_BUILD=AArch64", + "-DCMAKE_C_COMPILER_TARGET=aarch64-linux-gnu", + "-DDEFAULT_SYSROOT=C:/buildbot/.aarch64-ubuntu", + "-DLLVM_LIT_ARGS=-v -vv --threads=32", + WithProperties("%(remote_test_host:+-DREMOTE_TEST_HOST=)s%(remote_test_host:-)s"), + WithProperties("%(remote_test_user:+-DREMOTE_TEST_USER=)s%(remote_test_user:-)s"), + ], + cmake_cache="../llvm-project/clang/cmake/caches/CrossWinToARMLinux.cmake")}, + +# LLD builders. + + {'name' : "lld-x86_64-win-release", + 'tags' : ["lld", "release"], + 'workernames' : ["as-worker-93"], + 'builddir': "lld-x86_64-win-rel", + 'factory' : UnifiedTreeBuilder.getCmakeWithNinjaWithMSVCBuildFactory( + depends_on_projects=['llvm', 'lld'], + vs="autodetect", + extra_configure_args = [ + '-DLLVM_ENABLE_WERROR=OFF'])}, + + {'name' : "lld-x86_64-ubuntu-release", + 'tags' : ["lld", "release"], + 'workernames' : ["as-builder-4"], + 'builddir' : "lld-x86_64-ubuntu-rel", + 'factory': UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + clean=True, + extra_configure_args=[ + '-DLLVM_ENABLE_WERROR=OFF'], + depends_on_projects=['llvm', 'lld'])}, + +# LTO and ThinLTO builders. + + {'name' : "clang-with-thin-lto-ubuntu-release", + 'tags' : ["clang", "lld", "LTO", "release"], + 'workernames' : ["as-worker-92"], + 'builddir': "clang-with-thin-lto-ubuntu-rel", + 'factory' : ClangLTOBuilder.getClangWithLTOBuildFactory(jobs=72, lto='thin')}, + + {'name' : "clang-with-lto-ubuntu-release", + 'tags' : ["clang", "lld", "LTO", "release"], + 'workernames' : ["as-worker-91"], + 'builddir': "clang-with-lto-ubuntu-rel", + 'factory' : ClangLTOBuilder.getClangWithLTOBuildFactory(jobs=72)}, + + +# Libc++ builders. + + {'name': 'libcxx-libcxxabi-x86_64-linux-debian-release', + 'tags' : ["libcxx", "release"], + 'workernames': ['gribozavr4'], + 'builddir': 'libcxx-libcxxabi-x86_64-linux-debian-rel', + 'factory': LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder( + env={'CC': 'clang', 'CXX': 'clang++'}, + lit_extra_args=['--shuffle'], + check_libcxx_abilist=True)}, + +# OpenMP builders. + + {'name' : "openmp-clang-x86_64-linux-debian-release", + 'tags' : ["openmp", "release"], + 'workernames' : ["gribozavr4"], + 'builddir': "openmp-clang-x86_64-linux-debian-rel", + 'factory' : OpenMPBuilder.getOpenMPCMakeBuildFactory( + env={ + 'PATH':'/home/llvmbb/bin/clang-latest/bin:/home/llvmbb/bin:/usr/local/bin:/usr/local/bin:/usr/bin:/bin', + 'CC': 'ccache clang', 'CXX': 'ccache clang++', 'CCACHE_CPP2': 'yes', + })}, + +] diff --git a/buildbot/osuosl/master/config/schedulers.py b/buildbot/osuosl/master/config/schedulers.py --- a/buildbot/osuosl/master/config/schedulers.py +++ b/buildbot/osuosl/master/config/schedulers.py @@ -19,7 +19,7 @@ # Since we have many parametric builders, we dynamically build the minimum set # of schedulers, which covers all actually used combinations of dependencies. -def getSingleBranchSchedulers( +def getMainBranchSchedulers( builders, explicitly_set_schedulers = None, **kwargs): @@ -43,9 +43,10 @@ builder for builder in builders if builder.name not in builders_with_explicit_schedulers if getattr(builder.factory, 'depends_on_projects', None) + if 'release' not in getattr(builder, 'tags', []) ] - filter_branch = 'master' + filter_branch = 'main' treeStableTimer = kwargs.get('treeStableTimer', None) automatic_schedulers = [] @@ -60,7 +61,7 @@ ]) for projects in set_of_dependencies: - sch_builders = [ + main_builders = [ b.name for b in builders_with_automatic_schedulers if frozenset(getattr(b.factory, 'depends_on_projects')) == projects @@ -71,9 +72,9 @@ automatic_schedulers.append( schedulers.SingleBranchScheduler( name=automatic_scheduler_name, - treeStableTimer=kwargs.get('treeStableTimer', None), - reason="Merge to github %s branch" % filter_branch, - builderNames=sch_builders, + treeStableTimer=treeStableTimer, + reason="Merge to github {} branch".format(filter_branch), + builderNames=main_builders, change_filter=util.ChangeFilter( project_fn= \ lambda c, projects_of_interest=frozenset(projects): @@ -83,18 +84,90 @@ ) log.msg( - "Generated SingleBranchScheduler: { name='%s'" % automatic_scheduler_name, - ", builderNames=", sch_builders, - ", change_filter=", projects, " (branch: %s)" % filter_branch, - ", treeStableTimer=%s" % treeStableTimer, + "Generated SingleBranchScheduler: {{ name='{}'".format(automatic_scheduler_name), + ", builderNames=", main_builders, + ", change_filter=", projects, " (branch: {})".format(filter_branch), + ", treeStableTimer={}".format(treeStableTimer), "}") return automatic_schedulers -# TODO: Move these settings to the configuration file. -_repourl = "https://github.com/llvm/llvm-project" -_branch = "master" +def getReleaseBranchSchedulers( + builders, + explicitly_set_schedulers = None, + **kwargs): + """ + I'm taking over all of not yet assigned builders with the + declared source code dependencies and the 'release' tag, + and automatically generate a minimum set of SingleBranchSchedulers + to handle all the declared source code dependency combinations. + """ + + builders_with_explicit_schedulers = set() + if explicitly_set_schedulers: + # TODO: Make a list of builder names with already set schedulers. + # builders_with_explicit_schedulers.add(builder) + pass + + # For the builders created with LLVMBuildFactory or similar, + # we always use automatic schedulers, + # unless schedulers already explicitly set. + builders_with_automatic_schedulers = [ + builder for builder in builders + if builder.name not in builders_with_explicit_schedulers + if getattr(builder.factory, 'depends_on_projects', None) + if 'release' in getattr(builder, 'tags', []) + ] + + treeStableTimer = kwargs.get('treeStableTimer', None) + + automatic_schedulers = [] + + # Do we have any to take care of? + if builders_with_automatic_schedulers: + # Let's reconsile first to get a unique set of dependencies. + # We need a set of unique sets of dependent projects. + set_of_dependencies = set([ + frozenset(getattr(b.factory, 'depends_on_projects')) + for b in builders_with_automatic_schedulers + ]) + + for projects in set_of_dependencies: + release_builders = [ + b.name + for b in builders_with_automatic_schedulers + if frozenset(getattr(b.factory, 'depends_on_projects')) == projects + ] + + automatic_scheduler_name = "release:" + ",".join(sorted(projects)) + + automatic_schedulers.append( + schedulers.SingleBranchScheduler( + name=automatic_scheduler_name, + treeStableTimer=treeStableTimer, + reason="Merge to github release branch", + builderNames=release_builders, + change_filter=util.ChangeFilter( + project_fn= \ + lambda c, projects_of_interest=frozenset(projects): + isProjectOfInterest(c, projects_of_interest), + branch_fn= \ + lambda branch: branch.startswith('release/')) + ) + ) + + log.msg( + "Generated release SingleBranchScheduler: {{ name='{}'".format(automatic_scheduler_name), + ", builderNames=", release_builders, + ", change_filter=", projects, " (branch: {release/*})", + ", treeStableTimer={}".format(treeStableTimer), + "}") + return automatic_schedulers def getForceSchedulers(builders): + # TODO: Move these settings to the configuration file. + _repourl = "https://github.com/llvm/llvm-project" + _branch = "main" + # Walk over all builders and collect their names. scheduler_builders = [ builder.name for builder in builders @@ -120,8 +193,10 @@ codebases = [ util.CodebaseParameter( codebase = "", - branch = util.FixedParameter( + branch = util.StringParameter( name = "branch", + label = "branch:", + size = 64, default = _branch ), revision = util.StringParameter( @@ -157,19 +232,21 @@ # TODO: Abstract this kind of scheduler better. def getLntSchedulers(): - project = "lnt" - lnt_builders = [ + _project = "lnt" + _branch = 'master' + _repourl = "https://github.com/llvm/llvm-lnt" + _lnt_builders = [ "publish-lnt-sphinx-docs", ] return [ schedulers.SingleBranchScheduler( name="lnt-scheduler", treeStableTimer=kwargs.get('treeStableTimer', None), - reason="Merge to LNT github {} branch".format(filter_branch), - builderNames=lnt_builders, + reason="Merge to LNT github {} branch".format(_branch), + builderNames=_lnt_builders, change_filter=util.ChangeFilter( - project_fn=project, - branch=filter_branch)), + project_fn=_project, + branch=_branch)), schedulers.ForceScheduler( name = "force-build-scheduler", @@ -186,7 +263,7 @@ ], default = "Build a particular revision" ), - builderNames = lnt_builders, + builderNames = _lnt_builders, codebases = [ util.CodebaseParameter( codebase = "", @@ -206,7 +283,7 @@ ), project = util.FixedParameter( name = "project", - default = project + default = _project ) ) ], diff --git a/buildbot/osuosl/master/config/status.py b/buildbot/osuosl/master/config/status.py --- a/buildbot/osuosl/master/config/status.py +++ b/buildbot/osuosl/master/config/status.py @@ -9,18 +9,22 @@ all = [ - # Note: reporters.GitHubStatusPush requires txrequests package to allow - # interaction with GitHub REST API. + # Report github status for all the release builders, + # i.e. those with the "release" tag. reporters.GitHubStatusPush( str(config.options.get('GitHub Status', 'token')), context = Interpolate("%(prop:buildername)s"), verbose = True, # TODO: Turn off the verbosity once this is working reliably. builders = [ - "llvm-clang-x86_64-expensive-checks-ubuntu", - "llvm-clang-x86_64-win-fast", - "clang-x86_64-debian-fast", - "llvm-clang-x86_64-expensive-checks-debian", - ]), + "llvm-clang-x86_64-expensive-checks-ubuntu", + "llvm-clang-x86_64-win-fast", + "clang-x86_64-debian-fast", + "llvm-clang-x86_64-expensive-checks-debian", + ] + [ + b.get('name') for b in config.release_builders.all + if 'release' in b.get('tags', []) + ] + ), reporters.IRC( useColors = False, @@ -44,15 +48,6 @@ dumpMailsToLog = True, ), - # In addition to that the following notifiers are defined for special - # cases. - reporters.MailNotifier( - fromaddr = "llvm.buildmaster@lab.llvm.org", - sendToInterestedUsers = False, - extraRecipients = ["leandro.nunes@arm.com"], - subject = "Build %(builder)s Failure", - mode = "failing", - builders = ["clang-aarch64-linux-build-cache", "clang-armv7-linux-build-cache"]), reporters.MailNotifier( fromaddr = "llvm.buildmaster@lab.llvm.org", sendToInterestedUsers = False, @@ -141,8 +136,9 @@ reporters.MailNotifier( fromaddr = "llvm.buildmaster@lab.llvm.org", sendToInterestedUsers = False, - extraRecipients = ["stilis@microsoft.com", "jonas@devlieghere.com", - "diprou@microsoft.com", "makudrya@microsoft.com"], + extraRecipients = ["stilis@microsoft.com", + "diprou@microsoft.com", + "makudrya@microsoft.com"], subject = "Build %(builder)s Failure", mode = "failing", builders = ["lldb-x64-windows-ninja"]), @@ -211,7 +207,9 @@ subject = "Build %(builder)s Failure", mode = "failing", builders = ["libc-x86_64-debian", "libc-x86_64_debian-dbg", - "libc-x86_64-debian-dbg-asan"]), + "libc-x86_64-debian-dbg-asan", "libc-aarch64-ubuntu-dbg", + "libc-x86_64-debian-fullbuild-dbg", + "libc-x86_64-debian-fullbuild-dbg-asan"]), reporters.MailNotifier( fromaddr = "llvm.buildmaster@lab.llvm.org", sendToInterestedUsers = False, @@ -246,15 +244,36 @@ subject = "ThinLTO WPD Failure: %(builder)s", mode = "failing", builders = ["thinlto-x86-64-bot1"]), - + reporters.MailNotifier( + fromaddr = "llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers = False, + extraRecipients = ["douglas.yung@sony.com"], + subject = "Build %(builder)s Failure", + mode = "failing", + builders = ["llvm-clang-x86_64-sie-ubuntu-fast"]), + reporters.MailNotifier( + fromaddr="llvm.buildmaster@lab.llvm.org", + sendToInterestedUsers = False, + extraRecipients=[ + "mlir-bugs-external+buildbot@googlegroups.com"], + subject = "MLIR Build Failure: %(builder)s", + mode = "failing", + builders = ["mlir-nvidia", + "mlir-windows", + "ppc64le-mlir-rhel-clang"]), ] + +from twisted.python import log + # Returns a list of Status Targets. The results of each build will be # pushed to these targets. buildbot.plugins reporters has a variety # to choose from, including email senders, and IRC bots. def getReporters(): if config.options.getboolean('Master Options', 'is_production'): + log.msg(">>> getReporters: Production mode. All reporters are registered.") return all else: # Staging buildbot does not report issues. + log.msg(">>> getReporters: Staging buildbot does not report issues.") return [] diff --git a/buildbot/osuosl/master/config/workers.py b/buildbot/osuosl/master/config/workers.py --- a/buildbot/osuosl/master/config/workers.py +++ b/buildbot/osuosl/master/config/workers.py @@ -45,8 +45,14 @@ create_worker("linaro-aarch64-global-isel", properties={'jobs' : 32}, max_builds=1), create_worker("linaro-aarch64-lld", properties={'jobs' : 32}, max_builds=1), create_worker("linaro-aarch64-flang-oot", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-aarch64-flang-oot-new-driver", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-aarch64-flang-debug", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-aarch64-flang-latest-clang", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-aarch64-flang-release", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-aarch64-flang-rel-assert", properties={'jobs' : 32}, max_builds=1), create_worker("linaro-aarch64-flang-dylib", properties={'jobs' : 32}, max_builds=1), create_worker("linaro-aarch64-flang-sharedlibs", properties={'jobs' : 32}, max_builds=1), + create_worker("linaro-aarch64-flang-latest-gcc", properties={'jobs' : 32}, max_builds=1), # Libcxx testsuite has tests with timing assumptions. Run single-threaded to make # sure we have plenty CPU cycle to satisfy timing assumptions. create_worker("linaro-aarch64-libcxx", properties={'jobs' : 1}, max_builds=1), @@ -56,18 +62,15 @@ create_worker("linaro-armv8-windows-msvc-01", properties={'jobs' : 8}, max_builds=1), create_worker("linaro-armv8-windows-msvc-02", properties={'jobs' : 8}, max_builds=1), - # ARMv7 build cache workers - create_worker("packet-linux-armv7-slave-1", properties={'jobs' : 64}, max_builds=1), - - # AArch64 build cache worker - create_worker("packet-linux-aarch64-slave-1", properties={'jobs' : 64}, max_builds=1), - # Windows Server 2016 Intel Xeon(R) Quad 2.30 GHz, 56GB of RAM create_worker("win-py3-buildbot", properties={'jobs' : 64}, max_builds=1), # Windows Server 2016 Intel(R) Xeon(R) CPU @ 2.60GHz, 16 Core(s), 128 GB of RAM create_worker("win-mlir-buildbot", properties={'jobs' : 64}, max_builds=1), + # Motorola 68k 32-bit big endian (m68k) + create_worker("debian-akiko-m68k", properties={'jobs': 1}, max_builds=1), + # POWER7 PowerPC big endian (powerpc64) create_worker("ppc64be-clang-test", properties={'jobs': 16}, max_builds=1), create_worker("ppc64be-clang-lnt-test", properties={'jobs': 16, 'vcs_protocol': 'https'}, max_builds=1), @@ -123,6 +126,10 @@ # Debian 7.7 x86_64 GCE instance create_worker("sanitizer-buildbot8", properties={'jobs': 64}, max_builds=3), + # POWER 8 PowerPC AIX 7.2 + create_worker("aix-ppc64-ppc64le", properties={'jobs': 4}, max_builds=1), + create_worker("aix-ppc-libcxx", properties={'jobs': 4}, max_builds=1), + # IBM z13 (s390x), Ubuntu 16.04.2 create_worker("systemz-1", properties={'jobs': 4, 'vcs_protocol': 'https'}, max_builds=4), @@ -162,9 +169,15 @@ # Debian x86_64 Buster Xeon(R) Gold 6154 CPU @ 3.00GHz, 192GB RAM create_worker("lldb-x86_64-debian", properties={'jobs': 72}, max_builds=1), + # Ubuntu aarch64 128 CPUs, 125 GB RAM + create_worker("libc-aarch64-ubuntu", properties={'jobs': 32}, max_builds=2), + # Debian x86_64 Intel Broadwell 32 CPUs, 120 GB RAM create_worker("libc-x86_64-debian", properties={'jobs': 32}, max_builds=2), + # Debian x86_64 Intel Skylake 32 CPUs, 128 GB RAM + create_worker("libc-x86_64-debian-fullbuild", properties={'jobs': 32}, max_builds=2), + # Windows Server on Xeon Gold 6130 (2x2.1GHz), 128Gb of RAM create_worker("as-builder-1", properties={ 'remote_test_host': 'jetson6.lab.llvm.org', @@ -195,8 +208,8 @@ # Windows 10, Visual Studio 2019, Google Cloud 16 cores create_worker("windows10-vs2019", properties={'jobs': 32}, max_builds=1), - # CentOS 7.5.1804 on Intel(R) Xeon(R) Gold 6126 CPU @ 2.60GHz, 96GB RAM - create_worker("nec-arrproto41", properties={'jobs': 12}, max_builds=1), + # CentOS 7.8.1127 on Intel(R) Xeon(R) Gold 6126 CPU @ 2.60GHz, 96GB RAM + create_worker("hpce-aurora2", properties={'jobs': 8}, max_builds=1), # Ubuntu 18.04 amd64 on Google Cloud, 16 core, Nvidia Tesla T4 create_worker("mlir-nvidia", properties={'jobs': 16}, max_builds=1), @@ -228,4 +241,13 @@ # Ubuntu x86_64 create_worker("thinlto-x86-64-bot1", properties={'jobs': 64}, max_builds=1), + + # Ubuntu 20.04 on AWS, x86_64 PS4 target + create_worker("sie-linux-builder", properties={'jobs': 40}, max_builds=1), + + # XCore target, Ubuntu 20.04 x64 host + create_worker("xcore-ubuntu20-x64", properties={'jobs': 4}, max_builds=1), + + # Ziglang worker + create_worker("ziglang-carbon", properties={'jobs': 24}, max_builds=1), ] diff --git a/buildbot/osuosl/master/master.cfg b/buildbot/osuosl/master/master.cfg --- a/buildbot/osuosl/master/master.cfg +++ b/buildbot/osuosl/master/master.cfg @@ -64,13 +64,19 @@ c['builders'] = builders = [ util.BuilderConfig(**b) for b in config.builders.all ] +builders.extend([ + util.BuilderConfig(**b) for b in config.release_builders.all +]) ####### SCHEDULERS -c['schedulers'] = config.schedulers.getSingleBranchSchedulers( - c['builders']) +c['schedulers'] = config.schedulers.getMainBranchSchedulers( + builders) +c['schedulers'].extend(config.schedulers.getReleaseBranchSchedulers( + builders, + treeStableTimer=5*60)) c['schedulers'].extend(config.schedulers.getForceSchedulers( - c['builders'])) + builders)) ####### BUILDBOT SERVICES diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -1,29 +1,29 @@ -attrs==20.2.0 +attrs==20.3.0 six==1.15.0 Automat==20.2.0 constantly==15.1.0 idna==2.10 -hyperlink==20.0.1 +hyperlink==21.0.0 incremental==17.5.0 -Jinja2==2.11.2 +Jinja2==2.11.3 PyHamcrest==2.0.2 Tempita==0.5.2 decorator==4.4.2 pbr==5.5.1 sqlparse==0.4.1 -SQLAlchemy==1.3.20 +SQLAlchemy==1.3.23 sqlalchemy-migrate==0.13.0 -certifi==2020.6.20 -chardet==3.0.4 -urllib3==1.25.11 -requests==2.24.0 -zope.interface==5.1.2 +certifi==2020.12.5 +chardet==4.0.0 +urllib3==1.26.3 +requests==2.25.1 +zope.interface==5.2.0 pycparser==2.20 -pyOpenSSL==19.1.0 +pyOpenSSL==20.0.1 pyasn1==0.4.8 pyasn1-modules==0.2.8 service-identity==18.1.0 -cryptography==3.2.1 -cffi==1.14.3 +cryptography==3.4.5 +cffi==1.14.5 Twisted==20.3.0 -treq==20.9.0 +treq==21.1.0 diff --git a/tasks/tasktool/README.md b/tasks/tasktool/README.md --- a/tasks/tasktool/README.md +++ b/tasks/tasktool/README.md @@ -92,7 +92,7 @@ { "url": "https://github.com/my-project.git", "type": "git", - "default_rev": "refs/heads/master" + "default_rev": "refs/heads/main" } ``` @@ -166,7 +166,7 @@ : ${VARIABLE:='DefaultValue'} ``` -- Abstract artifacts like `refs/heads/master` in a git repository or +- Abstract artifacts like `refs/heads/main` in a git repository or `clang-stage2-cmake` on an A2 server are resolved to a concrete git revision or a specific URL when the task is started. - A task with a set of artifact and parameter inputs is called a build. diff --git a/tasks/tasktool/tasktool/repos/git.py b/tasks/tasktool/tasktool/repos/git.py --- a/tasks/tasktool/tasktool/repos/git.py +++ b/tasks/tasktool/tasktool/repos/git.py @@ -25,7 +25,7 @@ rev = config.get('rev') default_rev = config.pop('default_rev', None) if default_rev is None: - default_rev = "refs/heads/master" + default_rev = "refs/heads/main" if rev is not None: return url = config['url'] diff --git a/test/jenkins/test_monorepo_build.py b/test/jenkins/test_monorepo_build.py --- a/test/jenkins/test_monorepo_build.py +++ b/test/jenkins/test_monorepo_build.py @@ -3,7 +3,7 @@ # RUN: export TESTING=1 # RUN: export JOB_NAME="FOO" # RUN: export BUILD_NUMBER=321 -# RUN: export BRANCH=master +# RUN: export BRANCH=main # Tell monorepo_build.py to just print commands instead of running. # RUN: mkdir -p %t.SANDBOX/host-compiler/lib %t.SANDBOX/host-compiler/bin %t.SANDBOX/llvm-project/llvm %t.SANDBOX/llvm-project/clang %t.SANDBOX/llvm-project/libcxx %t.SANDBOX/llvm-project/compiler-rt %t.SANDBOX/llvm-project/debuginfo-tests %t.SANDBOX/llvm-project/clang-tools-extra %t.SANDBOX/llvm-project/lldb # RUN: touch %t.SANDBOX/host-compiler/bin/clang @@ -146,4 +146,4 @@ # Test to check if default timeout is being set to 600. # RUN: python %{src_root}/zorg/jenkins/monorepo_build.py cmake all > %t-timeout-default.log # RUN: FileCheck --check-prefix CHECK-TIMEOUT-DEFAULT < %t-timeout-default.log %s -# CHECK-TIMEOUT-DEFAULT: --timeout=600 \ No newline at end of file +# CHECK-TIMEOUT-DEFAULT: --timeout=600 diff --git a/zorg/buildbot/builders/ClangBuilder.py b/zorg/buildbot/builders/ClangBuilder.py --- a/zorg/buildbot/builders/ClangBuilder.py +++ b/zorg/buildbot/builders/ClangBuilder.py @@ -1,6 +1,7 @@ import copy from datetime import datetime +from buildbot.plugins import util from buildbot.process.properties import WithProperties, Property from buildbot.steps.shell import ShellCommand, SetProperty from buildbot.steps.shell import WarningCountingShellCommand @@ -118,9 +119,9 @@ vs_target_arch=vs_target_arch, useTwoStage=useTwoStage, testStage1=testStage1, stage1_config=stage1_config, stage2_config=stage2_config, runTestSuite=runTestSuite, - nt_flags=nt_flags, testsuite_flags=testsuite_flags, - submitURL=submitURL, testerName=testerName, - env=env, extra_cmake_args=extra_cmake_args, + cmake_test_suite=cmake_test_suite, nt_flags=nt_flags, + testsuite_flags=testsuite_flags, submitURL=submitURL, + testerName=testerName, env=env, extra_cmake_args=extra_cmake_args, checkout_clang_tools_extra=checkout_clang_tools_extra, checkout_compiler_rt=checkout_compiler_rt, checkout_lld=checkout_lld, @@ -149,6 +150,7 @@ # Test-suite runTestSuite=False, + cmake_test_suite=False, nt_flags=None, testsuite_flags=None, submitURL=None, @@ -170,9 +172,9 @@ vs_target_arch=vs_target_arch, useTwoStage=useTwoStage, testStage1=testStage1, stage1_config=stage1_config, stage2_config=stage2_config, runTestSuite=runTestSuite, - nt_flags=nt_flags, testsuite_flags=testsuite_flags, - submitURL=submitURL, testerName=testerName, - env=env, extra_cmake_args=extra_cmake_args, + cmake_test_suite=cmake_test_suite, nt_flags=nt_flags, + testsuite_flags=testsuite_flags, submitURL=submitURL, + testerName=testerName, env=env, extra_cmake_args=extra_cmake_args, checkout_clang_tools_extra=checkout_clang_tools_extra, checkout_lld=checkout_lld, checkout_compiler_rt=checkout_compiler_rt, @@ -199,6 +201,7 @@ # Test-suite runTestSuite=False, + cmake_test_suite=False, nt_flags=None, testsuite_flags=None, submitURL=None, @@ -269,10 +272,11 @@ # and the test-suite separately. Le's do this first, # so we wouldn't poison got_revision property. if runTestSuite or checkout_test_suite: - f.addGetSourcecodeForProject( - project='lnt', - src_dir='test/lnt', - alwaysUseLatest=True) + if not cmake_test_suite: + f.addGetSourcecodeForProject( + project='lnt', + src_dir='test/lnt', + alwaysUseLatest=True) f.addGetSourcecodeForProject( project='test-suite', src_dir='test/test-suite', @@ -382,9 +386,11 @@ if not vs: cc = 'clang' cxx = 'clang++' + fc = 'flang' else: cc = 'clang-cl.exe' cxx = 'clang-cl.exe' + fc = 'flang.exe' ############# STAGE 2 @@ -464,8 +470,9 @@ # Get generated python, lnt python = WithProperties('%(builddir)s/test/sandbox/bin/python') - lnt = WithProperties('%(builddir)s/test/sandbox/bin/lnt') - lnt_setup = WithProperties('%(builddir)s/test/lnt/setup.py') + if not cmake_test_suite: + lnt = WithProperties('%(builddir)s/test/sandbox/bin/lnt') + lnt_setup = WithProperties('%(builddir)s/test/lnt/setup.py') # Paths sandbox = WithProperties('%(builddir)s/test/sandbox') @@ -478,7 +485,18 @@ # LNT Command line (don't pass -jN. Users need to pass both --threads # and --build-threads in nt_flags/test_suite_flags to get the same effect) use_runtest_testsuite = len(nt_flags) == 0 - if not use_runtest_testsuite: + # with LNT, the workdir needs to be test/sandbox, + # but the CMakeList.txt and Makefiles are in %(builddir)s/test/test-suite. + test_suite_work_dir = 'test/sandbox' + if cmake_test_suite: + cmake_test_suite_cmd = ['cmake', '-G', 'Ninja', + '-DCMAKE_C_COMPILER=' + cc, + '-DCMAKE_CXX_COMPILER=' + cxx, + '-DTEST_SUITE_LIT:FILEPATH=' + lit, + test_suite_dir] + test_suite_build_cmd = ['ninja', '-k', '0'] + test_suite_cmd = ['ninja', 'check'] + elif not use_runtest_testsuite: test_suite_cmd = [python, lnt, 'runtest', 'nt', '--no-timestamp', '--sandbox', sandbox, @@ -496,6 +514,14 @@ '--cc', cc, '--cxx', cxx, '--use-lit', lit] + # Enable fortran if flang is checked out + if checkout_flang: + fortran_flags = [ + '--cmake-define=TEST_SUITE_FORTRAN:STRING=ON', + util.Interpolate( + '--cmake-define=CMAKE_Fortran_COMPILER=' + + '%(prop:builddir)s/'+compiler_path+'/bin/'+fc)] + test_suite_cmd.extend(fortran_flags) # Append any option provided by the user test_suite_cmd.extend(testsuite_flags) @@ -521,7 +547,7 @@ workdir='test', env=env)) f.addStep(ShellCommand(name='recreate sandbox', - command=['virtualenv', 'sandbox'], + command=['virtualenv', '--python=python3', 'sandbox'], haltOnFailure=True, description='recreating sandbox', workdir='test', @@ -532,12 +558,27 @@ description='setting up LNT in sandbox', workdir='test/sandbox', env=env)) + if cmake_test_suite: + f.addStep(ShellCommand( + name='cmake test-suite', + command=cmake_test_suite_cmd, + haltOnFailure=True, + description=['running cmake for test suite'], + workdir=test_suite_work_dir, + env=test_suite_env)) + f.addStep(ShellCommand( + name='Ninja test-suite', + command=test_suite_build_cmd, + haltOnFailure=True, + description=['Using ninja to build test suite'], + workdir=test_suite_work_dir, + env=test_suite_env)) f.addStep(LitTestCommand( name='test-suite', command=test_suite_cmd, haltOnFailure=True, description=['running the test suite'], - workdir='test/sandbox', + workdir=test_suite_work_dir, logfiles={'configure.log' : 'build/configure.log', 'build-tools.log' : 'build/build-tools.log', 'test.log' : 'build/test.log', diff --git a/zorg/buildbot/builders/ClangLTOBuilder.py b/zorg/buildbot/builders/ClangLTOBuilder.py --- a/zorg/buildbot/builders/ClangLTOBuilder.py +++ b/zorg/buildbot/builders/ClangLTOBuilder.py @@ -233,6 +233,7 @@ clean = False, jobs = None, extra_configure_args = None, + extra_configure_args_lto_stage = None, compare_last_2_stages = True, lto = None, # The string gets passed to -flto flag as is. Like -flto=thin. env = None, @@ -253,10 +254,16 @@ else: extra_configure_args = list(extra_configure_args) + if extra_configure_args_lto_stage is None: + extra_configure_args_lto_stage = [] + else: + extra_configure_args_lto_stage = list(extra_configure_args_lto_stage) + # Make sure CMAKE_INSTALL_PREFIX and -G are not specified # in the extra_configure_args. We set them internally as needed. # TODO: assert extra_configure_args. - install_prefix_specified = any(a.startswith('-DCMAKE_INSTALL_PREFIX=') for a in extra_configure_args) + install_prefix_specified = (any(a.startswith('-DCMAKE_INSTALL_PREFIX=') for a in extra_configure_args) or + any(a.startswith('-DCMAKE_INSTALL_PREFIX=') for a in extra_configure_args_lto_stage)) assert not install_prefix_specified, "Please do not explicitly specify the install prefix for multi-stage build." # Prepare environmental variables. Set here all env we want everywhere. @@ -321,7 +328,7 @@ s = f.staged_compiler_idx + 1 staged_install = f.stage_installdirs[f.staged_compiler_idx] for i in range(s, len(f.stage_objdirs[s:]) + s): - configure_args = extra_configure_args[:] + configure_args = extra_configure_args[:] + extra_configure_args_lto_stage[:] configure_args.append( WithProperties( diff --git a/zorg/buildbot/builders/DoxygenDocsBuilder.py b/zorg/buildbot/builders/DoxygenDocsBuilder.py new file mode 100644 --- /dev/null +++ b/zorg/buildbot/builders/DoxygenDocsBuilder.py @@ -0,0 +1,120 @@ +from collections import OrderedDict +from importlib import reload + +from buildbot.plugins import steps, util +from buildbot.steps.shell import ShellCommand + +from zorg.buildbot.builders import UnifiedTreeBuilder +from zorg.buildbot.commands import CmakeCommand + +from zorg.buildbot.process import factory +reload(factory) +reload(UnifiedTreeBuilder) +reload(CmakeCommand) + +llvm_docs = OrderedDict([ + # Project Build target Local path Remote path + ("llvm", ("doxygen", "docs/doxygen/html/", "llvm")), + ("clang", (None, "tools/clang/docs/doxygen/html/", "cfe")), + ("clang-tools-extra", (None, "tools/clang/tools/extra/docs/doxygen/html/", "cfe-extra")), + ("flang", (None, "tools/flang/docs/doxygen/html/", "flang")), + ("polly", (None, "tools/polly/docs/doxygen/html/", "polly")), + ("openmp", (None, "projects/openmp/docs/doxygen/html/", "openmp")), + ("lldb", ("lldb-cpp-doc", "tools/lldb/docs/cpp_reference/", "lldb/cpp_reference")), + # NOTE: 5/9/2020 lldb-python-doc fails to build. Disabled till be fixed. + #(None, ("lldb-python-doc", "tools/lldb/docs/python_reference/", "lldb")), +]) + +def getLLVMDocsBuildFactory( + clean = True, + depends_on_projects = None, + extra_configure_args = None, + env = None, + **kwargs): + + if depends_on_projects is None: + # All the projects from llvm_docs, and remove all duplicates. + _depends_on_projects=list(set( + [project for project in llvm_docs if project])) + else: + # Make a local copy of depends_on_projects, as we are going to modify + # that. + _depends_on_projects=depends_on_projects[:] + # Some projects are interdependent for the purpose of documentation. + # Enforce the dependencies. + # TODO: Check later the dependencies for doxygen docs and enforce them + # here if needed. + + # Make a local copy of the configure args, as we are going to modify that. + if extra_configure_args: + cmake_args = extra_configure_args[:] + else: + cmake_args = list() + + # Prepare environmental variables. Set here all env we want everywhere. + merged_env = { + 'TERM' : 'dumb' # Be cautious and disable color output from all tools. + } + if env is not None: + # Overwrite pre-set items with the given ones, so user can set anything. + merged_env.update(env) + + CmakeCommand.CmakeCommand.applyDefaultOptions(cmake_args, [ + ("-G", "Ninja"), + ("-DLLVM_ENABLE_DOXYGEN=", "ON"), + ("-DLLVM_BUILD_DOCS=", "ON"), + ("-DCLANG_TOOLS_EXTRA_INCLUDE_DOCS=", "ON"), + ("-DLLVM_ENABLE_ASSERTIONS=", "OFF"), + ("-DCMAKE_BUILD_TYPE=", "Release"), + ]) + + f = UnifiedTreeBuilder.getCmakeBuildFactory( + clean=clean, + depends_on_projects=_depends_on_projects, + extra_configure_args=cmake_args, + env=merged_env, + **kwargs) # Pass through all the extra arguments. + + # Build the documentation for all the projects. + for project in llvm_docs: + target = llvm_docs[project][0] + + # Build only those with specifies targets. + if target: + UnifiedTreeBuilder.addNinjaSteps( + f, + # Doxygen builds the final result for really + # long time without any output. + # We have to have a long timeout at this step. + timeout=21600, + targets=[target], + checks=[], + env=merged_env, + **kwargs) + + # Publish just built documentation + for project in llvm_docs: + target, local_path, remote_path = llvm_docs[project] + + f.addStep( + ShellCommand( + name="Publish {}".format(project or target), + description=[ + "Publish", "just", "built", "documentation", "for", + "{}".format(project or target) + ], + command=[ + 'rsync', + '-vrl', + '--delete', '--force', '--delay-updates', '--delete-delay', + '--ignore-times', + '--checksum', + '-p', '--chmod=Du=rwx,Dg=rwx,Do=rx,Fu=rw,Fg=rw,Fo=r', + "{}".format(local_path), + "lists.llvm.org:web/doxygen/{}".format(remote_path), + ], + env=merged_env, + ) + ) + + return f diff --git a/zorg/buildbot/builders/FlangBuilder.py b/zorg/buildbot/builders/FlangBuilder.py --- a/zorg/buildbot/builders/FlangBuilder.py +++ b/zorg/buildbot/builders/FlangBuilder.py @@ -14,9 +14,10 @@ env = dict() f = getCmakeWithNinjaBuildFactory( - depends_on_projects=['llvm','mlir'], + depends_on_projects=['llvm','clang','mlir'], obj_dir="build_llvm", checks=[], + install_dir="install", clean=clean, extra_configure_args=llvm_extra_configure_args, env=env, @@ -42,6 +43,7 @@ # Add LLVM_DIR and MLIR_DIR to the CMake invocation. llvm_dir = "{}/lib/cmake/llvm".format(f.obj_dir) mlir_dir = "{}/lib/cmake/mlir".format(f.obj_dir) + clang_dir = "{}/lib/cmake/clang".format(f.install_dir) CmakeCommand.applyRequiredOptions(flang_cmake_args, [ # We actually need the paths to be relative to the source directory, # otherwise find_package can't locate the config files. @@ -49,6 +51,8 @@ LLVMBuildFactory.pathRelativeTo(llvm_dir, flang_src_dir)), ('-DMLIR_DIR:PATH=', LLVMBuildFactory.pathRelativeTo(mlir_dir, flang_src_dir)), + ('-DCLANG_DIR:PATH=', + LLVMBuildFactory.pathRelativeTo(clang_dir, flang_src_dir)), ]) # We can't use addCmakeSteps as that would use the path in f.llvm_srcdir. diff --git a/zorg/buildbot/builders/HtmlDocsBuilder.py b/zorg/buildbot/builders/HtmlDocsBuilder.py new file mode 100644 --- /dev/null +++ b/zorg/buildbot/builders/HtmlDocsBuilder.py @@ -0,0 +1,105 @@ +from collections import OrderedDict +from importlib import reload + +from buildbot.plugins import steps, util +from buildbot.steps.shell import ShellCommand +from buildbot.steps.shell import WarningCountingShellCommand + +from zorg.buildbot.process import factory +reload(factory) + +llvm_docs = OrderedDict([ + # Project Build target Build path Local path Remote path + ("lnt", ("html", "docs", "_build/html/", "lnt")), +]) + +# We build with make for now. Change later if needed. +build_cmd = 'make' + +def getHtmlDocsBuildFactory( + depends_on_projects = None, + clean = False, + env = None, + **kwargs): + + if depends_on_projects is None: + # All the projects by default. + _depends_on_projects=llvm_docs.keys() + else: + # Make a local copy of depends_on_projects, as we are going to modify + # that. + _depends_on_projects=depends_on_projects[:] + + # Prepare environmental variables. Set here all env we want everywhere. + merged_env = { + 'TERM' : 'dumb' # Be cautious and disable color output from all tools. + } + if env is not None: + # Overwrite pre-set items with the given ones, so user can set anything. + merged_env.update(env) + + # HTML Sphinx documentation builds in tree, each in its own directory. + # For that, make sure the obj_dir is the same as llvm_srcdir. + src_dir = kwargs.pop('llvm_srcdir', '.') + f = factory.LLVMBuildFactory( + clean=clean, + depends_on_projects=_depends_on_projects, + llvm_srcdir=src_dir, + obj_dir=src_dir, + **kwargs) # Pass through all the extra arguments. + + # Build the documentation + for project in llvm_docs: + + # Checkout the source code and remove all the untracked files, so + # we would build a fresh new documentation. + f.addStep( + steps.Git( + name='Checkout the {} source code'.format(project), + repourl=f.repourl_prefix + "llvm-{}.git".format(project), + mode='full', + method='fresh', + progress=True, + workdir=util.WithProperties(project), + env=merged_env, + **kwargs)) + + target, build_path, local_path, remote_path = llvm_docs[project] + + build_dir = util.WithProperties( + "{}".format("/".join([ + project, + build_path]))) + f.addStep( + steps.WarningCountingShellCommand( + name="Build {} documentation".format(project), + command=[build_cmd, target], + haltOnFailure=True, + workdir=build_dir, + env=merged_env, + **kwargs)) + + # Publish just built documentation + f.addStep( + ShellCommand( + name="Publish {}".format(project), + description=[ + "Publish", "just", "built", "documentation", "for", + "{}".format(project) + ], + command=[ + 'rsync', + '-vrl', + '--delete', '--force', '--delay-updates', '--delete-delay', + '--ignore-times', + '--checksum', + '-p', '--chmod=Du=rwx,Dg=rwx,Do=rx,Fu=rw,Fg=rw,Fo=r', + "{}".format(local_path), + "lists.llvm.org:web/{}".format(remote_path), + ], + workdir=build_dir, + env=merged_env, + ) + ) + + return f diff --git a/zorg/buildbot/builders/LibcxxAndAbiBuilder.py b/zorg/buildbot/builders/LibcxxAndAbiBuilder.py --- a/zorg/buildbot/builders/LibcxxAndAbiBuilder.py +++ b/zorg/buildbot/builders/LibcxxAndAbiBuilder.py @@ -12,6 +12,7 @@ check_libcxx_benchmarks=None, depends_on_projects=None, use_cache=None, + build_standalone=False, **kwargs): if env is None: @@ -27,12 +28,19 @@ depends_on_projects = ['libcxx','libcxxabi','libunwind'] src_root = 'llvm' + build_path = 'build' + if build_standalone: + src_to_build_dir = 'runtimes' + else: + src_to_build_dir = None + if f is None: f = UnifiedTreeBuilder.getLLVMBuildFactoryAndSourcecodeSteps( depends_on_projects=depends_on_projects, llvm_srcdir=src_root, + src_to_build_dir=src_to_build_dir, obj_dir=build_path, **kwargs) # Pass through all the extra arguments. @@ -71,7 +79,12 @@ workdir=".", haltOnFailure=False)) - CmakeCommand.applyRequiredOptions(cmake_opts, [ + if build_standalone: + CmakeCommand.applyRequiredOptions(cmake_opts, [ + ('-DLLVM_ENABLE_RUNTIMES=', ";".join(f.depends_on_projects)), + ]) + else: + CmakeCommand.applyRequiredOptions(cmake_opts, [ ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)), ]) diff --git a/zorg/buildbot/builders/SanitizerBuilder.py b/zorg/buildbot/builders/SanitizerBuilder.py --- a/zorg/buildbot/builders/SanitizerBuilder.py +++ b/zorg/buildbot/builders/SanitizerBuilder.py @@ -6,6 +6,7 @@ def getSanitizerBuildFactory( clean=False, depends_on_projects=None, + extra_configure_args=None, env=None, timeout=1200): @@ -30,6 +31,11 @@ "libunwind", "lld"] + if extra_configure_args: + extra_configure_args = ['--'] + list(extra_configure_args) + else: + extra_configure_args = [] + # Explicitly use '/' as separator, because it works on *nix and Windows. sanitizer_script_dir = "sanitizer_buildbot/sanitizers" sanitizer_script = "../%s/zorg/buildbot/builders/sanitizers/%s" % (sanitizer_script_dir, "buildbot_selector.py") @@ -47,10 +53,11 @@ alwaysUseLatest=True) # Run annotated command for sanitizer. + command = ['python', sanitizer_script] + extra_configure_args f.addStep(AnnotatedCommand(name="annotate", description="annotate", timeout=timeout, haltOnFailure=True, - command="python " + sanitizer_script, + command=command, env=merged_env)) return f diff --git a/zorg/buildbot/builders/UnifiedTreeBuilder.py b/zorg/buildbot/builders/UnifiedTreeBuilder.py --- a/zorg/buildbot/builders/UnifiedTreeBuilder.py +++ b/zorg/buildbot/builders/UnifiedTreeBuilder.py @@ -50,6 +50,7 @@ def getLLVMBuildFactoryAndSourcecodeSteps( depends_on_projects = None, llvm_srcdir = None, + src_to_build_dir = None, obj_dir = None, install_dir = None, cleanBuildRequested = None, @@ -58,6 +59,7 @@ f = getLLVMBuildFactoryAndPrepareForSourcecodeSteps( depends_on_projects=depends_on_projects, llvm_srcdir=llvm_srcdir, + src_to_build_dir=src_to_build_dir, obj_dir=obj_dir, install_dir=install_dir, cleanBuildRequested=cleanBuildRequested, @@ -378,6 +380,7 @@ def getCmakeWithNinjaMultistageBuildFactory( depends_on_projects = None, llvm_srcdir = None, + src_to_build_dir = None, obj_dir = None, checks = None, install_dir = None, @@ -427,6 +430,7 @@ f = getLLVMBuildFactoryAndPrepareForSourcecodeSteps( depends_on_projects=depends_on_projects, llvm_srcdir=llvm_srcdir, + src_to_build_dir=src_to_build_dir, obj_dir=obj_dir, install_dir=install_dir, env=merged_env, diff --git a/zorg/buildbot/builders/ZigBuilder.py b/zorg/buildbot/builders/ZigBuilder.py new file mode 100644 --- /dev/null +++ b/zorg/buildbot/builders/ZigBuilder.py @@ -0,0 +1,72 @@ +import json +import urllib.request + +from buildbot.plugins import steps +from buildbot.steps.shell import ShellCommand, WarningCountingShellCommand +from buildbot.process.properties import WithProperties, Property +from buildbot.commands.CmakeCommand import CmakeCommand +from zorg.buildbot.process.factory import LLVMBuildFactory +from zorg.buildbot.builders import UnifiedTreeBuilder + +def getZiglangToolchainBuildFactory( + clean=False, + env=None, + extra_configure_args=None, + extra_cmake_args=None, + config='Release', + install=False, + testTimeout=2400, + test=False): + + build_env = { + "TERM": "dumb", + } + + if env is not None: + build_env.update(env) + + build_cmd = ["ninja"] + install_cmd = ["ninja", "install"] + test_cmd = ["./build/zig", "build", "test"] + + if jobs: + build_cmd.append(WithProperties("-j%s" % jobs)) + install_cmd.append(WithProperties("-j%s" % jobs)) + test_cmd.append(WithProperties("-j%s" % jobs)) + + cleanBuildRequested = lambda step: clean or step.build.getProperty("clean", + default=step.build.getProperty("clean_obj")) + + if extra_configure_args: + cmake_options += extra_cmake_args + + + llvm_src_dir = "llvm" + llvm_install_dir = "build" + + f = UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + depends_on_projects=[ + "llvm", + "clang", + "lld" + ], + llvm_src_dir=llvm_src_dir, + install_dir=llvm_install_dir + ) + + json_text = urllib.request.urlopen("https://ziglang.org/download/index.json").read().decode() + src_url = json.load(json_text)['master']['src']['tarball'] + + f.addStep(ShellCommand( + name="fetch-src", + command=["curl", "-Sf", "-o", "zig.tar.xz", src_url], + description=["Download Zig source code archive"], + workdir=zig_src_dir, + haltOnFailure=True + ) + + f.addStep(ShellCommand( + name="extract-src", + command=["tar", " xvf", "zig.tar.xz"], + description=["extract Zig source code"], + ) diff --git a/zorg/buildbot/builders/annotated/annotated_builder.py b/zorg/buildbot/builders/annotated/annotated_builder.py --- a/zorg/buildbot/builders/annotated/annotated_builder.py +++ b/zorg/buildbot/builders/annotated/annotated_builder.py @@ -13,6 +13,7 @@ from os.path import join as pjoin VSWHERE_PATH = "C:/Program Files (x86)/Microsoft Visual Studio/Installer/vswhere.exe" +BUILDTOOLS_VSDEVCMD = "C:/BuildTools/Common7/Tools/VsDevCmd.bat" def get_argument_parser(*args, **kwargs): ap = argparse.ArgumentParser(*args, **kwargs) @@ -211,10 +212,13 @@ cxx_compiler, linker, previous_stage_bin): + archiver = 'llvm-ar' + if os.name == 'nt': + archiver = 'llvm-lib' return ( cmake_args + [ '-DCMAKE_AR=%s' % ( - util.cmake_pjoin(previous_stage_bin, 'llvm-ar'),), + util.cmake_pjoin(previous_stage_bin, archiver),), '-DCMAKE_RANLIB=%s' % ( util.cmake_pjoin(previous_stage_bin, 'llvm-ranlib'),), ] + self.cmake_compiler_flags( @@ -323,9 +327,17 @@ def get_vcvars(vs_tools, arch): - """Get the VC tools environment using vswhere.exe from VS 2017 + """Get the VC tools environment using vswhere.exe or buildtools docker + + This is intended to work either when VS is in its standard installation + location, or when the docker instructions have been followed, and we can + find Visual C++ in C:/BuildTools. + + Visual Studio provides a docker image with instructions here: + https://docs.microsoft.com/en-us/visualstudio/install/build-tools-container?view=vs-2019 - This code is following the guidelines from strategy 1 in this blog post: + This vswhere code is following the guidelines from strategy 1 in this blog + post: https://blogs.msdn.microsoft.com/vcblog/2017/03/06/finding-the-visual-c-compiler-tools-in-visual-studio-2017/ It doesn't work when VS is not installed at the default location. @@ -342,13 +354,16 @@ # Use vswhere.exe if it exists. if os.path.exists(VSWHERE_PATH): cmd = [VSWHERE_PATH, "-latest", "-property", "installationPath"] - vs_path = subprocess.check_output(cmd).strip() + vs_path = subprocess.check_output(cmd).decode(sys.stdout.encoding) + vs_path = vs_path.strip() util.report("Running vswhere to find VS: " + repr(cmd)) util.report("vswhere output: " + vs_path) if not os.path.isdir(vs_path): raise ValueError("VS install path does not exist: " + vs_path) vcvars_path = pjoin(vs_path, 'VC', 'Auxiliary', 'Build', 'vcvarsall.bat') + elif os.path.exists(BUILDTOOLS_VSDEVCMD): + vcvars_path = BUILDTOOLS_VSDEVCMD elif vs_tools is None: vs_tools = os.path.expandvars('%VS140COMNTOOLS%') vcvars_path = pjoin(vs_tools, '..', '..', 'VC', 'vcvarsall.bat') @@ -357,7 +372,8 @@ # Windows /dev/null. cmd = util.shquote_cmd([vcvars_path, arch]) + ' > NUL && set' util.report("Running vcvars: " + cmd) - output = subprocess.check_output(cmd, shell=True) + output = \ + subprocess.check_output(cmd, shell=True).decode(sys.stdout.encoding) new_env = {} for line in output.splitlines(): var, val = line.split('=', 1) diff --git a/zorg/buildbot/builders/annotated/libc-linux.py b/zorg/buildbot/builders/annotated/libc-linux.py --- a/zorg/buildbot/builders/annotated/libc-linux.py +++ b/zorg/buildbot/builders/annotated/libc-linux.py @@ -9,6 +9,10 @@ from contextlib import contextmanager +def is_fullbuild_builder(builder_name): + return ('fullbuild' in builder_name.split('-')) + + def main(argv): ap = argparse.ArgumentParser() ap.add_argument('--asan', action='store_true', default=False, @@ -18,11 +22,18 @@ args, _ = ap.parse_known_args() source_dir = os.path.join('..', 'llvm-project') + fullbuild = is_fullbuild_builder(os.environ.get('BUILDBOT_BUILDERNAME')) with step('cmake', halt_on_fail=True): projects = ['llvm', 'libc', 'clang', 'clang-tools-extra'] - cmake_args = ['-GNinja'] + # On most systems the default generator is make and the default + # compilers are gcc and g++. We make it explicit here that we want + # clang and ninja which reduces one step of setting environment + # variables when setting up workers. + cmake_args = ['-GNinja', + '-DCMAKE_C_COMPILER=clang', + '-DCMAKE_CXX_COMPILER=clang++'] if args.debug: cmake_args.append('-DCMAKE_BUILD_TYPE=Debug') else: @@ -33,6 +44,10 @@ cmake_args.append('-DLLVM_ENABLE_PROJECTS={}'.format(';'.join(projects))) + if fullbuild: + cmake_args.extend(['-DLLVM_LIBC_FULL_BUILD=ON', + '-DLLVM_LIBC_ENABLE_LINTING=ON']) + run_command(['cmake', os.path.join(source_dir, 'llvm')] + cmake_args) with step('build llvmlibc', halt_on_fail=True): @@ -41,18 +56,16 @@ with step('check-libc'): run_command(['ninja', 'check-libc']) - if not args.asan: - with step('Loader Tests'): + if fullbuild and not args.asan: + with step('libc-loader-tests'): run_command(['ninja', 'libc_loader_tests']) - with step('Integration Tests'): + with step('libc-integration-test'): run_command(['ninja', 'libc-integration-test']) with step('AOR Tests'): aor_dir = os.path.join(source_dir, 'libc', 'AOR_v20.02') # Remove the AOR build dir. util.clean_dir(os.path.join(aor_dir, 'build')) run_command(['make', 'check'], directory=aor_dir) - - if not args.debug: with step('Benchmark Utils Tests'): run_command(['ninja', 'libc-benchmark-util-tests']) diff --git a/zorg/buildbot/builders/sanitizers/buildbot_android_functions.sh b/zorg/buildbot/builders/sanitizers/buildbot_android_functions.sh --- a/zorg/buildbot/builders/sanitizers/buildbot_android_functions.sh +++ b/zorg/buildbot/builders/sanitizers/buildbot_android_functions.sh @@ -17,7 +17,7 @@ [[ -d ${NDK_DIR} ]] && rm -rf ${NDK_DIR} [[ -d ${VERSION} ]] && rm -rf ${VERSION} [[ -f ${FILE_NAME} ]] && rm -f ${FILE_NAME} - wget ${NDK_URL} + wget -e dotbytes=10m ${NDK_URL} unzip ${FILE_NAME} > /dev/null mv ${VERSION} ${NDK_DIR} echo ${NDK_URL} > ${NDK_DIR}/android_ndk_url @@ -27,7 +27,7 @@ echo @@@BUILD_STEP downloading Android Platform Tools@@@ local FILE_NAME=platform-tools-latest-linux.zip [[ -f ${FILE_NAME} ]] && rm -f ${FILE_NAME} - wget https://dl.google.com/android/repository/${FILE_NAME} + wget -e dotbytes=10m https://dl.google.com/android/repository/${FILE_NAME} unzip ${FILE_NAME} > /dev/null fi export PATH=$ROOT/platform-tools/:$PATH @@ -36,7 +36,7 @@ function build_stage2_android() { # Build self-hosted tree with fresh Clang and -Werror. local CMAKE_OPTIONS="${CMAKE_COMMON_OPTIONS} -DLLVM_ENABLE_WERROR=ON ${STAGE1_AS_COMPILER} -DCMAKE_C_FLAGS=-gmlt -DCMAKE_CXX_FLAGS=-gmlt" - CMAKE_OPTIONS="${CMAKE_OPTIONS} -DLLVM_ENABLE_PROJECTS='clang;lld' -DLLVM_USE_LINKER=lld" + CMAKE_OPTIONS="${CMAKE_OPTIONS} -DLLVM_ENABLE_PROJECTS='clang;lld' -DLLVM_USE_LINKER=lld -DCLANG_DEFAULT_RTLIB=libgcc" if ccache -s ; then CMAKE_OPTIONS="${CMAKE_OPTIONS} -DLLVM_CCACHE_BUILD=ON" @@ -128,14 +128,18 @@ fi } -# If a multiarch device has x86 as the first arch, remove everything else from -# the list. This captures cases like [x86,armeabi-v7a], where the arm part is -# software emulation and incompatible with ASan. +# If a multiarch device's first arch starts with x86, remove archs that don't +# start with x86. Handle cases like: +# - x86,armeabi-v7a +# - x86_64,x86,arm64-v8a,armeabi-v7a,armeabi +# - x86_64,arm64-v8a,x86,armeabi-v7a,armeabi +# The arm part is software emulation and incompatible with ASan. function patch_abilist { # IN OUT local _abilist=$1 local _out=$2 - if [[ "$_abilist" == "x86,"* ]]; then - _abilist="x86" + if [[ "$_abilist" == "x86"* ]]; then + _abilist=$(echo $_abilist | tr , '\n' | grep '^x86') + _abilist=$(echo $_abilist | tr ' ' ,) fi eval $_out="'$_abilist'" } @@ -154,7 +158,7 @@ shift ABILIST=$(${ADB} -s $_serial shell getprop ro.product.cpu.abilist) - patch_abilist $ABILIST ABILIST + patch_abilist "$ABILIST" ABILIST for _arg in "$@"; do local _arch=${_arg%:*} local _abi=${_arg#*:} @@ -166,7 +170,12 @@ echo "$_serial" >> tested_arch_$_arch BUILD_ID=$(${ADB} -s $_serial shell getprop ro.build.id | tr -d '\r') BUILD_FLAVOR=$(${ADB} -s $_serial shell getprop ro.build.flavor | tr -d '\r') - test_arch_on_device "$_arch" "$_serial" "$BUILD_ID" "$BUILD_FLAVOR" + ( + # Test only one arch at a time to avoid simultaneously writes into the + # same compiler-rt build dir. + flock -x $lock_fd + test_arch_on_device "$_arch" "$_serial" "$BUILD_ID" "$BUILD_FLAVOR" + ) {lock_fd}>$ROOT/${_arch}.lock fi done } @@ -200,6 +209,9 @@ done tail_pids "$LOGS" + # Return to avoid exception if we already have error. + [[ $BUILD_RT_ERR == "" ]] || return + for _arg in "$@"; do local _arch=${_arg%:*} if [[ ! -f tested_arch_$_arch ]]; then diff --git a/zorg/buildbot/builders/sanitizers/buildbot_cmake.sh b/zorg/buildbot/builders/sanitizers/buildbot_cmake.sh --- a/zorg/buildbot/builders/sanitizers/buildbot_cmake.sh +++ b/zorg/buildbot/builders/sanitizers/buildbot_cmake.sh @@ -13,6 +13,15 @@ export PATH="/usr/local/bin:$PATH" export ANDROID_SDK_HOME=$ROOT/../../.. +CMAKE_ARGS="" +for arg in "$@" +do + case $arg in + --CMAKE_ARGS=*) + CMAKE_ARGS="${arg#*=}" + esac +done + # Always clobber bootstrap build trees. rm -rf compiler_rt_build llvm_build* symbolizer_build* @@ -27,7 +36,7 @@ LLVM=$ROOT/llvm ZLIB=$ROOT/zlib -CMAKE_COMMON_OPTIONS="${CMAKE_COMMON_OPTIONS:-} -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_PARALLEL_LINK_JOBS=10" +CMAKE_COMMON_OPTIONS="${CMAKE_COMMON_OPTIONS:-} -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_PARALLEL_LINK_JOBS=10 ${CMAKE_ARGS}" ENABLE_LIBCXX_FLAG= if [ "$PLATFORM" == "Darwin" ]; then CMAKE_COMMON_OPTIONS="${CMAKE_COMMON_OPTIONS} -DPYTHON_EXECUTABLE=/usr/bin/python" diff --git a/zorg/buildbot/builders/sanitizers/buildbot_functions.sh b/zorg/buildbot/builders/sanitizers/buildbot_functions.sh --- a/zorg/buildbot/builders/sanitizers/buildbot_functions.sh +++ b/zorg/buildbot/builders/sanitizers/buildbot_functions.sh @@ -21,7 +21,7 @@ echo @@@BUILD_STEP Prepare@@@ BUILDBOT_CLOBBER="${BUILDBOT_CLOBBER:-}" -BUILDBOT_REVISION="${BUILDBOT_REVISION:-origin/master}" +BUILDBOT_REVISION="${BUILDBOT_REVISION:-origin/main}" function rm_dirs { while ! rm -rf $@ ; do sleep 1; done @@ -54,17 +54,18 @@ cd llvm-project git init git remote add origin https://github.com/llvm/llvm-project.git + git config --local advice.detachedHead false ) cd llvm-project - git fetch --depth $DEPTH origin master + git fetch --depth $DEPTH origin main git clean -fd local REV=${BUILDBOT_REVISION} - if [[ "$REV" != "origin/master" ]] ; then + if [[ "$REV" != "origin/main" ]] ; then # "git fetch --depth 1 origin $REV" does not work with 2.11 on bots while true ; do git checkout $REV && break - git rev-list --pretty --max-count=1 origin/master - git rev-list --pretty --max-parents=0 origin/master + git rev-list --pretty --max-count=1 origin/main + git rev-list --pretty --max-parents=0 origin/main echo "DEPTH=$DEPTH is too small" [[ "$DEPTH" -le "1000000" ]] || exit 1 DEPTH=$(( $DEPTH * 10 )) diff --git a/zorg/buildbot/builders/sanitizers/buildbot_selector.py b/zorg/buildbot/builders/sanitizers/buildbot_selector.py --- a/zorg/buildbot/builders/sanitizers/buildbot_selector.py +++ b/zorg/buildbot/builders/sanitizers/buildbot_selector.py @@ -5,6 +5,7 @@ import sys THIS_DIR=os.path.dirname(sys.argv[0]) +extra_args = sys.argv[1:] def bash(path): @@ -44,7 +45,7 @@ def Main(): builder = os.environ.get('BUILDBOT_BUILDERNAME') print "builder name: %s" % (builder) - cmd = BOT_ASSIGNMENT.get(builder) + cmd = BOT_ASSIGNMENT.get(builder) + ' ' + ' '.join(extra_args) if not cmd: sys.stderr.write('ERROR - unset/invalid builder name\n') sys.exit(1) diff --git a/zorg/buildbot/changes/llvmgitpoller.py b/zorg/buildbot/changes/llvmgitpoller.py --- a/zorg/buildbot/changes/llvmgitpoller.py +++ b/zorg/buildbot/changes/llvmgitpoller.py @@ -11,31 +11,41 @@ class LLVMPoller(changes.GitPoller): """ - Poll LLVM repository for changes and submit them to the change master. + Poll LLVM repository for changes and submit them for builds scheduling. Following Multiple LLVM Projects. This source will poll a remote LLVM git _monorepo_ for changes and submit - them to the change master.""" + them for builds scheduling.""" _repourl = "https://github.com/llvm/llvm-project" - _branch = "master" - compare_attrs = ["repourl", "branch", "workdir", + compare_attrs = ["repourl", "branches", "workdir", "pollInterval", "gitbin", "usetimestamps", "category", "project", "projects"] + def _check_branches(branch): + log.msg("LLVMPoller: _check_branches: branch={}", branch) + + if branch == "refs/heads/main": + # Always listen for changes in the main branch. + return True + else: + # We are also interested in the release branches. + # Some builders will be building changes from there as well. + return re.search(r"refs\/heads\/release\/\d\d+.*", branch) + def __init__(self, - repourl=_repourl, branch=_branch, + repourl=_repourl, branches=_check_branches, **kwargs): self.cleanRe = re.compile(r"Require(?:s?)\s*.*\s*clean build", re.IGNORECASE + re.MULTILINE) self.cleanCfg = re.compile(r"(CMakeLists\.txt$|\.cmake$|\.cmake\.in$)") - # TODO: Add support for an optional list of projects. - # For now we always watch all the projects. + # Note: We always watch all the projects, then schedulers decide + # to build or not to build. - super().__init__(repourl=repourl, branch=branch, **kwargs) + super().__init__(repourl=repourl, branches=branches, **kwargs) def _transform_path(self, fileList): """ @@ -111,7 +121,7 @@ newRev, branch)) else: # This branch is known, but it now points to a different - # commit than last time we saw it, rebuild. + # commit than the last time we saw it, rebuild. log.msg('LLVMPoller: rebuilding {} for updated branch "{}"'.format( newRev, branch)) @@ -163,8 +173,9 @@ properties['clean_obj'] = (True, "change") log.msg("LLVMPoller: creating a change rev=%s" % rev) - log.msg(" >>> revision=%s, timestamp=%s, author=%s, committer=%s, project=%s, files=%s, comments=\"%s\", properties=%s" % \ - (bytes2unicode(rev, encoding=self.encoding), datetime.fromtimestamp(timestamp), author, committer, + log.msg(" >>> branch=%s, revision=%s, timestamp=%s, author=%s, committer=%s, project=%s, files=%s, comments=\"%s\", properties=%s" % \ + (bytes2unicode(self._removeHeads(branch)), + bytes2unicode(rev, encoding=self.encoding), datetime.fromtimestamp(timestamp), author, committer, projects, files, comments, properties)) yield self.master.data.updates.addChange( diff --git a/zorg/buildbot/commands/LitTestCommand.py b/zorg/buildbot/commands/LitTestCommand.py --- a/zorg/buildbot/commands/LitTestCommand.py +++ b/zorg/buildbot/commands/LitTestCommand.py @@ -4,7 +4,7 @@ from buildbot.process.results import SUCCESS from buildbot.process.results import FAILURE -from buildbot.steps.shell import Test +from buildbot.steps.shell import TestNewStyle from buildbot.process.logobserver import LogLineObserver @@ -140,7 +140,7 @@ self.resultCounts[code] = self.resultCounts.get(code, 0) + 1 return -class LitTestCommand(Test): +class LitTestCommand(TestNewStyle): resultNames = {'FAIL':'unexpected failures', 'PASS':'expected passes', 'XFAIL':'expected failures', @@ -159,8 +159,6 @@ super().__init__(*args, **kwargs) self.maxLogs = int(max_logs) self.logObserver = LitLogObserver(self.maxLogs, parseSummaryOnly) - self.addFactoryArguments(max_logs=max_logs) - self.addFactoryArguments(parseSummaryOnly=parseSummaryOnly) self.addLogObserver('stdio', self.logObserver) def evaluateCommand(self, cmd): @@ -175,7 +173,7 @@ return SUCCESS def describe(self, done=False): - description = Test.describe(self, done) or list() + description = TestNewStyle.describe(self, done) or list() for name, count in self.logObserver.resultCounts.items(): if name in self.resultNames: description.append('{0} {1}'.format(count, self.resultNames[name])) diff --git a/zorg/buildbot/process/factory.py b/zorg/buildbot/process/factory.py --- a/zorg/buildbot/process/factory.py +++ b/zorg/buildbot/process/factory.py @@ -20,7 +20,8 @@ self.depends_on_projects = frozenset(depends_on_projects) # Directories. - self.llvm_srcdir = kwargs.pop('llvm_srcdir', None) + self.monorepo_dir = kwargs.pop('llvm_srcdir', None) + self.src_to_build_dir = kwargs.pop('src_to_build_dir', None) self.obj_dir = kwargs.pop('obj_dir', None) self.install_dir = kwargs.pop('install_dir', None) @@ -29,9 +30,10 @@ for k,v in kwargs.items(): setattr(self, k, v) - self.monorepo_dir = self.llvm_srcdir or "llvm-project" + self.monorepo_dir = self.monorepo_dir or "llvm-project" + self.src_to_build_dir = self.src_to_build_dir or 'llvm' self.llvm_srcdir = \ - "%(monorepo_dir)s/llvm" % {'monorepo_dir' : self.monorepo_dir} + "{}/{}".format(self.monorepo_dir, self.src_to_build_dir) self.obj_dir = \ self.obj_dir or "build" diff --git a/zorg/buildbot/util/InformativeMailNotifier.py b/zorg/buildbot/util/InformativeMailNotifier.py --- a/zorg/buildbot/util/InformativeMailNotifier.py +++ b/zorg/buildbot/util/InformativeMailNotifier.py @@ -1,38 +1,104 @@ -# TODO: Add support for clang crash dumps. -# TODO: Better handle unit/regression tests failures - -# TODO: For debug purposes. Remove this later. -from twisted.python import log +import traceback from buildbot.plugins import reporters +from buildbot.process.results import CANCELLED +from buildbot.process.results import EXCEPTION +from buildbot.process.results import FAILURE +from buildbot.process.results import RETRY +from buildbot.process.results import SKIPPED +from buildbot.process.results import SUCCESS +from buildbot.process.results import WARNINGS +from buildbot.process.results import statusToString + +def get_log_details(build): + text = "" + try: + for step in build['steps']: + results = step['results'] + if results == SUCCESS or results == SKIPPED: + continue + + text += "Step {} ({}) {}: {}\n".format(step['number'], step['name'], statusToString(results), step['state_string']) + + logs = step['logs'] + if logs: + log_index = -1 + log_type = 0 + for i, _log in enumerate(logs): + if _log['name'].startswith("FAIL: "): # Use only first logchunk FAIL: + log_type = 3 + log_index = i + elif log_type < 2 and _log['name'].startswith("warnings "): + log_type = 2 + log_index = i + elif log_type < 1 and _log['type'] == "s": # stdio + log_type = 1 + log_index = i + + if log_index < 0: + continue + + log_text = logs[log_index]['content']['content'] + if log_type == 1: + # Parse stdio + lines = log_text.splitlines() + for line in lines[:]: + if line.startswith("h"): + lines.remove(line) + for j, line in enumerate(lines): + if line.startswith("o") or line.startswith("e"): + lines[j] = line[1:] + for j, line in enumerate(lines): + if line.find("FAIL:") != -1 or line.find("FAILED") != -1: + if j > 10: + del lines[:j-10] # Start 10 lines before FAIL + lines = ["..."] + lines + del lines[50:] # Keep up to 50 lines around FAIL + break + if len(lines) > 50: + del lines[:len(lines)-50] # Otherwise keep last 50 lines + lines = ["..."] + lines + + log_text = "\n".join(lines) + + elif logs[log_index]['num_lines'] > 50: + # Keep first 50 lines + lines = log_text.splitlines() + del lines[50:] + log_text = "\n".join(lines + ["..."]) + + text += log_text + "\n" -def _get_logs_and_tracebacks_from_build(build): - # TODO: Implement interesting parts of the logs and tracebacks extraction. - return dict() + except Exception as err: + print("Exception in LLVMMessageFormatter.get_log_details(): {}\n{}".format(err, traceback.format_exc())) + # TODO: We should send something useful in this case. + + return dict(details=text) + +# TODO: Add build reason if we have that valid and available +#Build Reason: {{ build['properties'].get('reason', [""])[0] }} MAIL_TEMPLATE = """\ The Buildbot has detected a {{ status_detected }} on builder {{ buildername }} while building {{ projects }}. + Full details are available at: {{ build_url }} -Buildbot URL: {{ buildbot_url }} + Worker for this Build: {{ workername }} -Build Reason: {{ build['properties'].get('reason', [""])[0] }} -Blamelist: {{ ", ".join(blamelist) }} +Blamelist: + {{ ",\n ".join(blamelist) }} + {{ summary }} + +{{ details }} Sincerely, LLVM Buildbot """ class LLVMMessageFormatter(reporters.MessageFormatter): def buildAdditionalContext(self, master, ctx): - #log.msg(">>> LLVMMessageFormatter.buildAdditionalContext got ctx={}".format(ctx)) ctx.update(self.ctx) - - build = ctx["build"] - build_interesting_data = _get_logs_and_tracebacks_from_build(build) - #log.msg(">>> LLVMMessageFormatter.buildAdditionalContext build_interesting_data={}",format(build_interesting_data)) - ctx["build"].update(build_interesting_data) - + ctx.update(get_log_details(ctx["build"])) LLVMInformativeMailNotifier = LLVMMessageFormatter( template=MAIL_TEMPLATE, diff --git a/zorg/jenkins/build.py b/zorg/jenkins/build.py --- a/zorg/jenkins/build.py +++ b/zorg/jenkins/build.py @@ -615,9 +615,8 @@ footer() header("Run Tests") - run_cmd(conf.lldbbuilddir(), [NINJA, '-v', 'check-debuginfo']) run_cmd(conf.lldbbuilddir(), ['/usr/bin/env', 'TERM=vt100', NINJA, '-v', - 'check-lldb']) + 'check-debuginfo', 'check-lldb', '-k2']) footer() diff --git a/zorg/jenkins/common.groovy b/zorg/jenkins/common.groovy --- a/zorg/jenkins/common.groovy +++ b/zorg/jenkins/common.groovy @@ -81,7 +81,7 @@ try { stage('main') { dir('config') { - git url: 'https://github.com/llvm/llvm-zorg.git', branch: 'master', poll: false + git url: 'https://github.com/llvm/llvm-zorg.git', branch: 'main', poll: false } body() } @@ -102,6 +102,7 @@ def benchmark_pipeline(label, body) { properties([ + disableResume(), parameters([ string(name: 'ARTIFACT'), string(name: 'GIT_DISTANCE'), @@ -119,10 +120,10 @@ def testsuite_pipeline(label, body) { benchmark_pipeline(label) { dir('lnt') { - git url: 'https://github.com/llvm/llvm-lnt.git', branch: 'master', poll: false + git url: 'https://github.com/llvm/llvm-lnt.git', branch: 'main', poll: false } dir('test-suite') { - git url: 'https://github.com/llvm/llvm-test-suite.git', branch: 'master', poll: false + git url: 'https://github.com/llvm/llvm-test-suite.git', branch: 'main', poll: false } body() } diff --git a/zorg/jenkins/inspect_log.py b/zorg/jenkins/inspect_log.py --- a/zorg/jenkins/inspect_log.py +++ b/zorg/jenkins/inspect_log.py @@ -26,7 +26,7 @@ '''Matching engine. Keeps data structures to match a single line against a fixed list of patterns.''' def __init__(self, patterns): - # Create a combined master regex combining all patterns. + # Create a combined main regex combining all patterns. combined = '' merge = '' for pattern in patterns: diff --git a/zorg/jenkins/jobs/jobs/clang-stage1-RA b/zorg/jenkins/jobs/jobs/clang-stage1-RA new file mode 100644 --- /dev/null +++ b/zorg/jenkins/jobs/jobs/clang-stage1-RA @@ -0,0 +1,113 @@ +pipeline { + agent { label 'green-dragon-20' } + + parameters { + string(name: 'GIT_REVISION', defaultValue: '*/main', description: 'Git revision to build') + } + + stages { + stage('Checkout') { + steps { + dir('llvm-project') { + checkout([$class: 'GitSCM', branches: [[name: params.GIT_REVISION]], userRemoteConfigs: [[url: 'https://github.com/llvm/llvm-project.git']]]) + } + dir('llvm-zorg') { + checkout([$class: 'GitSCM', branches: [[name: '*/main']], userRemoteConfigs: [[url: 'https://github.com/llvm/llvm-zorg.git']]]) + } + } + } + stage('Build') { + steps { + timeout(90) { + sh ''' + set -u + rm -rf build.properties + + cd llvm-project + git tag -a -m "First Commit" first_commit 97724f18c79c7cc81ced24239eb5e883bf1398ef || true + + git_desc=$(git describe --match "first_commit") + export GIT_DISTANCE=$(echo ${git_desc} | cut -f 2 -d "-") + + sha=$(echo ${git_desc} | cut -f 3 -d "-") + export GIT_SHA=${sha:1} + + # Also save the LLVM_REV until LNT server is taught about GIT + export LLVM_REV=$(git show -q | grep "llvm-svn:" | cut -f2 -d":" | tr -d " ") + + cd - + + echo "GIT_DISTANCE=$GIT_DISTANCE" > build.properties + echo "GIT_SHA=$GIT_SHA" >> build.properties + echo "ARTIFACT=$JOB_NAME/clang-d$GIT_DISTANCE-g$GIT_SHA-t$BUILD_ID-b$BUILD_NUMBER.tar.gz" >> build.properties + + export PATH=$PATH:/usr/bin:/usr/local/bin + + # Set a MacOS minimum deployment target + export MACOSX_DEPLOYMENT_TARGET=10.9 + + rm -rf clang-build clang-install *.tar.gz + python llvm-zorg/zorg/jenkins/monorepo_build.py cmake build \ + --assertions --cmake-type=RelWithDebInfo \ + --sccache --projects="clang;clang-tools-extra;compiler-rt;libcxx" \ + --cmake-flag='-DLIBCXX_ENABLE_SHARED=OFF'\ + --cmake-flag='-DLIBCXX_ENABLE_STATIC=OFF' \ + --cmake-flag='-DLIBCXX_INCLUDE_TESTS=OFF' \ + --cmake-flag='-DLIBCXX_ENABLE_EXPERIMENTAL_LIBRARY=OFF' + ''' + } + } + } + stage('Test') { + steps { + timeout(90) { + sh ''' + set -u + export PATH=$PATH:/usr/bin:/usr/local/bin + python llvm-zorg/zorg/jenkins/monorepo_build.py cmake testlong + ''' + } + junit 'clang-build/testresults.xunit.xml' + } + } + } + post { + always { + scanForIssues tool: clang() + } + regression { + emailext subject: '$DEFAULT_SUBJECT', + presendScript: '$DEFAULT_PRESEND_SCRIPT', + postsendScript: '$DEFAULT_POSTSEND_SCRIPT', + recipientProviders: [ + [$class: 'CulpritsRecipientProvider'], + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'], + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS', + body:'$DEFAULT_CONTENT' + } + success { + script { + def props = readProperties(file: 'build.properties'); + build job: 'upload_artifact', wait: false, parameters: [string(name: 'ARTIFACT', value: props.ARTIFACT)] + } + build job: 'clang-stage2-Rthinlto_relay', wait: false + build job: 'clang-stage2-cmake-RgSan_relay', wait: false + //build job: 'phase2_modules_relay', wait: false + build job: 'relay-lnt-test-suite', wait: false + build job: 'relay-lnt-ctmark', wait: false + build job: 'relay-test-suite-verify-machineinstrs', wait: false + } + unstable { + build job: 'clang-stage2-Rthinlto_relay', wait: false + build job: 'clang-stage2-cmake-RgSan_relay', wait: false + //build job: 'phase2_modules_relay', wait: false + build job: 'relay-lnt-test-suite', wait: false + build job: 'relay-lnt-ctmark', wait: false + build job: 'relay-test-suite-verify-machineinstrs', wait: false + } + } +} + diff --git a/zorg/jenkins/jobs/jobs/clang-stage1-cmake-RA-incremental b/zorg/jenkins/jobs/jobs/clang-stage1-cmake-RA-incremental new file mode 100644 --- /dev/null +++ b/zorg/jenkins/jobs/jobs/clang-stage1-cmake-RA-incremental @@ -0,0 +1,90 @@ +pipeline { + agent { label 'green-dragon-07' } + + parameters { + string(name: 'GIT_REVISION', defaultValue: '*/main', description: 'Git revision to build') + } + + stages { + stage('Checkout') { + steps { + dir('llvm-project') { + checkout([$class: 'GitSCM', branches: [[name: params.GIT_REVISION]], userRemoteConfigs: [[url: 'https://github.com/llvm/llvm-project.git']]]) + } + dir('llvm-zorg') { + checkout([$class: 'GitSCM', branches: [[name: '*/main']], userRemoteConfigs: [[url: 'https://github.com/llvm/llvm-zorg.git']]]) + } + } + } + stage('Build') { + steps { + timeout(90) { + sh ''' + set -u + rm -rf build.properties + + cd llvm-project + git tag -a -m "First Commit" first_commit 97724f18c79c7cc81ced24239eb5e883bf1398ef || true + + git_desc=$(git describe --match "first_commit") + + export GIT_DISTANCE=$(echo ${git_desc} | cut -f 2 -d "-") + + sha=$(echo ${git_desc} | cut -f 3 -d "-") + export GIT_SHA=${sha:1} + + # Also save the LLVM_REV until LNT server is taught about GIT + export LLVM_REV=$(git show -q | grep "llvm-svn:" | cut -f2 -d":" | tr -d " ") + + cd - + + export PATH=$PATH:/usr/bin:/usr/local/bin + + # Set a MacOS minimum deployment target + export MACOSX_DEPLOYMENT_TARGET=10.9 + + python llvm-zorg/zorg/jenkins/monorepo_build.py cmake build --assertions --projects="clang" + + # Removing the local artifact + rm -rf clang-*.tar.gz + ''' + } + } + } + stage('Test') { + steps { + timeout(90) { + sh ''' + set -u + export PATH=$PATH:/usr/bin:/usr/local/bin + + rm -rf clang-build/testresults.xunit.xml + + python llvm-zorg/zorg/jenkins/monorepo_build.py cmake testlong + ''' + } + junit 'clang-build/testresults.xunit.xml' + } + } + } + post { + always { + scanForIssues tool: clang() + } + /* Disabled email notification. + regression { + emailext subject: '$DEFAULT_SUBJECT', + presendScript: '$DEFAULT_PRESEND_SCRIPT', + postsendScript: '$DEFAULT_POSTSEND_SCRIPT', + recipientProviders: [ + [$class: 'CulpritsRecipientProvider'], + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'], + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS', + body:'$DEFAULT_CONTENT' + } + */ + } +} diff --git a/zorg/jenkins/jobs/jobs/clang-stage2-Rthinlto b/zorg/jenkins/jobs/jobs/clang-stage2-Rthinlto new file mode 100644 --- /dev/null +++ b/zorg/jenkins/jobs/jobs/clang-stage2-Rthinlto @@ -0,0 +1,100 @@ +pipeline { + agent { label 'green-dragon-22' } + + options { disableResume() } + + parameters { + string(name: 'GIT_SHA', defaultValue: '*/main', description: 'Git revision to build') + string(name: 'ARTIFACT', defaultValue: 'clang-stage1-RA/latest', description: 'description') + } + + stages { + stage('Checkout') { + steps { + dir('llvm-project') { + checkout([$class: 'GitSCM', branches: [[name: params.GIT_SHA]], userRemoteConfigs: [[url: 'https://github.com/llvm/llvm-project.git']]]) + } + dir('llvm-zorg') { + checkout([$class: 'GitSCM', branches: [[name: '*/main']], userRemoteConfigs: [[url: 'https://github.com/llvm/llvm-zorg.git']]]) + } + } + } + stage('Build') { + steps { + timeout(600) { + sh ''' + set -u + rm -rf build.properties + + cd llvm-project + git tag -a -m "First Commit" first_commit 97724f18c79c7cc81ced24239eb5e883bf1398ef || true + + git_desc=$(git describe --match "first_commit") + export GIT_DISTANCE=$(echo ${git_desc} | cut -f 2 -d "-") + + sha=$(echo ${git_desc} | cut -f 3 -d "-") + export GIT_SHA=${sha:1} + + cd - + + echo "ARTIFACT=$JOB_NAME/clang-d$GIT_DISTANCE-g$GIT_SHA-t$BUILD_ID-b$BUILD_NUMBER.tar.gz" > build.properties + + export PATH=$PATH:/usr/bin:/usr/local/bin + + rm -rf clang-build clang-install *.tar.gz + + python llvm-zorg/zorg/jenkins/monorepo_build.py fetch + python llvm-zorg/zorg/jenkins/monorepo_build.py clang build \ + --thinlto --projects="clang;compiler-rt;libcxx" \ + --cmake-flag="-DLIBCXX_ENABLE_SHARED=OFF" \ + --cmake-flag="-DLIBCXX_ENABLE_STATIC=OFF" \ + --cmake-flag="-DLIBCXX_INCLUDE_TESTS=OFF" \ + --cmake-flag="-DLIBCXX_ENABLE_EXPERIMENTAL_LIBRARY=OFF" \ + --cmake-flag="-DCMAKE_DSYMUTIL=$WORKSPACE/host-compiler/bin/dsymutil" \ + --sccache + ''' + } + } + } + + stage('Test') { + steps { + timeout(240) { + sh ''' + set -u + export PATH=$PATH:/usr/bin:/usr/local/bin + + rm -rf clang-build/testresults.xunit.xml + + python llvm-zorg/zorg/jenkins/monorepo_build.py clang test + ''' + } + junit 'clang-build/Build/testresults.xunit.xml' + } + } + } + post { + always { + scanForIssues tool: clang() + } + regression { + emailext subject: '$DEFAULT_SUBJECT', + presendScript: '$DEFAULT_PRESEND_SCRIPT', + postsendScript: '$DEFAULT_POSTSEND_SCRIPT', + recipientProviders: [ + [$class: 'CulpritsRecipientProvider'], + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'], + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS', + body:'$DEFAULT_CONTENT' + } + success { + script { + def props = readProperties(file: 'build.properties'); + build job: 'upload_artifact', wait: false, parameters: [string(name: 'ARTIFACT', value: props.ARTIFACT)] + } + } + } +} diff --git a/zorg/jenkins/jobs/jobs/lldb-cmake b/zorg/jenkins/jobs/jobs/lldb-cmake --- a/zorg/jenkins/jobs/jobs/lldb-cmake +++ b/zorg/jenkins/jobs/jobs/lldb-cmake @@ -2,8 +2,8 @@ pipeline { agent { label 'green-dragon-10' } parameters { - string(name: 'GIT_REVISION', defaultValue: '*/master', description: 'Git revision to build') - string(name: 'ARTIFACT', defaultValue: 'clang-stage1-RA/latest', description: 'Compiler artifact to use for building the project') + string(name: 'GIT_REVISION', defaultValue: '*/main', description: 'Git revision to build') + string(name: 'ARTIFACT', defaultValue: 'clang-stage2-Rthinlto/latest', description: 'Compiler artifact to use for building the project') string(name: 'BUILD_TYPE', defaultValue: 'Release', description: 'Default CMake build type; one of: Release, Debug, ...') string(name: 'CLEAN', defaultValue: "false", description: 'Whether or not to clean the build directory before building') } @@ -15,7 +15,7 @@ checkout([$class: 'GitSCM', branches: [[name: params.GIT_REVISION]], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-project.git']]]) } dir('llvm-zorg') { - checkout([$class: 'GitSCM', branches: [[name: '*/master']], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-zorg.git']]]) + checkout([$class: 'GitSCM', branches: [[name: '*/main']], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-zorg.git']]]) } } } @@ -68,7 +68,6 @@ sh ''' set -u export PATH=$PATH:/usr/bin:/usr/local/bin - export LLDB_CAPTURE_REPRODUCER=1 rm -rf test/results.xml diff --git a/zorg/jenkins/jobs/jobs/lldb-cmake-matrix b/zorg/jenkins/jobs/jobs/lldb-cmake-matrix --- a/zorg/jenkins/jobs/jobs/lldb-cmake-matrix +++ b/zorg/jenkins/jobs/jobs/lldb-cmake-matrix @@ -3,8 +3,8 @@ pipeline { agent { label 'green-dragon-23' } parameters { - string(name: 'GIT_REVISION', defaultValue: '*/master', description: 'Git revision to build') - string(name: 'ARTIFACT', defaultValue: 'clang-stage1-RA/latest', description: 'Compiler artifact to use for building the project') + string(name: 'GIT_REVISION', defaultValue: '*/main', description: 'Git revision to build') + string(name: 'ARTIFACT', defaultValue: 'clang-stage2-Rthinlto/latest', description: 'Compiler artifact to use for building the project') string(name: 'BUILD_TYPE', defaultValue: 'Release', description: 'Default CMake build type; one of: Release, Debug, ...') string(name: 'CLEAN', defaultValue: "false", description: 'Whether or not to clean the build directory before building') } @@ -16,7 +16,7 @@ checkout([$class: 'GitSCM', branches: [[name: params.GIT_REVISION]], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-project.git']]]) } dir('llvm-zorg') { - checkout([$class: 'GitSCM', branches: [[name: '*/master']], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-zorg.git']]]) + checkout([$class: 'GitSCM', branches: [[name: '*/main']], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-zorg.git']]]) } } } @@ -271,5 +271,19 @@ junit 'test/results.xml' } } + stage('Test Reproducers') { + steps { + timeout(120) { + sh ''' + export PATH=$PATH:/usr/bin:/usr/local/bin + + set +e + python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake \ + --cmake-test-target check-lldb-reproducers + set -e + ''' + } + } + } } } diff --git a/zorg/jenkins/jobs/jobs/lldb-cmake-reproducers b/zorg/jenkins/jobs/jobs/lldb-cmake-reproducers deleted file mode 100644 --- a/zorg/jenkins/jobs/jobs/lldb-cmake-reproducers +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env groovy -pipeline { - agent { label 'green-dragon-23' } - parameters { - string(name: 'GIT_REVISION', defaultValue: '*/master', description: 'Git revision to build') - string(name: 'ARTIFACT', defaultValue: 'clang-stage1-RA/latest', description: 'Compiler artifact to use for building the project') - string(name: 'BUILD_TYPE', defaultValue: 'Release', description: 'Default CMake build type; one of: Release, Debug, ...') - string(name: 'CLEAN', defaultValue: "false", description: 'Whether or not to clean the build directory before building') - } - stages { - stage('Checkout') { - steps { - timeout(30) { - dir('llvm-project') { - checkout([$class: 'GitSCM', branches: [[name: params.GIT_REVISION]], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-project.git']]]) - } - dir('llvm-zorg') { - checkout([$class: 'GitSCM', branches: [[name: '*/master']], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-zorg.git']]]) - } - } - } - } - stage('Fetch Host Compiler') { - steps { - timeout(10) { - sh ''' - python llvm-zorg/zorg/jenkins/monorepo_build.py fetch - ''' - } - } - } - stage('Build') { - steps { - timeout(90) { - sh ''' - set -u - rm -rf build.properties - - cd llvm-project - git tag -a -m "First Commit" first_commit 97724f18c79c7cc81ced24239eb5e883bf1398ef || true - - git_desc=$(git describe --match "first_commit") - - export GIT_DISTANCE=$(echo ${git_desc} | cut -f 2 -d "-") - - sha=$(echo ${git_desc} | cut -f 3 -d "-") - export GIT_SHA=${sha:1} - - cd - - - python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake build \ - --assertions \ - --projects="clang;libcxx;libcxxabi;compiler-rt;lld;lldb;debuginfo-tests" \ - --compiler-flag="-Wdocumentation" \ - --cmake-type=Release - ''' - } - } - } - stage('Test Reproducers') { - steps { - timeout(120) { - sh ''' - set -u - export PATH=$PATH:/usr/bin:/usr/local/bin - rm -rf test/results.xml - python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake \ - --cmake-test-target check-lldb-reproducers - ''' - } - } - } - } - post { - always { - scanForIssues tool: clang() - junit 'test/results.xml' - } - } -} diff --git a/zorg/jenkins/jobs/jobs/lldb-cmake-sanitized b/zorg/jenkins/jobs/jobs/lldb-cmake-sanitized --- a/zorg/jenkins/jobs/jobs/lldb-cmake-sanitized +++ b/zorg/jenkins/jobs/jobs/lldb-cmake-sanitized @@ -2,8 +2,8 @@ pipeline { agent { label 'green-dragon-23' } parameters { - string(name: 'GIT_REVISION', defaultValue: '*/master', description: 'Git revision to build') - string(name: 'ARTIFACT', defaultValue: 'clang-stage1-RA/latest', description: 'Compiler artifact to use for building the project') + string(name: 'GIT_REVISION', defaultValue: '*/main', description: 'Git revision to build') + string(name: 'ARTIFACT', defaultValue: 'clang-stage2-Rthinlto/latest', description: 'Compiler artifact to use for building the project') string(name: 'BUILD_TYPE', defaultValue: 'Release', description: 'Default CMake build type; one of: Release, Debug, ...') string(name: 'CLEAN', defaultValue: "false", description: 'Whether or not to clean the build directory before building') } @@ -15,7 +15,7 @@ checkout([$class: 'GitSCM', branches: [[name: params.GIT_REVISION]], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-project.git']]]) } dir('llvm-zorg') { - checkout([$class: 'GitSCM', branches: [[name: '*/master']], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-zorg.git']]]) + checkout([$class: 'GitSCM', branches: [[name: '*/main']], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-zorg.git']]]) } } } diff --git a/zorg/jenkins/jobs/jobs/lldb-cmake-standalone b/zorg/jenkins/jobs/jobs/lldb-cmake-standalone --- a/zorg/jenkins/jobs/jobs/lldb-cmake-standalone +++ b/zorg/jenkins/jobs/jobs/lldb-cmake-standalone @@ -2,8 +2,8 @@ pipeline { agent {label 'green-dragon-23'} parameters { - string(name: 'GIT_REVISION', defaultValue: '*/master', description: 'Git revision to build') - string(name: 'ARTIFACT', defaultValue: 'clang-stage1-RA/latest', description: 'Compiler artifact to use for building the project') + string(name: 'GIT_REVISION', defaultValue: '*/main', description: 'Git revision to build') + string(name: 'ARTIFACT', defaultValue: 'clang-stage2-Rthinlto/latest', description: 'Compiler artifact to use for building the project') string(name: 'BUILD_TYPE', defaultValue: 'Release', description: 'Default CMake build type; one of: Release, Debug, ...') string(name: 'CLEAN', defaultValue: "false", description: 'Whether or not to clean the build directory before building') } @@ -15,7 +15,7 @@ checkout([$class: 'GitSCM', branches: [[name: params.GIT_REVISION]], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-project.git']]]) } dir('llvm-zorg') { - checkout([$class: 'GitSCM', branches: [[name: '*/master']], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-zorg.git']]]) + checkout([$class: 'GitSCM', branches: [[name: '*/main']], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-zorg.git']]]) } } } diff --git a/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-O0-g b/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-O0-g --- a/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-O0-g +++ b/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-O0-g @@ -1,9 +1,11 @@ #!/usr/bin/env groovy def common = evaluate readTrusted('zorg/jenkins/common.groovy') common.testsuite_pipeline(label: 'green-dragon-12') { - sh """ + timeout(30) { + sh """ LNT_FLAGS+=" -C config/tasks/cmake/caches/target-arm64-iphoneos.cmake" LNT_FLAGS+=" -C config/tasks/cmake/caches/opt-O0-g.cmake" config/tasks/task jenkinsrun config/tasks/lnt-ctmark.sh -a compiler="${params.ARTIFACT}" -D LNT_FLAGS="\${LNT_FLAGS}" - """ + """ + } } diff --git a/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-O3-flto b/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-O3-flto --- a/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-O3-flto +++ b/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-O3-flto @@ -1,9 +1,11 @@ #!/usr/bin/env groovy def common = evaluate readTrusted('zorg/jenkins/common.groovy') common.testsuite_pipeline(label: 'green-dragon-12') { - sh """ + timeout(30) { + sh """ LNT_FLAGS+=" -C config/tasks/cmake/caches/target-arm64-iphoneos.cmake" LNT_FLAGS+=" -C config/tasks/cmake/caches/opt-O3-flto.cmake" config/tasks/task jenkinsrun config/tasks/lnt-ctmark.sh -a compiler="${params.ARTIFACT}" -D LNT_FLAGS="\${LNT_FLAGS}" - """ + """ + } } diff --git a/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-Os b/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-Os --- a/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-Os +++ b/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-Os @@ -1,9 +1,11 @@ #!/usr/bin/env groovy def common = evaluate readTrusted('zorg/jenkins/common.groovy') common.testsuite_pipeline(label: 'green-dragon-12') { - sh """ + timeout(30) { + sh """ LNT_FLAGS+=" -C config/tasks/cmake/caches/target-arm64-iphoneos.cmake" LNT_FLAGS+=" -C config/tasks/cmake/caches/opt-Os.cmake" config/tasks/task jenkinsrun config/tasks/lnt-ctmark.sh -a compiler="${params.ARTIFACT}" -D LNT_FLAGS="\${LNT_FLAGS}" - """ + """ + } } diff --git a/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-Oz b/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-Oz --- a/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-Oz +++ b/zorg/jenkins/jobs/jobs/lnt-ctmark-aarch64-Oz @@ -1,9 +1,11 @@ #!/usr/bin/env groovy def common = evaluate readTrusted('zorg/jenkins/common.groovy') common.testsuite_pipeline(label: 'green-dragon-12') { - sh """ + timeout(30) { + sh """ LNT_FLAGS+=" -C config/tasks/cmake/caches/target-arm64-iphoneos.cmake" LNT_FLAGS+=" -C config/tasks/cmake/caches/opt-Oz.cmake" config/tasks/task jenkinsrun config/tasks/lnt-ctmark.sh -a compiler="${params.ARTIFACT}" -D LNT_FLAGS="\${LNT_FLAGS}" - """ + """ + } } diff --git a/zorg/jenkins/jobs/util/make_pipeline.py b/zorg/jenkins/jobs/util/make_pipeline.py --- a/zorg/jenkins/jobs/util/make_pipeline.py +++ b/zorg/jenkins/jobs/util/make_pipeline.py @@ -32,7 +32,7 @@ - */master + */main false diff --git a/zorg/jenkins/monorepo_build.py b/zorg/jenkins/monorepo_build.py --- a/zorg/jenkins/monorepo_build.py +++ b/zorg/jenkins/monorepo_build.py @@ -202,7 +202,7 @@ def branch(self): """Figure out the source branch name. - Not using GIT_BRANCH env var from Jenkins as that includes the + Not using GIT_BRANCH env var from Jenkins as that includes the remote name too. """ if not os.environ.get('TESTING', False): @@ -213,16 +213,16 @@ cmd = ['git', '-C', conf.srcdir(), 'rev-parse', 'HEAD'] out = run_collect_output(cmd, stderr=subprocess.STDOUT).strip() return out - return 'master' + return 'main' def link_memory_usage(self): """Guesstimate the maximum link memory usage for this build. - We are only building master here so we will just use that value + We are only building main here so we will just use that value """ # Determinited experimentally. - usages = {'master': 3.5} - if self.branch() == 'master': - return usages['master'] + usages = {'main': 3.5} + if self.branch() == 'main': + return usages['main'] else: raise NotImplementedError( "Unknown link memory usage." + self.branch()) @@ -538,7 +538,7 @@ # Construct lit arguments. lit_args = ['-v', '--time-tests', '--shuffle', - '--xunit-xml-output={}'.format(results_file), '-v'] + '--xunit-xml-output={}'.format(results_file), '-v'] if conf.max_parallel_tests: lit_args.extend(['-j', conf.max_parallel_tests]) if variant == 'sanitized': @@ -609,15 +609,15 @@ run_cmd(conf.lldbbuilddir(), [NINJA, '-v', 'install']) footer() - if target == 'all' or target == 'testlong': - header("Run Debug Info Tests") - run_cmd(conf.lldbbuilddir(), [NINJA, '-v', 'check-debuginfo']) - footer() - if target == 'all' or target == 'test' or target == 'testlong': header("Run Tests") - run_cmd(conf.lldbbuilddir(), - ['/usr/bin/env', 'TERM=vt100', NINJA, '-v', 'check-lldb']) + if variant == 'matrix' or variant == 'sanitized': + test_command = ['/usr/bin/env', 'TERM=vt100', NINJA, + '-v', 'check-lldb'] + else: + test_command = ['/usr/bin/env', 'TERM=vt100', NINJA, + '-v', 'check-debuginfo', 'check-lldb', '-k2'] + run_cmd(conf.lldbbuilddir(), test_command) footer() for test_target in conf.cmake_test_targets: @@ -778,11 +778,11 @@ with open(dest, "wb") as local_file: local_file.write(f.read()) - except urllib.HTTPError as e: + except urllib.error.HTTPError as e: print("HTTP Error:", e.code, url) sys.exit(1) - except urllib.URLError as e: + except urllib.error.URLError as e: print("URL Error:", e.reason, url) sys.exit(1) print("done.")