diff --git a/buildbot/google/docker/README.md b/buildbot/google/docker/README.md --- a/buildbot/google/docker/README.md +++ b/buildbot/google/docker/README.md @@ -21,3 +21,6 @@ the container. The secret file shall just contain the password in plain text. Kubernetes offers a [mechanism to handle secrets](https://kubernetes.io/docs/concepts/configuration/secret/). + +# Setting up Windows VM for development +See [windows.md](windows.md). diff --git a/buildbot/google/docker/build_deploy.sh b/buildbot/google/docker/build_deploy.sh --- a/buildbot/google/docker/build_deploy.sh +++ b/buildbot/google/docker/build_deploy.sh @@ -15,19 +15,17 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" IMAGE_NAME="${1%/}" - -# increment version number cd "${DIR}/${IMAGE_NAME}" -# get version numbers from repository -# FIXME: use variables to configure URL -ALL_VERSIONS=$(gcloud container images list-tags gcr.io/sanitizer-bots/${IMAGE_NAME} --format=text | \ - awk '/tags.*:\W+[0-9]+$/ {print $2}') -# read local version number from file and add it to the array -ALL_VERSIONS+=($(cat VERSION)) -# find maximum version number and increment it -VERSION=$(echo "${ALL_VERSIONS[*]}" | sort -nr | head -n1) -VERSION=$(( ${VERSION} + 1 )) +# read local version number from file and increment it +# TODO: maybe also get latest version number from repository and take the +# maximum of local and repo versions. +VERSION=$(( $(cat VERSION) + 1 )) + +# on Windows: USER=$USERNAME +if [[ "${OS}" == "Windows_NT" ]] ; then + USER="${USERNAME}" +fi # get the git hash and add some suffixes GIT_HASH=$(git rev-parse HEAD) diff --git a/buildbot/google/docker/build_run.sh b/buildbot/google/docker/build_run.sh --- a/buildbot/google/docker/build_run.sh +++ b/buildbot/google/docker/build_run.sh @@ -12,7 +12,7 @@ # optional: #===----------------------------------------------------------------------===// -set -eux +set -eu DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" IMAGE_NAME="${1%/}" @@ -25,5 +25,24 @@ cd "${DIR}/${IMAGE_NAME}" +# Mount a volume "workertest" to persit across test runs. +# Use this to keep e.g. a git checkout or partial build across runs +if [[ $(docker volume ls | grep workertest | wc -l) == 0 ]] ; then + docker volume create workertest +fi + +# Volume to presist the build cache e.g. ccache or sccache. +# This will speed up local testing. +if [[ $(docker volume ls | grep workercache | wc -l) == 0 ]] ; then + docker volume create workercache +fi + +# Define arguments for mounting the volumes +# These differ on Windows and Linux +VOLUMES="-v ${SECRET_STORAGE}:/secrets -v workertest:/test" +if [[ "${OS}" == "Windows_NT" ]] ; then + VOLUMES="-v ${SECRET_STORAGE}:c:\\volumes\\secrets -v workertest:c:\volumes\\test -v workercache:c:\sccache" +fi + docker build -t "${IMAGE_NAME}:latest" . -docker run -it -v "${SECRET_STORAGE}":/secrets "${IMAGE_NAME}" ${CMD} +docker run -it ${VOLUMES} "${IMAGE_NAME}:latest" ${CMD} diff --git a/buildbot/google/docker/buildbot-windows10-vs2019/Dockerfile b/buildbot/google/docker/buildbot-windows10-vs2019/Dockerfile new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-windows10-vs2019/Dockerfile @@ -0,0 +1,22 @@ +# escape=` +FROM gcr.io/sanitizer-bots/windows-base-vscode2019:5 + +ENV WORKER_NAME=windows10-vs2019 + +# copy admin information +RUN md %WORKER_NAME%\info +COPY admin ${WORKER_NAME}\info + +# copy script to start the agent +COPY run.ps1 . + +# Set location for sccache cache +ENV SCCACHE_DIR="C:\volumes\sccache" + +# Set the maximum sccache local cache size +ENV SCCACHE_CACHE_SIZE="5G" + +# Configure 32bit tools ("x86") instead of 64bit ("amd64") +ENTRYPOINT ["C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat", "-arch=x86", "-host_arch=amd64", "&&", "powershell.exe", "-NoLogo", "-ExecutionPolicy", "Bypass"] + +CMD ["c:\\buildbot\\run.ps1"] diff --git a/buildbot/google/docker/buildbot-windows10-vs2019/VERSION b/buildbot/google/docker/buildbot-windows10-vs2019/VERSION new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-windows10-vs2019/VERSION @@ -0,0 +1 @@ +4 diff --git a/buildbot/google/docker/buildbot-windows10-vs2019/admin b/buildbot/google/docker/buildbot-windows10-vs2019/admin new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-windows10-vs2019/admin @@ -0,0 +1 @@ +Christian Kühnel diff --git a/buildbot/google/docker/buildbot-windows10-vs2019/run.ps1 b/buildbot/google/docker/buildbot-windows10-vs2019/run.ps1 new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-windows10-vs2019/run.ps1 @@ -0,0 +1,33 @@ +# Stop the script on the first error +$ErrorActionPreference = "Stop" + +# Read password from file +$WORKER_PASSWORD=Get-Content "C:\volumes\secrets\token" + +# Write host information +# configure powershell output to use UTF-8, otherwiese buildbot can't read the file +$HOST_FILE="$env:WORKER_NAME\info\host" +$PSDefaultParameterValues['Out-File:Encoding'] = 'utf8'; ` +Write-Output "Windows version: $(cmd /c ver)" > $HOST_FILE +Write-Output "VCTools version: $env:VCToolsVersion" >> $HOST_FILE +Write-Output "Cores : $((Get-CimInstance Win32_ComputerSystem).NumberOfLogicalProcessors)" >> $HOST_FILE +Write-Output "RAM : $([int][Math]::Round((Get-CimInstance Win32_ComputerSystem).TotalPhysicalMemory / 1073741824)) GB" >> $HOST_FILE + +# create the worker +Write-Output "creating worker..." +buildslave create-slave --keepalive=200 "$env:WORKER_NAME" ` + "lab.llvm.org:9994" "$env:WORKER_NAME" "$WORKER_PASSWORD" + +# Note: Powershell does NOT exit on non-zero return codes, so we need to check manually. +if ($LASTEXITCODE -ne 0) { throw "Exit code is $LASTEXITCODE" } + +# start the daemon, as this does not print and logs to std out, run it in the background +Write-Output "starting worker..." +cmd /c Start /B buildslave start "$env:WORKER_NAME" + +# Wait a bit until the logfile exists +Start-Sleep -s 5 + +# To keep the container running and produce log outputs: dump the worker +# log to stdout, this is the windows equivalent of `tail -f` +Get-Content -Path "$env:WORKER_NAME\twistd.log" -Wait diff --git a/buildbot/google/docker/pod_login.sh b/buildbot/google/docker/pod_login.sh --- a/buildbot/google/docker/pod_login.sh +++ b/buildbot/google/docker/pod_login.sh @@ -13,15 +13,23 @@ # pod name fragement : # part of the name of the pod to log in, eg. name of the # deployment, if we have only one of them +# command (optional): +# Command to be run in the container. Default: /bin/bash #===----------------------------------------------------------------------===// set -eu WORKLOAD_NAME=$1 +CMD="/bin/bash" +if [ "$#" -eq 2 ]; +then + CMD="$2" +fi + # get name of the pod POD=$(kubectl get pod -o name | grep "$1") # FIXME: exit if more than one pod is returned # login to the pod -kubectl exec --stdin --tty "${POD}" -- /bin/bash +kubectl exec --stdin --tty "${POD}" -- "${CMD}" diff --git a/buildbot/google/docker/windows-base-vscode2019/Dockerfile b/buildbot/google/docker/windows-base-vscode2019/Dockerfile new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/windows-base-vscode2019/Dockerfile @@ -0,0 +1,82 @@ +# escape=` +# Windows is picky about host and container OS versions. Recommondation: +# Use "LTSC" releases for both. +# https://docs.microsoft.com/en-us/virtualization/windowscontainers/deploy-containers/version-compatibility +# https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#choose_your_windows_server_node_image +FROM mcr.microsoft.com/dotnet/framework/sdk:4.8-windowsservercore-ltsc2019 + +# Restore the default Windows shell for correct batch processing. +SHELL ["cmd", "/S", "/C"] + +# Download the Build Tools bootstrapper. +ADD https://aka.ms/vs/16/release/vs_buildtools.exe C:\TEMP\vs_buildtools.exe + +# Download channel for fixed install. +ARG CHANNEL_URL=https://aka.ms/vs/16/release/channel +ADD ${CHANNEL_URL} C:\TEMP\VisualStudio.chman + +# Install Build Tools with C++ workload. +# - Documentation for docker installation +# https://docs.microsoft.com/en-us/visualstudio/install/build-tools-container?view=vs-2019 +# - Documentation on workloads +# https://docs.microsoft.com/en-us/visualstudio/install/workload-component-id-vs-build-tools?view=vs-2019#c-build-tools +# - Documentation on flags +# https://docs.microsoft.com/en-us/visualstudio/install/use-command-line-parameters-to-install-visual-studio?view=vs-2019 +RUN C:\TEMP\vs_buildtools.exe --quiet --wait --norestart --nocache ` + --channelUri C:\TEMP\VisualStudio.chman ` + --installChannelUri C:\TEMP\VisualStudio.chman ` + --installPath C:\BuildTools ` + --add Microsoft.VisualStudio.Workload.VCTools --includeRecommended ` + --add Microsoft.VisualStudio.Component.VC.ATL ` + || IF "%ERRORLEVEL%"=="3010" EXIT 0 + +# install chocolately as package manager +RUN powershell -NoProfile -InputFormat None -Command ` + iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')) ; ` + choco feature disable --name showDownloadProgress + +# install tools as described in https://llvm.org/docs/GettingStartedVS.html +# and a few more that were not documented... +RUN choco install -y ninja +RUN choco install -y git +RUN choco install -y cmake +RUN choco install -y gnuwin +# lit runs best with Python3, Buildbot worker needs Python2 +RUN choco install -y python2 python3 +RUN choco install -y sccache +# Install psutils for python3 for lit +RUN pip3 install psutil +# Install buildbot using python2 +RUN pip2 install --trusted-host pythonhosted.org buildbot-slave==0.8.11 pywin32 + +# configure Python encoding +ENV PYTHONIOENCODING=UTF-8 + +# update the path variable +RUN powershell -NoProfile -InputFormat None -Command ` + $path = $env:path + ';c:\Program Files (x86)\GnuWin32\bin;C:\Program Files\CMake\bin'; ` + Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment\' -Name Path -Value $path + +# use this folder to store the worksapce' +WORKDIR C:\buildbot + +# Root folder for all docker volumes. Make it explicite that all subfolders +# are volumes. Volumes are a common pitfall for config errors. +RUN md c:\volumes + +# storage for cached builds +VOLUME C:\volumes\sccache + +# storage for secrets +VOLUME C:\volumes\secrets + +# support long file names during git checkout +RUN git config --system core.longpaths true & ` + git config --global core.autocrlf false + +# Define the entry point for the docker container. +# This entry point starts the developer command prompt and launches the PowerShell shell. +# +# For running manually: +# C:\BuildTools\Common7\Tools\VsDevCmd.bat -arch=amd64 -host_arch=amd64 +ENTRYPOINT ["C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat", "-arch=amd64", "-host_arch=amd64", "&&", "powershell.exe", "-NoLogo", "-ExecutionPolicy", "Bypass"] diff --git a/buildbot/google/docker/windows-base-vscode2019/VERSION b/buildbot/google/docker/windows-base-vscode2019/VERSION new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/windows-base-vscode2019/VERSION @@ -0,0 +1 @@ +5 diff --git a/buildbot/google/docker/windows.md b/buildbot/google/docker/windows.md new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/windows.md @@ -0,0 +1,68 @@ +# Overview + +Running a Windows buildbot differs from Linux a bit. So this document will give +hints on the differences. + +# Create VM +To create a new Dockerfile and you do not have a local Windows machine, create +a VM in the the cloud. + +* Use the latest Windows image with "Desktop experience" in the description. +* Use something with at least 16 cores, to get reasonable build speed while + testing your images. +* Get 200GB of persistent SSD, this is much faster than the normal presistent + storage. +* In the "Cloud API access scopes" select "Read Write" for "Storage" to be able + to upload docker images form the VM. + +# setup your workstation +For frequent access to your Windows VM it's conveneint to install Remmina, a +remote desktop client for Linux, to connect the the Windows VM. + +# Configure the VM +1. install [Chocolately](https://chocolatey.org/docs/installation) as package + manager. +1. Install your development tools, a good starting point is: + ``` + choco install -y git vscode googlechrome arcanist vim + `` +1. Setup your github account and push access. Start "Git bash" and run + `ssh-keygen`, upload your public key to Github. +1. Setup your Phabricator account for `arc diff`. +1. `git clone ssh://git@github.com/llvm/llvm-zorg.git` +1. To install docker, run in an powershell with admin rights: + ```powershell + Install-PackageProvider -Name NuGet -Force + Install-Module -Name DockerMsftProvider -Repository PSGallery -Force + Install-Package -Name docker -ProviderName DockerMsftProvider -Force + sc.exe config docker start=delayed-auto + ``` + Then reboot your VM to start the Docker service. +1. Maybe disable Windows Defender anti vrius "realtime protection" to get better + IO performance. +1. Install [Google Cloud SDK](https://cloud.google.com/sdk/install). Installing + via chocolately is sometimes broken. Then run `gcloud auth login` and + `gcloud auth configure-docker` to be able to push images to the registry. +1. Create a file `token` somewhere and store your worker password there. You + will need this to test your worker locally. + +# General Hints + +## Get ideas from LLVM premerge-checks +[Premerge-testing](https://github.com/google/llvm-premerge-checks/) is also +running LLVM builds on Windows. You might be able to get some ideas from there. + +## Bash on Windows +Git on Windows includes a bash. Right-click the Start menu und launch a +"Windows Powershell (Admin)" (with admin rights, as docker requires that) and +run `bash` from there. Then you should be able to use the helper scripts +(e.g. `build_run.sh`) on Windows as well. + +## Resource Usage +If Windows seems stuck, use the `Resource Monitor` to check what the processes +are doing. If you need even more insight, e.g. debugging builds or test, use the +`Process Explorer` (`choco install procexp`) to figure out what's going on. + +## Windows base image with Visual Studio 2019 +Reuse the `windows-base-vscode2019` image, it contains everything you need to +build and test with Visual Studio 2019. diff --git a/buildbot/google/terraform/README.md b/buildbot/google/terraform/buildbot-mlir-nvidia/README.md rename from buildbot/google/terraform/README.md rename to buildbot/google/terraform/buildbot-mlir-nvidia/README.md diff --git a/buildbot/google/terraform/deployment-mlir-nvidia.yaml b/buildbot/google/terraform/buildbot-mlir-nvidia/deployment-mlir-nvidia.yaml rename from buildbot/google/terraform/deployment-mlir-nvidia.yaml rename to buildbot/google/terraform/buildbot-mlir-nvidia/deployment-mlir-nvidia.yaml diff --git a/buildbot/google/terraform/main.tf b/buildbot/google/terraform/buildbot-mlir-nvidia/main.tf rename from buildbot/google/terraform/main.tf rename to buildbot/google/terraform/buildbot-mlir-nvidia/main.tf diff --git a/buildbot/google/terraform/outputs.tf b/buildbot/google/terraform/buildbot-mlir-nvidia/outputs.tf rename from buildbot/google/terraform/outputs.tf rename to buildbot/google/terraform/buildbot-mlir-nvidia/outputs.tf diff --git a/buildbot/google/terraform/terraform.tfvars b/buildbot/google/terraform/buildbot-mlir-nvidia/terraform.tfvars rename from buildbot/google/terraform/terraform.tfvars rename to buildbot/google/terraform/buildbot-mlir-nvidia/terraform.tfvars diff --git a/buildbot/google/terraform/variables.tf b/buildbot/google/terraform/buildbot-mlir-nvidia/variables.tf rename from buildbot/google/terraform/variables.tf rename to buildbot/google/terraform/buildbot-mlir-nvidia/variables.tf diff --git a/buildbot/osuosl/master/config/builders.py b/buildbot/osuosl/master/config/builders.py --- a/buildbot/osuosl/master/config/builders.py +++ b/buildbot/osuosl/master/config/builders.py @@ -1,18 +1,13 @@ from buildbot.process.properties import WithProperties from zorg.buildbot.builders import ClangBuilder -from zorg.buildbot.builders import LLVMBuilder from zorg.buildbot.builders import PollyBuilder from zorg.buildbot.builders import LLDBBuilder -from zorg.buildbot.builders import LLDBuilder -from zorg.buildbot.builders import ClangAndLLDBuilder from zorg.buildbot.builders import SanitizerBuilder -from zorg.buildbot.builders import SanitizerBuilderII from zorg.buildbot.builders import OpenMPBuilder from zorg.buildbot.builders import LibcxxAndAbiBuilder from zorg.buildbot.builders import SphinxDocsBuilder from zorg.buildbot.builders import ABITestsuitBuilder -from zorg.buildbot.builders import ClangLTOBuilder3Stage from zorg.buildbot.builders import ClangLTOBuilder from zorg.buildbot.builders import UnifiedTreeBuilder from zorg.buildbot.builders import CUDATestsuiteBuilder @@ -819,6 +814,9 @@ '-DLLVM_TARGETS_TO_BUILD=X86', '-DCMAKE_C_COMPILER_LAUNCHER=sccache', '-DCMAKE_CXX_COMPILER_LAUNCHER=sccache', + # FIXME: DIA is not working on this machine, so disabling it until we + # have a proper fix + '-D LLVM_ENABLE_DIA_SDK=OFF', ])}, ] @@ -1569,30 +1567,24 @@ def _get_experimental_scheduled_builders(): return [ - {'name' : "clang-cuda-build", - 'slavenames' : ["cuda-build-test-01"], - 'builddir' : "clang-cuda-build", - 'factory' : CUDATestsuiteBuilder.getCUDATestsuiteBuildFactory( - useTwoStage=False, - test=True, - stage1_config='Release', - extra_cmake_args=[ - '-DLLVM_ENABLE_ASSERTIONS=ON', - "-DCMAKE_C_COMPILER:FILEPATH=/usr/bin/clang", - "-DCMAKE_CXX_COMPILER:FILEPATH=/usr/bin/clang++", - "-DLIBUNWIND_LIBCXX_PATH=../llvm/libcxx", - ], - externals="/home/botanist/bots/externals", - gpu_arch_list=["sm_35", "sm_61"], - gpu_devices=["GPU-af66efa4", # K40c(sm_35), - "GPU-44fe2444" # GTX1070(sm_61) - ], - cuda_jobs = 4, # Speeds up test execution time by ~2x. - extra_ts_cmake_args=[], - enable_thrust_tests=False, - ), + {'name': "clang-cuda-k80", + 'slavenames' :["cuda-k80-0"], + 'builddir': "clang-cuda-k80", + 'factory': AnnotatedBuilder.getAnnotatedBuildFactory(script="/buildbot/cuda-build", + checkout_llvm_sources=False), + 'category' : 'clang'}, + {'name': "clang-cuda-p4", + 'slavenames' :["cuda-p4-0"], + 'builddir': "clang-cuda-p4", + 'factory': AnnotatedBuilder.getAnnotatedBuildFactory(script="/buildbot/cuda-build", + checkout_llvm_sources=False), + 'category' : 'clang'}, + {'name': "clang-cuda-t4", + 'slavenames' :["cuda-t4-0"], + 'builddir': "clang-cuda-t4", + 'factory': AnnotatedBuilder.getAnnotatedBuildFactory(script="/buildbot/cuda-build", + checkout_llvm_sources=False), 'category' : 'clang'}, - {'name': "clang-ve-ninja", 'slavenames':["nec-arrproto41"], 'builddir':"clang-ve-ninja", diff --git a/buildbot/osuosl/master/config/slaves.py b/buildbot/osuosl/master/config/slaves.py --- a/buildbot/osuosl/master/config/slaves.py +++ b/buildbot/osuosl/master/config/slaves.py @@ -155,6 +155,10 @@ # Ubuntu 16.04 x86_64, 2 x Intel(R) Xeon(R) CPU E5-2690 v3 @ 2.60GHz, 64GB of RAM create_slave("cuda-build-test-01", properties={'jobs': 72}, max_builds=1), + # WIP migration of the CUDA buildbot to GCE. + create_slave("cuda-k80-0", max_builds=1), + create_slave("cuda-p4-0", max_builds=1), + create_slave("cuda-t4-0", max_builds=1), # Ubuntu 14.04 x86_64, AMD Athlon(tm) 5150 APU with Radeon(tm) R3, 8GiB RAM create_slave("am1i-slv1", properties={'jobs': 8}), diff --git a/buildbot/osuosl/master/config/status.py b/buildbot/osuosl/master/config/status.py --- a/buildbot/osuosl/master/config/status.py +++ b/buildbot/osuosl/master/config/status.py @@ -182,7 +182,7 @@ extraRecipients = ["tra+buildbot@google.com"], subject="Build %(builder)s Failure", mode = "failing", - builders = ["clang-cuda-build"], + builders = ["clang-cuda-k80", "clang-cuda-p4", "clang-cuda-t4"], addLogs=False, num_lines = 15), InformativeMailNotifier( diff --git a/test/buildbot/builders/Import.py b/test/buildbot/builders/Import.py --- a/test/buildbot/builders/Import.py +++ b/test/buildbot/builders/Import.py @@ -1,18 +1,11 @@ # RUN: python %s import zorg -from zorg.buildbot.builders import ClangBuilder, LLVMBuilder, LLVMGCCBuilder -from zorg.buildbot.builders import NightlytestBuilder +from zorg.buildbot.builders import ClangBuilder from zorg.buildbot.builders import SanitizerBuilder # Just check that we can instantiate the build factors, what else can we do? print ClangBuilder.getClangBuildFactory() -print LLVMBuilder.getLLVMBuildFactory() - -print LLVMGCCBuilder.getLLVMGCCBuildFactory() - -print NightlytestBuilder.getFastNightlyTestBuildFactory('x86_64-apple-darwin10') - print SanitizerBuilder.getSanitizerBuildFactory() diff --git a/zorg/buildbot/builders/AnnotatedBuilder.py b/zorg/buildbot/builders/AnnotatedBuilder.py --- a/zorg/buildbot/builders/AnnotatedBuilder.py +++ b/zorg/buildbot/builders/AnnotatedBuilder.py @@ -11,7 +11,8 @@ env=None, extra_args=None, timeout=1200, - is_legacy_mode=False): + is_legacy_mode=False, + checkout_llvm_sources=True): """ Returns a new build factory that uses AnnotatedCommand, which allows the build to be run by version-controlled scripts that do @@ -76,19 +77,22 @@ src_dir='llvm-zorg', alwaysUseLatest=True) - f.addGetSourcecodeSteps() - + if checkout_llvm_sources: + f.addGetSourcecodeSteps() extra_args_with_props = [WithProperties(arg) for arg in extra_args] # Explicitly use '/' as separator, because it works on *nix and Windows. - script_path = "../llvm-zorg/zorg/buildbot/builders/annotated/%s" % (script) + if script.startswith('/'): + command = [script] + else: + script_path = "../llvm-zorg/zorg/buildbot/builders/annotated/%s" % (script) + command = ["python", script_path, WithProperties("--jobs=%(jobs:-)s")] + command += extra_args_with_props + f.addStep(AnnotatedCommand(name="annotate", description="annotate", timeout=timeout, haltOnFailure=True, - command=["python", - script_path, - WithProperties("--jobs=%(jobs:-)s")] - + extra_args_with_props, + command=command, env=merged_env)) return f diff --git a/zorg/buildbot/builders/ClangAndLLDBuilder.py b/zorg/buildbot/builders/ClangAndLLDBuilder.py deleted file mode 100644 --- a/zorg/buildbot/builders/ClangAndLLDBuilder.py +++ /dev/null @@ -1,177 +0,0 @@ -import os - -import buildbot -import buildbot.process.factory -from buildbot.steps.source import SVN -from buildbot.steps.shell import ShellCommand -from buildbot.steps.shell import WarningCountingShellCommand -from buildbot.steps.shell import SetProperty -from buildbot.process.properties import WithProperties, Property -from zorg.buildbot.commands.LitTestCommand import LitTestCommand -from zorg.buildbot.commands.NinjaCommand import NinjaCommand - -import zorg.buildbot.builders.Util as builders_util - -def getClangAndLLDBuildFactory( - clean=True, - env=None, - withLLD=True, - extraCmakeOptions=None, - extraCompilerOptions=None, - buildWithSanitizerOptions=None, - triple=None, - isMSVC=False, - # Choose VS tools to build with. For example, - # "autodetect" to find the latest installed Visual Studio, or - # %VS140COMNTOOLS% to selects the 2015 toolchain. - vs=None, - target_arch=None, - prefixCommand=["nice", "-n", "10"], # For backward compatibility. - extraLitArgs=None - ): - - llvm_srcdir = "llvm.src" - llvm_objdir = "llvm.obj" - - # Prepare environmental variables. Set here all env we want everywhere. - merged_env = { - 'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences. - } - if env is not None: - assert not (isMSVC and vs), "Can't have custom builder env vars with VS" - # Overwrite pre-set items with the given ones, so user can set anything. - merged_env.update(env) - - f = buildbot.process.factory.BuildFactory() - - # Determine the build directory. - f.addStep(buildbot.steps.shell.SetProperty(name="get_builddir", - command=["pwd"], - property="builddir", - description="set build dir", - workdir=".", - env=merged_env)) - # Get LLVM, Clang and LLD. - f.addStep(SVN(name='svn-llvm', - mode='update', - baseURL='http://llvm.org/svn/llvm-project/llvm/', - defaultBranch='trunk', - workdir=llvm_srcdir)) - - # Sanitizer runtime in compiler-rt, and cannot be built with sanitizer compiler. - if buildWithSanitizerOptions is None: - f.addStep(SVN(name='svn-compiler-rt', - mode='update', - baseURL='http://llvm.org/svn/llvm-project/compiler-rt/', - defaultBranch='trunk', - workdir='%s/projects/compiler-rt' % llvm_srcdir)) - - f.addStep(SVN(name='svn-clang', - mode='update', - baseURL='http://llvm.org/svn/llvm-project/cfe/', - defaultBranch='trunk', - workdir='%s/tools/clang' % llvm_srcdir)) - f.addStep(SVN(name='svn-clang-tools-extra', - mode='update', - baseURL='http://llvm.org/svn/llvm-project/clang-tools-extra/', - defaultBranch='trunk', - workdir='%s/tools/clang/tools/extra' % llvm_srcdir)) - if withLLD: - f.addStep(SVN(name='svn-lld', - mode='update', - baseURL='http://llvm.org/svn/llvm-project/lld/', - defaultBranch='trunk', - workdir='%s/tools/lld' % llvm_srcdir)) - - # Clean directory, if requested. - if clean: - shellCommand = ["rm", "-rf", llvm_objdir] - if isMSVC: - shellCommand = ["rmdir", "/S", "/Q", llvm_objdir] - f.addStep(ShellCommand(name="rm-llvm_objdir", - command=shellCommand, - haltOnFailure=False, - flunkOnFailure=False, - description=["rm build dir", "llvm"], - workdir=".", - env=merged_env)) - - # Create configuration files with cmake. - options = ["-Wdocumentation", "-Wno-documentation-deprecated-sync"] - if isMSVC: - options = [] - if extraCompilerOptions: - options += extraCompilerOptions - - if buildWithSanitizerOptions: - options += buildWithSanitizerOptions - - lit_args = ["-v"] - if extraLitArgs: - lit_args += extraLitArgs - - lit_args = ["-DLLVM_LIT_ARGS=\"%s\"" % " ".join(lit_args)] - - cmakeCommand = [ - "cmake", - "-DCMAKE_BUILD_TYPE=Release", - "-DLLVM_ENABLE_ASSERTIONS=ON" - ] - if buildWithSanitizerOptions: - cmakeCommand += [ - "-DCMAKE_C_COMPILER=clang", - "-DCMAKE_CXX_COMPILER=clang++" - ] - if triple: - cmakeCommand += ["-DLLVM_DEFAULT_TARGET_TRIPLE=%s" % triple] - - if extraCmakeOptions: - assert not any(a.startswith('-DLLVM_LIT_ARGS=') for a in extraCmakeOptions), \ - "Please use extraLitArgs for LIT arguments instead of defining them in extraCmakeOptions." - cmakeCommand += extraCmakeOptions - - if not isMSVC: - cmakeCommand += [ - "-DCMAKE_C_FLAGS=\"%s\"" % (" ".join(options)), - "-DCMAKE_CXX_FLAGS=\"-std=c++11 %s\"" % (" ".join(options)), - ] - cmakeCommand += lit_args - cmakeCommand += [ - "-GNinja", - "../%s" % llvm_srcdir - ] - - if isMSVC and vs: - # Set up VS environment, if requested. - f.addStep(SetProperty( - command=builders_util.getVisualStudioEnvironment(vs, target_arch), - extract_fn=builders_util.extractSlaveEnvironment)) - merged_env = Property('slave_env') - - # Note: ShellCommand does not pass the params with special symbols right. - # The " ".join is a workaround for this bug. - f.addStep(ShellCommand(name="cmake-configure", - description=["cmake configure"], - haltOnFailure=True, - command=WithProperties(" ".join(cmakeCommand)), - workdir=llvm_objdir, - env=merged_env)) - - # Build everything. - f.addStep(NinjaCommand(name="build", - prefixCommand=prefixCommand, - haltOnFailure=True, - description=["build"], - workdir=llvm_objdir, - env=merged_env)) - - # Test everything. - f.addStep(NinjaCommand(name="test", - prefixCommand=prefixCommand, - targets=["check-all"], - haltOnFailure=True, - description=["test"], - workdir=llvm_objdir, - env=merged_env)) - - return f diff --git a/zorg/buildbot/builders/ClangBuilder.py b/zorg/buildbot/builders/ClangBuilder.py --- a/zorg/buildbot/builders/ClangBuilder.py +++ b/zorg/buildbot/builders/ClangBuilder.py @@ -1,345 +1,17 @@ -import buildbot -import buildbot.process.factory import copy -import os from datetime import datetime from buildbot.process.properties import WithProperties, Property -from buildbot.steps.shell import Configure, ShellCommand, SetProperty +from buildbot.steps.shell import ShellCommand, SetProperty from buildbot.steps.shell import WarningCountingShellCommand -from buildbot.steps.source import SVN -from buildbot.steps.transfer import FileDownload -import zorg.buildbot.util.artifacts as artifacts import zorg.buildbot.builders.Util as builders_util -import zorg.buildbot.util.phasedbuilderutils as phasedbuilderutils -import zorg.buildbot.commands as commands -import zorg.buildbot.commands.BatchFileDownload as batch_file_download -import zorg.buildbot.commands.LitTestCommand as lit_test_command + +from zorg.buildbot.commands.LitTestCommand import LitTestCommand from zorg.buildbot.conditions.FileConditions import FileDoesNotExist from zorg.buildbot.commands.CmakeCommand import CmakeCommand from zorg.buildbot.process.factory import LLVMBuildFactory -# FIXME: This method is deprecated and will be removed. Please use getClangCMakeBuildFactory instead. -def getClangBuildFactory( - triple=None, - clean=True, - test=True, - package_dst=None, - run_cxx_tests=False, - examples=False, - valgrind=False, - valgrindLeakCheck=False, - useTwoStage=False, - completely_clean=False, - make='make', - jobs="%(jobs)s", - stage1_config='Debug+Asserts', - stage2_config='Release+Asserts', - env={}, # Environmental variables for all steps. - extra_configure_args=[], - stage2_extra_configure_args=[], - use_pty_in_tests=False, - trunk_revision=None, - force_checkout=False, - extra_clean_step=None, - checkout_compiler_rt=False, - checkout_lld=False, - run_gdb=False, - run_modern_gdb=False, - run_gcc=False): - # Prepare environmental variables. Set here all env we want everywhere. - merged_env = { - 'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences. - } - if env is not None: - # Overwrite pre-set items with the given ones, so user can set anything. - merged_env.update(env) - - llvm_srcdir = "llvm.src" - llvm_1_objdir = "llvm.obj" - llvm_1_installdir = "llvm.install.1" - llvm_2_objdir = "llvm.obj.2" - llvm_2_installdir = "llvm.install" - - f = buildbot.process.factory.BuildFactory() - - # Determine the build directory. - f.addStep(buildbot.steps.shell.SetProperty(name="get_builddir", - command=["pwd"], - property="builddir", - description="set build dir", - workdir=".", - env=merged_env)) - - # Blow away completely, if requested. - if completely_clean: - f.addStep(ShellCommand(name="rm-llvm.src", - command=["rm", "-rf", llvm_srcdir], - haltOnFailure=True, - description=["rm src dir", "llvm"], - workdir=".", - env=merged_env)) - - # Checkout sources. - if trunk_revision: - # The SVN build step provides no mechanism to check out a specific revision - # based on a property, so just run the commands directly here. - svn_co = ['svn', 'checkout'] - if force_checkout: - svn_co += ['--force'] - svn_co += ['--revision', WithProperties(trunk_revision)] - - svn_co_llvm = svn_co + \ - [WithProperties('http://llvm.org/svn/llvm-project/llvm/trunk@%s' % - trunk_revision), - llvm_srcdir] - svn_co_clang = svn_co + \ - [WithProperties('http://llvm.org/svn/llvm-project/cfe/trunk@%s' % - trunk_revision), - '%s/tools/clang' % llvm_srcdir] - svn_co_clang_tools_extra = svn_co + \ - [WithProperties('http://llvm.org/svn/llvm-project/clang-tools-extra/trunk@%s' % - trunk_revision), - '%s/tools/clang/tools/extra' % llvm_srcdir] - - f.addStep(ShellCommand(name='svn-llvm', - command=svn_co_llvm, - haltOnFailure=True, - workdir='.')) - f.addStep(ShellCommand(name='svn-clang', - command=svn_co_clang, - haltOnFailure=True, - workdir='.')) - f.addStep(ShellCommand(name='svn-clang-tools-extra', - command=svn_co_clang_tools_extra, - haltOnFailure=True, - workdir='.')) - else: - f.addStep(SVN(name='svn-llvm', - mode='update', - baseURL='http://llvm.org/svn/llvm-project/llvm/', - defaultBranch='trunk', - workdir=llvm_srcdir)) - f.addStep(SVN(name='svn-clang', - mode='update', - baseURL='http://llvm.org/svn/llvm-project/cfe/', - defaultBranch='trunk', - workdir='%s/tools/clang' % llvm_srcdir)) - f.addStep(SVN(name='svn-clang-tools-extra', - mode='update', - baseURL='http://llvm.org/svn/llvm-project/clang-tools-extra/', - defaultBranch='trunk', - workdir='%s/tools/clang/tools/extra' % llvm_srcdir)) - if checkout_compiler_rt: - f.addStep(SVN(name='svn-compiler-rt', - mode='update', - baseURL='http://llvm.org/svn/llvm-project/compiler-rt/', - defaultBranch='trunk', - workdir='%s/projects/compiler-rt' % llvm_srcdir)) - - # Clean up llvm (stage 1); unless in-dir. - if clean and llvm_srcdir != llvm_1_objdir: - f.addStep(ShellCommand(name="rm-llvm.obj.stage1", - command=["rm", "-rf", llvm_1_objdir], - haltOnFailure=True, - description=["rm build dir", "llvm"], - workdir=".", - env=merged_env)) - - if not clean: - expected_makefile = 'Makefile' - f.addStep(SetProperty(name="Makefile_isready", - workdir=llvm_1_objdir, - command=["sh", "-c", - "test -e %s && echo OK || echo Missing" % expected_makefile], - flunkOnFailure=False, - property="exists_Makefile")) - - cmake_triple_arg = [] - if triple: - cmake_triple_arg = ['-DLLVM_HOST_TRIPLE=%s' % triple] - f.addStep(ShellCommand(name='cmake', - command=['cmake', - '-DLLVM_BUILD_TESTS=ON', - '-DCMAKE_BUILD_TYPE=%s' % stage1_config] + - cmake_triple_arg + - extra_configure_args + - ["../" + llvm_srcdir], - description='cmake stage1', - workdir=llvm_1_objdir, - env=merged_env, - doStepIf=lambda step: step.build.getProperty("exists_Makefile") != "OK")) - - # Make clean if using in-dir builds. - if clean and llvm_srcdir == llvm_1_objdir: - f.addStep(WarningCountingShellCommand(name="clean-llvm", - command=[make, "clean"], - haltOnFailure=True, - description="cleaning llvm", - descriptionDone="clean llvm", - workdir=llvm_1_objdir, - doStepIf=clean, - env=merged_env)) - - if extra_clean_step: - f.addStep(extra_clean_step) - - f.addStep(WarningCountingShellCommand(name="compile", - command=['nice', '-n', '10', - make, WithProperties("-j%s" % jobs)], - haltOnFailure=True, - flunkOnFailure=not run_gdb, - description=["compiling", stage1_config], - descriptionDone=["compile", stage1_config], - workdir=llvm_1_objdir, - env=merged_env)) - - if examples: - f.addStep(WarningCountingShellCommand(name="compile.examples", - command=['nice', '-n', '10', - make, WithProperties("-j%s" % jobs), - "BUILD_EXAMPLES=1"], - haltOnFailure=True, - description=["compiling", stage1_config, "examples"], - descriptionDone=["compile", stage1_config, "examples"], - workdir=llvm_1_objdir, - env=merged_env)) - - clangTestArgs = '-v -j %s' % jobs - if valgrind: - clangTestArgs += ' --vg' - if valgrindLeakCheck: - clangTestArgs += ' --vg-leak' - clangTestArgs += ' --vg-arg --suppressions=%(builddir)s/llvm/tools/clang/utils/valgrind/x86_64-pc-linux-gnu_gcc-4.3.3.supp --vg-arg --suppressions=%(builddir)s/llvm/utils/valgrind/x86_64-pc-linux-gnu.supp' - extraTestDirs = '' - if run_cxx_tests: - extraTestDirs += '%(builddir)s/llvm/tools/clang/utils/C++Tests' - if test: - f.addStep(lit_test_command.LitTestCommand(name='check-all', - command=[make, "check-all", "VERBOSE=1", - WithProperties("LIT_ARGS=%s" % clangTestArgs), - WithProperties("EXTRA_TESTDIRS=%s" % extraTestDirs)], - flunkOnFailure=not run_gdb, - description=["checking"], - descriptionDone=["checked"], - workdir=llvm_1_objdir, - usePTY=use_pty_in_tests, - env=merged_env)) - - # TODO: Install llvm and clang for stage1. - - if run_gdb or run_gcc or run_modern_gdb: - ignores = getClangTestsIgnoresFromPath(os.path.expanduser('~/public/clang-tests'), 'clang-x86_64-darwin10') - install_prefix = "%%(builddir)s/%s" % llvm_1_installdir - if run_gdb: - addClangGDBTests(f, ignores, install_prefix) - if run_modern_gdb: - addModernClangGDBTests(f, jobs, install_prefix) - if run_gcc: - addClangGCCTests(f, ignores, install_prefix) - - if not useTwoStage: - if package_dst: - name = WithProperties( - "%(builddir)s/" + llvm_1_objdir + - "/clang-r%(got_revision)s-b%(buildnumber)s.tgz") - f.addStep(ShellCommand(name='pkg.tar', - description="tar root", - command=["tar", "zcvf", name, "./"], - workdir=llvm_1_installdir, - warnOnFailure=True, - flunkOnFailure=False, - haltOnFailure=False, - env=merged_env)) - f.addStep(ShellCommand(name='pkg.upload', - description="upload root", - command=["scp", name, - WithProperties( - package_dst + "/%(buildername)s")], - workdir=".", - warnOnFailure=True, - flunkOnFailure=False, - haltOnFailure=False, - env=merged_env)) - - return f - - # Clean up llvm (stage 2). - # - # We always cleanly build the stage 2. If the compiler has been - # changed on the stage 1, we cannot trust any of the intermediate file - # from the old compiler. And if the stage 1 compiler is the same, we should - # not build in the first place. - f.addStep(ShellCommand(name="rm-llvm.obj.stage2", - command=["rm", "-rf", llvm_2_objdir], - haltOnFailure=True, - description=["rm build dir", "llvm", "(stage 2)"], - workdir=".", - env=merged_env)) - - # Configure llvm (stage 2). - f.addStep(ShellCommand(name='cmake', - command=['cmake'] + stage2_extra_configure_args + [ - '-DLLVM_BUILD_TESTS=ON', - WithProperties('-DCMAKE_C_COMPILER=%%(builddir)s/%s/bin/clang' % llvm_1_objdir), # FIXME use installdir - WithProperties('-DCMAKE_CXX_COMPILER=%%(builddir)s/%s/bin/clang++' % llvm_1_objdir), - '-DCMAKE_BUILD_TYPE=%s' % stage2_config, - "../" + llvm_srcdir], - description='cmake stage2', - workdir=llvm_2_objdir, - env=merged_env)) - - # Build llvm (stage 2). - f.addStep(WarningCountingShellCommand(name="compile.llvm.stage2", - command=['nice', '-n', '10', - make, WithProperties("-j%s" % jobs)], - haltOnFailure=True, - description=["compiling", "(stage 2)", - stage2_config], - descriptionDone=["compile", "(stage 2)", - stage2_config], - workdir=llvm_2_objdir, - env=merged_env)) - - if test: - f.addStep(lit_test_command.LitTestCommand(name='check-all', - command=[make, "check-all", "VERBOSE=1", - WithProperties("LIT_ARGS=%s" % clangTestArgs), - WithProperties("EXTRA_TESTDIRS=%s" % extraTestDirs)], - description=["checking"], - descriptionDone=["checked"], - workdir=llvm_2_objdir, - usePTY=use_pty_in_tests, - env=merged_env)) - - # TODO: Install llvm and clang for stage2. - - if package_dst: - name = WithProperties( - "%(builddir)s/" + llvm_2_objdir + - "/clang-r%(got_revision)s-b%(buildnumber)s.tgz") - f.addStep(ShellCommand(name='pkg.tar', - description="tar root", - command=["tar", "zcvf", name, "./"], - workdir=llvm_2_installdir, - warnOnFailure=True, - flunkOnFailure=False, - haltOnFailure=False, - env=merged_env)) - f.addStep(ShellCommand(name='pkg.upload', - description="upload root", - command=["scp", name, - WithProperties( - package_dst + "/%(buildername)s")], - workdir=".", - warnOnFailure=True, - flunkOnFailure=False, - haltOnFailure=False, - env=merged_env)) - - return f - def addGCSUploadSteps(f, package_name, install_prefix, gcs_directory, env, gcs_url_property=None, use_pixz_compression=False, xz_compression_factor=6): @@ -682,13 +354,13 @@ if test and testStage1: haltOnStage1Check = not useTwoStage and not runTestSuite - f.addStep(lit_test_command.LitTestCommand(name='ninja check 1', - command=ninja_check_cmd, - haltOnFailure=haltOnStage1Check, - description=["checking stage 1"], - descriptionDone=["stage 1 checked"], - workdir=stage1_build, - env=env)) + f.addStep(LitTestCommand(name='ninja check 1', + command=ninja_check_cmd, + haltOnFailure=haltOnStage1Check, + description=["checking stage 1"], + descriptionDone=["stage 1 checked"], + workdir=stage1_build, + env=env)) if useTwoStage or runTestSuite or stage1_upload_directory: f.addStep(ShellCommand(name='clean stage 1 install', @@ -763,13 +435,13 @@ env=env)) if test: - f.addStep(lit_test_command.LitTestCommand(name='ninja check 2', - command=ninja_check_cmd, - haltOnFailure=not runTestSuite, - description=["checking stage 2"], - descriptionDone=["stage 2 checked"], - workdir=stage2_build, - env=env)) + f.addStep(LitTestCommand(name='ninja check 2', + command=ninja_check_cmd, + haltOnFailure=not runTestSuite, + description=["checking stage 2"], + descriptionDone=["stage 2 checked"], + workdir=stage2_build, + env=env)) ############# TEST SUITE ## Test-Suite (stage 2 if built, stage 1 otherwise) @@ -859,7 +531,7 @@ description='setting up LNT in sandbox', workdir='test/sandbox', env=env)) - f.addStep(commands.LitTestCommand.LitTestCommand( + f.addStep(LitTestCommand( name='test-suite', command=test_suite_cmd, haltOnFailure=True, @@ -872,100 +544,3 @@ env=test_suite_env)) return f - -# FIXME: Deprecated. -def addClangGCCTests(f, ignores={}, install_prefix="%(builddir)s/llvm.install", - languages = ('gcc', 'g++', 'objc', 'obj-c++')): - make_vars = [WithProperties( - 'CC_UNDER_TEST=%s/bin/clang' % install_prefix), - WithProperties( - 'CXX_UNDER_TEST=%s/bin/clang++' % install_prefix)] - f.addStep(SVN(name='svn-clang-gcc-tests', mode='update', - baseURL='http://llvm.org/svn/llvm-project/clang-tests/', - defaultBranch='trunk', workdir='clang-tests')) - gcc_dg_ignores = ignores.get('gcc-4_2-testsuite', {}) - for lang in languages: - f.addStep(commands.SuppressionDejaGNUCommand.SuppressionDejaGNUCommand( - name='test-gcc-4_2-testsuite-%s' % lang, - command=["make", "-k", "check-%s" % lang] + make_vars, - description="gcc-4_2-testsuite (%s)" % lang, - workdir='clang-tests/gcc-4_2-testsuite', - logfiles={ 'dg.sum' : 'obj/%s/%s.sum' % (lang, lang), - '%s.log' % lang : 'obj/%s/%s.log' % (lang, lang)}, - ignore=gcc_dg_ignores.get(lang, []))) - -# FIXME: Deprecated. -def addClangGDBTests(f, ignores={}, install_prefix="%(builddir)s/llvm.install"): - make_vars = [WithProperties( - 'CC_UNDER_TEST=%s/bin/clang' % install_prefix), - WithProperties( - 'CXX_UNDER_TEST=%s/bin/clang++' % install_prefix)] - f.addStep(SVN(name='svn-clang-gdb-tests', mode='update', - baseURL='http://llvm.org/svn/llvm-project/clang-tests/', - defaultBranch='trunk', workdir='clang-tests')) - f.addStep(commands.SuppressionDejaGNUCommand.SuppressionDejaGNUCommand( - name='test-gdb-1472-testsuite', - command=["make", "-k", "check"] + make_vars, - description="gdb-1472-testsuite", - workdir='clang-tests/gdb-1472-testsuite', - logfiles={ 'dg.sum' : 'obj/filtered.gdb.sum', - 'gdb.log' : 'obj/gdb.log' })) - -# FIXME: Deprecated. -def addModernClangGDBTests(f, jobs, install_prefix): - make_vars = [WithProperties('RUNTESTFLAGS=CC_FOR_TARGET=\'{0}/bin/clang\' ' - 'CXX_FOR_TARGET=\'{0}/bin/clang++\' ' - 'CFLAGS_FOR_TARGET=\'-w -fno-limit-debug-info\'' - .format(install_prefix))] - f.addStep(SVN(name='svn-clang-modern-gdb-tests', mode='update', - svnurl='http://llvm.org/svn/llvm-project/clang-tests-external/trunk/gdb/7.5', - workdir='clang-tests/src')) - f.addStep(Configure(command='../src/configure', - workdir='clang-tests/build/')) - f.addStep(WarningCountingShellCommand(name='gdb-75-build', - command=['make', WithProperties('-j%s' % jobs)], - haltOnFailure=True, - workdir='clang-tests/build')) - f.addStep(commands.DejaGNUCommand.DejaGNUCommand( - name='gdb-75-check', - command=['make', '-k', WithProperties('-j%s' % jobs), 'check'] + make_vars, - workdir='clang-tests/build', - logfiles={'dg.sum':'gdb/testsuite/gdb.sum', - 'gdb.log':'gdb/testsuite/gdb.log'})) - - - -# FIXME: Deprecated. -addClangTests = addClangGCCTests - -def getClangTestsIgnoresFromPath(path, key): - def readList(path): - if not os.path.exists(path): - return [] - - f = open(path) - lines = [ln.strip() for ln in f] - f.close() - return lines - - ignores = {} - - gcc_dg_ignores = {} - for lang in ('gcc', 'g++', 'objc', 'obj-c++'): - lang_path = os.path.join(path, 'gcc-4_2-testsuite', 'expected_results', - key, lang) - gcc_dg_ignores[lang] = ( - readList(os.path.join(lang_path, 'FAIL.txt')) + - readList(os.path.join(lang_path, 'UNRESOLVED.txt')) + - readList(os.path.join(lang_path, 'XPASS.txt'))) - ignores['gcc-4_2-testsuite' ] = gcc_dg_ignores - - ignores_path = os.path.join(path, 'gdb-1472-testsuite', 'expected_results', - key) - gdb_dg_ignores = ( - readList(os.path.join(ignores_path, 'FAIL.txt')) + - readList(os.path.join(ignores_path, 'UNRESOLVED.txt')) + - readList(os.path.join(ignores_path, 'XPASS.txt'))) - ignores['gdb-1472-testsuite' ] = gdb_dg_ignores - - return ignores diff --git a/zorg/buildbot/builders/ClangLTOBuilder3Stage.py b/zorg/buildbot/builders/ClangLTOBuilder3Stage.py deleted file mode 100644 --- a/zorg/buildbot/builders/ClangLTOBuilder3Stage.py +++ /dev/null @@ -1,138 +0,0 @@ -from buildbot.steps.source import SVN -from buildbot.steps.shell import SetProperty -from buildbot.steps.shell import ShellCommand -from buildbot.steps.shell import WarningCountingShellCommand -from buildbot.process.factory import BuildFactory -from buildbot.process.properties import WithProperties -from zorg.buildbot.commands.NinjaCommand import NinjaCommand - -def get3StageClangLTOBuildFactory( - clean=True, - env=None, - build_gold=False, - cmake_cache_file=None, - extra_cmake_options=None, - ): - - llvm_srcdir = "llvm.src" - llvm_objdir = "llvm-stage1.obj" - - merged_env = { - 'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences. - } - if env is not None: - merged_env.update(env) - - f = BuildFactory() - - f.addStep( - SVN( - name='svn-llvm', - mode='update', - baseURL='http://llvm.org/svn/llvm-project/llvm/', - defaultBranch='trunk', - workdir=llvm_srcdir - ) - ) - - f.addStep( - SVN( - name='svn-clang', - mode='update', - baseURL='http://llvm.org/svn/llvm-project/cfe/', - defaultBranch='trunk', - workdir='%s/tools/clang' % llvm_srcdir - ) - ) - - # Clean directory, if requested. - cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean - f.addStep( - ShellCommand( - doStepIf=cleanBuildRequested, - name="rm-llvm_objdir", - command=["rm", "-rf", llvm_objdir], - haltOnFailure=True, - description=["rm build dir", "llvm"], - workdir=".", - env=merged_env - ) - ) - - cmake_command = ["cmake"] - - if cmake_cache_file: - cmake_command += ['-C', cmake_cache_file] - - if extra_cmake_options: - cmake_command += extra_cmake_options - - cmake_command += [ - "../%s" % llvm_srcdir - ] - - # Note: ShellCommand does not pass the params with special symbols right. - # The " ".join is a workaround for this bug. - f.addStep( - WarningCountingShellCommand( - name="cmake-configure", - description=["cmake configure"], - haltOnFailure=True, - command=WithProperties(" ".join(cmake_command)), - workdir=llvm_objdir, - env=merged_env - ) - ) - - if build_gold: - f.addStep( - NinjaCommand(name='build-LLVMgold.so', - targets=['lib/LLVMgold.so'], - haltOnFailure=True, - warnOnWarnings=True, - description=["3 Stage Build Clang"], - workdir=llvm_objdir, - env=merged_env) - ) - - f.addStep( - NinjaCommand(name='build-stage3-clang', - targets=['stage3-clang'], - haltOnFailure=True, - warnOnWarnings=True, - description=["3 Stage Build Clang"], - timeout=3600, # Default value is not enough. - workdir=llvm_objdir, - env=merged_env) - ) - - f.addStep( - NinjaCommand(name='build-stage3-check-clang', - targets=['stage3-check-clang'], - haltOnFailure=True, - warnOnWarnings=True, - description=["Check Clang"], - timeout=3600, # Default value is not enough. - workdir=llvm_objdir, - env=merged_env) - ) - - # Compare stage2 & stage3 clang - shell_command = [ - "diff", - "-q", - "tools/clang/stage2-bins/bin/clang", - "tools/clang/stage2-bins/tools/clang/stage3-bins/bin/clang" - ] - f.addStep( - ShellCommand( - name="compare-clang", - description=["comapre stage2 & stage3 clang"], - haltOnFailure=True, - command=WithProperties(" ".join(shell_command)), - workdir=llvm_objdir, - env=merged_env - ) - ) - - return f diff --git a/zorg/buildbot/builders/LLDBuilder.py b/zorg/buildbot/builders/LLDBuilder.py deleted file mode 100644 --- a/zorg/buildbot/builders/LLDBuilder.py +++ /dev/null @@ -1,173 +0,0 @@ -import os - -import buildbot -import buildbot.process.factory -from buildbot.steps.source import SVN -from buildbot.steps.shell import ShellCommand, SetProperty -from buildbot.steps.slave import RemoveDirectory -from buildbot.process.properties import WithProperties, Property -from zorg.buildbot.builders.Util import getVisualStudioEnvironment -from zorg.buildbot.builders.Util import extractSlaveEnvironment -from zorg.buildbot.commands.CmakeCommand import CmakeCommand -from zorg.buildbot.commands.NinjaCommand import NinjaCommand -from zorg.buildbot.conditions.FileConditions import FileDoesNotExist -from zorg.buildbot.process.factory import LLVMBuildFactory - -def getLLDBuildFactory( - clean = True, - jobs = None, - extra_configure_args = None, - env = None): - - # Set defaults - if jobs is None: - jobs = "%(jobs)s" - if extra_configure_args is None: - extra_configure_args = [] - - # Prepare environmental variables. Set here all env we want everywhere. - merged_env = { - 'CC' : "clang", - 'CXX' : "clang++", - 'TERM' : 'dumb' # Be cautious and disable color output from all tools. - } - if env is not None: - # Overwrite pre-set items with the given ones, so user can set anything. - merged_env.update(env) - - f = LLVMBuildFactory( - depends_on_projects=['llvm', 'lld'], - llvm_srcdir="llvm.src", - llvm_objdir="llvm.obj") - - # Get LLVM and Lld - f.addSVNSteps() - - # Clean directory, if requested. - cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean - f.addStep(RemoveDirectory(name='clean ' + f.llvm_objdir, - dir=f.llvm_objdir, - haltOnFailure=False, - flunkOnFailure=False, - doStepIf=cleanBuildRequested - )) - - # Create configuration files with cmake - f.addStep(CmakeCommand(name="cmake-configure", - description=["cmake configure"], - haltOnFailure=True, - options=extra_configure_args, - path="../%s" % f.llvm_srcdir, - env=merged_env, - workdir=f.llvm_objdir, - doStepIf=FileDoesNotExist( - "./%s/CMakeCache.txt" % f.llvm_objdir))) - - # Build Lld - f.addStep(ShellCommand(name="build_Lld", - command=['nice', '-n', '10', - 'make', WithProperties("-j%s" % jobs)], - haltOnFailure=True, - description=["build lld"], - env=merged_env, - workdir=f.llvm_objdir)) - - # Test Lld - f.addStep(ShellCommand(name="test_lld", - command=["make", "lld-test"], - haltOnFailure=True, - description=["test lld"], - env=merged_env, - workdir=f.llvm_objdir)) - - return f - - -def getLLDWinBuildFactory( - clean = True, - - # Default values for VS devenv and build configuration - vs = None, # What to run to configure Visual Studio utils. - target_arch = None, # Native. - - extra_configure_args = None, - env = None): - - # Set defaults - if vs is None: - vs = r"""%VS140COMNTOOLS%""" # Visual Studio 2015. - if extra_configure_args is None: - extra_configure_args = [] - if env is None: - env = {} - - f = LLVMBuildFactory( - depends_on_projects=['llvm', 'lld'], - llvm_srcdir="llvm.src", - llvm_objdir="llvm.obj") - - # Get LLVM and Lld - f.addSVNSteps() - - # Clean directory, if requested. - cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean - f.addStep(RemoveDirectory(name='clean ' + f.llvm_objdir, - dir=f.llvm_objdir, - haltOnFailure=False, - flunkOnFailure=False, - doStepIf=cleanBuildRequested - )) - - # If set up environment step is requested, do this now. - if vs: - f.addStep(SetProperty( - command=getVisualStudioEnvironment(vs, target_arch), - extract_fn=extractSlaveEnvironment)) - assert not env, "Can't have custom builder env vars with VS" - env = Property('slave_env') - - # Always build with ninja. - cmake_options = ["-G", "Ninja"] - # Reconsile configure args with the defaults we want. - if not any(a.startswith('-DCMAKE_BUILD_TYPE=') for a in extra_configure_args): - cmake_options.append('-DCMAKE_BUILD_TYPE=Release') - if not any(a.startswith('-DLLVM_ENABLE_WERROR=') for a in extra_configure_args): - cmake_options.append('-DLLVM_ENABLE_WERROR=ON') - if not any(a.startswith('-DLLVM_ENABLE_ASSERTIONS=') for a in extra_configure_args): - cmake_options.append('-DLLVM_ENABLE_ASSERTIONS=ON') - if not any(a.startswith('-DLLVM_LIT_ARGS=') for a in extra_configure_args): - cmake_options.append('-DLLVM_LIT_ARGS=\"-v\"') - - cmake_options += extra_configure_args - - # Note: ShellCommand does not pass the params with special symbols right. - # The " ".join is a workaround for this bug. - f.addStep(CmakeCommand(name="cmake-configure", - description=["cmake configure"], - haltOnFailure=True, - warnOnWarnings=True, - options=cmake_options, - path="../%s" % f.llvm_srcdir, - env=env, - workdir=f.llvm_objdir, - doStepIf=FileDoesNotExist( - "./%s/CMakeCache.txt" % f.llvm_objdir))) - - # Build Lld. - f.addStep(NinjaCommand(name='build lld', - haltOnFailure=True, - warnOnWarnings=True, - description='build lld', - workdir=f.llvm_objdir, - env=env)) - - # Test Lld - f.addStep(NinjaCommand(name='test lld', - targets=['lld-test'], - haltOnFailure=True, - warnOnWarnings=True, - description='test lld', - workdir=f.llvm_objdir, - env=env)) - - return f diff --git a/zorg/buildbot/builders/LLVMBuilder.py b/zorg/buildbot/builders/LLVMBuilder.py deleted file mode 100644 --- a/zorg/buildbot/builders/LLVMBuilder.py +++ /dev/null @@ -1,101 +0,0 @@ -import os - -import buildbot -import buildbot.process.factory -from buildbot.steps.source import SVN -from buildbot.steps.shell import Configure, ShellCommand -from buildbot.steps.shell import WarningCountingShellCommand -from buildbot.process.properties import WithProperties - -from zorg.buildbot.commands.LitTestCommand import LitTestCommand - -from Util import getConfigArgs - -def getLLVMCMakeBuildFactory( - clean = True, # "clean-llvm" step is requested if true. - test = True, # "test-llvm" step is requested if true. - jobs = '%(jobs)s', # Number of concurrent jobs. - timeout = 20, # Timeout if no activity seen (minutes). - make = 'make', # Make command. - enable_shared = False, # Enable shared (-DBUILD_SHARED_LIBS=ON configure parameters added) if true. - defaultBranch = 'trunk', # Branch to build. - config_name = 'Debug', # Configuration name. - env = None, # Environmental variables for all steps. - extra_cmake_args = []): # Extra args for the cmake step. - # Prepare environmental variables. Set here all env we want everywhere. - merged_env = { - 'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences. - } - if env is not None: - merged_env.update(env) # Overwrite pre-set items with the given ones, so user can set anything. - - llvm_srcdir = "llvm.src" - llvm_objdir = "llvm.obj" - - f = buildbot.process.factory.BuildFactory() - - # Determine the build directory. - f.addStep( - buildbot.steps.shell.SetProperty( - name = "get_builddir", - command = ["pwd"], - property = "builddir", - description = "set build dir", - workdir = ".", - env = merged_env)) - - # Checkout sources. - f.addStep( - SVN( - name = 'svn-llvm', - mode = 'update', baseURL='http://llvm.org/svn/llvm-project/llvm/', - defaultBranch = defaultBranch, - workdir = llvm_srcdir)) - - cmake_args = ['cmake'] - cmake_args += ["-DCMAKE_BUILD_TYPE="+config_name] - if enable_shared: - cmake_args.append('-DBUILD_SHARED_LIBS=ON') - cmake_args.extend(extra_cmake_args) - cmake_args += ['../' + llvm_srcdir] - f.addStep( - Configure( - command = cmake_args, - description = ['configuring', config_name], - descriptionDone = ['configure', config_name], - workdir = llvm_objdir, - env = merged_env)) - if clean: - f.addStep( - WarningCountingShellCommand( - name = "clean-llvm", - command = [make, 'clean'], - haltOnFailure = True, - description = "cleaning llvm", - descriptionDone = "clean llvm", - workdir = llvm_objdir, - env = merged_env)) - f.addStep( - WarningCountingShellCommand( - name = "compile", - command = ['nice', '-n', '10', - make, WithProperties("-j%s" % jobs)], - haltOnFailure = True, - description = "compiling llvm", - descriptionDone = "compile llvm", - workdir = llvm_objdir, - env = merged_env, - timeout = timeout * 60)) - if test: - litTestArgs = '-v -j %s' % jobs - f.addStep( - LitTestCommand( - name = 'test-llvm', - command = [make, "check-all", "VERBOSE=1", - WithProperties("-j%s" % jobs), - WithProperties("LIT_ARGS=%s" % litTestArgs)], - description = ["testing", "llvm"], - descriptionDone = ["test", "llvm"], - workdir = llvm_objdir, - env = merged_env)) - return f diff --git a/zorg/buildbot/builders/LLVMGCCBuilder.py b/zorg/buildbot/builders/LLVMGCCBuilder.py deleted file mode 100644 --- a/zorg/buildbot/builders/LLVMGCCBuilder.py +++ /dev/null @@ -1,321 +0,0 @@ -import buildbot -from buildbot.steps.source import SVN -from buildbot.steps.shell import Configure, WarningCountingShellCommand -from buildbot.steps.shell import ShellCommand, SetProperty -from buildbot.process.properties import WithProperties - -from zorg.buildbot.commands.LitTestCommand import LitTestCommand - -from Util import getConfigArgs - -def getLLVMGCCBuildFactory(jobs='%(jobs)s', update=True, clean=True, - gxxincludedir=None, - triple=None, build=None, host=None, target=None, - useTwoStage=True, stage1_config='Release+Asserts', - stage2_config='Release+Asserts', make='make', - extra_configure_args=[], extra_languages=None, - verbose=False, env = {}, defaultBranch='trunk', - timeout=20, package_dst=None): - if build or host or target: - if not build or not host or not target: - raise ValueError,"Must specify all of 'build', 'host', 'target' if used." - if triple: - raise ValueError,"Cannot specify 'triple' and 'build', 'host', 'target' options." - elif triple: - build = host = target = triple - - # Prepare environmental variables. Set here all env we want everywhere. - merged_env = { - 'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences. - } - if env is not None: - merged_env.update(env) # Overwrite pre-set items with the given ones, so user can set anything. - - f = buildbot.process.factory.BuildFactory() - - # Determine the build directory. - f.addStep(buildbot.steps.shell.SetProperty(name="get_builddir", - command=["pwd"], - property = "builddir", - description = "set build dir", - workdir = ".", - env = merged_env)) - - # Get the sources. - if update: - f.addStep(SVN(name='svn-llvm', - mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', - defaultBranch = defaultBranch, - workdir = "llvm.src")) - - f.addStep(SVN(name='svn-llvm-gcc', - mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm-gcc-4.2/', - defaultBranch = defaultBranch, - workdir = "llvm-gcc.src")) - - # Clean up llvm (stage 1). - if clean: - f.addStep(ShellCommand(name="rm-llvm.obj.stage1", - command=["rm", "-rf", "llvm.obj"], - haltOnFailure = True, - description = ["rm build dir", - "llvm", - "(stage 1)"], - workdir = ".", - env = merged_env)) - - # Configure llvm (stage 1). - base_llvm_configure_args = [WithProperties("%(builddir)s/llvm.src/configure")] - if build: - base_llvm_configure_args.append('--build=' + build) - base_llvm_configure_args.append('--host=' + host) - base_llvm_configure_args.append('--target=' + target) - stage_configure_args = getConfigArgs(stage1_config) - f.addStep(Configure(name='configure.llvm.stage1', - command=base_llvm_configure_args + - stage_configure_args + - ["--without-llvmgcc", - "--without-llvmgxx"], - description = [ "configure", - "llvm", - "(stage 1)", - stage1_config ], - workdir = "llvm.obj", - env = merged_env)) - - # Build llvm (stage 1). - base_llvm_make_args = ['nice', '-n', '10', - make, WithProperties("-j%s" % jobs)] - if verbose: - base_llvm_make_args.append('VERBOSE=1') - f.addStep(WarningCountingShellCommand(name = "compile.llvm.stage1", - command = base_llvm_make_args, - haltOnFailure = True, - description = ["compile", - "llvm", - "(stage 1)", - stage1_config], - workdir = "llvm.obj", - env = merged_env, - timeout = timeout * 60)) - - # Run LLVM tests (stage 1). - f.addStep(LitTestCommand(name = 'test.llvm.stage1', - command = [make, "check-lit", "VERBOSE=1"], - description = ["testing", "llvm"], - descriptionDone = ["test", "llvm"], - workdir = 'llvm.obj', - env = merged_env)) - - # Clean up llvm-gcc. - if clean: - f.addStep(ShellCommand(name="rm-llvm-gcc.obj.stage1", - command=["rm", "-rf", "llvm-gcc.obj"], - haltOnFailure = True, - description = ["rm build dir", - "llvm-gcc"], - workdir = ".", - env = merged_env)) - - # Configure llvm-gcc. - base_llvmgcc_configure_args = ["../llvm-gcc.src/configure"] - llvmgcc_languages = "--enable-languages=c,c++" - if extra_languages: - llvmgcc_languages = llvmgcc_languages + "," + extra_languages - base_llvmgcc_configure_args.append(llvmgcc_languages) - if gxxincludedir: - base_llvmgcc_configure_args.append('--with-gxx-include-dir=' + gxxincludedir) - base_llvmgcc_configure_args.extend(extra_configure_args) - if build: - base_llvmgcc_configure_args.append('--build=' + build) - base_llvmgcc_configure_args.append('--host=' + host) - base_llvmgcc_configure_args.append('--target=' + target) - f.addStep(Configure(name='configure.llvm-gcc.stage1', - command=(base_llvmgcc_configure_args + - ["--program-prefix=llvm-", - WithProperties("--prefix=%(builddir)s/llvm-gcc.install"), - WithProperties("--enable-llvm=%(builddir)s/llvm.obj")]), - haltOnFailure = True, - description = ["configure", - "llvm-gcc", - "(stage 1)"], - workdir = "llvm-gcc.obj", - env = merged_env)) - - # Build llvm-gcc. - f.addStep(WarningCountingShellCommand(name="compile.llvm-gcc.stage1", - command=['nice', '-n', '10', - make, WithProperties("-j%s" % jobs)], - haltOnFailure = True, - description = ["compile", - "llvm-gcc"], - workdir = "llvm-gcc.obj", - env = merged_env, - timeout = timeout * 60)) - - # Clean up llvm-gcc install. - if clean: - f.addStep(ShellCommand(name="rm-llvm-gcc.install.stage1", - command=["rm", "-rf", "llvm-gcc.install"], - haltOnFailure = True, - description = ["rm install dir", - "llvm-gcc"], - workdir = ".", - env = merged_env)) - - # Install llvm-gcc. - f.addStep(WarningCountingShellCommand(name="install.llvm-gcc.stage1", - command=['nice', '-n', '10', - make, 'install'], - haltOnFailure = True, - description = ["install", - "llvm-gcc"], - workdir = "llvm-gcc.obj", - env = merged_env)) - - # We are done if not doing a two-stage build. - if not useTwoStage: - return f - - # Clean up llvm (stage 2). - if clean: - f.addStep(ShellCommand(name="rm-llvm.obj.stage2", - command=["rm", "-rf", "llvm.obj.2"], - haltOnFailure = True, - description = ["rm build dir", - "llvm", - "(stage 2)"], - workdir = ".", - env = merged_env)) - - # Configure llvm (stage 2). - stage_configure_args = getConfigArgs(stage2_config) - local_env = dict(merged_env) - local_env['CC'] = WithProperties("%(builddir)s/llvm-gcc.install/bin/llvm-gcc") - local_env['CXX'] = WithProperties("%(builddir)s/llvm-gcc.install/bin/llvm-g++") - f.addStep(Configure(name="configure.llvm.stage2", - command=base_llvm_configure_args + - stage_configure_args + - [WithProperties("--with-llvmgcc=%(builddir)s/llvm-gcc.install/bin/llvm-gcc"), - WithProperties("--with-llvmgxx=%(builddir)s/llvm-gcc.install/bin/llvm-g++")], - haltOnFailure = True, - description = ["configure", - "llvm", - "(stage 2)", - stage2_config], - workdir = "llvm.obj.2", - env = local_env)) - - # Build LLVM (stage 2). - f.addStep(WarningCountingShellCommand(name = "compile.llvm.stage2", - command = base_llvm_make_args, - haltOnFailure = True, - description = ["compile", - "llvm", - "(stage 2)", - stage2_config], - workdir = "llvm.obj.2", - env = merged_env, - timeout = timeout * 60)) - - # Run LLVM tests (stage 2). - f.addStep(LitTestCommand(name = 'test.llvm.stage2', - command = [make, "check-lit", "VERBOSE=1"], - description = ["testing", "llvm", "(stage 2)"], - descriptionDone = ["test", "llvm", "(stage 2)"], - workdir = 'llvm.obj.2', - env = merged_env)) - - # Clean up llvm-gcc (stage 2). - if clean: - f.addStep(ShellCommand(name="rm-llvm-gcc.obj.stage2", - command=["rm", "-rf", "llvm-gcc.obj.2"], - haltOnFailure = True, - description = ["rm build dir", - "llvm-gcc", - "(stage 2)"], - workdir = ".", - env = merged_env)) - - # Configure llvm-gcc (stage 2). - local_env = dict(merged_env) - local_env['CC'] = WithProperties("%(builddir)s/llvm-gcc.install/bin/llvm-gcc") - local_env['CXX'] = WithProperties("%(builddir)s/llvm-gcc.install/bin/llvm-g++") - f.addStep(Configure(name = 'configure.llvm-gcc.stage2', - command=base_llvmgcc_configure_args + [ - "--program-prefix=llvm.2-", - WithProperties("--prefix=%(builddir)s/llvm-gcc.install.2"), - WithProperties("--enable-llvm=%(builddir)s/llvm.obj.2")], - haltOnFailure = True, - description = ["configure", - "llvm-gcc", - "(stage 2)"], - workdir = "llvm-gcc.obj.2", - env = local_env)) - - # Build llvm-gcc (stage 2). - f.addStep(WarningCountingShellCommand(name="compile.llvm-gcc.stage2", - command=['nice', '-n', '10', - make, WithProperties("-j%s" % jobs)], - haltOnFailure = True, - description = ["compile", - "llvm-gcc", - "(stage 2)"], - workdir = "llvm-gcc.obj.2", - env = merged_env, - timeout = timeout * 60)) - - # Clean up llvm-gcc install (stage 2). - if clean: - f.addStep(ShellCommand(name="rm-llvm-gcc.install.stage2", - command=["rm", "-rf", "llvm-gcc.install.2"], - haltOnFailure = True, - description = ["rm install dir", - "llvm-gcc", - "(stage 2)"], - workdir = ".", - env = merged_env)) - - # Install llvm-gcc. - f.addStep(WarningCountingShellCommand(name="install.llvm-gcc.stage2", - command = ['nice', '-n', '10', - make, 'install'], - haltOnFailure = True, - description = ["install", - "llvm-gcc", - "(stage 2)"], - workdir = "llvm-gcc.obj.2", - env = merged_env)) - if package_dst: - addPackageStep(f, package_dst, obj_path = 'llvm-gcc.install.2', env = merged_env) - - return f - -import os -def addPackageStep(f, package_dst, - obj_path, - info_string = '%(phase_id)s', - env = {}): - - # Package and upload. - name = WithProperties( - os.path.join("%(builddir)s", obj_path, - "llvm-gcc-%s.tar.gz" % info_string)) - - f.addStep(ShellCommand(name = 'pkg.tar', - description = "tar root", - command = ["tar", "zcvf", name, "./"], - workdir = obj_path, - env = env, - warnOnFailure = True, - flunkOnFailure = False, - haltOnFailure = False)) - - f.addStep(ShellCommand(name = 'pkg.upload', - description = "upload root", - command = ["scp", name, package_dst], - workdir = ".", - env = env, - warnOnFailure = True, - flunkOnFailure = False, - haltOnFailure = False)) diff --git a/zorg/buildbot/builders/LNTBuilder.py b/zorg/buildbot/builders/LNTBuilder.py deleted file mode 100644 --- a/zorg/buildbot/builders/LNTBuilder.py +++ /dev/null @@ -1,325 +0,0 @@ -""" -Builders for using LNT to test LLVM/Clang. -""" - -import os - -import buildbot -from buildbot.steps.source.svn import SVN -from buildbot.process.properties import WithProperties - -import zorg -from zorg.buildbot.builders import ClangBuilder -from zorg.buildbot.util.phasedbuilderutils import getBuildDir, setProperty -from zorg.buildbot.util.artifacts import GetCompilerRoot, package_url - -def _get_cc(status, stdin, stdout): - lines = filter(bool, stdin.split('\n')) - for line in lines: - if 'bin/clang' in line: - cc_path = line - return { 'cc_path' : cc_path } - return {} - -def _get_cxx(status, stdin, stdout): - lines = filter(bool, stdin.split('\n')) - for line in lines: - if 'bin/clang++' in line: - cxx_path = line - return { 'cxx_path' : cxx_path } - return {} - -def _get_liblto(status, stdin, stdout): - lines = filter(bool, stdin.split('\n')) - for line in lines: - if 'lib/libLTO.dylib' in line: - lto_path = line - return { 'lto_path' : lto_path } - return {} - -def getLNTFactory(triple, nt_flags, xfails=[], clean=True, test=False, - reportBuildslave = True, **kwargs): - lnt_args = {} - lnt_arg_names = ['submitURL', 'package_cache', 'testerName', - 'reportBuildslave'] - - for argname in lnt_arg_names: - if argname in kwargs: - lnt_args[argname] = kwargs.pop(argname) - - # Build compiler to test. - f = ClangBuilder.getClangBuildFactory( - triple, clean=clean, test=test, - stage1_config='Release+Asserts', **kwargs) - - # Add an LNT test runner. - AddLNTTestsToFactory(f, nt_flags, - cc_path="llvm.install.1/bin/clang", - cxx_path="llvm.install.1/bin/clang++", - **lnt_args); - - return f - -def AddLNTTestsToFactory(f, nt_flags, cc_path, cxx_path, **kwargs): - """ - Add the buildbot steps necessary to run an LNT driven test of a compiler. - - This assumes at a minimum that the factory has already been set up to - contain a builddir property which points at the full path to the build - directory. - """ - - parallel = kwargs.pop('parallel', False) - jobs = kwargs.pop('jobs', '$(jobs)s') - submitURL = kwargs.pop('submitURL', None) - package_cache = kwargs.pop('package_cache', 'http://lab.llvm.org/packages') - testerName = kwargs.pop('testerName', None) - reportBuildslave = kwargs.pop('reportBuildslave', True) - env = kwargs.pop('env', {}) - - # Create variables to refer to the compiler-under-test. - # - # We assume any relative paths are relative to the build directory (which - # prior steps will have presumably populated with a compiler). - cc_path = WithProperties(os.path.join('%(builddir)s', cc_path)) - cxx_path = WithProperties(os.path.join('%(builddir)s', cxx_path)) - - # Add --liblto-path if necessary. We assume it will be in a lib directory - # adjacent to cc_path. - # - # FIXME: This is currently only going to work on Darwin. - if '-flto' in nt_flags: - base_directory = os.path.dirname(os.path.dirname(cc_path)) - nt_flags.extend(['--liblto-path', WithProperties( - os.path.join('%(builddir)s', base_directory, 'lib', - 'libLTO.dylib'))]) - - # Get the LNT sources. - f.addStep(SVN(name='pull.lnt', mode='incremental', method='fresh', - baseURL='http://llvm.org/svn/llvm-project/lnt/', - defaultBranch='trunk', workdir='lnt', alwaysUseLatest=True)) - - # Get the LLVM test-suite sources. - f.addStep(SVN(name='pull.test-suite', mode='incremental', method='fresh', - baseURL='http://llvm.org/svn/llvm-project/test-suite/', - defaultBranch='trunk', workdir='test-suite', - alwaysUseLatest=False)) - - # Create the LNT virtual env. - f.addStep(buildbot.steps.shell.ShellCommand( - name='venv.lnt.clean', command=['rm', '-rf', 'lnt.venv'], - haltOnFailure=True, description=['clean', 'LNT', 'venv'], - workdir=WithProperties('%(builddir)s'))) - f.addStep(buildbot.steps.shell.ShellCommand( - name='venv.lnt.create', command=['virtualenv', 'lnt.venv'], - haltOnFailure=True, description=['create', 'LNT', 'venv'], - workdir=WithProperties('%(builddir)s'), - env={'PATH' : '${PATH}:/usr/local/bin'})) - f.addStep(buildbot.steps.shell.ShellCommand( - name='venv.lnt.install', haltOnFailure=True, - command=[WithProperties('%(builddir)s/lnt.venv/bin/pip'), 'install', - '--no-index', - '--find-links', package_cache, - '-e', '.'], - description=['install', 'LNT'], workdir='lnt', - env={'ARCHFLAGS' : '-arch i386 -arch x86_64'})) - - # Clean up the sandbox dir. - f.addStep(buildbot.steps.shell.ShellCommand( - name='lnt.nightly-test.clean', command=['rm', '-rf', 'nt'], - haltOnFailure=True, description=['clean', 'LNT', 'sandbox'], - workdir='tests')) - - if reportBuildslave: - reportName = '%(slavename)s' - - if testerName: - reportName += '__' + testerName - reportName = WithProperties(reportName) - else: - reportName = testerName - - # Run the nightly test. - args = [WithProperties('%(builddir)s/lnt.venv/bin/python'), - WithProperties('%(builddir)s/lnt.venv/bin/lnt'), - 'runtest', 'nt', '--verbose'] - - - # Only submit if a URL has been specified - if submitURL is not None: - if type(submitURL) != type([]): - submitURL = [submitURL] - - for url in submitURL: - args.extend(['--submit', url]) - - args.extend(['--commit', '1', - '--sandbox', 'nt', - '--no-timestamp', - '--cc', cc_path, '--cxx', cxx_path, - '--without-llvm', - '--test-suite', WithProperties('%(builddir)s/test-suite'), - '--no-machdep-info', reportName]) - if parallel: - args.extend(['-j', WithProperties(jobs)]) - args.extend(nt_flags) - f.addStep(zorg.buildbot.commands.LitTestCommand.LitTestCommand( - name='lnt.nightly-test', command=args, haltOnFailure=True, - description=['nightly test'], workdir='tests', - logfiles={'configure.log' : 'nt/build/configure.log', - 'build-tools.log' : 'nt/build/build-tools.log', - 'test.log' : 'nt/build/test.log', - 'report.json' : 'nt/build/report.json'}, - env=env)) - return f - -def CreateLNTNightlyFactory(nt_flags, cc_path=None, cxx_path=None, - parallel = False, jobs = '%(jobs)s', - db_url=None, external_URL=None): - # Paramaters used by this method: - # nt_flags : a list of flags passed to the lnt process - # cc_path : explicit path to c compiler - # cxx_path : explicit path to c++ compiler - # parallel : set to True if using multiple cores for faster turnaround - # set to False if measuring performance - # db_url : set to the submission URL for an LNT database when measuring - # performance - # external_URL : Used to pull additional tests from a separate svn - # repository - # Properties set externally but used by this method: - # builddir : This property is set below - # jobs : This property is set by the slave, it indicates the number of - # cores availble to use. - # revision : This property should be set by an upstream builder. - # slavename : This property is set by the slave - # buildername : This property is set by the master - - f = buildbot.process.factory.BuildFactory() - # Determine the build directory. - f = getBuildDir(f) - f = GetCompilerRoot(f) - if cc_path: - cc_command = ['echo', cc_path] - else: - cc_command = ['find', 'host-compiler', '-name', 'clang'] - f.addStep(buildbot.steps.shell.SetProperty( - name='find.cc', - command=cc_command, - extract_fn=_get_cc, - workdir=WithProperties('%(builddir)s'))) - if cxx_path: - cc_command = ['echo', cxx_path] - else: - cc_command = ['find', 'host-compiler', '-name', 'clang++'] - f.addStep(buildbot.steps.shell.SetProperty( - name='find.cxx', - command=cc_command, - extract_fn=_get_cxx, - workdir=WithProperties('%(builddir)s'))) - f.addStep(buildbot.steps.shell.ShellCommand( - name='sanity.test', haltOnFailure=True, - command=[WithProperties('%(builddir)s/%(cc_path)s'), '-v'], - description=['sanity test'])) - args = [WithProperties('%(builddir)s/lnt.venv/bin/python'), - WithProperties('%(builddir)s/lnt.venv/bin/lnt'), - 'runtest', '--verbose'] - if db_url: - f.addStep(buildbot.steps.shell.SetProperty( - name='db_url', - command=['echo', db_url, ':', WithProperties('%(buildername)s')], - extract_fn=_get_db_url, - workdir=WithProperties('%(builddir)s'))) - args.extend(['--submit', WithProperties('%(db_url)s')]) - # Add --liblto-path if necessary. - if '-flto' in nt_flags: - f.addStep(buildbot.steps.shell.SetProperty( - name='find.liblto', - command=['find', 'host-compiler', '-name', 'libLTO.dylib'], - extract_fn=_get_liblto, - workdir=WithProperties('%(builddir)s'))) - nt_flags.extend(['--liblto-path', WithProperties('%(builddir)s/%(lto_path)s')]) - # Get the LNT sources. - f.addStep(buildbot.steps.shell.ShellCommand( - name = 'svn.clean.lnt', - command = ['svn', 'cleanup'], - haltOnFailure=False, flunkOnFailure=False, - description = ['svn clean lnt'], - workdir='lnt')) - f.addStep(SVN(name='pull.lnt', mode='full', method='fresh', - repourl='http://llvm.org/svn/llvm-project/lnt/trunk', - haltOnFailure=False, flunkOnFailure=False, - workdir='lnt', alwaysUseLatest=True, retry = (60, 5), - description='pull.lnt')) - # Get the LLVM test-suite sources. - f.addStep(buildbot.steps.shell.ShellCommand( - name = 'svn.clean.tests', - command = ['svn', 'cleanup'], - haltOnFailure=False, flunkOnFailure=False, - description = ['svn clean test-suite'], - workdir='test-suite')) - f.addStep(SVN(name='pull.test-suite', mode='full', method='fresh', - repourl='http://llvm.org/svn/llvm-project/test-suite/trunk', - haltOnFailure=False, flunkOnFailure=False, - workdir='test-suite', retry = (60, 5), - description='pull.test-suite')) - if external_URL: - external_dir = WithProperties('%(builddir)s/test-suite-externals') - f.addStep(buildbot.steps.shell.ShellCommand( - name = 'svn.clean.externals', - command = ['svn', 'cleanup'], - haltOnFailure=False, flunkOnFailure=False, - description = ['svn clean externals'], - workdir='test-suite-externals')) - f.addStep(SVN(name='pull.test-suite-externals', mode='full', - repourl=external_URL, retry = (60, 5), method='fresh', - workdir='test-suite-externals', alwaysUseLatest=True, - haltOnFailure=False, flunkOnFailure=False, - description='pull.test-suite-externals', - timeout=300)) - # Buildbot uses got_revision instead of revision to identify builds. - # The previous step will set it incorrectly - # We set it to the correct value in th following step - setProperty(f, 'got_revision', WithProperties('%(revision)s')) - # Create the LNT virtual env. - f.addStep(buildbot.steps.shell.ShellCommand( - name='venv.lnt.clean', command=['rm', '-rfv', 'lnt.venv'], - haltOnFailure=True, description=['clean', 'LNT', 'venv'], - workdir=WithProperties('%(builddir)s'))) - f.addStep(buildbot.steps.shell.ShellCommand( - name='venv.lnt.create', command=['virtualenv', 'lnt.venv'], - haltOnFailure=True, description=['create', 'LNT', 'venv'], - workdir=WithProperties('%(builddir)s'), - env={'PATH' : '${PATH}:/usr/local/bin'})) - f.addStep(buildbot.steps.shell.ShellCommand( - name='venv.lnt.install', haltOnFailure=True, - command=[WithProperties('%(builddir)s/lnt.venv/bin/pip'), 'install', - '--index-url', package_url, - '-e', '.'], - description=['install', 'LNT'], workdir='lnt')) - # Clean up the sandbox dir. - f.addStep(buildbot.steps.shell.ShellCommand( - name='lnt.nightly-test.clean', command=['rm', '-rfv', 'nt'], - haltOnFailure=True, description=['clean', 'LNT', 'sandbox'], - workdir='tests')) - # Run the nightly test. - nick = '%(slavename)s-%(buildername)s' - args.extend(['nt', '--sandbox', 'nt', '--cc', - WithProperties('%(builddir)s/%(cc_path)s'), '--cxx', - WithProperties('%(builddir)s/%(cxx_path)s'), '--without-llvm', - '--test-suite', WithProperties('%(builddir)s/test-suite'), - '--no-timestamp', '--no-machdep-info', '--no-auto-name', - '--no-configure']) - if external_URL: - args.extend(['--test-externals', external_dir]) - if parallel: - args.extend(['-j', WithProperties(jobs)]) - args.extend(nt_flags) - f.addStep(zorg.buildbot.commands.LitTestCommand.LitTestCommand( - name='lnt.nightly-test', command=args, haltOnFailure=True, - description=['nightly test'], workdir='tests', - logfiles={'configure.log' : 'nt/build/configure.log', - 'build-tools.log' : 'nt/build/build-tools.log', - 'test.log' : 'nt/build/test.log', - 'report.json' : 'nt/build/report.json'})) - - return f diff --git a/zorg/buildbot/builders/LibCXXBuilder.py b/zorg/buildbot/builders/LibCXXBuilder.py deleted file mode 100644 --- a/zorg/buildbot/builders/LibCXXBuilder.py +++ /dev/null @@ -1,107 +0,0 @@ - -import os - -import buildbot -import buildbot.process.factory -import buildbot.steps.shell -import buildbot.steps.source as source -import buildbot.steps.source.svn as svn -import buildbot.process.properties as properties - -import zorg.buildbot.commands.LitTestCommand as lit_test_command -import zorg.buildbot.util.artifacts as artifacts -import zorg.buildbot.util.phasedbuilderutils as phased_builder_utils - -reload(lit_test_command) -reload(artifacts) -reload(phased_builder_utils) - -def getLibCXXBuilder(f=None, source_path=None, - lit_dir=None): - if f is None: - f = buildbot.process.factory.BuildFactory() - # Find the build directory. We assume if f is passed in that the build - # directory has already been found. - f = phased_builder_utils.getBuildDir(f) - - # Grab the sources if we are not passed in any. - if source_path is None: - source_path = 'sources' - src_url = 'http://llvm.org/svn/llvm-project/libcxx/trunk' - f = phased_builder_utils.SVNCleanupStep(f, source_path) - f.addStep(svn.SVN(name='pull.src', mode='full', repourl=src_url, - workdir=source_path, method='fresh', - alwaysUseLatest=False, retry = (60, 5), - description='pull.src')) - - # Grab the artifacts for our build. - f = artifacts.GetCompilerArtifacts(f) - host_compiler_dir = properties.WithProperties('%(builddir)s/host-compiler') - f = artifacts.GetCCFromCompilerArtifacts(f, host_compiler_dir) - f = artifacts.GetCXXFromCompilerArtifacts(f, host_compiler_dir) - - # Build libcxx. - CC = properties.WithProperties('%(cc_path)s') - CXX = properties.WithProperties('%(cxx_path)s') - HEADER_INCLUDE = \ - properties.WithProperties('-I %s' % os.path.join('%(builddir)s', - source_path, - 'include')) - SOURCE_LIB = \ - properties.WithProperties(os.path.join('%(builddir)s', - source_path, 'lib', - 'libc++.1.dylib')) - - f.addStep(buildbot.steps.shell.ShellCommand( - name='build.libcxx', command=['./buildit'], haltOnFailure=True, - workdir=os.path.join(source_path, 'lib'), - env={ 'CC' : CC, 'CXX' : CXX, 'TRIPLE' : '-apple-'})) - - # Get the 'lit' sources if we need to. - if lit_dir is None: - lit_dir = 'lit.src' - f.addStep(svn.SVN( - name='pull.lit', mode='incremental', method='fresh', - repourl='http://llvm.org/svn/llvm-project/llvm/trunk/utils/lit', - workdir=lit_dir, alwaysUseLatest=False)) - - # Install a copy of 'lit' in a virtualenv. - f.addStep(buildbot.steps.shell.ShellCommand( - name='venv.lit.clean', - command=['rm', '-rf', 'lit.venv'], - workdir='.', haltOnFailure=True)) - f.addStep(buildbot.steps.shell.ShellCommand( - name='venv.lit.make', - command=['/usr/local/bin/virtualenv', 'lit.venv'], - workdir='.', haltOnFailure=True)) - f.addStep(buildbot.steps.shell.ShellCommand( - name='venv.lit.install', - command=[ - properties.WithProperties('%(builddir)s/lit.venv/bin/python'), - 'setup.py', 'install'], - workdir=lit_dir, haltOnFailure=True)) - - # Run the tests with the system's dylib - f.addStep(lit_test_command.LitTestCommand( - name='test.libcxx.system', - command=[ - properties.WithProperties('%(builddir)s/lit.venv/bin/lit'), - '-v', '--show-xfail', '--show-unsupported', - properties.WithProperties( - '--param=cxx_under_test=%(cxx_path)s'), - '--param=use_system_lib=true', - 'sources/test'], - workdir='.')) - # Run the tests with the newly built dylib - f.addStep(lit_test_command.LitTestCommand( - name='test.libcxx.new', - command=[ - properties.WithProperties('%(builddir)s/lit.venv/bin/lit'), - '-v', '--show-xfail', '--show-unsupported', - properties.WithProperties( - '--param=cxx_under_test=%(cxx_path)s'), - '--param=use_system_lib=false', - 'sources/test'], - workdir='.')) - - return f diff --git a/zorg/buildbot/builders/LibcxxAndAbiBuilder.py b/zorg/buildbot/builders/LibcxxAndAbiBuilder.py --- a/zorg/buildbot/builders/LibcxxAndAbiBuilder.py +++ b/zorg/buildbot/builders/LibcxxAndAbiBuilder.py @@ -1,19 +1,11 @@ import buildbot.steps.shell import buildbot.process.properties as properties -import zorg.buildbot.commands.LitTestCommand as lit_test_command -import zorg.buildbot.util.artifacts as artifacts -import zorg.buildbot.util.phasedbuilderutils as phased_builder_utils - from zorg.buildbot.commands.LitTestCommand import LitTestCommand from zorg.buildbot.commands.CmakeCommand import CmakeCommand from zorg.buildbot.process.factory import LLVMBuildFactory from zorg.buildbot.builders import UnifiedTreeBuilder -reload(lit_test_command) -reload(artifacts) -reload(phased_builder_utils) - def getLibcxxAndAbiBuilder(f=None, env=None, cmake_extra_opts=None, lit_extra_opts=None, lit_extra_args=None, check_libcxx_abilist=False, diff --git a/zorg/buildbot/builders/NightlytestBuilder.py b/zorg/buildbot/builders/NightlytestBuilder.py deleted file mode 100644 --- a/zorg/buildbot/builders/NightlytestBuilder.py +++ /dev/null @@ -1,124 +0,0 @@ -from buildbot.steps.shell import Configure, ShellCommand -from buildbot.process.properties import WithProperties -from buildbot.steps.source import SVN - -from zorg.buildbot.commands.NightlyTestCommand import NightlyTestCommand - -import LLVMGCCBuilder -import ClangBuilder - -def getNightlytestBuildFactory(submitAux=None, *args, **kwargs): - f = LLVMGCCBuilder.getLLVMGCCBuildFactory(*args, **kwargs) - - # Prepare environmental variables. Set here all env we want everywhere. - merged_env = { - 'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences. - } - - env = kwargs.pop('env', None) - if env is not None: - merged_env.update(env) # Overwrite pre-set items with the given ones, so user can set anything. - - # Copy NT script. - f.addStep(ShellCommand(name="cp test script", - command=["cp", - WithProperties("%(builddir)s/llvm.src/utils/NewNightlyTest.pl"), - "."], - haltOnFailure = True, - description = "cp test script", - workdir = "llvm.nt", - env = merged_env)) - - submitCommand = [] - if submitAux is not None: - submitCommand = ['-submit-aux', - submitAux] - - f.addStep(ShellCommand(name="nightlytest", - command=["./NewNightlyTest.pl", - "-parallel-jobs", WithProperties("%(jobs)s"), - "-parallel", - "-noremoveatend", - "-noremoveresults", - "-release", - "-enable-llcbeta", - "-verbose", - "-nickname", WithProperties("%(slavename)s"), - "-test-cxxflags", "-I/usr/include/c++/4.2.1/i686-apple-darwin10 -I/usr/include/c++/4.2.1", - "-nosubmit", - "-teelogs"] + submitCommand, - haltOnFailure = True, - description = "nightlytest", - workdir = "llvm.nt", - env = { - 'LLVMGCCDIR' : WithProperties("%(builddir)s/llvm-gcc.install"), - 'BUILDDIR' : WithProperties("%(builddir)s/llvm.nt/build"), - 'WEBDIR' : WithProperties("%(builddir)s/llvm.nt/testresults"), - }.update(merged_env))) - return f - -def getFastNightlyTestBuildFactory(triple, xfails=[], clean=True, test=False, make='make', **kwargs): - # Build compiler to test. - f = ClangBuilder.getClangBuildFactory( - triple, clean=clean, test=test, - make=make, **kwargs) - - # Prepare environmental variables. Set here all env we want everywhere. - merged_env = { - 'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences. - } - env = kwargs.pop('env', None) - if env is not None: - merged_env.update(env) # Overwrite pre-set items with the given ones, so user can set anything. - - # Get the test-suite sources. - f.addStep(SVN(name = 'svn-test-suite', - mode = 'update', - baseURL = 'http://llvm.org/svn/llvm-project/test-suite/', - defaultBranch = 'trunk', - workdir = 'test-suite.src')) - - # Clean up. - if clean: - f.addStep(ShellCommand(name="rm.test-suite", - command=["rm", "-rf", "test-suite.obj"], - haltOnFailure = True, - description = "rm test-suite build dir", - workdir = ".", - env = merged_env)) - - # Configure. - f.addStep(Configure(name="configure.test-suite", - command=['../test-suite.src/configure', - WithProperties("--with-llvmsrc=%(builddir)s/llvm.src"), - WithProperties("--with-llvmobj=%(builddir)s/llvm.obj"), - WithProperties("--with-built-clang")], - haltOnFailure = True, - description = ["configuring", "test-suite"], - descriptionDone = ["configure", "test-suite"], - workdir = 'test-suite.obj', - env = merged_env)) - - # Build and test. - f.addStep(ShellCommand(name="rm.test-suite.report", - command=["rm", "-rf", - "test-suite.obj/report.nightly.raw.out", - "test-suite.obj/report.nightly.txt"], - haltOnFailure = True, - description = "rm test-suite report", - workdir = ".", - env = merged_env)) - f.addStep(NightlyTestCommand(name="make.test-suite", - command=[make, WithProperties("-j%(jobs)s"), - "ENABLE_PARALLEL_REPORT=1", - "DISABLE_CBE=1", "DISABLE_JIT=1", - "TEST=nightly", "report"], - haltOnFailure = True, - logfiles = { 'report' : 'report.nightly.txt' }, - xfails = xfails, - description = ["running", "test-suite"], - descriptionDone = ["run", "test-suite"], - workdir = 'test-suite.obj', - env = merged_env)) - - return f diff --git a/zorg/buildbot/builders/SanitizerBuilderII.py b/zorg/buildbot/builders/SanitizerBuilderII.py deleted file mode 100644 --- a/zorg/buildbot/builders/SanitizerBuilderII.py +++ /dev/null @@ -1,164 +0,0 @@ -from buildbot.process.factory import BuildFactory -from buildbot.process.properties import WithProperties -from buildbot.steps.source import SVN -from buildbot.steps.shell import ShellCommand, SetProperty -from buildbot.steps.shell import WarningCountingShellCommand -from buildbot.steps.slave import RemoveDirectory -from zorg.buildbot.commands.AnnotatedCommand import AnnotatedCommand -from zorg.buildbot.commands.NinjaCommand import NinjaCommand -from zorg.buildbot.conditions.FileConditions import FileDoesNotExist - -def getSanitizerBuildFactoryII( - clean=False, - sanity_check=True, - sanitizers=['sanitizer','asan','lsan','msan','tsan','ubsan','dfsan'], - common_cmake_options=None, # FIXME: For backward compatibility. Will be removed. - extra_configure_args=[], - prefixCommand=["nice", "-n", "10"], # For backward compatibility. - env=None, - jobs="%(jobs)s", - timeout=1200): - - llvm_srcdir = "llvm.src" - llvm_objdir = "llvm.obj" - - # Prepare environmental variables. Set here all env we want everywhere. - merged_env = { - 'TERM' : 'dumb', # Make sure Clang doesn't use color escape sequences. - } - if env: - # Overwrite pre-set items with the given ones, so user can set anything. - merged_env.update(env) - - f = BuildFactory() - - # Clean directory, if requested. - cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean - f.addStep(RemoveDirectory(name='clean '+llvm_objdir, - dir=llvm_objdir, - haltOnFailure=False, - flunkOnFailure=False, - doStepIf=cleanBuildRequested - )) - - # Get llvm, clang, ompiler-rt, libcxx, libcxxabi, libunwind - f.addStep(SVN(name='svn-llvm', - mode='update', - description='svn-llvm', - descriptionDone='svn-llvm', - baseURL='http://llvm.org/svn/llvm-project/llvm/', - defaultBranch='trunk', - workdir=llvm_srcdir)) - - f.addStep(SVN(name='svn-clang', - mode='update', - description='svn-clang', - descriptionDone='svn-clang', - baseURL='http://llvm.org/svn/llvm-project/cfe/', - defaultBranch='trunk', - workdir='%s/tools/clang' % llvm_srcdir)) - - f.addStep(SVN(name='svn-compiler-rt', - mode='update', - description='svn-compiler-rt', - descriptionDone='svn--compiler-rt', - baseURL='http://llvm.org/svn/llvm-project/compiler-rt/', - defaultBranch='trunk', - workdir='%s/projects/compiler-rt' % llvm_srcdir)) - - f.addStep(SVN(name='svn-libcxx', - mode='update', - description='svn-libcxx', - descriptionDone='svn-libcxx', - baseURL='http://llvm.org/svn/llvm-project/libcxx/', - defaultBranch='trunk', - workdir='%s/projects/libcxx' % llvm_srcdir)) - - f.addStep(SVN(name='svn-libcxxabi', - mode='update', - description='svn-libcxxabi', - descriptionDone='svn-libcxxabi', - baseURL='http://llvm.org/svn/llvm-project/libcxxabi/', - defaultBranch='trunk', - workdir='%s/projects/libcxxabi' % llvm_srcdir)) - - f.addStep(SVN(name='svn-libunwind', - mode='update', - description='svn-libunwind', - descriptionDone='svn-libunwind', - baseURL='http://llvm.org/svn/llvm-project/libunwind/', - defaultBranch='trunk', - workdir='%s/projects/libunwind' % llvm_srcdir)) - - # Run annotated command for sanitizer. - if sanity_check: - f.addStep( - AnnotatedCommand( - name="lint check", - description="lint check", - timeout=timeout, - haltOnFailure=False, #True, - warnOnWarnings=True, - command=["./check_lint.sh"], - workdir="%s/projects/compiler-rt/lib/sanitizer_common/scripts" % llvm_srcdir, - env=merged_env)) - - # Always build with ninja. - cmakeCommand = ["cmake", "-G", "Ninja"] - - # Reconsile configure args with the defaults we want. - if not any(a.startswith('-DCMAKE_BUILD_TYPE=') for a in extra_configure_args): - cmakeCommand.append('-DCMAKE_BUILD_TYPE=Release') - if not any(a.startswith('-DLLVM_ENABLE_WERROR=') for a in extra_configure_args): - cmakeCommand.append('-DLLVM_ENABLE_WERROR=OFF') - if not any(a.startswith('-DLLVM_ENABLE_ASSERTIONS=') for a in extra_configure_args): - cmakeCommand.append('-DLLVM_ENABLE_ASSERTIONS=ON') - if not any(a.startswith('-DCMAKE_C_COMPILER') for a in extra_configure_args): - cmakeCommand.append('-DCMAKE_C_COMPILER=clang') - if not any(a.startswith('-DCMAKE_CXX_COMPILER') for a in extra_configure_args): - cmakeCommand.append('-DCMAKE_CXX_COMPILER=clang++') - if not any(a.startswith('-DCMAKE_CXX_FLAGS') for a in extra_configure_args): - cmakeCommand.append('-DCMAKE_CXX_FLAGS=\"-std=c++11 -stdlib=libc++\"') - if not any(a.startswith('-DCMAKE_EXE_LINKER_FLAGS') for a in extra_configure_args): - cmakeCommand.append('-DCMAKE_EXE_LINKER_FLAGS=-lcxxrt') - if not any(a.startswith('-DLIBCXXABI_USE_LLVM_UNWINDER=') for a in extra_configure_args): - cmakeCommand.append('-DLIBCXXABI_USE_LLVM_UNWINDER=ON') - if not any(a.startswith('-DLLVM_LIT_ARGS=') for a in extra_configure_args): - cmakeCommand.append('-DLLVM_LIT_ARGS=\"-v\"') - - cmakeCommand += extra_configure_args + ["../%s" % llvm_srcdir] - - # Note: ShellCommand does not pass the params with special symbols right. - # The " ".join is a workaround for this bug. - f.addStep(ShellCommand( - name="cmake-configure", - description=["cmake configure"], - haltOnFailure=False, #True, - warnOnWarnings=True, - command=WithProperties(" ".join(cmakeCommand)), - env=merged_env, - workdir=llvm_objdir, - doStepIf=FileDoesNotExist("./%s/CMakeCache.txt" % llvm_objdir))) - - # Build everything. - f.addStep(NinjaCommand(name='build', - haltOnFailure=False, #True, - warnOnWarnings=True, - description=['building', 'with', 'ninja'], - descriptionDone=['built', 'with', 'ninja'], - workdir=llvm_objdir, - env=merged_env)) - - # Run tests for each of the requested sanitizers. - if sanitizers: - for s in sanitizers: - f.addStep( - NinjaCommand(name='test %s' % s, - targets=['check-%s' % s], - haltOnFailure=False, #True, - description=['testing', '%s' % s], - descriptionDone=['test', '%s' % s], - workdir=llvm_objdir, - env=merged_env)) - - return f diff --git a/zorg/buildbot/builders/Util.py b/zorg/buildbot/builders/Util.py --- a/zorg/buildbot/builders/Util.py +++ b/zorg/buildbot/builders/Util.py @@ -19,12 +19,14 @@ It doesn't work when VS is not installed at the default location. """ - # TODO: Implement autodetect for VS versions other than 2017 + # LLVM code base requires VS 2017 or later. + # This means vswhere.exe later than 1.0.40, so we add `-products *` to include Build Tools in the search + # to support build-tools only installations. vcvars_command = "for /f \"tokens=* USEBACKQ\" %%F in " \ - "(`\"%ProgramFiles(x86)%\\Microsoft Visual Studio\\Installer\\vswhere.exe\" -latest -property installationPath`) DO " \ + "(`\"%ProgramFiles(x86)%\\Microsoft Visual Studio\\Installer\\vswhere.exe\" -products * -latest -property installationPath`) DO " \ "\"%%F\"\\VC\\Auxiliary\\Build\\vcvarsall.bat" else: - # Older versions of VS + # Note: Support for older versions of VS is deprecated and will be removed. vcvars_command = "\"" + "\\".join((vs, '..','..','VC', 'vcvarsall.bat')) + "\"" vcvars_command = "%s %s && set" % (vcvars_command, arch_arg) diff --git a/zorg/buildbot/process/factory.py b/zorg/buildbot/process/factory.py --- a/zorg/buildbot/process/factory.py +++ b/zorg/buildbot/process/factory.py @@ -1,28 +1,9 @@ from collections import OrderedDict from buildbot.process.factory import BuildFactory -from buildbot.steps.source import SVN, Git +from buildbot.steps.source import Git from buildbot.steps.shell import WithProperties -# NOTE: svn_repos is deprecated and will be removed. -svn_repos = OrderedDict([ - ('llvm' , ("%(llvm_srcdir)s", '%(vcs_protocol:-http)s://llvm.org/svn/llvm-project/llvm/')), - ('clang' , ("%(llvm_srcdir)s/tools/clang", '%(vcs_protocol:-http)s://llvm.org/svn/llvm-project/cfe/')), - ('clang-tools-extra', ("%(llvm_srcdir)s/tools/clang/tools/extra", '%(vcs_protocol:-http)s://llvm.org/svn/llvm-project/clang-tools-extra/')), - ('compiler-rt' , ("%(llvm_srcdir)s/projects/compiler-rt", '%(vcs_protocol:-http)s://llvm.org/svn/llvm-project/compiler-rt/')), - ('libcxx' , ("%(llvm_srcdir)s/projects/libcxx", '%(vcs_protocol:-http)s://llvm.org/svn/llvm-project/libcxx/')), - ('libcxxabi' , ("%(llvm_srcdir)s/projects/libcxxabi", '%(vcs_protocol:-http)s://llvm.org/svn/llvm-project/libcxxabi/')), - ('libunwind' , ("%(llvm_srcdir)s/projects/libunwind", '%(vcs_protocol:-http)s://llvm.org/svn/llvm-project/libunwind/')), - ('lld' , ("%(llvm_srcdir)s/tools/lld", '%(vcs_protocol:-http)s://llvm.org/svn/llvm-project/lld/')), - ('lnt' , ("test/lnt", '%(vcs_protocol:-http)s://llvm.org/svn/llvm-project/lnt/')), - ('test-suite' , ("test/test-suite", '%(vcs_protocol:-http)s://llvm.org/svn/llvm-project/test-suite/')), - ('lldb' , ("%(llvm_srcdir)s/tools/lldb", '%(vcs_protocol:-http)s://llvm.org/svn/llvm-project/lldb/')), - ('llgo' , ("%(llvm_srcdir)s/tools/llgo", '%(vcs_protocol:-http)s://llvm.org/svn/llvm-project/llgo/')), - ('polly' , ("%(llvm_srcdir)s/tools/polly", '%(vcs_protocol:-http)s://llvm.org/svn/llvm-project/polly/')), - ('openmp' , ("%(llvm_srcdir)s/tools/openmp", '%(vcs_protocol:-http)s://llvm.org/svn/llvm-project/openmp/')), - ('zorg' , ("zorg", '%(vcs_protocol:-http)s://llvm.org/svn/llvm-project/zorg/')), - ]) - class LLVMBuildFactory(BuildFactory): """ TODO: Document @@ -37,7 +18,7 @@ else: self.depends_on_projects = frozenset(depends_on_projects) - # By default LLVMBuildFactory works in the legacy mode. + # FIXME: legacy mode is deprecated and will be removed. self.is_legacy_mode = kwargs.pop('is_legacy_mode', False) # Directories. @@ -50,18 +31,14 @@ for k,v in kwargs.items(): setattr(self, k, v) - if self.is_legacy_mode: - self.llvm_srcdir = self.llvm_srcdir or "llvm" - self.obj_dir = self.obj_dir or "build" - else: - self.monorepo_dir = self.llvm_srcdir or "llvm-project" - self.llvm_srcdir = \ + self.monorepo_dir = self.llvm_srcdir or "llvm-project" + self.llvm_srcdir = \ "%(monorepo_dir)s/llvm" % {'monorepo_dir' : self.monorepo_dir} - self.obj_dir = \ + self.obj_dir = \ self.obj_dir or "build" - # Repourl_prefix could be specified per builder. Otherwise we use github. - self.repourl_prefix = kwargs.pop('repourl_prefix', 'https://github.com/llvm/') + # Repourl_prefix could be specified per builder. Otherwise we use github. + self.repourl_prefix = kwargs.pop('repourl_prefix', 'https://github.com/llvm/') @staticmethod @@ -94,40 +71,10 @@ return rel_path - # llvm_srcdir - Path to the root of the unified source tree. - # mode - SVN checkout mode. - # defaultBranch - the default branch to checkout. - # and so on, see the list of the SVN params. - # NOTE: addSVNSteps is deprecated and will be removed. Please use addGetSourcecodeSteps instead. - def addSVNSteps(self, llvm_srcdir=None, **kwargs): - if llvm_srcdir is None: - llvm_srcdir = self.llvm_srcdir - if not kwargs.get('mode', None): - kwargs['mode'] = 'update' - if not kwargs.get('defaultBranch', None): - kwargs['defaultBranch'] = 'trunk' - - # Add a SVM step for each project this builder depends on. - # We want the projects be always checked out in a certain order. - for project in svn_repos.keys(): - if project in self.depends_on_projects: - workdir, baseURL = svn_repos[project] - self.addStep( - SVN(name='svn-%s' % project, - workdir=workdir % {'llvm_srcdir' : llvm_srcdir}, - baseURL=WithProperties(baseURL), - **kwargs)) - - def addGetSourcecodeSteps(self, **kwargs): # Remove 'is_legacy_mode' if it leaked in to kwargs. kwargs.pop('is_legacy_mode', None) - # Bail out if we are in the legacy mode and SVN checkout is required. - if self.is_legacy_mode: - self.addSVNSteps(**kwargs) - return - # Checkout the monorepo. self.addStep( Git(name='Checkout the source code', @@ -143,49 +90,26 @@ # Remove 'is_legacy_mode' if it leaked in to kwargs. kwargs.pop('is_legacy_mode', None) - # Bail out if we are in the legacy mode and SVN checkout is required. - if self.is_legacy_mode: - workdir, baseURL = svn_repos[project] - - if not name: - name = 'svn-%s' % project - - # Check out to the given directory if any. - # Otherwise this is a part of the unified source tree. - if src_dir is None: - src_dir = workdir % {'llvm_srcdir' : self.llvm_srcdir} - - if not kwargs.get('mode', None): - kwargs['mode'] = 'update' - if not kwargs.get('defaultBranch', None): - kwargs['defaultBranch'] = 'trunk' - - self.addStep( - SVN(name=name, - workdir=src_dir, - baseURL=WithProperties(baseURL), - **kwargs)) - else: - # project contains a repo name which is not a part of the monorepo. - # We do not enforce it here, though. - _repourl = kwargs.pop('repourl', None) - if not _repourl: - _repourl = self.repourl_prefix + "llvm-%s.git" % project - - if not name: - name = 'Checkout %s' % project - - # Check out to the given directory if any. - # Otherwise this is a part of the unified source tree. - if src_dir is None: - src_dir = 'llvm-%s' % project - - # Ignore workdir if given. We check out to src_dir. - kwargs.pop('workdir', None) - - self.addStep( - Git(name=name, - repourl=_repourl, - progress=True, - workdir=WithProperties(src_dir), - **kwargs)) + # project contains a repo name which is not a part of the monorepo. + # We do not enforce it here, though. + _repourl = kwargs.pop('repourl', None) + if not _repourl: + _repourl = self.repourl_prefix + "llvm-%s.git" % project + + if not name: + name = 'Checkout %s' % project + + # Check out to the given directory if any. + # Otherwise this is a part of the unified source tree. + if src_dir is None: + src_dir = 'llvm-%s' % project + + # Ignore workdir if given. We check out to src_dir. + kwargs.pop('workdir', None) + + self.addStep( + Git(name=name, + repourl=_repourl, + progress=True, + workdir=WithProperties(src_dir), + **kwargs))