diff --git a/buildbot/google/docker/README.md b/buildbot/google/docker/README.md --- a/buildbot/google/docker/README.md +++ b/buildbot/google/docker/README.md @@ -21,3 +21,6 @@ the container. The secret file shall just contain the password in plain text. Kubernetes offers a [mechanism to handle secrets](https://kubernetes.io/docs/concepts/configuration/secret/). + +# Setting up Windows VM for development +See [windows.md](windows.md). diff --git a/buildbot/google/docker/build_deploy.sh b/buildbot/google/docker/build_deploy.sh --- a/buildbot/google/docker/build_deploy.sh +++ b/buildbot/google/docker/build_deploy.sh @@ -15,19 +15,17 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" IMAGE_NAME="${1%/}" - -# increment version number cd "${DIR}/${IMAGE_NAME}" -# get version numbers from repository -# FIXME: use variables to configure URL -ALL_VERSIONS=$(gcloud container images list-tags gcr.io/sanitizer-bots/${IMAGE_NAME} --format=text | \ - awk '/tags.*:\W+[0-9]+$/ {print $2}') -# read local version number from file and add it to the array -ALL_VERSIONS+=($(cat VERSION)) -# find maximum version number and increment it -VERSION=$(echo "${ALL_VERSIONS[*]}" | sort -nr | head -n1) -VERSION=$(( ${VERSION} + 1 )) +# read local version number from file and increment it +# TODO: maybe also get latest version number from repository and take the +# maximum of local and repo versions. +VERSION=$(( $(cat VERSION) + 1 )) + +# on Windows: USER=$USERNAME +if [[ "${OS}" == "Windows_NT" ]] ; then + USER="${USERNAME}" +fi # get the git hash and add some suffixes GIT_HASH=$(git rev-parse HEAD) diff --git a/buildbot/google/docker/build_run.sh b/buildbot/google/docker/build_run.sh --- a/buildbot/google/docker/build_run.sh +++ b/buildbot/google/docker/build_run.sh @@ -12,7 +12,7 @@ # optional: #===----------------------------------------------------------------------===// -set -eux +set -eu DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" IMAGE_NAME="${1%/}" @@ -25,5 +25,24 @@ cd "${DIR}/${IMAGE_NAME}" +# Mount a volume "workertest" to persit across test runs. +# Use this to keep e.g. a git checkout or partial build across runs +if [[ $(docker volume ls | grep workertest | wc -l) == 0 ]] ; then + docker volume create workertest +fi + +# Volume to presist the build cache e.g. ccache or sccache. +# This will speed up local testing. +if [[ $(docker volume ls | grep workercache | wc -l) == 0 ]] ; then + docker volume create workercache +fi + +# Define arguments for mounting the volumes +# These differ on Windows and Linux +VOLUMES="-v ${SECRET_STORAGE}:/secrets -v workertest:/test" +if [[ "${OS}" == "Windows_NT" ]] ; then + VOLUMES="-v ${SECRET_STORAGE}:c:\\volumes\\secrets -v workertest:c:\volumes\\test -v workercache:c:\sccache" +fi + docker build -t "${IMAGE_NAME}:latest" . -docker run -it -v "${SECRET_STORAGE}":/secrets "${IMAGE_NAME}" ${CMD} +docker run -it ${VOLUMES} "${IMAGE_NAME}:latest" ${CMD} diff --git a/buildbot/google/docker/buildbot-windows10-vs2019/Dockerfile b/buildbot/google/docker/buildbot-windows10-vs2019/Dockerfile new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-windows10-vs2019/Dockerfile @@ -0,0 +1,19 @@ +# escape=` +FROM gcr.io/sanitizer-bots/windows-base-vscode2019:5 + +ENV WORKER_NAME=windows10-vs2019 + +# copy admin information +RUN md %WORKER_NAME%\info +COPY admin ${WORKER_NAME}\info + +# copy script to start the agent +COPY run.ps1 . + +# Set location for sccache cache +ENV SCCACHE_DIR="C:\volumes\sccache" + +# Configure 32bit tools ("x86") instead of 64bit ("amd64") +ENTRYPOINT ["C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat", "-arch=x86", "-host_arch=amd64", "&&", "powershell.exe", "-NoLogo", "-ExecutionPolicy", "Bypass"] + +CMD ["c:\\buildbot\\run.ps1"] diff --git a/buildbot/google/docker/buildbot-windows10-vs2019/VERSION b/buildbot/google/docker/buildbot-windows10-vs2019/VERSION new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-windows10-vs2019/VERSION @@ -0,0 +1 @@ +3 diff --git a/buildbot/google/docker/buildbot-windows10-vs2019/admin b/buildbot/google/docker/buildbot-windows10-vs2019/admin new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-windows10-vs2019/admin @@ -0,0 +1 @@ +Christian Kühnel diff --git a/buildbot/google/docker/buildbot-windows10-vs2019/run.ps1 b/buildbot/google/docker/buildbot-windows10-vs2019/run.ps1 new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-windows10-vs2019/run.ps1 @@ -0,0 +1,33 @@ +# Stop the script on the first error +$ErrorActionPreference = "Stop" + +# Read password from file +$WORKER_PASSWORD=Get-Content "C:\volumes\secrets\token" + +# Write host information +# configure powershell output to use UTF-8, otherwiese buildbot can't read the file +$HOST_FILE="$env:WORKER_NAME\info\host" +$PSDefaultParameterValues['Out-File:Encoding'] = 'utf8'; ` +Write-Output "Windows version: $(cmd /c ver)" > $HOST_FILE +Write-Output "VCTools version: $env:VCToolsVersion" >> $HOST_FILE +Write-Output "Cores : $((Get-CimInstance Win32_ComputerSystem).NumberOfLogicalProcessors)" >> $HOST_FILE +Write-Output "RAM : $([int][Math]::Round((Get-CimInstance Win32_ComputerSystem).TotalPhysicalMemory / 1073741824)) GB" >> $HOST_FILE + +# create the worker +Write-Output "creating worker..." +buildslave create-slave --keepalive=200 "$env:WORKER_NAME" ` + "lab.llvm.org:9994" "$env:WORKER_NAME" "$WORKER_PASSWORD" + +# Note: Powershell does NOT exit on non-zero return codes, so we need to check manually. +if ($LASTEXITCODE -ne 0) { throw "Exit code is $LASTEXITCODE" } + +# start the daemon, as this does not print and logs to std out, run it in the background +Write-Output "starting worker..." +cmd /c Start /B buildslave start "$env:WORKER_NAME" + +# Wait a bit until the logfile exists +Start-Sleep -s 5 + +# To keep the container running and produce log outputs: dump the worker +# log to stdout, this is the windows equivalent of `tail -f` +Get-Content -Path "$env:WORKER_NAME\twistd.log" -Wait diff --git a/buildbot/google/docker/pod_login.sh b/buildbot/google/docker/pod_login.sh --- a/buildbot/google/docker/pod_login.sh +++ b/buildbot/google/docker/pod_login.sh @@ -13,15 +13,23 @@ # pod name fragement : # part of the name of the pod to log in, eg. name of the # deployment, if we have only one of them +# command (optional): +# Command to be run in the container. Default: /bin/bash #===----------------------------------------------------------------------===// set -eu WORKLOAD_NAME=$1 +CMD="/bin/bash" +if [ "$#" -eq 2 ]; +then + CMD="$2" +fi + # get name of the pod POD=$(kubectl get pod -o name | grep "$1") # FIXME: exit if more than one pod is returned # login to the pod -kubectl exec --stdin --tty "${POD}" -- /bin/bash +kubectl exec --stdin --tty "${POD}" -- "${CMD}" diff --git a/buildbot/google/docker/windows-base-vscode2019/Dockerfile b/buildbot/google/docker/windows-base-vscode2019/Dockerfile new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/windows-base-vscode2019/Dockerfile @@ -0,0 +1,82 @@ +# escape=` +# Windows is picky about host and container OS versions. Recommondation: +# Use "LTSC" releases for both. +# https://docs.microsoft.com/en-us/virtualization/windowscontainers/deploy-containers/version-compatibility +# https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#choose_your_windows_server_node_image +FROM mcr.microsoft.com/dotnet/framework/sdk:4.8-windowsservercore-ltsc2019 + +# Restore the default Windows shell for correct batch processing. +SHELL ["cmd", "/S", "/C"] + +# Download the Build Tools bootstrapper. +ADD https://aka.ms/vs/16/release/vs_buildtools.exe C:\TEMP\vs_buildtools.exe + +# Download channel for fixed install. +ARG CHANNEL_URL=https://aka.ms/vs/16/release/channel +ADD ${CHANNEL_URL} C:\TEMP\VisualStudio.chman + +# Install Build Tools with C++ workload. +# - Documentation for docker installation +# https://docs.microsoft.com/en-us/visualstudio/install/build-tools-container?view=vs-2019 +# - Documentation on workloads +# https://docs.microsoft.com/en-us/visualstudio/install/workload-component-id-vs-build-tools?view=vs-2019#c-build-tools +# - Documentation on flags +# https://docs.microsoft.com/en-us/visualstudio/install/use-command-line-parameters-to-install-visual-studio?view=vs-2019 +RUN C:\TEMP\vs_buildtools.exe --quiet --wait --norestart --nocache ` + --channelUri C:\TEMP\VisualStudio.chman ` + --installChannelUri C:\TEMP\VisualStudio.chman ` + --installPath C:\BuildTools ` + --add Microsoft.VisualStudio.Workload.VCTools --includeRecommended ` + --add Microsoft.VisualStudio.Component.VC.ATL ` + || IF "%ERRORLEVEL%"=="3010" EXIT 0 + +# install chocolately as package manager +RUN powershell -NoProfile -InputFormat None -Command ` + iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')) ; ` + choco feature disable --name showDownloadProgress + +# install tools as described in https://llvm.org/docs/GettingStartedVS.html +# and a few more that were not documented... +RUN choco install -y ninja +RUN choco install -y git +RUN choco install -y cmake +RUN choco install -y gnuwin +# lit runs best with Python3, Buildbot worker needs Python2 +RUN choco install -y python2 python3 +RUN choco install -y sccache +# Install psutils for python3 for lit +RUN pip3 install psutil +# Install buildbot using python2 +RUN pip2 install --trusted-host pythonhosted.org buildbot-slave==0.8.11 pywin32 + +# configure Python encoding +ENV PYTHONIOENCODING=UTF-8 + +# update the path variable +RUN powershell -NoProfile -InputFormat None -Command ` + $path = $env:path + ';c:\Program Files (x86)\GnuWin32\bin;C:\Program Files\CMake\bin'; ` + Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment\' -Name Path -Value $path + +# use this folder to store the worksapce' +WORKDIR C:\buildbot + +# Root folder for all docker volumes. Make it explicite that all subfolders +# are volumes. Volumes are a common pitfall for config errors. +RUN md c:\volumes + +# storage for cached builds +VOLUME C:\volumes\sccache + +# storage for secrets +VOLUME C:\volumes\secrets + +# support long file names during git checkout +RUN git config --system core.longpaths true & ` + git config --global core.autocrlf false + +# Define the entry point for the docker container. +# This entry point starts the developer command prompt and launches the PowerShell shell. +# +# For running manually: +# C:\BuildTools\Common7\Tools\VsDevCmd.bat -arch=amd64 -host_arch=amd64 +ENTRYPOINT ["C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat", "-arch=amd64", "-host_arch=amd64", "&&", "powershell.exe", "-NoLogo", "-ExecutionPolicy", "Bypass"] diff --git a/buildbot/google/docker/windows-base-vscode2019/VERSION b/buildbot/google/docker/windows-base-vscode2019/VERSION new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/windows-base-vscode2019/VERSION @@ -0,0 +1 @@ +5 diff --git a/buildbot/google/docker/windows.md b/buildbot/google/docker/windows.md new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/windows.md @@ -0,0 +1,68 @@ +# Overview + +Running a Windows buildbot differs from Linux a bit. So this document will give +hints on the differences. + +# Create VM +To create a new Dockerfile and you do not have a local Windows machine, create +a VM in the the cloud. + +* Use the latest Windows image with "Desktop experience" in the description. +* Use something with at least 16 cores, to get reasonable build speed while + testing your images. +* Get 200GB of persistent SSD, this is much faster than the normal presistent + storage. +* In the "Cloud API access scopes" select "Read Write" for "Storage" to be able + to upload docker images form the VM. + +# setup your workstation +For frequent access to your Windows VM it's conveneint to install Remmina, a +remote desktop client for Linux, to connect the the Windows VM. + +# Configure the VM +1. install [Chocolately](https://chocolatey.org/docs/installation) as package + manager. +1. Install your development tools, a good starting point is: + ``` + choco install -y git vscode googlechrome arcanist vim + `` +1. Setup your github account and push access. Start "Git bash" and run + `ssh-keygen`, upload your public key to Github. +1. Setup your Phabricator account for `arc diff`. +1. `git clone ssh://git@github.com/llvm/llvm-zorg.git` +1. To install docker, run in an powershell with admin rights: + ```powershell + Install-PackageProvider -Name NuGet -Force + Install-Module -Name DockerMsftProvider -Repository PSGallery -Force + Install-Package -Name docker -ProviderName DockerMsftProvider -Force + sc.exe config docker start=delayed-auto + ``` + Then reboot your VM to start the Docker service. +1. Maybe disable Windows Defender anti vrius "realtime protection" to get better + IO performance. +1. Install [Google Cloud SDK](https://cloud.google.com/sdk/install). Installing + via chocolately is sometimes broken. Then run `gcloud auth login` and + `gcloud auth configure-docker` to be able to push images to the registry. +1. Create a file `token` somewhere and store your worker password there. You + will need this to test your worker locally. + +# General Hints + +## Get ideas from LLVM premerge-checks +[Premerge-testing](https://github.com/google/llvm-premerge-checks/) is also +running LLVM builds on Windows. You might be able to get some ideas from there. + +## Bash on Windows +Git on Windows includes a bash. Right-click the Start menu und launch a +"Windows Powershell (Admin)" (with admin rights, as docker requires that) and +run `bash` from there. Then you should be able to use the helper scripts +(e.g. `build_run.sh`) on Windows as well. + +## Resource Usage +If Windows seems stuck, use the `Resource Monitor` to check what the processes +are doing. If you need even more insight, e.g. debugging builds or test, use the +`Process Explorer` (`choco install procexp`) to figure out what's going on. + +## Windows base image with Visual Studio 2019 +Reuse the `windows-base-vscode2019` image, it contains everything you need to +build and test with Visual Studio 2019.