diff --git a/buildbot/google/docker/buildbot-sanitizer-windows/Dockerfile b/buildbot/google/docker/buildbot-sanitizer-windows/Dockerfile new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-sanitizer-windows/Dockerfile @@ -0,0 +1,21 @@ +# escape=` +FROM gcr.io/sanitizer-bots/windows-base-vscode2019:8 + +ENV WORKER_NAME=sanitizer-windows + +# copy script to start the agent +COPY run.ps1 . +COPY fix_buildbot_tac_paths.py . + +# Set location for sccache cache +ENV SCCACHE_DIR="C:\volumes\sccache" + +# Set the maximum sccache local cache size +ENV SCCACHE_CACHE_SIZE="10G" + +# Move buildbot work dir to a volume to avoid of disc issues +VOLUME C:\volumes\buildbot + +# Configure 32bit tools ("x86") instead of 64bit ("amd64") +CMD ["powershell.exe", "-NoLogo", "-ExecutionPolicy", "Bypass", ` + "c:\\buildbot\\run.ps1"] diff --git a/buildbot/google/docker/buildbot-sanitizer-windows/VERSION b/buildbot/google/docker/buildbot-sanitizer-windows/VERSION new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-sanitizer-windows/VERSION @@ -0,0 +1 @@ +1 diff --git a/buildbot/google/docker/buildbot-sanitizer-windows/admin b/buildbot/google/docker/buildbot-sanitizer-windows/admin new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-sanitizer-windows/admin @@ -0,0 +1 @@ +Reid Kleckner diff --git a/buildbot/google/docker/buildbot-sanitizer-windows/fix_buildbot_tac_paths.py b/buildbot/google/docker/buildbot-sanitizer-windows/fix_buildbot_tac_paths.py new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-sanitizer-windows/fix_buildbot_tac_paths.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +# Replace backward slashes in the paths in a "buildbot.tac" file + +import argparse +import re +import shutil + + +def fix_slashes(buildbot_tac_path : str): + shutil.copy(buildbot_tac_path, buildbot_tac_path + ".bak") + result = [] + with open(buildbot_tac_path) as buildbot_tac_file: + buildbot_tac = buildbot_tac_file.readlines() + for line in buildbot_tac: + if line.lstrip().startswith('basedir'): + line = line.replace(r'\\','/') + result.append(line) + with open(buildbot_tac_path, 'w') as buildbot_tac_file: + buildbot_tac_file.writelines(result) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('buildbot_tac_path', help="path of 'buildbot.tac' file") + args = parser.parse_args() + fix_slashes(args.buildbot_tac_path) diff --git a/buildbot/google/docker/buildbot-sanitizer-windows/run.ps1 b/buildbot/google/docker/buildbot-sanitizer-windows/run.ps1 new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-sanitizer-windows/run.ps1 @@ -0,0 +1,51 @@ +# Stop the script on the first error +$ErrorActionPreference = "Stop" + +# Read password from file +$WORKER_PASSWORD=Get-Content "C:\volumes\secrets\token" + +#Create short drive alias "Q:" for worker dir, to keep path lengths <260 +SUBST Q: C:\volumes\buildbot +Get-PSDrive + +# chdir to the volume storing the temporary data +# This is a mounted volume +Set-Location Q:\ + +# Write host and admin information +# delete old folder if it exists +# configure powershell output to use UTF-8, otherwiese buildbot can't read the file +if ( Test-Path -Path 'info\' -PathType Container ) { + Remove-Item -Recurse -Force info +} +mkdir "info\" +$HOST_FILE="info\host" +$PSDefaultParameterValues['Out-File:Encoding'] = 'utf8' +Write-Output "Windows version: $(cmd /c ver)" > $HOST_FILE +Write-Output "VCTools version: $(ls C:\BuildTools\VC\Tools\MSVC | Select -ExpandProperty Name)" >> $HOST_FILE +Write-Output "Cores : $((Get-CimInstance Win32_ComputerSystem).NumberOfLogicalProcessors)" >> $HOST_FILE +Write-Output "RAM : $([int][Math]::Round((Get-CimInstance Win32_ComputerSystem).TotalPhysicalMemory / 1073741824)) GB" >> $HOST_FILE +Write-Output "Reid Kleckner " > "info\admin" + +# create the worker +Write-Output "creating worker..." +buildslave create-slave --keepalive=200 Q: ` + "lab.llvm.org:9994" "$env:WORKER_NAME" "$WORKER_PASSWORD" +# Note: Powershell does NOT exit on non-zero return codes, so we need to check manually. +if ($LASTEXITCODE -ne 0) { throw "Exit code of 'buildslave create' is $LASTEXITCODE" } + +# Replace backward slashes with forward slashes on buildbot.tac +# Otherwise Cmake will go into infinite re-configuration loops +python C:\buildbot\fix_buildbot_tac_paths.py buildbot.tac +if ($LASTEXITCODE -ne 0) { throw "Exit code of 'fix_buildbot_tac_paths' is $LASTEXITCODE" } + +# start the daemon, as this does not print and logs to std out, run it in the background +Write-Output "starting worker..." +cmd /c Start /B buildslave start Q: + +# Wait a bit until the logfile exists +Start-Sleep -s 5 + +# To keep the container running and produce log outputs: dump the worker +# log to stdout, this is the windows equivalent of `tail -f` +Get-Content -Path "Q:\twistd.log" -Wait diff --git a/zorg/buildbot/builders/annotated/annotated_builder.py b/zorg/buildbot/builders/annotated/annotated_builder.py --- a/zorg/buildbot/builders/annotated/annotated_builder.py +++ b/zorg/buildbot/builders/annotated/annotated_builder.py @@ -13,6 +13,7 @@ from os.path import join as pjoin VSWHERE_PATH = "C:/Program Files (x86)/Microsoft Visual Studio/Installer/vswhere.exe" +BUILDTOOLS_VSDEVCMD = "C:/BuildTools/Common7/Tools/VsDevCmd.bat" def get_argument_parser(*args, **kwargs): ap = argparse.ArgumentParser(*args, **kwargs) @@ -323,9 +324,17 @@ def get_vcvars(vs_tools, arch): - """Get the VC tools environment using vswhere.exe from VS 2017 + """Get the VC tools environment using vswhere.exe or buildtools docker - This code is following the guidelines from strategy 1 in this blog post: + This is intended to work either when VS is in its standard installation + location, or when the docker instructions have been followed, and we can + find Visual C++ in C:/BuildTools. + + Visual Studio provides a docker image with instructions here: + https://docs.microsoft.com/en-us/visualstudio/install/build-tools-container?view=vs-2019 + + This vswhere code is following the guidelines from strategy 1 in this blog + post: https://blogs.msdn.microsoft.com/vcblog/2017/03/06/finding-the-visual-c-compiler-tools-in-visual-studio-2017/ It doesn't work when VS is not installed at the default location. @@ -349,6 +358,8 @@ raise ValueError("VS install path does not exist: " + vs_path) vcvars_path = pjoin(vs_path, 'VC', 'Auxiliary', 'Build', 'vcvarsall.bat') + elif os.path.exists(BUILDTOOLS_VSDEVCMD): + vcvars_path = BUILDTOOLS_VSDEVCMD elif vs_tools is None: vs_tools = os.path.expandvars('%VS140COMNTOOLS%') vcvars_path = pjoin(vs_tools, '..', '..', 'VC', 'vcvarsall.bat')