diff --git a/buildbot/google/docker/buildbot-cpp20/Dockerfile b/buildbot/google/docker/buildbot-cpp20/Dockerfile new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-cpp20/Dockerfile @@ -0,0 +1,70 @@ +#===-- Dockerfile --------------------------------------------------------===// +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +#===----------------------------------------------------------------------===// +# Environment variables configurable at runtime: +# BUILDBOT_PORT - server port to connect to +#===----------------------------------------------------------------------===// + +FROM debian:bullseye + +# Install build tools. +RUN apt-get update && \ + apt-get install -y \ + software-properties-common apt-transport-https ca-certificates \ + git wget gnupg \ + cmake ninja-build \ + ccache \ + python3 python3-pip python3-psutil && \ + # Install clang-15 to make the minimal C++20 support in the toolchain. + echo "deb http://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-15 main" >> /etc/apt/sources.list.d/llvm-15.list && \ + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key >> /etc/apt/trusted.gpg.d/llvm.asc && \ + apt-get update && \ + apt-get install -y clang-15 lld-15 && \ + apt-get clean + +# Install build bot (server was at 2.8.5-dev at time of writing). +RUN pip3 install buildbot-worker==2.8.4 + +# Workaround permissions issues when writing to named volumes +# https://github.com/docker/compose/issues/3270#issuecomment-206214034 +RUN mkdir -p /vol/test /vol/ccache /vol/worker ; \ + chmod -R 777 /vol + +# Volume to mount secrets into the container. +VOLUME /vol/secrets +# Volume to store data for local, manual testing of the container. +VOLUME /vol/test +# Volume to store ccache. +VOLUME /vol/ccache +ENV CCACHE_DIR=/vol/ccache +# Volume for worker working directory. +VOLUME /vol/worker + +# Create user account, some tests fail if run as root. +RUN useradd buildbot --create-home +WORKDIR /vol/worker + +# Copy startup script. +COPY run.sh /home/buildbot/ +RUN chmod a+rx /home/buildbot/run.sh + +USER buildbot +ENV WORKER_NAME="clang-debian-cpp20" + +# Allow the server port of this agent to be configurable during deployment. +# This way we can connect the same image to production and integration. +# Ports: +# 9990 - production +# 9994 - integration +ENV BUILDBOT_PORT="9994" + +# Configure LLVM tools. +ENV CC=clang-15 +ENV CXX=clang++-15 +ENV LD=lld-15 + +# Run startup script. +CMD /home/buildbot/run.sh diff --git a/buildbot/google/docker/buildbot-cpp20/VERSION b/buildbot/google/docker/buildbot-cpp20/VERSION new file mode 100644 --- /dev/null +++ b/buildbot/google/docker/buildbot-cpp20/VERSION @@ -0,0 +1 @@ +1 diff --git a/buildbot/google/docker/buildbot-cpp20/run.sh b/buildbot/google/docker/buildbot-cpp20/run.sh new file mode 100755 --- /dev/null +++ b/buildbot/google/docker/buildbot-cpp20/run.sh @@ -0,0 +1,53 @@ +#!/bin/bash +#===-- run.sh -------------------------------------------------------------===// +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +#===----------------------------------------------------------------------===// +# This script will start the buildbot worker +# +#===----------------------------------------------------------------------===// + +set -eu + +# Read the worker password from a mounted file. +WORKER_PASSWORD=$(cat /vol/secrets/token) + +# Set up buildbot host and maintainer info. +mkdir -p "${WORKER_NAME}/info/" +echo "Ilya Biryukov " > "${WORKER_NAME}/info/admin" + +# generate the host information of this worker +( + uname -a ; \ + cat /proc/cpuinfo | grep "model name" | head -n1 | cut -d " " -f 3- ;\ + echo "number of cores: $(nproc)" ;\ + cat /proc/meminfo | grep MemTo ;\ + lsb_release -d | cut -f 2- ; \ + clang --version | head -n1 ; \ + ld.lld-11 --version ; \ + cmake --version | head -n1 ; \ +) > ${WORKER_NAME}/info/host + +echo "Host information:" +cat ${WORKER_NAME}/info/host + +# create the folder structure +echo "creating worker ${WORKER_NAME} at port ${BUILDBOT_PORT}..." +buildbot-worker create-worker --keepalive=200 "${WORKER_NAME}" \ + lab.llvm.org:${BUILDBOT_PORT} "${WORKER_NAME}" "${WORKER_PASSWORD}" + +# start the daemon, this command returns immediately +echo "starting worker..." +set +e +if ! buildbot-worker start "${WORKER_NAME}"; then + echo ERROR: starting worker failed. contents of twistd.log: + cat "${WORKER_NAME}/twistd.log" + exit 1 +fi + +# To keep the container running and produce log outputs: dump the worker +# log to stdout +echo "Following worker log..." +tail -f ${WORKER_NAME}/twistd.log diff --git a/buildbot/google/terraform/main.tf b/buildbot/google/terraform/main.tf --- a/buildbot/google/terraform/main.tf +++ b/buildbot/google/terraform/main.tf @@ -257,3 +257,103 @@ } } } + +resource "kubernetes_deployment" "clang-debian-cpp20" { + metadata { + name = "clang-debian-cpp20" + labels = { + app = "clang-debian-cpp20" + } + } + + spec { + # create one instance of this container + replicas = 1 + + selector { + match_labels = { + app = "clang-debian-cpp20" + } + } + strategy{ + rolling_update{ + # do not deploy more replicas, as the buildbot server + # can't handle multiple workers with the same credentials + max_surge = 0 + # Allow to have 0 replicas during updates. + max_unavailable = 1 + } + type = "RollingUpdate" + } + template { + metadata { + labels = { + app = "clang-debian-cpp20" + } + } + + spec { + container { + image = "${var.gcp_config.gcr_prefix}/buildbot-clang-debian-cpp20:1" + name = "buildbot-clang-debian-cpp20" + + # reserve "-1" for this image, kubernetes also + # needs <1 core for management tools + resources { + limits { + cpu = "15" + memory = "32G" + } + requests { + cpu = "15" + memory = "32G" + } + } + + volume_mount { + mount_path = "/vol/secrets" + name = "buildbot-token" + } + volume_mount { + mount_path = "/vol/cccache" + name = "ccache-vol" + } + volume_mount { + mount_path = "/vol/worker" + name = "worker-vol" + } + + env { + # connect to staging at port 9994. + # port for production is 9990. + name = "BUILDBOT_PORT" + value = "9994" + } + } + # select which node pool to deploy to + node_selector = { + pool = "linux-16-core-pool" + } + # restart in case of any crashes + restart_policy = "Always" + + # select the secret to be mounted + volume { + name = "buildbot-token" + secret { + optional = false + secret_name = "password-clang-debian-cpp20" + } + } + volume { + name = "ccache-vol" + empty_dir {} + } + volume { + name = "worker-vol" + empty_dir {} + } + } + } + } +} diff --git a/buildbot/osuosl/master/config/builders.py b/buildbot/osuosl/master/config/builders.py --- a/buildbot/osuosl/master/config/builders.py +++ b/buildbot/osuosl/master/config/builders.py @@ -2446,6 +2446,23 @@ "-DGRPC_INSTALL_PATH=/usr/local/lib/grpc" ])}, + # Build in C++20 configuration. + {'name': "clang-debian-cpp20", + 'tags': ["clang", "c++20"], + 'workernames': ["clang-debian-cpp20"], + 'builddir': "clang-debian-cpp20", + 'factory': UnifiedTreeBuilder.getCmakeWithNinjaBuildFactory( + clean=True, + depends_on_projects=["llvm", "clang", "clang-tools-extra"], + checks=["check-clang", "check-clang-tools"], + targets=["clang", "clangd"], + extra_configure_args=[ + "-DCMAKE_CXX_STANDARD=20" + "-DLLVM_CCACHE_BUILD=ON", + "-DCMAKE_BUILD_TYPE=Release", + "-DLLVM_ENABLE_ASSERTIONS=ON", + ])}, + # Target ARC from Synopsys {'name': "arc-builder", 'tags': ["clang", "lld"], diff --git a/buildbot/osuosl/master/config/workers.py b/buildbot/osuosl/master/config/workers.py --- a/buildbot/osuosl/master/config/workers.py +++ b/buildbot/osuosl/master/config/workers.py @@ -238,6 +238,10 @@ # buildbot/google/terraform/main.tf create_worker("clangd-ubuntu-clang", max_builds=1), + # Debian bullseye, build in C++20 configuration, for worker + # configuration check buildbot/google/terraform/main.tf + create_worker("clang-debian-cpp20", max_builds=1), + # Ubuntu 18.04.LTS x86_64, GCE instance create_worker("polly-x86_64-gce1", properties={'jobs': 2}, max_builds=1), create_worker("polly-x86_64-gce2", properties={'jobs': 2}, max_builds=1),