Index: utils/docker/build_docker_image.sh =================================================================== --- utils/docker/build_docker_image.sh +++ utils/docker/build_docker_image.sh @@ -13,6 +13,8 @@ DOCKER_REPOSITORY="" DOCKER_TAG="" BUILDSCRIPT_ARGS="" +CHECKOUT_ARGS="" +CMAKE_ENABLED_PROJECTS="" function show_usage() { cat << EOF @@ -25,7 +27,7 @@ -s|--source image source dir (i.e. debian8, nvidia-cuda, etc) -d|--docker-repository docker repository for the image -t|--docker-tag docker tag for the image - LLVM-specific: + Checkout arguments: -b|--branch svn branch to checkout, i.e. 'trunk', 'branches/release_40' (default: 'trunk') @@ -40,11 +42,12 @@ Project 'llvm' is always included and ignored, if specified. Can be specified multiple times. - -i|--install-target name of a cmake install target to build and include in - the resulting archive. Can be specified multiple times. -c|--checksums name of a file, containing checksums of llvm checkout. Script will fail if checksums of the checkout do not match. + Build-specific: + -i|--install-target name of a cmake install target to build and include in + the resulting archive. Can be specified multiple times. Required options: --source and --docker-repository, at least one --install-target. @@ -75,6 +78,7 @@ CHECKSUMS_FILE="" SEEN_INSTALL_TARGET=0 +SEEN_CMAKE_ARGS=0 while [[ $# -gt 0 ]]; do case "$1" in -h|--help) @@ -96,11 +100,26 @@ DOCKER_TAG="$1" shift ;; - -i|--install-target|-r|--revision|-c|-cherrypick|-b|--branch|-p|--llvm-project) - if [ "$1" == "-i" ] || [ "$1" == "--install-target" ]; then - SEEN_INSTALL_TARGET=1 - fi + -r|--revision|-c|-cherrypick|-b|--branch) + CHECKOUT_ARGS="$CHECKOUT_ARGS $1 $2" + shift 2 + ;; + -i|--install-target) + SEEN_INSTALL_TARGET=1 BUILDSCRIPT_ARGS="$BUILDSCRIPT_ARGS $1 $2" + shift 2 + ;; + -p|--llvm-project) + PROJ="$2" + if [ "$PROJ" == "cfe" ]; then + PROJ="clang" + fi + + CHECKOUT_ARGS="$CHECKOUT_ARGS $1 $PROJ" + if [ "$PROJ" != "clang-tools-extra" ]; then + CMAKE_ENABLED_PROJECTS="$CMAKE_ENABLED_PROJECTS:$PROJ" + fi + shift 2 ;; -c|--checksums) @@ -111,6 +130,7 @@ --) shift BUILDSCRIPT_ARGS="$BUILDSCRIPT_ARGS -- $*" + SEEN_CMAKE_ARGS=1 shift $# ;; *) @@ -120,6 +140,17 @@ esac done + +if [ "$CMAKE_ENABLED_PROJECTS" != "" ]; then + # Remove the leading ':' character. + CMAKE_ENABLED_PROJECTS="${CMAKE_ENABLED_PROJECTS:1}" + + if [[ $SEEN_CMAKE_ARGS -eq 0 ]]; then + BUILDSCRIPT_ARGS="$BUILDSCRIPT_ARGS --" + fi + BUILDSCRIPT_ARGS="$BUILDSCRIPT_ARGS -DLLVM_ENABLE_PROJECTS=$CMAKE_ENABLED_PROJECTS" +fi + command -v docker >/dev/null || { echo "Docker binary cannot be found. Please install Docker to use this script." @@ -165,6 +196,7 @@ echo "Building ${DOCKER_REPOSITORY}${DOCKER_TAG} from $IMAGE_SOURCE" docker build -t "${DOCKER_REPOSITORY}${DOCKER_TAG}" \ + --build-arg "checkout_args=$CHECKOUT_ARGS" \ --build-arg "buildscript_args=$BUILDSCRIPT_ARGS" \ -f "$BUILD_DIR/$IMAGE_SOURCE/Dockerfile" \ "$BUILD_DIR" Index: utils/docker/debian8/Dockerfile =================================================================== --- utils/docker/debian8/Dockerfile +++ utils/docker/debian8/Dockerfile @@ -41,10 +41,13 @@ ADD checksums /tmp/checksums ADD scripts /tmp/scripts -# Arguments passed to build_install_clang.sh. -ARG buildscript_args + +# Checkout the source code. +ARG checkout_args +RUN /tmp/scripts/checkout.sh ${checkout_args} # Run the build. Results of the build will be available at /tmp/clang-install/. -RUN /tmp/scripts/build_install_llvm.sh ${buildscript_args} +ARG buildscript_args +RUN /tmp/scripts/build_install_llvm.sh --to /tmp/clang-install ${buildscript_args} # Stage 2. Produce a minimal release image with build results. Index: utils/docker/example/Dockerfile =================================================================== --- utils/docker/example/Dockerfile +++ utils/docker/example/Dockerfile @@ -19,10 +19,13 @@ ADD checksums /tmp/checksums ADD scripts /tmp/scripts -# Arguments passed to build_install_clang.sh. + +# Checkout the source code. +ARG checkout_args +RUN /tmp/scripts/checkout.sh ${checkout_args} +# Run the build. Results of the build will be available at /tmp/clang-install/. ARG buildscript_args -# Run the build. Results of the build will be available as /tmp/clang-install. -RUN /tmp/scripts/build_install_llvm.sh ${buildscript_args} +RUN /tmp/scripts/build_install_llvm.sh --to /tmp/clang-install ${buildscript_args} # Stage 2. Produce a minimal release image with build results. Index: utils/docker/nvidia-cuda/Dockerfile =================================================================== --- utils/docker/nvidia-cuda/Dockerfile +++ utils/docker/nvidia-cuda/Dockerfile @@ -17,10 +17,13 @@ ADD checksums /tmp/checksums ADD scripts /tmp/scripts -# Arguments passed to build_install_clang.sh. -ARG buildscript_args + +# Checkout the source code. +ARG checkout_args +RUN /tmp/scripts/checkout.sh ${checkout_args} # Run the build. Results of the build will be available at /tmp/clang-install/. -RUN /tmp/scripts/build_install_llvm.sh ${buildscript_args} +ARG buildscript_args +RUN /tmp/scripts/build_install_llvm.sh --to /tmp/clang-install ${buildscript_args} # Stage 2. Produce a minimal release image with build results. Index: utils/docker/scripts/build_install_llvm.sh =================================================================== --- utils/docker/scripts/build_install_llvm.sh +++ utils/docker/scripts/build_install_llvm.sh @@ -14,112 +14,35 @@ cat << EOF Usage: build_install_llvm.sh [options] -- [cmake-args] -Checkout svn sources and run cmake with the specified arguments. Used -inside docker container. +Run cmake with the specified arguments. Used inside docker container. Passes additional -DCMAKE_INSTALL_PREFIX and puts the build results into -/tmp/clang-install/ directory. +the directory specified by --to option. Available options: -h|--help show this help message - -b|--branch svn branch to checkout, i.e. 'trunk', - 'branches/release_40' - (default: 'trunk') - -r|--revision svn revision to checkout - -c|--cherrypick revision to cherry-pick. Can be specified multiple times. - Cherry-picks are performed in the sorted order using the - following command: - 'svn patch <(svn diff -c \$rev)'. - -p|--llvm-project name of an svn project to checkout. Will also add the - project to a list LLVM_ENABLE_PROJECTS, passed to CMake. - For clang, please use 'clang', not 'cfe'. - Project 'llvm' is always included and ignored, if - specified. - Can be specified multiple times. -i|--install-target name of a cmake install target to build and include in the resulting archive. Can be specified multiple times. -Required options: At least one --install-target. + --to destination directory where to install the targets. +Required options: --to, at least one --install-target. All options after '--' are passed to CMake invocation. EOF } -LLVM_SVN_REV="" -CHERRYPICKS="" -LLVM_BRANCH="" CMAKE_ARGS="" CMAKE_INSTALL_TARGETS="" -# We always checkout llvm -LLVM_PROJECTS="llvm" -CMAKE_LLVM_ENABLE_PROJECTS="" -CLANG_TOOLS_EXTRA_ENABLED=0 - -function contains_project() { - local TARGET_PROJ="$1" - local PROJ - for PROJ in $LLVM_PROJECTS; do - if [ "$PROJ" == "$TARGET_PROJ" ]; then - return 0 - fi - done - return 1 -} - -function append_project() { - local PROJ="$1" - - LLVM_PROJECTS="$LLVM_PROJECTS $PROJ" - if [ "$CMAKE_LLVM_ENABLE_PROJECTS" != "" ]; then - CMAKE_LLVM_ENABLE_PROJECTS="$CMAKE_LLVM_ENABLE_PROJECTS;$PROJ" - else - CMAKE_LLVM_ENABLE_PROJECTS="$PROJ" - fi -} +CLANG_INSTALL_DIR="" while [[ $# -gt 0 ]]; do case "$1" in - -r|--revision) - shift - LLVM_SVN_REV="$1" - shift - ;; - -c|--cherrypick) - shift - CHERRYPICKS="$CHERRYPICKS $1" - shift - ;; - -b|--branch) + -i|--install-target) shift - LLVM_BRANCH="$1" + CMAKE_INSTALL_TARGETS="$CMAKE_INSTALL_TARGETS $1" shift ;; - -p|--llvm-project) + --to) shift - PROJ="$1" - shift - - if [ "$PROJ" == "cfe" ]; then - PROJ="clang" - fi - - if [ "$PROJ" == "clang-tools-extra" ]; then - if [ $CLANG_TOOLS_EXTRA_ENABLED -ne 0 ]; then - echo "Project 'clang-tools-extra' is already enabled, ignoring extra occurrences." - else - CLANG_TOOLS_EXTRA_ENABLED=1 - fi - - continue - fi - - if ! contains_project "$PROJ" ; then - append_project "$PROJ" - else - echo "Project '$PROJ' is already enabled, ignoring extra occurrences." - fi - ;; - -i|--install-target) - shift - CMAKE_INSTALL_TARGETS="$CMAKE_INSTALL_TARGETS $1" + CLANG_INSTALL_DIR="$1" shift ;; --) @@ -142,102 +65,22 @@ exit 1 fi -if [ $CLANG_TOOLS_EXTRA_ENABLED -ne 0 ]; then - if ! contains_project "clang"; then - echo "Project 'clang-tools-extra' was enabled without 'clang'." - echo "Adding 'clang' to a list of projects." - - append_project "clang" - fi -fi - -if [ "$LLVM_BRANCH" == "" ]; then - LLVM_BRANCH="trunk" -fi - -if [ "$LLVM_SVN_REV" != "" ]; then - SVN_REV_ARG="-r$LLVM_SVN_REV" - echo "Checking out svn revision r$LLVM_SVN_REV." -else - SVN_REV_ARG="" - echo "Checking out latest svn revision." +if [ "$CLANG_INSTALL_DIR" == "" ]; then + echo "No install directory. Please specify the --to argument." + exit 1 fi -# Sort cherrypicks and remove duplicates. -CHERRYPICKS="$(echo "$CHERRYPICKS" | xargs -n1 | sort | uniq | xargs)" - -function apply_cherrypicks() { - local CHECKOUT_DIR="$1" - - [ "$CHERRYPICKS" == "" ] || echo "Applying cherrypicks" - pushd "$CHECKOUT_DIR" - - # This function is always called on a sorted list of cherrypicks. - for CHERRY_REV in $CHERRYPICKS; do - echo "Cherry-picking r$CHERRY_REV into $CHECKOUT_DIR" - - local PATCH_FILE="$(mktemp)" - svn diff -c $CHERRY_REV > "$PATCH_FILE" - svn patch "$PATCH_FILE" - rm "$PATCH_FILE" - done - - popd -} - CLANG_BUILD_DIR=/tmp/clang-build -CLANG_INSTALL_DIR=/tmp/clang-install - -mkdir "$CLANG_BUILD_DIR" - -# Get the sources from svn. -echo "Checking out sources from svn" -mkdir "$CLANG_BUILD_DIR/src" -for LLVM_PROJECT in $LLVM_PROJECTS; do - if [ "$LLVM_PROJECT" == "clang" ]; then - SVN_PROJECT="cfe" - else - SVN_PROJECT="$LLVM_PROJECT" - fi - - echo "Checking out https://llvm.org/svn/llvm-project/$SVN_PROJECT to $CLANG_BUILD_DIR/src/$LLVM_PROJECT" - svn co -q $SVN_REV_ARG \ - "https://llvm.org/svn/llvm-project/$SVN_PROJECT/$LLVM_BRANCH" \ - "$CLANG_BUILD_DIR/src/$LLVM_PROJECT" - - # We apply cherrypicks to all repositories regardless of whether the revision - # changes this repository or not. For repositories not affected by the - # cherrypick, applying the cherrypick is a no-op. - apply_cherrypicks "$CLANG_BUILD_DIR/src/$LLVM_PROJECT" -done - -if [ $CLANG_TOOLS_EXTRA_ENABLED -ne 0 ]; then - echo "Checking out https://llvm.org/svn/llvm-project/clang-tools-extra to $CLANG_BUILD_DIR/src/clang/tools/extra" - svn co -q $SVN_REV_ARG \ - "https://llvm.org/svn/llvm-project/clang-tools-extra/$LLVM_BRANCH" \ - "$CLANG_BUILD_DIR/src/clang/tools/extra" - apply_cherrypicks "$CLANG_BUILD_DIR/src/clang/tools/extra" -fi - -CHECKSUMS_FILE="/tmp/checksums/checksums.txt" - -if [ -f "$CHECKSUMS_FILE" ]; then - echo "Validating checksums for LLVM checkout..." - python "$(dirname $0)/llvm_checksum/llvm_checksum.py" -c "$CHECKSUMS_FILE" \ - --partial --multi_dir "$CLANG_BUILD_DIR/src" -else - echo "Skipping checksumming checks..." -fi +mkdir -p "$CLANG_INSTALL_DIR" -mkdir "$CLANG_BUILD_DIR/build" +mkdir -p "$CLANG_BUILD_DIR/build" pushd "$CLANG_BUILD_DIR/build" # Run the build as specified in the build arguments. echo "Running build" cmake -GNinja \ -DCMAKE_INSTALL_PREFIX="$CLANG_INSTALL_DIR" \ - -DLLVM_ENABLE_PROJECTS="$CMAKE_LLVM_ENABLE_PROJECTS" \ $CMAKE_ARGS \ "$CLANG_BUILD_DIR/src/llvm" ninja $CMAKE_INSTALL_TARGETS @@ -245,6 +88,6 @@ popd # Cleanup. -rm -rf "$CLANG_BUILD_DIR" +rm -rf "$CLANG_BUILD_DIR/build" echo "Done" Index: utils/docker/scripts/checkout.sh =================================================================== --- utils/docker/scripts/checkout.sh +++ utils/docker/scripts/checkout.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -#===- llvm/utils/docker/scripts/build_install_llvm.sh ---------------------===// +#===- llvm/utils/docker/scripts/checkout.sh ---------------------===// # # The LLVM Compiler Infrastructure # @@ -12,12 +12,9 @@ function show_usage() { cat << EOF -Usage: build_install_llvm.sh [options] -- [cmake-args] +Usage: checkout.sh [options] -Checkout svn sources and run cmake with the specified arguments. Used -inside docker container. -Passes additional -DCMAKE_INSTALL_PREFIX and puts the build results into -/tmp/clang-install/ directory. +Checkout svn sources into /tmp/clang-build/src. Used inside a docker container. Available options: -h|--help show this help message @@ -29,29 +26,19 @@ Cherry-picks are performed in the sorted order using the following command: 'svn patch <(svn diff -c \$rev)'. - -p|--llvm-project name of an svn project to checkout. Will also add the - project to a list LLVM_ENABLE_PROJECTS, passed to CMake. + -p|--llvm-project name of an svn project to checkout. For clang, please use 'clang', not 'cfe'. Project 'llvm' is always included and ignored, if specified. Can be specified multiple times. - -i|--install-target name of a cmake install target to build and include in - the resulting archive. Can be specified multiple times. -Required options: At least one --install-target. - -All options after '--' are passed to CMake invocation. EOF } LLVM_SVN_REV="" CHERRYPICKS="" LLVM_BRANCH="" -CMAKE_ARGS="" -CMAKE_INSTALL_TARGETS="" # We always checkout llvm LLVM_PROJECTS="llvm" -CMAKE_LLVM_ENABLE_PROJECTS="" -CLANG_TOOLS_EXTRA_ENABLED=0 function contains_project() { local TARGET_PROJ="$1" @@ -64,17 +51,6 @@ return 1 } -function append_project() { - local PROJ="$1" - - LLVM_PROJECTS="$LLVM_PROJECTS $PROJ" - if [ "$CMAKE_LLVM_ENABLE_PROJECTS" != "" ]; then - CMAKE_LLVM_ENABLE_PROJECTS="$CMAKE_LLVM_ENABLE_PROJECTS;$PROJ" - else - CMAKE_LLVM_ENABLE_PROJECTS="$PROJ" - fi -} - while [[ $# -gt 0 ]]; do case "$1" in -r|--revision) @@ -101,32 +77,16 @@ PROJ="clang" fi - if [ "$PROJ" == "clang-tools-extra" ]; then - if [ $CLANG_TOOLS_EXTRA_ENABLED -ne 0 ]; then - echo "Project 'clang-tools-extra' is already enabled, ignoring extra occurrences." - else - CLANG_TOOLS_EXTRA_ENABLED=1 - fi - - continue - fi - if ! contains_project "$PROJ" ; then - append_project "$PROJ" + if [ "$PROJ" == "clang-tools-extra" ] && [ ! contains_project "clang" ]; then + echo "Project 'clang-tools-extra' specified before 'clang'. Adding 'clang' to a list of projects first." + LLVM_PROJECTS="$LLVM_PROJECTS clang" + fi + LLVM_PROJECTS="$LLVM_PROJECTS $PROJ" else echo "Project '$PROJ' is already enabled, ignoring extra occurrences." fi ;; - -i|--install-target) - shift - CMAKE_INSTALL_TARGETS="$CMAKE_INSTALL_TARGETS $1" - shift - ;; - --) - shift - CMAKE_ARGS="$*" - shift $# - ;; -h|--help) show_usage exit 0 @@ -137,20 +97,6 @@ esac done -if [ "$CMAKE_INSTALL_TARGETS" == "" ]; then - echo "No install targets. Please pass one or more --install-target." - exit 1 -fi - -if [ $CLANG_TOOLS_EXTRA_ENABLED -ne 0 ]; then - if ! contains_project "clang"; then - echo "Project 'clang-tools-extra' was enabled without 'clang'." - echo "Adding 'clang' to a list of projects." - - append_project "clang" - fi -fi - if [ "$LLVM_BRANCH" == "" ]; then LLVM_BRANCH="trunk" fi @@ -186,13 +132,10 @@ } CLANG_BUILD_DIR=/tmp/clang-build -CLANG_INSTALL_DIR=/tmp/clang-install - -mkdir "$CLANG_BUILD_DIR" # Get the sources from svn. echo "Checking out sources from svn" -mkdir "$CLANG_BUILD_DIR/src" +mkdir -p "$CLANG_BUILD_DIR/src" for LLVM_PROJECT in $LLVM_PROJECTS; do if [ "$LLVM_PROJECT" == "clang" ]; then SVN_PROJECT="cfe" @@ -200,26 +143,23 @@ SVN_PROJECT="$LLVM_PROJECT" fi - echo "Checking out https://llvm.org/svn/llvm-project/$SVN_PROJECT to $CLANG_BUILD_DIR/src/$LLVM_PROJECT" + if [ "$SVN_PROJECT" != "clang-tools-extra" ]; then + CHECKOUT_DIR="$CLANG_BUILD_DIR/src/$LLVM_PROJECT" + else + CHECKOUT_DIR="$CLANG_BUILD_DIR/src/clang/tools/extra" + fi + + echo "Checking out https://llvm.org/svn/llvm-project/$SVN_PROJECT to $CHECKOUT_DIR" svn co -q $SVN_REV_ARG \ "https://llvm.org/svn/llvm-project/$SVN_PROJECT/$LLVM_BRANCH" \ - "$CLANG_BUILD_DIR/src/$LLVM_PROJECT" + "$CHECKOUT_DIR" # We apply cherrypicks to all repositories regardless of whether the revision # changes this repository or not. For repositories not affected by the # cherrypick, applying the cherrypick is a no-op. - apply_cherrypicks "$CLANG_BUILD_DIR/src/$LLVM_PROJECT" + apply_cherrypicks "$CHECKOUT_DIR" done -if [ $CLANG_TOOLS_EXTRA_ENABLED -ne 0 ]; then - echo "Checking out https://llvm.org/svn/llvm-project/clang-tools-extra to $CLANG_BUILD_DIR/src/clang/tools/extra" - svn co -q $SVN_REV_ARG \ - "https://llvm.org/svn/llvm-project/clang-tools-extra/$LLVM_BRANCH" \ - "$CLANG_BUILD_DIR/src/clang/tools/extra" - - apply_cherrypicks "$CLANG_BUILD_DIR/src/clang/tools/extra" -fi - CHECKSUMS_FILE="/tmp/checksums/checksums.txt" if [ -f "$CHECKSUMS_FILE" ]; then @@ -230,21 +170,4 @@ echo "Skipping checksumming checks..." fi -mkdir "$CLANG_BUILD_DIR/build" -pushd "$CLANG_BUILD_DIR/build" - -# Run the build as specified in the build arguments. -echo "Running build" -cmake -GNinja \ - -DCMAKE_INSTALL_PREFIX="$CLANG_INSTALL_DIR" \ - -DLLVM_ENABLE_PROJECTS="$CMAKE_LLVM_ENABLE_PROJECTS" \ - $CMAKE_ARGS \ - "$CLANG_BUILD_DIR/src/llvm" -ninja $CMAKE_INSTALL_TARGETS - -popd - -# Cleanup. -rm -rf "$CLANG_BUILD_DIR" - echo "Done"