Index: utils/docker/build_docker_image.sh =================================================================== --- utils/docker/build_docker_image.sh +++ utils/docker/build_docker_image.sh @@ -38,6 +38,9 @@ Can be specified multiple times. -i|--install-target name of a cmake install target to build and include in the resulting archive. Can be specified multiple times. + -c|--checksums name of a file, containing checksums of llvm checkout. + Script will fail if checksums of the checkout do not + match. Required options: --source and --docker-repository, at least one --install-target. @@ -66,6 +69,7 @@ EOF } +CHECKSUMS_FILE="" SEEN_INSTALL_TARGET=0 while [[ $# -gt 0 ]]; do case "$1" in @@ -95,6 +99,11 @@ BUILDSCRIPT_ARGS="$BUILDSCRIPT_ARGS $1 $2" shift 2 ;; + -c|--checksums) + shift + CHECKSUMS_FILE="$1" + shift + ;; --) shift BUILDSCRIPT_ARGS="$BUILDSCRIPT_ARGS -- $*" @@ -141,6 +150,11 @@ cp -r "$SOURCE_DIR/$IMAGE_SOURCE" "$BUILD_DIR/$IMAGE_SOURCE" cp -r "$SOURCE_DIR/scripts" "$BUILD_DIR/scripts" +mkdir "$BUILD_DIR/checksums" +if [ "$CHECKSUMS_FILE" != "" ]; then + cp "$CHECKSUMS_FILE" "$BUILD_DIR/checksums/checksums.txt" +fi + if [ "$DOCKER_TAG" != "" ]; then DOCKER_TAG=":$DOCKER_TAG" fi Index: utils/docker/debian8/build/Dockerfile =================================================================== --- utils/docker/debian8/build/Dockerfile +++ utils/docker/debian8/build/Dockerfile @@ -19,7 +19,7 @@ # Install compiler, python and subversion. RUN apt-get update && \ apt-get install -y --no-install-recommends ca-certificates gnupg \ - build-essential python2.7 wget subversion ninja-build && \ + build-essential python wget subversion ninja-build && \ rm -rf /var/lib/apt/lists/* # Import public key required for verifying signature of cmake download. @@ -37,9 +37,11 @@ tar xzf cmake-3.7.2-Linux-x86_64.tar.gz -C /usr/local --strip-components=1 && \ cd / && rm -rf /tmp/cmake-install +ADD checksums /tmp/checksums +ADD scripts /tmp/scripts + # Arguments passed to build_install_clang.sh. ARG buildscript_args # Run the build. Results of the build will be available as /tmp/clang.tar.gz. -ADD scripts/build_install_llvm.sh /tmp -RUN /tmp/build_install_llvm.sh ${buildscript_args} +RUN /tmp/scripts/build_install_llvm.sh ${buildscript_args} Index: utils/docker/example/build/Dockerfile =================================================================== --- utils/docker/example/build/Dockerfile +++ utils/docker/example/build/Dockerfile @@ -18,9 +18,11 @@ # FIXME: Install llvm/clang build dependencies. Including compiler to # build stage1, cmake, subversion, ninja, etc. -# Arguments to pass to build_install_clang.sh. +ADD checksums /tmp/checksums +ADD scripts /tmp/scripts + +# Arguments passed to build_install_clang.sh. ARG buildscript_args # Run the build. Results of the build will be available as /tmp/clang.tar.gz. -ADD scripts/build_install_llvm.sh /tmp -RUN /tmp/build_install_llvm.sh ${buildscript_args} +RUN /tmp/scripts/build_install_llvm.sh ${buildscript_args} Index: utils/docker/nvidia-cuda/build/Dockerfile =================================================================== --- utils/docker/nvidia-cuda/build/Dockerfile +++ utils/docker/nvidia-cuda/build/Dockerfile @@ -17,10 +17,15 @@ # Install llvm build dependencies. RUN apt-get update && \ - apt-get install -y --no-install-recommends ca-certificates cmake python2.7 \ + apt-get install -y --no-install-recommends ca-certificates cmake python \ subversion ninja-build && \ rm -rf /var/lib/apt/lists/* +ADD checksums /tmp/checksums +ADD scripts /tmp/scripts + +# Arguments passed to build_install_clang.sh. +ARG buildscript_args + # Run the build. Results of the build will be available as /tmp/clang.tar.gz. -ADD scripts/build_install_llvm.sh /tmp -RUN /tmp/build_install_llvm.sh ${buildscript_args} +RUN /tmp/scripts/build_install_llvm.sh ${buildscript_args} Index: utils/docker/scripts/build_install_llvm.sh =================================================================== --- utils/docker/scripts/build_install_llvm.sh +++ utils/docker/scripts/build_install_llvm.sh @@ -181,6 +181,16 @@ "$CLANG_BUILD_DIR/src/clang/tools/extra" fi +CHECKSUMS_FILE="/tmp/checksums/checksums.txt" + +if [ -f "$CHECKSUMS_FILE" ]; then + echo "Validating checksums for LLVM checkout..." + python "$(dirname $0)/llvm_checksum/llvm_checksum.py" -c "$CHECKSUMS_FILE" \ + --partial --multi_dir "$CLANG_BUILD_DIR/src" +else + echo "Skipping checksumming checks..." +fi + mkdir "$CLANG_BUILD_DIR/build" pushd "$CLANG_BUILD_DIR/build" Index: utils/docker/scripts/llvm_checksum/llvm_checksum.py =================================================================== --- /dev/null +++ utils/docker/scripts/llvm_checksum/llvm_checksum.py @@ -0,0 +1,198 @@ +#!/usr/bin/python +""" A small program to compute checksums of LLVM checkout. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import hashlib +import logging +import re +import sys +from argparse import ArgumentParser +from project_tree import * + +SVN_DATES_REGEX = re.compile(r"\$(Date|LastChangedDate)[^\$]+\$") + + +def main(): + parser = ArgumentParser() + parser.add_argument( + "-v", "--verbose", action="store_true", help="enable debug logging") + parser.add_argument( + "-c", + "--check", + metavar="reference_file", + help="read checksums from reference_file and " + + "check they match checksums of llvm_path.") + parser.add_argument( + "--partial", + action="store_true", + help="ignore projects from reference_file " + + "that are not checked out in llvm_path.") + parser.add_argument( + "--multi_dir", + action="store_true", + help="indicates llvm_path contains llvm, checked out " + + "into multiple directories, as opposed to a " + + "typical single source tree checkout.") + parser.add_argument("llvm_path") + + args = parser.parse_args() + if args.check is not None: + with open(args.check, "r") as f: + reference_checksums = ReadLLVMChecksums(f) + else: + reference_checksums = None + + if args.verbose: + logging.basicConfig(level=logging.DEBUG) + + llvm_projects = CreateLLVMProjects(not args.multi_dir) + checksums = ComputeLLVMChecksums(args.llvm_path, llvm_projects) + + if reference_checksums is None: + WriteLLVMChecksums(checksums, sys.stdout) + sys.exit(0) + + if not ValidateChecksums(reference_checksums, checksums, args.partial): + sys.stdout.write("Checksums differ.\nNew checksums:\n") + WriteLLVMChecksums(checksums, sys.stdout) + sys.stdout.write("Reference checksums:\n") + WriteLLVMChecksums(reference_checksums, sys.stdout) + sys.exit(1) + else: + sys.stdout.write("Checksums match.") + + +def ComputeLLVMChecksums(root_path, projects): + """Compute checksums for LLVM sources checked out using svn. + + Args: + root_path: a directory of llvm checkout. + projects: a list of LLVMProject instances, which describe checkout paths, + relative to root_path. + + Returns: + A dict mapping from project name to project checksum. + """ + hash_algo = hashlib.sha256 + + def collapse_svn_substitutions(contents): + # Replace svn substitutions for $Date$ and $LastChangedDate$. + # Unfortunately, these are locale-specific. + return SVN_DATES_REGEX.sub("$\1$", contents) + + def read_and_collapse_svn_subsitutions(file_path): + with open(file_path, "rb") as f: + contents = f.read() + new_contents = collapse_svn_substitutions(contents) + if contents != new_contents: + logging.debug("Replaced svn keyword substitutions in %s", file_path) + logging.debug("\n\tBefore\n%s\n\tAfter\n%s", contents, new_contents) + return new_contents + + project_checksums = dict() + # Hash each project using dir_checksum. + for proj in projects: + project_root = os.path.join(root_path, proj.relpath) + if not os.path.exists(project_root): + logging.info("Folder %s doesn't exist, skipping project %s", proj.relpath, + proj.name) + continue + + files = list() + + def add_file_hash(file_path): + if os.path.islink(file_path) and not os.path.exists(file_path): + content = os.readlink(file_path) + else: + content = read_and_collapse_svn_subsitutions(file_path) + hasher = hash_algo() + hasher.update(content) + file_digest = hasher.hexdigest() + logging.debug("Checksum %s for file %s", file_digest, file_path) + files.append((file_path, file_digest)) + + logging.info("Computing checksum for %s", proj.name) + WalkProjectFiles(root_path, projects, proj, add_file_hash) + + # Compute final checksum. + files.sort(key=lambda x: x[0]) + hasher = hash_algo() + for file_path, file_digest in files: + file_path = os.path.relpath(file_path, project_root) + hasher.update(file_path) + hasher.update(file_digest) + project_checksums[proj.name] = hasher.hexdigest() + return project_checksums + + +def WriteLLVMChecksums(checksums, f): + """Writes checksums to a text file. + + Args: + checksums: a dict mapping from project name to project checksum (result of + ComputeLLVMChecksums). + f: a file object to write into. + """ + + for proj in sorted(checksums.keys()): + f.write("{} {}\n".format(checksums[proj], proj)) + + +def ReadLLVMChecksums(f): + """Reads checksums from a text file, produced by WriteLLVMChecksums. + + Returns: + A dict, mapping from project name to project checksum. + """ + checksums = {} + while True: + line = f.readline() + if line == "": + break + checksum, proj = line.split() + checksums[proj] = checksum + return checksums + + +def ValidateChecksums(reference_checksums, + new_checksums, + allow_missing_projects=False): + """Validates that reference_checksums and new_checksums match. + + Args: + reference_checksums: a dict of reference checksums, mapping from a project + name to a project checksum. + new_checksums: a dict of checksums to be checked, mapping from a project + name to a project checksum. + allow_missing_projects: + When True, reference_checksums may contain more projects than + new_checksums. Projects missing from new_checksums are ignored. + When False, new_checksums and reference_checksums must contain checksums + for the same set of projects. If there is a project in + reference_checksums, missing from new_checksums, ValidateChecksums + will return False. + + Returns: + True, if checksums match with regards to allow_missing_projects flag value. + False, otherwise. + """ + if not allow_missing_projects: + if len(new_checksums) != len(reference_checksums): + return False + + for proj, checksum in new_checksums.iteritems(): + # We never computed a checksum for this project. + if proj not in reference_checksums: + return False + # Checksum did not match. + if reference_checksums[proj] != checksum: + return False + + return True + + +if __name__ == "__main__": + main() Index: utils/docker/scripts/llvm_checksum/project_tree.py =================================================================== --- /dev/null +++ utils/docker/scripts/llvm_checksum/project_tree.py @@ -0,0 +1,95 @@ +"""Contains helper functions to compute checksums for LLVM checkouts. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import logging +import os +import os.path +import sys + + +class LLVMProject(object): + """An LLVM project with a descriptive name and a relative checkout path. + """ + + def __init__(self, name, relpath): + self.name = name + self.relpath = relpath + + def is_subproject(self, other_project): + """ Check if self is checked out as a subdirectory of other_project. + """ + return self.relpath.startswith(other_project.relpath) + + +def WalkProjectFiles(checkout_root, all_projects, project, visitor): + """ Walk over all files inside a project without recursing into subprojects, '.git' and '.svn' subfolders. + + checkout_root: root of the LLVM checkout. + all_projects: projects in the LLVM checkout. + project: a project to walk the files of. Must be inside all_projects. + visitor: a function called on each visited file. + """ + assert project in all_projects + + ignored_paths = set() + for other_project in all_projects: + if other_project != project and other_project.is_subproject(project): + ignored_paths.add(os.path.join(checkout_root, other_project.relpath)) + + def raise_error(err): + raise err + + project_root = os.path.join(checkout_root, project.relpath) + for root, dirs, files in os.walk(project_root, onerror=raise_error): + dirs[:] = [ + d for d in dirs + if d != ".svn" and d != ".git" and + os.path.join(root, d) not in ignored_paths + ] + for f in files: + visitor(os.path.join(root, f)) + + +def CreateLLVMProjects(single_tree_checkout): + """Returns a list of LLVMProject instances, describing relative paths of a typical LLVM checkout. + + Args: + single_tree_checkout: + When True, relative paths for each project points to a typical single + source tree checkout. + When False, relative paths for each projects points to a separate + directory. However, clang-tools-extra is an exception, its relative path + will always be 'clang/tools/extra'. + """ + # FIXME: cover all of llvm projects. + + # Projects that reside inside 'projects/' in a single source tree checkout. + ORDINARY_PROJECTS = [ + "compiler-rt", "dragonegg", "libcxx", "libcxxabi", "libunwind", + "parallel-libs", "test-suite" + ] + # Projects that reside inside 'tools/' in a single source tree checkout. + TOOLS_PROJECTS = ["clang", "lld", "lldb", "llgo"] + + if single_tree_checkout: + projects = [LLVMProject("llvm", "")] + projects += [ + LLVMProject(p, os.path.join("projects", p)) for p in ORDINARY_PROJECTS + ] + projects += [ + LLVMProject(p, os.path.join("tools", p)) for p in TOOLS_PROJECTS + ] + projects.append( + LLVMProject("clang-tools-extra", + os.path.join("tools", "clang", "tools", "extra"))) + else: + projects = [LLVMProject("llvm", "llvm")] + projects += [LLVMProject(p, p) for p in ORDINARY_PROJECTS] + projects += [LLVMProject(p, p) for p in TOOLS_PROJECTS] + projects.append( + LLVMProject("clang-tools-extra", os.path.join("clang", "tools", + "extra"))) + return projects