Index: test/jenkins/test_monorepo_build.py =================================================================== --- /dev/null +++ test/jenkins/test_monorepo_build.py @@ -0,0 +1,132 @@ +# Testing for the Jenkins monorepo_build.py script +# +# RUN: export TESTING=1 +# RUN: export JOB_NAME="FOO" +# RUN: export BUILD_NUMBER=321 +# RUN: export BRANCH=master +# Tell monorepo_build.py to just print commands instead of running. +# RUN: mkdir -p %t.SANDBOX/host-compiler/lib %t.SANDBOX/host-compiler/bin %t.SANDBOX/llvm-project/llvm %t.SANDBOX/llvm-project/clang %t.SANDBOX/llvm-project/libcxx %t.SANDBOX/llvm-project/compiler-rt %t.SANDBOX/llvm-project/debuginfo-tests %t.SANDBOX/llvm-project/clang-tools-extra %t.SANDBOX/llvm-project/lldb +# RUN: touch %t.SANDBOX/host-compiler/bin/clang +# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py clang all > %t.log +# RUN: FileCheck --check-prefix CHECK-SIMPLE < %t.log %s +# CHECK-SIMPLE: @@@ Build Directory @@@ +# CHECK-SIMPLE: cd +# CHECK-SIMPLE: 'mkdir' '-p' +# CHECK-SIMPLE: @@@@@@ +# CHECK-SIMPLE: @@@ Build Clang @@@ +# CHECK-SIMPLE: cd +# CHECK-SIMPLE: 'mkdir' './Build' './Root' +# CHECK-SIMPLE: cd +# CHECK-SIMPLE: '/usr/local/bin/cmake' '-G' 'Ninja' '-C' +# CHECK-SIMPLE: '-DLLVM_ENABLE_ASSERTIONS:BOOL=FALSE' +# CHECK-SIMPLE: '-DCMAKE_BUILD_TYPE=RelWithDebInfo' +# CHECK-SIMPLE: '-DLLVM_ENABLE_PROJECTS=clang;clang-tools-extra;compiler-rt;libcxx' +# CHECK-SIMPLE: '-DCMAKE_MAKE_PROGRAM=/usr/local/bin/ninja' +# CHECK-SIMPLE: '-DLLVM_VERSION_PATCH=99' +# CHECK-SIMPLE: '-DLLVM_VERSION_SUFFIX=""' +# CHECK-SIMPLE: '-DLLVM_BUILD_EXTERNAL_COMPILER_RT=On' +# CHECK-SIMPLE: '-DCLANG_COMPILER_RT_CMAKE_ARGS +# CHECK-SIMPLE: Apple.cmake' +# CHECK-SIMPLE: '-DCOMPILER_RT_BUILD_SANITIZERS=On' +# CHECK-SIMPLE: '-DCMAKE_INSTALL_PREFIX +# CHECK-SIMPLE: '-DCLANG_APPEND_VC_REV=On' +# CHECK-SIMPLE: '-DLLVM_BUILD_TESTS=On' +# CHECK-SIMPLE: '-DLLVM_INCLUDE_TESTS=On' +# CHECK-SIMPLE: '-DCLANG_INCLUDE_TESTS=On' +# CHECK-SIMPLE: '-DLLVM_INCLUDE_UTILS=On' +# CHECK-SIMPLE: '-DCMAKE_MACOSX_RPATH=On' +# CHECK-SIMPLE: '-DLLVM_ENABLE_LTO=Off +# CHECK-SIMPLE-NOT: -DLLVM_PARALLEL_LINK_JOBS +# CHECK-SIMPLE: @@@@@@ +# CHECK-SIMPLE: @@@ Ninja @@@ +# CHECK-SIMPLE: cd +# CHECK-SIMPLE: '/usr/local/bin/ninja' '-v' 'install' +# CHECK-SIMPLE: @@@@@@ +# CHECK-SIMPLE: @@@ Tests @@@ + +# CHECK-SIMPLE: cd +# CHECK-SIMPLE: 'env' 'MALLOC_LOG_FILE=/dev/null' '/usr/local/bin/ninja' '-v' '-k' '0' 'check-all' + +# Now Check Assertion Builds have --enable assertions + +# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py clang all --assertions > %t-assert.log +# RUN: FileCheck --check-prefix CHECK-ASSERT < %t-assert.log %s +# CHECK-ASSERT: '/usr/local/bin/cmake' '-G' 'Ninja' '-C' +# CHECK-ASSERT: '-DLLVM_ENABLE_ASSERTIONS:BOOL=TRUE' + +# Check LTO + +# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py clang all --lto > %t-lto.log +# RUN: FileCheck --check-prefix CHECK-LTO < %t-lto.log %s +# CHECK-LTO: -DLLVM_PARALLEL_LINK_JOBS +# CHECK-LTO-NOT:: '-DLLVM_ENABLE_LTO=Off + +# Now try just a build +# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py clang build --lto +# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py clang build + +# Just a test +# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py clang test + +# CMake + +# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py cmake all --debug > %t-cmake.log +# RUN: FileCheck --check-prefix CHECK-CMAKE < %t-cmake.log %s +# CHECK-CMAKE: '/usr/local/bin/cmake' '-G' 'Ninja' +# CHECK-CMAKE: -DLLVM_BUILD_EXAMPLES=On +# CHECK-CMAKE: '-DCMAKE_BUILD_TYPE=Debug' +# CHECK-CMAKE: '-DLLVM_ENABLE_ASSERTIONS=Off' +# CHECK-CMAKE: -DLLVM_LIT_ARGS=--xunit-xml-output=testresults.xunit.xml -v --timeout=600 +# CHECK-CMAKE: '/usr/local/bin/ninja' '-v' 'all' +# CHECK-CMAKE: '/usr/local/bin/ninja' '-v' '-k' '0' 'check-all' + + +# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py cmake build +# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py cmake test +# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py cmake testlong + +# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py cmake all --lto | FileCheck --check-prefix CHECK-CMAKELTO %s +# CHECK-CMAKELTO: '/usr/local/bin/cmake' '-G' 'Ninja' +# CHECK-CMAKELTO: '-DLLVM_BUILD_EXAMPLES=Off' +# CHECK-CMAKELTO-NOT:: '-DLLVM_ENABLE_LTO=Off +# CHECK-CMAKELTO: '-DLLVM_PARALLEL_LINK_JOBS=1' +# CHECK-CMAKELTO: '-DCMAKE_BUILD_TYPE=Release' + +# RUN: env MAX_PARALLEL_LINKS=2 python %{src_root}/zorg/jenkins/monorepo_build.py cmake all --lto | FileCheck --check-prefix CHECK-CMAKE-PAR-LTO %s +# CHECK-CMAKE-PAR-LTO: '/usr/local/bin/cmake' '-G' 'Ninja' +# CHECK-CMAKE-PAR-LTO: '-DLLVM_BUILD_EXAMPLES=Off' +# CHECK-CMAKE-PAR-LTO-NOT:: '-DLLVM_ENABLE_LTO=Off +# CHECK-CMAKE-PAR-LTO: '-DLLVM_PARALLEL_LINK_JOBS=2' +# CHECK-CMAKE-PAR-LTO: '-DCMAKE_BUILD_TYPE=Release' + +# RUN: env MAX_PARALLEL_TESTS=2 python %{src_root}/zorg/jenkins/monorepo_build.py cmake all | FileCheck --check-prefix CHECK-CMAKE-2-TESTS %s +# CHECK-CMAKE-2-TESTS: '/usr/local/bin/cmake' '-G' 'Ninja' +# CHECK-CMAKE-2-TESTS: '-DLLVM_LIT_ARGS=--xunit-xml-output=testresults.xunit.xml -v --timeout=600 -j 2' + +# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py cmake all --cmake-type=RelWithDebugInfo | FileCheck --check-prefix CHECK-CMAKE-UPLOADS %s +# CHECK-CMAKE-UPLOADS: @@@ Uploading Artifact @@@ + +# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py static-analyzer-benchmarks | FileCheck --check-prefix CHECK-STATIC-ANALYZER-BENCHMARKS %s +# CHECK-STATIC-ANALYZER-BENCHMARKS: @@@ Static Analyzer Benchmarks @@@ +# CHECK-STATIC-ANALYZER-BENCHMARKS: cd [[WORKSPACE:.*]]/test-suite-ClangAnalyzer/ +# CHECK-STATIC-ANALYZER-BENCHMARKS: '[[WORKSPACE]]/utils-analyzer/SATestBuild.py' '--strictness' '0' +# CHECK-STATIC-ANALYZER-BENCHMARKS: @@@@@@ + +# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py cmake all --globalisel | FileCheck --check-prefix CHECK-GISEL %s +# CHECK-GISEL: '/usr/local/bin/cmake' '-G' 'Ninja' +# CHECK-GISEL: '-DLLVM_BUILD_GLOBAL_ISEL=ON' + +# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py clang all --lto --cmake-flag="-DFOO" | FileCheck --check-prefix CHECK-CMAKEFLAGS %s +# CHECK-CMAKEFLAGS: '-DFOO' + +# Make sure you can pass new build targetss. +# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py cmake build --cmake-build-target foo --cmake-build-target bar | FileCheck --check-prefix CHECK-BTARGETS %s +# CHECK-BTARGETS: '/usr/local/bin/ninja' '-v' 'foo' 'bar' + +# Make sure you can pass new test targets. +# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py cmake test --cmake-test-target foo --cmake-test-target bar | FileCheck --check-prefix CHECK-TTARGETS %s +# CHECK-TTARGETS: '/usr/local/bin/ninja' '-v' '-k' '0' 'foo' 'bar' + +# Test long should always do check-all, since that is what many bots expect. +# RUN: python %{src_root}/zorg/jenkins/monorepo_build.py cmake testlong | FileCheck --check-prefix CHECK-TTARGETS2 %s +# CHECK-TTARGETS2: '/usr/local/bin/ninja' '-v' '-k' '0' 'check-all' Index: zorg/jenkins/monorepo_build.py =================================================================== --- /dev/null +++ zorg/jenkins/monorepo_build.py @@ -0,0 +1,914 @@ +"""Build and test clangs.""" + +import sys +import logging +import os +import subprocess +import datetime +import time +import argparse +import shutil +import math +import re +import xml.etree.ElementTree as ET +from contextlib import contextmanager +from urllib2 import urlopen, URLError, HTTPError + +SERVER = "labmaster2.lab.llvm.org" + +NINJA = "/usr/local/bin/ninja" + +# Add dependency checker to the Python path. +# For relative reference to the dependency file. +here = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.abspath(here + "/../../dep/")) +import dep # noqa + + + +def readme_name(repo): + """Given a repo, return the name of the readme file.""" + if repo == "libcxx": + return "LICENSE.TXT" + return "README.txt" + + +def next_section(name): + """Jenkins is setup to parse @@@ xyz @@@ as a new section of the buildlog + with title xyz. The section ends with @@@@@@ """ + footer() + header(name) + + +def header(name): + print "@@@", name, "@@@" + + +def footer(): + print "Completed at: " + time.strftime("%FT%T") + print "@@@@@@" + + +def quote_sh_string(string): + """Make things that we print shell safe for copy and paste.""" + return "\\'".join("'" + p + "'" for p in string.split("'")) + + +class Configuration(object): + """docstring for Configuration""" + + def __init__(self, args): + super(Configuration, self).__init__() + self._args = args + self.workspace = os.environ.get('WORKSPACE', os.getcwd()) + self._src_dir = os.environ.get('SRC_DIR', 'llvm-project') + self._llvm_src_dir = os.environ.get('LLVM_SRC_DIR', 'llvm') + self._lldb_src_dir = os.environ.get('LLDB_SRC_DIR', 'lldb') + self._build_dir = os.environ.get('BUILD_DIR', 'clang-build') + self._lldb_build_dir = os.environ.get('LLDB_BUILD_DIR', 'lldb-build') + self._install_dir = os.environ.get('INSTALL_DIR', 'clang-install') + self.j_level = os.environ.get('J_LEVEL', None) + self.max_parallel_tests = os.environ.get('MAX_PARALLEL_TESTS', None) + self.max_parallel_links = os.environ.get('MAX_PARALLEL_LINKS', None) + self.host_compiler_url = os.environ.get('HOST_URL', + 'http://labmaster2.local/artifacts/') + self.artifact_url = os.environ.get('ARTIFACT', 'NONE') + self.job_name = os.environ.get('JOB_NAME', 'NONE') + self.build_id = os.environ.get('BUILD_ID', 'NONE') + self.build_number = os.environ.get('BUILD_NUMBER', 'NONE') + self.git_sha = os.environ.get('GIT_SHA', 'NONE') + self.nobootstrap = True + self.device = None + self.node_name = os.environ.get('NODE_NAME', None) + self.lldb_test_archs = os.environ.get('LLDB_TEST_ARCHS', 'x86_64').split() + + # Import all of the command line arguments into the config object + self.__dict__.update(vars(args)) + + def builddir(self): + """The build output directory for this compile.""" + return os.path.join(self.workspace, self._build_dir) + + def srcdir(self): + """The derived source directory for this build.""" + return os.path.join(self.workspace, self._src_dir) + + def llvmsrcdir(self): + """The llvm source directory for this build.""" + return os.path.join(self.workspace, self._src_dir, self._llvm_src_dir) + + def lldbbuilddir(self): + """The derived source directory for this lldb build.""" + return os.path.join(self.workspace, self._lldb_build_dir) + + def lldbsrcdir(self): + """The derived source directory for this lldb build.""" + return os.path.join(self.workspace, self._lldb_src_dir) + + def installdir(self): + """The install directory for the compile.""" + return os.path.join(self.workspace, self._install_dir) + + def CC(self): + """Location of the host compiler, if one is present in this build.""" + cc_basedir = os.path.join(self.workspace, 'host-compiler/') + if os.path.exists(cc_basedir): + clang_exec_path = os.path.join(cc_basedir, 'bin/clang') + assert os.path.exists(clang_exec_path), "host-compiler present," \ + " but has no clang executable." + return clang_exec_path + else: + return False + + def liblto(self): + """Location of the host compiler, if one is present""" + cc_basedir = os.path.join(self.workspace, 'host-compiler/') + if os.path.exists(cc_basedir): + clang_liblto_path = os.path.join(cc_basedir, 'lib/') + assert os.path.exists(clang_liblto_path), "host-compiler present," \ + " but has no liblto." + return clang_liblto_path + else: + return False + + def branch(self): + """Figure out the source branch name. + Not using GIT_BRANCH env var from Jenkins as that includes the + remote name too. + """ + if not os.environ.get('TESTING', False): + cmd = ['git', '-C', conf.srcdir(), 'symbolic-ref', '--short', 'HEAD'] + out = run_collect_output(cmd).strip() + return out + return 'master' + + def link_memory_usage(self): + """Guesstimate the maximum link memory usage for this build. + We are only building master here so we will just use that value + """ + # Determinited experimentally. + usages = {'master': 3.5} + if self.branch() == 'master': + return usages['master'] + else: + raise NotImplementedError( + "Unknown link memory usage." + self.branch()) + + +# Global storage for configuration object. +conf = None # type: Configuration + +def cmake_builder(target): + if not os.getenv("TESTING"): + dep.parse_dependencies([here + "/clang_build_dependencies.dep"]) + + env = [] + dyld_path = "" + if conf.lto and conf.liblto(): + dyld_path = conf.liblto() + env.extend(["env", "DYLD_LIBRARY_PATH=" + dyld_path]) + + cmake_cmd = env + ["/usr/local/bin/cmake", "-G", "Ninja", + '-DCMAKE_MAKE_PROGRAM=' + NINJA, + "-DCMAKE_INSTALL_PREFIX=" + conf.installdir(), + "-DLLVM_ENABLE_PROJECTS=" + conf.llvm_enable_projects, + conf.llvmsrcdir()] + + compiler_flags = conf.compiler_flags + max_parallel_links = conf.max_parallel_links + + if conf.lto: + if conf.thinlto: + cmake_cmd += ["-DLLVM_PARALLEL_LINK_JOBS=1"] + else: + cmake_cmd += ["-DLLVM_PARALLEL_LINK_JOBS=" + str(max_link_jobs())] + cmake_cmd += ['-DLLVM_BUILD_EXAMPLES=Off'] + if not max_parallel_links: + max_parallel_links = 1 + if dyld_path: + cmake_cmd += ['-DDYLD_LIBRARY_PATH=' + dyld_path] + else: + cmake_cmd += ['-DLLVM_ENABLE_LTO=Off'] + cmake_cmd += ['-DLLVM_BUILD_EXAMPLES=On'] + + cmake_cmd += ["-DCMAKE_MACOSX_RPATH=On"] + + libtool_path = query_sys_tool("macosx", "libtool") + if libtool_path: + cmake_cmd += ['-DCMAKE_LIBTOOL=' + libtool_path] + + if compiler_flags: + cmake_cmd += ["-DCMAKE_C_FLAGS={}".format(' '.join(compiler_flags)), + "-DCMAKE_CXX_FLAGS={}".format(' '.join(compiler_flags))] + + if max_parallel_links is not None: + cmake_cmd += ["-DLLVM_PARALLEL_LINK_JOBS={}".format(max_parallel_links)] + + if conf.CC(): + cmake_cmd += ['-DCMAKE_C_COMPILER=' + conf.CC(), + '-DCMAKE_CXX_COMPILER=' + conf.CC() + "++"] + + if conf.cmake_build_type: + cmake_cmd += ["-DCMAKE_BUILD_TYPE=" + conf.cmake_build_type] + elif conf.debug: + cmake_cmd += ["-DCMAKE_BUILD_TYPE=Debug"] + else: + cmake_cmd += ["-DCMAKE_BUILD_TYPE=Release"] + + cmake_cmd += ["-DLLVM_BUILD_EXTERNAL_COMPILER_RT=On"] + + for flag in conf.cmake_flags: + cmake_cmd += [flag] + + if conf.assertions: + cmake_cmd += ["-DLLVM_ENABLE_ASSERTIONS=On"] + else: + cmake_cmd += ["-DLLVM_ENABLE_ASSERTIONS=Off"] + + if conf.globalisel: + cmake_cmd += ["-DLLVM_BUILD_GLOBAL_ISEL=ON"] + + lit_flags = ['--xunit-xml-output=testresults.xunit.xml', '-v', '--timeout=600'] + if conf.max_parallel_tests: + lit_flags += ['-j', conf.max_parallel_tests] + cmake_cmd += ['-DLLVM_LIT_ARGS={}'.format(' '.join(lit_flags))] + + ninja_cmd = env + ["/usr/local/bin/ninja", '-v'] + if conf.j_level is not None: + ninja_cmd += ["-j", conf.j_level] + + if target == 'all' or target == 'build': + header("Cmake") + run_cmd(conf.builddir(), cmake_cmd) + footer() + header("Ninja build") + + # Build all if nothing is passed by the user. + passed_target = conf.cmake_build_targets + build_target = passed_target if passed_target else ['all'] + run_cmd(conf.builddir(), ninja_cmd + build_target) + footer() + if conf.noinstall: + header("Skip install") + else: + header("Ninja install") + run_cmd(conf.builddir(), ninja_cmd + ['install']) + build_upload_artifact() + footer() + # Run all the test targets. + ninja_cmd.extend(['-k', '0']) + if target == 'all' or target == 'test' or target == 'testlong': + header("Ninja test") + + targets = [ + 'check-all'] if target == 'testlong' or target == 'all' else conf.cmake_test_targets + + if not targets: + # testlong and all do check all, otherwise check and check-clang + # unless the user asked for something else. + targets = ['check', 'check-clang'] + + run_cmd(conf.builddir(), ninja_cmd + targets) + footer() + + +def clang_builder(target): + """Build to set of commands to compile and test apple-clang""" + + # get rid of old archives from prior builds + run_ws(['sh', '-c', 'rm -rfv *gz']) + + if target == "all" or target == "build": + # Clean the build directory. + run_ws(['rm', '-rf', 'clang.roots']) + + sdk_name = 'macosx' + + sdkroot = query_sdk_path(sdk_name) + libtool_path = query_sys_tool(sdk_name, "libtool") + + clang_br = os.path.join(conf.workspace, conf._build_dir) + next_section("Build Directory") + run_ws(["mkdir", "-p", clang_br]) + + toolchain = '/Applications/Xcode.app/Contents/Developer' \ + '/Toolchains/XcodeDefault.xctoolchain' + + env = [] + dyld_path = "" + if conf.lto and conf.liblto(): + dyld_path = conf.liblto() + env.extend(["env", "DYLD_LIBRARY_PATH=" + dyld_path]) + + next_section("Build Clang") + if conf.nobootstrap: + if conf.debug or conf.device: + assert False, "Invalid parameter for clang-builder." + run_cmd(clang_br, ['mkdir', + './Build', + './Root']) + install_prefix = conf.installdir() + + # Infer which CMake cache file to use. If ThinLTO we select a specific one. + cmake_cachefile_thinlto = '' + if conf.thinlto: + cmake_cachefile_thinlto = '-ThinLTO' + cmake_cachefile = '{}/clang/cmake/caches/Apple-stage2{}.cmake'.format( + conf.srcdir(), cmake_cachefile_thinlto) + + cmake_command = env + ["/usr/local/bin/cmake", '-G', 'Ninja', '-C', + cmake_cachefile, + '-DLLVM_ENABLE_ASSERTIONS:BOOL={}'.format( + "TRUE" if conf.assertions else "FALSE"), + '-DCMAKE_BUILD_TYPE=RelWithDebInfo', + '-DLLVM_ENABLE_PROJECTS={}'.format(conf.llvm_enable_projects), + '-DCMAKE_MAKE_PROGRAM=' + NINJA, + '-DLLVM_VERSION_PATCH=99', + '-DLLVM_VERSION_SUFFIX=""', + '-DLLVM_BUILD_EXTERNAL_COMPILER_RT=On', + '-DCLANG_COMPILER_RT_CMAKE_ARGS={}/compiler-rt/cmake/caches/Apple.cmake'.format( + conf.srcdir()), + '-DCOMPILER_RT_BUILD_SANITIZERS=On', + '-DCMAKE_INSTALL_PREFIX={}'.format( + install_prefix), + '-DCLANG_REPOSITORY_STRING={}'.format( + conf.branch()), + '-DCLANG_APPEND_VC_REV=On', + '-DLLVM_BUILD_TESTS=On', + '-DLLVM_INCLUDE_TESTS=On', + '-DCLANG_INCLUDE_TESTS=On', + '-DLLVM_INCLUDE_UTILS=On', + '-DLIBCXX_INSTALL_HEADERS=On', + '-DLIBCXX_OVERRIDE_DARWIN_INSTALL=On', + '-DLIBCXX_INSTALL_LIBRARY=Off', + '-DCMAKE_MACOSX_RPATH=On', + ] + + if dyld_path: + cmake_command += ['-DDYLD_LIBRARY_PATH=' + dyld_path] + + if libtool_path: + cmake_command += ['-DCMAKE_LIBTOOL=' + libtool_path] + + if conf.CC(): + cmake_command.extend(['-DCMAKE_C_COMPILER=' + conf.CC(), + '-DCMAKE_CXX_COMPILER=' + conf.CC() + "++"]) + + lit_flags = ['--xunit-xml-output=testresults.xunit.xml', '-v', '--timeout=600'] + if conf.max_parallel_tests: + lit_flags += ['-j', conf.max_parallel_tests] + cmake_command.extend( + ['-DLLVM_LIT_ARGS={}'.format(' '.join(lit_flags))]) + + if conf.thinlto: + cmake_command.extend(["-DLLVM_PARALLEL_LINK_JOBS=1"]) + elif conf.lto: + cmake_command.extend( + ["-DLLVM_PARALLEL_LINK_JOBS=" + str(max_link_jobs())]) + else: + cmake_command.extend(['-DLLVM_ENABLE_LTO=Off']) + cmake_command.extend([ + '-DCMAKE_C_FLAGS_RELWITHDEBINFO:STRING=-O2 -gline-tables-only -DNDEBUG', + '-DCMAKE_CXX_FLAGS_RELWITHDEBINFO:STRING=-O2 -gline-tables-only -DNDEBUG']) + + for flag in conf.cmake_flags: + cmake_command += [flag] + + cmake_command.append(conf.llvmsrcdir()) + run_cmd(os.path.join(clang_br, 'Build'), cmake_command) + next_section("Ninja") + run_cmd(os.path.join(clang_br, 'Build'), [NINJA, '-v', 'install']) + + build_upload_artifact() + + else: + # Two stage build, via the make files. + print 'Stage two compile TBD in near future' + + if not conf.device and (target == "test" or target == "all"): + # Add steps to run the tests. + next_section("Tests") + # Auto detect bootstrap and non-bootstrap. + obj_dir = os.path.join(conf._build_dir, + 'Objects/obj-llvm/tools/clang/stage2-bins/') + if not os.path.exists(obj_dir): + obj_dir = os.path.join(conf._build_dir, 'Build/') + obj_dir = os.path.join(conf.workspace, obj_dir) + + cmd = [NINJA, '-v', '-k', '0', 'check-all'] + + if conf.assertions: + cmd[-1] += ' --param use_gmalloc=1 ' \ + '--param gmalloc_path=$(xcodebuild -find-library' \ + ' libgmalloc.dylib)' + run_cmd(obj_dir, cmd, env={'MALLOC_LOG_FILE': '/dev/null'}) + + +def parse_settings_from_output(working_dir, cmd): + old_dir = os.getcwd() + try: + os.chdir(working_dir) + assignment_regex = re.compile(r"^\s+([^\s=]+)\s*=\s*(.+)$") + settings = {} + for line in subprocess.check_output(cmd).splitlines(True): + match = assignment_regex.match(line) + if match: + settings[match.group(1)] = match.group(2) + return settings + finally: + os.chdir(old_dir) + + +def lldb_builder(): + """Do an Xcode build of lldb.""" + + # Wipe the build folder + + header("Clean LLDB build directory") + if os.path.exists(conf.lldbbuilddir()): + shutil.rmtree(conf.lldbbuilddir()) + footer() + + # Build into the build folder + build_configuration = "Release" + + xcodebuild_cmd = [ + "xcodebuild", + "-arch", "x86_64", + "-configuration", build_configuration, + "-scheme", "desktop", + "-derivedDataPath", conf.lldbbuilddir() + # It is too fragile to use the Xcode debugserver. If we add new + # command line arguments to debugserver, the older Xcode debugserver + # will fall over and not run. By commenting out this flag, we + # are requiring the builder to have the lldb_codesign code signing + # certificate and we are ensuring we are testing the latest debugserver + # from lldb. + # "DEBUGSERVER_USE_FROM_SYSTEM=1" + ] + + header("Build Xcode desktop scheme") + run_cmd("lldb", xcodebuild_cmd) + footer() + + header("Gather Xcode build settings") + xcodebuild_cmd.append("-showBuildSettings") + settings = parse_settings_from_output("lldb", xcodebuild_cmd) + footer() + + build_dir = settings.get("BUILD_DIR", None) + built_products_dir = settings.get("BUILT_PRODUCTS_DIR", None) + if build_dir is None or built_products_dir is None: + raise Exception("failed to retrieve build-related directories " + "from Xcode") + + llvm_build_dir = settings.get("LLVM_BUILD_DIR", None) + llvm_build_dir_arch = settings.get("LLVM_BUILD_DIR_ARCH", None) + if llvm_build_dir is None or llvm_build_dir_arch is None: + raise Exception("failed to retrieve LLVM build-related settings " + "from Xcode") + llvm_build_bin_dir = os.path.join(llvm_build_dir, llvm_build_dir_arch, "bin") + built_clang_path = os.path.join(llvm_build_bin_dir, "clang") + built_filecheck_path = os.path.join(llvm_build_bin_dir, "FileCheck") + effective_clang = os.environ.get("LLDB_PYTHON_TESTSUITE_CC", + built_clang_path) + + # Run C++ test suite (gtests) + + xcodebuild_cmd = [ + "xcodebuild", + "-arch", "x86_64", + "-configuration", build_configuration, + "-scheme", "lldb-gtest", + "-derivedDataPath", conf.lldbbuilddir(), + # See notes above. + # "DEBUGSERVER_USE_FROM_SYSTEM=1" + ] + + header("Build Xcode lldb-gtest scheme") + run_cmd("lldb", xcodebuild_cmd) + footer() + + # Run LLDB Python test suite for archs defined in LLDB_TEST_ARCHS + for arch in conf.lldb_test_archs: + results_file = os.path.join(build_dir, + "test-results-{}.xml".format(arch)) + python_testsuite_cmd = [ + "/usr/bin/python", + "test/dotest.py", + "--executable", os.path.join(built_products_dir, "lldb"), + "-C", effective_clang, + "--arch", arch, + "--results-formatter", + "lldbsuite.test_event.formatter.xunit.XunitFormatter", + "--results-file", results_file, + "--rerun-all-issues", + "--env", "TERM=vt100", + "-O--xpass=ignore", + "--dsymutil="+os.path.join(os.path.dirname(effective_clang), 'dsymutil'), + "--filecheck="+built_filecheck_path + ] + + header("Run LLDB Python-based test suite ({} targets)".format(arch)) + # For the unit tests, we don't want to stop the build if there are + # build errors. We allow the JUnit/xUnit parser to pick this up. + print repr(python_testsuite_cmd) + run_cmd_errors_okay("lldb", python_testsuite_cmd) + footer() + + +def lldb_cmake_builder(): + """Do a CMake build of lldb.""" + + test_dir = os.path.join(conf.workspace, 'test') + log_dir = os.path.join(test_dir, 'logs') + results_file = os.path.join(test_dir, 'results.xml') + dest_dir = os.path.join(conf.workspace, 'results', 'lldb') + run_ws(["mkdir", "-p", conf.lldbbuilddir()]) + cmake_build_type = conf.cmake_build_type if conf.cmake_build_type else 'RelWithDebInfo' + header("Configure") + dotest_args=['--arch', 'x86_64', '--build-dir', + conf.lldbbuilddir()+'/lldb-test-build.noindex', + '-s='+log_dir, + '-t', + '--env', 'TERM=vt100'] + dotest_args.extend(conf.dotest_flags) + cmake_cmd = ["/usr/local/bin/cmake", '-G', 'Ninja', + conf.srcdir(), + '-DLLVM_ENABLE_ASSERTIONS:BOOL={}'.format( + "TRUE" if conf.assertions else "FALSE"), + '-DLLVM_ENABLE_PROJECTS='+conf.llvm_enable_projects, + '-DCMAKE_BUILD_TYPE='+cmake_build_type, + '-DCMAKE_MAKE_PROGRAM=' + NINJA, + '-DLLVM_VERSION_PATCH=99', + '-DLLVM_ENABLE_MODULES=On', + '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON', + '-DCMAKE_INSTALL_PREFIX="%s"'%dest_dir, + '-DLLDB_TEST_USER_ARGS='+';'.join(dotest_args), + '-DLLVM_LIT_ARGS=--xunit-xml-output=%s -v'%results_file] + cmake_cmd.extend(conf.cmake_flags) + + if conf.CC(): + cmake_cmd.extend(['-DCMAKE_C_COMPILER=' + conf.CC(), + '-DCMAKE_CXX_COMPILER=' + conf.CC() + "++"]) + + run_cmd(conf.lldbbuilddir(), cmake_cmd) + footer() + + header("Build") + run_cmd(conf.lldbbuilddir(), [NINJA, '-v']) + footer() + + header("Run Tests") + run_cmd(conf.lldbbuilddir(), [NINJA, '-v', 'check-debuginfo']) + run_cmd(conf.lldbbuilddir(), ['/usr/bin/env', 'TERM=vt100', NINJA, '-v', + 'check-lldb']) + footer() + + +def static_analyzer_benchmarks_builder(): + """Run static analyzer benchmarks""" + header("Static Analyzer Benchmarks") + + benchmark_script = conf.workspace + "/utils-analyzer/SATestBuild.py" + benchmarks_dir = conf.workspace + "/test-suite-ClangAnalyzer/" + + compiler_bin_dir = conf.workspace + "/host-compiler/bin/" + scanbuild_bin_dir = conf.workspace + "/tools-scan-build/bin/" + + old_path = os.environ.get("PATH", "") + env = dict(os.environ, PATH=compiler_bin_dir + os.pathsep + + scanbuild_bin_dir + os.pathsep + + old_path) + + benchmark_cmd = [benchmark_script, + "--strictness", "0" + ] + run_cmd(benchmarks_dir, benchmark_cmd, env=env) + + footer() + + +def http_download(url, dest): + """Safely download url to dest. + + Print error and exit if download fails. + """ + try: + print "GETting", url, "to", dest, "...", + f = urlopen(url) + # Open our local file for writing + with open(dest, "wb") as local_file: + local_file.write(f.read()) + + except HTTPError, e: + print + print "HTTP Error:", e.code, url + sys.exit(1) + + except URLError, e: + print + print "URL Error:", e.reason, url + sys.exit(1) + print "done." + + +def create_builddirs(): + paths = [conf.builddir(), conf.installdir()] + for p in paths: + if not os.path.exists(p): + os.makedirs(p) + + +def fetch_compiler(): + local_name = "host-compiler.tar.gz" + url = conf.host_compiler_url + "/" + conf.artifact_url + header("Fetching Compiler") + http_download(url, conf.workspace + "/" + local_name) + print "Decompressing..." + if os.path.exists(conf.workspace + "/host-compiler"): + shutil.rmtree(conf.workspace + "/host-compiler") + os.mkdir(conf.workspace + "/host-compiler") + run_cmd(conf.workspace + "/host-compiler/", + ['tar', 'zxf', "../" + local_name]) + os.unlink(local_name) + footer() + +def build_upload_artifact(): + """Create artifact for this build, and upload to server.""" + if conf.noupload: + print 'Not uploading artificats' + return + header("Uploading Artifact") + prop_file = "last_good_build.properties" + + # TODO : azhar : Update artifact name + artifact_name = "clang-t{}-b{}.tar.gz".format(conf.build_id, + conf.build_number) + new_url = conf.job_name + "/" + artifact_name + + with open(prop_file, 'w') as prop_fd: + prop_fd.write("GIT_SHA={}\n".format(conf.git_sha)) + prop_fd.write("ARTIFACT={}\n".format(new_url)) + + # The .a's are big and we don't need them later. Drop the LLVM and clang + # libraries, but keep the libraries from compiler-rt. + tar = ["tar", "zcvf", "../" + artifact_name, "--exclude=*libLLVM*.a", + "--exclude=*libclang[A-Z]*.a", "."] + + run_cmd(conf.installdir(), tar) + + mkdir_cmd = ["ssh", "buildslave@" + SERVER, "mkdir", "-p", "/Library/WebServer/Documents/artifacts/" + conf.job_name] + + run_cmd(conf.workspace, mkdir_cmd) + + upload_cmd = ["scp", artifact_name, + "buildslave@" + SERVER + ":/Library/WebServer/Documents/artifacts/" + + conf.job_name + "/"] + + run_cmd(conf.workspace, upload_cmd) + + upload_cmd = ["scp", prop_file, + "buildslave@" + SERVER + ":/Library/WebServer/Documents/artifacts/" + + conf.job_name + "/"] + + run_cmd(conf.workspace, upload_cmd) + + ln_cmd = ["ssh", "buildslave@" + SERVER, + "ln", "-fs", "/Library/WebServer/Documents/artifacts/" + + conf.job_name + "/" + artifact_name, + "/Library/WebServer/Documents/artifacts/" + + conf.job_name + "/latest"] + + run_cmd(conf.workspace, ln_cmd) + + lng_cmd = ["ssh", "buildslave@" + SERVER, + "ln", "-fs", "/Library/WebServer/Documents/artifacts/" + + conf.job_name + "/" + artifact_name, + "/Library/WebServer/Documents/artifacts/" + + conf.job_name + "/g" + conf.git_sha] + run_cmd(conf.workspace, lng_cmd) + + +def run_cmd(working_dir, cmd, env=None, sudo=False, err_okay=False): + """Run a command in a working directory, and make sure it returns zero.""" + assert type(cmd) == list, "Not a list: {}".format(type(cmd)) + old_cwd = os.getcwd() + if env: + envs = [] + for key, value in env.items(): + envs.append("{}={}".format(key, value)) + cmd = ["env"] + envs + cmd + if sudo: + cmd = ['sudo'] + cmd + + cmd_to_print = ' '.join([quote_sh_string(x) for x in cmd]) + sys.stdout.write("cd {}\n{}\n".format(working_dir, cmd_to_print)) + sys.stdout.flush() + return_code = 0 + start_time = datetime.datetime.now() + if not os.environ.get('TESTING', False): + try: + os.chdir(working_dir) + subprocess.check_call(cmd) + os.chdir(old_cwd) + except subprocess.CalledProcessError as excpt: + if not err_okay: + raise excpt + else: + logging.info("Ignoring failed command.") + return_code = excpt.returncode + end_time = datetime.datetime.now() + + logging.info("Command took {} seconds".format( + (end_time - start_time).seconds)) + return return_code + + +def run_cmd_errors_okay(working_dir, cmd, env=None): + """Run a command in a working directory, reporting return value. + Non-zero exit codes do not generate an exception. + """ + old_cwd = os.getcwd() + cmd_to_print = ' '.join([quote_sh_string(x) for x in cmd]) + sys.stdout.write("cd {}\n{}\n".format(working_dir, cmd_to_print)) + sys.stdout.flush() + + start_time = datetime.datetime.now() + if not os.environ.get('TESTING', False): + try: + os.chdir(working_dir) + result = subprocess.call(cmd, env=env) + finally: + os.chdir(old_cwd) + end_time = datetime.datetime.now() + + logging.info("Command took {} seconds: return code {}".format( + (end_time - start_time).seconds, result)) + + +KNOWN_TARGETS = ['all', 'build', 'test', 'testlong'] +KNOWN_BUILDS = ['clang', 'cmake', 'lldb', 'lldb-cmake', 'fetch', 'artifact', + 'static-analyzer-benchmarks'] + + +def query_sdk_path(sdk_name): + """Get the path to the sdk named using xcrun. + + When $TESTING define, just give a dummy back. We do this because xcrun + could fail if the sdk you want is not installed, and that is silly for + testing. + """ + + if not os.environ.get('TESTING', False): + cmd = ['xcrun', '--sdk', sdk_name, '--show-sdk-path'] + return run_collect_output(cmd).strip() + else: + return "/Applications/Xcode.app/Contents/Developer/Platforms/" \ + "MacOSX.platform/Developer/SDKs/MacOSX10.10.sdk" + + +def max_link_jobs(): + """Link jobs take about 3.6GB of memory, max.""" + mem_str = run_collect_output(["sysctl", "hw.memsize"]) + mem = float(mem_str.split()[1].strip()) + mem = mem / (1024.0 ** 3) # Conver to GBs. + return int(math.ceil(mem / conf.link_memory_usage())) + + +TEST_VALS = {"sysctl hw.ncpu": "hw.ncpu: 8\n", + "sysctl hw.memsize": "hw.memsize: 8589934592\n", + "xcrun --sdk iphoneos --show-sdk-path": "/Foo/bar", + "/usr/bin/xcrun svn upgrade": "", + } + + +@contextmanager +def cwd(path): + last_cwd = os.getcwd() + if path: + os.chdir(path) + try: + yield + finally: + os.chdir(last_cwd) + + +def run_collect_output(cmd, working_dir=None, stderr=None): + """Run cmd and then return the output. + + If working_dir is supplied the cmd will run in + with a context manager in working_dir. + """ + if os.getenv("TESTING"): + print 'TV: ' + ' '.join(cmd) + return TEST_VALS[' '.join(cmd)] + + with cwd(working_dir): + return subprocess.check_output(cmd, stderr=stderr) + + +def query_sys_tool(sdk_name, tool_name): + """Get the path of system tool + + When $TESTING define, just give a dummy back. + """ + + if not os.environ.get('TESTING', False): + cmd = ['xcrun', '--sdk', sdk_name, '--find', tool_name] + return run_collect_output(cmd).strip() + else: + return "/usr/bin/" + tool_name + + +def run_ws(cmd, env=None): + """Wrapper to call run_cmd in local workspace. + + Since 99 percent of the time, that is where you want to call things from. + """ + return run_cmd(conf.workspace, cmd, env) + + +def parse_args(): + """Get the command line arguments, and make sure they are correct.""" + + parser = argparse.ArgumentParser( + description='Build and test compilers and other things.') + + parser.add_argument("build_type", + help="The kind of build to trigger.", + choices=KNOWN_BUILDS) + + parser.add_argument("build_target", + nargs='?', + help="The targets to call (build, check, etc).", + choices=KNOWN_TARGETS) + + parser.add_argument('--assertions', dest='assertions', action='store_true') + parser.add_argument('--lto', dest='lto', action='store_true') + parser.add_argument('--thinlto', dest='thinlto', action='store_true') + parser.add_argument('--debug', dest='debug', action='store_true') + parser.add_argument('--cmake-type', dest='cmake_build_type', + help="Override cmake type Release, Debug, " + "RelWithDebInfo and MinSizeRel") + parser.add_argument('--cmake-flag', dest='cmake_flags', + action='append', default=[], + help='Set an arbitrary cmake flag') + parser.add_argument('--dotest-flag', dest='dotest_flags', + action='append', default=[], + help='Set an arbitrary lldb dotest.py flag') + parser.add_argument('--cmake-test-target', dest='cmake_test_targets', + action='append', default=[], + help='Targets to build during testing') + parser.add_argument('--cmake-build-target', dest='cmake_build_targets', + action='append', default=[], + help='Targets to build during building.') + parser.add_argument('--compiler-flag', dest='compiler_flags', + action='append', default=[], + help='Set an arbitrary compiler flag') + parser.add_argument('--noupload', dest='noupload', action='store_true') + parser.add_argument('--noinstall', dest='noinstall', action='store_true', + help="Disable the install stage, build only.") + parser.add_argument('--globalisel', dest='globalisel', + action='store_true', help="Turn on the experimental" + " GlobalISel CMake flag.") + parser.add_argument('--projects', dest='llvm_enable_projects', + default="clang;clang-tools-extra;compiler-rt;libcxx", + help="Semicolon seperated list of projects to build.") + + args = parser.parse_args() + if args.thinlto: + args.lto = True + return args + + +def main(): + """Run a build based on command line args and ENV.""" + global conf + args = parse_args() + conf = Configuration(args) + + create_builddirs() + try: + if args.build_type == 'clang': + clang_builder(args.build_target) + elif args.build_type == 'lldb': + lldb_builder() + elif args.build_type == 'lldb-cmake': + lldb_cmake_builder() + elif args.build_type == 'cmake': + cmake_builder(args.build_target) + elif args.build_type == 'fetch': + fetch_compiler() + elif args.build_type == 'artifact': + build_upload_artifact() + elif args.build_type == 'static-analyzer-benchmarks': + static_analyzer_benchmarks_builder() + except subprocess.CalledProcessError as exct: + print "Command failed", exct.message + print "Command:", exct.cmd + sys.exit(1) + + +if __name__ == '__main__': + main()