Index: buildbot/osuosl/master/config/builders.py =================================================================== --- buildbot/osuosl/master/config/builders.py +++ buildbot/osuosl/master/config/builders.py @@ -16,6 +16,7 @@ from zorg.buildbot.builders import ClangLTOBuilder from zorg.buildbot.builders import UnifiedTreeBuilder from zorg.buildbot.builders import CUDATestsuiteBuilder +from zorg.buildbot.builders import AOSPBuilder # Plain LLVM builders. def _get_llvm_builders(): @@ -667,6 +668,27 @@ "-DCMAKE_CXX_COMPILER:FILEPATH=/local/clang+llvm-3.7.1-x86_64-linux-gnu-ubuntu-14.04/bin/clang++"])} ] +# AOSP builders. +def _get_aosp_builders(): + return [ + {'name': "aosp-O3-polly-before-vectorizer-unprofitable", + 'slavenames': ["hexagon-build-03"], + 'builddir': "aosp", + 'factory': AOSPBuilder.getAOSPBuildFactory( + device="angler", + build_clang=True, + extra_cmake_args=["-DCMAKE_C_COMPILER:FILEPATH=/local/clang+llvm-3.7.1-x86_64-linux-gnu-ubuntu-14.04/bin/clang", + "-DCMAKE_CXX_COMPILER:FILEPATH=/local/clang+llvm-3.7.1-x86_64-linux-gnu-ubuntu-14.04/bin/clang++"], + timeout=180, + target_clang=None, + target_flags="-Wno-error -O3 -mllvm -polly -mllvm -polly-position=before-vectorizer -mllvm -polly-process-unprofitable", + jobs=16, + extra_make_args=None, + env={}, + sync=True, + patch="aosp-O3-polly-before-vectorizer-unprofitable")} + ] + # LLDB builders. def _get_lldb_builders(): return [ @@ -1338,6 +1360,10 @@ b['category'] = 'polly' yield b + for b in _get_aosp_builders(): + b['category'] = 'aosp' + yield b + for b in _get_lld_builders(): b['category'] = 'lld' yield b Index: buildbot/osuosl/master/config/status.py =================================================================== --- buildbot/osuosl/master/config/status.py +++ buildbot/osuosl/master/config/status.py @@ -174,7 +174,8 @@ extraRecipients = ["efriedma@codeaurora.org"], subject="Build %(builder)s Failure", mode = "failing", - builders = ["polly-arm-linux"], + builders = ["polly-arm-linux", + "aosp-O3-polly-before-vectorizer-unprofitable"], addLogs=False, num_lines = 15), InformativeMailNotifier( Index: buildbot/osuosl/master/master.cfg =================================================================== --- buildbot/osuosl/master/master.cfg +++ buildbot/osuosl/master/master.cfg @@ -109,6 +109,7 @@ from buildbot.schedulers import triggerable from buildbot.schedulers.filter import ChangeFilter from buildbot.schedulers.basic import SingleBranchScheduler +from buildbot.schedulers.timed import Nightly def depends_on(projects): """ @@ -225,6 +226,12 @@ "cfe", "llgo"]))) +c['schedulers'].append(Nightly(name="aosp_scheduler", + builderNames=get_all_for("aosp"), + hour=19, + minute=0, + branch=None)) + c['schedulers'].extend( config.schedulers.getSingleBranchSchedulers(c['builders'], c['schedulers']) ) Index: zorg/buildbot/builders/AOSPBuilder.py =================================================================== --- /dev/null +++ zorg/buildbot/builders/AOSPBuilder.py @@ -0,0 +1,182 @@ +import buildbot +from buildbot.steps.shell import ShellCommand +from buildbot.steps.source import SVN + + +def addLLVMBuildSteps( + f, # The BuildFactory to modify + jobs=None, # Number of concurrent jobs + extra_cmake_args=[]): # Extra args for the cmake command + llvm_srcdir = "llvm.src" + llvm_objdir = "llvm.obj" + llvm_instdir = "llvm.inst" + polly_srcdir = '%s/tools/polly' % llvm_srcdir + clang_srcdir = '%s/tools/clang' % llvm_srcdir + jobs_cmd = [] + if jobs is not None: + jobs_cmd = ["-j" + str(jobs)] + build_cmd = ['ninja'] + jobs_cmd + install_cmd = ['ninja', 'install'] + jobs_cmd + + # Get LLVM, Clang and Polly + f.addStep(SVN(name='svn-llvm', + mode='update', + baseURL='http://llvm.org/svn/llvm-project/llvm/', + defaultBranch='trunk', + workdir=llvm_srcdir)) + f.addStep(SVN(name='svn-clang', + mode='update', + baseURL='http://llvm.org/svn/llvm-project/cfe/', + defaultBranch='trunk', + workdir=clang_srcdir)) + f.addStep(SVN(name='svn-polly', + mode='update', + baseURL='http://llvm.org/svn/llvm-project/polly/', + defaultBranch='trunk', + workdir=polly_srcdir)) + + # Clean LLVM build dir + f.addStep(ShellCommand(name='clean-llvm-build-dir', + command=['rm', '-rf', llvm_objdir], + warnOnFailure=True, + description=["clean llvm build dir"], + workdir='.')) + + # Create configuration files with cmake + f.addStep(ShellCommand(name="create-llvm-build-dir", + command=["mkdir", "-p", llvm_objdir], + haltOnFailure=False, + description=["create llvm build dir"], + workdir=".")) + cmakeCommand = ["cmake", "../%s" % llvm_srcdir, + "-G", "Ninja", + "-DLLVM_TARGETS_TO_BUILD='ARM;AArch64'", + "-DLLVM_DEFAULT_TARGET_TRIPLE=arm-linux-androideabi", + "-DLLVM_TARGET_ARCH=arm-linux-androideabi", + "-DCMAKE_BUILD_TYPE=Release", + "-DLLVM_ENABLE_ASSERTIONS=True", + "-DCMAKE_INSTALL_PREFIX=../%s" % llvm_instdir] + extra_cmake_args + f.addStep(ShellCommand(name="cmake-configure", + command=cmakeCommand, + haltOnFailure=False, + description=["cmake configure"], + workdir=llvm_objdir)) + + # Build LLVM + f.addStep(ShellCommand(name="build llvm", + command=build_cmd, + haltOnFailure=True, + description=["build llvm"], + workdir=llvm_objdir)) + + # Clean LLVM install dir + f.addStep(ShellCommand(name='clean-llvm-install-dir', + command=['rm', '-rf', llvm_instdir], + haltOnFailure=False, + description=["clean llvm install dir"], + workdir='.')) + + # Install + f.addStep(ShellCommand(name="install llvm", + command=install_cmd, + haltOnFailure=False, + description=["install llvm"], + workdir=llvm_objdir)) + + +def getAOSPBuildCommand( + device, # Target device for AOSP build + timeout=None, # Maximum CPU time in seconds, umlimited if 'None' + target_clang=None, # Path to the Clang used for AOSP target build + # Set target_clang to None to use AOSP's default Clang + target_flags=None, # Extra C/CXX flags for AOSP target build + jobs=None, # Number of concurrent jobs + extra_make_args=None): # Extra args for the make command + command = "source build/envsetup.sh" + command += " && lunch aosp_%s-userdebug" % device + command += " && make -k" + if timeout: + command += " TIMEOUT=%s" % timeout + if target_clang: + command += " TARGET_CLANG=%s" % target_clang + if target_flags: + command += " TARGET_FLAGS='%s'" % target_flags + if extra_make_args: + command += " " + extra_make_args + if jobs: + command += " -j" + str(jobs) + return command + + +def getAOSPBuildFactory( + device, # Target device for AOSP build + build_clang=False, # Flag to control building Clang for AOSP target build + extra_cmake_args=[], # Extra args for the LLVM cmake command + # This flag is ignored if build_clang is False + timeout=None, # Maximum CPU time in seconds, umlimited if 'None' + target_clang=None, # Path to the Clang used for AOSP target build + # Set target_clang to None to use AOSP's default Clang + # This flag is ignored if build_clang is True + target_flags=None, # Extra C/CXX flags for AOSP target build + jobs=None, # Number of concurrent jobs + extra_make_args=None, # Extra args for the make command + env={}, # Environmen variables for all steps + sync=False, # Flag to control whether AOSP repo is synced + patch=None): # Name of the patch to apply to AOSP source + f = buildbot.process.factory.BuildFactory() + clang_dir = target_clang + + # Build Clang for AOSP target build + if build_clang: + addLLVMBuildSteps(f, jobs=jobs, extra_cmake_args=extra_cmake_args) + clang_dir = 'llvm.inst/bin' + + # Restore AOSP repo to a clean state + f.addStep(ShellCommand(name="clean-repo", + command=['repo', 'forall', '-c', + 'git reset --hard; git clean -fdx'], + haltOnFailure=False, + description=["clean repo"], + workdir=".", + env=env)) + + # Sync AOSP repo + if sync: + f.addStep(ShellCommand(name="sync-repo", + command=['repo', 'sync', '-c', '--no-tags'], + haltOnFailure=False, + description=["sync repo"], + workdir=".", + env=env)) + + # Patch AOSP build system to allow switching Clang for target build + if patch: + f.addStep(ShellCommand(name="patch-aosp", + command=['patch', '-p1', '-i', + 'patches/%s.patch' % patch], + haltOnFailure=True, + description=["patch aosp"], + workdir=".", + env=env)) + + # Delete existing output dir + f.addStep(ShellCommand(name="delete-out-dir", + command=['rm', '-rf', 'out'], + haltOnFailure=False, + description=["delete out dir"], + workdir=".", + env=env)) + + # Build AOSP + f.addStep(ShellCommand(name="build-aosp", + command=getAOSPBuildCommand(device=device, + timeout=timeout, + target_clang=clang_dir, + target_flags=target_flags, + jobs=jobs, + extra_make_args=extra_make_args), + haltOnFailure=True, + description=["build aosp"], + workdir=".", + env=env)) + return f