Index: libcxxabi/CMakeLists.txt =================================================================== --- libcxxabi/CMakeLists.txt +++ libcxxabi/CMakeLists.txt @@ -144,6 +144,7 @@ ${LIBCXXABI_LIBCXX_PATH}/include ${CMAKE_BINARY_DIR}/${LIBCXXABI_LIBCXX_INCLUDES} ${LLVM_MAIN_SRC_DIR}/projects/libcxx/include + ${LLVM_MAIN_SRC_DIR}/../libcxx/include ${LLVM_INCLUDE_DIR}/c++/v1 ) @@ -156,6 +157,7 @@ PATHS ${LIBCXXABI_LIBCXX_PATH} ${LIBCXXABI_LIBCXX_INCLUDES}/../ ${LLVM_MAIN_SRC_DIR}/projects/libcxx/ + ${LLVM_MAIN_SRC_DIR}/../libcxx/ NO_DEFAULT_PATH ) Index: llvm/CMakeLists.txt =================================================================== --- llvm/CMakeLists.txt +++ llvm/CMakeLists.txt @@ -93,6 +93,25 @@ endif() endif() +# Git Mono-Repo handling: automatically set the LLVM_EXTERNAL_${project}_SOURCE_DIR +set(LLVM_ALL_PROJECTS "clang;libcxx;libcxxabi;lldb;compiler-rt;lld;polly") +set(LLVM_ENABLE_PROJECTS "a" CACHE STRING + "Semicolon-separated list of projects to build (${LLVM_ALL_PROJECTS}), or \"all\".") +if( LLVM_ENABLE_PROJECTS STREQUAL "all" ) + set( LLVM_ENABLE_PROJECTS ${LLVM_ALL_PROJECTS}) +endif() +foreach(proj ${LLVM_ENABLE_PROJECTS}) + set(PROJ_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../${proj}") + if(NOT EXISTS "${PROJ_DIR}" OR NOT IS_DIRECTORY "${PROJ_DIR}") + message(FATAL_ERROR "LLVM_ENABLE_PROJECTS requests ${proj} but directory not found: ${PROJ_DIR}") + endif() + string(TOUPPER "${proj}" upper_proj) + set(LLVM_EXTERNAL_${upper_proj}_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../${proj}") + if (proj STREQUAL "clang") + set(LLVM_EXTERNAL_CLANG_TOOLS_EXTRA_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../clang-tools-extra") + endif() +endforeach() + # The following only works with the Ninja generator in CMake >= 3.0. set(LLVM_PARALLEL_COMPILE_JOBS "" CACHE STRING "Define the maximum number of concurrent compilation jobs.") Index: llvm/docs/GettingStarted.rst =================================================================== --- llvm/docs/GettingStarted.rst +++ llvm/docs/GettingStarted.rst @@ -680,6 +680,60 @@ Please, refer to the Git-SVN manual (``man git-svn``) for more information. +For developers to work with a git monorepo +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. note:: + + This set-up is using unofficial mirror hosted on GitHub, use with caution. + +To set up a clone of all the llvm projects using a unified repository: + +.. code-block:: console + + % export TOP_LEVEL_DIR=`pwd` + % git clone https://github.com/llvm-project/llvm-project/ + % cd llvm-project + % git config branch.master.rebase true + +You can configure various build directory from this clone, starting with a build +of LLVM alone: + +.. code-block:: console + + % cd $TOP_LEVEL_DIR + % mkdir llvm-build && cd llvm-build + % cmake -GNinja ../llvm-project/llvm + +Or lldb: + +.. code-block:: console + + % cd $TOP_LEVEL_DIR + % mkdir lldb-build && cd lldb-build + % cmake -GNinja ../llvm-project/llvm -DLLVM_ENABLE_PROJECTS=lldb + +Or a combination of multiple projects: + +.. code-block:: console + + % cd $TOP_LEVEL_DIR + % mkdir clang-build && cd clang-build + % cmake -GNinja ../llvm-project/llvm -DLLVM_ENABLE_PROJECTS="clang;libcxx;compiler-rt" + +A helper script is provided in `llvm/utils/git-svn/git-llvm`. Adding it to your +path enables an easy way to push back changes after commit: + +.. code-block:: console + + % export PATH=$PATH:$TOP_LEVEL_DIR/llvm-project/llvm/utils/git-svn/ + % git llvm push + +While this is using SVN under the hood, it does not require any interaction from +you with git-svn. +After a few minutes, `git pull` should get back the changes as they were +commited. + Local LLVM Configuration ------------------------ Index: llvm/utils/git-svn/git-llvm =================================================================== --- /dev/null +++ llvm/utils/git-svn/git-llvm @@ -0,0 +1,309 @@ +#!/usr/bin/env python +# +#=======- git-llvm - LLVM Git Help Integration ---------*- python -*--=========# +# +# The LLVM Compiler Infrastructure +# +# This file is distributed under the University of Illinois Open Source +# License. See LICENSE.TXT for details. +# +#===------------------------------------------------------------------------===# + +r""" +git-llvm integration +==================== + +This file provides integration for git. + +For further details, run: + + git llvm -h + +Requires Python 2.7 +""" + +from __future__ import print_function +import argparse +import collections +import contextlib +import errno +import os +import re +import subprocess +import sys +import tempfile +import time + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + +desc = ''' +TODO +''' + +# It's *almost* a straightforward mapping from the monorepo to svn... +GIT_TO_SVN_DIR = { + d: (d + '/trunk') + for d in [ + 'clang-tools-extra', + 'compiler-rt', + 'dragonegg', + 'klee', + 'libclc', + 'libcxx', + 'libcxxabi', + 'lld', + 'lldb', + 'llvm', + 'polly', + ] +} +GIT_TO_SVN_DIR.update({'clang': 'cfe/trunk'}) + + + +def main(): + # In order to keep '--' yet allow options after positionals, we need to + # check for '--' ourselves. (Setting nargs='*' throws away the '--', while + # nargs=argparse.REMAINDER disallows options after positionals.) + argv = sys.argv[1:] + try: + idx = argv.index('--') + except ValueError: + dash_dash = [] + else: + dash_dash = argv[idx:] + argv = argv[:idx] + + p = argparse.ArgumentParser( + prog='git llvm', formatter_class=argparse.RawDescriptionHelpFormatter, + description=desc) + subparsers = p.add_subparsers(title='subcommands', + description='valid subcommands', + help='additional help') + p.add_argument('-q', '--quiet', action='count', default=0, + help='print less information') + + parser_push = subparsers.add_parser('push', help='push back changes to the LLVM SVN repository') + parser_push.add_argument( + '-n', + '--dry-run', + dest='dry_run', + action='store_true', + help='Do everything other than commit to svn. Leaves junk in the svn ' + 'repo, so probably will not work well if you try to commit more ' + 'than one rev.') + parser_push.add_argument( + '-v', + '--verbose', + dest='verbose', + action='store_true', + help='Output every external command as we run it.') + parser_push.add_argument( + 'rev_range', + metavar='GIT_REVS', + type=str, + nargs='?', + help="revs to push (default: everything not in the branch's " + "upstream, or not in origin/master if the branch lacks " + "an explicit upstream)") + parser_push.set_defaults(func=svn_push) + args = p.parse_args(argv) + VERBOSE = not args.quiet + args.func(args) + +def svn_push(args): + + # Get the git root + git_root = run('git','rev-parse','--show-toplevel') + if not os.path.isdir(git_root): + eprint("Can't get git root dir") + sys.exit(1) + + # We need a staging area for SVN, let's hide it in the .git directory. + svn_root = os.path.join(git_root, ".git", "upstream-svn") + + if not os.path.exists(svn_root): + eprint("Creating svn staging directory: (%s)" % (svn_root)) + os.makedirs(svn_root) + eprint("This is a one-time initialization, please be patient for a few minutes...") + os.chdir(svn_root) + run('svn', 'checkout', '--depth=immediates', 'https://llvm.org/svn/llvm-project/', '.') + svn_dirs = list(GIT_TO_SVN_DIR.values()) + run('svn', 'update', *svn_dirs) + eprint("svn staging area ready in '%s'" % (svn_root)) + if not os.path.isdir(svn_root): + eprint("Can't initialize svn staging dir (%s)" % (svn_root)) + sys.exit(1) + + # Push from the root of the git repo + os.chdir(git_root) + + rev_range=args.rev_range + dry_run=args.dry_run + revs = get_revs_to_push(rev_range) + print('Pushing %d commit%s:\n%s' % + (len(revs), 's' if len(revs) != 1 + else '', '\n'.join(' ' + git('show', '--oneline', '--quiet', c) + for c in revs))) + clean_and_update_svn(svn_root) + for r in revs: + push(svn_root, r, dry_run) + + +VERBOSE = False + +def first_dirname(d): + while True: + (head, tail) = os.path.split(d) + if not head or head == '/': + return tail + d = head + +def shell(cmd, strip=True, cwd=None, stdin=None): + start = time.time() + if VERBOSE: + print('%r' % cmd) + + ret = subprocess.check_output(cmd, cwd=cwd, stdin=stdin) + if strip: + ret = ret.strip() + + elapsed = time.time() - start + if VERBOSE and elapsed >= .1: + print('Command took %0.1fs' % elapsed) + return ret + +def git(*cmd, **kwargs): + return shell(['git'] + list(cmd), kwargs.get('strip', True)) + + +def svn(cwd, *cmd, **kwargs): + # TODO: Better way to do default arg when we have *cmd? + return shell(['svn'] + list(cmd), cwd=cwd, stdin=kwargs.get('stdin', None)) + + +def get_default_rev_range(): + # Get the branch tracked by the current branch, as set by + # git branch --set-upstream-to See http://serverfault.com/a/352236/38694. + cur_branch = git('rev-parse', '--symbolic-full-name', 'HEAD') + upstream_branch = git('for-each-ref', '--format=%(upstream:short)', + cur_branch) + if not upstream_branch: + upstream_branch = 'origin/master' + + # Get the newest common ancestor between HEAD and our upstream branch. + upstream_rev = git('merge-base', 'HEAD', upstream_branch) + return '%s..' % upstream_rev + + +def get_revs_to_push(rev_range): + if not rev_range: + rev_range = get_default_rev_range() + # Use git show rather than some plumbing command to figure out which revs are + # in rev_range because it handles single revs (HEAD^) and ranges (foo..bar) + # like we want. + print(rev_range) + revs = git('show', '--reverse', '--quiet', '--pretty=%h', rev_range).split('\n') + # filter empty entries + revs = [r for r in revs if r != ''] + if not revs: + eprint('Nothing to push: No revs in range %s.' % rev_range) + sys.exit(1) + return revs + + +def clean_and_update_svn(svn_repo): + svn(svn_repo, 'revert', '-R', '.') + + # Unfortunately it appears there's no svn equivalent for git clean, so we have + # to do it ourselves. Because I'm paranoid, prompt before deleting anything. + to_delete = [] + for line in svn(svn_repo, 'status').split('\n'): + if not line.startswith('?'): + continue + sp = line.split(' ', 1) + if len(sp) != 2: + raise RuntimeError('Unexpected svn status line: %s' % line) + to_delete.append(os.path.join(svn_repo, sp[1].strip())) + + if to_delete: + print('Need to delete %d files to clean SVN repo: ' % len(to_delete)) + for f in to_delete: + print(' %s' % f) + if raw_input('Delete [yN]? ').lower() != 'y': + sys.exit(1) + for f in to_delete: + os.remove(f) + + svn(svn_repo, 'update') + + +def push(svn_repo, rev, dry_run): + files = git('diff-tree', '--no-commit-id', '--name-only', '-r', + rev).split('\n') + subrepos = {first_dirname(f) for f in files} + if not subrepos: + raise RuntimeError('Empty diff for rev %s?' % rev) + + status = svn(svn_repo, 'status') + if status: + print("Can't push git rev %s because svn status is not empty:\n%s" % + (rev, status)) + sys.exit(1) + + for sr in subrepos: + diff = git('show', '--binary', rev, '--', sr, strip=False) + svn_sr_path = os.path.join(svn_repo, GIT_TO_SVN_DIR[sr]) + with tempfile.TemporaryFile() as tf: + tf.write(diff) + tf.flush() + tf.seek(0) + # git is the only thing that can handle its own patches... + shell(['git', 'apply', '-p2', '-'], cwd=svn_sr_path, stdin=tf) + + status_lines = svn(svn_repo, 'status').split('\n') + + # Check that patch didn't dump any .orig or .rej files. + for line in status_lines: + if line.endswith('.rej') or line.endswith('.orig'): + raise RuntimeError('patch left a .orig/.rej file in the svn repo: %s.' % + line) + for l in (l for l in status_lines if l.startswith('?')): + svn(svn_repo, 'add', l[1:].strip()) + for l in (l for l in status_lines if l.startswith('!')): + svn(svn_repo, 'remove', l[1:].strip()) + + # Now we're ready to commit. + commit_msg = git('show', '--pretty=%B', '--quiet', rev) + if not dry_run: + print(svn(svn_repo, 'commit', '-m', commit_msg)) + print('Committed %s to svn.' % rev) + else: + print("Would have committed %s to svn, if this weren't a dry run." % rev) + +def run(*args, **kwargs): + stdin = kwargs.pop('stdin', '') + verbose = kwargs.pop('verbose', True) + strip = kwargs.pop('strip', True) + for name in kwargs: + raise TypeError("run() got an unexpected keyword argument '%s'" % name) + p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + stdin=subprocess.PIPE) + stdout, stderr = p.communicate(input=stdin) + if p.returncode == 0: + if stderr: + if verbose: + print >>sys.stderr, '`%s` printed to stderr:' % ' '.join(args) + print >>sys.stderr, stderr.rstrip() + if strip: + stdout = stdout.rstrip('\r\n') + return stdout + if verbose: + print >>sys.stderr, '`%s` returned %s' % (' '.join(args), p.returncode) + if stderr: + print >>sys.stderr, stderr.rstrip() + sys.exit(2) + +if __name__ == '__main__': + main()