Speed up git-llvm script by only svn up'ing affected directories.

[jyknight]

Summary

Also, support modifications to toplevel files in git (which need to be
committed to "monorepo-top" in svn).

[rnk]

I guess you should update the commit message to say "monorepo-root" instead of "monorepo-top". Otherwise, neat!

[zturner]

This fails with this on Python 3.7:

Pushing 1 commit:

71b82b6e1cb Add missing triple from llvm-mc command line.

Traceback (most recent call last):

File "D:\src\llvm-mono\llvm\utils\git-svn\git-llvm", line 418, in <module>
  args.func(args)
File "D:\src\llvm-mono\llvm\utils\git-svn\git-llvm", line 373, in cmd_push
  svn_push_one_rev(svn_root, r, dry_run)
File "D:\src\llvm-mono\llvm\utils\git-svn\git-llvm", line 290, in

svn_push_one_rev

for sr, files in subrepo_files.iteritems():

AttributeError: 'collections.defaultdict' object has no attribute
'iteritems'

So at the moment I'm blocked from pushing commits :D

Any chance you could take a look?

[jyknight]

Hopefully resolved the python3 compatibility with r347113.

[zturner]

r347114 suggests that your fix worked :)

llvm/trunk/utils/git-svn/git-llvm

for d in [
         'libunwind',
         'lld',
         'lldb',
         'llgo',
         'llvm',
         'openmp',
         'parallel-libs',
         'polly',
+        'pstl',
     ]
 }
 GIT_TO_SVN_DIR.update({'clang': 'cfe/trunk'})
+GIT_TO_SVN_DIR.update({'': 'monorepo-root/trunk'})
 
 VERBOSE = False
 QUIET = False
 dev_null_fd = None
 
 
 def eprint(*args, **kwargs):
     print(*args, file=sys.stderr, **kwargs)
def log_verbose(*args, **kwargs):
     print(*args, **kwargs)
 
 
 def die(msg):
     eprint(msg)
     sys.exit(1)
 
 
-def first_dirname(d):
-    while True:
-        (head, tail) = os.path.split(d)
-        if not head or head == '/':
-            return tail
-        d = head
+def split_first_path_component(d):
+    # Assuming we have a git path, it'll use slashes even on windows...I hope.
+    if '/' in d:
+        return d.split('/', 1)
+    else:
+        return (d, None)
 
 
 def get_dev_null():
     """Lazily create a /dev/null fd for use in shell()"""
     global dev_null_fd
     if dev_null_fd is None:
         dev_null_fd = open(os.devnull, 'w')
     return dev_null_fd
 
 
 def shell(cmd, strip=True, cwd=None, stdin=None, die_on_failure=True,
           ignore_errors=False, force_binary_stdin=False):
-    log_verbose('Running: %s' % ' '.join(cmd))
+    log_verbose('Running in %s: %s' % (cwd, ' '.join(cmd)))
 
     err_pipe = subprocess.PIPE
     if ignore_errors:
         # Silence errors if requested.
         err_pipe = get_dev_null()
 
     if force_binary_stdin and stdin:
         stdin = stdin.encode('utf-8')
if not text:
         stderr = stderr.decode('utf-8')
 
     if p.returncode == 0 or ignore_errors:
         if stderr and not ignore_errors:
             eprint('`%s` printed to stderr:' % ' '.join(cmd))
             eprint(stderr.rstrip())
         if strip:
             stdout = stdout.rstrip('\r\n')
+        if VERBOSE:
+            for l in stdout.splitlines():
+                log_verbose("STDOUT: %s" % l)
         return stdout
     err_msg = '`%s` returned %s' % (' '.join(cmd), p.returncode)
     eprint(err_msg)
     if stderr:
         eprint(stderr.rstrip())
     if die_on_failure:
         sys.exit(2)
     raise RuntimeError(err_msg)
def get_revs_to_push(rev_range):
     # (foo..bar) like we want.
     revs = git('show', '--reverse', '--quiet',
                '--pretty=%h', rev_range).splitlines()
     if not revs:
         die('Nothing to push: No revs in range %s.' % rev_range)
     return revs
 
 
-def clean_and_update_svn(svn_repo):
+def clean_svn(svn_repo):
     svn(svn_repo, 'revert', '-R', '.')
 
     # Unfortunately it appears there's no svn equivalent for git clean, so we
     # have to do it ourselves.
     for line in svn(svn_repo, 'status', '--no-ignore').split('\n'):
         if not line.startswith('?'):
             continue
         filename = line[1:].strip()
         os.remove(os.path.join(svn_repo, filename))
 
-    svn(svn_repo, 'update', *list(GIT_TO_SVN_DIR.values()))
-
 
 def svn_init(svn_root):
     if not os.path.exists(svn_root):
         log('Creating svn staging directory: (%s)' % (svn_root))
         os.makedirs(svn_root)
-        log('This is a one-time initialization, please be patient for a few'
-            ' minutes...')
         svn(svn_root, 'checkout', '--depth=immediates',
             'https://llvm.org/svn/llvm-project/', '.')
-        svn(svn_root, 'update', *list(GIT_TO_SVN_DIR.values()))
         log("svn staging area ready in '%s'" % svn_root)
     if not os.path.isdir(svn_root):
         die("Can't initialize svn staging dir (%s)" % svn_root)
 
 
-def fix_eol_style_native(rev, sr, svn_sr_path):
+def fix_eol_style_native(rev, svn_sr_path, files):
     """Fix line endings before applying patches with Unix endings
 
     SVN on Windows will check out files with CRLF for files with the
     svn:eol-style property set to "native". This breaks `git apply`, which
     typically works with Unix-line ending patches. Work around the problem here
     by doing a dos2unix up front for files with svn:eol-style set to "native".
     SVN will not commit a mass line ending re-doing because it detects the line
     ending format for files with this property.
     """
-    files = git('diff-tree', '--no-commit-id', '--name-only', '-r', rev, '--',
-                sr).split('\n')
-    files = [f.split('/', 1)[1] for f in files]
     # Skip files that don't exist in SVN yet.
     files = [f for f in files if os.path.exists(os.path.join(svn_sr_path, f))]
     # Use ignore_errors because 'svn propget' prints errors if the file doesn't
     # have the named property. There doesn't seem to be a way to suppress that.
     eol_props = svn(svn_sr_path, 'propget', 'svn:eol-style', *files,
                     ignore_errors=True)
     crlf_files = []
     if len(files) == 1:
else:
             if len(prop_parts) != 2:
                 eprint("unable to parse svn propget line:")
                 eprint(eol_prop)
                 continue
             (f, eol_style) = prop_parts
             if eol_style == 'native':
                 crlf_files.append(f)
     if crlf_files:
-        # Reformat all files with native SVN line endings to Unix format. SVN knows
-        # files with native line endings are text files. It will commit just the
-        # diff, and not a mass line ending change.
+        # Reformat all files with native SVN line endings to Unix format. SVN
+        # knows files with native line endings are text files. It will commit
+        # just the diff, and not a mass line ending change.
         shell(['dos2unix'] + crlf_files, ignore_errors=True, cwd=svn_sr_path)
 
+def get_all_parent_dirs(name):
+    parts = []
+    head, tail = os.path.split(name)
+    while head:
+        parts.append(head)
+        head, tail = os.path.split(head)
+    return parts
+
+def split_subrepo(f):
+    # Given a path, splits it into (subproject, rest-of-path). If the path is
+    # not in a subproject, returns ('', full-path).
+
+    subproject, remainder = split_first_path_component(f)
+
+    if subproject in GIT_TO_SVN_DIR:
+        return subproject, remainder
+    else:
+        return '', f
+
 def svn_push_one_rev(svn_repo, rev, dry_run):
     files = git('diff-tree', '--no-commit-id', '--name-only', '-r',
                 rev).split('\n')
-    subrepos = {first_dirname(f) for f in files}
-    if not subrepos:
+    if not files:
         raise RuntimeError('Empty diff for rev %s?' % rev)
 
+    # Split files by subrepo
+    subrepo_files = collections.defaultdict(list)
+    for f in files:
+        subrepo, remainder = split_subrepo(f)
+        subrepo_files[subrepo].append(remainder)
+
     status = svn(svn_repo, 'status', '--no-ignore')
     if status:
         die("Can't push git rev %s because svn status is not empty:\n%s" %
             (rev, status))
 
-    for sr in subrepos:
+    svn_dirs_to_update = set()
+    for sr, files in subrepo_files.iteritems():
+        svn_sr_path = GIT_TO_SVN_DIR[sr]
+        for f in files:
+            svn_dirs_to_update.update(
+                get_all_parent_dirs(os.path.join(svn_sr_path, f)))
+
+    # Sort by length to ensure that the parent directories are passed to svn
+    # before child directories.
+    sorted_dirs_to_update = sorted(svn_dirs_to_update,
+                                   cmp=lambda x,y: cmp(len(x), len(y)))
+
+    # SVN update only in the affected directories.
+    svn(svn_repo, 'update', '--depth=immediates', *sorted_dirs_to_update)
+
+    for sr, files in subrepo_files.iteritems():
         svn_sr_path = os.path.join(svn_repo, GIT_TO_SVN_DIR[sr])
         if os.name == 'nt':
-            fix_eol_style_native(rev, sr, svn_sr_path)
-        diff = git('show', '--binary', rev, '--', sr, strip=False)
+            fix_eol_style_native(rev, svn_sr_path, files)
+        diff = git('show', '--binary', rev, '--',
+                   *(os.path.join(sr, f) for f in files),
+                   strip=False)
         # git is the only thing that can handle its own patches...
         log_verbose('Apply patch: %s' % diff)
+        if sr == '':
+            prefix_strip = '-p1'
+        else:
+            prefix_strip = '-p2'
         try:
-            # If we allow python to apply the diff in text mode, it will silently
-            # convert \n to \r\n which git doesn't like.
-            shell(['git', 'apply', '-p2', '-'], cwd=svn_sr_path, stdin=diff,
-                  die_on_failure=False, force_binary_stdin=True)
+            # If we allow python to apply the diff in text mode, it will
+            # silently convert \n to \r\n which git doesn't like.
+            shell(['git', 'apply', prefix_strip, '-'], cwd=svn_sr_path,
+                  stdin=diff, die_on_failure=False, force_binary_stdin=True)
         except RuntimeError as e:
             eprint("Patch doesn't apply: maybe you should try `git pull -r` "
                    "first?")
             sys.exit(2)
 
     status_lines = svn(svn_repo, 'status', '--no-ignore').split('\n')
 
     for l in (l for l in status_lines if (l.startswith('?') or
def cmd_push(args):
     rev_range = args.rev_range
     dry_run = args.dry_run
     revs = get_revs_to_push(rev_range)
     log('Pushing %d commit%s:\n%s' %
         (len(revs), 's' if len(revs) != 1
          else '', '\n'.join('  ' + git('show', '--oneline', '--quiet', c)
                             for c in revs)))
     for r in revs:
-        clean_and_update_svn(svn_root)
+        clean_svn(svn_root)
         svn_push_one_rev(svn_root, r, dry_run)
 
 
 if __name__ == '__main__':
     if not program_exists('svn'):
         die('error: git-llvm needs svn command, but svn is not installed.')
 
     argv = sys.argv[1:]