Get the branch specifiers from a .json file rather than out of raw Python

[spyffe]

Summary

LLDB has many branches in a variety of repositories, and the build-script.py file is subtly different for each set.
This is unnecessary and causes merge headaches.
This patch makes build-script.py consult a directory full of .json files, each one of which matches a particular branch using a regular expression.

[spyffe]

• Removed some debug print statements.
• Added the option of an OVERRIDE.json file to disable regex-based matching if the user knows what they are doing.

[beanz]

LGTM!

[spyffe]

• Added a FALLBACK configuration file that reflects what the branch was cut from, allowing e.g. pull requests or branches without losing auto-checkout.
• Re-generate FALLBACK whenever we successfully detect a branch configuration so that it is as close as possible to what we last used

[jingham]

There should be some doc describing this process. Yes I know none of this is really documented, but since this seems actually useful, we should document it.

scripts/Xcode/build-llvm.py

48–49

Do you really want to exit here? You should really fall back to the fallback file.

50–51

It looks like find exits in all the bad cases, and with a better error message since it knows what it did wrong, so this in unnecessary.

scripts/Xcode/repo.py

43

Put in the actual path that you looked for the fallback file in.

[jingham]

Probably README.md is the right place to mention this.

[spyffe]

Updated to reflect Jim's comments:
• Removed redundant errors
• Improved existing errors
• Added documentation

[jingham]

LGTM

INSTALL.txt

 LLDB Installation Instructions
 ==============================
 
 LLDB builds on Mac OS X (with Xcode) and Linux (with GCC or Clang).
 
 On Mac OS X, in addition to using Xcode you'll need to enable code signing
 on your system to either build lldb or debug using lldb.  Please see the code
 signing documentation in docs/code-signing.txt for more detailed directions.
 
+If you are building on Mac OS X and LLVM is not present in llvm/, then LLDB
+will check it out automatically.  The files in scripts/Xcode/repos determine
+which branches of LLVM/Clang are checked out, depending on the current
+LLDB branch, according to the algorithm in scripts/Xcode/repo.py.
+
 For instructions to build LLDB on Linux, or more details about supported
 compiler versions, other dependencies, and build flags, see:
 
     http://lldb.llvm.org/build.html

scripts/Xcode/build-llvm.py

 #!/usr/bin/env python
 
 import errno
 import hashlib
 import fnmatch
 import os
 import platform
 import re
+import repo
 import subprocess
 import sys
 
 from lldbbuild import *
 
 #### SETTINGS ####
 
 
 def LLVM_HASH_INCLUDES_DIFFS():
     return False
 
-# The use of "x = "..."; return x" here is important because tooling looks for
-# it with regexps.  Only change how this works if you know what you are doing.
-
-
-def LLVM_REF():
-    llvm_ref = "master"
-    return llvm_ref
-
+# For use with Xcode-style builds
 
-def CLANG_REF():
-    clang_ref = "master"
-    return clang_ref
+def process_vcs(vcs):
+    return {
+        "svn": VCS.svn,
+        "git": VCS.git
+    }[vcs]
 
-# For use with Xcode-style builds
+def process_root(name):
+    return {
+        "llvm": llvm_source_path(),
+        "clang": clang_source_path(),
+        "ninja": ninja_source_path()
+    }[name]
 
+def process_repo(r):
+    return {
+        'name': r["name"],
+        'vcs': process_vcs(r["vcs"]),
+        'root': process_root(r["name"]),
+        'url': r["url"],
+        'ref': r["ref"]
+    }
 
 def XCODE_REPOSITORIES():
-    return [
-        {'name': "llvm",
-         'vcs': VCS.git,
-         'root': llvm_source_path(),
-         'url': "http://llvm.org/git/llvm.git",
-         'ref': LLVM_REF()},
-
-        {'name': "clang",
-         'vcs': VCS.git,
-         'root': clang_source_path(),
-         'url': "http://llvm.org/git/clang.git",
-         'ref': CLANG_REF()},
-
-        {'name': "ninja",
-         'vcs': VCS.git,
-         'root': ninja_source_path(),
-         'url': "https://github.com/ninja-build/ninja.git",
-         'ref': "master"}
-    ]
+    identifier = repo.identifier()
+    if identifier == None:
+        identifier = "<invalid>" # repo.find will just use the fallback file
+    set = repo.find(identifier)
+    return [process_repo(r) for r in set]
 
 
 def get_c_compiler():
     return subprocess.check_output([
         'xcrun',
         '--sdk', 'macosx',
         '-find', 'clang'
     ]).rstrip()
 

scripts/Xcode/repo.py

+import json
+import os
+import re
+import shutil
+import subprocess
+
+def identifier():
+  try:
+    svn_output = subprocess.check_output(["svn", "info", "--show-item", "url"], stderr=subprocess.STDOUT).rstrip()
+    return svn_output
+  except:
+    pass
+  try:
+    git_remote_and_branch = subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"]).rstrip()
+    git_remote = git_remote_and_branch.split("/")[0]
+    git_branch = "/".join(git_remote_and_branch.split("/")[1:])
+    git_url = subprocess.check_output(["git", "remote", "get-url", git_remote]).rstrip()
+    return git_url + ":" + git_branch
+  except:
+    pass
+  return None
+
+def find(identifier):
+  dir = os.path.dirname(os.path.realpath(__file__))
+  repos_dir = os.path.join(dir, "repos")
+  json_regex = re.compile(r"^.*.json$")
+  override_path = os.path.join(repos_dir, "OVERRIDE")
+  if os.path.isfile(override_path):
+    override_set = json.load(open(override_path))
+    return override_set["repos"]
+  fallback_path = os.path.join(repos_dir, "FALLBACK")
+  for path in [os.path.join(repos_dir, f) for f in filter(json_regex.match, os.listdir(repos_dir))]:
+    fd = open(path)
+    set = json.load(fd)
+    fd.close()
+    if any(re.match(set_regex, identifier) for set_regex in set["regexs"]):
+      shutil.copyfile(path, fallback_path)
+      return set["repos"]
+  if os.path.isfile(fallback_path):
+    fallback_set = json.load(open(fallback_path))
+    return fallback_set["repos"]
+  sys.exit("Couldn't find a branch configuration for " + identifier + " and there was no " + fallback_path)

scripts/Xcode/repos/FALLBACK

+{
+    "regexs" : [".*llvm\\.org.*"],
+    "repos" : [
+        {"name": "llvm",
+         "vcs": "git",
+         "url": "http://llvm.org/git/llvm.git",
+         "ref": "master"},
+
+        {"name": "clang",
+         "vcs": "git",
+         "url": "http://llvm.org/git/clang.git",
+         "ref": "master"},
+
+        {"name": "ninja",
+         "vcs": "git",
+         "url": "https://github.com/ninja-build/ninja.git",
+         "ref": "master"}
+    ]
+}

scripts/Xcode/repos/svn-trunk.json

+{
+    "regexs" : [".*llvm\\.org.*"],
+    "repos" : [
+        {"name": "llvm",
+         "vcs": "git",
+         "url": "http://llvm.org/git/llvm.git",
+         "ref": "master"},
+
+        {"name": "clang",
+         "vcs": "git",
+         "url": "http://llvm.org/git/clang.git",
+         "ref": "master"},
+
+        {"name": "ninja",
+         "vcs": "git",
+         "url": "https://github.com/ninja-build/ninja.git",
+         "ref": "master"}
+    ]
+}