Migrate the annotated buildbots to github

[rnk]

Summary

Reuse the base LLVMBuildFactory logic for checking out and updating
llvm-project instead of implementing it in the script.

There are advantages to putting the 'git' logic in the script, but since
we need a master restart anyway to move to git, we might as well reuse
the buildbot git step for now. I'm not sure what the best practices are
for incrementally updating a git repository anyway.

[inglorion]

LGTM.

I'm assuming that addGetSourcecodeSteps takes care of checking out the correct revision. From a cursory look at LLVMBuildFactory and the Buildbot documentation, this appears to be the case.

[rnk]

After two more follow-ons, one of the two bots is now green: http://lab.llvm.org:8011/builders/clang-x64-windows-msvc/builds/11758

I'm not sure why sanitizer-windows still thinks it should use svn.

[rnk]

Looks like Phab doesn't monitor llvm-zorg commits. This landed here:
https://github.com/llvm/llvm-zorg/commit/c5fbd2ca11d0625e659a1f45f6a083e83c137954

zorg/buildbot/builders/AnnotatedBuilder.py

merged_env = {
         'TERM': 'dumb'  # Be cautious and disable color output from all tools.
     }
     if env is not None:
         # Overwrite pre-set items with the given ones, so user can set
         # anything.
         merged_env.update(env)
 
     scripts_dir = "annotated"
-    f.addStep(SVN(name='update-annotate-scripts',
-                  mode='update',
-                  svnurl='http://llvm.org/svn/llvm-project/zorg/trunk/'
-                         'zorg/buildbot/builders/annotated',
-                  workdir=scripts_dir,
-                  alwaysUseLatest=True))
+
+    # Check out zorg so we can run the annotator scripts.
+    f.addGetSourcecodeForProject(
+        project='llvm-zorg',
+        src_dir='llvm-zorg',
+        alwaysUseLatest=True)
+
+    f.addGetSourcecodeSteps()
 
     # Explicitly use '/' as separator, because it works on *nix and Windows.
-    script_path = "../%s/%s" % (scripts_dir, script)
+    script_path = "llvm-zorg/zorg/buildbot/builders/annotated/%s" % (script)
     f.addStep(AnnotatedCommand(name="annotate",
                                description="annotate",
                                timeout=timeout,
                                haltOnFailure=True,
                                command=WithProperties(
                                    "python %(script)s --jobs=%(jobs:-)s",
                                    script=lambda _: script_path),
                                env=merged_env))
     return f

zorg/buildbot/builders/annotated/annotated_builder.py

def cmake(
         source_dir,
         cmake='cmake',
         cmake_args=None):
         self.report_build_step('%s cmake' % (stage_name,))
         self.halt_on_failure()
         cmd = [cmake]
         if cmake_args is not None:
             cmd += cmake_args
-        cmd += [source_dir]
+        cmd += [util.cmake_pjoin(source_dir, 'llvm')]
         util.mkdirp(build_dir)
         util.report_run_cmd(cmd, cwd=build_dir)
 
     def cmake_compiler_flags(
         self,
         c_compiler,
         cxx_compiler,
         linker=None,
def set_environment(self, env=None, vs_tools=None, arch=None):
                 os.environ[var] = val
 
             for var in sorted(os.environ.keys()):
                 util.report('%s=%s' % (var, os.environ[var]))
         except Exception as e:
             self.report_step_exception(e)
             raise
 
-    def update_sources(self, source_dir, projects, revision, svn='svn'):
-        self.report_build_step('update-sources')
-        self.halt_on_failure()
-
-        # TODO: This needs to be updated to use the monorepo.
-        # Where to check the project out relative to an LLVM checkout.
-        checkout_locations = {
-            'llvm': '',
-            'clang': 'tools/clang',
-            'lld': 'tools/lld',
-            'compiler-rt': 'projects/compiler-rt',
-            'debuginfo-tests': 'projects/debuginfo-tests',
-            }
-        # If the project is named differently in svn, put it here.
-        svn_locations = { 'clang': 'cfe' }
-        svn_uri_pattern = 'https://llvm.org/svn/llvm-project/%s/trunk'
-
-        for project in projects:
-            # TODO: Fail the build and report an error if we don't know the
-            # checkout location.
-            path = checkout_locations[project]
-            if not path:
-                path = source_dir
-            elif not os.path.isabs(path):
-                path = pjoin(source_dir, path)
-            uri = svn_uri_pattern % (svn_locations.get(project, project),)
-            util.report_run_cmd([svn, 'cleanup'], cwd=path)
-            util.report("Updating %s to %s at %s from %s" %
-                        (project, revision, util.shquote(path), uri))
-            if os.path.exists(pjoin(path, '.svn')):
-                util.report("Cleaning up in case of svn errors...")
-                util.report_run_cmd([svn, 'cleanup'], cwd=path)
-                cmd = [svn, 'up', '-r', revision]
-            else:
-                util.mkdirp(path)
-                cmd = [svn, 'co', '-r', revision, uri, '.']
-            util.report_run_cmd(cmd, cwd=path)
-
     def run_steps(
         self,
         stages=1,
         projects=None,
         check_targets=None,
         check_stages=None,
         extra_cmake_args=None,
         stage1_extra_cmake_args=None,
-        revision=None,
         compiler='clang',
         linker='ld.lld',
         env=None,
         jobs=None):
         """
         stages: number of stages to run (default: 1)
-        projects: which subprojects to check out from SVN
+        projects: which subprojects to enable
             llvm must be first in the list (default: ['llvm', 'clang', 'lld'])
         check_targets: targets to run during the check phase (default: ['check-all'])
         check_stages: stages for which to run the check phase
             (array of bool, default: all True)
         extra_cmake_args: extra arguments to pass to cmake (default: [])
         stage1_extra_cmake_args: extra arguments to pass to cmake for stage 1
             (default: use extra_cmake_args)
-        revision: revision to check out (default: os.environ['BUILDBOT_REVISION'],
-            or, if that is unset, the latest revision)
         compiler: compiler to use after stage 1
             ('clang' or 'clang-cl'; default 'clang')
         linker: linker to use after stage 1
             (None (let cmake choose) or 'lld' (default))
         env: environment overrides (map; default is no overrides)
         jobs: number of jobs to run concurrently (default: determine automatically)
         """
 
         # Set defaults.
         if check_targets is None:
             check_targets = ['check-all']
         if check_stages is None:
             check_stages = [True] * stages
         if extra_cmake_args is None:
             extra_cmake_args = []
-        if not revision:
-            revision = os.environ.get('BUILDBOT_REVISION')
         if stage1_extra_cmake_args is None:
             stage1_extra_cmake_args = extra_cmake_args
         if projects is None:
             projects = ['llvm', 'clang', 'lld']
 
         c_compiler, cxx_compiler = self.compiler_binaries(compiler)
 
         self.set_environment(env)
 
         # On Windows, if we're building clang-cl, make sure stage1 is built with
         # MSVC (cl.exe), and not gcc from mingw. CMake will prefer gcc if it is
         # available.
         if c_compiler == 'clang-cl':
             stage1_extra_cmake_args += ['-DCMAKE_C_COMPILER=cl',
                                         '-DCMAKE_CXX_COMPILER=cl']
 
-        if not revision:
-            cmd = ['svn', 'info', 'https://llvm.org/svn/llvm-project/']
-            try:
-                svninfo = subprocess.check_output(cmd)
-            except subprocess.CalledProcessError as e:
-                util.report("Failed to get most recent SVN rev: " + str(e))
-                return 1
-            m = re.search('Revision: ([0-9]+)', svninfo)
-            if m:
-                revision = m.group(1)
-            else:
-                util.report("Failed to find svn revision in svn info output:\n"
-                            + svninfo)
-                return 1
-        if not revision.isdigit():
-            util.report("SVN revision %s is not a positive integer" % (revision,))
-
         # Update sources.
         cwd = os.getcwd()
-        source_dir = pjoin(cwd, 'llvm.src')
+        source_dir = pjoin(cwd, 'llvm-project')
         build_dir = pjoin(cwd, 'build')
-        cmake_args = ['-GNinja']
+        cmake_args = ['-GNinja', '-DLLVM_ENABLE_PROJECTS=' + ';'.join(projects)]
 
         try:
-            self.update_sources(source_dir, projects, revision)
-
             # Build and check stages.
             self.build_and_check_stages(
                 stages,
                 build_dir,
                 source_dir,
                 cmake_args,
                 extra_cmake_args,
                 c_compiler,