Added XToolchainBuilder to build cross toolchains.

[aorlov]

Summary

The factory provides build steps to build host-side tools and then cross-build target libraries.

Only host-side tests are executed yet.
The factory will be later expanded to also run remote tests.

[andreil99]

Thanks, Alex!

[gkistanova]

LGTM

zorg/buildbot/builders/XToolchainBuilder.py

+from buildbot.steps.shell import ShellCommand, SetProperty
+from buildbot.process.properties import WithProperties, Property
+
+from zorg.buildbot.commands.CmakeCommand   import CmakeCommand
+from zorg.buildbot.commands.NinjaCommand   import NinjaCommand
+from zorg.buildbot.commands.LitTestCommand import LitTestCommand
+from zorg.buildbot.process.factory         import LLVMBuildFactory
+
+import zorg.buildbot.builders.Util as builders_util
+
+def getCmakeWithMSVCBuildFactory(
+        clean = True,                # False for incremental builds.
+        depends_on_projects  = None, # List of projects to listen.
+        cmake_cache = None,          # Path to a cmake cache file.
+        extra_configure_args = None, # Extra CMake args if any.
+        llvm_srcdir = None,          # Source code root directory.
+        obj_dir = None,              # Build tree root directory.
+        install_dir = None,          # Directory to install the results to.
+        checks = None,               # List of checks to test the build.
+        jobs = None,                 # Restrict a degree of parallelism.
+        env  = None,                 # Environmental variables for all steps.
+        # VS tools environment variable if using MSVC.
+        # For example, "autodetect" to auto detect, %VS140COMNTOOLS% to select
+        # the VS 2015 toolchain, or empty string if environment is already set.
+        vs=None,
+        **kwargs):
+
+    if vs is None:
+        # We autodetect Visual Studio, unless otherwise is requested.
+        vs = "autodetect"
+
+    if install_dir is None:
+        install_dir = 'install'
+
+    # Prepare environmental variables. Set here all env we want for all steps.
+    merged_env = {
+        'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences.
+        }
+    if env is not None:
+        assert not vs, "Cannot have custom builder env vars with VS setup."
+        # Overwrite pre-set items with the given ones, so user can set anything.
+        merged_env.update(env)
+
+    # Make a local copy of the configure args, as we are going to modify that.
+    if extra_configure_args:
+        cmake_args = extra_configure_args[:]
+    else:
+        cmake_args = list()
+
+    if depends_on_projects is None:
+        depends_on_projects = [
+            'llvm',
+            'compiler-rt',
+            'clang',
+            'clang-tools-extra',
+            'libunwind',
+            'libcxx',
+            'libcxxabi',
+            'lld',
+        ]
+
+    if checks is None:
+        # Check only host-side tools. Target-side tests should run on a target.
+        checks=[
+            "check-llvm",
+            "check-clang",
+            "check-lld",
+        ]
+
+    source_remove_requested = lambda step: step.build.getProperty("clean")
+    clean_build_requested = lambda step: \
+        clean or \
+        step.build.getProperty("clean", \
+            default=step.build.getProperty("clean_obj") \
+        )
+
+    f = LLVMBuildFactory(
+            depends_on_projects=depends_on_projects,
+            llvm_srcdir=llvm_srcdir,
+            obj_dir=obj_dir,
+            install_dir=install_dir,
+            cleanBuildRequested=clean_build_requested,
+            **kwargs) # Pass through all the extra arguments.
+
+    # Remove the source code tree if requested.
+    # NOTE: Somehow RemoveDirectory buildbot command often fails on Windows,
+    # as somthing keeps a lock. We use rm command instead realying on a shell
+    # to support that.
+    f.addStep(ShellCommand(name='clean-%s-dir' % f.monorepo_dir,
+                           command=['rm','-rf', f.monorepo_dir],
+                           warnOnFailure=True,
+                           haltOnFailure=False,
+                           flunkOnFailure=False,
+                           description='Remove the source code',
+                           workdir='.',
+                           env=merged_env,
+                           doStepIf=source_remove_requested))
+
+    # Get the source code.
+    f.addGetSourcecodeSteps(**kwargs)
+
+    # Remove the build directory if requested.
+    f.addStep(ShellCommand(name='clean-%s-dir' % f.obj_dir,
+                           command=['rm','-rf', f.obj_dir],
+                           warnOnFailure=True,
+                           haltOnFailure=False,
+                           flunkOnFailure=False,
+                           description='Remove build directory',
+                           workdir='.',
+                           env=merged_env,
+                           doStepIf=clean_build_requested))
+
+    if vs:
+        # Configure MSVC environment if requested.
+        f.addStep(SetProperty(
+            command=builders_util.getVisualStudioEnvironment(vs, None),
+            extract_fn=builders_util.extractSlaveEnvironment))
+        merged_env = Property('slave_env')
+
+    # Since this is a build of a cross toolchain, we build only the host-side
+    # tools first by the host system compiler. Libraries will be cross-compiled.
+    cmake_args.append(WithProperties(
+        '-DLLVM_AR=%(workdir)s/' + f.obj_dir + '/bin/llvm-ar.exe')),
+
+    CmakeCommand.applyDefaultOptions(cmake_args, [
+        ('-G', 'Ninja'),
+        ('-DLLVM_ENABLE_PROJECTS=', 'llvm;clang;clang-tools-extra;lld'),
+        ('-DCMAKE_BUILD_TYPE=', 'Release'),
+        ('-DCMAKE_CXX_FLAGS=', '-D__OPTIMIZE__'),
+        ('-DLLVM_ENABLE_ASSERTIONS=', 'ON'),
+        ('-DLLVM_LIT_ARGS=', '-v -vv'),
+        ])
+
+    if install_dir:
+        install_dir_rel = LLVMBuildFactory.pathRelativeToBuild(
+                              install_dir,
+                              f.obj_dir)
+        CmakeCommand.applyRequiredOptions(cmake_args, [
+            ('-DCMAKE_INSTALL_PREFIX=', install_dir_rel),
+            ])
+
+    # Remove the build directory if requested.
+    f.addStep(ShellCommand(name='clean-%s-dir' % install_dir,
+                           command=['rm','-rf', install_dir],
+                           warnOnFailure=True,
+                           haltOnFailure=False,
+                           flunkOnFailure=False,
+                           description='Remove install directory',
+                           workdir='.',
+                           env=merged_env,
+                           doStepIf=clean_build_requested))
+
+    src_dir_rel = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, f.obj_dir)
+
+    # Add given cmake cache at the very end.
+    if cmake_cache:
+        cmake_args.append('-C%s' % cmake_cache)
+
+    f.addStep(CmakeCommand(name="cmake-configure",
+                           haltOnFailure=True,
+                           description=["Cmake", "configure"],
+                           options=cmake_args,
+                           path=src_dir_rel,
+                           workdir=f.obj_dir,
+                           env=merged_env,
+                           **kwargs # Pass through all the extra arguments.
+                           ))
+
+    f.addStep(NinjaCommand(name="build-%s" % f.monorepo_dir,
+                           haltOnFailure=True,
+                           description=["Build", f.monorepo_dir],
+                           workdir=f.obj_dir,
+                           env=merged_env,
+                           **kwargs # Pass through all the extra arguments.
+                           ))
+
+    # Test the components if requested one check at a time.
+    for check in checks:
+        f.addStep(
+            LitTestCommand(
+                haltOnFailure=False, # We want to test as much as we could.
+                name='test-%s' % check,
+                command=["ninja", "%e [%u/%r/%f]", WithProperties(check)],
+                description=[
+                    "Testing", "just", "built", "components", "for", check],
+                descriptionDone=[
+                    "Test", "just", "built", "components", "for", check,
+                     "completed"],
+                env=merged_env,
+                **kwargs # Pass through all the extra arguments.
+                ))
+
+    # Install just built components
+    if install_dir:
+        f.addStep(NinjaCommand(name="install-all",
+                               haltOnFailure=True,
+                               targets=["install"],
+                               description=[
+                                   "Install", "just", "built", "components"],
+                               workdir=f.obj_dir,
+                               env=merged_env,
+                               **kwargs # Pass through all the extra arguments.
+                               ))
+
+    return f