[zorg] Add libc++ and libc++abi builders for Linux.

[danalbert]

Summary

The existing libc++ builder is based on buildit/testit and assumes that
the platform has its own libc++abi, and as such it only works for OS X.
Add a new one that uses an in-tree build and cmake to be compatible with
the other platforms.

This builder will actually build and test both libc++ and libc++abi.
Since most platforms don't have either of these libraries installed and
depend on each other, we can't test them in isolation.

[danalbert]

zorg/buildbot/builders/LibcxxAndAbiBuilder.py

63

gribozavr: can I assume there's a relatively recent clang installed on your buildslave?

[gribozavr]

buildbot/osuosl/master/master.cfg

166–170

I don't think it depends on llvm and cfe, unless we are rebuilding a trunk clang every time.

zorg/buildbot/builders/LibcxxAndAbiBuilder.py

44

Please add an env var dictionary parameter like in other builder factories.

Then 'clang' and 'clangxx' parameters are not needed. When adding a builder one would just add CC='clang-3.5' or CC='clang-3.6'.

51–53

I am not sure if a non-fully qualified path will work. All other builder factories have a 'get_builddir' step.

Also, can we move 'build' one level up?

Do we need to rm -rf the previous build?

63

Of course, there is Clang 3.5 installed.

[danalbert]

• Address review comments

[gribozavr]

zorg/buildbot/builders/LibcxxAndAbiBuilder.py

68–70

I think this relative path is not correct now. You could use the build_path variable here.

[danalbert]

• Missed one relative path

[gribozavr]

LGTM.

[danalbert]

Buildbot is up and running!

buildbot/osuosl/master/config/builders.py

 from zorg.buildbot.builders import SanitizerBuilderII
 reload(SanitizerBuilderII)
 from zorg.buildbot.builders import SanitizerBuilderII
 
 from zorg.buildbot.builders import Libiomp5Builder
 reload(Libiomp5Builder)
 from zorg.buildbot.builders import Libiomp5Builder
 
+from zorg.buildbot.builders import LibcxxAndAbiBuilder
+reload(LibcxxAndAbiBuilder)
+from zorg.buildbot.builders import LibcxxAndAbiBuilder
+
 # Plain LLVM builders.
 def _get_llvm_builders():
     return [
 #        {'name': "llvm-x86_64-ubuntu",
 #         'slavenames':["arxan_davinci"],
 #         'builddir':"llvm-x86_64-ubuntu",
 #         'factory': LLVMBuilder.getLLVMBuildFactory("x86_64-pc-linux-gnu", jobs=4,
 #                                                    timeout=30)},
return [
          'builddir':"libiomp5-clang-x86_64-linux-debian",
          'factory' : Libiomp5Builder.getLibiomp5BuildFactory(
                          buildcompiler="clang",
                          env={'PATH':'/home/llvmbb/bin/clang-latest/bin:/home/llvmbb/bin:/usr/local/bin:/usr/local/bin:/usr/bin:/bin'}),
          'category' : 'libiomp5'},
         ]
 
 
+def _get_libcxx_builders():
+    return [
+        {'name': 'libcxx-libcxxabi-x86_64-linux-debian',
+         'slavenames': ['gribozavr4'],
+         'builddir': 'libcxx-libcxxabi-x86_64-linux-debian',
+         'factory': LibcxxAndAbiBuilder.getLibcxxAndAbiBuilder(),
+         'category': 'libcxx'},
+    ]
+
+
 # Experimental and stopped builders
 def _get_experimental_builders():
     return [
         {'name': "llvm-ppc64-linux2",
          'slavenames':["coho"],
          'builddir':"llvm-ppc64-2",
          'factory': LLVMBuilder.getLLVMBuildFactory("ppc64-linux-gnu", jobs=2, clean=False, timeout=20),
          'category' : 'llvm'},
def get_builders():
     for b in _get_sanitizer_builders():
         b['category'] = 'sanitizer'
         yield b
 
     for b in _get_openmp_builders():
         b['category'] = 'openmp'
         yield b
 
+    for b in _get_libcxx_builders():
+        b['category'] = 'libcxx'
+        yield b
+
     for b in _get_experimental_builders():
         yield b
 
 # Random other unused builders...
 {'name': "clang-x86_64-openbsd",
  'slavenames':["ocean1"],
  'builddir':"clang-x86_64-openbsd",
  'factory': ClangBuilder.getClangBuildFactory(),

buildbot/osuosl/master/master.cfg

c['schedulers'].append(SingleBranchScheduler(name="sanitizer_scheduler",
                                 "compiler-rt"])))
 
 c['schedulers'].append(SingleBranchScheduler(name="openmp_scheduler",
                             treeStableTimer=2*60,
                             builderNames=get_all_for("openmp"),
                             change_filter=depends_on([
                                 "openmp"])))
 
+c['schedulers'].append(SingleBranchScheduler(name="libcxx_scheduler",
+                            treeStableTimer=2*60,
+                            builderNames=get_all_for("libcxx"),
+                            change_filter=depends_on([
+                                "libcxx",
+                                "libcxxabi"])))
+
 ####### PROJECT IDENTITY
 
 c['title'] = "LLVM"
 c['titleURL'] = "http://llvm.org/"
 c['buildbotURL'] = "http://lab.llvm.org:8011/"
 
 ####### DB URL
 
 c['db_url'] = "postgresql://localhost/buildmasterdb"

zorg/buildbot/builders/LibcxxAndAbiBuilder.py

+import os
+
+import buildbot
+import buildbot.process.factory
+import buildbot.steps.shell
+import buildbot.process.properties as properties
+
+from buildbot.steps.source.svn import SVN
+
+import zorg.buildbot.commands.LitTestCommand as lit_test_command
+import zorg.buildbot.util.artifacts as artifacts
+import zorg.buildbot.util.phasedbuilderutils as phased_builder_utils
+
+reload(lit_test_command)
+reload(artifacts)
+reload(phased_builder_utils)
+
+
+def getLibcxxWholeTree(f, src_root):
+    llvm_path = src_root
+    libcxx_path = os.path.join(llvm_path, 'projects/libcxx')
+    libcxxabi_path = os.path.join(llvm_path, 'projects/libcxxabi')
+
+    f = phased_builder_utils.SVNCleanupStep(f, llvm_path)
+    f.addStep(SVN(name='svn-llvm',
+                  mode='full',
+                  baseURL='http://llvm.org/svn/llvm-project/llvm/',
+                  defaultBranch='trunk',
+                  workdir=llvm_path))
+    f.addStep(SVN(name='svn-libcxx',
+                  mode='full',
+                  baseURL='http://llvm.org/svn/llvm-project/libcxx/',
+                  defaultBranch='trunk',
+                  workdir=libcxx_path))
+    f.addStep(SVN(name='svn-libcxxabi',
+                  mode='full',
+                  baseURL='http://llvm.org/svn/llvm-project/libcxxabi/',
+                  defaultBranch='trunk',
+                  workdir=libcxxabi_path))
+    return f
+
+
+def getLibcxxAndAbiBuilder(f=None, env={}):
+    if f is None:
+        f = buildbot.process.factory.BuildFactory()
+
+    # Determine the build directory.
+    f.addStep(properties.SetProperty(name="get_builddir",
+              command=["pwd"],
+              property="builddir",
+              description="set build dir",
+              workdir="."))
+
+    src_root = os.path.join(properties.Property('builddir'), 'llvm')
+    build_path = os.path.join(properties.Property('builddir'), 'build')
+
+    f = getLibcxxWholeTree(f, src_root)
+
+    # Nuke/remake build directory and run CMake
+    f.addStep(buildbot.steps.shell.ShellCommand(
+        name='rm.builddir', command=['rm', '-rf', build_path],
+        haltOnFailure=False, workdir=src_root))
+    f.addStep(buildbot.steps.shell.ShellCommand(
+        name='make.builddir', command=['mkdir', build_path],
+        haltOnFailure=True, workdir=src_root))
+
+    f.addStep(buildbot.steps.shell.ShellCommand(
+        name='cmake', command=['cmake', src_root], haltOnFailure=True,
+        workdir=build_path, env=env))
+
+    # Build libcxxabi
+    jobs_flag = properties.WithProperties('-j%(jobs)s')
+    f.addStep(buildbot.steps.shell.ShellCommand(
+              name='build.libcxxabi', command=['make', jobs_flag, 'cxxabi'],
+              haltOnFailure=True, workdir=build_path))
+
+    # Build libcxx
+    f.addStep(buildbot.steps.shell.ShellCommand(
+              name='build.libcxx', command=['make', jobs_flag, 'cxx'],
+              haltOnFailure=True, workdir=build_path))
+
+    # Test libc++abi
+    f.addStep(buildbot.steps.shell.ShellCommand(
+        name='test.libcxxabi', command=['make', 'check-libcxxabi'],
+        workdir=build_path))
+
+    # Test libc++
+    f.addStep(buildbot.steps.shell.ShellCommand(
+        name='test.libcxx', command=['make', 'check-libcxx'],
+        workdir=build_path))
+
+    return f