Add android test to darwin builder

[chying]

Summary

• Add remote test to andriod-arm
• Clean SVN tree for lldb/test folder
• Remote -f option when cleaning testing folder on Android, as it's not supported on API 16
• Add step to get tblgen binaries path, it's different from lldb binary when built with xcode

zorg/trunk/buildbot/osuosl/master/config/builders.py

#         'category' : 'clang'},
                     local_test_archs=['x86_64', 'i386'],
                     local_test_compilers=['clang', 'totclang', 'gcc4.8.2', 'gcc4.9.2'],
                     #remote_configs=[RemoteConfig('linux','x86_64',['clang'],['x86_64'])],
                     env={'SHELL':"/bin/bash"})},
         {'name': "lldb-x86_64-darwin-13.4",
          'slavenames': ["lldb-x86_64-darwin-13.4"],
          'builddir': "buildDir",
          'category' : 'lldb',
-         'factory': LLDBBuilder.getLLDBxcodebuildFactory()},
+         'factory': LLDBBuilder.getLLDBxcodebuildFactory(
+                    build_type='Release',
+                    remote_configs=[RemoteConfig('android', 'arm', ['arm-linux-androideabi-gcc'], ['arm'])],
+                    env={'SHELL':"/bin/bash"})},
         {'name': "lldb-x86_64-ubuntu-14.04-android",
          'slavenames': ["lldb-lab-linux01"],
          'builddir': "lldb-cross-compile",
          'category' : 'lldb',
          'factory': LLDBBuilder.getLLDBUbuntuCMakeBuildFactory(
                     build_compiler="clang",
                     build_type="Release",
                     remote_configs= [

zorg/trunk/zorg/buildbot/builders/LLDBBuilder.py

for compiler in test_compilers:
                                               WithProperties(DOTEST_OPTS)],
                                      description="test lldb",
                                      parseSummaryOnly=True,
                                      flunkOnFailure=flunkTestFailure,
                                      warnOnFailure=flunkTestFailure,
                                      workdir='%s' % llvm_builddir,
                                      timeout=1800,
                                      env=testenv))
-            f=cleanSVNSourceTree(f, '%s/tools/lldb' % llvm_srcdir)
+            f=cleanSVNSourceTree(f, '%s/tools/lldb/test' % llvm_srcdir)
     return f
 
 # Define a structure to describe remote target
 # For example, RemoteConfig('linux','x86_64',['gcc4.8.2','clang'],['i386'])
 RemoteConfig = collections.namedtuple("RemoteConfig", ["platform", "host_arch", "test_compilers", "test_archs"])
 
 # Add steps to run lldb test on remote target
 def getLLDBRemoteTestSteps(f,
f.addStep(SetProperty(name="get remote target " + stepDesc,
                           workdir="."))
     # rsync
     if remote_config.platform is 'linux':
         shellcmd = ['ssh',
                     WithProperties('%(remote_host)s')]
         hostname = '%(slave_hostname)s'
         launchcmd = shellcmd + ['screen', '-d', '-m']
         terminatecmd = shellcmd + ['pkill', 'lldb-server']
-        cleandircmd = WithProperties('ssh %(remote_host)s rm -rf %(remote_dir)s/*')
+        cleandircmd = WithProperties('ssh %(remote_host)s rm -r %(remote_dir)s/*')
         f.addStep(ShellCommand(name="rsync lldb-server",
                                command=['rsync',
                                         '-havL',
                                         'bin/lldb-server',
                                         WithProperties('%(remote_host)s:%(remote_dir)s')],
                                description="rsync lldb-server " + stepDesc,
                                haltOnFailure=True,
                                env=env,
def getLLDBUbuntuCMakeBuildFactory(build_compiler,
     f = buildbot.process.factory.BuildFactory()
 
     # Determine the build directory.
     f.addStep(SetProperty(name="get_builddir",
                           command=["pwd"],
                           property="builddir",
                           description="set build dir",
                           workdir="."))
+    # Determine the binary directory of *-tblgen.
+    f.addStep(SetProperty(name="get tblgen dir",
+                          command=["echo", WithProperties(bindir)],
+                          property="tblgen_bindir",
+                          description="set tblgen dir",
+                          workdir="."))
     # Get source code
     f = getLLDBSource(f,llvm_srcdir)
 
     # Clean Build Folder
     f.addStep(ShellCommand(name="clean",
                            command="rm -rf *",
                            description="clear build folder",
                            env=env,
def getLLDBAndroidCMakeStep(f,
     cmake_args.append(WithProperties("-DCMAKE_BUILD_TYPE=%s" % build_type))
     cmake_args.append(WithProperties('%(builddir)s/' + llvm_srcdir))
     cmake_args.append(WithProperties('-DCMAKE_TOOLCHAIN_FILE=' + '%(builddir)s/' + llvm_srcdir + '/tools/lldb/cmake/platforms/Android.cmake'))
     cmake_args.append(WithProperties('-DANDROID_TOOLCHAIN_DIR=' + '%(toolchain)s'))
     cmake_args.append('-DANDROID_ABI=' + abiMap[target_arch])
     cmake_args.append('-DCMAKE_CXX_COMPILER_VERSION=4.9')
     cmake_args.append('-DLLVM_TARGET_ARCH=' + target_arch)
     cmake_args.append('-DLLVM_HOST_TRIPLE=' + target_arch + '-unknown-linux-android')
-    cmake_args.append(WithProperties('-DLLVM_TABLEGEN=' + bindir + '/llvm-tblgen'))
-    cmake_args.append(WithProperties('-DCLANG_TABLEGEN=' + bindir + '/clang-tblgen'))
+    cmake_args.append(WithProperties('-DLLVM_TABLEGEN=%(tblgen_bindir)s/llvm-tblgen'))
+    cmake_args.append(WithProperties('-DCLANG_TABLEGEN=%(tblgen_bindir)s/clang-tblgen'))
 
     f.addStep(Configure(name='cmake-android-%s' % target_arch,
                         command=cmake_args,
                         env=env,
                         haltOnFailure=True,
                         workdir=llvm_builddir))
     return f
+# Set symbolic links, so the folder structure will be llvm, llvm/tools/clang, llvm/tools/lldb
+def getSymbLinkSteps(f, lldb_srcdir):
+    f.addStep(ShellCommand(name='set symbolic link clang',
+                           command=['ln', '-nfs',
+                                    WithProperties('%(builddir)s/' + lldb_srcdir + '/llvm/tools/clang'),
+                                    'clang'],
+                           workdir=WithProperties('%(builddir)s')))
+    f.addStep(ShellCommand(name='set symbolic link lldb',
+                           command=['ln', '-nfs',
+                                    WithProperties('%(builddir)s/' + lldb_srcdir),
+                                    lldb_srcdir + '/llvm/tools/lldb'],
+                           workdir=WithProperties('%(builddir)s')))
+    f.addStep(ShellCommand(name='set symbolic link llvm',
+                           command=['ln', '-nfs',
+                                    WithProperties('%(builddir)s/' + lldb_srcdir + '/llvm'),
+                                    'llvm'],
+                           workdir=WithProperties('%(builddir)s')))
+    return f
 
-def getLLDBxcodebuildFactory(use_cc=None):
+def getLLDBxcodebuildFactory(use_cc=None,
+                             build_type='Debug',
+                             remote_configs=None,
+                             env=None):
+    if env is None:
+        env = {}
     f = buildbot.process.factory.BuildFactory()
     f.addStep(SetProperty(name='get_builddir',
                           command=['pwd'],
                           property='builddir',
                           description='set build dir',
                           workdir='.'))
-    lldb_srcdir = 'lldb.src'
+    lldb_srcdir = 'lldb'
     OBJROOT='%(builddir)s/' + lldb_srcdir + '/build'
     f.addStep(SetProperty(name='get_bindir',
                           command=['echo',
-                                   WithProperties('%(builddir)s/' + lldb_srcdir + '/build/Debug')],
+                                   WithProperties('%(builddir)s/' + lldb_srcdir + '/build/' + build_type)],
                           property='lldb_bindir',
                           description='set bin dir',
                           workdir='.'))
     # cleaning out the build directory is vital for codesigning.
     f.addStep(ShellCommand(name='clean.lldb-buid',
                            command=['rm', '-rf', WithProperties(OBJROOT)],
                            haltOnFailure=True,
                            workdir=WithProperties('%(builddir)s')))
     f.addStep(ShellCommand(name='clean.llvm-buid',
                            command=['rm', '-rf', '%s/llvm-build' % lldb_srcdir ],
                            haltOnFailure=True,
                            workdir=WithProperties('%(builddir)s')))
     f.addStep(ShellCommand(name='clean.test trace',
-                           command=['rm', '-rf', '%s/DerivedData/lldb-test-results' % lldb_srcdir ],
+                           command='rm -rf %s/build/*' % lldb_srcdir,
                            haltOnFailure=True,
                            workdir=WithProperties('%(builddir)s')))
     f.addStep(SVN(name='svn-lldb',
                   mode='update',
                   baseURL='http://llvm.org/svn/llvm-project/lldb/',
                   defaultBranch='trunk',
                   workdir=lldb_srcdir))
     f.addStep(SVN(name='svn-llvm',
                   mode='update',
                   baseURL='http://llvm.org/svn/llvm-project/llvm/',
                   defaultBranch='trunk',
                   workdir='%s/llvm' % lldb_srcdir))
     f.addStep(SVN(name='svn-clang',
                   mode='update',
                   baseURL='http://llvm.org/svn/llvm-project/cfe/',
                   defaultBranch='trunk',
                   workdir='%s/llvm/tools/clang' % lldb_srcdir))
 # setup keychain for codesign
 # In order for the codesigning to work inside of buildbot, security must be
 # called to unlock the keychain, which requires a password.
 # I've set up a special keychain for this purpose, so as to not compromise
 # the login password of the buildslave.
 # This means I have to set the special keychain as the default and unlock it
 # prior to building the sources.
-
     f.addStep(ShellCommand(name='check.keychain',
                            command=['security', 'default-keychain'],
                            haltOnFailure=True,
                            workdir=WithProperties('%(builddir)s')))
     f.addStep(ShellCommand(name='find.certificate',
                            command=['security', 'find-certificate', '-c',
                                     'lldb_codesign'],
                            haltOnFailure=True,
                            workdir=WithProperties('%(builddir)s')))
 # Building the sources
 #
     buildcmd = ' '.join(['xcrun',
                          'xcodebuild',
                          '-workspace',
                          'lldb.xcworkspace',
                          '-scheme',
                          'lldb-tool',
                          '-configuration',
-                         'Debug',
+                         build_type,
                          'SYMROOT=' + OBJROOT,
                          'OBJROOT=' + OBJROOT])
     f.addStep(ShellCommand(name='lldb-build',
                            command=WithProperties(buildcmd + " || " + buildcmd),
                            haltOnFailure=True,
                            workdir=lldb_srcdir))
 
 # Testing
else:
                   description='set cc',
                   workdir=lldb_srcdir))
     DOTEST_OPTS = ' '.join(['-m',
                             '--executable',
                             '%(lldb_bindir)s/lldb',
                             '--framework', '%(lldb_bindir)s/LLDB.framework',
                             '-A', 'x86_64',
                             '-C', 'clang',
-                            '-s', '../DerivedData/lldb-test-results'])
+                            '-s', '../../build/lldb-test-traces'])
     f.addStep(LitTestCommand(name='lldb-test',
                              command=['./dosep.py',
                                       '--options',
                                       WithProperties(DOTEST_OPTS)],
                              haltOnFailure=False,
                              workdir='%s/test' % lldb_srcdir,
                              env={'DYLD_FRAMEWORK_PATH' : WithProperties('%(lldb_bindir)s')}))
+# Remote test steps
+    if remote_configs is not None:
+        # Source structure to use cmake command
+        f.addStep(SetProperty(name='get tblgen bindir',
+                              command=['echo',
+                                       WithProperties('%(builddir)s/' + lldb_srcdir + '/llvm-build/Release+Asserts/x86_64/Release+Asserts/bin')],
+                              property='tblgen_bindir',
+                              description='set tblgen binaries dir',
+                              workdir='.'))
+        f = getSymbLinkSteps(f, lldb_srcdir)
+        for config in remote_configs:
+            f = getLLDBRemoteTestSteps(f,
+                                       '%(lldb_bindir)s',
+                                       build_type,
+                                       config,
+                                       env={'DYLD_FRAMEWORK_PATH' : WithProperties('%(lldb_bindir)s')})
+        # Remove symbolic link to lldb, otherwise xcodebuild will have circular dependency in next build
+        f.addStep(ShellCommand(name='remove symbolic link lldb',
+                               command=['rm',
+                                        lldb_srcdir + '/llvm/tools/lldb'],
+                               haltOnFailure=False,
+                               flunkOnFailure=False,
+                               workdir=WithProperties('%(builddir)s')))
 # Compress and upload test log
-    f = archiveLLDBTestTraces(f, "lldb.src/DerivedData/lldb-test-results")
+    f = archiveLLDBTestTraces(f, "build/lldb-test-traces*")
 
 # Results go in a directory coded named according to the date and time of the test run, e.g.:
 #
 # 2012-10-16-11_26_48/Failure-x86_64-_Applications_Xcode.app_Contents_Developer_Toolchains_XcodeDefault.xctoolchain_usr_bin_clang-TestLogging.LogTestCase.test_with_dsym.log
 #
 # Possible results are ExpectedFailure, Failure, SkippedTest, UnexpectedSuccess, and Error.    return f
     return f