[buildbot] Updated CUDA buildbot config.

[tra]

Summary

• use default clang version for builds.
• build tests for sm_35 and sm_61
• run tests on multiple GPUS
• Run tests per each GPU specified in gpu_devices with only that GPU visible. This way tests don't have to implement any loop-over-all-visible-GPUs logic.

[gkistanova]

LGTM.
There is a cosmetic issue, but your change seems in line with the rest of the file, so, not sure if you would want to do this just for 2 ninja commands you are adding, or address it everywhere in this file as a separate patch.

zorg/buildbot/builders/CUDATestsuiteBuilder.py

157 ↗

(On Diff #132466)

Here and in all the similar cases. It is better to have the description as an array of words, as it helps buildbot to format multiline text for compact views.

[tra]

zorg/buildbot/builders/CUDATestsuiteBuilder.py

157 ↗

(On Diff #132466)

Would string.split("some long description goes here") do ?
I'll do it in another patch.

[tra]

zorg/trunk/buildbot/osuosl/master/config/builders.py

1379

@gkistanova This chould've been clang++.
I've fixed the error in rL324267.

zorg/trunk/buildbot/osuosl/master/config/builders.py

return [
          'slavenames' : ["cuda-build-test-01"],
          'builddir' : "clang-cuda-build",
          'factory' : CUDATestsuiteBuilder.getCUDATestsuiteBuildFactory(
                      useTwoStage=False,
                      test=True,
                      stage1_config='Release',
                      extra_cmake_args=[
                          '-DLLVM_ENABLE_ASSERTIONS=ON',
-                         "-DCMAKE_C_COMPILER:FILEPATH=/usr/bin/clang-3.8",
-                         "-DCMAKE_CXX_COMPILER:FILEPATH=/usr/bin/clang++-3.8"
+                         "-DCMAKE_C_COMPILER:FILEPATH=/usr/bin/clang",
+                         "-DCMAKE_CXX_COMPILER:FILEPATH=/usr/bin/clang"
                      ],
                      externals="/home/botanist/bots/externals",
-                     gpu_arch_list=["sm_35"],
-                     gpu_devices=[0],   # K40c.
+                     gpu_arch_list=["sm_35", "sm_61"],
+                     gpu_devices=[2, 0], # K40c(sm_35), GTX1070(sm_61)
                      extra_ts_cmake_args=[],
                      enable_thrust_tests=False,
          ),
          'category' : 'clang'},
 
         # lldb builders
         {'name': "lldb-x86_64-darwin-13.4",
          'slavenames': ["lldb-x86_64-darwin-13.4"],

zorg/trunk/zorg/buildbot/builders/CUDATestsuiteBuilder.py

f = ClangBuilder.getClangCMakeBuildFactory(
             checkout_test_suite=True)
 
     cuda_test_env = {
         'PYTHONPATH': WithProperties("%(workdir)s/" + source_dir +
                                      "/utils/lit:${PYTHONPATH}"),
         'PATH': WithProperties("%(workdir)s/" + clang_build_dir +
                                "/bin:${PATH}"),
     }
-    # Limit GPUs visible to CUDA.
-    if gpu_devices:
-        cuda_test_env["CUDA_VISIBLE_DEVICES"] = ",".join(
-            str(x) for x in gpu_devices)
     merged_env.update(cuda_test_env)
-
     ts_build_dir = 'test-suite-build'
 
     # Completely remove test suite build dir.
     f.addStep(
         RemoveDirectory(name="Remove old test-suite build directory",
                         dir=ts_build_dir))
 
     if extra_ts_cmake_args:
f.addStep(NinjaCommand(
         targets=["cuda-tests-simple"],
         jobs=jobs,
         haltOnFailure=True,
         description=["building simple CUDA tests"],
         descriptionDone=["simple CUDA tests built."],
         workdir=ts_build_dir,
         env=merged_env))
 
+    # Limit GPUs visible to CUDA.
+    if gpu_devices:
+        for gpu_id in gpu_devices:
+            # make ID a string as it may be either an integer or a UUID string.
+            gpu_id = str(gpu_id)
+            gpu_env = dict(merged_env)
+            gpu_env["CUDA_VISIBLE_DEVICES"] = gpu_id
+            f.addStep(NinjaCommand(
+                name='run simple CUDA tests on gpu %s' % gpu_id,
+                targets=["check-cuda-simple"],
+                jobs=1, # lit will parallelize the jobs
+                haltOnFailure=True,
+                description=["Running simple CUDA tests on GPU %s" % gpu_id],
+                descriptionDone=["simple CUDA tests on GPU %s done." % gpu_id],
+                workdir=ts_build_dir,
+                env=gpu_env))
+    else:
         f.addStep(NinjaCommand(
             name='run simple CUDA tests',
             targets=["check-cuda-simple"],
             jobs=1, # lit will parallelize the jobs
             haltOnFailure=True,
             description=["Running simple CUDA tests"],
             descriptionDone=["simple CUDA tests done."],
             workdir=ts_build_dir,
             env=merged_env))
 
     # If we've enabled thrust tests, build them now.
     # WARNING: This takes a lot of time to build.
     if (enable_thrust_tests):
         f.addStep(NinjaCommand(
             name='ninja build thrust',
             targets=["cuda-tests-thrust"],
             jobs=jobs,