[zorg] Enable running 'lnt runtest test-suite' instead of 'lnt runtest nt'.

[kristof.beyls]

Summary

With this patch, a typical configuration for a builder using 'lnt runtest test-suite' to track performance
of code generated by LLVM could be:

{

'name': "builder-name",
'slavenames': ["slave-name"],
'builddir': "builder-builddir",
'factory': ClangBuilder.getClangCMakeBuildFactory(
              jobs=8,
              clean=False,
              checkout_compiler_rt=False,
              checkout_lld=False,
              test=False,
              useTwoStage=False,
              runTestSuite=True,
              env={'PATH':'/usr/lib/ccache:/usr/local/sbin:'+
                          '/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'},
              testsuite_flags=['--cppflags', '-O3',
                               '--threads=1',
                               '--build-threads=8',
                               '--use-perf=all',
                               '--run-under', 'taskset -c 1',
                               '--benchmarking-only',
                               '--exec-multisample=3',
                               '--exclude-stat-from-submission=compile'],
              extra_cmake_args=["-DLLVM_TARGETS_TO_BUILD='AArch64'",
                                "-DLLVM_PARALLEL_LINK_JOBS=4"],
              submitURL='http://lnt.llvm.org/submitRun',
              testerName='LNT-AArch64-O3'),
'category': 'clang'

}

The only difference to using the somewhat deprecated 'lnt runtest nt'
is that a "testsuite_flags" parameter is defined instead of an
"nt_flags" parameter.

[rovka]

Hi, I tested this on our internal buildmaster and it seems to work. LGTM.

zorg/buildbot/builders/ClangBuilder.py

779

Could you throw an error or assert or something to make sure people don't pass in both nt_flags and testsuite_flags?

[kristof.beyls]

zorg/buildbot/builders/ClangBuilder.py

779

I tried adding

assert len(nt_flags) == 0 or len(testsuite_flags) == 0

but when you let it trigger by a bot defining both, you get an assertion failure when trying to start up the buildbot master.
That means that Galina or @vleschuk would fail to start the buildbot master if just one slave configuration triggered the assert.
Overall, it feels to me that it's better to not assert on the buildbot master, and put the onus on the bot owners to make sure their bot does what they intend them to do.

zorg/buildbot/builders/ClangBuilder.py

def getClangCMakeGCSBuildFactory(
             useTwoStage=False,
             testStage1=True,
             stage1_config='Release',
             stage2_config='Release',
 
             # Test-suite
             runTestSuite=False,
             nt_flags=[],
+            testsuite_flags=[],
             submitURL=None,
             testerName=None,
 
             # Environmental variables for all steps.
             env={},
             extra_cmake_args=[],
 
             # Extra repositories
             checkout_clang_tools_extra=True,
             checkout_compiler_rt=True,
             checkout_lld=True,
 
             # Upload artifacts to Google Cloud Storage (for the llvmbisect tool)
             stage1_upload_directory=None,
 
             # Triggers
             trigger_after_stage1=[]):
     return _getClangCMakeBuildFactory(
                clean=clean, test=test, cmake=cmake, jobs=jobs, vs=vs,
                vs_target_arch=vs_target_arch, useTwoStage=useTwoStage,
                testStage1=testStage1, stage1_config=stage1_config,
                stage2_config=stage2_config, runTestSuite=runTestSuite,
-               nt_flags=nt_flags, submitURL=submitURL, testerName=testerName,
+               nt_flags=nt_flags, testsuite_flags=testsuite_flags,
+               submitURL=submitURL, testerName=testerName,
                env=env, extra_cmake_args=extra_cmake_args,
                checkout_clang_tools_extra=checkout_clang_tools_extra,
                checkout_compiler_rt=checkout_compiler_rt,
                checkout_lld=checkout_lld,
                stage1_upload_directory=stage1_upload_directory,
                trigger_after_stage1=trigger_after_stage1)
 
 def getClangCMakeBuildFactory(
def getClangCMakeBuildFactory(
             useTwoStage=False,
             testStage1=True,
             stage1_config='Release',
             stage2_config='Release',
 
             # Test-suite
             runTestSuite=False,
             nt_flags=[],
+            testsuite_flags=[],
             submitURL=None,
             testerName=None,
 
             # Environmental variables for all steps.
             env={},
             extra_cmake_args=[],
 
             # Extra repositories
             checkout_clang_tools_extra=True,
             checkout_compiler_rt=True,
             checkout_lld=True,
             checkout_libcxx=False,
             checkout_test_suite=False):
     return _getClangCMakeBuildFactory(
                clean=clean, test=test, cmake=cmake, jobs=jobs, vs=vs,
                vs_target_arch=vs_target_arch, useTwoStage=useTwoStage,
                testStage1=testStage1, stage1_config=stage1_config,
                stage2_config=stage2_config, runTestSuite=runTestSuite,
-               nt_flags=nt_flags, submitURL=submitURL, testerName=testerName,
+               nt_flags=nt_flags, testsuite_flags=testsuite_flags,
+               submitURL=submitURL, testerName=testerName,
                env=env, extra_cmake_args=extra_cmake_args,
                checkout_clang_tools_extra=checkout_clang_tools_extra,
                checkout_lld=checkout_lld,
                checkout_compiler_rt=checkout_compiler_rt,
                checkout_libcxx=checkout_libcxx,
                checkout_test_suite=checkout_test_suite)
 
 def _getClangCMakeBuildFactory(
def _getClangCMakeBuildFactory(
             useTwoStage=False,
             testStage1=True,
             stage1_config='Release',
             stage2_config='Release',
 
             # Test-suite
             runTestSuite=False,
             nt_flags=[],
+            testsuite_flags=[],
             submitURL=None,
             testerName=None,
 
             # Environmental variables for all steps.
             env={},
             extra_cmake_args=[],
 
             # Extra repositories
if runTestSuite:
         sandbox = WithProperties('%(workdir)s/test/sandbox')
         test_suite_dir = WithProperties('%(workdir)s/test/test-suite')
 
         # Get latest built Clang (stage1 or stage2)
         cc = WithProperties('%(workdir)s/'+compiler_path+'/bin/'+cc)
         cxx = WithProperties('%(workdir)s/'+compiler_path+'/bin/'+cxx)
 
         # LNT Command line (don't pass -jN. Users need to pass both --threads
-        # and --build-threads in nt_flags to get the same effect)
+        # and --build-threads in nt_flags/test_suite_flags to get the same effect)
+        use_runtest_testsuite = len(nt_flags) == 0
+        if not use_runtest_testsuite:
             test_suite_cmd = [python, lnt, 'runtest', 'nt',
                               '--no-timestamp',
                               '--sandbox', sandbox,
                               '--test-suite', test_suite_dir,
                               '--cc', cc,
                               '--cxx', cxx]
             # Append any option provided by the user
             test_suite_cmd.extend(nt_flags)
+        else:
+            lit = WithProperties('%(workdir)s/'+stage1_build+'/bin/llvm-lit')
+            test_suite_cmd = [python, lnt, 'runtest', 'test-suite',
+                              '--no-timestamp',
+                              '--sandbox', sandbox,
+                              '--test-suite', test_suite_dir,
+                              '--cc', cc,
+                              '--cxx', cxx,
+                              '--use-lit', lit]
+            # Append any option provided by the user
+            test_suite_cmd.extend(testsuite_flags)
+
         # Only submit if a URL has been specified
         if submitURL is not None:
             if not isinstance(submitURL, list):
                 submitURL = [submitURL]
             for url in submitURL:
                 test_suite_cmd.extend(['--submit', url])
-            if testerName:
+            # lnt runtest test-suite doesn't understand --no-machdep-info:
+            if testerName and not use_runtest_testsuite:
                 test_suite_cmd.extend(['--no-machdep-info', testerName])
         # CC and CXX are needed as env for build-tools
         test_suite_env = copy.deepcopy(env)
         test_suite_env['CC'] = cc
         test_suite_env['CXX'] = cxx
 
         # Steps to prepare, build and run LNT
         f.addStep(ShellCommand(name='clean sandbox',