[llvm-exegesis] Improve test flexibility by using available_features

[kpdev42]

Summary

Currently, all llvm-exegesis regression tests are target-specific and
they are organized so that the entire subdirectories of
llvm/test/tools/llvm-exegesis are enabled or disabled as a whole.

After the commit e0ad2af691 ("Skip codegen" dry-run mode), at least two
kinds of tests are possible:

1. ensure that a snippet can be *generated* for the particular opcode
2. tests involving actual snippet execution inside the llvm-exegesis process

Thus, for the particular target subdirectory, some tests should run only
on the particular host architecture and other tests only need the target
being among LLVM_TARGETS_TO_BUILD.

This commit defines a bunch of exegesis-specific features that can be
referenced in REQUIRES, UNSUPPORTED, etc. on a test-by-test basis.

~~

Huawei RRI

[courbet]

This is great, thanks for the patch.

[stuij]

This was breaking for us because can_use_perf_counters() in lit.local.cfg couldn't find llvm-exegesis in the first place.

I made a fix: https://reviews.llvm.org/D146964

llvm/test/tools/llvm-exegesis/AArch64/latency-by-opcode-name.s

 # RUN: llvm-exegesis -mode=latency -opcode-name=ADDXrr | FileCheck %s
+# REQUIRES: exegesis-can-execute-aarch64, exegesis-can-measure-latency
 
 CHECK:      ---
 CHECK-NEXT: mode: latency
 CHECK-NEXT: key:
 CHECK-NEXT:   instructions:
 CHECK-NEXT:     ADDXrr [[REG1:X[0-9]+|LR]] [[REG2:X[0-9]+|LR]] [[REG3:X[0-9]+|LR]]
 CHECK-NEXT: config: ''
 CHECK-NEXT: register_initial_values:
 CHECK-DAG: - '[[REG2]]=0x0'
 # We don't check REG3 because in the case that REG2=REG3 the check would fail
 CHECK-LAST: ...

llvm/test/tools/llvm-exegesis/AArch64/lit.local.cfg

-import subprocess
-import lit.util
-
-if 'native' not in config.available_features:
-    config.unsupported = True
-
-elif not ('AArch64' in config.root.targets):
+if not ('AArch64' in config.root.targets):
     # We need support for AArch64.
     config.unsupported = True
-
-elif not ('aarch64' in config.root.host_triple):
-    # We need to be running on an AArch64 host.
-    config.unsupported = True
-
-else:
-    # We need libpfm to be installed and allow reading perf counters. We can
-    # only know that at runtime, so we try to measure the latency of an empty
-    # code snippet and bail out on error.
-    llvm_exegesis_exe = lit.util.which('llvm-exegesis', config.llvm_tools_dir)
-    if not llvm_exegesis_exe:
-        print('llvm-exegesis not found')
-        config.unsupported = True
-    else:
-      try:
-          with open(os.devnull, 'w') as quiet:
-              check_llvm_exegesis_result = subprocess.call(
-                [llvm_exegesis_exe, '-mode', 'latency', '-snippets-file', '/dev/null'], stdout=quiet, stderr=quiet)
-      except OSError:
-          print('could not exec llvm-exegesis')
-          config.unsupported = True
-      if not check_llvm_exegesis_result == 0:
-        config.unsupported = True

llvm/test/tools/llvm-exegesis/Mips/latency-GPR64.s

 # RUN: llvm-exegesis -mode=latency -opcode-name=AND64 | FileCheck %s
+# REQUIRES: exegesis-can-execute-mips, exegesis-can-measure-latency
 
 CHECK:      ---
 CHECK-NEXT: mode: latency
 CHECK-NEXT: key:
 CHECK-NEXT:   instructions:
 CHECK-NEXT:     AND64
 CHECK-NEXT: config: ''
 CHECK-NEXT: register_initial_values:
 CHECK-DAG: - '[[REG1:[A-Z0-9]+_64]]=0x0'
 CHECK-LAST: ...

llvm/test/tools/llvm-exegesis/Mips/latency-by-opcode-name.s

 # RUN: llvm-exegesis -mode=latency -opcode-name=ADD | FileCheck %s
+# REQUIRES: exegesis-can-execute-mips, exegesis-can-measure-latency
 
 CHECK:      ---
 CHECK-NEXT: mode: latency
 CHECK-NEXT: key:
 CHECK-NEXT:   instructions:
 CHECK-NEXT:     ADD
 CHECK-NEXT: config: ''
 CHECK-NEXT: register_initial_values:
 CHECK-DAG: - '[[REG1:[A-Z0-9]+]]=0x0'
 CHECK-LAST: ...

llvm/test/tools/llvm-exegesis/Mips/lit.local.cfg

-import subprocess
-import lit.util
-
-if 'native' not in config.available_features:
-    config.unsupported = True
-
-elif not ('Mips' in config.root.targets):
+if not ('Mips' in config.root.targets):
     # We need support for Mips.
     config.unsupported = True
-
-elif not ('mips' in config.root.host_triple):
-    # We need to be running on an Mips host.
-    config.unsupported = True
-
-else:
-    # We need libpfm to be installed and allow reading perf counters. We can
-    # only know that at runtime, so we try to measure the latency of an empty
-    # code snippet and bail out on error.
-    llvm_exegesis_exe = lit.util.which('llvm-exegesis', config.llvm_tools_dir)
-    if not llvm_exegesis_exe:
-        print('llvm-exegesis not found')
-        config.unsupported = True
-    else:
-      try:
-          with open(os.devnull, 'w') as quiet:
-              check_llvm_exegesis_result = subprocess.call(
-                [llvm_exegesis_exe, '-mode', 'latency', '-snippets-file', '/dev/null'], stdout=quiet, stderr=quiet)
-      except OSError:
-          print('could not exec llvm-exegesis')
-          config.unsupported = True
-      if not check_llvm_exegesis_result == 0:
-        config.unsupported = True

llvm/test/tools/llvm-exegesis/X86/lbr/lit.local.cfg

-import subprocess
-import lit.util
-
-if 'native' not in config.available_features:
-    config.unsupported = True
-
-elif not ('X86' in config.root.targets):
-    # We need support for X86.
-    config.unsupported = True
-
-elif not ('x86_64' in config.root.host_triple):
-    # We need to be running on an X86 host.
-    config.unsupported = True
-
-else:
-    # We need libpfm to be installed and the host to be support LBR format with cycles.
-    llvm_exegesis_exe = lit.util.which('llvm-exegesis', config.llvm_tools_dir)
-    if not llvm_exegesis_exe:
-        print('llvm-exegesis not found')
-        config.unsupported = True
-    else:
-      try:
-          with open(os.devnull, 'w') as quiet:
-              check_llvm_exegesis_latency_result = subprocess.call(
-                [llvm_exegesis_exe, '-mode', 'latency', '-x86-lbr-sample-period', '123', '-repetition-mode', 'loop', '-snippets-file', '/dev/null'], stdout=quiet, stderr=quiet)
-      except OSError:
-          print('could not exec llvm-exegesis')
-          config.unsupported = True
-      if not check_llvm_exegesis_latency_result == 0:
-        config.unsupported = True

llvm/test/tools/llvm-exegesis/X86/lbr/mov-add.s

 # RUN: llvm-exegesis -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mode=latency --repetition-mode=loop --x86-lbr-sample-period=521 --snippets-file=%p/Inputs/mov_add.att
+# REQUIRES: exegesis-can-execute-x86_64, exegesis-can-measure-latency-lbr
 
 
 CHECK:      ---
 CHECK-NEXT: mode: latency
 CHECK-NEXT: key:
 CHECK-NEXT:   instructions:
 CHECK-NEXT:     'MOV64ri32 RDI i_0x2'
 CHECK-NEXT:     'ADD64ri8 RDI RDI i_0x10'

llvm/test/tools/llvm-exegesis/lit.local.cfg

+import subprocess
+import lit.util
+
+
+def can_execute_generated_snippets(arch):
+    is_host_arch = arch in config.root.host_triple
+    # 'native' feature is defined as "host arch == default triple arch"
+    is_native_codegen = 'native' in config.available_features
+    return is_host_arch and is_native_codegen
+
+
+def can_use_perf_counters(mode, extra_options=[]):
+    # We need libpfm to be installed and allow reading perf counters. We can
+    # only know that at runtime, so we try to measure an empty code snippet
+    # and bail out on error.
+    llvm_exegesis_exe = lit.util.which('llvm-exegesis', config.llvm_tools_dir)
+    try:
+        return_code = subprocess.call(
+            [llvm_exegesis_exe, '-mode', mode, '-snippets-file', '/dev/null'] + extra_options,
+            stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        return return_code == 0
+    except OSError:
+        print('could not exec llvm-exegesis')
+        return False
+
+
+for arch in ['aarch64', 'mips', 'powerpc', 'x86_64']:
+    if can_execute_generated_snippets(arch):
+        config.available_features.add('exegesis-can-execute-%s' % arch)
+
+if can_use_perf_counters('latency'):
+    config.available_features.add('exegesis-can-measure-latency')
+
+if can_use_perf_counters('uops'):
+    config.available_features.add('exegesis-can-measure-uops')
+
+if can_execute_generated_snippets('x86_64'):
+    # Check for support of LBR format with cycles.
+    if can_use_perf_counters('latency', ['-x86-lbr-sample-period', '123', '-repetition-mode', 'loop']):
+        config.available_features.add('exegesis-can-measure-latency-lbr')