[llvm-exegesis] Add lit tests (v2).

[courbet]

Summary

This revisits rL342953 by adding detection of host support.

[courbet]

Swallow normal output on success.

[lebedev.ri]

That looks saner to me.
I assume you have tried running this, and it avoids failing tests, as expected?

[gchatelet]

test/tools/llvm-exegesis/X86/lit.local.cfg

15

"and bail out on error" ?

[courbet]

In D52464#1244779, @lebedev.ri wrote:

That looks saner to me.
I assume you have tried running this, and it avoids failing tests, as expected?

Yes :)

[courbet]

bail -> bail out

test/CMakeLists.txt

set(LLVM_TEST_DEPENDS
           llvm-cxxfilt
           llvm-cxxmap
           llvm-diff
           llvm-dis
           llvm-dlltool
           dsymutil
           llvm-dwarfdump
           llvm-dwp
+          llvm-exegesis
           llvm-extract
           llvm-isel-fuzzer
           llvm-lib
           llvm-link
           llvm-lto2
           llvm-mc
           llvm-mca
           llvm-modextract

test/lit.cfg.py

tools = [
     ToolSubst('%ocamlopt', ocamlopt_command, unresolved='ignore'),
     ToolSubst('%opt-viewer', opt_viewer_cmd),
     ToolSubst('%llvm-objcopy', FindTool('llvm-objcopy')),
     ToolSubst('%llvm-strip', FindTool('llvm-strip')),
 ]
 
 # FIXME: Why do we have both `lli` and `%lli` that do slightly different things?
 tools.extend([
-    'dsymutil', 'lli', 'lli-child-target', 'llvm-ar', 'llvm-as', 'llvm-bcanalyzer',
-    'llvm-config', 'llvm-cov', 'llvm-cxxdump', 'llvm-cvtres', 'llvm-diff', 'llvm-dis',
-    'llvm-dwarfdump', 'llvm-extract', 'llvm-isel-fuzzer', 'llvm-opt-fuzzer', 'llvm-lib',
-    'llvm-link', 'llvm-lto', 'llvm-lto2', 'llvm-mc', 'llvm-mca',
-    'llvm-modextract', 'llvm-nm', 'llvm-objcopy', 'llvm-objdump',
-    'llvm-pdbutil', 'llvm-profdata', 'llvm-ranlib', 'llvm-readobj',
-    'llvm-rtdyld', 'llvm-size', 'llvm-split', 'llvm-strings', 'llvm-strip', 'llvm-tblgen',
-    'llvm-undname', 'llvm-c-test', 'llvm-cxxfilt', 'llvm-xray', 'yaml2obj', 'obj2yaml',
-    'yaml-bench', 'verify-uselistorder',
-    'bugpoint', 'llc', 'llvm-symbolizer', 'opt', 'sancov', 'sanstats'])
+    'dsymutil', 'lli', 'lli-child-target', 'llvm-ar', 'llvm-as',
+    'llvm-bcanalyzer', 'llvm-config', 'llvm-cov', 'llvm-cxxdump', 'llvm-cvtres',
+    'llvm-diff', 'llvm-dis', 'llvm-dwarfdump', 'llvm-exegesis', 'llvm-extract',
+    'llvm-isel-fuzzer', 'llvm-opt-fuzzer', 'llvm-lib', 'llvm-link', 'llvm-lto',
+    'llvm-lto2', 'llvm-mc', 'llvm-mca', 'llvm-modextract', 'llvm-nm',
+    'llvm-objcopy', 'llvm-objdump', 'llvm-pdbutil', 'llvm-profdata',
+    'llvm-ranlib', 'llvm-readobj', 'llvm-rtdyld', 'llvm-size', 'llvm-split',
+    'llvm-strings', 'llvm-strip', 'llvm-tblgen', 'llvm-undname', 'llvm-c-test',
+    'llvm-cxxfilt', 'llvm-xray', 'yaml2obj', 'obj2yaml', 'yaml-bench',
+    'verify-uselistorder', 'bugpoint', 'llc', 'llvm-symbolizer', 'opt',
+    'sancov', 'sanstats'])
 
 # The following tools are optional
 tools.extend([
     ToolSubst('llvm-go', unresolved='ignore'),
     ToolSubst('llvm-mt', unresolved='ignore'),
     ToolSubst('Kaleidoscope-Ch3', unresolved='ignore'),
     ToolSubst('Kaleidoscope-Ch4', unresolved='ignore'),
     ToolSubst('Kaleidoscope-Ch5', unresolved='ignore'),

test/tools/llvm-exegesis/X86/latency-by-opcode-name.s

+# RUN: llvm-exegesis -mode=latency -opcode-name=ADD32rr | FileCheck %s
+
+CHECK:      mode:            latency
+CHECK-NEXT: key:
+CHECK-NEXT:   instructions:
+CHECK-NEXT:     ADD32rr

test/tools/llvm-exegesis/X86/lit.local.cfg

+import subprocess
+import lit.util
+
+if not ('X86' in config.root.targets):
+    # We need support for X86.
+    config.unsupported = True
+
+elif not ('x86_64' in config.root.host_triple):
+    # We need to be running on an X86 host.
+    config.unsupported = True
+
+else:
+    # We need libpfm to be installed and allow reading perf counters. We can
+    # only know that at runtime, so we try to measure the latency of an empty
+    # code snippet and bail on error.
+    llvm_exegesis_exe = lit.util.which('llvm-exegesis', config.llvm_tools_dir)
+    if not llvm_exegesis_exe:
+        print('llvm-exegesis not found')
+        config.unsupported = True
+    else:
+      try:
+          with open(os.devnull, 'w') as quiet:
+              check_llvm_exegesis_result = subprocess.call(
+                [llvm_exegesis_exe, '-mode', 'uops', '-snippets-file', '/dev/null'], stdout=quiet)
+      except OSError:
+          print('could not exec llvm-exegesis')
+          config.unsupported = True
+      if not check_llvm_exegesis_result == 0:
+        config.unsupported = True

test/tools/llvm-exegesis/X86/uops-by-opcode-name.s

+# RUN: llvm-exegesis -mode=uops -opcode-name=ADD32rr | FileCheck %s
+
+CHECK:      mode:            uops
+CHECK-NEXT: key:
+CHECK-NEXT:   instructions:
+CHECK-NEXT:     ADD32rr

test/tools/llvm-exegesis/lit.local.cfg

+if 'native' not in config.available_features:
+    config.unsupported = True

tools/llvm-exegesis/lib/BenchmarkRunner.cpp

BenchmarkRunner::BenchmarkRunner(const LLVMState &State,
     : State(State), Mode(Mode), Scratch(llvm::make_unique<ScratchSpace>()) {}
 
 BenchmarkRunner::~BenchmarkRunner() = default;
 
 // Repeat the snippet until there are at least MinInstructions in the resulting
 // code.
 static std::vector<llvm::MCInst>
 GenerateInstructions(const BenchmarkCode &BC, const size_t MinInstructions) {
+  if (BC.Instructions.empty())
+    return {};
   std::vector<llvm::MCInst> Code = BC.Instructions;
   for (int I = 0; Code.size() < MinInstructions; ++I)
     Code.push_back(BC.Instructions[I % BC.Instructions.size()]);
   return Code;
 }
 
 InstructionBenchmark
 BenchmarkRunner::runConfiguration(const BenchmarkCode &BC,
                                   unsigned NumRepetitions) const {
   InstructionBenchmark InstrBenchmark;
   InstrBenchmark.Mode = Mode;
   InstrBenchmark.CpuName = State.getTargetMachine().getTargetCPU();
   InstrBenchmark.LLVMTriple =
       State.getTargetMachine().getTargetTriple().normalize();
   InstrBenchmark.NumRepetitions = NumRepetitions;
   InstrBenchmark.Info = BC.Info;
 
   const std::vector<llvm::MCInst> &Instructions = BC.Instructions;
-  if (Instructions.empty()) {
-    InstrBenchmark.Error = "Empty snippet";
-    return InstrBenchmark;
-  }
 
   InstrBenchmark.Key.Instructions = Instructions;
 
   // Assemble at least kMinInstructionsForSnippet instructions by repeating the
   // snippet for debug/analysis. This is so that the user clearly understands
   // that the inside instructions are repeated.
   constexpr const int kMinInstructionsForSnippet = 16;
   {