[update_(llc_)test_checks.py] Support pre-processing commands

[arichardson]

Summary

This has been rather useful in our downstream CHERI target where we want
to run tests both with addrspace(0) and addrspace(200) pointers.
With this patch we can prefix the opt command with
sed -e 's/addrspace(200)/addrspace(0)/g' -e 's/-A200-P200-G200//g' to
test both cases using the same IR input.

[arichardson]

Add tests and also allow it in update_llc_test_checks.py

[jdoerfert]

LGTM. Maybe wait a bit if someone else wants to take a look.

Now we also need to execute RUN lines without FileCheck and we are almost there ;)

[lebedev.ri]

Is sed even allowed to be used in tests? Is it available in the required tool set?

[jrtc27]

In D95137#2515647, @lebedev.ri wrote:

Is sed even allowed to be used in tests? Is it available in the required tool set?

There are lots of uses in llvm/test/CodeGen/ for various architectures already.

[lebedev.ri]

In D95137#2515669, @jrtc27 wrote:

In D95137#2515647, @lebedev.ri wrote:

Is sed even allowed to be used in tests? Is it available in the required tool set?

There are lots of uses in llvm/test/CodeGen/ for various architectures already.

Touché.
I'm uneasy about such tests, but the UpdateTestChecks is fine i guess.

[arichardson]

rebase after latest changes

[arichardson]

• fix pre-processing test after output changes

[rtereshin]

Hi,

Doesn't look like this just handles --llc-binary command line argument of update_llc_test_checks.py correctly: I assume references to llc from within the preprocess_cmd should also be replaced with the value of --llc-binary if any, similarly to the substitution done for the main part of the command?

Thanks!

[arichardson]

Unless there are any

In D95137#3517661, @rtereshin wrote:

Hi,

Doesn't look like this just handles --llc-binary command line argument of update_llc_test_checks.py correctly: I assume references to llc from within the preprocess_cmd should also be replaced with the value of --llc-binary if any, similarly to the substitution done for the main part of the command?

Thanks!

Yes that would be ideal (and shouldn't be too hard to implement), I just never had the need for that so far.

llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/pre-process.ll

+; Test that update_llc_test_checks.py can run pre-processing commands.
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=CHECK-ADD
+; RUN: sed 's/add /sub /g' %s | llc -mtriple=x86_64-unknown-unknown \
+; RUN:   | FileCheck %s --check-prefix=CHECK-SUB
+; Check that multiple pre-processing commands are handled
+; RUN: sed 's/add /sub /g' %s | sed 's/i64 /i16 /g' | cat \
+; RUN:  | llc -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=CHECK-SUB-I16
+
+define i64 @test_add_constant(i64 %arg) nounwind {
+entry:
+  %a1 = add i64 %arg, 1
+  %a2 = add i64 %a1, 2
+  %a3 = add i64 %a2, 3
+  ret i64 %a3
+}

llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/pre-process.ll.expected

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; Test that update_llc_test_checks.py can run pre-processing commands.
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=CHECK-ADD
+; RUN: sed 's/add /sub /g' %s | llc -mtriple=x86_64-unknown-unknown \
+; RUN:   | FileCheck %s --check-prefix=CHECK-SUB
+; Check that multiple pre-processing commands are handled
+; RUN: sed 's/add /sub /g' %s | sed 's/i64 /i16 /g' | cat \
+; RUN:  | llc -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=CHECK-SUB-I16
+
+define i64 @test_add_constant(i64 %arg) nounwind {
+; CHECK-ADD-LABEL: test_add_constant:
+; CHECK-ADD:       # %bb.0: # %entry
+; CHECK-ADD-NEXT:    leaq 6(%rdi), %rax
+; CHECK-ADD-NEXT:    retq
+;
+; CHECK-SUB-LABEL: test_add_constant:
+; CHECK-SUB:       # %bb.0: # %entry
+; CHECK-SUB-NEXT:    leaq -6(%rdi), %rax
+; CHECK-SUB-NEXT:    retq
+;
+; CHECK-SUB-I16-LABEL: test_add_constant:
+; CHECK-SUB-I16:       # %bb.0: # %entry
+; CHECK-SUB-I16-NEXT:    # kill: def $edi killed $edi def $rdi
+; CHECK-SUB-I16-NEXT:    leal -6(%rdi), %eax
+; CHECK-SUB-I16-NEXT:    # kill: def $ax killed $ax killed $eax
+; CHECK-SUB-I16-NEXT:    retq
+entry:
+  %a1 = add i64 %arg, 1
+  %a2 = add i64 %a1, 2
+  %a3 = add i64 %a2, 3
+  ret i64 %a3
+}

llvm/test/tools/UpdateTestChecks/update_llc_test_checks/pre-process.test

+# REQUIRES: x86-registered-target
+## Test that update_llc_test_checks.py can run pre-processing commands.
+
+## Test that update_test_checks.py can run pre-processing commands.
+# RUN: cp -f %S/Inputs/pre-process.ll %t.ll && %update_llc_test_checks %t.ll
+# RUN: diff -u %t.ll %S/Inputs/pre-process.ll.expected
+## Check that running the script again does not change the result:
+# RUN: %update_llc_test_checks %t.ll
+# RUN: diff -u %t.ll %S/Inputs/pre-process.ll.expected
+

llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/pre-process.ll

+; Test that update_test_checks.py can run pre-processing commands.
+; RUN: opt < %s -instsimplify -S | FileCheck %s --check-prefix=CHECK-AS200
+; RUN: sed -e 's/addrspace(200)/addrspace(0)/g' -e 's/-A200-P200-G200//g' %s \
+; RUN:   | opt -instsimplify -S | FileCheck %s --check-prefix=CHECK-AS0
+; Check that multiple pre-processing commands are handled
+; RUN: sed 's/addrspace(200)/addrspace(1)/g' %s | sed 's/-A1-P1-G1//g' \
+; RUN:   | opt -instsimplify -S | FileCheck %s --check-prefix=CHECK-AS1
+; More than two commands should also be fine
+; RUN: cat %s | cat | cat | cat | opt < %s -instsimplify -S \
+; RUN:   | FileCheck %s --check-prefix=CHECK-AS200-NOOP-PRE-PROCESS
+
+target datalayout = "e-m:e-p200:128:128:128:64-p:64:64-A200-P200-G200"
+
+define i8 addrspace(200)* @test_zerogep_in_different_as(i8 addrspace(200)* %arg) addrspace(200) nounwind {
+entry:
+  %ret = getelementptr inbounds i8, i8 addrspace(200)* %arg, i64 0
+  ret i8 addrspace(200)* %ret
+}

llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/pre-process.ll.expected

+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature
+; Test update_test_checks.py can run sed as a pre-processing command.
+; RUN: opt < %s -instsimplify -S | FileCheck %s --check-prefix=CHECK-AS200
+; RUN: sed -e 's/addrspace(200)/addrspace(0)/g' -e 's/-A200-P200-G200//g' %s \
+; RUN:   | opt -instsimplify -S | FileCheck %s --check-prefix=CHECK-AS0
+; Check that multiple pre-processing commands are handled
+; RUN: sed 's/addrspace(200)/addrspace(1)/g' %s | sed 's/-A1-P1-G1//g' \
+; RUN:   | opt -instsimplify -S | FileCheck %s --check-prefix=CHECK-AS1
+; More than two commands should also be fine
+; RUN: cat %s | cat | cat | cat | opt < %s -instsimplify -S \
+; RUN:   | FileCheck %s --check-prefix=CHECK-AS200-NOOP-PRE-PROCESS
+
+target datalayout = "e-m:e-p200:128:128:128:64-p:64:64-A200-P200-G200"
+
+define i8 addrspace(200)* @test_zerogep_in_different_as(i8 addrspace(200)* %arg) addrspace(200) nounwind {
+; CHECK-AS200-LABEL: define {{[^@]+}}@test_zerogep_in_different_as
+; CHECK-AS200-SAME: (i8 addrspace(200)* [[ARG:%.*]]) addrspace(200) [[ATTR0:#.*]] {
+; CHECK-AS200-NEXT:  entry:
+; CHECK-AS200-NEXT:    ret i8 addrspace(200)* [[ARG]]
+;
+; CHECK-AS0-LABEL: define {{[^@]+}}@test_zerogep_in_different_as
+; CHECK-AS0-SAME: (i8* [[ARG:%.*]]) [[ATTR0:#.*]] {
+; CHECK-AS0-NEXT:  entry:
+; CHECK-AS0-NEXT:    ret i8* [[ARG]]
+;
+; CHECK-AS1-LABEL: define {{[^@]+}}@test_zerogep_in_different_as
+; CHECK-AS1-SAME: (i8 addrspace(1)* [[ARG:%.*]]) addrspace(1) [[ATTR0:#.*]] {
+; CHECK-AS1-NEXT:  entry:
+; CHECK-AS1-NEXT:    ret i8 addrspace(1)* [[ARG]]
+;
+; CHECK-AS200-NOOP-PRE-PROCESS-LABEL: define {{[^@]+}}@test_zerogep_in_different_as
+; CHECK-AS200-NOOP-PRE-PROCESS-SAME: (i8 addrspace(200)* [[ARG:%.*]]) addrspace(200) [[ATTR0:#.*]] {
+; CHECK-AS200-NOOP-PRE-PROCESS-NEXT:  entry:
+; CHECK-AS200-NOOP-PRE-PROCESS-NEXT:    ret i8 addrspace(200)* [[ARG]]
+;
+entry:
+  %ret = getelementptr inbounds i8, i8 addrspace(200)* %arg, i64 0
+  ret i8 addrspace(200)* %ret
+}

llvm/test/tools/UpdateTestChecks/update_test_checks/pre-process.test

+## Test that update_test_checks.py can run pre-processing commands.
+# RUN: cp -f %S/Inputs/pre-process.ll %t.ll && %update_test_checks --function-signature %t.ll
+# RUN: diff -u %t.ll %S/Inputs/pre-process.ll.expected
+## Check that running the script again does not change the result:
+# RUN: %update_test_checks %t.ll
+# RUN: diff -u %t.ll %S/Inputs/pre-process.ll.expected

llvm/utils/UpdateTestChecks/common.py

 from __future__ import print_function
 
 import copy
 import glob
+import os
 import re
 import subprocess
 import sys
 
 if sys.version_info[0] > 2:
   class string:
     expandtabs = str.expandtabs
 else:
def should_add_line_to_output(input_line, prefix_set):
   # And skip any CHECK lines. We're building our own.
   m = CHECK_RE.match(input_line)
   if m and m.group(1) in prefix_set:
     return False
 
   return True
 
 # Invoke the tool that is being tested.
-def invoke_tool(exe, cmd_args, ir):
+def invoke_tool(exe, cmd_args, ir, preprocess_cmd=None, verbose=False):
   with open(ir) as ir_file:
     # TODO Remove the str form which is used by update_test_checks.py and
     # update_llc_test_checks.py
     # The safer list form is used by update_cc_test_checks.py
+    if preprocess_cmd:
+      # Allow pre-processing the IR file (e.g. using sed):
+      assert isinstance(preprocess_cmd, str)  # TODO: use a list instead of using shell
+      preprocess_cmd = preprocess_cmd.replace('%s', ir).strip()
+      if verbose:
+        print('Pre-processing input file: ', ir, " with command '",
+              preprocess_cmd, "'", sep="", file=sys.stderr)
+      # Python 2.7 doesn't have subprocess.DEVNULL:
+      with open(os.devnull, 'w') as devnull:
+        pp = subprocess.Popen(preprocess_cmd, shell=True, stdin=devnull,
+                              stdout=subprocess.PIPE)
+        ir_file = pp.stdout
     if isinstance(cmd_args, list):
       stdout = subprocess.check_output([exe] + cmd_args, stdin=ir_file)
     else:
       stdout = subprocess.check_output(exe + ' ' + cmd_args,
                                        shell=True, stdin=ir_file)
     if sys.version_info[0] > 2:
       stdout = stdout.decode()
   # Fix line endings to unix CR style.

llvm/utils/update_llc_test_checks.py

for l in ti.input_lines:
         break
 
     run_list = []
     for l in ti.run_lines:
       if '|' not in l:
         common.warn('Skipping unparseable RUN line: ' + l)
         continue
 
-      commands = [cmd.strip() for cmd in l.split('|', 1)]
-      llc_cmd = commands[0]
+      commands = [cmd.strip() for cmd in l.split('|')]
+      assert len(commands) >= 2
+      preprocess_cmd = None
+      if len(commands) > 2:
+        preprocess_cmd = " | ".join(commands[:-2])
+      llc_cmd = commands[-2]
+      filecheck_cmd = commands[-1]
       llc_tool = llc_cmd.split(' ')[0]
 
       triple_in_cmd = None
       m = common.TRIPLE_ARG_RE.search(llc_cmd)
       if m:
         triple_in_cmd = m.groups()[0]
 
       march_in_cmd = None
       m = common.MARCH_ARG_RE.search(llc_cmd)
       if m:
         march_in_cmd = m.groups()[0]
 
-      filecheck_cmd = ''
-      if len(commands) > 1:
-        filecheck_cmd = commands[1]
       common.verify_filecheck_prefixes(filecheck_cmd)
       if llc_tool not in LLC_LIKE_TOOLS:
         common.warn('Skipping non-llc RUN line: ' + l)
         continue
 
       if not filecheck_cmd.startswith('FileCheck '):
         common.warn('Skipping non-FileChecked RUN line: ' + l)
         continue
 
       llc_cmd_args = llc_cmd[len(llc_tool):].strip()
       llc_cmd_args = llc_cmd_args.replace('< %s', '').replace('%s', '').strip()
       if ti.path.endswith('.mir'):
         llc_cmd_args += ' -x mir'
       check_prefixes = [item for m in common.CHECK_PREFIX_RE.finditer(filecheck_cmd)
                                for item in m.group(1).split(',')]
       if not check_prefixes:
         check_prefixes = ['CHECK']
 
       # FIXME: We should use multiple check prefixes to common check lines. For
       # now, we just ignore all but the last.
-      run_list.append((check_prefixes, llc_cmd_args, triple_in_cmd, march_in_cmd))
+      run_list.append((check_prefixes, llc_tool, llc_cmd_args, preprocess_cmd,
+                       triple_in_cmd, march_in_cmd))
 
     if ti.path.endswith('.mir'):
       check_indent = '  '
     else:
       check_indent = ''
 
     builder = common.FunctionTestBuilder(
         run_list=run_list, 
         flags=type('', (object,), {
             'verbose': ti.args.verbose,
             'function_signature': False,
             'check_attributes': False}),
         scrubber_args=[ti.args])
 
-    for prefixes, llc_args, triple_in_cmd, march_in_cmd in run_list:
+    for prefixes, llc_tool, llc_args, preprocess_cmd, triple_in_cmd, march_in_cmd in run_list:
       common.debug('Extracted LLC cmd:', llc_tool, llc_args)
       common.debug('Extracted FileCheck prefixes:', str(prefixes))
 
       raw_tool_output = common.invoke_tool(ti.args.llc_binary or llc_tool,
-                                           llc_args, ti.path)
+                                           llc_args, ti.path, preprocess_cmd,
+                                           verbose=ti.args.verbose)
       triple = triple_in_cmd or triple_in_ir
       if not triple:
         triple = asm.get_triple_from_march(march_in_cmd)
 
       scrubber, function_re = asm.get_run_handler(triple)
       builder.process_run_line(function_re, scrubber, raw_tool_output, prefixes)
 
     func_dict = builder.finish_and_get_func_dict()

llvm/utils/update_test_checks.py

else:
       common.SCRUB_TRAILING_WHITESPACE_TEST_RE = common.SCRUB_TRAILING_WHITESPACE_RE
 
     prefix_list = []
     for l in ti.run_lines:
       if '|' not in l:
         common.warn('Skipping unparseable RUN line: ' + l)
         continue
 
-      (tool_cmd, filecheck_cmd) = tuple([cmd.strip() for cmd in l.split('|', 1)])
+      commands = [cmd.strip() for cmd in l.split('|')]
+      assert len(commands) >= 2
+      preprocess_cmd = None
+      if len(commands) > 2:
+        preprocess_cmd = " | ".join(commands[:-2])
+      tool_cmd = commands[-2]
+      filecheck_cmd = commands[-1]
       common.verify_filecheck_prefixes(filecheck_cmd)
       if not tool_cmd.startswith(opt_basename + ' '):
         common.warn('Skipping non-%s RUN line: %s' % (opt_basename, l))
         continue
 
       if not filecheck_cmd.startswith('FileCheck '):
         common.warn('Skipping non-FileChecked RUN line: ' + l)
         continue
 
       tool_cmd_args = tool_cmd[len(opt_basename):].strip()
       tool_cmd_args = tool_cmd_args.replace('< %s', '').replace('%s', '').strip()
 
       check_prefixes = [item for m in
                         common.CHECK_PREFIX_RE.finditer(filecheck_cmd)
                         for item in m.group(1).split(',')]
       if not check_prefixes:
         check_prefixes = ['CHECK']
 
       # FIXME: We should use multiple check prefixes to common check lines. For
       # now, we just ignore all but the last.
-      prefix_list.append((check_prefixes, tool_cmd_args))
+      prefix_list.append((check_prefixes, tool_cmd_args, preprocess_cmd))
 
     global_vars_seen_dict = {}
     builder = common.FunctionTestBuilder(
       run_list=prefix_list,
       flags=ti.args,
       scrubber_args=[])
 
-    for prefixes, opt_args in prefix_list:
+    for prefixes, opt_args, preprocess_cmd in prefix_list:
       common.debug('Extracted opt cmd: ' + opt_basename + ' ' + opt_args)
       common.debug('Extracted FileCheck prefixes: ' + str(prefixes))
 
-      raw_tool_output = common.invoke_tool(ti.args.opt_binary, opt_args, 
-                                           ti.path)
+      raw_tool_output = common.invoke_tool(ti.args.opt_binary, opt_args,
+                                           ti.path, preprocess_cmd=preprocess_cmd,
+                                           verbose=ti.args.verbose)
       builder.process_run_line(common.OPT_FUNCTION_RE, common.scrub_body,
               raw_tool_output, prefixes)
 
     func_dict = builder.finish_and_get_func_dict()
     is_in_function = False
     is_in_function_start = False
-    prefix_set = set([prefix for prefixes, _ in prefix_list for prefix in prefixes])
+    prefix_set = set([prefix for prefixes, _, _ in prefix_list for prefix in prefixes])
     common.debug('Rewriting FileCheck prefixes:', str(prefix_set))
     output_lines = []
 
     include_generated_funcs = common.find_arg_in_test(ti,
                                                       lambda args: ti.args.include_generated_funcs,
                                                       '--include-generated-funcs',
                                                       True)