[Utils] Run non-filecheck runlines in-order in update_cc_test_checks

[ggeorgakoudis]

Summary

The script update_cc_test_checks runs all non-filechecked runlines before the filechecked ones. This creates problems since outputs of those non-filechecked runlines may conflict and that will fail the execution of update_cc_test_checks. This patch executes non-filechecked in the order specified in the test file to avoid this issue.

[jdoerfert]

Add a commit message explaining why/what this does. I run into the problem before so I know but other won't.

clang/test/utils/update_cc_test_checks/Inputs/exec-all-runlines.c

 // Check that the non-clang/non-filechecked runlines execute
 // RUN: cp %s %S/Output/execute-all-runlines.copy.c
 // RUN: cp %S/Output/execute-all-runlines.copy.c %s.copy.c
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fopenmp %s.copy.c -emit-llvm-bc -o %t-host.bc
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fopenmp -fopenmp-host-ir-file-path %t-host.bc %s.copy.c -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fopenmp -fopenmp-host-ir-file-path %t-host.bc %s.copy.c -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-pch %s -o %t
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -include-pch %t %s.copy.c -emit-llvm -o - | FileCheck %s --check-prefix=CHECK2
+// RUN: %clang_cc1 -triple powerpc64le-unknown-linux-gnu -emit-pch %s -o %t
+// RUN: %clang_cc1 -triple powerpc64le-unknown-linux-gnu -include-pch %t %s.copy.c -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
+
+
+#ifndef HEADER
+#define HEADER
 
 void use(int);
 
 void test(int a)
 {
 }
+
+#endif

clang/test/utils/update_cc_test_checks/Inputs/exec-all-runlines.c.expected

 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 // Check that the non-clang/non-filechecked runlines execute
 // RUN: cp %s %S/Output/execute-all-runlines.copy.c
 // RUN: cp %S/Output/execute-all-runlines.copy.c %s.copy.c
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fopenmp %s.copy.c -emit-llvm-bc -o %t-host.bc
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fopenmp -fopenmp-host-ir-file-path %t-host.bc %s.copy.c -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fopenmp -fopenmp-host-ir-file-path %t-host.bc %s.copy.c -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-pch %s -o %t
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -include-pch %t %s.copy.c -emit-llvm -o - | FileCheck %s --check-prefix=CHECK2
+// RUN: %clang_cc1 -triple powerpc64le-unknown-linux-gnu -emit-pch %s -o %t
+// RUN: %clang_cc1 -triple powerpc64le-unknown-linux-gnu -include-pch %t %s.copy.c -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
+
+
+#ifndef HEADER
+#define HEADER
 
 void use(int);
 
-// CHECK-LABEL: @test(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[A:%.*]], i32* [[A_ADDR]], align 4
-// CHECK-NEXT:    ret void
+// CHECK1-LABEL: @test(
+// CHECK1-NEXT:  entry:
+// CHECK1-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    store i32 [[A:%.*]], i32* [[A_ADDR]], align 4
+// CHECK1-NEXT:    ret void
+//
+// CHECK2-LABEL: @test(
+// CHECK2-NEXT:  entry:
+// CHECK2-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    store i32 [[A:%.*]], i32* [[A_ADDR]], align 4
+// CHECK2-NEXT:    ret void
+//
+// CHECK3-LABEL: @test(
+// CHECK3-NEXT:  entry:
+// CHECK3-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
+// CHECK3-NEXT:    store i32 [[A:%.*]], i32* [[A_ADDR]], align 4
+// CHECK3-NEXT:    ret void
 //
 void test(int a)
 {
 }
+
+#endif

llvm/utils/update_cc_test_checks.py

if popen.returncode != 0:
     sys.exit(3)
 
 def main():
   initial_args, parser = config()
   script_name = os.path.basename(__file__)
 
   for ti in common.itertests(initial_args.tests, parser, 'utils/' + script_name,
                              comment_prefix='//', argparse_callback=infer_dependent_args):
-    # Build a list of clang command lines and check prefixes from RUN lines.
+    # Build a list of filechecked and non-filechecked RUN lines.
     run_list = []
     line2spell_and_mangled_list = collections.defaultdict(list)
 
     subs = {
       '%s' : ti.path,
       '%t' : tempfile.NamedTemporaryFile().name,
       '%S' : os.getcwd(),
     }
 
     for l in ti.run_lines:
       commands = [cmd.strip() for cmd in l.split('|')]
 
       triple_in_cmd = None
       m = common.TRIPLE_ARG_RE.search(commands[0])
       if m:
         triple_in_cmd = m.groups()[0]
 
       # Parse executable args.
       exec_args = shlex.split(commands[0])
       # Execute non-clang runline.
       if exec_args[0] not in SUBST:
-        print('NOTE: Executing non-clang RUN line: ' + l, file=sys.stderr)
         # Do lit-like substitutions.
         for s in subs:
           exec_args = [i.replace(s, subs[s]) if s in i else i for i in exec_args]
-        exec_run_line(exec_args)
+        run_list.append((None, exec_args, None, None))
         continue
       # This is a clang runline, apply %clang substitution rule, do lit-like substitutions,
       # and append args.clang_args
       clang_args = exec_args
       clang_args[0:1] = SUBST[clang_args[0]]
       for s in subs:
         clang_args = [i.replace(s, subs[s]) if s in i else i for i in clang_args]
       clang_args += ti.args.clang_args
 
       # Extract -check-prefix in FileCheck args
       filecheck_cmd = commands[-1]
       common.verify_filecheck_prefixes(filecheck_cmd)
       if not filecheck_cmd.startswith('FileCheck '):
-        print('NOTE: Executing non-FileChecked clang RUN line: ' + l, file=sys.stderr)
         # Execute non-filechecked clang runline.
         exe = [ti.args.clang] + clang_args
-        exec_run_line(exe)
+        run_list.append((None, exe, None, None))
         continue
 
       check_prefixes = [item for m in common.CHECK_PREFIX_RE.finditer(filecheck_cmd)
                                for item in m.group(1).split(',')]
       if not check_prefixes:
         check_prefixes = ['CHECK']
       run_list.append((check_prefixes, clang_args, commands[1:-1], triple_in_cmd))
 
     # Execute clang, generate LLVM IR, and extract functions.
 
+    # Store only filechecked runlines.
+    filecheck_run_list = [i for i in run_list if i[0]]
     builder = common.FunctionTestBuilder(
-      run_list=run_list,
+      run_list=filecheck_run_list,
       flags=ti.args,
       scrubber_args=[])
 
-    for prefixes, clang_args, extra_commands, triple_in_cmd in run_list:
+    for prefixes, args, extra_commands, triple_in_cmd in run_list:
+      # Execute non-filechecked runline.
+      if not prefixes:
+        print('NOTE: Executing non-FileChecked RUN line: ' + ' '.join(args), file=sys.stderr)
+        exec_run_line(args)
+        continue
+
+      clang_args = args
       common.debug('Extracted clang cmd: clang {}'.format(clang_args))
       common.debug('Extracted FileCheck prefixes: {}'.format(prefixes))
 
       get_function_body(builder, ti.args, ti.path, clang_args, extra_commands,
                         prefixes)
 
       # Invoke clang -Xclang -ast-dump=json to get mapping from start lines to
       # mangled names. Forward all clang args for now.
       for k, v in get_line2spell_and_mangled(ti.args, clang_args).items():
         line2spell_and_mangled_list[k].append(v)
 
     func_dict = builder.finish_and_get_func_dict()
     global_vars_seen_dict = {}
-    prefix_set = set([prefix for p in run_list for prefix in p[0]])
+    prefix_set = set([prefix for p in filecheck_run_list for prefix in p[0]])
     output_lines = []
 
     include_generated_funcs = common.find_arg_in_test(ti,
                                                       lambda args: ti.args.include_generated_funcs,
                                                       '--include-generated-funcs',
                                                       True)
 
     if include_generated_funcs:
if include_generated_funcs:
                                func_dict, func, False,
                                ti.args.function_signature,
                                global_vars_seen_dict)
         else:
           asm.add_asm_checks(my_output_lines, '//',
                              prefixes,
                              func_dict, func)
 
-      common.add_checks_at_end(output_lines, run_list, builder.func_order(),
+      common.add_checks_at_end(output_lines, filecheck_run_list, builder.func_order(),
                                '//', lambda my_output_lines, prefixes, func:
                                check_generator(my_output_lines,
                                                prefixes, func))
     else:
       # Normal mode.  Put checks before each source function.
       for line_info in ti.iterlines(output_lines):
         idx = line_info.line_number
         line = line_info.line
else:
               while last_line == '//':
                 # Remove the comment line since we will generate a new  comment
                 # line as part of common.add_ir_checks()
                 output_lines.pop()
                 last_line = output_lines[-1].strip()
               if added:
                 output_lines.append('//')
               added.add(mangled)
-              common.add_ir_checks(output_lines, '//', run_list, func_dict, mangled,
+              common.add_ir_checks(output_lines, '//', filecheck_run_list, func_dict, mangled,
                                    False, args.function_signature, global_vars_seen_dict)
               if line.rstrip('\n') == '//':
                 include_line = False
 
         if include_line:
           output_lines.append(line.rstrip('\n'))
 
     common.debug('Writing %d lines to %s...' % (len(output_lines), ti.path))
     with open(ti.path, 'wb') as f:
       f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])
 
   return 0
 
 
 if __name__ == '__main__':
   sys.exit(main())