[test] Use host platform specific error message substitution in lit tests - continued

[abhina.sreeskantharajan]

Summary

On z/OS, other error messages are not matched correctly in lit tests.

EDC5121I Invalid argument.
EDC5111I Permission denied.

This patch adds a lit substitution to fix it.

[abhina.sreeskantharajan]

I've previously changed Permission denied errors before. Fixing these testcases with a lit substitution as well.

[jhenderson]

I think you need to add the new values to the list of recognised substitutions in the documentation?

[abhina.sreeskantharajan]

Thank you for reminding me, I've updated the TestingGuide.rst.

[jhenderson]

LGTM.

[ASDenysPetrov]

Hi, @abhina.sreeskantharajan.
I've just updated my repo and got this patch. After your changes I caught a test fail specifically in my domain of Clang Static Analyzer.
Here is output of Analysis/taint-generic.c

output_taint_test.txt8 KBDownload

Could you please check it and try to fix?
While testing I'm using build on Win10 + GCC config.

[jhenderson]

@ASDenysPetrov, I think you need to mark this patch as Accepted so that someone can close this review.

[ASDenysPetrov]

In D95808#2657744, @jhenderson wrote:

@ASDenysPetrov, I think you need to mark this patch as Accepted so that someone can close this review.

Done. Sorry for the delay.

clang/test/Analysis/taint-generic.c

 // CHECK-INVALID-FILE-SAME:        'alpha.security.taint.TaintPropagation:Config',
 // CHECK-INVALID-FILE-SAME:        that expects a valid filename instead of
 // CHECK-INVALID-FILE-SAME:        'justguessit'
 
 // RUN: not %clang_analyze_cc1 -verify %s \
 // RUN:   -analyzer-checker=alpha.security.taint \
 // RUN:   -analyzer-config \
 // RUN:     alpha.security.taint.TaintPropagation:Config=%S/Inputs/taint-generic-config-ill-formed.yaml \
-// RUN:   2>&1 | FileCheck %s -check-prefix=CHECK-ILL-FORMED
+// RUN:   2>&1 | FileCheck -DMSG=%errc_EINVAL %s -check-prefix=CHECK-ILL-FORMED
 
 // CHECK-ILL-FORMED: (frontend): invalid input for checker option
 // CHECK-ILL-FORMED-SAME:        'alpha.security.taint.TaintPropagation:Config',
-// CHECK-ILL-FORMED-SAME:        that expects a valid yaml file: {{[Ii]}}nvalid argument
+// CHECK-ILL-FORMED-SAME:        that expects a valid yaml file: [[MSG]]
 
 // RUN: not %clang_analyze_cc1 -verify %s \
 // RUN:   -analyzer-checker=alpha.security.taint \
 // RUN:   -analyzer-config \
 // RUN:     alpha.security.taint.TaintPropagation:Config=%S/Inputs/taint-generic-config-invalid-arg.yaml \
 // RUN:   2>&1 | FileCheck %s -check-prefix=CHECK-INVALID-ARG
 
 // CHECK-INVALID-ARG: (frontend): invalid input for checker option

clang/test/Format/style-on-command-line.cpp

 // RUN: clang-format -style="{BasedOnStyle: Google, IndentWidth: 8}" %s | FileCheck -strict-whitespace -check-prefix=CHECK1 %s
 // RUN: clang-format -style="{BasedOnStyle: LLVM, IndentWidth: 7}" %s | FileCheck -strict-whitespace -check-prefix=CHECK2 %s
-// RUN: not clang-format -style="{BasedOnStyle: invalid, IndentWidth: 7}" -fallback-style=LLVM %s 2>&1 | FileCheck -strict-whitespace -check-prefix=CHECK3 %s
-// RUN: not clang-format -style="{lsjd}" %s -fallback-style=LLVM 2>&1 | FileCheck -strict-whitespace -check-prefix=CHECK4 %s
+// RUN: not clang-format -style="{BasedOnStyle: invalid, IndentWidth: 7}" -fallback-style=LLVM %s 2>&1 | FileCheck -DMSG=%errc_EINVAL -strict-whitespace -check-prefix=CHECK3 %s
+// RUN: not clang-format -style="{lsjd}" %s -fallback-style=LLVM 2>&1 | FileCheck -DMSG=%errc_EINVAL -strict-whitespace -check-prefix=CHECK4 %s
 // RUN: mkdir -p %t
 // RUN: printf "BasedOnStyle: google\nIndentWidth: 5\n" > %t/.clang-format
 // RUN: clang-format -style=file -assume-filename=%t/foo.cpp < %s | FileCheck -strict-whitespace -check-prefix=CHECK5 %s
 // RUN: printf "\n" > %t/.clang-format
 // RUN: not clang-format -style=file -fallback-style=webkit -assume-filename=%t/foo.cpp < %s 2>&1 | FileCheck -strict-whitespace -check-prefix=CHECK6 %s
 // RUN: rm %t/.clang-format
 // RUN: printf "BasedOnStyle: google\nIndentWidth: 6\n" > %t/_clang-format
 // RUN: clang-format -style=file -assume-filename=%t/foo.cpp < %s | FileCheck -strict-whitespace -check-prefix=CHECK7 %s
 // RUN: clang-format -style="{BasedOnStyle: LLVM, PointerBindsToType: true}" %s | FileCheck -strict-whitespace -check-prefix=CHECK8 %s
 // RUN: clang-format -style="{BasedOnStyle: WebKit, PointerBindsToType: false}" %s | FileCheck -strict-whitespace -check-prefix=CHECK9 %s
 
 // Fallback style tests
 // Test config file with no based style, and fallback style "none", formatting is applied
 // RUN: printf "IndentWidth: 6\n" > %t/_clang-format
 // RUN: clang-format -style=file -fallback-style=none -assume-filename=%t/foo.cpp < %s 2>&1 | FileCheck -strict-whitespace -check-prefix=CHECK10 %s
 // Test yaml with no based style, and fallback style "none", LLVM formatting applied
 // RUN: clang-format -style="{IndentWidth: 7}" -fallback-style=none %s | FileCheck -strict-whitespace -check-prefix=CHECK11 %s
 
 void f() {
 // CHECK1: {{^        int\* i;$}}
 // CHECK2: {{^       int \*i;$}}
 // CHECK3: Unknown value for BasedOnStyle: invalid
-// CHECK3: Error parsing -style: {{I|i}}nvalid argument
-// CHECK4: Error parsing -style: {{I|i}}nvalid argument
+// CHECK3: Error parsing -style: [[MSG]]
+// CHECK4: Error parsing -style: [[MSG]]
 // CHECK5: {{^     int\* i;$}}
 // CHECK6: {{^Error reading .*\.clang-format: (I|i)nvalid argument}}
 // CHECK7: {{^      int\* i;$}}
 // CHECK8: {{^  int\* i;$}}
 // CHECK9: {{^    int \*i;$}}
 // CHECK10: {{^      int \*i;$}}
 // CHECK11: {{^       int \*i;$}}
 int*i;
 int j;
 }
 
 // On Windows, the 'rm' commands fail when the previous process is still alive.
 // This happens enough to make the test useless.
 // UNSUPPORTED: system-windows

llvm/test/tools/yaml2obj/ELF/DWARF/debug-gnu-pubnames.yaml

debug_gnu_pubnames:
     Length:     0x1234
     Version:    2
     UnitOffset: 0x1234
     UnitSize:   0x4321
     Entries:    []
 
 ## h) Test that yaml2obj emits an error if 'Descriptor' is missing.
 
-# RUN: not yaml2obj --docnum=8 %s -o %t8.o 2>&1 | FileCheck %s --check-prefix=MISSING-KEY --ignore-case
+# RUN: not yaml2obj --docnum=8 %s -o %t8.o 2>&1 | FileCheck -DMSG=%errc_EINVAL %s --check-prefix=MISSING-KEY --ignore-case
 
 #      MISSING-KEY: YAML:{{.*}}:9: error: missing required key 'Descriptor'
 # MISSING-KEY-NEXT:       - DieOffset: 0x12345678
 # MISSING-KEY-NEXT:         ^
-# MISSING-KEY-NEXT: yaml2obj: error: failed to parse YAML input: Invalid argument
+# MISSING-KEY-NEXT: yaml2obj: error: failed to parse YAML input: [[MSG]]
 
 --- !ELF
 FileHeader:
   Class: ELFCLASS64
   Data:  ELFDATA2LSB
   Type:  ET_EXEC
 DWARF:
   debug_gnu_pubnames:
     Length:     0x1234
     Version:    2
     UnitOffset: 0x1234
     UnitSize:   0x4321
     Entries:
       - DieOffset: 0x12345678
         Name:      abc

llvm/utils/lit/lit/llvm/config.py

def add_tool_substitutions(self, tools, search_dirs=None):
 
         self.config.substitutions.extend(substitutions)
         return True
 
     def add_err_msg_substitutions(self):
         if (sys.platform == 'zos'):
             self.config.substitutions.append(('%errc_ENOENT', '\'EDC5129I No such file or directory.\''))
             self.config.substitutions.append(('%errc_EISDIR', '\'EDC5123I Is a directory.\''))
+            self.config.substitutions.append(('%errc_EINVAL', '\'EDC5121I Invalid argument.\''))
         elif (sys.platform == 'win32'):
             self.config.substitutions.append(('%errc_ENOENT', '\'no such file or directory\''))
             self.config.substitutions.append(('%errc_EISDIR', '\'is a directory\''))
+            self.config.substitutions.append(('%errc_EINVAL', '\'invalid argument\''))
         else:
             self.config.substitutions.append(('%errc_ENOENT', '\'No such file or directory\''))
             self.config.substitutions.append(('%errc_EISDIR', '\'Is a directory\''))
+            self.config.substitutions.append(('%errc_EINVAL', '\'Invalid argument\''))
 
     def use_default_substitutions(self):
         tool_patterns = [
             ToolSubst('FileCheck', unresolved='fatal'),
             # Handle these specially as they are strings searched for during testing.
             ToolSubst(r'\| \bcount\b', command=FindTool(
                 'count'), verbatim=True, unresolved='fatal'),
             ToolSubst(r'\| \bnot\b', command=FindTool('not'), verbatim=True, unresolved='fatal')]