[test] Use host platform specific error message substitution in lit tests - continued

[abhina.sreeskantharajan]

Summary

On z/OS, other error messages are not matched correctly in lit tests.

EDC5121I Invalid argument.
EDC5111I Permission denied.

This patch adds a lit substitution to fix it.

[abhina.sreeskantharajan]

I've previously changed Permission denied errors before. Fixing these testcases with a lit substitution as well.

[jhenderson]

I think you need to add the new values to the list of recognised substitutions in the documentation?

[abhina.sreeskantharajan]

Thank you for reminding me, I've updated the TestingGuide.rst.

[jhenderson]

LGTM.

[ASDenysPetrov]

Hi, @abhina.sreeskantharajan.
I've just updated my repo and got this patch. After your changes I caught a test fail specifically in my domain of Clang Static Analyzer.
Here is output of Analysis/taint-generic.c

output_taint_test.txt8 KBDownload

Could you please check it and try to fix?
While testing I'm using build on Win10 + GCC config.

[jhenderson]

@ASDenysPetrov, I think you need to mark this patch as Accepted so that someone can close this review.

[ASDenysPetrov]

In D95808#2657744, @jhenderson wrote:

@ASDenysPetrov, I think you need to mark this patch as Accepted so that someone can close this review.

Done. Sorry for the delay.

clang/test/Analysis/taint-generic.c

 // CHECK-INVALID-FILE-SAME:        'alpha.security.taint.TaintPropagation:Config',
 // CHECK-INVALID-FILE-SAME:        that expects a valid filename instead of
 // CHECK-INVALID-FILE-SAME:        'justguessit'
 
 // RUN: not %clang_analyze_cc1 -verify %s \
 // RUN:   -analyzer-checker=alpha.security.taint \
 // RUN:   -analyzer-config \
 // RUN:     alpha.security.taint.TaintPropagation:Config=%S/Inputs/taint-generic-config-ill-formed.yaml \
-// RUN:   2>&1 | FileCheck %s -check-prefix=CHECK-ILL-FORMED
+// RUN:   2>&1 | FileCheck -DMSG=%errc_EINVAL %s -check-prefix=CHECK-ILL-FORMED
 
 // CHECK-ILL-FORMED: (frontend): invalid input for checker option
 // CHECK-ILL-FORMED-SAME:        'alpha.security.taint.TaintPropagation:Config',
-// CHECK-ILL-FORMED-SAME:        that expects a valid yaml file: {{[Ii]}}nvalid argument
+// CHECK-ILL-FORMED-SAME:        that expects a valid yaml file: [[MSG]]
 
 // RUN: not %clang_analyze_cc1 -verify %s \
 // RUN:   -analyzer-checker=alpha.security.taint \
 // RUN:   -analyzer-config \
 // RUN:     alpha.security.taint.TaintPropagation:Config=%S/Inputs/taint-generic-config-invalid-arg.yaml \
 // RUN:   2>&1 | FileCheck %s -check-prefix=CHECK-INVALID-ARG
 
 // CHECK-INVALID-ARG: (frontend): invalid input for checker option

clang/test/Format/style-on-command-line.cpp

 // RUN: clang-format -style="{BasedOnStyle: Google, IndentWidth: 8}" %s | FileCheck -strict-whitespace -check-prefix=CHECK1 %s
 // RUN: clang-format -style="{BasedOnStyle: LLVM, IndentWidth: 7}" %s | FileCheck -strict-whitespace -check-prefix=CHECK2 %s
-// RUN: not clang-format -style="{BasedOnStyle: invalid, IndentWidth: 7}" -fallback-style=LLVM %s 2>&1 | FileCheck -strict-whitespace -check-prefix=CHECK3 %s
-// RUN: not clang-format -style="{lsjd}" %s -fallback-style=LLVM 2>&1 | FileCheck -strict-whitespace -check-prefix=CHECK4 %s
+// RUN: not clang-format -style="{BasedOnStyle: invalid, IndentWidth: 7}" -fallback-style=LLVM %s 2>&1 | FileCheck -DMSG=%errc_EINVAL -strict-whitespace -check-prefix=CHECK3 %s
+// RUN: not clang-format -style="{lsjd}" %s -fallback-style=LLVM 2>&1 | FileCheck -DMSG=%errc_EINVAL -strict-whitespace -check-prefix=CHECK4 %s
 // RUN: mkdir -p %t
 // RUN: printf "BasedOnStyle: google\nIndentWidth: 5\n" > %t/.clang-format
 // RUN: clang-format -style=file -assume-filename=%t/foo.cpp < %s | FileCheck -strict-whitespace -check-prefix=CHECK5 %s
 // RUN: printf "\n" > %t/.clang-format
 // RUN: not clang-format -style=file -fallback-style=webkit -assume-filename=%t/foo.cpp < %s 2>&1 | FileCheck -strict-whitespace -check-prefix=CHECK6 %s
 // RUN: rm %t/.clang-format
 // RUN: printf "BasedOnStyle: google\nIndentWidth: 6\n" > %t/_clang-format
 // RUN: clang-format -style=file -assume-filename=%t/foo.cpp < %s | FileCheck -strict-whitespace -check-prefix=CHECK7 %s
 // RUN: clang-format -style="{BasedOnStyle: LLVM, PointerBindsToType: true}" %s | FileCheck -strict-whitespace -check-prefix=CHECK8 %s
 // RUN: clang-format -style="{BasedOnStyle: WebKit, PointerBindsToType: false}" %s | FileCheck -strict-whitespace -check-prefix=CHECK9 %s
 
 // Fallback style tests
 // Test config file with no based style, and fallback style "none", formatting is applied
 // RUN: printf "IndentWidth: 6\n" > %t/_clang-format
 // RUN: clang-format -style=file -fallback-style=none -assume-filename=%t/foo.cpp < %s 2>&1 | FileCheck -strict-whitespace -check-prefix=CHECK10 %s
 // Test yaml with no based style, and fallback style "none", LLVM formatting applied
 // RUN: clang-format -style="{IndentWidth: 7}" -fallback-style=none %s | FileCheck -strict-whitespace -check-prefix=CHECK11 %s
 
 void f() {
 // CHECK1: {{^        int\* i;$}}
 // CHECK2: {{^       int \*i;$}}
 // CHECK3: Unknown value for BasedOnStyle: invalid
-// CHECK3: Error parsing -style: {{I|i}}nvalid argument
-// CHECK4: Error parsing -style: {{I|i}}nvalid argument
+// CHECK3: Error parsing -style: [[MSG]]
+// CHECK4: Error parsing -style: [[MSG]]
 // CHECK5: {{^     int\* i;$}}
 // CHECK6: {{^Error reading .*\.clang-format: (I|i)nvalid argument}}
 // CHECK7: {{^      int\* i;$}}
 // CHECK8: {{^  int\* i;$}}
 // CHECK9: {{^    int \*i;$}}
 // CHECK10: {{^      int \*i;$}}
 // CHECK11: {{^       int \*i;$}}
 int*i;
 int j;
 }
 
 // On Windows, the 'rm' commands fail when the previous process is still alive.
 // This happens enough to make the test useless.
 // UNSUPPORTED: system-windows

lld/test/COFF/thinlto-emit-imports.ll

 ; RUN: ls %t3.obj.thinlto.bc
 
 ; Ensure lld generates error if unable to write to imports file.
 ; RUN: rm -f %t3.obj.imports
 ; RUN: touch %t3.obj.imports
 ; RUN: chmod 400 %t3.obj.imports
 ; RUN: not lld-link -entry:main -thinlto-index-only \
 ; RUN:     -thinlto-emit-imports-files %t1.obj %t2.obj %t3.obj \
-; RUN:     -out:%t4.exe 2>&1 | FileCheck %s --check-prefix=ERR
-; ERR: cannot open {{.*}}3.obj.imports: {{P|p}}ermission denied
+; RUN:     -out:%t4.exe 2>&1 | FileCheck -DMSG=%errc_EACCES %s --check-prefix=ERR
+; ERR: cannot open {{.*}}3.obj.imports: [[MSG]]
 
 ; Ensure lld doesn't generate import files when thinlto-index-only is not enabled
 ; RUN: rm -f %t1.obj.imports
 ; RUN: rm -f %t2.obj.imports
 ; RUN: rm -f %t3.obj.imports
 ; RUN: lld-link -entry:main -thinlto-emit-imports-files \
 ; RUN:     %t1.obj %t2.obj %t3.obj -out:%t4.exe
 ; RUN: not ls %t1.obj.imports

lld/test/ELF/lto/resolution-err.ll

 ; UNSUPPORTED: system-windows
 ; REQUIRES: shell
 ; RUN: llvm-as %s -o %t.bc
 ; RUN: touch %t.resolution.txt
 ; RUN: chmod -w %t.resolution.txt
-; RUN: not ld.lld -save-temps %t.bc -o %t 2>&1 | FileCheck %s
+; RUN: not ld.lld -save-temps %t.bc -o %t 2>&1 | FileCheck -DMSG=%errc_EACCES %s
 ; RUN: rm -f %t.resolution.txt
 
-; CHECK: error: {{[Pp]}}ermission denied{{$}}
+; CHECK: error: [[MSG]]
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @_start() {
   ret void
 }

lld/test/ELF/lto/thinlto-cant-write-index.ll

 ; REQUIRES: x86
 
 ; Basic ThinLTO tests.
 ; RUN: opt -module-summary %s -o %t1.o
 ; RUN: opt -module-summary %p/Inputs/thinlto.ll -o %t2.o
 
 ; Ensure lld generates error if unable to write to index files
 ; RUN: rm -f %t2.o.thinlto.bc
 ; RUN: touch %t2.o.thinlto.bc
 ; RUN: chmod u-w %t2.o.thinlto.bc
-; RUN: not ld.lld --plugin-opt=thinlto-index-only -shared %t1.o %t2.o -o /dev/null 2>&1 | FileCheck %s
+; RUN: not ld.lld --plugin-opt=thinlto-index-only -shared %t1.o %t2.o -o /dev/null 2>&1 | FileCheck -DMSG=%errc_EACCES %s
 ; RUN: chmod u+w %t2.o.thinlto.bc
-; CHECK: cannot open {{.*}}2.o.thinlto.bc: {{P|p}}ermission denied
+; CHECK: cannot open {{.*}}2.o.thinlto.bc: [[MSG]]
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 declare void @g(...)
 
 define void @f() {
 entry:
   call void (...) @g()
   ret void
 }

lld/test/ELF/lto/thinlto-emit-imports.ll

 
 ; The index file should be created even for the input with an empty summary.
 ; RUN: ls %t3.o.thinlto.bc
 
 ; Ensure lld generates error if unable to write to imports file.
 ; RUN: rm -f %t3.o.imports
 ; RUN: touch %t3.o.imports
 ; RUN: chmod 400 %t3.o.imports
-; RUN: not ld.lld --plugin-opt=thinlto-index-only --plugin-opt=thinlto-emit-imports-files -shared %t1.o %t2.o %t3.o -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR
-; ERR: cannot open {{.*}}3.o.imports: {{P|p}}ermission denied
+; RUN: not ld.lld --plugin-opt=thinlto-index-only --plugin-opt=thinlto-emit-imports-files -shared %t1.o %t2.o %t3.o -o /dev/null 2>&1 | FileCheck -DMSG=%errc_EACCES %s --check-prefix=ERR
+; ERR: cannot open {{.*}}3.o.imports: [[MSG]]
 
 ; Ensure lld doesn't generate import files when thinlto-index-only is not enabled
 ; RUN: rm -f %t1.o.imports
 ; RUN: rm -f %t2.o.imports
 ; RUN: rm -f %t3.o.imports
 ; RUN: ld.lld --plugin-opt=thinlto-emit-imports-files -shared %t1.o %t2.o %t3.o -o %t4
 ; RUN: not ls %t1.o.imports
 ; RUN: not ls %t2.o.imports

llvm/docs/TestingGuide.rst

``%:s, %:S, %:t, %:T:``
 
    Example: ``%:s: C\Desktop Files\foo_test.s.tmp``
 
 ``%errc_<ERRCODE>``
 
  Some error messages may be substituted to allow different spellings
  based on the host platform.
 
-   The following error codes are currently supported: ENOENT, EISDIR.
+   The following error codes are currently supported:
+   ENOENT, EISDIR, EINVAL, EACCES.
 
    Example: ``Linux %errc_ENOENT: No such file or directory``
 
    Example: ``Windows %errc_ENOENT: no such file or directory``
 
 **LLVM-specific substitutions:**
 
 ``%shlibext``

llvm/test/tools/llvm-ar/error-opening-permission.test

 ## Unsupported on windows as marking files "unreadable"
 ## is non-trivial on windows.
 # UNSUPPORTED: system-windows
 
 # RUN: rm -rf %t && mkdir -p %t
 # RUN: echo file1 > %t/1.txt
 
 ## No Permission:
 # RUN: llvm-ar rc %t/permission.b %t/1.txt
 # RUN: chmod 100 %t/permission.b
 # RUN: not llvm-ar p %t/permission.b 2>&1 | \
-# RUN:   FileCheck %s --check-prefix=NO-PERMISSION -DARCHIVE=%t/permission.b
+# RUN:   FileCheck %s --check-prefix=NO-PERMISSION -DARCHIVE=%t/permission.b -DMSG=%errc_EACCES
 
-# NO-PERMISSION: error: unable to open '[[ARCHIVE]]': {{.*}}{{[pP]}}ermission denied
+# NO-PERMISSION: error: unable to open '[[ARCHIVE]]': [[MSG]]

llvm/test/tools/llvm-elfabi/fail-file-write.test

 ## Test failing to write output file on non-windows platforms.
 
 # UNSUPPORTED: system-windows
 # RUN: rm -rf %t.TestDir
 # RUN: mkdir %t.TestDir
 # RUN: touch %t.TestDir/Output.TestFile
 # RUN: chmod 400 %t.TestDir
-# RUN: not llvm-elfabi %s --output-target=elf64-little %t.TestDir/Output.TestFile 2>&1 | FileCheck %s --check-prefix=ERR
+# RUN: not llvm-elfabi %s --output-target=elf64-little %t.TestDir/Output.TestFile 2>&1 | FileCheck -DMSG=%errc_EACCES %s --check-prefix=ERR
 # RUN: chmod 777 %t.TestDir
 # RUN: rm -rf %t.TestDir
 
 --- !tapi-tbe
 TbeVersion: 1.0
 Arch: AArch64
 Symbols: {}
 ...
 
-# ERR: {{.*}}Permission denied{{.*}} when trying to open `{{.*}}.TestDir/Output.TestFile` for writing
+# ERR: [[MSG]] when trying to open `{{.*}}.TestDir/Output.TestFile` for writing

llvm/test/tools/yaml2obj/ELF/DWARF/debug-gnu-pubnames.yaml

debug_gnu_pubnames:
     Length:     0x1234
     Version:    2
     UnitOffset: 0x1234
     UnitSize:   0x4321
     Entries:    []
 
 ## h) Test that yaml2obj emits an error if 'Descriptor' is missing.
 
-# RUN: not yaml2obj --docnum=8 %s -o %t8.o 2>&1 | FileCheck %s --check-prefix=MISSING-KEY --ignore-case
+# RUN: not yaml2obj --docnum=8 %s -o %t8.o 2>&1 | FileCheck -DMSG=%errc_EINVAL %s --check-prefix=MISSING-KEY --ignore-case
 
 #      MISSING-KEY: YAML:{{.*}}:9: error: missing required key 'Descriptor'
 # MISSING-KEY-NEXT:       - DieOffset: 0x12345678
 # MISSING-KEY-NEXT:         ^
-# MISSING-KEY-NEXT: yaml2obj: error: failed to parse YAML input: Invalid argument
+# MISSING-KEY-NEXT: yaml2obj: error: failed to parse YAML input: [[MSG]]
 
 --- !ELF
 FileHeader:
   Class: ELFCLASS64
   Data:  ELFDATA2LSB
   Type:  ET_EXEC
 DWARF:
   debug_gnu_pubnames:
     Length:     0x1234
     Version:    2
     UnitOffset: 0x1234
     UnitSize:   0x4321
     Entries:
       - DieOffset: 0x12345678
         Name:      abc

llvm/utils/lit/lit/llvm/config.py

def add_tool_substitutions(self, tools, search_dirs=None):
 
         self.config.substitutions.extend(substitutions)
         return True
 
     def add_err_msg_substitutions(self):
         if (sys.platform == 'zos'):
             self.config.substitutions.append(('%errc_ENOENT', '\'EDC5129I No such file or directory.\''))
             self.config.substitutions.append(('%errc_EISDIR', '\'EDC5123I Is a directory.\''))
+            self.config.substitutions.append(('%errc_EINVAL', '\'EDC5121I Invalid argument.\''))
+            self.config.substitutions.append(('%errc_EACCES', '\'EDC5111I Permission denied.\''))
         elif (sys.platform == 'win32'):
             self.config.substitutions.append(('%errc_ENOENT', '\'no such file or directory\''))
             self.config.substitutions.append(('%errc_EISDIR', '\'is a directory\''))
+            self.config.substitutions.append(('%errc_EINVAL', '\'invalid argument\''))
+            self.config.substitutions.append(('%errc_EACCES', '\'permission denied\''))
         else:
             self.config.substitutions.append(('%errc_ENOENT', '\'No such file or directory\''))
             self.config.substitutions.append(('%errc_EISDIR', '\'Is a directory\''))
+            self.config.substitutions.append(('%errc_EINVAL', '\'Invalid argument\''))
+            self.config.substitutions.append(('%errc_EACCES', '\'Permission denied\''))
 
     def use_default_substitutions(self):
         tool_patterns = [
             ToolSubst('FileCheck', unresolved='fatal'),
             # Handle these specially as they are strings searched for during testing.
             ToolSubst(r'\| \bcount\b', command=FindTool(
                 'count'), verbatim=True, unresolved='fatal'),
             ToolSubst(r'\| \bnot\b', command=FindTool('not'), verbatim=True, unresolved='fatal')]