[clang][NFC] reformats the SARIF diagnostic test so it's human readable

[cjdb]

Summary

The current FileCheck test output is very difficult to read, which makes
adding new tests to SARIF output rather painful. I'd ideally like to
replace this test with a far more robust form of JSON testing, but am
unable to allocate resources to paying down the tech debt at present.

[cjdb]

clang/test/Frontend/sarif-diagnostics.cpp

31

This was split out for future tests where the COMMON stuff isn't written to file. Perhaps I should check it in as STDERR?

43

Aside from the reformatting, the only change to this file is the use of regex for the URIs.

[cjdb]

clang/test/Frontend/sarif-diagnostics.cpp

31

Further thoughts: everything is common, it's a case of where it's output that's not the same. I think with this in mind I should absolutely change COMMON to STDERR, and possibly CHECK to SARIF.

[cjdb]

renames check identifiers

[cjdb]

removes redundant line

[cjdb]

removes something from a future commit

This should be the final alteration pre-review.

[aaron.ballman]

LGTM, this is incremental progress. Hopefully we won't be adding too many more RUN lines to this file though (sticking too many tests into one file is also tech debt).

[cjdb]

In D145178#4167103, @aaron.ballman wrote:

LGTM, this is incremental progress. Hopefully we won't be adding too many more RUN lines to this file though (sticking too many tests into one file is also tech debt).

Thanks! 100% of the RUN lines I intend to add are bug regressions, but there will be more RUN lines in the future.

clang/test/Frontend/sarif-diagnostics.cpp

-// RUN: %clang -fsyntax-only -Wall -Wextra -fdiagnostics-format=sarif %s > %t 2>&1 || true
-// RUN: FileCheck -dump-input=always %s --input-file=%t
-
 // FIXME: this test is incredibly fragile because the `main()` function
 // must be on line 12 in order for the CHECK lines to get the correct line
 // number values.
 //
 // So these comment lines are being used to ensure the code below happens
 // to work properly for the test coverage, which as you can imagine, is not
 // the best way to structure the test. We really need to introduce a better
 // tool than FileCheck for diff'ing JSON output like SARIF.
+//
+// In order to add more tests over time, we've put the RUN lines **underneath**
+// the code.
 void main() {
   int i = hello;
 
   float test = 1a.0;
 
   if (true)
     bool Yes = true;
     return;
 
   bool j = hi;
 }
 }
 
 struct t1 { };
 void f1(t1 x, t1 y) {
     x + y;
 }
 
-// CHECK: warning: diagnostic formatting in SARIF mode is currently unstable [-Wsarif-format-unstable]
-// CHECK: {"$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json","runs":[{"artifacts":[{"length":
-// Omit exact length of this file
-// CHECK: ,"location":{"index":0,"uri":"file://
-// Omit filepath to llvm project directory
-// CHECK: test/Frontend/sarif-diagnostics.cpp"},"mimeType":"text/plain","roles":["resultFile"]}],"columnKind":"unicodeCodePoints","results":
-// CHECK: [{"level":"error","locations":[{"physicalLocation":{"artifactLocation":{"index":0,"uri":"file://
-// CHECK: {"endColumn":1,"startColumn":1,"startLine":12}}}],"message":{"text":"'main' must return 'int'"},"ruleId":"{{[0-9]+}}","ruleIndex":0},
-// CHECK: {"level":"error","locations":[{"physicalLocation":{"artifactLocation":{"index":0,"uri":"file://
-// CHECK: {"endColumn":11,"startColumn":11,"startLine":13}}}],"message":{"text":"use of undeclared identifier
-// CHECK: 'hello'"},"ruleId":"{{[0-9]+}}","ruleIndex":1},{"level":"error","locations":[{"physicalLocation":{"artifactLocation":
-// CHECK: {"index":0,"uri":"file://{{.+}}"},"region":{"endColumn":17,"startColumn":17,"startLine":15}}}],"message":{"text":"invalid digit 'a' in decimal
-// CHECK: constant"},"ruleId":"{{[0-9]+}}","ruleIndex":2},{"level":"warning","locations":[{"physicalLocation":{"artifactLocation":
-// CHECK: {"index":0,"uri":"file://{{.+}}"},"region":{"endColumn":5,"startColumn":5,"startLine":19}}}],"message":{"text":"misleading indentation; statement is not part
-// CHECK: of the previous 'if'"},"ruleId":"{{[0-9]+}}","ruleIndex":3},{"level":"note","locations":[{"physicalLocation":{"artifactLocation":
-// CHECK: {"index":0,"uri":"file://{{.+}}"},"region":{"endColumn":3,"startColumn":3,"startLine":17}}}],"message":{"text":"previous statement is
-// CHECK: here"},"ruleId":"{{[0-9]+}}","ruleIndex":4},{"level":"warning","locations":[{"physicalLocation":{"artifactLocation":
-// CHECK: {"index":0,"uri":"file://{{.+}}"},"region":{"endColumn":10,"startColumn":10,"startLine":18}}}],"message":{"text":"unused variable
-// CHECK: 'Yes'"},"ruleId":"{{[0-9]+}}","ruleIndex":5},{"level":"error","locations":[{"physicalLocation":{"artifactLocation":{"index":0,"uri":"file://
-// CHECK: {"endColumn":12,"startColumn":12,"startLine":21}}}],"message":{"text":"use of undeclared identifier
-// CHECK: 'hi'"},"ruleId":"{{[0-9]+}}","ruleIndex":6},{"level":"error","locations":[{"physicalLocation":{"artifactLocation":{"index":0,"uri":"file://
-// CHECK: {"endColumn":1,"startColumn":1,"startLine":23}}}],"message":{"text":"extraneous closing brace
-// CHECK: ('}')"},"ruleId":"{{[0-9]+}}","ruleIndex":7},{"level":"error","locations":[{"physicalLocation":{"artifactLocation":{"index":0,"uri":"file://
-// CHECK: {"endColumn":6,"endLine":27,"startColumn":5,"startLine":27}}},{"physicalLocation":{"artifactLocation":{"index":0,"uri":"file://
-// CHECK: {"endColumn":10,"endLine":27,"startColumn":9,"startLine":27}}},{"physicalLocation":{"artifactLocation":{"index":0,"uri":"file://
-// CHECK: {"endColumn":7,"startColumn":7,"startLine":27}}}],"message":{"text":"invalid operands to binary expression ('t1' and
-// CHECK: 't1')"},"ruleId":"{{[0-9]+}}","ruleIndex":8}],"tool":{"driver":{"fullName":"","informationUri":"https://clang.llvm.org/docs/
-// CHECK: UsersManual.html","language":"en-US","name":"clang","rules":[{"defaultConfiguration":
-// CHECK: {"enabled":true,"level":"error","rank":50},"fullDescription":{"text":""},"id":"{{[0-9]+}}","name":""},{"defaultConfiguration":
-// CHECK: {"enabled":true,"level":"error","rank":50},"fullDescription":{"text":""},"id":"{{[0-9]+}}","name":""},{"defaultConfiguration":
-// CHECK: {"enabled":true,"level":"error","rank":50},"fullDescription":{"text":""},"id":"{{[0-9]+}}","name":""},{"defaultConfiguration":
-// CHECK: {"enabled":true,"level":"warning","rank":-1},"fullDescription":{"text":""},"id":"{{[0-9]+}}","name":""},{"defaultConfiguration":
-// CHECK: {"enabled":true,"level":"note","rank":-1},"fullDescription":{"text":""},"id":"{{[0-9]+}}","name":""},{"defaultConfiguration":
-// CHECK: {"enabled":true,"level":"warning","rank":-1},"fullDescription":{"text":""},"id":"{{[0-9]+}}","name":""},{"defaultConfiguration":
-// CHECK: {"enabled":true,"level":"error","rank":50},"fullDescription":{"text":""},"id":"{{[0-9]+}}","name":""},{"defaultConfiguration":
-// CHECK: {"enabled":true,"level":"error","rank":50},"fullDescription":{"text":""},"id":"{{[0-9]+}}","name":""},{"defaultConfiguration":
-// CHECK: {"enabled":true,"level":"error","rank":50},"fullDescription":
-// CHECK: {"text":""},"id":"{{[0-9]+}}","name":""}],"version":"{{[0-9]+\.[0-9]+\.[0-9]+}}"}}}],"version":"2.1.0"}
-// CHECK: 2 warnings and 6 errors generated.
+// RUN: %clang -fsyntax-only -Wall -Wextra -fdiagnostics-format=sarif-stderr %s > %t.txt 2>&1 || true
+// RUN: FileCheck -dump-input=always %s --input-file=%t.txt --check-prefixes=STDERR
+// RUN: FileCheck -dump-input=always %s --input-file=%t.txt --check-prefixes=STDERR,SARIF
+
+// STDERR: warning: diagnostic formatting in SARIF mode is currently unstable [-Wsarif-format-unstable]
+// SARIF: {
+// SARIF:   "$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json",
+// SARIF:   "runs":[
+// SARIF:     {
+// SARIF:       "artifacts":[
+// SARIF:         {
+// SARIF:           "length":{{[0-9]+}},
+// SARIF:           "location":{
+// SARIF:             "index":0,
+// SARIF:             "uri":{{"file://[^"]+/clang/test/Frontend/sarif-diagnostics.cpp"}}
+// SARIF:           },
+// SARIF:           "mimeType":"text/plain",
+// SARIF:           "roles":[
+// SARIF:             "resultFile"
+// SARIF:           ]
+// SARIF:         }
+// SARIF:       ],
+// SARIF:       "columnKind":"unicodeCodePoints",
+// SARIF:       "results":[
+// SARIF:         {
+// SARIF:           "level":"error",
+// SARIF:           "locations":[
+// SARIF:             {
+// SARIF:               "physicalLocation":{
+// SARIF:                 "artifactLocation":{
+// SARIF:                   "index":0,
+// SARIF:                   "uri":{{"file://[^"]+/clang/test/Frontend/sarif-diagnostics.cpp"}}
+// SARIF:                 },
+// SARIF:                 "region":{
+// SARIF:                   "endColumn":1,
+// SARIF:                   "startColumn":1,
+// SARIF:                   "startLine":12
+// SARIF:                 }
+// SARIF:               }
+// SARIF:             }
+// SARIF:           ],
+// SARIF:           "message":{
+// SARIF:             "text":"'main' must return 'int'"
+// SARIF:           },
+// SARIF:           "ruleId":"3485",
+// SARIF:           "ruleIndex":0
+// SARIF:         },
+// SARIF:         {
+// SARIF:           "level":"error",
+// SARIF:           "locations":[
+// SARIF:             {
+// SARIF:               "physicalLocation":{
+// SARIF:                 "artifactLocation":{
+// SARIF:                   "index":0,
+// SARIF:                   "uri":{{"file://[^"]+/clang/test/Frontend/sarif-diagnostics.cpp"}}
+// SARIF:                 },
+// SARIF:                 "region":{
+// SARIF:                   "endColumn":11,
+// SARIF:                   "startColumn":11,
+// SARIF:                   "startLine":13
+// SARIF:                 }
+// SARIF:               }
+// SARIF:             }
+// SARIF:           ],
+// SARIF:           "message":{
+// SARIF:             "text":"use of undeclared identifier 'hello'"
+// SARIF:           },
+// SARIF:           "ruleId":"4632",
+// SARIF:           "ruleIndex":1
+// SARIF:         },
+// SARIF:         {
+// SARIF:           "level":"error",
+// SARIF:           "locations":[
+// SARIF:             {
+// SARIF:               "physicalLocation":{
+// SARIF:                 "artifactLocation":{
+// SARIF:                   "index":0,
+// SARIF:                   "uri":{{"file://[^"]+/clang/test/Frontend/sarif-diagnostics.cpp"}}
+// SARIF:                 },
+// SARIF:                 "region":{
+// SARIF:                   "endColumn":17,
+// SARIF:                   "startColumn":17,
+// SARIF:                   "startLine":15
+// SARIF:                 }
+// SARIF:               }
+// SARIF:             }
+// SARIF:           ],
+// SARIF:           "message":{
+// SARIF:             "text":"invalid digit 'a' in decimal constant"
+// SARIF:           },
+// SARIF:           "ruleId":"898",
+// SARIF:           "ruleIndex":2
+// SARIF:         },
+// SARIF:         {
+// SARIF:           "level":"warning",
+// SARIF:           "locations":[
+// SARIF:             {
+// SARIF:               "physicalLocation":{
+// SARIF:                 "artifactLocation":{
+// SARIF:                   "index":0,
+// SARIF:                   "uri":{{"file://[^"]+clang/test/Frontend/sarif-diagnostics.cpp"}}
+// SARIF:                 },
+// SARIF:                 "region":{
+// SARIF:                   "endColumn":5,
+// SARIF:                   "startColumn":5,
+// SARIF:                   "startLine":19
+// SARIF:                 }
+// SARIF:               }
+// SARIF:             }
+// SARIF:           ],
+// SARIF:           "message":{
+// SARIF:             "text":"misleading indentation; statement is not part of the previous 'if'"
+// SARIF:           },
+// SARIF:           "ruleId":"1826",
+// SARIF:           "ruleIndex":3
+// SARIF:         },
+// SARIF:         {
+// SARIF:           "level":"note",
+// SARIF:           "locations":[
+// SARIF:             {
+// SARIF:               "physicalLocation":{
+// SARIF:                 "artifactLocation":{
+// SARIF:                   "index":0,
+// SARIF:                   "uri":{{"file://[^"]+/clang/test/Frontend/sarif-diagnostics.cpp"}}
+// SARIF:                 },
+// SARIF:                 "region":{
+// SARIF:                   "endColumn":3,
+// SARIF:                   "startColumn":3,
+// SARIF:                   "startLine":17
+// SARIF:                 }
+// SARIF:               }
+// SARIF:             }
+// SARIF:           ],
+// SARIF:           "message":{
+// SARIF:             "text":"previous statement is here"
+// SARIF:           },
+// SARIF:           "ruleId":"1746",
+// SARIF:           "ruleIndex":4
+// SARIF:         },
+// SARIF:         {
+// SARIF:           "level":"warning",
+// SARIF:           "locations":[
+// SARIF:             {
+// SARIF:               "physicalLocation":{
+// SARIF:                 "artifactLocation":{
+// SARIF:                   "index":0,
+// SARIF:                   "uri":{{"file://[^"]+/clang/test/Frontend/sarif-diagnostics.cpp"}}
+// SARIF:                 },
+// SARIF:                 "region":{
+// SARIF:                   "endColumn":10,
+// SARIF:                   "startColumn":10,
+// SARIF:                   "startLine":18
+// SARIF:                 }
+// SARIF:               }
+// SARIF:             }
+// SARIF:           ],
+// SARIF:           "message":{
+// SARIF:             "text":"unused variable 'Yes'"
+// SARIF:           },
+// SARIF:           "ruleId":"6593",
+// SARIF:           "ruleIndex":5
+// SARIF:         },
+// SARIF:         {
+// SARIF:           "level":"error",
+// SARIF:           "locations":[
+// SARIF:             {
+// SARIF:               "physicalLocation":{
+// SARIF:                 "artifactLocation":{
+// SARIF:                   "index":0,
+// SARIF:                   "uri":{{"file://[^"]+/clang/test/Frontend/sarif-diagnostics.cpp"}}
+// SARIF:                 },
+// SARIF:                 "region":{
+// SARIF:                   "endColumn":12,
+// SARIF:                   "startColumn":12,
+// SARIF:                   "startLine":21
+// SARIF:                 }
+// SARIF:               }
+// SARIF:             }
+// SARIF:           ],
+// SARIF:           "message":{
+// SARIF:             "text":"use of undeclared identifier 'hi'"
+// SARIF:           },
+// SARIF:           "ruleId":"4632",
+// SARIF:           "ruleIndex":6
+// SARIF:         },
+// SARIF:         {
+// SARIF:           "level":"error",
+// SARIF:           "locations":[
+// SARIF:             {
+// SARIF:               "physicalLocation":{
+// SARIF:                 "artifactLocation":{
+// SARIF:                   "index":0,
+// SARIF:                   "uri":{{"file://[^"]+/clang/test/Frontend/sarif-diagnostics.cpp"}}
+// SARIF:                 },
+// SARIF:                 "region":{
+// SARIF:                   "endColumn":1,
+// SARIF:                   "startColumn":1,
+// SARIF:                   "startLine":23
+// SARIF:                 }
+// SARIF:               }
+// SARIF:             }
+// SARIF:           ],
+// SARIF:           "message":{
+// SARIF:             "text":"extraneous closing brace ('}')"
+// SARIF:           },
+// SARIF:           "ruleId":"1400",
+// SARIF:           "ruleIndex":7
+// SARIF:         },
+// SARIF:         {
+// SARIF:           "level":"error",
+// SARIF:           "locations":[
+// SARIF:             {
+// SARIF:               "physicalLocation":{
+// SARIF:                 "artifactLocation":{
+// SARIF:                   "index":0,
+// SARIF:                   "uri":{{"file://[^"]+/clang/test/Frontend/sarif-diagnostics.cpp"}}
+// SARIF:                 },
+// SARIF:                 "region":{
+// SARIF:                   "endColumn":6,
+// SARIF:                   "endLine":27,
+// SARIF:                   "startColumn":5,
+// SARIF:                   "startLine":27
+// SARIF:                 }
+// SARIF:               }
+// SARIF:             },
+// SARIF:             {
+// SARIF:               "physicalLocation":{
+// SARIF:                 "artifactLocation":{
+// SARIF:                   "index":0,
+// SARIF:                   "uri":{{"file://[^"]+/clang/test/Frontend/sarif-diagnostics.cpp"}}
+// SARIF:                 },
+// SARIF:                 "region":{
+// SARIF:                   "endColumn":10,
+// SARIF:                   "endLine":27,
+// SARIF:                   "startColumn":9,
+// SARIF:                   "startLine":27
+// SARIF:                 }
+// SARIF:               }
+// SARIF:             },
+// SARIF:             {
+// SARIF:               "physicalLocation":{
+// SARIF:                 "artifactLocation":{
+// SARIF:                   "index":0,
+// SARIF:                   "uri":{{"file://[^"]+/clang/test/Frontend/sarif-diagnostics.cpp"}}
+// SARIF:                 },
+// SARIF:                 "region":{
+// SARIF:                   "endColumn":7,
+// SARIF:                   "startColumn":7,
+// SARIF:                   "startLine":27
+// SARIF:                 }
+// SARIF:               }
+// SARIF:             }
+// SARIF:           ],
+// SARIF:           "message":{
+// SARIF:             "text":"invalid operands to binary expression ('t1' and 't1')"
+// SARIF:           },
+// SARIF:           "ruleId":"4567",
+// SARIF:           "ruleIndex":8
+// SARIF:         }
+// SARIF:       ],
+// SARIF:       "tool":{
+// SARIF:         "driver":{
+// SARIF:           "fullName":"",
+// SARIF:           "informationUri":"https://clang.llvm.org/docs/UsersManual.html",
+// SARIF:           "language":"en-US",
+// SARIF:           "name":"clang",
+// SARIF:           "rules":[
+// SARIF:             {
+// SARIF:               "defaultConfiguration":{
+// SARIF:                 "enabled":true,
+// SARIF:                 "level":"error",
+// SARIF:                 "rank":50
+// SARIF:               },
+// SARIF:               "fullDescription":{
+// SARIF:                 "text":""
+// SARIF:               },
+// SARIF:               "id":"3485",
+// SARIF:               "name":""
+// SARIF:             },
+// SARIF:             {
+// SARIF:               "defaultConfiguration":{
+// SARIF:                 "enabled":true,
+// SARIF:                 "level":"error",
+// SARIF:                 "rank":50
+// SARIF:               },
+// SARIF:               "fullDescription":{
+// SARIF:                 "text":""
+// SARIF:               },
+// SARIF:               "id":"4632",
+// SARIF:               "name":""
+// SARIF:             },
+// SARIF:             {
+// SARIF:               "defaultConfiguration":{
+// SARIF:                 "enabled":true,
+// SARIF:                 "level":"error",
+// SARIF:                 "rank":50
+// SARIF:               },
+// SARIF:               "fullDescription":{
+// SARIF:                 "text":""
+// SARIF:               },
+// SARIF:               "id":"898",
+// SARIF:               "name":""
+// SARIF:             },
+// SARIF:             {
+// SARIF:               "defaultConfiguration":{
+// SARIF:                 "enabled":true,
+// SARIF:                 "level":"warning",
+// SARIF:                 "rank":-1
+// SARIF:               },
+// SARIF:               "fullDescription":{
+// SARIF:                 "text":""
+// SARIF:               },
+// SARIF:               "id":"1826",
+// SARIF:               "name":""
+// SARIF:             },
+// SARIF:             {
+// SARIF:               "defaultConfiguration":{
+// SARIF:                 "enabled":true,
+// SARIF:                 "level":"note",
+// SARIF:                 "rank":-1
+// SARIF:               },
+// SARIF:               "fullDescription":{
+// SARIF:                 "text":""
+// SARIF:               },
+// SARIF:               "id":"1746",
+// SARIF:               "name":""
+// SARIF:             },
+// SARIF:             {
+// SARIF:               "defaultConfiguration":{
+// SARIF:                 "enabled":true,
+// SARIF:                 "level":"warning",
+// SARIF:                 "rank":-1
+// SARIF:               },
+// SARIF:               "fullDescription":{
+// SARIF:                 "text":""
+// SARIF:               },
+// SARIF:               "id":"6593",
+// SARIF:               "name":""
+// SARIF:             },
+// SARIF:             {
+// SARIF:               "defaultConfiguration":{
+// SARIF:                 "enabled":true,
+// SARIF:                 "level":"error",
+// SARIF:                 "rank":50
+// SARIF:               },
+// SARIF:               "fullDescription":{
+// SARIF:                 "text":""
+// SARIF:               },
+// SARIF:               "id":"4632",
+// SARIF:               "name":""
+// SARIF:             },
+// SARIF:             {
+// SARIF:               "defaultConfiguration":{
+// SARIF:                 "enabled":true,
+// SARIF:                 "level":"error",
+// SARIF:                 "rank":50
+// SARIF:               },
+// SARIF:               "fullDescription":{
+// SARIF:                 "text":""
+// SARIF:               },
+// SARIF:               "id":"1400",
+// SARIF:               "name":""
+// SARIF:             },
+// SARIF:             {
+// SARIF:               "defaultConfiguration":{
+// SARIF:                 "enabled":true,
+// SARIF:                 "level":"error",
+// SARIF:                 "rank":50
+// SARIF:               },
+// SARIF:               "fullDescription":{
+// SARIF:                 "text":""
+// SARIF:               },
+// SARIF:               "id":"4567",
+// SARIF:               "name":""
+// SARIF:             }
+// SARIF:           ],
+// SARIF:           "version":"17.0.0"
+// SARIF:         }
+// SARIF:       }
+// SARIF:     }
+// SARIF:   ],
+// SARIF:   "version":"2.1.0"
+// SARIF: }
+// STDERR: 2 warnings and 6 errors generated.