[lit] Print full googletest commad line

[vitalybuka]

Summary

Similar to regular output of LIT tests:
https://github.com/llvm/llvm-project/blob/c162f086ba632ffaedfe92d63bf21571bc8ae4da/llvm/utils/lit/lit/TestRunner.py#L1569

[delcypher]

The patch seems reasonable. I have a minor nit about the use of the % operator but other than that it seems fine.

llvm/utils/lit/lit/formats/googletest.py

125

Minor nit. This patch is mixing the old python string interpolation (% operator) and the format function.

Given that Python 3.6 is the minimum maybe we should just use format strings in both places e.g.

header = f"Script:\n--\n{' '.join(cmd)}\n--\n"
...
return (lit.test.TIMEOUT, f"{header}Reached timeout of {litConfig.maxIndividualTestTime} seconds")

[vitalybuka]

upgrade formating

[vitalybuka]

Add UNRESOLVED test

[vitalybuka]

rebase

[rnk]

I see test failures on Windows that appear to be related to CRLF handling. I think the CHECK-NEXT checks aren't working because something is doubling up the newlines in the test output. That seems unrelated, so I suggest dropping the -NEXTs as necessary to get it past the presubmit testing.

[vitalybuka]

rebase

[vitalybuka]

it's just rebase so far, windows is not fixed
it does not reproduce for me so I will wait for another pre-merge

llvm/utils/lit/lit/formats/googletest.py

138–139

I assume we don't need it here

[vitalybuka]

fix test on Windows

[vitalybuka]

Issue is in the path separator.
lines like FAIL: googletest-upstream-format :: [[PATH:[Dd]ummy[Ss]ub[Dd]ir/]][[FILE:OneTest\.py]]/[[TEST:FirstTest\.subTestB]]
always have /
and command line is \

[vitalybuka]

Reuse vars in tests

[vitalybuka]

"x64 windows passed" now

[rnk]

Classic.

lgtm

llvm/utils/lit/lit/formats/googletest.py

def execute(self, test, litConfig):
         cmd = [testPath, '--gtest_filter=' + testName]
         cmd = self.prepareCmd(cmd)
         if litConfig.useValgrind:
             cmd = litConfig.valgrindArgs + cmd
 
         if litConfig.noExecute:
             return lit.Test.PASS, ''
 
+        header = f"Script:\n--\n{' '.join(cmd)}\n--\n"
+
         try:
             out, err, exitCode = lit.util.executeCommand(
                 cmd, env=test.config.environment,
                 timeout=litConfig.maxIndividualTestTime)
         except lit.util.ExecuteCommandTimeoutException:
             return (lit.Test.TIMEOUT,
-                    'Reached timeout of {} seconds'.format(
-                        litConfig.maxIndividualTestTime)
-                   )
+                    f'{header}Reached timeout of '
+                    f'{litConfig.maxIndividualTestTime} seconds')
 
         if exitCode:
-            return lit.Test.FAIL, out + err
+            return lit.Test.FAIL, header + out + err
 
         passing_test_line = '[  PASSED  ] 1 test.'
         if passing_test_line not in out:
-            msg = ('Unable to find %r in gtest output:\n\n%s%s' %
-                   (passing_test_line, out, err))
-            return lit.Test.UNRESOLVED, msg
+            return (lit.Test.UNRESOLVED,
+                    f'{header}Unable to find {passing_test_line} '
+                    f'in gtest output:\n\n{out}{err}')
 
         return lit.Test.PASS,''
 
     def prepareCmd(self, cmd):
         """Insert interpreter if needed.
 
         It inserts the python exe into the command if cmd[0] ends in .py or caller
         specified run_under.

llvm/utils/lit/tests/Inputs/googletest-format/DummySubDir/OneTest.py

 #!/usr/bin/env python
 
 import sys
 
 if len(sys.argv) != 2:
     raise ValueError("unexpected number of args")
 
 if sys.argv[1] == "--gtest_list_tests":
     print("""\
 FirstTest.
   subTestA
   subTestB
+  subTestC
 ParameterizedTest/0.
   subTest
 ParameterizedTest/1.
   subTest""")
     sys.exit(0)
 elif not sys.argv[1].startswith("--gtest_filter="):
     raise ValueError("unexpected argument: %r" % (sys.argv[1]))
 
 test_name = sys.argv[1].split('=',1)[1]
 if test_name == 'FirstTest.subTestA':
     print('I am subTest A, I PASS')
     print('[  PASSED  ] 1 test.')
     sys.exit(0)
 elif test_name == 'FirstTest.subTestB':
     print('I am subTest B, I FAIL')
     print('And I have two lines of output')
     sys.exit(1)
+elif test_name == 'FirstTest.subTestC':
+    print('I am subTest C, I am UNRESOLVED')
+    sys.exit(0)
 elif test_name in ('ParameterizedTest/0.subTest',
                    'ParameterizedTest/1.subTest'):
     print('I am a parameterized test, I also PASS')
     print('[  PASSED  ] 1 test.')
     sys.exit(0)
 else:
     raise SystemExit("error: invalid test name: %r" % (test_name,))

llvm/utils/lit/tests/Inputs/googletest-upstream-format/DummySubDir/OneTest.py

 #!/usr/bin/env python
 
 import sys
 
 if len(sys.argv) != 2:
     raise ValueError("unexpected number of args")
 
 if sys.argv[1] == "--gtest_list_tests":
     print("""\
 Running main() from gtest_main.cc
 FirstTest.
   subTestA
   subTestB
+  subTestC
 ParameterizedTest/0.
   subTest
 ParameterizedTest/1.
   subTest""")
     sys.exit(0)
 elif not sys.argv[1].startswith("--gtest_filter="):
     raise ValueError("unexpected argument: %r" % (sys.argv[1]))
 
 test_name = sys.argv[1].split('=',1)[1]
 print('Running main() from gtest_main.cc')
 if test_name == 'FirstTest.subTestA':
     print('I am subTest A, I PASS')
     print('[  PASSED  ] 1 test.')
     sys.exit(0)
 elif test_name == 'FirstTest.subTestB':
     print('I am subTest B, I FAIL')
     print('And I have two lines of output')
     sys.exit(1)
+elif test_name == 'FirstTest.subTestC':
+    print('I am subTest C, I am UNRESOLVED')
+    sys.exit(0)
 elif test_name in ('ParameterizedTest/0.subTest',
                    'ParameterizedTest/1.subTest'):
     print('I am a parameterized test, I also PASS')
     print('[  PASSED  ] 1 test.')
     sys.exit(0)
 else:
     raise SystemExit("error: invalid test name: %r" % (test_name,))

llvm/utils/lit/tests/googletest-format.py

 # Check the various features of the GoogleTest format.
 
 # FIXME: this test depends on order of tests
 # RUN: rm -f %{inputs}/googletest-format/.lit_test_times.txt
 
 # RUN: not %{lit} -j 1 -v %{inputs}/googletest-format > %t.out
 # FIXME: Temporarily dump test output so we can debug failing tests on
 # buildbots.
 # RUN: cat %t.out
 # RUN: FileCheck < %t.out %s
 #
 # END.
 
 # CHECK: -- Testing:
 # CHECK: PASS: googletest-format :: {{[Dd]ummy[Ss]ub[Dd]ir}}/OneTest.py/FirstTest.subTestA
-# CHECK: FAIL: googletest-format :: {{[Dd]ummy[Ss]ub[Dd]ir}}/OneTest.py/FirstTest.subTestB
-# CHECK-NEXT: *** TEST 'googletest-format :: {{[Dd]ummy[Ss]ub[Dd]ir}}/OneTest.py/FirstTest.subTestB' FAILED ***
+# CHECK: FAIL: googletest-format :: [[TEST:[Dd]ummy[Ss]ub[Dd]ir/OneTest.py]]/[[FILTER:FirstTest.subTestB]]
+# CHECK-NEXT: *** TEST 'googletest-format :: [[TEST]]/[[FILTER]]' FAILED ***
+# CHECK-NEXT: Script:
+# CHECK-NEXT: --
+# CHECK-NEXT: [[TEST]] --gtest_filter=[[FILTER]]
+# CHECK-NEXT: --
 # CHECK-NEXT: I am subTest B, I FAIL
 # CHECK-NEXT: And I have two lines of output
+# CHECK: UNRESOLVED: googletest-format :: [[TEST:[Dd]ummy[Ss]ub[Dd]ir/OneTest.py]]/[[FILTER:FirstTest.subTestC]]
+# CHECK-NEXT: *** TEST 'googletest-format :: [[TEST]]/[[FILTER]]' FAILED ***
+# CHECK-NEXT: Script:
+# CHECK-NEXT: --
+# CHECK-NEXT: [[TEST]] --gtest_filter=[[FILTER]]
+# CHECK-NEXT: --
+# CHECK-NEXT: Unable to find [ PASSED ] 1 test. in gtest output
+# CHECK: I am subTest C, I am UNRESOLVED
 # CHECK: ***
 # CHECK: PASS: googletest-format :: {{[Dd]ummy[Ss]ub[Dd]ir}}/OneTest.py/ParameterizedTest/0.subTest
 # CHECK: PASS: googletest-format :: {{[Dd]ummy[Ss]ub[Dd]ir}}/OneTest.py/ParameterizedTest/1.subTest
 # CHECK: Failed Tests (1)
-# CHECK: Passed: 3
-# CHECK: Failed: 1
+# CHECK: Passed : 3
+# CHECK: Unresolved: 1
+# CHECK: Failed : 1
 

llvm/utils/lit/tests/googletest-timeout.py

 # the configuration file
 #
 # RUN: not %{lit} -j 1 -v %{inputs}/googletest-timeout \
 # RUN:  --filter=InfiniteLoopSubTest  --param set_timeout=1 \
 # RUN:  > %t.cfgset.out
 # RUN: FileCheck --check-prefix=CHECK-INF < %t.cfgset.out %s
 
 # CHECK-INF: -- Testing:
-# CHECK-INF: TIMEOUT: googletest-timeout :: {{[Dd]ummy[Ss]ub[Dd]ir}}/OneTest.py/T.InfiniteLoopSubTest
+# CHECK-INF: TIMEOUT: googletest-timeout :: [[TEST:[Dd]ummy[Ss]ub[Dd]ir/OneTest\.py]]/[[FILTER:T\.InfiniteLoopSubTest]]
+# CHECK-INF-NEXT: ******************** TEST 'googletest-timeout :: [[TEST]]/[[FILTER]]' FAILED ********************
+# CHECK-INF-NEXT: Script:
+# CHECK-INF-NEXT: --
+# CHECK-INF-NEXT: [[TEST]] --gtest_filter=[[FILTER]]
+# CHECK-INF-NEXT: --
 # CHECK-INF: Timed Out: 1
 
 ###############################################################################
 # Check tests can complete with a timeout set
 #
 # `QuickSubTest` should execute quickly so we shouldn't wait anywhere near the
 # 3600 second timeout.
 ###############################################################################

llvm/utils/lit/tests/googletest-upstream-format.py

 # Check the various features of the GoogleTest format.
 
 # FIXME: this test depends on order of tests
 # RUN: rm -f %{inputs}/googletest-upstream-format/.lit_test_times.txt
 
 # RUN: not %{lit} -j 1 -v %{inputs}/googletest-upstream-format > %t.out
 # RUN: FileCheck < %t.out %s
 #
 # END.
 
 # CHECK: -- Testing:
 # CHECK: PASS: googletest-upstream-format :: {{[Dd]ummy[Ss]ub[Dd]ir}}/OneTest.py/FirstTest.subTestA
-# CHECK: FAIL: googletest-upstream-format :: {{[Dd]ummy[Ss]ub[Dd]ir}}/OneTest.py/FirstTest.subTestB
-# CHECK-NEXT: *** TEST 'googletest-upstream-format :: {{[Dd]ummy[Ss]ub[Dd]ir}}/OneTest.py/FirstTest.subTestB' FAILED ***
+# CHECK: FAIL: googletest-upstream-format :: [[TEST:[Dd]ummy[Ss]ub[Dd]ir/OneTest.py]]/[[FILTER:FirstTest.subTestB]]
+# CHECK-NEXT: *** TEST 'googletest-upstream-format :: [[TEST]]/[[FILTER]]' FAILED ***
+# CHECK-NEXT: Script:
+# CHECK-NEXT: --
+# CHECK-NEXT: [[TEST]] --gtest_filter=[[FILTER]]
+# CHECK-NEXT: --
 # CHECK-NEXT: Running main() from gtest_main.cc
 # CHECK-NEXT: I am subTest B, I FAIL
 # CHECK-NEXT: And I have two lines of output
+# CHECK: UNRESOLVED: googletest-upstream-format :: [[TEST:[Dd]ummy[Ss]ub[Dd]ir/OneTest.py]]/[[FILTER:FirstTest.subTestC]]
+# CHECK-NEXT: *** TEST 'googletest-upstream-format :: [[TEST]]/[[FILTER]]' FAILED ***
+# CHECK-NEXT: Script:
+# CHECK-NEXT: --
+# CHECK-NEXT: [[TEST]] --gtest_filter=[[FILTER]]
+# CHECK-NEXT: --
+# CHECK-NEXT: Unable to find [ PASSED ] 1 test. in gtest output
+# CHECK: I am subTest C, I am UNRESOLVED
 # CHECK: ***
 # CHECK: PASS: googletest-upstream-format :: {{[Dd]ummy[Ss]ub[Dd]ir}}/OneTest.py/ParameterizedTest/0.subTest
 # CHECK: PASS: googletest-upstream-format :: {{[Dd]ummy[Ss]ub[Dd]ir}}/OneTest.py/ParameterizedTest/1.subTest
 # CHECK: Failed Tests (1)
-# CHECK: Passed: 3
-# CHECK: Failed: 1
+# CHECK: Passed : 3
+# CHECK: Unresolved: 1
+# CHECK: Failed : 1