[lit] Port googletest lit tests to Windows

[rnk]

Summary

The technique of directly calling subprocess.Popen on a python script
doesn't work on Windows. The executable path of the command must refer
to a valid win32 executable.

Instead, rename all the python scripts masquerading as gtest executables
to have .py extensions, so we can easily detect then and call the python
executable for them. Do this on Linux as well as Windows for
consistency.

Fixes PR33933

[modocache]

LGTM, but I want to double-check this passes for me locally. Also, I think you may have left some debugging code in here.

llvm/utils/lit/lit/formats/googletest.py

92 ↗

(On Diff #108340)

My guess is that you may have added this for your own debugging? Or is this intentional?

llvm/utils/lit/tests/googletest-format.py

9 ↗

(On Diff #108340)

This is interesting, why do we need to handle upper- and lower-case here? Could you explain this in the commit message?

[rnk]

llvm/utils/lit/lit/formats/googletest.py

92 ↗

(On Diff #108340)

Yep, removed.

llvm/utils/lit/tests/googletest-format.py

9 ↗

(On Diff #108340)

Because of this code:

class GoogleTest(TestFormat):
    def __init__(self, test_sub_dirs, test_suffix):
        self.test_sub_dirs = os.path.normcase(str(test_sub_dirs)).split(';')

Maybe we don't need it?

[rnk]

• remove debugging print

[rnk]

In D35909#821935, @modocache wrote:

LGTM, but I want to double-check this passes for me locally. Also, I think you may have left some debugging code in here.

Should I land the updated patch?

[modocache]

Yup! Sorry for the wait. This works for me! :)

[rnk]

I see some failure on the modules buildbot: http://lab.llvm.org:8011/builders/clang-x86_64-linux-selfhost-modules-2/builds/10162

llvm/trunk/utils/lit/lit/formats/googletest.py

 from __future__ import absolute_import
 import os
 import subprocess
 import sys
 
 import lit.Test
 import lit.TestRunner
 import lit.util
 from .base import TestFormat
 
 kIsWindows = sys.platform in ['win32', 'cygwin']
 
 class GoogleTest(TestFormat):
     def __init__(self, test_sub_dirs, test_suffix):
         self.test_sub_dirs = os.path.normcase(str(test_sub_dirs)).split(';')
-        self.test_suffix = str(test_suffix)
 
         # On Windows, assume tests will also end in '.exe'.
+        exe_suffix = str(test_suffix)
         if kIsWindows:
-            self.test_suffix += '.exe'
+            exe_suffix += '.exe'
+
+        # Also check for .py files for testing purposes.
+        self.test_suffixes = {exe_suffix, test_suffix + '.py'}
 
     def getGTestTests(self, path, litConfig, localConfig):
         """getGTestTests(path) - [name]
 
         Return the tests available in gtest executable.
 
         Args:
           path: String path to a gtest executable
           litConfig: LitConfig instance
           localConfig: TestingConfig instance"""
 
+        list_test_cmd = self.maybeAddPythonToCmd([path, '--gtest_list_tests'])
+
         try:
-            output = subprocess.check_output([path, '--gtest_list_tests'],
+            output = subprocess.check_output(list_test_cmd,
                                              env=localConfig.environment)
         except subprocess.CalledProcessError as exc:
             litConfig.warning(
                 "unable to discover google-tests in %r: %s. Process output: %s"
                 % (path, sys.exc_info()[1], exc.output))
             raise StopIteration
 
         nested_tests = []
class GoogleTest(TestFormat):
     def getTestsInDirectory(self, testSuite, path_in_suite,
                             litConfig, localConfig):
         source_path = testSuite.getSourcePath(path_in_suite)
         for subdir in self.test_sub_dirs:
             dir_path = os.path.join(source_path, subdir)
             if not os.path.isdir(dir_path):
                 continue
             for fn in lit.util.listdir_files(dir_path,
-                                             suffixes={self.test_suffix}):
+                                             suffixes=self.test_suffixes):
                 # Discover the tests in this executable.
                 execpath = os.path.join(source_path, subdir, fn)
                 testnames = self.getGTestTests(execpath, litConfig, localConfig)
                 for testname in testnames:
                     testPath = path_in_suite + (subdir, fn, testname)
                     yield lit.Test.Test(testSuite, testPath, localConfig,
                                         file_path=execpath)
 
     def execute(self, test, litConfig):
         testPath,testName = os.path.split(test.getSourcePath())
         while not os.path.exists(testPath):
             # Handle GTest parametrized and typed tests, whose name includes
             # some '/'s.
             testPath, namePrefix = os.path.split(testPath)
             testName = namePrefix + '/' + testName
 
         cmd = [testPath, '--gtest_filter=' + testName]
+        cmd = self.maybeAddPythonToCmd(cmd)
         if litConfig.useValgrind:
             cmd = litConfig.valgrindArgs + cmd
 
         if litConfig.noExecute:
             return lit.Test.PASS, ''
 
         try:
             out, err, exitCode = lit.util.executeCommand(
def execute(self, test, litConfig):
 
         passing_test_line = '[  PASSED  ] 1 test.'
         if passing_test_line not in out:
             msg = ('Unable to find %r in gtest output:\n\n%s%s' %
                    (passing_test_line, out, err))
             return lit.Test.UNRESOLVED, msg
 
         return lit.Test.PASS,''
+
+    def maybeAddPythonToCmd(self, cmd):
+        """Insert the python exe into the command if cmd[0] ends in .py
+
+        We cannot rely on the system to interpret shebang lines for us on
+        Windows, so add the python executable to the command if this is a .py
+        script.
+        """
+        if cmd[0].endswith('.py'):
+            return [sys.executable] + cmd
+        return cmd

llvm/trunk/utils/lit/tests/Inputs/googletest-format/DummySubDir/OneTest

-#!/usr/bin/env python
-
-import sys
-
-if len(sys.argv) != 2:
-    raise ValueError("unexpected number of args")
-
-if sys.argv[1] == "--gtest_list_tests":
-    print("""\
-FirstTest.
-  subTestA
-  subTestB
-ParameterizedTest/0.
-  subTest
-ParameterizedTest/1.
-  subTest""")
-    sys.exit(0)
-elif not sys.argv[1].startswith("--gtest_filter="):
-    raise ValueError("unexpected argument: %r" % (sys.argv[1]))
-
-test_name = sys.argv[1].split('=',1)[1]
-if test_name == 'FirstTest.subTestA':
-    print('I am subTest A, I PASS')
-    print('[  PASSED  ] 1 test.')
-    sys.exit(0)
-elif test_name == 'FirstTest.subTestB':
-    print('I am subTest B, I FAIL')
-    print('And I have two lines of output')
-    sys.exit(1)
-elif test_name in ('ParameterizedTest/0.subTest',
-                   'ParameterizedTest/1.subTest'):
-    print('I am a parameterized test, I also PASS')
-    print('[  PASSED  ] 1 test.')
-    sys.exit(0)
-else:
-    raise SystemExit("error: invalid test name: %r" % (test_name,))

llvm/trunk/utils/lit/tests/Inputs/googletest-format/DummySubDir/OneTest.py

+#!/usr/bin/env python
+
+import sys
+
+if len(sys.argv) != 2:
+    raise ValueError("unexpected number of args")
+
+if sys.argv[1] == "--gtest_list_tests":
+    print("""\
+FirstTest.
+  subTestA
+  subTestB
+ParameterizedTest/0.
+  subTest
+ParameterizedTest/1.
+  subTest""")
+    sys.exit(0)
+elif not sys.argv[1].startswith("--gtest_filter="):
+    raise ValueError("unexpected argument: %r" % (sys.argv[1]))
+
+test_name = sys.argv[1].split('=',1)[1]
+if test_name == 'FirstTest.subTestA':
+    print('I am subTest A, I PASS')
+    print('[  PASSED  ] 1 test.')
+    sys.exit(0)
+elif test_name == 'FirstTest.subTestB':
+    print('I am subTest B, I FAIL')
+    print('And I have two lines of output')
+    sys.exit(1)
+elif test_name in ('ParameterizedTest/0.subTest',
+                   'ParameterizedTest/1.subTest'):
+    print('I am a parameterized test, I also PASS')
+    print('[  PASSED  ] 1 test.')
+    sys.exit(0)
+else:
+    raise SystemExit("error: invalid test name: %r" % (test_name,))

llvm/trunk/utils/lit/tests/Inputs/googletest-timeout/DummySubDir/OneTest

-#!/usr/bin/env python
-
-import sys
-import time
-
-if len(sys.argv) != 2:
-    raise ValueError("unexpected number of args")
-
-if sys.argv[1] == "--gtest_list_tests":
-    print("""\
-FirstTest.
-  subTestA
-  subTestB
-  subTestC
-""")
-    sys.exit(0)
-elif not sys.argv[1].startswith("--gtest_filter="):
-    raise ValueError("unexpected argument: %r" % (sys.argv[1]))
-
-test_name = sys.argv[1].split('=',1)[1]
-if test_name == 'FirstTest.subTestA':
-    print('I am subTest A, I PASS')
-    print('[  PASSED  ] 1 test.')
-    sys.exit(0)
-elif test_name == 'FirstTest.subTestB':
-    print('I am subTest B, I am slow')
-    time.sleep(6)
-    print('[  PASSED  ] 1 test.')
-    sys.exit(0)
-elif test_name == 'FirstTest.subTestC':
-    print('I am subTest C, I will hang')
-    while True:
-        pass
-else:
-    raise SystemExit("error: invalid test name: %r" % (test_name,))

llvm/trunk/utils/lit/tests/Inputs/googletest-timeout/DummySubDir/OneTest.py

+#!/usr/bin/env python
+
+import sys
+import time
+
+if len(sys.argv) != 2:
+    raise ValueError("unexpected number of args")
+
+if sys.argv[1] == "--gtest_list_tests":
+    print("""\
+FirstTest.
+  subTestA
+  subTestB
+  subTestC
+""")
+    sys.exit(0)
+elif not sys.argv[1].startswith("--gtest_filter="):
+    raise ValueError("unexpected argument: %r" % (sys.argv[1]))
+
+test_name = sys.argv[1].split('=',1)[1]
+if test_name == 'FirstTest.subTestA':
+    print('I am subTest A, I PASS')
+    print('[  PASSED  ] 1 test.')
+    sys.exit(0)
+elif test_name == 'FirstTest.subTestB':
+    print('I am subTest B, I am slow')
+    time.sleep(6)
+    print('[  PASSED  ] 1 test.')
+    sys.exit(0)
+elif test_name == 'FirstTest.subTestC':
+    print('I am subTest C, I will hang')
+    while True:
+        pass
+else:
+    raise SystemExit("error: invalid test name: %r" % (test_name,))

llvm/trunk/utils/lit/tests/Inputs/googletest-upstream-format/DummySubDir/OneTest

-#!/usr/bin/env python
-
-import sys
-
-if len(sys.argv) != 2:
-    raise ValueError("unexpected number of args")
-
-if sys.argv[1] == "--gtest_list_tests":
-    print("""\
-Running main() from gtest_main.cc
-FirstTest.
-  subTestA
-  subTestB
-ParameterizedTest/0.
-  subTest
-ParameterizedTest/1.
-  subTest""")
-    sys.exit(0)
-elif not sys.argv[1].startswith("--gtest_filter="):
-    raise ValueError("unexpected argument: %r" % (sys.argv[1]))
-
-test_name = sys.argv[1].split('=',1)[1]
-print('Running main() from gtest_main.cc')
-if test_name == 'FirstTest.subTestA':
-    print('I am subTest A, I PASS')
-    print('[  PASSED  ] 1 test.')
-    sys.exit(0)
-elif test_name == 'FirstTest.subTestB':
-    print('I am subTest B, I FAIL')
-    print('And I have two lines of output')
-    sys.exit(1)
-elif test_name in ('ParameterizedTest/0.subTest',
-                   'ParameterizedTest/1.subTest'):
-    print('I am a parameterized test, I also PASS')
-    print('[  PASSED  ] 1 test.')
-    sys.exit(0)
-else:
-    raise SystemExit("error: invalid test name: %r" % (test_name,))

llvm/trunk/utils/lit/tests/Inputs/googletest-upstream-format/DummySubDir/OneTest.py

+#!/usr/bin/env python
+
+import sys
+
+if len(sys.argv) != 2:
+    raise ValueError("unexpected number of args")
+
+if sys.argv[1] == "--gtest_list_tests":
+    print("""\
+Running main() from gtest_main.cc
+FirstTest.
+  subTestA
+  subTestB
+ParameterizedTest/0.
+  subTest
+ParameterizedTest/1.
+  subTest""")
+    sys.exit(0)
+elif not sys.argv[1].startswith("--gtest_filter="):
+    raise ValueError("unexpected argument: %r" % (sys.argv[1]))
+
+test_name = sys.argv[1].split('=',1)[1]
+print('Running main() from gtest_main.cc')
+if test_name == 'FirstTest.subTestA':
+    print('I am subTest A, I PASS')
+    print('[  PASSED  ] 1 test.')
+    sys.exit(0)
+elif test_name == 'FirstTest.subTestB':
+    print('I am subTest B, I FAIL')
+    print('And I have two lines of output')
+    sys.exit(1)
+elif test_name in ('ParameterizedTest/0.subTest',
+                   'ParameterizedTest/1.subTest'):
+    print('I am a parameterized test, I also PASS')
+    print('[  PASSED  ] 1 test.')
+    sys.exit(0)
+else:
+    raise SystemExit("error: invalid test name: %r" % (test_name,))

llvm/trunk/utils/lit/tests/googletest-format.py

 # Check the various features of the GoogleTest format.
 #
-# PR33933
-# XFAIL: windows
-#
 # RUN: not %{lit} -j 1 -v %{inputs}/googletest-format > %t.out
 # RUN: FileCheck < %t.out %s
 #
 # END.
 
 # CHECK: -- Testing:
-# CHECK: PASS: googletest-format :: DummySubDir/OneTest/FirstTest.subTestA
-# CHECK: FAIL: googletest-format :: DummySubDir/OneTest/FirstTest.subTestB
-# CHECK-NEXT: *** TEST 'googletest-format :: DummySubDir/OneTest/FirstTest.subTestB' FAILED ***
+# CHECK: PASS: googletest-format :: {{[Dd]ummy[Ss]ub[Dd]ir}}/OneTest.py/FirstTest.subTestA
+# CHECK: FAIL: googletest-format :: {{[Dd]ummy[Ss]ub[Dd]ir}}/OneTest.py/FirstTest.subTestB
+# CHECK-NEXT: *** TEST 'googletest-format :: {{[Dd]ummy[Ss]ub[Dd]ir}}/OneTest.py/FirstTest.subTestB' FAILED ***
 # CHECK-NEXT: I am subTest B, I FAIL
 # CHECK-NEXT: And I have two lines of output
 # CHECK: ***
-# CHECK: PASS: googletest-format :: DummySubDir/OneTest/ParameterizedTest/0.subTest
-# CHECK: PASS: googletest-format :: DummySubDir/OneTest/ParameterizedTest/1.subTest
+# CHECK: PASS: googletest-format :: {{[Dd]ummy[Ss]ub[Dd]ir}}/OneTest.py/ParameterizedTest/0.subTest
+# CHECK: PASS: googletest-format :: {{[Dd]ummy[Ss]ub[Dd]ir}}/OneTest.py/ParameterizedTest/1.subTest
 # CHECK: Failing Tests (1)
 # CHECK: Expected Passes    : 3
 # CHECK: Unexpected Failures: 1
 

llvm/trunk/utils/lit/tests/googletest-timeout.py

 # REQUIRES: python-psutil
 
-# PR33934
-# XFAIL: windows
-
 # Check that the per test timeout is enforced when running GTest tests.
 #
 # RUN: not %{lit} -j 1 -v %{inputs}/googletest-timeout --timeout=1 > %t.cmd.out
 # RUN: FileCheck < %t.cmd.out %s
 
 # Check that the per test timeout is enforced when running GTest tests via
 # the configuration file
 #
 # RUN: not %{lit} -j 1 -v %{inputs}/googletest-timeout \
 # RUN: --param set_timeout=1 > %t.cfgset.out 2> %t.cfgset.err
 # RUN: FileCheck < %t.cfgset.out %s
 
 # CHECK: -- Testing:
-# CHECK: PASS: googletest-timeout :: DummySubDir/OneTest/FirstTest.subTestA
-# CHECK: TIMEOUT: googletest-timeout :: DummySubDir/OneTest/FirstTest.subTestB
-# CHECK: TIMEOUT: googletest-timeout :: DummySubDir/OneTest/FirstTest.subTestC
+# CHECK: PASS: googletest-timeout :: {{[Dd]ummy[Ss]ub[Dd]ir}}/OneTest.py/FirstTest.subTestA
+# CHECK: TIMEOUT: googletest-timeout :: {{[Dd]ummy[Ss]ub[Dd]ir}}/OneTest.py/FirstTest.subTestB
+# CHECK: TIMEOUT: googletest-timeout :: {{[Dd]ummy[Ss]ub[Dd]ir}}/OneTest.py/FirstTest.subTestC
 # CHECK: Expected Passes    : 1
 # CHECK: Individual Timeouts: 2
 
 # Test per test timeout via a config file and on the command line.
 # The value set on the command line should override the config file.
 # RUN: not %{lit} -j 1 -v %{inputs}/googletest-timeout \
 # RUN: --param set_timeout=1 --timeout=2 > %t.cmdover.out 2> %t.cmdover.err
 # RUN: FileCheck < %t.cmdover.out %s
 # RUN: FileCheck --check-prefix=CHECK-CMDLINE-OVERRIDE-ERR < %t.cmdover.err %s
 
 # CHECK-CMDLINE-OVERRIDE-ERR: Forcing timeout to be 2 seconds

llvm/trunk/utils/lit/tests/googletest-upstream-format.py

 # Check the various features of the GoogleTest format.
 #
-# PR33935
-# XFAIL: windows
-#
 # RUN: not %{lit} -j 1 -v %{inputs}/googletest-upstream-format > %t.out
 # RUN: FileCheck < %t.out %s
 #
 # END.
 
 # CHECK: -- Testing:
-# CHECK: PASS: googletest-upstream-format :: DummySubDir/OneTest/FirstTest.subTestA
-# CHECK: FAIL: googletest-upstream-format :: DummySubDir/OneTest/FirstTest.subTestB
-# CHECK-NEXT: *** TEST 'googletest-upstream-format :: DummySubDir/OneTest/FirstTest.subTestB' FAILED ***
+# CHECK: PASS: googletest-upstream-format :: {{[Dd]ummy[Ss]ub[Dd]ir}}/OneTest.py/FirstTest.subTestA
+# CHECK: FAIL: googletest-upstream-format :: {{[Dd]ummy[Ss]ub[Dd]ir}}/OneTest.py/FirstTest.subTestB
+# CHECK-NEXT: *** TEST 'googletest-upstream-format :: {{[Dd]ummy[Ss]ub[Dd]ir}}/OneTest.py/FirstTest.subTestB' FAILED ***
 # CHECK-NEXT: Running main() from gtest_main.cc
 # CHECK-NEXT: I am subTest B, I FAIL
 # CHECK-NEXT: And I have two lines of output
 # CHECK: ***
-# CHECK: PASS: googletest-upstream-format :: DummySubDir/OneTest/ParameterizedTest/0.subTest
-# CHECK: PASS: googletest-upstream-format :: DummySubDir/OneTest/ParameterizedTest/1.subTest
+# CHECK: PASS: googletest-upstream-format :: {{[Dd]ummy[Ss]ub[Dd]ir}}/OneTest.py/ParameterizedTest/0.subTest
+# CHECK: PASS: googletest-upstream-format :: {{[Dd]ummy[Ss]ub[Dd]ir}}/OneTest.py/ParameterizedTest/1.subTest
 # CHECK: Failing Tests (1)
 # CHECK: Expected Passes    : 3
 # CHECK: Unexpected Failures: 1