[libc++] Add support for generated tests in the libc++ test format

[ldionne]

Summary

A recurring problem recently has been that libc++ has several generated
tests which all need to be re-generated before committing a change. This
creates noise during code reviews and friction for contributors.

Furthermore, the way we generated most of these tests resulted in
extremely bad compilation times when using modules, because we defined
a macro before compiling each file.

This commit introduces a new kind of test called a '.gen' test. These
tests are normal shell tests, however the Lit test format will run the
test to discover the actual Lit tests it should run. This basically
allows generating a Lit test suite on the fly using arbitrary code,
which can be used in the future to generate tests like our __verbose_abort
tests and several others.

[ldionne]

Fix CI and rebase.

libcxx/test/libcxx/selftest/gen.cpp/empty.gen.cpp

+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Make sure we can generate no tests at all
+
+// RUN: true

libcxx/test/libcxx/selftest/gen.cpp/one.gen.cpp

+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Make sure we can generate one test
+
+// RUN: echo "//--- test1.compile.pass.cpp"

libcxx/test/libcxx/selftest/gen.cpp/two.gen.cpp

+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Make sure we can generate two tests
+
+// RUN: echo "//--- test1.compile.pass.cpp"
+// RUN: echo "//--- test2.compile.pass.cpp"

libcxx/utils/libcxx/test/dsl.py

with _makeConfigTest(config) as test:
         out, err, exitCode, _, runcmd = _executeWithFakeConfig(
             test, ["%{{run}} {}".format(" ".join(args))]
         )
         if exitCode != 0:
             raise ConfigurationRuntimeError(
                 "Failed to run program, cmd:\n{}\nstderr is:\n{}".format(runcmd, err)
             )
 
-        actualOut = re.search("# command output:\n(.+)\n$", out, flags=re.DOTALL)
-        actualOut = actualOut.group(1) if actualOut else ""
-        return actualOut
+        return libcxx.test.format._parseLitOutput(out)
 
 
 @_memoizeExpensiveOperation(
     lambda c, p, args=None: (c.substitutions, c.environment, p, args)
 )
 def programSucceeds(config, program, args=None):
     """
     Compiles a program for the test target, run it on the test target and return

libcxx/utils/libcxx/test/format.py

 # ===----------------------------------------------------------------------===##
 #
 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
 # ===----------------------------------------------------------------------===##
 
+import contextlib
+import io
 import lit
 import lit.formats
 import os
 import pipes
 import re
 import shutil
 
 
def _getTempPaths(test):
     return tmpDir, tmpBase
 
 
 def _checkBaseSubstitutions(substitutions):
     substitutions = [s for (s, _) in substitutions]
     for s in ["%{cxx}", "%{compile_flags}", "%{link_flags}", "%{flags}", "%{exec}"]:
         assert s in substitutions, "Required substitution {} was not provided".format(s)
 
+def _parseLitOutput(fullOutput):
+    """
+    Parse output of a Lit ShTest to extract the actual output of the contained commands.
+
+    This takes output of the form
+
+        $ ":" "RUN: at line 11"
+        $ "echo" "OUTPUT1"
+        # command output:
+        OUTPUT1
+
+        $ ":" "RUN: at line 12"
+        $ "echo" "OUTPUT2"
+        # command output:
+        OUTPUT2
+
+    and returns a string containing
+
+        OUTPUT1
+        OUTPUT2
+
+    as-if the commands had been run directly. This is a workaround for the fact
+    that Lit doesn't let us execute ShTest and retrieve the raw output without
+    injecting additional Lit output around it.
+    """
+    parsed = ''
+    for output in re.split('[$]\s*":"\s*"RUN: at line \d+"', fullOutput):
+        if output: # skip blank lines
+            commandOutput = re.search("# command output:\n(.+)\n$", output, flags=re.DOTALL)
+            if commandOutput:
+                parsed += commandOutput.group(1)
+    return parsed
 
 def _executeScriptInternal(test, litConfig, commands):
     """
     Returns (stdout, stderr, exitCode, timeoutInfo, parsedCommands)
 
     TODO: This really should be easier to access from Lit itself
     """
     parsedCommands = parseScript(test, preamble=commands)
class CxxStandardLibraryTest(lit.formats.TestFormat):
     FOO.compile.fail.cpp    - Does not compile successfully
 
     FOO.link.pass.cpp       - Compiles and links successfully, run not attempted
     FOO.link.pass.mm        - Same as .link.pass.cpp, but for Objective-C++
     FOO.link.fail.cpp       - Compiles successfully, but fails to link
 
     FOO.sh.<anything>       - A builtin Lit Shell test
 
+    FOO.gen.<anything>      - A .sh test that generates one or more Lit tests on the
+                              fly. Executing this test must generate one or more files
+                              as expected by LLVM split-file, and each generated file
+                              leads to a separate Lit test that runs that file as
+                              defined by the test format. This can be used to generate
+                              multiple Lit tests from a single source file, which is
+                              useful for testing repetitive properties in the library.
+                              Be careful not to abuse this since this is not a replacement
+                              for usual code reuse techniques.
+
     FOO.verify.cpp          - Compiles with clang-verify. This type of test is
                               automatically marked as UNSUPPORTED if the compiler
                               does not support Clang-verify.
 
     FOO.fail.cpp            - Compiled with clang-verify if clang-verify is
                               supported, and equivalent to a .compile.fail.cpp
                               test otherwise. This is supported only for backwards
                               compatibility with the test suite.
def getTestsInDirectory(self, testSuite, pathInSuite, litConfig, localConfig):
             "[.]pass[.]mm$",
             "[.]compile[.]pass[.]cpp$",
             "[.]compile[.]pass[.]mm$",
             "[.]compile[.]fail[.]cpp$",
             "[.]link[.]pass[.]cpp$",
             "[.]link[.]pass[.]mm$",
             "[.]link[.]fail[.]cpp$",
             "[.]sh[.][^.]+$",
+            "[.]gen[.][^.]+$",
             "[.]verify[.]cpp$",
             "[.]fail[.]cpp$",
         ]
         sourcePath = testSuite.getSourcePath(pathInSuite)
         for filename in os.listdir(sourcePath):
             # Ignore dot files and excluded tests.
             if filename.startswith(".") or filename in localConfig.excludes:
                 continue
 
             filepath = os.path.join(sourcePath, filename)
             if not os.path.isdir(filepath):
                 if any([re.search(ext, filename) for ext in SUPPORTED_SUFFIXES]):
-                    yield lit.Test.Test(
-                        testSuite, pathInSuite + (filename,), localConfig
-                    )
+                    # If this is a generated test, run the generation step and add
+                    # as many Lit tests as necessary.
+                    if re.search('[.]gen[.][^.]+$', filename):
+                        for test in self._generateGenTest(testSuite, pathInSuite + (filename,), litConfig, localConfig):
+                            yield test
+                    else:
+                        yield lit.Test.Test(testSuite, pathInSuite + (filename,), localConfig)
 
     def execute(self, test, litConfig):
         VERIFY_FLAGS = (
             "-Xclang -verify -Xclang -verify-ignore-unexpected=note -ferror-limit=0"
         )
         supportsVerify = "verify-support" in test.config.available_features
         filename = test.path_in_suite[-1]
 
def _executeShTest(self, test, litConfig, steps):
                 lit.Test.XFAIL if test.isExpectedToFail() else lit.Test.PASS
             )
         else:
             _, tmpBase = _getTempPaths(test)
             useExternalSh = False
             return lit.TestRunner._runShTest(
                 test, litConfig, useExternalSh, script, tmpBase
             )
+
+    def _generateGenTest(self, testSuite, pathInSuite, litConfig, localConfig):
+        generator = lit.Test.Test(testSuite, pathInSuite, localConfig)
+
+        # Make sure we have a directory to execute the generator test in
+        generatorExecDir = os.path.dirname(testSuite.getExecPath(pathInSuite))
+        os.makedirs(generatorExecDir, exist_ok=True)
+
+        # Run the generator test
+        steps = [] # Steps must already be in the script
+        (out, err, exitCode, _, _) = _executeScriptInternal(generator, litConfig, steps)
+        if exitCode != 0:
+            raise RuntimeError(f"Error while trying to generate gen test\nstdout:\n{out}\n\nstderr:\n{err}")
+
+        # Split the generated output into multiple files and generate one test for each file
+        parsed = _parseLitOutput(out)
+        for (subfile, content) in self._splitFile(parsed):
+            generatedFile = testSuite.getExecPath(pathInSuite + (subfile, ))
+            os.makedirs(os.path.dirname(generatedFile), exist_ok=True)
+            with open(generatedFile, 'w') as f:
+                f.write(content)
+            yield lit.Test.Test(testSuite, (generatedFile,), localConfig)
+
+    def _splitFile(self, input):
+        DELIM = r'^(//|#)---(.+)'
+        lines = input.splitlines()
+        currentFile = None
+        thisFileContent = []
+        for line in lines:
+            match = re.match(DELIM, line)
+            if match:
+                if currentFile is not None:
+                    yield (currentFile, '\n'.join(thisFileContent))
+                currentFile = match.group(2).strip()
+                thisFileContent = []
+            assert currentFile is not None, f"Some input to split-file doesn't belong to any file, input was:\n{input}"
+            thisFileContent.append(line)
+        if currentFile is not None:
+            yield (currentFile, '\n'.join(thisFileContent))