diff --git a/libcxx/test/libcxx/selftest/gen.cpp/empty.gen.cpp b/libcxx/test/libcxx/selftest/gen.cpp/empty.gen.cpp new file mode 100644 --- /dev/null +++ b/libcxx/test/libcxx/selftest/gen.cpp/empty.gen.cpp @@ -0,0 +1,11 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// Make sure we can generate no tests at all + +// RUN: true diff --git a/libcxx/test/libcxx/selftest/gen.cpp/one.gen.cpp b/libcxx/test/libcxx/selftest/gen.cpp/one.gen.cpp new file mode 100644 --- /dev/null +++ b/libcxx/test/libcxx/selftest/gen.cpp/one.gen.cpp @@ -0,0 +1,11 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// Make sure we can generate one test + +// RUN: echo "//--- test1.compile.pass.cpp" diff --git a/libcxx/test/libcxx/selftest/gen.cpp/two.gen.cpp b/libcxx/test/libcxx/selftest/gen.cpp/two.gen.cpp new file mode 100644 --- /dev/null +++ b/libcxx/test/libcxx/selftest/gen.cpp/two.gen.cpp @@ -0,0 +1,12 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// Make sure we can generate two tests + +// RUN: echo "//--- test1.compile.pass.cpp" +// RUN: echo "//--- test2.compile.pass.cpp" diff --git a/libcxx/utils/libcxx/test/dsl.py b/libcxx/utils/libcxx/test/dsl.py --- a/libcxx/utils/libcxx/test/dsl.py +++ b/libcxx/utils/libcxx/test/dsl.py @@ -179,9 +179,7 @@ "Failed to run program, cmd:\n{}\nstderr is:\n{}".format(runcmd, err) ) - actualOut = re.search("# command output:\n(.+)\n$", out, flags=re.DOTALL) - actualOut = actualOut.group(1) if actualOut else "" - return actualOut + return libcxx.test.format._parseLitOutput(out) @_memoizeExpensiveOperation( diff --git a/libcxx/utils/libcxx/test/format.py b/libcxx/utils/libcxx/test/format.py --- a/libcxx/utils/libcxx/test/format.py +++ b/libcxx/utils/libcxx/test/format.py @@ -6,6 +6,8 @@ # # ===----------------------------------------------------------------------===## +import contextlib +import io import lit import lit.formats import os @@ -33,6 +35,38 @@ for s in ["%{cxx}", "%{compile_flags}", "%{link_flags}", "%{flags}", "%{exec}"]: assert s in substitutions, "Required substitution {} was not provided".format(s) +def _parseLitOutput(fullOutput): + """ + Parse output of a Lit ShTest to extract the actual output of the contained commands. + + This takes output of the form + + $ ":" "RUN: at line 11" + $ "echo" "OUTPUT1" + # command output: + OUTPUT1 + + $ ":" "RUN: at line 12" + $ "echo" "OUTPUT2" + # command output: + OUTPUT2 + + and returns a string containing + + OUTPUT1 + OUTPUT2 + + as-if the commands had been run directly. This is a workaround for the fact + that Lit doesn't let us execute ShTest and retrieve the raw output without + injecting additional Lit output around it. + """ + parsed = '' + for output in re.split('[$]\s*":"\s*"RUN: at line \d+"', fullOutput): + if output: # skip blank lines + commandOutput = re.search("# command output:\n(.+)\n$", output, flags=re.DOTALL) + if commandOutput: + parsed += commandOutput.group(1) + return parsed def _executeScriptInternal(test, litConfig, commands): """ @@ -170,6 +204,16 @@ FOO.sh. - A builtin Lit Shell test + FOO.gen. - A .sh test that generates one or more Lit tests on the + fly. Executing this test must generate one or more files + as expected by LLVM split-file, and each generated file + leads to a separate Lit test that runs that file as + defined by the test format. This can be used to generate + multiple Lit tests from a single source file, which is + useful for testing repetitive properties in the library. + Be careful not to abuse this since this is not a replacement + for usual code reuse techniques. + FOO.verify.cpp - Compiles with clang-verify. This type of test is automatically marked as UNSUPPORTED if the compiler does not support Clang-verify. @@ -245,6 +289,7 @@ "[.]link[.]pass[.]mm$", "[.]link[.]fail[.]cpp$", "[.]sh[.][^.]+$", + "[.]gen[.][^.]+$", "[.]verify[.]cpp$", "[.]fail[.]cpp$", ] @@ -257,9 +302,13 @@ filepath = os.path.join(sourcePath, filename) if not os.path.isdir(filepath): if any([re.search(ext, filename) for ext in SUPPORTED_SUFFIXES]): - yield lit.Test.Test( - testSuite, pathInSuite + (filename,), localConfig - ) + # If this is a generated test, run the generation step and add + # as many Lit tests as necessary. + if re.search('[.]gen[.][^.]+$', filename): + for test in self._generateGenTest(testSuite, pathInSuite + (filename,), litConfig, localConfig): + yield test + else: + yield lit.Test.Test(testSuite, pathInSuite + (filename,), localConfig) def execute(self, test, litConfig): VERIFY_FLAGS = ( @@ -356,3 +405,42 @@ return lit.TestRunner._runShTest( test, litConfig, useExternalSh, script, tmpBase ) + + def _generateGenTest(self, testSuite, pathInSuite, litConfig, localConfig): + generator = lit.Test.Test(testSuite, pathInSuite, localConfig) + + # Make sure we have a directory to execute the generator test in + generatorExecDir = os.path.dirname(testSuite.getExecPath(pathInSuite)) + os.makedirs(generatorExecDir, exist_ok=True) + + # Run the generator test + steps = [] # Steps must already be in the script + (out, err, exitCode, _, _) = _executeScriptInternal(generator, litConfig, steps) + if exitCode != 0: + raise RuntimeError(f"Error while trying to generate gen test\nstdout:\n{out}\n\nstderr:\n{err}") + + # Split the generated output into multiple files and generate one test for each file + parsed = _parseLitOutput(out) + for (subfile, content) in self._splitFile(parsed): + generatedFile = testSuite.getExecPath(pathInSuite + (subfile, )) + os.makedirs(os.path.dirname(generatedFile), exist_ok=True) + with open(generatedFile, 'w') as f: + f.write(content) + yield lit.Test.Test(testSuite, (generatedFile,), localConfig) + + def _splitFile(self, input): + DELIM = r'^(//|#)---(.+)' + lines = input.splitlines() + currentFile = None + thisFileContent = [] + for line in lines: + match = re.match(DELIM, line) + if match: + if currentFile is not None: + yield (currentFile, '\n'.join(thisFileContent)) + currentFile = match.group(2).strip() + thisFileContent = [] + assert currentFile is not None, f"Some input to split-file doesn't belong to any file, input was:\n{input}" + thisFileContent.append(line) + if currentFile is not None: + yield (currentFile, '\n'.join(thisFileContent))