Index: llvm/test/Other/lit-globbing.ll =================================================================== --- llvm/test/Other/lit-globbing.ll +++ llvm/test/Other/lit-globbing.ll @@ -1,28 +1,28 @@ -RUN: echo TA > %T/TA.txt -RUN: echo TB > %T/TB.txt -RUN: echo TAB > %T/TAB.txt - -RUN: echo %T/TA* | FileCheck -check-prefix=STAR %s -RUN: echo %T/'TA'* | FileCheck -check-prefix=STAR %s -RUN: echo %T/T'A'* | FileCheck -check-prefix=STAR %s - -RUN: echo %T/T?.txt | FileCheck -check-prefix=QUESTION %s -RUN: echo %T/'T'?.txt | FileCheck -check-prefix=QUESTION %s - -RUN: echo %T/T??.txt | FileCheck -check-prefix=QUESTION2 %s -RUN: echo %T/'T'??.txt | FileCheck -check-prefix=QUESTION2 %s - -RUN: echo 'T*' 'T?.txt' 'T??.txt' | FileCheck -check-prefix=QUOTEDARGS %s - -STAR-NOT: TB.txt -STAR: {{(TA.txt.*TAB.txt|TAB.txt.*TA.txt)}} - -QUESTION-NOT: TAB.txt -QUESTION: {{(TA.txt.*TB.txt|TB.txt.*TA.txt)}} - -QUESTION2-NOT: TA.txt -QUESTION2-NOT: TB.txt -QUESTION2: TAB.txt - -QUOTEDARGS-NOT: .txt -QUOTEDARGS: T* T?.txt T??.txt +RUN: echo XXA > %T/XXA.txt +RUN: echo XXB > %T/XXB.txt +RUN: echo XXAB > %T/XXAB.txt + +RUN: echo %T/XXA* | FileCheck -check-prefix=STAR %s +RUN: echo %T/'XXA'* | FileCheck -check-prefix=STAR %s +RUN: echo %T/XX'A'* | FileCheck -check-prefix=STAR %s + +RUN: echo %T/XX?.txt | FileCheck -check-prefix=QUESTION %s +RUN: echo %T/'XX'?.txt | FileCheck -check-prefix=QUESTION %s + +RUN: echo %T/XX??.txt | FileCheck -check-prefix=QUESTION2 %s +RUN: echo %T/'XX'??.txt | FileCheck -check-prefix=QUESTION2 %s + +RUN: echo 'XX*' 'XX?.txt' 'XX??.txt' | FileCheck -check-prefix=QUOTEDARGS %s + +STAR-NOT: XXB.txt +STAR: {{(XXA.txt.*XXAB.txt|XXAB.txt.*XXA.txt)}} + +QUESTION-NOT: XXAB.txt +QUESTION: {{(XXA.txt.*XXB.txt|XXB.txt.*XXA.txt)}} + +QUESTION2-NOT: XXA.txt +QUESTION2-NOT: XXB.txt +QUESTION2: XXAB.txt + +QUOTEDARGS-NOT: .txt +QUOTEDARGS: XX* XX?.txt XX??.txt Index: llvm/test/Unit/lit.cfg =================================================================== --- llvm/test/Unit/lit.cfg +++ llvm/test/Unit/lit.cfg @@ -39,9 +39,11 @@ config.environment[symbolizer] = os.environ[symbolizer] # Win32 seeks DLLs along %PATH%. -if sys.platform in ['win32', 'cygwin'] and os.path.isdir(config.shlibdir): - config.environment['PATH'] = os.path.pathsep.join(( - config.shlibdir, config.environment['PATH'])) +if sys.platform in ['win32', 'cygwin']: + shlibdir = getattr(config, 'shlibdir', None) + if shlibdir is not None and os.path.isdir(shlibdir): + config.environment['PATH'] = os.path.pathsep.join(( + config.shlibdir, config.environment['PATH'])) # Win32 may use %SYSTEMDRIVE% during file system shell operations, so propogate. if sys.platform == 'win32' and 'SYSTEMDRIVE' in os.environ: Index: llvm/utils/FileEdit/CMakeLists.txt =================================================================== --- /dev/null +++ llvm/utils/FileEdit/CMakeLists.txt @@ -0,0 +1,5 @@ +add_llvm_utility(FileEdit + FileEdit.cpp + ) + +target_link_libraries(FileEdit LLVMSupport) Index: llvm/utils/FileEdit/FileEdit.cpp =================================================================== --- /dev/null +++ llvm/utils/FileEdit/FileEdit.cpp @@ -0,0 +1,235 @@ +//===- FileEdit.cpp - Split and transform file into multiple sub-files ----===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// FileEdit takes an input file and performs various operations on the file, +// such as replacing certain sequences or splitting the file into multiple +// sub-files. This is useful for keeping tests self-contained, as it allows us +// to write a single file containing both test input and test check lines even +// when the test input itself must span multiple files (e.g. a test which tests +// some functionality involving compiling and linking 2 separate compilation +// units). +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/LineIterator.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/PrettyStackTrace.h" +#include "llvm/Support/Regex.h" +#include "llvm/Support/Signals.h" +#include "llvm/Support/raw_ostream.h" + +using namespace llvm; + +namespace { +enum class EditResult { Discard, UseModified, UseUnmodified }; +struct LineReplacement { + StringRef Pattern; + StringRef Repl; +}; +} // namespace + +static cl::opt + InputFilename(cl::Positional, cl::desc("input file (defaults to stdin)"), + cl::Required); + +static cl::list StripPrefixes( + "strip-prefix", cl::CommaSeparated, + cl::desc("Exclude lines beginning with the given prefix from the output")); + +static cl::list Replacements( + "replace", cl::ZeroOrMore, + cl::desc("Apply the specified replacement regex to each line")); + +static cl::opt + DestFolder("dest", cl::desc("Write split files to the specified folder")); + +static size_t findNextUnescapedSlash(StringRef Str) { + size_t Pos = 0; + while (Pos != StringRef::npos) { + Pos = Str.find_first_of('/'); + if (Pos == StringRef::npos) + return Pos; + if (Pos == 0 || Str[Pos - 1] != '\\') + return Pos; + } + return StringRef::npos; +} + +static bool parseOneReplacementItem(StringRef Str, StringRef &Pattern, + StringRef &Repl) { + if (!Str.consume_front("s/")) + return false; + + for (StringRef *Dest : {&Pattern, &Repl}) { + size_t NextSlash = findNextUnescapedSlash(Str); + if (NextSlash == StringRef::npos) + return false; + *Dest = Str.take_front(NextSlash); + Str = Str.drop_front(NextSlash + 1); + } + + return Str.empty(); +} + +static void buildReplacementVector(std::vector &Repls) { + std::string UniversalRegexStr; + std::vector Patterns; + for (StringRef S : Replacements) { + StringRef Pattern; + StringRef Repl; + if (!parseOneReplacementItem(S, Pattern, Repl)) { + outs() << formatv("FileEdit: Invalid replacement string `{0}`", S); + continue; + } + + Patterns.push_back(Pattern); + Repls.push_back({Pattern, Repl}); + } +} + +static bool handleFileDirective(StringRef Line, StringRef &OutFile) { + if (!Line.consume_front("{!--")) + return false; + OutFile = Line.trim(); + return true; +} + +static std::error_code writeOutput(StringRef FileName, StringRef Text) { + if (FileName.empty()) { + llvm::outs() << Text; + return std::error_code(); + } + + SmallString<128> Path; + if (DestFolder.getNumOccurrences() == 0) + llvm::sys::fs::current_path(Path); + else + Path = DestFolder; + llvm::sys::path::append(Path, FileName); + llvm::sys::path::native(Path); + auto Directory = llvm::sys::path::parent_path(Path); + if (auto EC = llvm::sys::fs::create_directories(Directory, true)) + return EC; + + int FD; + if (auto EC = + llvm::sys::fs::openFileForWrite(Path, FD, llvm::sys::fs::F_Text)) + return EC; + llvm::raw_fd_ostream OS(FD, true); + OS << Text; + OS.flush(); + return std::error_code(); +} + +static EditResult editLine(StringRef Line, + ArrayRef Replacements, + SmallVectorImpl &ModifiedLine) { + if (Line.empty()) + return EditResult::UseUnmodified; + + for (StringRef Prefix : StripPrefixes) { + if (Line.startswith(Prefix)) + return EditResult::Discard; + } + + // Perform all replacements from left to right in the input string. + ModifiedLine.clear(); + bool WasModified = false; + while (!Line.empty()) { + const LineReplacement *UseRepl = nullptr; + size_t FirstPos = StringRef::npos; + + // Find the leftmost replacement in the input string, and map it back to + // the text that should replace it. + for (const LineReplacement &LR : Replacements) { + size_t Pos = Line.find(LR.Pattern); + if (Pos < FirstPos) { + UseRepl = &LR; + FirstPos = Pos; + } + } + if (!UseRepl) { + if (!WasModified) { + // If we didn't find anything to replace on this or any previous + // iteration we can just exit and use the original line. + return EditResult::UseUnmodified; + } + + ModifiedLine.append(Line.begin(), Line.end()); + return EditResult::UseModified; + } + + // The line can be split into 3 groups. Line = Left | Repl | Right. Append + // `Left` as is, use the replacement value for `Repl`, and keep `Right` as + // the remainder of the string for further replacement. + StringRef Left = Line.take_front(FirstPos); + + ModifiedLine.append(Left.begin(), Left.end()); + Line = Line.drop_front(FirstPos); + + ModifiedLine.append(UseRepl->Repl.begin(), UseRepl->Repl.end()); + WasModified = true; + Line = Line.drop_front(UseRepl->Pattern.size()); + } + return EditResult::UseModified; +} + +int main(int argc, char **argv) { + sys::PrintStackTraceOnErrorSignal(argv[0]); + PrettyStackTraceProgram X(argc, argv); + cl::ParseCommandLineOptions(argc, argv); + + // Read the expected strings from the check file. + ErrorOr> InFileOrErr = + MemoryBuffer::getFileOrSTDIN(InputFilename); + if (std::error_code EC = InFileOrErr.getError()) { + errs() << "Could not open input file '" << InputFilename + << "': " << EC.message() << '\n'; + return 1; + } + MemoryBuffer &InFile = *InFileOrErr.get(); + + std::vector Replacements; + + buildReplacementVector(Replacements); + + SmallString<4096> InputFileBuffer; + + line_iterator Iter(InFile, false); + line_iterator End; + StringRef OutFile; + std::string OutputText; + SmallString<80> ModifiedLine; + for (StringRef Line : make_range(Iter, End)) { + StringRef NewOutFile; + if (handleFileDirective(Line, NewOutFile)) { + writeOutput(OutFile, OutputText); + OutFile = NewOutFile; + OutputText.clear(); + continue; + } + + EditResult Result = editLine(Line, Replacements, ModifiedLine); + if (Result == EditResult::Discard) + continue; + if (Result == EditResult::UseModified) + Line = ModifiedLine; + if (!OutputText.empty()) + OutputText += "\n"; + OutputText += Line; + } + + writeOutput(OutFile, OutputText); + return 0; +} Index: llvm/utils/lit/lit/Test.py =================================================================== --- llvm/utils/lit/lit/Test.py +++ llvm/utils/lit/lit/Test.py @@ -172,7 +172,7 @@ return os.path.join(self.source_root, *components) def getExecPath(self, components): - return os.path.join(self.exec_root, *components) + return os.path.join(self.exec_root, "Output", *components) class Test: """Test - Information on a single test instance.""" @@ -226,6 +226,9 @@ def getFullName(self): return self.suite.config.name + ' :: ' + '/'.join(self.path_in_suite) + def getTestBaseName(self): + return self.path_in_suite[-1] + def getFilePath(self): if self.file_path: return self.file_path @@ -234,8 +237,11 @@ def getSourcePath(self): return self.suite.getSourcePath(self.path_in_suite) - def getExecPath(self): - return self.suite.getExecPath(self.path_in_suite) + def getTempFilePrefix(self): + return self.suite.getExecPath(self.path_in_suite) + ".tmp" + + def getTempFileDir(self): + return os.path.dirname(self.getTempFilePrefix()) def isExpectedToFail(self): """ Index: llvm/utils/lit/lit/TestRunner.py =================================================================== --- llvm/utils/lit/lit/TestRunner.py +++ llvm/utils/lit/lit/TestRunner.py @@ -690,37 +690,28 @@ finally: f.close() -def getTempPaths(test): - """Get the temporary location, this is always relative to the test suite - root, not test source root.""" - execpath = test.getExecPath() - execdir,execbase = os.path.split(execpath) - tmpDir = os.path.join(execdir, 'Output') - tmpBase = os.path.join(tmpDir, execbase) - return tmpDir, tmpBase - -def getDefaultSubstitutions(test, tmpDir, tmpBase, normalize_slashes=False): +def getDefaultSubstitutions(test, normalize_slashes=False): sourcepath = test.getSourcePath() sourcedir = os.path.dirname(sourcepath) + tmpDir = test.getTempFileDir() + tmpPrefix = test.getTempFilePrefix() + baseName = test.getTestBaseName() # Normalize slashes, if requested. if normalize_slashes: sourcepath = sourcepath.replace('\\', '/') sourcedir = sourcedir.replace('\\', '/') tmpDir = tmpDir.replace('\\', '/') - tmpBase = tmpBase.replace('\\', '/') # We use #_MARKER_# to hide %% while we do the other substitutions. substitutions = [] substitutions.extend([('%%', '#_MARKER_#')]) substitutions.extend(test.config.substitutions) - tmpName = tmpBase + '.tmp' - baseName = os.path.basename(tmpBase) substitutions.extend([('%s', sourcepath), ('%S', sourcedir), ('%p', sourcedir), ('%{pathsep}', os.pathsep), - ('%t', tmpName), + ('%t', tmpPrefix), ('%basename_t', baseName), ('%T', tmpDir), ('#_MARKER_#', '%')]) @@ -730,7 +721,7 @@ ('%/s', sourcepath.replace('\\', '/')), ('%/S', sourcedir.replace('\\', '/')), ('%/p', sourcedir.replace('\\', '/')), - ('%/t', tmpBase.replace('\\', '/') + '.tmp'), + ('%/t', tmpPrefix.replace('\\', '/')), ('%/T', tmpDir.replace('\\', '/')), ]) @@ -740,7 +731,7 @@ ('%:s', re.sub(r'^(.):', r'\1', sourcepath)), ('%:S', re.sub(r'^(.):', r'\1', sourcedir)), ('%:p', re.sub(r'^(.):', r'\1', sourcedir)), - ('%:t', re.sub(r'^(.):', r'\1', tmpBase) + '.tmp'), + ('%:t', re.sub(r'^(.):', r'\1', tmpPrefix)), ('%:T', re.sub(r'^(.):', r'\1', tmpDir)), ]) else: @@ -748,7 +739,7 @@ ('%:s', sourcepath), ('%:S', sourcedir), ('%:p', sourcedir), - ('%:t', tmpBase + '.tmp'), + ('%:t', tmpPrefix), ('%:T', tmpDir), ]) return substitutions @@ -1014,12 +1005,8 @@ return script - def _runShTest(test, litConfig, useExternalSh, script, tmpBase): - # Create the output directory if it does not already exist. - lit.util.mkdir_p(os.path.dirname(tmpBase)) - - execdir = os.path.dirname(test.getExecPath()) + execdir = os.path.dirname(test.getTempFileDir()) if useExternalSh: res = executeScript(test, litConfig, tmpBase, script, execdir) else: @@ -1063,10 +1050,8 @@ return script if litConfig.noExecute: return lit.Test.Result(Test.PASS) - - tmpDir, tmpBase = getTempPaths(test) substitutions = list(extra_substitutions) - substitutions += getDefaultSubstitutions(test, tmpDir, tmpBase, + substitutions += getDefaultSubstitutions(test, normalize_slashes=useExternalSh) script = applySubstitutions(script, substitutions) @@ -1075,7 +1060,8 @@ if hasattr(test.config, 'test_retry_attempts'): attempts += test.config.test_retry_attempts for i in range(attempts): - res = _runShTest(test, litConfig, useExternalSh, script, tmpBase) + res = _runShTest(test, litConfig, useExternalSh, script, + test.getTempFilePrefix()) if res.code != Test.FAIL: break # If we had to run the test more than once, count it as a flaky pass. These Index: llvm/utils/lit/lit/run.py =================================================================== --- llvm/utils/lit/lit/run.py +++ llvm/utils/lit/lit/run.py @@ -248,6 +248,28 @@ if not self.tests or jobs == 0: return + # Create fresh output directories for each test we're going to run. + # This guarantees that test runs will not remnants of previous test + # runs' output. + import time + start = time.time() + clean_paths = set() + for test in self.tests: + clean_paths.add(os.path.normpath(test.getTempFileDir())) + clean_paths = list(clean_paths) + clean_paths.sort(key=lambda x: len(x.split(os.sep))) + for base in clean_paths: + if os.path.exists(base): + if not os.path.islink(base) and os.path.isdir(base): + from shutil import rmtree + rmtree(base, True) + else: + os.unlink(os.path) + if not os.path.exists(base): + lit.util.mkdir_p(base) + end = time.time() + print("Cleanup took {} seconds".format(end - start)) + # Set up semaphores to limit parallelism of certain classes of tests. # For example, some ASan tests require lots of virtual memory and run # faster with less parallelism on OS X.