diff --git a/clang/include/clang/Basic/DiagnosticLexKinds.td b/clang/include/clang/Basic/DiagnosticLexKinds.td --- a/clang/include/clang/Basic/DiagnosticLexKinds.td +++ b/clang/include/clang/Basic/DiagnosticLexKinds.td @@ -899,11 +899,11 @@ } -let CategoryName = "Dependency Directive Source Minimization Issue" in { +let CategoryName = "Dependency Directive Source Scanner Issue" in { -def err_dep_source_minimizer_missing_sema_after_at_import : Error< +def err_dep_source_scanner_missing_semi_after_at_import : Error< "could not find ';' after @import">; -def err_dep_source_minimizer_unexpected_tokens_at_import : Error< +def err_dep_source_scanner_unexpected_tokens_at_import : Error< "unexpected extra tokens at end of @import declaration">; } diff --git a/clang/include/clang/Lex/DependencyDirectivesSourceMinimizer.h b/clang/include/clang/Lex/DependencyDirectivesScanner.h rename from clang/include/clang/Lex/DependencyDirectivesSourceMinimizer.h rename to clang/include/clang/Lex/DependencyDirectivesScanner.h --- a/clang/include/clang/Lex/DependencyDirectivesSourceMinimizer.h +++ b/clang/include/clang/Lex/DependencyDirectivesScanner.h @@ -1,4 +1,4 @@ -//===- clang/Lex/DependencyDirectivesSourceMinimizer.h - ----------*- C++ -*-// +//===- clang/Lex/DependencyDirectivesScanner.h ---------------------*- C++ -*-// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -7,15 +7,15 @@ //===----------------------------------------------------------------------===// /// /// \file -/// This is the interface for minimizing header and source files to the +/// This is the interface for scanning header and source files to get the /// minimum necessary preprocessor directives for evaluating includes. It /// reduces the source down to #define, #include, #import, @import, and any /// conditional preprocessor logic that contains one of those. /// //===----------------------------------------------------------------------===// -#ifndef LLVM_CLANG_LEX_DEPENDENCYDIRECTIVESSOURCEMINIMIZER_H -#define LLVM_CLANG_LEX_DEPENDENCYDIRECTIVESSOURCEMINIMIZER_H +#ifndef LLVM_CLANG_LEX_DEPENDENCYDIRECTIVESSCANNER_H +#define LLVM_CLANG_LEX_DEPENDENCYDIRECTIVESSCANNER_H #include "clang/Basic/SourceLocation.h" #include "llvm/ADT/ArrayRef.h" @@ -26,11 +26,11 @@ class DiagnosticsEngine; -namespace minimize_source_to_dependency_directives { +namespace dependency_directives_scan { /// Represents the kind of preprocessor directive or a module declaration that -/// is tracked by the source minimizer in its token output. -enum TokenKind { +/// is tracked by the scanner in its token output. +enum DirectiveKind : uint8_t { pp_none, pp_include, pp___include_macros, @@ -58,17 +58,17 @@ pp_eof, }; -/// Represents a simplified token that's lexed as part of the source -/// minimization. It's used to track the location of various preprocessor -/// directives that could potentially have an effect on the depedencies. -struct Token { +/// Represents a directive that's lexed as part of the dependency directives +/// scanning. It's used to track various preprocessor directives that could +/// potentially have an effect on the depedencies. +struct Directive { /// The kind of token. - TokenKind K = pp_none; + DirectiveKind Kind = pp_none; /// Offset into the output byte stream of where the directive begins. int Offset = -1; - Token(TokenKind K, int Offset) : K(K), Offset(Offset) {} + Directive(DirectiveKind K, int Offset) : Kind(K), Offset(Offset) {} }; /// Simplified token range to track the range of a potentially skippable PP @@ -86,10 +86,10 @@ /// when skipping a directive like #if, #ifdef or #elsif. /// /// \returns false on success, true on error. -bool computeSkippedRanges(ArrayRef Input, +bool computeSkippedRanges(ArrayRef Input, llvm::SmallVectorImpl &Range); -} // end namespace minimize_source_to_dependency_directives +} // end namespace dependency_directives_scan /// Minimize the input down to the preprocessor directives that might have /// an effect on the dependencies for a compilation unit. @@ -103,13 +103,12 @@ /// \returns false on success, true on error. If the diagnostic engine is not /// null, an appropriate error is reported using the given input location /// with the offset that corresponds to the minimizer's current buffer offset. -bool minimizeSourceToDependencyDirectives( +bool scanSourceForDependencyDirectives( llvm::StringRef Input, llvm::SmallVectorImpl &Output, - llvm::SmallVectorImpl - &Tokens, + llvm::SmallVectorImpl &Directives, DiagnosticsEngine *Diags = nullptr, SourceLocation InputSourceLoc = SourceLocation()); } // end namespace clang -#endif // LLVM_CLANG_LEX_DEPENDENCYDIRECTIVESSOURCEMINIMIZER_H +#endif // LLVM_CLANG_LEX_DEPENDENCYDIRECTIVESSCANNER_H diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h @@ -28,14 +28,14 @@ CachedFileContents(std::unique_ptr Original) : Original(std::move(Original)), MinimizedAccess(nullptr) {} - /// Owning storage for the minimized contents. + /// Owning storage for the original contents. std::unique_ptr Original; - /// The mutex that must be locked before mutating minimized contents. + /// The mutex that must be locked before mutating directive tokens. std::mutex ValueLock; /// Owning storage for the minimized contents. std::unique_ptr MinimizedStorage; - /// Accessor to the minimized contents that's atomic to avoid data races. + /// Accessor to the directive tokens that's atomic to avoid data races. std::atomic MinimizedAccess; /// Skipped range mapping of the minimized contents. /// This is initialized iff `MinimizedAccess != nullptr`. @@ -46,8 +46,8 @@ /// the dependency scanning filesystem. /// /// It represents one of the following: -/// - opened file with original contents and a stat value, -/// - opened file with original contents, minimized contents and a stat value, +/// - opened file with contents and a stat value, +/// - opened file with contents, directive tokens and a stat value, /// - directory entry with its stat value, /// - filesystem error. /// @@ -84,8 +84,9 @@ return Contents->Original->getBuffer(); } - /// \returns Minimized contents of the file. - StringRef getMinimizedContents() const { + /// \returns The scanned preprocessor directive tokens of the file that are + /// used to speed up preprocessing, if available. + StringRef getDirectiveTokens() const { assert(!isError() && "error"); assert(!MaybeStat->isDirectory() && "not a file"); assert(Contents && "contents not initialized"); @@ -119,8 +120,8 @@ return Contents->PPSkippedRangeMapping; } - /// \returns The data structure holding both original and minimized contents. - CachedFileContents *getContents() const { + /// \returns The data structure holding both contents and directive tokens. + CachedFileContents *getCachedContents() const { assert(!isError() && "error"); assert(!isDirectory() && "not a file"); return Contents; @@ -145,7 +146,7 @@ }; /// This class is a shared cache, that caches the 'stat' and 'open' calls to the -/// underlying real file system. It distinguishes between minimized and original +/// underlying real file system, and the scanned preprocessor directives of /// files. /// /// It is sharded based on the hash of the key to reduce the lock contention for @@ -210,8 +211,7 @@ }; /// This class is a local cache, that caches the 'stat' and 'open' calls to the -/// underlying real file system. It distinguishes between minimized and original -/// files. +/// underlying real file system. class DependencyScanningFilesystemLocalCache { llvm::StringMap Cache; @@ -234,9 +234,8 @@ }; /// Reference to a CachedFileSystemEntry. -/// If the underlying entry is an opened file, this wrapper returns the correct -/// contents (original or minimized) and ensures consistency with file size -/// reported by status. +/// If the underlying entry is an opened file, this wrapper returns the file +/// contents and the scanned preprocessor directives. class EntryRef { /// For entry that is an opened file, this bit signifies whether its contents /// are minimized. @@ -270,8 +269,7 @@ } StringRef getContents() const { - return Minimized ? Entry.getMinimizedContents() - : Entry.getOriginalContents(); + return Minimized ? Entry.getDirectiveTokens() : Entry.getOriginalContents(); } const PreprocessorSkippedRangeMapping *getPPSkippedRangeMapping() const { @@ -301,14 +299,14 @@ llvm::ErrorOr> openFileForRead(const Twine &Path) override; - /// Disable minimization of the given file. - void disableMinimization(StringRef Filename); - /// Enable minimization of all files. - void enableMinimizationOfAllFiles() { NotToBeMinimized.clear(); } + /// Disable directives scanning of the given file. + void disableDirectivesScanning(StringRef Filename); + /// Enable directives scanning of all files. + void enableDirectivesScanningOfAllFiles() { NotToBeScanned.clear(); } private: - /// Check whether the file should be minimized. - bool shouldMinimize(StringRef Filename, llvm::sys::fs::UniqueID UID); + /// Check whether the file should be scanned for preprocessor directives. + bool shouldScanForDirectives(StringRef Filename, llvm::sys::fs::UniqueID UID); /// Returns entry for the given filename. /// @@ -316,7 +314,7 @@ /// using the underlying filesystem. llvm::ErrorOr getOrCreateFileSystemEntry(StringRef Filename, - bool DisableMinimization = false); + bool DisableDirectivesScanning = false); /// For a filename that's not yet associated with any entry in the caches, /// uses the underlying filesystem to either look up the entry based in the @@ -324,10 +322,10 @@ llvm::ErrorOr computeAndStoreResult(StringRef Filename); - /// Minimizes the given entry if necessary and returns a wrapper object with - /// reference semantics. - EntryRef minimizeIfNecessary(const CachedFileSystemEntry &Entry, - StringRef Filename, bool Disable); + /// Scan for preprocessor directives for the given entry if necessary and + /// returns a wrapper object with reference semantics. + EntryRef scanForDirectivesIfNecessary(const CachedFileSystemEntry &Entry, + StringRef Filename, bool Disable); /// Represents a filesystem entry that has been stat-ed (and potentially read) /// and that's about to be inserted into the cache as `CachedFileSystemEntry`. @@ -402,8 +400,8 @@ /// excluded conditional directive skip mappings that are used by the /// currently active preprocessor. ExcludedPreprocessorDirectiveSkipMapping &PPSkipMappings; - /// The set of files that should not be minimized. - llvm::DenseSet NotToBeMinimized; + /// The set of files that should not be scanned for PP directives. + llvm::DenseSet NotToBeScanned; }; } // end namespace dependencies diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h @@ -19,15 +19,13 @@ /// dependencies. enum class ScanningMode { /// This mode is used to compute the dependencies by running the preprocessor - /// over - /// the unmodified source files. + /// over the source files. CanonicalPreprocessing, /// This mode is used to compute the dependencies by running the preprocessor - /// over - /// the source files that have been minimized to contents that might affect - /// the dependencies. - MinimizedSourcePreprocessing + /// with special kind of lexing after scanning header and source files to get + /// the minimum necessary preprocessor directives for evaluating includes. + DependencyDirectivesScan, }; /// The format that is output by the dependency scanner. diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp --- a/clang/lib/Frontend/FrontendActions.cpp +++ b/clang/lib/Frontend/FrontendActions.cpp @@ -18,7 +18,7 @@ #include "clang/Frontend/FrontendDiagnostic.h" #include "clang/Frontend/MultiplexConsumer.h" #include "clang/Frontend/Utils.h" -#include "clang/Lex/DependencyDirectivesSourceMinimizer.h" +#include "clang/Lex/DependencyDirectivesScanner.h" #include "clang/Lex/HeaderSearch.h" #include "clang/Lex/Preprocessor.h" #include "clang/Lex/PreprocessorOptions.h" @@ -1158,9 +1158,9 @@ llvm::MemoryBufferRef FromFile = SM.getBufferOrFake(SM.getMainFileID()); llvm::SmallString<1024> Output; - llvm::SmallVector Toks; - if (minimizeSourceToDependencyDirectives( - FromFile.getBuffer(), Output, Toks, &CI.getDiagnostics(), + llvm::SmallVector Directives; + if (scanSourceForDependencyDirectives( + FromFile.getBuffer(), Output, Directives, &CI.getDiagnostics(), SM.getLocForStartOfFile(SM.getMainFileID()))) { assert(CI.getDiagnostics().hasErrorOccurred() && "no errors reported for failure"); diff --git a/clang/lib/Lex/CMakeLists.txt b/clang/lib/Lex/CMakeLists.txt --- a/clang/lib/Lex/CMakeLists.txt +++ b/clang/lib/Lex/CMakeLists.txt @@ -3,7 +3,7 @@ set(LLVM_LINK_COMPONENTS support) add_clang_library(clangLex - DependencyDirectivesSourceMinimizer.cpp + DependencyDirectivesScanner.cpp HeaderMap.cpp HeaderSearch.cpp InitHeaderSearch.cpp diff --git a/clang/lib/Lex/DependencyDirectivesSourceMinimizer.cpp b/clang/lib/Lex/DependencyDirectivesScanner.cpp rename from clang/lib/Lex/DependencyDirectivesSourceMinimizer.cpp rename to clang/lib/Lex/DependencyDirectivesScanner.cpp --- a/clang/lib/Lex/DependencyDirectivesSourceMinimizer.cpp +++ b/clang/lib/Lex/DependencyDirectivesScanner.cpp @@ -1,4 +1,4 @@ -//===- DependencyDirectivesSourceMinimizer.cpp - -------------------------===// +//===- DependencyDirectivesScanner.cpp ------------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -7,14 +7,14 @@ //===----------------------------------------------------------------------===// /// /// \file -/// This is the implementation for minimizing header and source files to the +/// This is the interface for scanning header and source files to get the /// minimum necessary preprocessor directives for evaluating includes. It /// reduces the source down to #define, #include, #import, @import, and any /// conditional preprocessor logic that contains one of those. /// //===----------------------------------------------------------------------===// -#include "clang/Lex/DependencyDirectivesSourceMinimizer.h" +#include "clang/Lex/DependencyDirectivesScanner.h" #include "clang/Basic/CharInfo.h" #include "clang/Basic/Diagnostic.h" #include "clang/Lex/LexDiagnostic.h" @@ -24,26 +24,26 @@ using namespace llvm; using namespace clang; -using namespace clang::minimize_source_to_dependency_directives; +using namespace clang::dependency_directives_scan; namespace { -struct Minimizer { +struct Scanner { /// Minimized output. SmallVectorImpl &Out; /// The known tokens encountered during the minimization. - SmallVectorImpl &Tokens; + SmallVectorImpl &Directives; - Minimizer(SmallVectorImpl &Out, SmallVectorImpl &Tokens, - StringRef Input, DiagnosticsEngine *Diags, - SourceLocation InputSourceLoc) - : Out(Out), Tokens(Tokens), Input(Input), Diags(Diags), + Scanner(SmallVectorImpl &Out, SmallVectorImpl &Directives, + StringRef Input, DiagnosticsEngine *Diags, + SourceLocation InputSourceLoc) + : Out(Out), Directives(Directives), Input(Input), Diags(Diags), InputSourceLoc(InputSourceLoc) {} - /// Lex the provided source and emit the minimized output. + /// Lex the provided source and emit the directive tokens. /// /// \returns True on error. - bool minimize(); + bool scan(); private: struct IdInfo { @@ -57,31 +57,33 @@ LLVM_NODISCARD IdInfo lexIdentifier(const char *First, const char *const End); LLVM_NODISCARD bool isNextIdentifier(StringRef Id, const char *&First, const char *const End); - LLVM_NODISCARD bool minimizeImpl(const char *First, const char *const End); + LLVM_NODISCARD bool scanImpl(const char *First, const char *const End); LLVM_NODISCARD bool lexPPLine(const char *&First, const char *const End); LLVM_NODISCARD bool lexAt(const char *&First, const char *const End); LLVM_NODISCARD bool lexModule(const char *&First, const char *const End); LLVM_NODISCARD bool lexDefine(const char *&First, const char *const End); LLVM_NODISCARD bool lexPragma(const char *&First, const char *const End); LLVM_NODISCARD bool lexEndif(const char *&First, const char *const End); - LLVM_NODISCARD bool lexDefault(TokenKind Kind, StringRef Directive, + LLVM_NODISCARD bool lexDefault(DirectiveKind Kind, StringRef Directive, const char *&First, const char *const End); - Token &makeToken(TokenKind K) { - Tokens.emplace_back(K, Out.size()); - return Tokens.back(); + Directive &pushDirective(DirectiveKind K) { + Directives.emplace_back(K, Out.size()); + return Directives.back(); } - void popToken() { - Out.resize(Tokens.back().Offset); - Tokens.pop_back(); + void popDirective() { + Out.resize(Directives.back().Offset); + Directives.pop_back(); + } + DirectiveKind topDirective() const { + return Directives.empty() ? pp_none : Directives.back().Kind; } - TokenKind top() const { return Tokens.empty() ? pp_none : Tokens.back().K; } - Minimizer &put(char Byte) { + Scanner &put(char Byte) { Out.push_back(Byte); return *this; } - Minimizer &append(StringRef S) { return append(S.begin(), S.end()); } - Minimizer &append(const char *First, const char *Last) { + Scanner &append(StringRef S) { return append(S.begin(), S.end()); } + Scanner &append(const char *First, const char *Last) { Out.append(First, Last); return *this; } @@ -106,7 +108,7 @@ } // end anonymous namespace -bool Minimizer::reportError(const char *CurPtr, unsigned Err) { +bool Scanner::reportError(const char *CurPtr, unsigned Err) { if (!Diags) return true; assert(CurPtr >= Input.data() && "invalid buffer ptr"); @@ -279,8 +281,7 @@ return Last; } -static const char *findFirstTrailingSpace(const char *First, - const char *Last) { +static const char *findFirstTrailingSpace(const char *First, const char *Last) { const char *LastNonSpace = findLastNonSpace(First, Last); if (Last == LastNonSpace) return Last; @@ -395,13 +396,14 @@ skipLine(First, End); } -void Minimizer::printToNewline(const char *&First, const char *const End) { +void Scanner::printToNewline(const char *&First, const char *const End) { while (First != End && !isVerticalWhitespace(*First)) { const char *Last = First; do { // Iterate over strings correctly to avoid comments and newlines. if (*Last == '"' || *Last == '\'' || - (*Last == '<' && (top() == pp_include || top() == pp_import))) { + (*Last == '<' && + (topDirective() == pp_include || topDirective() == pp_import))) { if (LLVM_UNLIKELY(isRawStringLiteral(First, Last))) skipRawString(Last, End); else @@ -487,8 +489,8 @@ } } -void Minimizer::printAdjacentModuleNameParts(const char *&First, - const char *const End) { +void Scanner::printAdjacentModuleNameParts(const char *&First, + const char *const End) { // Skip over parts of the body. const char *Last = First; do @@ -498,7 +500,7 @@ First = Last; } -bool Minimizer::printAtImportBody(const char *&First, const char *const End) { +bool Scanner::printAtImportBody(const char *&First, const char *const End) { for (;;) { skipWhitespace(First, End); if (First == End) @@ -523,7 +525,7 @@ } } -void Minimizer::printDirectiveBody(const char *&First, const char *const End) { +void Scanner::printDirectiveBody(const char *&First, const char *const End) { skipWhitespace(First, End); // Skip initial whitespace. printToNewline(First, End); while (Out.back() == ' ') @@ -552,8 +554,8 @@ return isAsciiIdentifierContinue(First[0]) ? First : nullptr; } -Minimizer::IdInfo Minimizer::lexIdentifier(const char *First, - const char *const End) { +Scanner::IdInfo Scanner::lexIdentifier(const char *First, + const char *const End) { const char *Last = lexRawIdentifier(First, End); const char *Next = getIdentifierContinuation(Last, End); if (LLVM_LIKELY(!Next)) @@ -571,8 +573,8 @@ SplitIds.try_emplace(StringRef(Id.begin(), Id.size()), 0).first->first()}; } -void Minimizer::printAdjacentMacroArgs(const char *&First, - const char *const End) { +void Scanner::printAdjacentMacroArgs(const char *&First, + const char *const End) { // Skip over parts of the body. const char *Last = First; do @@ -583,7 +585,7 @@ First = Last; } -bool Minimizer::printMacroArgs(const char *&First, const char *const End) { +bool Scanner::printMacroArgs(const char *&First, const char *const End) { assert(*First == '('); put(*First++); for (;;) { @@ -608,8 +610,8 @@ /// /// Updates "First" to just past the next identifier, if any. Returns true iff /// the identifier matches "Id". -bool Minimizer::isNextIdentifier(StringRef Id, const char *&First, - const char *const End) { +bool Scanner::isNextIdentifier(StringRef Id, const char *&First, + const char *const End) { skipWhitespace(First, End); if (First == End || !isAsciiIdentifierStart(*First)) return false; @@ -619,29 +621,29 @@ return FoundId.Name == Id; } -bool Minimizer::lexAt(const char *&First, const char *const End) { +bool Scanner::lexAt(const char *&First, const char *const End) { // Handle "@import". const char *ImportLoc = First++; if (!isNextIdentifier("import", First, End)) { skipLine(First, End); return false; } - makeToken(decl_at_import); + pushDirective(decl_at_import); append("@import "); if (printAtImportBody(First, End)) return reportError( - ImportLoc, diag::err_dep_source_minimizer_missing_sema_after_at_import); + ImportLoc, diag::err_dep_source_scanner_missing_semi_after_at_import); skipWhitespace(First, End); if (First == End) return false; if (!isVerticalWhitespace(*First)) return reportError( - ImportLoc, diag::err_dep_source_minimizer_unexpected_tokens_at_import); + ImportLoc, diag::err_dep_source_scanner_unexpected_tokens_at_import); skipNewline(First, End); return false; } -bool Minimizer::lexModule(const char *&First, const char *const End) { +bool Scanner::lexModule(const char *&First, const char *const End) { IdInfo Id = lexIdentifier(First, End); First = Id.Last; bool Export = false; @@ -679,14 +681,14 @@ } if (Export) { - makeToken(cxx_export_decl); + pushDirective(cxx_export_decl); append("export "); } if (Id.Name == "module") - makeToken(cxx_module_decl); + pushDirective(cxx_module_decl); else - makeToken(cxx_import_decl); + pushDirective(cxx_import_decl); append(Id.Name); append(" "); printToNewline(First, End); @@ -694,8 +696,8 @@ return false; } -bool Minimizer::lexDefine(const char *&First, const char *const End) { - makeToken(pp_define); +bool Scanner::lexDefine(const char *&First, const char *const End) { + pushDirective(pp_define); append("#define "); skipWhitespace(First, End); @@ -728,7 +730,7 @@ return false; } -bool Minimizer::lexPragma(const char *&First, const char *const End) { +bool Scanner::lexPragma(const char *&First, const char *const End) { // #pragma. skipWhitespace(First, End); if (First == End || !isAsciiIdentifierStart(*First)) @@ -739,27 +741,27 @@ if (FoundId.Name == "once") { // #pragma once skipLine(First, End); - makeToken(pp_pragma_once); + pushDirective(pp_pragma_once); append("#pragma once\n"); return false; } if (FoundId.Name == "push_macro") { // #pragma push_macro - makeToken(pp_pragma_push_macro); + pushDirective(pp_pragma_push_macro); append("#pragma push_macro"); printDirectiveBody(First, End); return false; } if (FoundId.Name == "pop_macro") { // #pragma pop_macro - makeToken(pp_pragma_pop_macro); + pushDirective(pp_pragma_pop_macro); append("#pragma pop_macro"); printDirectiveBody(First, End); return false; } if (FoundId.Name == "include_alias") { // #pragma include_alias - makeToken(pp_pragma_include_alias); + pushDirective(pp_pragma_include_alias); append("#pragma include_alias"); printDirectiveBody(First, End); return false; @@ -783,16 +785,16 @@ } // #pragma clang module import. - makeToken(pp_pragma_import); + pushDirective(pp_pragma_import); append("#pragma clang module import "); printDirectiveBody(First, End); return false; } -bool Minimizer::lexEndif(const char *&First, const char *const End) { +bool Scanner::lexEndif(const char *&First, const char *const End) { // Strip out "#else" if it's empty. - if (top() == pp_else) - popToken(); + if (topDirective() == pp_else) + popDirective(); // If "#ifdef" is empty, strip it and skip the "#endif". // @@ -800,8 +802,8 @@ // we can skip empty `#if` and `#elif` blocks as well after scanning for a // literal __has_include in the condition. Even without that rule we could // drop the tokens if we scan for identifiers in the condition and find none. - if (top() == pp_ifdef || top() == pp_ifndef) { - popToken(); + if (topDirective() == pp_ifdef || topDirective() == pp_ifndef) { + popDirective(); skipLine(First, End); return false; } @@ -809,9 +811,9 @@ return lexDefault(pp_endif, "endif", First, End); } -bool Minimizer::lexDefault(TokenKind Kind, StringRef Directive, - const char *&First, const char *const End) { - makeToken(Kind); +bool Scanner::lexDefault(DirectiveKind Kind, StringRef Directive, + const char *&First, const char *const End) { + pushDirective(Kind); put('#').append(Directive).put(' '); printDirectiveBody(First, End); return false; @@ -829,7 +831,7 @@ return false; } -bool Minimizer::lexPPLine(const char *&First, const char *const End) { +bool Scanner::lexPPLine(const char *&First, const char *const End) { assert(First != End); skipWhitespace(First, End); @@ -869,7 +871,7 @@ if (Id.Name == "pragma") return lexPragma(First, End); - auto Kind = llvm::StringSwitch(Id.Name) + auto Kind = llvm::StringSwitch(Id.Name) .Case("include", pp_include) .Case("__include_macros", pp___include_macros) .Case("define", pp_define) @@ -906,7 +908,7 @@ First += 3; } -bool Minimizer::minimizeImpl(const char *First, const char *const End) { +bool Scanner::scanImpl(const char *First, const char *const End) { skipUTF8ByteOrderMark(First, End); while (First != End) if (lexPPLine(First, End)) @@ -914,14 +916,14 @@ return false; } -bool Minimizer::minimize() { - bool Error = minimizeImpl(Input.begin(), Input.end()); +bool Scanner::scan() { + bool Error = scanImpl(Input.begin(), Input.end()); if (!Error) { // Add a trailing newline and an EOF on success. if (!Out.empty() && Out.back() != '\n') Out.push_back('\n'); - makeToken(pp_eof); + pushDirective(pp_eof); } // Null-terminate the output. This way the memory buffer that's passed to @@ -931,9 +933,9 @@ return Error; } -bool clang::minimize_source_to_dependency_directives::computeSkippedRanges( - ArrayRef Input, llvm::SmallVectorImpl &Range) { - struct Directive { +bool clang::dependency_directives_scan::computeSkippedRanges( + ArrayRef Input, llvm::SmallVectorImpl &Range) { + struct IfElseDirective { enum DirectiveKind { If, // if/ifdef/ifndef Else // elif/elifdef/elifndef, else @@ -941,13 +943,13 @@ int Offset; DirectiveKind Kind; }; - llvm::SmallVector Offsets; - for (const Token &T : Input) { - switch (T.K) { + llvm::SmallVector Offsets; + for (const Directive &T : Input) { + switch (T.Kind) { case pp_if: case pp_ifdef: case pp_ifndef: - Offsets.push_back({T.Offset, Directive::If}); + Offsets.push_back({T.Offset, IfElseDirective::If}); break; case pp_elif: @@ -958,7 +960,7 @@ return true; int PreviousOffset = Offsets.back().Offset; Range.push_back({PreviousOffset, T.Offset - PreviousOffset}); - Offsets.push_back({T.Offset, Directive::Else}); + Offsets.push_back({T.Offset, IfElseDirective::Else}); break; } @@ -968,8 +970,8 @@ int PreviousOffset = Offsets.back().Offset; Range.push_back({PreviousOffset, T.Offset - PreviousOffset}); do { - Directive::DirectiveKind Kind = Offsets.pop_back_val().Kind; - if (Kind == Directive::If) + IfElseDirective::DirectiveKind Kind = Offsets.pop_back_val().Kind; + if (Kind == IfElseDirective::If) break; } while (!Offsets.empty()); break; @@ -981,11 +983,11 @@ return false; } -bool clang::minimizeSourceToDependencyDirectives( +bool clang::scanSourceForDependencyDirectives( StringRef Input, SmallVectorImpl &Output, - SmallVectorImpl &Tokens, DiagnosticsEngine *Diags, + SmallVectorImpl &Directives, DiagnosticsEngine *Diags, SourceLocation InputSourceLoc) { Output.clear(); - Tokens.clear(); - return Minimizer(Output, Tokens, Input, Diags, InputSourceLoc).minimize(); + Directives.clear(); + return Scanner(Output, Directives, Input, Diags, InputSourceLoc).scan(); } diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp @@ -7,7 +7,7 @@ //===----------------------------------------------------------------------===// #include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h" -#include "clang/Lex/DependencyDirectivesSourceMinimizer.h" +#include "clang/Lex/DependencyDirectivesScanner.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/SmallVectorMemoryBuffer.h" #include "llvm/Support/Threading.h" @@ -41,13 +41,13 @@ return TentativeEntry(Stat, std::move(Buffer)); } -EntryRef DependencyScanningWorkerFilesystem::minimizeIfNecessary( +EntryRef DependencyScanningWorkerFilesystem::scanForDirectivesIfNecessary( const CachedFileSystemEntry &Entry, StringRef Filename, bool Disable) { if (Entry.isError() || Entry.isDirectory() || Disable || - !shouldMinimize(Filename, Entry.getUniqueID())) + !shouldScanForDirectives(Filename, Entry.getUniqueID())) return EntryRef(/*Minimized=*/false, Filename, Entry); - CachedFileContents *Contents = Entry.getContents(); + CachedFileContents *Contents = Entry.getCachedContents(); assert(Contents && "contents not initialized"); // Double-checked locking. @@ -62,9 +62,9 @@ llvm::SmallString<1024> MinimizedFileContents; // Minimize the file down to directives that might affect the dependencies. - SmallVector Tokens; - if (minimizeSourceToDependencyDirectives(Contents->Original->getBuffer(), - MinimizedFileContents, Tokens)) { + SmallVector Tokens; + if (scanSourceForDependencyDirectives(Contents->Original->getBuffer(), + MinimizedFileContents, Tokens)) { // FIXME: Propagate the diagnostic if desired by the client. // Use the original file if the minimization failed. Contents->MinimizedStorage = @@ -79,10 +79,8 @@ // Compute the skipped PP ranges that speedup skipping over inactive // preprocessor blocks. - llvm::SmallVector - SkippedRanges; - minimize_source_to_dependency_directives::computeSkippedRanges(Tokens, - SkippedRanges); + llvm::SmallVector SkippedRanges; + dependency_directives_scan::computeSkippedRanges(Tokens, SkippedRanges); PreprocessorSkippedRangeMapping Mapping; for (const auto &Range : SkippedRanges) { if (Range.Length < 16) { @@ -189,7 +187,7 @@ /// /// This is kinda hacky, it would be better if we knew what kind of file Clang /// was expecting instead. -static bool shouldMinimizeBasedOnExtension(StringRef Filename) { +static bool shouldScanForDirectivesBasedOnExtension(StringRef Filename) { StringRef Ext = llvm::sys::path::extension(Filename); if (Ext.empty()) return true; // C++ standard library @@ -207,22 +205,22 @@ if (Ext.empty()) return false; // This may be the module cache directory. // Only cache stat failures on source files. - return shouldMinimizeBasedOnExtension(Filename); + return shouldScanForDirectivesBasedOnExtension(Filename); } -void DependencyScanningWorkerFilesystem::disableMinimization( +void DependencyScanningWorkerFilesystem::disableDirectivesScanning( StringRef Filename) { - // Since we're not done setting up `NotToBeMinimized` yet, we need to disable - // minimization explicitly. - if (llvm::ErrorOr Result = - getOrCreateFileSystemEntry(Filename, /*DisableMinimization=*/true)) - NotToBeMinimized.insert(Result->getStatus().getUniqueID()); + // Since we're not done setting up `NotToBeScanned` yet, we need to disable + // directive scanning explicitly. + if (llvm::ErrorOr Result = getOrCreateFileSystemEntry( + Filename, /*DisableDirectivesScanning=*/true)) + NotToBeScanned.insert(Result->getStatus().getUniqueID()); } -bool DependencyScanningWorkerFilesystem::shouldMinimize( +bool DependencyScanningWorkerFilesystem::shouldScanForDirectives( StringRef Filename, llvm::sys::fs::UniqueID UID) { - return shouldMinimizeBasedOnExtension(Filename) && - !NotToBeMinimized.contains(UID); + return shouldScanForDirectivesBasedOnExtension(Filename) && + !NotToBeScanned.contains(UID); } const CachedFileSystemEntry & @@ -275,14 +273,16 @@ llvm::ErrorOr DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry( - StringRef Filename, bool DisableMinimization) { + StringRef Filename, bool DisableDirectivesScanning) { if (const auto *Entry = findEntryByFilenameWithWriteThrough(Filename)) - return minimizeIfNecessary(*Entry, Filename, DisableMinimization) + return scanForDirectivesIfNecessary(*Entry, Filename, + DisableDirectivesScanning) .unwrapError(); auto MaybeEntry = computeAndStoreResult(Filename); if (!MaybeEntry) return MaybeEntry.getError(); - return minimizeIfNecessary(*MaybeEntry, Filename, DisableMinimization) + return scanForDirectivesIfNecessary(*MaybeEntry, Filename, + DisableDirectivesScanning) .unwrapError(); } @@ -301,10 +301,10 @@ /// The VFS that is used by clang consumes the \c CachedFileSystemEntry using /// this subclass. -class MinimizedVFSFile final : public llvm::vfs::File { +class DepScanFile final : public llvm::vfs::File { public: - MinimizedVFSFile(std::unique_ptr Buffer, - llvm::vfs::Status Stat) + DepScanFile(std::unique_ptr Buffer, + llvm::vfs::Status Stat) : Buffer(std::move(Buffer)), Stat(std::move(Stat)) {} static llvm::ErrorOr> @@ -328,14 +328,15 @@ } // end anonymous namespace -llvm::ErrorOr> MinimizedVFSFile::create( - EntryRef Entry, ExcludedPreprocessorDirectiveSkipMapping &PPSkipMappings) { +llvm::ErrorOr> +DepScanFile::create(EntryRef Entry, + ExcludedPreprocessorDirectiveSkipMapping &PPSkipMappings) { assert(!Entry.isError() && "error"); if (Entry.isDirectory()) return std::make_error_code(std::errc::is_a_directory); - auto Result = std::make_unique( + auto Result = std::make_unique( llvm::MemoryBuffer::getMemBuffer(Entry.getContents(), Entry.getStatus().getName(), /*RequiresNullTerminator=*/false), @@ -357,5 +358,5 @@ llvm::ErrorOr Result = getOrCreateFileSystemEntry(Filename); if (!Result) return Result.getError(); - return MinimizedVFSFile::create(Result.get(), PPSkipMappings); + return DepScanFile::create(Result.get(), PPSkipMappings); } diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -183,19 +183,19 @@ // Use the dependency scanning optimized file system if requested to do so. if (DepFS) { - DepFS->enableMinimizationOfAllFiles(); + DepFS->enableDirectivesScanningOfAllFiles(); // Don't minimize any files that contributed to prebuilt modules. The // implicit build validates the modules by comparing the reported sizes of // their inputs to the current state of the filesystem. Minimization would // throw this mechanism off. for (const auto &File : PrebuiltModulesInputFiles) - DepFS->disableMinimization(File.getKey()); + DepFS->disableDirectivesScanning(File.getKey()); // Don't minimize any files that were explicitly passed in the build // settings and that might be opened. for (const auto &E : ScanInstance.getHeaderSearchOpts().UserEntries) - DepFS->disableMinimization(E.Path); + DepFS->disableDirectivesScanning(E.Path); for (const auto &F : ScanInstance.getHeaderSearchOpts().VFSOverlayFiles) - DepFS->disableMinimization(F); + DepFS->disableDirectivesScanning(F); // Support for virtual file system overlays on top of the caching // filesystem. @@ -287,7 +287,7 @@ OverlayFS->pushOverlay(InMemoryFS); RealFS = OverlayFS; - if (Service.getMode() == ScanningMode::MinimizedSourcePreprocessing) + if (Service.getMode() == ScanningMode::DependencyDirectivesScan) DepFS = new DependencyScanningWorkerFilesystem(Service.getSharedCache(), RealFS, PPSkipMappings); if (Service.canReuseFileManager()) diff --git a/clang/test/ClangScanDeps/has_include_if_elif.cpp b/clang/test/ClangScanDeps/has_include_if_elif.cpp --- a/clang/test/ClangScanDeps/has_include_if_elif.cpp +++ b/clang/test/ClangScanDeps/has_include_if_elif.cpp @@ -10,7 +10,7 @@ // RUN: cp %S/Inputs/header.h %t.dir/Inputs/header4.h // RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/has_include_if_elif.json > %t.cdb // -// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -mode preprocess-minimized-sources | \ +// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -mode preprocess-dependency-directives | \ // RUN: FileCheck %s // RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -mode preprocess | \ // RUN: FileCheck %s diff --git a/clang/test/ClangScanDeps/macro-expansions.cpp b/clang/test/ClangScanDeps/macro-expansions.cpp new file mode 100644 --- /dev/null +++ b/clang/test/ClangScanDeps/macro-expansions.cpp @@ -0,0 +1,40 @@ +// This checks that there's no issue with the preprocessor handling user or built-in macro +// expansion during dependency scanning. + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed -e "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json + +// RUN: clang-scan-deps -compilation-database %t/cdb.json | FileCheck %s + +// CHECK: test.o: +// CHECK-NEXT: test.cpp +// CHECK-NEXT: header1.h +// CHECK-NEXT: header2.h + +//--- cdb.json.template +[{ + "directory" : "DIR", + "command" : "clang -c DIR/test.cpp -o DIR/test.o", + "file" : "DIR/test.o" +}] + +//--- test.cpp +#define FN_MACRO(x) 1 +#if FN_MACRO(a) +#include "header1.h" +#endif + +#if __has_cpp_attribute(clang::fallthrough) +#include "header2.h" +#endif + +//--- header1.h +#ifndef _HEADER1_H_ +#define _HEADER1_H_ +#endif + +//--- header2.h +#ifndef _HEADER2_H_ +#define _HEADER2_H_ +#endif diff --git a/clang/test/ClangScanDeps/modulemap-via-vfs.m b/clang/test/ClangScanDeps/modulemap-via-vfs.m --- a/clang/test/ClangScanDeps/modulemap-via-vfs.m +++ b/clang/test/ClangScanDeps/modulemap-via-vfs.m @@ -3,7 +3,7 @@ // RUN: sed -e "s|DIR|%/t.dir|g" %t.dir/build/compile-commands.json.in > %t.dir/build/compile-commands.json // RUN: sed -e "s|DIR|%/t.dir|g" %t.dir/build/vfs.yaml.in > %t.dir/build/vfs.yaml // RUN: clang-scan-deps -compilation-database %t.dir/build/compile-commands.json -j 1 -format experimental-full \ -// RUN: -mode preprocess-minimized-sources -generate-modules-path-args > %t.db +// RUN: -mode preprocess-dependency-directives -generate-modules-path-args > %t.db // RUN: %deps-to-rsp %t.db --module-name=A > %t.A.cc1.rsp // RUN: cat %t.A.cc1.rsp | sed 's:\\\\\?:/:g' | FileCheck %s diff --git a/clang/test/ClangScanDeps/modules-fmodule-name-no-module-built.m b/clang/test/ClangScanDeps/modules-fmodule-name-no-module-built.m --- a/clang/test/ClangScanDeps/modules-fmodule-name-no-module-built.m +++ b/clang/test/ClangScanDeps/modules-fmodule-name-no-module-built.m @@ -10,7 +10,7 @@ // RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/module_fmodule_name_cdb.json > %t.cdb // RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -format experimental-full \ -// RUN: -generate-modules-path-args -mode preprocess-minimized-sources > %t.result +// RUN: -generate-modules-path-args -mode preprocess-dependency-directives > %t.result // RUN: cat %t.result | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t.dir --check-prefixes=CHECK %s #import "header3.h" diff --git a/clang/test/ClangScanDeps/modules-full-by-mod-name.cpp b/clang/test/ClangScanDeps/modules-full-by-mod-name.cpp --- a/clang/test/ClangScanDeps/modules-full-by-mod-name.cpp +++ b/clang/test/ClangScanDeps/modules-full-by-mod-name.cpp @@ -12,11 +12,11 @@ // RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/modules_cdb_clangcl_by_mod_name.json > %t_clangcl.cdb // // RUN: clang-scan-deps -compilation-database %t.cdb -j 4 -format experimental-full \ -// RUN: -mode preprocess-minimized-sources -module-name=header1 > %t.result +// RUN: -mode preprocess-dependency-directives -module-name=header1 > %t.result // RUN: cat %t.result | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t.dir --check-prefixes=CHECK %s // // RUN: clang-scan-deps -compilation-database %t_clangcl.cdb -j 4 -format experimental-full \ -// RUN: -mode preprocess-minimized-sources -module-name=header1 > %t_clangcl.result +// RUN: -mode preprocess-dependency-directives -module-name=header1 > %t_clangcl.result // RUN: cat %t_clangcl.result | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t.dir --check-prefixes=CHECK %s // CHECK: { diff --git a/clang/test/ClangScanDeps/modules-full.cpp b/clang/test/ClangScanDeps/modules-full.cpp --- a/clang/test/ClangScanDeps/modules-full.cpp +++ b/clang/test/ClangScanDeps/modules-full.cpp @@ -11,20 +11,20 @@ // RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/modules_cdb_clangcl.json > %t_clangcl.cdb // // RUN: clang-scan-deps -compilation-database %t.cdb -j 4 -format experimental-full \ -// RUN: -mode preprocess-minimized-sources > %t.result +// RUN: -mode preprocess-dependency-directives > %t.result // RUN: cat %t.result | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t.dir --check-prefixes=CHECK,CHECK-NO-ABS %s // // RUN: clang-scan-deps -compilation-database %t.cdb -j 4 -format experimental-full \ -// RUN: -generate-modules-path-args -mode preprocess-minimized-sources > %t.result +// RUN: -generate-modules-path-args -mode preprocess-dependency-directives > %t.result // RUN: cat %t.result | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t.dir --check-prefixes=CHECK,CHECK-ABS %s // // RUN: clang-scan-deps -compilation-database %t.cdb -j 4 -format experimental-full \ // RUN: -generate-modules-path-args -module-files-dir %t.dir/custom \ -// RUN: -mode preprocess-minimized-sources > %t.result +// RUN: -mode preprocess-dependency-directives > %t.result // RUN: cat %t.result | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t.dir --check-prefixes=CHECK,CHECK-CUSTOM %s // // RUN: clang-scan-deps -compilation-database %t_clangcl.cdb -j 4 -format experimental-full \ -// RUN: -mode preprocess-minimized-sources > %t_clangcl.result +// RUN: -mode preprocess-dependency-directives > %t_clangcl.result // RUN: cat %t_clangcl.result | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t.dir --check-prefixes=CHECK,CHECK-NO-ABS %s #include "header.h" diff --git a/clang/test/ClangScanDeps/modules-inferred-explicit-build.m b/clang/test/ClangScanDeps/modules-inferred-explicit-build.m --- a/clang/test/ClangScanDeps/modules-inferred-explicit-build.m +++ b/clang/test/ClangScanDeps/modules-inferred-explicit-build.m @@ -6,7 +6,7 @@ // RUN: %S/Inputs/modules_inferred_cdb.json > %t.cdb // // RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -format experimental-full \ -// RUN: -mode preprocess-minimized-sources -generate-modules-path-args > %t.db +// RUN: -mode preprocess-dependency-directives -generate-modules-path-args > %t.db // RUN: %deps-to-rsp %t.db --module-name=Inferred > %t.inferred.cc1.rsp // RUN: %deps-to-rsp %t.db --module-name=System > %t.system.cc1.rsp // RUN: %deps-to-rsp %t.db --tu-index=0 > %t.tu.rsp diff --git a/clang/test/ClangScanDeps/modules-inferred.m b/clang/test/ClangScanDeps/modules-inferred.m --- a/clang/test/ClangScanDeps/modules-inferred.m +++ b/clang/test/ClangScanDeps/modules-inferred.m @@ -6,7 +6,7 @@ // RUN: %/S/Inputs/modules_inferred_cdb.json > %t.cdb // // RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -format experimental-full \ -// RUN: -generate-modules-path-args -mode preprocess-minimized-sources > %t.result +// RUN: -generate-modules-path-args -mode preprocess-dependency-directives > %t.result // RUN: cat %t.result | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t.dir -DSOURCEDIR=%/S --check-prefixes=CHECK #include diff --git a/clang/test/ClangScanDeps/modules.cpp b/clang/test/ClangScanDeps/modules.cpp --- a/clang/test/ClangScanDeps/modules.cpp +++ b/clang/test/ClangScanDeps/modules.cpp @@ -13,9 +13,9 @@ // RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/modules_cdb.json > %t.cdb // RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/modules_cdb_clangcl.json > %t_clangcl.cdb // -// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -mode preprocess-minimized-sources | \ +// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -mode preprocess-dependency-directives | \ // RUN: FileCheck --check-prefixes=CHECK1,CHECK2,CHECK2NO %s -// RUN: clang-scan-deps -compilation-database %t_clangcl.cdb -j 1 -mode preprocess-minimized-sources | \ +// RUN: clang-scan-deps -compilation-database %t_clangcl.cdb -j 1 -mode preprocess-dependency-directives | \ // RUN: FileCheck --check-prefixes=CHECK1,CHECK2,CHECK2NO %s // // The output order is non-deterministic when using more than one thread, @@ -23,17 +23,17 @@ // as it might fail if the results for `modules_cdb_input.cpp` are reported before // `modules_cdb_input2.cpp`. // -// RUN: clang-scan-deps -compilation-database %t.cdb -j 2 -mode preprocess-minimized-sources | \ +// RUN: clang-scan-deps -compilation-database %t.cdb -j 2 -mode preprocess-dependency-directives | \ // RUN: FileCheck --check-prefix=CHECK1 %s -// RUN: clang-scan-deps -compilation-database %t_clangcl.cdb -j 2 -mode preprocess-minimized-sources | \ +// RUN: clang-scan-deps -compilation-database %t_clangcl.cdb -j 2 -mode preprocess-dependency-directives | \ // RUN: FileCheck --check-prefix=CHECK1 %s // RUN: clang-scan-deps -compilation-database %t.cdb -j 2 -mode preprocess | \ // RUN: FileCheck --check-prefix=CHECK1 %s // RUN: clang-scan-deps -compilation-database %t_clangcl.cdb -j 2 -mode preprocess | \ // RUN: FileCheck --check-prefix=CHECK1 %s -// RUN: clang-scan-deps -compilation-database %t.cdb -j 2 -mode preprocess-minimized-sources | \ +// RUN: clang-scan-deps -compilation-database %t.cdb -j 2 -mode preprocess-dependency-directives | \ // RUN: FileCheck --check-prefix=CHECK2 %s -// RUN: clang-scan-deps -compilation-database %t_clangcl.cdb -j 2 -mode preprocess-minimized-sources | \ +// RUN: clang-scan-deps -compilation-database %t_clangcl.cdb -j 2 -mode preprocess-dependency-directives | \ // RUN: FileCheck --check-prefix=CHECK2 %s // RUN: clang-scan-deps -compilation-database %t.cdb -j 2 -mode preprocess | \ // RUN: FileCheck --check-prefix=CHECK2 %s diff --git a/clang/test/ClangScanDeps/preprocess_minimized_pragmas.cpp b/clang/test/ClangScanDeps/preprocess_minimized_pragmas.cpp --- a/clang/test/ClangScanDeps/preprocess_minimized_pragmas.cpp +++ b/clang/test/ClangScanDeps/preprocess_minimized_pragmas.cpp @@ -11,7 +11,7 @@ // RUN: touch %t.dir/Inputs/c_alias.h // RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/preprocess_minimized_pragmas_cdb.json > %t.cdb // -// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -mode preprocess-minimized-sources | \ +// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -mode preprocess-dependency-directives | \ // RUN: FileCheck %s // RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -mode preprocess | \ // RUN: FileCheck %s diff --git a/clang/test/ClangScanDeps/regular_cdb.cpp b/clang/test/ClangScanDeps/regular_cdb.cpp --- a/clang/test/ClangScanDeps/regular_cdb.cpp +++ b/clang/test/ClangScanDeps/regular_cdb.cpp @@ -10,9 +10,9 @@ // RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/regular_cdb.json > %t.cdb // RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/regular_cdb_clangcl.json > %t_clangcl.cdb // -// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -mode preprocess-minimized-sources | \ +// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -mode preprocess-dependency-directives | \ // RUN: FileCheck --check-prefixes=CHECK1,CHECK2,CHECK2NO,CHECK3 %s -// RUN: clang-scan-deps -compilation-database %t_clangcl.cdb -j 1 -mode preprocess-minimized-sources | \ +// RUN: clang-scan-deps -compilation-database %t_clangcl.cdb -j 1 -mode preprocess-dependency-directives | \ // RUN: FileCheck --check-prefixes=CHECK1,CHECK2,CHECK2NO,CHECK3 %s // RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -mode preprocess | \ @@ -31,9 +31,9 @@ // as it might fail if the results for `regular_cdb_input.cpp` are reported before // `regular_cdb_input2.cpp`. // -// RUN: clang-scan-deps -compilation-database %t.cdb -j 2 -mode preprocess-minimized-sources | \ +// RUN: clang-scan-deps -compilation-database %t.cdb -j 2 -mode preprocess-dependency-directives | \ // RUN: FileCheck --check-prefix=CHECK1 %s -// RUN: clang-scan-deps -compilation-database %t_clangcl.cdb -j 2 -mode preprocess-minimized-sources | \ +// RUN: clang-scan-deps -compilation-database %t_clangcl.cdb -j 2 -mode preprocess-dependency-directives | \ // RUN: FileCheck --check-prefix=CHECK1 %s // RUN: clang-scan-deps -compilation-database %t.cdb -j 2 -mode preprocess | \ @@ -41,9 +41,9 @@ // RUN: clang-scan-deps -compilation-database %t_clangcl.cdb -j 2 -mode preprocess | \ // RUN: FileCheck --check-prefix=CHECK1 %s -// RUN: clang-scan-deps -compilation-database %t.cdb -j 2 -mode preprocess-minimized-sources | \ +// RUN: clang-scan-deps -compilation-database %t.cdb -j 2 -mode preprocess-dependency-directives | \ // RUN: FileCheck --check-prefix=CHECK2 %s -// RUN: clang-scan-deps -compilation-database %t_clangcl.cdb -j 2 -mode preprocess-minimized-sources | \ +// RUN: clang-scan-deps -compilation-database %t_clangcl.cdb -j 2 -mode preprocess-dependency-directives | \ // RUN: FileCheck --check-prefix=CHECK2 %s // RUN: clang-scan-deps -compilation-database %t.cdb -j 2 -mode preprocess | \ diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp --- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp +++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp @@ -116,15 +116,15 @@ "mode", llvm::cl::desc("The preprocessing mode used to compute the dependencies"), llvm::cl::values( - clEnumValN(ScanningMode::MinimizedSourcePreprocessing, - "preprocess-minimized-sources", - "The set of dependencies is computed by preprocessing the " - "source files that were minimized to only include the " - "contents that might affect the dependencies"), + clEnumValN(ScanningMode::DependencyDirectivesScan, + "preprocess-dependency-directives", + "The set of dependencies is computed by preprocessing with " + "special lexing after scanning the source files to get the " + "directives that might affect the dependencies"), clEnumValN(ScanningMode::CanonicalPreprocessing, "preprocess", "The set of dependencies is computed by preprocessing the " - "unmodified source files")), - llvm::cl::init(ScanningMode::MinimizedSourcePreprocessing), + "source files")), + llvm::cl::init(ScanningMode::DependencyDirectivesScan), llvm::cl::cat(DependencyScannerCategory)); static llvm::cl::opt Format( diff --git a/clang/unittests/Lex/CMakeLists.txt b/clang/unittests/Lex/CMakeLists.txt --- a/clang/unittests/Lex/CMakeLists.txt +++ b/clang/unittests/Lex/CMakeLists.txt @@ -3,7 +3,7 @@ ) add_clang_unittest(LexTests - DependencyDirectivesSourceMinimizerTest.cpp + DependencyDirectivesScannerTest.cpp HeaderMapTest.cpp HeaderSearchTest.cpp LexerTest.cpp diff --git a/clang/unittests/Lex/DependencyDirectivesSourceMinimizerTest.cpp b/clang/unittests/Lex/DependencyDirectivesScannerTest.cpp rename from clang/unittests/Lex/DependencyDirectivesSourceMinimizerTest.cpp rename to clang/unittests/Lex/DependencyDirectivesScannerTest.cpp --- a/clang/unittests/Lex/DependencyDirectivesSourceMinimizerTest.cpp +++ b/clang/unittests/Lex/DependencyDirectivesScannerTest.cpp @@ -1,4 +1,4 @@ -//===- unittests/Lex/DependencyDirectivesSourceMinimizer.cpp - -----------===// +//===- unittests/Lex/DependencyDirectivesScannerTest.cpp ------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,45 +6,48 @@ // //===----------------------------------------------------------------------===// -#include "clang/Lex/DependencyDirectivesSourceMinimizer.h" +#include "clang/Lex/DependencyDirectivesScanner.h" #include "llvm/ADT/SmallString.h" #include "gtest/gtest.h" using namespace llvm; using namespace clang; -using namespace clang::minimize_source_to_dependency_directives; +using namespace clang::dependency_directives_scan; -namespace clang { - -bool minimizeSourceToDependencyDirectives(StringRef Input, - SmallVectorImpl &Out) { - SmallVector Tokens; - return minimizeSourceToDependencyDirectives(Input, Out, Tokens); +static bool minimizeSourceToDependencyDirectives(StringRef Input, + SmallVectorImpl &Out) { + SmallVector Directives; + return scanSourceForDependencyDirectives(Input, Out, Directives); } -} // end namespace clang +static bool +minimizeSourceToDependencyDirectives(StringRef Input, + SmallVectorImpl &Out, + SmallVectorImpl &Directives) { + return scanSourceForDependencyDirectives(Input, Out, Directives); +} namespace { TEST(MinimizeSourceToDependencyDirectivesTest, Empty) { SmallVector Out; - SmallVector Tokens; + SmallVector Directives; - ASSERT_FALSE(minimizeSourceToDependencyDirectives("", Out, Tokens)); + ASSERT_FALSE(minimizeSourceToDependencyDirectives("", Out, Directives)); EXPECT_TRUE(Out.empty()); - ASSERT_EQ(1u, Tokens.size()); - ASSERT_EQ(pp_eof, Tokens.back().K); + ASSERT_EQ(1u, Directives.size()); + ASSERT_EQ(pp_eof, Directives.back().Kind); ASSERT_FALSE( - minimizeSourceToDependencyDirectives("abc def\nxyz", Out, Tokens)); + minimizeSourceToDependencyDirectives("abc def\nxyz", Out, Directives)); EXPECT_TRUE(Out.empty()); - ASSERT_EQ(1u, Tokens.size()); - ASSERT_EQ(pp_eof, Tokens.back().K); + ASSERT_EQ(1u, Directives.size()); + ASSERT_EQ(pp_eof, Directives.back().Kind); } -TEST(MinimizeSourceToDependencyDirectivesTest, AllTokens) { +TEST(MinimizeSourceToDependencyDirectivesTest, AllDirectives) { SmallVector Out; - SmallVector Tokens; + SmallVector Directives; ASSERT_FALSE( minimizeSourceToDependencyDirectives("#define A\n" @@ -68,41 +71,41 @@ "#pragma include_alias(, )\n" "export module m;\n" "import m;\n", - Out, Tokens)); - EXPECT_EQ(pp_define, Tokens[0].K); - EXPECT_EQ(pp_undef, Tokens[1].K); - EXPECT_EQ(pp_endif, Tokens[2].K); - EXPECT_EQ(pp_if, Tokens[3].K); - EXPECT_EQ(pp_ifdef, Tokens[4].K); - EXPECT_EQ(pp_ifndef, Tokens[5].K); - EXPECT_EQ(pp_elifdef, Tokens[6].K); - EXPECT_EQ(pp_elifndef, Tokens[7].K); - EXPECT_EQ(pp_elif, Tokens[8].K); - EXPECT_EQ(pp_else, Tokens[9].K); - EXPECT_EQ(pp_include, Tokens[10].K); - EXPECT_EQ(pp_include_next, Tokens[11].K); - EXPECT_EQ(pp___include_macros, Tokens[12].K); - EXPECT_EQ(pp_import, Tokens[13].K); - EXPECT_EQ(decl_at_import, Tokens[14].K); - EXPECT_EQ(pp_pragma_import, Tokens[15].K); - EXPECT_EQ(pp_pragma_push_macro, Tokens[16].K); - EXPECT_EQ(pp_pragma_pop_macro, Tokens[17].K); - EXPECT_EQ(pp_pragma_include_alias, Tokens[18].K); - EXPECT_EQ(cxx_export_decl, Tokens[19].K); - EXPECT_EQ(cxx_module_decl, Tokens[20].K); - EXPECT_EQ(cxx_import_decl, Tokens[21].K); - EXPECT_EQ(pp_eof, Tokens[22].K); + Out, Directives)); + EXPECT_EQ(pp_define, Directives[0].Kind); + EXPECT_EQ(pp_undef, Directives[1].Kind); + EXPECT_EQ(pp_endif, Directives[2].Kind); + EXPECT_EQ(pp_if, Directives[3].Kind); + EXPECT_EQ(pp_ifdef, Directives[4].Kind); + EXPECT_EQ(pp_ifndef, Directives[5].Kind); + EXPECT_EQ(pp_elifdef, Directives[6].Kind); + EXPECT_EQ(pp_elifndef, Directives[7].Kind); + EXPECT_EQ(pp_elif, Directives[8].Kind); + EXPECT_EQ(pp_else, Directives[9].Kind); + EXPECT_EQ(pp_include, Directives[10].Kind); + EXPECT_EQ(pp_include_next, Directives[11].Kind); + EXPECT_EQ(pp___include_macros, Directives[12].Kind); + EXPECT_EQ(pp_import, Directives[13].Kind); + EXPECT_EQ(decl_at_import, Directives[14].Kind); + EXPECT_EQ(pp_pragma_import, Directives[15].Kind); + EXPECT_EQ(pp_pragma_push_macro, Directives[16].Kind); + EXPECT_EQ(pp_pragma_pop_macro, Directives[17].Kind); + EXPECT_EQ(pp_pragma_include_alias, Directives[18].Kind); + EXPECT_EQ(cxx_export_decl, Directives[19].Kind); + EXPECT_EQ(cxx_module_decl, Directives[20].Kind); + EXPECT_EQ(cxx_import_decl, Directives[21].Kind); + EXPECT_EQ(pp_eof, Directives[22].Kind); } TEST(MinimizeSourceToDependencyDirectivesTest, Define) { SmallVector Out; - SmallVector Tokens; + SmallVector Directives; ASSERT_FALSE( - minimizeSourceToDependencyDirectives("#define MACRO", Out, Tokens)); + minimizeSourceToDependencyDirectives("#define MACRO", Out, Directives)); EXPECT_STREQ("#define MACRO\n", Out.data()); - ASSERT_EQ(2u, Tokens.size()); - ASSERT_EQ(pp_define, Tokens.front().K); + ASSERT_EQ(2u, Directives.size()); + ASSERT_EQ(pp_define, Directives.front().Kind); } TEST(MinimizeSourceToDependencyDirectivesTest, DefineSpacing) { @@ -679,18 +682,17 @@ TEST(MinimizeSourceToDependencyDirectivesTest, PragmaOnce) { SmallVector Out; - SmallVector Tokens; + SmallVector Directives; StringRef Source = R"(// comment #pragma once // another comment #include )"; - ASSERT_FALSE(minimizeSourceToDependencyDirectives(Source, Out, Tokens)); + ASSERT_FALSE(minimizeSourceToDependencyDirectives(Source, Out, Directives)); EXPECT_STREQ("#pragma once\n#include \n", Out.data()); - ASSERT_EQ(Tokens.size(), 3u); - EXPECT_EQ(Tokens[0].K, - minimize_source_to_dependency_directives::pp_pragma_once); + ASSERT_EQ(Directives.size(), 3u); + EXPECT_EQ(Directives[0].Kind, dependency_directives_scan::pp_pragma_once); Source = R"(// comment #pragma once extra tokens @@ -758,7 +760,7 @@ TEST(MinimizeSourceToDependencyDirectivesTest, CxxModules) { SmallVector Out; - SmallVector Tokens; + SmallVector Directives; StringRef Source = R"( module; @@ -787,29 +789,28 @@ import f(->a = 3); } )"; - ASSERT_FALSE(minimizeSourceToDependencyDirectives(Source, Out, Tokens)); + ASSERT_FALSE(minimizeSourceToDependencyDirectives(Source, Out, Directives)); EXPECT_STREQ("#include \"textual-header.h\"\nexport module m;\n" "export import :l [[rename]];\n" "import <<= 3;\nimport a b d e d e f e;\n" "import foo [[no_unique_address]];\nimport foo();\n" - "import f(:sefse);\nimport f(->a = 3);\n", Out.data()); - ASSERT_EQ(Tokens.size(), 12u); - EXPECT_EQ(Tokens[0].K, - minimize_source_to_dependency_directives::pp_include); - EXPECT_EQ(Tokens[2].K, - minimize_source_to_dependency_directives::cxx_module_decl); + "import f(:sefse);\nimport f(->a = 3);\n", + Out.data()); + ASSERT_EQ(Directives.size(), 12u); + EXPECT_EQ(Directives[0].Kind, dependency_directives_scan::pp_include); + EXPECT_EQ(Directives[2].Kind, dependency_directives_scan::cxx_module_decl); } TEST(MinimizeSourceToDependencyDirectivesTest, SkippedPPRangesBasic) { SmallString<128> Out; - SmallVector Toks; + SmallVector Directives; StringRef Source = "#ifndef GUARD\n" "#define GUARD\n" "void foo();\n" "#endif\n"; - ASSERT_FALSE(minimizeSourceToDependencyDirectives(Source, Out, Toks)); + ASSERT_FALSE(minimizeSourceToDependencyDirectives(Source, Out, Directives)); SmallVector Ranges; - ASSERT_FALSE(computeSkippedRanges(Toks, Ranges)); + ASSERT_FALSE(computeSkippedRanges(Directives, Ranges)); EXPECT_EQ(Ranges.size(), 1u); EXPECT_EQ(Ranges[0].Offset, 0); EXPECT_EQ(Ranges[0].Length, (int)Out.find("#endif")); @@ -817,7 +818,7 @@ TEST(MinimizeSourceToDependencyDirectivesTest, SkippedPPRangesBasicElifdef) { SmallString<128> Out; - SmallVector Toks; + SmallVector Directives; StringRef Source = "#ifdef BLAH\n" "void skip();\n" "#elifdef BLAM\n" @@ -826,9 +827,9 @@ "#define GUARD\n" "void foo();\n" "#endif\n"; - ASSERT_FALSE(minimizeSourceToDependencyDirectives(Source, Out, Toks)); + ASSERT_FALSE(minimizeSourceToDependencyDirectives(Source, Out, Directives)); SmallVector Ranges; - ASSERT_FALSE(computeSkippedRanges(Toks, Ranges)); + ASSERT_FALSE(computeSkippedRanges(Directives, Ranges)); EXPECT_EQ(Ranges.size(), 3u); EXPECT_EQ(Ranges[0].Offset, 0); EXPECT_EQ(Ranges[0].Length, (int)Out.find("#elifdef")); @@ -840,7 +841,7 @@ TEST(MinimizeSourceToDependencyDirectivesTest, SkippedPPRangesNested) { SmallString<128> Out; - SmallVector Toks; + SmallVector Directives; StringRef Source = "#ifndef GUARD\n" "#define GUARD\n" "#if FOO\n" @@ -851,9 +852,9 @@ "#else\n" "#include nothing\n" "#endif\n"; - ASSERT_FALSE(minimizeSourceToDependencyDirectives(Source, Out, Toks)); + ASSERT_FALSE(minimizeSourceToDependencyDirectives(Source, Out, Directives)); SmallVector Ranges; - ASSERT_FALSE(computeSkippedRanges(Toks, Ranges)); + ASSERT_FALSE(computeSkippedRanges(Directives, Ranges)); EXPECT_EQ(Ranges.size(), 4u); EXPECT_EQ(Ranges[0].Offset, (int)Out.find("#if FOO")); EXPECT_EQ(Ranges[0].Offset + Ranges[0].Length, (int)Out.find("#elif")); diff --git a/clang/unittests/Tooling/DependencyScannerTest.cpp b/clang/unittests/Tooling/DependencyScannerTest.cpp --- a/clang/unittests/Tooling/DependencyScannerTest.cpp +++ b/clang/unittests/Tooling/DependencyScannerTest.cpp @@ -215,9 +215,9 @@ ExcludedPreprocessorDirectiveSkipMapping Mappings; DependencyScanningWorkerFilesystem DepFS(SharedCache, VFS, Mappings); - DepFS.enableMinimizationOfAllFiles(); // Let's be explicit for clarity. + DepFS.enableDirectivesScanningOfAllFiles(); // Let's be explicit for clarity. auto StatusMinimized0 = DepFS.status("/mod.h"); - DepFS.disableMinimization("/mod.h"); + DepFS.disableDirectivesScanning("/mod.h"); auto StatusFull1 = DepFS.status("/mod.h"); EXPECT_TRUE(StatusMinimized0); @@ -238,9 +238,9 @@ ExcludedPreprocessorDirectiveSkipMapping Mappings; DependencyScanningWorkerFilesystem DepFS(SharedCache, VFS, Mappings); - DepFS.disableMinimization("/mod.h"); + DepFS.disableDirectivesScanning("/mod.h"); auto StatusFull0 = DepFS.status("/mod.h"); - DepFS.enableMinimizationOfAllFiles(); + DepFS.enableDirectivesScanningOfAllFiles(); auto StatusMinimized1 = DepFS.status("/mod.h"); EXPECT_TRUE(StatusFull0);