diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h --- a/clang/include/clang/Lex/Preprocessor.h +++ b/clang/include/clang/Lex/Preprocessor.h @@ -33,6 +33,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/FoldingSet.h" +#include "llvm/ADT/FunctionExtras.h" #include "llvm/ADT/None.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/PointerUnion.h" @@ -48,8 +49,8 @@ #include #include #include -#include #include +#include #include #include #include @@ -124,6 +125,7 @@ friend class VAOptDefinitionContext; friend class VariadicMacroScopeGuard; + llvm::unique_function OnToken; std::shared_ptr PPOpts; DiagnosticsEngine *Diags; LangOptions &LangOpts; @@ -997,6 +999,13 @@ } /// \} + /// Register a function that would be called on each token in the final + /// expanded token stream. + /// This also reports annotation tokens produced by the parser. + void setTokenWatcher(llvm::unique_function F) { + OnToken = std::move(F); + } + bool isMacroDefined(StringRef Id) { return isMacroDefined(&Identifiers.get(Id)); } @@ -1281,6 +1290,7 @@ void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro, MacroArgs *Args); +private: /// Add a "macro" context to the top of the include stack, /// which will cause the lexer to start returning the specified tokens. /// @@ -1292,18 +1302,24 @@ /// of tokens has a permanent owner somewhere, so they do not need to be /// copied. If it is true, it assumes the array of tokens is allocated with /// \c new[] and the Preprocessor will delete[] it. -private: + /// + /// If \p IsReinject the resulting tokens will have Token::IsReinjected flag + /// set, see the flag documentation for details. void EnterTokenStream(const Token *Toks, unsigned NumToks, - bool DisableMacroExpansion, bool OwnsTokens); + bool DisableMacroExpansion, bool OwnsTokens, + bool IsReinject); public: void EnterTokenStream(std::unique_ptr Toks, unsigned NumToks, - bool DisableMacroExpansion) { - EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion, true); + bool DisableMacroExpansion, bool IsReinject) { + EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion, true, + IsReinject); } - void EnterTokenStream(ArrayRef Toks, bool DisableMacroExpansion) { - EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion, false); + void EnterTokenStream(ArrayRef Toks, bool DisableMacroExpansion, + bool IsReinject) { + EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion, false, + IsReinject); } /// Pop the current lexer/macro exp off the top of the lexer stack. @@ -1448,15 +1464,18 @@ /// /// If BackTrack() is called afterwards, the token will remain at the /// insertion point. - void EnterToken(const Token &Tok) { + /// If \p IsReinject is true, resulting token will have Token::IsReinjected + /// flag set. See the flag documentation for details. + void EnterToken(const Token &Tok, bool IsReinject) { if (LexLevel) { // It's not correct in general to enter caching lex mode while in the // middle of a nested lexing action. auto TokCopy = llvm::make_unique(1); TokCopy[0] = Tok; - EnterTokenStream(std::move(TokCopy), 1, true); + EnterTokenStream(std::move(TokCopy), 1, true, IsReinject); } else { EnterCachingLexMode(); + assert(IsReinject && "new tokens in the middle of cached stream"); CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok); } } @@ -2135,7 +2154,7 @@ //===--------------------------------------------------------------------===// // Caching stuff. - void CachingLex(Token &Result, bool &IsNewToken); + void CachingLex(Token &Result); bool InCachingLexMode() const { // If the Lexer pointers are 0 and IncludeMacroStack is empty, it means diff --git a/clang/include/clang/Lex/Token.h b/clang/include/clang/Lex/Token.h --- a/clang/include/clang/Lex/Token.h +++ b/clang/include/clang/Lex/Token.h @@ -70,20 +70,23 @@ public: // Various flags set per token: enum TokenFlags { - StartOfLine = 0x01, // At start of line or only after whitespace - // (considering the line after macro expansion). - LeadingSpace = 0x02, // Whitespace exists before this token (considering - // whitespace after macro expansion). - DisableExpand = 0x04, // This identifier may never be macro expanded. - NeedsCleaning = 0x08, // Contained an escaped newline or trigraph. + StartOfLine = 0x01, // At start of line or only after whitespace + // (considering the line after macro expansion). + LeadingSpace = 0x02, // Whitespace exists before this token (considering + // whitespace after macro expansion). + DisableExpand = 0x04, // This identifier may never be macro expanded. + NeedsCleaning = 0x08, // Contained an escaped newline or trigraph. LeadingEmptyMacro = 0x10, // Empty macro exists before this token. - HasUDSuffix = 0x20, // This string or character literal has a ud-suffix. - HasUCN = 0x40, // This identifier contains a UCN. - IgnoredComma = 0x80, // This comma is not a macro argument separator (MS). + HasUDSuffix = 0x20, // This string or character literal has a ud-suffix. + HasUCN = 0x40, // This identifier contains a UCN. + IgnoredComma = 0x80, // This comma is not a macro argument separator (MS). StringifiedInMacro = 0x100, // This string or character literal is formed by // macro stringizing or charizing operator. CommaAfterElided = 0x200, // The comma following this token was elided (MS). IsEditorPlaceholder = 0x400, // This identifier is a placeholder. + IsReinjected = 0x800, // A phase 4 token that was produced before and + // re-added, e.g. via EnterTokenStream. Annotation + // tokens are *not* reinjected. }; tok::TokenKind getKind() const { return Kind; } diff --git a/clang/include/clang/Lex/TokenLexer.h b/clang/include/clang/Lex/TokenLexer.h --- a/clang/include/clang/Lex/TokenLexer.h +++ b/clang/include/clang/Lex/TokenLexer.h @@ -96,6 +96,10 @@ /// should not be subject to further macro expansion. bool DisableMacroExpansion : 1; + /// When true, the produced tokens have Token::IsReinjected flag set. + /// See the flag documentation for details. + bool IsReinject : 1; + public: /// Create a TokenLexer for the specified macro with the specified actual /// arguments. Note that this ctor takes ownership of the ActualArgs pointer. @@ -111,9 +115,9 @@ /// specified, this takes ownership of the tokens and delete[]'s them when /// the token lexer is empty. TokenLexer(const Token *TokArray, unsigned NumToks, bool DisableExpansion, - bool ownsTokens, Preprocessor &pp) + bool ownsTokens, bool isReinject, Preprocessor &pp) : PP(pp), OwnsTokens(false) { - Init(TokArray, NumToks, DisableExpansion, ownsTokens); + Init(TokArray, NumToks, DisableExpansion, ownsTokens, isReinject); } TokenLexer(const TokenLexer &) = delete; @@ -132,8 +136,8 @@ /// /// DisableExpansion is true when macro expansion of tokens lexed from this /// stream should be disabled. - void Init(const Token *TokArray, unsigned NumToks, - bool DisableMacroExpansion, bool OwnsTokens); + void Init(const Token *TokArray, unsigned NumToks, bool DisableMacroExpansion, + bool OwnsTokens, bool IsReinject); /// If the next token lexed will pop this macro off the /// expansion stack, return 2. If the next unexpanded token is a '(', return diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h --- a/clang/include/clang/Parse/Parser.h +++ b/clang/include/clang/Parse/Parser.h @@ -254,7 +254,7 @@ Depth = Depth - AddedLevels + D; AddedLevels = D; } - + unsigned getDepth() const { return Depth; } unsigned getOriginalDepth() const { return Depth - AddedLevels; } }; @@ -536,9 +536,9 @@ /// token the current token. void UnconsumeToken(Token &Consumed) { Token Next = Tok; - PP.EnterToken(Consumed); + PP.EnterToken(Consumed, /*IsReinject*/true); PP.Lex(Tok); - PP.EnterToken(Next); + PP.EnterToken(Next, /*IsReinject*/true); } SourceLocation ConsumeAnnotationToken() { diff --git a/clang/lib/Frontend/PrintPreprocessedOutput.cpp b/clang/lib/Frontend/PrintPreprocessedOutput.cpp --- a/clang/lib/Frontend/PrintPreprocessedOutput.cpp +++ b/clang/lib/Frontend/PrintPreprocessedOutput.cpp @@ -678,7 +678,8 @@ auto Toks = llvm::make_unique(1); Toks[0] = PragmaTok; PP.EnterTokenStream(std::move(Toks), /*NumToks=*/1, - /*DisableMacroExpansion=*/false); + /*DisableMacroExpansion=*/false, + /*IsReinject=*/false); PP.Lex(PragmaTok); } Token PrevToken; diff --git a/clang/lib/Lex/MacroArgs.cpp b/clang/lib/Lex/MacroArgs.cpp --- a/clang/lib/Lex/MacroArgs.cpp +++ b/clang/lib/Lex/MacroArgs.cpp @@ -181,7 +181,7 @@ // list. With this installed, we lex expanded tokens until we hit the EOF // token at the end of the unexp list. PP.EnterTokenStream(AT, NumToks, false /*disable expand*/, - false /*owns tokens*/); + false /*owns tokens*/, false /*is reinject*/); // Lex all of the macro-expanded tokens into Result. do { diff --git a/clang/lib/Lex/PPCaching.cpp b/clang/lib/Lex/PPCaching.cpp --- a/clang/lib/Lex/PPCaching.cpp +++ b/clang/lib/Lex/PPCaching.cpp @@ -45,7 +45,7 @@ recomputeCurLexerKind(); } -void Preprocessor::CachingLex(Token &Result, bool &IsNewToken) { +void Preprocessor::CachingLex(Token &Result) { if (!InCachingLexMode()) return; @@ -55,7 +55,7 @@ if (CachedLexPos < CachedTokens.size()) { Result = CachedTokens[CachedLexPos++]; - IsNewToken = false; + Result.setFlag(Token::IsReinjected); return; } diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp --- a/clang/lib/Lex/PPDirectives.cpp +++ b/clang/lib/Lex/PPDirectives.cpp @@ -1035,7 +1035,7 @@ // Enter this token stream so that we re-lex the tokens. Make sure to // enable macro expansion, in case the token after the # is an identifier // that is expanded. - EnterTokenStream(std::move(Toks), 2, false); + EnterTokenStream(std::move(Toks), 2, false, /*IsReinject*/false); return; } @@ -1518,7 +1518,7 @@ Tok[0].setLocation(Range.getBegin()); Tok[0].setAnnotationEndLoc(Range.getEnd()); Tok[0].setAnnotationValue(AnnotationVal); - EnterTokenStream(std::move(Tok), 1, true); + EnterTokenStream(std::move(Tok), 1, true, /*IsReinject*/ false); } /// Produce a diagnostic informing the user that a #include or similar diff --git a/clang/lib/Lex/PPLexerChange.cpp b/clang/lib/Lex/PPLexerChange.cpp --- a/clang/lib/Lex/PPLexerChange.cpp +++ b/clang/lib/Lex/PPLexerChange.cpp @@ -154,10 +154,11 @@ /// must be freed. /// void Preprocessor::EnterTokenStream(const Token *Toks, unsigned NumToks, - bool DisableMacroExpansion, - bool OwnsTokens) { + bool DisableMacroExpansion, bool OwnsTokens, + bool IsReinject) { if (CurLexerKind == CLK_CachingLexer) { if (CachedLexPos < CachedTokens.size()) { + assert(IsReinject && "new tokens in the middle of cached stream"); // We're entering tokens into the middle of our cached token stream. We // can't represent that, so just insert the tokens into the buffer. CachedTokens.insert(CachedTokens.begin() + CachedLexPos, @@ -170,7 +171,8 @@ // New tokens are at the end of the cached token sequnece; insert the // token stream underneath the caching lexer. ExitCachingLexMode(); - EnterTokenStream(Toks, NumToks, DisableMacroExpansion, OwnsTokens); + EnterTokenStream(Toks, NumToks, DisableMacroExpansion, OwnsTokens, + IsReinject); EnterCachingLexMode(); return; } @@ -179,10 +181,11 @@ std::unique_ptr TokLexer; if (NumCachedTokenLexers == 0) { TokLexer = llvm::make_unique( - Toks, NumToks, DisableMacroExpansion, OwnsTokens, *this); + Toks, NumToks, DisableMacroExpansion, OwnsTokens, IsReinject, *this); } else { TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]); - TokLexer->Init(Toks, NumToks, DisableMacroExpansion, OwnsTokens); + TokLexer->Init(Toks, NumToks, DisableMacroExpansion, OwnsTokens, + IsReinject); } // Save our current state. diff --git a/clang/lib/Lex/PPMacroExpansion.cpp b/clang/lib/Lex/PPMacroExpansion.cpp --- a/clang/lib/Lex/PPMacroExpansion.cpp +++ b/clang/lib/Lex/PPMacroExpansion.cpp @@ -804,7 +804,7 @@ // Do not lose the EOF/EOD. auto Toks = llvm::make_unique(1); Toks[0] = Tok; - EnterTokenStream(std::move(Toks), 1, true); + EnterTokenStream(std::move(Toks), 1, true, /*IsReinject*/ false); break; } else if (Tok.is(tok::r_paren)) { // If we found the ) token, the macro arg list is done. diff --git a/clang/lib/Lex/Pragma.cpp b/clang/lib/Lex/Pragma.cpp --- a/clang/lib/Lex/Pragma.cpp +++ b/clang/lib/Lex/Pragma.cpp @@ -189,7 +189,8 @@ std::copy(Tokens.begin() + 1, Tokens.end(), Toks.get()); Toks[Tokens.size() - 1] = Tok; Self.EnterTokenStream(std::move(Toks), Tokens.size(), - /*DisableMacroExpansion*/ true); + /*DisableMacroExpansion*/ true, + /*IsReinject*/ true); // ... and return the _Pragma token unchanged. Tok = *Tokens.begin(); @@ -366,7 +367,8 @@ std::copy(PragmaToks.begin(), PragmaToks.end(), TokArray); // Push the tokens onto the stack. - EnterTokenStream(TokArray, PragmaToks.size(), true, true); + EnterTokenStream(TokArray, PragmaToks.size(), true, true, + /*IsReinject*/ false); // With everything set up, lex this as a #pragma directive. HandlePragmaDirective(PragmaLoc, PIK___pragma); @@ -1028,7 +1030,7 @@ Crasher.startToken(); Crasher.setKind(tok::annot_pragma_parser_crash); Crasher.setAnnotationRange(SourceRange(Tok.getLocation())); - PP.EnterToken(Crasher); + PP.EnterToken(Crasher, /*IsReinject*/false); } else if (II->isStr("dump")) { Token Identifier; PP.LexUnexpandedToken(Identifier); @@ -1040,7 +1042,7 @@ SourceRange(Tok.getLocation(), Identifier.getLocation())); DumpAnnot.setAnnotationValue(DumpII); PP.DiscardUntilEndOfDirective(); - PP.EnterToken(DumpAnnot); + PP.EnterToken(DumpAnnot, /*IsReinject*/false); } else { PP.Diag(Identifier, diag::warn_pragma_debug_missing_argument) << II->getName(); @@ -1123,7 +1125,8 @@ Toks[0].setKind(tok::annot_pragma_captured); Toks[0].setLocation(NameLoc); - PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true); + PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true, + /*IsReinject=*/false); } // Disable MSVC warning about runtime stack overflow. diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp --- a/clang/lib/Lex/Preprocessor.cpp +++ b/clang/lib/Lex/Preprocessor.cpp @@ -635,8 +635,7 @@ CurTokenLexer->Lex(Tok); break; case CLK_CachingLexer: - bool IsNewToken; - CachingLex(Tok, IsNewToken); + CachingLex(Tok); break; case CLK_LexAfterModuleImport: LexAfterModuleImport(Tok); @@ -881,7 +880,6 @@ // We loop here until a lex function returns a token; this avoids recursion. bool ReturnedToken; - bool IsNewToken = true; do { switch (CurLexerKind) { case CLK_Lexer: @@ -891,7 +889,7 @@ ReturnedToken = CurTokenLexer->Lex(Result); break; case CLK_CachingLexer: - CachingLex(Result, IsNewToken); + CachingLex(Result); ReturnedToken = true; break; case CLK_LexAfterModuleImport: @@ -911,7 +909,8 @@ // Update ImportSeqState to track our position within a C++20 import-seq // if this token is being produced as a result of phase 4 of translation. - if (getLangOpts().CPlusPlusModules && LexLevel == 1 && IsNewToken) { + if (getLangOpts().CPlusPlusModules && LexLevel == 1 && + !Result.getFlag(Token::IsReinjected)) { switch (Result.getKind()) { case tok::l_paren: case tok::l_square: case tok::l_brace: ImportSeqState.handleOpenBracket(); @@ -952,6 +951,8 @@ LastTokenWasAt = Result.is(tok::at); --LexLevel; + if (OnToken && LexLevel == 0 && !Result.getFlag(Token::IsReinjected)) + OnToken(Result); } /// Lex a header-name token (including one formed from header-name-tokens if @@ -1131,7 +1132,7 @@ auto ToksCopy = llvm::make_unique(Toks.size()); std::copy(Toks.begin(), Toks.end(), ToksCopy.get()); EnterTokenStream(std::move(ToksCopy), Toks.size(), - /*DisableMacroExpansion*/ true); + /*DisableMacroExpansion*/ true, /*IsReinject*/ false); }; // Check for a header-name. diff --git a/clang/lib/Lex/TokenLexer.cpp b/clang/lib/Lex/TokenLexer.cpp --- a/clang/lib/Lex/TokenLexer.cpp +++ b/clang/lib/Lex/TokenLexer.cpp @@ -53,6 +53,7 @@ Tokens = &*Macro->tokens_begin(); OwnsTokens = false; DisableMacroExpansion = false; + IsReinject = false; NumTokens = Macro->tokens_end()-Macro->tokens_begin(); MacroExpansionStart = SourceLocation(); @@ -91,7 +92,9 @@ /// Create a TokenLexer for the specified token stream. This does not /// take ownership of the specified token vector. void TokenLexer::Init(const Token *TokArray, unsigned NumToks, - bool disableMacroExpansion, bool ownsTokens) { + bool disableMacroExpansion, bool ownsTokens, + bool isReinject) { + assert(!isReinject || disableMacroExpansion); // If the client is reusing a TokenLexer, make sure to free any memory // associated with it. destroy(); @@ -101,6 +104,7 @@ Tokens = TokArray; OwnsTokens = ownsTokens; DisableMacroExpansion = disableMacroExpansion; + IsReinject = isReinject; NumTokens = NumToks; CurTokenIdx = 0; ExpandLocStart = ExpandLocEnd = SourceLocation(); @@ -647,6 +651,8 @@ // Get the next token to return. Tok = Tokens[CurTokenIdx++]; + if (IsReinject) + Tok.setFlag(Token::IsReinjected); bool TokenIsFromPaste = false; diff --git a/clang/lib/Parse/ParseCXXInlineMethods.cpp b/clang/lib/Parse/ParseCXXInlineMethods.cpp --- a/clang/lib/Parse/ParseCXXInlineMethods.cpp +++ b/clang/lib/Parse/ParseCXXInlineMethods.cpp @@ -323,7 +323,7 @@ // Parse the default argument from its saved token stream. Toks->push_back(Tok); // So that the current token doesn't get lost - PP.EnterTokenStream(*Toks, true); + PP.EnterTokenStream(*Toks, true, /*IsReinject*/ true); // Consume the previously-pushed token. ConsumeAnyToken(); @@ -396,7 +396,7 @@ // Parse the default argument from its saved token stream. Toks->push_back(Tok); // So that the current token doesn't get lost - PP.EnterTokenStream(*Toks, true); + PP.EnterTokenStream(*Toks, true, /*IsReinject*/true); // Consume the previously-pushed token. ConsumeAnyToken(); @@ -503,7 +503,7 @@ // Append the current token at the end of the new token stream so that it // doesn't get lost. LM.Toks.push_back(Tok); - PP.EnterTokenStream(LM.Toks, true); + PP.EnterTokenStream(LM.Toks, true, /*IsReinject*/true); // Consume the previously pushed token. ConsumeAnyToken(/*ConsumeCodeCompletionTok=*/true); @@ -617,7 +617,7 @@ // Append the current token at the end of the new token stream so that it // doesn't get lost. MI.Toks.push_back(Tok); - PP.EnterTokenStream(MI.Toks, true); + PP.EnterTokenStream(MI.Toks, true, /*IsReinject*/true); // Consume the previously pushed token. ConsumeAnyToken(/*ConsumeCodeCompletionTok=*/true); @@ -989,7 +989,8 @@ auto Buffer = llvm::make_unique(Toks.size()); std::copy(Toks.begin() + 1, Toks.end(), Buffer.get()); Buffer[Toks.size() - 1] = Self.Tok; - Self.PP.EnterTokenStream(std::move(Buffer), Toks.size(), true); + Self.PP.EnterTokenStream(std::move(Buffer), Toks.size(), true, + /*IsReinject*/ true); Self.Tok = Toks.front(); } diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp --- a/clang/lib/Parse/ParseDecl.cpp +++ b/clang/lib/Parse/ParseDecl.cpp @@ -1469,7 +1469,7 @@ // Append the current token at the end of the new token stream so that it // doesn't get lost. LA.Toks.push_back(Tok); - PP.EnterTokenStream(LA.Toks, true); + PP.EnterTokenStream(LA.Toks, true, /*IsReinject=*/true); // Consume the previously pushed token. ConsumeAnyToken(/*ConsumeCodeCompletionTok=*/true); @@ -4427,7 +4427,7 @@ if (Tok.isNot(tok::semi)) { // A semicolon was missing after this declaration. Diagnose and recover. ExpectAndConsume(tok::semi, diag::err_expected_after, "enum"); - PP.EnterToken(Tok); + PP.EnterToken(Tok, /*IsReinject=*/true); Tok.setKind(tok::semi); } } else { @@ -4705,7 +4705,7 @@ // Push this token back into the preprocessor and change our current token // to ';' so that the rest of the code recovers as though there were an // ';' after the definition. - PP.EnterToken(Tok); + PP.EnterToken(Tok, /*IsReinject=*/true); Tok.setKind(tok::semi); } } diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp --- a/clang/lib/Parse/ParseDeclCXX.cpp +++ b/clang/lib/Parse/ParseDeclCXX.cpp @@ -1040,7 +1040,7 @@ if (PP.isBacktrackEnabled()) PP.RevertCachedTokens(1); else - PP.EnterToken(Tok); + PP.EnterToken(Tok, /*IsReinject*/true); Tok.setKind(tok::annot_decltype); setExprAnnotation(Tok, @@ -1736,7 +1736,7 @@ // A semicolon was missing after this declaration. Diagnose and recover. ExpectAndConsume(tok::semi, diag::err_expected_after, DeclSpec::getSpecifierName(TagType, PPol)); - PP.EnterToken(Tok); + PP.EnterToken(Tok, /*IsReinject*/true); Tok.setKind(tok::semi); } } else @@ -2013,7 +2013,7 @@ // Push this token back into the preprocessor and change our current token // to ';' so that the rest of the code recovers as though there were an // ';' after the definition. - PP.EnterToken(Tok); + PP.EnterToken(Tok, /*IsReinject=*/true); Tok.setKind(tok::semi); } } @@ -3280,7 +3280,7 @@ if (SuggestFixIt) { LBraceDiag << FixItHint::CreateInsertion(BraceLoc, " {"); // Try recovering from missing { after base-clause. - PP.EnterToken(Tok); + PP.EnterToken(Tok, /*IsReinject*/true); Tok.setKind(tok::l_brace); } else { if (TagDecl) @@ -3376,12 +3376,12 @@ diag::note_missing_end_of_definition_before) << D; // Push '};' onto the token stream to recover. - PP.EnterToken(Tok); + PP.EnterToken(Tok, /*IsReinject*/ true); Tok.startToken(); Tok.setLocation(PP.getLocForEndOfToken(PrevTokLocation)); Tok.setKind(tok::semi); - PP.EnterToken(Tok); + PP.EnterToken(Tok, /*IsReinject*/ true); Tok.setKind(tok::r_brace); } diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp --- a/clang/lib/Parse/ParseExpr.cpp +++ b/clang/lib/Parse/ParseExpr.cpp @@ -303,7 +303,7 @@ // We can't do this before consuming the comma, because // isNotExpressionStart() looks at the token stream. if (OpToken.is(tok::comma) && isNotExpressionStart()) { - PP.EnterToken(Tok); + PP.EnterToken(Tok, /*IsReinject*/true); Tok = OpToken; return LHS; } @@ -313,7 +313,7 @@ if (isFoldOperator(NextTokPrec) && Tok.is(tok::ellipsis)) { // FIXME: We can't check this via lookahead before we consume the token // because that tickles a lexer bug. - PP.EnterToken(Tok); + PP.EnterToken(Tok, /*IsReinject*/true); Tok = OpToken; return LHS; } @@ -326,7 +326,7 @@ if (getLangOpts().ObjC && getLangOpts().CPlusPlus && Tok.isOneOf(tok::colon, tok::r_square) && OpToken.getIdentifierInfo() != nullptr) { - PP.EnterToken(Tok); + PP.EnterToken(Tok, /*IsReinject*/true); Tok = OpToken; return LHS; } diff --git a/clang/lib/Parse/ParseExprCXX.cpp b/clang/lib/Parse/ParseExprCXX.cpp --- a/clang/lib/Parse/ParseExprCXX.cpp +++ b/clang/lib/Parse/ParseExprCXX.cpp @@ -69,9 +69,9 @@ DigraphToken.setLength(1); // Push new tokens back to token stream. - PP.EnterToken(ColonToken); + PP.EnterToken(ColonToken, /*IsReinject*/ true); if (!AtDigraph) - PP.EnterToken(DigraphToken); + PP.EnterToken(DigraphToken, /*IsReinject*/ true); } // Check for '<::' which should be '< ::' instead of '[:' when following @@ -434,7 +434,7 @@ Token ColonColon; PP.Lex(ColonColon); ColonColon.setKind(tok::colon); - PP.EnterToken(ColonColon); + PP.EnterToken(ColonColon, /*IsReinject*/ true); break; } } @@ -460,8 +460,8 @@ // mistyped '::' instead of ':'. if (CorrectionFlagPtr && IsCorrectedToColon) { ColonColon.setKind(tok::colon); - PP.EnterToken(Tok); - PP.EnterToken(ColonColon); + PP.EnterToken(Tok, /*IsReinject*/ true); + PP.EnterToken(ColonColon, /*IsReinject*/ true); Tok = Identifier; break; } @@ -3300,7 +3300,8 @@ Toks.push_back(Tok); // Re-enter the stored parenthesized tokens into the token stream, so we may // parse them now. - PP.EnterTokenStream(Toks, true /*DisableMacroExpansion*/); + PP.EnterTokenStream(Toks, /*DisableMacroExpansion*/ true, + /*IsReinject*/ true); // Drop the current token and bring the first cached one. It's the same token // as when we entered this function. ConsumeAnyToken(); diff --git a/clang/lib/Parse/ParseObjc.cpp b/clang/lib/Parse/ParseObjc.cpp --- a/clang/lib/Parse/ParseObjc.cpp +++ b/clang/lib/Parse/ParseObjc.cpp @@ -1943,7 +1943,7 @@ Tok.setLocation(Tok.getLocation().getLocWithOffset(-1)); Tok.setKind(tok::at); Tok.setLength(1); - PP.EnterToken(Tok); + PP.EnterToken(Tok, /*IsReinject*/true); HelperActionsForIvarDeclarations(interfaceDecl, atLoc, T, AllIvarDecls, true); return; @@ -3654,7 +3654,7 @@ // Append the current token at the end of the new token stream so that it // doesn't get lost. LM.Toks.push_back(Tok); - PP.EnterTokenStream(LM.Toks, true); + PP.EnterTokenStream(LM.Toks, true, /*IsReinject*/true); // Consume the previously pushed token. ConsumeAnyToken(/*ConsumeCodeCompletionTok=*/true); diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp --- a/clang/lib/Parse/ParseOpenMP.cpp +++ b/clang/lib/Parse/ParseOpenMP.cpp @@ -747,8 +747,9 @@ Parser::DeclGroupPtrTy Parser::ParseOMPDeclareSimdClauses(Parser::DeclGroupPtrTy Ptr, CachedTokens &Toks, SourceLocation Loc) { - PP.EnterToken(Tok); - PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true); + PP.EnterToken(Tok, /*IsReinject*/ true); + PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true, + /*IsReinject*/ true); // Consume the previously pushed token. ConsumeAnyToken(/*ConsumeCodeCompletionTok=*/true); @@ -1319,7 +1320,7 @@ FlushHasClause = true; // Push copy of the current token back to stream to properly parse // pseudo-clause OMPFlushClause. - PP.EnterToken(Tok); + PP.EnterToken(Tok, /*IsReinject*/ true); } LLVM_FALLTHROUGH; case OMPD_taskyield: diff --git a/clang/lib/Parse/ParsePragma.cpp b/clang/lib/Parse/ParsePragma.cpp --- a/clang/lib/Parse/ParsePragma.cpp +++ b/clang/lib/Parse/ParsePragma.cpp @@ -117,7 +117,8 @@ Toks[0].setAnnotationEndLoc(Tok.getLocation()); Toks[0].setAnnotationValue(reinterpret_cast( static_cast(OOS))); - PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true); + PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true, + /*IsReinject=*/false); } }; @@ -739,7 +740,8 @@ // Grab the tokens out of the annotation and enter them into the stream. auto TheTokens = (std::pair, size_t> *)Tok.getAnnotationValue(); - PP.EnterTokenStream(std::move(TheTokens->first), TheTokens->second, true); + PP.EnterTokenStream(std::move(TheTokens->first), TheTokens->second, true, + /*IsReinject=*/true); SourceLocation PragmaLocation = ConsumeAnnotationToken(); assert(Tok.isAnyIdentifier()); StringRef PragmaName = Tok.getIdentifierInfo()->getName(); @@ -1111,7 +1113,8 @@ Hint.StateLoc = IdentifierLoc::create(Actions.Context, StateLoc, StateInfo); } else { // Enter constant expression including eof terminator into token stream. - PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/false); + PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/false, + /*IsReinject=*/false); ConsumeAnnotationToken(); ExprResult R = ParseConstantExpression(); @@ -1414,7 +1417,8 @@ return; } - PP.EnterTokenStream(Info->Tokens, /*DisableMacroExpansion=*/false); + PP.EnterTokenStream(Info->Tokens, /*DisableMacroExpansion=*/false, + /*IsReinject=*/false); ConsumeAnnotationToken(); ParsedAttributes &Attrs = Info->Attributes; @@ -1623,8 +1627,9 @@ Toks[0].setLocation(VisLoc); Toks[0].setAnnotationEndLoc(EndLoc); Toks[0].setAnnotationValue( - const_cast(static_cast(VisType))); - PP.EnterTokenStream(std::move(Toks), 1, /*DisableMacroExpansion=*/true); + const_cast(static_cast(VisType))); + PP.EnterTokenStream(std::move(Toks), 1, /*DisableMacroExpansion=*/true, + /*IsReinject=*/false); } // #pragma pack(...) comes in the following delicious flavors: @@ -1737,7 +1742,8 @@ Toks[0].setLocation(PackLoc); Toks[0].setAnnotationEndLoc(RParenLoc); Toks[0].setAnnotationValue(static_cast(Info)); - PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true); + PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true, + /*IsReinject=*/false); } // #pragma ms_struct on @@ -1780,7 +1786,8 @@ Toks[0].setAnnotationEndLoc(EndLoc); Toks[0].setAnnotationValue(reinterpret_cast( static_cast(Kind))); - PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true); + PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true, + /*IsReinject=*/false); } // #pragma clang section bss="abc" data="" rodata="def" text="" @@ -1893,7 +1900,8 @@ Toks[0].setAnnotationEndLoc(EndLoc); Toks[0].setAnnotationValue(reinterpret_cast( static_cast(Kind))); - PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true); + PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true, + /*IsReinject=*/false); } void PragmaAlignHandler::HandlePragma(Preprocessor &PP, @@ -1985,7 +1993,8 @@ pragmaUnusedTok.setLocation(UnusedLoc); idTok = Identifiers[i]; } - PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true); + PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true, + /*IsReinject=*/false); } // #pragma weak identifier @@ -2034,7 +2043,8 @@ pragmaUnusedTok.setAnnotationEndLoc(AliasName.getLocation()); Toks[1] = WeakName; Toks[2] = AliasName; - PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true); + PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true, + /*IsReinject=*/false); } else { MutableArrayRef Toks( PP.getPreprocessorAllocator().Allocate(2), 2); @@ -2044,7 +2054,8 @@ pragmaUnusedTok.setLocation(WeakLoc); pragmaUnusedTok.setAnnotationEndLoc(WeakLoc); Toks[1] = WeakName; - PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true); + PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true, + /*IsReinject=*/false); } } @@ -2089,7 +2100,8 @@ pragmaRedefTok.setAnnotationEndLoc(AliasName.getLocation()); Toks[1] = RedefName; Toks[2] = AliasName; - PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true); + PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true, + /*IsReinject=*/false); } @@ -2109,7 +2121,8 @@ Toks[0].setAnnotationEndLoc(Tok.getLocation()); Toks[0].setAnnotationValue(reinterpret_cast( static_cast(OOS))); - PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true); + PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true, + /*IsReinject=*/false); } void @@ -2171,7 +2184,8 @@ Toks[0].setLocation(NameLoc); Toks[0].setAnnotationValue(static_cast(Info)); Toks[0].setAnnotationEndLoc(StateLoc); - PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true); + PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true, + /*IsReinject=*/false); if (PP.getPPCallbacks()) PP.getPPCallbacks()->PragmaOpenCLExtension(NameLoc, Ext, @@ -2230,7 +2244,7 @@ auto Toks = llvm::make_unique(Pragma.size()); std::copy(Pragma.begin(), Pragma.end(), Toks.get()); PP.EnterTokenStream(std::move(Toks), Pragma.size(), - /*DisableMacroExpansion=*/false); + /*DisableMacroExpansion=*/false, /*IsReinject=*/false); } /// Handle '#pragma pointers_to_members' @@ -2328,7 +2342,7 @@ AnnotTok.setAnnotationEndLoc(EndLoc); AnnotTok.setAnnotationValue( reinterpret_cast(static_cast(RepresentationMethod))); - PP.EnterToken(AnnotTok); + PP.EnterToken(AnnotTok, /*IsReinject=*/true); } /// Handle '#pragma vtordisp' @@ -2421,7 +2435,7 @@ AnnotTok.setAnnotationEndLoc(EndLoc); AnnotTok.setAnnotationValue(reinterpret_cast( static_cast((Action << 16) | (Value & 0xFFFF)))); - PP.EnterToken(AnnotTok); + PP.EnterToken(AnnotTok, /*IsReinject=*/false); } /// Handle all MS pragmas. Simply forwards the tokens after inserting @@ -2452,7 +2466,7 @@ std::pair, size_t>(std::move(TokenArray), TokenVector.size()); AnnotTok.setAnnotationValue(Value); - PP.EnterToken(AnnotTok); + PP.EnterToken(AnnotTok, /*IsReinject*/ false); } /// Handle the Microsoft \#pragma detect_mismatch extension. @@ -2736,7 +2750,7 @@ std::copy(TokenList.begin(), TokenList.end(), TokenArray.get()); PP.EnterTokenStream(std::move(TokenArray), TokenList.size(), - /*DisableMacroExpansion=*/false); + /*DisableMacroExpansion=*/false, /*IsReinject=*/false); } void Parser::HandlePragmaFP() { @@ -2919,7 +2933,7 @@ std::copy(TokenList.begin(), TokenList.end(), TokenArray.get()); PP.EnterTokenStream(std::move(TokenArray), TokenList.size(), - /*DisableMacroExpansion=*/false); + /*DisableMacroExpansion=*/false, /*IsReinject=*/false); } /// Handle the loop unroll optimization pragmas. @@ -2994,7 +3008,7 @@ TokenArray[0].setAnnotationEndLoc(PragmaName.getLocation()); TokenArray[0].setAnnotationValue(static_cast(Info)); PP.EnterTokenStream(std::move(TokenArray), 1, - /*DisableMacroExpansion=*/false); + /*DisableMacroExpansion=*/false, /*IsReinject=*/false); } /// Handle the Microsoft \#pragma intrinsic extension. @@ -3266,5 +3280,5 @@ TokenArray[0].setAnnotationEndLoc(FirstToken.getLocation()); TokenArray[0].setAnnotationValue(static_cast(Info)); PP.EnterTokenStream(std::move(TokenArray), 1, - /*DisableMacroExpansion=*/false); + /*DisableMacroExpansion=*/false, /*IsReinject=*/false); } diff --git a/clang/lib/Parse/ParseStmtAsm.cpp b/clang/lib/Parse/ParseStmtAsm.cpp --- a/clang/lib/Parse/ParseStmtAsm.cpp +++ b/clang/lib/Parse/ParseStmtAsm.cpp @@ -213,7 +213,8 @@ // Also copy the current token over. LineToks.push_back(Tok); - PP.EnterTokenStream(LineToks, /*DisableMacroExpansions*/ true); + PP.EnterTokenStream(LineToks, /*DisableMacroExpansions*/ true, + /*IsReinject*/ true); // Clear the current token and advance to the first token in LineToks. ConsumeAnyToken(); diff --git a/clang/lib/Parse/ParseTemplate.cpp b/clang/lib/Parse/ParseTemplate.cpp --- a/clang/lib/Parse/ParseTemplate.cpp +++ b/clang/lib/Parse/ParseTemplate.cpp @@ -919,7 +919,7 @@ PrevTokLocation = RAngleLoc; } else { PrevTokLocation = TokBeforeGreaterLoc; - PP.EnterToken(Tok); + PP.EnterToken(Tok, /*IsReinject=*/true); Tok = Greater; } @@ -1402,7 +1402,7 @@ // Append the current token at the end of the new token stream so that it // doesn't get lost. LPT.Toks.push_back(Tok); - PP.EnterTokenStream(LPT.Toks, true); + PP.EnterTokenStream(LPT.Toks, true, /*IsReinject*/true); // Consume the previously pushed token. ConsumeAnyToken(/*ConsumeCodeCompletionTok=*/true); diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp --- a/clang/lib/Parse/Parser.cpp +++ b/clang/lib/Parse/Parser.cpp @@ -1545,7 +1545,7 @@ if (PP.isBacktrackEnabled()) PP.RevertCachedTokens(1); else - PP.EnterToken(Tok); + PP.EnterToken(Tok, /*IsReinject=*/true); Tok.setKind(tok::annot_cxxscope); Tok.setAnnotationValue(Actions.SaveNestedNameSpecifierAnnotation(SS)); Tok.setAnnotationRange(SS.getRange()); @@ -1764,7 +1764,7 @@ Token TypedefToken; PP.Lex(TypedefToken); bool Result = TryAnnotateTypeOrScopeToken(); - PP.EnterToken(Tok); + PP.EnterToken(Tok, /*IsReinject=*/true); Tok = TypedefToken; if (!Result) Diag(Tok.getLocation(), diag::warn_expected_qualified_after_typename); diff --git a/clang/lib/Rewrite/HTMLRewrite.cpp b/clang/lib/Rewrite/HTMLRewrite.cpp --- a/clang/lib/Rewrite/HTMLRewrite.cpp +++ b/clang/lib/Rewrite/HTMLRewrite.cpp @@ -572,7 +572,7 @@ // Enter the tokens we just lexed. This will cause them to be macro expanded // but won't enter sub-files (because we removed #'s). - TmpPP.EnterTokenStream(TokenStream, false); + TmpPP.EnterTokenStream(TokenStream, false, /*IsReinject=*/false); TokenConcatenation ConcatInfo(TmpPP);