diff --git a/clang/lib/Parse/ParsePragma.cpp b/clang/lib/Parse/ParsePragma.cpp --- a/clang/lib/Parse/ParsePragma.cpp +++ b/clang/lib/Parse/ParsePragma.cpp @@ -14,11 +14,13 @@ #include "clang/Basic/PragmaKinds.h" #include "clang/Basic/TargetInfo.h" #include "clang/Lex/Preprocessor.h" +#include "clang/Lex/Token.h" #include "clang/Parse/LoopHint.h" #include "clang/Parse/ParseDiagnostic.h" #include "clang/Parse/Parser.h" #include "clang/Parse/RAIIObjectsForParser.h" #include "clang/Sema/Scope.h" +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringSwitch.h" using namespace clang; @@ -292,6 +294,10 @@ Token &FirstToken) override; }; +void markAsReinjectedForRelexing(llvm::MutableArrayRef Toks) { + for (auto &T : Toks) + T.setFlag(clang::Token::IsReinjected); +} } // end namespace void Parser::initializePragmaHandlers() { @@ -2619,6 +2625,7 @@ TokenVector.push_back(EoF); // We must allocate this array with new because EnterTokenStream is going to // delete it later. + markAsReinjectedForRelexing(TokenVector); auto TokenArray = std::make_unique(TokenVector.size()); std::copy(TokenVector.begin(), TokenVector.end(), TokenArray.get()); auto Value = new (PP.getPreprocessorAllocator()) @@ -3176,6 +3183,7 @@ EOFTok.setLocation(Tok.getLocation()); ValueList.push_back(EOFTok); // Terminates expression for parsing. + markAsReinjectedForRelexing(ValueList); Info.Toks = llvm::makeArrayRef(ValueList).copy(PP.getPreprocessorAllocator()); Info.PragmaName = PragmaName; @@ -3632,6 +3640,7 @@ EOFTok.setLocation(EndLoc); AttributeTokens.push_back(EOFTok); + markAsReinjectedForRelexing(AttributeTokens); Info->Tokens = llvm::makeArrayRef(AttributeTokens).copy(PP.getPreprocessorAllocator()); } diff --git a/clang/unittests/Tooling/Syntax/TokensTest.cpp b/clang/unittests/Tooling/Syntax/TokensTest.cpp --- a/clang/unittests/Tooling/Syntax/TokensTest.cpp +++ b/clang/unittests/Tooling/Syntax/TokensTest.cpp @@ -1037,4 +1037,13 @@ IsEmpty()); } +TEST_F(TokenCollectorTest, Pragmas) { + // Tokens coming from concatenations. + recordTokens(R"cpp( + void foo() { + #pragma unroll 4 + for(int i=0;i<4;++i); + } + )cpp"); +} } // namespace