diff --git a/clang/lib/Parse/ParsePragma.cpp b/clang/lib/Parse/ParsePragma.cpp --- a/clang/lib/Parse/ParsePragma.cpp +++ b/clang/lib/Parse/ParsePragma.cpp @@ -2607,6 +2607,7 @@ Token EoF, AnnotTok; EoF.startToken(); EoF.setKind(tok::eof); + EoF.setFlag(clang::Token::IsReinjected); AnnotTok.startToken(); AnnotTok.setKind(tok::annot_pragma_ms_pragma); AnnotTok.setLocation(Tok.getLocation()); @@ -2614,6 +2615,7 @@ SmallVector TokenVector; // Suck up all of the tokens before the eod. for (; Tok.isNot(tok::eod); PP.Lex(Tok)) { + Tok.setFlag(clang::Token::IsReinjected); TokenVector.push_back(Tok); AnnotTok.setAnnotationEndLoc(Tok.getLocation()); } @@ -3159,6 +3161,7 @@ break; } + Tok.setFlag(clang::Token::IsReinjected); ValueList.push_back(Tok); PP.Lex(Tok); } @@ -3174,6 +3177,7 @@ Token EOFTok; EOFTok.startToken(); + EOFTok.setFlag(clang::Token::IsReinjected); EOFTok.setKind(tok::eof); EOFTok.setLocation(Tok.getLocation()); ValueList.push_back(EOFTok); // Terminates expression for parsing. @@ -3612,6 +3616,7 @@ break; } + Tok.setFlag(clang::Token::IsReinjected); AttributeTokens.push_back(Tok); PP.Lex(Tok); } @@ -3632,6 +3637,7 @@ EOFTok.startToken(); EOFTok.setKind(tok::eof); EOFTok.setLocation(EndLoc); + EOFTok.setFlag(clang::Token::IsReinjected); AttributeTokens.push_back(EOFTok); Info->Tokens = diff --git a/clang/unittests/Tooling/Syntax/TokensTest.cpp b/clang/unittests/Tooling/Syntax/TokensTest.cpp --- a/clang/unittests/Tooling/Syntax/TokensTest.cpp +++ b/clang/unittests/Tooling/Syntax/TokensTest.cpp @@ -1037,4 +1037,13 @@ IsEmpty()); } +TEST_F(TokenCollectorTest, Pragmas) { + // Tokens coming from concatenations. + recordTokens(R"cpp( + void foo() { + #pragma unroll 4 + for(int i=0;i<4;++i); + } + )cpp"); +} } // namespace