diff --git a/clang/include/clang/Lex/Lexer.h b/clang/include/clang/Lex/Lexer.h --- a/clang/include/clang/Lex/Lexer.h +++ b/clang/include/clang/Lex/Lexer.h @@ -554,7 +554,8 @@ /// Returns the next token, or none if the location is inside a macro. static Optional findNextToken(SourceLocation Loc, const SourceManager &SM, - const LangOptions &LangOpts); + const LangOptions &LangOpts, + bool IncludeComments = false); /// Checks that the given token is the first token that occurs after /// the given location (this excludes comments and whitespace). Returns the diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp --- a/clang/lib/Lex/Lexer.cpp +++ b/clang/lib/Lex/Lexer.cpp @@ -1258,7 +1258,8 @@ Optional Lexer::findNextToken(SourceLocation Loc, const SourceManager &SM, - const LangOptions &LangOpts) { + const LangOptions &LangOpts, + bool IncludeComments) { if (Loc.isMacroID()) { if (!Lexer::isAtEndOfMacroExpansion(Loc, SM, LangOpts, &Loc)) return None; @@ -1274,11 +1275,13 @@ if (InvalidTemp) return None; - const char *TokenBegin = File.data() + LocInfo.second; + const char *const TokenBegin = File.data() + LocInfo.second; // Lex from the start of the given location. Lexer lexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, File.begin(), TokenBegin, File.end()); + lexer.SetCommentRetentionState(IncludeComments); + // Find the token. Token Tok; lexer.LexFromRawLexer(Tok); diff --git a/clang/unittests/Lex/LexerTest.cpp b/clang/unittests/Lex/LexerTest.cpp --- a/clang/unittests/Lex/LexerTest.cpp +++ b/clang/unittests/Lex/LexerTest.cpp @@ -622,6 +622,24 @@ "xyz", "=", "abcd", ";")); } +TEST_F(LexerTest, FindNextTokenWithComments) { + Lex("int /* type */ abcd = 0; // vardecl\n" + "int /*other type*/ xyz = abcd; //other vardecl \n"); + std::vector GeneratedByNextToken; + SourceLocation Loc = + SourceMgr.getLocForStartOfFile(SourceMgr.getMainFileID()); + while (true) { + auto T = Lexer::findNextToken(Loc, SourceMgr, LangOpts, true); + ASSERT_TRUE(T); + if (T->is(tok::eof)) + break; + GeneratedByNextToken.push_back(getSourceText(*T, *T)); + Loc = T->getLocation(); + } + EXPECT_THAT(GeneratedByNextToken, ElementsAre("/* type */", "abcd", "=", "0", ";", "// vardecl", "int", "/*other type*/", + "xyz", "=", "abcd", ";", "//other vardecl ")); +} + TEST_F(LexerTest, CreatedFIDCountForPredefinedBuffer) { TrivialModuleLoader ModLoader; auto PP = CreatePP("", ModLoader);