[clang] Add test for FindNextToken in Lexer.

[usaxena95]

Summary

Currently Lexer::FindNextToken does not have any unittests associated with it.

• This patch adds a test for this method.
• This test acts as a documentation for how to use this.
• This method is used at various places in clang-tidy, clang and clangd using a similar pattern as in the tests. Thus this will prevent those usages to break in the future.

[usaxena95]

Revert unintended formatting.

[ilya-biryukov]

Could you add a rationale in the change description? E.g. something like we do not have any tests for findNextToken

clang/unittests/Lex/LexerTest.cpp

558

NIT: could be replaced with

EXPECT_THAT(GeneratedByNextToken, ElementsAre("abcd", "=", ....))

To avoid creating an extra variable.
But up to you, this version also looks ok.

[usaxena95]

Addressed comments and fixed build failure.

[ilya-biryukov]

Thanks!

clang/unittests/Lex/LexerTest.cpp

 //===- unittests/Lex/LexerTest.cpp ------ Lexer tests ---------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "clang/Lex/Lexer.h"
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/DiagnosticOptions.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/LangOptions.h"
+#include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Basic/TargetOptions.h"
+#include "clang/Basic/TokenKinds.h"
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Lex/HeaderSearchOptions.h"
 #include "clang/Lex/MacroArgs.h"
 #include "clang/Lex/MacroInfo.h"
 #include "clang/Lex/ModuleLoader.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Lex/PreprocessorOptions.h"
+#include "gmock/gmock.h"
 #include "gtest/gtest.h"
-
-using namespace clang;
+#include <vector>
 
 namespace {
+using namespace clang;
+using testing::ElementsAre;
 
 // The test fixture.
 class LexerTest : public ::testing::Test {
 protected:
   LexerTest()
     : FileMgr(FileMgrOpts),
       DiagID(new DiagnosticIDs()),
       Diags(DiagID, new DiagnosticOptions, new IgnoringDiagConsumer()),
TEST_F(LexerTest, CharRangeOffByOne) {
   EXPECT_TRUE(
       Lexer::isAtEndOfMacroExpansion(R.getEnd(), SourceMgr, LangOpts));
 
   CharSourceRange CR = Lexer::getAsCharRange(R, SourceMgr, LangOpts);
 
   EXPECT_EQ(Lexer::getSourceText(CR, SourceMgr, LangOpts), "MOO"); // Was "MO".
 }
 
+TEST_F(LexerTest, FindNextToken) {
+  Lex("int abcd = 0;\n"
+      "int xyz = abcd;\n");
+  std::vector<std::string> GeneratedByNextToken;
+  SourceLocation Loc =
+      SourceMgr.getLocForStartOfFile(SourceMgr.getMainFileID());
+  while (true) {
+    auto T = Lexer::findNextToken(Loc, SourceMgr, LangOpts);
+    ASSERT_TRUE(T.hasValue());
+    if (T->is(tok::eof))
+      break;
+    GeneratedByNextToken.push_back(getSourceText(*T, *T));
+    Loc = T->getLocation();
+  }
+  EXPECT_THAT(GeneratedByNextToken, ElementsAre("abcd", "=", "0", ";", "int",
+                                                "xyz", "=", "abcd", ";"));
+}
 } // anonymous namespace