[clangd] Implement a function to lex the file to find candidate occurrences.

[hokein]

Summary

This will be used for incoming cross-file rename (to detect index
staleness issue).

[merge_guards_bot]

Build result: pass - 59741 tests passed, 0 failed and 762 were skipped.
Log files: console-log.txt, CMakeCache.txt

[ilya-biryukov]

clang-tools-extra/clangd/SourceCode.cpp

760

Why do we have both raw_identifier and identifier?
We should run lexer in the raw mode, in which case we should always get raw_identifier.

clang-tools-extra/clangd/SourceCode.h

236

NIT: maybe use a shorter name for the paramter? Identifier conveys the same meaning.

clang-tools-extra/clangd/unittests/SourceCodeTests.cpp

690

Could you add a few more occurrences?
Maybe also with the weird things like multi-line tokens:

F\
o\
o

[hokein]

address comments, and simplify the code.

clang-tools-extra/clangd/SourceCode.cpp

760

I just borrowed from collectIdentifiers. You are right, we only run a raw lexer, updated the code.

clang-tools-extra/clangd/unittests/SourceCodeTests.cpp

690

Done, note that multi-line identifier is not supported -- getRawIdentifier returns a literal name F\o\o which makes the check Tok.getRawIdentifier() == Identifier fail. I think it is fine to not support it.

[merge_guards_bot]

Build result: pass - 59741 tests passed, 0 failed and 762 were skipped.
Log files: console-log.txt, CMakeCache.txt

[ilya-biryukov]

clang-tools-extra/clangd/SourceCode.cpp

757

NIT: use early exits to keep the code flat.

if (Tok.getKind() != ...)
  return;
auto Range = ...;
if (!Range)
  return;
...

clang-tools-extra/clangd/unittests/SourceCodeTests.cpp

690

Yeah, it's probably ok.
I'd still add this as a test-case to make sure we don't crash and fix the behavior.

Although agree it's not very important, this is super rare.

[hokein]

address comment.

[merge_guards_bot]

Build result: fail - 59859 tests passed, 22 failed and 763 were skipped.

failed: libc++.std/thread/thread_mutex/thread_mutex_requirements/thread_mutex_requirements_mutex/thread_mutex_class/try_lock.pass.cpp
failed: lld.ELF/linkerscript/filename-spec.s
failed: Clang.Index/index-module-with-vfs.m
failed: Clang.Modules/double-quotes.m
failed: Clang.Modules/framework-public-includes-private.m
failed: Clang.VFS/external-names.c
failed: Clang.VFS/framework-import.m
failed: Clang.VFS/implicit-include.c
failed: Clang.VFS/include-mixed-real-and-virtual.c
failed: Clang.VFS/include-real-from-virtual.c
failed: Clang.VFS/include-virtual-from-real.c
failed: Clang.VFS/include.c
failed: Clang.VFS/incomplete-umbrella.m
failed: Clang.VFS/module-import.m
failed: Clang.VFS/module_missing_vfs.m
failed: Clang.VFS/real-path-found-first.m
failed: Clang.VFS/relative-path.c
failed: Clang.VFS/test_nonmodular.c
failed: Clang.VFS/umbrella-framework-import-skipnonexist.m
failed: Clang.VFS/vfsroot-include.c
failed: Clang.VFS/vfsroot-module.m
failed: Clang.VFS/vfsroot-with-overlay.c

Log files: console-log.txt, CMakeCache.txt

clang-tools-extra/clangd/SourceCode.h

 /// Formats the edits and code around it according to Style. Changes
 /// Replacements to formatted ones if succeeds.
 llvm::Error reformatEdit(Edit &E, const format::FormatStyle &Style);
 
 /// Collects identifiers with counts in the source code.
 llvm::StringMap<unsigned> collectIdentifiers(llvm::StringRef Content,
                                              const format::FormatStyle &Style);
 
+/// Collects all ranges of the given identifier in the source code.
+std::vector<Range> collectIdentifierRanges(llvm::StringRef Identifier,
+                                           llvm::StringRef Content,
+                                           const LangOptions &LangOpts);
+
 /// Collects words from the source code.
 /// Unlike collectIdentifiers:
 /// - also finds text in comments:
 /// - splits text into words
 /// - drops stopwords like "get" and "for"
 llvm::StringSet<> collectWords(llvm::StringRef Content);
 
 /// Heuristically determine namespaces visible at a point, without parsing Code.

clang-tools-extra/clangd/SourceCode.cpp

 cleanupAndFormat(StringRef Code, const tooling::Replacements &Replaces,
                  const format::FormatStyle &Style) {
   auto CleanReplaces = cleanupAroundReplacements(Code, Replaces, Style);
   if (!CleanReplaces)
     return CleanReplaces;
   return formatReplacements(Code, std::move(*CleanReplaces), Style);
 }
 
-void lex(llvm::StringRef Code, const format::FormatStyle &Style,
-         llvm::function_ref<void(const clang::Token &, Position)> Action) {
+void lex(llvm::StringRef Code, const LangOptions &LangOpts,
+         llvm::function_ref<void(const clang::Token &, const SourceManager &SM)>
+             Action) {
   // FIXME: InMemoryFileAdapter crashes unless the buffer is null terminated!
   std::string NullTerminatedCode = Code.str();
   SourceManagerForFile FileSM("dummy.cpp", NullTerminatedCode);
   auto &SM = FileSM.get();
   auto FID = SM.getMainFileID();
-  Lexer Lex(FID, SM.getBuffer(FID), SM, format::getFormattingLangOpts(Style));
+  // Create a raw lexer (with no associated preprocessor object).
+  Lexer Lex(FID, SM.getBuffer(FID), SM, LangOpts);
   Token Tok;
 
   while (!Lex.LexFromRawLexer(Tok))
-    Action(Tok, sourceLocToPosition(SM, Tok.getLocation()));
+    Action(Tok, SM);
   // LexFromRawLexer returns true after it lexes last token, so we still have
   // one more token to report.
-  Action(Tok, sourceLocToPosition(SM, Tok.getLocation()));
+  Action(Tok, SM);
 }
 
 llvm::StringMap<unsigned> collectIdentifiers(llvm::StringRef Content,
                                              const format::FormatStyle &Style) {
   llvm::StringMap<unsigned> Identifiers;
-  lex(Content, Style, [&](const clang::Token &Tok, Position) {
-    switch (Tok.getKind()) {
-    case tok::identifier:
-      ++Identifiers[Tok.getIdentifierInfo()->getName()];
-      break;
-    case tok::raw_identifier:
+  auto LangOpt = format::getFormattingLangOpts(Style);
+  lex(Content, LangOpt, [&](const clang::Token &Tok, const SourceManager &) {
+    if (Tok.getKind() == tok::raw_identifier)
       ++Identifiers[Tok.getRawIdentifier()];
-      break;
-    default:
-      break;
-    }
   });
   return Identifiers;
 }
 
+std::vector<Range> collectIdentifierRanges(llvm::StringRef Identifier,
+                                           llvm::StringRef Content,
+                                           const LangOptions &LangOpts) {
+  std::vector<Range> Ranges;
+  lex(Content, LangOpts, [&](const clang::Token &Tok, const SourceManager &SM) {
+    if (Tok.getKind() != tok::raw_identifier)
+      return;
+    if (Tok.getRawIdentifier() != Identifier)
+      return;
+    auto Range = getTokenRange(SM, LangOpts, Tok.getLocation());
+    if (!Range)
+      return;
+    Ranges.push_back(*Range);
+  });
+  return Ranges;
+}
+
 namespace {
 struct NamespaceEvent {
   enum {
     BeginNamespace, // namespace <ns> {.     Payload is resolved <ns>.
     EndNamespace,   // } // namespace <ns>.  Payload is resolved *outer*
                     // namespace.
     UsingDirective  // using namespace <ns>. Payload is unresolved <ns>.
   } Trigger;
enum {
     NamespaceName,      // just saw 'namespace' NSName
     Using,              // just saw 'using'
     UsingNamespace,     // just saw 'using namespace'
     UsingNamespaceName, // just saw 'using namespace' NSName
   } State = Default;
   std::string NSName;
 
   NamespaceEvent Event;
-  lex(Code, Style, [&](const clang::Token &Tok, Position P) {
-    Event.Pos = std::move(P);
+  lex(Code, format::getFormattingLangOpts(Style),
+      [&](const clang::Token &Tok,const SourceManager &SM) {
+    Event.Pos = sourceLocToPosition(SM, Tok.getLocation());
     switch (Tok.getKind()) {
     case tok::raw_identifier:
       // In raw mode, this could be a keyword or a name.
       switch (State) {
       case UsingNamespace:
       case UsingNamespaceName:
         NSName.append(Tok.getRawIdentifier());
         State = UsingNamespaceName;

clang-tools-extra/clangd/unittests/SourceCodeTests.cpp

for (auto Case : Cases) {
 
     auto Res = getEligiblePoints(Test.code(), Case.FullyQualifiedName,
                                  format::getLLVMStyle());
     EXPECT_THAT(Res.EligiblePoints, testing::ElementsAreArray(Test.points()))
         << Test.code();
     EXPECT_EQ(Res.EnclosingNamespace, Case.EnclosingNamespace) << Test.code();
   }
 }
+
+TEST(SourceCodeTests, IdentifierRanges) {
+  Annotations Code(R"cpp(
+   class [[Foo]] {};
+   // Foo
+   /* Foo */
+   void f([[Foo]]* foo1) {
+     [[Foo]] foo2;
+     auto S = [[Foo]]();
+// cross-line identifier is not supported.
+F\
+o\
+o foo2;
+   }
+  )cpp");
+  LangOptions LangOpts;
+  LangOpts.CPlusPlus = true;
+  EXPECT_EQ(Code.ranges(),
+            collectIdentifierRanges("Foo", Code.code(), LangOpts));
+}
+
 } // namespace
 } // namespace clangd
 } // namespace clang