[clangd] Get rid of lexer usage in locateMacroAt

[kadircet]

• Update forgotten call site

[hokein]

clang-tools-extra/clangd/SourceCode.h

284–285

nit: update the comment.

285–287

nit: would be clearer to name it SpelledTok.

[kadircet]

• Address comments

clang-tools-extra/clangd/Hover.cpp

llvm::Optional<HoverInfo> getHover(ParsedAST &AST, Position Pos,
   const SourceManager &SM = AST.getSourceManager();
   auto CurLoc = sourceLocationInMainFile(SM, Pos);
   if (!CurLoc) {
     llvm::consumeError(CurLoc.takeError());
     return llvm::None;
   }
   auto TokensTouchingCursor =
       syntax::spelledTokensTouching(*CurLoc, AST.getTokens());
+  // Early exit if there were no tokens around the cursor.
   if (TokensTouchingCursor.empty())
     return llvm::None;
 
-  // In general we prefer the touching token that works over the one that
-  // doesn't, see SelectionTree::create(). The following locations are used only
-  // for triggering on macros and auto/decltype, so simply choosing the lone
-  // identifier-or-keyword token is equivalent.
+  // To be used as a backup for highlighting the selected token.
   SourceLocation IdentLoc;
-  SourceLocation AutoLoc;
+  llvm::Optional<HoverInfo> HI;
+  // Macros and deducedtype only works on identifiers and auto/decltype keywords
+  // respectively. Therefore they are only trggered on whichever works for them,
+  // similar to SelectionTree::create().
   for (const auto &Tok : TokensTouchingCursor) {
-    if (Tok.kind() == tok::identifier)
+    if (Tok.kind() == tok::identifier) {
       IdentLoc = Tok.location();
-    if (Tok.kind() == tok::kw_auto || Tok.kind() == tok::kw_decltype)
-      AutoLoc = Tok.location();
+      if (auto M = locateMacroAt(Tok, AST.getPreprocessor())) {
+        HI = getHoverContents(*M, AST);
+        HI->SymRange = getTokenRange(AST.getSourceManager(), AST.getLangOpts(),
+                                     Tok.location());
+        break;
       }
-
-  llvm::Optional<HoverInfo> HI;
-  if (auto Deduced = getDeducedType(AST.getASTContext(), AutoLoc)) {
+    } else if (Tok.kind() == tok::kw_auto || Tok.kind() == tok::kw_decltype) {
+      if (auto Deduced = getDeducedType(AST.getASTContext(), Tok.location())) {
         HI = getHoverContents(*Deduced, AST.getASTContext(), Index);
-    HI->SymRange =
-        getTokenRange(AST.getSourceManager(), AST.getLangOpts(), AutoLoc);
-  } else if (auto M = locateMacroAt(IdentLoc, AST.getPreprocessor())) {
-    HI = getHoverContents(*M, AST);
-    HI->SymRange =
-        getTokenRange(AST.getSourceManager(), AST.getLangOpts(), IdentLoc);
-  } else {
+        HI->SymRange = getTokenRange(AST.getSourceManager(), AST.getLangOpts(),
+                                     Tok.location());
+        break;
+      }
+    }
+  }
+
+  // If it wasn't auto/decltype or macro, look for decls and expressions.
+  if (!HI) {
     auto Offset = SM.getFileOffset(*CurLoc);
     // Editors send the position on the left of the hovered character.
     // So our selection tree should be biased right. (Tested with VSCode).
     SelectionTree ST = SelectionTree::createRight(
         AST.getASTContext(), AST.getTokens(), Offset, Offset);
     std::vector<const Decl *> Result;
     if (const SelectionTree::Node *N = ST.commonAncestor()) {
       auto Decls = explicitReferenceTargets(N->ASTNode, DeclRelation::Alias);

clang-tools-extra/clangd/SourceCode.h

 #include "Context.h"
 #include "Protocol.h"
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/LangOptions.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Format/Format.h"
 #include "clang/Tooling/Core/Replacement.h"
+#include "clang/Tooling/Syntax/Tokens.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/SHA1.h"
 #include <string>
 
 namespace clang {
 class SourceManager;

 /// \p FullyQualifiedName should not contain anonymous namespaces.
 EligibleRegion getEligiblePoints(llvm::StringRef Code,
                                  llvm::StringRef FullyQualifiedName,
                                  const format::FormatStyle &Style);
 
 struct DefinedMacro {
   llvm::StringRef Name;
   const MacroInfo *Info;
 };
-/// Gets the macro at a specified \p Loc. It must be a spelling location and
-/// point to the beginning of identifier.
-llvm::Optional<DefinedMacro> locateMacroAt(SourceLocation Loc,
+/// Gets the macro referenced by \p SpelledTok. It must be a spelled token
+/// aligned to the beginning of an identifier.
+llvm::Optional<DefinedMacro> locateMacroAt(const syntax::Token &SpelledTok,
                                            Preprocessor &PP);
 
 /// Infers whether this is a header from the FileName and LangOpts (if
 /// presents).
 bool isHeaderFile(llvm::StringRef FileName,
                   llvm::Optional<LangOptions> LangOpts = llvm::None);
 
 /// Returns true if the given location is in a generated protobuf file.
 bool isProtoFile(SourceLocation Loc, const SourceManager &SourceMgr);
 
 } // namespace clangd
 } // namespace clang
 #endif

clang-tools-extra/clangd/SourceCode.cpp

case Separator:
       break;
     }
   }
   Flush();
 
   return Result;
 }
 
-llvm::Optional<DefinedMacro> locateMacroAt(SourceLocation Loc,
+llvm::Optional<DefinedMacro> locateMacroAt(const syntax::Token &SpelledTok,
                                            Preprocessor &PP) {
+  SourceLocation Loc = SpelledTok.location();
   assert(Loc.isFileID());
   const auto &SM = PP.getSourceManager();
-  const auto &LangOpts = PP.getLangOpts();
-  Token Result;
-  if (Lexer::getRawToken(Loc, Result, SM, LangOpts, false))
-    return None;
-  if (Result.is(tok::raw_identifier))
-    PP.LookUpIdentifierInfo(Result);
-  IdentifierInfo *IdentifierInfo = Result.getIdentifierInfo();
+  IdentifierInfo *IdentifierInfo = PP.getIdentifierInfo(SpelledTok.text(SM));
   if (!IdentifierInfo || !IdentifierInfo->hadMacroDefinition())
     return None;
 
   // Get the definition just before the searched location so that a macro
   // referenced in a '#undef MACRO' can still be found. Note that we only do
   // that if Loc is not pointing at start of file.
   if (SM.getLocForStartOfFile(SM.getFileID(Loc)) != Loc)
     Loc = Loc.getLocWithOffset(-1);

clang-tools-extra/clangd/XRefs.cpp

std::vector<LocatedSymbol> locateSymbolAt(ParsedAST &AST, Position Pos,
   }
 
   // Macros are simple: there's no declaration/definition distinction.
   // As a consequence, there's no need to look them up in the index either.
   std::vector<LocatedSymbol> Result;
   const auto *TouchedIdentifier =
       syntax::spelledIdentifierTouching(*CurLoc, AST.getTokens());
   if (TouchedIdentifier) {
-    if (auto M = locateMacroAt(TouchedIdentifier->location(),
-                               AST.getPreprocessor())) {
+    if (auto M = locateMacroAt(*TouchedIdentifier, AST.getPreprocessor())) {
       if (auto Loc = makeLocation(AST.getASTContext(),
                                   M->Info->getDefinitionLoc(), *MainFilePath)) {
         LocatedSymbol Macro;
         Macro.Name = std::string(M->Name);
         Macro.PreferredDeclaration = *Loc;
         Macro.Definition = Loc;
         Result.push_back(std::move(Macro));
 
if (!MainFilePath) {
     return Results;
   }
   auto URIMainFile = URIForFile::canonicalize(*MainFilePath, *MainFilePath);
   auto CurLoc = sourceLocationInMainFile(SM, Pos);
   if (!CurLoc) {
     llvm::consumeError(CurLoc.takeError());
     return {};
   }
-  SourceLocation SLocId;
+  llvm::Optional<DefinedMacro> Macro;
   if (const auto *IdentifierAtCursor =
-          syntax::spelledIdentifierTouching(*CurLoc, AST.getTokens()))
-    SLocId = IdentifierAtCursor->location();
-  RefsRequest Req;
+          syntax::spelledIdentifierTouching(*CurLoc, AST.getTokens())) {
+    Macro = locateMacroAt(*IdentifierAtCursor, AST.getPreprocessor());
+  }
 
-  if (auto Macro = locateMacroAt(SLocId, AST.getPreprocessor())) {
+  RefsRequest Req;
+  if (Macro) {
     // Handle references to macro.
     if (auto MacroSID = getSymbolID(Macro->Name, Macro->Info, SM)) {
       // Collect macro references from main file.
       const auto &IDToRefs = AST.getMacros().MacroRefs;
       auto Refs = IDToRefs.find(*MacroSID);
       if (Refs != IDToRefs.end()) {
         for (const auto Ref : Refs->second) {
           Location Result;
for (const NamedDecl *D : getDeclAtPosition(AST, *CurLoc, Relations)) {
     Results.push_back(std::move(NewSymbol));
   }
 
   const auto *IdentifierAtCursor =
       syntax::spelledIdentifierTouching(*CurLoc, AST.getTokens());
   if (!IdentifierAtCursor)
     return Results;
 
-  if (auto M = locateMacroAt(IdentifierAtCursor->location(),
-                             AST.getPreprocessor())) {
+  if (auto M = locateMacroAt(*IdentifierAtCursor, AST.getPreprocessor())) {
     SymbolDetails NewMacro;
     NewMacro.name = std::string(M->Name);
     llvm::SmallString<32> USR;
     if (!index::generateUSRForMacro(NewMacro.name, M->Info->getDefinitionLoc(),
                                     SM, USR)) {
       NewMacro.USR = std::string(USR.str());
       NewMacro.ID = SymbolID(NewMacro.USR);
     }

clang-tools-extra/clangd/refactor/Rename.cpp

if (!Loc)
     return Loc.takeError();
   const syntax::Token *IdentifierToken =
       spelledIdentifierTouching(*Loc, AST.getTokens());
   // Renames should only triggered on identifiers.
   if (!IdentifierToken)
     return makeError(ReasonToReject::NoSymbolFound);
   // FIXME: Renaming macros is not supported yet, the macro-handling code should
   // be moved to rename tooling library.
-  if (locateMacroAt(IdentifierToken->location(), AST.getPreprocessor()))
+  if (locateMacroAt(*IdentifierToken, AST.getPreprocessor()))
     return makeError(ReasonToReject::UnsupportedSymbol);
 
   auto DeclsUnderCursor = locateDeclAt(AST, IdentifierToken->location());
   if (DeclsUnderCursor.empty())
     return makeError(ReasonToReject::NoSymbolFound);
   if (DeclsUnderCursor.size() > 1)
     return makeError(ReasonToReject::AmbiguousSymbol);
 

clang-tools-extra/clangd/unittests/CollectMacrosTests.cpp

for (const char *Test : Tests) {
     // Known macros.
     for (int I = 1;; I++) {
       const auto ExpectedRefs = T.ranges(llvm::to_string(I));
       if (ExpectedRefs.empty())
         break;
 
       auto Loc = sourceLocationInMainFile(SM, ExpectedRefs.begin()->start);
       ASSERT_TRUE(bool(Loc));
-      auto Macro = locateMacroAt(*Loc, PP);
+      const auto *Id = syntax::spelledIdentifierTouching(*Loc, AST.getTokens());
+      ASSERT_TRUE(Id);
+      auto Macro = locateMacroAt(*Id, PP);
       assert(Macro);
       auto SID = getSymbolID(Macro->Name, Macro->Info, SM);
 
       EXPECT_THAT(ExpectedRefs,
                   UnorderedElementsAreArray(ActualMacroRefs.MacroRefs[*SID]))
           << "Annotation=" << I << ", MacroName=" << Macro->Name
           << ", Test = " << Test;
     }

clang-tools-extra/clangd/unittests/SourceCodeTests.cpp

Annotations Code(R"cpp(
      int abc = MA^CRO;
    )cpp");
   TestTU TU = TestTU::withCode(Code.code());
   auto AST = TU.build();
   auto CurLoc = sourceLocationInMainFile(AST.getSourceManager(), Code.point());
   ASSERT_TRUE(bool(CurLoc));
   const auto *Id = syntax::spelledIdentifierTouching(*CurLoc, AST.getTokens());
   ASSERT_TRUE(Id);
-  auto Result = locateMacroAt(Id->location(), AST.getPreprocessor());
+  auto Result = locateMacroAt(*Id, AST.getPreprocessor());
   ASSERT_TRUE(Result);
   EXPECT_THAT(*Result, MacroName("MACRO"));
 }
 
 TEST(SourceCodeTests, WorksAtBeginOfFile) {
   Annotations Code("^MACRO");
   TestTU TU = TestTU::withCode(Code.code());
   TU.HeaderCode = "#define MACRO int x;";
   auto AST = TU.build();
   auto CurLoc = sourceLocationInMainFile(AST.getSourceManager(), Code.point());
   ASSERT_TRUE(bool(CurLoc));
   const auto *Id = syntax::spelledIdentifierTouching(*CurLoc, AST.getTokens());
   ASSERT_TRUE(Id);
-  auto Result = locateMacroAt(Id->location(), AST.getPreprocessor());
+  auto Result = locateMacroAt(*Id, AST.getPreprocessor());
   ASSERT_TRUE(Result);
   EXPECT_THAT(*Result, MacroName("MACRO"));
 }
 
 TEST(SourceCodeTests, IsInsideMainFile){
   TestTU TU;
   TU.HeaderCode = R"cpp(
     #define DEFINE_CLASS(X) class X {};