[clang-tidy] NFC refactor lexer-utils to be usable without ASTContext

[JonasToth]

Summary

This patch is a small refactoring necessary for
'readability-isolate-declaration' and does not introduce functional changes.
It allows to use the utility functions without a full ASTContext and requires only the SourceManager and the LangOpts.

[JonasToth]

• fix missing file

[alexfh]

So far the change doesn't look really helpful. Are you going to use the function from code where no ASTContext is available? Can you give an example?

[JonasToth]

This patch is related to https://reviews.llvm.org/D51949

To isolate variable declarations (split int * p, v; up) it is necessary to do a lot of work with source location and requires some forward and backwards lexing. The functions there just use the LangOpts and the SourceManager and don't have a ASTContext available, that why I changed this interface.

[alexfh]

In D52684#1250885, @JonasToth wrote:

This patch is related to https://reviews.llvm.org/D51949

To isolate variable declarations (split int * p, v; up) it is necessary to do a lot of work with source location and requires some forward and backwards lexing. The functions there just use the LangOpts and the SourceManager and don't have a ASTContext available, that why I changed this interface.

Makes sense. But then I'd change "to be easier to use" to "to be callable without ASTContext" or something similar in the patch description.

Otherwise LG.

clang-tidy/bugprone/ArgumentCommentCheck.cpp

getCommentsInRange(ASTContext *Ctx, CharSourceRange Range) {
 
   return Comments;
 }
 
 static std::vector<std::pair<SourceLocation, StringRef>>
 getCommentsBeforeLoc(ASTContext *Ctx, SourceLocation Loc) {
   std::vector<std::pair<SourceLocation, StringRef>> Comments;
   while (Loc.isValid()) {
-    clang::Token Tok =
-        utils::lexer::getPreviousToken(*Ctx, Loc, /*SkipComments=*/false);
+    clang::Token Tok = utils::lexer::getPreviousToken(
+        Loc, Ctx->getSourceManager(), Ctx->getLangOpts(),
+        /*SkipComments=*/false);
     if (Tok.isNot(tok::comment))
       break;
     Loc = Tok.getLocation();
     Comments.emplace_back(
         Loc,
         Lexer::getSourceText(CharSourceRange::getCharRange(
                                  Loc, Loc.getLocWithOffset(Tok.getLength())),
                              Ctx->getSourceManager(), Ctx->getLangOpts()));

clang-tidy/bugprone/SuspiciousSemicolonCheck.cpp

void SuspiciousSemicolonCheck::check(const MatchFinder::MatchResult &Result) {
 
   const auto *Semicolon = Result.Nodes.getNodeAs<NullStmt>("semi");
   SourceLocation LocStart = Semicolon->getBeginLoc();
 
   if (LocStart.isMacroID())
     return;
 
   ASTContext &Ctxt = *Result.Context;
-  auto Token = utils::lexer::getPreviousToken(Ctxt, LocStart);
+  auto Token = utils::lexer::getPreviousToken(LocStart, Ctxt.getSourceManager(),
+                                              Ctxt.getLangOpts());
   auto &SM = *Result.SourceManager;
   unsigned SemicolonLine = SM.getSpellingLineNumber(LocStart);
 
   const auto *Statement = Result.Nodes.getNodeAs<Stmt>("stmt");
   const bool IsIfStmt = isa<IfStmt>(Statement);
 
   if (!IsIfStmt &&
       SM.getSpellingLineNumber(Token.getLocation()) != SemicolonLine)

clang-tidy/cppcoreguidelines/ProTypeMemberInitCheck.cpp

SourceLocation getLocation(const ASTContext &Context,
                              const CXXConstructorDecl &Constructor) const {
     assert((Where != nullptr || Placement == InitializerPlacement::New) &&
            "Location should be relative to an existing initializer or this "
            "insertion represents a new initializer list.");
     SourceLocation Location;
     switch (Placement) {
     case InitializerPlacement::New:
       Location = utils::lexer::getPreviousToken(
-                     Context, Constructor.getBody()->getBeginLoc())
+                     Constructor.getBody()->getBeginLoc(),
+                     Context.getSourceManager(), Context.getLangOpts())
                      .getLocation();
       break;
     case InitializerPlacement::Before:
       Location = utils::lexer::getPreviousToken(
-                     Context, Where->getSourceRange().getBegin())
+                     Where->getSourceRange().getBegin(),
+                     Context.getSourceManager(), Context.getLangOpts())
                      .getLocation();
       break;
     case InitializerPlacement::After:
       Location = Where->getRParenLoc();
       break;
     }
     return getLocationForEndOfToken(Context, Location);
   }

clang-tidy/utils/FixItHintUtils.cpp

 
 namespace clang {
 namespace tidy {
 namespace utils {
 namespace fixit {
 
 FixItHint changeVarDeclToReference(const VarDecl &Var, ASTContext &Context) {
   SourceLocation AmpLocation = Var.getLocation();
-  auto Token = utils::lexer::getPreviousToken(Context, AmpLocation);
+  auto Token = utils::lexer::getPreviousToken(
+      AmpLocation, Context.getSourceManager(), Context.getLangOpts());
   if (!Token.is(tok::unknown))
     AmpLocation = Lexer::getLocForEndOfToken(Token.getLocation(), 0,
                                              Context.getSourceManager(),
                                              Context.getLangOpts());
   return FixItHint::CreateInsertion(AmpLocation, "&");
 }
 
 FixItHint changeVarDeclToConst(const VarDecl &Var) {
   return FixItHint::CreateInsertion(Var.getTypeSpecStartLoc(), "const ");
 }
 
 } // namespace fixit
 } // namespace utils
 } // namespace tidy
 } // namespace clang

clang-tidy/utils/LexerUtils.h

 #include "clang/Lex/Lexer.h"
 
 namespace clang {
 namespace tidy {
 namespace utils {
 namespace lexer {
 
 /// Returns previous token or ``tok::unknown`` if not found.
-Token getPreviousToken(const ASTContext &Context, SourceLocation Location,
-                       bool SkipComments = true);
+Token getPreviousToken(SourceLocation Location, const SourceManager &SM,
+                       const LangOptions &LangOpts, bool SkipComments = true);
 
 } // namespace lexer
 } // namespace utils
 } // namespace tidy
 } // namespace clang
 
 #endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_UTILS_LEXER_UTILS_H

clang-tidy/utils/LexerUtils.cpp

 //===--- LexerUtils.cpp - clang-tidy---------------------------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #include "LexerUtils.h"
 
 namespace clang {
 namespace tidy {
 namespace utils {
 namespace lexer {
 
-Token getPreviousToken(const ASTContext &Context, SourceLocation Location,
-                       bool SkipComments) {
-  const auto &SourceManager = Context.getSourceManager();
+Token getPreviousToken(SourceLocation Location, const SourceManager &SM,
+                       const LangOptions &LangOpts, bool SkipComments) {
   Token Token;
   Token.setKind(tok::unknown);
   Location = Location.getLocWithOffset(-1);
-  auto StartOfFile =
-      SourceManager.getLocForStartOfFile(SourceManager.getFileID(Location));
+  auto StartOfFile = SM.getLocForStartOfFile(SM.getFileID(Location));
   while (Location != StartOfFile) {
-    Location = Lexer::GetBeginningOfToken(Location, SourceManager,
-                                          Context.getLangOpts());
-    if (!Lexer::getRawToken(Location, Token, SourceManager,
-                            Context.getLangOpts()) &&
+    Location = Lexer::GetBeginningOfToken(Location, SM, LangOpts);
+    if (!Lexer::getRawToken(Location, Token, SM, LangOpts) &&
         (!SkipComments || !Token.is(tok::comment))) {
       break;
     }
     Location = Location.getLocWithOffset(-1);
   }
   return Token;
 }
 
 } // namespace lexer
 } // namespace utils
 } // namespace tidy
 } // namespace clang