diff --git a/clang/include/clang/Tooling/Transformer/SourceCode.h b/clang/include/clang/Tooling/Transformer/SourceCode.h --- a/clang/include/clang/Tooling/Transformer/SourceCode.h +++ b/clang/include/clang/Tooling/Transformer/SourceCode.h @@ -20,9 +20,10 @@ namespace clang { namespace tooling { -/// Extends \p Range to include the token \p Next, if it immediately follows the -/// end of the range. Otherwise, returns \p Range unchanged. -CharSourceRange maybeExtendRange(CharSourceRange Range, tok::TokenKind Next, +/// Extends \p Range to include the token \p Terminator, if it immediately +/// follows the end of the range. Otherwise, returns \p Range unchanged. +CharSourceRange maybeExtendRange(CharSourceRange Range, + tok::TokenKind Terminator, ASTContext &Context); /// Returns the source range spanning the node, extended to include \p Next, if @@ -35,6 +36,11 @@ Next, Context); } +/// Returns the logical source range of the node, extended to include associated +/// comments and whitespace before and after the node, and associated +/// terminators. +CharSourceRange getAssociatedRange(const Decl &D, ASTContext &Context); + /// Returns the source-code text in the specified range. StringRef getText(CharSourceRange Range, const ASTContext &Context); diff --git a/clang/lib/Tooling/Transformer/SourceCode.cpp b/clang/lib/Tooling/Transformer/SourceCode.cpp --- a/clang/lib/Tooling/Transformer/SourceCode.cpp +++ b/clang/lib/Tooling/Transformer/SourceCode.cpp @@ -10,6 +10,13 @@ // //===----------------------------------------------------------------------===// #include "clang/Tooling/Transformer/SourceCode.h" +#include "clang/AST/ASTContext.h" +#include "clang/AST/Attr.h" +#include "clang/AST/Comment.h" +#include "clang/AST/Decl.h" +#include "clang/AST/DeclCXX.h" +#include "clang/AST/DeclTemplate.h" +#include "clang/AST/Expr.h" #include "clang/Lex/Lexer.h" using namespace clang; @@ -63,3 +70,296 @@ return Range; } + +static std::unique_ptr initLexer(const SourceManager &SM, + SourceLocation Loc, + const LangOptions &LangOpts) { + bool Invalid = false; + auto FileOffset = SM.getDecomposedLoc(Loc); + llvm::StringRef File = SM.getBufferData(FileOffset.first, &Invalid); + assert(!Invalid && "Cannot get file/offset"); + return std::make_unique(SM.getLocForStartOfFile(FileOffset.first), + LangOpts, File.begin(), + File.data() + FileOffset.second, File.end()); +} + +static bool startsWithNewline(const SourceManager &SM, const Token &Tok) { + return isVerticalWhitespace(SM.getCharacterData(Tok.getLocation())[0]); +} + +static bool contains(const std::set &Terminators, + const Token &Tok) { + return Terminators.count(Tok.getKind()) > 0; +} + +// Returns the exlusive, *file* location of the entity whose last token is at +// location 'EntityLast'. That is, it returns the location one past the last +// relevant character. +// +// Associated tokens include comments, horizontal whitespace and 'Terminators' +// -- optional tokens, which, if any are found, will be included; if +// 'Terminators' is empty, we will not include any extra tokens beyond comments +// and horizontal whitespace. +static SourceLocation +getEntityEndLoc(const SourceManager &SM, SourceLocation EntityLast, + const std::set &Terminators, + const LangOptions &LangOpts) { + assert(EntityLast.isValid() && "Invalid end location found."); + + // We remember the last location of a non-horizontal-whitespace token we have + // lexed; this is the location up to which we will want to delete. + // FIXME: Support using the spelling loc here for cases where we want to + // analyze the macro text. + + // FIXME: Should check isTokenRange(), for the (rare) case that + // `ExpansionRange` is a character range. + CharSourceRange ExpansionRange = SM.getExpansionRange(EntityLast); + std::unique_ptr Lexer = + initLexer(SM, ExpansionRange.getEnd(), LangOpts); + // Tell Lexer to return whitespace as pseudo-tokens (kind is tok::unknown). + Lexer->SetKeepWhitespaceMode(true); + + // Generally, the code we want to include looks like this ([] are optional), + // If Terminators is empty: + // [ ] [ ] + // Otherwise: + // ... [ ] [ ] + + Token Tok; + bool Terminated = false; + + // First, lex to the current token (which is the last token of the range that + // we know to be deleted. Then, we process the first token separately from the + // rest based on conditions that hold specifically for that first token. + // + // We do not search for a terminator if none is required or we've already + // encountered it. Also, if the original `EntityLast` location was in a macro + // expansion, we don't have visibility into the text, so we assume we've + // already terminated. + // + // FIXME: This handling of macros is more conservative than necessary. When + // the end of the expansion coincides with the end of the node, we can still + // safely analyze the code. But, it is more complicated, because we need to + // start by lexing the spelling loc for the first token and then switch to the + // expansion loc. + Lexer->LexFromRawLexer(Tok); + if (Terminators.empty() || EntityLast.isMacroID() || + contains(Terminators, Tok)) { + Terminated = true; + } + + // We save the most recent candidate for the exclusive end location. + SourceLocation End = Tok.getEndLoc(); + + while (!Terminated) { + // Lex the next token we want to possibly expand the range with. + Lexer->LexFromRawLexer(Tok); + + switch (Tok.getKind()) { + case tok::eof: + // Unexpected separators. + case tok::l_brace: + case tok::r_brace: + case tok::comma: + return End; + // Whitespace pseudo-tokens. + case tok::unknown: + if (startsWithNewline(SM, Tok)) + // Include at least until the end of the line. + End = Tok.getEndLoc(); + break; + default: + if (contains(Terminators, Tok)) + Terminated = true; + End = Tok.getEndLoc(); + break; + } + } + + do { + // Lex the next token we want to possibly expand the range with. + Lexer->LexFromRawLexer(Tok); + + switch (Tok.getKind()) { + case tok::unknown: + if (startsWithNewline(SM, Tok)) + // We're done, but include this newline. + return Tok.getEndLoc(); + break; + case tok::comment: + // Include any comments we find on the way. + End = Tok.getEndLoc(); + break; + // Special case including of extra semicolons or commas if any terminator + // is a semicolon or comma. + // Note that extra commas only happen when the end location is a macro + // location; we are safe to remove the comma, as removing the comma + // will not break anything that removing the entity wouldn't have + // already broken. + case tok::semi: + case tok::comma: + if (contains(Terminators, Tok)) { + End = Tok.getEndLoc(); + break; + } + // Found an unrelated token; stop and don't include it. + return End; + default: + // Found an unrelated token; stop and don't include it. + return End; + } + } while (true); +} + +// Returns the expected terminator tokens for the given declaration. +// +// If we do not know the correct terminator token, returns an empty set. +// +// There are cases where we have more than one possible terminator (for example, +// we find either a comma or a semicolon after a VarDecl). +static std::set getTerminators(const Decl &D) { + if (llvm::isa(D) || llvm::isa(D)) + return {tok::semi}; + + if (llvm::isa(D) || llvm::isa(D)) + return {tok::r_brace, tok::semi}; + + if (llvm::isa(D) || llvm::isa(D)) + return {tok::comma, tok::semi}; + + return {}; +} + +// Starting from `Loc`, skips whitespace up to, and including, a single +// newline. Returns the (exclusive) end of any skipped whitespace (that is, the +// location immediately after the whitespace). +static SourceLocation skipEmptyLine(const SourceManager &SM, SourceLocation Loc, + const LangOptions &LangOpts) { + const char *LocChars = SM.getCharacterData(Loc); + int i = 0; + while (isHorizontalWhitespace(LocChars[i])) + ++i; + if (isVerticalWhitespace(LocChars[i])) + ++i; + return Loc.getLocWithOffset(i); +} + +// Is `Loc` "separate" from any following syntactic entity? That is, either +// there is no following entity, or it is separated by something meaningful +// (e.g. an empty line, a comment). Since this is a heuristic, we return false +// when in doubt. `Loc` should point at the beginning of a separating or +// terminating element. +static bool startsSeparation(const SourceManager &SM, SourceLocation Loc, + const LangOptions &LangOpts) { + // If the first character is a newline, we'll check for an empty line as a + // separator. However, we can't identify an empty line using tokens, so we + // analyse the characters. If we try to use tokens, we'll just end up with a + // whitespace token, whose characters we'd have to analyse anyhow. + const char *LocChars = SM.getCharacterData(Loc); + int i = 0; + for (; isWhitespace(LocChars[i]); ++i) + if (isVerticalWhitespace(LocChars[i])) + return true; + // We didn't find an empty line, so lex the next token (skipping the newline + // at `End`). + Token Tok; + bool Failed = Lexer::getRawToken(Loc.getLocWithOffset(i), Tok, SM, LangOpts, + /*IgnoreWhiteSpace=*/true); + if (Failed) + // Any text that confuses the lexer seems fair to consider a separation. + return true; + + switch (Tok.getKind()) { + case tok::comment: + case tok::l_brace: + case tok::r_brace: + case tok::eof: + return true; + default: + return false; + } +} + +CharSourceRange tooling::getAssociatedRange(const Decl &Decl, + ASTContext &Context) { + const SourceManager &SM = Context.getSourceManager(); + const LangOptions &LangOpts = Context.getLangOpts(); + CharSourceRange Range = CharSourceRange::getTokenRange(Decl.getSourceRange()); + + // First, expand to the start of the template<> declaration if necessary. + if (const auto *Record = llvm::dyn_cast(&Decl)) { + if (const auto *T = Record->getDescribedClassTemplate()) + if (SM.isBeforeInTranslationUnit(T->getBeginLoc(), Range.getBegin())) + Range.setBegin(T->getBeginLoc()); + } else if (const auto *F = llvm::dyn_cast(&Decl)) { + if (const auto *T = F->getDescribedFunctionTemplate()) + if (SM.isBeforeInTranslationUnit(T->getBeginLoc(), Range.getBegin())) + Range.setBegin(T->getBeginLoc()); + } + + // Next, expand the end location past trailing comments to include a potential + // newline at the end of the decl's line. + Range.setEnd( + getEntityEndLoc(SM, Decl.getEndLoc(), getTerminators(Decl), LangOpts)); + Range.setTokenRange(false); + + // Expand to include preceeding associated comments. We ignore any comments + // that are not preceeding the decl, since we've already skipped trailing + // comments with getEntityEndLoc. + if (const RawComment *Comment = + Decl.getASTContext().getRawCommentForDeclNoCache(&Decl)) + // Only include a preceding comment if: + // * it is *not* separate from the declaration (not including any newline + // that immediately follows the comment), + // * the decl *is* separate from any following entity (so, there are no + // other entities the comment could refer to), and + // * it is not a IfThisThenThat lint check. + if (SM.isBeforeInTranslationUnit(Comment->getBeginLoc(), + Range.getBegin()) && + !startsSeparation(SM, skipEmptyLine(SM, Comment->getEndLoc(), LangOpts), + LangOpts) && + startsSeparation(SM, Range.getEnd(), LangOpts)) { + const StringRef CommentText = Comment->getRawText(SM); + if (!CommentText.contains("LINT.IfChange") && + !CommentText.contains("LINT.ThenChange")) + Range.setBegin(Comment->getBeginLoc()); + } + + // Add leading attributes. + for (auto *Attr : Decl.attrs()) { + if (Attr->getLocation().isInvalid() || + !SM.isBeforeInTranslationUnit(Attr->getLocation(), Range.getBegin())) + continue; + Range.setBegin(Attr->getLocation()); + + // Extend to the left '[[' or '__attribute((' if we saw the attribute, + // unless it is not a valid location. + bool Invalid; + StringRef Source = + SM.getBufferData(SM.getFileID(Range.getBegin()), &Invalid); + if (Invalid) + continue; + llvm::StringRef BeforeAttr = + Source.substr(0, SM.getFileOffset(Range.getBegin())); + llvm::StringRef BeforeAttrStripped = BeforeAttr.rtrim(); + + for (llvm::StringRef Prefix : {"[[", "__attribute__(("}) { + // Handle whitespace between attribute prefix and attribute value. + if (BeforeAttrStripped.endswith(Prefix)) { + // Move start to start position of prefix, which is + // length(BeforeAttr) - length(BeforeAttrStripped) + length(Prefix) + // positions to the left. + Range.setBegin(Range.getBegin().getLocWithOffset(static_cast( + -BeforeAttr.size() + BeforeAttrStripped.size() - Prefix.size()))); + break; + // If we didn't see '[[' or '__attribute' it's probably coming from a + // macro expansion which is already handled by getExpansionRange(), + // below. + } + } + } + + // Range.getEnd() is already fully un-expanded by getEntityEndLoc. But, + // Range.getBegin() may be inside an expansion. + return Lexer::makeFileCharRange(Range, SM, LangOpts); +} diff --git a/clang/unittests/Tooling/SourceCodeTest.cpp b/clang/unittests/Tooling/SourceCodeTest.cpp --- a/clang/unittests/Tooling/SourceCodeTest.cpp +++ b/clang/unittests/Tooling/SourceCodeTest.cpp @@ -17,6 +17,7 @@ using namespace clang; using llvm::ValueIs; +using tooling::getAssociatedRange; using tooling::getExtendedText; using tooling::getRangeForEdit; using tooling::getText; @@ -47,6 +48,28 @@ arg.getBegin() == R.getBegin() && arg.getEnd() == R.getEnd(); } +MATCHER_P2(EqualsAnnotatedRange, SM, R, "") { + if (arg.getBegin().isMacroID()) { + *result_listener << "which starts in a macro"; + return false; + } + if (arg.getEnd().isMacroID()) { + *result_listener << "which ends in a macro"; + return false; + } + + unsigned Begin = SM->getFileOffset(arg.getBegin()); + unsigned End = SM->getFileOffset(arg.getEnd()); + + *result_listener << "which is [" << Begin << ","; + if (arg.isTokenRange()) { + *result_listener << End << "]"; + return Begin == R.Begin && End + 1 == R.End; + } + *result_listener << End << ")"; + return Begin == R.Begin && End == R.End; +} + static ::testing::Matcher AsRange(const SourceManager &SM, llvm::Annotations::Range R) { return EqualsRange(CharSourceRange::getCharRange( @@ -122,6 +145,194 @@ Visitor.runOver("int foo() { return foo() + 3; }"); } +TEST(SourceCodeTest, getAssociatedRange) { + struct VarDeclsVisitor : TestVisitor { + llvm::Annotations Code; + + VarDeclsVisitor() : Code("$r[[]]") {} + bool VisitVarDecl(VarDecl *Decl) { + EXPECT_THAT( + getAssociatedRange(*Decl, *Context), + EqualsAnnotatedRange(&Context->getSourceManager(), Code.range("r"))) + << Code.code(); + return true; + } + }; + + VarDeclsVisitor Visitor; + + // Includes newline. + Visitor.Code = llvm::Annotations("$r[[int x = 4;]]"); + Visitor.runOver(Visitor.Code.code()); + + // Includes newline and semicolon. + Visitor.Code = llvm::Annotations("$r[[int x = 4;\n]]"); + Visitor.runOver(Visitor.Code.code()); + + // Includes trailing comments. + Visitor.Code = llvm::Annotations("$r[[int x = 4; // Comment\n]]"); + Visitor.runOver(Visitor.Code.code()); + Visitor.Code = llvm::Annotations("$r[[int x = 4; /* Comment */\n]]"); + Visitor.runOver(Visitor.Code.code()); + + // Does *not* include trailing comments when another entity appears between + // the decl and the comment. + Visitor.Code = llvm::Annotations("$r[[int x = 4;]] class C {}; // Comment\n"); + Visitor.runOver(Visitor.Code.code()); + + // Includes attributes. + Visitor.Code = llvm::Annotations(R"cpp( + #define ATTR __attribute__((deprecated("message"))) + $r[[ATTR + int x;]])cpp"); + Visitor.runOver(Visitor.Code.code()); + + // Includes attributes and comments together. + Visitor.Code = llvm::Annotations(R"cpp( + #define ATTR __attribute__((deprecated("message"))) + $r[[ATTR + // Commment. + int x;]])cpp"); + Visitor.runOver(Visitor.Code.code()); +} + +TEST(SourceCodeTest, getAssociatedRangeWithComments) { + struct VarDeclsVisitor : TestVisitor { + llvm::Annotations Code; + + VarDeclsVisitor() : Code("$r[[]]") {} + bool VisitVarDecl(VarDecl *Decl) { + EXPECT_THAT( + getAssociatedRange(*Decl, *Context), + EqualsAnnotatedRange(&Context->getSourceManager(), Code.range("r"))) + << Code.code(); + return true; + } + bool runOverWithComments(StringRef Code) { + std::vector Args = {"-std=c++11", "-fparse-all-comments"}; + return tooling::runToolOnCodeWithArgs(CreateTestAction(), Code, Args); + } + }; + + VarDeclsVisitor Visitor; + + // Includes leading comments. + Visitor.Code = llvm::Annotations("$r[[// Comment.\nint x = 4;]]"); + Visitor.runOverWithComments(Visitor.Code.code()); + Visitor.Code = llvm::Annotations("$r[[// Comment.\nint x = 4;\n]]"); + Visitor.runOverWithComments(Visitor.Code.code()); + Visitor.Code = llvm::Annotations("$r[[/* Comment.*/\nint x = 4;\n]]"); + Visitor.runOverWithComments(Visitor.Code.code()); + // ... even if separated by (extra) horizontal whitespace. + Visitor.Code = llvm::Annotations("$r[[/* Comment.*/ \nint x = 4;\n]]"); + Visitor.runOverWithComments(Visitor.Code.code()); + + // Includes comments even in the presence of trailing whitespace. + Visitor.Code = llvm::Annotations("$r[[// Comment.\nint x = 4;]] "); + Visitor.runOverWithComments(Visitor.Code.code()); + + + // Includes comments when the declaration is followed by the beginning or end + // of a compound statement. + Visitor.Code = llvm::Annotations(R"cpp( + class Foo { + $r[[/* C */ + int x = 4;]] + };)cpp"); + Visitor.runOverWithComments(Visitor.Code.code()); + Visitor.Code = llvm::Annotations(R"cpp( + void foo() { + $r[[/* C */ + int x = 4; + ]]{ class Foo {}; } + })cpp"); + Visitor.runOverWithComments(Visitor.Code.code()); + + // Includes comments inside macros (when decl is in the same macro). + Visitor.Code = llvm::Annotations(R"cpp( + #define DECL /* Comment */ int x + $r[[DECL;]])cpp"); + Visitor.runOverWithComments(Visitor.Code.code()); + + // Does not include comments when either decl or comment are inside macros + // (unless both are in the same macro). + // FIXME: Change code to allow this. + Visitor.Code = llvm::Annotations(R"cpp( + #define DECL int x + // Comment + $r[[DECL;]])cpp"); + Visitor.runOverWithComments(Visitor.Code.code()); + Visitor.Code = llvm::Annotations(R"cpp( + #define COMMENT /* Comment */ + COMMENT + $r[[int x;]])cpp"); + Visitor.runOverWithComments(Visitor.Code.code()); + + // Includes multi-line comments. + Visitor.Code = llvm::Annotations(R"cpp( + $r[[/* multi + * line + * comment + */ + int x;]])cpp"); + Visitor.runOverWithComments(Visitor.Code.code()); + Visitor.Code = llvm::Annotations(R"cpp( + $r[[// multi + // line + // comment + int x;]])cpp"); + Visitor.runOverWithComments(Visitor.Code.code()); + + // Does not include comments separated by multiple empty lines. + Visitor.Code = llvm::Annotations("// Comment.\n\n\n$r[[int x = 4;\n]]"); + Visitor.runOverWithComments(Visitor.Code.code()); + Visitor.Code = llvm::Annotations("/* Comment.*/\n\n\n$r[[int x = 4;\n]]"); + Visitor.runOverWithComments(Visitor.Code.code()); + + // Does not include comments before a *series* of declarations. + Visitor.Code = llvm::Annotations(R"cpp( + // Comment. + $r[[int x = 4; + ]]class foo {};)cpp"); + Visitor.runOverWithComments(Visitor.Code.code()); + + // Includes attributes. + Visitor.Code = llvm::Annotations(R"cpp( + #define ATTR __attribute__((deprecated("message"))) + $r[[ATTR + int x;]])cpp"); + Visitor.runOverWithComments(Visitor.Code.code()); + + // Includes attributes and comments together. + Visitor.Code = llvm::Annotations(R"cpp( + #define ATTR __attribute__((deprecated("message"))) + $r[[ATTR + // Commment. + int x;]])cpp"); + Visitor.runOverWithComments(Visitor.Code.code()); +} + +TEST(SourceCodeTest, getAssociatedRangeInvalidForPartialExpansions) { + struct FailingVarDeclsVisitor : TestVisitor { + FailingVarDeclsVisitor() {} + bool VisitVarDecl(VarDecl *Decl) { + EXPECT_TRUE(getAssociatedRange(*Decl, *Context).isInvalid()); + return true; + } + bool runOverWithComments(StringRef Code) { + std::vector Args = {"-std=c++11", "-fparse-all-comments"}; + return tooling::runToolOnCodeWithArgs(CreateTestAction(), Code, Args); + } + }; + + FailingVarDeclsVisitor Visitor; + // Should fail because it only includes a part of the expansion. + std::string Code = R"cpp( + #define DECL class foo { }; int x + DECL;)cpp"; + Visitor.runOver(Code); +} + TEST(SourceCodeTest, EditRangeWithMacroExpansionsShouldSucceed) { // The call expression, whose range we are extracting, includes two macro // expansions.