diff --git a/clang/include/clang/Tooling/Syntax/BaseToken.h b/clang/include/clang/Tooling/Syntax/BaseToken.h new file mode 100644 --- /dev/null +++ b/clang/include/clang/Tooling/Syntax/BaseToken.h @@ -0,0 +1,44 @@ +//===- BaseToken.h - base token class for syntax tree ------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Defines common interfaces for the Token class. The syntax-tree Leaf node +// points to a single token, this allows using different underlying token +// implementations (syntax::Token, pseudo::Token) in the syntax-tree. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLING_SYNTAX_BASE_TOKEN_H +#define LLVM_CLANG_TOOLING_SYNTAX_BASE_TOKEN_H + +#include "clang/Basic/TokenKinds.h" +#include "llvm/ADT/StringRef.h" + +namespace clang { +namespace syntax { + +/// Base class that represents a token for the C++ source code. +/// +/// Depending on the subclass implementation, it can be a spelled token from +/// the original source file or an expanded token. +class BaseToken { +public: + BaseToken(tok::TokenKind K) : Kind(K) {} + + tok::TokenKind kind() const { return Kind; } + + /// Returns the text of the token. + virtual llvm::StringRef text(const void *Ctx = nullptr) const = 0; + +private: + tok::TokenKind Kind; +}; + +} // namespace syntax +} // namespace clang + +#endif // LLVM_CLANG_TOOLING_SYNTAX_BASE_TOKEN_H diff --git a/clang/include/clang/Tooling/Syntax/Tokens.h b/clang/include/clang/Tooling/Syntax/Tokens.h --- a/clang/include/clang/Tooling/Syntax/Tokens.h +++ b/clang/include/clang/Tooling/Syntax/Tokens.h @@ -33,6 +33,7 @@ #include "clang/Basic/SourceManager.h" #include "clang/Basic/TokenKinds.h" #include "clang/Lex/Token.h" +#include "clang/Tooling/Syntax/BaseToken.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/Optional.h" @@ -102,13 +103,12 @@ /// A token coming directly from a file or from a macro invocation. Has just /// enough information to locate the token in the source code. /// Can represent both expanded and spelled tokens. -class Token { +class Token : public BaseToken { public: Token(SourceLocation Location, unsigned Length, tok::TokenKind Kind); /// EXPECTS: clang::Token is not an annotation token. explicit Token(const clang::Token &T); - tok::TokenKind kind() const { return Kind; } /// Location of the first character of a token. SourceLocation location() const { return Location; } /// Location right after the last character of a token. @@ -122,7 +122,10 @@ /// in\ /// t /// both have the same kind tok::kw_int, but results of text() are different. - llvm::StringRef text(const SourceManager &SM) const; + llvm::StringRef text(const SourceManager& SM) const; + /// A wrapper of the above text(). + /// EXPECTS: The opauqe parameter must be a `const clang::SourceManager*`. + llvm::StringRef text(const void* SM) const override; /// Gets a range of this token. /// EXPECTS: token comes from a file, not from a macro expansion. diff --git a/clang/include/clang/Tooling/Syntax/Tree.h b/clang/include/clang/Tooling/Syntax/Tree.h --- a/clang/include/clang/Tooling/Syntax/Tree.h +++ b/clang/include/clang/Tooling/Syntax/Tree.h @@ -122,9 +122,9 @@ Node *getPreviousSibling() { return PreviousSibling; } /// Dumps the structure of a subtree. For debugging and testing purposes. - std::string dump(const SourceManager &SM) const; + std::string dump(const void *SM) const; /// Dumps the tokens forming this subtree. - std::string dumpTokens(const SourceManager &SM) const; + std::string dumpTokens(const void *SM) const; /// Asserts invariants on this node of the tree and its immediate children. /// Will not recurse into the subtree. No-op if NDEBUG is set. @@ -153,16 +153,19 @@ unsigned CanModify : 1; }; -/// A leaf node points to a single token inside the expanded token stream. +/// A leaf node points to a single token. class Leaf final : public Node { public: - Leaf(const Token *T); + Leaf(const BaseToken *T); static bool classof(const Node *N); - const Token *getToken() const { return Tok; } + const BaseToken *getToken() const { return Tok; } + template const T *getTokenAs() const { + return static_cast(Tok); + } private: - const Token *Tok; + const BaseToken *Tok; }; /// A node that has children and represents a syntactic language construct. diff --git a/clang/lib/Tooling/Syntax/BuildTree.cpp b/clang/lib/Tooling/Syntax/BuildTree.cpp --- a/clang/lib/Tooling/Syntax/BuildTree.cpp +++ b/clang/lib/Tooling/Syntax/BuildTree.cpp @@ -671,7 +671,7 @@ R += std::string( formatv("- '{0}' covers '{1}'+{2} tokens\n", It->second->getKind(), It->first->text(A.getSourceManager()), CoveredTokens)); - R += It->second->dump(A.getSourceManager()); + R += It->second->dump(&A.getSourceManager()); } return R; } @@ -896,7 +896,7 @@ // token kind, we run `NumericLiteralParser` again to distinguish them. auto TokLoc = S->getBeginLoc(); auto TokSpelling = - Builder.findToken(TokLoc)->text(Context.getSourceManager()); + Builder.findToken(TokLoc)->text(&Context.getSourceManager()); auto Literal = NumericLiteralParser(TokSpelling, TokLoc, Context.getSourceManager(), Context.getLangOpts(), Context.getTargetInfo(), diff --git a/clang/lib/Tooling/Syntax/ComputeReplacements.cpp b/clang/lib/Tooling/Syntax/ComputeReplacements.cpp --- a/clang/lib/Tooling/Syntax/ComputeReplacements.cpp +++ b/clang/lib/Tooling/Syntax/ComputeReplacements.cpp @@ -48,7 +48,7 @@ if (SpanBegin) Callback(llvm::makeArrayRef(SpanBegin, SpanEnd), SpanIsOriginal); // Start recording a new span. - SpanBegin = L->getToken(); + SpanBegin = L->getTokenAs(); SpanEnd = SpanBegin + 1; SpanIsOriginal = L->isOriginal(); } diff --git a/clang/lib/Tooling/Syntax/Synthesis.cpp b/clang/lib/Tooling/Syntax/Synthesis.cpp --- a/clang/lib/Tooling/Syntax/Synthesis.cpp +++ b/clang/lib/Tooling/Syntax/Synthesis.cpp @@ -213,7 +213,7 @@ // `L->getToken()` gives us the expanded token, thus we implicitly expand // any macros here. return createLeaf(A, L->getToken()->kind(), - L->getToken()->text(A.getSourceManager())); + L->getToken()->text(&A.getSourceManager())); const auto *T = cast(N); std::vector> Children; diff --git a/clang/lib/Tooling/Syntax/Tokens.cpp b/clang/lib/Tooling/Syntax/Tokens.cpp --- a/clang/lib/Tooling/Syntax/Tokens.cpp +++ b/clang/lib/Tooling/Syntax/Tokens.cpp @@ -100,7 +100,7 @@ syntax::Token::Token(SourceLocation Location, unsigned Length, tok::TokenKind Kind) - : Location(Location), Length(Length), Kind(Kind) { + : BaseToken(Kind), Location(Location), Length(Length) { assert(Location.isValid()); } @@ -115,6 +115,10 @@ assert(!Invalid); return llvm::StringRef(Start, length()); } +llvm::StringRef syntax::Token::text(const void* Ctx) const { + const auto* SM = reinterpret_cast(Ctx); + return text(*SM); +} FileRange syntax::Token::range(const SourceManager &SM) const { assert(location().isFileID() && "must be a spelled token"); @@ -860,7 +864,7 @@ } std::string syntax::Token::dumpForTests(const SourceManager &SM) const { - return std::string(llvm::formatv("Token(`{0}`, {1}, length = {2})", text(SM), + return std::string(llvm::formatv("Token(`{0}`, {1}, length = {2})", text(&SM), tok::getTokenName(kind()), length())); } @@ -868,7 +872,7 @@ auto PrintToken = [this](const syntax::Token &T) -> std::string { if (T.kind() == tok::eof) return ""; - return std::string(T.text(*SourceMgr)); + return std::string(T.text(SourceMgr)); }; auto DumpTokens = [this, &PrintToken](llvm::raw_ostream &OS, @@ -877,7 +881,7 @@ OS << ""; return; } - OS << Tokens[0].text(*SourceMgr); + OS << Tokens[0].text(SourceMgr); for (unsigned I = 1; I < Tokens.size(); ++I) { if (Tokens[I].kind() == tok::eof) continue; diff --git a/clang/lib/Tooling/Syntax/Tree.cpp b/clang/lib/Tooling/Syntax/Tree.cpp --- a/clang/lib/Tooling/Syntax/Tree.cpp +++ b/clang/lib/Tooling/Syntax/Tree.cpp @@ -49,7 +49,7 @@ return {FID, It.first->second}; } -syntax::Leaf::Leaf(const syntax::Token *Tok) : Node(NodeKind::Leaf), Tok(Tok) { +syntax::Leaf::Leaf(const syntax::BaseToken *Tok) : Node(NodeKind::Leaf), Tok(Tok) { assert(Tok != nullptr); } @@ -191,7 +191,7 @@ namespace { static void dumpLeaf(raw_ostream &OS, const syntax::Leaf *L, - const SourceManager &SM) { + const void* SM) { assert(L); const auto *Token = L->getToken(); assert(Token); @@ -203,7 +203,7 @@ } static void dumpNode(raw_ostream &OS, const syntax::Node *N, - const SourceManager &SM, llvm::BitVector IndentMask) { + const void* SM, llvm::BitVector IndentMask) { auto DumpExtraInfo = [&OS](const syntax::Node *N) { if (N->getRole() != syntax::NodeRole::Unknown) OS << " " << N->getRole(); @@ -248,14 +248,14 @@ } } // namespace -std::string syntax::Node::dump(const SourceManager &SM) const { +std::string syntax::Node::dump(const void* SM) const { std::string Str; llvm::raw_string_ostream OS(Str); dumpNode(OS, this, SM, /*IndentMask=*/{}); return std::move(OS.str()); } -std::string syntax::Node::dumpTokens(const SourceManager &SM) const { +std::string syntax::Node::dumpTokens(const void* SM) const { std::string Storage; llvm::raw_string_ostream OS(Storage); traverse(this, [&](const syntax::Node *N) { diff --git a/clang/tools/clang-check/ClangCheck.cpp b/clang/tools/clang-check/ClangCheck.cpp --- a/clang/tools/clang-check/ClangCheck.cpp +++ b/clang/tools/clang-check/ClangCheck.cpp @@ -159,7 +159,7 @@ llvm::outs() << TB.dumpForTests(); clang::syntax::Arena A(AST.getSourceManager(), AST.getLangOpts(), TB); llvm::outs() << clang::syntax::buildSyntaxTree(A, AST)->dump( - AST.getSourceManager()); + &AST.getSourceManager()); } private: diff --git a/clang/unittests/Tooling/Syntax/BuildTreeTest.cpp b/clang/unittests/Tooling/Syntax/BuildTreeTest.cpp --- a/clang/unittests/Tooling/Syntax/BuildTreeTest.cpp +++ b/clang/unittests/Tooling/Syntax/BuildTreeTest.cpp @@ -26,7 +26,7 @@ auto ErrorOK = errorOK(Code); if (!ErrorOK) return ErrorOK; - auto Actual = StringRef(Root->dump(Arena->getSourceManager())).trim().str(); + auto Actual = StringRef(Root->dump(&Arena->getSourceManager())).trim().str(); // EXPECT_EQ shows the diff between the two strings if they are different. EXPECT_EQ(Tree.trim().str(), Actual); if (Actual != Tree.trim().str()) { @@ -59,7 +59,7 @@ auto *AnnotatedNode = nodeByRange(AnnotatedRanges[i], Root); assert(AnnotatedNode); auto AnnotatedNodeDump = - StringRef(AnnotatedNode->dump(Arena->getSourceManager())) + StringRef(AnnotatedNode->dump(&Arena->getSourceManager())) .trim() .str(); // EXPECT_EQ shows the diff between the two strings if they are different. diff --git a/clang/unittests/Tooling/Syntax/SynthesisTest.cpp b/clang/unittests/Tooling/Syntax/SynthesisTest.cpp --- a/clang/unittests/Tooling/Syntax/SynthesisTest.cpp +++ b/clang/unittests/Tooling/Syntax/SynthesisTest.cpp @@ -27,7 +27,8 @@ return ::testing::AssertionFailure() << "Root was not built successfully."; - auto Actual = StringRef(Root->dump(Arena->getSourceManager())).trim().str(); + auto Actual = + StringRef(Root->dump(&Arena->getSourceManager())).trim().str(); auto Expected = Dump.trim().str(); // EXPECT_EQ shows the diff between the two strings if they are different. EXPECT_EQ(Expected, Actual); @@ -175,7 +176,7 @@ auto *Copy = deepCopyExpandingMacros(*Arena, StatementContinue); EXPECT_TRUE( - treeDumpEqual(Copy, StatementContinue->dump(Arena->getSourceManager()))); + treeDumpEqual(Copy, StatementContinue->dump(&Arena->getSourceManager()))); // FIXME: Test that copy is independent of original, once the Mutations API is // more developed. } diff --git a/clang/unittests/Tooling/Syntax/TokensTest.cpp b/clang/unittests/Tooling/Syntax/TokensTest.cpp --- a/clang/unittests/Tooling/Syntax/TokensTest.cpp +++ b/clang/unittests/Tooling/Syntax/TokensTest.cpp @@ -220,7 +220,7 @@ } // An equality test for search. auto TextMatches = [this](llvm::StringRef Q, const syntax::Token &T) { - return Q == T.text(*SourceMgr); + return Q == T.text(&*SourceMgr); }; // Find a match. auto Found = @@ -906,7 +906,7 @@ SourceLocation Loc = SourceMgr->getComposedLoc(SourceMgr->getMainFileID(), Code.points()[Index]); const syntax::Token *Tok = spelledIdentifierTouching(Loc, Buffer); - return Tok ? Tok->text(*SourceMgr) : ""; + return Tok ? Tok->text(&*SourceMgr) : ""; }; EXPECT_THAT(Touching(0), SameRange(findSpelled("int"))); diff --git a/clang/unittests/Tooling/Syntax/TreeTest.cpp b/clang/unittests/Tooling/Syntax/TreeTest.cpp --- a/clang/unittests/Tooling/Syntax/TreeTest.cpp +++ b/clang/unittests/Tooling/Syntax/TreeTest.cpp @@ -180,7 +180,7 @@ private: std::string dumpQuotedTokensOrNull(const Node *N) { return N ? "'" + - StringRef(N->dumpTokens(Arena->getSourceManager())) + StringRef(N->dumpTokens(&Arena->getSourceManager())) .trim() .str() + "'" diff --git a/clang/unittests/Tooling/Syntax/TreeTestBase.cpp b/clang/unittests/Tooling/Syntax/TreeTestBase.cpp --- a/clang/unittests/Tooling/Syntax/TreeTestBase.cpp +++ b/clang/unittests/Tooling/Syntax/TreeTestBase.cpp @@ -38,10 +38,10 @@ ArrayRef tokens(syntax::Node *N) { assert(N->isOriginal() && "tokens of modified nodes are not well-defined"); if (auto *L = dyn_cast(N)) - return llvm::makeArrayRef(L->getToken(), 1); + return llvm::makeArrayRef(L->getTokenAs(), 1); auto *T = cast(N); - return llvm::makeArrayRef(T->findFirstLeaf()->getToken(), - T->findLastLeaf()->getToken() + 1); + return llvm::makeArrayRef(T->findFirstLeaf()->getTokenAs(), + T->findLastLeaf()->getTokenAs() + 1); } } // namespace