diff --git a/clang/include/clang/Tooling/Syntax/Tokens.h b/clang/include/clang/Tooling/Syntax/Tokens.h --- a/clang/include/clang/Tooling/Syntax/Tokens.h +++ b/clang/include/clang/Tooling/Syntax/Tokens.h @@ -200,6 +200,21 @@ llvm::Optional> spelledForExpanded(llvm::ArrayRef Expanded) const; + /// An expansion produced by the preprocessor, includes macro expansions and + /// macro directives. Preprocessor always maps a non-empty range of spelled + /// tokens to a (possibly empty) range of expanded tokens. + /// Here is a few examples of expansions: + /// #pragme once // Expansion from "#pragma once" to an empty range. + /// #define FOO 1 2 3 // Expansion from "#define FOO 1" to an empty range. + /// FOO // Expansion from "FOO" to "1 2 3". + struct Expansion { + llvm::ArrayRef Spelled; + llvm::ArrayRef Expanded; + }; + /// If \p Spelled starts a mapping (e.g. if it's a macro name) return the + /// subrange of expanded tokens that the macro expands to. + llvm::Optional findExpansion(const syntax::Token *Spelled) const; + /// Lexed tokens of a file before preprocessing. E.g. for the following input /// #define DECL(name) int name = 10 /// DECL(a); @@ -292,6 +307,7 @@ /// finished, i.e. after running Execute(). LLVM_NODISCARD TokenBuffer consume() &&; + private: /// Maps from a start location to an end location of each mapping. /// 1. range from '#' to the last token in the line for PP directives, diff --git a/clang/lib/Tooling/Syntax/Tokens.cpp b/clang/lib/Tooling/Syntax/Tokens.cpp --- a/clang/lib/Tooling/Syntax/Tokens.cpp +++ b/clang/lib/Tooling/Syntax/Tokens.cpp @@ -200,6 +200,32 @@ : LastSpelled + 1); } +llvm::Optional +TokenBuffer::findExpansion(const syntax::Token *Spelled) const { + assert(Spelled); + assert(Spelled->location().isFileID() && "not a spelled token"); + auto FileIt = Files.find(SourceMgr->getFileID(Spelled->location())); + assert(FileIt != Files.end()); + + auto &File = FileIt->second; + assert(File.SpelledTokens.data() <= Spelled && + Spelled < (File.SpelledTokens.data() + File.SpelledTokens.size())); + + unsigned SpelledIndex = Spelled - File.SpelledTokens.data(); + auto M = llvm::bsearch(File.Mappings, [&](const Mapping &M) { + return SpelledIndex <= M.BeginSpelled; + }); + if (M == File.Mappings.end() || M->BeginSpelled != SpelledIndex) + return llvm::None; + + Expansion E; + E.Spelled = llvm::makeArrayRef(File.SpelledTokens.data() + M->BeginSpelled, + File.SpelledTokens.data() + M->EndSpelled); + E.Expanded = llvm::makeArrayRef(ExpandedTokens.data() + M->BeginExpanded, + ExpandedTokens.data() + M->EndExpanded); + return E; +} + std::vector syntax::tokenize(FileID FID, const SourceManager &SM, const LangOptions &LO) { std::vector Tokens;