[Syntax] Added function to get macro expansion tokens to TokenBuffer.

[jvikstrom]

Summary

Returns the first token in every mapping where the token is an identifier.
This API is required to be able to highlight macro expansions in clangd.

[jvikstrom]

Readded dissapeared newline.

[ilya-biryukov]

LGTM

clang/include/clang/Tooling/Syntax/Tokens.h

239

NIT: add the doxygen tag for parameter in \p FID

245

NIT: FOO and FOO2 are also mentioned in the #define lines. Maybe clarify that the results are at lines 3 and 5, not 1 and 2?

clang/include/clang/Tooling/Syntax/Tokens.h

public:
   /// Lexed tokens of a file before preprocessing. E.g. for the following input
   ///     #define DECL(name) int name = 10
   ///     DECL(a);
   /// spelledTokens() returns {"#", "define", "DECL", "(", "name", ")", "eof"}.
   /// FIXME: we do not yet store tokens of directives, like #include, #define,
   ///        #pragma, etc.
   llvm::ArrayRef<syntax::Token> spelledTokens(FileID FID) const;
 
+  /// Get all tokens that expand a macro in FID. For the following input
+  ///     #define FOO B
+  ///     #define FOO2(X) int X
+  ///     FOO2(XY)
+  ///     int B;
+  ///     FOO;
+  /// macroExpansions() returns {"FOO2", "FOO"}.
+  std::vector<const syntax::Token *> macroExpansions(FileID FID) const;
+
   const SourceManager &sourceManager() const { return *SourceMgr; }
 
   std::string dumpForTests() const;
 
 private:
   /// Describes a mapping between a continuous subrange of spelled tokens and
   /// expanded tokens. Represents macro expansions, preprocessor directives,
   /// conditionally disabled pp regions, etc.

clang/lib/Tooling/Syntax/Tokens.cpp

TokenBuffer::expansionStartingAt(const syntax::Token *Spelled) const {
   Expansion E;
   E.Spelled = llvm::makeArrayRef(File.SpelledTokens.data() + M->BeginSpelled,
                                  File.SpelledTokens.data() + M->EndSpelled);
   E.Expanded = llvm::makeArrayRef(ExpandedTokens.data() + M->BeginExpanded,
                                   ExpandedTokens.data() + M->EndExpanded);
   return E;
 }
 
+std::vector<const syntax::Token *>
+TokenBuffer::macroExpansions(FileID FID) const {
+  auto FileIt = Files.find(FID);
+  assert(FileIt != Files.end() && "file not tracked by token buffer");
+  auto &File = FileIt->second;
+  std::vector<const syntax::Token *> Expansions;
+  auto &Spelled = File.SpelledTokens;
+  for (auto Mapping : File.Mappings) {
+    const syntax::Token *Token = &Spelled[Mapping.BeginSpelled];
+    if (Token->kind() == tok::TokenKind::identifier)
+      Expansions.push_back(Token);
+  }
+  return Expansions;
+}
+
 std::vector<syntax::Token> syntax::tokenize(FileID FID, const SourceManager &SM,
                                             const LangOptions &LO) {
   std::vector<syntax::Token> Tokens;
   IdentifierTable Identifiers(LO);
   auto AddToken = [&](clang::Token T) {
     // Fill the proper token kind for keywords, etc.
     if (T.getKind() == tok::raw_identifier && !T.needsCleaning() &&
         !T.hasUCN()) { // FIXME: support needsCleaning and hasUCN cases.

clang/unittests/Tooling/Syntax/TokensTest.cpp

 using llvm::ValueIs;
 using ::testing::AllOf;
 using ::testing::Contains;
 using ::testing::ElementsAre;
 using ::testing::Field;
 using ::testing::Matcher;
 using ::testing::Not;
 using ::testing::StartsWith;
-
 namespace {
 // Checks the passed ArrayRef<T> has the same begin() and end() iterators as the
 // argument.
 MATCHER_P(SameRange, A, "") {
   return A.begin() == arg.begin() && A.end() == arg.end();
 }
 
 Matcher<TokenBuffer::Expansion>
TEST_F(TokenBufferTest, TokensToFileRange) {
   EXPECT_EQ(Int.range(SM), FileRange(SM.getMainFileID(), Code.range("i").Begin,
                                      Code.range("i").End));
   EXPECT_EQ(syntax::Token::range(SM, Int, Semi),
             FileRange(SM.getMainFileID(), Code.range("all").Begin,
                       Code.range("all").End));
   // We don't test assertion failures because death tests are slow.
 }
 
+TEST_F(TokenBufferTest, macroExpansions) {
+  llvm::Annotations Code(R"cpp(
+    #define FOO B
+    #define FOO2 BA
+    #define CALL(X) int X
+    #define G CALL(FOO2)
+    int B;
+    $macro[[FOO]];
+    $macro[[CALL]](A);
+    $macro[[G]];
+  )cpp");
+  recordTokens(Code.code());
+  auto &SM = *SourceMgr;
+  auto Expansions = Buffer.macroExpansions(SM.getMainFileID());
+  std::vector<FileRange> ExpectedMacroRanges;
+  for (auto Range : Code.ranges("macro"))
+    ExpectedMacroRanges.push_back(
+        FileRange(SM.getMainFileID(), Range.Begin, Range.End));
+  std::vector<FileRange> ActualMacroRanges;
+  for (auto Expansion : Expansions)
+    ActualMacroRanges.push_back(Expansion->range(SM));
+  EXPECT_EQ(ExpectedMacroRanges, ActualMacroRanges);
+}
 } // namespace