[index] Improve macro indexing support

[benlangmuir]

Summary

The major change here is to index macro occurrences in more places than
before, specifically

• In non-expansion references such as #if, #ifdef, etc.
• When the macro is a reference to a builtin macro such as __LINE__.
• When using the preprocessor state instead of callbacks, we now include all definition locations and undefinitions instead of just the latest one (which may also have had the wrong location previously).
• When indexing an existing module file (.pcm), we now include module macros, and we no longer report unrelated preprocessor macros during indexing the module, which could have caused duplication.

Additionally, we now correctly obey the system symbol filter for macros,
so by default in system headers only definition/undefinition occurrences
are reported, but it can be configured to report references as well if
desired.

Extends FileIndexRecord to support occurrences of macros. Since the
design of this type is to keep a single list of entities organized by
source location, we incorporate macros into the existing DeclOccurrence
struct.

[benlangmuir]

Fixed clang-format issue.

[benlangmuir]

Fix the incomplete type issue with PointerUnion in DeclOccurrence. h. I'm not actually sure how that code compiles without error with other versions of clang.

[benlangmuir]

Fix clang-tidy warnings

clang/include/clang/Index/DeclOccurrence.h

 //===- DeclOccurrence.h - An occurrence of a decl within a file -*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_INDEX_DECLOCCURRENCE_H
 #define LLVM_CLANG_INDEX_DECLOCCURRENCE_H
 
+#include "clang/AST/DeclBase.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Index/IndexSymbol.h"
+#include "clang/Lex/MacroInfo.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/PointerUnion.h"
 #include "llvm/ADT/SmallVector.h"
 
 namespace clang {
-class Decl;
-
 namespace index {
 
 struct DeclOccurrence {
   SymbolRoleSet Roles;
   unsigned Offset;
-  const Decl *Dcl;
+  llvm::PointerUnion<const Decl *, const MacroInfo *> DeclOrMacro;
+  const IdentifierInfo *MacroName = nullptr;
   SmallVector<SymbolRelation, 3> Relations;
 
   DeclOccurrence(SymbolRoleSet R, unsigned Offset, const Decl *D,
                  ArrayRef<SymbolRelation> Relations)
-      : Roles(R), Offset(Offset), Dcl(D),
+      : Roles(R), Offset(Offset), DeclOrMacro(D),
         Relations(Relations.begin(), Relations.end()) {}
+  DeclOccurrence(SymbolRoleSet R, unsigned Offset, const IdentifierInfo *Name,
+                 const MacroInfo *MI)
+      : Roles(R), Offset(Offset), DeclOrMacro(MI), MacroName(Name) {}
 
   friend bool operator<(const DeclOccurrence &LHS, const DeclOccurrence &RHS) {
     return LHS.Offset < RHS.Offset;
   }
 };
 
 } // namespace index
 } // namespace clang
 
 #endif // LLVM_CLANG_INDEX_DECLOCCURRENCE_H

clang/include/clang/Index/IndexingOptions.h

enum class SystemSymbolFilterKind {
     DeclarationsOnly,
     All,
   };
 
   SystemSymbolFilterKind SystemSymbolFilter =
       SystemSymbolFilterKind::DeclarationsOnly;
   bool IndexFunctionLocals = false;
   bool IndexImplicitInstantiation = false;
+  bool IndexMacros = true;
   // Whether to index macro definitions in the Preprocesor when preprocessor
   // callback is not available (e.g. after parsing has finished). Note that
   // macro references are not available in Proprocessor.
   bool IndexMacrosInPreprocessor = false;
   // Has no effect if IndexFunctionLocals are false.
   bool IndexParametersInDeclarations = false;
   bool IndexTemplateParameters = false;
 

clang/lib/Index/FileIndexRecord.h

public:
   /// \c Offset
   ///
   /// \param Roles the roles the occurrence fulfills in this position.
   /// \param Offset the offset in the file of this occurrence.
   /// \param D the canonical declaration this is an occurrence of.
   /// \param Relations the set of symbols related to this occurrence.
   void addDeclOccurence(SymbolRoleSet Roles, unsigned Offset, const Decl *D,
                         ArrayRef<SymbolRelation> Relations);
-  void print(llvm::raw_ostream &OS) const;
+
+  /// Adds an occurrence of the given macro at the supplied \c Offset.
+  ///
+  /// \param Roles the roles the occurrence fulfills in this position.
+  /// \param Offset the offset in the file of this occurrence.
+  /// \param Name the name of the macro.
+  /// \param MI the canonical declaration this is an occurrence of.
+  void addMacroOccurence(SymbolRoleSet Roles, unsigned Offset,
+                         const IdentifierInfo *Name, const MacroInfo *MI);
+
+  /// Remove any macro occurrences for header guards. When preprocessing, this
+  /// will only be accurate after HandleEndOfFile.
+  void removeHeaderGuardMacros();
+
+  void print(llvm::raw_ostream &OS, SourceManager &SM) const;
 };
 
 } // end namespace index
 } // end namespace clang
 
 #endif // LLVM_CLANG_LIB_INDEX_FILEINDEXRECORD_H

clang/lib/Index/FileIndexRecord.cpp

 #include "clang/AST/DeclTemplate.h"
 #include "clang/Basic/SourceManager.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/Support/Path.h"
 
 using namespace clang;
 using namespace clang::index;
 
-void FileIndexRecord::addDeclOccurence(SymbolRoleSet Roles, unsigned Offset,
-                                       const Decl *D,
-                                       ArrayRef<SymbolRelation> Relations) {
-  assert(D->isCanonicalDecl() &&
-         "Occurrences should be associated with their canonical decl");
-
+static void addOccurrence(std::vector<DeclOccurrence> &Decls,
+                          DeclOccurrence Info) {
   auto IsNextOccurence = [&]() -> bool {
     if (Decls.empty())
       return true;
     auto &Last = Decls.back();
-    return Last.Offset < Offset;
+    return Last.Offset < Info.Offset;
   };
 
   if (IsNextOccurence()) {
-    Decls.emplace_back(Roles, Offset, D, Relations);
+    Decls.push_back(std::move(Info));
     return;
   }
 
-  DeclOccurrence NewInfo(Roles, Offset, D, Relations);
   // We keep Decls in order as we need to access them in this order in all cases.
-  auto It = llvm::upper_bound(Decls, NewInfo);
-  Decls.insert(It, std::move(NewInfo));
+  auto It = llvm::upper_bound(Decls, Info);
+  Decls.insert(It, std::move(Info));
 }
 
-void FileIndexRecord::print(llvm::raw_ostream &OS) const {
+void FileIndexRecord::addDeclOccurence(SymbolRoleSet Roles, unsigned Offset,
+                                       const Decl *D,
+                                       ArrayRef<SymbolRelation> Relations) {
+  assert(D->isCanonicalDecl() &&
+         "Occurrences should be associated with their canonical decl");
+  addOccurrence(Decls, DeclOccurrence(Roles, Offset, D, Relations));
+}
+
+void FileIndexRecord::addMacroOccurence(SymbolRoleSet Roles, unsigned Offset,
+                                        const IdentifierInfo *Name,
+                                        const MacroInfo *MI) {
+  addOccurrence(Decls, DeclOccurrence(Roles, Offset, Name, MI));
+}
+
+void FileIndexRecord::removeHeaderGuardMacros() {
+  auto It =
+      std::remove_if(Decls.begin(), Decls.end(), [](const DeclOccurrence &D) {
+        if (const auto *MI = D.DeclOrMacro.dyn_cast<const MacroInfo *>())
+          return MI->isUsedForHeaderGuard();
+        return false;
+      });
+  Decls.erase(It, Decls.end());
+}
+
+void FileIndexRecord::print(llvm::raw_ostream &OS, SourceManager &SM) const {
   OS << "DECLS BEGIN ---\n";
   for (auto &DclInfo : Decls) {
-    const Decl *D = DclInfo.Dcl;
-    SourceManager &SM = D->getASTContext().getSourceManager();
+    if (const auto *D = DclInfo.DeclOrMacro.dyn_cast<const Decl *>()) {
       SourceLocation Loc = SM.getFileLoc(D->getLocation());
       PresumedLoc PLoc = SM.getPresumedLoc(Loc);
-    OS << llvm::sys::path::filename(PLoc.getFilename()) << ':' << PLoc.getLine()
-       << ':' << PLoc.getColumn();
+      OS << llvm::sys::path::filename(PLoc.getFilename()) << ':'
+         << PLoc.getLine() << ':' << PLoc.getColumn();
 
-    if (auto ND = dyn_cast<NamedDecl>(D)) {
+      if (const auto *ND = dyn_cast<NamedDecl>(D)) {
         OS << ' ' << ND->getDeclName();
       }
+    } else {
+      const auto *MI = DclInfo.DeclOrMacro.get<const MacroInfo *>();
+      SourceLocation Loc = SM.getFileLoc(MI->getDefinitionLoc());
+      PresumedLoc PLoc = SM.getPresumedLoc(Loc);
+      OS << llvm::sys::path::filename(PLoc.getFilename()) << ':'
+         << PLoc.getLine() << ':' << PLoc.getColumn();
+      OS << ' ' << DclInfo.MacroName->getName();
+    }
 
     OS << '\n';
   }
   OS << "DECLS END ---\n";
 }

clang/lib/Index/IndexingAction.cpp

public:
   void MacroUndefined(const Token &MacroNameTok, const MacroDefinition &MD,
                       const MacroDirective *Undef) override {
     if (!MD.getMacroInfo())  // Ignore noop #undef.
       return;
     IndexCtx->handleMacroUndefined(*MacroNameTok.getIdentifierInfo(),
                                    MacroNameTok.getLocation(),
                                    *MD.getMacroInfo());
   }
+
+  void Defined(const Token &MacroNameTok, const MacroDefinition &MD,
+               SourceRange Range) override {
+    if (!MD.getMacroInfo()) // Ignore nonexistent macro.
+      return;
+    // Note: this is defined(M), not #define M
+    IndexCtx->handleMacroReference(*MacroNameTok.getIdentifierInfo(),
+                                   MacroNameTok.getLocation(),
+                                   *MD.getMacroInfo());
+  }
+  void Ifdef(SourceLocation Loc, const Token &MacroNameTok,
+             const MacroDefinition &MD) override {
+    if (!MD.getMacroInfo()) // Ignore non-existent macro.
+      return;
+    IndexCtx->handleMacroReference(*MacroNameTok.getIdentifierInfo(),
+                                   MacroNameTok.getLocation(),
+                                   *MD.getMacroInfo());
+  }
+  void Ifndef(SourceLocation Loc, const Token &MacroNameTok,
+              const MacroDefinition &MD) override {
+    if (!MD.getMacroInfo()) // Ignore nonexistent macro.
+      return;
+    IndexCtx->handleMacroReference(*MacroNameTok.getIdentifierInfo(),
+                                   MacroNameTok.getLocation(),
+                                   *MD.getMacroInfo());
+  }
 };
 
 class IndexASTConsumer final : public ASTConsumer {
   std::shared_ptr<IndexDataConsumer> DataConsumer;
   std::shared_ptr<IndexingContext> IndexCtx;
   std::shared_ptr<Preprocessor> PP;
   std::function<bool(const Decl *)> ShouldSkipFunctionBody;
 
static bool topLevelDeclVisitor(void *context, const Decl *D) {
   IndexingContext &IndexCtx = *static_cast<IndexingContext*>(context);
   return IndexCtx.indexTopLevelDecl(D);
 }
 
 static void indexTranslationUnit(ASTUnit &Unit, IndexingContext &IndexCtx) {
   Unit.visitLocalTopLevelDecls(&IndexCtx, topLevelDeclVisitor);
 }
 
-static void indexPreprocessorMacros(const Preprocessor &PP,
-                                    IndexDataConsumer &DataConsumer) {
-  for (const auto &M : PP.macros())
-    if (MacroDirective *MD = M.second.getLatest()) {
-      auto *MI = MD->getMacroInfo();
-      // When using modules, it may happen that we find #undef of a macro that
-      // was defined in another module. In such case, MI may be nullptr, since
-      // we only look for macro definitions in the current TU. In that case,
-      // there is nothing to index.
-      if (!MI)
-        continue;
+static void indexPreprocessorMacro(const IdentifierInfo *II,
+                                   const MacroInfo *MI,
+                                   MacroDirective::Kind DirectiveKind,
+                                   SourceLocation Loc,
+    return;
 
-      DataConsumer.handleMacroOccurrence(
-          M.first, MD->getMacroInfo(),
-          static_cast<unsigned>(index::SymbolRole::Definition),
-          MD->getLocation());
-    }
+  // Skip implicit visibility change.
+  if (DirectiveKind == MacroDirective::MD_Visibility)
+    return;
+
+  auto Role = DirectiveKind == MacroDirective::MD_Define
+                  ? SymbolRole::Definition
+                  : SymbolRole::Undefinition;
+  DataConsumer.handleMacroOccurrence(II, MI, static_cast<unsigned>(Role), Loc);
+}
+
+static void indexPreprocessorMacros(Preprocessor &PP,
+                                    IndexDataConsumer &DataConsumer) {
+  for (const auto &M : PP.macros()) {
+    for (auto *MD = M.second.getLatest(); MD; MD = MD->getPrevious()) {
+      indexPreprocessorMacro(M.first, MD->getMacroInfo(), MD->getKind(),
+                             MD->getLocation(), DataConsumer);
+    }
+  }
+}
+
+static void indexPreprocessorModuleMacros(Preprocessor &PP,
+                                          serialization::ModuleFile &Mod,
+                                          IndexDataConsumer &DataConsumer) {
+  for (const auto &M : PP.macros()) {
+    if (M.second.getLatest() == nullptr) {
+      for (auto *MM : PP.getLeafModuleMacros(M.first)) {
+        auto *OwningMod = MM->getOwningModule();
+        if (OwningMod && OwningMod->getASTFile() == Mod.File) {
+          if (auto *MI = MM->getMacroInfo()) {
+            indexPreprocessorMacro(M.first, MI, MacroDirective::MD_Define,
+                                   MI->getDefinitionLoc(), DataConsumer);
+          }
+        }
+      }
+    }
 }
 
 void index::indexASTUnit(ASTUnit &Unit, IndexDataConsumer &DataConsumer,
                          IndexingOptions Opts) {
   IndexingContext IndexCtx(Opts, DataConsumer);
   IndexCtx.setASTContext(Unit.getASTContext());
   DataConsumer.initialize(Unit.getASTContext());
   DataConsumer.setPreprocessor(Unit.getPreprocessorPtr());

 void index::indexModuleFile(serialization::ModuleFile &Mod, ASTReader &Reader,
                             IndexDataConsumer &DataConsumer,
                             IndexingOptions Opts) {
   ASTContext &Ctx = Reader.getContext();
   IndexingContext IndexCtx(Opts, DataConsumer);
   IndexCtx.setASTContext(Ctx);
   DataConsumer.initialize(Ctx);
 
-  if (Opts.IndexMacrosInPreprocessor)
-    indexPreprocessorMacros(Reader.getPreprocessor(), DataConsumer);
+  if (Opts.IndexMacrosInPreprocessor) {
+    indexPreprocessorModuleMacros(Reader.getPreprocessor(), Mod, DataConsumer);
+  }
 
   for (const Decl *D : Reader.getModuleFileLevelDecls(Mod)) {
     IndexCtx.indexTopLevelDecl(D);
   }
   DataConsumer.finish();
 }

clang/lib/Index/IndexingContext.h

void indexBody(const Stmt *S, const NamedDecl *Parent,
                  const DeclContext *DC = nullptr);
 
   bool indexTopLevelDecl(const Decl *D);
   bool indexDeclGroupRef(DeclGroupRef DG);
 
 private:
   bool shouldIgnoreIfImplicit(const Decl *D);
 
+  bool shouldIndexMacroOccurrence(bool IsRef, SourceLocation Loc);
+
   bool handleDeclOccurrence(const Decl *D, SourceLocation Loc,
                             bool IsRef, const Decl *Parent,
                             SymbolRoleSet Roles,
                             ArrayRef<SymbolRelation> Relations,
                             const Expr *RefE,
                             const Decl *RefD,
                             const DeclContext *ContainerDC);
 };
 
 } // end namespace index
 } // end namespace clang
 
 #endif

clang/lib/Index/IndexingContext.cpp

bool IndexingContext::handleDeclOccurrence(const Decl *D, SourceLocation Loc,
 
   IndexDataConsumer::ASTNodeInfo Node{OrigE, OrigD, Parent, ContainerDC};
   return DataConsumer.handleDeclOccurrence(D, Roles, FinalRelations, Loc, Node);
 }
 
 void IndexingContext::handleMacroDefined(const IdentifierInfo &Name,
                                          SourceLocation Loc,
                                          const MacroInfo &MI) {
+  if (!shouldIndexMacroOccurrence(/*IsRef=*/false, Loc))
+    return;
   SymbolRoleSet Roles = (unsigned)SymbolRole::Definition;
   DataConsumer.handleMacroOccurrence(&Name, &MI, Roles, Loc);
 }
 
 void IndexingContext::handleMacroUndefined(const IdentifierInfo &Name,
                                            SourceLocation Loc,
                                            const MacroInfo &MI) {
+  if (!shouldIndexMacroOccurrence(/*IsRef=*/false, Loc))
+    return;
   SymbolRoleSet Roles = (unsigned)SymbolRole::Undefinition;
   DataConsumer.handleMacroOccurrence(&Name, &MI, Roles, Loc);
 }
 
 void IndexingContext::handleMacroReference(const IdentifierInfo &Name,
                                            SourceLocation Loc,
                                            const MacroInfo &MI) {
+  if (!shouldIndexMacroOccurrence(/*IsRef=*/true, Loc))
+    return;
   SymbolRoleSet Roles = (unsigned)SymbolRole::Reference;
   DataConsumer.handleMacroOccurrence(&Name, &MI, Roles, Loc);
 }
+
+bool IndexingContext::shouldIndexMacroOccurrence(bool IsRef,
+                                                 SourceLocation Loc) {
+  if (!IndexOpts.IndexMacros)
+    return false;
+
+  switch (IndexOpts.SystemSymbolFilter) {
+  case IndexingOptions::SystemSymbolFilterKind::None:
+    break;
+  case IndexingOptions::SystemSymbolFilterKind::DeclarationsOnly:
+    if (!IsRef)
+      return true;
+    break;
+  case IndexingOptions::SystemSymbolFilterKind::All:
+    return true;
+  }
+
+  SourceManager &SM = Ctx->getSourceManager();
+  FileID FID = SM.getFileID(SM.getFileLoc(Loc));
+  if (FID.isInvalid())
+    return false;
+
+  bool Invalid = false;
+  const SrcMgr::SLocEntry &SEntry = SM.getSLocEntry(FID, &Invalid);
+  if (Invalid || !SEntry.isFile())
+    return false;
+
+  return SEntry.getFile().getFileCharacteristic() == SrcMgr::C_User;
+}

clang/lib/Index/USRGeneration.cpp

bool clang::index::generateUSRForMacro(const MacroDefinitionRecord *MD,
   return generateUSRForMacro(MD->getName()->getName(), MD->getLocation(),
                              SM, Buf);
 
 }
 
 bool clang::index::generateUSRForMacro(StringRef MacroName, SourceLocation Loc,
                                        const SourceManager &SM,
                                        SmallVectorImpl<char> &Buf) {
-  // Don't generate USRs for things with invalid locations.
-  if (MacroName.empty() || Loc.isInvalid())
+  if (MacroName.empty())
     return true;
 
   llvm::raw_svector_ostream Out(Buf);
 
   // Assume that system headers are sane.  Don't put source location
   // information into the USR if the macro comes from a system header.
-  bool ShouldGenerateLocation = !SM.isInSystemHeader(Loc);
+  bool ShouldGenerateLocation = Loc.isValid() && !SM.isInSystemHeader(Loc);
 
   Out << getUSRSpacePrefix();
   if (ShouldGenerateLocation)
     printLoc(Out, Loc, SM, /*IncludeOffset=*/true);
   Out << "@macro@";
   Out << MacroName;
   return false;
 }

clang/test/Index/Core/Inputs/module/ModA.h

 
 void ModA_func(void);
+#define MODA_MACRO 1

clang/test/Index/Core/Inputs/module/SubModA.h

 
 void SubModA_func(void);
+#define SUBMODA_MACRO 1

clang/test/Index/Core/Inputs/sys/system-head.h

 
 // CHECK: [[@LINE+3]]:7 | class/C++ | SubCls1 | [[SubCls1_USR:.*]] | {{.*}} | Def | rel: 0
 // CHECK: [[@LINE+2]]:24 | class/C++ | Cls | [[Cls_USR]] | {{.*}} | Ref,RelBase,RelCont | rel: 1
 // CHECK-NEXT: RelBase,RelCont | SubCls1 | [[SubCls1_USR]]
 class SubCls1 : public Cls {
   // CHECK-NOT: [[@LINE+1]]:3 | class/C++ | SubCls1 |
   SubCls1 *f;
 };
+
+// FIXME: this decl gets reported after the macro definitions, immediately
+// before the next declaration. Add a dummy declaration so that the checks work.
+void reset_parser();
+
+// CHECK: [[@LINE+1]]:9 | macro/C | SYSTEM_MACRO | c:@macro@SYSTEM_MACRO | Def
+#define SYSTEM_MACRO 1
+// CHECK: [[@LINE+1]]:8 | macro/C | SYSTEM_MACRO | c:@macro@SYSTEM_MACRO | Undef
+#undef SYSTEM_MACRO
+// CHECK: [[@LINE+1]]:9 | macro/C | SYSTEM_MACRO | c:@macro@SYSTEM_MACRO | Def
+#define SYSTEM_MACRO int fromSystemMacro = 1
+
+// CHECK-NOT: [[@LINE+2]]:1 | macro/C
+// CHECK: [[@LINE+1]]:1 | variable/C | fromSystemMacro
+SYSTEM_MACRO;

clang/test/Index/Core/index-macros.c

 // RUN: c-index-test core -print-source-symbols -- %s | FileCheck %s
+// RUN: c-index-test core -print-source-symbols -ignore-macros -- %s | FileCheck %s -check-prefix=DISABLED
+// DISABLED-NOT: macro/C
+// DISABLED-NOT: XI
 
-// CHECK: [[@LINE+1]]:9 | macro/C | X1 | c:index-macros.c@157@macro@X1 | Def |
+// CHECK: [[@LINE+1]]:9 | macro/C | X1 | [[X1_USR:.*@macro@X1]] | Def |
 #define X1 1
-// CHECK: [[@LINE+1]]:9 | macro/C | DEF | c:index-macros.c@251@macro@DEF | Def |
+// CHECK: [[@LINE+1]]:9 | macro/C | DEF | [[DEF_USR:.*@macro@DEF]] | Def |
 #define DEF(x) int x
-// CHECK: [[@LINE+1]]:8 | macro/C | X1 | c:index-macros.c@157@macro@X1 | Undef |
+// CHECK: [[@LINE+1]]:8 | macro/C | X1 | [[X1_USR]] | Undef |
 #undef X1
 
-// CHECK: [[@LINE+2]]:1 | macro/C | DEF | c:index-macros.c@251@macro@DEF | Ref |
+// CHECK: [[@LINE+1]]:9 | macro/C | C | [[C_USR:.*@macro@C]] | Def |
+#define C 1
+// CHECK: [[@LINE+1]]:5 | macro/C | C | [[C_USR]] | Ref |
+#if C
+#endif
+// CHECK: [[@LINE+1]]:8 | macro/C | C | [[C_USR]] | Ref |
+#ifdef C
+#endif
+// CHECK: [[@LINE+1]]:9 | macro/C | C | [[C_USR]] | Ref |
+#ifndef C
+#endif
+// CHECK: [[@LINE+1]]:13 | macro/C | C | [[C_USR]] | Ref |
+#if defined(C)
+#endif
+// CHECK: [[@LINE+1]]:14 | macro/C | C | [[C_USR]] | Ref |
+#if !defined(C)
+#endif
+
+// Nonexistent macros should not be included.
+// CHECK-NOT: NOT_DEFINED
+#ifdef NOT_DEFINED
+#endif
+#ifndef NOT_DEFINED
+#endif
+#if defined(NOT_DEFINED) && NOT_DEFINED
+#elif !defined(NOT_DEFINED)
+#endif
+
+// CHECK: [[@LINE+1]]:5 | macro/C | __LINE__ | c:@macro@__LINE__ | Ref |
+#if __LINE__ == 41
+#endif
+
+// CHECK: [[@LINE+2]]:1 | macro/C | DEF | [[DEF_USR]] | Ref |
 // CHECK: [[@LINE+1]]:5 | variable/C | i | c:@i | {{.*}} | Def | rel: 0
 DEF(i);

clang/test/Index/Core/index-with-module.m

 #include "SubSubModA.h" // CHECK: [[@LINE]]:1 | module/C | ModA.SubModA.SubSubModA | [[SubSubModA_USR]] | Decl,Impl |
 
 void foo() {
   // CHECK: [[@LINE+1]]:3 | function/C | ModA_func | c:@F@ModA_func | {{.*}} | Ref,Call,RelCall,RelCont | rel: 1
   ModA_func();
 }
 
 // CHECK: ==== Module ModA ====
-// CHECK: 2:6 | function/C | ModA_func | c:@F@ModA_func | {{.*}} | Decl | rel: 0
+// CHECK-DAG: 3:9 | macro/C | MODA_MACRO | c:{{.*}}@macro@MODA_MACRO | Def |
+// CHECK-DAG: 3:9 | macro/C | SUBMODA_MACRO | c:{{.*}}@macro@SUBMODA_MACRO | Def |
+// CHECK-DAG: 2:6 | function/C | ModA_func | c:@F@ModA_func | {{.*}} | Decl | rel: 0
+// CHECK-DAG: 2:6 | function/C | SubModA_func | c:@F@SubModA_func | {{.*}} | Decl | rel: 0
 // CHECK: ---- Module Inputs ----
 // CHECK: user | {{.*}}ModA.h
 // CHECK: user | {{.*}}module.modulemap

clang/tools/c-index-test/core_main.cpp

 
 static cl::opt<bool>
 DumpModuleImports("dump-imported-module-files",
                cl::desc("Print symbols and input files from imported modules"));
 
 static cl::opt<bool>
 IncludeLocals("include-locals", cl::desc("Print local symbols"));
 
+static cl::opt<bool> IgnoreMacros("ignore-macros",
+                                  cl::desc("Skip indexing macros"));
+
 static cl::opt<std::string>
 ModuleFilePath("module-file",
                cl::desc("Path to module file to print symbols from"));
 static cl::opt<std::string>
   ModuleFormat("fmodule-format", cl::init("raw"),
         cl::desc("Container format for clang modules and PCH, 'raw' or 'obj'"));
 
 }
Reader.visitInputFiles(Mod, /*IncludeSystem=*/true, /*Complain=*/false,
                         [&](const serialization::InputFile &IF, bool isSystem) {
     OS << (isSystem ? "system" : "user") << " | ";
     OS << IF.getFile()->getName() << '\n';
   });
 }
 
 static bool printSourceSymbols(const char *Executable,
                                ArrayRef<const char *> Args,
-                               bool dumpModuleImports, bool indexLocals) {
+                               bool dumpModuleImports, bool indexLocals,
+                               bool ignoreMacros) {
   SmallVector<const char *, 4> ArgsWithProgName;
   ArgsWithProgName.push_back(Executable);
   ArgsWithProgName.append(Args.begin(), Args.end());
   IntrusiveRefCntPtr<DiagnosticsEngine>
     Diags(CompilerInstance::createDiagnostics(new DiagnosticOptions));
   auto CInvok = createInvocationFromCommandLine(ArgsWithProgName, Diags);
   if (!CInvok)
     return true;
 
   raw_ostream &OS = outs();
   auto DataConsumer = std::make_shared<PrintIndexDataConsumer>(OS);
   IndexingOptions IndexOpts;
   IndexOpts.IndexFunctionLocals = indexLocals;
+  IndexOpts.IndexMacros = !ignoreMacros;
+  IndexOpts.IndexMacrosInPreprocessor = !ignoreMacros;
   std::unique_ptr<FrontendAction> IndexAction =
       createIndexingAction(DataConsumer, IndexOpts);
 
   auto PCHContainerOps = std::make_shared<PCHContainerOperations>();
   std::unique_ptr<ASTUnit> Unit(ASTUnit::LoadFromCompilerInvocationAction(
       std::move(CInvok), PCHContainerOps, Diags, IndexAction.get()));
 
   if (!Unit)
if (!options::ModuleFilePath.empty()) {
                                           options::ModuleFormat);
     }
     if (CompArgs.empty()) {
       errs() << "error: missing compiler args; pass '-- <compiler arguments>'\n";
       return 1;
     }
     return printSourceSymbols(Executable.c_str(), CompArgs,
                               options::DumpModuleImports,
-                              options::IncludeLocals);
+                              options::IncludeLocals, options::IgnoreMacros);
   }
 
   return 0;
 }
 
 //===----------------------------------------------------------------------===//
 // Utility functions
 //===----------------------------------------------------------------------===//

clang/unittests/Index/IndexTests.cpp

 TEST(IndexTest, Simple) {
   auto Index = std::make_shared<Indexer>();
   tooling::runToolOnCode(std::make_unique<IndexAction>(Index),
                          "class X {}; void f() {}");
   EXPECT_THAT(Index->Symbols, UnorderedElementsAre(QName("X"), QName("f")));
 }
 
 TEST(IndexTest, IndexPreprocessorMacros) {
-  std::string Code = "#define INDEX_MAC 1";
+  std::string Code = R"cpp(
+    #define INDEX_MAC 1
+    #define INDEX_MAC_UNDEF 1
+    #undef INDEX_MAC_UNDEF
+    #define INDEX_MAC_REDEF 1
+    #undef INDEX_MAC_REDEF
+    #define INDEX_MAC_REDEF 2
+  )cpp";
   auto Index = std::make_shared<Indexer>();
   IndexingOptions Opts;
   Opts.IndexMacrosInPreprocessor = true;
   tooling::runToolOnCode(std::make_unique<IndexAction>(Index, Opts), Code);
-  EXPECT_THAT(Index->Symbols, Contains(QName("INDEX_MAC")));
+  EXPECT_THAT(Index->Symbols,
+              Contains(AllOf(QName("INDEX_MAC"), WrittenAt(Position(2, 13)),
+                             DeclAt(Position(2, 13)),
+                             HasRole(SymbolRole::Definition))));
+  EXPECT_THAT(
+      Index->Symbols,
+      AllOf(Contains(AllOf(QName("INDEX_MAC_UNDEF"), WrittenAt(Position(3, 13)),
+                           DeclAt(Position(3, 13)),
+                           HasRole(SymbolRole::Definition))),
+            Contains(AllOf(QName("INDEX_MAC_UNDEF"), WrittenAt(Position(4, 12)),
+                           DeclAt(Position(3, 13)),
+                           HasRole(SymbolRole::Undefinition)))));
+  EXPECT_THAT(
+      Index->Symbols,
+      AllOf(Contains(AllOf(QName("INDEX_MAC_REDEF"), WrittenAt(Position(5, 13)),
+                           DeclAt(Position(5, 13)),
+                           HasRole(SymbolRole::Definition))),
+            Contains(AllOf(QName("INDEX_MAC_REDEF"), WrittenAt(Position(6, 12)),
+                           DeclAt(Position(5, 13)),
+                           HasRole(SymbolRole::Undefinition))),
+            Contains(AllOf(QName("INDEX_MAC_REDEF"), WrittenAt(Position(7, 13)),
+                           DeclAt(Position(7, 13)),
+                           HasRole(SymbolRole::Definition)))));
 
   Opts.IndexMacrosInPreprocessor = false;
   Index->Symbols.clear();
   tooling::runToolOnCode(std::make_unique<IndexAction>(Index, Opts), Code);
   EXPECT_THAT(Index->Symbols, UnorderedElementsAre());
 }
 
 TEST(IndexTest, IndexParametersInDecls) {