[find-all-symbol] Add macro support.

[hokein]

Fix code-style.

[ioeric]

include-fixer/find-all-symbols/FindAllMacros.h

23

nit: "A preprocessor that collects..."

37

nit: "allowing clients to include..."

40

This comment does not introduce useful information IMO.

include-fixer/find-all-symbols/SymbolReporter.h

19

This doesn't make sense to me?

Also, I think it should be "An interface for classes that/to collect symbols" grammatically.

[hokein]

Address comments.

[bkramer]

In the future I'd prefer to do renaming changes (ResultReporter->SymbolReporter) in a separate change, but this is fine now.

include-fixer/find-all-symbols/SymbolReporter.h

1

find all symbols?

[hokein]

Correct the header comments.

[hokein]

Remove an unneeded file.

include-fixer/find-all-symbols/CMakeLists.txt

 set(LLVM_LINK_COMPONENTS
   Support
   )
 
 add_clang_library(findAllSymbols
   FindAllSymbols.cpp
+  FindAllMacros.cpp
   PragmaCommentHandler.cpp
   SymbolInfo.cpp
 
   LINK_LIBS
   clangAST
   clangASTMatchers
   clangBasic
   clangLex
   )
 
 add_subdirectory(tool)

include-fixer/find-all-symbols/FindAllMacros.h

+//===-- FindAllMacros.h - find all symbols----------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_FIND_ALL_SYMBOLS_FIND_ALL_MACROS_H
+#define LLVM_CLANG_TOOLS_EXTRA_FIND_ALL_SYMBOLS_FIND_ALL_MACROS_H
+
+#include "SymbolInfo.h"
+#include "SymbolReporter.h"
+#include "clang/Lex/PPCallbacks.h"
+
+namespace clang {
+namespace find_all_symbols {
+
+class HeaderMapCollector;
+
+/// \brief A preprocessor that collects all macro symbols.
+/// The contexts of a macro will be ignored since they are not available during
+/// preprocessing period.
+class FindAllMacros : public clang::PPCallbacks {
+public:
+  explicit FindAllMacros(SymbolReporter *Reporter,
+                         HeaderMapCollector *Collector, SourceManager *SM)
+      : Reporter(Reporter), Collector(Collector), SM(SM) {}
+
+  void MacroDefined(const Token &MacroNameTok,
+                    const MacroDirective *MD) override;
+
+private:
+  // Reporter for SymbolInfo.
+  SymbolReporter *const Reporter;
+  // A remapping header file collector allowing clients to include a different
+  // header.
+  HeaderMapCollector *const Collector;
+
+  SourceManager *const SM;
+};
+
+} // namespace find_all_symbols
+} // namespace clang
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_FIND_ALL_SYMBOLS_FIND_ALL_MACROS_H

include-fixer/find-all-symbols/FindAllMacros.cpp

+//===--- FindAllMacros.cpp - find all symbols------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "FindAllMacros.h"
+#include "HeaderMapCollector.h"
+#include "SymbolInfo.h"
+#include "clang/Basic/IdentifierTable.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Lex/Token.h"
+
+namespace clang {
+namespace find_all_symbols {
+
+void FindAllMacros::MacroDefined(const Token &MacroNameTok,
+                                 const MacroDirective *MD) {
+  SourceLocation Loc = SM->getExpansionLoc(MacroNameTok.getLocation());
+  if (Loc.isInvalid() || SM->isInMainFile(Loc))
+    return;
+
+  llvm::StringRef FilePath = SM->getFilename(Loc);
+  if (FilePath.empty())
+    return;
+
+  // Check pragma remapping header.
+  auto HeaderMappingTable = Collector->getHeaderMappingTable();
+  auto Iter = HeaderMappingTable.find(FilePath);
+  if (Iter != HeaderMappingTable.end())
+    FilePath = Iter->second;
+
+  SymbolInfo Symbol(MacroNameTok.getIdentifierInfo()->getName(),
+                    SymbolInfo::SymbolKind::Macro, FilePath.str(),
+                    SM->getSpellingLineNumber(Loc), {});
+
+  Reporter->reportSymbol(SM->getFileEntryForID(SM->getMainFileID())->getName(),
+                         Symbol);
+}
+
+} // namespace find_all_symbols
+} // namespace clang

include-fixer/find-all-symbols/FindAllSymbols.h

 //===-- FindAllSymbols.h - find all symbols----------------------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_TOOLS_EXTRA_FIND_ALL_SYMBOLS_SYMBOL_MATCHER_H
 #define LLVM_CLANG_TOOLS_EXTRA_FIND_ALL_SYMBOLS_SYMBOL_MATCHER_H
 
 #include "SymbolInfo.h"
+#include "SymbolReporter.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
 #include <string>
 
 namespace clang {
 namespace find_all_symbols {
 
 class HeaderMapCollector;
 
 /// \brief FindAllSymbols collects all classes, free standing functions and
 /// global variables with some extra information such as the path of the header
 /// file, the namespaces they are contained in, the type of variables and the
 /// parameter types of functions.
 ///
 /// NOTE:
 ///   - Symbols declared in main files are not collected since they can not be
 ///   included.
 ///   - Member functions are not collected because accessing them must go
 ///   through the class. #include fixer only needs the class name to find
 ///   headers.
 ///
 class FindAllSymbols : public clang::ast_matchers::MatchFinder::MatchCallback {
 public:
-  class ResultReporter {
-  public:
-    virtual ~ResultReporter() = default;
-
-    virtual void reportResult(llvm::StringRef FileName,
-                              const SymbolInfo &Symbol) = 0;
-  };
-
-  explicit FindAllSymbols(ResultReporter *Reporter,
+  explicit FindAllSymbols(SymbolReporter *Reporter,
                           HeaderMapCollector *Collector)
       : Reporter(Reporter), Collector(Collector) {}
 
   void registerMatchers(clang::ast_matchers::MatchFinder *MatchFinder);
 
   void
   run(const clang::ast_matchers::MatchFinder::MatchResult &result) override;
 
 private:
   // Reporter for SymbolInfo.
-  ResultReporter *const Reporter;
+  SymbolReporter *const Reporter;
   // A remapping header file collector allowing clients include a different
   // header.
   HeaderMapCollector *const Collector;
 };
 
 } // namespace find_all_symbols
 } // namespace clang
 
 #endif // LLVM_CLANG_TOOLS_EXTRA_FIND_ALL_SYMBOLS_SYMBOL_MATCHER_H

include-fixer/find-all-symbols/FindAllSymbols.cpp

void FindAllSymbols::run(const MatchFinder::MatchResult &Result) {
 
   const NamedDecl *ND = Result.Nodes.getNodeAs<NamedDecl>("decl");
   assert(ND && "Matched declaration must be a NamedDecl!");
   const SourceManager *SM = Result.SourceManager;
 
   llvm::Optional<SymbolInfo> Symbol =
       CreateSymbolInfo(ND, *SM, Collector->getHeaderMappingTable());
   if (Symbol)
-    Reporter->reportResult(
+    Reporter->reportSymbol(
         SM->getFileEntryForID(SM->getMainFileID())->getName(), *Symbol);
 }
 
 } // namespace find_all_symbols
 } // namespace clang

include-fixer/find-all-symbols/SymbolInfo.h

public:
   /// \brief The SymbolInfo Type.
   enum class SymbolKind {
     Function,
     Class,
     Variable,
     TypedefName,
     EnumDecl,
     EnumConstantDecl,
+    Macro,
     Unknown,
   };
 
   /// \brief The Context Type.
   enum class ContextType {
     Namespace, // Symbols declared in a namespace.
     Record,    // Symbols declared in a class.
     EnumDecl,  // Enum constants declared in a enum declaration.

include-fixer/find-all-symbols/SymbolInfo.cpp

 template <> struct ScalarEnumerationTraits<SymbolKind> {
   static void enumeration(IO &io, SymbolKind &value) {
     io.enumCase(value, "Variable", SymbolKind::Variable);
     io.enumCase(value, "Function", SymbolKind::Function);
     io.enumCase(value, "Class", SymbolKind::Class);
     io.enumCase(value, "TypedefName", SymbolKind::TypedefName);
     io.enumCase(value, "EnumDecl", SymbolKind::EnumDecl);
     io.enumCase(value, "EnumConstantDecl", SymbolKind::EnumConstantDecl);
+    io.enumCase(value, "Macro", SymbolKind::Macro);
     io.enumCase(value, "Unknown", SymbolKind::Unknown);
   }
 };
 
 template <> struct MappingTraits<SymbolInfo::Context> {
   static void mapping(IO &io, SymbolInfo::Context &Context) {
     io.mapRequired("ContextType", Context.first);
     io.mapRequired("ContextName", Context.second);

include-fixer/find-all-symbols/SymbolReporter.h

+//===-- SymbolReporter.h - find all symbols----------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_FIND_ALL_SYMBOLS_SYMBOL_REPORTER_H
+#define LLVM_CLANG_TOOLS_EXTRA_FIND_ALL_SYMBOLS_SYMBOL_REPORTER_H
+
+#include "SymbolInfo.h"
+
+namespace clang {
+namespace find_all_symbols {
+
+/// \brief An interface for classes that collect symbols.
+class SymbolReporter {
+public:
+  virtual ~SymbolReporter() = default;
+
+  virtual void reportSymbol(llvm::StringRef FileName,
+                            const SymbolInfo &Symbol) = 0;
+};
+
+} // namespace find_all_symbols
+} // namespace clang
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_FIND_ALL_SYMBOLS_SYMBOL_REPORTER_H

include-fixer/find-all-symbols/tool/FindAllSymbolsMain.cpp

 //===-- FindAllSymbolsMain.cpp --------------------------------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
+#include "FindAllMacros.h"
 #include "FindAllSymbols.h"
 #include "HeaderMapCollector.h"
 #include "PragmaCommentHandler.h"
 #include "SymbolInfo.h"
+#include "SymbolReporter.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
 #include "clang/ASTMatchers/ASTMatchers.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/FrontendActions.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Tooling/CommonOptionsParser.h"
 #include "clang/Tooling/Tooling.h"
 #include "llvm/ADT/ArrayRef.h"

 static cl::opt<std::string> MergeDir("merge-dir", cl::desc(R"(
 The directory for merging symbols.)"),
                                      cl::init(""),
                                      cl::cat(FindAllSymbolsCategory));
 
 namespace clang {
 namespace find_all_symbols {
 
-class YamlReporter
-    : public clang::find_all_symbols::FindAllSymbols::ResultReporter {
+class YamlReporter : public clang::find_all_symbols::SymbolReporter {
 public:
   ~YamlReporter() override {}
 
-  void reportResult(StringRef FileName, const SymbolInfo &Symbol) override {
+  void reportSymbol(StringRef FileName, const SymbolInfo &Symbol) override {
     Symbols[FileName].insert(Symbol);
   }
 
   void Write(const std::string &Dir) {
     for (const auto &Symbol : Symbols) {
       int FD;
       SmallString<128> ResultPath;
       llvm::sys::fs::createUniqueFile(
           Dir + "/" + llvm::sys::path::filename(Symbol.first) + "-%%%%%%.yaml",
           FD, ResultPath);
       llvm::raw_fd_ostream OS(FD, /*shouldClose=*/true);
       WriteSymbolInfosToStream(OS, Symbol.second);
     }
   }
 
 private:
   std::map<std::string, std::set<SymbolInfo>> Symbols;
 };
 
+// FIXME: Move this out from the main file, make it reusable in unittest.
 class FindAllSymbolsAction : public clang::ASTFrontendAction {
 public:
   FindAllSymbolsAction()
       : Reporter(), MatchFinder(), Collector(), Handler(&Collector),
         Matcher(&Reporter, &Collector) {
     Matcher.registerMatchers(&MatchFinder);
   }
 
   std::unique_ptr<clang::ASTConsumer>
   CreateASTConsumer(clang::CompilerInstance &Compiler,
                     StringRef InFile) override {
     Compiler.getPreprocessor().addCommentHandler(&Handler);
+    Compiler.getPreprocessor().addPPCallbacks(llvm::make_unique<FindAllMacros>(
+        &Reporter, &Collector, &Compiler.getSourceManager()));
     return MatchFinder.newASTConsumer();
   }
 
   void EndSourceFileAction() override { Reporter.Write(OutputDir); }
 
 private:
   YamlReporter Reporter;
   clang::ast_matchers::MatchFinder MatchFinder;

unittests/include-fixer/find-all-symbols/FindAllSymbolsTests.cpp

 //===-- FindAllSymbolsTests.cpp - find all symbols unit tests -------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
+#include "FindAllMacros.h"
 #include "FindAllSymbols.h"
 #include "HeaderMapCollector.h"
 #include "PragmaCommentHandler.h"
 #include "SymbolInfo.h"
+#include "SymbolReporter.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/FileSystemOptions.h"
 #include "clang/Basic/VirtualFileSystem.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/PCHContainerOperations.h"
 #include "clang/Tooling/Tooling.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "gtest/gtest.h"
 #include <memory>
 #include <string>
 #include <vector>
 
 namespace clang {
 namespace find_all_symbols {
 
 static const char HeaderName[] = "symbols.h";
 
-class MockReporter
-    : public clang::find_all_symbols::FindAllSymbols::ResultReporter {
+class TestSymbolReporter : public clang::find_all_symbols::SymbolReporter {
 public:
-  ~MockReporter() override {}
+  ~TestSymbolReporter() override {}
 
-  void reportResult(llvm::StringRef FileName,
+  void reportSymbol(llvm::StringRef FileName,
                     const SymbolInfo &Symbol) override {
     Symbols.push_back(Symbol);
   }
 
-  bool hasSymbol(const SymbolInfo &Symbol) {
+  bool hasSymbol(const SymbolInfo &Symbol) const {
     for (const auto &S : Symbols) {
       if (S == Symbol)
         return true;
     }
     return false;
   }
 
 private:
   std::vector<SymbolInfo> Symbols;
 };
 
 class TestFindAllSymbolsAction : public clang::ASTFrontendAction {
 public:
-  TestFindAllSymbolsAction(FindAllSymbols::ResultReporter *Reporter)
-      : MatchFinder(), Collector(), Handler(&Collector),
+  TestFindAllSymbolsAction(SymbolReporter *Reporter)
+      : Reporter(Reporter), MatchFinder(), Collector(), Handler(&Collector),
         Matcher(Reporter, &Collector) {
     Matcher.registerMatchers(&MatchFinder);
   }
 
   std::unique_ptr<clang::ASTConsumer>
   CreateASTConsumer(clang::CompilerInstance &Compiler,
                     StringRef InFile) override {
     Compiler.getPreprocessor().addCommentHandler(&Handler);
+    Compiler.getPreprocessor().addPPCallbacks(llvm::make_unique<FindAllMacros>(
+        Reporter, &Collector, &Compiler.getSourceManager()));
     return MatchFinder.newASTConsumer();
   }
 
 private:
+  SymbolReporter *const Reporter;
   ast_matchers::MatchFinder MatchFinder;
   HeaderMapCollector Collector;
   PragmaCommentHandler Handler;
   FindAllSymbols Matcher;
 };
 
 class TestFindAllSymbolsActionFactory
     : public clang::tooling::FrontendActionFactory {
 public:
-  TestFindAllSymbolsActionFactory(MockReporter *Reporter)
+  TestFindAllSymbolsActionFactory(TestSymbolReporter *Reporter)
       : Reporter(Reporter) {}
   clang::FrontendAction *create() override {
     return new TestFindAllSymbolsAction(Reporter);
   }
 
 private:
-  MockReporter *const Reporter;
+  TestSymbolReporter *const Reporter;
 };
 
 class FindAllSymbolsTest : public ::testing::Test {
 public:
   bool hasSymbol(const SymbolInfo &Symbol) {
     return Reporter.hasSymbol(Symbol);
   }
 
bool runFindAllSymbols(StringRef Code) {
     std::string Content = "#include\"" + std::string(HeaderName) + "\"";
     InMemoryFileSystem->addFile(FileName, 0,
                                 llvm::MemoryBuffer::getMemBuffer(Content));
     Invocation.run();
     return true;
   }
 
 private:
-  MockReporter Reporter;
+  TestSymbolReporter Reporter;
 };
 
 TEST_F(FindAllSymbolsTest, VariableSymbols) {
   static const char Code[] = R"(
       extern int xargc;
       namespace na {
       static bool SSSS = false;
       namespace nb { const long long *XXXX; }
TEST_F(FindAllSymbolsTest, IWYUPrivatePragmaTest) {
   )";
   runFindAllSymbols(Code);
 
   SymbolInfo Symbol =
       SymbolInfo("Bar", SymbolInfo::SymbolKind::Class, "bar.h", 3, {});
   EXPECT_TRUE(hasSymbol(Symbol));
 }
 
+TEST_F(FindAllSymbolsTest, MacroTest) {
+  static const char Code[] = R"(
+    #define X
+    #define Y 1
+    #define MAX(X, Y) ((X) > (Y) ? (X) : (Y))
+  )";
+  runFindAllSymbols(Code);
+  SymbolInfo Symbol =
+      SymbolInfo("X", SymbolInfo::SymbolKind::Macro, HeaderName, 2, {});
+  EXPECT_TRUE(hasSymbol(Symbol));
+
+  Symbol = SymbolInfo("Y", SymbolInfo::SymbolKind::Macro, HeaderName, 3, {});
+  EXPECT_TRUE(hasSymbol(Symbol));
+
+  Symbol = SymbolInfo("MAX", SymbolInfo::SymbolKind::Macro, HeaderName, 4, {});
+  EXPECT_TRUE(hasSymbol(Symbol));
+}
+
+TEST_F(FindAllSymbolsTest, MacroTestWithIWYU) {
+  static const char Code[] = R"(
+    // IWYU pragma: private, include "bar.h"
+    #define X
+    #define Y 1
+    #define MAX(X, Y) ((X) > (Y) ? (X) : (Y))
+  )";
+  runFindAllSymbols(Code);
+  SymbolInfo Symbol =
+      SymbolInfo("X", SymbolInfo::SymbolKind::Macro, "bar.h", 3, {});
+  EXPECT_TRUE(hasSymbol(Symbol));
+
+  Symbol = SymbolInfo("Y", SymbolInfo::SymbolKind::Macro, "bar.h", 4, {});
+  EXPECT_TRUE(hasSymbol(Symbol));
+
+  Symbol = SymbolInfo("MAX", SymbolInfo::SymbolKind::Macro, "bar.h", 5, {});
+  EXPECT_TRUE(hasSymbol(Symbol));
+}
+
 } // namespace find_all_symbols
 } // namespace clang