[include-cleaner] add macro symbols and implicit refs to HTML report

[sammccall]

Summary

Demo: http://htmlpreview.github.io/?https://gist.githubusercontent.com/sam-mccall/fec9b77c726cfb3cc7c424b197e3f68c/raw/8c5c5ad927b0485db76c34c25250b0ae8f7637f8/ASTTests.html

[hokein]

clang-tools-extra/include-cleaner/lib/HTMLReport.cpp

243

It seems noisy to print it even there is only one candidate, maybe exclude it if there is only one candidate?

[sammccall]

• hover card follows the internal data model: symbol -> location -> header (->include, later)
• don't use public API, so we can get at internal details

[sammccall]

format

[sammccall]

Updated the demo link, PTAL (significant changes since last review)

[hokein]

The new UI looks good.

clang-tools-extra/include-cleaner/lib/Analysis.cpp

24 ↗

(On Diff #475068)

nit: mention the writeHTMLReport function explicitly.

clang-tools-extra/include-cleaner/lib/HTMLReport.cpp

303

We need a special logic (if (auto SS = Recognizer(&ND)) {...}) for the stdlib symbols, otherwise, they will be treated as normal Decl symbols.

clang-tools-extra/include-cleaner/lib/AnalysisInternal.h

 ///
 /// walkAST is typically called once per top-level declaration in the file
 /// being analyzed, in order to find all references within it.
 void walkAST(Decl &Root,
              llvm::function_ref<void(SourceLocation, NamedDecl &, RefType)>);
 
 /// Write an HTML summary of the analysis to the given stream.
 /// FIXME: Once analysis has a public API, this should be public too.
-void writeHTMLReport(FileID File, llvm::ArrayRef<Decl *> Roots, ASTContext &Ctx,
+void writeHTMLReport(FileID File, llvm::ArrayRef<Decl *> Roots,
+                     llvm::ArrayRef<SymbolReference> MacroRefs, ASTContext &Ctx,
                      llvm::raw_ostream &OS);
 
 } // namespace include_cleaner
 } // namespace clang
 
 #endif

clang-tools-extra/include-cleaner/lib/HTMLReport.cpp

 // be able to identify specific facts about the code and the inferences made.
 //
 // This library prints an annotated version of the code
 //
 //===----------------------------------------------------------------------===//
 
 #include "AnalysisInternal.h"
 #include "clang-include-cleaner/Analysis.h"
+#include "clang-include-cleaner/Types.h"
 #include "clang/AST/ASTContext.h"
-#include "clang/AST/Decl.h"
 #include "clang/AST/PrettyPrinter.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Lex/Lexer.h"
+#include "llvm/Support/ScopedPrinter.h"
 #include "llvm/Support/raw_ostream.h"
 
 namespace clang::include_cleaner {
 namespace {
 
 constexpr llvm::StringLiteral CSS = R"css(
   body { margin: 0; }
   pre { line-height: 1.5em; counter-reset: line; margin: 0; }
   pre .line { counter-increment: line; }
   pre .line::before {
     content: counter(line);
     display: inline-block;
     background-color: #eee; border-right: 1px solid #ccc;
     text-align: right;
     width: 3em; padding-right: 0.5em; margin-right: 0.5em;
   }
   .ref { text-decoration: underline; color: #008; }
   .sel { position: relative; cursor: pointer; }
+  .ref.implicit { background-color: #ff8; }
   #hover {
     background-color: #aaccff; border: 1px solid black;
     z-index: 1;
     position: absolute; top: 100%; left: 0;
     font-family: sans-serif;
     padding: 0.5em;
   }
   #hover p, #hover pre { margin: 0; }
   #hover section header { font-weight: bold; }
+  #hover section.implicit { background-color: #bbb; }
+  #hover section.ambiguous { background-color: #caf; }
   #hover section:not(:first-child) { margin-top: 1em; }
 )css";
 
 constexpr llvm::StringLiteral JS = R"js(
   // Recreate the #hover div inside whichever target .sel element was clicked.
   function select(event) {
     var target = event.target.closest('.sel');
     var hover = document.getElementById('hover');
function fillHover(hover, target) {
     target.dataset.hover?.split(',').forEach(function(id) {
       for (c of document.getElementById(id).content.childNodes)
         hover.appendChild(c.cloneNode(true));
     })
   }
 )js";
 
 // Print the declaration tersely, but enough to identify e.g. which overload.
-std::string printDecl(const NamedDecl &ND) {
+std::string printSymbol(const Symbol &Sym) {
   std::string S;
   llvm::raw_string_ostream OS(S);
-  PrintingPolicy PP = ND.getASTContext().getPrintingPolicy();
+  switch (Sym.kind()) {
+  case Symbol::Declaration: {
+    const auto &D = Sym.declaration();
+    PrintingPolicy PP = D.getASTContext().getPrintingPolicy();
     PP.FullyQualifiedName = true;
     PP.TerseOutput = true;
     PP.SuppressInitializers = true;
-  ND.print(OS, PP);
+    D.print(OS, PP);
+    break;
+  }
+  case Symbol::Macro:
+    OS << "#define " << Sym.macro().Name->getName() << " ...";
+    break;
+  case Symbol::Standard:
+    OS << "(defined in stdlib)";
+    break;
+  }
   llvm::erase_value(S, '\n');
   return S;
 }
 
+// Categorize the symbol, like FunctionDecl or Macro
+llvm::StringRef describeSymbol(const Symbol &Sym) {
+  switch (Sym.kind()) {
+  case Symbol::Declaration:
+    return Sym.declaration().getDeclKindName();
+  case Symbol::Macro:
+    return "Macro";
+  case Symbol::Standard:
+    return "Standard symbol";
+  }
+  llvm_unreachable("unhandled symbol kind");
+}
+
+// Categorize the symbol, like FunctionDecl or Macro
+std::string locateSymbol(const Symbol &Sym, const SourceManager &SM) {
+  switch (Sym.kind()) {
+  case Symbol::Declaration:
+    return Sym.declaration().getLocation().printToString(SM);
+  case Symbol::Macro:
+    return Sym.macro().Definition.printToString(SM);
+  case Symbol::Standard:
+    return ("stdlib header " + Sym.standard().header().name()).str();
+  }
+  llvm_unreachable("unhandled symbol kind");
+}
+
+llvm::StringRef refType(RefType T) {
+  switch (T) {
+  case RefType::Explicit:
+    return "";
+  case RefType::Implicit:
+    return "implicit";
+  case RefType::Ambiguous:
+    return "ambiguous";
+  }
+}
+
 class Reporter {
   llvm::raw_ostream &OS;
   const ASTContext &Ctx;
   const SourceManager &SM;
   FileID File;
 
   // Symbols that are referenced from the main file.
+  // FIXME: should we deduplicate these?
   struct Target {
-    const NamedDecl *D;
+    Symbol Sym;
+    RefType Type;
+    SmallVector<Header> Providers;
   };
   std::vector<Target> Targets;
   // Points within the main file that reference a Target.
-  std::vector<std::pair</*Offset*/ unsigned, /*TargetIndex*/ unsigned>> Refs;
+  // Implicit refs will be marked with a symbol just before the token.
+  struct Ref {
+    unsigned Offset;
+    bool Implicit;
+    size_t TargetIndex;
+    bool operator<(const Ref &Other) const {
+      return std::forward_as_tuple(Offset, !Implicit, TargetIndex) <
+             std::forward_as_tuple(Other.Offset, !Other.Implicit, TargetIndex);
+    }
+  };
+  std::vector<Ref> Refs;
 
 public:
   Reporter(llvm::raw_ostream &OS, ASTContext &Ctx, FileID File)
       : OS(OS), Ctx(Ctx), SM(Ctx.getSourceManager()), File(File) {}
 
-  void addRef(SourceLocation Loc, const NamedDecl &D) {
-    auto [File, Offset] = SM.getDecomposedLoc(SM.getFileLoc(Loc));
+  void addRef(const SymbolReference &SR, llvm::ArrayRef<Header> Headers) {
+    auto [File, Offset] = SM.getDecomposedLoc(SM.getFileLoc(SR.RefLocation));
     if (File != this->File) {
       // Can get here e.g. if there's an #include inside a root Decl.
       // FIXME: do something more useful than this.
-      llvm::errs() << "Ref location outside file! "
-                   << D.getQualifiedNameAsString() << " at "
-                   << Loc.printToString(SM) << "\n";
+      llvm::errs() << "Ref location outside file! " << SR.Target << " at "
+                   << SR.RefLocation.printToString(SM) << "\n";
       return;
     }
-    Targets.push_back({&D});
-    Refs.push_back({Offset, Targets.size() - 1});
+    Targets.push_back({SR.Target, SR.RT, {Headers.begin(), Headers.end()}});
+    Refs.push_back({Offset, SR.RT == RefType::Implicit, Targets.size() - 1});
   }
 
   void write() {
     OS << "<!doctype html>\n";
     OS << "<html>\n";
     OS << "<head>\n";
     OS << "<style>" << CSS << "</style>\n";
     OS << "<script>" << JS << "</script>\n";
     for (unsigned I = 0; I < Targets.size(); ++I) {
-      OS << "<template id='t" << I << "'><section>";
+      OS << "<template id='t" << I << "'><section class='"
+         << refType(Targets[I].Type) << "'>";
       writeTarget(Targets[I]);
       OS << "</section></template>\n";
     }
     OS << "</head>\n";
     OS << "<body>\n";
     writeCode();
     OS << "</body>\n";
     OS << "</html>\n";
private:
   }
 
   void escapeString(llvm::StringRef S) {
     for (char C : S)
       escapeChar(C);
   }
 
   void writeTarget(const Target &T) {
-    OS << "<header>" << T.D->getDeclKindName() << " ";
-    escapeString(T.D->getQualifiedNameAsString());
-    OS << "</header>";
+    OS << "<header>";
+    if (T.Type != RefType::Explicit)
+      OS << refType(T.Type) << " ";
+    OS << describeSymbol(T.Sym) << " ";
+    escapeString(llvm::to_string(T.Sym));
+    OS << "</header>\n";
 
     OS << "<p>declared at ";
-    escapeString(SM.getFileLoc(T.D->getLocation()).printToString(SM));
-    OS << "</p><pre>";
-    escapeString(printDecl(*T.D));
-    OS << "</pre>";
+    escapeString(locateSymbol(T.Sym, SM));
+    OS << "</p>\n";
+
+    OS << "<pre>";
+    escapeString(printSymbol(T.Sym));
+    OS << "</pre>\n";
+
+    OS << "<ul>";
+    for (Header H : T.Providers) {
+      OS << "Candidate: ";
+      escapeString(llvm::to_string(H));
+    }
+    OS << "</ul>\n";
   }
 
   void writeCode() {
     llvm::sort(Refs);
     llvm::StringRef Code = SM.getBufferData(File);
 
     OS << "<pre onclick='select(event)' class='code'>";
     OS << "<code class='line'>";
     auto Rest = llvm::makeArrayRef(Refs);
     unsigned End = 0;
     for (unsigned I = 0; I < Code.size(); ++I) {
       // Finish refs early at EOL to avoid dealing with splitting the span.
       if (End && (End == I || Code[I] == '\n')) {
         OS << "</span>";
         End = 0;
       }
+      // Handle implicit refs, which are rendered *before* the token.
+      while (!Rest.empty() && Rest.front().Offset == I &&
+             Rest.front().Implicit) {
+        const Ref &R = Rest.front();
+        OS << "<span class='ref sel implicit' data-hover='t" << R.TargetIndex
+           << "'>&loz;</span>";
+        Rest = Rest.drop_front();
+      };
+      // Accumulate all explicit refs that appear on the same token.
       std::string TargetList;
-      Rest = Rest.drop_while([&](auto &R) {
-        if (R.first != I)
+      Rest = Rest.drop_while([&](const Ref &R) {
+        if (R.Offset != I)
           return false;
         if (!TargetList.empty())
           TargetList.push_back(',');
         TargetList.push_back('t');
-        TargetList.append(std::to_string(R.second));
+        TargetList.append(std::to_string(R.TargetIndex));
         return true;
       });
       if (!TargetList.empty()) {
         assert(End == 0 && "Overlapping tokens!");
         OS << "<span class='ref sel' data-hover='" << TargetList << "'>";
         End = I + Lexer::MeasureTokenLength(SM.getComposedLoc(File, I), SM,
                                             Ctx.getLangOpts());
       }
       if (Code[I] == '\n')
         OS << "</code>\n<code class='line'>";
       else
         escapeChar(Code[I]);
     }
     OS << "</code></pre>\n";
   }
 };
 
 } // namespace
 
-void writeHTMLReport(FileID File, llvm::ArrayRef<Decl *> Roots, ASTContext &Ctx,
+void writeHTMLReport(FileID File, llvm::ArrayRef<Decl *> Roots,
+                     llvm::ArrayRef<SymbolReference> MacroRefs, ASTContext &Ctx,
                      llvm::raw_ostream &OS) {
   Reporter R(OS, Ctx, File);
-  for (Decl *Root : Roots)
-    walkAST(*Root, [&](SourceLocation Loc, const NamedDecl &D, RefType) {
-      R.addRef(Loc, D);
+  walkUsed(Roots, MacroRefs, Ctx.getSourceManager(),
+           [&](const SymbolReference &Ref, llvm::ArrayRef<Header> Headers) {
+             R.addRef(Ref, Headers);
            });
   R.write();
 }
 
 } // namespace clang::include_cleaner

clang-tools-extra/include-cleaner/lib/Types.cpp

 
 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Symbol &S) {
   switch (S.kind()) {
   case Symbol::Declaration:
     if (const auto *ND = llvm::dyn_cast<NamedDecl>(&S.declaration()))
       return OS << ND->getNameAsString();
     return OS << S.declaration().getDeclKindName();
   case Symbol::Macro:
-    return OS << S.macro().Name;
+    return OS << S.macro().Name->getName();
   case Symbol::Standard:
     return OS << S.standard().scope() << S.standard().name();
   }
   llvm_unreachable("Unhandled Symbol kind");
 }
 
 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Header &H) {
   switch (H.kind()) {

clang-tools-extra/include-cleaner/tool/IncludeCleaner.cpp

cl::opt<std::string> HTMLReportPath{
     "html",
     cl::desc("Specify an output filename for an HTML report. "
              "This describes both recommendations and reasons for changes."),
     cl::cat(IncludeCleaner),
 };
 
 class HTMLReportAction : public clang::ASTFrontendAction {
   RecordedAST AST;
+  RecordedPP PP;
+
+  void ExecuteAction() override {
+    auto &P = getCompilerInstance().getPreprocessor();
+    P.addPPCallbacks(PP.record(P));
+    ASTFrontendAction::ExecuteAction();
+  }
 
   std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
                                                  StringRef File) override {
     return AST.record();
   }
 
   void EndSourceFile() override {
     std::error_code EC;
     llvm::raw_fd_ostream OS(HTMLReportPath, EC);
     if (EC) {
       llvm::errs() << "Unable to write HTML report to " << HTMLReportPath
                    << ": " << EC.message() << "\n";
       exit(1);
     }
     writeHTMLReport(AST.Ctx->getSourceManager().getMainFileID(), AST.Roots,
-                    *AST.Ctx, OS);
+                    PP.MacroReferences, *AST.Ctx, OS);
   }
 };
 
 } // namespace
 } // namespace include_cleaner
 } // namespace clang
 
 int main(int argc, const char **argv) {