[include-cleaner] Add a simple heuristic to handle token-pasting symbols.

[hokein]

Summary

include-cleaner didn't report the usage of token-pasting symbols, as
these symbols are spelled in a "scratch space" file which is not a real
file, thus we have some false postives of unused includes for these
symbols (e.g. ABSL_FLAG).

This patch adds a simple heuristic to handle this case, we use the
expansion location as a "proxy" reference location, this should work
well on token-pasting formed from macro arguemtns.

clang-tools-extra/include-cleaner/lib/Analysis.cpp

 void walkUsed(llvm::ArrayRef<Decl *> ASTRoots,
               llvm::ArrayRef<SymbolReference> MacroRefs,
               const PragmaIncludes *PI, const SourceManager &SM,
               UsedSymbolCB CB) {
   // This is duplicated in writeHTMLReport, changes should be mirrored there.
   tooling::stdlib::Recognizer Recognizer;
   for (auto *Root : ASTRoots) {
     walkAST(*Root, [&](SourceLocation Loc, NamedDecl &ND, RefType RT) {
+      // If the symbol is spelled as a token paste, it spelling location points
+      // to <scratch space> file which is not a real file. We fallback to use
+      // the expansion location, this herustic is based on the assumption that
+      // most of token pastings are formed with the macro arguement, and it can
+      // allow us to detect uses of symbol defined by the common macro like
+      // `ABSL_FLAG`.
+      if (SM.isWrittenInScratchSpace(SM.getSpellingLoc(Loc)))
+        Loc = SM.getExpansionLoc(Loc);
+
       auto FID = SM.getFileID(SM.getSpellingLoc(Loc));
       if (FID != SM.getMainFileID() && FID != SM.getPreambleFileID())
         return;
       // FIXME: Most of the work done here is repetitive. It might be useful to
       // have a cache/batching.
       SymbolReference SymRef{ND, Loc, RT};
       return CB(SymRef, headersForSymbol(ND, SM, PI));
     });

clang-tools-extra/include-cleaner/lib/HTMLReport.cpp

void writeHTMLReport(FileID File, const include_cleaner::Includes &Includes,
                      llvm::ArrayRef<Decl *> Roots,
                      llvm::ArrayRef<SymbolReference> MacroRefs, ASTContext &Ctx,
                      const HeaderSearch &HS, PragmaIncludes *PI,
                      llvm::raw_ostream &OS) {
   Reporter R(OS, Ctx, HS, Includes, PI, File);
   const auto& SM = Ctx.getSourceManager();
   for (Decl *Root : Roots)
     walkAST(*Root, [&](SourceLocation Loc, const NamedDecl &D, RefType T) {
+      if (SM.isWrittenInScratchSpace( SM.getSpellingLoc(Loc)))
+        Loc = SM.getExpansionLoc(Loc);
+
       if(!SM.isWrittenInMainFile(SM.getSpellingLoc(Loc)))
         return;
       R.addRef(SymbolReference{D, Loc, T});
     });
   for (const SymbolReference &Ref : MacroRefs) {
     if (!SM.isWrittenInMainFile(SM.getSpellingLoc(Ref.RefLocation)))
       continue;
     R.addRef(Ref);
   }
   R.write();
 }
 
 } // namespace clang::include_cleaner

clang-tools-extra/include-cleaner/unittests/AnalysisTest.cpp

EXPECT_THAT(
                RefType::Explicit}}),
       UnorderedElementsAre(
           Pair(Code.point("1"), UnorderedElementsAre(HdrFile)),
           Pair(Code.point("2"), UnorderedElementsAre(HdrFile)),
           Pair(Code.point("3"), UnorderedElementsAre(MainFile)),
           Pair(Code.point("4"), UnorderedElementsAre(MainFile))));
 }
 
+TEST_F(WalkUsedTest, TokenPasting) {
+  llvm::Annotations Code(R"cpp(
+    #define DEFINE_FLAG(name) int FLAG_##name = 0;
+    #define MY_FLAG(name) DEFINE_FLAG(MY_##name)
+
+    #define PASTED_TOKEN X##2
+
+    $1^DEFINE_FLAG(abc);
+    $2^MY_FLAG(abc);
+
+    int $3^TPASTED_TOKEN = 3;
+  )cpp");
+  Inputs.Code = Code.code();
+
+  TestAST AST(Inputs);
+  auto &SM = AST.sourceManager();
+  auto MainFile = Header(SM.getFileEntryForID(SM.getMainFileID()));
+
+  EXPECT_THAT(offsetToProviders(AST, SM),
+              UnorderedElementsAre(
+                  Pair(Code.point("1"), UnorderedElementsAre(MainFile)),
+                  Pair(Code.point("2"), UnorderedElementsAre(MainFile)),
+                  Pair(Code.point("3"), UnorderedElementsAre(MainFile))));
+}
+
 class AnalyzeTest : public testing::Test {
 protected:
   TestInputs Inputs;
   PragmaIncludes PI;
   RecordedPP PP;
   AnalyzeTest() {
     Inputs.MakeAction = [this] {
       struct Hook : public SyntaxOnlyAction {