[clang][extract-api] Process only APIs declared in inputs

[dang]

Summary

We should only process APIs declared in the command line inputs to avoid
drowning the ExtractAPI output with symbols the user doesn't care about.
This is achieved by keeping track of the provided input files and
checking that the associated Decl or Macro is declared in one of those files.

[cishida]

clang/lib/ExtractAPI/ExtractAPIConsumer.cpp

67

s/externsal/external

73

How is this searching by the way the header was included? It looks like comparisons are based on file locations as KnownIncludes stores file paths.

clang/test/ExtractAPI/known_files_only.c

7

It may be useful to add a test for the headermap case to ensure the remapped location is recognized.

[zixuw]

clang/test/ExtractAPI/known_files_only.c

21

I would just include another header split from this test file but not passed into the command line to avoid pulling in extra dependencies. Makes the test self-contained and more consistent, also simpler.

167

uri not normalized.

[dang]

Address code review feedback:

• Remove unnecessary code
• Add a test to validate that everything works in the presence of header maps
• Simplify existing test
• Fix small nits

[dang]

clang/lib/ExtractAPI/ExtractAPIConsumer.cpp

73

Upon second thought and further testing all this stuff isn't needed because we always construct the input buffer using #include "ABSOLUTE_PATH". I have added a test to validate that this works when using a header map. If we ever move to relative includes, then we should change how this to use only include string comparisons (potentially caching the FileEntries as a performance optimization).

[cishida]

LGTM

[dang]

Rebase on top of latest changes.

clang/include/clang/ExtractAPI/FrontendActions.h

private:
 
   /// The product this action is extracting API information for.
   std::string ProductName;
 
   /// The synthesized input buffer that contains all the provided input header
   /// files.
   std::unique_ptr<llvm::MemoryBuffer> Buffer;
 
+  /// The input file originally provided on the command line.
+  std::vector<std::string> KnownInputFiles;
+
   /// Prepare to execute the action on the given CompilerInstance.
   ///
   /// This is called before executing the action on any inputs. This generates a
   /// single header that includes all of CI's inputs and replaces CI's input
   /// list with it before actually executing the action.
   bool PrepareToExecuteAction(CompilerInstance &CI) override;
 
   /// Called after executing the action on the synthesized input buffer.

clang/lib/ExtractAPI/ExtractAPIConsumer.cpp

 
 #include "clang/AST/ASTConsumer.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/ParentMapContext.h"
 #include "clang/AST/RawCommentList.h"
 #include "clang/AST/RecursiveASTVisitor.h"
+#include "clang/Basic/SourceLocation.h"
+#include "clang/Basic/SourceManager.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/ExtractAPI/API.h"
 #include "clang/ExtractAPI/AvailabilityInfo.h"
 #include "clang/ExtractAPI/DeclarationFragments.h"
 #include "clang/ExtractAPI/FrontendActions.h"
 #include "clang/ExtractAPI/Serialization/SymbolGraphSerializer.h"
 #include "clang/Frontend/ASTConsumers.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/FrontendOptions.h"
 #include "clang/Lex/MacroInfo.h"
 #include "clang/Lex/PPCallbacks.h"
+#include "clang/Lex/Preprocessor.h"
 #include "clang/Lex/PreprocessorOptions.h"
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/raw_ostream.h"
+#include <memory>
+#include <utility>
 
 using namespace clang;
 using namespace extractapi;
 
 namespace {
 
+struct LocationFileChecker {
+  bool isLocationInKnownFile(SourceLocation Loc) {
+    // If the loc refers to a macro expansion we need to first get the file
+    // location of the expansion.
+    auto FileLoc = SM.getFileLoc(Loc);
+    FileID FID = SM.getFileID(FileLoc);
+    if (FID.isInvalid())
+      return false;
+
+    const auto *File = SM.getFileEntryForID(FID);
+    if (!File)
+      return false;
+
+    if (KnownFileEntries.count(File))
+      return true;
+
+    return false;
+  }
+
+  LocationFileChecker(const SourceManager &SM,
+                      const std::vector<std::string> &KnownFiles)
+      : SM(SM) {
+    for (const auto &KnownFilePath : KnownFiles)
+      if (auto FileEntry = SM.getFileManager().getFile(KnownFilePath))
+        KnownFileEntries.insert(*FileEntry);
+  }
+
+private:
+  const SourceManager &SM;
+  llvm::DenseSet<const FileEntry *> KnownFileEntries;
+};
+
 /// The RecursiveASTVisitor to traverse symbol declarations and collect API
 /// information.
 class ExtractAPIVisitor : public RecursiveASTVisitor<ExtractAPIVisitor> {
 public:
-  ExtractAPIVisitor(ASTContext &Context, APISet &API)
-      : Context(Context), API(API) {}
+  ExtractAPIVisitor(ASTContext &Context, LocationFileChecker &LCF, APISet &API)
+      : Context(Context), API(API), LCF(LCF) {}
 
   const APISet &getAPI() const { return API; }
 
   bool VisitVarDecl(const VarDecl *Decl) {
     // Skip function parameters.
     if (isa<ParmVarDecl>(Decl))
       return true;
 
     // Skip non-global variables in records (struct/union/class).
     if (Decl->getDeclContext()->isRecord())
       return true;
 
     // Skip local variables inside function or method.
     if (!Decl->isDefinedOutsideFunctionOrMethod())
       return true;
 
     // If this is a template but not specialization or instantiation, skip.
     if (Decl->getASTContext().getTemplateOrSpecializationInfo(Decl) &&
         Decl->getTemplateSpecializationKind() == TSK_Undeclared)
       return true;
 
+    if (!LCF.isLocationInKnownFile(Decl->getLocation()))
+      return true;
+
     // Collect symbol information.
     StringRef Name = Decl->getName();
     StringRef USR = API.recordUSR(Decl);
     PresumedLoc Loc =
         Context.getSourceManager().getPresumedLoc(Decl->getLocation());
     AvailabilityInfo Availability = getAvailability(Decl);
     LinkageInfo Linkage = Decl->getLinkageAndVisibility();
     DocComment Comment;
case FunctionDecl::TK_FunctionTemplateSpecialization:
           return true;
       }
       break;
     case FunctionDecl::TK_FunctionTemplate:
     case FunctionDecl::TK_DependentFunctionTemplateSpecialization:
       return true;
     }
 
+    if (!LCF.isLocationInKnownFile(Decl->getLocation()))
+      return true;
+
     // Collect symbol information.
     StringRef Name = Decl->getName();
     StringRef USR = API.recordUSR(Decl);
     PresumedLoc Loc =
         Context.getSourceManager().getPresumedLoc(Decl->getLocation());
     AvailabilityInfo Availability = getAvailability(Decl);
     LinkageInfo Linkage = Decl->getLinkageAndVisibility();
     DocComment Comment;
public:
   bool VisitEnumDecl(const EnumDecl *Decl) {
     if (!Decl->isComplete())
       return true;
 
     // Skip forward declaration.
     if (!Decl->isThisDeclarationADefinition())
       return true;
 
+    if (!LCF.isLocationInKnownFile(Decl->getLocation()))
+      return true;
+
     // Collect symbol information.
     StringRef Name = Decl->getName();
     StringRef USR = API.recordUSR(Decl);
     PresumedLoc Loc =
         Context.getSourceManager().getPresumedLoc(Decl->getLocation());
     AvailabilityInfo Availability = getAvailability(Decl);
     DocComment Comment;
     if (auto *RawComment = Context.getRawCommentForDeclNoCache(Decl))
bool VisitRecordDecl(const RecordDecl *Decl) {
     if (!Decl->isCompleteDefinition())
       return true;
 
     // Skip C++ structs/classes/unions
     // TODO: support C++ records
     if (isa<CXXRecordDecl>(Decl))
       return true;
 
+    if (!LCF.isLocationInKnownFile(Decl->getLocation()))
+      return true;
+
     // Collect symbol information.
     StringRef Name = Decl->getName();
     StringRef USR = API.recordUSR(Decl);
     PresumedLoc Loc =
         Context.getSourceManager().getPresumedLoc(Decl->getLocation());
     AvailabilityInfo Availability = getAvailability(Decl);
     DocComment Comment;
     if (auto *RawComment = Context.getRawCommentForDeclNoCache(Decl))
bool VisitRecordDecl(const RecordDecl *Decl) {
     return true;
   }
 
   bool VisitObjCInterfaceDecl(const ObjCInterfaceDecl *Decl) {
     // Skip forward declaration for classes (@class)
     if (!Decl->isThisDeclarationADefinition())
       return true;
 
+    if (!LCF.isLocationInKnownFile(Decl->getLocation()))
+      return true;
+
     // Collect symbol information.
     StringRef Name = Decl->getName();
     StringRef USR = API.recordUSR(Decl);
     PresumedLoc Loc =
         Context.getSourceManager().getPresumedLoc(Decl->getLocation());
     AvailabilityInfo Availability = getAvailability(Decl);
     LinkageInfo Linkage = Decl->getLinkageAndVisibility();
     DocComment Comment;
bool VisitObjCInterfaceDecl(const ObjCInterfaceDecl *Decl) {
     return true;
   }
 
   bool VisitObjCProtocolDecl(const ObjCProtocolDecl *Decl) {
     // Skip forward declaration for protocols (@protocol).
     if (!Decl->isThisDeclarationADefinition())
       return true;
 
+    if (!LCF.isLocationInKnownFile(Decl->getLocation()))
+      return true;
+
     // Collect symbol information.
     StringRef Name = Decl->getName();
     StringRef USR = API.recordUSR(Decl);
     PresumedLoc Loc =
         Context.getSourceManager().getPresumedLoc(Decl->getLocation());
     AvailabilityInfo Availability = getAvailability(Decl);
     DocComment Comment;
     if (auto *RawComment = Context.getRawCommentForDeclNoCache(Decl))
void recordObjCProtocols(ObjCContainerRecord *Container,
                            ObjCInterfaceDecl::protocol_range Protocols) {
     for (const auto *Protocol : Protocols)
       Container->Protocols.emplace_back(Protocol->getName(),
                                         API.recordUSR(Protocol));
   }
 
   ASTContext &Context;
   APISet &API;
+  LocationFileChecker &LCF;
 };
 
 class ExtractAPIConsumer : public ASTConsumer {
 public:
-  ExtractAPIConsumer(ASTContext &Context, APISet &API)
-      : Visitor(Context, API) {}
+  ExtractAPIConsumer(ASTContext &Context,
+                     std::unique_ptr<LocationFileChecker> LCF, APISet &API)
+      : Visitor(Context, *LCF, API), LCF(std::move(LCF)) {}
 
   void HandleTranslationUnit(ASTContext &Context) override {
     // Use ExtractAPIVisitor to traverse symbol declarations in the context.
     Visitor.TraverseDecl(Context.getTranslationUnitDecl());
   }
 
 private:
   ExtractAPIVisitor Visitor;
+  std::unique_ptr<LocationFileChecker> LCF;
 };
 
 class MacroCallback : public PPCallbacks {
 public:
-  MacroCallback(const SourceManager &SM, APISet &API) : SM(SM), API(API) {}
+  MacroCallback(const SourceManager &SM, LocationFileChecker &LCF, APISet &API)
+      : SM(SM), LCF(LCF), API(API) {}
 
   void MacroDefined(const Token &MacroNameToken,
                     const MacroDirective *MD) override {
     auto *MacroInfo = MD->getMacroInfo();
 
     if (MacroInfo->isBuiltinMacro())
       return;
 
public:
 
   void EndOfMainFile() override {
     for (auto &PM : PendingMacros) {
       // `isUsedForHeaderGuard` is only set when the preprocessor leaves the
       // file so check for it here.
       if (PM.MD->getMacroInfo()->isUsedForHeaderGuard())
         continue;
 
+      if (!LCF.isLocationInKnownFile(PM.MacroNameToken.getLocation()))
+        continue;
+
       StringRef Name = PM.MacroNameToken.getIdentifierInfo()->getName();
       PresumedLoc Loc = SM.getPresumedLoc(PM.MacroNameToken.getLocation());
       StringRef USR =
           API.recordUSRForMacro(Name, PM.MacroNameToken.getLocation(), SM);
 
       API.addMacroDefinition(
           Name, USR, Loc,
           DeclarationFragmentsBuilder::getFragmentsForMacro(Name, PM.MD),
           DeclarationFragmentsBuilder::getSubHeadingForMacro(Name));
     }
 
     PendingMacros.clear();
   }
 
 private:
   struct PendingMacro {
     Token MacroNameToken;
     const MacroDirective *MD;
 
     PendingMacro(const Token &MacroNameToken, const MacroDirective *MD)
         : MacroNameToken(MacroNameToken), MD(MD) {}
   };
 
   const SourceManager &SM;
+  LocationFileChecker &LCF;
   APISet &API;
   llvm::SmallVector<PendingMacro> PendingMacros;
 };
 
 } // namespace
 
 std::unique_ptr<ASTConsumer>
 ExtractAPIAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {
   OS = CreateOutputFile(CI, InFile);
   if (!OS)
     return nullptr;
 
   ProductName = CI.getFrontendOpts().ProductName;
 
   // Now that we have enough information about the language options and the
   // target triple, let's create the APISet before anyone uses it.
   API = std::make_unique<APISet>(
       CI.getTarget().getTriple(),
       CI.getFrontendOpts().Inputs.back().getKind().getLanguage());
 
+  auto LCF = std::make_unique<LocationFileChecker>(CI.getSourceManager(),
+                                                   KnownInputFiles);
+
   // Register preprocessor callbacks that will add macro definitions to API.
   CI.getPreprocessor().addPPCallbacks(
-      std::make_unique<MacroCallback>(CI.getSourceManager(), *API));
+      std::make_unique<MacroCallback>(CI.getSourceManager(), *LCF, *API));
 
-  return std::make_unique<ExtractAPIConsumer>(CI.getASTContext(), *API);
+  return std::make_unique<ExtractAPIConsumer>(CI.getASTContext(),
+                                              std::move(LCF), *API);
 }
 
 bool ExtractAPIAction::PrepareToExecuteAction(CompilerInstance &CI) {
   auto &Inputs = CI.getFrontendOpts().Inputs;
   if (Inputs.empty())
     return true;
 
   auto Kind = Inputs[0].getKind();
 
   // Convert the header file inputs into a single input buffer.
   SmallString<256> HeaderContents;
   for (const FrontendInputFile &FIF : Inputs) {
     if (Kind.isObjectiveC())
       HeaderContents += "#import";
     else
       HeaderContents += "#include";
     HeaderContents += " \"";
     HeaderContents += FIF.getFile();
     HeaderContents += "\"\n";
+
+    KnownInputFiles.emplace_back(FIF.getFile());
   }
 
   Buffer = llvm::MemoryBuffer::getMemBufferCopy(HeaderContents,
                                                 getInputBufferName());
 
   // Set that buffer up as our "real" input in the CompilerInstance.
   Inputs.clear();
   Inputs.emplace_back(Buffer->getMemBufferRef(), Kind, /*IsSystem*/ false);

clang/test/ExtractAPI/known_files_only.c

+// RUN: rm -rf %t
+// RUN: split-file %s %t
+// RUN: sed -e "s@INPUT_DIR@%/t@g" %t/reference.output.json.in >> \
+// RUN: %t/reference.output.json
+// RUN: %clang -extract-api --product-name=GlobalRecord -target arm64-apple-macosx \
+// RUN: %t/input1.h -o %t/output.json | FileCheck -allow-empty %s
+
+// Generator version is not consistent across test runs, normalize it.
+// RUN: sed -e "s@\"generator\": \".*\"@\"generator\": \"?\"@g" \
+// RUN: %t/output.json >> %t/output-normalized.json
+// RUN: diff %t/reference.output.json %t/output-normalized.json
+
+// CHECK-NOT: error:
+// CHECK-NOT: warning:
+
+//--- input1.h
+int num;
+#include "input2.h"
+
+//--- input2.h
+// Ensure that these symbols are not emitted in the Symbol Graph.
+#define HELLO 1
+char not_emitted;
+void foo(int);
+struct Foo { int a; };
+
+//--- reference.output.json.in
+{
+  "metadata": {
+    "formatVersion": {
+      "major": 0,
+      "minor": 5,
+      "patch": 3
+    },
+    "generator": "?"
+  },
+  "module": {
+    "name": "GlobalRecord",
+    "platform": {
+      "architecture": "arm64",
+      "operatingSystem": {
+        "minimumVersion": {
+          "major": 11,
+          "minor": 0,
+          "patch": 0
+        },
+        "name": "macosx"
+      },
+      "vendor": "apple"
+    }
+  },
+  "relationhips": [],
+  "symbols": [
+    {
+      "declarationFragments": [
+        {
+          "kind": "typeIdentifier",
+          "preciseIdentifier": "c:I",
+          "spelling": "int"
+        },
+        {
+          "kind": "text",
+          "spelling": " "
+        },
+        {
+          "kind": "identifier",
+          "spelling": "num"
+        }
+      ],
+      "identifier": {
+        "interfaceLanguage": "c",
+        "precise": "c:@num"
+      },
+      "kind": {
+        "displayName": "Global Variable",
+        "identifier": "c.var"
+      },
+      "location": {
+        "character": 5,
+        "line": 1,
+        "uri": "file://INPUT_DIR/input1.h"
+      },
+      "names": {
+        "subHeading": [
+          {
+            "kind": "identifier",
+            "spelling": "num"
+          }
+        ],
+        "title": "num"
+      }
+    }
+  ]
+}

clang/test/ExtractAPI/known_files_only_hmap.c

+// RUN: rm -rf %t
+// RUN: split-file %s %t
+// RUN: sed -e "s@INPUT_DIR@%/t@g" %t/reference.output.json.in >> \
+// RUN: %t/reference.output.json
+// RUN: sed -e "s@INPUT_DIR@%/t@g" %t/known_files_only.hmap.json.in >> \
+// RUN: %t/known_files_only.hmap.json
+// RUN: %hmaptool write %t/known_files_only.hmap.json %t/known_files_only.hmap
+// RUN: %clang -extract-api --product-name=KnownFilesOnlyHmap -target arm64-apple-macosx \
+// RUN: -I%t/known_files_only.hmap -I%t/subdir %t/subdir/subdir1/input.h \
+// RUN: %t/subdir/subdir2/known_file.h -o %t/output.json | FileCheck -allow-empty %s
+
+// Generator version is not consistent across test runs, normalize it.
+// RUN: sed -e "s@\"generator\": \".*\"@\"generator\": \"?\"@g" \
+// RUN: %t/output.json >> %t/output-normalized.json
+// RUN: diff %t/reference.output.json %t/output-normalized.json
+
+// CHECK-NOT: error:
+// CHECK-NOT: warning:
+//--- known_files_only.hmap.json.in
+{
+  "mappings" :
+    {
+     "subdir2/known_file.h" : "INPUT_DIR/subdir/subdir3/unknown.h"
+    }
+}
+
+//--- subdir/subdir1/input.h
+int num;
+#include "subdir2/known_file.h"
+
+//--- subdir/subdir2/known_file.h
+int known_num;
+
+//--- subdir/subdir3/unknown.h
+// Ensure that these symbols are not emitted in the Symbol Graph.
+#ifndef INPUT4_H
+#define INPUT4_H
+
+#define HELLO 1
+char not_emitted;
+void foo(int);
+struct Foo { int a; };
+
+#endif
+
+//--- reference.output.json.in
+{
+  "metadata": {
+    "formatVersion": {
+      "major": 0,
+      "minor": 5,
+      "patch": 3
+    },
+    "generator": "?"
+  },
+  "module": {
+    "name": "KnownFilesOnlyHmap",
+    "platform": {
+      "architecture": "arm64",
+      "operatingSystem": {
+        "minimumVersion": {
+          "major": 11,
+          "minor": 0,
+          "patch": 0
+        },
+        "name": "macosx"
+      },
+      "vendor": "apple"
+    }
+  },
+  "relationhips": [],
+  "symbols": [
+    {
+      "declarationFragments": [
+        {
+          "kind": "typeIdentifier",
+          "preciseIdentifier": "c:I",
+          "spelling": "int"
+        },
+        {
+          "kind": "text",
+          "spelling": " "
+        },
+        {
+          "kind": "identifier",
+          "spelling": "num"
+        }
+      ],
+      "identifier": {
+        "interfaceLanguage": "c",
+        "precise": "c:@num"
+      },
+      "kind": {
+        "displayName": "Global Variable",
+        "identifier": "c.var"
+      },
+      "location": {
+        "character": 5,
+        "line": 1,
+        "uri": "file://INPUT_DIR/subdir/subdir1/input.h"
+      },
+      "names": {
+        "subHeading": [
+          {
+            "kind": "identifier",
+            "spelling": "num"
+          }
+        ],
+        "title": "num"
+      }
+    },
+    {
+      "declarationFragments": [
+        {
+          "kind": "typeIdentifier",
+          "preciseIdentifier": "c:I",
+          "spelling": "int"
+        },
+        {
+          "kind": "text",
+          "spelling": " "
+        },
+        {
+          "kind": "identifier",
+          "spelling": "known_num"
+        }
+      ],
+      "identifier": {
+        "interfaceLanguage": "c",
+        "precise": "c:@known_num"
+      },
+      "kind": {
+        "displayName": "Global Variable",
+        "identifier": "c.var"
+      },
+      "location": {
+        "character": 5,
+        "line": 1,
+        "uri": "file://INPUT_DIR/subdir/subdir2/known_file.h"
+      },
+      "names": {
+        "subHeading": [
+          {
+            "kind": "identifier",
+            "spelling": "known_num"
+          }
+        ],
+        "title": "known_num"
+      }
+    }
+  ]
+}