[clang][ExtractAPI] Allow users to specify a list of symbols to ignore

[dang]

Summary

Adds a --extract-api-ignores= command line option that allows users to
provide a file containing a new line separated list of symbols to
unconditionally ignore when extracting API information.

clang/include/clang/Basic/DiagnosticFrontendKinds.td

def warn_profile_data_unprofiled : Warning<
   InGroup<ProfileInstrUnprofiled>;
 def warn_profile_data_misexpect : Warning<
   "Potential performance regression from use of __builtin_expect(): "
   "Annotation was correct on %0 of profiled executions.">,
   BackendInfo,
   InGroup<MisExpect>;
 } // end of instrumentation issue category
 
+def err_extract_api_ignores_file_not_found :
+  Error<"file '%0' specified by '--extract-api-ignores=' not found">, DefaultFatal;
+
 }

clang/include/clang/Driver/Options.td

 def end_no_unused_arguments : Flag<["--"], "end-no-unused-arguments">, Flags<[CoreOption]>,
   HelpText<"Start emitting warnings for unused driver arguments">;
 def interface_stub_version_EQ : JoinedOrSeparate<["-"], "interface-stub-version=">, Flags<[CC1Option]>;
 def exported__symbols__list : Separate<["-"], "exported_symbols_list">;
 def extract_api : Flag<["-"], "extract-api">, Flags<[CC1Option]>, Group<Action_Group>,
   HelpText<"Extract API information">;
 def product_name_EQ: Joined<["--"], "product-name=">, Flags<[CC1Option]>,
   MarshallingInfoString<FrontendOpts<"ProductName">>;
+def extract_api_ignores_EQ: Joined<["--"], "extract-api-ignores=">, Flags<[CC1Option]>,
+    HelpText<"File containing a new line separated list of API symbols to ignore when extracting API information.">,
+    MarshallingInfoString<FrontendOpts<"ExtractAPIIgnoresFile">>;
 def e : JoinedOrSeparate<["-"], "e">, Flags<[LinkerInput]>, Group<Link_Group>;
 def fmax_tokens_EQ : Joined<["-"], "fmax-tokens=">, Group<f_Group>, Flags<[CC1Option]>,
   HelpText<"Max total number of preprocessed tokens for -Wmax-tokens.">,
   MarshallingInfoInt<LangOpts<"MaxTokens">>;
 def fPIC : Flag<["-"], "fPIC">, Group<f_Group>;
 def fno_PIC : Flag<["-"], "fno-PIC">, Group<f_Group>;
 def fPIE : Flag<["-"], "fPIE">, Group<f_Group>;
 def fno_PIE : Flag<["-"], "fno-PIE">, Group<f_Group>;

clang/include/clang/ExtractAPI/APIIgnoresList.h

+//===- ExtractAPI/APIIgnoresList.h ---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file This file defines APIIgnoresList which is a type that allows querying
+/// a file containing symbols to ignore when extracting API information.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_API_IGNORES_LIST_H
+#define LLVM_CLANG_API_IGNORES_LIST_H
+
+#include "clang/Basic/FileManager.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include <memory>
+#include <system_error>
+
+namespace llvm {
+class MemoryBuffer;
+} // namespace llvm
+
+namespace clang {
+namespace extractapi {
+
+struct IgnoresFileNotFound : public llvm::ErrorInfo<IgnoresFileNotFound> {
+  std::string Path;
+  static char ID;
+
+  explicit IgnoresFileNotFound(StringRef Path) : Path(Path) {}
+
+  virtual void log(llvm::raw_ostream &os) const override;
+
+  virtual std::error_code convertToErrorCode() const override;
+};
+
+/// A type that provides access to a new line separated list of symbol names to
+/// ignore when extracting API information.
+struct APIIgnoresList {
+  /// The API to use for generating from the file at \p IgnoresFilePath.
+  ///
+  /// \returns an initialized APIIgnoresList or an Error.
+  static llvm::Expected<APIIgnoresList> create(llvm::StringRef IgnoresFilePath,
+                                               FileManager &FM);
+
+  APIIgnoresList() = default;
+
+  /// Check if \p SymbolName is specified in the APIIgnoresList and if it should
+  /// therefore be ignored.
+  bool shouldIgnore(llvm::StringRef SymbolName) const;
+
+private:
+  using SymbolNameList = llvm::SmallVector<llvm::StringRef, 32>;
+
+  APIIgnoresList(SymbolNameList SymbolsToIgnore,
+                 std::unique_ptr<llvm::MemoryBuffer> Buffer)
+      : SymbolsToIgnore(std::move(SymbolsToIgnore)), Buffer(std::move(Buffer)) {
+  }
+
+  SymbolNameList SymbolsToIgnore;
+  std::unique_ptr<llvm::MemoryBuffer> Buffer;
+};
+
+} // namespace extractapi
+} // namespace clang
+
+#endif // LLVM_CLANG_API_IGNORES_LIST_H

clang/include/clang/ExtractAPI/FrontendActions.h

 /// This file defines the ExtractAPIAction frontend action.
 ///
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_EXTRACTAPI_FRONTEND_ACTIONS_H
 #define LLVM_CLANG_EXTRACTAPI_FRONTEND_ACTIONS_H
 
 #include "clang/ExtractAPI/API.h"
+#include "clang/ExtractAPI/APIIgnoresList.h"
 #include "clang/Frontend/FrontendAction.h"
 
 namespace clang {
 
 /// ExtractAPIAction sets up the output file and creates the ExtractAPIVisitor.
 class ExtractAPIAction : public ASTFrontendAction {
 protected:
   std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
                                                  StringRef InFile) override;
 
 private:
   /// A representation of the APIs this action extracts.
   std::unique_ptr<extractapi::APISet> API;
 
   /// A stream to the output file of this action.
   std::unique_ptr<raw_pwrite_stream> OS;
 
   /// The product this action is extracting API information for.
   std::string ProductName;
 
   /// The synthesized input buffer that contains all the provided input header
   /// files.
   std::unique_ptr<llvm::MemoryBuffer> Buffer;
 
+  /// The list of symbols to ignore during serialization
+  extractapi::APIIgnoresList IgnoresList;
+
   /// The input file originally provided on the command line.
   ///
   /// This captures the spelling used to include the file and whether the
   /// include is quoted or not.
   SmallVector<std::pair<SmallString<32>, bool>> KnownInputFiles;
 
   /// Prepare to execute the action on the given CompilerInstance.
   ///

clang/include/clang/ExtractAPI/Serialization/SerializerBase.h

 /// This file defines the ExtractAPI APISerializer interface.
 ///
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_EXTRACTAPI_SERIALIZATION_SERIALIZERBASE_H
 #define LLVM_CLANG_EXTRACTAPI_SERIALIZATION_SERIALIZERBASE_H
 
 #include "clang/ExtractAPI/API.h"
+#include "clang/ExtractAPI/APIIgnoresList.h"
 #include "llvm/Support/raw_ostream.h"
 
 namespace clang {
 namespace extractapi {
 
 /// Common options to customize the serializer output.
 struct APISerializerOption {
   /// Do not include unnecessary whitespaces to save space.

 protected:
   const APISet &API;
 
   /// The product name of API.
   ///
   /// Note: This should be used for populating metadata about the API.
   StringRef ProductName;
 
+  /// The list of symbols to ignore.
+  ///
+  /// Note: This should be consulted before emitting a symbol.
+  const APIIgnoresList &IgnoresList;
+
   APISerializerOption Options;
 
 public:
   APISerializer() = delete;
   APISerializer(const APISerializer &) = delete;
   APISerializer(APISerializer &&) = delete;
   APISerializer &operator=(const APISerializer &) = delete;
   APISerializer &operator=(APISerializer &&) = delete;
 
 protected:
   APISerializer(const APISet &API, StringRef ProductName,
+                const APIIgnoresList &IgnoresList,
                 APISerializerOption Options = {})
-      : API(API), ProductName(ProductName), Options(Options) {}
+      : API(API), ProductName(ProductName), IgnoresList(IgnoresList),
+        Options(Options) {}
 
   virtual ~APISerializer() = default;
 };
 
 } // namespace extractapi
 } // namespace clang
 
 #endif // LLVM_CLANG_EXTRACTAPI_SERIALIZATION_SERIALIZERBASE_H

clang/include/clang/ExtractAPI/Serialization/SymbolGraphSerializer.h

 /// See https://github.com/apple/swift-docc-symbolkit.
 ///
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_EXTRACTAPI_SERIALIZATION_SYMBOLGRAPHSERIALIZER_H
 #define LLVM_CLANG_EXTRACTAPI_SERIALIZATION_SYMBOLGRAPHSERIALIZER_H
 
 #include "clang/ExtractAPI/API.h"
+#include "clang/ExtractAPI/APIIgnoresList.h"
 #include "clang/ExtractAPI/Serialization/SerializerBase.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/JSON.h"
 #include "llvm/Support/VersionTuple.h"
 #include "llvm/Support/raw_ostream.h"
 
 namespace clang {
 namespace extractapi {
private:
   ///
   /// \param Component The component to push onto the path components stack.
   /// \return A PathComponentGuard responsible for removing the latest
   /// component from the stack on scope exit.
   [[nodiscard]] PathComponentGuard makePathComponentGuard(StringRef Component);
 
 public:
   SymbolGraphSerializer(const APISet &API, StringRef ProductName,
+                        const APIIgnoresList &IgnoresList,
                         APISerializerOption Options = {})
-      : APISerializer(API, ProductName, Options) {}
+      : APISerializer(API, ProductName, IgnoresList, Options) {}
 };
 
 } // namespace extractapi
 } // namespace clang
 
 #endif // LLVM_CLANG_EXTRACTAPI_SERIALIZATION_SYMBOLGRAPHSERIALIZER_H

clang/include/clang/Frontend/FrontendOptions.h

public:
 
   /// The name of the action to run when using a plugin action.
   std::string ActionName;
 
   // Currently this is only used as part of the `-extract-api` action.
   /// The name of the product the input files belong too.
   std::string ProductName;
 
+  // Currently this is only used as part of the `-extract-api` action.
+  /// The file providing a list of APIs to ignore when extracting documentation
+  std::string ExtractAPIIgnoresFile;
+
   /// Args to pass to the plugins
   std::map<std::string, std::vector<std::string>> PluginArgs;
 
   /// The list of plugin actions to run in addition to the normal action.
   std::vector<std::string> AddPluginActions;
 
   /// The list of plugins to load.
   std::vector<std::string> Plugins;

clang/lib/Driver/ToolChains/Clang.cpp

void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   } else if (isa<VerifyPCHJobAction>(JA)) {
     CmdArgs.push_back("-verify-pch");
   } else if (isa<ExtractAPIJobAction>(JA)) {
     assert(JA.getType() == types::TY_API_INFO &&
            "Extract API actions must generate a API information.");
     CmdArgs.push_back("-extract-api");
     if (Arg *ProductNameArg = Args.getLastArg(options::OPT_product_name_EQ))
       ProductNameArg->render(Args, CmdArgs);
+    if (Arg *ExtractAPIIgnoresFileArg =
+            Args.getLastArg(options::OPT_extract_api_ignores_EQ))
+      ExtractAPIIgnoresFileArg->render(Args, CmdArgs);
   } else {
     assert((isa<CompileJobAction>(JA) || isa<BackendJobAction>(JA)) &&
            "Invalid action for clang tool.");
     if (JA.getType() == types::TY_Nothing) {
       CmdArgs.push_back("-fsyntax-only");
     } else if (JA.getType() == types::TY_LLVM_IR ||
                JA.getType() == types::TY_LTO_IR) {
       CmdArgs.push_back("-emit-llvm");

clang/lib/ExtractAPI/APIIgnoresList.cpp

+//===- ExtractAPI/APIIgnoresList.cpp -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file implements APIIgnoresList that allows users to specifiy a file
+/// containing symbols to ignore during API extraction.
+///
+//===----------------------------------------------------------------------===//
+
+#include "clang/ExtractAPI/APIIgnoresList.h"
+#include "clang/Basic/FileManager.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/Error.h"
+
+using namespace clang;
+using namespace clang::extractapi;
+using namespace llvm;
+
+char IgnoresFileNotFound::ID;
+
+void IgnoresFileNotFound::log(llvm::raw_ostream &os) const {
+  os << "Could not find API ignores file " << Path;
+}
+
+std::error_code IgnoresFileNotFound::convertToErrorCode() const {
+  return llvm::inconvertibleErrorCode();
+}
+
+Expected<APIIgnoresList> APIIgnoresList::create(StringRef IgnoresFilePath,
+                                                FileManager &FM) {
+  auto BufferOrErr = FM.getBufferForFile(IgnoresFilePath);
+  if (!BufferOrErr)
+    return make_error<IgnoresFileNotFound>(IgnoresFilePath);
+
+  auto Buffer = std::move(BufferOrErr.get());
+  SmallVector<StringRef, 32> Lines;
+  Buffer->getBuffer().split(Lines, '\n', /*MaxSplit*/ -1, /*KeepEmpty*/ false);
+  // Symbol names don't have spaces in them, let's just remove these in case the
+  // input is slighlty malformed.
+  transform(Lines, Lines.begin(), [](StringRef Line) { return Line.trim(); });
+  sort(Lines);
+  return APIIgnoresList(std::move(Lines), std::move(Buffer));
+}
+
+bool APIIgnoresList::shouldIgnore(StringRef SymbolName) const {
+  auto It = lower_bound(SymbolsToIgnore, SymbolName);
+  return (It != SymbolsToIgnore.end()) && (*It == SymbolName);
+}

clang/lib/ExtractAPI/CMakeLists.txt

 set(LLVM_LINK_COMPONENTS
   Support
   )
 
 add_clang_library(clangExtractAPI
   API.cpp
+  APIIgnoresList.cpp
   AvailabilityInfo.cpp
   ExtractAPIConsumer.cpp
   DeclarationFragments.cpp
   Serialization/SerializerBase.cpp
   Serialization/SymbolGraphSerializer.cpp
   TypedefUnderlyingTypeResolver.cpp
 
   LINK_LIBS
   clangAST
   clangBasic
   clangFrontend
   clangIndex
   clangLex
   )

clang/lib/ExtractAPI/ExtractAPIConsumer.cpp

 #include "TypedefUnderlyingTypeResolver.h"
 #include "clang/AST/ASTConsumer.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/ParentMapContext.h"
 #include "clang/AST/RawCommentList.h"
 #include "clang/AST/RecursiveASTVisitor.h"
+#include "clang/Basic/DiagnosticFrontend.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/ExtractAPI/API.h"
+#include "clang/ExtractAPI/APIIgnoresList.h"
 #include "clang/ExtractAPI/AvailabilityInfo.h"
 #include "clang/ExtractAPI/DeclarationFragments.h"
 #include "clang/ExtractAPI/FrontendActions.h"
 #include "clang/ExtractAPI/Serialization/SymbolGraphSerializer.h"
 #include "clang/Frontend/ASTConsumers.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/FrontendOptions.h"
 #include "clang/Lex/MacroInfo.h"
 #include "clang/Lex/PPCallbacks.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Lex/PreprocessorOptions.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/Error.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Regex.h"
 #include "llvm/Support/raw_ostream.h"
 #include <memory>
 #include <utility>
 
ExtractAPIAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {
   CI.getPreprocessor().addPPCallbacks(std::make_unique<MacroCallback>(
       CI.getSourceManager(), *LCF, *API, CI.getPreprocessor()));
 
   // Do not include location in anonymous decls.
   PrintingPolicy Policy = CI.getASTContext().getPrintingPolicy();
   Policy.AnonymousTagLocations = false;
   CI.getASTContext().setPrintingPolicy(Policy);
 
+  if (!CI.getFrontendOpts().ExtractAPIIgnoresFile.empty()) {
+    llvm::handleAllErrors(
+        APIIgnoresList::create(CI.getFrontendOpts().ExtractAPIIgnoresFile,
+                               CI.getFileManager())
+            .moveInto(IgnoresList),
+        [&CI](const IgnoresFileNotFound &Err) {
+          CI.getDiagnostics().Report(
+              diag::err_extract_api_ignores_file_not_found)
+              << Err.Path;
+        });
+  }
+
   return std::make_unique<ExtractAPIConsumer>(CI.getASTContext(),
                                               std::move(LCF), *API);
 }
 
 bool ExtractAPIAction::PrepareToExecuteAction(CompilerInstance &CI) {
   auto &Inputs = CI.getFrontendOpts().Inputs;
   if (Inputs.empty())
     return true;

 
 void ExtractAPIAction::EndSourceFileAction() {
   if (!OS)
     return;
 
   // Setup a SymbolGraphSerializer to write out collected API information in
   // the Symbol Graph format.
   // FIXME: Make the kind of APISerializer configurable.
-  SymbolGraphSerializer SGSerializer(*API, ProductName);
+  SymbolGraphSerializer SGSerializer(*API, ProductName, IgnoresList);
   SGSerializer.serialize(*OS);
   OS.reset();
 }
 
 std::unique_ptr<raw_pwrite_stream>
 ExtractAPIAction::CreateOutputFile(CompilerInstance &CI, StringRef InFile) {
   std::unique_ptr<raw_pwrite_stream> OS =
       CI.createDefaultOutputFile(/*Binary=*/false, InFile, /*Extension=*/"json",
                                  /*RemoveFileOnSignal=*/false);
   if (!OS)
     return nullptr;
   return OS;
 }

clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp

 //===- ExtractAPI/Serialization/SymbolGraphSerializer.cpp -------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
 /// \file
 /// This file implements the SymbolGraphSerializer.
 ///
 //===----------------------------------------------------------------------===//
 
 #include "clang/ExtractAPI/Serialization/SymbolGraphSerializer.h"
 #include "clang/Basic/Version.h"
 #include "clang/ExtractAPI/API.h"
+#include "clang/ExtractAPI/APIIgnoresList.h"
 #include "clang/ExtractAPI/DeclarationFragments.h"
 #include "llvm/Support/JSON.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/VersionTuple.h"
 #include <type_traits>
 
 using namespace clang;
 using namespace clang::extractapi;
Object SymbolGraphSerializer::serializeModule() const {
   // The user is expected to always pass `--product-name=` on the command line
   // to populate this field.
   Module["name"] = ProductName;
   serializeObject(Module, "platform", serializePlatform(API.getTarget()));
   return Module;
 }
 
 bool SymbolGraphSerializer::shouldSkip(const APIRecord &Record) const {
+  // Skip explicitly ignored symbols.
+  if (IgnoresList.shouldIgnore(Record.Name))
+    return true;
+
   // Skip unconditionally unavailable symbols
   if (Record.Availabilities.isUnconditionallyUnavailable())
     return true;
 
   // Filter out symbols prefixed with an underscored as they are understood to
   // be symbols clients should not use.
   if (Record.Name.startswith("_"))
     return true;

clang/test/Driver/extract-api-unknown-ignore-diag.h

+// RUN: rm -rf %t
+// RUN: not %clang -target x86_64-unknown-unknown -extract-api --extract-api-ignores=does-not-exist %s 2>&1 | FileCheck %s
+
+// CHECK: fatal error: file 'does-not-exist' specified by '--extract-api-ignores=' not found
+
+void dummy_function(void);

clang/test/ExtractAPI/ignored-symbols.c

+// RUN: rm -rf %t
+// RUN: split-file %s %t
+// RUN: %clang_cc1 -extract-api -triple arm64-apple-macosx \
+// RUN:   --extract-api-ignores=%t/ignores-list            \
+// RUN:   -x c-header %t/input.h -verify -o - | FileCheck %t/input.h
+
+//--- input.h
+#define IGNORED_1 1
+#define IGNORED_2 2
+#define IGNORED_3 3
+#define IGNORED_4 4
+typedef int Ignored;
+typedef float NonIgnored;
+
+// CHECK-NOT: IGNORED_1
+// CHECK-NOT: IGNORED_2
+// CHECK-NOT: IGNORED_3
+// CHECK: NonIgnored
+
+// expected-no-diagnostics
+
+//--- ignores-list
+Ignored
+IGNORED_4
+IGNORED_3   
+IGNORED_2
+IGNORED_1