[clang][ExtractAPI] Add multiple file support to --extract-api-ignores

[Arsenic]

Summary

• Modify -extract-api-ignores command line option to accept multiple arguments
• Update APIIgnoresList to operate on a file list instead of a single file
• Add new test verifying the functionality
• fix #61242 on GitHub issue tracker

[dang]

clang/test/ExtractAPI/ignored-symbols-multifile.c

35

This test doesn't check the symbol name sorting behavior across multiple files, which is crucial APIIgnoresList::shouldIgnore(StringRef SymbolName) because it uses binary search to find the symbol in the list. Would you be able to shuffle the order of the macros so that they aren't pre-sorted?

[Arsenic]

Update clang/test/ExtractAPI/ignored-symbols-multifile.c :
Shuffle the order of ignored symbols in test to check the sorting behaviour of APIIgnoresList

[dang]

clang/test/ExtractAPI/ignored-symbols-multifile.c

28

Should this not be a 6?

[dang]

LGTM once you fix the test

[Arsenic]

fix test

clang/include/clang/Driver/Options.td

 def end_no_unused_arguments : Flag<["--"], "end-no-unused-arguments">, Flags<[CoreOption]>,
   HelpText<"Start emitting warnings for unused driver arguments">;
 def interface_stub_version_EQ : JoinedOrSeparate<["-"], "interface-stub-version=">, Flags<[CC1Option]>;
 def exported__symbols__list : Separate<["-"], "exported_symbols_list">;
 def extract_api : Flag<["-"], "extract-api">, Flags<[CC1Option]>, Group<Action_Group>,
   HelpText<"Extract API information">;
 def product_name_EQ: Joined<["--"], "product-name=">, Flags<[CC1Option]>,
   MarshallingInfoString<FrontendOpts<"ProductName">>;
-def extract_api_ignores_EQ: Joined<["--"], "extract-api-ignores=">, Flags<[CC1Option]>,
-    HelpText<"File containing a new line separated list of API symbols to ignore when extracting API information.">,
-    MarshallingInfoString<FrontendOpts<"ExtractAPIIgnoresFile">>;
+def extract_api_ignores_EQ: CommaJoined<["--"], "extract-api-ignores=">, Flags<[CC1Option]>,
+    HelpText<"Comma separated list of files containing a new line separated list of API symbols to ignore when extracting API information.">,
+    MarshallingInfoStringVector<FrontendOpts<"ExtractAPIIgnoresFileList">>;
 def e : JoinedOrSeparate<["-"], "e">, Flags<[LinkerInput]>, Group<Link_Group>;
 def fmax_tokens_EQ : Joined<["-"], "fmax-tokens=">, Group<f_Group>, Flags<[CC1Option]>,
   HelpText<"Max total number of preprocessed tokens for -Wmax-tokens.">,
   MarshallingInfoInt<LangOpts<"MaxTokens">>;
 def fPIC : Flag<["-"], "fPIC">, Group<f_Group>;
 def fno_PIC : Flag<["-"], "fno-PIC">, Group<f_Group>;
 def fPIE : Flag<["-"], "fPIE">, Group<f_Group>;
 def fno_PIE : Flag<["-"], "fno-PIE">, Group<f_Group>;

clang/include/clang/ExtractAPI/APIIgnoresList.h

 //===- ExtractAPI/APIIgnoresList.h ---------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
 /// \file This file defines APIIgnoresList which is a type that allows querying
-/// a file containing symbols to ignore when extracting API information.
+/// files containing symbols to ignore when extracting API information.
 ///
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_API_IGNORES_LIST_H
 #define LLVM_CLANG_API_IGNORES_LIST_H
 
 #include "clang/Basic/FileManager.h"
 #include "llvm/ADT/SmallVector.h"
struct IgnoresFileNotFound : public llvm::ErrorInfo<IgnoresFileNotFound> {
   virtual void log(llvm::raw_ostream &os) const override;
 
   virtual std::error_code convertToErrorCode() const override;
 };
 
 /// A type that provides access to a new line separated list of symbol names to
 /// ignore when extracting API information.
 struct APIIgnoresList {
-  /// The API to use for generating from the file at \p IgnoresFilePath.
+  using FilePathList = std::vector<std::string>;
+
+  /// The API to use for generating from the files at \p IgnoresFilePathList.
   ///
   /// \returns an initialized APIIgnoresList or an Error.
-  static llvm::Expected<APIIgnoresList> create(llvm::StringRef IgnoresFilePath,
-                                               FileManager &FM);
+  static llvm::Expected<APIIgnoresList>
+  create(const FilePathList &IgnoresFilePathList, FileManager &FM);
 
   APIIgnoresList() = default;
 
   /// Check if \p SymbolName is specified in the APIIgnoresList and if it should
   /// therefore be ignored.
   bool shouldIgnore(llvm::StringRef SymbolName) const;
 
 private:
   using SymbolNameList = llvm::SmallVector<llvm::StringRef, 32>;
+  using BufferList = llvm::SmallVector<std::unique_ptr<llvm::MemoryBuffer>>;
 
-  APIIgnoresList(SymbolNameList SymbolsToIgnore,
-                 std::unique_ptr<llvm::MemoryBuffer> Buffer)
-      : SymbolsToIgnore(std::move(SymbolsToIgnore)), Buffer(std::move(Buffer)) {
-  }
+  APIIgnoresList(SymbolNameList SymbolsToIgnore, BufferList Buffers)
+      : SymbolsToIgnore(std::move(SymbolsToIgnore)),
+        Buffers(std::move(Buffers)) {}
 
   SymbolNameList SymbolsToIgnore;
-  std::unique_ptr<llvm::MemoryBuffer> Buffer;
+  BufferList Buffers;
 };
 
 } // namespace extractapi
 } // namespace clang
 
 #endif // LLVM_CLANG_API_IGNORES_LIST_H

clang/include/clang/Frontend/FrontendOptions.h

public:
   /// The name of the action to run when using a plugin action.
   std::string ActionName;
 
   // Currently this is only used as part of the `-extract-api` action.
   /// The name of the product the input files belong too.
   std::string ProductName;
 
   // Currently this is only used as part of the `-extract-api` action.
-  /// The file providing a list of APIs to ignore when extracting documentation
-  std::string ExtractAPIIgnoresFile;
+  // A comma seperated list of files providing a list of APIs to
+  // ignore when extracting documentation.
+  std::vector<std::string> ExtractAPIIgnoresFileList;
 
   /// Args to pass to the plugins
   std::map<std::string, std::vector<std::string>> PluginArgs;
 
   /// The list of plugin actions to run in addition to the normal action.
   std::vector<std::string> AddPluginActions;
 
   /// The list of plugins to load.

clang/lib/ExtractAPI/APIIgnoresList.cpp

 void IgnoresFileNotFound::log(llvm::raw_ostream &os) const {
   os << "Could not find API ignores file " << Path;
 }
 
 std::error_code IgnoresFileNotFound::convertToErrorCode() const {
   return llvm::inconvertibleErrorCode();
 }
 
-Expected<APIIgnoresList> APIIgnoresList::create(StringRef IgnoresFilePath,
-                                                FileManager &FM) {
-  auto BufferOrErr = FM.getBufferForFile(IgnoresFilePath);
+Expected<APIIgnoresList>
+APIIgnoresList::create(const FilePathList &IgnoresFilePathList,
+  SmallVector<StringRef, 32> Lines;
+  BufferList symbolBufferList;
+
+  for (const auto &CurrentIgnoresFilePath : IgnoresFilePathList) {
+    auto BufferOrErr = FM.getBufferForFile(CurrentIgnoresFilePath);
+
     if (!BufferOrErr)
-    return make_error<IgnoresFileNotFound>(IgnoresFilePath);
+      return make_error<IgnoresFileNotFound>(CurrentIgnoresFilePath);
 
     auto Buffer = std::move(BufferOrErr.get());
-  SmallVector<StringRef, 32> Lines;
-  Buffer->getBuffer().split(Lines, '\n', /*MaxSplit*/ -1, /*KeepEmpty*/ false);
-  // Symbol names don't have spaces in them, let's just remove these in case the
-  // input is slighlty malformed.
+    Buffer->getBuffer().split(Lines, '\n', /*MaxSplit*/ -1,
+                              /*KeepEmpty*/ false);
+    symbolBufferList.push_back(std::move(Buffer));
+  }
+
+  // Symbol names don't have spaces in them, let's just remove these in case
+  // the input is slighlty malformed.
   transform(Lines, Lines.begin(), [](StringRef Line) { return Line.trim(); });
   sort(Lines);
-  return APIIgnoresList(std::move(Lines), std::move(Buffer));
+  return APIIgnoresList(std::move(Lines), std::move(symbolBufferList));
 }
 
 bool APIIgnoresList::shouldIgnore(StringRef SymbolName) const {
   auto It = lower_bound(SymbolsToIgnore, SymbolName);
   return (It != SymbolsToIgnore.end()) && (*It == SymbolName);
 }

clang/lib/ExtractAPI/ExtractAPIConsumer.cpp

ExtractAPIAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {
   CI.getPreprocessor().addPPCallbacks(std::make_unique<MacroCallback>(
       CI.getSourceManager(), *LCF, *API, CI.getPreprocessor()));
 
   // Do not include location in anonymous decls.
   PrintingPolicy Policy = CI.getASTContext().getPrintingPolicy();
   Policy.AnonymousTagLocations = false;
   CI.getASTContext().setPrintingPolicy(Policy);
 
-  if (!CI.getFrontendOpts().ExtractAPIIgnoresFile.empty()) {
+  if (!CI.getFrontendOpts().ExtractAPIIgnoresFileList.empty()) {
     llvm::handleAllErrors(
-        APIIgnoresList::create(CI.getFrontendOpts().ExtractAPIIgnoresFile,
+        APIIgnoresList::create(CI.getFrontendOpts().ExtractAPIIgnoresFileList,
                                CI.getFileManager())
             .moveInto(IgnoresList),
         [&CI](const IgnoresFileNotFound &Err) {
           CI.getDiagnostics().Report(
               diag::err_extract_api_ignores_file_not_found)
               << Err.Path;
         });
   }

clang/test/ExtractAPI/ignored-symbols-multifile.c

+// RUN: rm -rf %t
+// RUN: split-file %s %t
+// RUN: %clang_cc1 -extract-api -triple arm64-apple-macosx \
+// RUN:   --extract-api-ignores=%t/ignores-list1,%t/ignores-list2,%t/ignores-list3 \
+// RUN:   -x c-header %t/input.h -verify -o - | FileCheck %t/input.h
+
+//--- input.h
+#define IGNORED_6_FILE1 6
+#define IGNORED_2_FILE1 2
+#define IGNORED_5_FILE1 5
+
+#define IGNORED_4_FILE2 4
+#define IGNORED_3_FILE2 3
+
+typedef double IGNORED_1_FILE3;
+typedef int IGNORED_7_FILE3;
+
+typedef float NonIgnored;
+
+// CHECK-NOT: IGNORED_6_FILE1
+// CHECK-NOT: IGNORED_2_FILE1
+// CHECK-NOT: IGNORED_5_FILE1
+
+// CHECK-NOT: IGNORED_4_FILE2
+// CHECK-NOT: IGNORED_3_FILE2
+
+// CHECK-NOT: IGNORED_1_FILE3
+// CHECK-NOT: IGNORED_7_FILE3
+// CHECK: NonIgnored
+
+// expected-no-diagnostics
+
+//--- ignores-list1
+IGNORED_6_FILE1
+IGNORED_2_FILE1
+IGNORED_5_FILE1
+
+//--- ignores-list2
+IGNORED_4_FILE2
+IGNORED_3_FILE2
+
+//--- ignores-list3
+IGNORED_1_FILE3
+IGNORED_7_FILE3