[clang][extract-api] Enable processing of multiple headers

[dang]

Summary

Before actually executing the ExtractAPIAction, clear the
CompilationInstance's input list and replace it with a single
synthesized file that just includes (or imports in ObjC) all the inputs.

Depends on D122141

[dang]

This is up as a "draft" right now. I am waiting on https://reviews.llvm.org/D122160 to land before doing a final rebase.

[ributzka]

clang/include/clang/SymbolGraph/FrontendActions.h

35

This comment confused me a bit. By looking at the code further down, it only adds include statements for each header, which is different from merging all headers into a single header/buffer.

[dang]

Rebase on top of latest changes.

[dang]

Clarify doc comment for ExtractAPIAction::PrepareToExecuteAction

[dang]

clang/include/clang/SymbolGraph/FrontendActions.h

35

You are right in that is a bit confusing. I have reworded this to reflect what is actually happening more accurately.

[ributzka]

LGTM

clang/include/clang/SymbolGraph/FrontendActions.h

 
 namespace clang {
 
 class ExtractAPIAction : public ASTFrontendAction {
 protected:
   std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
                                                  StringRef InFile) override;
 
+private:
+  /// The synthesized input buffer that contains all the provided input header
+  /// files.
+  std::unique_ptr<llvm::MemoryBuffer> Buffer;
+
 public:
+  /// Prepare to execut the action on the given CompilerInstance.
+  ///
+  /// This is called before executing the action on any inputs. This merges all
+  /// the provided headers into a single header for processing.
+  bool PrepareToExecuteAction(CompilerInstance &CI) override;
+
   static std::unique_ptr<llvm::raw_pwrite_stream>
   CreateOutputFile(CompilerInstance &CI, StringRef InFile);
+  static StringRef getInputBufferName() { return "<extract-api-includes>"; }
 };
 
 } // namespace clang
 
 #endif // LLVM_CLANG_SYMBOLGRAPH_FRONTEND_ACTIONS_H

clang/lib/SymbolGraph/ExtractAPIConsumer.cpp

 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/ParentMapContext.h"
 #include "clang/AST/RawCommentList.h"
 #include "clang/AST/RecursiveASTVisitor.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Frontend/ASTConsumers.h"
 #include "clang/Frontend/CompilerInstance.h"
+#include "clang/Frontend/FrontendDiagnostic.h"
+#include "clang/Frontend/FrontendOptions.h"
 #include "clang/SymbolGraph/API.h"
 #include "clang/SymbolGraph/AvailabilityInfo.h"
 #include "clang/SymbolGraph/DeclarationFragments.h"
 #include "clang/SymbolGraph/FrontendActions.h"
 #include "clang/SymbolGraph/Serialization.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace clang;
 using namespace symbolgraph;
 
 namespace {
 class ExtractAPIVisitor : public RecursiveASTVisitor<ExtractAPIVisitor> {
 public:
ExtractAPIAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {
   std::unique_ptr<raw_pwrite_stream> OS = CreateOutputFile(CI, InFile);
   if (!OS)
     return nullptr;
   return std::make_unique<ExtractAPIConsumer>(
       CI.getASTContext(), CI.getInvocation().getFrontendOpts().ProductName,
       std::move(OS));
 }
 
+bool ExtractAPIAction::PrepareToExecuteAction(CompilerInstance &CI) {
+  auto &Inputs = CI.getFrontendOpts().Inputs;
+  if (Inputs.empty())
+    return true;
+
+  auto Kind = Inputs[0].getKind();
+
+  // Convert the header file inputs into a single input buffer.
+  SmallString<256> HeaderContents;
+  for (const FrontendInputFile &FIF : Inputs) {
+    if (Kind.isObjectiveC())
+      HeaderContents += "#import";
+    else
+      HeaderContents += "#include";
+    HeaderContents += " \"";
+    HeaderContents += FIF.getFile();
+    HeaderContents += "\"\n";
+  }
+
+  Buffer = llvm::MemoryBuffer::getMemBufferCopy(HeaderContents,
+                                                getInputBufferName());
+
+  // Set that buffer up as our "real" input in the CompilerInstance.
+  Inputs.clear();
+  Inputs.emplace_back(Buffer->getMemBufferRef(), Kind, /*IsSystem*/ false);
+
+  return true;
+}
+
 std::unique_ptr<raw_pwrite_stream>
 ExtractAPIAction::CreateOutputFile(CompilerInstance &CI, StringRef InFile) {
   std::unique_ptr<raw_pwrite_stream> OS =
       CI.createDefaultOutputFile(/*Binary=*/false, InFile, /*Extension=*/"json",
                                  /*RemoveFileOnSignal=*/false);
   if (!OS)
     return nullptr;
   return OS;
 }

clang/test/SymbolGraph/global_record_multifile.c

+// RUN: rm -rf %t
+// RUN: split-file %s %t
+// RUN: sed -e "s@INPUT_DIR@%/t@g" %t/reference.output.json.in >> \
+// RUN: %t/reference.output.json
+// RUN: %clang -extract-api --product-name=GlobalRecord -target arm64-apple-macosx \
+// RUN: %t/input1.h %t/input2.h %t/input3.h -o %t/output.json | FileCheck -allow-empty %s
+
+// Generator version is not consistent across test runs, normalize it.
+// RUN: sed -e "s@\"generator\": \".*\"@\"generator\": \"?\"@g" \
+// RUN: %t/output.json >> %t/output-normalized.json
+// RUN: diff %t/reference.output.json %t/output-normalized.json
+
+// CHECK-NOT: error:
+// CHECK-NOT: warning:
+
+//--- input1.h
+int num;
+
+//--- input2.h
+/**
+ * \brief Add two numbers.
+ * \param [in]  x   A number.
+ * \param [in]  y   Another number.
+ * \param [out] res The result of x + y.
+ */
+void add(const int x, const int y, int *res);
+
+//--- input3.h
+char unavailable __attribute__((unavailable));
+
+//--- reference.output.json.in
+{
+  "metadata": {
+    "formatVersion": {
+      "major": 0,
+      "minor": 5,
+      "patch": 3
+    },
+    "generator": "?"
+  },
+  "module": {
+    "name": "GlobalRecord",
+    "platform": {
+      "architecture": "arm64",
+      "operatingSystem": {
+        "minimumVersion": {
+          "major": 11,
+          "minor": 0,
+          "patch": 0
+        },
+        "name": "macosx"
+      },
+      "vendor": "apple"
+    }
+  },
+  "relationhips": [],
+  "symbols": [
+    {
+      "declarationFragments": [
+        {
+          "kind": "typeIdentifier",
+          "preciseIdentifier": "c:I",
+          "spelling": "int"
+        },
+        {
+          "kind": "text",
+          "spelling": " "
+        },
+        {
+          "kind": "identifier",
+          "spelling": "num"
+        }
+      ],
+      "identifier": {
+        "interfaceLanguage": "c",
+        "precise": "c:@num"
+      },
+      "kind": {
+        "displayName": "Global Variable",
+        "identifier": "c.var"
+      },
+      "location": {
+        "character": 5,
+        "line": 1,
+        "uri": "file://INPUT_DIR/input1.h"
+      },
+      "names": {
+        "subHeading": [
+          {
+            "kind": "identifier",
+            "spelling": "num"
+          }
+        ],
+        "title": "num"
+      }
+    },
+    {
+      "declarationFragments": [
+        {
+          "kind": "typeIdentifier",
+          "preciseIdentifier": "c:v",
+          "spelling": "void"
+        },
+        {
+          "kind": "text",
+          "spelling": " "
+        },
+        {
+          "kind": "identifier",
+          "spelling": "add"
+        },
+        {
+          "kind": "text",
+          "spelling": "("
+        },
+        {
+          "kind": "keyword",
+          "spelling": "const"
+        },
+        {
+          "kind": "text",
+          "spelling": " "
+        },
+        {
+          "kind": "typeIdentifier",
+          "preciseIdentifier": "c:I",
+          "spelling": "int"
+        },
+        {
+          "kind": "text",
+          "spelling": " "
+        },
+        {
+          "kind": "internalParam",
+          "spelling": "x"
+        },
+        {
+          "kind": "text",
+          "spelling": ", "
+        },
+        {
+          "kind": "keyword",
+          "spelling": "const"
+        },
+        {
+          "kind": "text",
+          "spelling": " "
+        },
+        {
+          "kind": "typeIdentifier",
+          "preciseIdentifier": "c:I",
+          "spelling": "int"
+        },
+        {
+          "kind": "text",
+          "spelling": " "
+        },
+        {
+          "kind": "internalParam",
+          "spelling": "y"
+        },
+        {
+          "kind": "text",
+          "spelling": ", "
+        },
+        {
+          "kind": "typeIdentifier",
+          "preciseIdentifier": "c:I",
+          "spelling": "int"
+        },
+        {
+          "kind": "text",
+          "spelling": " *"
+        },
+        {
+          "kind": "internalParam",
+          "spelling": "res"
+        },
+        {
+          "kind": "text",
+          "spelling": ")"
+        }
+      ],
+      "docComment": {
+        "lines": [
+          {
+            "range": {
+              "end": {
+                "character": 4,
+                "line": 1
+              },
+              "start": {
+                "character": 4,
+                "line": 1
+              }
+            },
+            "text": ""
+          },
+          {
+            "range": {
+              "end": {
+                "character": 27,
+                "line": 2
+              },
+              "start": {
+                "character": 3,
+                "line": 2
+              }
+            },
+            "text": " \\brief Add two numbers."
+          },
+          {
+            "range": {
+              "end": {
+                "character": 30,
+                "line": 3
+              },
+              "start": {
+                "character": 3,
+                "line": 3
+              }
+            },
+            "text": " \\param [in]  x   A number."
+          },
+          {
+            "range": {
+              "end": {
+                "character": 36,
+                "line": 4
+              },
+              "start": {
+                "character": 3,
+                "line": 4
+              }
+            },
+            "text": " \\param [in]  y   Another number."
+          },
+          {
+            "range": {
+              "end": {
+                "character": 41,
+                "line": 5
+              },
+              "start": {
+                "character": 3,
+                "line": 5
+              }
+            },
+            "text": " \\param [out] res The result of x + y."
+          },
+          {
+            "range": {
+              "end": {
+                "character": 4,
+                "line": 6
+              },
+              "start": {
+                "character": 1,
+                "line": 6
+              }
+            },
+            "text": " "
+          }
+        ]
+      },
+      "identifier": {
+        "interfaceLanguage": "c",
+        "precise": "c:@F@add"
+      },
+      "kind": {
+        "displayName": "Function",
+        "identifier": "c.func"
+      },
+      "location": {
+        "character": 6,
+        "line": 7,
+        "uri": "file://INPUT_DIR/input2.h"
+      },
+      "names": {
+        "subHeading": [
+          {
+            "kind": "identifier",
+            "spelling": "add"
+          }
+        ],
+        "title": "add"
+      },
+      "parameters": {
+        "parameters": [
+          {
+            "declarationFragments": [
+              {
+                "kind": "keyword",
+                "spelling": "const"
+              },
+              {
+                "kind": "text",
+                "spelling": " "
+              },
+              {
+                "kind": "typeIdentifier",
+                "preciseIdentifier": "c:I",
+                "spelling": "int"
+              },
+              {
+                "kind": "text",
+                "spelling": " "
+              },
+              {
+                "kind": "internalParam",
+                "spelling": "x"
+              }
+            ],
+            "name": "x"
+          },
+          {
+            "declarationFragments": [
+              {
+                "kind": "keyword",
+                "spelling": "const"
+              },
+              {
+                "kind": "text",
+                "spelling": " "
+              },
+              {
+                "kind": "typeIdentifier",
+                "preciseIdentifier": "c:I",
+                "spelling": "int"
+              },
+              {
+                "kind": "text",
+                "spelling": " "
+              },
+              {
+                "kind": "internalParam",
+                "spelling": "y"
+              }
+            ],
+            "name": "y"
+          },
+          {
+            "declarationFragments": [
+              {
+                "kind": "typeIdentifier",
+                "preciseIdentifier": "c:I",
+                "spelling": "int"
+              },
+              {
+                "kind": "text",
+                "spelling": " *"
+              },
+              {
+                "kind": "internalParam",
+                "spelling": "res"
+              }
+            ],
+            "name": "res"
+          }
+        ],
+        "returns": [
+          {
+            "kind": "typeIdentifier",
+            "preciseIdentifier": "c:v",
+            "spelling": "void"
+          }
+        ]
+      }
+    }
+  ]
+}