diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -1179,6 +1179,9 @@ HelpText<"Extract API information">; def product_name_EQ: Joined<["--"], "product-name=">, Flags<[CC1Option]>, MarshallingInfoString>; +def emit_symbol_graph_EQ: JoinedOrSeparate<["--"], "emit-symbol-graph=">, Flags<[CC1Option]>, + HelpText<"Generate Extract API information as a side effect of compilation.">, + MarshallingInfoString>; def extract_api_ignores_EQ: CommaJoined<["--"], "extract-api-ignores=">, Flags<[CC1Option]>, HelpText<"Comma separated list of files containing a new line separated list of API symbols to ignore when extracting API information.">, MarshallingInfoStringVector>; diff --git a/clang/include/clang/ExtractAPI/FrontendActions.h b/clang/include/clang/ExtractAPI/FrontendActions.h --- a/clang/include/clang/ExtractAPI/FrontendActions.h +++ b/clang/include/clang/ExtractAPI/FrontendActions.h @@ -7,7 +7,8 @@ //===----------------------------------------------------------------------===// /// /// \file -/// This file defines the ExtractAPIAction frontend action. +/// This file defines the ExtractAPIAction and WrappingExtractAPIAction frontend +/// actions. /// //===----------------------------------------------------------------------===// @@ -17,16 +18,19 @@ #include "clang/ExtractAPI/API.h" #include "clang/ExtractAPI/APIIgnoresList.h" #include "clang/Frontend/FrontendAction.h" +#include "clang/Frontend/MultiplexConsumer.h" namespace clang { -/// ExtractAPIAction sets up the output file and creates the ExtractAPIVisitor. -class ExtractAPIAction : public ASTFrontendAction { +/// Base class to be used by front end actions to generate ExtarctAPI info +/// +/// Deriving from this class equipts an action with all the necessary tools to +/// generate ExractAPI information in form of symbol-graphs +class ExtractAPIActionBase { protected: - std::unique_ptr CreateASTConsumer(CompilerInstance &CI, - StringRef InFile) override; + std::unique_ptr CreateExtractAPIConsumer(CompilerInstance &CI, + StringRef InFile); -private: /// A representation of the APIs this action extracts. std::unique_ptr API; @@ -49,6 +53,24 @@ /// include is quoted or not. SmallVector, bool>> KnownInputFiles; + /// Implements EndSourceFileAction for Symbol-Graph generation + /// + /// Use the serializer to generate output symbol graph files from + /// the information gathered during the execution of Action. + void ImplEndSourceFileAction(); + + static std::unique_ptr + CreateOutputFile(CompilerInstance &CI, StringRef InFile); +}; + +/// ExtractAPIAction sets up the output file and creates the ExtractAPIVisitor. +class ExtractAPIAction : public ASTFrontendAction, + private ExtractAPIActionBase { +protected: + std::unique_ptr CreateASTConsumer(CompilerInstance &CI, + StringRef InFile) override; + +private: /// Prepare to execute the action on the given CompilerInstance. /// /// This is called before executing the action on any inputs. This generates a @@ -62,12 +84,49 @@ /// emit them in this callback. void EndSourceFileAction() override; - static std::unique_ptr - CreateOutputFile(CompilerInstance &CI, StringRef InFile); - static StringRef getInputBufferName() { return ""; } }; +/// Wrap ExtractAPIAction on top of a pre-existing action +/// +/// Used when the ExtractAPI action needs to be executed as a side effect of a +/// regular compilation job. Unlike ExtarctAPIAction, this is meant to be used +/// on regular source files ( .m , .c files) instead of header files +class WrappingExtractAPIAction : public WrapperFrontendAction, + private ExtractAPIActionBase { +public: + WrappingExtractAPIAction(std::unique_ptr WrappedAction) + : WrapperFrontendAction(std::move(WrappedAction)) {} + +protected: + /// Create ExtractAPI consumer multiplexed on another consumer. + /// + /// This allows us to execute ExtractAPI action while on top of + std::unique_ptr CreateASTConsumer(CompilerInstance &CI, + StringRef InFile) override; + +private: + /// Flag to check if the wrapper front end action's consumer is + /// craeted or not + bool createdASTConsumer = false; + + /// Prepare to execute the action on the given CompilerInstance. + /// + /// This is called before executing the action on any inputs. This + /// is the place where all the input files that are provided by the + /// command line are identified and stored in "KnownInputFiles" + bool PrepareToExecuteAction(CompilerInstance &CI) override; + + void EndSourceFile() override { FrontendAction::EndSourceFile(); } + + /// Called after executing the action on the synthesized input buffer. + /// + /// Executes both Wrapper and ExtractAPIBase end source file + /// actions. This is the place where all the gathered symbol graph + /// information is emited. + void EndSourceFileAction() override; +}; + } // namespace clang #endif // LLVM_CLANG_EXTRACTAPI_FRONTEND_ACTIONS_H diff --git a/clang/include/clang/Frontend/FrontendOptions.h b/clang/include/clang/Frontend/FrontendOptions.h --- a/clang/include/clang/Frontend/FrontendOptions.h +++ b/clang/include/clang/Frontend/FrontendOptions.h @@ -456,6 +456,12 @@ // ignore when extracting documentation. std::vector ExtractAPIIgnoresFileList; + // Currently this is only used as part of the `-emit-symbol-graph` + // action. + // Location of output directory where symbol graph information would + // be dumped + std::string SymbolGraphOutputDir; + /// Args to pass to the plugins std::map> PluginArgs; diff --git a/clang/lib/ExtractAPI/ExtractAPIConsumer.cpp b/clang/lib/ExtractAPI/ExtractAPIConsumer.cpp --- a/clang/lib/ExtractAPI/ExtractAPIConsumer.cpp +++ b/clang/lib/ExtractAPI/ExtractAPIConsumer.cpp @@ -34,6 +34,7 @@ #include "clang/Lex/PreprocessorOptions.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallString.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Support/Casting.h" #include "llvm/Support/Error.h" @@ -342,7 +343,8 @@ } // namespace std::unique_ptr -ExtractAPIAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) { +ExtractAPIActionBase::CreateExtractAPIConsumer(CompilerInstance &CI, + StringRef InFile) { OS = CreateOutputFile(CI, InFile); if (!OS) return nullptr; @@ -381,6 +383,53 @@ std::move(LCF), *API); } +void ExtractAPIActionBase::ImplEndSourceFileAction() { + if (!OS) + return; + + // Setup a SymbolGraphSerializer to write out collected API information in + // the Symbol Graph format. + // FIXME: Make the kind of APISerializer configurable. + SymbolGraphSerializer SGSerializer(*API, IgnoresList); + SGSerializer.serialize(*OS); + OS.reset(); +} + +std::unique_ptr +ExtractAPIActionBase::CreateOutputFile(CompilerInstance &CI, StringRef InFile) { + std::unique_ptr OS; + std::string OutputDir = CI.getFrontendOpts().SymbolGraphOutputDir; + if (OutputDir.empty()) + OS = CI.createDefaultOutputFile(/*Binary=*/false, InFile, + /*Extension=*/"json", + /*RemoveFileOnSignal=*/false); + else { + // The symbol graphs need to be generated as a side effect of regular + // compilation so the output should be dumped in the directory provided with + // the command line option. + llvm::SmallString<128> OutFilePath(OutputDir); + auto Seperator = llvm::sys::path::get_separator(); + auto Infilename = llvm::sys::path::filename(InFile); + OutFilePath.append({Seperator, Infilename}); + llvm::sys::path::replace_extension(OutFilePath, "json"); + // StringRef outputFilePathref = *OutFilePath; + + // don't use the default output file + OS = CI.createOutputFile(/*OutputPath=*/OutFilePath, /*Binary=*/false, + /*RemoveFileOnSignal=*/true, + /*UseTemporary=*/true, + /*CreateMissingDirectories=*/true); + } + if (!OS) + return nullptr; + return OS; +} + +std::unique_ptr +ExtractAPIAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) { + return CreateExtractAPIConsumer(CI, InFile); +} + bool ExtractAPIAction::PrepareToExecuteAction(CompilerInstance &CI) { auto &Inputs = CI.getFrontendOpts().Inputs; if (Inputs.empty()) @@ -438,24 +487,54 @@ return true; } -void ExtractAPIAction::EndSourceFileAction() { - if (!OS) - return; +void ExtractAPIAction::EndSourceFileAction() { ImplEndSourceFileAction(); } - // Setup a SymbolGraphSerializer to write out collected API information in - // the Symbol Graph format. - // FIXME: Make the kind of APISerializer configurable. - SymbolGraphSerializer SGSerializer(*API, IgnoresList); - SGSerializer.serialize(*OS); - OS.reset(); +std::unique_ptr +WrappingExtractAPIAction::CreateASTConsumer(CompilerInstance &CI, + StringRef InFile) { + auto OtherConsumer = WrapperFrontendAction::CreateASTConsumer(CI, InFile); + if (!OtherConsumer) + return nullptr; + + createdASTConsumer = true; + std::vector> Consumers; + Consumers.push_back(std::move(OtherConsumer)); + Consumers.push_back(CreateExtractAPIConsumer(CI, InFile)); + return std::make_unique(std::move(Consumers)); } -std::unique_ptr -ExtractAPIAction::CreateOutputFile(CompilerInstance &CI, StringRef InFile) { - std::unique_ptr OS = - CI.createDefaultOutputFile(/*Binary=*/false, InFile, /*Extension=*/"json", - /*RemoveFileOnSignal=*/false); - if (!OS) - return nullptr; - return OS; +bool WrappingExtractAPIAction::PrepareToExecuteAction(CompilerInstance &CI) { + + bool Prepared = WrapperFrontendAction::PrepareToExecuteAction(CI); + + auto &Inputs = CI.getFrontendOpts().Inputs; + if (Inputs.empty()) + Prepared = false; + + else if (!CI.hasFileManager()) { + if (!CI.createFileManager()) + Prepared = false; + } + + if (Prepared) { + bool IsQuoted = false; + for (const FrontendInputFile &FIF : Inputs) { + StringRef FilePath = FIF.getFile(); + if (auto RelativeName = getRelativeIncludeName(CI, FilePath, &IsQuoted)) + KnownInputFiles.emplace_back( + static_cast>(*RelativeName), IsQuoted); + else + KnownInputFiles.emplace_back(FilePath, true); + } + } + return Prepared; +} + +void WrappingExtractAPIAction::EndSourceFileAction() { + // Invoke wrapped action's method. + WrapperFrontendAction::EndSourceFileAction(); + + if (createdASTConsumer) { + ImplEndSourceFileAction(); + } } diff --git a/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp b/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp --- a/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp +++ b/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp @@ -178,6 +178,14 @@ } #endif + // Wrap the base FE action in an extract api action to generate + // symbol graph as a biproduct of comilation ( enabled with + // --emit-symbol-graph option ) + if (!FEOpts.SymbolGraphOutputDir.empty()) { + CI.getCodeGenOpts().ClearASTBeforeBackend = false; + Act = std::make_unique(std::move(Act)); + } + // If there are any AST files to merge, create a frontend action // adaptor to perform the merge. if (!FEOpts.ASTMergeFiles.empty()) diff --git a/clang/test/ExtractAPI/emit-symbol-graph/multi_file.c b/clang/test/ExtractAPI/emit-symbol-graph/multi_file.c new file mode 100644 --- /dev/null +++ b/clang/test/ExtractAPI/emit-symbol-graph/multi_file.c @@ -0,0 +1,364 @@ +// RUN: rm -rf %t +// RUN: mkdir %t +// RUN: split-file %s %t +// RUN: sed -e "s@INPUT_DIR@%{/t:regex_replacement}@g" \ +// RUN: %t/reference.main.json.in >> %t/reference.main.json +// RUN: sed -e "s@INPUT_DIR@%{/t:regex_replacement}@g" \ +// RUN: %t/reference.test.json.in >> %t/reference.test.json +// RUN: %clang %t/test.c %t/main.c -Xclang --emit-symbol-graph=%t/SymbolGraphs -Xclang --product-name=multifile_hello_world -Xclang -triple=x86_64-apple-macosx12.0.0 + +// Test main.json +// Generator version is not consistent across test runs, normalize it. +// RUN: sed -e "s@\"generator\": \".*\"@\"generator\": \"?\"@g" \ +// RUN: %t/SymbolGraphs/main.json > %t/output-normalized.json +// RUN: diff %t/reference.main.json %t/output-normalized.json + +// Test test.json +// Generator version is not consistent across test runs, normalize it. +// RUN: sed -e "s@\"generator\": \".*\"@\"generator\": \"?\"@g" \ +// RUN: %t/SymbolGraphs/test.json > %t/output-normalized.json +// RUN: diff %t/reference.test.json %t/output-normalized.json + +// CHECK-NOT: error: +// CHECK-NOT: warning: + +//--- test.h +#ifndef TEST_H +#define TEST_H + +int testfunc (int param1, int param2); +void testfunc2 (); + +#endif /* TEST_H */ + +//--- test.c +#include "test.h" +#include + +int testfunc(int param1, int param2) { return param1 + param2; } + +void testfunc2() { + printf("hello world\n"); +} + +//--- main.c +#include "test.h" + +int main () +{ + testfunc2(); + return 0; +} + +//--- reference.main.json.in +{ + "metadata": { + "formatVersion": { + "major": 0, + "minor": 5, + "patch": 3 + }, + "generator": "?" + }, + "module": { + "name": "multifile_hello_world", + "platform": { + "architecture": "x86_64", + "operatingSystem": { + "name": "macosx" + }, + "vendor": "apple" + } + }, + "relationships": [], + "symbols": [ + { + "accessLevel": "public", + "declarationFragments": [ + { + "kind": "typeIdentifier", + "preciseIdentifier": "c:I", + "spelling": "int" + }, + { + "kind": "text", + "spelling": " " + }, + { + "kind": "identifier", + "spelling": "main" + }, + { + "kind": "text", + "spelling": "();" + } + ], + "functionSignature": { + "returns": [ + { + "kind": "typeIdentifier", + "preciseIdentifier": "c:I", + "spelling": "int" + } + ] + }, + "identifier": { + "interfaceLanguage": "c", + "precise": "c:@F@main" + }, + "kind": { + "displayName": "Function", + "identifier": "c.func" + }, + "location": { + "position": { + "character": 5, + "line": 3 + }, + "uri": "file://INPUT_DIR/main.c" + }, + "names": { + "navigator": [ + { + "kind": "identifier", + "spelling": "main" + } + ], + "subHeading": [ + { + "kind": "identifier", + "spelling": "main" + } + ], + "title": "main" + }, + "pathComponents": [ + "main" + ] + } + ] +} +//--- reference.test.json.in +{ + "metadata": { + "formatVersion": { + "major": 0, + "minor": 5, + "patch": 3 + }, + "generator": "?" + }, + "module": { + "name": "multifile_hello_world", + "platform": { + "architecture": "x86_64", + "operatingSystem": { + "name": "macosx" + }, + "vendor": "apple" + } + }, + "relationships": [], + "symbols": [ + { + "accessLevel": "public", + "declarationFragments": [ + { + "kind": "typeIdentifier", + "preciseIdentifier": "c:I", + "spelling": "int" + }, + { + "kind": "text", + "spelling": " " + }, + { + "kind": "identifier", + "spelling": "testfunc" + }, + { + "kind": "text", + "spelling": "(" + }, + { + "kind": "typeIdentifier", + "preciseIdentifier": "c:I", + "spelling": "int" + }, + { + "kind": "text", + "spelling": " " + }, + { + "kind": "internalParam", + "spelling": "param1" + }, + { + "kind": "text", + "spelling": ", " + }, + { + "kind": "typeIdentifier", + "preciseIdentifier": "c:I", + "spelling": "int" + }, + { + "kind": "text", + "spelling": " " + }, + { + "kind": "internalParam", + "spelling": "param2" + }, + { + "kind": "text", + "spelling": ");" + } + ], + "functionSignature": { + "parameters": [ + { + "declarationFragments": [ + { + "kind": "typeIdentifier", + "preciseIdentifier": "c:I", + "spelling": "int" + }, + { + "kind": "text", + "spelling": " " + }, + { + "kind": "internalParam", + "spelling": "param1" + } + ], + "name": "param1" + }, + { + "declarationFragments": [ + { + "kind": "typeIdentifier", + "preciseIdentifier": "c:I", + "spelling": "int" + }, + { + "kind": "text", + "spelling": " " + }, + { + "kind": "internalParam", + "spelling": "param2" + } + ], + "name": "param2" + } + ], + "returns": [ + { + "kind": "typeIdentifier", + "preciseIdentifier": "c:I", + "spelling": "int" + } + ] + }, + "identifier": { + "interfaceLanguage": "c", + "precise": "c:@F@testfunc" + }, + "kind": { + "displayName": "Function", + "identifier": "c.func" + }, + "location": { + "position": { + "character": 5, + "line": 4 + }, + "uri": "file://INPUT_DIR/test.c" + }, + "names": { + "navigator": [ + { + "kind": "identifier", + "spelling": "testfunc" + } + ], + "subHeading": [ + { + "kind": "identifier", + "spelling": "testfunc" + } + ], + "title": "testfunc" + }, + "pathComponents": [ + "testfunc" + ] + }, + { + "accessLevel": "public", + "declarationFragments": [ + { + "kind": "typeIdentifier", + "preciseIdentifier": "c:v", + "spelling": "void" + }, + { + "kind": "text", + "spelling": " " + }, + { + "kind": "identifier", + "spelling": "testfunc2" + }, + { + "kind": "text", + "spelling": "();" + } + ], + "functionSignature": { + "returns": [ + { + "kind": "typeIdentifier", + "preciseIdentifier": "c:v", + "spelling": "void" + } + ] + }, + "identifier": { + "interfaceLanguage": "c", + "precise": "c:@F@testfunc2" + }, + "kind": { + "displayName": "Function", + "identifier": "c.func" + }, + "location": { + "position": { + "character": 6, + "line": 6 + }, + "uri": "file://INPUT_DIR/test.c" + }, + "names": { + "navigator": [ + { + "kind": "identifier", + "spelling": "testfunc2" + } + ], + "subHeading": [ + { + "kind": "identifier", + "spelling": "testfunc2" + } + ], + "title": "testfunc2" + }, + "pathComponents": [ + "testfunc2" + ] + } + ] +} diff --git a/clang/test/ExtractAPI/emit-symbol-graph/single_file.c b/clang/test/ExtractAPI/emit-symbol-graph/single_file.c new file mode 100644 --- /dev/null +++ b/clang/test/ExtractAPI/emit-symbol-graph/single_file.c @@ -0,0 +1,213 @@ +// RUN: rm -rf %t +// RUN: mkdir %t +// RUN: split-file %s %t +// RUN: sed -e "s@INPUT_DIR@%{/t:regex_replacement}@g" \ +// RUN: %t/reference.output.json.in >> %t/reference.output.json +// RUN: %clang_cc1 %t/main.c --emit-symbol-graph=%t/SymbolGraphs --product-name=basicfile -triple=x86_64-apple-macosx12.0.0 + +// Generator version is not consistent across test runs, normalize it. +// RUN: sed -e "s@\"generator\": \".*\"@\"generator\": \"?\"@g" \ +// RUN: %t/SymbolGraphs/main.json >> %t/output-normalized.json +// RUN: diff %t/reference.output.json %t/output-normalized.json + +// CHECK-NOT: error: +// CHECK-NOT: warning: + +//--- main.c +#define TESTMACRO1 2 +#define TESTMARCRO2 5 + +int main () +{ + return 0; +} + + +//--- reference.output.json.in +{ + "metadata": { + "formatVersion": { + "major": 0, + "minor": 5, + "patch": 3 + }, + "generator": "?" + }, + "module": { + "name": "basicfile", + "platform": { + "architecture": "x86_64", + "operatingSystem": { + "name": "macosx" + }, + "vendor": "apple" + } + }, + "relationships": [], + "symbols": [ + { + "accessLevel": "public", + "declarationFragments": [ + { + "kind": "typeIdentifier", + "preciseIdentifier": "c:I", + "spelling": "int" + }, + { + "kind": "text", + "spelling": " " + }, + { + "kind": "identifier", + "spelling": "main" + }, + { + "kind": "text", + "spelling": "();" + } + ], + "functionSignature": { + "returns": [ + { + "kind": "typeIdentifier", + "preciseIdentifier": "c:I", + "spelling": "int" + } + ] + }, + "identifier": { + "interfaceLanguage": "c", + "precise": "c:@F@main" + }, + "kind": { + "displayName": "Function", + "identifier": "c.func" + }, + "location": { + "position": { + "character": 5, + "line": 4 + }, + "uri": "file://INPUT_DIR/main.c" + }, + "names": { + "navigator": [ + { + "kind": "identifier", + "spelling": "main" + } + ], + "subHeading": [ + { + "kind": "identifier", + "spelling": "main" + } + ], + "title": "main" + }, + "pathComponents": [ + "main" + ] + }, + { + "accessLevel": "public", + "declarationFragments": [ + { + "kind": "keyword", + "spelling": "#define" + }, + { + "kind": "text", + "spelling": " " + }, + { + "kind": "identifier", + "spelling": "TESTMACRO1" + } + ], + "identifier": { + "interfaceLanguage": "c", + "precise": "c:main.c@8@macro@TESTMACRO1" + }, + "kind": { + "displayName": "Macro", + "identifier": "c.macro" + }, + "location": { + "position": { + "character": 9, + "line": 1 + }, + "uri": "file://INPUT_DIR/main.c" + }, + "names": { + "navigator": [ + { + "kind": "identifier", + "spelling": "TESTMACRO1" + } + ], + "subHeading": [ + { + "kind": "identifier", + "spelling": "TESTMACRO1" + } + ], + "title": "TESTMACRO1" + }, + "pathComponents": [ + "TESTMACRO1" + ] + }, + { + "accessLevel": "public", + "declarationFragments": [ + { + "kind": "keyword", + "spelling": "#define" + }, + { + "kind": "text", + "spelling": " " + }, + { + "kind": "identifier", + "spelling": "TESTMARCRO2" + } + ], + "identifier": { + "interfaceLanguage": "c", + "precise": "c:main.c@29@macro@TESTMARCRO2" + }, + "kind": { + "displayName": "Macro", + "identifier": "c.macro" + }, + "location": { + "position": { + "character": 9, + "line": 2 + }, + "uri": "file://INPUT_DIR/main.c" + }, + "names": { + "navigator": [ + { + "kind": "identifier", + "spelling": "TESTMARCRO2" + } + ], + "subHeading": [ + { + "kind": "identifier", + "spelling": "TESTMARCRO2" + } + ], + "title": "TESTMARCRO2" + }, + "pathComponents": [ + "TESTMARCRO2" + ] + } + ] +}