diff --git a/clang/include/clang-c/Dependencies.h b/clang/include/clang-c/Dependencies.h new file mode 100644 --- /dev/null +++ b/clang/include/clang-c/Dependencies.h @@ -0,0 +1,223 @@ +/*==-- clang-c/Dependencies.h - Dependency Discovery C Interface --*- C -*-===*\ +|* *| +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| +|* Exceptions. *| +|* See https://llvm.org/LICENSE.txt for license information. *| +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| +|* *| +|*===----------------------------------------------------------------------===*| +|* *| +|* This header provides a dependency discovery interface similar to *| +|* clang-scan-deps. *| +|* *| +|* An example of its usage is available in c-index-test/core_main.cpp. *| +|* *| +|* EXPERIMENTAL: These interfaces are experimental and will change. If you *| +|* use these be prepared for them to change without notice on any commit. *| +|* *| +\*===----------------------------------------------------------------------===*/ + +#ifndef LLVM_CLANG_C_DEPENDENCIES_H +#define LLVM_CLANG_C_DEPENDENCIES_H + +#include "clang-c/BuildSystem.h" +#include "clang-c/CXErrorCode.h" +#include "clang-c/CXString.h" +#include "clang-c/Platform.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * \defgroup SCAN_DEPS Dependency scanning service. + * @{ + */ + +typedef struct { + CXString Name; + /** + * The context hash of a module represents the set of compiler options that + * may make one version of a module incompatible from another. This includes + * things like language mode, predefined macros, header search paths, etc... + * + * Modules with the same name but a different \c ContextHash should be treated + * as separate modules for the purpose of a build. + */ + CXString ContextHash; + + /** + * The path to the modulemap file which defines this module. + * + * This can be used to explicitly build this module. This file will + * additionally appear in \c FileDeps as a dependency. + */ + CXString ModuleMapPath; + + /** + * The list of files which this module directly depends on. + * + * If any of these change then the module needs to be rebuilt. + */ + CXStringSet *FileDeps; + + /** + * The list of modules which this module direct depends on. + * + * This does not include the \c ContextHash, as all modules in a single + * translation unit all have the same context hash. + */ + CXStringSet *ModuleDeps; + + /** + * The full command line needed to build this module. + * + * Not including `-fmodule-file=` or `-o`. + */ + CXStringSet *BuildArguments; +} CXModuleDependency; + +typedef struct { + int Count; + CXModuleDependency *Modules; +} CXModuleDependencySet; + +/** + * See \c CXModuleDependency for the meaning of these fields, with the addition + * that they represent only the direct dependencies for \c CXDependencyMode_Full + * mode. + */ +typedef struct { + CXString ContextHash; + CXStringSet *FileDeps; + CXStringSet *ModuleDeps; + + /** + * Additional arguments to append to the build of this file. + * + * This contains things like disabling implicit modules. This does not include + * the `-fmodule-file=` arguments that are needed. + */ + CXStringSet *AdditionalArguments; +} CXFileDependencies; + +CINDEX_LINKAGE void +clang_experimental_ModuleDependencySet_dispose(CXModuleDependencySet *MD); + +CINDEX_LINKAGE void +clang_experimental_FileDependencies_dispose(CXFileDependencies *ID); + +/** + * Object encapsulating instance of a dependency scanner service. + * + * The dependency scanner service is a global instance that owns the + * global cache and other global state that's shared between the dependency + * scanner workers. The service APIs are thread safe. + */ +typedef struct CXOpaqueDependencyScannerService *CXDependencyScannerService; + +/** + * The mode to report module dependencies in. + */ +typedef enum { + /** + * Flatten all module dependencies. This reports the full transitive set of + * header and module map dependencies needed to do an implicit module build. + */ + CXDependencyMode_Flat, + + /** + * Report the full module graph. This reports only the direct dependencies of + * each file, and calls a callback for each module that is discovered. + */ + CXDependencyMode_Full, +} CXDependencyMode; + +/** + * Create a \c CXDependencyScannerService object. + * Must be disposed with \c clang_DependencyScannerService_dispose(). + */ +CINDEX_LINKAGE CXDependencyScannerService +clang_experimental_DependencyScannerService_create_v0(CXDependencyMode Format); + +/** + * Dispose of a \c CXDependencyScannerService object. + * + * The service object must be disposed of after the workers are disposed of. + */ +CINDEX_LINKAGE void clang_experimental_DependencyScannerService_dispose_v0( + CXDependencyScannerService); + +/** + * Object encapsulating instance of a dependency scanner worker. + * + * The dependency scanner workers are expected to be used in separate worker + * threads. An individual worker is not thread safe. + * + * Operations on a worker are not thread-safe and should only be used from a + * single thread at a time. They are intended to be used by a single dedicated + * thread in a thread pool, but they are not inherently pinned to a thread. + */ +typedef struct CXOpaqueDependencyScannerWorker *CXDependencyScannerWorker; + +/** + * Create a \c CXDependencyScannerWorker object. + * Must be disposed with + * \c clang_experimental_DependencyScannerWorker_dispose_v0(). + */ +CINDEX_LINKAGE CXDependencyScannerWorker + clang_experimental_DependencyScannerWorker_create_v0( + CXDependencyScannerService); + +CINDEX_LINKAGE void clang_experimental_DependencyScannerWorker_dispose_v0( + CXDependencyScannerWorker); + +/** + * A callback that is called whenever a module is discovered when in + * \c CXDependencyMode_Full mode. + * + * \param Context the context that was passed to + * \c clang_experimental_DependencyScannerWorker_getFileDependencies_v0. + * \param MDS the list of discovered modules. Must be freed by calling + * \c clang_experimental_ModuleDependencySet_dispose. + */ +typedef void CXModuleDiscoveredCallback(void *Context, + CXModuleDependencySet *MDS); + +/** + * Returns the list of file dependencies for a particular compiler invocation. + * + * \param argc the number of compiler invocation arguments (including argv[0]). + * \param argv the compiler invocation arguments (including argv[0]). + * the invocation may be a -cc1 clang invocation or a driver + * invocation. + * \param WorkingDirectory the directory in which the invocation runs. + * \param MDC a callback that is called whenever a new module is discovered. + * This may receive the same module on different workers. This should + * be NULL if + * \c clang_experimental_DependencyScannerService_create_v0 was + * called with \c CXDependencyMode_Flat. This callback will be called + * on the same thread that called this function. + * \param Context the context that will be passed to \c MDC each time it is + * called. + * \param [out] error the error string to pass back to client (if any). + * + * \returns A pointer to a CXFileDependencies on success, NULL otherwise. The + * CXFileDependencies must be freed by calling + * \c clang_experimental_FileDependencies_dispose. + */ +CINDEX_LINKAGE CXFileDependencies * +clang_experimental_DependencyScannerWorker_getFileDependencies_v0( + CXDependencyScannerWorker Worker, int argc, const char *const *argv, + const char *WorkingDirectory, CXModuleDiscoveredCallback *MDC, + void *Context, CXString *error); + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#endif // LLVM_CLANG_C_DEPENDENCIES_H diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h @@ -11,13 +11,69 @@ #include "clang/Tooling/DependencyScanning/DependencyScanningService.h" #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h" +#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h" #include "clang/Tooling/JSONCompilationDatabase.h" +#include "llvm/ADT/StringSet.h" #include namespace clang{ namespace tooling{ namespace dependencies{ +/// The full dependencies and module graph for a specific input. +struct FullDependencies { + /// The name of the C++20 module this translation unit exports. This may + /// include `:` for C++20 module partitons. + /// + /// If the translation unit is not a module then this will be empty. + std::string ExportedModuleName; + + /// The context hash represents the set of compiler options that may make one + /// version of a module incompatible with another. This includes things like + /// language mode, predefined macros, header search paths, etc... + /// + /// Modules with the same name but a different \c ContextHash should be + /// treated as separate modules for the purpose of a build. + std::string ContextHash; + + /// A collection of absolute paths to files that this translation unit + /// directly depends on, not including transitive dependencies. + std::vector FileDeps; + + /// A list of modules this translation unit directly depends on, not including + /// transitive dependencies. + /// + /// This may include modules with a different context hash when it can be + /// determined that the differences are benign for this compilation. + std::vector ClangModuleDeps; + + /// A partial addtional set of command line arguments that can be used to + /// build this translation unit. + /// + /// Call \c getFullAdditionalCommandLine() to get a command line suitable for + /// appending to the original command line to pass to clang. + std::vector AdditionalNonPathCommandLine; + + /// Gets the full addtional command line suitable for appending to the + /// original command line to pass to clang. + /// + /// \param LookupPCMPath this function is called to fill in `-fmodule-file=` + /// flags and for the `-o` flag. It needs to return a + /// path for where the PCM for the given module is to + /// be located. + /// \param LookupModuleDeps this fucntion is called to collect the full + /// transitive set of dependencies for this + /// compilation. + std::vector getAdditionalCommandLine( + std::function LookupPCMPath, + std::function LookupModuleDeps) const; +}; + +struct FullDependenciesResult { + FullDependencies FullDeps; + std::vector DiscoveredModules; +}; + /// The high-level implementation of the dependency discovery tool that runs on /// an individual worker thread. class DependencyScanningTool { @@ -35,8 +91,23 @@ getDependencyFile(const tooling::CompilationDatabase &Compilations, StringRef CWD); + /// Collect the full module depenedency graph for the input, ignoring any + /// modules which have already been seen. + /// + /// \param AlreadySeen this is used to not report modules that have previously + /// been reported. Use the same `llvm::StringSet<>` for all + /// calls to `getFullDependencies` for a single + /// `DependencyScanningTool` for a single build. Use a + /// different one for different tools, and clear it between + /// builds. + /// + /// \returns a \c StringError with the diagnostic output if clang errors + /// occurred, \c FullDependencies otherwise. + llvm::Expected + getFullDependencies(const tooling::CompilationDatabase &Compilations, + StringRef CWD, const llvm::StringSet<> &AlreadySeen); + private: - const ScanningOutputFormat Format; DependencyScanningWorker Worker; }; diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h @@ -63,6 +63,15 @@ const CompilationDatabase &CDB, DependencyConsumer &Consumer); + llvm::Error + computeDependenciesForClangInvocation(StringRef WorkingDirectory, + ArrayRef Arguments, + DependencyConsumer &Consumer); + + ScanningOutputFormat getFormat() const { return Format; } + + llvm::StringSet<> AlreadySeen; + private: IntrusiveRefCntPtr DiagOpts; std::shared_ptr PCHContainerOps; diff --git a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h b/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h --- a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h +++ b/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h @@ -28,14 +28,69 @@ class DependencyConsumer; +/// This is used to refer to a specific module. +/// +/// See \c ModuleDeps for details about what these members mean. +struct ClangModuleDep { + std::string ModuleName; + std::string ContextHash; +}; + struct ModuleDeps { + /// The name of the module. This may include `:` for C++20 module partitons, + /// or a header-name for C++20 header units. std::string ModuleName; - std::string ClangModuleMapFile; - std::string ModulePCMPath; + + /// The context hash of a module represents the set of compiler options that + /// may make one version of a module incompatible with another. This includes + /// things like language mode, predefined macros, header search paths, etc... + /// + /// Modules with the same name but a different \c ContextHash should be + /// treated as separate modules for the purpose of a build. std::string ContextHash; + + /// The path to the modulemap file which defines this module. + /// + /// This can be used to explicitly build this module. This file will + /// additionally appear in \c FileDeps as a dependency. + std::string ClangModuleMapFile; + + /// The path to where an implicit build would put the PCM for this module. + std::string ImplicitModulePCMPath; + + /// A collection of absolute paths to files that this module directly depends + /// on, not including transitive dependencies. llvm::StringSet<> FileDeps; - llvm::StringSet<> ClangModuleDeps; + + /// A list of modules this module directly depends on, not including + /// transitive dependencies. + /// + /// This may include modules with a different context hash when it can be + /// determined that the differences are benign for this compilation. + std::vector ClangModuleDeps; + + /// A partial command line that can be used to build this module. + /// + /// Call \c getFullCommandLine() to get a command line suitable for passing to + /// clang. + std::vector NonPathCommandLine; + + // Used to track which modules that were discovered were directly imported by + // the primary TU. bool ImportedByMainFile = false; + + /// Gets the full command line suitable for passing to clang. + /// + /// \param LookupPCMPath this function is called to fill in `-fmodule-file=` + /// flags and for the `-o` flag. It needs to return a + /// path for where the PCM for the given module is to + /// be located. + /// \param LookupModuleDeps this fucntion is called to collect the full + /// transitive set of dependencies for this + /// compilation. + std::vector getFullCommandLine( + std::function LookupPCMPath, + std::function LookupModuleDeps) const; }; class ModuleDepCollector; @@ -54,6 +109,8 @@ StringRef SearchPath, StringRef RelativePath, const Module *Imported, SrcMgr::CharacteristicKind FileType) override; + void moduleImport(SourceLocation ImportLoc, ModuleIdPath Path, + const Module *Imported) override; void EndOfMainFile() override; @@ -62,16 +119,18 @@ ModuleDepCollector &MDC; llvm::DenseSet DirectDeps; + void handleImport(const Module *Imported); void handleTopLevelModule(const Module *M); - void addAllSubmoduleDeps(const Module *M, ModuleDeps &MD); - void addModuleDep(const Module *M, ModuleDeps &MD); - - void addDirectDependencies(const Module *Mod); + void addAllSubmoduleDeps(const Module *M, ModuleDeps &MD, + llvm::DenseSet &AddedModules); + void addModuleDep(const Module *M, ModuleDeps &MD, + llvm::DenseSet &AddedModules); }; class ModuleDepCollector final : public DependencyCollector { public: - ModuleDepCollector(CompilerInstance &I, DependencyConsumer &C); + ModuleDepCollector(std::unique_ptr Opts, + CompilerInstance &I, DependencyConsumer &C); void attachToPreprocessor(Preprocessor &PP) override; void attachToASTReader(ASTReader &R) override; @@ -85,6 +144,7 @@ std::string ContextHash; std::vector MainDeps; std::unordered_map Deps; + std::unique_ptr Opts; }; } // end namespace dependencies diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp @@ -8,24 +8,45 @@ #include "clang/Tooling/DependencyScanning/DependencyScanningTool.h" #include "clang/Frontend/Utils.h" -#include "llvm/Support/JSON.h" - -static llvm::json::Array toJSONSorted(const llvm::StringSet<> &Set) { - std::vector Strings; - for (auto &&I : Set) - Strings.push_back(I.getKey()); - std::sort(Strings.begin(), Strings.end()); - return llvm::json::Array(Strings); -} namespace clang{ namespace tooling{ namespace dependencies{ +std::vector FullDependencies::getAdditionalCommandLine( + std::function LookupPCMPath, + std::function LookupModuleDeps) const { + std::vector Ret = AdditionalNonPathCommandLine; + llvm::StringSet<> AlreadyAdded; + + Ret.push_back("-fno-implicit-modules"); + Ret.push_back("-fno-implicit-module-maps"); + + std::function AddArgs = [&](const ModuleDeps &Mod) { + for (const ClangModuleDep &CMD : Mod.ClangModuleDeps) { + if (!AlreadyAdded.insert(CMD.ModuleName + CMD.ContextHash).second) + continue; + const ModuleDeps &M = LookupModuleDeps(CMD); + // Depth first traversal. + AddArgs(M); + Ret.push_back(("-fmodule-file=" + LookupPCMPath(CMD)).str()); + if (!M.ClangModuleMapFile.empty()) { + Ret.push_back("-fmodule-map-file=" + M.ClangModuleMapFile); + } + } + }; + + for (const ClangModuleDep &CMD : ClangModuleDeps) { + const ModuleDeps &MD = LookupModuleDeps(CMD); + AddArgs(MD); + } + + return Ret; +} + DependencyScanningTool::DependencyScanningTool( DependencyScanningService &Service) - : Format(Service.getFormat()), Worker(Service) { -} + : Worker(Service) {} llvm::Expected DependencyScanningTool::getDependencyFile( const tooling::CompilationDatabase &Compilations, StringRef CWD) { @@ -75,8 +96,33 @@ std::vector Dependencies; }; + // We expect a single command here because if a source file occurs multiple + // times in the original CDB, then `computeDependencies` would run the + // `DependencyScanningAction` once for every time the input occured in the + // CDB. Instead we split up the CDB into single command chunks to avoid this + // behavior. + assert(Compilations.getAllCompileCommands().size() == 1 && + "Expected a compilation database with a single command!"); + std::string Input = Compilations.getAllCompileCommands().front().Filename; + + MakeDependencyPrinterConsumer Consumer; + auto Result = Worker.computeDependencies(Input, CWD, Compilations, Consumer); + if (Result) + return std::move(Result); + std::string Output; + Consumer.printDependencies(Output); + return Output; +} + +llvm::Expected +DependencyScanningTool::getFullDependencies( + const tooling::CompilationDatabase &Compilations, StringRef CWD, + const llvm::StringSet<> &AlreadySeen) { class FullDependencyPrinterConsumer : public DependencyConsumer { public: + FullDependencyPrinterConsumer(const llvm::StringSet<> &AlreadySeen) + : AlreadySeen(AlreadySeen) {} + void handleFileDependency(const DependencyOutputOptions &Opts, StringRef File) override { Dependencies.push_back(File); @@ -90,55 +136,41 @@ ContextHash = std::move(Hash); } - void printDependencies(std::string &S, StringRef MainFile) { - // Sort the modules by name to get a deterministic order. - std::vector Modules; - for (auto &&Dep : ClangModuleDeps) - Modules.push_back(Dep.first); - std::sort(Modules.begin(), Modules.end()); + FullDependenciesResult getFullDependencies() const { + FullDependencies FD; - llvm::raw_string_ostream OS(S); + FD.ContextHash = std::move(ContextHash); - using namespace llvm::json; + FD.FileDeps.assign(Dependencies.begin(), Dependencies.end()); - Array Imports; - for (auto &&ModName : Modules) { - auto &MD = ClangModuleDeps[ModName]; + for (auto &&M : ClangModuleDeps) { + auto &MD = M.second; if (MD.ImportedByMainFile) - Imports.push_back(MD.ModuleName); + FD.ClangModuleDeps.push_back({MD.ModuleName, ContextHash}); } - Array Mods; - for (auto &&ModName : Modules) { - auto &MD = ClangModuleDeps[ModName]; - Object Mod{ - {"name", MD.ModuleName}, - {"file-deps", toJSONSorted(MD.FileDeps)}, - {"clang-module-deps", toJSONSorted(MD.ClangModuleDeps)}, - {"clang-modulemap-file", MD.ClangModuleMapFile}, - }; - Mods.push_back(std::move(Mod)); - } + FullDependenciesResult FDR; - Object O{ - {"input-file", MainFile}, - {"clang-context-hash", ContextHash}, - {"file-deps", Dependencies}, - {"clang-module-deps", std::move(Imports)}, - {"clang-modules", std::move(Mods)}, - }; + for (auto &&M : ClangModuleDeps) { + // TODO: Avoid handleModuleDependency even being called for modules + // we've already seen. + if (AlreadySeen.count(M.first)) + continue; + FDR.DiscoveredModules.push_back(std::move(M.second)); + } - S = llvm::formatv("{0:2},\n", Value(std::move(O))).str(); - return; + FDR.FullDeps = std::move(FD); + return FDR; } private: std::vector Dependencies; std::unordered_map ClangModuleDeps; std::string ContextHash; + std::vector OutputPaths; + const llvm::StringSet<> &AlreadySeen; }; - // We expect a single command here because if a source file occurs multiple // times in the original CDB, then `computeDependencies` would run the // `DependencyScanningAction` once for every time the input occured in the @@ -147,26 +179,13 @@ assert(Compilations.getAllCompileCommands().size() == 1 && "Expected a compilation database with a single command!"); std::string Input = Compilations.getAllCompileCommands().front().Filename; - - if (Format == ScanningOutputFormat::Make) { - MakeDependencyPrinterConsumer Consumer; - auto Result = - Worker.computeDependencies(Input, CWD, Compilations, Consumer); - if (Result) - return std::move(Result); - std::string Output; - Consumer.printDependencies(Output); - return Output; - } else { - FullDependencyPrinterConsumer Consumer; - auto Result = - Worker.computeDependencies(Input, CWD, Compilations, Consumer); - if (Result) - return std::move(Result); - std::string Output; - Consumer.printDependencies(Output, Input); - return Output; - } + + FullDependencyPrinterConsumer Consumer(AlreadySeen); + llvm::Error Result = + Worker.computeDependencies(Input, CWD, Compilations, Consumer); + if (Result) + return std::move(Result); + return Consumer.getFullDependencies(); } } // end namespace dependencies diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -142,11 +142,18 @@ Consumer)); break; case ScanningOutputFormat::Full: - Compiler.addDependencyCollector( - std::make_shared(Compiler, Consumer)); + Compiler.addDependencyCollector(std::make_shared( + std::move(Opts), Compiler, Consumer)); break; } + // Consider different header search and diagnostic options to create + // different modules. This avoids the unsound aliasing of module PCMs. + // + // TODO: Implement diagnostic bucketing and header search pruning to reduce + // the impact of strict context hashing. + Compiler.getHeaderSearchOpts().ModulesStrictContextHash = false; + Consumer.handleContextHash(Compiler.getInvocation().getModuleHash()); auto Action = std::make_unique(); @@ -215,3 +222,29 @@ return !Tool.run(&Action); }); } + +llvm::Error DependencyScanningWorker::computeDependenciesForClangInvocation( + StringRef WorkingDirectory, ArrayRef Arguments, + DependencyConsumer &Consumer) { + RealFS->setCurrentWorkingDirectory(WorkingDirectory); + return runWithDiags(DiagOpts.get(), [&](DiagnosticConsumer &DC) { + IntrusiveRefCntPtr DiagID = new DiagnosticIDs(); + IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); + DiagnosticsEngine Diags(DiagID, &*DiagOpts, &DC, /*ShouldOwnClient=*/false); + + llvm::opt::ArgStringList CC1Args; + for (const auto &Arg : Arguments) + CC1Args.push_back(Arg.c_str()); + std::unique_ptr Invocation( + newInvocation(&Diags, CC1Args)); + + DependencyScanningAction Action(WorkingDirectory, Consumer, DepFS, + PPSkipMappings.get(), Format); + + llvm::IntrusiveRefCntPtr FM = Files; + if (!FM) + FM = new FileManager(FileSystemOptions(), RealFS); + return Action.runInvocation(std::move(Invocation), FM.get(), + PCHContainerOps, &DC); + }); +} diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp --- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp +++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp @@ -17,6 +17,37 @@ using namespace tooling; using namespace dependencies; +std::vector ModuleDeps::getFullCommandLine( + std::function LookupPCMPath, + std::function LookupModuleDeps) const { + std::vector Ret = NonPathCommandLine; + llvm::StringSet<> AlreadyAdded; + + // TODO: Build full command line. That also means capturing the original + // command line into NonPathCommandLine. + + Ret.push_back("-fno-implicit-modules"); + Ret.push_back("-fno-implicit-module-maps"); + + std::function AddArgs = [&](const ModuleDeps &Mod) { + for (const ClangModuleDep &CMD : Mod.ClangModuleDeps) { + if (!AlreadyAdded.insert(CMD.ModuleName + CMD.ContextHash).second) + continue; + const ModuleDeps &M = LookupModuleDeps(CMD); + // Depth first traversal. + AddArgs(M); + Ret.push_back(("-fmodule-file=" + LookupPCMPath(CMD)).str()); + if (!M.ClangModuleMapFile.empty()) { + Ret.push_back("-fmodule-map-file=" + M.ClangModuleMapFile); + } + } + }; + + AddArgs(*this); + + return Ret; +} + void ModuleDepCollectorPP::FileChanged(SourceLocation Loc, FileChangeReason Reason, SrcMgr::CharacteristicKind FileType, @@ -50,7 +81,16 @@ // here as `FileChanged` will never see it. MDC.MainDeps.push_back(FileName); } + handleImport(Imported); +} + +void ModuleDepCollectorPP::moduleImport(SourceLocation ImportLoc, + ModuleIdPath Path, + const Module *Imported) { + handleImport(Imported); +} +void ModuleDepCollectorPP::handleImport(const Module *Imported) { if (!Imported) return; @@ -71,9 +111,8 @@ for (auto &&I : MDC.Deps) MDC.Consumer.handleModuleDependency(I.second); - DependencyOutputOptions Opts; for (auto &&I : MDC.MainDeps) - MDC.Consumer.handleFileDependency(Opts, I); + MDC.Consumer.handleFileDependency(*MDC.Opts, I); } void ModuleDepCollectorPP::handleTopLevelModule(const Module *M) { @@ -94,7 +133,7 @@ MD.ClangModuleMapFile = ModuleMap ? ModuleMap->getName() : ""; MD.ModuleName = M->getFullModuleName(); - MD.ModulePCMPath = M->getASTFile()->getName(); + MD.ImplicitModulePCMPath = M->getASTFile()->getName(); MD.ContextHash = MDC.ContextHash; serialization::ModuleFile *MF = MDC.Instance.getASTReader()->getModuleManager().lookup(M->getASTFile()); @@ -103,30 +142,38 @@ MD.FileDeps.insert(IF.getFile()->getName()); }); - addAllSubmoduleDeps(M, MD); + llvm::DenseSet AddedModules; + addAllSubmoduleDeps(M, MD, AddedModules); } -void ModuleDepCollectorPP::addAllSubmoduleDeps(const Module *M, - ModuleDeps &MD) { - addModuleDep(M, MD); +void ModuleDepCollectorPP::addAllSubmoduleDeps( + const Module *M, ModuleDeps &MD, + llvm::DenseSet &AddedModules) { + addModuleDep(M, MD, AddedModules); for (const Module *SubM : M->submodules()) - addAllSubmoduleDeps(SubM, MD); + addAllSubmoduleDeps(SubM, MD, AddedModules); } -void ModuleDepCollectorPP::addModuleDep(const Module *M, ModuleDeps &MD) { +void ModuleDepCollectorPP::addModuleDep( + const Module *M, ModuleDeps &MD, + llvm::DenseSet &AddedModules) { for (const Module *Import : M->Imports) { if (Import->getTopLevelModule() != M->getTopLevelModule()) { - MD.ClangModuleDeps.insert(Import->getTopLevelModuleName()); + if (AddedModules.insert(Import->getTopLevelModule()).second) + MD.ClangModuleDeps.push_back( + {Import->getTopLevelModuleName(), + Instance.getInvocation().getModuleHash()}); handleTopLevelModule(Import->getTopLevelModule()); } } } -ModuleDepCollector::ModuleDepCollector(CompilerInstance &I, - DependencyConsumer &C) - : Instance(I), Consumer(C), ContextHash(I.getInvocation().getModuleHash()) { -} +ModuleDepCollector::ModuleDepCollector( + std::unique_ptr Opts, CompilerInstance &I, + DependencyConsumer &C) + : Instance(I), Consumer(C), ContextHash(I.getInvocation().getModuleHash()), + Opts(std::move(Opts)) {} void ModuleDepCollector::attachToPreprocessor(Preprocessor &PP) { PP.addPPCallbacks(std::make_unique(Instance, *this)); diff --git a/clang/test/ClangScanDeps/Inputs/modules_cdb.json b/clang/test/ClangScanDeps/Inputs/modules_cdb.json --- a/clang/test/ClangScanDeps/Inputs/modules_cdb.json +++ b/clang/test/ClangScanDeps/Inputs/modules_cdb.json @@ -1,13 +1,22 @@ [ { "directory": "DIR", - "command": "clang -E -fsyntax-only DIR/modules_cdb_input2.cpp -IInputs -D INCLUDE_HEADER2 -MD -MF DIR/modules_cdb2.d -fmodules -fcxx-modules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps", + "command": "clang -E DIR/modules_cdb_input2.cpp -IInputs -D INCLUDE_HEADER2 -MD -MF DIR/modules_cdb2.d -fmodules -fcxx-modules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps", "file": "DIR/modules_cdb_input2.cpp" }, { "directory": "DIR", "command": "clang -E DIR/modules_cdb_input.cpp -IInputs -fmodules -fcxx-modules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps", "file": "DIR/modules_cdb_input.cpp" +}, +{ + "directory": "DIR", + "command": "clang -E DIR/modules_cdb_input.cpp -IInputs -fmodules -fcxx-modules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps -o a.o", + "file": "DIR/modules_cdb_input.cpp" +}, +{ + "directory": "DIR", + "command": "clang -E DIR/modules_cdb_input.cpp -IInputs -fmodules -fcxx-modules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps -o b.o", + "file": "DIR/modules_cdb_input.cpp" } ] - diff --git a/clang/test/ClangScanDeps/modules-full.cpp b/clang/test/ClangScanDeps/modules-full.cpp --- a/clang/test/ClangScanDeps/modules-full.cpp +++ b/clang/test/ClangScanDeps/modules-full.cpp @@ -11,67 +11,113 @@ // RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/modules_cdb.json > %t.cdb // // RUN: echo %t.dir > %t.result -// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 \ +// RUN: clang-scan-deps -compilation-database %t.cdb -j 4 \ // RUN: -mode preprocess-minimized-sources -format experimental-full >> %t.result -// RUN: cat %t.result | FileCheck --check-prefixes=CHECK %s +// RUN: cat %t.result | sed 's/\\/\//g' | FileCheck --check-prefixes=CHECK %s // FIXME: Backslash issues. // XFAIL: system-windows #include "header.h" -// CHECK: [[PREFIX:(.*[/\\])+[a-zA-Z0-9.-]+]] +// CHECK: [[PREFIX:.*]] +// CHECK-NEXT: { +// CHECK-NEXT: "modules": [ // CHECK-NEXT: { -// CHECK-NEXT: "clang-context-hash": "[[CONTEXT_HASH:[A-Z0-9]+]]", -// CHECK-NEXT: "clang-module-deps": [ -// CHECK-NEXT: "header1" -// CHECK-NEXT: ], -// CHECK-NEXT: "clang-modules": [ -// CHECK-NEXT: { -// CHECK-NEXT: "clang-module-deps": [ -// CHECK-NEXT: "header2" -// CHECK-NEXT: ], -// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap", -// CHECK-NEXT: "file-deps": [ -// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}header.h", -// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap" -// CHECK-NEXT: ], -// CHECK-NEXT: "name": "header1" -// CHECK-NEXT: }, -// CHECK-NEXT: { -// CHECK-NEXT: "clang-module-deps": [], -// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap", -// CHECK-NEXT: "file-deps": [ -// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}header2.h", -// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap" -// CHECK-NEXT: ], -// CHECK-NEXT: "name": "header2" -// CHECK-NEXT: } -// CHECK-NEXT: ], -// CHECK-NEXT: "file-deps": [ -// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}modules_cdb_input2.cpp" -// CHECK-NEXT: ], -// CHECK-NEXT: "input-file": "[[PREFIX]]{{[/\\]}}modules_cdb_input2.cpp" -// CHECK-NEXT:}, -// CHECK-NEXT:{ -// CHECK-NOT: "clang-context-hash": "[[CONTEXT_HASH]]", -// CHECK-NEXT: "clang-context-hash": "{{[A-Z0-9]+}}", -// CHECK-NEXT: "clang-module-deps": [ -// CHECK-NEXT: "header1" -// CHECK-NEXT: ], -// CHECK-NEXT: "clang-modules": [ -// CHECK-NEXT: { -// CHECK-NEXT: "clang-module-deps": [], -// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap", -// CHECK-NEXT: "file-deps": [ -// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}header.h", -// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap" -// CHECK-NEXT: ], -// CHECK-NEXT: "name": "header1" -// CHECK-NEXT: } -// CHECK-NEXT: ], -// CHECK-NEXT: "file-deps": [ -// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}modules_cdb_input.cpp" -// CHECK-NEXT: ], -// CHECK-NEXT: "input-file": "[[PREFIX]]{{[/\\]}}modules_cdb_input.cpp" -// CHECK-NEXT:}, +// CHECK-NEXT: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H1:[A-Z0-9]+]]", +// CHECK-NEXT: "module-name": "header2" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/Inputs/module.modulemap", +// CHECK-NEXT: "command-line": [], +// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H1]]", +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/Inputs/header.h", +// CHECK-NEXT: "[[PREFIX]]/Inputs/module.modulemap" +// CHECK-NEXT: ], +// CHECK-NEXT: "name": "header1" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "clang-module-deps": [], +// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/Inputs/module.modulemap", +// CHECK-NEXT: "command-line": [], +// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H2:[A-Z0-9]+]]", +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/Inputs/header.h", +// CHECK-NEXT: "[[PREFIX]]/Inputs/module.modulemap" +// CHECK-NEXT: ], +// CHECK-NEXT: "name": "header1" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "clang-module-deps": [], +// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/Inputs/module.modulemap", +// CHECK-NEXT: "command-line": [], +// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H1]]", +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/Inputs/header2.h", +// CHECK-NEXT: "[[PREFIX]]/Inputs/module.modulemap" +// CHECK-NEXT: ], +// CHECK-NEXT: "name": "header2" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "translation-units": [ +// CHECK-NEXT: { +// CHECK-NEXT: "clang-context-hash": "[[CONTEXT_HASH_H2]]", +// CHECK-NEXT: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H2]]", +// CHECK-NEXT: "module-name": "header1" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "command-line": [], +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/modules_cdb_input.cpp" +// CHECK-NEXT: ], +// CHECK-NEXT: "input-file": "[[PREFIX]]/modules_cdb_input.cpp" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "clang-context-hash": "[[CONTEXT_HASH_H2]]", +// CHECK-NEXT: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H2]]", +// CHECK-NEXT: "module-name": "header1" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "command-line": [], +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/modules_cdb_input.cpp" +// CHECK-NEXT: ], +// CHECK-NEXT: "input-file": "[[PREFIX]]/modules_cdb_input.cpp" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "clang-context-hash": "[[CONTEXT_HASH_H2]]", +// CHECK-NEXT: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H2]]", +// CHECK-NEXT: "module-name": "header1" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "command-line": [], +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/modules_cdb_input.cpp" +// CHECK-NEXT: ], +// CHECK-NEXT: "input-file": "[[PREFIX]]/modules_cdb_input.cpp" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "clang-context-hash": "[[CONTEXT_HASH_H1]]", +// CHECK-NEXT: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H1]]", +// CHECK-NEXT: "module-name": "header1" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "command-line": [], +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/modules_cdb_input2.cpp" +// CHECK-NEXT: ], +// CHECK-NEXT: "input-file": "[[PREFIX]]/modules_cdb_input2.cpp" +// CHECK-NEXT: } +// CHECK-NEXT: ] +// CHECK-NEXT: } diff --git a/clang/test/Index/Core/scan-deps.m b/clang/test/Index/Core/scan-deps.m new file mode 100644 --- /dev/null +++ b/clang/test/Index/Core/scan-deps.m @@ -0,0 +1,26 @@ +// RUN: rm -rf %t.mcp +// RUN: echo %S > %t.result +// RUN: c-index-test core --scan-deps %S -- %clang -cc1 -I %S/Inputs/module \ +// RUN: -fmodules -fmodules-cache-path=%t.mcp -fimplicit-module-maps \ +// RUN: -o FoE.o -x objective-c %s >> %t.result +// RUN: cat %t.result | sed 's/\\/\//g' | FileCheck %s + +@import ModA; + +// CHECK: [[PREFIX:.*]] +// CHECK-NEXT: module: +// CHECK-NEXT: name: ModA +// CHECK-NEXT: context-hash: [[CONTEXT_HASH:[A-Z0-9]+]] +// CHECK-NEXT: module-map-path: [[PREFIX]]/Inputs/module/module.modulemap +// CHECK-NEXT: module-deps: +// CHECK-NEXT: file-deps: +// CHECK-NEXT: [[PREFIX]]/Inputs/module/SubModA.h +// CHECK-NEXT: [[PREFIX]]/Inputs/module/ModA.h +// CHECK-NEXT: [[PREFIX]]/Inputs/module/module.modulemap +// CHECK-NEXT: [[PREFIX]]/Inputs/module/SubSubModA.h +// CHECK-NEXT: dependencies: +// CHECK-NEXT: context-hash: [[CONTEXT_HASH]] +// CHECK-NEXT: module-deps: +// CHECK-NEXT: ModA:[[CONTEXT_HASH]] +// CHECK-NEXT: file-deps: +// CHECK-NEXT: [[PREFIX]]/scan-deps.m diff --git a/clang/tools/c-index-test/CMakeLists.txt b/clang/tools/c-index-test/CMakeLists.txt --- a/clang/tools/c-index-test/CMakeLists.txt +++ b/clang/tools/c-index-test/CMakeLists.txt @@ -28,6 +28,7 @@ clangAST clangBasic clangCodeGen + clangDependencyScanning clangFrontend clangIndex clangSerialization diff --git a/clang/tools/c-index-test/core_main.cpp b/clang/tools/c-index-test/core_main.cpp --- a/clang/tools/c-index-test/core_main.cpp +++ b/clang/tools/c-index-test/core_main.cpp @@ -6,6 +6,7 @@ // //===----------------------------------------------------------------------===// +#include "clang-c/Dependencies.h" #include "clang/AST/Mangle.h" #include "clang/Basic/LangOptions.h" #include "clang/CodeGen/ObjectFilePCHContainerOperations.h" @@ -13,16 +14,17 @@ #include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/CompilerInvocation.h" #include "clang/Frontend/FrontendAction.h" -#include "clang/Index/IndexingAction.h" #include "clang/Index/IndexDataConsumer.h" +#include "clang/Index/IndexingAction.h" #include "clang/Index/USRGeneration.h" #include "clang/Lex/Preprocessor.h" #include "clang/Serialization/ASTReader.h" +#include "llvm/ADT/FunctionExtras.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/FileSystem.h" +#include "llvm/Support/PrettyStackTrace.h" #include "llvm/Support/Signals.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/Support/PrettyStackTrace.h" using namespace clang; using namespace clang::index; @@ -35,18 +37,23 @@ enum class ActionType { None, PrintSourceSymbols, + ScanDeps, }; namespace options { static cl::OptionCategory IndexTestCoreCategory("index-test-core options"); -static cl::opt -Action(cl::desc("Action:"), cl::init(ActionType::None), - cl::values( - clEnumValN(ActionType::PrintSourceSymbols, - "print-source-symbols", "Print symbols from source")), - cl::cat(IndexTestCoreCategory)); +static cl::opt Action( + cl::desc("Action:"), cl::init(ActionType::None), + cl::values(clEnumValN(ActionType::PrintSourceSymbols, + "print-source-symbols", "Print symbols from source"), + clEnumValN(ActionType::ScanDeps, "scan-deps", + "Get file dependencies")), + cl::cat(IndexTestCoreCategory)); + +static cl::list InputFiles(cl::Positional, + cl::desc("...")); static cl::extrahelp MoreHelp( "\nAdd \"-- \" at the end to setup the compiler " @@ -315,6 +322,65 @@ generateFullUSRForModule(Mod, OS); } +static int scanDeps(ArrayRef Args, std::string WorkingDirectory) { + CXDependencyScannerService Service = + clang_experimental_DependencyScannerService_create_v0( + CXDependencyMode_Full); + CXDependencyScannerWorker Worker = + clang_experimental_DependencyScannerWorker_create_v0(Service); + CXString Error; + + auto Callback = [&](CXModuleDependencySet *MDS) { + for (const auto &M : llvm::makeArrayRef(MDS->Modules, MDS->Count)) { + llvm::outs() << "module:\n" + << " name: " << clang_getCString(M.Name) << "\n" + << " context-hash: " << clang_getCString(M.ContextHash) + << "\n" + << " module-map-path: " << clang_getCString(M.ModuleMapPath) + << "\n" + << " module-deps:\n"; + for (const auto &ModuleName : + llvm::makeArrayRef(M.ModuleDeps->Strings, M.ModuleDeps->Count)) + llvm::outs() << " " << clang_getCString(ModuleName) << "\n"; + llvm::outs() << " file-deps:\n"; + for (const auto &FileName : + llvm::makeArrayRef(M.FileDeps->Strings, M.FileDeps->Count)) + llvm::outs() << " " << clang_getCString(FileName) << "\n"; + } + clang_experimental_ModuleDependencySet_dispose(MDS); + }; + + auto CB = + functionObjectToCCallbackRef(Callback); + + CXFileDependencies *Result = + clang_experimental_DependencyScannerWorker_getFileDependencies_v0( + Worker, Args.size(), Args.data(), WorkingDirectory.c_str(), + CB.Callback, CB.Context, &Error); + if (!Result) { + llvm::errs() << "error: failed to get dependencies\n"; + llvm::errs() << clang_getCString(Error) << "\n"; + clang_disposeString(Error); + return 1; + } + llvm::outs() << "dependencies:\n"; + llvm::outs() << " context-hash: " << clang_getCString(Result->ContextHash) + << "\n" + << " module-deps:\n"; + for (const auto &ModuleName : llvm::makeArrayRef(Result->ModuleDeps->Strings, + Result->ModuleDeps->Count)) + llvm::outs() << " " << clang_getCString(ModuleName) << "\n"; + llvm::outs() << " file-deps:\n"; + for (const auto &FileName : + llvm::makeArrayRef(Result->FileDeps->Strings, Result->FileDeps->Count)) + llvm::outs() << " " << clang_getCString(FileName) << "\n"; + + clang_experimental_FileDependencies_dispose(Result); + clang_experimental_DependencyScannerWorker_dispose_v0(Worker); + clang_experimental_DependencyScannerService_dispose_v0(Service); + return 0; +} + //===----------------------------------------------------------------------===// // Command line processing. //===----------------------------------------------------------------------===// @@ -358,5 +424,13 @@ options::IncludeLocals); } + if (options::Action == ActionType::ScanDeps) { + if (options::InputFiles.empty()) { + errs() << "error: missing working directory\n"; + return 1; + } + return scanDeps(CompArgs, options::InputFiles[0]); + } + return 0; } diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp --- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp +++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp @@ -15,6 +15,7 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/FileUtilities.h" #include "llvm/Support/InitLLVM.h" +#include "llvm/Support/JSON.h" #include "llvm/Support/Program.h" #include "llvm/Support/Signals.h" #include "llvm/Support/Threading.h" @@ -129,6 +130,11 @@ llvm::cl::init(ScanningOutputFormat::Make), llvm::cl::cat(DependencyScannerCategory)); +static llvm::cl::opt FullCommandLine( + "full-command-line", + llvm::cl::desc("Include the full command lines to use to build modules"), + llvm::cl::init(false), llvm::cl::cat(DependencyScannerCategory)); + llvm::cl::opt NumThreads("j", llvm::cl::Optional, llvm::cl::desc("Number of worker threads to use (default: use " @@ -189,9 +195,10 @@ /// based on the result. /// /// \returns True on error. -static bool handleDependencyToolResult(const std::string &Input, - llvm::Expected &MaybeFile, - SharedStream &OS, SharedStream &Errs) { +static bool +handleMakeDependencyToolResult(const std::string &Input, + llvm::Expected &MaybeFile, + SharedStream &OS, SharedStream &Errs) { if (!MaybeFile) { llvm::handleAllErrors( MaybeFile.takeError(), [&Input, &Errs](llvm::StringError &Err) { @@ -206,6 +213,184 @@ return false; } +static llvm::json::Array toJSONSorted(const llvm::StringSet<> &Set) { + std::vector Strings; + for (auto &&I : Set) + Strings.push_back(I.getKey()); + std::sort(Strings.begin(), Strings.end()); + return llvm::json::Array(Strings); +} + +static llvm::json::Array toJSONSorted(std::vector V) { + std::sort(V.begin(), V.end(), + [](const ClangModuleDep &A, const ClangModuleDep &B) { + return std::tie(A.ModuleName, A.ContextHash) < + std::tie(B.ModuleName, B.ContextHash); + }); + + llvm::json::Array Ret; + for (const ClangModuleDep &CMD : V) + Ret.push_back(llvm::json::Object( + {{"module-name", CMD.ModuleName}, {"context-hash", CMD.ContextHash}})); + return Ret; +} + +// Thread safe. +class FullDeps { +public: + void mergeDeps(StringRef Input, FullDependenciesResult FDR, + size_t InputIndex) { + const FullDependencies &FD = FDR.FullDeps; + + InputDeps ID; + ID.FileName = Input; + ID.ContextHash = std::move(FD.ContextHash); + ID.FileDeps = std::move(FD.FileDeps); + ID.ModuleDeps = std::move(FD.ClangModuleDeps); + + std::unique_lock ul(Lock); + for (const ModuleDeps &MD : FDR.DiscoveredModules) { + auto I = Modules.find({MD.ContextHash, MD.ModuleName, 0}); + if (I != Modules.end()) { + I->first.InputIndex = std::min(I->first.InputIndex, InputIndex); + continue; + } + Modules.insert( + I, {{MD.ContextHash, MD.ModuleName, InputIndex}, std::move(MD)}); + } + + if (FullCommandLine) + ID.AdditonalCommandLine = FD.getAdditionalCommandLine( + [&](ClangModuleDep CMD) { return lookupPCMPath(CMD); }, + [&](ClangModuleDep CMD) -> const ModuleDeps & { + return lookupModuleDeps(CMD); + }); + + Inputs.push_back(std::move(ID)); + } + + void printFullOutput(raw_ostream &OS) { + // Sort the modules by name to get a deterministic order. + std::vector ModuleNames; + for (auto &&M : Modules) + ModuleNames.push_back(M.first); + std::sort(ModuleNames.begin(), ModuleNames.end(), + [](const ContextModulePair &A, const ContextModulePair &B) { + return std::tie(A.ModuleName, A.InputIndex) < + std::tie(B.ModuleName, B.InputIndex); + }); + + std::sort(Inputs.begin(), Inputs.end(), + [](const InputDeps &A, const InputDeps &B) { + return A.FileName < B.FileName; + }); + + using namespace llvm::json; + + Array OutModules; + for (auto &&ModName : ModuleNames) { + auto &MD = Modules[ModName]; + Object O{ + {"name", MD.ModuleName}, + {"context-hash", MD.ContextHash}, + {"file-deps", toJSONSorted(MD.FileDeps)}, + {"clang-module-deps", toJSONSorted(MD.ClangModuleDeps)}, + {"clang-modulemap-file", MD.ClangModuleMapFile}, + {"command-line", + FullCommandLine + ? MD.getFullCommandLine( + [&](ClangModuleDep CMD) { return lookupPCMPath(CMD); }, + [&](ClangModuleDep CMD) -> const ModuleDeps & { + return lookupModuleDeps(CMD); + }) + : MD.NonPathCommandLine}, + }; + OutModules.push_back(std::move(O)); + } + + Array TUs; + for (auto &&I : Inputs) { + Object O{ + {"input-file", I.FileName}, + {"clang-context-hash", I.ContextHash}, + {"file-deps", I.FileDeps}, + {"clang-module-deps", toJSONSorted(I.ModuleDeps)}, + {"command-line", I.AdditonalCommandLine}, + }; + TUs.push_back(std::move(O)); + } + + Object Output{ + {"modules", std::move(OutModules)}, + {"translation-units", std::move(TUs)}, + }; + + OS << llvm::formatv("{0:2}\n", Value(std::move(Output))); + } + +private: + StringRef lookupPCMPath(ClangModuleDep CMD) { + return Modules[ContextModulePair{CMD.ContextHash, CMD.ModuleName, 0}] + .ImplicitModulePCMPath; + } + + const ModuleDeps &lookupModuleDeps(ClangModuleDep CMD) { + auto I = + Modules.find(ContextModulePair{CMD.ContextHash, CMD.ModuleName, 0}); + assert(I != Modules.end()); + return I->second; + }; + + struct ContextModulePair { + std::string ContextHash; + std::string ModuleName; + mutable size_t InputIndex; + + bool operator==(const ContextModulePair &Other) const { + return ContextHash == Other.ContextHash && ModuleName == Other.ModuleName; + } + }; + + struct ContextModulePairHasher { + std::size_t operator()(const ContextModulePair &CMP) const { + using llvm::hash_combine; + + return hash_combine(CMP.ContextHash, CMP.ModuleName); + } + }; + + struct InputDeps { + std::string FileName; + std::string ContextHash; + std::vector FileDeps; + std::vector ModuleDeps; + std::vector AdditonalCommandLine; + }; + + std::mutex Lock; + std::unordered_map + Modules; + std::vector Inputs; +}; + +static bool handleFullDependencyToolResult( + const std::string &Input, + llvm::Expected &MaybeFullDeps, FullDeps &FD, + size_t InputIndex, SharedStream &OS, SharedStream &Errs) { + if (!MaybeFullDeps) { + llvm::handleAllErrors( + MaybeFullDeps.takeError(), [&Input, &Errs](llvm::StringError &Err) { + Errs.applyLocked([&](raw_ostream &OS) { + OS << "Error while scanning dependencies for " << Input << ":\n"; + OS << Err.getMessage(); + }); + }); + return true; + } + FD.mergeDeps(Input, std::move(*MaybeFullDeps), InputIndex); + return false; +} + int main(int argc, const char **argv) { llvm::InitLLVM X(argc, argv); llvm::cl::HideUnrelatedOptions(DependencyScannerCategory); @@ -316,6 +501,7 @@ std::vector WorkerThreads; std::atomic HadErrors(false); + FullDeps FD; std::mutex Lock; size_t Index = 0; @@ -324,26 +510,38 @@ << " files using " << NumWorkers << " workers\n"; } for (unsigned I = 0; I < NumWorkers; ++I) { - auto Worker = [I, &Lock, &Index, &Inputs, &HadErrors, &WorkerTools, + auto Worker = [I, &Lock, &Index, &Inputs, &HadErrors, &FD, &WorkerTools, &DependencyOS, &Errs]() { + llvm::StringSet<> AlreadySeenModules; while (true) { const SingleCommandCompilationDatabase *Input; std::string Filename; std::string CWD; + size_t LocalIndex; // Take the next input. { std::unique_lock LockGuard(Lock); if (Index >= Inputs.size()) return; + LocalIndex = Index; Input = &Inputs[Index++]; tooling::CompileCommand Cmd = Input->getAllCompileCommands()[0]; Filename = std::move(Cmd.Filename); CWD = std::move(Cmd.Directory); } // Run the tool on it. - auto MaybeFile = WorkerTools[I]->getDependencyFile(*Input, CWD); - if (handleDependencyToolResult(Filename, MaybeFile, DependencyOS, Errs)) - HadErrors = true; + if (Format == ScanningOutputFormat::Make) { + auto MaybeFile = WorkerTools[I]->getDependencyFile(*Input, CWD); + if (handleMakeDependencyToolResult(Filename, MaybeFile, DependencyOS, + Errs)) + HadErrors = true; + } else { + auto MaybeFullDeps = WorkerTools[I]->getFullDependencies( + *Input, CWD, AlreadySeenModules); + if (handleFullDependencyToolResult(Filename, MaybeFullDeps, FD, + LocalIndex, DependencyOS, Errs)) + HadErrors = true; + } } }; #if LLVM_ENABLE_THREADS @@ -356,5 +554,8 @@ for (auto &W : WorkerThreads) W.join(); + if (Format == ScanningOutputFormat::Full) + FD.printFullOutput(llvm::outs()); + return HadErrors; } diff --git a/clang/tools/libclang/CDependencies.cpp b/clang/tools/libclang/CDependencies.cpp new file mode 100644 --- /dev/null +++ b/clang/tools/libclang/CDependencies.cpp @@ -0,0 +1,222 @@ +//===- CDependencies.cpp - Dependency Discovery C Interface ---------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Implements the dependency discovery interface. It provides a C library for +// the functionality that clang-scan-deps provides. +// +//===----------------------------------------------------------------------===// + +#include "CXString.h" + +#include "clang-c/Dependencies.h" + +#include "clang/Tooling/DependencyScanning/DependencyScanningService.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningTool.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h" + +using namespace clang; +using namespace clang::tooling::dependencies; + +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScanningService, + CXDependencyScannerService) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScanningWorker, + CXDependencyScannerWorker) + +inline ScanningOutputFormat unwrap(CXDependencyMode Format) { + switch (Format) { + case CXDependencyMode_Flat: + return ScanningOutputFormat::Make; + case CXDependencyMode_Full: + return ScanningOutputFormat::Full; + } +} + +void clang_experimental_ModuleDependencySet_dispose( + CXModuleDependencySet *MDS) { + for (int I = 0; I < MDS->Count; ++I) { + CXModuleDependency &MD = MDS->Modules[I]; + clang_disposeString(MD.Name); + clang_disposeString(MD.ContextHash); + clang_disposeString(MD.ModuleMapPath); + clang_disposeStringSet(MD.FileDeps); + clang_disposeStringSet(MD.ModuleDeps); + } + delete[] MDS->Modules; + delete MDS; +} + +CXDependencyScannerService +clang_experimental_DependencyScannerService_create_v0(CXDependencyMode Format) { + return wrap(new DependencyScanningService( + ScanningMode::MinimizedSourcePreprocessing, unwrap(Format), + /*ReuseFilemanager=*/false)); +} + +void clang_experimental_DependencyScannerService_dispose_v0( + CXDependencyScannerService Service) { + delete unwrap(Service); +} + +void clang_experimental_FileDependencies_dispose(CXFileDependencies *ID) { + clang_disposeString(ID->ContextHash); + clang_disposeStringSet(ID->FileDeps); + clang_disposeStringSet(ID->ModuleDeps); + delete ID; +} + +CXDependencyScannerWorker clang_experimental_DependencyScannerWorker_create_v0( + CXDependencyScannerService Service) { + return wrap(new DependencyScanningWorker(*unwrap(Service))); +} + +void clang_experimental_DependencyScannerWorker_dispose_v0( + CXDependencyScannerWorker Worker) { + delete unwrap(Worker); +} + +static CXFileDependencies * +getFlatDependencies(DependencyScanningWorker *Worker, + ArrayRef Compilation, + const char *WorkingDirectory, CXString *error) { + // TODO: Implement flat deps. + return nullptr; +} + +namespace { +class FullDependencyConsumer : public DependencyConsumer { +public: + FullDependencyConsumer(const llvm::StringSet<> &AlreadySeen) + : AlreadySeen(AlreadySeen) {} + + void handleFileDependency(const DependencyOutputOptions &Opts, + StringRef File) override { + if (OutputPaths.empty()) + OutputPaths = Opts.Targets; + Dependencies.push_back(File); + } + + void handleModuleDependency(ModuleDeps MD) override { + ClangModuleDeps[MD.ContextHash + MD.ModuleName] = std::move(MD); + } + + void handleContextHash(std::string Hash) override { + ContextHash = std::move(Hash); + } + + FullDependenciesResult getFullDependencies() const { + FullDependencies FD; + + FD.ContextHash = std::move(ContextHash); + + FD.FileDeps.assign(Dependencies.begin(), Dependencies.end()); + + for (auto &&M : ClangModuleDeps) { + auto &MD = M.second; + if (MD.ImportedByMainFile) + FD.ClangModuleDeps.push_back({MD.ModuleName, ContextHash}); + } + + FullDependenciesResult FDR; + + for (auto &&M : ClangModuleDeps) { + // TODO: Avoid handleModuleDependency even being called for modules + // we've already seen. + if (AlreadySeen.count(M.first)) + continue; + FDR.DiscoveredModules.push_back(std::move(M.second)); + } + + FDR.FullDeps = std::move(FD); + return FDR; + } + +private: + std::vector Dependencies; + std::unordered_map ClangModuleDeps; + std::string ContextHash; + std::vector OutputPaths; + const llvm::StringSet<> &AlreadySeen; +}; +} // namespace + +static CXFileDependencies *getFullDependencies( + DependencyScanningWorker *Worker, ArrayRef Compilation, + const char *WorkingDirectory, CXModuleDiscoveredCallback *MDC, + void *Context, CXString *error) { + FullDependencyConsumer Consumer(Worker->AlreadySeen); + llvm::Error Result = Worker->computeDependenciesForClangInvocation( + WorkingDirectory, Compilation, Consumer); + + if (Result) { + std::string Str; + llvm::raw_string_ostream OS(Str); + llvm::handleAllErrors(std::move(Result), + [&](const llvm::ErrorInfoBase &EI) { EI.log(OS); }); + *error = cxstring::createDup(OS.str()); + return nullptr; + } + + FullDependenciesResult FDR = Consumer.getFullDependencies(); + + if (!FDR.DiscoveredModules.empty()) { + CXModuleDependencySet *MDS = new CXModuleDependencySet; + MDS->Count = FDR.DiscoveredModules.size(); + MDS->Modules = new CXModuleDependency[MDS->Count]; + for (int I = 0; I < MDS->Count; ++I) { + CXModuleDependency &M = MDS->Modules[I]; + const ModuleDeps &MD = FDR.DiscoveredModules[I]; + M.Name = cxstring::createDup(MD.ModuleName); + M.ContextHash = cxstring::createDup(MD.ContextHash); + M.ModuleMapPath = cxstring::createDup(MD.ClangModuleMapFile); + M.FileDeps = cxstring::createSet(MD.FileDeps); + std::vector Modules; + for (const ClangModuleDep &CMD : MD.ClangModuleDeps) + Modules.push_back(CMD.ModuleName + ":" + CMD.ContextHash); + M.ModuleDeps = cxstring::createSet(Modules); + M.BuildArguments = cxstring::createSet(std::vector{}); + } + MDC(Context, MDS); + } + + const FullDependencies &FD = FDR.FullDeps; + CXFileDependencies *FDeps = new CXFileDependencies; + FDeps->ContextHash = cxstring::createDup(FD.ContextHash); + FDeps->FileDeps = cxstring::createSet(FD.FileDeps); + std::vector Modules; + for (const ClangModuleDep &CMD : FD.ClangModuleDeps) + Modules.push_back(CMD.ModuleName + ":" + CMD.ContextHash); + FDeps->ModuleDeps = cxstring::createSet(Modules); + FDeps->AdditionalArguments = cxstring::createSet(std::vector{}); + return FDeps; +} + +CXFileDependencies * +clang_experimental_DependencyScannerWorker_getFileDependencies_v0( + CXDependencyScannerWorker W, int argc, const char *const *argv, + const char *WorkingDirectory, CXModuleDiscoveredCallback *MDC, + void *Context, CXString *error) { + if (!W || argc < 2) + return nullptr; + if (error) + *error = cxstring::createEmpty(); + + DependencyScanningWorker *Worker = unwrap(W); + + std::vector Compilation; + if (StringRef(argv[1]) == "-cc1") + for (int i = 2; i < argc; ++i) + Compilation.push_back(argv[i]); + else { + return nullptr; // TODO: Run the driver to get -cc1 args. + } + + if (Worker->getFormat() == ScanningOutputFormat::Full) + return getFullDependencies(Worker, Compilation, WorkingDirectory, MDC, + Context, error); + return getFlatDependencies(Worker, Compilation, WorkingDirectory, error); +} diff --git a/clang/tools/libclang/CMakeLists.txt b/clang/tools/libclang/CMakeLists.txt --- a/clang/tools/libclang/CMakeLists.txt +++ b/clang/tools/libclang/CMakeLists.txt @@ -1,6 +1,7 @@ set(SOURCES ARCMigrate.cpp BuildSystem.cpp + CDependencies.cpp CIndex.cpp CIndexCXX.cpp CIndexCodeCompletion.cpp @@ -37,6 +38,7 @@ set(LIBS clangAST clangBasic + clangDependencyScanning clangDriver clangFrontend clangIndex diff --git a/clang/tools/libclang/CXString.h b/clang/tools/libclang/CXString.h --- a/clang/tools/libclang/CXString.h +++ b/clang/tools/libclang/CXString.h @@ -17,6 +17,7 @@ #include "clang/Basic/LLVM.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSet.h" #include "llvm/Support/Compiler.h" #include #include @@ -69,6 +70,8 @@ CXStringSet *createSet(const std::vector &Strings); +CXStringSet *createSet(const llvm::StringSet<> &Strings); + /// A string pool used for fast allocation/deallocation of strings. class CXStringPool { public: diff --git a/clang/tools/libclang/CXString.cpp b/clang/tools/libclang/CXString.cpp --- a/clang/tools/libclang/CXString.cpp +++ b/clang/tools/libclang/CXString.cpp @@ -119,6 +119,15 @@ return Set; } +CXStringSet *createSet(const llvm::StringSet<> &Strings) { + CXStringSet *Set = new CXStringSet; + Set->Count = Strings.size(); + Set->Strings = new CXString[Set->Count]; + int I = 0; + for (auto SI = Strings.begin(), SE = Strings.end(); SI != SE; ++SI) + Set->Strings[I++] = createDup(SI->getKey()); + return Set; +} //===----------------------------------------------------------------------===// // String pools. diff --git a/clang/tools/libclang/libclang.exports b/clang/tools/libclang/libclang.exports --- a/clang/tools/libclang/libclang.exports +++ b/clang/tools/libclang/libclang.exports @@ -157,6 +157,13 @@ clang_equalRanges clang_equalTypes clang_executeOnThread +clang_experimental_DependencyScannerService_create_v0 +clang_experimental_DependencyScannerService_dispose_v0 +clang_experimental_DependencyScannerWorker_create_v0 +clang_experimental_DependencyScannerWorker_dispose_v0 +clang_experimental_DependencyScannerWorker_getFileDependencies_v0 +clang_experimental_FileDependencies_dispose +clang_experimental_ModuleDependencySet_dispose clang_findIncludesInFile clang_findIncludesInFileWithBlock clang_findReferencesInFile diff --git a/llvm/include/llvm/ADT/FunctionExtras.h b/llvm/include/llvm/ADT/FunctionExtras.h --- a/llvm/include/llvm/ADT/FunctionExtras.h +++ b/llvm/include/llvm/ADT/FunctionExtras.h @@ -287,6 +287,37 @@ } }; +template struct FunctionObjectCallback { + void *Context; + CallTy *Callback; +}; + +namespace detail { +template +struct functionObjectToCCallbackRefImpl; + +template +struct functionObjectToCCallbackRefImpl { + static FunctionObjectCallback impl(FuncTy &F) { + auto Func = +[](void *C, Args... V) -> Ret { + return (*reinterpret_cast *>(C))( + std::forward(V)...); + }; + + return {&F, Func}; + } +}; +} // namespace detail + +/// Returns a function pointer and context pair suitable for use as a C +/// callback. +/// +/// \param F the function object to turn into a C callback. The returned +/// callback has the same lifetime as F. +template +auto functionObjectToCCallbackRef(FuncTy &F) { + return detail::functionObjectToCCallbackRefImpl::impl(F); +} } // end namespace llvm #endif // LLVM_ADT_FUNCTION_H