diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h @@ -28,7 +28,17 @@ llvm::function_ref; /// Graph of modular dependencies. -using ModuleDepsGraph = std::vector; +class ModuleDepsGraph { + /// This is keeping the scan instance alive so that \c ModuleDeps can use it + /// to compute some properties lazily. + std::shared_ptr ScanInstance; + +public: + ModuleDepsGraph(std::shared_ptr ScanInstance = nullptr) + : ScanInstance(std::move(ScanInstance)) {} + + std::vector MDs; +}; /// The full dependencies and module graph for a specific input. struct TranslationUnitDeps { @@ -170,6 +180,10 @@ ContextHash = std::move(Hash); } + void handleScanInstance(std::shared_ptr CI) override { + ScanInstance = std::move(CI); + } + TranslationUnitDeps takeTranslationUnitDeps(); ModuleDepsGraph takeModuleGraphDeps(); @@ -180,6 +194,7 @@ std::vector DirectModuleDeps; std::vector Commands; std::string ContextHash; + std::shared_ptr ScanInstance; std::vector OutputPaths; const llvm::DenseSet &AlreadySeen; }; diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h @@ -58,6 +58,8 @@ virtual void handleDirectModuleDependency(ModuleID MD) = 0; + virtual void handleScanInstance(std::shared_ptr CI) = 0; + virtual void handleContextHash(std::string Hash) = 0; }; diff --git a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h b/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h --- a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h +++ b/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h @@ -104,7 +104,21 @@ DiagnosticSerializationFile, }; -struct ModuleDeps { +class ModuleDepCollector; +class ModuleDepCollectorPP; + +class ModuleDeps { + /// This facilitates the lazy computation of file deps and build arguments. + ModuleDepCollector *MDC; + /// Storage for the lazily-computed file dependencies. + std::optional> FileDeps; + /// Storage for the lazily-computed build arguments. + std::optional> BuildArguments; + + friend ModuleDepCollector; + friend ModuleDepCollectorPP; + +public: /// The identifier of the module. ModuleID ID; @@ -117,9 +131,10 @@ /// additionally appear in \c FileDeps as a dependency. std::string ClangModuleMapFile; - /// A collection of absolute paths to files that this module directly depends - /// on, not including transitive dependencies. - llvm::StringSet<> FileDeps; + /// Compute/get the set of absolute paths to files that this module directly + /// depends on, not including transitive dependencies. Must be first called + /// during the lifetime of the parent dependency graph. + const llvm::StringSet<> &getFileDeps(); /// A collection of absolute paths to module map files that this module needs /// to know about. The ordering is significant. @@ -136,13 +151,12 @@ /// determined that the differences are benign for this compilation. std::vector ClangModuleDeps; - /// Compiler invocation that can be used to build this module. Does not - /// include argv[0]. - std::vector BuildArguments; + /// Compute/get the compiler invocation that can be used to build this module. + /// Does not include argv[0]. Must be first called during the lifetime of the + /// parent dependency graph. + const std::vector &getBuildArguments(); }; -class ModuleDepCollector; - /// Callback that records textual includes and direct modular includes/imports /// during preprocessing. At the end of the main file, it also collects /// transitive modular dependencies and passes everything to the @@ -216,6 +230,13 @@ private: friend ModuleDepCollectorPP; + friend ModuleDeps; + + /// Information we keep to be able to compute some ModuleDeps info lazily. + struct LazyModuleDepsInfo { + serialization::ModuleFile *MF; + CompilerInvocation CI; + }; /// The compiler instance for scanning the current translation unit. CompilerInstance &ScanInstance; @@ -235,6 +256,8 @@ /// Secondary mapping for \c ModularDeps allowing lookup by ModuleID without /// a preprocessor. Storage owned by \c ModularDeps. llvm::DenseMap ModuleDepsByID; + /// Mapping for lazy computation of some ModuleDeps info. + llvm::DenseMap LazyModuleDepsInfoByID; /// Direct modular dependencies that have already been built. llvm::MapVector DirectPrebuiltModularDeps; /// Working set of direct modular dependencies. @@ -280,6 +303,11 @@ void addModuleFiles(CompilerInvocation &CI, ArrayRef ClangModuleDeps) const; + /// Compute the file deps and store them into \c MD. + void addFileDeps(ModuleDeps &MD); + /// Compute the build arguments and store them into \c MD. + void addBuildArguments(ModuleDeps &MD); + /// Add paths that require looking up outputs to the given dependencies. void addOutputPaths(CompilerInvocation &CI, ModuleDeps &Deps); diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp @@ -41,6 +41,7 @@ void handleModuleDependency(ModuleDeps MD) override {} void handleDirectModuleDependency(ModuleID ID) override {} void handleContextHash(std::string Hash) override {} + void handleScanInstance(std::shared_ptr CI) override {} void printDependencies(std::string &S) { assert(Opts && "Handled dependency output options."); @@ -172,22 +173,14 @@ TU.FileDeps = std::move(Dependencies); TU.PrebuiltModuleDeps = std::move(PrebuiltModuleDeps); TU.Commands = std::move(Commands); - - for (auto &&M : ClangModuleDeps) { - auto &MD = M.second; - // TODO: Avoid handleModuleDependency even being called for modules - // we've already seen. - if (AlreadySeen.count(M.first)) - continue; - TU.ModuleGraph.push_back(std::move(MD)); - } + TU.ModuleGraph = takeModuleGraphDeps(); TU.ClangModuleDeps = std::move(DirectModuleDeps); return TU; } ModuleDepsGraph FullDependencyConsumer::takeModuleGraphDeps() { - ModuleDepsGraph ModuleGraph; + ModuleDepsGraph ModuleGraph(std::move(ScanInstance)); for (auto &&M : ClangModuleDeps) { auto &MD = M.second; @@ -195,7 +188,7 @@ // we've already seen. if (AlreadySeen.count(M.first)) continue; - ModuleGraph.push_back(std::move(MD)); + ModuleGraph.MDs.push_back(std::move(MD)); } return ModuleGraph; diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -165,7 +165,8 @@ Scanned = true; // Create a compiler instance to handle the actual work. - ScanInstanceStorage.emplace(std::move(PCHContainerOps)); + ScanInstanceStorage = + std::make_shared(std::move(PCHContainerOps)); CompilerInstance &ScanInstance = *ScanInstanceStorage; ScanInstance.setInvocation(std::move(Invocation)); @@ -268,6 +269,8 @@ if (Result) setLastCC1Arguments(std::move(OriginalInvocation)); + Consumer.handleScanInstance(ScanInstanceStorage); + return Result; } @@ -299,7 +302,7 @@ bool EagerLoadModules; bool DisableFree; std::optional ModuleName; - std::optional ScanInstanceStorage; + std::shared_ptr ScanInstanceStorage; std::shared_ptr MDC; std::vector LastCC1Arguments; bool Scanned = false; diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp --- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp +++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp @@ -467,20 +467,6 @@ serialization::ModuleFile *MF = MDC.ScanInstance.getASTReader()->getModuleManager().lookup( M->getASTFile()); - MDC.ScanInstance.getASTReader()->visitInputFileInfos( - *MF, /*IncludeSystem=*/true, - [&](const serialization::InputFileInfo &IFI, bool IsSystem) { - // __inferred_module.map is the result of the way in which an implicit - // module build handles inferred modules. It adds an overlay VFS with - // this file in the proper directory and relies on the rest of Clang to - // handle it like normal. With explicitly built modules we don't need - // to play VFS tricks, so replace it with the correct module map. - if (StringRef(IFI.Filename).endswith("__inferred_module.map")) { - MDC.addFileDep(MD, ModuleMap->getName()); - return; - } - MDC.addFileDep(MD, IFI.Filename); - }); llvm::DenseSet SeenDeps; addAllSubmodulePrebuiltDeps(M, MD, SeenDeps); @@ -510,7 +496,9 @@ // Finish the compiler invocation. Requires dependencies and the context hash. MDC.addOutputPaths(CI, MD); - MD.BuildArguments = CI.getCC1CommandLine(); + // Wire up lazy info computation. + MD.MDC = &MDC; + MDC.LazyModuleDepsInfoByID.insert({MD.ID, {MF, std::move(CI)}}); return MD.ID; } @@ -643,5 +631,50 @@ void ModuleDepCollector::addFileDep(ModuleDeps &MD, StringRef Path) { llvm::SmallString<256> Storage; Path = makeAbsoluteAndPreferred(ScanInstance, Path, Storage); - MD.FileDeps.insert(Path); + MD.FileDeps->insert(Path); +} + +void ModuleDepCollector::addFileDeps(ModuleDeps &MD) { + auto It = LazyModuleDepsInfoByID.find(MD.ID); + assert(It != LazyModuleDepsInfoByID.end()); + + MD.FileDeps = llvm::StringSet<>{}; + + ScanInstance.getASTReader()->visitInputFileInfos( + *It->second.MF, /*IncludeSystem=*/true, + [&](const serialization::InputFileInfo &IFI, bool IsSystem) { + // __inferred_module.map is the result of the way in which an implicit + // module build handles inferred modules. It adds an overlay VFS with + // this file in the proper directory and relies on the rest of Clang to + // handle it like normal. With explicitly built modules we don't need + // to play VFS tricks, so replace it with the correct module map. + if (StringRef(IFI.Filename).endswith("__inferred_module.map")) { + FileManager &FileMgr = ScanInstance.getFileManager(); + auto ModuleMap = FileMgr.getOptionalFileRef(MD.ClangModuleMapFile); + assert(ModuleMap && "Module map file of a dependency still exists"); + addFileDep(MD, ModuleMap->getName()); + return; + } + addFileDep(MD, IFI.Filename); + }); +} + +void ModuleDepCollector::addBuildArguments(ModuleDeps &MD) { + auto It = LazyModuleDepsInfoByID.find(MD.ID); + assert(It != LazyModuleDepsInfoByID.end()); + MD.BuildArguments = It->second.CI.getCC1CommandLine(); +} + +const llvm::StringSet<> &ModuleDeps::getFileDeps() { + if (FileDeps) + return *FileDeps; + MDC->addFileDeps(*this); + return *FileDeps; +} + +const std::vector &ModuleDeps::getBuildArguments() { + if (BuildArguments) + return *BuildArguments; + MDC->addBuildArguments(*this); + return *BuildArguments; } diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp --- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp +++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp @@ -351,14 +351,24 @@ } void mergeDeps(ModuleDepsGraph Graph, size_t InputIndex) { - std::unique_lock ul(Lock); - for (const ModuleDeps &MD : Graph) { - auto I = Modules.find({MD.ID, 0}); - if (I != Modules.end()) { - I->first.InputIndex = std::min(I->first.InputIndex, InputIndex); - continue; + std::vector NewMDs; + { + std::unique_lock ul(Lock); + for (const ModuleDeps &MD : Graph.MDs) { + auto I = Modules.find({MD.ID, 0}); + if (I != Modules.end()) { + I->first.InputIndex = std::min(I->first.InputIndex, InputIndex); + continue; + } + auto NewIt = Modules.insert(I, {{MD.ID, InputIndex}, std::move(MD)}); + NewMDs.push_back(&NewIt->second); } - Modules.insert(I, {{MD.ID, InputIndex}, std::move(MD)}); + } + // Eagerly compute the lazy members before the graph goes out of scope. + // This is somewhat costly, so do it outside the critical section. + for (ModuleDeps *MD : NewMDs) { + (void)MD->getBuildArguments(); + (void)MD->getFileDeps(); } } @@ -382,7 +392,7 @@ /*ShouldOwnClient=*/false); for (auto &&M : Modules) - if (roundTripCommand(M.second.BuildArguments, *Diags)) + if (roundTripCommand(M.second.getBuildArguments(), *Diags)) return true; for (auto &&I : Inputs) @@ -408,10 +418,10 @@ Object O{ {"name", MD.ID.ModuleName}, {"context-hash", MD.ID.ContextHash}, - {"file-deps", toJSONSorted(MD.FileDeps)}, + {"file-deps", toJSONSorted(MD.getFileDeps())}, {"clang-module-deps", toJSONSorted(MD.ClangModuleDeps)}, {"clang-modulemap-file", MD.ClangModuleMapFile}, - {"command-line", MD.BuildArguments}, + {"command-line", MD.getBuildArguments()}, }; OutModules.push_back(std::move(O)); }