[clang][deps] Minor ModuleDepCollector refactorings NFC

[benlangmuir]

Summary

• Factor module map and module file path functions out
• Use a secondary mapping to lookup module deps by ID instead of the preprocessor module map.
• Sink DirectPrebuiltModularDeps into MDC.

[jansvoboda11]

LGTM with a nit.

clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp

277

Nit: I think you should be able to drop std::make_pair() and use brace-initialization. Also applies to the other call to insert above.

clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h

struct ModuleID {
   /// treated as separate modules for the purpose of a build.
   std::string ContextHash;
 
   bool operator==(const ModuleID &Other) const {
     return ModuleName == Other.ModuleName && ContextHash == Other.ContextHash;
   }
 };
 
-struct ModuleIDHasher {
-  std::size_t operator()(const ModuleID &MID) const {
-    return llvm::hash_combine(MID.ModuleName, MID.ContextHash);
-  }
-};
-
 /// An output from a module compilation, such as the path of the module file.
 enum class ModuleOutputKind {
   /// The module file (.pcm). Required.
   ModuleFile,
   /// The path of the dependency file (.d), if any.
   DependencyFile,
   /// The null-separated list of names to use as the targets in the dependency
   /// file, if any. Defaults to the value of \c ModuleFile, as in the driver.
public:
 
   void EndOfMainFile() override;
 
 private:
   /// The parent dependency collector.
   ModuleDepCollector &MDC;
   /// Working set of direct modular dependencies.
   llvm::SetVector<const Module *> DirectModularDeps;
-  /// Working set of direct modular dependencies that have already been built.
-  llvm::SetVector<const Module *> DirectPrebuiltModularDeps;
 
   void handleImport(const Module *Imported);
 
   /// Adds direct modular dependencies that have already been built to the
   /// ModuleDeps instance.
   void
   addAllSubmodulePrebuiltDeps(const Module *M, ModuleDeps &MD,
                               llvm::DenseSet<const Module *> &SeenSubmodules);
private:
   std::string MainFile;
   /// Hash identifying the compilation conditions of the current TU.
   std::string ContextHash;
   /// Non-modular file dependencies. This includes the main source file and
   /// textually included header files.
   std::vector<std::string> FileDeps;
   /// Direct and transitive modular dependencies of the main source file.
   llvm::MapVector<const Module *, std::unique_ptr<ModuleDeps>> ModularDeps;
+  /// Secondary mapping for \c ModularDeps allowing lookup by ModuleID without
+  /// a preprocessor. Storage owned by \c ModularDeps.
+  llvm::DenseMap<ModuleID, ModuleDeps *> ModuleDepsByID;
+  /// Direct modular dependencies that have already been built.
+  llvm::MapVector<const Module *, PrebuiltModuleDep> DirectPrebuiltModularDeps;
   /// Options that control the dependency output generation.
   std::unique_ptr<DependencyOutputOptions> Opts;
   /// The original Clang invocation passed to dependency scanner.
   CompilerInvocation OriginalInvocation;
   /// Whether to optimize the modules' command-line arguments.
   bool OptimizeArgs;
   /// Whether to set up command-lines to load PCM files eagerly.
   bool EagerLoadModules;
 
   /// Checks whether the module is known as being prebuilt.
   bool isPrebuiltModule(const Module *M);
 
   /// Adds \p Path to \c FileDeps, making it absolute if necessary.
   void addFileDep(StringRef Path);
   /// Adds \p Path to \c MD.FileDeps, making it absolute if necessary.
   void addFileDep(ModuleDeps &MD, StringRef Path);
 
   /// Constructs a CompilerInvocation that can be used to build the given
   /// module, excluding paths to discovered modular dependencies that are yet to
   /// be built.
   CompilerInvocation makeInvocationForModuleBuildWithoutOutputs(
       const ModuleDeps &Deps,
       llvm::function_ref<void(CompilerInvocation &)> Optimize) const;
 
+  /// Add module map files to the invocation, if needed.
+  void addModuleMapFiles(CompilerInvocation &CI,
+                         ArrayRef<ModuleID> ClangModuleDeps) const;
+  /// Add module files (pcm) to the invocation, if needed.
+  void addModuleFiles(CompilerInvocation &CI,
+                      ArrayRef<ModuleID> ClangModuleDeps) const;
+
   /// Add paths that require looking up outputs to the given dependencies.
   void addOutputPaths(CompilerInvocation &CI, ModuleDeps &Deps);
+
+  /// Compute the context hash for \p Deps, and create the mapping
+  /// \c ModuleDepsByID[Deps.ID] = &Deps.
+  void associateWithContextHash(const CompilerInvocation &CI, ModuleDeps &Deps);
 };
 
 } // end namespace dependencies
 } // end namespace tooling
 } // end namespace clang
 
+namespace llvm {
+template <> struct DenseMapInfo<clang::tooling::dependencies::ModuleID> {
+  using ModuleID = clang::tooling::dependencies::ModuleID;
+  static inline ModuleID getEmptyKey() { return ModuleID{"", ""}; }
+  static inline ModuleID getTombstoneKey() {
+    return ModuleID{"~", "~"}; // ~ is not a valid module name or context hash
+  }
+  static unsigned getHashValue(const ModuleID &ID) {
+    return hash_combine(ID.ModuleName, ID.ContextHash);
+  }
+  static bool isEqual(const ModuleID &LHS, const ModuleID &RHS) {
+    return LHS == RHS;
+  }
+};
+} // namespace llvm
+
 #endif // LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_MODULEDEPCOLLECTOR_H

clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp

static std::vector<std::string> splitString(std::string S, char Separator) {
   return Result;
 }
 
 void ModuleDepCollector::addOutputPaths(CompilerInvocation &CI,
                                         ModuleDeps &Deps) {
   // These are technically *inputs* to the compilation, but we populate them
   // here in order to make \c getModuleContextHash() independent of
   // \c lookupModuleOutput().
-  for (ModuleID MID : Deps.ClangModuleDeps) {
-    auto PCMPath =
-        Consumer.lookupModuleOutput(MID, ModuleOutputKind::ModuleFile);
-    if (EagerLoadModules)
-      CI.getFrontendOpts().ModuleFiles.push_back(PCMPath);
-    else
-      CI.getHeaderSearchOpts().PrebuiltModuleFiles.insert(
-          {MID.ModuleName, PCMPath});
-  }
+  addModuleFiles(CI, Deps.ClangModuleDeps);
 
   CI.getFrontendOpts().OutputFile =
       Consumer.lookupModuleOutput(Deps.ID, ModuleOutputKind::ModuleFile);
   if (!CI.getDiagnosticOpts().DiagnosticSerializationFile.empty())
     CI.getDiagnosticOpts().DiagnosticSerializationFile =
         Consumer.lookupModuleOutput(
             Deps.ID, ModuleOutputKind::DiagnosticSerializationFile);
   if (!CI.getDependencyOutputOpts().OutputFile.empty()) {
ModuleDepCollector::makeInvocationForModuleBuildWithoutOutputs(
   CI.getFrontendOpts().IsSystemModule = Deps.IsSystem;
 
   // Inputs
   InputKind ModuleMapInputKind(CI.getFrontendOpts().DashX.getLanguage(),
                                InputKind::Format::ModuleMap);
   CI.getFrontendOpts().Inputs.emplace_back(Deps.ClangModuleMapFile,
                                            ModuleMapInputKind);
   CI.getFrontendOpts().ModuleMapFiles = Deps.ModuleMapFileDeps;
+  addModuleMapFiles(CI, Deps.ClangModuleDeps);
 
   // Report the prebuilt modules this module uses.
   for (const auto &PrebuiltModule : Deps.PrebuiltModuleDeps)
     CI.getFrontendOpts().ModuleFiles.push_back(PrebuiltModule.PCMFile);
 
-  if (!EagerLoadModules) {
-    ModuleMap &ModMap =
-        ScanInstance.getPreprocessor().getHeaderSearchInfo().getModuleMap();
-    for (ModuleID MID : Deps.ClangModuleDeps) {
-      const Module *M = ModMap.findModule(MID.ModuleName);
-      assert(M && "Modular dependency not found");
-      auto MDeps = ModularDeps.find(M);
-      assert(MDeps != ModularDeps.end() && "Inconsistent dependency info");
-      CI.getFrontendOpts().ModuleMapFiles.push_back(
-          MDeps->second->ClangModuleMapFile);
-    }
-  }
-
   // Remove any macro definitions that are explicitly ignored.
   if (!CI.getHeaderSearchOpts().ModulesIgnoreMacros.empty()) {
     llvm::erase_if(
         CI.getPreprocessorOpts().Macros,
         [&CI](const std::pair<std::string, bool> &Def) {
           StringRef MacroDef = Def.first;
           return CI.getHeaderSearchOpts().ModulesIgnoreMacros.contains(
               llvm::CachedHashString(MacroDef.split('=').first));
ModuleDepCollector::makeInvocationForModuleBuildWithoutOutputs(
   // reporting dependencies to the clients. Let's make sure we're using
   // **strict** context hash in order to prevent accidental sharing of
   // incompatible modules (e.g. with differences in search paths).
   CI.getHeaderSearchOpts().ModulesStrictContextHash = true;
 
   return CI;
 }
 
+void ModuleDepCollector::addModuleMapFiles(
+    CompilerInvocation &CI, ArrayRef<ModuleID> ClangModuleDeps) const {
+  if (EagerLoadModules)
+    return; // Only pcm is needed for eager load.
+
+  for (const ModuleID &MID : ClangModuleDeps) {
+    ModuleDeps *MD = ModuleDepsByID.lookup(MID);
+    assert(MD && "Inconsistent dependency info");
+    CI.getFrontendOpts().ModuleMapFiles.push_back(MD->ClangModuleMapFile);
+  }
+}
+
+void ModuleDepCollector::addModuleFiles(
+    CompilerInvocation &CI, ArrayRef<ModuleID> ClangModuleDeps) const {
+  for (const ModuleID &MID : ClangModuleDeps) {
+    std::string PCMPath =
+        Consumer.lookupModuleOutput(MID, ModuleOutputKind::ModuleFile);
+    if (EagerLoadModules)
+      CI.getFrontendOpts().ModuleFiles.push_back(std::move(PCMPath));
+    else
+      CI.getHeaderSearchOpts().PrebuiltModuleFiles.insert(
+          {MID.ModuleName, std::move(PCMPath)});
+  }
+}
+
 static std::string getModuleContextHash(const ModuleDeps &MD,
                                         const CompilerInvocation &CI,
                                         bool EagerLoadModules) {
   llvm::HashBuilder<llvm::TruncatedBLAKE3<16>,
                     llvm::support::endianness::native>
       HashBuilder;
   SmallString<32> Scratch;
 
static std::string getModuleContextHash(const ModuleDeps &MD,
 
   llvm::BLAKE3Result<16> Hash = HashBuilder.final();
   std::array<uint64_t, 2> Words;
   static_assert(sizeof(Hash) == sizeof(Words), "Hash must match Words");
   std::memcpy(Words.data(), Hash.data(), sizeof(Hash));
   return toString(llvm::APInt(sizeof(Words) * 8, Words), 36, /*Signed=*/false);
 }
 
+void ModuleDepCollector::associateWithContextHash(const CompilerInvocation &CI,
+                                                  ModuleDeps &Deps) {
+  Deps.ID.ContextHash = getModuleContextHash(Deps, CI, EagerLoadModules);
+  bool Inserted = ModuleDepsByID.insert({Deps.ID, &Deps}).second;
+  (void)Inserted;
+  assert(Inserted && "duplicate module mapping");
+}
+
 void ModuleDepCollectorPP::FileChanged(SourceLocation Loc,
                                        FileChangeReason Reason,
                                        SrcMgr::CharacteristicKind FileType,
                                        FileID PrevFID) {
   if (Reason != PPCallbacks::EnterFile)
     return;
 
   // This has to be delayed as the context hash can change at the start of

 
 void ModuleDepCollectorPP::handleImport(const Module *Imported) {
   if (!Imported)
     return;
 
   const Module *TopLevelModule = Imported->getTopLevelModule();
 
   if (MDC.isPrebuiltModule(TopLevelModule))
-    DirectPrebuiltModularDeps.insert(TopLevelModule);
+    MDC.DirectPrebuiltModularDeps.insert(
+        {TopLevelModule, PrebuiltModuleDep{TopLevelModule}});
   else
     DirectModularDeps.insert(TopLevelModule);
 }
 
 void ModuleDepCollectorPP::EndOfMainFile() {
   FileID MainFileID = MDC.ScanInstance.getSourceManager().getMainFileID();
   MDC.MainFile = std::string(MDC.ScanInstance.getSourceManager()
                                  .getFileEntryForID(MainFileID)
void ModuleDepCollectorPP::EndOfMainFile() {
   MDC.Consumer.handleDependencyOutputOpts(*MDC.Opts);
 
   for (auto &&I : MDC.ModularDeps)
     MDC.Consumer.handleModuleDependency(*I.second);
 
   for (auto &&I : MDC.FileDeps)
     MDC.Consumer.handleFileDependency(I);
 
-  for (auto &&I : DirectPrebuiltModularDeps)
-    MDC.Consumer.handlePrebuiltModuleDependency(PrebuiltModuleDep{I});
+  for (auto &&I : MDC.DirectPrebuiltModularDeps)
+    MDC.Consumer.handlePrebuiltModuleDependency(I.second);
 }
 
 ModuleID ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
   assert(M == M->getTopLevelModule() && "Expected top level module!");
 
   // If this module has been handled already, just return its ID.
   auto ModI = MDC.ModularDeps.insert({M, nullptr});
   if (!ModI.second)
ModuleID ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
 
   CompilerInvocation CI = MDC.makeInvocationForModuleBuildWithoutOutputs(
       MD, [&](CompilerInvocation &BuildInvocation) {
         if (MDC.OptimizeArgs)
           optimizeHeaderSearchOpts(BuildInvocation.getHeaderSearchOpts(),
                                    *MDC.ScanInstance.getASTReader(), *MF);
       });
 
-  // Compute the context hash from the inputs. Requires dependencies.
-  MD.ID.ContextHash = getModuleContextHash(MD, CI, MDC.EagerLoadModules);
+  MDC.associateWithContextHash(CI, MD);
+
   // Finish the compiler invocation. Requires dependencies and the context hash.
   MDC.addOutputPaths(CI, MD);
 
   MD.BuildArguments = CI.getCC1CommandLine();
 
   return MD.ID;
 }