[clang][deps] Generate '-fmodule-file=' only for direct dependencies

[jansvoboda11]

Summary

The clang-scan-deps tool currently generates -fmodule-file= command-line arguments for the whole transitive closure of modular dependencies. This is not necessary, we only need to provide the direct dependencies on the command line. Information about transitive dependencies is stored within the .pcm files of direct dependencies. This makes the command lines shorter, but should be a NFC otherwise (unless there are bugs in the loading mechanism for explicit modules).

Depends on D120465.

[Bigcheese]

lgtm

[jansvoboda11]

Rebase

[jansvoboda11]

Rebase on top of D120465

clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h

struct FullDependencies {
   std::vector<std::string> OriginalCommandLine;
 
   /// Get the full command line.
   ///
   /// \param LookupPCMPath This function is called to fill in "-fmodule-file="
   ///                      arguments and the "-o" argument. It needs to return
   ///                      a path for where the PCM for the given module is to
   ///                      be located.
-  /// \param LookupModuleDeps This function is called to collect the full
-  ///                         transitive set of dependencies for this
-  ///                         compilation.
-  std::vector<std::string> getCommandLine(
-      std::function<StringRef(ModuleID)> LookupPCMPath,
-      std::function<const ModuleDeps &(ModuleID)> LookupModuleDeps) const;
+  std::vector<std::string>
+  getCommandLine(std::function<StringRef(ModuleID)> LookupPCMPath) const;
 
   /// Get the full command line, excluding -fmodule-file=" arguments.
   std::vector<std::string> getCommandLineWithoutModulePaths() const;
 
   /// Get additional arguments suitable for appending to the original Clang
   /// command line, excluding "-fmodule-file=" arguments.
   std::vector<std::string> getAdditionalArgsWithoutModulePaths() const;
 };

clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h

struct ModuleDeps {
   CompilerInvocation BuildInvocation;
 
   /// Gets the canonical command line suitable for passing to clang.
   ///
   /// \param LookupPCMPath This function is called to fill in "-fmodule-file="
   ///                      arguments and the "-o" argument. It needs to return
   ///                      a path for where the PCM for the given module is to
   ///                      be located.
-  /// \param LookupModuleDeps This function is called to collect the full
-  ///                         transitive set of dependencies for this
-  ///                         compilation.
   std::vector<std::string> getCanonicalCommandLine(
-      std::function<StringRef(ModuleID)> LookupPCMPath,
-      std::function<const ModuleDeps &(ModuleID)> LookupModuleDeps) const;
+      std::function<StringRef(ModuleID)> LookupPCMPath) const;
 
   /// Gets the canonical command line suitable for passing to clang, excluding
   /// "-fmodule-file=" and "-o" arguments.
   std::vector<std::string> getCanonicalCommandLineWithoutModulePaths() const;
 };
 
-namespace detail {
-/// Collect the paths of PCM for the modules in \c Modules transitively.
-void collectPCMPaths(
-    llvm::ArrayRef<ModuleID> Modules,
-    std::function<StringRef(ModuleID)> LookupPCMPath,
-    std::function<const ModuleDeps &(ModuleID)> LookupModuleDeps,
-    std::vector<std::string> &PCMPaths);
-} // namespace detail
-
 class ModuleDepCollector;
 
 /// Callback that records textual includes and direct modular includes/imports
 /// during preprocessing. At the end of the main file, it also collects
 /// transitive modular dependencies and passes everything to the
 /// \c DependencyConsumer of the parent \c ModuleDepCollector.
 class ModuleDepCollectorPP final : public PPCallbacks {
 public:

clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp

 //===- DependencyScanningTool.cpp - clang-scan-deps service ---------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "clang/Tooling/DependencyScanning/DependencyScanningTool.h"
 #include "clang/Frontend/Utils.h"
 
 namespace clang {
 namespace tooling {
 namespace dependencies {
 
 std::vector<std::string> FullDependencies::getCommandLine(
-    std::function<StringRef(ModuleID)> LookupPCMPath,
-    std::function<const ModuleDeps &(ModuleID)> LookupModuleDeps) const {
+    std::function<StringRef(ModuleID)> LookupPCMPath) const {
   std::vector<std::string> Ret = getCommandLineWithoutModulePaths();
 
-  std::vector<std::string> PCMPaths;
-  dependencies::detail::collectPCMPaths(ClangModuleDeps, LookupPCMPath,
-                                        LookupModuleDeps, PCMPaths);
-  for (const std::string &PCMPath : PCMPaths)
-    Ret.push_back("-fmodule-file=" + PCMPath);
+  for (ModuleID MID : ClangModuleDeps)
+    Ret.push_back(("-fmodule-file=" + LookupPCMPath(MID)).str());
 
   return Ret;
 }
 
 std::vector<std::string>
 FullDependencies::getCommandLineWithoutModulePaths() const {
   std::vector<std::string> Args = OriginalCommandLine;
 

clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp

serializeCompilerInvocation(const CompilerInvocation &CI) {
   SmallVector<const char *, 32> Args{"-cc1"};
   CI.generateCC1CommandLine(Args, SA);
 
   // Convert arguments to the return type.
   return std::vector<std::string>{Args.begin(), Args.end()};
 }
 
 std::vector<std::string> ModuleDeps::getCanonicalCommandLine(
-    std::function<StringRef(ModuleID)> LookupPCMPath,
-    std::function<const ModuleDeps &(ModuleID)> LookupModuleDeps) const {
+    std::function<StringRef(ModuleID)> LookupPCMPath) const {
   CompilerInvocation CI(BuildInvocation);
   FrontendOptions &FrontendOpts = CI.getFrontendOpts();
 
   InputKind ModuleMapInputKind(FrontendOpts.DashX.getLanguage(),
                                InputKind::Format::ModuleMap);
   FrontendOpts.Inputs.emplace_back(ClangModuleMapFile, ModuleMapInputKind);
   FrontendOpts.OutputFile = std::string(LookupPCMPath(ID));
 
-  dependencies::detail::collectPCMPaths(ClangModuleDeps, LookupPCMPath,
-                                        LookupModuleDeps,
-                                        FrontendOpts.ModuleFiles);
+  for (ModuleID MID : ClangModuleDeps)
+    FrontendOpts.ModuleFiles.emplace_back(LookupPCMPath(MID));
 
   return serializeCompilerInvocation(CI);
 }
 
 std::vector<std::string>
 ModuleDeps::getCanonicalCommandLineWithoutModulePaths() const {
   return serializeCompilerInvocation(BuildInvocation);
 }
 
-void dependencies::detail::collectPCMPaths(
-    llvm::ArrayRef<ModuleID> Modules,
-    std::function<StringRef(ModuleID)> LookupPCMPath,
-    std::function<const ModuleDeps &(ModuleID)> LookupModuleDeps,
-    std::vector<std::string> &PCMPaths) {
-  llvm::StringSet<> AlreadyAdded;
-
-  std::function<void(llvm::ArrayRef<ModuleID>)> AddArgs =
-      [&](llvm::ArrayRef<ModuleID> Modules) {
-        for (const ModuleID &MID : Modules) {
-          if (!AlreadyAdded.insert(MID.ModuleName + MID.ContextHash).second)
-            continue;
-          const ModuleDeps &M = LookupModuleDeps(MID);
-          // Depth first traversal.
-          AddArgs(M.ClangModuleDeps);
-          PCMPaths.push_back(LookupPCMPath(MID).str());
-        }
-      };
-
-  AddArgs(Modules);
-}
-
 void ModuleDepCollectorPP::FileChanged(SourceLocation Loc,
                                        FileChangeReason Reason,
                                        SrcMgr::CharacteristicKind FileType,
                                        FileID PrevFID) {
   if (Reason != PPCallbacks::EnterFile)
     return;
 
   // This has to be delayed as the context hash can change at the start of

clang/test/ClangScanDeps/modules-full.cpp

 // CHECK-NEXT:           "context-hash": "[[HASH_H1_DINCLUDE]]",
 // CHECK-NEXT:           "module-name": "header1"
 // CHECK-NEXT:         }
 // CHECK-NEXT:       ],
 // CHECK-NEXT:       "command-line": [
 // CHECK:              "-fno-implicit-modules"
 // CHECK-NEXT:         "-fno-implicit-module-maps"
 // CHECK-NO-ABS-NOT:   "-fmodule-file={{.*}}"
-// CHECK-ABS-NEXT:     "-fmodule-file=[[PREFIX]]/module-cache{{(_clangcl)?}}/[[HASH_H2_DINCLUDE]]/header2-{{[A-Z0-9]+}}.pcm"
 // CHECK-ABS-NEXT:     "-fmodule-file=[[PREFIX]]/module-cache{{(_clangcl)?}}/[[HASH_H1_DINCLUDE]]/header1-{{[A-Z0-9]+}}.pcm"
-// CHECK-CUSTOM-NEXT:  "-fmodule-file=[[PREFIX]]/custom/[[HASH_H2_DINCLUDE]]/header2-{{[A-Z0-9]+}}.pcm"
 // CHECK-CUSTOM-NEXT:  "-fmodule-file=[[PREFIX]]/custom/[[HASH_H1_DINCLUDE]]/header1-{{[A-Z0-9]+}}.pcm"
 // CHECK-NEXT:       ],
 // CHECK-NEXT:       "file-deps": [
 // CHECK-NEXT:         "[[PREFIX]]/modules_cdb_input2.cpp"
 // CHECK-NEXT:       ],
 // CHECK-NEXT:       "input-file": "[[PREFIX]]/modules_cdb_input2.cpp"
 // CHECK-NEXT:     }
 // CHECK-NEXT:   ]
 // CHECK-NEXT: }

clang/test/ClangScanDeps/modules-pch.c

 // CHECK-PCH-NEXT:           "context-hash": "[[HASH_MOD_PCH]]",
 // CHECK-PCH-NEXT:           "module-name": "ModPCH"
 // CHECK-PCH-NEXT:         }
 // CHECK-PCH-NEXT:       ],
 // CHECK-PCH-NEXT:       "command-line": [
 // CHECK-PCH:              "-fno-implicit-modules",
 // CHECK-PCH-NEXT:         "-fno-implicit-module-maps",
 // CHECK-PCH-NEXT:         "-fmodule-file=[[PREFIX]]/build/[[HASH_MOD_COMMON_1]]/ModCommon1-{{.*}}.pcm",
-// CHECK-PCH-NEXT:         "-fmodule-file=[[PREFIX]]/build/[[HASH_MOD_COMMON_2]]/ModCommon2-{{.*}}.pcm",
 // CHECK-PCH-NEXT:         "-fmodule-file=[[PREFIX]]/build/[[HASH_MOD_PCH]]/ModPCH-{{.*}}.pcm"
 // CHECK-PCH-NEXT:       ],
 // CHECK-PCH-NEXT:       "file-deps": [
 // CHECK-PCH-NEXT:         "[[PREFIX]]/pch.h"
 // CHECK-PCH-NEXT:       ],
 // CHECK-PCH-NEXT:       "input-file": "[[PREFIX]]/pch.h"
 // CHECK-PCH-NEXT:     }
 // CHECK-PCH-NEXT:   ]

clang/tools/clang-scan-deps/ClangScanDeps.cpp

for (const ModuleDeps &MD : FDR.DiscoveredModules) {
         I->first.InputIndex = std::min(I->first.InputIndex, InputIndex);
         continue;
       }
       Modules.insert(I, {{MD.ID, InputIndex}, std::move(MD)});
     }
 
     ID.CommandLine = GenerateModulesPathArgs
                          ? FD.getCommandLine(
-                               [&](ModuleID MID) { return lookupPCMPath(MID); },
-                               [&](ModuleID MID) -> const ModuleDeps & {
-                                 return lookupModuleDeps(MID);
-                               })
+                               [&](ModuleID MID) { return lookupPCMPath(MID); })
                          : FD.getCommandLineWithoutModulePaths();
 
     Inputs.push_back(std::move(ID));
   }
 
   void printFullOutput(raw_ostream &OS) {
     // Sort the modules by name to get a deterministic order.
     std::vector<IndexedModuleID> ModuleIDs;
for (auto &&ModID : ModuleIDs) {
           {"name", MD.ID.ModuleName},
           {"context-hash", MD.ID.ContextHash},
           {"file-deps", toJSONSorted(MD.FileDeps)},
           {"clang-module-deps", toJSONSorted(MD.ClangModuleDeps)},
           {"clang-modulemap-file", MD.ClangModuleMapFile},
           {"command-line",
            GenerateModulesPathArgs
                ? MD.getCanonicalCommandLine(
-                     [&](ModuleID MID) { return lookupPCMPath(MID); },
-                     [&](ModuleID MID) -> const ModuleDeps & {
-                       return lookupModuleDeps(MID);
-                     })
+                     [&](ModuleID MID) { return lookupPCMPath(MID); })
                : MD.getCanonicalCommandLineWithoutModulePaths()},
       };
       OutModules.push_back(std::move(O));
     }
 
     Array TUs;
     for (auto &&I : Inputs) {
       Object O{
void printFullOutput(raw_ostream &OS) {
 
     OS << llvm::formatv("{0:2}\n", Value(std::move(Output)));
   }
 
 private:
   StringRef lookupPCMPath(ModuleID MID) {
     auto PCMPath = PCMPaths.insert({MID, ""});
     if (PCMPath.second)
-      PCMPath.first->second = constructPCMPath(lookupModuleDeps(MID));
+      PCMPath.first->second = constructPCMPath(MID);
     return PCMPath.first->second;
   }
 
   /// Construct a path for the explicitly built PCM.
-  std::string constructPCMPath(const ModuleDeps &MD) const {
+  std::string constructPCMPath(ModuleID MID) const {
+    auto MDIt = Modules.find(IndexedModuleID{MID, 0});
+    assert(MDIt != Modules.end());
+    const ModuleDeps &MD = MDIt->second;
+
     StringRef Filename = llvm::sys::path::filename(MD.ImplicitModulePCMPath);
 
     SmallString<256> ExplicitPCMPath(
         !ModuleFilesDir.empty()
             ? ModuleFilesDir
             : MD.BuildInvocation.getHeaderSearchOpts().ModuleCachePath);
     llvm::sys::path::append(ExplicitPCMPath, MD.ID.ContextHash, Filename);
     return std::string(ExplicitPCMPath);
   }
 
-  const ModuleDeps &lookupModuleDeps(ModuleID MID) {
-    auto I = Modules.find(IndexedModuleID{MID, 0});
-    assert(I != Modules.end());
-    return I->second;
-  };
-
   struct IndexedModuleID {
     ModuleID ID;
     mutable size_t InputIndex;
 
     bool operator==(const IndexedModuleID &Other) const {
       return ID.ModuleName == Other.ID.ModuleName &&
              ID.ContextHash == Other.ID.ContextHash;
     }