[clang][deps] Compute command-lines for dependencies immediately

[benlangmuir]

Summary

Instead of delaying the generation of command-lines to after all
dependencies are reported, compute them immediately. This is partly in
preparation for splitting the TU driver command into its constituent cc1
and other jobs, but it also just simplifies working with the compiler
invocation for modules if they are not "without paths".

Also change the computation of the default output path in
clang-scan-deps to scrape the implicit module cache from the
command-line rather than get it from the dependency, since that is now
unavailable at the time we make the callback.

[jansvoboda11]

LGTM overall with some minor nitpicks.

clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h

103

Creating a type alias might make this more readable & easier to refactor later.

clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp

102

Can you point out in a comment that this value is just temporary for context hash computation and will be replaced by real path later on?

clang/tools/clang-scan-deps/ClangScanDeps.cpp

403

Can you split this into separate patch?

[benlangmuir]

Per review

• Add typedef for callback type
• Add comment about mapping paths to "-"
• Update commit message for module cache path computation change

[benlangmuir]

clang/tools/clang-scan-deps/ClangScanDeps.cpp

403

This is hard because we don't have the association between input command-line and module dependencies in the previous code. Per our offline discussion I've mentioned this in the commit message instead.

clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h

 #include "llvm/ADT/StringMap.h"
 #include <string>
 #include <vector>
 
 namespace clang {
 namespace tooling {
 namespace dependencies {
 
+/// A callback to lookup module outputs for "-fmodule-file=", "-o" etc.
+using LookupModuleOutputCallback =
+    llvm::function_ref<std::string(const ModuleID &, ModuleOutputKind)>;
+
 /// The full dependencies and module graph for a specific input.
 struct FullDependencies {
   /// The identifier of the C++20 module this translation unit exports.
   ///
   /// If the translation unit is not a module then \c ID.ModuleName is empty.
   ModuleID ID;
 
   /// A collection of absolute paths to files that this translation unit
   /// directly depends on, not including transitive dependencies.
   std::vector<std::string> FileDeps;
 
   /// A collection of prebuilt modules this translation unit directly depends
   /// on, not including transitive dependencies.
   std::vector<PrebuiltModuleDep> PrebuiltModuleDeps;
 
   /// A list of modules this translation unit directly depends on, not including
   /// transitive dependencies.
   ///
   /// This may include modules with a different context hash when it can be
   /// determined that the differences are benign for this compilation.
   std::vector<ModuleID> ClangModuleDeps;
 
-  /// The original command line of the TU (excluding the compiler executable).
-  std::vector<std::string> OriginalCommandLine;
-
-  /// Get the full command line.
-  ///
-  /// \param LookupModuleOutput This function is called to fill in
-  ///                           "-fmodule-file=", "-o" and other output
-  ///                           arguments for dependencies.
-  std::vector<std::string> getCommandLine(
-      llvm::function_ref<std::string(const ModuleID &, ModuleOutputKind)>
-          LookupModuleOutput) const;
+  /// The command line of the TU (excluding the compiler executable).
+  std::vector<std::string> CommandLine;
 };
 
 struct FullDependenciesResult {
   FullDependencies FullDeps;
   std::vector<ModuleDeps> DiscoveredModules;
 };
 
 /// The high-level implementation of the dependency discovery tool that runs on
public:
   /// modules which have already been seen. If \p ModuleName isn't empty, this
   /// function returns the full dependency information of module \p ModuleName.
   ///
   /// \param AlreadySeen This stores modules which have previously been
   ///                    reported. Use the same instance for all calls to this
   ///                    function for a single \c DependencyScanningTool in a
   ///                    single build. Use a different one for different tools,
   ///                    and clear it between builds.
+  /// \param LookupModuleOutput This function is called to fill in
+  ///                           "-fmodule-file=", "-o" and other output
+  ///                           arguments for dependencies.
   ///
   /// \returns a \c StringError with the diagnostic output if clang errors
   /// occurred, \c FullDependencies otherwise.
   llvm::Expected<FullDependenciesResult>
   getFullDependencies(const std::vector<std::string> &CommandLine,
                       StringRef CWD, const llvm::StringSet<> &AlreadySeen,
+                      LookupModuleOutputCallback LookupModuleOutput,
                       llvm::Optional<StringRef> ModuleName = None);
 
 private:
   DependencyScanningWorker Worker;
 };
 
 class FullDependencyConsumer : public DependencyConsumer {
 public:
-  FullDependencyConsumer(const llvm::StringSet<> &AlreadySeen)
-      : AlreadySeen(AlreadySeen) {}
+  FullDependencyConsumer(const llvm::StringSet<> &AlreadySeen,
+                         LookupModuleOutputCallback LookupModuleOutput)
+      : AlreadySeen(AlreadySeen), LookupModuleOutput(LookupModuleOutput) {}
 
   void handleDependencyOutputOpts(const DependencyOutputOptions &) override {}
 
   void handleFileDependency(StringRef File) override {
     Dependencies.push_back(std::string(File));
   }
 
   void handlePrebuiltModuleDependency(PrebuiltModuleDep PMD) override {
     PrebuiltModuleDeps.emplace_back(std::move(PMD));
   }
 
   void handleModuleDependency(ModuleDeps MD) override {
     ClangModuleDeps[MD.ID.ContextHash + MD.ID.ModuleName] = std::move(MD);
   }
 
   void handleContextHash(std::string Hash) override {
     ContextHash = std::move(Hash);
   }
 
+  std::string lookupModuleOutput(const ModuleID &ID,
+                                 ModuleOutputKind Kind) override {
+    return LookupModuleOutput(ID, Kind);
+  }
+
   FullDependenciesResult getFullDependencies(
       const std::vector<std::string> &OriginalCommandLine) const;
 
 private:
   std::vector<std::string> Dependencies;
   std::vector<PrebuiltModuleDep> PrebuiltModuleDeps;
   llvm::MapVector<std::string, ModuleDeps, llvm::StringMap<unsigned>>
       ClangModuleDeps;
   std::string ContextHash;
   std::vector<std::string> OutputPaths;
   const llvm::StringSet<> &AlreadySeen;
+  LookupModuleOutputCallback LookupModuleOutput;
 };
 
 } // end namespace dependencies
 } // end namespace tooling
 } // end namespace clang
 
 #endif // LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGTOOL_H

clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h

public:
 
   virtual void handleFileDependency(StringRef Filename) = 0;
 
   virtual void handlePrebuiltModuleDependency(PrebuiltModuleDep PMD) = 0;
 
   virtual void handleModuleDependency(ModuleDeps MD) = 0;
 
   virtual void handleContextHash(std::string Hash) = 0;
+
+  virtual std::string lookupModuleOutput(const ModuleID &ID,
+                                         ModuleOutputKind Kind) = 0;
 };
 
 /// An individual dependency scanning worker that is able to run on its own
 /// thread.
 ///
 /// The worker computes the dependencies for the input files by preprocessing
 /// sources either using a fast mode where the source files are minimized, or
 /// using the regular processing run.

clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h

struct ModuleDeps {
   /// This may include modules with a different context hash when it can be
   /// determined that the differences are benign for this compilation.
   std::vector<ModuleID> ClangModuleDeps;
 
   // Used to track which modules that were discovered were directly imported by
   // the primary TU.
   bool ImportedByMainFile = false;
 
-  /// Whether the TU had a dependency file. The path in \c BuildInvocation is
-  /// cleared to avoid leaking the specific path from the TU into the module.
-  bool HadDependencyFile = false;
-
-  /// Whether the TU had serialized diagnostics. The path in \c BuildInvocation
-  /// is cleared to avoid leaking the specific path from the TU into the module.
-  bool HadSerializedDiagnostics = false;
-
   /// Compiler invocation that can be used to build this module (without paths).
   CompilerInvocation BuildInvocation;
 
   /// Gets the canonical command line suitable for passing to clang.
-  ///
-  /// \param LookupModuleOutput This function is called to fill in
-  ///                           "-fmodule-file=", "-o" and other output
-  ///                           arguments.
-  std::vector<std::string> getCanonicalCommandLine(
-      llvm::function_ref<std::string(const ModuleID &, ModuleOutputKind)>
-          LookupModuleOutput) const;
+  std::vector<std::string> getCanonicalCommandLine() const;
 };
 
 class ModuleDepCollector;
 
 /// Callback that records textual includes and direct modular includes/imports
 /// during preprocessing. At the end of the main file, it also collects
 /// transitive modular dependencies and passes everything to the
 /// \c DependencyConsumer of the parent \c ModuleDepCollector.
private:
   /// Adds \p Path to \c FileDeps, making it absolute if necessary.
   void addFileDep(StringRef Path);
   /// Adds \p Path to \c MD.FileDeps, making it absolute if necessary.
   void addFileDep(ModuleDeps &MD, StringRef Path);
 
   /// Constructs a CompilerInvocation that can be used to build the given
   /// module, excluding paths to discovered modular dependencies that are yet to
   /// be built.
-  CompilerInvocation makeInvocationForModuleBuildWithoutPaths(
+  CompilerInvocation makeInvocationForModuleBuildWithoutOutputs(
       const ModuleDeps &Deps,
       llvm::function_ref<void(CompilerInvocation &)> Optimize) const;
+
+  /// Add paths that require looking up outputs to the given dependencies.
+  void addOutputPaths(ModuleDeps &Deps);
 };
 
 } // end namespace dependencies
 } // end namespace tooling
 } // end namespace clang
 
 #endif // LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_MODULEDEPCOLLECTOR_H

clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp

 //===- DependencyScanningTool.cpp - clang-scan-deps service ---------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "clang/Tooling/DependencyScanning/DependencyScanningTool.h"
 #include "clang/Frontend/Utils.h"
 
 using namespace clang;
 using namespace tooling;
 using namespace dependencies;
 
-std::vector<std::string> FullDependencies::getCommandLine(
-    llvm::function_ref<std::string(const ModuleID &, ModuleOutputKind)>
-        LookupModuleOutput) const {
+static std::vector<std::string>
+makeTUCommandLineWithoutPaths(ArrayRef<std::string> OriginalCommandLine) {
   std::vector<std::string> Args = OriginalCommandLine;
 
   Args.push_back("-fno-implicit-modules");
   Args.push_back("-fno-implicit-module-maps");
-  for (const PrebuiltModuleDep &PMD : PrebuiltModuleDeps)
-    Args.push_back("-fmodule-file=" + PMD.PCMFile);
-  for (ModuleID MID : ClangModuleDeps)
-    Args.push_back("-fmodule-file=" +
-                   LookupModuleOutput(MID, ModuleOutputKind::ModuleFile));
 
   // These arguments are unused in explicit compiles.
   llvm::erase_if(Args, [](StringRef Arg) {
     if (Arg.consume_front("-fmodules-")) {
       return Arg.startswith("cache-path=") ||
              Arg.startswith("prune-interval=") ||
              Arg.startswith("prune-after=") ||
              Arg == "validate-once-per-build-session";
public:
     void handleModuleDependency(ModuleDeps MD) override {
       // These are ignored for the make format as it can't support the full
       // set of deps, and handleFileDependency handles enough for implicitly
       // built modules to work.
     }
 
     void handleContextHash(std::string Hash) override {}
 
+    std::string lookupModuleOutput(const ModuleID &ID,
+                                   ModuleOutputKind Kind) override {
+      llvm::report_fatal_error("unexpected call to lookupModuleOutput");
+    }
+
     void printDependencies(std::string &S) {
       assert(Opts && "Handled dependency output options.");
 
       class DependencyPrinter : public DependencyFileGenerator {
       public:
         DependencyPrinter(DependencyOutputOptions &Opts,
                           ArrayRef<std::string> Dependencies)
             : DependencyFileGenerator(Opts) {
llvm::Expected<std::string> DependencyScanningTool::getDependencyFile(
   Consumer.printDependencies(Output);
   return Output;
 }
 
 llvm::Expected<FullDependenciesResult>
 DependencyScanningTool::getFullDependencies(
     const std::vector<std::string> &CommandLine, StringRef CWD,
     const llvm::StringSet<> &AlreadySeen,
+    LookupModuleOutputCallback LookupModuleOutput,
     llvm::Optional<StringRef> ModuleName) {
-  FullDependencyConsumer Consumer(AlreadySeen);
+  FullDependencyConsumer Consumer(AlreadySeen, LookupModuleOutput);
   llvm::Error Result =
       Worker.computeDependencies(CWD, CommandLine, Consumer, ModuleName);
   if (Result)
     return std::move(Result);
   return Consumer.getFullDependencies(CommandLine);
 }
 
 FullDependenciesResult FullDependencyConsumer::getFullDependencies(
     const std::vector<std::string> &OriginalCommandLine) const {
   FullDependencies FD;
 
-  FD.OriginalCommandLine = ArrayRef<std::string>(OriginalCommandLine).slice(1);
+  FD.CommandLine = makeTUCommandLineWithoutPaths(
+      ArrayRef<std::string>(OriginalCommandLine).slice(1));
 
   FD.ID.ContextHash = std::move(ContextHash);
 
   FD.FileDeps.assign(Dependencies.begin(), Dependencies.end());
 
+  for (const PrebuiltModuleDep &PMD : PrebuiltModuleDeps)
+    FD.CommandLine.push_back("-fmodule-file=" + PMD.PCMFile);
+
   for (auto &&M : ClangModuleDeps) {
     auto &MD = M.second;
-    if (MD.ImportedByMainFile)
+    if (MD.ImportedByMainFile) {
       FD.ClangModuleDeps.push_back(MD.ID);
+      FD.CommandLine.push_back(
+          "-fmodule-file=" +
+          LookupModuleOutput(MD.ID, ModuleOutputKind::ModuleFile));
+    }
   }
 
   FD.PrebuiltModuleDeps = std::move(PrebuiltModuleDeps);
 
   FullDependenciesResult FDR;
 
   for (auto &&M : ClangModuleDeps) {
     // TODO: Avoid handleModuleDependency even being called for modules

clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp

std::function<void(const serialization::ModuleFile *)> VisitMF =
             VisitMF(Import);
       };
   VisitMF(&MF);
 
   for (auto Idx : SearchPathUsage.set_bits())
     Opts.UserEntries.push_back(Entries[Idx]);
 }
 
-CompilerInvocation ModuleDepCollector::makeInvocationForModuleBuildWithoutPaths(
+static std::vector<std::string> splitString(std::string S, char Separator) {
+  SmallVector<StringRef> Segments;
+  StringRef(S).split(Segments, Separator, /*MaxSplit=*/-1, /*KeepEmpty=*/false);
+  std::vector<std::string> Result;
+  Result.reserve(Segments.size());
+  for (StringRef Segment : Segments)
+    Result.push_back(Segment.str());
+  return Result;
+}
+
+void ModuleDepCollector::addOutputPaths(ModuleDeps &Deps) {
+  CompilerInvocation &CI = Deps.BuildInvocation;
+  for (ModuleID MID : Deps.ClangModuleDeps)
+    CI.getFrontendOpts().ModuleFiles.push_back(
+        Consumer.lookupModuleOutput(MID, ModuleOutputKind::ModuleFile));
+
+  CI.getFrontendOpts().OutputFile =
+      Consumer.lookupModuleOutput(Deps.ID, ModuleOutputKind::ModuleFile);
+  if (!CI.getDiagnosticOpts().DiagnosticSerializationFile.empty())
+    CI.getDiagnosticOpts().DiagnosticSerializationFile =
+        Consumer.lookupModuleOutput(
+            Deps.ID, ModuleOutputKind::DiagnosticSerializationFile);
+  if (!CI.getDependencyOutputOpts().OutputFile.empty()) {
+    CI.getDependencyOutputOpts().OutputFile =
+        Consumer.lookupModuleOutput(Deps.ID, ModuleOutputKind::DependencyFile);
+    CI.getDependencyOutputOpts().Targets =
+        splitString(Consumer.lookupModuleOutput(
+                        Deps.ID, ModuleOutputKind::DependencyTargets),
+                    '\0');
+    if (!CI.getDependencyOutputOpts().OutputFile.empty() &&
+        CI.getDependencyOutputOpts().Targets.empty()) {
+      // Fallback to -o as dependency target, as in the driver.
+      SmallString<128> Target;
+      quoteMakeTarget(CI.getFrontendOpts().OutputFile, Target);
+      CI.getDependencyOutputOpts().Targets.push_back(std::string(Target));
+    }
+  }
+}
+
+CompilerInvocation
+ModuleDepCollector::makeInvocationForModuleBuildWithoutOutputs(
     const ModuleDeps &Deps,
     llvm::function_ref<void(CompilerInvocation &)> Optimize) const {
   // Make a deep copy of the original Clang invocation.
   CompilerInvocation CI(OriginalInvocation);
 
   CI.getLangOpts()->resetNonModularOptions();
   CI.getPreprocessorOpts().resetNonModularOptions();
 
   // Remove options incompatible with explicit module build or are likely to
   // differ between identical modules discovered from different translation
   // units.
   CI.getFrontendOpts().Inputs.clear();
   CI.getFrontendOpts().OutputFile.clear();
   CI.getCodeGenOpts().MainFileName.clear();
   CI.getCodeGenOpts().DwarfDebugFlags.clear();
-  CI.getDiagnosticOpts().DiagnosticSerializationFile.clear();
-  CI.getDependencyOutputOpts().OutputFile.clear();
+  // Map output paths that affect behaviour to "-" so their existence is in the
+  // context hash. The final path will be computed in addOutputPaths.
+  if (!CI.getDiagnosticOpts().DiagnosticSerializationFile.empty())
+    CI.getDiagnosticOpts().DiagnosticSerializationFile = "-";
+  if (!CI.getDependencyOutputOpts().OutputFile.empty())
+    CI.getDependencyOutputOpts().OutputFile = "-";
   CI.getDependencyOutputOpts().Targets.clear();
 
   CI.getFrontendOpts().ProgramAction = frontend::GenerateModule;
   CI.getLangOpts()->ModuleName = Deps.ID.ModuleName;
   CI.getFrontendOpts().IsSystemModule = Deps.IsSystem;
 
   // Disable implicit modules and canonicalize options that are only used by
   // implicit modules.
   CI.getLangOpts()->ImplicitModules = false;
   CI.getHeaderSearchOpts().ImplicitModuleMaps = false;
   CI.getHeaderSearchOpts().ModuleCachePath.clear();
   CI.getHeaderSearchOpts().ModulesValidateOncePerBuildSession = false;
   CI.getHeaderSearchOpts().BuildSessionTimestamp = 0;
   // The specific values we canonicalize to for pruning don't affect behaviour,
   /// so use the default values so they will be dropped from the command-line.
   CI.getHeaderSearchOpts().ModuleCachePruneInterval = 7 * 24 * 60 * 60;
   CI.getHeaderSearchOpts().ModuleCachePruneAfter = 31 * 24 * 60 * 60;
 
+  // Inputs
+  InputKind ModuleMapInputKind(CI.getFrontendOpts().DashX.getLanguage(),
+                               InputKind::Format::ModuleMap);
+  CI.getFrontendOpts().Inputs.emplace_back(Deps.ClangModuleMapFile,
+                                           ModuleMapInputKind);
+  CI.getFrontendOpts().ModuleMapFiles = Deps.ModuleMapFileDeps;
+
+  // Report the prebuilt modules this module uses.
+  for (const auto &PrebuiltModule : Deps.PrebuiltModuleDeps)
+    CI.getFrontendOpts().ModuleFiles.push_back(PrebuiltModule.PCMFile);
+
   // Remove any macro definitions that are explicitly ignored.
   if (!CI.getHeaderSearchOpts().ModulesIgnoreMacros.empty()) {
     llvm::erase_if(
         CI.getPreprocessorOpts().Macros,
         [&CI](const std::pair<std::string, bool> &Def) {
           StringRef MacroDef = Def.first;
           return CI.getHeaderSearchOpts().ModulesIgnoreMacros.contains(
               llvm::CachedHashString(MacroDef.split('=').first));
         });
     // Remove the now unused option.
     CI.getHeaderSearchOpts().ModulesIgnoreMacros.clear();
   }
 
-  // Report the prebuilt modules this module uses.
-  for (const auto &PrebuiltModule : Deps.PrebuiltModuleDeps)
-    CI.getFrontendOpts().ModuleFiles.push_back(PrebuiltModule.PCMFile);
-
-  CI.getFrontendOpts().ModuleMapFiles = Deps.ModuleMapFileDeps;
-
   Optimize(CI);
 
   // The original invocation probably didn't have strict context hash enabled.
   // We will use the context hash of this invocation to distinguish between
   // multiple incompatible versions of the same module and will use it when
   // reporting dependencies to the clients. Let's make sure we're using
   // **strict** context hash in order to prevent accidental sharing of
   // incompatible modules (e.g. with differences in search paths).
serializeCompilerInvocation(const CompilerInvocation &CI) {
   // Synthesize full command line from the CompilerInvocation, including "-cc1".
   SmallVector<const char *, 32> Args{"-cc1"};
   CI.generateCC1CommandLine(Args, SA);
 
   // Convert arguments to the return type.
   return std::vector<std::string>{Args.begin(), Args.end()};
 }
 
-static std::vector<std::string> splitString(std::string S, char Separator) {
-  SmallVector<StringRef> Segments;
-  StringRef(S).split(Segments, Separator, /*MaxSplit=*/-1, /*KeepEmpty=*/false);
-  std::vector<std::string> Result;
-  Result.reserve(Segments.size());
-  for (StringRef Segment : Segments)
-    Result.push_back(Segment.str());
-  return Result;
-}
-
-std::vector<std::string> ModuleDeps::getCanonicalCommandLine(
-    llvm::function_ref<std::string(const ModuleID &, ModuleOutputKind)>
-        LookupModuleOutput) const {
-  CompilerInvocation CI(BuildInvocation);
-  FrontendOptions &FrontendOpts = CI.getFrontendOpts();
-
-  InputKind ModuleMapInputKind(FrontendOpts.DashX.getLanguage(),
-                               InputKind::Format::ModuleMap);
-  FrontendOpts.Inputs.emplace_back(ClangModuleMapFile, ModuleMapInputKind);
-  FrontendOpts.OutputFile =
-      LookupModuleOutput(ID, ModuleOutputKind::ModuleFile);
-  if (HadSerializedDiagnostics)
-    CI.getDiagnosticOpts().DiagnosticSerializationFile =
-        LookupModuleOutput(ID, ModuleOutputKind::DiagnosticSerializationFile);
-  if (HadDependencyFile) {
-    DependencyOutputOptions &DepOpts = CI.getDependencyOutputOpts();
-    DepOpts.OutputFile =
-        LookupModuleOutput(ID, ModuleOutputKind::DependencyFile);
-    DepOpts.Targets = splitString(
-        LookupModuleOutput(ID, ModuleOutputKind::DependencyTargets), '\0');
-    if (!DepOpts.OutputFile.empty() && DepOpts.Targets.empty()) {
-      // Fallback to -o as dependency target, as in the driver.
-      SmallString<128> Target;
-      quoteMakeTarget(FrontendOpts.OutputFile, Target);
-      DepOpts.Targets.push_back(std::string(Target));
-    }
-  }
-
-  for (ModuleID MID : ClangModuleDeps)
-    FrontendOpts.ModuleFiles.push_back(
-        LookupModuleOutput(MID, ModuleOutputKind::ModuleFile));
-
-  return serializeCompilerInvocation(CI);
+std::vector<std::string> ModuleDeps::getCanonicalCommandLine() const {
+  return serializeCompilerInvocation(BuildInvocation);
 }
 
 static std::string getModuleContextHash(const ModuleDeps &MD) {
   llvm::HashBuilder<llvm::TruncatedBLAKE3<16>,
                     llvm::support::endianness::native>
       HashBuilder;
   SmallString<32> Scratch;
 
   // Hash the compiler version and serialization version to ensure the module
   // will be readable.
   HashBuilder.add(getClangFullRepositoryVersion());
   HashBuilder.add(serialization::VERSION_MAJOR, serialization::VERSION_MINOR);
 
   // Hash the BuildInvocation without any input files.
   SmallVector<const char *, 32> DummyArgs;
   MD.BuildInvocation.generateCC1CommandLine(DummyArgs, [&](const Twine &Arg) {
     Scratch.clear();
     StringRef Str = Arg.toStringRef(Scratch);
     HashBuilder.add(Str);
     return "<unused>";
   });
 
-  // Hash the input file paths and module dependencies. These paths may differ
-  // even if the invocation is identical if they depend on the contents of the
-  // files in the TU -- for example, case-insensitive paths to modulemap files.
-  // Usually such a case would indicate a missed optimization to canonicalize,
-  // but it may be difficult to canonicalize all cases when there is a VFS.
-  HashBuilder.add(MD.ClangModuleMapFile);
-  for (const auto &Dep : MD.PrebuiltModuleDeps)
-    HashBuilder.add(Dep.PCMFile);
+  // Hash the module dependencies. These paths may differ even if the invocation
+  // is identical if they depend on the contents of the files in the TU -- for
+  // example, case-insensitive paths to modulemap files. Usually such a case
+  // would indicate a missed optimization to canonicalize, but it may be
+  // difficult to canonicalize all cases when there is a VFS.
   for (const auto &ID : MD.ClangModuleDeps) {
     HashBuilder.add(ID.ModuleName);
     HashBuilder.add(ID.ContextHash);
   }
 
-  // Hash options that affect which callbacks are made for outputs.
-  HashBuilder.add(MD.HadDependencyFile);
-  HashBuilder.add(MD.HadSerializedDiagnostics);
-
   llvm::BLAKE3Result<16> Hash = HashBuilder.final();
   std::array<uint64_t, 2> Words;
   static_assert(sizeof(Hash) == sizeof(Words), "Hash must match Words");
   std::memcpy(Words.data(), Hash.data(), sizeof(Hash));
   return toString(llvm::APInt(sizeof(Words) * 8, Words), 36, /*Signed=*/false);
 }
 
 void ModuleDepCollectorPP::FileChanged(SourceLocation Loc,
ModuleID ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
   }
 
   // Add direct prebuilt module dependencies now, so that we can use them when
   // creating a CompilerInvocation and computing context hash for this
   // ModuleDeps instance.
   llvm::DenseSet<const Module *> SeenModules;
   addAllSubmodulePrebuiltDeps(M, MD, SeenModules);
 
-  MD.BuildInvocation = MDC.makeInvocationForModuleBuildWithoutPaths(
+  MD.BuildInvocation = MDC.makeInvocationForModuleBuildWithoutOutputs(
       MD, [&](CompilerInvocation &BuildInvocation) {
         if (MDC.OptimizeArgs)
           optimizeHeaderSearchOpts(BuildInvocation.getHeaderSearchOpts(),
                                    *MDC.ScanInstance.getASTReader(), *MF);
       });
-  MD.HadSerializedDiagnostics = !MDC.OriginalInvocation.getDiagnosticOpts()
-                                     .DiagnosticSerializationFile.empty();
-  MD.HadDependencyFile =
-      !MDC.OriginalInvocation.getDependencyOutputOpts().OutputFile.empty();
 
   llvm::DenseSet<const Module *> AddedModules;
   addAllSubmoduleDeps(M, MD, AddedModules);
 
-  // Do this last since it requires the dependencies.
+  // Compute the context hash from the inputs. Requires dependencies.
   MD.ID.ContextHash = getModuleContextHash(MD);
+  // Finish the compiler invocation. Requires dependencies and the context hash.
+  MDC.addOutputPaths(MD);
   return MD.ID;
 }
 
 static void forEachSubmoduleSorted(const Module *M,
                                    llvm::function_ref<void(const Module *)> F) {
   // Submodule order depends on order of header includes for inferred submodules
   // we don't care about the exact order, so sort so that it's consistent across
   // TUs to improve sharing.

clang/tools/clang-scan-deps/ClangScanDeps.cpp

 //===- ClangScanDeps.cpp - Implementation of clang-scan-deps --------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
+#include "clang/Driver/Driver.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Tooling/CommonOptionsParser.h"
 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
 #include "clang/Tooling/DependencyScanning/DependencyScanningTool.h"
 #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
 #include "clang/Tooling/JSONCompilationDatabase.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/Twine.h"
for (const ModuleDeps &MD : FDR.DiscoveredModules) {
       auto I = Modules.find({MD.ID, 0});
       if (I != Modules.end()) {
         I->first.InputIndex = std::min(I->first.InputIndex, InputIndex);
         continue;
       }
       Modules.insert(I, {{MD.ID, InputIndex}, std::move(MD)});
     }
 
-    ID.CommandLine =
-        FD.getCommandLine([&](const ModuleID &MID, ModuleOutputKind MOK) {
-          return lookupModuleOutput(MID, MOK);
-        });
+    ID.CommandLine = FD.CommandLine;
     Inputs.push_back(std::move(ID));
   }
 
   void printFullOutput(raw_ostream &OS) {
     // Sort the modules by name to get a deterministic order.
     std::vector<IndexedModuleID> ModuleIDs;
     for (auto &&M : Modules)
       ModuleIDs.push_back(M.first);
void printFullOutput(raw_ostream &OS) {
     for (auto &&ModID : ModuleIDs) {
       auto &MD = Modules[ModID];
       Object O{
           {"name", MD.ID.ModuleName},
           {"context-hash", MD.ID.ContextHash},
           {"file-deps", toJSONSorted(MD.FileDeps)},
           {"clang-module-deps", toJSONSorted(MD.ClangModuleDeps)},
           {"clang-modulemap-file", MD.ClangModuleMapFile},
-          {"command-line", MD.getCanonicalCommandLine(
-                               [&](const ModuleID &MID, ModuleOutputKind MOK) {
-                                 return lookupModuleOutput(MID, MOK);
-                               })},
+          {"command-line", MD.getCanonicalCommandLine()},
       };
       OutModules.push_back(std::move(O));
     }
 
     Array TUs;
     for (auto &&I : Inputs) {
       Object O{
           {"input-file", I.FileName},
Object Output{
         {"modules", std::move(OutModules)},
         {"translation-units", std::move(TUs)},
     };
 
     OS << llvm::formatv("{0:2}\n", Value(std::move(Output)));
   }
 
 private:
-  std::string lookupModuleOutput(const ModuleID &MID, ModuleOutputKind MOK) {
-    // Cache the PCM path, since it will be queried repeatedly for each module.
-    // The other outputs are only queried once during getCanonicalCommandLine.
-    auto PCMPath = PCMPaths.insert({MID, ""});
-    if (PCMPath.second)
-      PCMPath.first->second = constructPCMPath(MID);
-    switch (MOK) {
-    case ModuleOutputKind::ModuleFile:
-      return PCMPath.first->second;
-    case ModuleOutputKind::DependencyFile:
-      return PCMPath.first->second + ".d";
-    case ModuleOutputKind::DependencyTargets:
-      // Null-separate the list of targets.
-      return join(ModuleDepTargets, StringRef("\0", 1));
-    case ModuleOutputKind::DiagnosticSerializationFile:
-      return PCMPath.first->second + ".diag";
-    }
-    llvm_unreachable("Fully covered switch above!");
-  }
-
-  /// Construct a path for the explicitly built PCM.
-  std::string constructPCMPath(ModuleID MID) const {
-    auto MDIt = Modules.find(IndexedModuleID{MID, 0});
-    assert(MDIt != Modules.end());
-    const ModuleDeps &MD = MDIt->second;
-
-    StringRef Filename = llvm::sys::path::filename(MD.ImplicitModulePCMPath);
-    StringRef ModuleCachePath = llvm::sys::path::parent_path(
-        llvm::sys::path::parent_path(MD.ImplicitModulePCMPath));
-
-    SmallString<256> ExplicitPCMPath(!ModuleFilesDir.empty() ? ModuleFilesDir
-                                                             : ModuleCachePath);
-    llvm::sys::path::append(ExplicitPCMPath, MD.ID.ContextHash, Filename);
-    return std::string(ExplicitPCMPath);
-  }
-
   struct IndexedModuleID {
     ModuleID ID;
     mutable size_t InputIndex;
 
     bool operator==(const IndexedModuleID &Other) const {
       return ID.ModuleName == Other.ID.ModuleName &&
              ID.ContextHash == Other.ID.ContextHash;
     }
struct InputDeps {
     std::vector<std::string> FileDeps;
     std::vector<ModuleID> ModuleDeps;
     std::vector<std::string> CommandLine;
   };
 
   std::mutex Lock;
   std::unordered_map<IndexedModuleID, ModuleDeps, IndexedModuleIDHasher>
       Modules;
-  std::unordered_map<ModuleID, std::string, ModuleIDHasher> PCMPaths;
   std::vector<InputDeps> Inputs;
 };
 
 static bool handleFullDependencyToolResult(
     const std::string &Input,
     llvm::Expected<FullDependenciesResult> &MaybeFullDeps, FullDeps &FD,
     size_t InputIndex, SharedStream &OS, SharedStream &Errs) {
   if (!MaybeFullDeps) {
     llvm::handleAllErrors(
         MaybeFullDeps.takeError(), [&Input, &Errs](llvm::StringError &Err) {
           Errs.applyLocked([&](raw_ostream &OS) {
             OS << "Error while scanning dependencies for " << Input << ":\n";
             OS << Err.getMessage();
           });
         });
     return true;
   }
   FD.mergeDeps(Input, std::move(*MaybeFullDeps), InputIndex);
   return false;
 }
 
+/// Construct a path for the explicitly built PCM.
+static std::string constructPCMPath(ModuleID MID, StringRef OutputDir) {
+  SmallString<256> ExplicitPCMPath(OutputDir);
+  llvm::sys::path::append(ExplicitPCMPath, MID.ContextHash,
+                          MID.ModuleName + "-" + MID.ContextHash + ".pcm");
+  return std::string(ExplicitPCMPath);
+}
+
+static std::string lookupModuleOutput(const ModuleID &MID, ModuleOutputKind MOK,
+                                      StringRef OutputDir) {
+  std::string PCMPath = constructPCMPath(MID, OutputDir);
+  switch (MOK) {
+  case ModuleOutputKind::ModuleFile:
+    return PCMPath;
+  case ModuleOutputKind::DependencyFile:
+    return PCMPath + ".d";
+  case ModuleOutputKind::DependencyTargets:
+    // Null-separate the list of targets.
+    return join(ModuleDepTargets, StringRef("\0", 1));
+  case ModuleOutputKind::DiagnosticSerializationFile:
+    return PCMPath + ".diag";
+  }
+  llvm_unreachable("Fully covered switch above!");
+}
+
+static std::string getModuleCachePath(ArrayRef<std::string> Args) {
+  for (StringRef Arg : llvm::reverse(Args)) {
+    Arg.consume_front("/clang:");
+    if (Arg.consume_front("-fmodules-cache-path="))
+      return std::string(Arg);
+  }
+  SmallString<128> Path;
+  driver::Driver::getDefaultModuleCachePath(Path);
+  return std::string(Path);
+}
+
 int main(int argc, const char **argv) {
   llvm::InitLLVM X(argc, argv);
   llvm::cl::HideUnrelatedOptions(DependencyScannerCategory);
   if (!llvm::cl::ParseCommandLineOptions(argc, argv))
     return 1;
 
   std::string ErrorMessage;
   std::unique_ptr<tooling::JSONCompilationDatabase> Compilations =
Pool.async([I, &Lock, &Index, &Inputs, &HadErrors, &FD, &WorkerTools,
           LocalIndex = Index;
           Input = &Inputs[Index++];
           Filename = std::move(Input->Filename);
           CWD = std::move(Input->Directory);
         }
         Optional<StringRef> MaybeModuleName;
         if (!ModuleName.empty())
           MaybeModuleName = ModuleName;
+
+        std::string OutputDir(ModuleFilesDir);
+        if (OutputDir.empty())
+          OutputDir = getModuleCachePath(Input->CommandLine);
+        auto LookupOutput = [&](const ModuleID &MID, ModuleOutputKind MOK) {
+          return ::lookupModuleOutput(MID, MOK, OutputDir);
+        };
+
         // Run the tool on it.
         if (Format == ScanningOutputFormat::Make) {
           auto MaybeFile = WorkerTools[I]->getDependencyFile(
               Input->CommandLine, CWD, MaybeModuleName);
           if (handleMakeDependencyToolResult(Filename, MaybeFile, DependencyOS,
                                              Errs))
             HadErrors = true;
         } else {
           auto MaybeFullDeps = WorkerTools[I]->getFullDependencies(
-              Input->CommandLine, CWD, AlreadySeenModules, MaybeModuleName);
+              Input->CommandLine, CWD, AlreadySeenModules, LookupOutput,
+              MaybeModuleName);
           if (handleFullDependencyToolResult(Filename, MaybeFullDeps, FD,
                                              LocalIndex, DependencyOS, Errs))
             HadErrors = true;
         }
       }
     });
   }
   Pool.wait();
 
   if (Format == ScanningOutputFormat::Full)
     FD.printFullOutput(llvm::outs());
 
   return HadErrors;
 }