[clang][deps] Add argument for customizing PCM paths

[jansvoboda11]

Summary

Dependency scanning currently performs an implicit build. When testing that Clang can build modules with the command-lines generated by clang-scan-deps, the actual compilation would overwrite artifacts created during the scan, which makes debugging harder than it should be and can lead to errors in multi-step builds.

To prevent this, this patch adds new flag to clang-scan-deps that allows developers to customize the directory to use when generating module map paths, instead of always using the module cache. Moreover, the explicit context hash in now part of the PCM path, which will be useful in D102488, where the context hash can change due to command-line pruning.

[Bigcheese]

lgtm with a few minor changes.

clang/tools/clang-scan-deps/ClangScanDeps.cpp

168–169

This was a bit confusing at first. I think it would be better to say that the paths that it returns are using this directory as a base. clang-scan-deps doesn't actually put files in here itself, it just returns paths that use it.

420

This doesn't need to use the IndexedModuleID. It can just use ModuleID.

clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h

struct ModuleID {
 
   /// The context hash of a module represents the set of compiler options that
   /// may make one version of a module incompatible with another. This includes
   /// things like language mode, predefined macros, header search paths, etc...
   ///
   /// Modules with the same name but a different \c ContextHash should be
   /// treated as separate modules for the purpose of a build.
   std::string ContextHash;
+
+  bool operator==(const ModuleID &Other) const {
+    return ModuleName == Other.ModuleName && ContextHash == Other.ContextHash;
+  }
+};
+
+struct ModuleIDHasher {
+  std::size_t operator()(const ModuleID &MID) const {
+    return llvm::hash_combine(MID.ModuleName, MID.ContextHash);
+  }
 };
 
 struct ModuleDeps {
   /// The identifier of the module.
   ModuleID ID;
 
   /// Whether this is a "system" module.
   bool IsSystem;

clang/test/ClangScanDeps/modules-full.cpp

 // RUN:   -mode preprocess-minimized-sources >> %t.result
 // RUN: cat %t.result | sed 's/\\/\//g' | FileCheck --check-prefixes=CHECK,CHECK-NO-ABS %s
 //
 // RUN: echo %t.dir > %t.result
 // RUN: clang-scan-deps -compilation-database %t.cdb -j 4 -format experimental-full \
 // RUN:   -generate-modules-path-args -mode preprocess-minimized-sources >> %t.result
 // RUN: cat %t.result | sed 's/\\/\//g' | FileCheck --check-prefixes=CHECK,CHECK-ABS %s
 //
+// RUN: echo %t.dir > %t.result
+// RUN: clang-scan-deps -compilation-database %t.cdb -j 4 -format experimental-full \
+// RUN:   -generate-modules-path-args -module-files-dir %t.dir/custom \
+// RUN:   -mode preprocess-minimized-sources >> %t.result
+// RUN: cat %t.result | sed 's/\\/\//g' | FileCheck --check-prefixes=CHECK,CHECK-CUSTOM %s
+//
 // RUN: echo %t.dir > %t_clangcl.result
 // RUN: clang-scan-deps -compilation-database %t_clangcl.cdb -j 4 -format experimental-full \
 // RUN:   -mode preprocess-minimized-sources >> %t_clangcl.result
 // RUN: cat %t_clangcl.result | sed 's/\\/\//g' | FileCheck --check-prefixes=CHECK,CHECK-NO-ABS %s
 
 // FIXME: Backslash issues.
 // XFAIL: system-windows
 

 // CHECK-NEXT:           "module-name": "header2"
 // CHECK-NEXT:         }
 // CHECK-NEXT:       ],
 // CHECK-NEXT:       "clang-modulemap-file": "[[PREFIX]]/Inputs/module.modulemap",
 // CHECK-NEXT:       "command-line": [
 // CHECK-NEXT:         "-cc1"
 // CHECK-NO-ABS-NOT:   "-fmodule-map-file={{.*}}"
 // CHECK-ABS:          "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap"
+// CHECK-CUSTOM:       "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap"
 // CHECK:              "-emit-module"
 // CHECK-NO-ABS-NOT:   "-fmodule-file={{.*}}"
 // CHECK-ABS:          "-fmodule-file=[[PREFIX]]/module-cache{{(_clangcl)?}}/[[CONTEXT_HASH_H1]]/header2-{{[A-Z0-9]+}}.pcm"
+// CHECK-CUSTOM:       "-fmodule-file=[[PREFIX]]/custom/[[CONTEXT_HASH_H1]]/header2-{{[A-Z0-9]+}}.pcm"
 // CHECK:              "-fmodule-name=header1"
 // CHECK:              "-fno-implicit-modules"
 // CHECK:            ],
 // CHECK-NEXT:       "context-hash": "[[CONTEXT_HASH_H1]]",
 // CHECK-NEXT:       "file-deps": [
 // CHECK-NEXT:         "[[PREFIX]]/Inputs/header.h",
 // CHECK-NEXT:         "[[PREFIX]]/Inputs/module.modulemap"
 // CHECK-NEXT:       ],

 // CHECK-NEXT:           "module-name": "header1"
 // CHECK-NEXT:         }
 // CHECK-NEXT:       ],
 // CHECK-NEXT:       "command-line": [
 // CHECK-NEXT:         "-fno-implicit-modules"
 // CHECK-NEXT:         "-fno-implicit-module-maps"
 // CHECK-NO-ABS-NOT:   "-fmodule-file={{.*}}"
 // CHECK-ABS-NEXT:     "-fmodule-file=[[PREFIX]]/module-cache{{(_clangcl)?}}/[[CONTEXT_HASH_H2]]/header1-{{[A-Z0-9]+}}.pcm"
+// CHECK-CUSTOM-NEXT:  "-fmodule-file=[[PREFIX]]/custom/[[CONTEXT_HASH_H2]]/header1-{{[A-Z0-9]+}}.pcm"
 // CHECK-NO-ABS-NOT:   "-fmodule-map-file={{.*}}"
 // CHECK-ABS-NEXT:     "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap"
+// CHECK-CUSTOM-NEXT:  "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap"
 // CHECK-NEXT:       ],
 // CHECK-NEXT:       "file-deps": [
 // CHECK-NEXT:         "[[PREFIX]]/modules_cdb_input.cpp"
 // CHECK-NEXT:       ],
 // CHECK-NEXT:       "input-file": "[[PREFIX]]/modules_cdb_input.cpp"
 // CHECK-NEXT:     },
 // CHECK-NEXT:     {
 // CHECK-NEXT:       "clang-context-hash": "[[CONTEXT_HASH_H2]]",
 // CHECK-NEXT:       "clang-module-deps": [
 // CHECK-NEXT:         {
 // CHECK-NEXT:           "context-hash": "[[CONTEXT_HASH_H2]]",
 // CHECK-NEXT:           "module-name": "header1"
 // CHECK-NEXT:         }
 // CHECK-NEXT:       ],
 // CHECK-NEXT:       "command-line": [
 // CHECK-NEXT:         "-fno-implicit-modules"
 // CHECK-NEXT:         "-fno-implicit-module-maps"
 // CHECK-NO-ABS-NOT:   "-fmodule-file={{.*}},
 // CHECK-ABS-NEXT:     "-fmodule-file=[[PREFIX]]/module-cache{{(_clangcl)?}}/[[CONTEXT_HASH_H2]]/header1-{{[A-Z0-9]+}}.pcm"
+// CHECK-CUSTOM-NEXT:  "-fmodule-file=[[PREFIX]]/custom/[[CONTEXT_HASH_H2]]/header1-{{[A-Z0-9]+}}.pcm"
 // CHECK-NO-ABS-NOT:   "-fmodule-map-file={{.*}}
 // CHECK-ABS-NEXT:     "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap"
+// CHECK-CUSTOM-NEXT:  "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap"
 // CHECK-NEXT:       ],
 // CHECK-NEXT:       "file-deps": [
 // CHECK-NEXT:         "[[PREFIX]]/modules_cdb_input.cpp"
 // CHECK-NEXT:       ],
 // CHECK-NEXT:       "input-file": "[[PREFIX]]/modules_cdb_input.cpp"
 // CHECK-NEXT:     },
 // CHECK-NEXT:     {
 // CHECK-NEXT:       "clang-context-hash": "[[CONTEXT_HASH_H2]]",
 // CHECK-NEXT:       "clang-module-deps": [
 // CHECK-NEXT:         {
 // CHECK-NEXT:           "context-hash": "[[CONTEXT_HASH_H2]]",
 // CHECK-NEXT:           "module-name": "header1"
 // CHECK-NEXT:         }
 // CHECK-NEXT:       ],
 // CHECK-NEXT:       "command-line": [
 // CHECK-NEXT:         "-fno-implicit-modules"
 // CHECK-NEXT:         "-fno-implicit-module-maps"
 // CHECK-NO-ABS-NOT:   "-fmodule-file={{.*}}"
 // CHECK-ABS-NEXT:     "-fmodule-file=[[PREFIX]]/module-cache{{(_clangcl)?}}/[[CONTEXT_HASH_H2]]/header1-{{[A-Z0-9]+}}.pcm"
+// CHECK-CUSTOM-NEXT:  "-fmodule-file=[[PREFIX]]/custom/[[CONTEXT_HASH_H2]]/header1-{{[A-Z0-9]+}}.pcm"
 // CHECK-NO-ABS-NOT:   "-fmodule-map-file={{.*}}"
 // CHECK-ABS-NEXT:     "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap"
+// CHECK-CUSTOM-NEXT:  "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap"
 // CHECK-NEXT:       ],
 // CHECK-NEXT:       "file-deps": [
 // CHECK-NEXT:         "[[PREFIX]]/modules_cdb_input.cpp"
 // CHECK-NEXT:       ],
 // CHECK-NEXT:       "input-file": "[[PREFIX]]/modules_cdb_input.cpp"
 // CHECK-NEXT:     },
 // CHECK-NEXT:     {
 // CHECK-NEXT:       "clang-context-hash": "[[CONTEXT_HASH_H1]]",
 // CHECK-NEXT:       "clang-module-deps": [
 // CHECK-NEXT:         {
 // CHECK-NEXT:           "context-hash": "[[CONTEXT_HASH_H1]]",
 // CHECK-NEXT:           "module-name": "header1"
 // CHECK-NEXT:         }
 // CHECK-NEXT:       ],
 // CHECK-NEXT:       "command-line": [
 // CHECK-NEXT:         "-fno-implicit-modules"
 // CHECK-NEXT:         "-fno-implicit-module-maps"
 // CHECK-NO-ABS-NOT:   "-fmodule-file={{.*}}"
 // CHECK-ABS-NEXT:     "-fmodule-file=[[PREFIX]]/module-cache{{(_clangcl)?}}/[[CONTEXT_HASH_H1]]/header2-{{[A-Z0-9]+}}.pcm"
 // CHECK-ABS-NEXT:     "-fmodule-file=[[PREFIX]]/module-cache{{(_clangcl)?}}/[[CONTEXT_HASH_H1]]/header1-{{[A-Z0-9]+}}.pcm"
+// CHECK-CUSTOM-NEXT:  "-fmodule-file=[[PREFIX]]/custom/[[CONTEXT_HASH_H1]]/header2-{{[A-Z0-9]+}}.pcm"
+// CHECK-CUSTOM-NEXT:  "-fmodule-file=[[PREFIX]]/custom/[[CONTEXT_HASH_H1]]/header1-{{[A-Z0-9]+}}.pcm"
 // CHECK-NO-ABS-NOT:   "-fmodule-map-file={{.*}}"
 // CHECK-ABS-NEXT:     "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap"
 // CHECK-ABS-NEXT:     "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap"
+// CHECK-CUSTOM-NEXT:  "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap"
+// CHECK-CUSTOM-NEXT:  "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap"
 // CHECK-NEXT:       ],
 // CHECK-NEXT:       "file-deps": [
 // CHECK-NEXT:         "[[PREFIX]]/modules_cdb_input2.cpp"
 // CHECK-NEXT:       ],
 // CHECK-NEXT:       "input-file": "[[PREFIX]]/modules_cdb_input2.cpp"
 // CHECK-NEXT:     }
 // CHECK-NEXT:   ]
 // CHECK-NEXT: }

clang/tools/clang-scan-deps/ClangScanDeps.cpp

 static llvm::cl::opt<bool> GenerateModulesPathArgs(
     "generate-modules-path-args",
     llvm::cl::desc(
         "With '-format experimental-full', include arguments specifying "
         "modules-related paths in the generated command lines: "
         "'-fmodule-file=', '-o', '-fmodule-map-file='."),
     llvm::cl::init(false), llvm::cl::cat(DependencyScannerCategory));
 
+static llvm::cl::opt<std::string> ModuleFilesDir(
+    "module-files-dir",
+    llvm::cl::desc("With '-generate-modules-path-args', paths to module files "
+                   "in the generated command lines will begin with the "
+                   "specified directory instead the module cache directory."),
+    llvm::cl::cat(DependencyScannerCategory));
+
 llvm::cl::opt<unsigned>
     NumThreads("j", llvm::cl::Optional,
                llvm::cl::desc("Number of worker threads to use (default: use "
                               "all concurrent threads)"),
                llvm::cl::init(0), llvm::cl::cat(DependencyScannerCategory));
 
 llvm::cl::opt<std::string>
     CompilationDB("compilation-database",
Object Output{
         {"translation-units", std::move(TUs)},
     };
 
     OS << llvm::formatv("{0:2}\n", Value(std::move(Output)));
   }
 
 private:
   StringRef lookupPCMPath(ModuleID MID) {
-    return Modules[IndexedModuleID{MID, 0}].ImplicitModulePCMPath;
+    auto PCMPath = PCMPaths.insert({MID, ""});
+    if (PCMPath.second)
+      PCMPath.first->second = constructPCMPath(lookupModuleDeps(MID));
+    return PCMPath.first->second;
+  }
+
+  /// Construct a path for the explicitly built PCM.
+  std::string constructPCMPath(const ModuleDeps &MD) const {
+    StringRef Filename = llvm::sys::path::filename(MD.ImplicitModulePCMPath);
+
+    SmallString<256> ExplicitPCMPath(
+        !ModuleFilesDir.empty()
+            ? ModuleFilesDir
+            : MD.Invocation.getHeaderSearchOpts().ModuleCachePath);
+    llvm::sys::path::append(ExplicitPCMPath, MD.ID.ContextHash, Filename);
+    return std::string(ExplicitPCMPath);
   }
 
   const ModuleDeps &lookupModuleDeps(ModuleID MID) {
     auto I = Modules.find(IndexedModuleID{MID, 0});
     assert(I != Modules.end());
     return I->second;
   };
 
struct InputDeps {
     std::vector<std::string> FileDeps;
     std::vector<ModuleID> ModuleDeps;
     std::vector<std::string> AdditionalCommandLine;
   };
 
   std::mutex Lock;
   std::unordered_map<IndexedModuleID, ModuleDeps, IndexedModuleIDHasher>
       Modules;
+  std::unordered_map<ModuleID, std::string, ModuleIDHasher> PCMPaths;
   std::vector<InputDeps> Inputs;
 };
 
 static bool handleFullDependencyToolResult(
     const std::string &Input,
     llvm::Expected<FullDependenciesResult> &MaybeFullDeps, FullDeps &FD,
     size_t InputIndex, SharedStream &OS, SharedStream &Errs) {
   if (!MaybeFullDeps) {