[clang][deps] Add argument for customizing PCM paths

[jansvoboda11]

Summary

Dependency scanning currently performs an implicit build. When testing that Clang can build modules with the command-lines generated by clang-scan-deps, the actual compilation would overwrite artifacts created during the scan, which makes debugging harder than it should be and can lead to errors in multi-step builds.

To prevent this, this patch adds new flag to clang-scan-deps that allows developers to customize the directory to use when generating module map paths, instead of always using the module cache. Moreover, the explicit context hash in now part of the PCM path, which will be useful in D102488, where the context hash can change due to command-line pruning.

[Bigcheese]

lgtm with a few minor changes.

clang/tools/clang-scan-deps/ClangScanDeps.cpp

168–169

This was a bit confusing at first. I think it would be better to say that the paths that it returns are using this directory as a base. clang-scan-deps doesn't actually put files in here itself, it just returns paths that use it.

419

This doesn't need to use the IndexedModuleID. It can just use ModuleID.

clang/test/ClangScanDeps/modules-full.cpp

 // RUN:   -mode preprocess-minimized-sources >> %t.result
 // RUN: cat %t.result | sed 's/\\/\//g' | FileCheck --check-prefixes=CHECK,CHECK-NO-ABS %s
 //
 // RUN: echo %t.dir > %t.result
 // RUN: clang-scan-deps -compilation-database %t.cdb -j 4 -format experimental-full \
 // RUN:   -generate-modules-path-args -mode preprocess-minimized-sources >> %t.result
 // RUN: cat %t.result | sed 's/\\/\//g' | FileCheck --check-prefixes=CHECK,CHECK-ABS %s
 //
+// RUN: echo %t.dir > %t.result
+// RUN: clang-scan-deps -compilation-database %t.cdb -j 4 -format experimental-full \
+// RUN:   -generate-modules-path-args -build-dir %t.dir/build \
+// RUN:   -mode preprocess-minimized-sources >> %t.result
+// RUN: cat %t.result | sed 's/\\/\//g' | FileCheck --check-prefixes=CHECK,CHECK-ABS-BUILD %s
+//
 // RUN: echo %t.dir > %t_clangcl.result
 // RUN: clang-scan-deps -compilation-database %t_clangcl.cdb -j 4 -format experimental-full \
 // RUN:   -mode preprocess-minimized-sources >> %t_clangcl.result
 // RUN: cat %t_clangcl.result | sed 's/\\/\//g' | FileCheck --check-prefixes=CHECK,CHECK-NO-ABS %s
 
 // FIXME: Backslash issues.
 // XFAIL: system-windows
 

 // CHECK-NEXT:           "module-name": "header2"
 // CHECK-NEXT:         }
 // CHECK-NEXT:       ],
 // CHECK-NEXT:       "clang-modulemap-file": "[[PREFIX]]/Inputs/module.modulemap",
 // CHECK-NEXT:       "command-line": [
 // CHECK-NEXT:         "-cc1"
 // CHECK-NO-ABS-NOT:   "-fmodule-map-file={{.*}}"
 // CHECK-ABS:          "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap"
+// CHECK-ABS-BUILD:    "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap"
 // CHECK:              "-emit-module"
 // CHECK-NO-ABS-NOT:   "-fmodule-file={{.*}}"
 // CHECK-ABS:          "-fmodule-file=[[PREFIX]]/module-cache{{(_clangcl)?}}/[[CONTEXT_HASH_H1]]/header2-{{[A-Z0-9]+}}.pcm"
+// CHECK-ABS-BUILD:    "-fmodule-file=[[PREFIX]]/build/[[CONTEXT_HASH_H1]]/header2-{{[A-Z0-9]+}}.pcm"
 // CHECK:              "-fmodule-name=header1"
 // CHECK:              "-fno-implicit-modules"
 // CHECK:            ],
 // CHECK-NEXT:       "context-hash": "[[CONTEXT_HASH_H1]]",
 // CHECK-NEXT:       "file-deps": [
 // CHECK-NEXT:         "[[PREFIX]]/Inputs/header.h",
 // CHECK-NEXT:         "[[PREFIX]]/Inputs/module.modulemap"
 // CHECK-NEXT:       ],

 // CHECK-NEXT:           "module-name": "header1"
 // CHECK-NEXT:         }
 // CHECK-NEXT:       ],
 // CHECK-NEXT:       "command-line": [
 // CHECK-NEXT:         "-fno-implicit-modules"
 // CHECK-NEXT:         "-fno-implicit-module-maps"
 // CHECK-NO-ABS-NOT:   "-fmodule-file={{.*}}"
 // CHECK-ABS-NEXT:     "-fmodule-file=[[PREFIX]]/module-cache{{(_clangcl)?}}/[[CONTEXT_HASH_H2]]/header1-{{[A-Z0-9]+}}.pcm"
+// CHECK-ABS-BUILD-NEXT: "-fmodule-file=[[PREFIX]]/build/[[CONTEXT_HASH_H2]]/header1-{{[A-Z0-9]+}}.pcm"
 // CHECK-NO-ABS-NOT:   "-fmodule-map-file={{.*}}"
 // CHECK-ABS-NEXT:     "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap"
+// CHECK-ABS-BUILD-NEXT: "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap"
 // CHECK-NEXT:       ],
 // CHECK-NEXT:       "file-deps": [
 // CHECK-NEXT:         "[[PREFIX]]/modules_cdb_input.cpp"
 // CHECK-NEXT:       ],
 // CHECK-NEXT:       "input-file": "[[PREFIX]]/modules_cdb_input.cpp"
 // CHECK-NEXT:     },
 // CHECK-NEXT:     {
 // CHECK-NEXT:       "clang-context-hash": "[[CONTEXT_HASH_H2]]",
 // CHECK-NEXT:       "clang-module-deps": [
 // CHECK-NEXT:         {
 // CHECK-NEXT:           "context-hash": "[[CONTEXT_HASH_H2]]",
 // CHECK-NEXT:           "module-name": "header1"
 // CHECK-NEXT:         }
 // CHECK-NEXT:       ],
 // CHECK-NEXT:       "command-line": [
 // CHECK-NEXT:         "-fno-implicit-modules"
 // CHECK-NEXT:         "-fno-implicit-module-maps"
 // CHECK-NO-ABS-NOT:   "-fmodule-file={{.*}},
 // CHECK-ABS-NEXT:     "-fmodule-file=[[PREFIX]]/module-cache{{(_clangcl)?}}/[[CONTEXT_HASH_H2]]/header1-{{[A-Z0-9]+}}.pcm"
+// CHECK-ABS-BUILD-NEXT: "-fmodule-file=[[PREFIX]]/build/[[CONTEXT_HASH_H2]]/header1-{{[A-Z0-9]+}}.pcm"
 // CHECK-NO-ABS-NOT:   "-fmodule-map-file={{.*}}
 // CHECK-ABS-NEXT:     "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap"
+// CHECK-ABS-BUILD-NEXT: "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap"
 // CHECK-NEXT:       ],
 // CHECK-NEXT:       "file-deps": [
 // CHECK-NEXT:         "[[PREFIX]]/modules_cdb_input.cpp"
 // CHECK-NEXT:       ],
 // CHECK-NEXT:       "input-file": "[[PREFIX]]/modules_cdb_input.cpp"
 // CHECK-NEXT:     },
 // CHECK-NEXT:     {
 // CHECK-NEXT:       "clang-context-hash": "[[CONTEXT_HASH_H2]]",
 // CHECK-NEXT:       "clang-module-deps": [
 // CHECK-NEXT:         {
 // CHECK-NEXT:           "context-hash": "[[CONTEXT_HASH_H2]]",
 // CHECK-NEXT:           "module-name": "header1"
 // CHECK-NEXT:         }
 // CHECK-NEXT:       ],
 // CHECK-NEXT:       "command-line": [
 // CHECK-NEXT:         "-fno-implicit-modules"
 // CHECK-NEXT:         "-fno-implicit-module-maps"
 // CHECK-NO-ABS-NOT:   "-fmodule-file={{.*}}"
 // CHECK-ABS-NEXT:     "-fmodule-file=[[PREFIX]]/module-cache{{(_clangcl)?}}/[[CONTEXT_HASH_H2]]/header1-{{[A-Z0-9]+}}.pcm"
+// CHECK-ABS-BUILD-NEXT: "-fmodule-file=[[PREFIX]]/build/[[CONTEXT_HASH_H2]]/header1-{{[A-Z0-9]+}}.pcm"
 // CHECK-NO-ABS-NOT:   "-fmodule-map-file={{.*}}"
 // CHECK-ABS-NEXT:     "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap"
+// CHECK-ABS-BUILD-NEXT: "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap"
 // CHECK-NEXT:       ],
 // CHECK-NEXT:       "file-deps": [
 // CHECK-NEXT:         "[[PREFIX]]/modules_cdb_input.cpp"
 // CHECK-NEXT:       ],
 // CHECK-NEXT:       "input-file": "[[PREFIX]]/modules_cdb_input.cpp"
 // CHECK-NEXT:     },
 // CHECK-NEXT:     {
 // CHECK-NEXT:       "clang-context-hash": "[[CONTEXT_HASH_H1]]",
 // CHECK-NEXT:       "clang-module-deps": [
 // CHECK-NEXT:         {
 // CHECK-NEXT:           "context-hash": "[[CONTEXT_HASH_H1]]",
 // CHECK-NEXT:           "module-name": "header1"
 // CHECK-NEXT:         }
 // CHECK-NEXT:       ],
 // CHECK-NEXT:       "command-line": [
 // CHECK-NEXT:         "-fno-implicit-modules"
 // CHECK-NEXT:         "-fno-implicit-module-maps"
 // CHECK-NO-ABS-NOT:   "-fmodule-file={{.*}}"
 // CHECK-ABS-NEXT:     "-fmodule-file=[[PREFIX]]/module-cache{{(_clangcl)?}}/[[CONTEXT_HASH_H1]]/header2-{{[A-Z0-9]+}}.pcm"
 // CHECK-ABS-NEXT:     "-fmodule-file=[[PREFIX]]/module-cache{{(_clangcl)?}}/[[CONTEXT_HASH_H1]]/header1-{{[A-Z0-9]+}}.pcm"
+// CHECK-ABS-BUILD-NEXT: "-fmodule-file=[[PREFIX]]/build/[[CONTEXT_HASH_H1]]/header2-{{[A-Z0-9]+}}.pcm"
+// CHECK-ABS-BUILD-NEXT: "-fmodule-file=[[PREFIX]]/build/[[CONTEXT_HASH_H1]]/header1-{{[A-Z0-9]+}}.pcm"
 // CHECK-NO-ABS-NOT:   "-fmodule-map-file={{.*}}"
 // CHECK-ABS-NEXT:     "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap"
 // CHECK-ABS-NEXT:     "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap"
+// CHECK-ABS-BUILD-NEXT: "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap"
+// CHECK-ABS-BUILD-NEXT: "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap"
 // CHECK-NEXT:       ],
 // CHECK-NEXT:       "file-deps": [
 // CHECK-NEXT:         "[[PREFIX]]/modules_cdb_input2.cpp"
 // CHECK-NEXT:       ],
 // CHECK-NEXT:       "input-file": "[[PREFIX]]/modules_cdb_input2.cpp"
 // CHECK-NEXT:     }
 // CHECK-NEXT:   ]
 // CHECK-NEXT: }

clang/tools/clang-scan-deps/ClangScanDeps.cpp

 static llvm::cl::opt<bool> GenerateModulesPathArgs(
     "generate-modules-path-args",
     llvm::cl::desc(
         "With '-format experimental-full', include arguments specifying "
         "modules-related paths in the generated command lines: "
         "'-fmodule-file=', '-o', '-fmodule-map-file='."),
     llvm::cl::init(false), llvm::cl::cat(DependencyScannerCategory));
 
+static llvm::cl::opt<std::string> BuildDir(
+    "build-dir",
+    llvm::cl::desc("With '-generate-modules-path-args', put the PCM files in"
+                   "the provided directory instead of the module cache."),
+    llvm::cl::cat(DependencyScannerCategory));
+
 llvm::cl::opt<unsigned>
     NumThreads("j", llvm::cl::Optional,
                llvm::cl::desc("Number of worker threads to use (default: use "
                               "all concurrent threads)"),
                llvm::cl::init(0), llvm::cl::cat(DependencyScannerCategory));
 
 llvm::cl::opt<std::string>
     CompilationDB("compilation-database",
Object Output{
         {"translation-units", std::move(TUs)},
     };
 
     OS << llvm::formatv("{0:2}\n", Value(std::move(Output)));
   }
 
 private:
   StringRef lookupPCMPath(ModuleID MID) {
-    return Modules[IndexedModuleID{MID, 0}].ImplicitModulePCMPath;
+    auto PCMPath = PCMPaths.insert({IndexedModuleID{MID, 0}, ""});
+    if (PCMPath.second)
+      PCMPath.first->second = constructPCMPath(lookupModuleDeps(MID));
+    return PCMPath.first->second;
+  }
+
+  /// Construct a path where to put the explicitly built PCM.
+  std::string constructPCMPath(const ModuleDeps &MD) const {
+    StringRef Filename = llvm::sys::path::filename(MD.ImplicitModulePCMPath);
+
+    SmallString<256> ExplicitPCMPath(
+        !BuildDir.empty()
+            ? BuildDir
+            : MD.Invocation.getHeaderSearchOpts().ModuleCachePath);
+    llvm::sys::path::append(ExplicitPCMPath, MD.ID.ContextHash, Filename);
+    return std::string(ExplicitPCMPath);
   }
 
   const ModuleDeps &lookupModuleDeps(ModuleID MID) {
     auto I = Modules.find(IndexedModuleID{MID, 0});
     assert(I != Modules.end());
     return I->second;
   };
 
struct InputDeps {
     std::vector<std::string> FileDeps;
     std::vector<ModuleID> ModuleDeps;
     std::vector<std::string> AdditionalCommandLine;
   };
 
   std::mutex Lock;
   std::unordered_map<IndexedModuleID, ModuleDeps, IndexedModuleIDHasher>
       Modules;
+  std::unordered_map<IndexedModuleID, std::string, IndexedModuleIDHasher>
+      PCMPaths;
   std::vector<InputDeps> Inputs;
 };
 
 static bool handleFullDependencyToolResult(
     const std::string &Input,
     llvm::Expected<FullDependenciesResult> &MaybeFullDeps, FullDeps &FD,
     size_t InputIndex, SharedStream &OS, SharedStream &Errs) {
   if (!MaybeFullDeps) {