[clang][deps] Move invocation adjustments from `clang-scan-deps` to `DependencyScanning` library

[jansvoboda11]

Summary

The clang-scan-deps tool has some logic that parses and modifies the original Clang command-line. The goal is to setup DependencyOutputOptions by injecting -M -MT <target> and prevent the creation of output files.

This patch moves the logic into the DependencyScanning library, and uses the parsed CompilerInvocation instead of the raw command-line. The code simpler and can be used from the C++ API as well.

The -o /dev/null arguments are not necessary, since the DependencyScanning library only runs a preprocessing action, so there's no way it'll produce an actual object file.

Related: The -M argument implies -w, which would appear on the command-line of modular dependencies even though it was not on the original TU command line (see D104036).

Some related tests were updated.

[jansvoboda11]

FYI @saudi.

clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp

public:
   void visitImport(StringRef ModuleName, StringRef Filename) override {
     PrebuiltModuleFiles[std::string(ModuleName)] = std::string(Filename);
   }
 
 private:
   std::map<std::string, std::string> &PrebuiltModuleFiles;
 };
 
+/// Transform arbitrary file name into an object-like file name.
+static std::string makeObjFileName(StringRef FileName) {
+  SmallString<128> ObjFileName(FileName);
+  llvm::sys::path::replace_extension(ObjFileName, "o");
+  return std::string(ObjFileName.str());
+}
+
+/// Deduce the dependency target based on the output file and input files.
+static std::string
+deduceDepTarget(const std::string &OutputFile,
+                const SmallVectorImpl<FrontendInputFile> &InputFiles) {
+  if (OutputFile != "-")
+    return OutputFile;
+
+  if (InputFiles.empty() || !InputFiles.front().isFile())
+    return "clang-scan-deps\\ dependency";
+
+  return makeObjFileName(InputFiles.front().getFile());
+}
+
 /// A clang tool that runs the preprocessor in a mode that's optimized for
 /// dependency scanning for the given compiler invocation.
 class DependencyScanningAction : public tooling::ToolAction {
 public:
   DependencyScanningAction(
       StringRef WorkingDirectory, DependencyConsumer &Consumer,
       llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS,
       ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings,
bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
     // dependencies.
     //
     // This also moves the existing dependency output options from the
     // invocation to the collector. The options in the invocation are reset,
     // which ensures that the compiler won't create new dependency collectors,
     // and thus won't write out the extra '.d' files to disk.
     auto Opts = std::make_unique<DependencyOutputOptions>(
         std::move(Compiler.getInvocation().getDependencyOutputOpts()));
-    // We need at least one -MT equivalent for the generator to work.
+    // We need at least one -MT equivalent for the generator of make dependency
+    // files to work.
     if (Opts->Targets.empty())
-      Opts->Targets = {"clang-scan-deps dependency"};
+      Opts->Targets = {deduceDepTarget(Compiler.getFrontendOpts().OutputFile,
+                                       Compiler.getFrontendOpts().Inputs)};
 
     switch (Format) {
     case ScanningOutputFormat::Make:
       Compiler.addDependencyCollector(
           std::make_shared<DependencyConsumerForwarder>(std::move(Opts),
                                                         Consumer));
       break;
     case ScanningOutputFormat::Full:

clang/test/ClangScanDeps/modules-pch.c

 // RUN: rm -rf %t && mkdir %t
 // RUN: cp %S/Inputs/modules-pch/* %t
 
 // Scan dependencies of the PCH:
 //
 // RUN: sed "s|DIR|%/t|g" %S/Inputs/modules-pch/cdb_pch.json > %t/cdb.json
 // RUN: echo -%t > %t/result_pch.json
 // RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full \
 // RUN:   -generate-modules-path-args -module-files-dir %t/build -mode preprocess >> %t/result_pch.json
 // RUN: cat %t/result_pch.json | sed 's:\\\\\?:/:g' | FileCheck %s -check-prefix=CHECK-PCH
 //
+// Check we didn't build the PCH during dependency scanning.
+// RUN: not cat %/t/pch.h.gch
+//
 // CHECK-PCH:      -[[PREFIX:.*]]
 // CHECK-PCH-NEXT: {
 // CHECK-PCH-NEXT:   "modules": [
 // CHECK-PCH-NEXT:     {
 // CHECK-PCH-NEXT:       "clang-module-deps": [],
 // CHECK-PCH-NEXT:       "clang-modulemap-file": "[[PREFIX]]/module.modulemap",
 // CHECK-PCH-NEXT:       "command-line": [
 // CHECK-PCH-NEXT:         "-cc1"

clang/test/ClangScanDeps/modules.cpp

 // RUN:   FileCheck --check-prefix=CHECK2 %s
 // RUN: clang-scan-deps -compilation-database %t.cdb -j 2 -mode preprocess | \
 // RUN:   FileCheck --check-prefix=CHECK2 %s
 // RUN: clang-scan-deps -compilation-database %t_clangcl.cdb -j 2 -mode preprocess | \
 // RUN:   FileCheck --check-prefix=CHECK2 %s
 
 #include "header.h"
 
-// CHECK1: modules_cdb_input2.cpp
+// CHECK1: modules_cdb_input2.o:
 // CHECK1-NEXT: modules_cdb_input2.cpp
 // CHECK1-NEXT: Inputs{{/|\\}}module.modulemap
 // CHECK1-NEXT: Inputs{{/|\\}}header2.h
 // CHECK1: Inputs{{/|\\}}header.h
 
-// CHECK2: modules_cdb_input.cpp
+// CHECK2: {{.*}}.o:
+// CHECK2-NEXT: modules_cdb_input.cpp
 // CHECK2-NEXT: Inputs{{/|\\}}module.modulemap
 // CHECK2-NEXT: Inputs{{/|\\}}header.h
 // CHECK2NO-NOT: header2

clang/test/ClangScanDeps/regular_cdb.cpp

 
 // RUN: clang-scan-deps -compilation-database %t.cdb -j 2 -mode preprocess | \
 // RUN:   FileCheck --check-prefix=CHECK2 %s
 // RUN: clang-scan-deps -compilation-database %t_clangcl.cdb -j 2 -mode preprocess | \
 // RUN:   FileCheck --check-prefix=CHECK2 %s
 
 #include "header.h"
 
-// CHECK1: regular_cdb_input2.cpp
+// CHECK1: regular_cdb_input2.o:
 // CHECK1-NEXT: regular_cdb_input2.cpp
 // CHECK1-NEXT: Inputs{{/|\\}}header.h
 // CHECK1-NEXT: Inputs{{/|\\}}header2.h
 
-// CHECK3: regular_cdb_input.o
-// CHECK2: regular_cdb_input.cpp
+// CHECK2: regular_cdb_input.o:
+// CHECK2-NEXT: regular_cdb_input.cpp
 // CHECK2-NEXT: Inputs{{/|\\}}header.h
 // CHECK2NO-NOT: header2
 
-// CHECK3: adena.o
+// CHECK3: adena.o:

clang/tools/clang-scan-deps/ClangScanDeps.cpp

 
 llvm::cl::opt<bool> Verbose("v", llvm::cl::Optional,
                             llvm::cl::desc("Use verbose output."),
                             llvm::cl::init(false),
                             llvm::cl::cat(DependencyScannerCategory));
 
 } // end anonymous namespace
 
-/// \returns object-file path derived from source-file path.
-static std::string getObjFilePath(StringRef SrcFile) {
-  SmallString<128> ObjFileName(SrcFile);
-  llvm::sys::path::replace_extension(ObjFileName, "o");
-  return std::string(ObjFileName.str());
-}
-
 class SingleCommandCompilationDatabase : public tooling::CompilationDatabase {
 public:
   SingleCommandCompilationDatabase(tooling::CompileCommand Cmd)
       : Command(std::move(Cmd)) {}
 
   std::vector<tooling::CompileCommand>
   getCompileCommands(StringRef FilePath) const override {
     return {Command};
int main(int argc, const char **argv) {
   llvm::cl::PrintOptionValues();
 
   // The command options are rewritten to run Clang in preprocessor only mode.
   auto AdjustingCompilations =
       std::make_unique<tooling::ArgumentsAdjustingCompilations>(
           std::move(Compilations));
   ResourceDirectoryCache ResourceDirCache;
 
-  // FIXME: Adjust the resulting CompilerInvocation in DependencyScanningAction
-  // instead of parsing and adjusting the raw command-line. This will make it
-  // possible to remove some code specific to clang-cl and Windows.
   AdjustingCompilations->appendArgumentsAdjuster(
       [&ResourceDirCache](const tooling::CommandLineArguments &Args,
                           StringRef FileName) {
         std::string LastO = "";
-        bool HasMT = false;
-        bool HasMQ = false;
-        bool HasMD = false;
         bool HasResourceDir = false;
         bool ClangCLMode = false;
         auto FlagsEnd = llvm::find(Args, "--");
         if (FlagsEnd != Args.begin()) {
           ClangCLMode =
               llvm::sys::path::stem(Args[0]).contains_lower("clang-cl") ||
               llvm::is_contained(Args, "--driver-mode=cl");
 
AdjustingCompilations->appendArgumentsAdjuster(
                 else if (Arg.startswith("/Fo") || Arg.startswith("-Fo"))
                   LastO = Arg.drop_front(3).str();
                 else if (Arg.startswith("/o") || Arg.startswith("-o"))
                   LastO = Arg.drop_front(2).str();
 
                 if (!LastO.empty() && !llvm::sys::path::has_extension(LastO))
                   LastO.append(".obj");
               }
-              if (Arg == "/clang:-MT")
-                HasMT = true;
-              if (Arg == "/clang:-MQ")
-                HasMQ = true;
-              if (Arg == "/clang:-MD")
-                HasMD = true;
-            } else {
-              if (LastO.empty()) {
-                if (Arg == "-o" && I != R)
-                  LastO = I[-1]; // Next argument (reverse iterator)
-                else if (Arg.startswith("-o"))
-                  LastO = Arg.drop_front(2).str();
-              }
-              if (Arg == "-MT")
-                HasMT = true;
-              if (Arg == "-MQ")
-                HasMQ = true;
-              if (Arg == "-MD")
-                HasMD = true;
             }
             if (Arg == "-resource-dir")
               HasResourceDir = true;
           }
         }
-        // If there's no -MT/-MQ Driver would add -MT with the value of the last
-        // -o option.
         tooling::CommandLineArguments AdjustedArgs(Args.begin(), FlagsEnd);
-        AdjustedArgs.push_back("-o");
-#ifdef _WIN32
-        AdjustedArgs.push_back("nul");
-#else
-        AdjustedArgs.push_back("/dev/null");
-#endif
-        if (!HasMT && !HasMQ && Format == ScanningOutputFormat::Make) {
-          // We're interested in source dependencies of an object file.
-          std::string FileNameArg;
-          if (!HasMD) {
-            // FIXME: We are missing the directory unless the -o value is an
-            // absolute path.
-            FileNameArg = !LastO.empty() ? LastO : getObjFilePath(FileName);
-          } else {
-            FileNameArg = std::string(FileName);
-          }
-          if (ClangCLMode) {
-            AdjustedArgs.push_back("/clang:-M");
-            AdjustedArgs.push_back("/clang:-MT");
-            AdjustedArgs.push_back(Twine("/clang:", FileNameArg).str());
-          } else {
-            AdjustedArgs.push_back("-M");
-            AdjustedArgs.push_back("-MT");
-            AdjustedArgs.push_back(std::move(FileNameArg));
-          }
+        // The clang-cl driver passes "-o -" to the frontend. Inject the real
+        // file here to ensure "-MT" can be deduced if need be.
+        if (ClangCLMode && !LastO.empty()) {
+          AdjustedArgs.push_back("/clang:-o");
+          AdjustedArgs.push_back("/clang:" + LastO);
         }
         AdjustedArgs.push_back("-Xclang");
         AdjustedArgs.push_back("-sys-header-deps");
         AdjustedArgs.push_back("-Wno-error");
 
         if (!HasResourceDir) {
           StringRef ResourceDir =
               ResourceDirCache.findResourceDir(Args, ClangCLMode);