[clang-scan-deps] Add an API for clang dependency scanner to perform module lookup by name alone

[ahatanak]

Summary

This removes the need to create a fake source file that imports a module.

rdar://64538073

[jansvoboda11]

LGTM, thanks!

clang/include/clang/Frontend/FrontendActions.h

 
 class PrintPreprocessedAction : public PreprocessorFrontendAction {
 protected:
   void ExecuteAction() override;
 
   bool hasPCHSupport() const override { return true; }
 };
 
+class GetDependenciesByModuleNameAction : public PreprocessOnlyAction {
+  StringRef ModuleName;
+  void ExecuteAction() override;
+
+public:
+  GetDependenciesByModuleNameAction(StringRef ModuleName)
+      : ModuleName(ModuleName) {}
+};
+
 }  // end namespace clang
 
 #endif

clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h

 /// an individual worker thread.
 class DependencyScanningTool {
 public:
   /// Construct a dependency scanning tool.
   DependencyScanningTool(DependencyScanningService &Service);
 
   /// Print out the dependency information into a string using the dependency
   /// file format that is specified in the options (-MD is the default) and
-  /// return it.
+  /// return it. If \p ModuleName isn't empty, this function returns the
+  /// dependency information of module \p ModuleName.
   ///
   /// \returns A \c StringError with the diagnostic output if clang errors
   /// occurred, dependency file contents otherwise.
   llvm::Expected<std::string>
   getDependencyFile(const tooling::CompilationDatabase &Compilations,
-                    StringRef CWD);
+                    StringRef CWD, llvm::Optional<StringRef> ModuleName = None);
 
   /// Collect the full module dependency graph for the input, ignoring any
-  /// modules which have already been seen.
+  /// modules which have already been seen. If \p ModuleName isn't empty, this
+  /// function returns the full dependency information of module \p ModuleName.
   ///
   /// \param AlreadySeen This stores modules which have previously been
   ///                    reported. Use the same instance for all calls to this
   ///                    function for a single \c DependencyScanningTool in a
   ///                    single build. Use a different one for different tools,
   ///                    and clear it between builds.
   ///
   /// \returns a \c StringError with the diagnostic output if clang errors
   /// occurred, \c FullDependencies otherwise.
   llvm::Expected<FullDependenciesResult>
   getFullDependencies(const tooling::CompilationDatabase &Compilations,
-                      StringRef CWD, const llvm::StringSet<> &AlreadySeen);
+                      StringRef CWD, const llvm::StringSet<> &AlreadySeen,
+                      llvm::Optional<StringRef> ModuleName = None);
 
 private:
   DependencyScanningWorker Worker;
 };
 
 } // end namespace dependencies
 } // end namespace tooling
 } // end namespace clang
 
 #endif // LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_TOOL_H

clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h

 /// sources either using a fast mode where the source files are minimized, or
 /// using the regular processing run.
 class DependencyScanningWorker {
 public:
   DependencyScanningWorker(DependencyScanningService &Service);
 
   /// Run the dependency scanning tool for a given clang driver invocation (as
   /// specified for the given Input in the CDB), and report the discovered
-  /// dependencies to the provided consumer.
+  /// dependencies to the provided consumer. If \p ModuleName isn't empty, this
+  /// function reports the dependencies of module \p ModuleName.
   ///
   /// \returns A \c StringError with the diagnostic output if clang errors
   /// occurred, success otherwise.
   llvm::Error computeDependencies(const std::string &Input,
                                   StringRef WorkingDirectory,
                                   const CompilationDatabase &CDB,
-                                  DependencyConsumer &Consumer);
+                                  DependencyConsumer &Consumer,
+                                  llvm::Optional<StringRef> ModuleName = None);
 
 private:
   IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts;
   std::shared_ptr<PCHContainerOperations> PCHContainerOps;
   std::unique_ptr<ExcludedPreprocessorDirectiveSkipMapping> PPSkipMappings;
 
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> RealFS;
   /// The file system that is used by each worker when scanning for

clang/lib/Frontend/FrontendActions.cpp

if (CI.getDiagnosticOpts().VerifyDiagnostics) {
       do {
         PP.Lex(Tok);
       } while (Tok.isNot(tok::eof));
     }
     return;
   }
   llvm::outs() << Output;
 }
+
+void GetDependenciesByModuleNameAction::ExecuteAction() {
+  CompilerInstance &CI = getCompilerInstance();
+  Preprocessor &PP = CI.getPreprocessor();
+  SourceManager &SM = PP.getSourceManager();
+  FileID MainFileID = SM.getMainFileID();
+  SourceLocation FileStart = SM.getLocForStartOfFile(MainFileID);
+  SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> Path;
+  IdentifierInfo *ModuleID = PP.getIdentifierInfo(ModuleName);
+  Path.push_back(std::make_pair(ModuleID, FileStart));
+  auto ModResult = CI.loadModule(FileStart, Path, Module::Hidden, false);
+  PPCallbacks *CB = PP.getPPCallbacks();
+  CB->moduleImport(SourceLocation(), Path, ModResult);
+}

clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp

FullDependencies::getAdditionalArgsWithoutModulePaths() const {
   return Args;
 }
 
 DependencyScanningTool::DependencyScanningTool(
     DependencyScanningService &Service)
     : Worker(Service) {}
 
 llvm::Expected<std::string> DependencyScanningTool::getDependencyFile(
-    const tooling::CompilationDatabase &Compilations, StringRef CWD) {
+    const tooling::CompilationDatabase &Compilations, StringRef CWD,
+    llvm::Optional<StringRef> ModuleName) {
   /// Prints out all of the gathered dependencies into a string.
   class MakeDependencyPrinterConsumer : public DependencyConsumer {
   public:
     void
     handleDependencyOutputOpts(const DependencyOutputOptions &Opts) override {
       this->Opts = std::make_unique<DependencyOutputOptions>(Opts);
     }
 
llvm::Expected<std::string> DependencyScanningTool::getDependencyFile(
   // `DependencyScanningAction` once for every time the input occured in the
   // CDB. Instead we split up the CDB into single command chunks to avoid this
   // behavior.
   assert(Compilations.getAllCompileCommands().size() == 1 &&
          "Expected a compilation database with a single command!");
   std::string Input = Compilations.getAllCompileCommands().front().Filename;
 
   MakeDependencyPrinterConsumer Consumer;
-  auto Result = Worker.computeDependencies(Input, CWD, Compilations, Consumer);
+  auto Result = Worker.computeDependencies(Input, CWD, Compilations, Consumer,
+                                           ModuleName);
   if (Result)
     return std::move(Result);
   std::string Output;
   Consumer.printDependencies(Output);
   return Output;
 }
 
 llvm::Expected<FullDependenciesResult>
 DependencyScanningTool::getFullDependencies(
     const tooling::CompilationDatabase &Compilations, StringRef CWD,
-    const llvm::StringSet<> &AlreadySeen) {
+    const llvm::StringSet<> &AlreadySeen,
+    llvm::Optional<StringRef> ModuleName) {
   class FullDependencyPrinterConsumer : public DependencyConsumer {
   public:
     FullDependencyPrinterConsumer(const llvm::StringSet<> &AlreadySeen)
         : AlreadySeen(AlreadySeen) {}
 
     void
     handleDependencyOutputOpts(const DependencyOutputOptions &Opts) override {}
 
DependencyScanningTool::getFullDependencies(
   // `DependencyScanningAction` once for every time the input occured in the
   // CDB. Instead we split up the CDB into single command chunks to avoid this
   // behavior.
   assert(Compilations.getAllCompileCommands().size() == 1 &&
          "Expected a compilation database with a single command!");
   std::string Input = Compilations.getAllCompileCommands().front().Filename;
 
   FullDependencyPrinterConsumer Consumer(AlreadySeen);
-  llvm::Error Result =
-      Worker.computeDependencies(Input, CWD, Compilations, Consumer);
+  llvm::Error Result = Worker.computeDependencies(Input, CWD, Compilations,
+                                                  Consumer, ModuleName);
   if (Result)
     return std::move(Result);
   return Consumer.getFullDependencies();
 }
 
 } // end namespace dependencies
 } // end namespace tooling
 } // end namespace clang

clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp

 /// A clang tool that runs the preprocessor in a mode that's optimized for
 /// dependency scanning for the given compiler invocation.
 class DependencyScanningAction : public tooling::ToolAction {
 public:
   DependencyScanningAction(
       StringRef WorkingDirectory, DependencyConsumer &Consumer,
       llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS,
       ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings,
-      ScanningOutputFormat Format)
+      ScanningOutputFormat Format, llvm::Optional<StringRef> ModuleName = None,
+      llvm::Optional<llvm::MemoryBufferRef> FakeMemBuffer = None)
       : WorkingDirectory(WorkingDirectory), Consumer(Consumer),
-        DepFS(std::move(DepFS)), PPSkipMappings(PPSkipMappings),
-        Format(Format) {}
+        DepFS(std::move(DepFS)), PPSkipMappings(PPSkipMappings), Format(Format),
+        ModuleName(ModuleName), FakeMemBuffer(FakeMemBuffer) {}
 
   bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
                      FileManager *FileMgr,
                      std::shared_ptr<PCHContainerOperations> PCHContainerOps,
                      DiagnosticConsumer *DiagConsumer) override {
     // Make a deep copy of the original Clang invocation.
     CompilerInvocation OriginalInvocation(*Invocation);
 
if (DepFS) {
 
       // Pass the skip mappings which should speed up excluded conditional block
       // skipping in the preprocessor.
       if (PPSkipMappings)
         Compiler.getPreprocessorOpts()
             .ExcludedConditionalDirectiveSkipMappings = PPSkipMappings;
     }
 
+    if (ModuleName.hasValue()) {
+      SmallString<128> FullPath(*ModuleName);
+      llvm::sys::fs::make_absolute(WorkingDirectory, FullPath);
+      SourceManager &SrcMgr = Compiler.getSourceManager();
+      FileMgr->getVirtualFile(FullPath.c_str(), FakeMemBuffer->getBufferSize(),
+                              0);
+      FileID MainFileID = SrcMgr.createFileID(*FakeMemBuffer);
+      SrcMgr.setMainFileID(MainFileID);
+    }
+
     // Create the dependency collector that will collect the produced
     // dependencies.
     //
     // This also moves the existing dependency output options from the
     // invocation to the collector. The options in the invocation are reset,
     // which ensures that the compiler won't create new dependency collectors,
     // and thus won't write out the extra '.d' files to disk.
     auto Opts = std::make_unique<DependencyOutputOptions>();
bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
 
     // Consider different header search and diagnostic options to create
     // different modules. This avoids the unsound aliasing of module PCMs.
     //
     // TODO: Implement diagnostic bucketing and header search pruning to reduce
     // the impact of strict context hashing.
     Compiler.getHeaderSearchOpts().ModulesStrictContextHash = true;
 
-    auto Action = std::make_unique<ReadPCHAndPreprocessAction>();
+    std::unique_ptr<FrontendAction> Action;
+
+    if (ModuleName.hasValue())
+      Action = std::make_unique<GetDependenciesByModuleNameAction>(*ModuleName);
+    else
+      Action = std::make_unique<ReadPCHAndPreprocessAction>();
+
     const bool Result = Compiler.ExecuteAction(*Action);
     if (!DepFS)
       FileMgr->clearStatCache();
     return Result;
   }
 
 private:
   StringRef WorkingDirectory;
   DependencyConsumer &Consumer;
   llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
   ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings;
   ScanningOutputFormat Format;
+  llvm::Optional<StringRef> ModuleName;
+  llvm::Optional<llvm::MemoryBufferRef> FakeMemBuffer;
 };
 
 } // end anonymous namespace
 
 DependencyScanningWorker::DependencyScanningWorker(
     DependencyScanningService &Service)
     : Format(Service.getFormat()) {
   DiagOpts = new DiagnosticOptions();
static llvm::Error runWithDiags(
   if (BodyShouldSucceed(DiagPrinter))
     return llvm::Error::success();
   return llvm::make_error<llvm::StringError>(DiagnosticsOS.str(),
                                              llvm::inconvertibleErrorCode());
 }
 
 llvm::Error DependencyScanningWorker::computeDependencies(
     const std::string &Input, StringRef WorkingDirectory,
-    const CompilationDatabase &CDB, DependencyConsumer &Consumer) {
+    const CompilationDatabase &CDB, DependencyConsumer &Consumer,
+    llvm::Optional<StringRef> ModuleName) {
   RealFS->setCurrentWorkingDirectory(WorkingDirectory);
+  std::unique_ptr<llvm::MemoryBuffer> FakeMemBuffer =
+      ModuleName.hasValue() ? llvm::MemoryBuffer::getMemBuffer(" ") : nullptr;
   return runWithDiags(DiagOpts.get(), [&](DiagnosticConsumer &DC) {
     /// Create the tool that uses the underlying file system to ensure that any
     /// file system requests that are made by the driver do not go through the
     /// dependency scanning filesystem.
-    tooling::ClangTool Tool(CDB, Input, PCHContainerOps, RealFS, Files);
+    SmallString<128> FullPath;
+    tooling::ClangTool Tool(CDB,
+                            ModuleName.hasValue() ? ModuleName->str() : Input,
+                            PCHContainerOps, RealFS, Files);
     Tool.clearArgumentsAdjusters();
     Tool.setRestoreWorkingDir(false);
     Tool.setPrintErrorMessage(false);
     Tool.setDiagnosticConsumer(&DC);
-    DependencyScanningAction Action(WorkingDirectory, Consumer, DepFS,
-                                    PPSkipMappings.get(), Format);
+    DependencyScanningAction Action(
+        WorkingDirectory, Consumer, DepFS, PPSkipMappings.get(), Format,
+        ModuleName,
+        FakeMemBuffer
+            ? llvm::Optional<llvm::MemoryBufferRef>(*FakeMemBuffer.get())
+            : None);
+
+    if (ModuleName.hasValue()) {
+      Tool.mapVirtualFile(*ModuleName, FakeMemBuffer->getBuffer());
+      FullPath = *ModuleName;
+      llvm::sys::fs::make_absolute(WorkingDirectory, FullPath);
+      Tool.appendArgumentsAdjuster(
+          [&](const tooling::CommandLineArguments &Args, StringRef FileName) {
+            tooling::CommandLineArguments AdjustedArgs(Args);
+            AdjustedArgs.push_back(std::string(FullPath));
+            return AdjustedArgs;
+          });
+    }
+
     return !Tool.run(&Action);
   });
 }

clang/test/ClangScanDeps/Inputs/modules_cdb_by_mod_name.json

+[
+{
+  "directory": "DIR",
+  "command": "clang -E -IInputs -D INCLUDE_HEADER2 -MD -MF DIR/modules_cdb2.d -fmodules -fcxx-modules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps -gmodules -x c++",
+  "file": ""
+},
+{
+  "directory": "DIR",
+  "command": "clang -E -IInputs -fmodules -fcxx-modules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps -x c++",
+  "file": ""
+},
+]

clang/test/ClangScanDeps/Inputs/modules_cdb_clangcl_by_mod_name.json

+[
+{
+  "directory": "DIR",
+  "command": "clang-cl /E /IInputs /D INCLUDE_HEADER2 /clang:-MD /clang:-MF /clang:DIR/modules_cdb2_clangcl.d /clang:-fmodules /clang:-fcxx-modules /clang:-fmodules-cache-path=DIR/module-cache_clangcl /clang:-fimplicit-modules /clang:-fimplicit-module-maps /clang:-x /clang:c++ --",
+  "file": ""
+},
+{
+  "directory": "DIR",
+  "command": "clang-cl /E /IInputs /clang:-fmodules /clang:-fcxx-modules /clang:-fmodules-cache-path=DIR/module-cache_clangcl /clang:-fimplicit-modules /clang:-fimplicit-module-maps /clang:-x /clang:c++ --",
+  "file": ""
+},
+]

clang/test/ClangScanDeps/modules-full-by-mod-name.cpp

+// RUN: rm -rf %t.dir
+// RUN: rm -rf %t.cdb
+// RUN: mkdir -p %t.dir
+// RUN: cp %s %t.dir/modules_cdb_input.cpp
+// RUN: cp %s %t.dir/modules_cdb_input2.cpp
+// RUN: mkdir %t.dir/Inputs
+// RUN: cp %S/Inputs/header.h %t.dir/Inputs/header.h
+// RUN: cp %S/Inputs/header2.h %t.dir/Inputs/header2.h
+// RUN: cp %S/Inputs/module.modulemap %t.dir/Inputs/module.modulemap
+// RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/modules_cdb_by_mod_name.json > %t.cdb
+// RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/modules_cdb_clangcl_by_mod_name.json > %t_clangcl.cdb
+//
+// RUN: echo %t.dir > %t.result
+// RUN: clang-scan-deps -compilation-database %t.cdb -j 4 -format experimental-full \
+// RUN:   -mode preprocess-minimized-sources -module-name=header1 >> %t.result
+// RUN: cat %t.result | sed 's:\\\\\?:/:g' | FileCheck --check-prefixes=CHECK %s
+//
+// RUN: echo %t.dir > %t_clangcl.result
+// RUN: clang-scan-deps -compilation-database %t_clangcl.cdb -j 4 -format experimental-full \
+// RUN:   -mode preprocess-minimized-sources -module-name=header1 >> %t_clangcl.result
+// RUN: cat %t_clangcl.result | sed 's:\\\\\?:/:g' | FileCheck --check-prefixes=CHECK %s
+
+// CHECK: [[PREFIX:.*]]
+// CHECK-NEXT: {
+// CHECK-NEXT:   "modules": [
+// CHECK-NEXT:     {
+// CHECK-NEXT:       "clang-module-deps": [
+// CHECK-NEXT:         {
+// CHECK-NEXT:           "context-hash": "[[HASH_H2_DINCLUDE:[A-Z0-9]+]]",
+// CHECK-NEXT:           "module-name": "header2"
+// CHECK-NEXT:         }
+// CHECK-NEXT:       ],
+// CHECK-NEXT:       "clang-modulemap-file": "[[PREFIX]]/Inputs/module.modulemap",
+// CHECK-NEXT:       "command-line": [
+// CHECK-NEXT:         "-cc1"
+// CHECK:              "-emit-module"
+// CHECK:              "-fmodule-name=header1"
+// CHECK:              "-fno-implicit-modules"
+// CHECK:            ],
+// CHECK-NEXT:       "context-hash": "[[HASH_H1_DINCLUDE:[A-Z0-9]+]]",
+// CHECK-NEXT:       "file-deps": [
+// CHECK-NEXT:         "[[PREFIX]]/Inputs/header.h",
+// CHECK-NEXT:         "[[PREFIX]]/Inputs/module.modulemap"
+// CHECK-NEXT:       ],
+// CHECK-NEXT:       "name": "header1"
+// CHECK-NEXT:     },
+// CHECK-NEXT:     {
+// CHECK-NEXT:       "clang-module-deps": [],
+// CHECK-NEXT:       "clang-modulemap-file": "[[PREFIX]]/Inputs/module.modulemap",
+// CHECK-NEXT:       "command-line": [
+// CHECK-NEXT:         "-cc1",
+// CHECK:              "-emit-module",
+// CHECK:              "-fmodule-name=header1",
+// CHECK:              "-fno-implicit-modules",
+// CHECK:            ],
+// CHECK-NEXT:       "context-hash": "[[HASH_H1:[A-Z0-9]+]]",
+// CHECK-NEXT:       "file-deps": [
+// CHECK-NEXT:         "[[PREFIX]]/Inputs/header.h",
+// CHECK-NEXT:         "[[PREFIX]]/Inputs/module.modulemap"
+// CHECK-NEXT:       ],
+// CHECK-NEXT:       "name": "header1"
+// CHECK-NEXT:     },
+// CHECK-NEXT:     {
+// CHECK-NEXT:       "clang-module-deps": [],
+// CHECK-NEXT:       "clang-modulemap-file": "[[PREFIX]]/Inputs/module.modulemap",
+// CHECK-NEXT:       "command-line": [
+// CHECK-NEXT:         "-cc1",
+// CHECK:              "-emit-module",
+// CHECK:              "-fmodule-name=header2",
+// CHECK:              "-fno-implicit-modules",
+// CHECK:            ],
+// CHECK-NEXT:       "context-hash": "[[HASH_H2_DINCLUDE]]",
+// CHECK-NEXT:       "file-deps": [
+// CHECK-NEXT:         "[[PREFIX]]/Inputs/header2.h",
+// CHECK-NEXT:         "[[PREFIX]]/Inputs/module.modulemap"
+// CHECK-NEXT:       ],
+// CHECK-NEXT:       "name": "header2"
+// CHECK-NEXT:     }
+// CHECK-NEXT:   ],

clang/tools/clang-scan-deps/ClangScanDeps.cpp

 llvm::cl::opt<bool> SkipExcludedPPRanges(
     "skip-excluded-pp-ranges",
     llvm::cl::desc(
         "Use the preprocessor optimization that skips excluded conditionals by "
         "bumping the buffer pointer in the lexer instead of lexing the tokens  "
         "until reaching the end directive."),
     llvm::cl::init(true), llvm::cl::cat(DependencyScannerCategory));
 
+llvm::cl::opt<std::string> ModuleName(
+    "module-name", llvm::cl::Optional,
+    llvm::cl::desc("the module of which the dependencies are to be computed"),
+    llvm::cl::cat(DependencyScannerCategory));
+
 llvm::cl::opt<bool> Verbose("v", llvm::cl::Optional,
                             llvm::cl::desc("Use verbose output."),
                             llvm::cl::init(false),
                             llvm::cl::cat(DependencyScannerCategory));
 
 } // end anonymous namespace
 
 /// Takes the result of a dependency scan and prints error / dependency files
Pool.async([I, &Lock, &Index, &Inputs, &HadErrors, &FD, &WorkerTools,
           LocalIndex = Index;
           Input = &Inputs[Index++];
           tooling::CompileCommand Cmd = Input->getAllCompileCommands()[0];
           Filename = std::move(Cmd.Filename);
           CWD = std::move(Cmd.Directory);
         }
         // Run the tool on it.
         if (Format == ScanningOutputFormat::Make) {
-          auto MaybeFile = WorkerTools[I]->getDependencyFile(*Input, CWD);
+          auto MaybeFile = WorkerTools[I]->getDependencyFile(
+              *Input, CWD,
+              ModuleName.empty()
+                  ? None
+                  : llvm::Optional<StringRef>(ModuleName.c_str()));
           if (handleMakeDependencyToolResult(Filename, MaybeFile, DependencyOS,
                                              Errs))
             HadErrors = true;
         } else {
           auto MaybeFullDeps = WorkerTools[I]->getFullDependencies(
-              *Input, CWD, AlreadySeenModules);
+              *Input, CWD, AlreadySeenModules,
+              ModuleName.empty()
+                  ? None
+                  : llvm::Optional<StringRef>(ModuleName.c_str()));
           if (handleFullDependencyToolResult(Filename, MaybeFullDeps, FD,
                                              LocalIndex, DependencyOS, Errs))
             HadErrors = true;
         }
       }
     });
   }
   Pool.wait();
 
   if (Format == ScanningOutputFormat::Full)
     FD.printFullOutput(llvm::outs());
 
   return HadErrors;
 }