[clang][clang-scan-deps] Add support for extracting full module dependencies.

[Bigcheese]

Summary

This adds experimental support for extracting a Clang module dependency graph
from a compilation database. The output format is experimental and will change.
It is currently a concatenation of JSON outputs for each compilation. Future
patches will change this to deduplicate modules between compilations.

This patch doesn't implement deduplication as the patch was already getting pretty
large. This also doesn't properly support C++20 modules yet.

[kousikk]

A few nit comments

clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h

34

"ScanningOutputFormat" or "DependencyOutputFormat" ?

35

IIUC, this format would NOT include module dependencies right?

If so, can you explicitly state that in the comment?

clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp

95

Should output arguments be at the end?

[Bigcheese]

Address review comments.

clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp

95

The point was to match up with MakeDependencyPrinterConsumer's printDependencies. This will also be going away with the patch to merge the dependencies.

[Bigcheese]

This fixes some issues I received externally.

• Fix issue with MSVC getting confused about ModuleDeps::ModuleDeps.
• Always generate absolute paths.

[Bigcheese]

I forgot to clang-format.

[arphaman]

clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h

44

It would be nice if the dependency collector could be reused outside of clang-scan-deps, like for a new -cc1 flag that computes the modular dependencies in the same fashion. However, its something that can be done as a follow-up, so for now this seems fine.

[doug.gregor]

It looks like Inputs/modules_cdb.json missing from this patch?

[Bigcheese]

It was already there from a previous commit. I see it in tree currently.

[doug.gregor]

In D69420#1757664, @Bigcheese wrote:

It was already there from a previous commit. I see it in tree currently.

Oh, sorry. I missed it in my cherry-picking and GitHub search failed to find it.

clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h

enum class ScanningMode {
 
   /// This mode is used to compute the dependencies by running the preprocessor
   /// over
   /// the source files that have been minimized to contents that might affect
   /// the dependencies.
   MinimizedSourcePreprocessing
 };
 
+/// The format that is output by the dependency scanner.
+enum class ScanningOutputFormat {
+  /// This is the Makefile compatible dep format. This will include all of the
+  /// deps necessary for an implicit modules build, but won't include any
+  /// intermodule dependency information.
+  Make,
+
+  /// This outputs the full module dependency graph suitable for use for
+  /// explicitly building modules.
+  Full,
+};
+
 /// The dependency scanning service contains the shared state that is used by
 /// the invidual dependency scanning workers.
 class DependencyScanningService {
 public:
-  DependencyScanningService(ScanningMode Mode, bool ReuseFileManager = true,
+  DependencyScanningService(ScanningMode Mode, ScanningOutputFormat Format,
+                            bool ReuseFileManager = true,
                             bool SkipExcludedPPRanges = true);
 
   ScanningMode getMode() const { return Mode; }
 
+  ScanningOutputFormat getFormat() const { return Format; }
+
   bool canReuseFileManager() const { return ReuseFileManager; }
 
   bool canSkipExcludedPPRanges() const { return SkipExcludedPPRanges; }
 
   DependencyScanningFilesystemSharedCache &getSharedCache() {
     return SharedCache;
   }
 
 private:
   const ScanningMode Mode;
+  const ScanningOutputFormat Format;
   const bool ReuseFileManager;
   /// Set to true to use the preprocessor optimization that skips excluded PP
   /// ranges by bumping the buffer pointer in the lexer instead of lexing the
   /// tokens in the range until reaching the corresponding directive.
   const bool SkipExcludedPPRanges;
   /// The global file system cache.
   DependencyScanningFilesystemSharedCache SharedCache;
 };
 
 } // end namespace dependencies
 } // end namespace tooling
 } // end namespace clang
 
 #endif // LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_SERVICE_H

clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h

public:
   /// return it.
   ///
   /// \returns A \c StringError with the diagnostic output if clang errors
   /// occurred, dependency file contents otherwise.
   llvm::Expected<std::string> getDependencyFile(const std::string &Input,
                                                 StringRef CWD);
 
 private:
+  const ScanningOutputFormat Format;
   DependencyScanningWorker Worker;
   const tooling::CompilationDatabase &Compilations;
 };
 
 } // end namespace dependencies
 } // end namespace tooling
 } // end namespace clang
 
 #endif // LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_TOOL_H

clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h

 #define LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_WORKER_H
 
 #include "clang/Basic/DiagnosticOptions.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Frontend/PCHContainerOperations.h"
 #include "clang/Lex/PreprocessorExcludedConditionalDirectiveSkipMapping.h"
 #include "clang/Tooling/CompilationDatabase.h"
+#include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
+#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/FileSystem.h"
 #include <string>
 
 namespace clang {
 
 class DependencyOutputOptions;
 
 namespace tooling {
 namespace dependencies {
 
-class DependencyScanningService;
 class DependencyScanningWorkerFilesystem;
 
 class DependencyConsumer {
 public:
   virtual ~DependencyConsumer() {}
 
   virtual void handleFileDependency(const DependencyOutputOptions &Opts,
                                     StringRef Filename) = 0;
 
-  // FIXME: Add support for reporting modular dependencies.
+  virtual void handleModuleDependency(ModuleDeps MD) = 0;
+
+  virtual void handleContextHash(std::string Hash) = 0;
 };
 
 /// An individual dependency scanning worker that is able to run on its own
 /// thread.
 ///
 /// The worker computes the dependencies for the input files by preprocessing
 /// sources either using a fast mode where the source files are minimized, or
 /// using the regular processing run.
private:
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> RealFS;
   /// The file system that is used by each worker when scanning for
   /// dependencies. This filesystem persists accross multiple compiler
   /// invocations.
   llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
   /// The file manager that is reused accross multiple invocations by this
   /// worker. If null, the file manager will not be reused.
   llvm::IntrusiveRefCntPtr<FileManager> Files;
+  ScanningOutputFormat Format;
 };
 
 } // end namespace dependencies
 } // end namespace tooling
 } // end namespace clang
 
 #endif // LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_WORKER_H

clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h

+//===- ModuleDepCollector.h - Callbacks to collect deps ---------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_MODULE_DEP_COLLECTOR_H
+#define LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_MODULE_DEP_COLLECTOR_H
+
+#include "clang/Basic/LLVM.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Frontend/Utils.h"
+#include "clang/Lex/HeaderSearch.h"
+#include "clang/Lex/PPCallbacks.h"
+#include "clang/Serialization/ASTReader.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/StringSet.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include <string>
+
+namespace clang {
+namespace tooling {
+namespace dependencies {
+
+class DependencyConsumer;
+
+struct ModuleDeps {
+  std::string ModuleName;
+  std::string ClangModuleMapFile;
+  std::string ModulePCMPath;
+  std::string ContextHash;
+  llvm::StringSet<> FileDeps;
+  llvm::StringSet<> ClangModuleDeps;
+  bool ImportedByMainFile = false;
+};
+
+class ModuleDepCollector;
+
+class ModuleDepCollectorPP final : public PPCallbacks {
+public:
+  ModuleDepCollectorPP(CompilerInstance &I, ModuleDepCollector &MDC)
+      : Instance(I), MDC(MDC) {}
+
+  void FileChanged(SourceLocation Loc, FileChangeReason Reason,
+                   SrcMgr::CharacteristicKind FileType,
+                   FileID PrevFID) override;
+  void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok,
+                          StringRef FileName, bool IsAngled,
+                          CharSourceRange FilenameRange, const FileEntry *File,
+                          StringRef SearchPath, StringRef RelativePath,
+                          const Module *Imported,
+                          SrcMgr::CharacteristicKind FileType) override;
+
+  void EndOfMainFile() override;
+
+private:
+  CompilerInstance &Instance;
+  ModuleDepCollector &MDC;
+  llvm::DenseSet<const Module *> DirectDeps;
+
+  void handleTopLevelModule(const Module *M);
+  void addAllSubmoduleDeps(const Module *M, ModuleDeps &MD);
+  void addModuleDep(const Module *M, ModuleDeps &MD);
+
+  void addDirectDependencies(const Module *Mod);
+};
+
+class ModuleDepCollector final : public DependencyCollector {
+public:
+  ModuleDepCollector(CompilerInstance &I, DependencyConsumer &C);
+
+  void attachToPreprocessor(Preprocessor &PP) override;
+  void attachToASTReader(ASTReader &R) override;
+
+private:
+  friend ModuleDepCollectorPP;
+
+  CompilerInstance &Instance;
+  DependencyConsumer &Consumer;
+  std::string MainFile;
+  std::string ContextHash;
+  std::vector<std::string> MainDeps;
+  std::unordered_map<std::string, ModuleDeps> Deps;
+};
+
+} // end namespace dependencies
+} // end namespace tooling
+} // end namespace clang
+
+#endif // LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_MODULE_DEP_COLLECTOR_H

clang/lib/Tooling/DependencyScanning/CMakeLists.txt

 set(LLVM_LINK_COMPONENTS
   Core
   Support
   )
 
 add_clang_library(clangDependencyScanning
   DependencyScanningFilesystem.cpp
   DependencyScanningService.cpp
   DependencyScanningWorker.cpp
   DependencyScanningTool.cpp
+  ModuleDepCollector.cpp
 
   DEPENDS
   ClangDriverOptions
 
   LINK_LIBS
   clangAST
   clangBasic
   clangDriver
   clangFrontend
   clangFrontendTool
   clangLex
   clangParse
   clangSerialization
   clangTooling
 )

clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp

 //===- DependencyScanningService.cpp - clang-scan-deps service ------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
 
 using namespace clang;
 using namespace tooling;
 using namespace dependencies;
 
-DependencyScanningService::DependencyScanningService(ScanningMode Mode,
-                                                     bool ReuseFileManager,
-                                                     bool SkipExcludedPPRanges)
-    : Mode(Mode), ReuseFileManager(ReuseFileManager),
+DependencyScanningService::DependencyScanningService(
+    ScanningMode Mode, ScanningOutputFormat Format, bool ReuseFileManager,
+    : Mode(Mode), Format(Format), ReuseFileManager(ReuseFileManager),
       SkipExcludedPPRanges(SkipExcludedPPRanges) {}

clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp

 //===- DependencyScanningTool.cpp - clang-scan-deps service ---------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "clang/Tooling/DependencyScanning/DependencyScanningTool.h"
 #include "clang/Frontend/Utils.h"
+#include "llvm/Support/JSON.h"
+
+static llvm::json::Array toJSON(const llvm::StringSet<> &Set) {
+  llvm::json::Array Ret;
+  for (auto &&I : Set) {
+    Ret.push_back(std::string(I.getKey()));
+  }
+  return Ret;
+}
 
 namespace clang{
 namespace tooling{
 namespace dependencies{
 
 DependencyScanningTool::DependencyScanningTool(
     DependencyScanningService &Service,
     const tooling::CompilationDatabase &Compilations)
-    : Worker(Service), Compilations(Compilations) {}
+    : Format(Service.getFormat()), Worker(Service), Compilations(Compilations) {
+}
 
 llvm::Expected<std::string>
 DependencyScanningTool::getDependencyFile(const std::string &Input,
                                           StringRef CWD) {
   /// Prints out all of the gathered dependencies into a string.
-  class DependencyPrinterConsumer : public DependencyConsumer {
+  class MakeDependencyPrinterConsumer : public DependencyConsumer {
   public:
     void handleFileDependency(const DependencyOutputOptions &Opts,
                               StringRef File) override {
       if (!this->Opts)
         this->Opts = std::make_unique<DependencyOutputOptions>(Opts);
       Dependencies.push_back(File);
     }
 
+    void handleModuleDependency(ModuleDeps MD) override {
+      // These are ignored for the make format as it can't support the full
+      // set of deps, and handleFileDependency handles enough for implicitly
+      // built modules to work.
+    }
+
+    void handleContextHash(std::string Hash) override {}
+
     void printDependencies(std::string &S) {
       if (!Opts)
         return;
 
       class DependencyPrinter : public DependencyFileGenerator {
       public:
         DependencyPrinter(DependencyOutputOptions &Opts,
                           ArrayRef<std::string> Dependencies)
void printDependencies(std::string &S) {
       Generator.printDependencies(S);
     }
 
   private:
     std::unique_ptr<DependencyOutputOptions> Opts;
     std::vector<std::string> Dependencies;
   };
 
-  DependencyPrinterConsumer Consumer;
+  class FullDependencyPrinterConsumer : public DependencyConsumer {
+  public:
+    void handleFileDependency(const DependencyOutputOptions &Opts,
+                              StringRef File) override {
+      Dependencies.push_back(File);
+    }
+
+    void handleModuleDependency(ModuleDeps MD) override {
+      ModuleDeps[MD.ContextHash + MD.ModuleName] = std::move(MD);
+    }
+
+    void handleContextHash(std::string Hash) override {
+      ContextHash = std::move(Hash);
+    }
+
+    void printDependencies(std::string &S, StringRef MainFile) {
+      // Sort the modules by name to get a deterministic order.
+      std::vector<StringRef> Modules;
+      for (auto &&Dep : ModuleDeps)
+        Modules.push_back(Dep.first);
+      std::sort(Modules.begin(), Modules.end());
+
+      llvm::raw_string_ostream OS(S);
+
+      using namespace llvm::json;
+
+      Array Imports;
+      for (auto &&ModName : Modules) {
+        auto &MD = ModuleDeps[ModName];
+        if (MD.ImportedByMainFile)
+          Imports.push_back(MD.ModuleName);
+      }
+
+      Array Mods;
+      for (auto &&ModName : Modules) {
+        auto &MD = ModuleDeps[ModName];
+        Object Mod{
+            {"name", MD.ModuleName},
+            {"file-deps", toJSON(MD.FileDeps)},
+            {"clang-module-deps", toJSON(MD.ClangModuleDeps)},
+            {"clang-modulemap-file", MD.ClangModuleMapFile},
+        };
+        Mods.push_back(std::move(Mod));
+      }
+
+      Object O{
+          {"input-file", MainFile},
+          {"clang-context-hash", ContextHash},
+          {"file-deps", Dependencies},
+          {"clang-module-deps", std::move(Imports)},
+          {"clang-modules", std::move(Mods)},
+      };
+
+      S = llvm::formatv("{0:2},\n", Value(std::move(O))).str();
+      return;
+    }
+
+  private:
+    std::vector<std::string> Dependencies;
+    std::unordered_map<std::string, ModuleDeps> ModuleDeps;
+    std::string ContextHash;
+  };
+
+  if (Format == ScanningOutputFormat::Make) {
+    MakeDependencyPrinterConsumer Consumer;
     auto Result =
         Worker.computeDependencies(Input, CWD, Compilations, Consumer);
     if (Result)
       return std::move(Result);
     std::string Output;
     Consumer.printDependencies(Output);
     return Output;
+  } else {
+    FullDependencyPrinterConsumer Consumer;
+    auto Result =
+        Worker.computeDependencies(Input, CWD, Compilations, Consumer);
+    if (Result)
+      return std::move(Result);
+    std::string Output;
+    Consumer.printDependencies(Output, Input);
+    return Output;
+  }
 }
 
 } // end namespace dependencies
 } // end namespace tooling
 } // end namespace clang

clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp

 //===- DependencyScanningWorker.cpp - clang-scan-deps worker --------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/CompilerInvocation.h"
 #include "clang/Frontend/FrontendActions.h"
 #include "clang/Frontend/TextDiagnosticPrinter.h"
 #include "clang/Frontend/Utils.h"
 #include "clang/Lex/PreprocessorOptions.h"
 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
+#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
 #include "clang/Tooling/Tooling.h"
 
 using namespace clang;
 using namespace tooling;
 using namespace dependencies;
 
 namespace {
 

 
 /// A clang tool that runs the preprocessor in a mode that's optimized for
 /// dependency scanning for the given compiler invocation.
 class DependencyScanningAction : public tooling::ToolAction {
 public:
   DependencyScanningAction(
       StringRef WorkingDirectory, DependencyConsumer &Consumer,
       llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS,
-      ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings)
+      ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings,
+      ScanningOutputFormat Format)
       : WorkingDirectory(WorkingDirectory), Consumer(Consumer),
-        DepFS(std::move(DepFS)), PPSkipMappings(PPSkipMappings) {}
+        DepFS(std::move(DepFS)), PPSkipMappings(PPSkipMappings),
+        Format(Format) {}
 
   bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
                      FileManager *FileMgr,
                      std::shared_ptr<PCHContainerOperations> PCHContainerOps,
                      DiagnosticConsumer *DiagConsumer) override {
     // Create a compiler instance to handle the actual work.
     CompilerInstance Compiler(std::move(PCHContainerOps));
     Compiler.setInvocation(std::move(Invocation));
bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
     // invocation to the collector. The options in the invocation are reset,
     // which ensures that the compiler won't create new dependency collectors,
     // and thus won't write out the extra '.d' files to disk.
     auto Opts = std::make_unique<DependencyOutputOptions>(
         std::move(Compiler.getInvocation().getDependencyOutputOpts()));
     // We need at least one -MT equivalent for the generator to work.
     if (Opts->Targets.empty())
       Opts->Targets = {"clang-scan-deps dependency"};
+
+    switch (Format) {
+    case ScanningOutputFormat::Make:
       Compiler.addDependencyCollector(
           std::make_shared<DependencyConsumerForwarder>(std::move(Opts),
                                                         Consumer));
+      break;
+    case ScanningOutputFormat::Full:
+      Compiler.addDependencyCollector(
+          std::make_shared<ModuleDepCollector>(Compiler, Consumer));
+      break;
+    }
+
+    Consumer.handleContextHash(Compiler.getInvocation().getModuleHash());
 
     auto Action = std::make_unique<PreprocessOnlyAction>();
     const bool Result = Compiler.ExecuteAction(*Action);
     if (!DepFS)
       FileMgr->clearStatCache();
     return Result;
   }
 
 private:
   StringRef WorkingDirectory;
   DependencyConsumer &Consumer;
   llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
   ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings;
+  ScanningOutputFormat Format;
 };
 
 } // end anonymous namespace
 
 DependencyScanningWorker::DependencyScanningWorker(
-    DependencyScanningService &Service) {
+    DependencyScanningService &Service)
+    : Format(Service.getFormat()) {
   DiagOpts = new DiagnosticOptions();
   PCHContainerOps = std::make_shared<PCHContainerOperations>();
   RealFS = new ProxyFileSystemWithoutChdir(llvm::vfs::getRealFileSystem());
   if (Service.canSkipExcludedPPRanges())
     PPSkipMappings =
         std::make_unique<ExcludedPreprocessorDirectiveSkipMapping>();
   if (Service.getMode() == ScanningMode::MinimizedSourcePreprocessing)
     DepFS = new DependencyScanningWorkerFilesystem(
return runWithDiags(DiagOpts.get(), [&](DiagnosticConsumer &DC) {
     /// file system requests that are made by the driver do not go through the
     /// dependency scanning filesystem.
     tooling::ClangTool Tool(CDB, Input, PCHContainerOps, RealFS, Files);
     Tool.clearArgumentsAdjusters();
     Tool.setRestoreWorkingDir(false);
     Tool.setPrintErrorMessage(false);
     Tool.setDiagnosticConsumer(&DC);
     DependencyScanningAction Action(WorkingDirectory, Consumer, DepFS,
-                                    PPSkipMappings.get());
+                                    PPSkipMappings.get(), Format);
     return !Tool.run(&Action);
   });
 }

clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp

+//===- ModuleDepCollector.cpp - Callbacks to collect deps -------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
+
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
+
+using namespace clang;
+using namespace tooling;
+using namespace dependencies;
+
+void ModuleDepCollectorPP::FileChanged(SourceLocation Loc,
+                                       FileChangeReason Reason,
+                                       SrcMgr::CharacteristicKind FileType,
+                                       FileID PrevFID) {
+  if (Reason != PPCallbacks::EnterFile)
+    return;
+
+  SourceManager &SM = Instance.getSourceManager();
+
+  // Dependency generation really does want to go all the way to the
+  // file entry for a source location to find out what is depended on.
+  // We do not want #line markers to affect dependency generation!
+  Optional<FileEntryRef> File =
+      SM.getFileEntryRefForID(SM.getFileID(SM.getExpansionLoc(Loc)));
+  if (!File)
+    return;
+
+  StringRef FileName =
+      llvm::sys::path::remove_leading_dotslash(File->getName());
+
+  MDC.MainDeps.push_back(FileName);
+}
+
+void ModuleDepCollectorPP::InclusionDirective(
+    SourceLocation HashLoc, const Token &IncludeTok, StringRef FileName,
+    bool IsAngled, CharSourceRange FilenameRange, const FileEntry *File,
+    StringRef SearchPath, StringRef RelativePath, const Module *Imported,
+    SrcMgr::CharacteristicKind FileType) {
+  if (!File && !Imported) {
+    // This is a non-modular include that HeaderSearch failed to find. Add it
+    // here as `FileChanged` will never see it.
+    MDC.MainDeps.push_back(FileName);
+  }
+
+  if (!Imported)
+    return;
+
+  MDC.Deps[MDC.ContextHash + Imported->getTopLevelModule()->getFullModuleName()]
+      .ImportedByMainFile = true;
+  DirectDeps.insert(Imported->getTopLevelModule());
+}
+
+void ModuleDepCollectorPP::EndOfMainFile() {
+  FileID MainFileID = Instance.getSourceManager().getMainFileID();
+  MDC.MainFile =
+      Instance.getSourceManager().getFileEntryForID(MainFileID)->getName();
+
+  for (const Module *M : DirectDeps) {
+    handleTopLevelModule(M);
+  }
+
+  for (auto &&I : MDC.Deps)
+    MDC.Consumer.handleModuleDependency(I.second);
+
+  DependencyOutputOptions Opts;
+  for (auto &&I : MDC.MainDeps)
+    MDC.Consumer.handleFileDependency(Opts, I);
+}
+
+void ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
+  assert(M == M->getTopLevelModule() && "Expected top level module!");
+
+  auto ModI = MDC.Deps.insert(
+      std::make_pair(MDC.ContextHash + M->getFullModuleName(), ModuleDeps{}));
+
+  if (!ModI.first->second.ModuleName.empty())
+    return;
+
+  ModuleDeps &MD = ModI.first->second;
+
+  const FileEntry *ModuleMap = Instance.getPreprocessor()
+                                   .getHeaderSearchInfo()
+                                   .getModuleMap()
+                                   .getContainingModuleMapFile(M);
+
+  MD.ClangModuleMapFile = ModuleMap ? ModuleMap->getName() : "";
+  MD.ModuleName = M->getFullModuleName();
+  MD.ModulePCMPath = M->getASTFile()->getName();
+  MD.ContextHash = MDC.ContextHash;
+  serialization::ModuleFile *MF =
+      MDC.Instance.getModuleManager()->getModuleManager().lookup(
+          M->getASTFile());
+  MDC.Instance.getModuleManager()->visitInputFiles(
+      *MF, true, true, [&](const serialization::InputFile &IF, bool isSystem) {
+        MD.FileDeps.insert(IF.getFile()->getName());
+      });
+
+  addAllSubmoduleDeps(M, MD);
+}
+
+void ModuleDepCollectorPP::addAllSubmoduleDeps(const Module *M,
+                                               ModuleDeps &MD) {
+  addModuleDep(M, MD);
+
+  for (const Module *SubM : M->submodules())
+    addAllSubmoduleDeps(SubM, MD);
+}
+
+void ModuleDepCollectorPP::addModuleDep(const Module *M, ModuleDeps &MD) {
+  for (const Module *Import : M->Imports) {
+    if (Import->getTopLevelModule() != M->getTopLevelModule()) {
+      MD.ClangModuleDeps.insert(Import->getTopLevelModuleName());
+      handleTopLevelModule(Import->getTopLevelModule());
+    }
+  }
+}
+
+ModuleDepCollector::ModuleDepCollector(CompilerInstance &I,
+                                       DependencyConsumer &C)
+    : Instance(I), Consumer(C), ContextHash(I.getInvocation().getModuleHash()) {
+}
+
+void ModuleDepCollector::attachToPreprocessor(Preprocessor &PP) {
+  PP.addPPCallbacks(std::make_unique<ModuleDepCollectorPP>(Instance, *this));
+}
+
+void ModuleDepCollector::attachToASTReader(ASTReader &R) {}

clang/test/ClangScanDeps/modules-full.cpp

+// RUN: rm -rf %t.dir
+// RUN: rm -rf %t.cdb
+// RUN: rm -rf %t.module-cache
+// RUN: mkdir -p %t.dir
+// RUN: cp %s %t.dir/modules_cdb_input.cpp
+// RUN: cp %s %t.dir/modules_cdb_input2.cpp
+// RUN: mkdir %t.dir/Inputs
+// RUN: cp %S/Inputs/header.h %t.dir/Inputs/header.h
+// RUN: cp %S/Inputs/header2.h %t.dir/Inputs/header2.h
+// RUN: cp %S/Inputs/module.modulemap %t.dir/Inputs/module.modulemap
+// RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/modules_cdb.json > %t.cdb
+//
+// RUN: echo %t.dir > %t.result
+// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 \
+// RUN:   -mode preprocess-minimized-sources -format experimental-full >> %t.result
+// RUN: cat %t.result | FileCheck --check-prefixes=CHECK %s
+
+#include "header.h"
+
+// CHECK: [[PREFIX:(.*[/\\])+[a-zA-Z0-9.-]+]]
+// CHECK-NEXT:     {
+// CHECK-NEXT:  "clang-context-hash": "[[CONTEXT_HASH:[A-Z0-9]+]]",
+// CHECK-NEXT:  "clang-module-deps": [
+// CHECK-NEXT:    "header1"
+// CHECK-NEXT:  ],
+// CHECK-NEXT:  "clang-modules": [
+// CHECK-NEXT:    {
+// CHECK-NEXT:      "clang-module-deps": [
+// CHECK-NEXT:        "header2"
+// CHECK-NEXT:      ],
+// CHECK-NEXT:      "clang-modulemap-file": "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap",
+// CHECK-NEXT:      "file-deps": [
+// CHECK-NEXT:        "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap",
+// CHECK-NEXT:        "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}header.h"
+// CHECK-NEXT:      ],
+// CHECK-NEXT:      "name": "header1"
+// CHECK-NEXT:    },
+// CHECK-NEXT:    {
+// CHECK-NEXT:      "clang-module-deps": [],
+// CHECK-NEXT:      "clang-modulemap-file": "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap",
+// CHECK-NEXT:      "file-deps": [
+// CHECK-NEXT:        "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}header2.h",
+// CHECK-NEXT:        "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap"
+// CHECK-NEXT:      ],
+// CHECK-NEXT:      "name": "header2"
+// CHECK-NEXT:    }
+// CHECK-NEXT:  ],
+// CHECK-NEXT:  "file-deps": [
+// CHECK-NEXT:    "[[PREFIX]]{{[/\\]}}modules_cdb_input2.cpp"
+// CHECK-NEXT:  ],
+// CHECK-NEXT:  "input-file": "[[PREFIX]]{{[/\\]}}modules_cdb_input2.cpp"
+// CHECK-NEXT:},
+// CHECK-NEXT:{
+// CHECK-NOT:   "clang-context-hash": "[[CONTEXT_HASH]]",
+// CHECK-NEXT:  "clang-context-hash": "{{[A-Z0-9]+}}",
+// CHECK-NEXT:  "clang-module-deps": [
+// CHECK-NEXT:    "header1"
+// CHECK-NEXT:  ],
+// CHECK-NEXT:  "clang-modules": [
+// CHECK-NEXT:    {
+// CHECK-NEXT:      "clang-module-deps": [],
+// CHECK-NEXT:      "clang-modulemap-file": "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap",
+// CHECK-NEXT:      "file-deps": [
+// CHECK-NEXT:        "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap",
+// CHECK-NEXT:        "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}header.h"
+// CHECK-NEXT:      ],
+// CHECK-NEXT:      "name": "header1"
+// CHECK-NEXT:    }
+// CHECK-NEXT:  ],
+// CHECK-NEXT:  "file-deps": [
+// CHECK-NEXT:    "[[PREFIX]]{{[/\\]}}modules_cdb_input.cpp"
+// CHECK-NEXT:  ],
+// CHECK-NEXT:  "input-file": "[[PREFIX]]{{[/\\]}}modules_cdb_input.cpp"
+// CHECK-NEXT:},

clang/tools/clang-scan-deps/ClangScanDeps.cpp

llvm::cl::values(
                    "source files that were minimized to only include the "
                    "contents that might affect the dependencies"),
         clEnumValN(ScanningMode::CanonicalPreprocessing, "preprocess",
                    "The set of dependencies is computed by preprocessing the "
                    "unmodified source files")),
     llvm::cl::init(ScanningMode::MinimizedSourcePreprocessing),
     llvm::cl::cat(DependencyScannerCategory));
 
+static llvm::cl::opt<ScanningOutputFormat> Format(
+    "format", llvm::cl::desc("The output format for the dependencies"),
+    llvm::cl::values(clEnumValN(ScanningOutputFormat::Make, "make",
+                                "Makefile compatible dep file"),
+                     clEnumValN(ScanningOutputFormat::Full, "experimental-full",
+                                "Full dependency graph suitable"
+                                " for explicitly building modules. This format "
+                                "is experimental and will change.")),
+    llvm::cl::init(ScanningOutputFormat::Make),
+    llvm::cl::cat(DependencyScannerCategory));
+
 llvm::cl::opt<unsigned>
     NumThreads("j", llvm::cl::Optional,
                llvm::cl::desc("Number of worker threads to use (default: use "
                               "all concurrent threads)"),
                llvm::cl::init(0), llvm::cl::cat(DependencyScannerCategory));
 
 llvm::cl::opt<std::string>
     CompilationDB("compilation-database",
AdjustingCompilations->appendArgumentsAdjuster(
       });
   AdjustingCompilations->appendArgumentsAdjuster(
       tooling::getClangStripSerializeDiagnosticAdjuster());
 
   SharedStream Errs(llvm::errs());
   // Print out the dependency results to STDOUT by default.
   SharedStream DependencyOS(llvm::outs());
 
-  DependencyScanningService Service(ScanMode, ReuseFileManager,
+  DependencyScanningService Service(ScanMode, Format, ReuseFileManager,
                                     SkipExcludedPPRanges);
 #if LLVM_ENABLE_THREADS
   unsigned NumWorkers =
       NumThreads == 0 ? llvm::hardware_concurrency() : NumThreads;
 #else
   unsigned NumWorkers = 1;
 #endif
   std::vector<std::unique_ptr<DependencyScanningTool>> WorkerTools;