[clang][deps] Handle precompiled headers' AST files

[jansvoboda11]

Summary

The PreprocessOnlyAction doesn't support loading the AST file of a precompiled header. This is problematic for dependency scanning, since the #include manufactured for the PCH is treated as textual. This means the PCH contents get scanned with each TU, which is redundant. Moreover, dependencies of the PCH end up being considered dependency of the TU.

To handle AST file of PCH properly, this patch creates new FrontendAction that behaves the same way PreprocessorOnlyAction does, but treats the manufactured PCH #include as a normal compilation would (by not claiming it only uses a preprocessor and creating the default AST consumer).

The AST file is now reported as a file dependency of the TU.

Depends on D103519.

[Bigcheese]

lgtm.

clang/test/ClangScanDeps/modules-pch.c

7

Is this needed for this commit?

11–60

You should mention that PCHs don't work with minimization yet.

clang/include/clang/Frontend/FrontendActions.h

std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
                                                  StringRef InFile) override;
 
 public:
   // Don't claim to only use the preprocessor, we want to follow the AST path,
   // but do nothing.
   bool usesPreprocessorOnly() const override { return false; }
 };
 
+/// Preprocessor-based frontend action that also loads PCH files.
+class ReadPCHAndPreprocessAction : public FrontendAction {
+  void ExecuteAction() override;
+
+  std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
+                                                 StringRef InFile) override;
+
+public:
+  bool usesPreprocessorOnly() const override { return false; }
+};
+
 class DumpCompilerOptionsAction : public FrontendAction {
   std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
                                                  StringRef InFile) override {
     return nullptr;
   }
 
   void ExecuteAction() override;
 

clang/lib/Frontend/FrontendActions.cpp

 std::unique_ptr<ASTConsumer>
 InitOnlyAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {
   return std::make_unique<ASTConsumer>();
 }
 
 void InitOnlyAction::ExecuteAction() {
 }
 
+// Basically PreprocessOnlyAction::ExecuteAction.
+void ReadPCHAndPreprocessAction::ExecuteAction() {
+  Preprocessor &PP = getCompilerInstance().getPreprocessor();
+
+  // Ignore unknown pragmas.
+  PP.IgnorePragmas();
+
+  Token Tok;
+  // Start parsing the specified input file.
+  PP.EnterMainSourceFile();
+  do {
+    PP.Lex(Tok);
+  } while (Tok.isNot(tok::eof));
+}
+
+std::unique_ptr<ASTConsumer>
+ReadPCHAndPreprocessAction::CreateASTConsumer(CompilerInstance &CI,
+                                              StringRef InFile) {
+  return std::make_unique<ASTConsumer>();
+}
+
 //===----------------------------------------------------------------------===//
 // AST Consumer Actions
 //===----------------------------------------------------------------------===//
 
 std::unique_ptr<ASTConsumer>
 ASTPrintAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {
   if (std::unique_ptr<raw_ostream> OS =
           CI.createDefaultOutputFile(false, InFile))

clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp

bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
 
     // Don't print 'X warnings and Y errors generated'.
     Compiler.getDiagnosticOpts().ShowCarets = false;
     // Create the compiler's actual diagnostics engine.
     Compiler.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false);
     if (!Compiler.hasDiagnostics())
       return false;
 
+    Compiler.getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath = true;
+
     // Use the dependency scanning optimized file system if we can.
     if (DepFS) {
       const CompilerInvocation &CI = Compiler.getInvocation();
       // Add any filenames that were explicity passed in the build settings and
       // that might be opened, as we want to ensure we don't run source
       // minimization on them.
       DepFS->IgnoredFiles.clear();
       for (const auto &Entry : CI.getHeaderSearchOpts().UserEntries)
bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
 
     // Consider different header search and diagnostic options to create
     // different modules. This avoids the unsound aliasing of module PCMs.
     //
     // TODO: Implement diagnostic bucketing and header search pruning to reduce
     // the impact of strict context hashing.
     Compiler.getHeaderSearchOpts().ModulesStrictContextHash = true;
 
-    auto Action = std::make_unique<PreprocessOnlyAction>();
+    auto Action = std::make_unique<ReadPCHAndPreprocessAction>();
     const bool Result = Compiler.ExecuteAction(*Action);
     if (!DepFS)
       FileMgr->clearStatCache();
     return Result;
   }
 
 private:
   StringRef WorkingDirectory;

clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp

void ModuleDepCollectorPP::handleImport(const Module *Imported) {
   DirectModularDeps.insert(TopLevelModule);
 }
 
 void ModuleDepCollectorPP::EndOfMainFile() {
   FileID MainFileID = Instance.getSourceManager().getMainFileID();
   MDC.MainFile = std::string(
       Instance.getSourceManager().getFileEntryForID(MainFileID)->getName());
 
+  if (!Instance.getPreprocessorOpts().ImplicitPCHInclude.empty())
+    MDC.FileDeps.push_back(Instance.getPreprocessorOpts().ImplicitPCHInclude);
+
   for (const Module *M : DirectModularDeps)
     handleTopLevelModule(M);
 
   for (auto &&I : MDC.ModularDeps)
     MDC.Consumer.handleModuleDependency(I.second);
 
   for (auto &&I : MDC.FileDeps)
     MDC.Consumer.handleFileDependency(*MDC.Opts, I);

clang/test/ClangScanDeps/modules-pch.c

 // RUN: rm -rf %t && mkdir %t
 // RUN: cp %S/Inputs/modules-pch/* %t
 
 // Explicitly build the PCH:
 //
 // RUN: %clang -x c-header %t/pch.h -fmodules -gmodules -fimplicit-module-maps \
 // RUN:   -fmodules-cache-path=%t/cache -o %t/pch.h.gch
 
 // Scan dependencies of the TU:
 //
 // RUN: sed "s|DIR|%/t|g" %S/Inputs/modules-pch/cdb_tu.json > %t/cdb.json
+// RUN: echo -%t > %t/result_tu.json
+// FIXME: Make this work with '-mode preprocess-minimized-sources'.
 // RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full \
-// RUN:   -generate-modules-path-args -module-files-dir %t/build
+// RUN:   -generate-modules-path-args -module-files-dir %t/build -mode preprocess >> %t/result_tu.json
+// RUN: cat %t/result_tu.json | sed 's:\\\\\?:/:g' | FileCheck %s -check-prefix=CHECK-TU
+//
+// CHECK-TU:      -[[PREFIX:.*]]
+// CHECK-TU-NEXT: {
+// CHECK-TU-NEXT:   "modules": [
+// CHECK-TU-NEXT:     {
+// CHECK-TU-NEXT:       "clang-module-deps": [],
+// CHECK-TU-NEXT:       "clang-modulemap-file": "[[PREFIX]]/module.modulemap",
+// CHECK-TU-NEXT:       "command-line": [
+// CHECK-TU-NEXT:         "-cc1",
+// CHECK-TU:              "-emit-module",
+// CHECK-TU:              "-fmodule-name=ModTU",
+// CHECK-TU:              "-fno-implicit-modules",
+// CHECK-TU:            ],
+// CHECK-TU-NEXT:       "context-hash": "[[HASH_MOD_TU:.*]]",
+// CHECK-TU-NEXT:       "file-deps": [
+// CHECK-TU-NEXT:         "[[PREFIX]]/mod_tu.h",
+// CHECK-TU-NEXT:         "[[PREFIX]]/module.modulemap"
+// CHECK-TU-NEXT:       ],
+// CHECK-TU-NEXT:       "name": "ModTU"
+// CHECK-TU-NEXT:     }
+// CHECK-TU-NEXT:   ],
+// CHECK-TU-NEXT:   "translation-units": [
+// CHECK-TU-NEXT:     {
+// CHECK-TU-NEXT:       "clang-context-hash": "[[HASH_TU:.*]]",
+// CHECK-TU-NEXT:       "clang-module-deps": [
+// CHECK-TU-NEXT:         {
+// CHECK-TU-NEXT:           "context-hash": "[[HASH_MOD_TU]]",
+// CHECK-TU-NEXT:           "module-name": "ModTU"
+// CHECK-TU-NEXT:         }
+// CHECK-TU-NEXT:       ],
+// CHECK-TU-NEXT:       "command-line": [
+// CHECK-TU-NEXT:         "-fno-implicit-modules",
+// CHECK-TU-NEXT:         "-fno-implicit-module-maps",
+// CHECK-TU-NEXT:         "-fmodule-file=[[PREFIX]]/build/[[HASH_MOD_TU]]/ModTU-{{.*}}.pcm",
+// CHECK-TU-NEXT:         "-fmodule-map-file=[[PREFIX]]/module.modulemap"
+// CHECK-TU-NEXT:       ],
+// CHECK-TU-NEXT:       "file-deps": [
+// CHECK-TU-NEXT:         "[[PREFIX]]/tu.c",
+// CHECK-TU-NEXT:         "[[PREFIX]]/pch.h.gch"
+// CHECK-TU-NEXT:       ],
+// CHECK-TU-NEXT:       "input-file": "[[PREFIX]]/tu.c"
+// CHECK-TU-NEXT:     }
+// CHECK-TU-NEXT:   ]
+// CHECK-TU-NEXT: }