[clang][deps] Avoid minimizing PCH input files

[jansvoboda11]

Summary

This patch avoid minimizing input files that contributed to a PCH or its modules. This prevents the implicit modular build to fail on unexpected file size. Depends on D106146.

[jansvoboda11]

Handle system input files.

[jansvoboda11]

I run a quick benchmark where an empty TU is configured with an explicitly built PCH that imports Cocoa. Dependency scanning without minimization takes 226 ms, with this patch 303 ms and 325 ms when padding minimized files to the original size. Since we'll eventually switch to a more sound implementation, I'm fine with this in the meantime.

[jansvoboda11]

Entirely disable DepFS when scanning PCHs.

[jansvoboda11]

Rebase on top of D106064, solving Windows CI failure.

[jansvoboda11]

Rebase on top of D106146.

[dexonsmith]

LGTM.

[thakis]

Looks like this breaks tests on Windows: http://45.33.8.238/win/42264/step_7.txt

Please take a look, and revert for now if it takes a while to fix.

[jansvoboda11]

In D104536#2889999, @thakis wrote:

Looks like this breaks tests on Windows: http://45.33.8.238/win/42264/step_7.txt

Please take a look, and revert for now if it takes a while to fix.

Thanks, that was a bad rebase. Should be fixed in c94a345a5c693b6c12a41e8f50e3fe96d1311991.

clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp

void finishedMainFile(DiagnosticsEngine &Diags) override {
     }
   }
 
 private:
   std::unique_ptr<DependencyOutputOptions> Opts;
   DependencyConsumer &C;
 };
 
-/// A listener that collects the names and paths to imported modules.
-class ImportCollectingListener : public ASTReaderListener {
-  using PrebuiltModuleFilesT =
-      decltype(HeaderSearchOptions::PrebuiltModuleFiles);
-
+/// A listener that collects the imported modules and optionally the input
+/// files.
+class PrebuiltModuleListener : public ASTReaderListener {
 public:
-  ImportCollectingListener(PrebuiltModuleFilesT &PrebuiltModuleFiles)
-      : PrebuiltModuleFiles(PrebuiltModuleFiles) {}
+  PrebuiltModuleListener(llvm::StringMap<std::string> &PrebuiltModuleFiles,
+                         llvm::StringSet<> &InputFiles, bool VisitInputFiles)
+      : PrebuiltModuleFiles(PrebuiltModuleFiles), InputFiles(InputFiles),
+        VisitInputFiles(VisitInputFiles) {}
 
   bool needsImportVisitation() const override { return true; }
+  bool needsInputFileVisitation() override { return VisitInputFiles; }
+  bool needsSystemInputFileVisitation() override { return VisitInputFiles; }
 
   void visitImport(StringRef ModuleName, StringRef Filename) override {
-    PrebuiltModuleFiles[std::string(ModuleName)] = std::string(Filename);
+    PrebuiltModuleFiles.insert({ModuleName, Filename.str()});
+  }
+
+  bool visitInputFile(StringRef Filename, bool isSystem, bool isOverridden,
+                      bool isExplicitModule) override {
+    InputFiles.insert(Filename);
+    return true;
   }
 
 private:
-  PrebuiltModuleFilesT &PrebuiltModuleFiles;
+  llvm::StringMap<std::string> &PrebuiltModuleFiles;
+  llvm::StringSet<> &InputFiles;
+  bool VisitInputFiles;
 };
 
+using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
+
+/// Visit the given prebuilt module and collect all of the modules it
+/// transitively imports and contributing input files.
+static void visitPrebuiltModule(StringRef PrebuiltModuleFilename,
+                                CompilerInstance &CI,
+                                PrebuiltModuleFilesT &ModuleFiles,
+                                llvm::StringSet<> &InputFiles,
+                                bool VisitInputFiles) {
+  // Maps the names of modules that weren't yet visited to their PCM path.
+  llvm::StringMap<std::string> ModuleFilesWorklist;
+  // Contains PCM paths of all visited modules.
+  llvm::StringSet<> VisitedModuleFiles;
+
+  PrebuiltModuleListener Listener(ModuleFilesWorklist, InputFiles,
+                                  VisitInputFiles);
+
+  auto GatherModuleFileInfo = [&](StringRef ASTFile) {
+    ASTReader::readASTFileControlBlock(
+        ASTFile, CI.getFileManager(), CI.getPCHContainerReader(),
+        /*FindModuleFileExtensions=*/false, Listener,
+        /*ValidateDiagnosticOptions=*/false);
+  };
+
+  GatherModuleFileInfo(PrebuiltModuleFilename);
+  while (!ModuleFilesWorklist.empty()) {
+    auto WorklistItemIt = ModuleFilesWorklist.begin();
+
+    if (!VisitedModuleFiles.contains(WorklistItemIt->getValue())) {
+      VisitedModuleFiles.insert(WorklistItemIt->getValue());
+      GatherModuleFileInfo(WorklistItemIt->getValue());
+      ModuleFiles[WorklistItemIt->getKey().str()] = WorklistItemIt->getValue();
+    }
+
+    ModuleFilesWorklist.erase(WorklistItemIt);
+  }
+}
+
 /// Transform arbitrary file name into an object-like file name.
 static std::string makeObjFileName(StringRef FileName) {
   SmallString<128> ObjFileName(FileName);
   llvm::sys::path::replace_extension(ObjFileName, "o");
   return std::string(ObjFileName.str());
 }
 
 /// Deduce the dependency target based on the output file and input files.
bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
     Compiler.getDiagnosticOpts().Warnings.push_back("no-error");
     // Create the compiler's actual diagnostics engine.
     Compiler.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false);
     if (!Compiler.hasDiagnostics())
       return false;
 
     Compiler.getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath = true;
 
-    // Use the dependency scanning optimized file system if we can.
+    FileMgr->getFileSystemOpts().WorkingDir = std::string(WorkingDirectory);
+    Compiler.setFileManager(FileMgr);
+    Compiler.createSourceManager(*FileMgr);
+
+    llvm::StringSet<> PrebuiltModulesInputFiles;
+    // Store the list of prebuilt module files into header search options. This
+    // will prevent the implicit build to create duplicate modules and will
+    // force reuse of the existing prebuilt module files instead.
+    if (!Compiler.getPreprocessorOpts().ImplicitPCHInclude.empty())
+      visitPrebuiltModule(
+          Compiler.getPreprocessorOpts().ImplicitPCHInclude, Compiler,
+          Compiler.getHeaderSearchOpts().PrebuiltModuleFiles,
+          PrebuiltModulesInputFiles, /*VisitInputFiles=*/DepFS != nullptr);
+
+    // Use the dependency scanning optimized file system if requested to do so.
     if (DepFS) {
       const CompilerInvocation &CI = Compiler.getInvocation();
+      DepFS->clearIgnoredFiles();
+      // Ignore any files that contributed to prebuilt modules. The implicit
+      // build validates the modules by comparing the reported sizes of their
+      // inputs to the current state of the filesystem. Minimization would throw
+      // this mechanism off.
+      for (const auto &File : PrebuiltModulesInputFiles)
+        DepFS->ignoreFile(File.getKey());
       // Add any filenames that were explicity passed in the build settings and
       // that might be opened, as we want to ensure we don't run source
       // minimization on them.
-      DepFS->clearIgnoredFiles();
       for (const auto &Entry : CI.getHeaderSearchOpts().UserEntries)
         DepFS->ignoreFile(Entry.Path);
       for (const auto &Entry : CI.getHeaderSearchOpts().VFSOverlayFiles)
         DepFS->ignoreFile(Entry);
 
       // Support for virtual file system overlays on top of the caching
       // filesystem.
       FileMgr->setVirtualFileSystem(createVFSFromCompilerInvocation(
           CI, Compiler.getDiagnostics(), DepFS));
 
       // Pass the skip mappings which should speed up excluded conditional block
       // skipping in the preprocessor.
       if (PPSkipMappings)
         Compiler.getPreprocessorOpts()
             .ExcludedConditionalDirectiveSkipMappings = PPSkipMappings;
     }
 
-    FileMgr->getFileSystemOpts().WorkingDir = std::string(WorkingDirectory);
-    Compiler.setFileManager(FileMgr);
-    Compiler.createSourceManager(*FileMgr);
-
-    if (!Compiler.getPreprocessorOpts().ImplicitPCHInclude.empty()) {
-      // Collect the modules that were prebuilt as part of the PCH and pass them
-      // to the compiler. This will prevent the implicit build to create
-      // duplicate modules and force reuse of existing prebuilt module files
-      // instead.
-      ImportCollectingListener Listener(
-          Compiler.getHeaderSearchOpts().PrebuiltModuleFiles);
-      ASTReader::readASTFileControlBlock(
-          Compiler.getPreprocessorOpts().ImplicitPCHInclude,
-          Compiler.getFileManager(), Compiler.getPCHContainerReader(),
-          /*FindModuleFileExtensions=*/false, Listener,
-          /*ValidateDiagnosticOptions=*/false);
-    }
-
     // Create the dependency collector that will collect the produced
     // dependencies.
     //
     // This also moves the existing dependency output options from the
     // invocation to the collector. The options in the invocation are reset,
     // which ensures that the compiler won't create new dependency collectors,
     // and thus won't write out the extra '.d' files to disk.
     auto Opts = std::make_unique<DependencyOutputOptions>();

clang/test/ClangScanDeps/modules-pch.c

 // RUN: rm -rf %t && mkdir %t
 // RUN: cp %S/Inputs/modules-pch/* %t
 
 // Scan dependencies of the PCH:
 //
 // RUN: sed "s|DIR|%/t|g" %S/Inputs/modules-pch/cdb_pch.json > %t/cdb.json
 // RUN: echo -%t > %t/result_pch.json
 // RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full \
-// RUN:   -generate-modules-path-args -module-files-dir %t/build -mode preprocess >> %t/result_pch.json
+// RUN:   -generate-modules-path-args -module-files-dir %t/build >> %t/result_pch.json
 // RUN: cat %t/result_pch.json | sed 's:\\\\\?:/:g' | FileCheck %s -check-prefix=CHECK-PCH
 //
 // Check we didn't build the PCH during dependency scanning.
 // RUN: not cat %/t/pch.h.gch
 //
 // CHECK-PCH:      -[[PREFIX:.*]]
 // CHECK-PCH-NEXT: {
 // CHECK-PCH-NEXT:   "modules": [

 // RUN: %clang @%t/mod_pch.cc1.rsp
 // RUN: %clang -x c-header %t/pch.h -fmodules -gmodules -fimplicit-module-maps \
 // RUN:   -fmodules-cache-path=%t/cache -o %t/pch.h.gch @%t/pch.rsp
 
 // Scan dependencies of the TU:
 //
 // RUN: sed "s|DIR|%/t|g" %S/Inputs/modules-pch/cdb_tu.json > %t/cdb.json
 // RUN: echo -%t > %t/result_tu.json
-// FIXME: Make this work with '-mode preprocess-minimized-sources'.
 // RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full \
-// RUN:   -generate-modules-path-args -module-files-dir %t/build -mode preprocess >> %t/result_tu.json
+// RUN:   -generate-modules-path-args -module-files-dir %t/build >> %t/result_tu.json
 // RUN: cat %t/result_tu.json | sed 's:\\\\\?:/:g' | FileCheck %s -check-prefix=CHECK-TU
 //
 // CHECK-TU:      -[[PREFIX:.*]]
 // CHECK-TU-NEXT: {
 // CHECK-TU-NEXT:   "modules": [
 // CHECK-TU-NEXT:     {
 // CHECK-TU-NEXT:       "clang-module-deps": [],
 // CHECK-TU-NEXT:       "clang-modulemap-file": "[[PREFIX]]/module.modulemap",

 // RUN: %clang -fsyntax-only %t/tu.c -fmodules -gmodules -fimplicit-module-maps \
 // RUN:   -fmodules-cache-path=%t/cache -include %t/pch.h -o %t/tu.o @%t/tu.rsp
 
 // Scan dependencies of the TU that has common modules with the PCH:
 //
 // RUN: sed "s|DIR|%/t|g" %S/Inputs/modules-pch/cdb_tu_with_common.json > %t/cdb.json
 // RUN: echo -%t > %t/result_tu_with_common.json
 // RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full \
-// RUN:   -generate-modules-path-args -module-files-dir %t/build -mode preprocess >> %t/result_tu_with_common.json
+// RUN:   -generate-modules-path-args -module-files-dir %t/build >> %t/result_tu_with_common.json
 // RUN: cat %t/result_tu_with_common.json | sed 's:\\\\\?:/:g' | FileCheck %s -check-prefix=CHECK-TU-WITH-COMMON
 //
 // CHECK-TU-WITH-COMMON:      -[[PREFIX:.*]]
 // CHECK-TU-WITH-COMMON-NEXT: {
 // CHECK-TU-WITH-COMMON-NEXT:   "modules": [
 // CHECK-TU-WITH-COMMON-NEXT:     {
 // CHECK-TU-WITH-COMMON-NEXT:       "clang-module-deps": [],
 // CHECK-TU-WITH-COMMON-NEXT:       "clang-modulemap-file": "[[PREFIX]]/module.modulemap",