diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -46,25 +46,73 @@ DependencyConsumer &C; }; -/// A listener that collects the names and paths to imported modules. -class ImportCollectingListener : public ASTReaderListener { - using PrebuiltModuleFilesT = - decltype(HeaderSearchOptions::PrebuiltModuleFiles); - +/// A listener that collects the imported modules and optionally the input +/// files. +class PrebuiltModuleListener : public ASTReaderListener { public: - ImportCollectingListener(PrebuiltModuleFilesT &PrebuiltModuleFiles) - : PrebuiltModuleFiles(PrebuiltModuleFiles) {} + PrebuiltModuleListener(llvm::StringMap &PrebuiltModuleFiles, + llvm::StringSet<> &InputFiles, bool VisitInputFiles) + : PrebuiltModuleFiles(PrebuiltModuleFiles), InputFiles(InputFiles), + VisitInputFiles(VisitInputFiles) {} bool needsImportVisitation() const override { return true; } + bool needsInputFileVisitation() override { return VisitInputFiles; } + bool needsSystemInputFileVisitation() override { return VisitInputFiles; } void visitImport(StringRef ModuleName, StringRef Filename) override { - PrebuiltModuleFiles[std::string(ModuleName)] = std::string(Filename); + PrebuiltModuleFiles.insert({ModuleName, Filename.str()}); + } + + bool visitInputFile(StringRef Filename, bool isSystem, bool isOverridden, + bool isExplicitModule) override { + InputFiles.insert(Filename); + return true; } private: - PrebuiltModuleFilesT &PrebuiltModuleFiles; + llvm::StringMap &PrebuiltModuleFiles; + llvm::StringSet<> &InputFiles; + bool VisitInputFiles; }; +using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles); + +/// Visit the given prebuilt module and collect all of the modules it +/// transitively imports and contributing input files. +static void visitPrebuiltModule(StringRef PrebuiltModuleFilename, + CompilerInstance &CI, + PrebuiltModuleFilesT &ModuleFiles, + llvm::StringSet<> &InputFiles, + bool VisitInputFiles) { + // Maps the names of modules that weren't yet visited to their PCM path. + llvm::StringMap ModuleFilesWorklist; + // Contains PCM paths of all visited modules. + llvm::StringSet<> VisitedModuleFiles; + + PrebuiltModuleListener Listener(ModuleFilesWorklist, InputFiles, + VisitInputFiles); + + auto GatherModuleFileInfo = [&](StringRef ASTFile) { + ASTReader::readASTFileControlBlock( + ASTFile, CI.getFileManager(), CI.getPCHContainerReader(), + /*FindModuleFileExtensions=*/false, Listener, + /*ValidateDiagnosticOptions=*/false); + }; + + GatherModuleFileInfo(PrebuiltModuleFilename); + while (!ModuleFilesWorklist.empty()) { + auto WorklistItemIt = ModuleFilesWorklist.begin(); + + if (!VisitedModuleFiles.contains(WorklistItemIt->getValue())) { + VisitedModuleFiles.insert(WorklistItemIt->getValue()); + GatherModuleFileInfo(WorklistItemIt->getValue()); + ModuleFiles[WorklistItemIt->getKey().str()] = WorklistItemIt->getValue(); + } + + ModuleFilesWorklist.erase(WorklistItemIt); + } +} + /// Transform arbitrary file name into an object-like file name. static std::string makeObjFileName(StringRef FileName) { SmallString<128> ObjFileName(FileName); @@ -122,13 +170,33 @@ Compiler.getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath = true; - // Use the dependency scanning optimized file system if we can. + FileMgr->getFileSystemOpts().WorkingDir = std::string(WorkingDirectory); + Compiler.setFileManager(FileMgr); + Compiler.createSourceManager(*FileMgr); + + llvm::StringSet<> PrebuiltModulesInputFiles; + // Store the list of prebuilt module files into header search options. This + // will prevent the implicit build to create duplicate modules and will + // force reuse of the existing prebuilt module files instead. + if (!Compiler.getPreprocessorOpts().ImplicitPCHInclude.empty()) + visitPrebuiltModule( + Compiler.getPreprocessorOpts().ImplicitPCHInclude, Compiler, + Compiler.getHeaderSearchOpts().PrebuiltModuleFiles, + PrebuiltModulesInputFiles, /*VisitInputFiles=*/DepFS != nullptr); + + // Use the dependency scanning optimized file system if requested to do so. if (DepFS) { const CompilerInvocation &CI = Compiler.getInvocation(); + DepFS->clearIgnoredFiles(); + // Ignore any files that contributed to prebuilt modules. The implicit + // build validates the modules by comparing the reported sizes of their + // inputs to the current state of the filesystem. Minimization would throw + // this mechanism off. + for (const auto &File : PrebuiltModulesInputFiles) + DepFS->ignoreFile(File.getKey()); // Add any filenames that were explicity passed in the build settings and // that might be opened, as we want to ensure we don't run source // minimization on them. - DepFS->clearIgnoredFiles(); for (const auto &Entry : CI.getHeaderSearchOpts().UserEntries) DepFS->ignoreFile(Entry.Path); for (const auto &Entry : CI.getHeaderSearchOpts().VFSOverlayFiles) @@ -146,24 +214,6 @@ .ExcludedConditionalDirectiveSkipMappings = PPSkipMappings; } - FileMgr->getFileSystemOpts().WorkingDir = std::string(WorkingDirectory); - Compiler.setFileManager(FileMgr); - Compiler.createSourceManager(*FileMgr); - - if (!Compiler.getPreprocessorOpts().ImplicitPCHInclude.empty()) { - // Collect the modules that were prebuilt as part of the PCH and pass them - // to the compiler. This will prevent the implicit build to create - // duplicate modules and force reuse of existing prebuilt module files - // instead. - ImportCollectingListener Listener( - Compiler.getHeaderSearchOpts().PrebuiltModuleFiles); - ASTReader::readASTFileControlBlock( - Compiler.getPreprocessorOpts().ImplicitPCHInclude, - Compiler.getFileManager(), Compiler.getPCHContainerReader(), - /*FindModuleFileExtensions=*/false, Listener, - /*ValidateDiagnosticOptions=*/false); - } - // Create the dependency collector that will collect the produced // dependencies. // diff --git a/clang/test/ClangScanDeps/modules-pch.c b/clang/test/ClangScanDeps/modules-pch.c --- a/clang/test/ClangScanDeps/modules-pch.c +++ b/clang/test/ClangScanDeps/modules-pch.c @@ -6,7 +6,7 @@ // RUN: sed "s|DIR|%/t|g" %S/Inputs/modules-pch/cdb_pch.json > %t/cdb.json // RUN: echo -%t > %t/result_pch.json // RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full \ -// RUN: -generate-modules-path-args -module-files-dir %t/build -mode preprocess >> %t/result_pch.json +// RUN: -generate-modules-path-args -module-files-dir %t/build >> %t/result_pch.json // RUN: cat %t/result_pch.json | sed 's:\\\\\?:/:g' | FileCheck %s -check-prefix=CHECK-PCH // // Check we didn't build the PCH during dependency scanning. @@ -127,9 +127,8 @@ // // RUN: sed "s|DIR|%/t|g" %S/Inputs/modules-pch/cdb_tu.json > %t/cdb.json // RUN: echo -%t > %t/result_tu.json -// FIXME: Make this work with '-mode preprocess-minimized-sources'. // RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full \ -// RUN: -generate-modules-path-args -module-files-dir %t/build -mode preprocess >> %t/result_tu.json +// RUN: -generate-modules-path-args -module-files-dir %t/build >> %t/result_tu.json // RUN: cat %t/result_tu.json | sed 's:\\\\\?:/:g' | FileCheck %s -check-prefix=CHECK-TU // // CHECK-TU: -[[PREFIX:.*]] @@ -193,7 +192,7 @@ // RUN: sed "s|DIR|%/t|g" %S/Inputs/modules-pch/cdb_tu_with_common.json > %t/cdb.json // RUN: echo -%t > %t/result_tu_with_common.json // RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full \ -// RUN: -generate-modules-path-args -module-files-dir %t/build -mode preprocess >> %t/result_tu_with_common.json +// RUN: -generate-modules-path-args -module-files-dir %t/build >> %t/result_tu_with_common.json // RUN: cat %t/result_tu_with_common.json | sed 's:\\\\\?:/:g' | FileCheck %s -check-prefix=CHECK-TU-WITH-COMMON // // CHECK-TU-WITH-COMMON: -[[PREFIX:.*]]