[Frontend] Add flag to allow PCM generation despite compiler errors

[bnbarham]

Summary

As with precompiled headers, it's useful for indexers to be able to
continue through compiler errors in dependent modules.

Resolves rdar://69816264

[bnbarham]

Noticed I had left in the -fdisable-module-hash flags in the test, removed now.

[akyrtzi]

I'd like if we only had to use one flag (-fallow-pcm-with-compiler-errors) and have it handle both modules and PCH. Could you make the flag also work for PCH and/or add a test that verifies it works?
You may only have to change

Opts.AllowPCHWithCompilerErrors = Args.hasArg(OPT_fallow_pch_with_errors);

to

Opts.AllowPCHWithCompilerErrors = Args.hasArg(OPT_fallow_pch_with_errors, OPT_fallow_pcm_with_errors);

[bnbarham]

Have allow-pcm also set allow-pch + test to make sure that works.

clang/include/clang/Driver/Options.td

 def pic_level : Separate<["-"], "pic-level">,
   HelpText<"Value for __PIC__">;
 def pic_is_pie : Flag<["-"], "pic-is-pie">,
   HelpText<"File is for a position independent executable">;
 def fno_validate_pch : Flag<["-"], "fno-validate-pch">,
   HelpText<"Disable validation of precompiled headers">;
 def fallow_pch_with_errors : Flag<["-"], "fallow-pch-with-compiler-errors">,
   HelpText<"Accept a PCH file that was created with compiler errors">;
+def fallow_pcm_with_errors : Flag<["-"], "fallow-pcm-with-compiler-errors">,
+  HelpText<"Accept a PCM file that was created with compiler errors">;
 def dump_deserialized_pch_decls : Flag<["-"], "dump-deserialized-decls">,
   HelpText<"Dump declarations that are deserialized from PCH, for testing">;
 def error_on_deserialized_pch_decl : Separate<["-"], "error-on-deserialized-decl">,
   HelpText<"Emit error if a specific declaration is deserialized from PCH, for testing">;
 def error_on_deserialized_pch_decl_EQ : Joined<["-"], "error-on-deserialized-decl=">,
   Alias<error_on_deserialized_pch_decl>;
 def static_define : Flag<["-"], "static-define">,
   HelpText<"Should __STATIC__ be defined">;

clang/include/clang/Frontend/ASTUnit.h

public:
   ///
   /// \returns - The initialized ASTUnit or null if the AST failed to load.
   static std::unique_ptr<ASTUnit> LoadFromASTFile(
       const std::string &Filename, const PCHContainerReader &PCHContainerRdr,
       WhatToLoad ToLoad, IntrusiveRefCntPtr<DiagnosticsEngine> Diags,
       const FileSystemOptions &FileSystemOpts, bool UseDebugInfo = false,
       bool OnlyLocalDecls = false, ArrayRef<RemappedFile> RemappedFiles = None,
       CaptureDiagsKind CaptureDiagnostics = CaptureDiagsKind::None,
-      bool AllowPCHWithCompilerErrors = false,
+      bool AllowASTWithCompilerErrors = false,
       bool UserFilesAreVolatile = false);
 
 private:
   /// Helper function for \c LoadFromCompilerInvocation() and
   /// \c LoadFromCommandLine(), which loads an AST from a compiler invocation.
   ///
   /// \param PrecompilePreambleAfterNParses After how many parses the preamble
   /// of this translation unit should be precompiled, to improve the performance

clang/include/clang/Frontend/FrontendActions.h

protected:
   std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
                                                  StringRef InFile) override;
 
   TranslationUnitKind getTranslationUnitKind() override {
     return TU_Module;
   }
 
   bool hasASTFileSupport() const override { return false; }
+
+  bool shouldEraseOutputFiles() override;
 };
 
 class GenerateInterfaceStubsAction : public ASTFrontendAction {
 protected:
   std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
                                                  StringRef InFile) override;
 
   TranslationUnitKind getTranslationUnitKind() override { return TU_Module; }

clang/include/clang/Frontend/FrontendOptions.h

public:
   unsigned IncludeTimestamps : 1;
 
   /// Should a temporary file be used during compilation.
   unsigned UseTemporary : 1;
 
   /// When using -emit-module, treat the modulemap as a system module.
   unsigned IsSystemModule : 1;
 
+  /// Output (and read) PCM files regardless of compiler errors.
+  unsigned AllowPCMWithCompilerErrors : 1;
+
   CodeCompleteOptions CodeCompleteOpts;
 
   /// Specifies the output format of the AST.
   ASTDumpOutputFormat ASTDumpFormat = ADOF_Default;
 
   enum {
     ARCMT_None,
     ARCMT_Check,
FrontendOptions()
       : DisableFree(false), RelocatablePCH(false), ShowHelp(false),
         ShowStats(false), ShowTimers(false), TimeTrace(false),
         ShowVersion(false), FixWhatYouCan(false), FixOnlyWarnings(false),
         FixAndRecompile(false), FixToTemporaries(false),
         ARCMTMigrateEmitARCErrors(false), SkipFunctionBodies(false),
         UseGlobalModuleIndex(true), GenerateGlobalModuleIndex(true),
         ASTDumpDecls(false), ASTDumpLookups(false),
         BuildingImplicitModule(false), ModulesEmbedAllFiles(false),
-        IncludeTimestamps(true), UseTemporary(true), TimeTraceGranularity(500) {}
+        IncludeTimestamps(true), UseTemporary(true),
+        AllowPCMWithCompilerErrors(false), TimeTraceGranularity(500) {}
 
   /// getInputKindForExtension - Return the appropriate input kind for a file
   /// extension. For example, "c" would return Language::C.
   ///
   /// \return The input kind for the extension, or Language::Unknown if the
   /// extension is not recognized.
   static InputKind getInputKindForExtension(StringRef Extension);
 };
 
 } // namespace clang
 
 #endif // LLVM_CLANG_FRONTEND_FRONTENDOPTIONS_H

clang/lib/Frontend/ASTUnit.cpp

Diags->setClient(new FilterAndStoreDiagnosticConsumer(
         CaptureDiagnostics != CaptureDiagsKind::AllWithoutNonErrorsFromIncludes));
 }
 
 std::unique_ptr<ASTUnit> ASTUnit::LoadFromASTFile(
     const std::string &Filename, const PCHContainerReader &PCHContainerRdr,
     WhatToLoad ToLoad, IntrusiveRefCntPtr<DiagnosticsEngine> Diags,
     const FileSystemOptions &FileSystemOpts, bool UseDebugInfo,
     bool OnlyLocalDecls, ArrayRef<RemappedFile> RemappedFiles,
-    CaptureDiagsKind CaptureDiagnostics, bool AllowPCHWithCompilerErrors,
+    CaptureDiagsKind CaptureDiagnostics, bool AllowASTWithCompilerErrors,
     bool UserFilesAreVolatile) {
   std::unique_ptr<ASTUnit> AST(new ASTUnit(true));
 
   // Recover resources if we crash before exiting this method.
   llvm::CrashRecoveryContextCleanupRegistrar<ASTUnit>
     ASTUnitCleanup(AST.get());
   llvm::CrashRecoveryContextCleanupRegistrar<DiagnosticsEngine,
     llvm::CrashRecoveryContextReleaseRefCleanup<DiagnosticsEngine>>
AST->Ctx = new ASTContext(*AST->LangOpts, AST->getSourceManager(),
                               PP.getBuiltinInfo());
 
   bool disableValid = false;
   if (::getenv("LIBCLANG_DISABLE_PCH_VALIDATION"))
     disableValid = true;
   AST->Reader = new ASTReader(
       PP, *AST->ModuleCache, AST->Ctx.get(), PCHContainerRdr, {},
       /*isysroot=*/"",
-      /*DisableValidation=*/disableValid, AllowPCHWithCompilerErrors);
+      /*DisableValidation=*/disableValid, AllowASTWithCompilerErrors);
 
   AST->Reader->setListener(std::make_unique<ASTInfoCollector>(
       *AST->PP, AST->Ctx.get(), *AST->HSOpts, *AST->PPOpts, *AST->LangOpts,
       AST->TargetOpts, AST->Target, Counter));
 
   // Attach the AST reader to the AST context as an external AST
   // source, so that declarations will be deserialized from the
   // AST file as needed.

clang/lib/Frontend/CompilerInstance.cpp

if (getSourceManager().getModuleBuildStack().empty() &&
       getHeaderSearchOpts().ModuleCachePruneInterval > 0 &&
       getHeaderSearchOpts().ModuleCachePruneAfter > 0) {
     pruneModuleCache(getHeaderSearchOpts());
   }
 
   HeaderSearchOptions &HSOpts = getHeaderSearchOpts();
   std::string Sysroot = HSOpts.Sysroot;
   const PreprocessorOptions &PPOpts = getPreprocessorOpts();
+  const FrontendOptions &FEOpts = getFrontendOpts();
   std::unique_ptr<llvm::Timer> ReadTimer;
+
   if (FrontendTimerGroup)
     ReadTimer = std::make_unique<llvm::Timer>("reading_modules",
                                                 "Reading modules",
                                                 *FrontendTimerGroup);
   TheASTReader = new ASTReader(
       getPreprocessor(), getModuleCache(), &getASTContext(),
       getPCHContainerReader(), getFrontendOpts().ModuleFileExtensions,
       Sysroot.empty() ? "" : Sysroot.c_str(), PPOpts.DisablePCHValidation,
-      /*AllowASTWithCompilerErrors=*/false,
+      /*AllowASTWithCompilerErrors=*/FEOpts.AllowPCMWithCompilerErrors,
       /*AllowConfigurationMismatch=*/false, HSOpts.ModulesValidateSystemHeaders,
       HSOpts.ValidateASTInputFilesContent,
       getFrontendOpts().UseGlobalModuleIndex, std::move(ReadTimer));
   if (hasASTConsumer()) {
     TheASTReader->setDeserializationListener(
         getASTConsumer().GetASTDeserializationListener());
     getASTContext().setASTMutationListener(
       getASTConsumer().GetASTMutationListener());

clang/lib/Frontend/CompilerInvocation.cpp

for (const auto *A : Args.filtered(OPT_fmodule_file)) {
     if (Val.find('=') == StringRef::npos)
       Opts.ModuleFiles.push_back(std::string(Val));
   }
   Opts.ModulesEmbedFiles = Args.getAllArgValues(OPT_fmodules_embed_file_EQ);
   Opts.ModulesEmbedAllFiles = Args.hasArg(OPT_fmodules_embed_all_files);
   Opts.IncludeTimestamps = !Args.hasArg(OPT_fno_pch_timestamp);
   Opts.UseTemporary = !Args.hasArg(OPT_fno_temp_file);
   Opts.IsSystemModule = Args.hasArg(OPT_fsystem_module);
+  Opts.AllowPCMWithCompilerErrors = Args.hasArg(OPT_fallow_pcm_with_errors);
 
   if (Opts.ProgramAction != frontend::GenerateModule && Opts.IsSystemModule)
     Diags.Report(diag::err_drv_argument_only_allowed_with) << "-fsystem-module"
                                                            << "-emit-module";
 
   Opts.CodeCompleteOpts.IncludeMacros
     = Args.hasArg(OPT_code_completion_macros);
   Opts.CodeCompleteOpts.IncludeCodePatterns

clang/lib/Frontend/FrontendActions.cpp

GenerateModuleAction::CreateASTConsumer(CompilerInstance &CI,
   std::string Sysroot;
 
   auto Buffer = std::make_shared<PCHBuffer>();
   std::vector<std::unique_ptr<ASTConsumer>> Consumers;
 
   Consumers.push_back(std::make_unique<PCHGenerator>(
       CI.getPreprocessor(), CI.getModuleCache(), OutputFile, Sysroot, Buffer,
       CI.getFrontendOpts().ModuleFileExtensions,
-      /*AllowASTWithErrors=*/false,
+      /*AllowASTWithErrors=*/
+      +CI.getFrontendOpts().AllowPCMWithCompilerErrors,
       /*IncludeTimestamps=*/
       +CI.getFrontendOpts().BuildingImplicitModule,
       /*ShouldCacheASTInMemory=*/
       +CI.getFrontendOpts().BuildingImplicitModule));
   Consumers.push_back(CI.getPCHContainerWriter().CreatePCHContainerGenerator(
       CI, std::string(InFile), OutputFile, std::move(OS), Buffer));
   return std::make_unique<MultiplexConsumer>(std::move(Consumers));
 }
 
+bool GenerateModuleAction::shouldEraseOutputFiles() {
+  return !getCompilerInstance().getFrontendOpts().AllowPCMWithCompilerErrors &&
+         ASTFrontendAction::shouldEraseOutputFiles();
+}
+
 bool GenerateModuleFromModuleMapAction::BeginSourceFileAction(
     CompilerInstance &CI) {
   if (!CI.getLangOpts().Modules) {
     CI.getDiagnostics().Report(diag::err_module_build_requires_fmodules);
     return false;
   }
 
   return GenerateModuleAction::BeginSourceFileAction(CI);
void VerifyPCHAction::ExecuteAction() {
   CompilerInstance &CI = getCompilerInstance();
   bool Preamble = CI.getPreprocessorOpts().PrecompiledPreambleBytes.first != 0;
   const std::string &Sysroot = CI.getHeaderSearchOpts().Sysroot;
   std::unique_ptr<ASTReader> Reader(new ASTReader(
       CI.getPreprocessor(), CI.getModuleCache(), &CI.getASTContext(),
       CI.getPCHContainerReader(), CI.getFrontendOpts().ModuleFileExtensions,
       Sysroot.empty() ? "" : Sysroot.c_str(),
       /*DisableValidation*/ false,
-      /*AllowPCHWithCompilerErrors*/ false,
+      /*AllowASTWithCompilerErrors*/ false,
       /*AllowConfigurationMismatch*/ true,
       /*ValidateSystemInputs*/ true));
 
   Reader->ReadAST(getCurrentFile(),
                   Preamble ? serialization::MK_Preamble
                            : serialization::MK_PCH,
                   SourceLocation(),
                   ASTReader::ARR_ConfigurationMismatch);

clang/test/Modules/Inputs/error.h

+@import undefined
+
+@interface Error
+- (int)method;
+undefined
+@end
+
+undefined

clang/test/Modules/Inputs/module.map

module template_nontrivial0 {
   header "template-nontrivial0.h"
   export *
 }
 
 module template_nontrivial1 {
   header "template-nontrivial1.h"
   export *
 }
+module error { header "error.h" }

clang/test/Modules/load-module-with-errors.m

+// RUN: rm -rf %t
+
+// RUN: %clang_cc1 -fmodules -fallow-pcm-with-compiler-errors -fmodules-cache-path=%t -emit-module -x objective-c -fmodule-name=error %S/Inputs/module.map
+// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fallow-pcm-with-compiler-errors -ast-print -x objective-c -fmodules-cache-path=%t -I %S/Inputs %s | FileCheck -check-prefix=CHECK-PRINT %s
+
+@import error;
+
+void test(id x) {
+  [x method];
+}
+
+// CHECK-PRINT: @interface Error
+// CHECK-PRINT-NEXT: - (int)method;
+// CHECK-PRINT: void test(id x)

clang/tools/c-index-test/core_main.cpp

static bool printSourceSymbolsFromModule(StringRef modulePath,
   }
 
   IntrusiveRefCntPtr<DiagnosticsEngine> Diags =
       CompilerInstance::createDiagnostics(new DiagnosticOptions());
   std::unique_ptr<ASTUnit> AU = ASTUnit::LoadFromASTFile(
       std::string(modulePath), *pchRdr, ASTUnit::LoadASTOnly, Diags,
       FileSystemOpts, /*UseDebugInfo=*/false,
       /*OnlyLocalDecls=*/true, None, CaptureDiagsKind::None,
-      /*AllowPCHWithCompilerErrors=*/true,
+      /*AllowASTWithCompilerErrors=*/true,
       /*UserFilesAreVolatile=*/false);
   if (!AU) {
     errs() << "failed to create TU for: " << modulePath << '\n';
     return true;
   }
 
   PrintIndexDataConsumer DataConsumer(outs());
   IndexingOptions IndexOpts;

clang/tools/libclang/CIndex.cpp

enum CXErrorCode clang_createTranslationUnit2(CXIndex CIdx,
   FileSystemOptions FileSystemOpts;
 
   IntrusiveRefCntPtr<DiagnosticsEngine> Diags =
       CompilerInstance::createDiagnostics(new DiagnosticOptions());
   std::unique_ptr<ASTUnit> AU = ASTUnit::LoadFromASTFile(
       ast_filename, CXXIdx->getPCHContainerOperations()->getRawReader(),
       ASTUnit::LoadEverything, Diags, FileSystemOpts, /*UseDebugInfo=*/false,
       CXXIdx->getOnlyLocalDecls(), None, CaptureDiagsKind::All,
-      /*AllowPCHWithCompilerErrors=*/true,
+      /*AllowASTWithCompilerErrors=*/true,
       /*UserFilesAreVolatile=*/true);
   *out_TU = MakeCXTranslationUnit(CXXIdx, std::move(AU));
   return *out_TU ? CXError_Success : CXError_Failure;
 }
 
 unsigned clang_defaultEditingTranslationUnitOptions() {
   return CXTranslationUnit_PrecompiledPreamble |
          CXTranslationUnit_CacheCompletionResults;