diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp --- a/clang/lib/CodeGen/BackendUtil.cpp +++ b/clang/lib/CodeGen/BackendUtil.cpp @@ -1057,67 +1057,6 @@ MPM.addPass(createModuleToFunctionPassAdaptor(CoroCleanupPass())); } -static void addSanitizersAtO0(ModulePassManager &MPM, - const Triple &TargetTriple, - const LangOptions &LangOpts, - const CodeGenOptions &CodeGenOpts) { - if (CodeGenOpts.SanitizeCoverageType || - CodeGenOpts.SanitizeCoverageIndirectCalls || - CodeGenOpts.SanitizeCoverageTraceCmp) { - auto SancovOpts = getSancovOptsFromCGOpts(CodeGenOpts); - MPM.addPass(ModuleSanitizerCoveragePass( - SancovOpts, CodeGenOpts.SanitizeCoverageAllowlistFiles, - CodeGenOpts.SanitizeCoverageBlocklistFiles)); - } - - auto ASanPass = [&](SanitizerMask Mask, bool CompileKernel) { - MPM.addPass(RequireAnalysisPass()); - bool Recover = CodeGenOpts.SanitizeRecover.has(Mask); - MPM.addPass(createModuleToFunctionPassAdaptor(AddressSanitizerPass( - CompileKernel, Recover, CodeGenOpts.SanitizeAddressUseAfterScope))); - bool ModuleUseAfterScope = asanUseGlobalsGC(TargetTriple, CodeGenOpts); - MPM.addPass( - ModuleAddressSanitizerPass(CompileKernel, Recover, ModuleUseAfterScope, - CodeGenOpts.SanitizeAddressUseOdrIndicator)); - }; - - if (LangOpts.Sanitize.has(SanitizerKind::Address)) { - ASanPass(SanitizerKind::Address, /*CompileKernel=*/false); - } - - if (LangOpts.Sanitize.has(SanitizerKind::KernelAddress)) { - ASanPass(SanitizerKind::KernelAddress, /*CompileKernel=*/true); - } - - if (LangOpts.Sanitize.has(SanitizerKind::HWAddress)) { - bool Recover = CodeGenOpts.SanitizeRecover.has(SanitizerKind::HWAddress); - MPM.addPass(HWAddressSanitizerPass( - /*CompileKernel=*/false, Recover)); - } - if (LangOpts.Sanitize.has(SanitizerKind::KernelHWAddress)) { - MPM.addPass(HWAddressSanitizerPass( - /*CompileKernel=*/true, /*Recover=*/true)); - } - - if (LangOpts.Sanitize.has(SanitizerKind::Memory)) { - bool Recover = CodeGenOpts.SanitizeRecover.has(SanitizerKind::Memory); - int TrackOrigins = CodeGenOpts.SanitizeMemoryTrackOrigins; - MPM.addPass(MemorySanitizerPass({TrackOrigins, Recover, false})); - MPM.addPass(createModuleToFunctionPassAdaptor( - MemorySanitizerPass({TrackOrigins, Recover, false}))); - } - - if (LangOpts.Sanitize.has(SanitizerKind::KernelMemory)) { - MPM.addPass(createModuleToFunctionPassAdaptor( - MemorySanitizerPass({0, false, /*Kernel=*/true}))); - } - - if (LangOpts.Sanitize.has(SanitizerKind::Thread)) { - MPM.addPass(ThreadSanitizerPass()); - MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass())); - } -} - /// A clean version of `EmitAssembly` that uses the new pass manager. /// /// Not all features are currently supported in this system, but where @@ -1254,30 +1193,134 @@ bool IsThinLTO = CodeGenOpts.PrepareForThinLTO; bool IsLTO = CodeGenOpts.PrepareForLTO; - if (CodeGenOpts.OptimizationLevel == 0) { - // If we reached here with a non-empty index file name, then the index - // file was empty and we are not performing ThinLTO backend compilation - // (used in testing in a distributed build environment). Drop any the type - // test assume sequences inserted for whole program vtables so that - // codegen doesn't complain. - if (!CodeGenOpts.ThinLTOIndexFile.empty()) - MPM.addPass(LowerTypeTestsPass(/*ExportSummary=*/nullptr, - /*ImportSummary=*/nullptr, - /*DropTypeTests=*/true)); - if (Optional Options = getGCOVOptions(CodeGenOpts, LangOpts)) - MPM.addPass(GCOVProfilerPass(*Options)); - if (Optional Options = - getInstrProfOptions(CodeGenOpts, LangOpts)) - MPM.addPass(InstrProfiling(*Options, false)); + // If we reached here with a non-empty index file name, then the index + // file was empty and we are not performing ThinLTO backend compilation + // (used in testing in a distributed build environment). Drop any the type + // test assume sequences inserted for whole program vtables so that + // codegen doesn't complain. + if (!CodeGenOpts.ThinLTOIndexFile.empty()) + PB.registerPipelineStartEPCallback( + [](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) { + MPM.addPass(LowerTypeTestsPass(/*ExportSummary=*/nullptr, + /*ImportSummary=*/nullptr, + /*DropTypeTests=*/true)); + }); + + PB.registerPipelineStartEPCallback( + [](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) { + if (Level != PassBuilder::OptimizationLevel::O0) { + MPM.addPass(createModuleToFunctionPassAdaptor( + EntryExitInstrumenterPass(/*PostInlining=*/false))); + } + }); + + // Register callbacks to schedule sanitizer passes at the appropriate part + // of the pipeline. + if (LangOpts.Sanitize.has(SanitizerKind::LocalBounds)) + PB.registerScalarOptimizerLateEPCallback( + [](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) { + FPM.addPass(BoundsCheckingPass()); + }); + + if (CodeGenOpts.SanitizeCoverageType || + CodeGenOpts.SanitizeCoverageIndirectCalls || + CodeGenOpts.SanitizeCoverageTraceCmp) { + PB.registerOptimizerLastEPCallback( + [this](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) { + auto SancovOpts = getSancovOptsFromCGOpts(CodeGenOpts); + MPM.addPass(ModuleSanitizerCoveragePass( + SancovOpts, CodeGenOpts.SanitizeCoverageAllowlistFiles, + CodeGenOpts.SanitizeCoverageBlocklistFiles)); + }); + } + if (LangOpts.Sanitize.has(SanitizerKind::Memory)) { + int TrackOrigins = CodeGenOpts.SanitizeMemoryTrackOrigins; + bool Recover = CodeGenOpts.SanitizeRecover.has(SanitizerKind::Memory); + PB.registerOptimizerLastEPCallback( + [TrackOrigins, Recover](ModulePassManager &MPM, + PassBuilder::OptimizationLevel Level) { + MPM.addPass(MemorySanitizerPass({TrackOrigins, Recover, false})); + MPM.addPass(createModuleToFunctionPassAdaptor( + MemorySanitizerPass({TrackOrigins, Recover, false}))); + }); + } + if (LangOpts.Sanitize.has(SanitizerKind::Thread)) { + PB.registerOptimizerLastEPCallback( + [](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) { + MPM.addPass(ThreadSanitizerPass()); + MPM.addPass( + createModuleToFunctionPassAdaptor(ThreadSanitizerPass())); + }); + } + auto ASanPass = [&](SanitizerMask Mask, bool CompileKernel) { + bool Recover = CodeGenOpts.SanitizeRecover.has(Mask); + bool UseAfterScope = CodeGenOpts.SanitizeAddressUseAfterScope; + bool ModuleUseAfterScope = asanUseGlobalsGC(TargetTriple, CodeGenOpts); + bool UseOdrIndicator = CodeGenOpts.SanitizeAddressUseOdrIndicator; + PB.registerOptimizerLastEPCallback( + [CompileKernel, Recover, UseAfterScope, ModuleUseAfterScope, + UseOdrIndicator](ModulePassManager &MPM, + PassBuilder::OptimizationLevel Level) { + MPM.addPass( + RequireAnalysisPass()); + MPM.addPass(ModuleAddressSanitizerPass( + CompileKernel, Recover, ModuleUseAfterScope, UseOdrIndicator)); + MPM.addPass(createModuleToFunctionPassAdaptor( + AddressSanitizerPass(CompileKernel, Recover, UseAfterScope))); + }); + }; + if (LangOpts.Sanitize.has(SanitizerKind::Address)) + ASanPass(SanitizerKind::Address, false); + if (LangOpts.Sanitize.has(SanitizerKind::KernelAddress)) + ASanPass(SanitizerKind::KernelAddress, true); + + if (LangOpts.Sanitize.has(SanitizerKind::HWAddress)) { + bool Recover = CodeGenOpts.SanitizeRecover.has(SanitizerKind::HWAddress); + PB.registerOptimizerLastEPCallback( + [Recover](ModulePassManager &MPM, + PassBuilder::OptimizationLevel Level) { + MPM.addPass(HWAddressSanitizerPass( + /*CompileKernel=*/false, Recover)); + }); + } + if (LangOpts.Sanitize.has(SanitizerKind::KernelHWAddress)) { + bool Recover = + CodeGenOpts.SanitizeRecover.has(SanitizerKind::KernelHWAddress); + PB.registerOptimizerLastEPCallback( + [Recover](ModulePassManager &MPM, + PassBuilder::OptimizationLevel Level) { + MPM.addPass(HWAddressSanitizerPass( + /*CompileKernel=*/true, Recover)); + }); + } + + if (Optional Options = getGCOVOptions(CodeGenOpts, LangOpts)) + PB.registerPipelineStartEPCallback( + [Options](ModulePassManager &MPM, + PassBuilder::OptimizationLevel Level) { + MPM.addPass(GCOVProfilerPass(*Options)); + }); + if (Optional Options = + getInstrProfOptions(CodeGenOpts, LangOpts)) + PB.registerPipelineStartEPCallback( + [Options](ModulePassManager &MPM, + PassBuilder::OptimizationLevel Level) { + MPM.addPass(InstrProfiling(*Options, false)); + }); + + if (CodeGenOpts.OptimizationLevel == 0) { // Build a minimal pipeline based on the semantics required by Clang, // which is just that always inlining occurs. Further, disable generating // lifetime intrinsics to avoid enabling further optimizations during // code generation. // However, we need to insert lifetime intrinsics to avoid invalid access // caused by multithreaded coroutines. - MPM.addPass( - AlwaysInlinerPass(/*InsertLifetimeIntrinsics=*/LangOpts.Coroutines)); + PB.registerPipelineStartEPCallback( + [this](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) { + MPM.addPass(AlwaysInlinerPass( + /*InsertLifetimeIntrinsics=*/LangOpts.Coroutines)); + }); // At -O0, we can still do PGO. Add all the requested passes for // instrumentation PGO, if requested. @@ -1289,140 +1332,28 @@ /* IsCS */ false, PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile); - // At -O0 we directly run necessary sanitizer passes. - if (LangOpts.Sanitize.has(SanitizerKind::LocalBounds)) - MPM.addPass(createModuleToFunctionPassAdaptor(BoundsCheckingPass())); - - // Lastly, add semantically necessary passes for LTO. - if (IsLTO || IsThinLTO) { - MPM.addPass(CanonicalizeAliasesPass()); - MPM.addPass(NameAnonGlobalPass()); - } - } else { - // If we reached here with a non-empty index file name, then the index - // file was empty and we are not performing ThinLTO backend compilation - // (used in testing in a distributed build environment). Drop any the type - // test assume sequences inserted for whole program vtables so that - // codegen doesn't complain. - if (!CodeGenOpts.ThinLTOIndexFile.empty()) - PB.registerPipelineStartEPCallback( - [](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) { - MPM.addPass(LowerTypeTestsPass(/*ExportSummary=*/nullptr, - /*ImportSummary=*/nullptr, - /*DropTypeTests=*/true)); - }); - - PB.registerPipelineStartEPCallback( - [](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) { - MPM.addPass(createModuleToFunctionPassAdaptor( - EntryExitInstrumenterPass(/*PostInlining=*/false))); - }); - - // Register callbacks to schedule sanitizer passes at the appropriate part of - // the pipeline. - if (LangOpts.Sanitize.has(SanitizerKind::LocalBounds)) - PB.registerScalarOptimizerLateEPCallback( - [](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) { - FPM.addPass(BoundsCheckingPass()); - }); - - if (CodeGenOpts.SanitizeCoverageType || - CodeGenOpts.SanitizeCoverageIndirectCalls || - CodeGenOpts.SanitizeCoverageTraceCmp) { - PB.registerOptimizerLastEPCallback( - [this](ModulePassManager &MPM, - PassBuilder::OptimizationLevel Level) { - auto SancovOpts = getSancovOptsFromCGOpts(CodeGenOpts); - MPM.addPass(ModuleSanitizerCoveragePass( - SancovOpts, CodeGenOpts.SanitizeCoverageAllowlistFiles, - CodeGenOpts.SanitizeCoverageBlocklistFiles)); - }); - } + PB.runRegisteredEPCallbacks(MPM, Level, CodeGenOpts.DebugPassManager); - if (LangOpts.Sanitize.has(SanitizerKind::Memory)) { - int TrackOrigins = CodeGenOpts.SanitizeMemoryTrackOrigins; - bool Recover = CodeGenOpts.SanitizeRecover.has(SanitizerKind::Memory); - PB.registerOptimizerLastEPCallback( - [TrackOrigins, Recover](ModulePassManager &MPM, - PassBuilder::OptimizationLevel Level) { - MPM.addPass(MemorySanitizerPass({TrackOrigins, Recover, false})); - MPM.addPass(createModuleToFunctionPassAdaptor( - MemorySanitizerPass({TrackOrigins, Recover, false}))); - }); - } - if (LangOpts.Sanitize.has(SanitizerKind::Thread)) { - PB.registerOptimizerLastEPCallback( - [](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) { - MPM.addPass(ThreadSanitizerPass()); - MPM.addPass( - createModuleToFunctionPassAdaptor(ThreadSanitizerPass())); - }); - } - if (LangOpts.Sanitize.has(SanitizerKind::Address)) { - bool Recover = CodeGenOpts.SanitizeRecover.has(SanitizerKind::Address); - bool UseAfterScope = CodeGenOpts.SanitizeAddressUseAfterScope; - bool ModuleUseAfterScope = asanUseGlobalsGC(TargetTriple, CodeGenOpts); - bool UseOdrIndicator = CodeGenOpts.SanitizeAddressUseOdrIndicator; - PB.registerOptimizerLastEPCallback( - [Recover, UseAfterScope, ModuleUseAfterScope, UseOdrIndicator]( - ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) { - MPM.addPass( - RequireAnalysisPass()); - MPM.addPass(ModuleAddressSanitizerPass( - /*CompileKernel=*/false, Recover, ModuleUseAfterScope, - UseOdrIndicator)); - MPM.addPass( - createModuleToFunctionPassAdaptor(AddressSanitizerPass( - /*CompileKernel=*/false, Recover, UseAfterScope))); - }); - } + // FIXME: the backends do not handle matrix intrinsics currently. Make + // sure they are also lowered in O0. A lightweight version of the pass + // should run in the backend pipeline on demand. + if (LangOpts.MatrixTypes) + MPM.addPass( + createModuleToFunctionPassAdaptor(LowerMatrixIntrinsicsPass())); - if (LangOpts.Sanitize.has(SanitizerKind::HWAddress)) { - bool Recover = - CodeGenOpts.SanitizeRecover.has(SanitizerKind::HWAddress); - PB.registerOptimizerLastEPCallback( - [Recover](ModulePassManager &MPM, - PassBuilder::OptimizationLevel Level) { - MPM.addPass(HWAddressSanitizerPass( - /*CompileKernel=*/false, Recover)); - }); - } - if (LangOpts.Sanitize.has(SanitizerKind::KernelHWAddress)) { - bool Recover = - CodeGenOpts.SanitizeRecover.has(SanitizerKind::KernelHWAddress); - PB.registerOptimizerLastEPCallback( - [Recover](ModulePassManager &MPM, - PassBuilder::OptimizationLevel Level) { - MPM.addPass(HWAddressSanitizerPass( - /*CompileKernel=*/true, Recover)); - }); - } + addCoroutinePassesAtO0(MPM, LangOpts, CodeGenOpts); + } else if (IsThinLTO) { + MPM = PB.buildThinLTOPreLinkDefaultPipeline(Level); + } else if (IsLTO) { + MPM = PB.buildLTOPreLinkDefaultPipeline(Level); + } else { + MPM = PB.buildPerModuleDefaultPipeline(Level); + } - if (Optional Options = getGCOVOptions(CodeGenOpts, LangOpts)) - PB.registerPipelineStartEPCallback( - [Options](ModulePassManager &MPM, - PassBuilder::OptimizationLevel Level) { - MPM.addPass(GCOVProfilerPass(*Options)); - }); - if (Optional Options = - getInstrProfOptions(CodeGenOpts, LangOpts)) - PB.registerPipelineStartEPCallback( - [Options](ModulePassManager &MPM, - PassBuilder::OptimizationLevel Level) { - MPM.addPass(InstrProfiling(*Options, false)); - }); - - if (IsThinLTO) { - MPM = PB.buildThinLTOPreLinkDefaultPipeline(Level); - MPM.addPass(CanonicalizeAliasesPass()); - MPM.addPass(NameAnonGlobalPass()); - } else if (IsLTO) { - MPM = PB.buildLTOPreLinkDefaultPipeline(Level); - MPM.addPass(CanonicalizeAliasesPass()); - MPM.addPass(NameAnonGlobalPass()); - } else { - MPM = PB.buildPerModuleDefaultPipeline(Level); - } + // Lastly, add semantically necessary passes for LTO. + if (IsLTO || IsThinLTO) { + MPM.addPass(CanonicalizeAliasesPass()); + MPM.addPass(NameAnonGlobalPass()); } // Add UniqueInternalLinkageNames Pass which renames internal linkage @@ -1434,20 +1365,6 @@ MPM.addPass(createModuleToFunctionPassAdaptor(MemProfilerPass())); MPM.addPass(ModuleMemProfilerPass()); } - - if (CodeGenOpts.OptimizationLevel == 0) { - PB.runRegisteredEPCallbacks(MPM, Level, CodeGenOpts.DebugPassManager); - - // FIXME: the backends do not handle matrix intrinsics currently. Make - // sure they are also lowered in O0. A lightweight version of the pass - // should run in the backend pipeline on demand. - if (LangOpts.MatrixTypes) - MPM.addPass( - createModuleToFunctionPassAdaptor(LowerMatrixIntrinsicsPass())); - - addCoroutinePassesAtO0(MPM, LangOpts, CodeGenOpts); - addSanitizersAtO0(MPM, TargetTriple, LangOpts, CodeGenOpts); - } } // FIXME: We still use the legacy pass manager to do code generation. We