Index: clang/test/CodeGen/lto-newpm-pipeline.c =================================================================== --- clang/test/CodeGen/lto-newpm-pipeline.c +++ clang/test/CodeGen/lto-newpm-pipeline.c @@ -28,6 +28,8 @@ // CHECK-FULL-O0: Running pass: AlwaysInlinerPass // CHECK-FULL-O0-NEXT: Running analysis: InnerAnalysisManagerProxy // CHECK-FULL-O0-NEXT: Running analysis: ProfileSummaryAnalysis +// CHECK-FULL-O0: Running pass: CoroSplitPass +// CHECK-FULL-O0-NEXT: Running pass: CoroCleanupPass // CHECK-FULL-O0-NEXT: Running pass: CanonicalizeAliasesPass // CHECK-FULL-O0-NEXT: Running pass: NameAnonGlobalPass // CHECK-FULL-O0-NEXT: Running pass: BitcodeWriterPass @@ -35,6 +37,7 @@ // CHECK-THIN-O0: Running pass: AlwaysInlinerPass // CHECK-THIN-O0-NEXT: Running analysis: InnerAnalysisManagerProxy // CHECK-THIN-O0-NEXT: Running analysis: ProfileSummaryAnalysis +// CHECK-THIN-O0: Running pass: CoroCleanupPass // CHECK-THIN-O0-NEXT: Running pass: CanonicalizeAliasesPass // CHECK-THIN-O0-NEXT: Running pass: NameAnonGlobalPass // CHECK-THIN-O0-NEXT: Running pass: ThinLTOBitcodeWriterPass Index: clang/test/CodeGenCoroutines/coro-always-inline.cpp =================================================================== --- clang/test/CodeGenCoroutines/coro-always-inline.cpp +++ clang/test/CodeGenCoroutines/coro-always-inline.cpp @@ -40,15 +40,13 @@ // CHECK-LABEL: @_Z3foov // CHECK-LABEL: entry: -// CHECK-NEXT: %this.addr.i{{[0-9]*}} = alloca %"struct.std::experimental::awaitable"*, align 8 -// CHECK-NEXT: %this.addr.i{{[0-9]*}} = alloca %"struct.std::experimental::awaitable"*, align 8 -// CHECK: [[CAST0:%[0-9]+]] = bitcast %"struct.std::experimental::awaitable"** %this.addr.i{{[0-9]*}} to i8* -// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 8, i8* [[CAST0]]) -// CHECK: [[CAST1:%[0-9]+]] = bitcast %"struct.std::experimental::awaitable"** %this.addr.i{{[0-9]*}} to i8* -// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 8, i8* [[CAST1]]) - -// CHECK: [[CAST2:%[0-9]+]] = bitcast %"struct.std::experimental::awaitable"** %this.addr.i{{[0-9]*}} to i8* -// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 8, i8* [[CAST2]]) -// CHECK: [[CAST3:%[0-9]+]] = bitcast %"struct.std::experimental::awaitable"** %this.addr.i{{[0-9]*}} to i8* -// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 8, i8* [[CAST3]]) +// CHECK: [[CAST0:%[0-9]+]] = bitcast %"struct.std::experimental::awaitable"* %ref.tmp{{.*}} to i8* +// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 1, i8* [[CAST0]]) +// CHECK: [[CAST1:%[0-9]+]] = bitcast %"struct.std::experimental::awaitable"* %ref.tmp{{.*}} to i8* +// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 1, i8* [[CAST1]]) + +// CHECK: [[CAST2:%[0-9]+]] = bitcast %"struct.std::experimental::awaitable"* %ref.tmp{{.*}} to i8* +// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 1, i8* [[CAST2]]) +// CHECK: [[CAST3:%[0-9]+]] = bitcast %"struct.std::experimental::awaitable"* %ref.tmp{{.*}} to i8* +// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 1, i8* [[CAST3]]) void foo() { co_return; } Index: llvm/lib/Passes/PassBuilder.cpp =================================================================== --- llvm/lib/Passes/PassBuilder.cpp +++ llvm/lib/Passes/PassBuilder.cpp @@ -1926,10 +1926,8 @@ // which is just that always inlining occurs. Further, disable generating // lifetime intrinsics to avoid enabling further optimizations during // code generation. - // However, we need to insert lifetime intrinsics to avoid invalid access - // caused by multithreaded coroutines. MPM.addPass(AlwaysInlinerPass( - /*InsertLifetimeIntrinsics=*/true)); + /*InsertLifetimeIntrinsics=*/false)); if (PTO.MergeFunctions) MPM.addPass(MergeFunctionsPass()); Index: llvm/lib/Transforms/Utils/InlineFunction.cpp =================================================================== --- llvm/lib/Transforms/Utils/InlineFunction.cpp +++ llvm/lib/Transforms/Utils/InlineFunction.cpp @@ -2189,7 +2189,10 @@ // Leave lifetime markers for the static alloca's, scoping them to the // function we just inlined. - if (InsertLifetime && !IFI.StaticAllocas.empty()) { + // We need to insert lifetime intrinsics to avoid invalid access + // caused by multithreaded coroutines + if ((InsertLifetime || Caller->isPresplitCoroutine()) && + !IFI.StaticAllocas.empty()) { IRBuilder<> builder(&FirstNewBlock->front()); for (unsigned ai = 0, ae = IFI.StaticAllocas.size(); ai != ae; ++ai) { AllocaInst *AI = IFI.StaticAllocas[ai]; Index: llvm/test/Other/new-pm-O0-defaults.ll =================================================================== --- llvm/test/Other/new-pm-O0-defaults.ll +++ llvm/test/Other/new-pm-O0-defaults.ll @@ -51,7 +51,7 @@ ; CHECK-LTO-NEXT: Running pass: LowerTypeTestsPass ; CHECK-LTO-NEXT: Running pass: AnnotationRemarksPass ; CHECK-LTO-NEXT: Running analysis: TargetLibraryAnalysis -; CHECK-NEXT: Running pass: PrintModulePass +; CHECK: Running pass: PrintModulePass ; Make sure we get the IR back out without changes when we print the module. ; CHECK-LABEL: define void @foo(i32 %n) local_unnamed_addr { Index: llvm/test/Other/new-pm-defaults.ll =================================================================== --- llvm/test/Other/new-pm-defaults.ll +++ llvm/test/Other/new-pm-defaults.ll @@ -89,6 +89,7 @@ ; CHECK-O-NEXT: Running analysis: DominatorTreeAnalysis ; CHECK-O-NEXT: Running pass: EarlyCSEPass ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis +; CHECK-O-NEXT: Running pass: CoroEarlyPass ; CHECK-O3-NEXT: Running pass: CallSiteSplittingPass ; CHECK-O-NEXT: Running pass: OpenMPOptPass ; CHECK-EP-PIPELINE-EARLY-SIMPLIFICATION-NEXT: Running pass: NoOpModulePass @@ -183,6 +184,7 @@ ; CHECK-O23SZ-NEXT: Running analysis: LazyValueAnalysis ; CHECK-O23SZ-NEXT: Running pass: CorrelatedValuePropagationPass ; CHECK-O23SZ-NEXT: Invalidating analysis: LazyValueAnalysis +; CHECK-O1-NEXT: Running pass: CoroElidePass ; CHECK-O-NEXT: Running pass: ADCEPass ; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis ; CHECK-O23SZ-NEXT: Running pass: MemCpyOptPass @@ -190,10 +192,12 @@ ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass ; CHECK-O23SZ-NEXT: Running pass: LICMPass +; CHECK-O23SZ-NEXT: Running pass: CoroElidePass ; CHECK-EP-SCALAR-LATE-NEXT: Running pass: NoOpFunctionPass ; CHECK-O-NEXT: Running pass: SimplifyCFGPass ; CHECK-O-NEXT: Running pass: InstCombinePass ; CHECK-EP-PEEPHOLE-NEXT: Running pass: NoOpFunctionPass +; CHECK-O-NEXT: Running pass: CoroSplitPass ; CHECK-O-NEXT: Running pass: GlobalOptPass ; CHECK-O-NEXT: Running pass: GlobalDCEPass ; CHECK-DEFAULT-NEXT: Running pass: EliminateAvailableExternallyPass @@ -236,6 +240,7 @@ ; CHECK-O-NEXT: Running pass: InstSimplifyPass ; CHECK-O-NEXT: Running pass: DivRemPairsPass ; CHECK-O-NEXT: Running pass: SimplifyCFGPass +; CHECK-O-NEXT: Running pass: CoroCleanupPass ; CHECK-EP-OPTIMIZER-LAST: Running pass: NoOpFunctionPass ; CHECK-O-NEXT: Running pass: CGProfilePass ; CHECK-O-NEXT: Running pass: GlobalDCEPass Index: llvm/test/Other/new-pm-thinlto-defaults.ll =================================================================== --- llvm/test/Other/new-pm-thinlto-defaults.ll +++ llvm/test/Other/new-pm-thinlto-defaults.ll @@ -70,6 +70,7 @@ ; CHECK-O-NEXT: Running analysis: DominatorTreeAnalysis ; CHECK-O-NEXT: Running pass: EarlyCSEPass ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis +; CHECK-O-NEXT: Running pass: CoroEarlyPass ; CHECK-O3-NEXT: Running pass: CallSiteSplittingPass ; CHECK-O-NEXT: Running pass: OpenMPOptPass ; CHECK-POSTLINK-O-NEXT: Running pass: LowerTypeTestsPass @@ -170,6 +171,7 @@ ; CHECK-O23SZ-NEXT: Running analysis: LazyValueAnalysis ; CHECK-O23SZ-NEXT: Running pass: CorrelatedValuePropagationPass ; CHECK-O23SZ-NEXT: Invalidating analysis: LazyValueAnalysis +; CHECK-O1-NEXT: Running pass: CoroElidePass ; CHECK-O-NEXT: Running pass: ADCEPass ; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis ; CHECK-O23SZ-NEXT: Running pass: MemCpyOptPass @@ -177,8 +179,10 @@ ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass ; CHECK-O23SZ-NEXT: Running pass: LICMPass on Loop at depth 1 containing: %loop +; CHECK-O23SZ-NEXT: Running pass: CoroElidePass ; CHECK-O-NEXT: Running pass: SimplifyCFGPass ; CHECK-O-NEXT: Running pass: InstCombinePass +; CHECK-O-NEXT: Running pass: CoroSplitPass ; CHECK-PRELINK-O-NEXT: Running pass: GlobalOptPass ; CHECK-POSTLINK-O-NEXT: Running pass: GlobalOptPass ; CHECK-POSTLINK-O-NEXT: Running pass: GlobalDCEPass @@ -217,6 +221,7 @@ ; CHECK-POSTLINK-O-NEXT: Running pass: InstSimplifyPass ; CHECK-POSTLINK-O-NEXT: Running pass: DivRemPairsPass ; CHECK-POSTLINK-O-NEXT: Running pass: SimplifyCFGPass +; CHECK-O-NEXT: Running pass: CoroCleanupPass ; CHECK-POSTLINK-O-NEXT: Running pass: CGProfilePass ; CHECK-POSTLINK-O-NEXT: Running pass: GlobalDCEPass ; CHECK-POSTLINK-O-NEXT: Running pass: ConstantMergePass Index: llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll =================================================================== --- llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll +++ llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll @@ -40,6 +40,7 @@ ; CHECK-O-NEXT: Running analysis: DominatorTreeAnalysis ; CHECK-O-NEXT: Running pass: EarlyCSEPass ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis +; CHECK-O-NEXT: Running pass: CoroEarlyPass ; CHECK-O3-NEXT: Running pass: CallSiteSplittingPass ; CHECK-O-NEXT: Running pass: OpenMPOptPass ; CHECK-O-NEXT: Running pass: LowerTypeTestsPass @@ -143,14 +144,17 @@ ; CHECK-O23SZ-NEXT: Running analysis: LazyValueAnalysis ; CHECK-O23SZ-NEXT: Running pass: CorrelatedValuePropagationPass ; CHECK-O23SZ-NEXT: Invalidating analysis: LazyValueAnalysis +; CHECK-O1-NEXT: Running pass: CoroElidePass ; CHECK-O-NEXT: Running pass: ADCEPass ; CHECK-O23SZ-NEXT: Running pass: MemCpyOptPass ; CHECK-O23SZ-NEXT: Running pass: DSEPass ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass ; CHECK-O23SZ-NEXT: Running pass: LICMPass +; CHECK-O23SZ-NEXT: Running pass: CoroElidePass ; CHECK-O-NEXT: Running pass: SimplifyCFGPass ; CHECK-O-NEXT: Running pass: InstCombinePass +; CHECK-O-NEXT: Running pass: CoroSplitPass ; CHECK-O-NEXT: Running pass: GlobalOptPass ; CHECK-O-NEXT: Running pass: GlobalDCEPass ; CHECK-O-NEXT: Running pass: EliminateAvailableExternallyPass @@ -186,6 +190,7 @@ ; CHECK-O-NEXT: Running pass: InstSimplifyPass ; CHECK-O-NEXT: Running pass: DivRemPairsPass ; CHECK-O-NEXT: Running pass: SimplifyCFGPass +; CHECK-O-NEXT: Running pass: CoroCleanupPass ; CHECK-O-NEXT: Running pass: CGProfilePass ; CHECK-O-NEXT: Running pass: GlobalDCEPass ; CHECK-O-NEXT: Running pass: ConstantMergePass Index: llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll =================================================================== --- llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll +++ llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll @@ -42,6 +42,7 @@ ; CHECK-O-NEXT: Running analysis: DominatorTreeAnalysis ; CHECK-O-NEXT: Running pass: EarlyCSEPass ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis +; CHECK-O-NEXT: Running pass: CoroEarlyPass ; CHECK-O3-NEXT: Running pass: CallSiteSplittingPass ; CHECK-O-NEXT: Running pass: InstCombinePass on foo ; CHECK-O-NEXT: Running analysis: OptimizationRemarkEmitterAnalysis on foo @@ -152,17 +153,20 @@ ; CHECK-O23SZ-NEXT: Running analysis: LazyValueAnalysis ; CHECK-O23SZ-NEXT: Running pass: CorrelatedValuePropagationPass ; CHECK-O23SZ-NEXT: Invalidating analysis: LazyValueAnalysis +; CHECK-O1-NEXT: Running pass: CoroElidePass ; CHECK-O-NEXT: Running pass: ADCEPass ; CHECK-O23SZ-NEXT: Running pass: MemCpyOptPass ; CHECK-O23SZ-NEXT: Running pass: DSEPass ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass ; CHECK-O23SZ-NEXT: Running pass: LICMPass +; CHECK-O23SZ-NEXT: Running pass: CoroElidePass ; CHECK-O-NEXT: Running pass: SimplifyCFGPass ; CHECK-O-NEXT: Running pass: InstCombinePass ; CHECK-O3-NEXT: Running pass: ControlHeightReductionPass on foo ; CHECK-O3-NEXT: Running analysis: RegionInfoAnalysis on foo ; CHECK-O3-NEXT: Running analysis: DominanceFrontierAnalysis on foo +; CHECK-O-NEXT: Running pass: CoroSplitPass ; CHECK-O-NEXT: Running pass: GlobalOptPass ; CHECK-O-NEXT: Running pass: GlobalDCEPass ; CHECK-O-NEXT: Running pass: EliminateAvailableExternallyPass @@ -198,6 +202,7 @@ ; CHECK-O-NEXT: Running pass: InstSimplifyPass ; CHECK-O-NEXT: Running pass: DivRemPairsPass ; CHECK-O-NEXT: Running pass: SimplifyCFGPass +; CHECK-O-NEXT: Running pass: CoroCleanupPass ; CHECK-O-NEXT: Running pass: CGProfilePass ; CHECK-O-NEXT: Running pass: GlobalDCEPass ; CHECK-O-NEXT: Running pass: ConstantMergePass Index: llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll =================================================================== --- llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll +++ llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll @@ -41,6 +41,7 @@ ; CHECK-O-NEXT: Running analysis: DominatorTreeAnalysis ; CHECK-O-NEXT: Running pass: EarlyCSEPass ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis +; CHECK-O-NEXT: Running pass: CoroEarlyPass ; CHECK-O3-NEXT: Running pass: CallSiteSplittingPass ; CHECK-O-NEXT: Running pass: OpenMPOptPass ; CHECK-O-NEXT: Running pass: IPSCCPPass @@ -183,20 +184,24 @@ ; CHECK-O23SZ-NEXT: Running analysis: LazyValueAnalysis ; CHECK-O23SZ-NEXT: Running pass: CorrelatedValuePropagationPass ; CHECK-O23SZ-NEXT: Invalidating analysis: LazyValueAnalysis +; CHECK-O1-NEXT: Running pass: CoroElidePass ; CHECK-O-NEXT: Running pass: ADCEPass ; CHECK-O23SZ-NEXT: Running pass: MemCpyOptPass ; CHECK-O23SZ-NEXT: Running pass: DSEPass ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass ; CHECK-O23SZ-NEXT: Running pass: LICMPass +; CHECK-O23SZ-NEXT: Running pass: CoroElidePass ; CHECK-O-NEXT: Running pass: SimplifyCFGPass ; CHECK-O-NEXT: Running pass: InstCombinePass ; CHECK-O3-NEXT: Running pass: ControlHeightReductionPass on foo ; CHECK-O3-NEXT: Running analysis: RegionInfoAnalysis on foo ; CHECK-O3-NEXT: Running analysis: DominanceFrontierAnalysis on foo +; CHECK-O-NEXT: Running pass: CoroSplitPass ; CHECK-O-NEXT: Running pass: GlobalOptPass ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis on bar ; CHECK-EXT: Running pass: {{.*}}::Bye +; CHECK-O-NEXT: Running pass: CoroCleanupPass ; CHECK-O-NEXT: Running pass: AnnotationRemarksPass on foo ; CHECK-O-NEXT: Running pass: CanonicalizeAliasesPass ; CHECK-O-NEXT: Running pass: NameAnonGlobalPass Index: llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll =================================================================== --- llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll +++ llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll @@ -40,6 +40,7 @@ ; CHECK-O-NEXT: Running analysis: DominatorTreeAnalysis ; CHECK-O-NEXT: Running pass: EarlyCSEPass ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis +; CHECK-O-NEXT: Running pass: CoroEarlyPass ; CHECK-O3-NEXT: Running pass: CallSiteSplittingPass ; CHECK-O-NEXT: Running pass: InstCombinePass on foo ; CHECK-O-NEXT: Running analysis: OptimizationRemarkEmitterAnalysis on foo @@ -146,18 +147,22 @@ ; CHECK-O23SZ-NEXT: Running analysis: LazyValueAnalysis ; CHECK-O23SZ-NEXT: Running pass: CorrelatedValuePropagationPass ; CHECK-O23SZ-NEXT: Invalidating analysis: LazyValueAnalysis +; CHECK-O1-NEXT: Running pass: CoroElidePass ; CHECK-O-NEXT: Running pass: ADCEPass ; CHECK-O23SZ-NEXT: Running pass: MemCpyOptPass ; CHECK-O23SZ-NEXT: Running pass: DSEPass ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass ; CHECK-O23SZ-NEXT: Running pass: LICMPass +; CHECK-O23SZ-NEXT: Running pass: CoroElidePass ; CHECK-O-NEXT: Running pass: SimplifyCFGPass ; CHECK-O-NEXT: Running pass: InstCombinePass ; CHECK-O3-NEXT: Running pass: ControlHeightReductionPass on foo ; CHECK-O3-NEXT: Running analysis: RegionInfoAnalysis on foo ; CHECK-O3-NEXT: Running analysis: DominanceFrontierAnalysis on foo +; CHECK-O-NEXT: Running pass: CoroSplitPass ; CHECK-O-NEXT: Running pass: GlobalOptPass +; CHECK-O-NEXT: Running pass: CoroCleanupPass ; CHECK-O-NEXT: Running pass: AnnotationRemarksPass on foo ; CHECK-O-NEXT: Running pass: CanonicalizeAliasesPass ; CHECK-O-NEXT: Running pass: NameAnonGlobalPass