diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp --- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp +++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp @@ -668,16 +668,23 @@ auto *FramePtrTy = Shape.FrameTy->getPointerTo(); auto *ProjectionFunc = cast(ActiveSuspend) ->getAsyncContextProjectionFunction(); + auto DbgLoc = + cast(VMap[ActiveSuspend])->getDebugLoc(); // Calling i8* (i8*) auto *CallerContext = Builder.CreateCall( cast(ProjectionFunc->getType()->getPointerElementType()), ProjectionFunc, CalleeContext); CallerContext->setCallingConv(ProjectionFunc->getCallingConv()); + CallerContext->setDebugLoc(DbgLoc); // The frame is located after the async_context header. auto &Context = Builder.getContext(); auto *FramePtrAddr = Builder.CreateConstInBoundsGEP1_32( Type::getInt8Ty(Context), CallerContext, Shape.AsyncLowering.FrameOffset, "async.ctx.frameptr"); + // Inline the projection function. + InlineFunctionInfo InlineInfo; + auto InlineRes = InlineFunction(*CallerContext, InlineInfo); + assert(InlineRes.isSuccess()); return Builder.CreateBitCast(FramePtrAddr, FramePtrTy); } // In continuation-lowering, the argument is the opaque storage. @@ -1420,14 +1427,19 @@ IRBuilder<> Builder(ReturnBB); - // Insert the call to the tail call function. + // Insert the call to the tail call function and inline it. auto *Fun = Suspend->getMustTailCallFunction(); + auto DbgLoc = Suspend->getDebugLoc(); SmallVector Args(Suspend->operand_values()); auto *TailCall = Builder.CreateCall( cast(Fun->getType()->getPointerElementType()), Fun, ArrayRef(Args).drop_front(3).drop_back(1)); - TailCall->setTailCallKind(CallInst::TCK_MustTail); + TailCall->setDebugLoc(DbgLoc); + TailCall->setTailCall(); TailCall->setCallingConv(Fun->getCallingConv()); + InlineFunctionInfo FnInfo; + auto InlineRes = InlineFunction(*TailCall, FnInfo); + assert(InlineRes.isSuccess() && "Expected inlining to succeed"); Builder.CreateRetVoid(); // Replace the lvm.coro.async.resume intrisic call. diff --git a/llvm/test/Transforms/Coroutines/coro-async.ll b/llvm/test/Transforms/Coroutines/coro-async.ll --- a/llvm/test/Transforms/Coroutines/coro-async.ll +++ b/llvm/test/Transforms/Coroutines/coro-async.ll @@ -28,8 +28,9 @@ }> ; Function that implements the dispatch to the callee function. -define swiftcc void @my_async_function.my_other_async_function_fp.apply(i8* %async.ctxt, %async.task* %task, %async.actor* %actor) { - musttail call swiftcc void @asyncSuspend(i8* %async.ctxt, %async.task* %task, %async.actor* %actor) +define swiftcc void @my_async_function.my_other_async_function_fp.apply(i8* %fnPtr, i8* %async.ctxt, %async.task* %task, %async.actor* %actor) { + %callee = bitcast i8* %fnPtr to void(i8*, %async.task*, %async.actor*)* + tail call swiftcc void %callee(i8* %async.ctxt, %async.task* %task, %async.actor* %actor) ret void } @@ -78,11 +79,12 @@ %callee_context.caller_context.addr = getelementptr inbounds %async.ctxt, %async.ctxt* %callee_context.0, i32 0, i32 0 store i8* %async.ctxt, i8** %callee_context.caller_context.addr %resume_proj_fun = bitcast i8*(i8*)* @resume_context_projection to i8* + %callee = bitcast void(i8*, %async.task*, %async.actor*)* @asyncSuspend to i8* %res = call {i8*, i8*, i8*} (i8*, i8*, ...) @llvm.coro.suspend.async( i8* %resume.func_ptr, i8* %resume_proj_fun, - void (i8*, %async.task*, %async.actor*)* @my_async_function.my_other_async_function_fp.apply, - i8* %callee_context, %async.task* %task, %async.actor *%actor) + void (i8*, i8*, %async.task*, %async.actor*)* @my_async_function.my_other_async_function_fp.apply, + i8* %callee, i8* %callee_context, %async.task* %task, %async.actor *%actor) call void @llvm.coro.async.context.dealloc(i8* %callee_context) %continuation_task_arg = extractvalue {i8*, i8*, i8*} %res, 1 @@ -126,7 +128,7 @@ ; CHECK: store i8* bitcast (void (i8*, i8*, i8*)* @my_async_function.resume.0 to i8*), i8** [[RETURN_TO_CALLER_ADDR]] ; CHECK: [[CALLER_CONTEXT_ADDR:%.*]] = bitcast i8* [[CALLEE_CTXT]] to i8** ; CHECK: store i8* %async.ctxt, i8** [[CALLER_CONTEXT_ADDR]] -; CHECK: musttail call swiftcc void @asyncSuspend(i8* [[CALLEE_CTXT]], %async.task* %task, %async.actor* %actor) +; CHECK: tail call swiftcc void @asyncSuspend(i8* [[CALLEE_CTXT]], %async.task* %task, %async.actor* %actor) ; CHECK: ret void ; CHECK: } @@ -185,11 +187,12 @@ %callee_context.caller_context.addr = getelementptr inbounds %async.ctxt, %async.ctxt* %callee_context.0, i32 0, i32 0 store i8* %async.ctxt, i8** %callee_context.caller_context.addr %resume_proj_fun = bitcast i8*(i8*)* @resume_context_projection to i8* + %callee = bitcast void(i8*, %async.task*, %async.actor*)* @asyncSuspend to i8* %res = call {i8*, i8*, i8*} (i8*, i8*, ...) @llvm.coro.suspend.async( i8* %resume.func_ptr, i8* %resume_proj_fun, - void (i8*, %async.task*, %async.actor*)* @my_async_function.my_other_async_function_fp.apply, - i8* %callee_context, %async.task* %task, %async.actor *%actor) + void (i8*, i8*, %async.task*, %async.actor*)* @my_async_function.my_other_async_function_fp.apply, + i8* %callee, i8* %callee_context, %async.task* %task, %async.actor *%actor) %continuation_task_arg = extractvalue {i8*, i8*, i8*} %res, 0 %task.2 = bitcast i8* %continuation_task_arg to %async.task* @@ -202,11 +205,12 @@ %callee_context.caller_context.addr.1 = getelementptr inbounds %async.ctxt, %async.ctxt* %callee_context.0.1, i32 0, i32 0 store i8* %async.ctxt, i8** %callee_context.caller_context.addr.1 %resume_proj_fun.2 = bitcast i8*(i8*)* @resume_context_projection to i8* + %callee.2 = bitcast void(i8*, %async.task*, %async.actor*)* @asyncSuspend to i8* %res.2 = call {i8*, i8*, i8*} (i8*, i8*, ...) @llvm.coro.suspend.async( i8* %resume.func_ptr.1, i8* %resume_proj_fun.2, - void (i8*, %async.task*, %async.actor*)* @my_async_function.my_other_async_function_fp.apply, - i8* %callee_context, %async.task* %task, %async.actor *%actor) + void (i8*, i8*, %async.task*, %async.actor*)* @my_async_function.my_other_async_function_fp.apply, + i8* %callee.2, i8* %callee_context, %async.task* %task, %async.actor *%actor) call void @llvm.coro.async.context.dealloc(i8* %callee_context) %continuation_actor_arg = extractvalue {i8*, i8*, i8*} %res.2, 1 @@ -225,7 +229,7 @@ ; CHECK: store i8* [[CALLEE_CTXT]], ; CHECK: store i8* bitcast (void (i8*, i8*, i8*)* @my_async_function2.resume.0 to i8*), ; CHECK: store i8* %async.ctxt, -; CHECK: musttail call swiftcc void @asyncSuspend(i8* [[CALLEE_CTXT]], %async.task* %task, %async.actor* %actor) +; CHECK: tail call swiftcc void @asyncSuspend(i8* [[CALLEE_CTXT]], %async.task* %task, %async.actor* %actor) ; CHECK: ret void ; CHECK-LABEL: define internal swiftcc void @my_async_function2.resume.0(i8* %0, i8* nocapture readnone %1, i8* nocapture readonly %2) { @@ -235,7 +239,7 @@ ; CHECK: [[CALLEE_CTXT_SPILL_ADDR2:%.*]] = bitcast i8* [[CALLEE_CTXT_SPILL_ADDR]] to i8** ; CHECK: store i8* bitcast (void (i8*, i8*, i8*)* @my_async_function2.resume.1 to i8*), ; CHECK: [[CALLLE_CTXT_RELOAD:%.*]] = load i8*, i8** [[CALLEE_CTXT_SPILL_ADDR2]] -; CHECK: musttail call swiftcc void @asyncSuspend(i8* [[CALLEE_CTXT_RELOAD]] +; CHECK: tail call swiftcc void @asyncSuspend(i8* [[CALLEE_CTXT_RELOAD]] ; CHECK: ret void ; CHECK-LABEL: define internal swiftcc void @my_async_function2.resume.1(i8* nocapture readnone %0, i8* %1, i8* nocapture readonly %2) {