[coro] Async coroutines: Allow more than 3 arguments in the dispatch function

[aschwaighofer]

Summary

We need to be able to call function pointers. Inline the dispatch
function.

rdar://70097093

[rjmccall]

I don't quite understand the need to inline here.

[aschwaighofer]

Semantically, it is not needed. But without inlining here this helper dispatch function thunk will remain at O0 because inlining is not run again.

[aschwaighofer]

I am worried about introducing unnecessary thunks that the debugger has to step through.

[aschwaighofer]

Also inline the context project function used by the resume function.

[rjmccall]

What is this function being inlined? We expect the frontend to emit a thunk that glues things together?

[aschwaighofer]

Yes, frontend is expected to emit a thunk that models the tail call to the function at the suspend point. This prevents code moving in between the tail call and return before coroutine splitting and allows for specifying how to perform the call (e.g pointer authentication).

define internal hidden void @__suspend_dispatch_3.1(i8* %0, %task* %1, %executor* %2, %context* %3)  {
entry:
  %4 = bitcast i8* %0 to void (%task*, %executor*, %context*)*
  tail call swiftcc void %4(%task* %1, %executor* %2, %context* %3)
  ret void
}

define linkonce_odr hidden i8* @__resume_project_context(i8* %0) {
entry:
  %1 = bitcast i8* %0 to i8**
  %2 = load i8*, i8** %1, align 8
  ret i8* %2
}

%resume_func = call i8* @llvm.coro.async.resume()
%24 = call { i8*, i8*, i8* } (i8*, i8*, ...) @llvm.coro.suspend.async(
  i8* %resume_func, 
  i8* bitcast (i8* (i8*)* @__resume_project_context to i8*), 
  i8* bitcast (void (i8*, %task*, %executor*, %context*)* @__suspend_dispatch_3.1 to i8*), 
  i8* %fun_ptr, %task* %0, %executor* %1, %context* %22)

[aschwaighofer]

Transfer the debug location from the suspend point to the functions called at and after the suspend point (context projection).

[rjmccall]

I see. This LGTM as an immediate fix, then.

I wonder if we shouldn't have a different IR design, though. Maybe we should lower directly to IR that uses a pattern more like the SIL get_unsafe_continuation pattern? i.e.

%0 = call token @llvm.coro.async.continuation()  // returns the continuation function pointer; must be used by exactly one suspend
... // somewhere in here we store the continuation into the child context
call void @llvm.coro.async.suspend(token %0)

I can't remember if we discussed and discarded this.

[rjmccall]

Oh, unfortunately llvm.coro.async.continuation wouldn't be able to just return the continuation function pointer; there would have to be a llvm.coro.async.get_continuation_function intrinsic. But we do something similar with the alloc intrinsics.

[aschwaighofer]

Use the function argument index instead of the function argument in
coro.id.async. This solves any spurious use issues.

Coerce the arguments of the tail call function at a suspend point. The LLVM
optimizer seems to drop casts leading to a vararg intrinsic.

[lxfind]

llvm/lib/Transforms/Coroutines/CoroSplit.cpp

686

InlineRes will be unused in a release build. Add (void)?

llvm/include/llvm/IR/Intrinsics.td

def int_coro_id_retcon : Intrinsic<[llvm_token_ty],
      llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty],
     []>;
 def int_coro_id_retcon_once : Intrinsic<[llvm_token_ty],
     [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty,
      llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty],
     []>;
 def int_coro_alloc : Intrinsic<[llvm_i1_ty], [llvm_token_ty], []>;
 def int_coro_id_async : Intrinsic<[llvm_token_ty],
-  [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty, llvm_ptr_ty],
+  [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty],
   []>;
 def int_coro_async_context_alloc : Intrinsic<[llvm_ptr_ty],
     [llvm_ptr_ty, llvm_ptr_ty],
     []>;
 def int_coro_async_context_dealloc : Intrinsic<[],
     [llvm_ptr_ty],
     []>;
 def int_coro_async_resume : Intrinsic<[llvm_ptr_ty],

llvm/lib/Transforms/Coroutines/CoroFrame.cpp

static void sinkSpillUsesAfterCoroBegin(Function &F,
   SmallSetVector<Instruction *, 32> ToMove;
   SmallVector<Instruction *, 32> Worklist;
 
   // Collect all users that precede coro.begin.
   for (auto *Def : FrameData.getAllDefs()) {
     for (User *U : Def->users()) {
       auto Inst = cast<Instruction>(U);
       if (Inst->getParent() != CoroBegin->getParent() ||
-          Dom.dominates(CoroBegin, Inst) ||
-          isa<CoroIdAsyncInst>(Inst) /*'fake' use of async context argument*/)
+          Dom.dominates(CoroBegin, Inst))
         continue;
       if (ToMove.insert(Inst))
         Worklist.push_back(Inst);
     }
   }
   // Recursively collect users before coro.begin.
   while (!Worklist.empty()) {
     auto *Def = Worklist.back();

llvm/lib/Transforms/Coroutines/CoroInstr.h

public:
   }
 
   /// The alignment of the initial async function context.
   Align getStorageAlignment() const {
     return cast<ConstantInt>(getArgOperand(AlignArg))->getAlignValue();
   }
 
   /// The async context parameter.
-  Value *getStorage() const { return getArgOperand(StorageArg); }
+  Value *getStorage() const {
+    return getParent()->getParent()->getArg(getStorageArgumentIndex());
+  }
 
   unsigned getStorageArgumentIndex() const {
-    auto *Arg = cast<Argument>(getArgOperand(StorageArg)->stripPointerCasts());
-    return Arg->getArgNo();
+    auto *Arg = cast<ConstantInt>(getArgOperand(StorageArg));
+    return Arg->getZExtValue();
   }
 
   /// Return the async function pointer address. This should be the address of
   /// a async function pointer struct for the current async function.
   /// struct async_function_pointer {
   ///   uint32_t context_size;
   ///   uint32_t relative_async_function_pointer;
   ///  };

llvm/lib/Transforms/Coroutines/CoroSplit.cpp

Value *CoroCloner::deriveNewFramePointer() {
   // the resume function from the async context projection function associated
   // with the active suspend. The frame is located as a tail to the async
   // context header.
   case coro::ABI::Async: {
     auto *CalleeContext = NewF->getArg(Shape.AsyncLowering.ContextArgNo);
     auto *FramePtrTy = Shape.FrameTy->getPointerTo();
     auto *ProjectionFunc = cast<CoroSuspendAsyncInst>(ActiveSuspend)
                                ->getAsyncContextProjectionFunction();
+    auto DbgLoc =
+        cast<CoroSuspendAsyncInst>(VMap[ActiveSuspend])->getDebugLoc();
     // Calling i8* (i8*)
     auto *CallerContext = Builder.CreateCall(
         cast<FunctionType>(ProjectionFunc->getType()->getPointerElementType()),
         ProjectionFunc, CalleeContext);
     CallerContext->setCallingConv(ProjectionFunc->getCallingConv());
+    CallerContext->setDebugLoc(DbgLoc);
     // The frame is located after the async_context header.
     auto &Context = Builder.getContext();
     auto *FramePtrAddr = Builder.CreateConstInBoundsGEP1_32(
         Type::getInt8Ty(Context), CallerContext,
         Shape.AsyncLowering.FrameOffset, "async.ctx.frameptr");
+    // Inline the projection function.
+    InlineFunctionInfo InlineInfo;
+    auto InlineRes = InlineFunction(*CallerContext, InlineInfo);
+    assert(InlineRes.isSuccess());
     return Builder.CreateBitCast(FramePtrAddr, FramePtrTy);
   }
   // In continuation-lowering, the argument is the opaque storage.
   case coro::ABI::Retcon:
   case coro::ABI::RetconOnce: {
     Argument *NewStorage = &*NewF->arg_begin();
     auto FramePtrTy = Shape.FrameTy->getPointerTo();
 
static void replaceAsyncResumeFunction(CoroSuspendAsyncInst *Suspend,
 
   IRBuilder<> Builder(ResumeIntrinsic);
   auto *Val = Builder.CreateBitOrPointerCast(Continuation, Int8PtrTy);
   ResumeIntrinsic->replaceAllUsesWith(Val);
   ResumeIntrinsic->eraseFromParent();
   Suspend->setOperand(0, UndefValue::get(Int8PtrTy));
 }
 
+/// Coerce the arguments in \p FnArgs according to \p FnTy in \p CallArgs.
+static void coerceArguments(IRBuilder<> &Builder, FunctionType *FnTy,
+                            ArrayRef<Value *> FnArgs,
+                            SmallVectorImpl<Value *> &CallArgs) {
+  size_t ArgIdx = 0;
+  for (auto paramTy : FnTy->params()) {
+    assert(ArgIdx < FnArgs.size());
+    if (paramTy != FnArgs[ArgIdx]->getType())
+      CallArgs.push_back(
+          Builder.CreateBitOrPointerCast(FnArgs[ArgIdx], paramTy));
+    else
+      CallArgs.push_back(FnArgs[ArgIdx]);
+    ++ArgIdx;
+  }
+}
+
 static void splitAsyncCoroutine(Function &F, coro::Shape &Shape,
                                 SmallVectorImpl<Function *> &Clones) {
   assert(Shape.ABI == coro::ABI::Async);
   assert(Clones.empty());
   // Reset various things that the optimizer might have decided it
   // "knows" about the coroutine function due to not seeing a return.
   F.removeFnAttr(Attribute::NoReturn);
   F.removeAttribute(AttributeList::ReturnIndex, Attribute::NoAlias);
for (size_t Idx = 0, End = Shape.CoroSuspends.size(); Idx != End; ++Idx) {
 
     // Place it before the first suspend.
     auto *ReturnBB =
         BasicBlock::Create(F.getContext(), "coro.return", &F, NewSuspendBB);
     Branch->setSuccessor(0, ReturnBB);
 
     IRBuilder<> Builder(ReturnBB);
 
-    // Insert the call to the tail call function.
-    auto *Fun = Suspend->getMustTailCallFunction();
+    // Insert the call to the tail call function and inline it.
+    auto *Fn = Suspend->getMustTailCallFunction();
+    auto DbgLoc = Suspend->getDebugLoc();
     SmallVector<Value *, 8> Args(Suspend->operand_values());
-    auto *TailCall = Builder.CreateCall(
-        cast<FunctionType>(Fun->getType()->getPointerElementType()), Fun,
-        ArrayRef<Value *>(Args).drop_front(3).drop_back(1));
-    TailCall->setTailCallKind(CallInst::TCK_MustTail);
-    TailCall->setCallingConv(Fun->getCallingConv());
+    auto FnArgs = ArrayRef<Value *>(Args).drop_front(3).drop_back(1);
+    auto FnTy = cast<FunctionType>(Fn->getType()->getPointerElementType());
+    // Coerce the arguments, llvm optimizations seem to ignore the types in
+    // vaarg functions and throws away casts in optimized mode.
+    SmallVector<Value *, 8> CallArgs;
+    coerceArguments(Builder, FnTy, FnArgs, CallArgs);
+    auto *TailCall = Builder.CreateCall(FnTy, Fn, CallArgs);
+    TailCall->setDebugLoc(DbgLoc);
+    TailCall->setTailCall();
+    TailCall->setCallingConv(Fn->getCallingConv());
+    InlineFunctionInfo FnInfo;
+    auto InlineRes = InlineFunction(*TailCall, FnInfo);
+    assert(InlineRes.isSuccess() && "Expected inlining to succeed");
     Builder.CreateRetVoid();
 
     // Replace the lvm.coro.async.resume intrisic call.
     replaceAsyncResumeFunction(Suspend, Continuation);
   }
 
   assert(Clones.size() == Shape.CoroSuspends.size());
   for (size_t Idx = 0, End = Shape.CoroSuspends.size(); Idx != End; ++Idx) {

llvm/lib/Transforms/Coroutines/Coroutines.cpp

if (StructTy->isOpaque() || !StructTy->isPacked() ||
       !StructTy->getElementType(1)->isIntegerTy(32))
     fail(I,
          "llvm.coro.id.async async function pointer argument's type is not "
          "<{i32, i32}>",
          V);
 }
 
 void CoroIdAsyncInst::checkWellFormed() const {
-  // TODO: check that the StorageArg is a parameter of this function.
   checkConstantInt(this, getArgOperand(SizeArg),
                    "size argument to coro.id.async must be constant");
   checkConstantInt(this, getArgOperand(AlignArg),
                    "alignment argument to coro.id.async must be constant");
+  checkConstantInt(this, getArgOperand(StorageArg),
+                   "storage argument offset to coro.id.async must be constant");
   checkAsyncFuncPointer(this, getArgOperand(AsyncFuncPtrArg));
 }
 
 static void checkAsyncContextProjectFunction(const Instruction *I,
                                              Function *F) {
   auto *FunTy = cast<FunctionType>(F->getType()->getPointerElementType());
   if (!FunTy->getReturnType()->isPointerTy() ||
       !FunTy->getReturnType()->getPointerElementType()->isIntegerTy(8))

llvm/test/Transforms/Coroutines/coro-async.ll

<{ i32 trunc ( ; Relative pointer to async function
          i64 ptrtoint (void (i8*, %async.task*, %async.actor*)* @my_async_function to i64),
          i64 ptrtoint (i32* getelementptr inbounds (<{ i32, i32 }>, <{ i32, i32 }>* @my_async_function_fp, i32 0, i32 1) to i64)
        )
      to i32),
      i32 128    ; Initial async context size without space for frame
 }>
 
 ; Function that implements the dispatch to the callee function.
-define swiftcc void @my_async_function.my_other_async_function_fp.apply(i8* %async.ctxt, %async.task* %task, %async.actor* %actor) {
-  musttail call swiftcc void @asyncSuspend(i8* %async.ctxt, %async.task* %task, %async.actor* %actor)
+define swiftcc void @my_async_function.my_other_async_function_fp.apply(i8* %fnPtr, i8* %async.ctxt, %async.task* %task, %async.actor* %actor) {
+  %callee = bitcast i8* %fnPtr to void(i8*, %async.task*, %async.actor*)*
+  tail call swiftcc void %callee(i8* %async.ctxt, %async.task* %task, %async.actor* %actor)
   ret void
 }
 
 declare void @some_user(i64)
 declare void @some_may_write(i64*)
 
 define i8* @resume_context_projection(i8* %ctxt) {
 entry:
   %resume_ctxt_addr = bitcast i8* %ctxt to i8**
   %resume_ctxt = load i8*, i8** %resume_ctxt_addr, align 8
   ret i8* %resume_ctxt
 }
 
 
 define swiftcc void @my_async_function(i8* %async.ctxt, %async.task* %task, %async.actor* %actor)  {
 entry:
   %tmp = alloca { i64, i64 }, align 8
   %proj.1 = getelementptr inbounds { i64, i64 }, { i64, i64 }* %tmp, i64 0, i32 0
   %proj.2 = getelementptr inbounds { i64, i64 }, { i64, i64 }* %tmp, i64 0, i32 1
 
-  %id = call token @llvm.coro.id.async(i32 128, i32 16,
-          i8* %async.ctxt,
+  %id = call token @llvm.coro.id.async(i32 128, i32 16, i32 0,
           i8* bitcast (<{i32, i32}>* @my_async_function_fp to i8*))
   %hdl = call i8* @llvm.coro.begin(token %id, i8* null)
   store i64 0, i64* %proj.1, align 8
   store i64 1, i64* %proj.2, align 8
   call void @some_may_write(i64* %proj.1)
 
   ; Begin lowering: apply %my_other_async_function(%args...)
 
entry:
   %return_to_caller.addr = bitcast void(i8*, %async.task*, %async.actor*)** %callee_context.return_to_caller.addr to i8**
   %resume.func_ptr = call i8* @llvm.coro.async.resume()
   store i8* %resume.func_ptr, i8** %return_to_caller.addr
 
   ; store caller context into callee context
   %callee_context.caller_context.addr = getelementptr inbounds %async.ctxt, %async.ctxt* %callee_context.0, i32 0, i32 0
   store i8* %async.ctxt, i8** %callee_context.caller_context.addr
   %resume_proj_fun = bitcast i8*(i8*)* @resume_context_projection to i8*
+  %callee = bitcast void(i8*, %async.task*, %async.actor*)* @asyncSuspend to i8*
   %res = call {i8*, i8*, i8*} (i8*, i8*, ...) @llvm.coro.suspend.async(
                                                   i8* %resume.func_ptr,
                                                   i8* %resume_proj_fun,
-                                                  void (i8*, %async.task*, %async.actor*)* @my_async_function.my_other_async_function_fp.apply,
-                                                  i8* %callee_context, %async.task* %task, %async.actor *%actor)
+                                                  void (i8*, i8*, %async.task*, %async.actor*)* @my_async_function.my_other_async_function_fp.apply,
+                                                  i8* %callee, i8* %callee_context, %async.task* %task, %async.actor *%actor)
 
   call void @llvm.coro.async.context.dealloc(i8* %callee_context)
   %continuation_task_arg = extractvalue {i8*, i8*, i8*} %res, 1
   %task.2 =  bitcast i8* %continuation_task_arg to %async.task*
   %val = load i64, i64* %proj.1
   call void @some_user(i64 %val)
   %val.2 = load i64, i64* %proj.2
   call void @some_user(i64 %val.2)

 ; CHECK:   [[CALLEE_CTXT_SPILL:%.*]] = getelementptr inbounds i8, i8* %async.ctxt, i64 160
 ; CHECK:   [[CAST2:%.*]] = bitcast i8* [[CALLEE_CTXT_SPILL]] to i8**
 ; CHECK:   store i8* [[CALLEE_CTXT]], i8** [[CAST2]]
 ; CHECK:   [[TYPED_RETURN_TO_CALLER_ADDR:%.*]] = getelementptr inbounds i8, i8* [[CALLEE_CTXT]], i64 8
 ; CHECK:   [[RETURN_TO_CALLER_ADDR:%.*]] = bitcast i8* [[TYPED_RETURN_TO_CALLER_ADDR]] to i8**
 ; CHECK:   store i8* bitcast (void (i8*, i8*, i8*)* @my_async_function.resume.0 to i8*), i8** [[RETURN_TO_CALLER_ADDR]]
 ; CHECK:   [[CALLER_CONTEXT_ADDR:%.*]] = bitcast i8* [[CALLEE_CTXT]] to i8**
 ; CHECK:   store i8* %async.ctxt, i8** [[CALLER_CONTEXT_ADDR]]
-; CHECK:   musttail call swiftcc void @asyncSuspend(i8* [[CALLEE_CTXT]], %async.task* %task, %async.actor* %actor)
+; CHECK:   tail call swiftcc void @asyncSuspend(i8* [[CALLEE_CTXT]], %async.task* %task, %async.actor* %actor)
 ; CHECK:   ret void
 ; CHECK: }
 
 ; CHECK-LABEL: define internal swiftcc void @my_async_function.resume.0(i8* nocapture readonly %0, i8* %1, i8* nocapture readnone %2) {
 ; CHECK: entryresume.0:
 ; CHECK:   [[CALLER_CONTEXT_ADDR:%.*]] = bitcast i8* %0 to i8**
 ; CHECK:   [[CALLER_CONTEXT:%.*]] = load i8*, i8** [[CALLER_CONTEXT_ADDR]]
 ; CHECK:   [[FRAME_PTR:%.*]] = getelementptr inbounds i8, i8* [[CALLER_CONTEXT]], i64 128
<{ i32 trunc ( ; Relative pointer to async function
        )
      to i32),
      i32 128    ; Initial async context size without space for frame
   }>
 
 define swiftcc void @my_async_function2(%async.task* %task, %async.actor* %actor, i8* %async.ctxt)  {
 entry:
 
-  %id = call token @llvm.coro.id.async(i32 128, i32 16, i8* %async.ctxt, i8* bitcast (<{i32, i32}>* @my_async_function2_fp to i8*))
+  %id = call token @llvm.coro.id.async(i32 128, i32 16, i32 2, i8* bitcast (<{i32, i32}>* @my_async_function2_fp to i8*))
   %hdl = call i8* @llvm.coro.begin(token %id, i8* null)
   ; setup callee context
   %arg0 = bitcast %async.task* %task to i8*
   %arg1 = bitcast <{ i32, i32}>* @my_other_async_function_fp to i8*
   %callee_context = call i8* @llvm.coro.async.context.alloc(i8* %arg0, i8* %arg1)
 
   %callee_context.0 = bitcast i8* %callee_context to %async.ctxt*
   %callee_context.return_to_caller.addr = getelementptr inbounds %async.ctxt, %async.ctxt* %callee_context.0, i32 0, i32 1
   %return_to_caller.addr = bitcast void(i8*, %async.task*, %async.actor*)** %callee_context.return_to_caller.addr to i8**
   %resume.func_ptr = call i8* @llvm.coro.async.resume()
   store i8* %resume.func_ptr, i8** %return_to_caller.addr
   %callee_context.caller_context.addr = getelementptr inbounds %async.ctxt, %async.ctxt* %callee_context.0, i32 0, i32 0
   store i8* %async.ctxt, i8** %callee_context.caller_context.addr
   %resume_proj_fun = bitcast i8*(i8*)* @resume_context_projection to i8*
+  %callee = bitcast void(i8*, %async.task*, %async.actor*)* @asyncSuspend to i8*
   %res = call {i8*, i8*, i8*} (i8*, i8*, ...) @llvm.coro.suspend.async(
                                                   i8* %resume.func_ptr,
                                                   i8* %resume_proj_fun,
-                                                  void (i8*, %async.task*, %async.actor*)* @my_async_function.my_other_async_function_fp.apply,
-                                                  i8* %callee_context, %async.task* %task, %async.actor *%actor)
+                                                  void (i8*, i8*, %async.task*, %async.actor*)* @my_async_function.my_other_async_function_fp.apply,
+                                                  i8* %callee, i8* %callee_context, %async.task* %task, %async.actor *%actor)
 
   %continuation_task_arg = extractvalue {i8*, i8*, i8*} %res, 0
   %task.2 =  bitcast i8* %continuation_task_arg to %async.task*
 
   %callee_context.0.1 = bitcast i8* %callee_context to %async.ctxt*
   %callee_context.return_to_caller.addr.1 = getelementptr inbounds %async.ctxt, %async.ctxt* %callee_context.0.1, i32 0, i32 1
   %return_to_caller.addr.1 = bitcast void(i8*, %async.task*, %async.actor*)** %callee_context.return_to_caller.addr.1 to i8**
   %resume.func_ptr.1 = call i8* @llvm.coro.async.resume()
   store i8* %resume.func_ptr.1, i8** %return_to_caller.addr.1
   %callee_context.caller_context.addr.1 = getelementptr inbounds %async.ctxt, %async.ctxt* %callee_context.0.1, i32 0, i32 0
   store i8* %async.ctxt, i8** %callee_context.caller_context.addr.1
   %resume_proj_fun.2 = bitcast i8*(i8*)* @resume_context_projection to i8*
+  %callee.2 = bitcast void(i8*, %async.task*, %async.actor*)* @asyncSuspend to i8*
   %res.2 = call {i8*, i8*, i8*} (i8*, i8*, ...) @llvm.coro.suspend.async(
                                                   i8* %resume.func_ptr.1,
                                                   i8* %resume_proj_fun.2,
-                                                  void (i8*, %async.task*, %async.actor*)* @my_async_function.my_other_async_function_fp.apply,
-                                                  i8* %callee_context, %async.task* %task, %async.actor *%actor)
+                                                  void (i8*, i8*, %async.task*, %async.actor*)* @my_async_function.my_other_async_function_fp.apply,
+                                                  i8* %callee.2, i8* %callee_context, %async.task* %task, %async.actor *%actor)
 
   call void @llvm.coro.async.context.dealloc(i8* %callee_context)
   %continuation_actor_arg = extractvalue {i8*, i8*, i8*} %res.2, 1
   %actor.2 =  bitcast i8* %continuation_actor_arg to %async.actor*
 
   tail call swiftcc void @asyncReturn(i8* %async.ctxt, %async.task* %task.2, %async.actor* %actor.2)
   call i1 @llvm.coro.end(i8* %hdl, i1 0)
   unreachable
 }
 
 ; CHECK-LABEL: define swiftcc void @my_async_function2(%async.task* %task, %async.actor* %actor, i8* %async.ctxt) {
 ; CHECK: store i8* %async.ctxt,
 ; CHECK: store %async.actor* %actor,
 ; CHECK: store %async.task* %task,
 ; CHECK: [[CALLEE_CTXT:%.*]] =  tail call i8* @llvm.coro.async.context.alloc(
 ; CHECK: store i8* [[CALLEE_CTXT]],
 ; CHECK: store i8* bitcast (void (i8*, i8*, i8*)* @my_async_function2.resume.0 to i8*),
 ; CHECK: store i8* %async.ctxt,
-; CHECK: musttail call swiftcc void @asyncSuspend(i8* [[CALLEE_CTXT]], %async.task* %task, %async.actor* %actor)
+; CHECK: tail call swiftcc void @asyncSuspend(i8* [[CALLEE_CTXT]], %async.task* %task, %async.actor* %actor)
 ; CHECK: ret void
 
 ; CHECK-LABEL: define internal swiftcc void @my_async_function2.resume.0(i8* %0, i8* nocapture readnone %1, i8* nocapture readonly %2) {
 ; CHECK: [[CALLEE_CTXT_ADDR:%.*]] = bitcast i8* %2 to i8**
 ; CHECK: [[CALLEE_CTXT:%.*]] = load i8*, i8** [[CALLEE_CTXT_ADDR]]
 ; CHECK: [[CALLEE_CTXT_SPILL_ADDR:%.*]] = getelementptr inbounds i8, i8* [[CALLEE_CTXT]], i64 152
 ; CHECK: [[CALLEE_CTXT_SPILL_ADDR2:%.*]] = bitcast i8* [[CALLEE_CTXT_SPILL_ADDR]] to i8**
 ; CHECK: store i8* bitcast (void (i8*, i8*, i8*)* @my_async_function2.resume.1 to i8*),
 ; CHECK: [[CALLLE_CTXT_RELOAD:%.*]] = load i8*, i8** [[CALLEE_CTXT_SPILL_ADDR2]]
-; CHECK: musttail call swiftcc void @asyncSuspend(i8* [[CALLEE_CTXT_RELOAD]]
+; CHECK: tail call swiftcc void @asyncSuspend(i8* [[CALLEE_CTXT_RELOAD]]
 ; CHECK: ret void
 
 ; CHECK-LABEL: define internal swiftcc void @my_async_function2.resume.1(i8* nocapture readnone %0, i8* %1, i8* nocapture readonly %2) {
 ; CHECK: bitcast i8* %2 to i8**
 ; CHECK: [[ACTOR_ARG:%.*]] = bitcast i8* %1
 ; CHECK: tail call swiftcc void @asyncReturn({{.*}}[[ACTOR_ARG]])
 ; CHECK: ret void
 
 define swiftcc void @top_level_caller(i8* %ctxt, i8* %task, i8* %actor) {
   %prepare = call i8* @llvm.coro.prepare.async(i8* bitcast (void (i8*, %async.task*,  %async.actor*)* @my_async_function to i8*))
   %f = bitcast i8* %prepare to void (i8*, i8*, i8*)*
   call swiftcc void %f(i8* %ctxt, i8* %task, i8* %actor)
   ret void
 }
 
 ; CHECK-LABEL: define swiftcc void @top_level_caller(i8* %ctxt, i8* %task, i8* %actor)
 ; CHECK: store i8* bitcast (void (i8*, i8*, i8*)* @my_async_function.resume.0
 ; CHECK: store i8* %ctxt
 ; CHECK: tail call swiftcc void @asyncSuspend
 ; CHECK: ret void
 
 declare i8* @llvm.coro.prepare.async(i8*)
-declare token @llvm.coro.id.async(i32, i32, i8*, i8*)
+declare token @llvm.coro.id.async(i32, i32, i32, i8*)
 declare i8* @llvm.coro.begin(token, i8*)
 declare i1 @llvm.coro.end(i8*, i1)
 declare {i8*, i8*, i8*} @llvm.coro.suspend.async(i8*, i8*, ...)
 declare i8* @llvm.coro.async.context.alloc(i8*, i8*)
 declare void @llvm.coro.async.context.dealloc(i8*)
 declare swiftcc void @asyncReturn(i8*, %async.task*, %async.actor*)
 declare swiftcc void @asyncSuspend(i8*, %async.task*, %async.actor*)
 declare i8* @llvm.coro.async.resume()