Index: include/llvm/IR/Intrinsics.td =================================================================== --- include/llvm/IR/Intrinsics.td +++ include/llvm/IR/Intrinsics.td @@ -619,7 +619,7 @@ def int_coro_free : Intrinsic<[llvm_ptr_ty], [llvm_token_ty, llvm_ptr_ty], [IntrReadMem, IntrArgMemOnly, ReadOnly<1>, NoCapture<1>]>; -def int_coro_end : Intrinsic<[], [llvm_ptr_ty, llvm_i1_ty], []>; +def int_coro_end : Intrinsic<[llvm_i1_ty], [llvm_ptr_ty, llvm_i1_ty], []>; def int_coro_frame : Intrinsic<[llvm_ptr_ty], [], [IntrNoMem]>; def int_coro_size : Intrinsic<[llvm_anyint_ty], [], [IntrNoMem]>; Index: lib/Transforms/Coroutines/CoroSplit.cpp =================================================================== --- lib/Transforms/Coroutines/CoroSplit.cpp +++ lib/Transforms/Coroutines/CoroSplit.cpp @@ -145,6 +145,33 @@ BB->getTerminator()->eraseFromParent(); } +// In Resumers, we replace unwind coro.end with True to force the immediate +// unwind to caller. +static void replaceUnwindCoroEnds(coro::Shape &Shape, ValueToValueMapTy &VMap) { + if (Shape.CoroEnds.empty()) + return; + + LLVMContext &Context = Shape.CoroEnds.front()->getContext(); + auto *True = ConstantInt::getTrue(Context); + for (CoroEndInst *CE : Shape.CoroEnds) { + if (!CE->isUnwind()) + continue; + + auto *NewCE = cast(VMap[CE]); + + // If coro.end has an associated bundle, add cleanupret instruction. + if (auto Bundle = NewCE->getOperandBundle(LLVMContext::OB_funclet)) { + Value *FromPad = Bundle->Inputs[0]; + auto *CleanupRet = CleanupReturnInst::Create(FromPad, nullptr, NewCE); + NewCE->getParent()->splitBasicBlock(NewCE); + CleanupRet->getParent()->getTerminator()->eraseFromParent(); + } + + NewCE->replaceAllUsesWith(True); + NewCE->eraseFromParent(); + } +} + // Rewrite final suspend point handling. We do not use suspend index to // represent the final suspend point. Instead we zero-out ResumeFnAddr in the // coroutine frame, since it is undefined behavior to resume a coroutine @@ -271,8 +298,7 @@ // Remove coro.end intrinsics. replaceFallthroughCoroEnd(Shape.CoroEnds.front(), VMap); - // FIXME: coming in upcoming patches: - // replaceUnwindCoroEnds(Shape.CoroEnds, VMap); + replaceUnwindCoroEnds(Shape, VMap); // Eliminate coro.free from the clones, replacing it with 'null' in cleanup, // to suppress deallocation code. @@ -285,8 +311,16 @@ } static void removeCoroEnds(coro::Shape &Shape) { - for (CoroEndInst *CE : Shape.CoroEnds) + if (Shape.CoroEnds.empty()) + return; + + LLVMContext &Context = Shape.CoroEnds.front()->getContext(); + auto *False = ConstantInt::getFalse(Context); + + for (CoroEndInst *CE : Shape.CoroEnds) { + CE->replaceAllUsesWith(False); CE->eraseFromParent(); + } } static void replaceFrameSize(coro::Shape &Shape) { @@ -510,16 +544,6 @@ return; } - // If there are no suspend points, no split required, just remove - // the allocation and deallocation blocks, they are not needed. - if (Shape.CoroSuspends.empty()) { - handleNoSuspendCoroutine(Shape.CoroBegin, Shape.FrameTy); - removeCoroEnds(Shape); - postSplitCleanup(F); - coro::updateCallGraph(F, {}, CG, SCC); - return; - } - auto *ResumeEntry = createResumeEntryBlock(F, Shape); auto ResumeClone = createClone(F, ".resume", Shape, ResumeEntry, 0); auto DestroyClone = createClone(F, ".destroy", Shape, ResumeEntry, 1);