diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst --- a/clang/docs/LanguageExtensions.rst +++ b/clang/docs/LanguageExtensions.rst @@ -3007,6 +3007,7 @@ void *__builtin_coro_begin(void *memory) void __builtin_coro_end(void *coro_frame, bool unwind) char __builtin_coro_suspend(bool final) + void __builtin_coro_mark_done() bool __builtin_coro_param(void *original, void *copy) Note that there is no builtin matching the `llvm.coro.save` intrinsic. LLVM diff --git a/clang/include/clang/Basic/Builtins.def b/clang/include/clang/Basic/Builtins.def --- a/clang/include/clang/Basic/Builtins.def +++ b/clang/include/clang/Basic/Builtins.def @@ -1611,6 +1611,7 @@ LANGBUILTIN(__builtin_coro_begin, "v*v*", "n", COR_LANG) LANGBUILTIN(__builtin_coro_end, "bv*Ib", "n", COR_LANG) LANGBUILTIN(__builtin_coro_suspend, "cIb", "n", COR_LANG) +LANGBUILTIN(__builtin_coro_mark_done, "v", "n", COR_LANG) LANGBUILTIN(__builtin_coro_param, "bv*v*", "n", COR_LANG) // OpenCL v2.0 s6.13.16, s9.17.3.5 - Pipe functions. diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -4696,6 +4696,8 @@ return EmitCoroutineIntrinsic(E, Intrinsic::coro_suspend); case Builtin::BI__builtin_coro_param: return EmitCoroutineIntrinsic(E, Intrinsic::coro_param); + case Builtin::BI__builtin_coro_mark_done: + return EmitCoroutineIntrinsic(E, Intrinsic::coro_mark_done); // OpenCL v2.0 s6.13.16.2, Built-in pipe read and write functions case Builtin::BIread_pipe: diff --git a/clang/lib/Sema/SemaCoroutine.cpp b/clang/lib/Sema/SemaCoroutine.cpp --- a/clang/lib/Sema/SemaCoroutine.cpp +++ b/clang/lib/Sema/SemaCoroutine.cpp @@ -1476,7 +1476,43 @@ return false; } - this->OnException = UnhandledException.get(); + // [dcl.fct.def.coroutine]/p14 + // If the evaluation of the expression promise.unhandled_­exception() exits + // via an exception, the coroutine is considered suspended at the final + // suspend point. + // + // To implement this, it convert the call to promise.unhandled_exception() to: + // ``` + // try { + // promise.unhandled_exception(); + // } catch (...) { + // __builtin_coro_mark_done(); + // throw; // Re-throw the exception + // thrown from promise.unhandled_­exception() + // } + // ``` + // + // The call to __builtin_coro_mark_done would mark current coroutine as + // done. In other words, the coroutine is considered suspended at the final + // suspend point. + auto *TryBody = CompoundStmt::Create(S.getASTContext(), + UnhandledException.get(), Loc, Loc); + + Expr *MarkDone = + S.BuildBuiltinCallExpr(Loc, Builtin::BI__builtin_coro_mark_done, {}); + ExprResult Rethrow = + S.BuildCXXThrow(Loc, nullptr, /*IsThrownVarInScope=*/false); + if (Rethrow.isInvalid()) + return false; + auto *CatchBody = CompoundStmt::Create(S.getASTContext(), + {MarkDone, Rethrow.get()}, Loc, Loc); + if (!CatchBody) + return false; + + auto *Catch = new (S.getASTContext()) CXXCatchStmt(Loc, nullptr, CatchBody); + + this->OnException = + CXXTryStmt::Create(S.getASTContext(), Loc, TryBody, Catch); return true; } diff --git a/clang/test/CodeGenCoroutines/coro-await-resume-eh-exp-namespace.cpp b/clang/test/CodeGenCoroutines/coro-await-resume-eh-exp-namespace.cpp --- a/clang/test/CodeGenCoroutines/coro-await-resume-eh-exp-namespace.cpp +++ b/clang/test/CodeGenCoroutines/coro-await-resume-eh-exp-namespace.cpp @@ -53,6 +53,8 @@ // CHECK: invoke void @_ZN13throwing_task12promise_type19unhandled_exceptionEv // CHECK-NEXT: to label %[[RESUMEENDCATCH:.+]] unwind label // CHECK: [[RESUMEENDCATCH]]: + // CHECK-NEXT: br label %[[TRY_CONT:.+]] + // CHECK: [[TRY_CONT]] // CHECK-NEXT: invoke void @__cxa_end_catch() // CHECK-NEXT: to label %[[RESUMEENDCATCHCONT:.+]] unwind label // CHECK: [[RESUMEENDCATCHCONT]]: diff --git a/clang/test/CodeGenCoroutines/coro-await-resume-eh.cpp b/clang/test/CodeGenCoroutines/coro-await-resume-eh.cpp --- a/clang/test/CodeGenCoroutines/coro-await-resume-eh.cpp +++ b/clang/test/CodeGenCoroutines/coro-await-resume-eh.cpp @@ -51,6 +51,8 @@ // CHECK: invoke void @_ZN13throwing_task12promise_type19unhandled_exceptionEv // CHECK-NEXT: to label %[[RESUMEENDCATCH:.+]] unwind label // CHECK: [[RESUMEENDCATCH]]: + // CHECK-NEXT: br label %[[TRY_CONT:.+]] + // CHECK: [[TRY_CONT]] // CHECK-NEXT: invoke void @__cxa_end_catch() // CHECK-NEXT: to label %[[RESUMEENDCATCHCONT:.+]] unwind label // CHECK: [[RESUMEENDCATCHCONT]]: diff --git a/clang/test/CodeGenCoroutines/coro-throw-from-unhandled_exception.cpp b/clang/test/CodeGenCoroutines/coro-throw-from-unhandled_exception.cpp new file mode 100644 --- /dev/null +++ b/clang/test/CodeGenCoroutines/coro-throw-from-unhandled_exception.cpp @@ -0,0 +1,43 @@ +// RUN: %clang_cc1 -std=c++20 \ +// RUN: -triple=x86_64-unknown-linux-gnu -emit-llvm -o - %s \ +// RUN: -fexceptions -fcxx-exceptions -disable-llvm-passes \ +// RUN: | FileCheck %s + +#include "Inputs/coroutine.h" + +struct throwing_task { + struct promise_type { + auto get_return_object() { return throwing_task{}; } + auto initial_suspend() { return std::suspend_always{}; } + auto final_suspend() noexcept { return std::suspend_never{}; } + void return_void() {} + void unhandled_exception() { throw 43; } + }; +}; + +throwing_task f() { + // CHECK: invoke void @_ZN13throwing_task12promise_type19unhandled_exceptionEv + // CHECK-NEXT: to label %{{.+}} unwind label %[[LPAD:.+]] + // CHECK: [[LPAD]] + // CHECK: br label %[[CATCH:.+]] + // CHECK: [[CATCH]] + // CHECK-NOT: br label + // CHECK: call void @llvm.coro.mark.done + // CHECK-NEXT: invoke void @__cxa_rethrow() + co_return; +} + +struct nothrowing_task { + struct promise_type { + auto get_return_object() { return nothrowing_task{}; } + auto initial_suspend() { return std::suspend_always{}; } + auto final_suspend() noexcept { return std::suspend_never{}; } + void return_void() {} + void unhandled_exception() noexcept {} + }; +}; + +nothrowing_task f2() { + // CHECK: call void @_ZN15nothrowing_task12promise_type19unhandled_exceptionEv + co_return; +} diff --git a/llvm/docs/Coroutines.rst b/llvm/docs/Coroutines.rst --- a/llvm/docs/Coroutines.rst +++ b/llvm/docs/Coroutines.rst @@ -1644,6 +1644,54 @@ In a yield-once coroutine, it is undefined behavior if the coroutine executes a call to ``llvm.coro.suspend.retcon`` after resuming in any way. +.. _coro.mark.done: + +'llvm.coro.mark.done' Intrinsic +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:: + + declare void @llvm.coro.mark.done() + +Overview: +""""""""" + +The '``llvm.coro.mark.done``' is used by a frontend to mark a coroutine suspended +at the final suspend point. After '``llvm.coro.mark.done``' is called, `llvm.coro.done()` +should return true. All of the '``llvm.coro.mark.done``' would be lowered in CoroSplit pass. +This intrinsic is only supported for switched-resume coroutines. + +Arguments: +"""""""""" + +The '``llvm.coro.mark.done``' doesn't require any argument. The basic assumption here is that +the coroutines in LLVM wouldn't be inlined before splitted. So there is no ambiguousity about the +coroutine that the '``llvm.coro.mark.done``' refers to. + +Semantics: +"""""""""" + +After '``llvm.coro.mark.done``' is called in a coroutine function, the corresponding coroutine is +considered suspend at final suspend point. In other words, it isn't allowed to resume the coroutine +any more. + +Example: +"""""""" + +In C++'s specification, [dcl.fct.def.coroutine]/p14 says: +> If the evaluation of the expression promise.unhandled_­exception() exits via an exception, +> the coroutine is considered suspended at the final suspend point. + +In this case, the frontend could generate following code: + +.. code-block:: c++ + + try { + promise.unhandled_­exception(); + } catch (...) { + call void @llvm.coro.mark.done(); + throw; + } + .. _coro.param: 'llvm.coro.param' Intrinsic diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h --- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -626,6 +626,7 @@ case Intrinsic::coro_size: case Intrinsic::coro_suspend: case Intrinsic::coro_param: + case Intrinsic::coro_mark_done: case Intrinsic::coro_subfn_addr: // These intrinsics don't actually represent code after lowering. return 0; diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -1286,6 +1286,8 @@ [IntrNoMem, ReadNone>, ReadNone>]>; +def int_coro_mark_done : Intrinsic<[], [], []>; + // Coroutine Manipulation Intrinsics. def int_coro_resume : Intrinsic<[], [llvm_ptr_ty], [Throws]>; diff --git a/llvm/lib/Transforms/Coroutines/CoroInstr.h b/llvm/lib/Transforms/Coroutines/CoroInstr.h --- a/llvm/lib/Transforms/Coroutines/CoroInstr.h +++ b/llvm/lib/Transforms/Coroutines/CoroInstr.h @@ -599,6 +599,18 @@ } }; +/// This represents the llvm.coro.mark.done instruction. +class LLVM_LIBRARY_VISIBILITY CoroMarkDoneInst : public IntrinsicInst { +public: + // Methods to support type inquiry through isa, cast, and dyn_cast: + static bool classof(const IntrinsicInst *I) { + return I->getIntrinsicID() == Intrinsic::coro_mark_done; + } + static bool classof(const Value *V) { + return isa(V) && classof(cast(V)); + } +}; + class LLVM_LIBRARY_VISIBILITY AnyCoroEndInst : public IntrinsicInst { enum { FrameArg, UnwindArg }; diff --git a/llvm/lib/Transforms/Coroutines/CoroInternal.h b/llvm/lib/Transforms/Coroutines/CoroInternal.h --- a/llvm/lib/Transforms/Coroutines/CoroInternal.h +++ b/llvm/lib/Transforms/Coroutines/CoroInternal.h @@ -101,6 +101,7 @@ SmallVector CoroSizes; SmallVector CoroSuspends; SmallVector SwiftErrorOps; + SmallVector CoroMarkDones; // Field indexes for special fields in the switch lowering. struct SwitchFieldIndex { diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp --- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp +++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp @@ -1480,10 +1480,28 @@ S.resize(N); } +static void handleCoroMarkDone(Function &F, coro::Shape &Shape) { + IRBuilder<> Builder(F.getContext()); + for (auto *CMD : Shape.CoroMarkDones) { + Builder.SetInsertPoint(CMD); + // Done state is represented by storing zero in ResumeFnAddr. + auto *GepIndex = Builder.CreateStructGEP( + Shape.FrameTy, Shape.FramePtr, coro::Shape::SwitchFieldIndex::Resume, + "ResumeFn.addr"); + auto *NullPtr = ConstantPointerNull::get(cast( + Shape.FrameTy->getTypeAtIndex(coro::Shape::SwitchFieldIndex::Resume))); + Builder.CreateStore(NullPtr, GepIndex); + CMD->eraseFromParent(); + } + + Shape.CoroMarkDones.clear(); +} + static void splitSwitchCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl &Clones) { assert(Shape.ABI == coro::ABI::Switch); + handleCoroMarkDone(F, Shape); createResumeEntryBlock(F, Shape); auto ResumeClone = createClone(F, ".resume", Shape, CoroCloner::Kind::SwitchResume); diff --git a/llvm/lib/Transforms/Coroutines/Coroutines.cpp b/llvm/lib/Transforms/Coroutines/Coroutines.cpp --- a/llvm/lib/Transforms/Coroutines/Coroutines.cpp +++ b/llvm/lib/Transforms/Coroutines/Coroutines.cpp @@ -301,6 +301,9 @@ } break; } + case Intrinsic::coro_mark_done: + CoroMarkDones.push_back(cast(II)); + break; case Intrinsic::coro_begin: { auto CB = cast(II); diff --git a/llvm/test/Transforms/Coroutines/coro-mark-done.ll b/llvm/test/Transforms/Coroutines/coro-mark-done.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/Coroutines/coro-mark-done.ll @@ -0,0 +1,68 @@ +; Check that the @llvm.coro.mark.done could be lowered correctly +; RUN: opt < %s -passes='cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s + +define i8* @f(i1 %val) "coroutine.presplit"="1" personality i32 3 { +entry: + %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null) + %hdl = call i8* @llvm.coro.begin(token %id, i8* null) + call void @print(i32 0) + br i1 %val, label %resume, label %susp + +susp: + %0 = call i8 @llvm.coro.suspend(token none, i1 false) + switch i8 %0, label %suspend [i8 0, label %resume + i8 1, label %suspend] +resume: + invoke void @print(i32 1) to label %suspend unwind label %lpad + +suspend: + call i1 @llvm.coro.end(i8* %hdl, i1 false) + ret i8* %hdl + +lpad: + %lpval = landingpad { i8*, i32 } + cleanup + + call void @llvm.coro.mark.done() + %need.resume = call i1 @llvm.coro.end(i8* null, i1 true) + resume { i8*, i32 } %lpval +} +; CHECK-LABEL: define i8* @f( +; CHECK: lpad: +; CHECK: store void (%f.Frame*)* null, void (%f.Frame*)** %resume.addr, align 8 +; CHECK-NEXT: resume { i8*, i32 } %lpval + +; CHECK-LABEL: define internal fastcc void @f.resume +; CHECK: lpad: +; CHECK: %ResumeFn.addr = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 0 +; CHECK-NEXT: store void (%f.Frame*)* null, void (%f.Frame*)** %ResumeFn.addr, align 8 +; CHECK-NEXT: resume { i8*, i32 } %lpval + +; CHECK-LABEL: define internal fastcc void @f.destroy +; CHECK: lpad: +; CHECK: %ResumeFn.addr = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 0 +; CHECK-NEXT: store void (%f.Frame*)* null, void (%f.Frame*)** %ResumeFn.addr, align 8 +; CHECK-NEXT: resume { i8*, i32 } %lpval + +; CHECK-LABEL: define internal fastcc void @f.cleanup +; CHECK: lpad: +; CHECK: %ResumeFn.addr = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 0 +; CHECK-NEXT: store void (%f.Frame*)* null, void (%f.Frame*)** %ResumeFn.addr, align 8 +; CHECK-NEXT: resume { i8*, i32 } %lpval + +declare i8* @llvm.coro.free(token, i8*) +declare i32 @llvm.coro.size.i32() +declare i8 @llvm.coro.suspend(token, i1) +declare void @llvm.coro.resume(i8*) +declare void @llvm.coro.destroy(i8*) +declare void @llvm.coro.mark.done() + +declare token @llvm.coro.id(i32, i8*, i8*, i8*) +declare i8* @llvm.coro.alloc(token) +declare i8* @llvm.coro.begin(token, i8*) +declare i1 @llvm.coro.end(i8*, i1) + +declare noalias i8* @malloc(i32) +declare void @print(i32) +declare void @free(i8*) + \ No newline at end of file