Index: lib/CodeGen/CGCoroutine.cpp =================================================================== --- lib/CodeGen/CGCoroutine.cpp +++ lib/CodeGen/CGCoroutine.cpp @@ -12,35 +12,58 @@ //===----------------------------------------------------------------------===// #include "CodeGenFunction.h" +#include "llvm/ADT/ScopeExit.h" #include "clang/AST/StmtCXX.h" using namespace clang; using namespace CodeGen; -namespace clang { -namespace CodeGen { +using llvm::Value; +using llvm::BasicBlock; -struct CGCoroData { +namespace { +enum class AwaitKind { Init, Normal, Yield, Final }; +static constexpr llvm::StringLiteral AwaitKindStr[] = {"init", "await", "yield", + "final"}; +} - // Stores the jump destination just before the final suspend. Coreturn - // statements jumps to this point after calling return_xxx promise member. - CodeGenFunction::JumpDest FinalJD; +struct clang::CodeGen::CGCoroData { + // What is the current await expression kind and how many + // await/yield expressions were encountered so far. + // These are used to generate pretty labels for await expressions in LLVM IR. + AwaitKind CurrentAwaitKind = AwaitKind::Init; + unsigned AwaitNum = 0; + unsigned YieldNum = 0; + // How many co_return statements are in the coroutine. Used to decide whether + // we need to add co_return; equivalent at the end of the user authored body. unsigned CoreturnCount = 0; + // A branch to this block is emitted when coroutine needs to suspend. + llvm::BasicBlock *SuspendBB = nullptr; + + // Stores the jump destination just before the coroutine memory is freed. + // This is the destination that every suspend point jumps to for the cleanup + // branch. + CodeGenFunction::JumpDest CleanupJD; + + // Stores the jump destination just before the final suspend. The co_return + // statements jumps to this point after calling return_xxx promise member. + CodeGenFunction::JumpDest FinalJD; + // Stores the llvm.coro.id emitted in the function so that we can supply it // as the first argument to coro.begin, coro.alloc and coro.free intrinsics. // Note: llvm.coro.id returns a token that cannot be directly expressed in a // builtin. llvm::CallInst *CoroId = nullptr; + // If coro.id came from the builtin, remember the expression to give better // diagnostic. If CoroIdExpr is nullptr, the coro.id was created by // EmitCoroutineBody. CallExpr const *CoroIdExpr = nullptr; }; -} -} +// Defining these here allows to keep CGCoroData private to this file. clang::CodeGen::CodeGenFunction::CGCoroInfo::CGCoroInfo() {} CodeGenFunction::CGCoroInfo::~CGCoroInfo() {} @@ -66,6 +89,126 @@ CurCoro.Data->CoroIdExpr = CoroIdExpr; } +// Synthesize a pretty name for a suspend point. +static SmallString<32> buildSuspendPrefixStr(CGCoroData &Coro, AwaitKind Kind) { + unsigned No = 0; + switch (Kind) { + case AwaitKind::Init: + case AwaitKind::Final: + break; + case AwaitKind::Normal: + No = ++Coro.AwaitNum; + break; + case AwaitKind::Yield: + No = ++Coro.YieldNum; + break; + } + SmallString<32> Prefix(AwaitKindStr[static_cast(Kind)]); + if (No > 1) { + Twine(No).toVector(Prefix); + } + return Prefix; +} + +// Emit suspend expression which roughly looks like: +// +// auto && x = CommonExpr(); +// if (!x.await_ready()) { +// llvm_coro_save(); +// x.await_suspend(...); (*) +// llvm_coro_suspend(); (**) +// } +// x.await_resume(); +// +// where the result of the entire expression is the result of x.await_resume() +// +// (*) If x.await_suspend return type is bool, it allows to veto a suspend: +// if (x.await_suspend(...)) +// llvm_coro_suspend(); +// +// (**) llvm_coro_suspend() encodes three possible continuations as +// a switch instruction: +// +// %where-to = call i8 @llvm.coro.suspend(...) +// switch i8 %where-to, label %coro.ret [ ; jump to epilogue to suspend +// i8 0, label %yield.ready ; go here when resumed +// i8 1, label %yield.cleanup ; go here when destroyed +// ] +// +// See llvm's docs/Coroutines.rst for more details. +// +static RValue emitSuspendExpression(CodeGenFunction &CGF, CGCoroData &Coro, + CoroutineSuspendExpr const &S, + AwaitKind Kind, AggValueSlot aggSlot, + bool ignoreResult) { + auto *E = S.getCommonExpr(); + auto Binder = + CodeGenFunction::OpaqueValueMappingData::bind(CGF, S.getOpaqueValue(), E); + auto UnbindOnExit = llvm::make_scope_exit([&] { Binder.unbind(CGF); }); + + auto Prefix = buildSuspendPrefixStr(Coro, Kind); + BasicBlock *ReadyBlock = CGF.createBasicBlock(Prefix + Twine(".ready")); + BasicBlock *SuspendBlock = CGF.createBasicBlock(Prefix + Twine(".suspend")); + BasicBlock *CleanupBlock = CGF.createBasicBlock(Prefix + Twine(".cleanup")); + + // If expression is ready, no need to suspend. + CGF.EmitBranchOnBoolExpr(S.getReadyExpr(), ReadyBlock, SuspendBlock, 0); + + // Otherwise, emit suspend logic. + CGF.EmitBlock(SuspendBlock); + + auto &Builder = CGF.Builder; + llvm::Function *CoroSave = CGF.CGM.getIntrinsic(llvm::Intrinsic::coro_save); + auto *NullPtr = llvm::ConstantPointerNull::get(CGF.CGM.Int8PtrTy); + auto *SaveCall = Builder.CreateCall(CoroSave, {NullPtr}); + + auto *SuspendRet = CGF.EmitScalarExpr(S.getSuspendExpr()); + if (SuspendRet != nullptr) { + // Veto suspension if requested by bool returning await_suspend. + assert(SuspendRet->getType()->isIntegerTy(1) && + "Sema should have already checked that it is void or bool"); + BasicBlock *RealSuspendBlock = + CGF.createBasicBlock(Prefix + Twine(".suspend.bool")); + CGF.Builder.CreateCondBr(SuspendRet, RealSuspendBlock, ReadyBlock); + SuspendBlock = RealSuspendBlock; + CGF.EmitBlock(RealSuspendBlock); + } + + // Emit the suspend point. + const bool IsFinalSuspend = (Kind == AwaitKind::Final); + llvm::Function *CoroSuspend = + CGF.CGM.getIntrinsic(llvm::Intrinsic::coro_suspend); + auto *SuspendResult = Builder.CreateCall( + CoroSuspend, {SaveCall, Builder.getInt1(IsFinalSuspend)}); + + // Create a switch capturing three possible continuations. + auto *Switch = Builder.CreateSwitch(SuspendResult, Coro.SuspendBB, 2); + Switch->addCase(Builder.getInt8(0), ReadyBlock); + Switch->addCase(Builder.getInt8(1), CleanupBlock); + + // Emit cleanup for this suspend point. + CGF.EmitBlock(CleanupBlock); + CGF.EmitBranchThroughCleanup(Coro.CleanupJD); + + // Emit await_resume expression. + CGF.EmitBlock(ReadyBlock); + return CGF.EmitAnyExpr(S.getResumeExpr(), aggSlot, ignoreResult); +} + +RValue CodeGenFunction::EmitCoawaitExpr(const CoawaitExpr &E, + AggValueSlot aggSlot, + bool ignoreResult) { + return emitSuspendExpression(*this, *CurCoro.Data, E, + CurCoro.Data->CurrentAwaitKind, aggSlot, + ignoreResult); +} +RValue CodeGenFunction::EmitCoyieldExpr(const CoyieldExpr &E, + AggValueSlot aggSlot, + bool ignoreResult) { + return emitSuspendExpression(*this, *CurCoro.Data, E, AwaitKind::Yield, + aggSlot, ignoreResult); +} + void CodeGenFunction::EmitCoreturnStmt(CoreturnStmt const &S) { ++CurCoro.Data->CoreturnCount; EmitStmt(S.getPromiseCall()); @@ -78,11 +221,13 @@ unsigned NewAlign = TI.getNewAlign() / TI.getCharWidth(); auto *FinalBB = createBasicBlock("coro.final"); + auto *RetBB = createBasicBlock("coro.ret"); auto *CoroId = Builder.CreateCall( CGM.getIntrinsic(llvm::Intrinsic::coro_id), {Builder.getInt32(NewAlign), NullPtr, NullPtr, NullPtr}); createCoroData(*this, CurCoro, CoroId); + CurCoro.Data->SuspendBB = RetBB; EmitScalarExpr(S.getAllocate()); @@ -90,10 +235,12 @@ EmitStmt(S.getPromiseDeclStmt()); + CurCoro.Data->CleanupJD = getJumpDestInCurrentScope(RetBB); CurCoro.Data->FinalJD = getJumpDestInCurrentScope(FinalBB); // FIXME: Emit initial suspend and more before the body. + CurCoro.Data->CurrentAwaitKind = AwaitKind::Normal; EmitStmt(S.getBody()); // See if we need to generate final suspend. @@ -105,6 +252,8 @@ } EmitStmt(S.getDeallocate()); + EmitBlock(RetBB); + // FIXME: Emit return for the coroutine return object. } Index: lib/CodeGen/CGExprAgg.cpp =================================================================== --- lib/CodeGen/CGExprAgg.cpp +++ lib/CodeGen/CGExprAgg.cpp @@ -111,6 +111,13 @@ void VisitGenericSelectionExpr(GenericSelectionExpr *GE) { Visit(GE->getResultExpr()); } + void VisitCoawaitExpr(CoawaitExpr *E) { + CGF.EmitCoawaitExpr(*E, Dest, IsResultUnused); + } + void VisitCoyieldExpr(CoyieldExpr *E) { + CGF.EmitCoyieldExpr(*E, Dest, IsResultUnused); + } + void VisitUnaryCoawait(UnaryOperator *E) { Visit(E->getSubExpr()); } void VisitUnaryExtension(UnaryOperator *E) { Visit(E->getSubExpr()); } void VisitSubstNonTypeTemplateParmExpr(SubstNonTypeTemplateParmExpr *E) { return Visit(E->getReplacement()); Index: lib/CodeGen/CGExprComplex.cpp =================================================================== --- lib/CodeGen/CGExprComplex.cpp +++ lib/CodeGen/CGExprComplex.cpp @@ -110,6 +110,16 @@ VisitSubstNonTypeTemplateParmExpr(SubstNonTypeTemplateParmExpr *PE) { return Visit(PE->getReplacement()); } + ComplexPairTy VisitCoawaitExpr(CoawaitExpr *S) { + return CGF.EmitCoawaitExpr(*S).getComplexVal(); + } + ComplexPairTy VisitCoyieldExpr(CoyieldExpr *S) { + return CGF.EmitCoyieldExpr(*S).getComplexVal(); + } + ComplexPairTy VisitUnaryCoawait(const UnaryOperator *E) { + return Visit(E->getSubExpr()); + } + // l-values. ComplexPairTy VisitDeclRefExpr(DeclRefExpr *E) { Index: lib/CodeGen/CGExprScalar.cpp =================================================================== --- lib/CodeGen/CGExprScalar.cpp +++ lib/CodeGen/CGExprScalar.cpp @@ -276,6 +276,15 @@ Value *VisitGenericSelectionExpr(GenericSelectionExpr *GE) { return Visit(GE->getResultExpr()); } + Value *VisitCoawaitExpr(CoawaitExpr *S) { + return CGF.EmitCoawaitExpr(*S).getScalarVal(); + } + Value *VisitCoyieldExpr(CoyieldExpr *S) { + return CGF.EmitCoyieldExpr(*S).getScalarVal(); + } + Value *VisitUnaryCoawait(const UnaryOperator *E) { + return Visit(E->getSubExpr()); + } // Leaves. Value *VisitIntegerLiteral(const IntegerLiteral *E) { Index: lib/CodeGen/CodeGenFunction.h =================================================================== --- lib/CodeGen/CodeGenFunction.h +++ lib/CodeGen/CodeGenFunction.h @@ -1986,7 +1986,7 @@ /// pointer to a char. Address EmitMSVAListRef(const Expr *E); - /// EmitAnyExprToTemp - Similary to EmitAnyExpr(), however, the result will + /// EmitAnyExprToTemp - Similarly to EmitAnyExpr(), however, the result will /// always be accessible even if no aggregate location is provided. RValue EmitAnyExprToTemp(const Expr *E); @@ -2528,6 +2528,12 @@ void EmitCoroutineBody(const CoroutineBodyStmt &S); void EmitCoreturnStmt(const CoreturnStmt &S); + RValue EmitCoawaitExpr(const CoawaitExpr &E, + AggValueSlot aggSlot = AggValueSlot::ignored(), + bool ignoreResult = false); + RValue EmitCoyieldExpr(const CoyieldExpr &E, + AggValueSlot aggSlot = AggValueSlot::ignored(), + bool ignoreResult = false); RValue EmitCoroutineIntrinsic(const CallExpr *E, unsigned int IID); void EnterCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock = false); Index: test/CodeGenCoroutines/coro-await.cpp =================================================================== --- /dev/null +++ test/CodeGenCoroutines/coro-await.cpp @@ -0,0 +1,230 @@ +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fcoroutines-ts -std=c++14 -emit-llvm %s -o - -disable-llvm-passes | FileCheck %s + +namespace std { +namespace experimental { +template +struct coroutine_traits; + +template struct coroutine_handle; + +template <> +struct coroutine_handle { + void *ptr; + static coroutine_handle from_address(void *); +}; + +template +struct coroutine_handle : coroutine_handle<> { + static coroutine_handle from_address(void *); +}; + +} +} + +struct suspend_always { + int stuff; + bool await_ready(); + void await_suspend(std::experimental::coroutine_handle<>); + void await_resume(); +}; + +template<> +struct std::experimental::coroutine_traits { + struct promise_type { + void get_return_object(); + suspend_always initial_suspend(); + suspend_always final_suspend(); + void return_void(); + }; +}; + +// CHECK-LABEL: f0( +extern "C" void f0() { + + co_await suspend_always{}; + // See if we need to suspend: + // -------------------------- + // CHECK: %[[READY:.+]] = call zeroext i1 @_ZN14suspend_always11await_readyEv(%struct.suspend_always* %[[AWAITABLE:.+]]) + // CHECK: br i1 %[[READY]], label %[[READY_BB:.+]], label %[[SUSPEND_BB:.+]] + + // If we are suspending: + // --------------------- + // CHECK: [[SUSPEND_BB]]: + // CHECK: %[[SUSPEND_ID:.+]] = call token @llvm.coro.save( + // --------------------------- + // Build the coroutine handle and pass it to await_suspend + // --------------------------- + // CHECK: %[[FRAME:.+]] = call i8* @llvm.coro.frame() + // CHECK: call i8* @_ZNSt12experimental16coroutine_handleINS_16coroutine_traitsIJvEE12promise_typeEE12from_addressEPv(i8* %[[FRAME]]) + // ... many lines of code to coerce coroutine_handle into an i8* scalar + // CHECK: %[[CH:.+]] = load i8*, i8** %{{.+}} + // CHECK: call void @_ZN14suspend_always13await_suspendENSt12experimental16coroutine_handleIvEE(%struct.suspend_always* %[[AWAITABLE]], i8* %[[CH]]) + // ------------------------- + // Generate a suspend point: + // ------------------------- + // CHECK: %[[OUTCOME:.+]] = call i8 @llvm.coro.suspend(token %[[SUSPEND_ID]], i1 false) + // CHECK: switch i8 %[[OUTCOME]], label %[[RET_BB:.+]] [ + // CHECK: i8 0, label %[[READY_BB]] + // CHECK: i8 1, label %[[CLEANUP_BB:.+]] + // CHECK: ] + + // Cleanup code goes here: + // ----------------------- + // CHECK: [[CLEANUP_BB]]: + + // When coroutine is resumed, call await_resume + // -------------------------- + // CHECK: [[READY_BB]]: + // CHECK: call void @_ZN14suspend_always12await_resumeEv(%struct.suspend_always* %[[AWAITABLE]]) +} + +struct suspend_maybe { + float stuff; + ~suspend_maybe(); + bool await_ready(); + bool await_suspend(std::experimental::coroutine_handle<>); + void await_resume(); +}; + + +template<> +struct std::experimental::coroutine_traits { + struct promise_type { + void get_return_object(); + suspend_always initial_suspend(); + suspend_always final_suspend(); + void return_void(); + suspend_maybe yield_value(int); + }; +}; + +// CHECK-LABEL: f1( +extern "C" void f1(int) { + co_yield 42; + // CHECK: %[[PROMISE:.+]] = alloca %"struct.std::experimental::coroutine_traits::promise_type" + // CHECK: call void @_ZNSt12experimental16coroutine_traitsIJviEE12promise_type11yield_valueEi(%struct.suspend_maybe* sret %[[AWAITER:.+]], %"struct.std::experimental::coroutine_traits::promise_type"* %[[PROMISE]], i32 42) + + // See if we need to suspend: + // -------------------------- + // CHECK: %[[READY:.+]] = call zeroext i1 @_ZN13suspend_maybe11await_readyEv(%struct.suspend_maybe* %[[AWAITABLE]]) + // CHECK: br i1 %[[READY]], label %[[READY_BB:.+]], label %[[SUSPEND_BB:.+]] + + // If we are suspending: + // --------------------- + // CHECK: [[SUSPEND_BB]]: + // CHECK: %[[SUSPEND_ID:.+]] = call token @llvm.coro.save( + // --------------------------- + // Build the coroutine handle and pass it to await_suspend + // --------------------------- + // CHECK: %[[FRAME:.+]] = call i8* @llvm.coro.frame() + // CHECK: call i8* @_ZNSt12experimental16coroutine_handleINS_16coroutine_traitsIJviEE12promise_typeEE12from_addressEPv(i8* %[[FRAME]]) + // ... many lines of code to coerce coroutine_handle into an i8* scalar + // CHECK: %[[CH:.+]] = load i8*, i8** %{{.+}} + // CHECK: %[[YES:.+]] = call zeroext i1 @_ZN13suspend_maybe13await_suspendENSt12experimental16coroutine_handleIvEE(%struct.suspend_maybe* %[[AWAITABLE]], i8* %[[CH]]) + // ------------------------------------------- + // See if await_suspend decided not to suspend + // ------------------------------------------- + // CHECK: br i1 %[[YES]], label %[[SUSPEND_PLEASE:.+]], label %[[READY_BB]] + + // CHECK: [[SUSPEND_PLEASE]]: + // CHECK: call i8 @llvm.coro.suspend(token %[[SUSPEND_ID]], i1 false) + + // CHECK: [[READY_BB]]: + // CHECK: call void @_ZN13suspend_maybe12await_resumeEv(%struct.suspend_maybe* %[[AWAITABLE]]) +} + +struct ComplexAwaiter { + template void await_suspend(F); + bool await_ready(); + _Complex float await_resume(); +}; +extern "C" void UseComplex(_Complex float); + +// CHECK-LABEL: @TestComplex( +extern "C" void TestComplex() { + UseComplex(co_await ComplexAwaiter{}); + // CHECK: call <2 x float> @_ZN14ComplexAwaiter12await_resumeEv(%struct.ComplexAwaiter* + // CHECK: call void @UseComplex(<2 x float> %{{.+}}) + + co_await ComplexAwaiter{}; + // CHECK: call <2 x float> @_ZN14ComplexAwaiter12await_resumeEv(%struct.ComplexAwaiter* + + _Complex float Val = co_await ComplexAwaiter{}; + // CHECK: call <2 x float> @_ZN14ComplexAwaiter12await_resumeEv(%struct.ComplexAwaiter* +} + +struct Aggr { int X, Y, Z; ~Aggr(); }; +struct AggrAwaiter { + template void await_suspend(F); + bool await_ready(); + Aggr await_resume(); +}; + +extern "C" void Whatever(); +extern "C" void UseAggr(Aggr&&); + +// FIXME: Once the cleanup code is in, add testing that destructors for Aggr +// are invoked properly on the cleanup branches. + +// CHECK-LABEL: @TestAggr( +extern "C" void TestAggr() { + UseAggr(co_await AggrAwaiter{}); + Whatever(); + // CHECK: call void @_ZN11AggrAwaiter12await_resumeEv(%struct.Aggr* sret %[[AwaitResume:.+]], + // CHECK: call void @UseAggr(%struct.Aggr* dereferenceable(12) %[[AwaitResume]]) + // CHECK: call void @_ZN4AggrD1Ev(%struct.Aggr* %[[AwaitResume]]) + // CHECK: call void @Whatever() + + co_await AggrAwaiter{}; + Whatever(); + // CHECK: call void @_ZN11AggrAwaiter12await_resumeEv(%struct.Aggr* sret %[[AwaitResume2:.+]], + // CHECK: call void @_ZN4AggrD1Ev(%struct.Aggr* %[[AwaitResume2]]) + // CHECK: call void @Whatever() + + Aggr Val = co_await AggrAwaiter{}; + Whatever(); + // CHECK: call void @_ZN11AggrAwaiter12await_resumeEv(%struct.Aggr* sret %[[AwaitResume3:.+]], + // CHECK: call void @Whatever() + // CHECK: call void @_ZN4AggrD1Ev(%struct.Aggr* %[[AwaitResume3]]) +} + +struct ScalarAwaiter { + template void await_suspend(F); + bool await_ready(); + int await_resume(); +}; + +extern "C" void UseScalar(int); + +// CHECK-LABEL: @TestScalar( +extern "C" void TestScalar() { + UseScalar(co_await ScalarAwaiter{}); + // CHECK: %[[Result:.+]] = call i32 @_ZN13ScalarAwaiter12await_resumeEv(%struct.ScalarAwaiter* + // CHECK: call void @UseScalar(i32 %[[Result]]) + + int Val = co_await ScalarAwaiter{}; + // CHECK: %[[Result2:.+]] = call i32 @_ZN13ScalarAwaiter12await_resumeEv(%struct.ScalarAwaiter* + // CHECK: store i32 %[[Result2]], i32* %Val + + co_await ScalarAwaiter{}; + // CHECK: call i32 @_ZN13ScalarAwaiter12await_resumeEv(%struct.ScalarAwaiter* +} + +// Test operator co_await codegen. +enum class MyInt: int {}; +ScalarAwaiter operator co_await(MyInt); + +struct MyAgg { + AggrAwaiter operator co_await(); +}; + +// CHECK-LABEL: @TestOpAwait( +extern "C" void TestOpAwait() { + co_await MyInt(42); + // CHECK: call void @_Zaw5MyInt(i32 42) + // CHECK: call i32 @_ZN13ScalarAwaiter12await_resumeEv(%struct.ScalarAwaiter* % + + co_await MyAgg{}; + // CHECK: call void @_ZN5MyAggawEv(%struct.MyAgg* % + // CHECK: call void @_ZN11AggrAwaiter12await_resumeEv(%struct.Aggr* sret % +}