Index: lib/CodeGen/CGCoroutine.cpp =================================================================== --- lib/CodeGen/CGCoroutine.cpp +++ lib/CodeGen/CGCoroutine.cpp @@ -12,20 +12,34 @@ //===----------------------------------------------------------------------===// #include "CodeGenFunction.h" +#include "llvm/ADT/ScopeExit.h" #include "clang/AST/StmtCXX.h" using namespace clang; using namespace CodeGen; +using llvm::Value; +using llvm::BasicBlock; + +namespace { +enum class AwaitKind { Init, Normal, Yield, Final }; +char const *AwaitKindStr[] = {"init", "await", "yield", "final"}; +} + namespace clang { namespace CodeGen { struct CGCoroData { + AwaitKind CurrentAwaitKind = AwaitKind::Init; + llvm::BasicBlock *SuspendBB = nullptr; + CodeGenFunction::JumpDest CleanupJD; // Stores the jump destination just before the final suspend. Coreturn // statements jumps to this point after calling return_xxx promise member. CodeGenFunction::JumpDest FinalJD; + unsigned AwaitNum = 0; + unsigned YieldNum = 0; unsigned CoreturnCount = 0; // Stores the llvm.coro.id emitted in the function so that we can supply it @@ -66,6 +80,135 @@ CurCoro.Data->CoroIdExpr = CoroIdExpr; } +// Synthesize a pretty name for a suspend point. +static SmallString<32> buildSuspendSuffixStr(CGCoroData &Coro, AwaitKind Kind) { + unsigned No = 0; + switch (Kind) { + default: + break; + case AwaitKind::Normal: + No = ++Coro.AwaitNum; + break; + case AwaitKind::Yield: + No = ++Coro.YieldNum; + break; + } + SmallString<32> Suffix(AwaitKindStr[static_cast(Kind)]); + if (No > 1) { + Twine(No).toVector(Suffix); + } + return Suffix; +} + +// Emit suspend expression which roughly looks like: +// +// auto && x = CommonExpr(); +// if (!x.await_ready()) { +// llvm_coro_save(); +// x.await_suspend(...); (*) +// llvm_coro_suspend(); (**) +// } +// x.await_resume(); +// +// where the result of the entire expression is the result of x.await_resume() +// +// (*) If x.await_suspend return type is bool, it allows to veto a suspend: +// if (x.await_suspend(...)) +// llvm_coro_suspend(); +// +// (**) llvm_coro_suspend() encodes three possible continuation passes as +// a switch instruction: +// +// %where-to = call i8 @llvm.coro.suspend(...) +// switch i8 %where-to, label %coro.ret [ ; jump to epilogue to suspend +// i8 0, label %yield.ready ; go here when resumed +// i8 1, label %yield.cleanup ; go here when destroyed +// ] +// +// See llvm's docs/Coroutines.rst for more details. +// +static Value *emitSuspendExpression(CodeGenFunction &CGF, CGCoroData &Coro, + CoroutineSuspendExpr const &S, + AwaitKind Kind, + ReturnValueSlot ReturnValue) { + auto &Builder = CGF.Builder; + const bool IsFinalSuspend = Kind == AwaitKind::Final; + auto Suffix = buildSuspendSuffixStr(Coro, Kind); + + auto *E = S.getCommonExpr(); + if (auto *UO = dyn_cast(E)) + if (UO->getOpcode() == UO_Coawait) + E = UO->getSubExpr(); + + auto Binder = + CodeGenFunction::OpaqueValueMappingData::bind(CGF, S.getOpaqueValue(), E); + auto UnbindOnExit = llvm::make_scope_exit([&] { Binder.unbind(CGF); }); + + BasicBlock *ReadyBlock = CGF.createBasicBlock(Suffix + Twine(".ready")); + BasicBlock *SuspendBlock = CGF.createBasicBlock(Suffix + Twine(".suspend")); + BasicBlock *CleanupBlock = CGF.createBasicBlock(Suffix + Twine(".cleanup")); + + CGF.EmitBranchOnBoolExpr(S.getReadyExpr(), ReadyBlock, SuspendBlock, 0); + CGF.EmitBlock(SuspendBlock); + + llvm::Function *CoroSave = CGF.CGM.getIntrinsic(llvm::Intrinsic::coro_save); + auto *NullPtr = llvm::ConstantPointerNull::get(CGF.CGM.Int8PtrTy); + auto *SaveCall = Builder.CreateCall(CoroSave, {NullPtr}); + + auto *SuspendRet = CGF.EmitScalarExpr(S.getSuspendExpr()); + if (SuspendRet != nullptr) { + assert(SuspendRet->getType()->isIntegerTy(1) && + "Sema should have already checked that it is void or bool"); + BasicBlock *RealSuspendBlock = + CGF.createBasicBlock(Suffix + Twine(".suspend.bool")); + CGF.Builder.CreateCondBr(SuspendRet, RealSuspendBlock, ReadyBlock); + SuspendBlock = RealSuspendBlock; + CGF.EmitBlock(RealSuspendBlock); + } + + llvm::Function *CoroSuspend = + CGF.CGM.getIntrinsic(llvm::Intrinsic::coro_suspend); + auto *SuspendResult = Builder.CreateCall( + CoroSuspend, {SaveCall, Builder.getInt1(IsFinalSuspend)}); + auto *Switch = Builder.CreateSwitch(SuspendResult, Coro.SuspendBB, 2); + Switch->addCase(Builder.getInt8(0), ReadyBlock); + Switch->addCase(Builder.getInt8(1), CleanupBlock); + + CGF.EmitBlock(CleanupBlock); + + CGF.EmitBranchThroughCleanup(Coro.CleanupJD); + + // Emit await_resume expression. + CGF.EmitBlock(ReadyBlock); + QualType Type = S.getResumeExpr()->getType(); + switch (CGF.getEvaluationKind(Type)) { + case TEK_Scalar: + return CGF.EmitScalarExpr(S.getResumeExpr()); + case TEK_Aggregate: + CGF.EmitAggExpr(S.getResumeExpr(), + AggValueSlot::forAddr(ReturnValue.getValue(), Qualifiers(), + AggValueSlot::IsDestructed, + AggValueSlot::DoesNotNeedGCBarriers, + AggValueSlot::IsNotAliased)); + break; + case TEK_Complex: + CGF.CGM.ErrorUnsupported(S.getResumeExpr(), "_Complex await expression"); + break; + } + return nullptr; +} + +llvm::Value *CodeGenFunction::EmitCoawaitExpr(const CoawaitExpr &E, + ReturnValueSlot ReturnValue) { + return emitSuspendExpression(*this, *CurCoro.Data, E, + CurCoro.Data->CurrentAwaitKind, ReturnValue); +} +llvm::Value *CodeGenFunction::EmitCoyieldExpr(const CoyieldExpr &E, + ReturnValueSlot ReturnValue) { + return emitSuspendExpression(*this, *CurCoro.Data, E, AwaitKind::Yield, + ReturnValue); +} + void CodeGenFunction::EmitCoreturnStmt(CoreturnStmt const &S) { ++CurCoro.Data->CoreturnCount; EmitStmt(S.getPromiseCall()); @@ -78,11 +221,13 @@ unsigned NewAlign = TI.getNewAlign() / TI.getCharWidth(); auto *FinalBB = createBasicBlock("coro.final"); + auto *RetBB = createBasicBlock("coro.ret"); auto *CoroId = Builder.CreateCall( CGM.getIntrinsic(llvm::Intrinsic::coro_id), {Builder.getInt32(NewAlign), NullPtr, NullPtr, NullPtr}); createCoroData(*this, CurCoro, CoroId); + CurCoro.Data->SuspendBB = RetBB; EmitScalarExpr(S.getAllocate()); @@ -90,6 +235,7 @@ EmitStmt(S.getPromiseDeclStmt()); + CurCoro.Data->CleanupJD = getJumpDestInCurrentScope(RetBB); CurCoro.Data->FinalJD = getJumpDestInCurrentScope(FinalBB); // FIXME: Emit initial suspend and more before the body. @@ -105,6 +251,8 @@ } EmitStmt(S.getDeallocate()); + EmitBlock(RetBB); + // FIXME: Emit return for the coroutine return object. } Index: lib/CodeGen/CGExprAgg.cpp =================================================================== --- lib/CodeGen/CGExprAgg.cpp +++ lib/CodeGen/CGExprAgg.cpp @@ -111,6 +111,13 @@ void VisitGenericSelectionExpr(GenericSelectionExpr *GE) { Visit(GE->getResultExpr()); } + void VisitCoawaitExpr(CoawaitExpr *E) { + CGF.EmitCoawaitExpr(*E, getReturnValueSlot()); + } + void VisitCoyieldExpr(CoyieldExpr *E) { + CGF.EmitCoyieldExpr(*E, getReturnValueSlot()); + } + void VisitUnaryCoawait(UnaryOperator *E) { Visit(E->getSubExpr()); } void VisitUnaryExtension(UnaryOperator *E) { Visit(E->getSubExpr()); } void VisitSubstNonTypeTemplateParmExpr(SubstNonTypeTemplateParmExpr *E) { return Visit(E->getReplacement()); Index: lib/CodeGen/CGExprScalar.cpp =================================================================== --- lib/CodeGen/CGExprScalar.cpp +++ lib/CodeGen/CGExprScalar.cpp @@ -276,6 +276,15 @@ Value *VisitGenericSelectionExpr(GenericSelectionExpr *GE) { return Visit(GE->getResultExpr()); } + Value *VisitCoawaitExpr(CoawaitExpr* S) { + return CGF.EmitCoawaitExpr(*S); + } + Value *VisitCoyieldExpr(CoyieldExpr* S) { + return CGF.EmitCoyieldExpr(*S); + } + Value *VisitUnaryCoawait (const UnaryOperator *E) { + return Visit(E->getSubExpr()); + } // Leaves. Value *VisitIntegerLiteral(const IntegerLiteral *E) { @@ -1671,7 +1680,7 @@ } case CK_IntToOCLSampler: - return CGF.CGM.createOpenCLIntToSamplerConversion(E, CGF); + return CGF.CGM.createOpenCLIntToSamplerConversion(E, CGF); } // end of switch Index: lib/CodeGen/CodeGenFunction.h =================================================================== --- lib/CodeGen/CodeGenFunction.h +++ lib/CodeGen/CodeGenFunction.h @@ -2511,6 +2511,10 @@ void EmitCoroutineBody(const CoroutineBodyStmt &S); void EmitCoreturnStmt(const CoreturnStmt &S); + llvm::Value *EmitCoawaitExpr(const CoawaitExpr &E, + ReturnValueSlot ReturnValue = ReturnValueSlot()); + llvm::Value *EmitCoyieldExpr(const CoyieldExpr &E, + ReturnValueSlot ReturnValue = ReturnValueSlot()); RValue EmitCoroutineIntrinsic(const CallExpr *E, unsigned int IID); void EnterCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock = false); Index: test/CodeGenCoroutines/coro-await.cpp =================================================================== --- /dev/null +++ test/CodeGenCoroutines/coro-await.cpp @@ -0,0 +1,134 @@ +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fcoroutines-ts -std=c++14 -emit-llvm %s -o - -disable-llvm-passes | FileCheck %s + +namespace std { +namespace experimental { +template +struct coroutine_traits; + +template struct coroutine_handle; + +template <> +struct coroutine_handle { + void *ptr; + static coroutine_handle from_address(void *); +}; + +template +struct coroutine_handle : coroutine_handle<> { + static coroutine_handle from_address(void *); +}; + +} +} + +struct suspend_always { + int stuff; + bool await_ready(); + void await_suspend(std::experimental::coroutine_handle<>); + void await_resume(); +}; + +template<> +struct std::experimental::coroutine_traits { + struct promise_type { + void get_return_object(); + suspend_always initial_suspend(); + suspend_always final_suspend(); + void return_void(); + }; +}; + +// CHECK-LABEL: f0( +extern "C" void f0() { + + co_await suspend_always{}; + // See if we need to suspend: + // -------------------------- + // CHECK: %[[READY:.+]] = call zeroext i1 @_ZN14suspend_always11await_readyEv(%struct.suspend_always* %[[AWAITABLE:.+]]) + // CHECK: br i1 %[[READY]], label %[[READY_BB:.+]], label %[[SUSPEND_BB:.+]] + + // If we are suspending: + // --------------------- + // CHECK: [[SUSPEND_BB]]: + // CHECK: %[[SUSPEND_ID:.+]] = call token @llvm.coro.save( + // --------------------------- + // Build the coroutine handle and pass it to await_suspend + // --------------------------- + // CHECK: %[[FRAME:.+]] = call i8* @llvm.coro.frame() + // CHECK: call i8* @_ZNSt12experimental16coroutine_handleINS_16coroutine_traitsIJvEE12promise_typeEE12from_addressEPv(i8* %[[FRAME]]) + // ... many lines of code to coerce coroutine_handle into an i8* scalar + // CHECK: %[[CH:.+]] = load i8*, i8** %{{.+}} + // CHECK: call void @_ZN14suspend_always13await_suspendENSt12experimental16coroutine_handleIvEE(%struct.suspend_always* %[[AWAITABLE]], i8* %[[CH]]) + // ------------------------- + // Generate a suspend point: + // ------------------------- + // CHECK: %[[OUTCOME:.+]] = call i8 @llvm.coro.suspend(token %[[SUSPEND_ID]], i1 false) + // CHECK: switch i8 %[[OUTCOME]], label %[[RET_BB:.+]] [ + // CHECK: i8 0, label %[[READY_BB]] + // CHECK: i8 1, label %[[CLEANUP_BB:.+]] + // CHECK: ] + + // Cleanup code goes here: + // ----------------------- + // CHECK: [[CLEANUP_BB]]: + + // When coroutine is resumed, call await_resume + // -------------------------- + // CHECK: [[READY_BB]]: + // CHECK: call void @_ZN14suspend_always12await_resumeEv(%struct.suspend_always* %[[AWAITABLE]]) +} + +struct suspend_maybe { + float stuff; + ~suspend_maybe(); + bool await_ready(); + bool await_suspend(std::experimental::coroutine_handle<>); + void await_resume(); +}; + + +template<> +struct std::experimental::coroutine_traits { + struct promise_type { + void get_return_object(); + suspend_always initial_suspend(); + suspend_always final_suspend(); + void return_void(); + suspend_maybe yield_value(int); + }; +}; + +// CHECK-LABEL: f1( +extern "C" void f1(int) { + co_yield 42; + // CHECK: %[[PROMISE:.+]] = alloca %"struct.std::experimental::coroutine_traits::promise_type" + // CHECK: call void @_ZNSt12experimental16coroutine_traitsIJviEE12promise_type11yield_valueEi(%struct.suspend_maybe* sret %[[AWAITER:.+]], %"struct.std::experimental::coroutine_traits::promise_type"* %[[PROMISE]], i32 42) + + // See if we need to suspend: + // -------------------------- + // CHECK: %[[READY:.+]] = call zeroext i1 @_ZN13suspend_maybe11await_readyEv(%struct.suspend_maybe* %[[AWAITABLE]]) + // CHECK: br i1 %[[READY]], label %[[READY_BB:.+]], label %[[SUSPEND_BB:.+]] + + // If we are suspending: + // --------------------- + // CHECK: [[SUSPEND_BB]]: + // CHECK: %[[SUSPEND_ID:.+]] = call token @llvm.coro.save( + // --------------------------- + // Build the coroutine handle and pass it to await_suspend + // --------------------------- + // CHECK: %[[FRAME:.+]] = call i8* @llvm.coro.frame() + // CHECK: call i8* @_ZNSt12experimental16coroutine_handleINS_16coroutine_traitsIJviEE12promise_typeEE12from_addressEPv(i8* %[[FRAME]]) + // ... many lines of code to coerce coroutine_handle into an i8* scalar + // CHECK: %[[CH:.+]] = load i8*, i8** %{{.+}} + // CHECK: %[[YES:.+]] = call zeroext i1 @_ZN13suspend_maybe13await_suspendENSt12experimental16coroutine_handleIvEE(%struct.suspend_maybe* %[[AWAITABLE]], i8* %[[CH]]) + // ------------------------------------------- + // See if await_suspend decided not to suspend + // ------------------------------------------- + // CHECK: br i1 %[[YES]], label %[[SUSPEND_PLEASE:.+]], label %[[READY_BB]] + + // CHECK: [[SUSPEND_PLEASE]]: + // CHECK: call i8 @llvm.coro.suspend(token %[[SUSPEND_ID]], i1 false) + + // CHECK: [[READY_BB]]: + // CHECK: call void @_ZN13suspend_maybe12await_resumeEv(%struct.suspend_maybe* %[[AWAITABLE]]) +}