[C++20] [Coroutines] Implement return value optimization for get_return_object

[ChuanqiXu]

Summary

This patch tries to implement RVO for coroutine's return object got from get_return_object.
From [dcl.fct.def.coroutine]/p7 we could know that the return value of get_return_object is either a reference or a prvalue. So it makes sense to do copy elision for the return value. The return object should be constructed directly into the storage where they would otherwise be copied/moved to.

I have tested folly and our internal workloads.

[ChuanqiXu]

Update.

[ChuanqiXu]

clang/lib/CodeGen/CGCoroutine.cpp

481–485

Since we've deleted the gro variable, we could remove GetReturnObjectManager.

[ChuanqiXu]

gentle ping~

[ChuanqiXu]

gentle ping~

[junparser]

LGTM. Thanks!

clang/lib/CodeGen/CGCoroutine.cpp

650–653

can we just remove this?

[ChuanqiXu]

Update test.

clang/lib/CodeGen/CGCoroutine.cpp

650–653

We couldn't. Otherwise it would emit the return expression again.

[junparser]

clang/lib/CodeGen/CGCoroutine.cpp

654–655

I mean, remove the if statements here since the retuen expr is null.

[ChuanqiXu]

clang/lib/CodeGen/CGCoroutine.cpp

654–655

We couldn't. Since we need emit ret instruction if needed.

[junparser]

clang/lib/CodeGen/CGCoroutine.cpp

654–655

make sense to me.

[alkis]

Does this change the semantics of get_return_value() such that it is eagerly converted to R the return value of the coro? AFAIU before this change one could return T convertible to R and the conversion would happen late - after final_suspend.

[ChuanqiXu]

In D117087#4082454, @alkis wrote:

Does this change the semantics of get_return_value() such that it is eagerly converted to R the return value of the coro? AFAIU before this change one could return T convertible to R and the conversion would happen late - after final_suspend.

Yes, this changed the semantics. But the spec is ambiguous for this. Wg21 is tracking this in https://cplusplus.github.io/CWG/issues/2563.html.

clang/include/clang/AST/StmtCXX.h

enum SubStmt {
     Promise,       ///< The promise statement.
     InitSuspend,   ///< The initial suspend statement, run before the body.
     FinalSuspend,  ///< The final suspend statement, run after the body.
     OnException,   ///< Handler for exceptions thrown in the body.
     OnFallthrough, ///< Handler for control flow falling off the body.
     Allocate,      ///< Coroutine frame memory allocation.
     Deallocate,    ///< Coroutine frame memory deallocation.
     ReturnValue,   ///< Return value for thunk function: p.get_return_object().
-    ResultDecl,    ///< Declaration holding the result of get_return_object.
     ReturnStmt,    ///< Return statement for the thunk function.
     ReturnStmtOnAllocFailure, ///< Return statement if allocation failed.
     FirstParamMove ///< First offset for move construction of parameter copies.
   };
   unsigned NumParams;
 
   friend class ASTStmtReader;
   friend class ASTReader;
struct CtorArgs {
     Stmt *Promise = nullptr;
     Expr *InitialSuspend = nullptr;
     Expr *FinalSuspend = nullptr;
     Stmt *OnException = nullptr;
     Stmt *OnFallthrough = nullptr;
     Expr *Allocate = nullptr;
     Expr *Deallocate = nullptr;
     Expr *ReturnValue = nullptr;
-    Stmt *ResultDecl = nullptr;
     Stmt *ReturnStmt = nullptr;
     Stmt *ReturnStmtOnAllocFailure = nullptr;
     ArrayRef<Stmt *> ParamMoves;
   };
 
 private:
 
   CoroutineBodyStmt(CtorArgs const& Args);
Expr *getAllocate() const {
     return cast_or_null<Expr>(getStoredStmts()[SubStmt::Allocate]);
   }
   Expr *getDeallocate() const {
     return cast_or_null<Expr>(getStoredStmts()[SubStmt::Deallocate]);
   }
   Expr *getReturnValueInit() const {
     return cast<Expr>(getStoredStmts()[SubStmt::ReturnValue]);
   }
-  Stmt *getResultDecl() const { return getStoredStmts()[SubStmt::ResultDecl]; }
+  Expr *getReturnValue() const {
+    assert(getReturnStmt());
+    auto *RS = cast<clang::ReturnStmt>(getReturnStmt());
+    return RS->getRetValue();
+  }
   Stmt *getReturnStmt() const { return getStoredStmts()[SubStmt::ReturnStmt]; }
   Stmt *getReturnStmtOnAllocFailure() const {
     return getStoredStmts()[SubStmt::ReturnStmtOnAllocFailure];
   }
   ArrayRef<Stmt const *> getParamMoves() const {
     return {getStoredStmts() + SubStmt::FirstParamMove, NumParams};
   }
 

clang/lib/AST/StmtCXX.cpp

CoroutineBodyStmt::CoroutineBodyStmt(CoroutineBodyStmt::CtorArgs const &Args)
   SubStmts[CoroutineBodyStmt::Promise] = Args.Promise;
   SubStmts[CoroutineBodyStmt::InitSuspend] = Args.InitialSuspend;
   SubStmts[CoroutineBodyStmt::FinalSuspend] = Args.FinalSuspend;
   SubStmts[CoroutineBodyStmt::OnException] = Args.OnException;
   SubStmts[CoroutineBodyStmt::OnFallthrough] = Args.OnFallthrough;
   SubStmts[CoroutineBodyStmt::Allocate] = Args.Allocate;
   SubStmts[CoroutineBodyStmt::Deallocate] = Args.Deallocate;
   SubStmts[CoroutineBodyStmt::ReturnValue] = Args.ReturnValue;
-  SubStmts[CoroutineBodyStmt::ResultDecl] = Args.ResultDecl;
   SubStmts[CoroutineBodyStmt::ReturnStmt] = Args.ReturnStmt;
   SubStmts[CoroutineBodyStmt::ReturnStmtOnAllocFailure] =
       Args.ReturnStmtOnAllocFailure;
   std::copy(Args.ParamMoves.begin(), Args.ParamMoves.end(),
             const_cast<Stmt **>(getParamMoves().data()));
 }

clang/lib/CodeGen/CGCoroutine.cpp

void Emit(CodeGenFunction &CGF, Flags) override {
     // No longer need old terminator.
     InsertPt->eraseFromParent();
     CGF.Builder.SetInsertPoint(AfterFreeBB);
   }
   explicit CallCoroDelete(Stmt *DeallocStmt) : Deallocate(DeallocStmt) {}
 };
 }
 
-namespace {
-struct GetReturnObjectManager {
-  CodeGenFunction &CGF;
-  CGBuilderTy &Builder;
-  const CoroutineBodyStmt &S;
-
-  Address GroActiveFlag;
-  CodeGenFunction::AutoVarEmission GroEmission;
-
-  GetReturnObjectManager(CodeGenFunction &CGF, const CoroutineBodyStmt &S)
-      : CGF(CGF), Builder(CGF.Builder), S(S), GroActiveFlag(Address::invalid()),
-        GroEmission(CodeGenFunction::AutoVarEmission::invalid()) {}
-
-  // The gro variable has to outlive coroutine frame and coroutine promise, but,
-  // it can only be initialized after coroutine promise was created, thus, we
-  // split its emission in two parts. EmitGroAlloca emits an alloca and sets up
-  // cleanups. Later when coroutine promise is available we initialize the gro
-  // and sets the flag that the cleanup is now active.
-
-  void EmitGroAlloca() {
-    auto *GroDeclStmt = dyn_cast<DeclStmt>(S.getResultDecl());
-    if (!GroDeclStmt) {
-      // If get_return_object returns void, no need to do an alloca.
-      return;
-    }
-
-    auto *GroVarDecl = cast<VarDecl>(GroDeclStmt->getSingleDecl());
-
-    // Set GRO flag that it is not initialized yet
-    GroActiveFlag =
-      CGF.CreateTempAlloca(Builder.getInt1Ty(), CharUnits::One(), "gro.active");
-    Builder.CreateStore(Builder.getFalse(), GroActiveFlag);
-
-    GroEmission = CGF.EmitAutoVarAlloca(*GroVarDecl);
-
-    // Remember the top of EHStack before emitting the cleanup.
-    auto old_top = CGF.EHStack.stable_begin();
-    CGF.EmitAutoVarCleanups(GroEmission);
-    auto top = CGF.EHStack.stable_begin();
-
-    // Make the cleanup conditional on gro.active
-    for (auto b = CGF.EHStack.find(top), e = CGF.EHStack.find(old_top);
-      b != e; b++) {
-      if (auto *Cleanup = dyn_cast<EHCleanupScope>(&*b)) {
-        assert(!Cleanup->hasActiveFlag() && "cleanup already has active flag?");
-        Cleanup->setActiveFlag(GroActiveFlag);
-        Cleanup->setTestFlagInEHCleanup();
-        Cleanup->setTestFlagInNormalCleanup();
-      }
-    }
-  }
-
-  void EmitGroInit() {
-    if (!GroActiveFlag.isValid()) {
-      // No Gro variable was allocated. Simply emit the call to
-      // get_return_object.
-      CGF.EmitStmt(S.getResultDecl());
-      return;
-    }
-
-    CGF.EmitAutoVarInit(GroEmission);
-    Builder.CreateStore(Builder.getTrue(), GroActiveFlag);
-  }
-};
-}
-
 static void emitBodyAndFallthrough(CodeGenFunction &CGF,
                                    const CoroutineBodyStmt &S, Stmt *Body) {
   CGF.EmitStmt(Body);
   const bool CanFallthrough = CGF.Builder.GetInsertBlock();
   if (CanFallthrough)
     if (Stmt *OnFallthrough = S.getFallthroughHandler())
       CGF.EmitStmt(OnFallthrough);
 }
void CodeGenFunction::EmitCoroutineBody(const CoroutineBodyStmt &S) {
   // Pass the result of the allocation to coro.begin.
   auto *Phi = Builder.CreatePHI(VoidPtrTy, 2);
   Phi->addIncoming(NullPtr, EntryBB);
   Phi->addIncoming(AllocateCall, AllocOrInvokeContBB);
   auto *CoroBegin = Builder.CreateCall(
       CGM.getIntrinsic(llvm::Intrinsic::coro_begin), {CoroId, Phi});
   CurCoro.Data->CoroBegin = CoroBegin;
 
-  // We need to emit `get_­return_­object` first. According to:
-  // [dcl.fct.def.coroutine]p7
-  // The call to get_­return_­object is sequenced before the call to
-  // initial_­suspend and is invoked at most once.
-  GetReturnObjectManager GroManager(*this, S);
-  GroManager.EmitGroAlloca();
-
   CurCoro.Data->CleanupJD = getJumpDestInCurrentScope(RetBB);
   {
     CGDebugInfo *DI = getDebugInfo();
     ParamReferenceReplacerRAII ParamReplacer(LocalDeclMap);
     CodeGenFunction::RunCleanupsScope ResumeScope(*this);
     EHStack.pushCleanup<CallCoroDelete>(NormalAndEHCleanup, S.getDeallocate());
 
     // Create mapping between parameters and copy-params for coroutine function.
CurCoro.Data->CleanupJD = getJumpDestInCurrentScope(RetBB);
 
     Address PromiseAddr = GetAddrOfLocalVar(S.getPromiseDecl());
     auto *PromiseAddrVoidPtr =
         new llvm::BitCastInst(PromiseAddr.getPointer(), VoidPtrTy, "", CoroId);
     // Update CoroId to refer to the promise. We could not do it earlier because
     // promise local variable was not emitted yet.
     CoroId->setArgOperand(1, PromiseAddrVoidPtr);
 
-    // Now we have the promise, initialize the GRO
-    GroManager.EmitGroInit();
+    // ReturnValue should be valid as long as the coroutine's return type
+    // is not void. The assertion could help us to reduce the check later.
+    assert(ReturnValue.isValid() == (bool)S.getReturnStmt());
+    // Now we have the promise, initialize the GRO.
+    // We need to emit `get_return_object` first. According to:
+    // [dcl.fct.def.coroutine]p7
+    // The call to get_return_­object is sequenced before the call to
+    // initial_suspend and is invoked at most once.
+    //
+    // So we couldn't emit return value when we emit return statment,
+    // otherwise the call to get_return_object wouldn't be in front
+    // of initial_suspend.
+    if (ReturnValue.isValid()) {
+      EmitAnyExprToMem(S.getReturnValue(), ReturnValue,
+                       S.getReturnValue()->getType().getQualifiers(),
+                       /*IsInit*/ true);
+    }
 
     EHStack.pushCleanup<CallCoroEnd>(EHCleanup);
 
     CurCoro.Data->CurrentAwaitKind = AwaitKind::Init;
     CurCoro.Data->ExceptionHandler = S.getExceptionHandler();
     EmitStmt(S.getInitSuspendStmt());
     CurCoro.Data->FinalJD = getJumpDestInCurrentScope(FinalBB);
 
void CodeGenFunction::EmitCoroutineBody(const CoroutineBodyStmt &S) {
   }
 
   EmitBlock(RetBB);
   // Emit coro.end before getReturnStmt (and parameter destructors), since
   // resume and destroy parts of the coroutine should not include them.
   llvm::Function *CoroEnd = CGM.getIntrinsic(llvm::Intrinsic::coro_end);
   Builder.CreateCall(CoroEnd, {NullPtr, Builder.getFalse()});
 
-  if (Stmt *Ret = S.getReturnStmt())
+  if (Stmt *Ret = S.getReturnStmt()) {
+    // Since we already emitted the return value above, so we shouldn't
+    // emit it again here.
+    cast<ReturnStmt>(Ret)->setRetValue(nullptr);
     EmitStmt(Ret);
+  }
 
   // LLVM require the frontend to add the function attribute. See
   // Coroutines.rst.
   CurFn->addFnAttr("coroutine.presplit", "0");
 }
 
 // Emit coroutine intrinsic and patch up arguments of the token type.
 RValue CodeGenFunction::EmitCoroutineIntrinsic(const CallExpr *E,

clang/lib/Sema/SemaCoroutine.cpp

assert(!FnRetType->isDependentType() &&
          "get_return_object type must no longer be dependent");
 
   if (FnRetType->isVoidType()) {
     ExprResult Res =
         S.ActOnFinishFullExpr(this->ReturnValue, Loc, /*DiscardedValue*/ false);
     if (Res.isInvalid())
       return false;
 
-    this->ResultDecl = Res.get();
     return true;
   }
 
   if (GroType->isVoidType()) {
     // Trigger a nice error message.
     InitializedEntity Entity =
         InitializedEntity::InitializeResult(Loc, FnRetType);
     S.PerformCopyInitialization(Entity, SourceLocation(), ReturnValue);
     noteMemberDeclaredHere(S, ReturnValue, Fn);
     return false;
   }
 
-  auto *GroDecl = VarDecl::Create(
-      S.Context, &FD, FD.getLocation(), FD.getLocation(),
-      &S.PP.getIdentifierTable().get("__coro_gro"), GroType,
-      S.Context.getTrivialTypeSourceInfo(GroType, Loc), SC_None);
-  GroDecl->setImplicit();
-
-  S.CheckVariableDeclarationType(GroDecl);
-  if (GroDecl->isInvalidDecl())
-    return false;
-
-  InitializedEntity Entity = InitializedEntity::InitializeVariable(GroDecl);
-  ExprResult Res =
-      S.PerformCopyInitialization(Entity, SourceLocation(), ReturnValue);
-  if (Res.isInvalid())
-    return false;
-
-  Res = S.ActOnFinishFullExpr(Res.get(), /*DiscardedValue*/ false);
-  if (Res.isInvalid())
-    return false;
-
-  S.AddInitializerToDecl(GroDecl, Res.get(),
-                         /*DirectInit=*/false);
-
-  S.FinalizeDeclaration(GroDecl);
-
-  // Form a declaration statement for the return declaration, so that AST
-  // visitors can more easily find it.
-  StmtResult GroDeclStmt =
-      S.ActOnDeclStmt(S.ConvertDeclToDeclGroup(GroDecl), Loc, Loc);
-  if (GroDeclStmt.isInvalid())
-    return false;
-
-  this->ResultDecl = GroDeclStmt.get();
-
-  ExprResult declRef = S.BuildDeclRefExpr(GroDecl, GroType, VK_LValue, Loc);
-  if (declRef.isInvalid())
-    return false;
-
-  StmtResult ReturnStmt = S.BuildReturnStmt(Loc, declRef.get());
+  StmtResult ReturnStmt = S.BuildReturnStmt(Loc, ReturnValue);
   if (ReturnStmt.isInvalid()) {
     noteMemberDeclaredHere(S, ReturnValue, Fn);
     return false;
   }
-  if (cast<clang::ReturnStmt>(ReturnStmt.get())->getNRVOCandidate() == GroDecl)
-    GroDecl->setNRVOVariable(true);
 
   this->ReturnStmt = ReturnStmt.get();
   return true;
 }
 
 // Create a static_cast\<T&&>(expr).
 static Expr *castForMoving(Sema &S, Expr *E, QualType T = QualType()) {
   if (T.isNull())

clang/lib/Sema/TreeTransform.h

if (AllocRes.isInvalid())
       return StmtError();
     Builder.Allocate = AllocRes.get();
 
     ExprResult DeallocRes = getDerived().TransformExpr(S->getDeallocate());
     if (DeallocRes.isInvalid())
       return StmtError();
     Builder.Deallocate = DeallocRes.get();
 
-    assert(S->getResultDecl() && "ResultDecl must already be built");
-    StmtResult ResultDecl = getDerived().TransformStmt(S->getResultDecl());
-    if (ResultDecl.isInvalid())
-      return StmtError();
-    Builder.ResultDecl = ResultDecl.get();
-
     if (auto *ReturnStmt = S->getReturnStmt()) {
       StmtResult Res = getDerived().TransformStmt(ReturnStmt);
       if (Res.isInvalid())
         return StmtError();
       Builder.ReturnStmt = Res.get();
     }
   }
 

clang/test/CodeGenCoroutines/coro-alloc-exp-namespace.cpp

struct promise_type {
     void return_void() {}
     static int get_return_object_on_allocation_failure() { return -1; }
   };
 };
 
 // CHECK-LABEL: f4(
 extern "C" int f4(promise_on_alloc_failure_tag) {
   // CHECK: %[[RetVal:.+]] = alloca i32
-  // CHECK: %[[Gro:.+]] = alloca i32
   // CHECK: %[[ID:.+]] = call token @llvm.coro.id(i32 16
   // CHECK: %[[SIZE:.+]] = call i64 @llvm.coro.size.i64()
   // CHECK: %[[MEM:.+]] = call noalias noundef i8* @_ZnwmRKSt9nothrow_t(i64 noundef %[[SIZE]], %"struct.std::nothrow_t"* noundef nonnull align 1 dereferenceable(1) @_ZStL7nothrow)
   // CHECK: %[[OK:.+]] = icmp ne i8* %[[MEM]], null
   // CHECK: br i1 %[[OK]], label %[[OKBB:.+]], label %[[ERRBB:.+]]
 
   // CHECK: [[ERRBB]]:
   // CHECK:   %[[FailRet:.+]] = call noundef i32 @_ZNSt12experimental16coroutine_traitsIJi28promise_on_alloc_failure_tagEE12promise_type39get_return_object_on_allocation_failureEv(
   // CHECK:   store i32 %[[FailRet]], i32* %[[RetVal]]
   // CHECK:   br label %[[RetBB:.+]]
 
   // CHECK: [[OKBB]]:
   // CHECK:   %[[OkRet:.+]] = call noundef i32 @_ZNSt12experimental16coroutine_traitsIJi28promise_on_alloc_failure_tagEE12promise_type17get_return_objectEv(
-  // CHECK:   store i32 %[[OkRet]], i32* %[[Gro]]
-
-  // CHECK: %[[Tmp1:.*]] = load i32, i32* %[[Gro]]
-  // CHECK-NEXT: store i32 %[[Tmp1]], i32* %[[RetVal]]
-  // CHECK-NEXT: %[[Gro_CAST:.+]] = bitcast i32* %[[Gro]] to i8*
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 4, i8* %[[Gro_CAST]]) #2
-  // CHECK-NEXT: br label %[[RetBB]]
 
   // CHECK: [[RetBB]]:
   // CHECK:   %[[LoadRet:.+]] = load i32, i32* %[[RetVal]], align 4
   // CHECK:   ret i32 %[[LoadRet]]
   co_return;
 }

clang/test/CodeGenCoroutines/coro-alloc.cpp

struct promise_type {
     void return_void() {}
     static int get_return_object_on_allocation_failure() { return -1; }
   };
 };
 
 // CHECK-LABEL: f4(
 extern "C" int f4(promise_on_alloc_failure_tag) {
   // CHECK: %[[RetVal:.+]] = alloca i32
-  // CHECK: %[[Gro:.+]] = alloca i32
   // CHECK: %[[ID:.+]] = call token @llvm.coro.id(i32 16
   // CHECK: %[[SIZE:.+]] = call i64 @llvm.coro.size.i64()
   // CHECK: %[[MEM:.+]] = call noalias noundef i8* @_ZnwmRKSt9nothrow_t(i64 noundef %[[SIZE]], %"struct.std::nothrow_t"* noundef nonnull align 1 dereferenceable(1) @_ZStL7nothrow)
   // CHECK: %[[OK:.+]] = icmp ne i8* %[[MEM]], null
   // CHECK: br i1 %[[OK]], label %[[OKBB:.+]], label %[[ERRBB:.+]]
 
   // CHECK: [[ERRBB]]:
   // CHECK:   %[[FailRet:.+]] = call noundef i32 @_ZNSt16coroutine_traitsIJi28promise_on_alloc_failure_tagEE12promise_type39get_return_object_on_allocation_failureEv(
   // CHECK:   store i32 %[[FailRet]], i32* %[[RetVal]]
   // CHECK:   br label %[[RetBB:.+]]
 
   // CHECK: [[OKBB]]:
   // CHECK:   %[[OkRet:.+]] = call noundef i32 @_ZNSt16coroutine_traitsIJi28promise_on_alloc_failure_tagEE12promise_type17get_return_objectEv(
-  // CHECK:   store i32 %[[OkRet]], i32* %[[Gro]]
-
-  // CHECK: %[[Tmp1:.*]] = load i32, i32* %[[Gro]]
-  // CHECK-NEXT: store i32 %[[Tmp1]], i32* %[[RetVal]]
-  // CHECK-NEXT: %[[Gro_CAST:.+]] = bitcast i32* %[[Gro]] to i8*
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 4, i8* %[[Gro_CAST]]) #2
-  // CHECK-NEXT: br label %[[RetBB]]
 
   // CHECK: [[RetBB]]:
   // CHECK:   %[[LoadRet:.+]] = load i32, i32* %[[RetVal]], align 4
   // CHECK:   ret i32 %[[LoadRet]]
   co_return;
 }

clang/test/CodeGenCoroutines/coro-gro-exp-namespace.cpp

 struct Cleanup {
   ~Cleanup();
 };
 void doSomething() noexcept;
 
 // CHECK: define{{.*}} i32 @_Z1fv(
 int f() {
   // CHECK: %[[RetVal:.+]] = alloca i32
-  // CHECK: %[[GroActive:.+]] = alloca i1
 
   // CHECK: %[[Size:.+]] = call i64 @llvm.coro.size.i64()
   // CHECK: call noalias noundef nonnull i8* @_Znwm(i64 noundef %[[Size]])
-  // CHECK: store i1 false, i1* %[[GroActive]]
   // CHECK: call void @_ZNSt12experimental16coroutine_traitsIJiEE12promise_typeC1Ev(
-  // CHECK: call void @_ZNSt12experimental16coroutine_traitsIJiEE12promise_type17get_return_objectEv(
-  // CHECK: store i1 true, i1* %[[GroActive]]
+  // CHECK: call void @_ZNSt12experimental16coroutine_traitsIJiEE12promise_type17get_return_objectEv(%struct.GroType* sret(%struct.GroType) align {{[0-9]+}} %[[GRO:.+]],
+  // CHECK: %[[Conv:.+]] = call noundef i32 @_ZN7GroTypecviEv({{.*}}[[GRO]]
+  // CHECK: store i32 %[[Conv]], i32* %[[RetVal]]
 
   Cleanup cleanup;
   doSomething();
   co_return;
 
   // CHECK: call void @_Z11doSomethingv(
   // CHECK: call void @_ZNSt12experimental16coroutine_traitsIJiEE12promise_type11return_voidEv(
   // CHECK: call void @_ZN7CleanupD1Ev(
 
   // Destroy promise and free the memory.
 
   // CHECK: call void @_ZNSt12experimental16coroutine_traitsIJiEE12promise_typeD1Ev(
   // CHECK: %[[Mem:.+]] = call i8* @llvm.coro.free(
   // CHECK: call void @_ZdlPv(i8* noundef %[[Mem]])
 
-  // Initialize retval from Gro and destroy Gro
-
-  // CHECK: %[[Conv:.+]] = call noundef i32 @_ZN7GroTypecviEv(
-  // CHECK: store i32 %[[Conv]], i32* %[[RetVal]]
-  // CHECK: %[[IsActive:.+]] = load i1, i1* %[[GroActive]]
-  // CHECK: br i1 %[[IsActive]], label %[[CleanupGro:.+]], label %[[Done:.+]]
-
-  // CHECK: [[CleanupGro]]:
-  // CHECK:   call void @_ZN7GroTypeD1Ev(
-  // CHECK:   br label %[[Done]]
-
-  // CHECK: [[Done]]:
+  // CHECK: coro.ret:
   // CHECK:   %[[LoadRet:.+]] = load i32, i32* %[[RetVal]]
   // CHECK:   ret i32 %[[LoadRet]]
 }

clang/test/CodeGenCoroutines/coro-gro.cpp

 };
 
 struct Cleanup { ~Cleanup(); };
 void doSomething() noexcept;
 
 // CHECK: define{{.*}} i32 @_Z1fv(
 int f() {
   // CHECK: %[[RetVal:.+]] = alloca i32
-  // CHECK: %[[GroActive:.+]] = alloca i1
 
   // CHECK: %[[Size:.+]] = call i64 @llvm.coro.size.i64()
   // CHECK: call noalias noundef nonnull i8* @_Znwm(i64 noundef %[[Size]])
-  // CHECK: store i1 false, i1* %[[GroActive]]
   // CHECK: call void @_ZNSt16coroutine_traitsIJiEE12promise_typeC1Ev(
-  // CHECK: call void @_ZNSt16coroutine_traitsIJiEE12promise_type17get_return_objectEv(
-  // CHECK: store i1 true, i1* %[[GroActive]]
+  // CHECK: call void @_ZNSt16coroutine_traitsIJiEE12promise_type17get_return_objectEv(%struct.GroType* sret(%struct.GroType) align {{[0-9]+}} %[[GRO:.+]],
+  // CHECK: %[[Conv:.+]] = call noundef i32 @_ZN7GroTypecviEv({{.*}}[[GRO]]
+  // CHECK: store i32 %[[Conv]], i32* %[[RetVal]]
 
   Cleanup cleanup;
   doSomething();
   co_return;
 
   // CHECK: call void @_Z11doSomethingv(
   // CHECK: call void @_ZNSt16coroutine_traitsIJiEE12promise_type11return_voidEv(
   // CHECK: call void @_ZN7CleanupD1Ev(
 
   // Destroy promise and free the memory.
 
   // CHECK: call void @_ZNSt16coroutine_traitsIJiEE12promise_typeD1Ev(
   // CHECK: %[[Mem:.+]] = call i8* @llvm.coro.free(
   // CHECK: call void @_ZdlPv(i8* noundef %[[Mem]])
 
-  // Initialize retval from Gro and destroy Gro
-
-  // CHECK: %[[Conv:.+]] = call noundef i32 @_ZN7GroTypecviEv(
-  // CHECK: store i32 %[[Conv]], i32* %[[RetVal]]
-  // CHECK: %[[IsActive:.+]] = load i1, i1* %[[GroActive]]
-  // CHECK: br i1 %[[IsActive]], label %[[CleanupGro:.+]], label %[[Done:.+]]
-
-  // CHECK: [[CleanupGro]]:
-  // CHECK:   call void @_ZN7GroTypeD1Ev(
-  // CHECK:   br label %[[Done]]
-
-  // CHECK: [[Done]]:
+  // CHECK: coro.ret:
   // CHECK:   %[[LoadRet:.+]] = load i32, i32* %[[RetVal]]
   // CHECK:   ret i32 %[[LoadRet]]
 }

clang/test/CodeGenCoroutines/coro-gro2-exp-namespace.cpp

 
 struct coro {
   using promise_type = promise_type<coro>;
   coro(coro const &);
   struct Impl;
   Impl *impl;
 };
 
-// Verify that the NRVO is applied to the Gro object.
+// Verify that the RVO is applied.
 // CHECK-LABEL: define{{.*}} void @_Z1fi(%struct.coro* noalias sret(%struct.coro) align 8 %agg.result, i32 noundef %0)
 coro f(int) {
   // CHECK: %call = call noalias noundef nonnull i8* @_Znwm(
   // CHECK-NEXT: br label %[[CoroInit:.*]]
 
   // CHECK: {{.*}}[[CoroInit]]:
-  // CHECK: store i1 false, i1* %gro.active
   // CHECK: call void @{{.*get_return_objectEv}}(%struct.coro* sret(%struct.coro) align 8 %agg.result
-  // CHECK-NEXT: store i1 true, i1* %gro.active
   co_return;
 }
 
 template <class RetObject>
 struct promise_type_with_on_alloc_failure {
   static RetObject get_return_object_on_allocation_failure();
   RetObject get_return_object();
   suspend_always initial_suspend();
coro_two h(int) {
   // CHECK-NEXT: %[[CheckNull:.*]] = icmp ne i8* %call, null
   // CHECK-NEXT: br i1 %[[CheckNull]], label %[[InitOnSuccess:.*]], label %[[InitOnFailure:.*]]
 
   // CHECK: {{.*}}[[InitOnFailure]]:
   // CHECK-NEXT: call void @{{.*get_return_object_on_allocation_failureEv}}(%struct.coro_two* sret(%struct.coro_two) align 8 %agg.result
   // CHECK-NEXT: br label %[[RetLabel:.*]]
 
   // CHECK: {{.*}}[[InitOnSuccess]]:
-  // CHECK: store i1 false, i1* %gro.active
   // CHECK: call void @{{.*get_return_objectEv}}(%struct.coro_two* sret(%struct.coro_two) align 8 %agg.result
-  // CHECK-NEXT: store i1 true, i1* %gro.active
 
   // CHECK: [[RetLabel]]:
   // CHECK-NEXT: ret void
   co_return;
 }

clang/test/CodeGenCoroutines/coro-gro2.cpp

 
 struct coro {
   using promise_type = promise_type<coro>;
   coro(coro const&);
   struct Impl;
   Impl *impl;
 };
 
-// Verify that the NRVO is applied to the Gro object.
+// Verify that the RVO is applied.
 // CHECK-LABEL: define{{.*}} void @_Z1fi(%struct.coro* noalias sret(%struct.coro) align 8 %agg.result, i32 noundef %0)
 coro f(int) {
 // CHECK: %call = call noalias noundef nonnull i8* @_Znwm(
 // CHECK-NEXT: br label %[[CoroInit:.*]]
 
 // CHECK: {{.*}}[[CoroInit]]:
-// CHECK: store i1 false, i1* %gro.active
 // CHECK: call void @{{.*get_return_objectEv}}(%struct.coro* sret(%struct.coro) align 8 %agg.result
-// CHECK-NEXT: store i1 true, i1* %gro.active
-  co_return;
 }
 
 
 template <class RetObject>
 struct promise_type_with_on_alloc_failure {
     static RetObject get_return_object_on_allocation_failure();
     RetObject get_return_object();
     suspend_always initial_suspend();
     suspend_never final_suspend() noexcept;
     void return_void();
     static void unhandled_exception();
 };
 
 struct coro_two {
   using promise_type = promise_type_with_on_alloc_failure<coro_two>;
   coro_two(coro_two const&);
   struct Impl;
   Impl *impl;
 };
 
-// Verify that the NRVO is applied to the Gro object.
+// Verify that the RVO is applied.
 // CHECK-LABEL: define{{.*}} void @_Z1hi(%struct.coro_two* noalias sret(%struct.coro_two) align 8 %agg.result, i32 noundef %0)
- coro_two h(int) {
 
   // CHECK: %call = call noalias noundef i8* @_ZnwmRKSt9nothrow_t
   // CHECK-NEXT: %[[CheckNull:.*]] = icmp ne i8* %call, null
   // CHECK-NEXT: br i1 %[[CheckNull]], label %[[InitOnSuccess:.*]], label %[[InitOnFailure:.*]]
 
   // CHECK: {{.*}}[[InitOnFailure]]:
   // CHECK-NEXT: call void @{{.*get_return_object_on_allocation_failureEv}}(%struct.coro_two* sret(%struct.coro_two) align 8 %agg.result
   // CHECK-NEXT: br label %[[RetLabel:.*]]
 
   // CHECK: {{.*}}[[InitOnSuccess]]:
-// CHECK: store i1 false, i1* %gro.active
   // CHECK: call void @{{.*get_return_objectEv}}(%struct.coro_two* sret(%struct.coro_two) align 8 %agg.result
-// CHECK-NEXT: store i1 true, i1* %gro.active
 
   // CHECK: [[RetLabel]]:
   // CHECK-NEXT: ret void
   co_return;
 }

clang/test/CodeGenCoroutines/coro-promise-dtor-exp-namespace.cpp

 
 coro_t f() {
   Cleanup cleanup;
   may_throw();
   co_return;
 }
 
 // CHECK-LABEL: define dso_local void @"?f@@YA?AUcoro_t@@XZ"(
-// CHECK:  %gro.active = alloca i1
-// CHECK:  store i1 false, i1* %gro.active
 
 // CHECK:  invoke noundef %"struct.coro_t::promise_type"* @"??0promise_type@coro_t@@QEAA@XZ"(
 // CHECK:  invoke void @"?get_return_object@promise_type@coro_t@@QEAA?AU2@XZ"(
-// CHECK:  store i1 true, i1* %gro.active
 
-// CHECK:  %[[IS_ACTIVE:.+]] = load i1, i1* %gro.active
-// CHECK:  br i1 %[[IS_ACTIVE]], label %[[CLEANUP1:.+]], label
-
-// CHECK: [[CLEANUP1]]:
-// CHECK:  %[[NRVO:.+]] = load i1, i1* %nrvo
-// CHECK:  br i1 %[[NRVO]], label %{{.+}}, label %[[DTOR:.+]]
-
-// CHECK: [[DTOR]]:
-// CHECK:  call void @"??1coro_t@@QEAA@XZ"(
+// CHECK:  call void @"??1promise_type@coro_t@@QEAA@XZ"

clang/test/CodeGenCoroutines/coro-promise-dtor.cpp

 
 coro_t f() {
   Cleanup cleanup;
   may_throw();
   co_return;
 }
 
 // CHECK-LABEL: define dso_local void @"?f@@YA?AUcoro_t@@XZ"(
-// CHECK:  %gro.active = alloca i1
-// CHECK:  store i1 false, i1* %gro.active
 
 // CHECK:  invoke noundef %"struct.coro_t::promise_type"* @"??0promise_type@coro_t@@QEAA@XZ"(
 // CHECK:  invoke void @"?get_return_object@promise_type@coro_t@@QEAA?AU2@XZ"(
-// CHECK:  store i1 true, i1* %gro.active
 
-// CHECK:  %[[IS_ACTIVE:.+]] = load i1, i1* %gro.active
-// CHECK:  br i1 %[[IS_ACTIVE]], label %[[CLEANUP1:.+]], label
-
-// CHECK: [[CLEANUP1]]:
-// CHECK:  %[[NRVO:.+]] = load i1, i1* %nrvo
-// CHECK:  br i1 %[[NRVO]], label %{{.+}}, label %[[DTOR:.+]]
-
-// CHECK: [[DTOR]]:
-// CHECK:  call void @"??1coro_t@@QEAA@XZ"(
+// CHECK:  call void @"??1promise_type@coro_t@@QEAA@XZ"

clang/test/SemaCXX/coroutine-no-move-ctor.cpp

+// RUN: %clang_cc1 -triple x86_64-apple-darwin9 %s -std=c++20 -fsyntax-only -verify
+// expected-no-diagnostics
+
+#include "Inputs/std-coroutine.h"
+
+class invoker {
+public:
+  class invoker_promise {
+  public:
+    invoker get_return_object() { return invoker{}; }
+    auto initial_suspend() { return std::suspend_never{}; }
+    auto final_suspend() noexcept { return std::suspend_never{}; }
+    void return_void() {}
+    void unhandled_exception() {}
+  };
+  using promise_type = invoker_promise;
+  invoker() {}
+  invoker(const invoker &) = delete;
+  invoker &operator=(const invoker &) = delete;
+  invoker(invoker &&) = delete;
+  invoker &operator=(invoker &&) = delete;
+};
+
+invoker f() {
+  co_return;
+}

clang/test/SemaCXX/coroutines-exp-namespace.cpp

struct promise_type {
     suspend_always final_suspend() noexcept { return {}; }
     void return_void() {}
     void unhandled_exception();
   };
 };
 
 extern "C" int f(mismatch_gro_type_tag2) {
   // cxx2b-error@-1 {{cannot initialize return object of type 'int' with an rvalue of type 'void *'}}
-  // cxx14_20-error@-2 {{cannot initialize return object of type 'int' with an lvalue of type 'void *'}}
+  // cxx14_20-error@-2 {{cannot initialize return object of type 'int' with an rvalue of type 'void *'}}
   co_return; //expected-note {{function is a coroutine due to use of 'co_return' here}}
 }
 
 struct mismatch_gro_type_tag3 {};
 template <>
 struct std::experimental::coroutine_traits<int, mismatch_gro_type_tag3> {
   struct promise_type {
     int get_return_object() {}

clang/test/SemaCXX/coroutines.cpp

struct promise_type {
     suspend_always final_suspend() noexcept { return {}; }
     void return_void() {}
     void unhandled_exception();
   };
 };
 
 extern "C" int f(mismatch_gro_type_tag2) {
   // cxx2b-error@-1 {{cannot initialize return object of type 'int' with an rvalue of type 'void *'}}
-  // cxx14_20-error@-2 {{cannot initialize return object of type 'int' with an lvalue of type 'void *'}}
+  // cxx14_20-error@-2 {{cannot initialize return object of type 'int' with an rvalue of type 'void *'}}
   co_return; //expected-note {{function is a coroutine due to use of 'co_return' here}}
 }
 
 struct mismatch_gro_type_tag3 {};
 template <>
 struct std::coroutine_traits<int, mismatch_gro_type_tag3> {
   struct promise_type {
     int get_return_object() {}