[coroutines] Replace all coro.frame builtins with an SSA value of coro.begin

[GorNishanov]

Summary

SemaCoroutine forms expressions referring to the coroutine frame of the enclosing coroutine using coro.frame builtin.
During codegen, we emit llvm.coro.begin intrinsic that returns the address of the coroutine frame.
When coro.frame is emitted, we replace it with SSA value of coro.begin.

[GorNishanov]

gentle ping

[GorNishanov]

merged with top of the trunk. preparing to land

[GorNishanov]

Committed: r303598 = cd8cc3ea48a887f846b3cbfc6dcb546a7b4de3af

lib/CodeGen/CGCoroutine.cpp

struct clang::CodeGen::CGCoroData {
   CodeGenFunction::JumpDest FinalJD;
 
   // Stores the llvm.coro.id emitted in the function so that we can supply it
   // as the first argument to coro.begin, coro.alloc and coro.free intrinsics.
   // Note: llvm.coro.id returns a token that cannot be directly expressed in a
   // builtin.
   llvm::CallInst *CoroId = nullptr;
 
+  // Stores the llvm.coro.begin emitted in the function so that we can replace
+  // all coro.frame intrinsics with direct SSA value of coro.begin that returns
+  // the address of the coroutine frame of the current coroutine.
+  llvm::CallInst *CoroBegin = nullptr;
+
   // If coro.id came from the builtin, remember the expression to give better
   // diagnostic. If CoroIdExpr is nullptr, the coro.id was created by
   // EmitCoroutineBody.
   CallExpr const *CoroIdExpr = nullptr;
 };
 
 // Defining these here allows to keep CGCoroData private to this file.
 clang::CodeGen::CodeGenFunction::CGCoroInfo::CGCoroInfo() {}
void CodeGenFunction::EmitCoroutineBody(const CoroutineBodyStmt &S) {
   }
 
   EmitBlock(InitBB);
 
   // Pass the result of the allocation to coro.begin.
   auto *Phi = Builder.CreatePHI(VoidPtrTy, 2);
   Phi->addIncoming(NullPtr, EntryBB);
   Phi->addIncoming(AllocateCall, AllocOrInvokeContBB);
-  Builder.CreateCall(
+  auto *CoroBegin = Builder.CreateCall(
       CGM.getIntrinsic(llvm::Intrinsic::coro_begin), {CoroId, Phi});
+  CurCoro.Data->CoroBegin = CoroBegin;
 
   CurCoro.Data->CleanupJD = getJumpDestInCurrentScope(RetBB);
   {
     CodeGenFunction::RunCleanupsScope ResumeScope(*this);
     EHStack.pushCleanup<CallCoroDelete>(NormalAndEHCleanup, S.getDeallocate());
 
     EmitStmt(S.getPromiseDeclStmt());
 

 
 // Emit coroutine intrinsic and patch up arguments of the token type.
 RValue CodeGenFunction::EmitCoroutineIntrinsic(const CallExpr *E,
                                                unsigned int IID) {
   SmallVector<llvm::Value *, 8> Args;
   switch (IID) {
   default:
     break;
+  // The coro.frame builtin is replaced with an SSA value of the coro.begin
+  // intrinsic.
+  case llvm::Intrinsic::coro_frame: {
+    if (CurCoro.Data && CurCoro.Data->CoroBegin) {
+      return RValue::get(CurCoro.Data->CoroBegin);
+    }
+    CGM.Error(E->getLocStart(), "this builtin expect that __builtin_coro_begin "
+      "has been used earlier in this function");
+    auto NullPtr = llvm::ConstantPointerNull::get(Builder.getInt8PtrTy());
+    return RValue::get(NullPtr);
+  }
   // The following three intrinsics take a token parameter referring to a token
   // returned by earlier call to @llvm.coro.id. Since we cannot represent it in
   // builtins, we patch it up here.
   case llvm::Intrinsic::coro_alloc:
   case llvm::Intrinsic::coro_begin:
   case llvm::Intrinsic::coro_free: {
     if (CurCoro.Data && CurCoro.Data->CoroId) {
       Args.push_back(CurCoro.Data->CoroId);
case llvm::Intrinsic::coro_suspend:
     break;
   }
   for (auto &Arg : E->arguments())
     Args.push_back(EmitScalarExpr(Arg));
 
   llvm::Value *F = CGM.getIntrinsic(IID);
   llvm::CallInst *Call = Builder.CreateCall(F, Args);
 
+  // Note: The following code is to enable to emit coroutine intrinsics by
+  // hand to experiment with coroutines in C.
   // If we see @llvm.coro.id remember it in the CoroData. We will update
   // coro.alloc, coro.begin and coro.free intrinsics to refer to it.
   if (IID == llvm::Intrinsic::coro_id) {
     createCoroData(*this, CurCoro, Call, E);
   }
+  else if (IID == llvm::Intrinsic::coro_begin) {
+    if (CurCoro.Data)
+      CurCoro.Data->CoroBegin = Call;
+  }
   return RValue::get(Call);
 }

test/CodeGenCoroutines/coro-alloc.cpp

extern "C" void f0(global_new_delete_tag) {
 
   // CHECK: [[AllocBB]]:
   // CHECK: %[[SIZE:.+]] = call i64 @llvm.coro.size.i64()
   // CHECK: %[[MEM:.+]] = call i8* @_Znwm(i64 %[[SIZE]])
   // CHECK: br label %[[InitBB]]
 
   // CHECK: [[InitBB]]:
   // CHECK: %[[PHI:.+]] = phi i8* [ null, %{{.+}} ], [ %call, %[[AllocBB]] ]
-  // CHECK: call i8* @llvm.coro.begin(token %[[ID]], i8* %[[PHI]])
+  // CHECK: %[[FRAME:.+]] = call i8* @llvm.coro.begin(token %[[ID]], i8* %[[PHI]])
 
-  // CHECK: %[[FRAME:.+]] = call i8* @llvm.coro.frame()
   // CHECK: %[[MEM:.+]] = call i8* @llvm.coro.free(token %[[ID]], i8* %[[FRAME]])
   // CHECK: call void @_ZdlPv(i8* %[[MEM]])
   co_return;
 }
 
 struct promise_new_tag {};
 
 template<>
 struct std::experimental::coroutine_traits<void, promise_new_tag> {
   struct promise_type {
     void *operator new(unsigned long);
     void get_return_object() {}
     suspend_always initial_suspend() { return {}; }
     suspend_always final_suspend() { return {}; }
     void return_void() {}
   };
 };
 
 // CHECK-LABEL: f1(
 extern "C" void f1(promise_new_tag ) {
   // CHECK: %[[ID:.+]] = call token @llvm.coro.id(i32 16
   // CHECK: %[[SIZE:.+]] = call i64 @llvm.coro.size.i64()
   // CHECK: call i8* @_ZNSt12experimental16coroutine_traitsIJv15promise_new_tagEE12promise_typenwEm(i64 %[[SIZE]])
 
-  // CHECK: %[[FRAME:.+]] = call i8* @llvm.coro.frame()
+  // CHECK: %[[FRAME:.+]] = call i8* @llvm.coro.begin(
   // CHECK: %[[MEM:.+]] = call i8* @llvm.coro.free(token %[[ID]], i8* %[[FRAME]])
   // CHECK: call void @_ZdlPv(i8* %[[MEM]])
   co_return;
 }
 
 struct promise_delete_tag {};
 
 template<>
 struct std::experimental::coroutine_traits<void, promise_delete_tag> {
   struct promise_type {
     void operator delete(void*);
     void get_return_object() {}
     suspend_always initial_suspend() { return {}; }
     suspend_always final_suspend() { return {}; }
     void return_void() {}
   };
 };
 
 // CHECK-LABEL: f2(
 extern "C" void f2(promise_delete_tag) {
   // CHECK: %[[ID:.+]] = call token @llvm.coro.id(i32 16
   // CHECK: %[[SIZE:.+]] = call i64 @llvm.coro.size.i64()
   // CHECK: call i8* @_Znwm(i64 %[[SIZE]])
 
-  // CHECK: %[[FRAME:.+]] = call i8* @llvm.coro.frame()
+  // CHECK: %[[FRAME:.+]] = call i8* @llvm.coro.begin(
   // CHECK: %[[MEM:.+]] = call i8* @llvm.coro.free(token %[[ID]], i8* %[[FRAME]])
   // CHECK: call void @_ZNSt12experimental16coroutine_traitsIJv18promise_delete_tagEE12promise_typedlEPv(i8* %[[MEM]])
   co_return;
 }
 
 struct promise_sized_delete_tag {};
 
 template<>
 struct std::experimental::coroutine_traits<void, promise_sized_delete_tag> {
   struct promise_type {
     void operator delete(void*, unsigned long);
     void get_return_object() {}
     suspend_always initial_suspend() { return {}; }
     suspend_always final_suspend() { return {}; }
     void return_void() {}
   };
 };
 
 // CHECK-LABEL: f3(
 extern "C" void f3(promise_sized_delete_tag) {
   // CHECK: %[[ID:.+]] = call token @llvm.coro.id(i32 16
   // CHECK: %[[SIZE:.+]] = call i64 @llvm.coro.size.i64()
   // CHECK: call i8* @_Znwm(i64 %[[SIZE]])
 
-  // CHECK: %[[FRAME:.+]] = call i8* @llvm.coro.frame()
+  // CHECK: %[[FRAME:.+]] = call i8* @llvm.coro.begin(
   // CHECK: %[[MEM:.+]] = call i8* @llvm.coro.free(token %[[ID]], i8* %[[FRAME]])
   // CHECK: %[[SIZE2:.+]] = call i64 @llvm.coro.size.i64()
   // CHECK: call void @_ZNSt12experimental16coroutine_traitsIJv24promise_sized_delete_tagEE12promise_typedlEPvm(i8* %[[MEM]], i64 %[[SIZE2]])
   co_return;
 }
 
 struct promise_on_alloc_failure_tag {};
 

test/CodeGenCoroutines/coro-await.cpp

struct promise_type {
     suspend_always initial_suspend();
     suspend_always final_suspend();
     void return_void();
   };
 };
 
 // CHECK-LABEL: f0(
 extern "C" void f0() {
+  // CHECK: %[[FRAME:.+]] = call i8* @llvm.coro.begin(
 
   co_await suspend_always{};
   // See if we need to suspend:
   // --------------------------
   // CHECK: %[[READY:.+]] = call zeroext i1 @_ZN14suspend_always11await_readyEv(%struct.suspend_always* %[[AWAITABLE:.+]])
   // CHECK: br i1 %[[READY]], label %[[READY_BB:.+]], label %[[SUSPEND_BB:.+]]
 
   // If we are suspending:
   // ---------------------
   // CHECK: [[SUSPEND_BB]]:
   // CHECK: %[[SUSPEND_ID:.+]] = call token @llvm.coro.save(
   // ---------------------------
   // Build the coroutine handle and pass it to await_suspend
   // ---------------------------
-  // CHECK: %[[FRAME:.+]] = call i8* @llvm.coro.frame()
   // CHECK: call i8* @_ZNSt12experimental16coroutine_handleINS_16coroutine_traitsIJvEE12promise_typeEE12from_addressEPv(i8* %[[FRAME]])
   //   ... many lines of code to coerce coroutine_handle into an i8* scalar
   // CHECK: %[[CH:.+]] = load i8*, i8** %{{.+}}
   // CHECK: call void @_ZN14suspend_always13await_suspendENSt12experimental16coroutine_handleIvEE(%struct.suspend_always* %[[AWAITABLE]], i8* %[[CH]])
   // -------------------------
   // Generate a suspend point:
   // -------------------------
   // CHECK: %[[OUTCOME:.+]] = call i8 @llvm.coro.suspend(token %[[SUSPEND_ID]], i1 false)
struct promise_type {
     suspend_always final_suspend();
     void return_void();
     suspend_maybe yield_value(int);
   };
 };
 
 // CHECK-LABEL: f1(
 extern "C" void f1(int) {
-  co_yield 42;
   // CHECK: %[[PROMISE:.+]] = alloca %"struct.std::experimental::coroutine_traits<void, int>::promise_type"
+  // CHECK: %[[FRAME:.+]] = call i8* @llvm.coro.begin(
+  co_yield 42;
   // CHECK: call void @_ZNSt12experimental16coroutine_traitsIJviEE12promise_type11yield_valueEi(%struct.suspend_maybe* sret %[[AWAITER:.+]], %"struct.std::experimental::coroutine_traits<void, int>::promise_type"* %[[PROMISE]], i32 42)
 
   // See if we need to suspend:
   // --------------------------
   // CHECK: %[[READY:.+]] = call zeroext i1 @_ZN13suspend_maybe11await_readyEv(%struct.suspend_maybe* %[[AWAITABLE]])
   // CHECK: br i1 %[[READY]], label %[[READY_BB:.+]], label %[[SUSPEND_BB:.+]]
 
   // If we are suspending:
   // ---------------------
   // CHECK: [[SUSPEND_BB]]:
   // CHECK: %[[SUSPEND_ID:.+]] = call token @llvm.coro.save(
   // ---------------------------
   // Build the coroutine handle and pass it to await_suspend
   // ---------------------------
-  // CHECK: %[[FRAME:.+]] = call i8* @llvm.coro.frame()
   // CHECK: call i8* @_ZNSt12experimental16coroutine_handleINS_16coroutine_traitsIJviEE12promise_typeEE12from_addressEPv(i8* %[[FRAME]])
   //   ... many lines of code to coerce coroutine_handle into an i8* scalar
   // CHECK: %[[CH:.+]] = load i8*, i8** %{{.+}}
   // CHECK: %[[YES:.+]] = call zeroext i1 @_ZN13suspend_maybe13await_suspendENSt12experimental16coroutine_handleIvEE(%struct.suspend_maybe* %[[AWAITABLE]], i8* %[[CH]])
   // -------------------------------------------
   // See if await_suspend decided not to suspend
   // -------------------------------------------
   // CHECK: br i1 %[[YES]], label %[[SUSPEND_PLEASE:.+]], label %[[READY_BB]]

test/CodeGenCoroutines/coro-builtins.c

 // RUN: %clang_cc1 -triple x86_64-pc-windows-msvc18.0.0 -fcoroutines-ts -emit-llvm %s -o - -disable-llvm-passes | FileCheck %s
 
 void *myAlloc(long long);
 
-// CHECK-LABEL: f( 
+// CHECK-LABEL: f(
 void f(int n) {
   // CHECK: %n.addr = alloca i32
   // CHECK: %n_copy = alloca i32
   // CHECK: %promise = alloca i32
   int n_copy;
   int promise;
 
   // CHECK: %[[PROM_ADDR:.+]] = bitcast i32* %promise to i8*
   // CHECK-NEXT: %[[COROID:.+]] = call token @llvm.coro.id(i32 32, i8* %[[PROM_ADDR]], i8* null, i8* null)
   __builtin_coro_id(32, &promise, 0, 0);
 
   // CHECK-NEXT: call i1 @llvm.coro.alloc(token %[[COROID]])
   __builtin_coro_alloc();
 
   // CHECK-NEXT: %[[SIZE:.+]] = call i64 @llvm.coro.size.i64()
   // CHECK-NEXT: %[[MEM:.+]] = call i8* @myAlloc(i64 %[[SIZE]])
-  // CHECK-NEXT: %[[BEG:.+]] = call i8* @llvm.coro.begin(token %[[COROID]], i8* %[[MEM]])
+  // CHECK-NEXT: %[[FRAME:.+]] = call i8* @llvm.coro.begin(token %[[COROID]], i8* %[[MEM]])
   __builtin_coro_begin(myAlloc(__builtin_coro_size()));
 
-  // CHECK-NEXT: %[[FRAME1:.+]] = call i8* @llvm.coro.frame() 
-  // CHECK-NEXT: call void @llvm.coro.resume(i8* %[[FRAME1]])
+  // CHECK-NEXT: call void @llvm.coro.resume(i8* %[[FRAME]])
   __builtin_coro_resume(__builtin_coro_frame());
 
-  // CHECK-NEXT: %[[FRAME2:.+]] = call i8* @llvm.coro.frame() 
-  // CHECK-NEXT: call void @llvm.coro.destroy(i8* %[[FRAME2]])
+  // CHECK-NEXT: call void @llvm.coro.destroy(i8* %[[FRAME]])
   __builtin_coro_destroy(__builtin_coro_frame());
 
-  // CHECK-NEXT: %[[FRAME3:.+]] = call i8* @llvm.coro.frame() 
-  // CHECK-NEXT: call i1 @llvm.coro.done(i8* %[[FRAME3]])
+  // CHECK-NEXT: call i1 @llvm.coro.done(i8* %[[FRAME]])
   __builtin_coro_done(__builtin_coro_frame());
 
-  // CHECK-NEXT: %[[FRAME4:.+]] = call i8* @llvm.coro.frame() 
-  // CHECK-NEXT: call i8* @llvm.coro.promise(i8* %[[FRAME4]], i32 48, i1 false)
+  // CHECK-NEXT: call i8* @llvm.coro.promise(i8* %[[FRAME]], i32 48, i1 false)
   __builtin_coro_promise(__builtin_coro_frame(), 48, 0);
 
-  // CHECK-NEXT: %[[FRAME5:.+]] = call i8* @llvm.coro.frame() 
-  // CHECK-NEXT: call i8* @llvm.coro.free(token %[[COROID]], i8* %[[FRAME5]])
+  // CHECK-NEXT: call i8* @llvm.coro.free(token %[[COROID]], i8* %[[FRAME]])
   __builtin_coro_free(__builtin_coro_frame());
 
-  // CHECK-NEXT: %[[FRAME6:.+]] = call i8* @llvm.coro.frame() 
-  // CHECK-NEXT: call i1 @llvm.coro.end(i8* %[[FRAME6]], i1 false)
+  // CHECK-NEXT: call i1 @llvm.coro.end(i8* %[[FRAME]], i1 false)
   __builtin_coro_end(__builtin_coro_frame(), 0);
 
   // CHECK-NEXT: call i8 @llvm.coro.suspend(token none, i1 true)
   __builtin_coro_suspend(1);
 
   // CHECK-NEXT: %[[N_ADDR:.+]] = bitcast i32* %n.addr to i8*
   // CHECK-NEXT: %[[N_COPY_ADDR:.+]] = bitcast i32* %n_copy to i8*
   // CHECK-NEXT: call i1 @llvm.coro.param(i8* %[[N_ADDR]], i8* %[[N_COPY_ADDR]])
   __builtin_coro_param(&n, &n_copy);
 }