diff --git a/clang/lib/CodeGen/CGCoroutine.cpp b/clang/lib/CodeGen/CGCoroutine.cpp --- a/clang/lib/CodeGen/CGCoroutine.cpp +++ b/clang/lib/CodeGen/CGCoroutine.cpp @@ -556,6 +556,8 @@ {Builder.getInt32(NewAlign), NullPtr, NullPtr, NullPtr}); createCoroData(*this, CurCoro, CoroId); CurCoro.Data->SuspendBB = RetBB; + assert(ShouldEmitLifetimeMarkers && + "Must emit lifetime intrinsics for coroutines"); // Backend is allowed to elide memory allocations, to help it, emit // auto mem = coro.alloc() ? 0 : ... allocation code ...; diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h --- a/clang/lib/CodeGen/CodeGenFunction.h +++ b/clang/lib/CodeGen/CodeGenFunction.h @@ -1886,8 +1886,9 @@ /// function attribute. unsigned LargestVectorWidth = 0; - /// True if we need emit the life-time markers. - const bool ShouldEmitLifetimeMarkers; + /// True if we need emit the life-time markers. This is initially set in + /// the constructor, but could be overwritten to true if this is a coroutine. + bool ShouldEmitLifetimeMarkers; /// Add OpenCL kernel arg metadata and the kernel attribute metadata to /// the function metadata. diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp --- a/clang/lib/CodeGen/CodeGenFunction.cpp +++ b/clang/lib/CodeGen/CodeGenFunction.cpp @@ -1318,10 +1318,16 @@ Stmt *Body = FD->getBody(); - // Initialize helper which will detect jumps which can cause invalid lifetime - // markers. - if (Body && ShouldEmitLifetimeMarkers) - Bypasses.Init(Body); + if (Body) { + // Coroutines always emit lifetime markers. + if (isa(Body)) + ShouldEmitLifetimeMarkers = true; + + // Initialize helper which will detect jumps which can cause invalid + // lifetime markers. + if (ShouldEmitLifetimeMarkers) + Bypasses.Init(Body); + } // Emit the standard function prologue. StartFunction(GD, ResTy, Fn, FnInfo, Args, Loc, BodyRange.getBegin()); diff --git a/clang/test/CodeGenCoroutines/coro-alloc.cpp b/clang/test/CodeGenCoroutines/coro-alloc.cpp --- a/clang/test/CodeGenCoroutines/coro-alloc.cpp +++ b/clang/test/CodeGenCoroutines/coro-alloc.cpp @@ -245,6 +245,8 @@ // CHECK: %[[Tmp1:.*]] = load i32, i32* %[[Gro]] // CHECK-NEXT: store i32 %[[Tmp1]], i32* %[[RetVal]] + // CHECK-NEXT: %[[Gro_CAST:.+]] = bitcast i32* %[[Gro]] to i8* + // CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 4, i8* %[[Gro_CAST]]) #2 // CHECK-NEXT: br label %[[RetBB]] // CHECK: [[RetBB]]: diff --git a/clang/test/CodeGenCoroutines/coro-await-resume-eh.cpp b/clang/test/CodeGenCoroutines/coro-await-resume-eh.cpp --- a/clang/test/CodeGenCoroutines/coro-await-resume-eh.cpp +++ b/clang/test/CodeGenCoroutines/coro-await-resume-eh.cpp @@ -57,12 +57,18 @@ // CHECK-NEXT: to label %[[RESUMEENDCATCHCONT:.+]] unwind label // CHECK: [[RESUMEENDCATCHCONT]]: // CHECK-NEXT: br label %[[RESUMETRYCONT]] + // CHECK: [[RESUMETRYCONT]]: + // CHECK-NEXT: br label %[[CLEANUP:.+]] + // CHECK: [[CLEANUP]]: + // CHECK: switch i32 %{{.+}}, label %{{.+}} [ + // CHECK-NEXT: i32 0, label %[[CLEANUPCONT:.+]] + // CHECK-NEXT: ] // The variable RESUMETHREW is loaded and if true, then 'await_resume' // threw an exception and the coroutine body is skipped, and the final // suspend is executed immediately. Otherwise, the coroutine body is // executed, and then the final suspend. - // CHECK: [[RESUMETRYCONT]]: + // CHECK: [[CLEANUPCONT]]: // CHECK-NEXT: %[[RESUMETHREWLOAD:.+]] = load i1, i1* %[[RESUMETHREW]] // CHECK-NEXT: br i1 %[[RESUMETHREWLOAD]], label %[[RESUMEDCONT:.+]], label %[[RESUMEDBODY:.+]] @@ -76,7 +82,7 @@ // CHECK-NEXT: br label %[[COROFINAL]] // CHECK: [[COROFINAL]]: - // CHECK-NEXT: call void @_ZN13throwing_task12promise_type13final_suspendEv + // CHECK: call void @_ZN13throwing_task12promise_type13final_suspendEv co_return; } diff --git a/clang/test/CodeGenCoroutines/coro-await.cpp b/clang/test/CodeGenCoroutines/coro-await.cpp --- a/clang/test/CodeGenCoroutines/coro-await.cpp +++ b/clang/test/CodeGenCoroutines/coro-await.cpp @@ -231,7 +231,9 @@ int Val = co_await ScalarAwaiter{}; // CHECK: %[[Result2:.+]] = call i32 @_ZN13ScalarAwaiter12await_resumeEv(%struct.ScalarAwaiter* - // CHECK: store i32 %[[Result2]], i32* %Val + // CHECK: store i32 %[[Result2]], i32* %[[TMP_EXPRCLEANUP:.+]], + // CHECK: %[[TMP:.+]] = load i32, i32* %[[TMP_EXPRCLEANUP]], + // CHECK: store i32 %[[TMP]], i32* %Val, co_await ScalarAwaiter{}; // CHECK: call i32 @_ZN13ScalarAwaiter12await_resumeEv(%struct.ScalarAwaiter* @@ -312,19 +314,25 @@ // CHECK: %[[YVAR:.+]] = alloca %struct.RefTag*, // CHECK-NEXT: %[[TMP1:.+]] = alloca %struct.AwaitResumeReturnsLValue, + // CHECK: %[[TMP_EXPRCLEANUP1:.+]] = alloca %struct.RefTag*, // CHECK: %[[ZVAR:.+]] = alloca %struct.RefTag*, // CHECK-NEXT: %[[TMP2:.+]] = alloca %struct.AwaitResumeReturnsLValue, + // CHECK: %[[TMP_EXPRCLEANUP2:.+]] = alloca %struct.RefTag*, // CHECK: %[[RES1:.+]] = call nonnull align 1 dereferenceable({{.*}}) %struct.RefTag* @_ZN24AwaitResumeReturnsLValue12await_resumeEv(%struct.AwaitResumeReturnsLValue* {{[^,]*}} %[[AVAR]]) // CHECK-NEXT: store %struct.RefTag* %[[RES1]], %struct.RefTag** %[[XVAR]], RefTag& x = co_await a; // CHECK: %[[RES2:.+]] = call nonnull align 1 dereferenceable({{.*}}) %struct.RefTag* @_ZN24AwaitResumeReturnsLValue12await_resumeEv(%struct.AwaitResumeReturnsLValue* {{[^,]*}} %[[TMP1]]) - // CHECK-NEXT: store %struct.RefTag* %[[RES2]], %struct.RefTag** %[[YVAR]], + // CHECK-NEXT: store %struct.RefTag* %[[RES2]], %struct.RefTag** %[[TMP_EXPRCLEANUP1]], + // CHECK: %[[LOAD_TMP1:.+]] = load %struct.RefTag*, %struct.RefTag** %[[TMP_EXPRCLEANUP1]], + // CHECK: store %struct.RefTag* %[[LOAD_TMP1]], %struct.RefTag** %[[YVAR]], RefTag& y = co_await AwaitResumeReturnsLValue{}; // CHECK: %[[RES3:.+]] = call nonnull align 1 dereferenceable({{.*}}) %struct.RefTag* @_ZN24AwaitResumeReturnsLValue12await_resumeEv(%struct.AwaitResumeReturnsLValue* {{[^,]*}} %[[TMP2]]) - // CHECK-NEXT: store %struct.RefTag* %[[RES3]], %struct.RefTag** %[[ZVAR]], + // CHECK-NEXT: store %struct.RefTag* %[[RES3]], %struct.RefTag** %[[TMP_EXPRCLEANUP2]], + // CHECK: %[[LOAD_TMP2:.+]] = load %struct.RefTag*, %struct.RefTag** %[[TMP_EXPRCLEANUP2]], + // CHECK: store %struct.RefTag* %[[LOAD_TMP2]], %struct.RefTag** %[[ZVAR]], RefTag& z = co_yield 42; } diff --git a/clang/test/CodeGenCoroutines/coro-dest-slot.cpp b/clang/test/CodeGenCoroutines/coro-dest-slot.cpp --- a/clang/test/CodeGenCoroutines/coro-dest-slot.cpp +++ b/clang/test/CodeGenCoroutines/coro-dest-slot.cpp @@ -17,10 +17,24 @@ extern "C" coro f(int) { co_return; } // Verify that cleanup.dest.slot is eliminated in a coroutine. // CHECK-LABEL: f( +// CHECK: %[[INIT_SUSPEND:.+]] = call i8 @llvm.coro.suspend( +// CHECK-NEXT: switch i8 %[[INIT_SUSPEND]], label +// CHECK-NEXT: i8 0, label %[[INIT_READY:.+]] +// CHECK-NEXT: i8 1, label %[[INIT_CLEANUP:.+]] +// CHECK-NEXT: ] +// CHECK: %[[CLEANUP_DEST0:.+]] = phi i32 [ 0, %[[INIT_READY]] ], [ 2, %[[INIT_CLEANUP]] ] + +// CHECK: %[[FINAL_SUSPEND:.+]] = call i8 @llvm.coro.suspend( +// CHECK-NEXT: switch i8 %29, label %coro.ret [ +// CHECK-NEXT: i8 0, label %[[FINAL_READY:.+]] +// CHECK-NEXT: i8 1, label %[[FINAL_CLEANUP:.+]] +// CHECK-NEXT: ] + // CHECK: call void @_ZNSt12experimental13coroutines_v113suspend_never12await_resumeEv( -// CHECK: %[[CLEANUP_DEST:.+]] = phi i32 [ 0, %{{.+}} ], [ 2, %{{.+}} ], [ 2, %{{.+}} ] +// CHECK: %[[CLEANUP_DEST1:.+]] = phi i32 [ 0, %[[FINAL_READY]] ], [ 2, %[[FINAL_CLEANUP]] ] +// CHECK: %[[CLEANUP_DEST2:.+]] = phi i32 [ %[[CLEANUP_DEST0]], %{{.+}} ], [ %[[CLEANUP_DEST1]], %{{.+}} ], [ 0, %{{.+}} ] // CHECK: call i8* @llvm.coro.free( -// CHECK: switch i32 %cleanup.dest.slot.0, label %{{.+}} [ +// CHECK: switch i32 %[[CLEANUP_DEST2]], label %{{.+}} [ // CHECK-NEXT: i32 0 // CHECK-NEXT: i32 2 // CHECK-NEXT: ] diff --git a/clang/test/CodeGenCoroutines/coro-params.cpp b/clang/test/CodeGenCoroutines/coro-params.cpp --- a/clang/test/CodeGenCoroutines/coro-params.cpp +++ b/clang/test/CodeGenCoroutines/coro-params.cpp @@ -70,7 +70,11 @@ // CHECK: call i8* @llvm.coro.begin( // CHECK: call void @_ZN8MoveOnlyC1EOS_(%struct.MoveOnly* {{[^,]*}} %[[MoCopy]], %struct.MoveOnly* nonnull align 4 dereferenceable(4) %[[MoParam]]) + // CHECK-NEXT: bitcast %struct.MoveAndCopy* %[[McCopy]] to i8* + // CHECK-NEXT: call void @llvm.lifetime.start.p0i8( // CHECK-NEXT: call void @_ZN11MoveAndCopyC1EOS_(%struct.MoveAndCopy* {{[^,]*}} %[[McCopy]], %struct.MoveAndCopy* nonnull align 4 dereferenceable(4) %[[McParam]]) # + // CHECK-NEXT: bitcast %"struct.std::experimental::coroutine_traits::promise_type"* %__promise to i8* + // CHECK-NEXT: call void @llvm.lifetime.start.p0i8( // CHECK-NEXT: invoke void @_ZNSt12experimental16coroutine_traitsIJvi8MoveOnly11MoveAndCopyEE12promise_typeC1Ev( // CHECK: call void @_ZN14suspend_always12await_resumeEv( @@ -89,9 +93,17 @@ // CHECK: call void @_ZN14suspend_always12await_resumeEv( // Destroy promise, then parameter copies: - // CHECK: call void @_ZNSt12experimental16coroutine_traitsIJvi8MoveOnly11MoveAndCopyEE12promise_typeD1Ev(%"struct.std::experimental::coroutine_traits::promise_type"* {{[^,]*}} %__promise) #2 + // CHECK: call void @_ZNSt12experimental16coroutine_traitsIJvi8MoveOnly11MoveAndCopyEE12promise_typeD1Ev(%"struct.std::experimental::coroutine_traits::promise_type"* {{[^,]*}} %__promise) + // CHECK-NEXT: bitcast %"struct.std::experimental::coroutine_traits::promise_type"* %__promise to i8* + // CHECK-NEXT: call void @llvm.lifetime.end.p0i8( // CHECK-NEXT: call void @_ZN11MoveAndCopyD1Ev(%struct.MoveAndCopy* {{[^,]*}} %[[McCopy]]) + // CHECK-NEXT: bitcast %struct.MoveAndCopy* %[[McCopy]] to i8* + // CHECK-NEXT: call void @llvm.lifetime.end.p0i8( // CHECK-NEXT: call void @_ZN8MoveOnlyD1Ev(%struct.MoveOnly* {{[^,]*}} %[[MoCopy]] + // CHECK-NEXT: bitcast %struct.MoveOnly* %[[MoCopy]] to i8* + // CHECK-NEXT: call void @llvm.lifetime.end.p0i8( + // CHECK-NEXT: bitcast i32* %{{.+}} to i8* + // CHECK-NEXT: call void @llvm.lifetime.end.p0i8( // CHECK-NEXT: call i8* @llvm.coro.free( } @@ -103,9 +115,17 @@ // CHECK-NEXT: %[[y_copy:.+]] = alloca %struct.B // CHECK: call i8* @llvm.coro.begin + // CHECK-NEXT: bitcast %struct.A* %[[x_copy]] to i8* + // CHECK-NEXT: call void @llvm.lifetime.start.p0i8( // CHECK-NEXT: call void @_ZN1AC1EOS_(%struct.A* {{[^,]*}} %[[x_copy]], %struct.A* nonnull align 4 dereferenceable(512) %x) + // CHECK-NEXT: bitcast %struct.B* %[[unnamed_copy]] to i8* + // CHECK-NEXT: call void @llvm.lifetime.start.p0i8( // CHECK-NEXT: call void @_ZN1BC1EOS_(%struct.B* {{[^,]*}} %[[unnamed_copy]], %struct.B* nonnull align 4 dereferenceable(512) %0) + // CHECK-NEXT: %10 = bitcast %struct.B* %[[y_copy]] to i8* + // CHECK-NEXT: call void @llvm.lifetime.start.p0i8( // CHECK-NEXT: call void @_ZN1BC1EOS_(%struct.B* {{[^,]*}} %[[y_copy]], %struct.B* nonnull align 4 dereferenceable(512) %y) + // CHECK-NEXT: bitcast %"struct.std::experimental::coroutine_traits::promise_type"* %__promise to i8* + // CHECK-NEXT: call void @llvm.lifetime.start.p0i8( // CHECK-NEXT: invoke void @_ZNSt12experimental16coroutine_traitsIJv1A1BS2_EE12promise_typeC1Ev( co_return; diff --git a/clang/test/CodeGenCoroutines/coro-symmetric-transfer-01.cpp b/clang/test/CodeGenCoroutines/coro-symmetric-transfer-01.cpp --- a/clang/test/CodeGenCoroutines/coro-symmetric-transfer-01.cpp +++ b/clang/test/CodeGenCoroutines/coro-symmetric-transfer-01.cpp @@ -1,4 +1,4 @@ -// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fcoroutines-ts -std=c++14 -O1 -emit-llvm %s -o - -disable-llvm-passes | FileCheck %s +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fcoroutines-ts -std=c++14 -O0 -emit-llvm %s -o - -disable-llvm-passes | FileCheck %s #include "Inputs/coroutine.h" @@ -50,8 +50,13 @@ // check that the lifetime of the coroutine handle used to obtain the address is contained within single basic block, and hence does not live across suspension points. // CHECK-LABEL: final.suspend: -// CHECK: %[[PTR1:.+]] = bitcast %"struct.std::experimental::coroutines_v1::coroutine_handle.0"* %[[ADDR_TMP:.+]] to i8* -// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 8, i8* %[[PTR1]]) -// CHECK: call i8* @{{.*address.*}}(%"struct.std::experimental::coroutines_v1::coroutine_handle.0"* {{[^,]*}} %[[ADDR_TMP]]) -// CHECK-NEXT: %[[PTR2:.+]] = bitcast %"struct.std::experimental::coroutines_v1::coroutine_handle.0"* %[[ADDR_TMP]] to i8* -// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 8, i8* %[[PTR2]]) +// CHECK: %{{.+}} = call token @llvm.coro.save(i8* null) +// CHECK: %[[HDL_CAST1:.+]] = bitcast %"struct.std::experimental::coroutines_v1::coroutine_handle.0"* %[[HDL:.+]] to i8* +// CHECK: call void @llvm.lifetime.start.p0i8(i64 8, i8* %[[HDL_CAST1]]) +// CHECK: %[[CALL:.+]] = call i8* @_ZN13detached_task12promise_type13final_awaiter13await_suspendENSt12experimental13coroutines_v116coroutine_handleIS0_EE( +// CHECK: %[[HDL_CAST2:.+]] = getelementptr inbounds %"struct.std::experimental::coroutines_v1::coroutine_handle.0", %"struct.std::experimental::coroutines_v1::coroutine_handle.0"* %[[HDL]], i32 0, i32 0 +// CHECK: store i8* %[[CALL]], i8** %[[HDL_CAST2]], align 8 +// CHECK: %[[HDL_TRANSFER:.+]] = call i8* @_ZNKSt12experimental13coroutines_v116coroutine_handleIvE7addressEv(%"struct.std::experimental::coroutines_v1::coroutine_handle.0"* nonnull dereferenceable(8) %[[HDL]]) +// CHECK: %[[HDL_CAST3:.+]] = bitcast %"struct.std::experimental::coroutines_v1::coroutine_handle.0"* %[[HDL]] to i8* +// CHECK: call void @llvm.lifetime.end.p0i8(i64 8, i8* %[[HDL_CAST3]]) +// CHECK: call void @llvm.coro.resume(i8* %[[HDL_TRANSFER]]) diff --git a/clang/test/CodeGenCoroutines/coro-unhandled-exception.cpp b/clang/test/CodeGenCoroutines/coro-unhandled-exception.cpp --- a/clang/test/CodeGenCoroutines/coro-unhandled-exception.cpp +++ b/clang/test/CodeGenCoroutines/coro-unhandled-exception.cpp @@ -50,6 +50,8 @@ // CHECK: [[TRYCONT]]: // CHECK-NEXT: br label %[[COROFIN:.+]] // CHECK: [[COROFIN]]: +// CHECK-NEXT: bitcast %"struct.std::experimental::coroutines_v1::suspend_never"* %{{.+}} to i8* +// CHECK-NEXT: call void @llvm.lifetime.start.p0i8( // CHECK-NEXT: call void @"?final_suspend@promise_type@coro_t@@QEAA?AUsuspend_never@coroutines_v1@experimental@std@@XZ"( // CHECK-LPAD: @_Z1fv( @@ -69,4 +71,6 @@ // CHECK-LPAD: [[TRYCONT]]: // CHECK-LPAD: br label %[[COROFIN:.+]] // CHECK-LPAD: [[COROFIN]]: +// CHECK-LPAD-NEXT: bitcast %"struct.std::experimental::coroutines_v1::suspend_never"* %{{.+}} to i8* +// CHECK-LPAD-NEXT: call void @llvm.lifetime.start.p0i8( // CHECK-LPAD-NEXT: call void @_ZN6coro_t12promise_type13final_suspendEv(