Index: llvm/docs/Coroutines.rst =================================================================== --- llvm/docs/Coroutines.rst +++ llvm/docs/Coroutines.rst @@ -1557,4 +1557,7 @@ #. Make required changes to make sure that coroutine optimizations work with LTO. +#. Make required changes to make sure not to insert or move any instructions + into edges from suspended(-1) case of coro.suspend intrinsics to their label. + #. More tests, more tests, more tests Index: llvm/lib/Transforms/Scalar/LICM.cpp =================================================================== --- llvm/lib/Transforms/Scalar/LICM.cpp +++ llvm/lib/Transforms/Scalar/LICM.cpp @@ -385,10 +385,40 @@ SmallVector ExitBlocks; L->getUniqueExitBlocks(ExitBlocks); - // We can't insert into a catchswitch. - bool HasCatchSwitch = llvm::any_of(ExitBlocks, [](BasicBlock *Exit) { - return isa(Exit->getTerminator()); - }); + bool HasCatchSwitch = false; + for (auto *BIter = ExitBlocks.end(); BIter != ExitBlocks.begin();) { + BasicBlock *ExitBB = *--BIter; + // We can't insert into a catchswitch. + HasCatchSwitch |= isa(ExitBB->getTerminator()); + + // Find exit edge from coroutine suspend block. + SmallSetVector SuspendBBs; + SmallSetVector PredBBs(pred_begin(ExitBB), + pred_end(ExitBB)); + for (auto *PredBB : PredBBs) { + + if (auto *SWI = dyn_cast(PredBB->getTerminator())) { + IntrinsicInst *II = dyn_cast(SWI->getCondition()); + if (II && (II->getIntrinsicID() == Intrinsic::coro_suspend) && + SWI->getDefaultDest() == ExitBB) + SuspendBBs.insert(PredBB); + } + } + + if (!SuspendBBs.empty()) { + // Split all of the predcessor blocks to keep dedicated form. + for (auto *PredBB : PredBBs) { + BasicBlock *NewPred = SplitBlockPredecessors( + ExitBB, PredBB, ".split.loop.exit", DT, LI, MSSAU.get(), true); + // Skip exit blocks comes from coroutine suspend block since we + // do not want sink anything into them. + if (!SuspendBBs.count(PredBB)) + ExitBlocks.push_back(NewPred); + } + + ExitBlocks.erase(BIter); + } + } if (!HasCatchSwitch) { SmallVector InsertPts; Index: llvm/test/Transforms/LICM/sink-with-coroutine.ll =================================================================== --- /dev/null +++ llvm/test/Transforms/LICM/sink-with-coroutine.ll @@ -0,0 +1,58 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -S < %s -licm | FileCheck %s + +; LICM across a @coro.suspend. + +define i64 @licm(i64 %n) #0 { +; CHECK-LABEL: @licm( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[P:%.*]] = alloca i64, align 8 +; CHECK-NEXT: br label [[BB0:%.*]] +; CHECK: bb0: +; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, [[BB0]] ], [ [[T5:%.*]], [[AWAIT_READY:%.*]] ] +; CHECK-NEXT: [[T5]] = add i64 [[I]], 1 +; CHECK-NEXT: [[SUSPEND:%.*]] = call i8 @llvm.coro.suspend(token none, i1 false) +; CHECK-NEXT: switch i8 [[SUSPEND]], label [[BB2_SPLIT_LOOP_EXIT1:%.*]] [ +; CHECK-NEXT: i8 0, label [[AWAIT_READY]] +; CHECK-NEXT: ] +; CHECK: await.ready: +; CHECK-NEXT: [[T6:%.*]] = icmp ult i64 [[T5]], [[N:%.*]] +; CHECK-NEXT: br i1 [[T6]], label [[LOOP]], label [[BB2_SPLIT_LOOP_EXIT:%.*]] +; CHECK: bb2.split.loop.exit: +; CHECK-NEXT: store i64 1, i64* [[P]], align 1 +; CHECK-NEXT: br label [[BB2:%.*]] +; CHECK: bb2.split.loop.exit1: +; CHECK-NEXT: br label [[BB2]] +; CHECK: bb2: +; CHECK-NEXT: [[RES:%.*]] = call i1 @llvm.coro.end(i8* null, i1 false) +; CHECK-NEXT: ret i64 0 +; +entry: + %p = alloca i64 + br label %bb0 + +bb0: + br label %loop + +loop: + %i = phi i64 [ 0, %bb0 ], [ %t5, %await.ready ] + %t5 = add i64 %i, 1 + %suspend = call i8 @llvm.coro.suspend(token none, i1 false) + switch i8 %suspend, label %bb2 [ + i8 0, label %await.ready + ] + +await.ready: + store i64 1, i64* %p + %t6 = icmp ult i64 %t5, %n + br i1 %t6, label %loop, label %bb2 + +bb2: + %res = call i1 @llvm.coro.end(i8* null, i1 false) + ret i64 0 +} + +declare i8 @llvm.coro.suspend(token, i1) +declare i1 @llvm.coro.end(i8*, i1)