diff --git a/llvm/lib/Transforms/Utils/LoopVersioning.cpp b/llvm/lib/Transforms/Utils/LoopVersioning.cpp --- a/llvm/lib/Transforms/Utils/LoopVersioning.cpp +++ b/llvm/lib/Transforms/Utils/LoopVersioning.cpp @@ -117,6 +117,11 @@ // Adds the necessary PHI nodes for the versioned loops based on the // loop-defined values used outside of the loop. addPHINodes(DefsUsedOutside); + formDedicatedExitBlocks(NonVersionedLoop, DT, LI, nullptr, true); + formDedicatedExitBlocks(VersionedLoop, DT, LI, nullptr, true); + assert(NonVersionedLoop->isLoopSimplifyForm() && + VersionedLoop->isLoopSimplifyForm() && + "The versioned loops should be in simplify form."); } void LoopVersioning::addPHINodes( diff --git a/llvm/test/Transforms/LoopDistribute/basic-with-memchecks.ll b/llvm/test/Transforms/LoopDistribute/basic-with-memchecks.ll --- a/llvm/test/Transforms/LoopDistribute/basic-with-memchecks.ll +++ b/llvm/test/Transforms/LoopDistribute/basic-with-memchecks.ll @@ -61,7 +61,7 @@ ; CHECK: for.body.ph.lver.orig: ; CHECK: br label %for.body.lver.orig ; CHECK: for.body.lver.orig: -; CHECK: br i1 %exitcond.lver.orig, label %for.end, label %for.body.lver.orig +; CHECK: br i1 %exitcond.lver.orig, label %for.end.loopexit, label %for.body.lver.orig ; Verify the two distributed loops. diff --git a/llvm/test/Transforms/LoopDistribute/followup.ll b/llvm/test/Transforms/LoopDistribute/followup.ll --- a/llvm/test/Transforms/LoopDistribute/followup.ll +++ b/llvm/test/Transforms/LoopDistribute/followup.ll @@ -51,11 +51,15 @@ ; CHECK-LABEL: for.body.lver.orig: -; CHECK: br i1 %exitcond.lver.orig, label %for.end, label %for.body.lver.orig, !llvm.loop ![[LOOP_ORIG:[0-9]+]] +; CHECK: br i1 %exitcond.lver.orig, label %for.end.loopexit, label %for.body.lver.orig, !llvm.loop ![[LOOP_ORIG:[0-9]+]] ; CHECK-LABEL: for.body.ldist1: ; CHECK: br i1 %exitcond.ldist1, label %for.body.ph, label %for.body.ldist1, !llvm.loop ![[LOOP_SEQUENTIAL:[0-9]+]] ; CHECK-LABEL: for.body: -; CHECK: br i1 %exitcond, label %for.end, label %for.body, !llvm.loop ![[LOOP_COINCIDENT:[0-9]+]] +; CHECK: br i1 %exitcond, label %for.end.loopexit26, label %for.body, !llvm.loop ![[LOOP_COINCIDENT:[0-9]+]] +; CHECK-LABEL: for.end.loopexit: +; CHECK: br label %for.end +; CHECK-LABEL: for.end.loopexit26: +; CHECK: br label %for.end ; CHECK: ![[LOOP_ORIG]] = distinct !{![[LOOP_ORIG]], ![[FOLLOWUP_ALL:[0-9]+]], ![[FOLLOUP_FALLBACK:[0-9]+]]} ; CHECK: ![[FOLLOWUP_ALL]] = !{!"FollowupAll"} diff --git a/llvm/test/Transforms/LoopDistribute/outside-use.ll b/llvm/test/Transforms/LoopDistribute/outside-use.ll --- a/llvm/test/Transforms/LoopDistribute/outside-use.ll +++ b/llvm/test/Transforms/LoopDistribute/outside-use.ll @@ -36,8 +36,12 @@ ; CHECK: for.body.ph: ; CHECK: for.body: ; CHECK: %sum_add = add nuw nsw i32 %sum, %loadC +; CHECK: for.end.loopexit: +; CHECK: %sum_add.lver.ph = phi i32 [ %sum_add.lver.orig, %for.body.lver.orig ] +; CHECK: for.end.loopexit6: +; CHECK: %sum_add.lver.ph7 = phi i32 [ %sum_add, %for.body ] ; CHECK: for.end: -; CHECK: %sum_add.lver = phi i32 [ %sum_add, %for.body ], [ %sum_add.lver.orig, %for.body.lver.orig ] +; CHECK: %sum_add.lver = phi i32 [ %sum_add.lver.ph, %for.end.loopexit ], [ %sum_add.lver.ph7, %for.end.loopexit6 ] for.body: ; preds = %for.body, %entry %ind = phi i64 [ 0, %entry ], [ %add, %for.body ] diff --git a/llvm/test/Transforms/LoopDistribute/scev-inserted-runtime-check.ll b/llvm/test/Transforms/LoopDistribute/scev-inserted-runtime-check.ll --- a/llvm/test/Transforms/LoopDistribute/scev-inserted-runtime-check.ll +++ b/llvm/test/Transforms/LoopDistribute/scev-inserted-runtime-check.ll @@ -61,7 +61,7 @@ ; CHECK-NEXT: [[ARRAYIDXC_LVER_ORIG:%.*]] = getelementptr inbounds i32, i32* [[C:%.*]], i64 [[MUL_EXT_LVER_ORIG]] ; CHECK-NEXT: store i32 [[MULC_LVER_ORIG]], i32* [[ARRAYIDXC_LVER_ORIG]], align 4 ; CHECK-NEXT: [[EXITCOND_LVER_ORIG:%.*]] = icmp eq i64 [[ADD_LVER_ORIG]], [[N]] -; CHECK-NEXT: br i1 [[EXITCOND_LVER_ORIG]], label [[FOR_END:%.*]], label [[FOR_BODY_LVER_ORIG]] +; CHECK-NEXT: br i1 [[EXITCOND_LVER_ORIG]], label %[[FOR_END1:.*]], label [[FOR_BODY_LVER_ORIG]] ; CHECK: for.body.ph.ldist1: ; CHECK-NEXT: br label [[FOR_BODY_LDIST1:%.*]] ; CHECK: for.body.ldist1: @@ -97,7 +97,11 @@ ; CHECK-NEXT: [[ARRAYIDXC:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[MUL_EXT]] ; CHECK-NEXT: store i32 [[MULC]], i32* [[ARRAYIDXC]], align 4 ; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[ADD]], [[N]] -; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]] +; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END2:.*]], label [[FOR_BODY]] +; CHECK: [[FOR_END1]]: +; CHECK: br label %for.end +; CHECK: [[FOR_END2]]: +; CHECK: br label %for.end ; CHECK: for.end: ; CHECK-NEXT: ret void ; diff --git a/llvm/test/Transforms/LoopVersioning/add-phi-update-users.ll b/llvm/test/Transforms/LoopVersioning/add-phi-update-users.ll --- a/llvm/test/Transforms/LoopVersioning/add-phi-update-users.ll +++ b/llvm/test/Transforms/LoopVersioning/add-phi-update-users.ll @@ -16,8 +16,12 @@ ; added phi node. ; CHECK: define void @f1 +; CHECK: for.end.loopexit: +; CHECK-NEXT: %t2.lver.ph = phi i16 [ %t2.lver.orig, %for.body.lver.orig ] +; CHECK: for.end.loopexit2: +; CHECK-NEXT: %t2.lver.ph3 = phi i16 [ %t2, %for.body ] ; CHECK: for.end: -; CHECK-NEXT: %t2.lver = phi i16 [ %t2, %for.body ], [ %t2.lver.orig, %for.body.lver.orig ] +; CHECK-NEXT: %t2.lver = phi i16 [ %t2.lver.ph, %for.end.loopexit ], [ %t2.lver.ph3, %for.end.loopexit2 ] ; CHECK-NEXT: %tobool = icmp eq i16 %t2.lver, 0 ; CHECK: if.then: ; CHECK-NEXT: store i16 %t2.lver diff --git a/llvm/test/Transforms/LoopVersioning/basic.ll b/llvm/test/Transforms/LoopVersioning/basic.ll --- a/llvm/test/Transforms/LoopVersioning/basic.ll +++ b/llvm/test/Transforms/LoopVersioning/basic.ll @@ -19,10 +19,14 @@ ; CHECK: for.body.ph.lver.orig: ; CHECK: for.body.lver.orig: -; CHECK: br i1 %exitcond.lver.orig, label %for.end, label %for.body.lver.orig +; CHECK: br i1 %exitcond.lver.orig, label %for.end.loopexit, label %for.body.lver.orig ; CHECK: for.body.ph: ; CHECK: for.body: -; CHECK: br i1 %exitcond, label %for.end, label %for.body +; CHECK: br i1 %exitcond, label %for.end.loopexit12, label %for.body +; CHECK: for.end.loopexit: +; CHECK: br label %for.end +; CHECK: for.end.loopexit12: +; CHECK: br label %for.end ; CHECK: for.end: for.body: ; preds = %for.body, %entry diff --git a/llvm/test/Transforms/LoopVersioning/incorrect-phi.ll b/llvm/test/Transforms/LoopVersioning/incorrect-phi.ll --- a/llvm/test/Transforms/LoopVersioning/incorrect-phi.ll +++ b/llvm/test/Transforms/LoopVersioning/incorrect-phi.ll @@ -26,7 +26,9 @@ loop.exit: ; preds = %bb6 %_tmp142.lcssa = phi i64 [ %_tmp142, %bb6 ] %split = phi i16 [ undef, %bb6 ] -; CHECK: %split = phi i16 [ undef, %bb6 ], [ undef, %bb6.lver.orig ] +; CHECK: %split.ph = phi i16 [ undef, %bb6.lver.orig ] +; CHECK: %split.ph3 = phi i16 [ undef, %bb6 ] +; CHECK: %split = phi i16 [ %split.ph, %loop.exit.loopexit ], [ %split.ph3, %loop.exit.loopexit1 ] br label %bb9 bb9: ; preds = %bb9.loopexit, %bb1 @@ -52,7 +54,9 @@ loop.exit: ; preds = %bb6 %_tmp142.lcssa = phi i64 [ %_tmp142, %bb6 ] %split = phi i16 [ %t, %bb6 ] -; CHECK: %split = phi i16 [ %t, %bb6 ], [ %t, %bb6.lver.orig ] +; CHECK: %split.ph = phi i16 [ %t, %bb6.lver.orig ] +; CHECK: %split.ph3 = phi i16 [ %t, %bb6 ] +; CHECK: %split = phi i16 [ %split.ph, %loop.exit.loopexit ], [ %split.ph3, %loop.exit.loopexit1 ] br label %bb9 bb9: ; preds = %bb9.loopexit, %bb1