diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp --- a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp +++ b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp @@ -1115,6 +1115,11 @@ BasicBlock *ExitingBlock = L->getLoopLatch(); if (!ExitingBlock || !L->isLoopExiting(ExitingBlock)) ExitingBlock = L->getExitingBlock(); + // If the latch is not exiting and there's no single exiting block, check if + // the header is exiting and use it to estimate the trip count. This can + // happen when unrolling non-rotated loops. + if (!ExitingBlock && L->isLoopExiting(L->getHeader())) + ExitingBlock = L->getHeader(); if (ExitingBlock) { TripCount = SE.getSmallConstantTripCount(L, ExitingBlock); TripMultiple = SE.getSmallConstantTripMultiple(L, ExitingBlock); diff --git a/llvm/lib/Transforms/Utils/LoopUnroll.cpp b/llvm/lib/Transforms/Utils/LoopUnroll.cpp --- a/llvm/lib/Transforms/Utils/LoopUnroll.cpp +++ b/llvm/lib/Transforms/Utils/LoopUnroll.cpp @@ -247,14 +247,17 @@ /// branch instruction. However, if the trip count (and multiple) are not known, /// loop unrolling will mostly produce more code that is no faster. /// -/// TripCount is the upper bound of the iteration on which control exits -/// LatchBlock. Control may exit the loop prior to TripCount iterations either -/// via an early branch in other loop block or via LatchBlock terminator. This -/// is relaxed from the general definition of trip count which is the number of -/// times the loop header executes. Note that UnrollLoop assumes that the loop -/// counter test is in LatchBlock in order to remove unnecesssary instances of -/// the test. If control can exit the loop from the LatchBlock's terminator -/// prior to TripCount iterations, flag PreserveCondBr needs to be set. +/// TripCount is the upper bound of the iteration on which control exits via +/// latch block (if it exits), the header (if it exits) or the unique exiting +/// block of the loop, if there is one. Control may exit the loop prior to +/// TripCount iterations via any of its exits. This is relaxed from the general +/// definition of trip count which is the number of times the loop header +/// executes. Note that UnrollLoop assumes that the loop counter test is in +/// LatchBlock or in the unique exiting block in order to remove unnecesssary +/// instances of the test. Otherwise the conditional branches are preserved, +/// pending simplification after unrolling. If control can exit the loop from +/// the LatchBlock's terminator prior to TripCount iterations, flag +/// PreserveCondBr needs to be set. /// /// PreserveCondBr indicates whether the conditional branch of the LatchBlock /// needs to be preserved. It is needed when we use trip count upper bound to diff --git a/llvm/test/Transforms/LoopUnroll/unroll-header-exiting-with-phis-multiple-exiting-blocks.ll b/llvm/test/Transforms/LoopUnroll/unroll-header-exiting-with-phis-multiple-exiting-blocks.ll --- a/llvm/test/Transforms/LoopUnroll/unroll-header-exiting-with-phis-multiple-exiting-blocks.ll +++ b/llvm/test/Transforms/LoopUnroll/unroll-header-exiting-with-phis-multiple-exiting-blocks.ll @@ -8,25 +8,53 @@ ; CHECK-NEXT: entry: ; CHECK-NEXT: br label [[HEADER:%.*]] ; CHECK: header: -; CHECK-NEXT: [[RES:%.*]] = phi i16 [ 123, [[ENTRY:%.*]] ], [ [[RES_NEXT:%.*]], [[LATCH:%.*]] ] -; CHECK-NEXT: [[I_0:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INC9:%.*]], [[LATCH]] ] -; CHECK-NEXT: [[PTR:%.*]] = getelementptr inbounds i16, i16* [[A:%.*]], i64 [[I_0]] -; CHECK-NEXT: [[LV:%.*]] = load i16, i16* [[PTR]], align 2 -; CHECK-NEXT: [[RES_NEXT]] = add i16 [[RES]], [[LV]] -; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[I_0]], 3 -; CHECK-NEXT: br i1 [[CMP]], label [[EXITING_1:%.*]], label [[EXIT:%.*]] +; CHECK-NEXT: [[LV:%.*]] = load i16, i16* [[A:%.*]], align 2 +; CHECK-NEXT: [[RES_NEXT:%.*]] = add i16 123, [[LV]] +; CHECK-NEXT: br i1 true, label [[EXITING_1:%.*]], label [[EXIT:%.*]] ; CHECK: exiting.1: ; CHECK-NEXT: [[EC_1:%.*]] = icmp eq i16 [[LV]], [[X:%.*]] ; CHECK-NEXT: br i1 [[EC_1]], label [[EXIT]], label [[EXITING_2:%.*]] ; CHECK: exiting.2: ; CHECK-NEXT: [[EC_2:%.*]] = icmp eq i16 [[LV]], [[Y:%.*]] -; CHECK-NEXT: br i1 [[EC_2]], label [[EXIT]], label [[LATCH]] +; CHECK-NEXT: br i1 [[EC_2]], label [[EXIT]], label [[LATCH:%.*]] ; CHECK: latch: -; CHECK-NEXT: [[INC9]] = add i64 [[I_0]], 1 -; CHECK-NEXT: br label [[HEADER]] +; CHECK-NEXT: [[PTR_1:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 1 +; CHECK-NEXT: [[LV_1:%.*]] = load i16, i16* [[PTR_1]], align 2 +; CHECK-NEXT: [[RES_NEXT_1:%.*]] = add i16 [[RES_NEXT]], [[LV_1]] +; CHECK-NEXT: br i1 true, label [[EXITING_1_1:%.*]], label [[EXIT]] ; CHECK: exit: -; CHECK-NEXT: [[RES_LCSSA:%.*]] = phi i16 [ [[RES_NEXT]], [[HEADER]] ], [ 0, [[EXITING_1]] ], [ 1, [[EXITING_2]] ] +; CHECK-NEXT: [[RES_LCSSA:%.*]] = phi i16 [ [[RES_NEXT]], [[HEADER]] ], [ 0, [[EXITING_1]] ], [ 1, [[EXITING_2]] ], [ [[RES_NEXT_1]], [[LATCH]] ], [ 0, [[EXITING_1_1]] ], [ 1, [[EXITING_2_1:%.*]] ], [ [[RES_NEXT_2:%.*]], [[LATCH_1:%.*]] ], [ 0, [[EXITING_1_2:%.*]] ], [ 1, [[EXITING_2_2:%.*]] ], [ [[RES_NEXT_3:%.*]], [[LATCH_2:%.*]] ], [ 0, [[EXITING_1_3:%.*]] ], [ 1, [[EXITING_2_3:%.*]] ] ; CHECK-NEXT: ret i16 [[RES_LCSSA]] +; CHECK: exiting.1.1: +; CHECK-NEXT: [[EC_1_1:%.*]] = icmp eq i16 [[LV_1]], [[X]] +; CHECK-NEXT: br i1 [[EC_1_1]], label [[EXIT]], label [[EXITING_2_1]] +; CHECK: exiting.2.1: +; CHECK-NEXT: [[EC_2_1:%.*]] = icmp eq i16 [[LV_1]], [[Y]] +; CHECK-NEXT: br i1 [[EC_2_1]], label [[EXIT]], label [[LATCH_1]] +; CHECK: latch.1: +; CHECK-NEXT: [[PTR_2:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 2 +; CHECK-NEXT: [[LV_2:%.*]] = load i16, i16* [[PTR_2]], align 2 +; CHECK-NEXT: [[RES_NEXT_2]] = add i16 [[RES_NEXT_1]], [[LV_2]] +; CHECK-NEXT: br i1 true, label [[EXITING_1_2]], label [[EXIT]] +; CHECK: exiting.1.2: +; CHECK-NEXT: [[EC_1_2:%.*]] = icmp eq i16 [[LV_2]], [[X]] +; CHECK-NEXT: br i1 [[EC_1_2]], label [[EXIT]], label [[EXITING_2_2]] +; CHECK: exiting.2.2: +; CHECK-NEXT: [[EC_2_2:%.*]] = icmp eq i16 [[LV_2]], [[Y]] +; CHECK-NEXT: br i1 [[EC_2_2]], label [[EXIT]], label [[LATCH_2]] +; CHECK: latch.2: +; CHECK-NEXT: [[PTR_3:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 3 +; CHECK-NEXT: [[LV_3:%.*]] = load i16, i16* [[PTR_3]], align 2 +; CHECK-NEXT: [[RES_NEXT_3]] = add i16 [[RES_NEXT_2]], [[LV_3]] +; CHECK-NEXT: br i1 false, label [[EXITING_1_3]], label [[EXIT]] +; CHECK: exiting.1.3: +; CHECK-NEXT: [[EC_1_3:%.*]] = icmp eq i16 [[LV_3]], [[X]] +; CHECK-NEXT: br i1 [[EC_1_3]], label [[EXIT]], label [[EXITING_2_3]] +; CHECK: exiting.2.3: +; CHECK-NEXT: [[EC_2_3:%.*]] = icmp eq i16 [[LV_3]], [[Y]] +; CHECK-NEXT: br i1 [[EC_2_3]], label [[EXIT]], label [[LATCH_3:%.*]] +; CHECK: latch.3: +; CHECK-NEXT: unreachable ; entry: br label %header @@ -63,25 +91,49 @@ ; CHECK-NEXT: entry: ; CHECK-NEXT: br label [[HEADER:%.*]] ; CHECK: header: -; CHECK-NEXT: [[RES:%.*]] = phi i16 [ 123, [[ENTRY:%.*]] ], [ [[RES_NEXT:%.*]], [[LATCH:%.*]] ] -; CHECK-NEXT: [[I_0:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INC9:%.*]], [[LATCH]] ] -; CHECK-NEXT: [[PTR:%.*]] = getelementptr inbounds i16, i16* [[A:%.*]], i64 [[I_0]] -; CHECK-NEXT: [[LV:%.*]] = load i16, i16* [[PTR]], align 2 -; CHECK-NEXT: [[RES_NEXT]] = add i16 [[RES]], [[LV]] -; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[I_0]], 3 -; CHECK-NEXT: br i1 [[CMP]], label [[EXITING_1:%.*]], label [[EXIT:%.*]] +; CHECK-NEXT: [[LV:%.*]] = load i16, i16* [[A:%.*]], align 2 +; CHECK-NEXT: [[RES_NEXT:%.*]] = add i16 123, [[LV]] +; CHECK-NEXT: br i1 true, label [[EXITING_1:%.*]], label [[EXIT:%.*]] ; CHECK: exiting.1: -; CHECK-NEXT: [[EC_1:%.*]] = icmp eq i64 [[I_0]], 1 -; CHECK-NEXT: br i1 [[EC_1]], label [[EXIT]], label [[EXITING_2:%.*]] +; CHECK-NEXT: br i1 false, label [[EXIT]], label [[EXITING_2:%.*]] ; CHECK: exiting.2: ; CHECK-NEXT: [[EC_2:%.*]] = icmp eq i16 [[LV]], [[Y:%.*]] -; CHECK-NEXT: br i1 [[EC_2]], label [[EXIT]], label [[LATCH]] +; CHECK-NEXT: br i1 [[EC_2]], label [[EXIT]], label [[LATCH:%.*]] ; CHECK: latch: -; CHECK-NEXT: [[INC9]] = add i64 [[I_0]], 1 -; CHECK-NEXT: br label [[HEADER]] +; CHECK-NEXT: [[PTR_1:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 1 +; CHECK-NEXT: [[LV_1:%.*]] = load i16, i16* [[PTR_1]], align 2 +; CHECK-NEXT: [[RES_NEXT_1:%.*]] = add i16 [[RES_NEXT]], [[LV_1]] +; CHECK-NEXT: br i1 true, label [[EXITING_1_1:%.*]], label [[EXIT]] ; CHECK: exit: -; CHECK-NEXT: [[RES_LCSSA:%.*]] = phi i16 [ [[RES_NEXT]], [[HEADER]] ], [ 0, [[EXITING_1]] ], [ 1, [[EXITING_2]] ] +; CHECK-NEXT: [[RES_LCSSA:%.*]] = phi i16 [ [[RES_NEXT]], [[HEADER]] ], [ 0, [[EXITING_1]] ], [ 1, [[EXITING_2]] ], [ [[RES_NEXT_1]], [[LATCH]] ], [ 0, [[EXITING_1_1]] ], [ 1, [[EXITING_2_1:%.*]] ], [ [[RES_NEXT_2:%.*]], [[LATCH_1:%.*]] ], [ 0, [[EXITING_1_2:%.*]] ], [ 1, [[EXITING_2_2:%.*]] ], [ [[RES_NEXT_3:%.*]], [[LATCH_2:%.*]] ], [ 0, [[EXITING_1_3:%.*]] ], [ 1, [[EXITING_2_3:%.*]] ] ; CHECK-NEXT: ret i16 [[RES_LCSSA]] +; CHECK: exiting.1.1: +; CHECK-NEXT: br i1 true, label [[EXIT]], label [[EXITING_2_1]] +; CHECK: exiting.2.1: +; CHECK-NEXT: [[EC_2_1:%.*]] = icmp eq i16 [[LV_1]], [[Y]] +; CHECK-NEXT: br i1 [[EC_2_1]], label [[EXIT]], label [[LATCH_1]] +; CHECK: latch.1: +; CHECK-NEXT: [[PTR_2:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 2 +; CHECK-NEXT: [[LV_2:%.*]] = load i16, i16* [[PTR_2]], align 2 +; CHECK-NEXT: [[RES_NEXT_2]] = add i16 [[RES_NEXT_1]], [[LV_2]] +; CHECK-NEXT: br i1 true, label [[EXITING_1_2]], label [[EXIT]] +; CHECK: exiting.1.2: +; CHECK-NEXT: br i1 false, label [[EXIT]], label [[EXITING_2_2]] +; CHECK: exiting.2.2: +; CHECK-NEXT: [[EC_2_2:%.*]] = icmp eq i16 [[LV_2]], [[Y]] +; CHECK-NEXT: br i1 [[EC_2_2]], label [[EXIT]], label [[LATCH_2]] +; CHECK: latch.2: +; CHECK-NEXT: [[PTR_3:%.*]] = getelementptr inbounds i16, i16* [[A]], i64 3 +; CHECK-NEXT: [[LV_3:%.*]] = load i16, i16* [[PTR_3]], align 2 +; CHECK-NEXT: [[RES_NEXT_3]] = add i16 [[RES_NEXT_2]], [[LV_3]] +; CHECK-NEXT: br i1 false, label [[EXITING_1_3]], label [[EXIT]] +; CHECK: exiting.1.3: +; CHECK-NEXT: br i1 false, label [[EXIT]], label [[EXITING_2_3]] +; CHECK: exiting.2.3: +; CHECK-NEXT: [[EC_2_3:%.*]] = icmp eq i16 [[LV_3]], [[Y]] +; CHECK-NEXT: br i1 [[EC_2_3]], label [[EXIT]], label [[LATCH_3:%.*]] +; CHECK: latch.3: +; CHECK-NEXT: unreachable ; entry: br label %header