diff --git a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp --- a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp +++ b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp @@ -50,6 +50,10 @@ "unroll-runtime-multi-exit", cl::init(false), cl::Hidden, cl::desc("Allow runtime unrolling for loops with multiple exits, when " "epilog is generated")); +static cl::opt UnrollRuntimeAtMostTwoExits( + "unroll-runtime-at-most-two-exits", cl::init(false), cl::Hidden, + cl::desc("Allow runtime unrolling for loops with at most two exits, when " + "epilog is generated")); /// Connect the unrolling prolog code to the original loop. /// The unrolling prolog code contains code to execute the @@ -493,6 +497,15 @@ if (ExitingBlocks.size() > 2) return false; + // Only enable unrolling of loops that have a single side exit other than the + // normal LatchExit. + if (OtherExits.size() > 1) + return false; + + // At this point, the loop must have at most two exiting/exit blocks. + if (UnrollRuntimeAtMostTwoExits) + return true; + // The second heuristic is that L has one exit other than the latchexit and // that exit is a deoptimize block. We know that deoptimize blocks are rarely // taken, which also implies the branch leading to the deoptimize block is diff --git a/llvm/test/Transforms/LoopUnroll/runtime-loop-at-most-two-exits.ll b/llvm/test/Transforms/LoopUnroll/runtime-loop-at-most-two-exits.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/LoopUnroll/runtime-loop-at-most-two-exits.ll @@ -0,0 +1,130 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt < %s -loop-unroll -unroll-runtime=true -unroll-count=2 -unroll-runtime-epilog=true -unroll-runtime-at-most-two-exits=true -verify-loop-lcssa -verify-dom-info -verify-loop-info -S | FileCheck %s --check-prefix=ENABLED +; RUN: opt < %s -loop-unroll -unroll-runtime=true -unroll-count=2 -unroll-runtime-epilog=true -unroll-runtime-at-most-two-exits=false -verify-loop-lcssa -verify-dom-info -verify-loop-info -S | FileCheck %s --check-prefix=DISABLED + +define i32 @test(i32* nocapture %a, i64 %n) { +; ENABLED-LABEL: @test( +; ENABLED-NEXT: entry: +; ENABLED-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], -1 +; ENABLED-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 1 +; ENABLED-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1 +; ENABLED-NEXT: br i1 [[TMP1]], label [[FOR_END_UNR_LCSSA:%.*]], label [[ENTRY_NEW:%.*]] +; ENABLED: entry.new: +; ENABLED-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]] +; ENABLED-NEXT: br label [[HEADER:%.*]] +; ENABLED: header: +; ENABLED-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY_NEW]] ], [ [[INDVARS_IV_NEXT_1:%.*]], [[FOR_BODY_1:%.*]] ] +; ENABLED-NEXT: [[SUM_02:%.*]] = phi i32 [ 0, [[ENTRY_NEW]] ], [ [[ADD_1:%.*]], [[FOR_BODY_1]] ] +; ENABLED-NEXT: [[NITER:%.*]] = phi i64 [ [[UNROLL_ITER]], [[ENTRY_NEW]] ], [ [[NITER_NSUB_1:%.*]], [[FOR_BODY_1]] ] +; ENABLED-NEXT: [[CMP:%.*]] = icmp eq i64 [[N]], 42 +; ENABLED-NEXT: br i1 [[CMP]], label [[FOR_EXIT2_LOOPEXIT:%.*]], label [[FOR_BODY:%.*]] +; ENABLED: for.body: +; ENABLED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]] +; ENABLED-NEXT: [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX]], align 4 +; ENABLED-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP2]], [[SUM_02]] +; ENABLED-NEXT: [[INDVARS_IV_NEXT:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 1 +; ENABLED-NEXT: [[NITER_NSUB:%.*]] = sub i64 [[NITER]], 1 +; ENABLED-NEXT: [[CMP_1:%.*]] = icmp eq i64 [[N]], 42 +; ENABLED-NEXT: br i1 [[CMP_1]], label [[FOR_EXIT2_LOOPEXIT]], label [[FOR_BODY_1]] +; ENABLED: for.end.unr-lcssa.loopexit: +; ENABLED-NEXT: [[SUM_0_LCSSA_PH_PH:%.*]] = phi i32 [ [[ADD_1]], [[FOR_BODY_1]] ] +; ENABLED-NEXT: [[INDVARS_IV_UNR_PH:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_1]], [[FOR_BODY_1]] ] +; ENABLED-NEXT: [[SUM_02_UNR_PH:%.*]] = phi i32 [ [[ADD_1]], [[FOR_BODY_1]] ] +; ENABLED-NEXT: br label [[FOR_END_UNR_LCSSA]] +; ENABLED: for.end.unr-lcssa: +; ENABLED-NEXT: [[SUM_0_LCSSA_PH:%.*]] = phi i32 [ undef, [[ENTRY:%.*]] ], [ [[SUM_0_LCSSA_PH_PH]], [[FOR_END_UNR_LCSSA_LOOPEXIT:%.*]] ] +; ENABLED-NEXT: [[INDVARS_IV_UNR:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDVARS_IV_UNR_PH]], [[FOR_END_UNR_LCSSA_LOOPEXIT]] ] +; ENABLED-NEXT: [[SUM_02_UNR:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[SUM_02_UNR_PH]], [[FOR_END_UNR_LCSSA_LOOPEXIT]] ] +; ENABLED-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; ENABLED-NEXT: br i1 [[LCMP_MOD]], label [[HEADER_EPIL_PREHEADER:%.*]], label [[FOR_END:%.*]] +; ENABLED: header.epil.preheader: +; ENABLED-NEXT: br label [[HEADER_EPIL:%.*]] +; ENABLED: header.epil: +; ENABLED-NEXT: [[INDVARS_IV_EPIL:%.*]] = phi i64 [ [[INDVARS_IV_UNR]], [[HEADER_EPIL_PREHEADER]] ] +; ENABLED-NEXT: [[SUM_02_EPIL:%.*]] = phi i32 [ [[SUM_02_UNR]], [[HEADER_EPIL_PREHEADER]] ] +; ENABLED-NEXT: [[CMP_EPIL:%.*]] = icmp eq i64 [[N]], 42 +; ENABLED-NEXT: br i1 [[CMP_EPIL]], label [[FOR_EXIT2:%.*]], label [[FOR_BODY_EPIL:%.*]] +; ENABLED: for.body.epil: +; ENABLED-NEXT: [[ARRAYIDX_EPIL:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV_EPIL]] +; ENABLED-NEXT: [[TMP3:%.*]] = load i32, i32* [[ARRAYIDX_EPIL]], align 4 +; ENABLED-NEXT: [[ADD_EPIL:%.*]] = add nsw i32 [[TMP3]], [[SUM_02_EPIL]] +; ENABLED-NEXT: [[INDVARS_IV_NEXT_EPIL:%.*]] = add i64 [[INDVARS_IV_EPIL]], 1 +; ENABLED-NEXT: [[EXITCOND_EPIL:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_EPIL]], [[N]] +; ENABLED-NEXT: br label [[FOR_END_EPILOG_LCSSA:%.*]] +; ENABLED: for.end.epilog-lcssa: +; ENABLED-NEXT: br label [[FOR_END]] +; ENABLED: for.end: +; ENABLED-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ [[SUM_0_LCSSA_PH]], [[FOR_END_UNR_LCSSA]] ], [ [[ADD_EPIL]], [[FOR_END_EPILOG_LCSSA]] ] +; ENABLED-NEXT: ret i32 [[SUM_0_LCSSA]] +; ENABLED: for.exit2.loopexit: +; ENABLED-NEXT: [[RETVAL_PH:%.*]] = phi i32 [ [[SUM_02]], [[HEADER]] ], [ [[ADD]], [[FOR_BODY]] ] +; ENABLED-NEXT: br label [[FOR_EXIT2]] +; ENABLED: for.exit2: +; ENABLED-NEXT: [[RETVAL:%.*]] = phi i32 [ [[SUM_02_EPIL]], [[HEADER_EPIL]] ], [ [[RETVAL_PH]], [[FOR_EXIT2_LOOPEXIT]] ] +; ENABLED-NEXT: ret i32 [[RETVAL]] +; ENABLED: for.body.1: +; ENABLED-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV_NEXT]] +; ENABLED-NEXT: [[TMP4:%.*]] = load i32, i32* [[ARRAYIDX_1]], align 4 +; ENABLED-NEXT: [[ADD_1]] = add nsw i32 [[TMP4]], [[ADD]] +; ENABLED-NEXT: [[INDVARS_IV_NEXT_1]] = add i64 [[INDVARS_IV_NEXT]], 1 +; ENABLED-NEXT: [[NITER_NSUB_1]] = sub i64 [[NITER_NSUB]], 1 +; ENABLED-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NSUB_1]], 0 +; ENABLED-NEXT: br i1 [[NITER_NCMP_1]], label [[FOR_END_UNR_LCSSA_LOOPEXIT]], label [[HEADER]], [[LOOP0:!llvm.loop !.*]] +; +; DISABLED-LABEL: @test( +; DISABLED-NEXT: entry: +; DISABLED-NEXT: br label [[HEADER:%.*]] +; DISABLED: header: +; DISABLED-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT_1:%.*]], [[FOR_BODY_1:%.*]] ] +; DISABLED-NEXT: [[SUM_02:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[ADD_1:%.*]], [[FOR_BODY_1]] ] +; DISABLED-NEXT: [[CMP:%.*]] = icmp eq i64 [[N:%.*]], 42 +; DISABLED-NEXT: br i1 [[CMP]], label [[FOR_EXIT2:%.*]], label [[FOR_BODY:%.*]] +; DISABLED: for.body: +; DISABLED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]] +; DISABLED-NEXT: [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4 +; DISABLED-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP0]], [[SUM_02]] +; DISABLED-NEXT: [[INDVARS_IV_NEXT:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 1 +; DISABLED-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] +; DISABLED-NEXT: br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[HEADER_1:%.*]] +; DISABLED: for.end: +; DISABLED-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[ADD_1]], [[FOR_BODY_1]] ] +; DISABLED-NEXT: ret i32 [[SUM_0_LCSSA]] +; DISABLED: for.exit2: +; DISABLED-NEXT: [[RETVAL:%.*]] = phi i32 [ [[SUM_02]], [[HEADER]] ], [ [[ADD]], [[HEADER_1]] ] +; DISABLED-NEXT: ret i32 [[RETVAL]] +; DISABLED: header.1: +; DISABLED-NEXT: [[CMP_1:%.*]] = icmp eq i64 [[N]], 42 +; DISABLED-NEXT: br i1 [[CMP_1]], label [[FOR_EXIT2]], label [[FOR_BODY_1]] +; DISABLED: for.body.1: +; DISABLED-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV_NEXT]] +; DISABLED-NEXT: [[TMP1:%.*]] = load i32, i32* [[ARRAYIDX_1]], align 4 +; DISABLED-NEXT: [[ADD_1]] = add nsw i32 [[TMP1]], [[ADD]] +; DISABLED-NEXT: [[INDVARS_IV_NEXT_1]] = add i64 [[INDVARS_IV_NEXT]], 1 +; DISABLED-NEXT: [[EXITCOND_1:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_1]], [[N]] +; DISABLED-NEXT: br i1 [[EXITCOND_1]], label [[FOR_END]], label [[HEADER]], [[LOOP0:!llvm.loop !.*]] +; +entry: + br label %header + +header: + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %sum.02 = phi i32 [ %add, %for.body ], [ 0, %entry ] + %cmp = icmp eq i64 %n, 42 + br i1 %cmp, label %for.exit2, label %for.body + +for.body: + %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv + %0 = load i32, i32* %arrayidx, align 4 + %add = add nsw i32 %0, %sum.02 + %indvars.iv.next = add i64 %indvars.iv, 1 + %exitcond = icmp eq i64 %indvars.iv.next, %n + br i1 %exitcond, label %for.end, label %header + +for.end: + %sum.0.lcssa = phi i32 [ %add, %for.body ] + ret i32 %sum.0.lcssa + +for.exit2: + %retval = phi i32 [ %sum.02, %header ] + ret i32 %retval +}