Index: lib/Transforms/Utils/LoopUnrollPeel.cpp =================================================================== --- lib/Transforms/Utils/LoopUnrollPeel.cpp +++ lib/Transforms/Utils/LoopUnrollPeel.cpp @@ -618,8 +618,12 @@ assert(DT->verify(DominatorTree::VerificationLevel::Fast)); } - updateBranchWeights(InsertBot, cast(VMap[LatchBR]), Iter, + auto *LatchBRCopy = cast(VMap[LatchBR]); + updateBranchWeights(InsertBot, LatchBRCopy, Iter, PeelCount, ExitWeight); + // Remove Loop metadata from the latch branch instruction + // because it is not the Loop's latch branch anymore. + LatchBRCopy->setMetadata(LLVMContext::MD_loop, nullptr); InsertTop = InsertBot; InsertBot = SplitBlock(InsertBot, InsertBot->getTerminator(), DT, LI); Index: test/Transforms/LoopUnroll/peel-loop-conditions.ll =================================================================== --- test/Transforms/LoopUnroll/peel-loop-conditions.ll +++ test/Transforms/LoopUnroll/peel-loop-conditions.ll @@ -23,7 +23,9 @@ ; CHECK: for.inc.peel: ; CHECK-NEXT: [[INC_PEEL:%.*]] = add nsw i32 0, 1 ; CHECK-NEXT: [[CMP_PEEL:%.*]] = icmp slt i32 [[INC_PEEL]], [[K:%.*]] -; CHECK-NEXT: br i1 [[CMP_PEEL]], label [[FOR_BODY_PEEL_NEXT:%.*]], label [[FOR_END:%.*]] +; CHECK-NEXT: br i1 [[CMP_PEEL]], label [[FOR_BODY_PEEL_NEXT:%.*]], label [[FOR_END:%[^,]*]] +; Verify that MD_loop metadata is dropped. +; CHECK-NOT: , !llvm.loop !{{[0-9]*}} ; CHECK: for.body.peel.next: ; CHECK-NEXT: br label [[FOR_BODY_PEEL2:%.*]] ; CHECK: for.body.peel2: @@ -39,6 +41,8 @@ ; CHECK-NEXT: [[INC_PEEL7:%.*]] = add nsw i32 [[INC_PEEL]], 1 ; CHECK-NEXT: [[CMP_PEEL8:%.*]] = icmp slt i32 [[INC_PEEL7]], [[K]] ; CHECK-NEXT: br i1 [[CMP_PEEL8]], label [[FOR_BODY_PEEL_NEXT1:%.*]], label [[FOR_END]] +; Verify that MD_loop metadata is dropped. +; CHECK-NOT: , !llvm.loop !{{[0-9]*}} ; CHECK: for.body.peel.next1: ; CHECK-NEXT: br label [[FOR_BODY_PEEL_NEXT9:%.*]] ; CHECK: for.body.peel.next9: @@ -57,7 +61,7 @@ ; CHECK: for.inc: ; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_05]], 1 ; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[INC]], [[K]] -; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END_LOOPEXIT:%.*]], !llvm.loop !0 +; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END_LOOPEXIT:%.*]], !llvm.loop !{{.*}} ; CHECK: for.end.loopexit: ; CHECK-NEXT: br label [[FOR_END]] ; CHECK: for.end: @@ -82,12 +86,14 @@ for.inc: %inc = add nsw i32 %i.05, 1 %cmp = icmp slt i32 %inc, %k - br i1 %cmp, label %for.body, label %for.end + br i1 %cmp, label %for.body, label %for.end, !llvm.loop !1 for.end: ret void } +!1 = distinct !{!1} + ; Check we peel off the maximum number of iterations that make conditions true. define void @test2(i32 %k) { ; CHECK-LABEL: @test2( @@ -113,7 +119,9 @@ ; CHECK: for.inc.peel: ; CHECK-NEXT: [[INC_PEEL:%.*]] = add nsw i32 0, 1 ; CHECK-NEXT: [[CMP_PEEL:%.*]] = icmp slt i32 [[INC_PEEL]], [[K:%.*]] -; CHECK-NEXT: br i1 [[CMP_PEEL]], label [[FOR_BODY_PEEL_NEXT:%.*]], label [[FOR_END:%.*]] +; CHECK-NEXT: br i1 [[CMP_PEEL]], label [[FOR_BODY_PEEL_NEXT:%.*]], label [[FOR_END:%[^,]*]] +; Verify that MD_loop metadata is dropped. +; CHECK-NOT: , !llvm.loop !{{[0-9]*}} ; CHECK: for.body.peel.next: ; CHECK-NEXT: br label [[FOR_BODY_PEEL2:%.*]] ; CHECK: for.body.peel2: @@ -135,6 +143,8 @@ ; CHECK-NEXT: [[INC_PEEL10:%.*]] = add nsw i32 [[INC_PEEL]], 1 ; CHECK-NEXT: [[CMP_PEEL11:%.*]] = icmp slt i32 [[INC_PEEL10]], [[K]] ; CHECK-NEXT: br i1 [[CMP_PEEL11]], label [[FOR_BODY_PEEL_NEXT1:%.*]], label [[FOR_END]] +; Verify that MD_loop metadata is dropped. +; CHECK-NOT: , !llvm.loop !{{[0-9]*}} ; CHECK: for.body.peel.next1: ; CHECK-NEXT: br label [[FOR_BODY_PEEL13:%.*]] ; CHECK: for.body.peel13: @@ -156,6 +166,8 @@ ; CHECK-NEXT: [[INC_PEEL21:%.*]] = add nsw i32 [[INC_PEEL10]], 1 ; CHECK-NEXT: [[CMP_PEEL22:%.*]] = icmp slt i32 [[INC_PEEL21]], [[K]] ; CHECK-NEXT: br i1 [[CMP_PEEL22]], label [[FOR_BODY_PEEL_NEXT12:%.*]], label [[FOR_END]] +; Verify that MD_loop metadata is dropped. +; CHECK-NOT: , !llvm.loop !{{[0-9]*}} ; CHECK: for.body.peel.next12: ; CHECK-NEXT: br label [[FOR_BODY_PEEL24:%.*]] ; CHECK: for.body.peel24: @@ -177,6 +189,8 @@ ; CHECK-NEXT: [[INC_PEEL32:%.*]] = add nsw i32 [[INC_PEEL21]], 1 ; CHECK-NEXT: [[CMP_PEEL33:%.*]] = icmp slt i32 [[INC_PEEL32]], [[K]] ; CHECK-NEXT: br i1 [[CMP_PEEL33]], label [[FOR_BODY_PEEL_NEXT23:%.*]], label [[FOR_END]] +; Verify that MD_loop metadata is dropped. +; CHECK-NOT: , !llvm.loop !{{[0-9]*}} ; CHECK: for.body.peel.next23: ; CHECK-NEXT: br label [[FOR_BODY_PEEL_NEXT34:%.*]] ; CHECK: for.body.peel.next34: @@ -200,7 +214,7 @@ ; CHECK: for.inc: ; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_05]], 1 ; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[INC]], [[K]] -; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END_LOOPEXIT:%.*]], !llvm.loop !2 +; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END_LOOPEXIT:%.*]], !llvm.loop !{{.*}} ; CHECK: for.end.loopexit: ; CHECK-NEXT: br label [[FOR_END]] ; CHECK: for.end: @@ -233,12 +247,14 @@ for.inc: %inc = add nsw i32 %i.05, 1 %cmp = icmp slt i32 %inc, %k - br i1 %cmp, label %for.body, label %for.end + br i1 %cmp, label %for.body, label %for.end, !llvm.loop !2 for.end: ret void } +!2 = distinct !{!2} + ; Check that we can peel off iterations that make a condition false. define void @test3(i32 %k) { ; CHECK-LABEL: @test3( @@ -258,7 +274,9 @@ ; CHECK: for.inc.peel: ; CHECK-NEXT: [[INC_PEEL:%.*]] = add nsw i32 0, 1 ; CHECK-NEXT: [[CMP_PEEL:%.*]] = icmp slt i32 [[INC_PEEL]], [[K:%.*]] -; CHECK-NEXT: br i1 [[CMP_PEEL]], label [[FOR_BODY_PEEL_NEXT:%.*]], label [[FOR_END:%.*]] +; CHECK-NEXT: br i1 [[CMP_PEEL]], label [[FOR_BODY_PEEL_NEXT:%.*]], label [[FOR_END:%[^,]*]] +; Verify that MD_loop metadata is dropped. +; CHECK-NOT: , !llvm.loop !{{[0-9]*}} ; CHECK: for.body.peel.next: ; CHECK-NEXT: br label [[FOR_BODY_PEEL2:%.*]] ; CHECK: for.body.peel2: @@ -274,6 +292,8 @@ ; CHECK-NEXT: [[INC_PEEL7:%.*]] = add nsw i32 [[INC_PEEL]], 1 ; CHECK-NEXT: [[CMP_PEEL8:%.*]] = icmp slt i32 [[INC_PEEL7]], [[K]] ; CHECK-NEXT: br i1 [[CMP_PEEL8]], label [[FOR_BODY_PEEL_NEXT1:%.*]], label [[FOR_END]] +; Verify that MD_loop metadata is dropped. +; CHECK-NOT: , !llvm.loop !{{[0-9]*}} ; CHECK: for.body.peel.next1: ; CHECK-NEXT: br label [[FOR_BODY_PEEL10:%.*]] ; CHECK: for.body.peel10: @@ -289,6 +309,8 @@ ; CHECK-NEXT: [[INC_PEEL15:%.*]] = add nsw i32 [[INC_PEEL7]], 1 ; CHECK-NEXT: [[CMP_PEEL16:%.*]] = icmp slt i32 [[INC_PEEL15]], [[K]] ; CHECK-NEXT: br i1 [[CMP_PEEL16]], label [[FOR_BODY_PEEL_NEXT9:%.*]], label [[FOR_END]] +; Verify that MD_loop metadata is dropped. +; CHECK-NOT: , !llvm.loop !{{[0-9]*}} ; CHECK: for.body.peel.next9: ; CHECK-NEXT: br label [[FOR_BODY_PEEL_NEXT17:%.*]] ; CHECK: for.body.peel.next17: @@ -307,7 +329,7 @@ ; CHECK: for.inc: ; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_05]], 1 ; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[INC]], [[K]] -; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END_LOOPEXIT:%.*]], !llvm.loop !3 +; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END_LOOPEXIT:%.*]], !llvm.loop !{{.*}} ; CHECK: for.end.loopexit: ; CHECK-NEXT: br label [[FOR_END]] ; CHECK: for.end: @@ -332,12 +354,14 @@ for.inc: %inc = add nsw i32 %i.05, 1 %cmp = icmp slt i32 %inc, %k - br i1 %cmp, label %for.body, label %for.end + br i1 %cmp, label %for.body, label %for.end, !llvm.loop !3 for.end: ret void } +!3 = distinct !{!3} + ; Test that we only peel off iterations if it simplifies a condition in the ; loop body after peeling at most MaxPeelCount iterations. define void @test4(i32 %k) { Index: test/Transforms/LoopUnroll/peel-loop.ll =================================================================== --- test/Transforms/LoopUnroll/peel-loop.ll +++ test/Transforms/LoopUnroll/peel-loop.ll @@ -8,17 +8,24 @@ ; CHECK: [[NEXT0]]: ; CHECK: store i32 0, i32* %p, align 4 ; CHECK: %[[CMP1:.*]] = icmp eq i32 %k, 1 -; CHECK: br i1 %[[CMP1]], label %for.end, label %[[NEXT1:.*]] +; CHECK: br i1 %[[CMP1]], label %for.end, label %[[NEXT1:[^,]*]] +; Verify that MD_loop metadata is dropped. +; CHECK-NOT: , !llvm.loop !{{[0-9]*}} ; CHECK: [[NEXT1]]: ; CHECK: %[[INC1:.*]] = getelementptr inbounds i32, i32* %p, i64 1 ; CHECK: store i32 1, i32* %[[INC1]], align 4 ; CHECK: %[[CMP2:.*]] = icmp sgt i32 %k, 2 ; CHECK: br i1 %[[CMP2]], label %[[NEXT2:.*]], label %for.end +; Verify that MD_loop metadata is dropped. +; CHECK-NOT: , !llvm.loop !{{[0-9]*}} ; CHECK: [[NEXT2]]: ; CHECK: %[[INC2:.*]] = getelementptr inbounds i32, i32* %p, i64 2 ; CHECK: store i32 2, i32* %[[INC2]], align 4 ; CHECK: %[[CMP3:.*]] = icmp eq i32 %k, 3 -; CHECK: br i1 %[[CMP3]], label %for.end, label %[[LOOP_PH:.*]] +; CHECK: br i1 %[[CMP3]], label %for.end, label %[[LOOP_PH:[^,]*]] +; Verify that MD_loop metadata is dropped. +; CHECK-NOT: , !llvm.loop !{{[0-9]*}} +; CHECK: br i1 %[[CMP4:.*]], label %[[LOOP_PH]], label %for.end, !llvm.loop !{{.*}} ; CHECK: for.end: ; CHECK: ret void @@ -37,7 +44,7 @@ store i32 %i.05, i32* %p.addr.04, align 4 %inc = add nsw i32 %i.05, 1 %cmp = icmp slt i32 %inc, %k - br i1 %cmp, label %for.body, label %for.cond.for.end_crit_edge + br i1 %cmp, label %for.body, label %for.cond.for.end_crit_edge, !llvm.loop !1 for.cond.for.end_crit_edge: ; preds = %for.body br label %for.end @@ -46,6 +53,8 @@ ret void } +!1 = distinct !{!1} + ; Make sure peeling works correctly when a value defined in a loop is used ; in later code - we need to correctly plumb the phi depending on which ; iteration is actually used. @@ -55,17 +64,24 @@ ; CHECK: [[NEXT0]]: ; CHECK: store i32 0, i32* %p, align 4 ; CHECK: %[[CMP1:.*]] = icmp eq i32 %k, 1 -; CHECK: br i1 %[[CMP1]], label %for.end, label %[[NEXT1:.*]] +; CHECK: br i1 %[[CMP1]], label %for.end, label %[[NEXT1:[^,]*]] +; Verify that MD_loop metadata is dropped. +; CHECK-NOT: , !llvm.loop !{{[0-9]*}} ; CHECK: [[NEXT1]]: ; CHECK: %[[INC1:.*]] = getelementptr inbounds i32, i32* %p, i64 1 ; CHECK: store i32 1, i32* %[[INC1]], align 4 ; CHECK: %[[CMP2:.*]] = icmp sgt i32 %k, 2 ; CHECK: br i1 %[[CMP2]], label %[[NEXT2:.*]], label %for.end +; Verify that MD_loop metadata is dropped. +; CHECK-NOT: , !llvm.loop !{{[0-9]*}} ; CHECK: [[NEXT2]]: ; CHECK: %[[INC2:.*]] = getelementptr inbounds i32, i32* %p, i64 2 ; CHECK: store i32 2, i32* %[[INC2]], align 4 ; CHECK: %[[CMP3:.*]] = icmp eq i32 %k, 3 -; CHECK: br i1 %[[CMP3]], label %for.end, label %[[LOOP_PH:.*]] +; CHECK: br i1 %[[CMP3]], label %for.end, label %[[LOOP_PH:[^,]*]] +; Verify that MD_loop metadata is dropped. +; CHECK-NOT: , !llvm.loop !{{[0-9]*}} +; CHECK: br i1 %[[CMP4:.*]], label %[[LOOP_PH]], label %for.end, !llvm.loop !{{.*}} ; CHECK: for.end: ; CHECK: %ret = phi i32 [ 0, %entry ], [ 1, %[[NEXT0]] ], [ 2, %[[NEXT1]] ], [ 3, %[[NEXT2]] ], [ %inc, %for.body ] ; CHECK: ret i32 %ret @@ -84,7 +100,7 @@ store i32 %i.05, i32* %p.addr.04, align 4 %inc = add nsw i32 %i.05, 1 %cmp = icmp slt i32 %inc, %k - br i1 %cmp, label %for.body, label %for.cond.for.end_crit_edge + br i1 %cmp, label %for.body, label %for.cond.for.end_crit_edge, !llvm.loop !2 for.cond.for.end_crit_edge: ; preds = %for.body br label %for.end @@ -93,3 +109,5 @@ %ret = phi i32 [ 0, %entry], [ %inc, %for.cond.for.end_crit_edge ] ret i32 %ret } + +!2 = distinct !{!2}