Index: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp =================================================================== --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -3131,12 +3131,6 @@ PHINode *OrigPhi = InductionEntry.first; InductionDescriptor II = InductionEntry.second; - // Create phi nodes to merge from the backedge-taken check block. - PHINode *BCResumeVal = - PHINode::Create(OrigPhi->getType(), 3, "bc.resume.val", - LoopScalarPreHeader->getTerminator()); - // Copy original phi DL over to the new one. - BCResumeVal->setDebugLoc(OrigPhi->getDebugLoc()); Value *&EndValue = IVEndValues[OrigPhi]; Value *EndValueFromAdditionalBypass = AdditionalBypass.second; if (OrigPhi == OldInduction) { @@ -3172,6 +3166,14 @@ EndValueFromAdditionalBypass->setName("ind.end"); } } + + // Create phi nodes to merge from the backedge-taken check block. + PHINode *BCResumeVal = + PHINode::Create(OrigPhi->getType(), 3, "bc.resume.val", + LoopScalarPreHeader->getTerminator()); + // Copy original phi DL over to the new one. + BCResumeVal->setDebugLoc(OrigPhi->getDebugLoc()); + // The new PHI merges the original incoming value, in case of a bypass, // or the value at the end of the vectorized loop. BCResumeVal->addIncoming(EndValue, LoopMiddleBlock); Index: llvm/test/Transforms/LoopVectorize/create-induction-resume.ll =================================================================== --- /dev/null +++ llvm/test/Transforms/LoopVectorize/create-induction-resume.ll @@ -0,0 +1,122 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -S -passes=loop-vectorize < %s | FileCheck %s + +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128-ni:1-p2:32:8:8:32-ni:2" +target triple = "x86_64-unknown-linux-gnu" +define void @hoge(i32 %arg) { +; CHECK-LABEL: @hoge( +; CHECK-NEXT: bb: +; CHECK-NEXT: [[TMP0:%.*]] = sub i32 -1, [[ARG:%.*]] +; CHECK-NEXT: br label [[BB1:%.*]] +; CHECK: bb1: +; CHECK-NEXT: [[INDUCTION_IV:%.*]] = phi i32 [ [[INDUCTION_IV_NEXT:%.*]], [[BB3:%.*]] ], [ [[TMP0]], [[BB:%.*]] ] +; CHECK-NEXT: [[INDVAR:%.*]] = phi i32 [ [[INDVAR_NEXT:%.*]], [[BB3]] ], [ 0, [[BB]] ] +; CHECK-NEXT: [[TMP:%.*]] = phi i32 [ [[ARG]], [[BB]] ], [ [[TMP4:%.*]], [[BB3]] ] +; CHECK-NEXT: [[TMP2:%.*]] = phi i32 [ 1, [[BB]] ], [ [[TMP5:%.*]], [[BB3]] ] +; CHECK-NEXT: [[TMP1:%.*]] = mul nuw nsw i32 [[INDVAR]], -1 +; CHECK-NEXT: [[TMP22:%.*]] = add i32 [[TMP1]], -2 +; CHECK-NEXT: br i1 false, label [[BB3]], label [[BB20:%.*]] +; CHECK: bb3: +; CHECK-NEXT: [[TMP4]] = add i32 [[TMP2]], [[TMP]] +; CHECK-NEXT: [[TMP5]] = add nuw nsw i32 [[TMP2]], 1 +; CHECK-NEXT: [[INDVAR_NEXT]] = add i32 [[INDVAR]], 1 +; CHECK-NEXT: [[INDUCTION_IV_NEXT]] = add i32 [[INDUCTION_IV]], [[TMP22]] +; CHECK-NEXT: br i1 false, label [[BB1]], label [[BB6:%.*]] +; CHECK: bb6: +; CHECK-NEXT: [[INDUCTION_IV_LCSSA3:%.*]] = phi i32 [ [[INDUCTION_IV]], [[BB3]] ] +; CHECK-NEXT: [[INDUCTION_IV_LCSSA1:%.*]] = phi i32 [ [[INDUCTION_IV]], [[BB3]] ] +; CHECK-NEXT: [[TMP7:%.*]] = phi i32 [ [[TMP4]], [[BB3]] ] +; CHECK-NEXT: br label [[BB8:%.*]] +; CHECK: bb8: +; CHECK-NEXT: switch i32 undef, label [[BB19:%.*]] [ +; CHECK-NEXT: i32 8, label [[BB10:%.*]] +; CHECK-NEXT: i32 14, label [[BB10]] +; CHECK-NEXT: i32 16, label [[BB10]] +; CHECK-NEXT: i32 20, label [[BB9:%.*]] +; CHECK-NEXT: i32 27, label [[BB10]] +; CHECK-NEXT: ] +; CHECK: bb9: +; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] +; CHECK: vector.ph: +; CHECK-NEXT: [[TMP3:%.*]] = mul i32 12, [[INDUCTION_IV_LCSSA1]] +; CHECK-NEXT: [[IND_END:%.*]] = add i32 undef, [[TMP3]] +; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] +; CHECK: vector.body: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 12 +; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK: middle.block: +; CHECK-NEXT: br i1 true, label [[BB18:%.*]], label [[SCALAR_PH]] +; CHECK: scalar.ph: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ undef, [[BB9]] ] +; CHECK-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i64 [ 13, [[MIDDLE_BLOCK]] ], [ 1, [[BB9]] ] +; CHECK-NEXT: br label [[BB11:%.*]] +; CHECK: bb10: +; CHECK-NEXT: ret void +; CHECK: bb11: +; CHECK-NEXT: [[TMP12:%.*]] = phi i32 [ [[TMP14:%.*]], [[BB11]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] +; CHECK-NEXT: [[TMP13:%.*]] = phi i64 [ [[TMP16:%.*]], [[BB11]] ], [ [[BC_RESUME_VAL2]], [[SCALAR_PH]] ] +; CHECK-NEXT: [[TMP14]] = sub i32 [[TMP12]], [[TMP7]] +; CHECK-NEXT: [[TMP15:%.*]] = sext i32 [[TMP14]] to i64 +; CHECK-NEXT: [[TMP16]] = add nuw nsw i64 [[TMP13]], 1 +; CHECK-NEXT: [[TMP17:%.*]] = icmp ugt i64 [[TMP13]], 11 +; CHECK-NEXT: br i1 [[TMP17]], label [[BB18]], label [[BB11]], !llvm.loop [[LOOP2:![0-9]+]] +; CHECK: bb18: +; CHECK-NEXT: br label [[BB19]] +; CHECK: bb19: +; CHECK-NEXT: br label [[BB8]] +; CHECK: bb20: +; CHECK-NEXT: ret void +; +bb: + br label %bb1 + +bb1: ; preds = %bb3, %bb + %tmp = phi i32 [ %arg, %bb ], [ %tmp4, %bb3 ] + %tmp2 = phi i32 [ 1, %bb ], [ %tmp5, %bb3 ] + br i1 undef, label %bb3, label %bb20 + +bb3: ; preds = %bb1 + %tmp4 = add i32 %tmp2, %tmp + %tmp5 = add nuw nsw i32 %tmp2, 1 + br i1 undef, label %bb1, label %bb6 + +bb6: ; preds = %bb3 + %tmp7 = phi i32 [ %tmp4, %bb3 ] + br label %bb8 + +bb8: ; preds = %bb19, %bb6 + switch i32 undef, label %bb19 [ + i32 8, label %bb10 + i32 14, label %bb10 + i32 16, label %bb10 + i32 20, label %bb9 + i32 27, label %bb10 + ] + +bb9: ; preds = %bb8 + br label %bb11 + +bb10: ; preds = %bb8, %bb8, %bb8, %bb8 + ret void + +bb11: ; preds = %bb11, %bb9 + %tmp12 = phi i32 [ %tmp14, %bb11 ], [ undef, %bb9 ] + %tmp13 = phi i64 [ %tmp16, %bb11 ], [ 1, %bb9 ] + %tmp14 = sub i32 %tmp12, %tmp7 + %tmp15 = sext i32 %tmp14 to i64 + %tmp16 = add nuw nsw i64 %tmp13, 1 + %tmp17 = icmp ugt i64 %tmp13, 11 + br i1 %tmp17, label %bb18, label %bb11 + +bb18: ; preds = %bb11 + br label %bb19 + +bb19: ; preds = %bb18, %bb8 + br label %bb8 + +bb20: ; preds = %bb1 + ret void +} +