[NFC][LoopVectorize] Add tail-folding RUN lines to existing SVE tests

[david-arm]

Summary

In this patch I am trying to increase our vectorisation test coverage
by adding extra RUN lines to all tests that don't currently use
tail-folding.

[david-arm]

Hi all, I realise this is a big patch. All I'm doing is adding RUN lines to existing tests so there is no functional change, although I did have to tweak some of the CHECK lines so that I could sometimes share the same CHECK line between RUN lines. If you feel this patch is too big to review I'm happy to split it up into smaller ones!

[david-arm]

• Updated tests to only include those tests that would fail with tail-folding enabled by default. These tests are a subset of those changed in D129137

[david-arm]

• Rebased on top of D132219 to make the review easier.

[CarolineConcatto]

Hey David,
About the manual changes, why not use the python script to add the extra check-lines for tail folding?

[david-arm]

In D129986#3805329, @CarolineConcatto wrote:

Hey David,
About the manual changes, why not use the python script to add the extra check-lines for tail folding?

So for many of the existing tests, e.g. scalable-reductions.ll, we've tried in the past to keep the number of CHECK lines to a minimum because most of the CHECK lines aren't required for the thing we want to test.The vectoriser generates lots of preheader instructions, plus it leaves in a lot of unnecessary code after the vector loop too. However, this does place more burden on the engineer (i.e. me in this case!) to hand-write the CHECK lines. I do think it makes the tests a bit more readable, although that's just my opinion!

llvm/test/Transforms/LoopVectorize/AArch64/scalable-reductions.ll

 ; RUN: opt < %s -loop-vectorize -prefer-predicate-over-epilogue=scalar-epilogue -pass-remarks=loop-vectorize -pass-remarks-analysis=loop-vectorize \
 ; RUN:   -pass-remarks-missed=loop-vectorize -mtriple aarch64-unknown-linux-gnu -mattr=+sve,+bf16 -S 2>%t | FileCheck %s -check-prefix=CHECK
 ; RUN: cat %t | FileCheck %s -check-prefix=CHECK-REMARK
+; RUN: opt < %s -loop-vectorize -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -pass-remarks=loop-vectorize -pass-remarks-analysis=loop-vectorize \
+; RUN:   -pass-remarks-missed=loop-vectorize -mtriple aarch64-unknown-linux-gnu -mattr=+sve,+bf16 -S | FileCheck %s --check-prefix=CHECK-TF
 
 ; Reduction can be vectorized
 
 ; ADD
 
 ; CHECK-REMARK: vectorized loop (vectorization width: vscale x 8, interleaved count: 2)
 define i32 @add(i32* nocapture %a, i32* nocapture readonly %b, i64 %n) {
 ; CHECK-LABEL: @add
 ; CHECK: vector.body:
 ; CHECK: %[[LOAD1:.*]] = load <vscale x 8 x i32>
 ; CHECK: %[[LOAD2:.*]] = load <vscale x 8 x i32>
 ; CHECK: %[[ADD1:.*]] = add <vscale x 8 x i32> %[[LOAD1]]
 ; CHECK: %[[ADD2:.*]] = add <vscale x 8 x i32> %[[LOAD2]]
 ; CHECK: middle.block:
 ; CHECK: %[[ADD:.*]] = add <vscale x 8 x i32> %[[ADD2]], %[[ADD1]]
 ; CHECK-NEXT: call i32 @llvm.vector.reduce.add.nxv8i32(<vscale x 8 x i32> %[[ADD]])
+
+; CHECK-TF-LABEL: @add
+; CHECK-TF: vector.body:
+; CHECK-TF: %[[ACTIVE_LANE_MASK1:.*]] = phi <vscale x 8 x i1>
+; CHECK-TF: %[[ACTIVE_LANE_MASK2:.*]] = phi <vscale x 8 x i1>
+; CHECK-TF: %[[VEC_PHI1:.*]] = phi <vscale x 8 x i32>
+; CHECK-TF: %[[VEC_PHI2:.*]] = phi <vscale x 8 x i32>
+; CHECK-TF: %[[LOAD1:.*]] = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32
+; CHECK-TF: %[[LOAD2:.*]] = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32
+; CHECK-TF: %[[ADD1:.*]] = add <vscale x 8 x i32> %[[LOAD1]]
+; CHECK-TF: %[[ADD2:.*]] = add <vscale x 8 x i32> %[[LOAD2]]
+; CHECK-TF: %[[SEL_ADD1:.*]] = select <vscale x 8 x i1> %[[ACTIVE_LANE_MASK1]], <vscale x 8 x i32> %[[ADD1]], <vscale x 8 x i32> %[[VEC_PHI1]]
+; CHECK-TF: %[[SEL_ADD2:.*]] = select <vscale x 8 x i1> %[[ACTIVE_LANE_MASK2]], <vscale x 8 x i32> %[[ADD2]], <vscale x 8 x i32> %[[VEC_PHI2]]
+; CHECK-TF: middle.block:
+; CHECK-TF: %[[ADD:.*]] = add <vscale x 8 x i32> %[[SEL_ADD2]], %[[SEL_ADD1]]
+; CHECK-TF-NEXT: call i32 @llvm.vector.reduce.add.nxv8i32(<vscale x 8 x i32> %[[ADD]])
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.body
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
   %sum.07 = phi i32 [ 2, %entry ], [ %add, %for.body ]
   %arrayidx = getelementptr inbounds i32, i32* %a, i64 %iv
   %0 = load i32, i32* %arrayidx, align 4

 ; CHECK: vector.body:
 ; CHECK: %[[LOAD1:.*]] = load <vscale x 8 x i32>
 ; CHECK: %[[LOAD2:.*]] = load <vscale x 8 x i32>
 ; CHECK: %[[OR1:.*]] = or <vscale x 8 x i32> %[[LOAD1]]
 ; CHECK: %[[OR2:.*]] = or <vscale x 8 x i32> %[[LOAD2]]
 ; CHECK: middle.block:
 ; CHECK: %[[OR:.*]] = or <vscale x 8 x i32> %[[OR2]], %[[OR1]]
 ; CHECK-NEXT: call i32 @llvm.vector.reduce.or.nxv8i32(<vscale x 8 x i32> %[[OR]])
+
+; CHECK-TF-LABEL: @or
+; CHECK-TF: vector.body:
+; CHECK-TF: %[[ACTIVE_LANE_MASK1:.*]] = phi <vscale x 8 x i1>
+; CHECK-TF: %[[ACTIVE_LANE_MASK2:.*]] = phi <vscale x 8 x i1>
+; CHECK-TF: %[[VEC_PHI1:.*]] = phi <vscale x 8 x i32>
+; CHECK-TF: %[[VEC_PHI2:.*]] = phi <vscale x 8 x i32>
+; CHECK-TF: %[[LOAD1:.*]] = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32
+; CHECK-TF: %[[LOAD2:.*]] = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32
+; CHECK-TF: %[[OR1:.*]] = or <vscale x 8 x i32> %[[LOAD1]]
+; CHECK-TF: %[[OR2:.*]] = or <vscale x 8 x i32> %[[LOAD2]]
+; CHECK-TF: %[[SEL_OR1:.*]] = select <vscale x 8 x i1> %[[ACTIVE_LANE_MASK1]], <vscale x 8 x i32> %[[OR1]], <vscale x 8 x i32> %[[VEC_PHI1]]
+; CHECK-TF: %[[SEL_OR2:.*]] = select <vscale x 8 x i1> %[[ACTIVE_LANE_MASK2]], <vscale x 8 x i32> %[[OR2]], <vscale x 8 x i32> %[[VEC_PHI2]]
+; CHECK-TF: middle.block:
+; CHECK-TF: %[[OR:.*]] = or <vscale x 8 x i32> %[[SEL_OR2]], %[[SEL_OR1]]
+; CHECK-TF-NEXT: call i32 @llvm.vector.reduce.or.nxv8i32(<vscale x 8 x i32> %[[OR]])
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.body
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
   %sum.07 = phi i32 [ 2, %entry ], [ %or, %for.body ]
   %arrayidx = getelementptr inbounds i32, i32* %a, i64 %iv
   %0 = load i32, i32* %arrayidx, align 4

 ; CHECK: vector.body:
 ; CHECK: %[[LOAD1:.*]] = load <vscale x 8 x i32>
 ; CHECK: %[[LOAD2:.*]] = load <vscale x 8 x i32>
 ; CHECK: %[[AND1:.*]] = and <vscale x 8 x i32> %[[LOAD1]]
 ; CHECK: %[[AND2:.*]] = and <vscale x 8 x i32> %[[LOAD2]]
 ; CHECK: middle.block:
 ; CHECK: %[[ABD:.*]] = and <vscale x 8 x i32> %[[ADD2]], %[[AND1]]
 ; CHECK-NEXT: call i32 @llvm.vector.reduce.and.nxv8i32(<vscale x 8 x i32> %[[ADD]])
+
+; CHECK-TF-LABEL: @and
+; CHECK-TF: vector.body:
+; CHECK-TF: %[[ACTIVE_LANE_MASK1:.*]] = phi <vscale x 8 x i1>
+; CHECK-TF: %[[ACTIVE_LANE_MASK2:.*]] = phi <vscale x 8 x i1>
+; CHECK-TF: %[[VEC_PHI1:.*]] = phi <vscale x 8 x i32>
+; CHECK-TF: %[[VEC_PHI2:.*]] = phi <vscale x 8 x i32>
+; CHECK-TF: %[[LOAD1:.*]] = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32
+; CHECK-TF: %[[LOAD2:.*]] = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32
+; CHECK-TF: %[[AND1:.*]] = and <vscale x 8 x i32> %[[LOAD1]]
+; CHECK-TF: %[[AND2:.*]] = and <vscale x 8 x i32> %[[LOAD2]]
+; CHECK-TF: %[[SEL_AND1:.*]] = select <vscale x 8 x i1> %[[ACTIVE_LANE_MASK1]], <vscale x 8 x i32> %[[AND1]], <vscale x 8 x i32> %[[VEC_PHI1]]
+; CHECK-TF: %[[SEL_AND2:.*]] = select <vscale x 8 x i1> %[[ACTIVE_LANE_MASK2]], <vscale x 8 x i32> %[[AND2]], <vscale x 8 x i32> %[[VEC_PHI2]]
+; CHECK-TF: middle.block:
+; CHECK-TF: %[[AND:.*]] = and <vscale x 8 x i32> %[[SEL_AND2]], %[[SEL_AND1]]
+; CHECK-TF-NEXT: call i32 @llvm.vector.reduce.and.nxv8i32(<vscale x 8 x i32> %[[AND]])
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.body
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
   %sum.07 = phi i32 [ 2, %entry ], [ %and, %for.body ]
   %arrayidx = getelementptr inbounds i32, i32* %a, i64 %iv
   %0 = load i32, i32* %arrayidx, align 4

 ; CHECK: vector.body:
 ; CHECK: %[[LOAD1:.*]] = load <vscale x 8 x i32>
 ; CHECK: %[[LOAD2:.*]] = load <vscale x 8 x i32>
 ; CHECK: %[[XOR1:.*]] = xor <vscale x 8 x i32> %[[LOAD1]]
 ; CHECK: %[[XOR2:.*]] = xor <vscale x 8 x i32> %[[LOAD2]]
 ; CHECK: middle.block:
 ; CHECK: %[[XOR:.*]] = xor <vscale x 8 x i32> %[[XOR2]], %[[XOR1]]
 ; CHECK-NEXT: call i32 @llvm.vector.reduce.xor.nxv8i32(<vscale x 8 x i32> %[[XOR]])
+
+; CHECK-TF-LABEL: @xor
+; CHECK-TF: vector.body:
+; CHECK-TF: %[[ACTIVE_LANE_MASK1:.*]] = phi <vscale x 8 x i1>
+; CHECK-TF: %[[ACTIVE_LANE_MASK2:.*]] = phi <vscale x 8 x i1>
+; CHECK-TF: %[[VEC_PHI1:.*]] = phi <vscale x 8 x i32>
+; CHECK-TF: %[[VEC_PHI2:.*]] = phi <vscale x 8 x i32>
+; CHECK-TF: %[[LOAD1:.*]] = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32
+; CHECK-TF: %[[LOAD2:.*]] = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32
+; CHECK-TF: %[[XOR1:.*]] = xor <vscale x 8 x i32> %[[LOAD1]]
+; CHECK-TF: %[[XOR2:.*]] = xor <vscale x 8 x i32> %[[LOAD2]]
+; CHECK-TF: %[[SEL_XOR1:.*]] = select <vscale x 8 x i1> %[[ACTIVE_LANE_MASK1]], <vscale x 8 x i32> %[[XOR1]], <vscale x 8 x i32> %[[VEC_PHI1]]
+; CHECK-TF: %[[SEL_XOR2:.*]] = select <vscale x 8 x i1> %[[ACTIVE_LANE_MASK2]], <vscale x 8 x i32> %[[XOR2]], <vscale x 8 x i32> %[[VEC_PHI2]]
+; CHECK-TF: middle.block:
+; CHECK-TF: %[[XOR:.*]] = xor <vscale x 8 x i32> %[[SEL_XOR2]], %[[SEL_XOR1]]
+; CHECK-TF-NEXT: call i32 @llvm.vector.reduce.xor.nxv8i32(<vscale x 8 x i32> %[[XOR]])
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.body
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
   %sum.07 = phi i32 [ 2, %entry ], [ %xor, %for.body ]
   %arrayidx = getelementptr inbounds i32, i32* %a, i64 %iv
   %0 = load i32, i32* %arrayidx, align 4

 ; CHECK: %[[ICMP1:.*]] = icmp slt <vscale x 8 x i32> %[[LOAD1]]
 ; CHECK: %[[ICMP2:.*]] = icmp slt <vscale x 8 x i32> %[[LOAD2]]
 ; CHECK: %[[SEL1:.*]] = select <vscale x 8 x i1> %[[ICMP1]], <vscale x 8 x i32> %[[LOAD1]]
 ; CHECK: %[[SEL2:.*]] = select <vscale x 8 x i1> %[[ICMP2]], <vscale x 8 x i32> %[[LOAD2]]
 ; CHECK: middle.block:
 ; CHECK: %[[ICMP:.*]] = icmp slt <vscale x 8 x i32> %[[SEL1]], %[[SEL2]]
 ; CHECK-NEXT: %[[SEL:.*]] = select <vscale x 8 x i1> %[[ICMP]], <vscale x 8 x i32> %[[SEL1]], <vscale x 8 x i32> %[[SEL2]]
 ; CHECK-NEXT: call i32 @llvm.vector.reduce.smin.nxv8i32(<vscale x 8 x i32>  %[[SEL]])
+
+; CHECK-TF-LABEL: @smin
+; CHECK-TF: vector.body:
+; CHECK-TF: %[[ACTIVE_LANE_MASK1:.*]] = phi <vscale x 8 x i1>
+; CHECK-TF: %[[ACTIVE_LANE_MASK2:.*]] = phi <vscale x 8 x i1>
+; CHECK-TF: %[[VEC_PHI1:.*]] = phi <vscale x 8 x i32>
+; CHECK-TF: %[[VEC_PHI2:.*]] = phi <vscale x 8 x i32>
+; CHECK-TF: %[[LOAD1:.*]] = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32
+; CHECK-TF: %[[LOAD2:.*]] = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32
+; CHECK-TF: %[[ICMP1:.*]] = icmp slt <vscale x 8 x i32> %[[LOAD1]]
+; CHECK-TF: %[[ICMP2:.*]] = icmp slt <vscale x 8 x i32> %[[LOAD2]]
+; CHECK-TF: %[[SEL1:.*]] = select <vscale x 8 x i1> %[[ICMP1]], <vscale x 8 x i32> %[[LOAD1]]
+; CHECK-TF: %[[SEL2:.*]] = select <vscale x 8 x i1> %[[ICMP2]], <vscale x 8 x i32> %[[LOAD2]]
+; CHECK-TF: %[[SEL_SEL1:.*]] = select <vscale x 8 x i1> %[[ACTIVE_LANE_MASK1]], <vscale x 8 x i32> %[[SEL1]], <vscale x 8 x i32> %[[VEC_PHI1]]
+; CHECK-TF: %[[SEL_SEL2:.*]] = select <vscale x 8 x i1> %[[ACTIVE_LANE_MASK2]], <vscale x 8 x i32> %[[SEL2]], <vscale x 8 x i32> %[[VEC_PHI2]]
+; CHECK-TF: middle.block:
+; CHECK-TF: %[[ICMP:.*]] = icmp slt <vscale x 8 x i32> %[[SEL_SEL1]], %[[SEL_SEL2]]
+; CHECK-TF-NEXT: %[[SEL:.*]] = select <vscale x 8 x i1> %[[ICMP]], <vscale x 8 x i32> %[[SEL_SEL1]], <vscale x 8 x i32> %[[SEL_SEL2]]
+; CHECK-TF-NEXT: call i32 @llvm.vector.reduce.smin.nxv8i32(<vscale x 8 x i32> %[[SEL]])
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.body
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
   %sum.010 = phi i32 [ 2, %entry ], [ %.sroa.speculated, %for.body ]
   %arrayidx = getelementptr inbounds i32, i32* %a, i64 %iv
   %0 = load i32, i32* %arrayidx, align 4

 ; CHECK: %[[ICMP1:.*]] = icmp ugt <vscale x 8 x i32> %[[LOAD1]]
 ; CHECK: %[[ICMP2:.*]] = icmp ugt <vscale x 8 x i32> %[[LOAD2]]
 ; CHECK: %[[SEL1:.*]] = select <vscale x 8 x i1> %[[ICMP1]], <vscale x 8 x i32> %[[LOAD1]]
 ; CHECK: %[[SEL2:.*]] = select <vscale x 8 x i1> %[[ICMP2]], <vscale x 8 x i32> %[[LOAD2]]
 ; CHECK: middle.block:
 ; CHECK: %[[ICMP:.*]] = icmp ugt <vscale x 8 x i32> %[[SEL1]], %[[SEL2]]
 ; CHECK-NEXT: %[[SEL:.*]] = select <vscale x 8 x i1> %[[ICMP]], <vscale x 8 x i32> %[[SEL1]], <vscale x 8 x i32> %[[SEL2]]
 ; CHECK-NEXT: call i32 @llvm.vector.reduce.umax.nxv8i32(<vscale x 8 x i32>  %[[SEL]])
+
+; CHECK-TF-LABEL: @umax
+; CHECK-TF: vector.body:
+; CHECK-TF: %[[ACTIVE_LANE_MASK1:.*]] = phi <vscale x 8 x i1>
+; CHECK-TF: %[[ACTIVE_LANE_MASK2:.*]] = phi <vscale x 8 x i1>
+; CHECK-TF: %[[VEC_PHI1:.*]] = phi <vscale x 8 x i32>
+; CHECK-TF: %[[VEC_PHI2:.*]] = phi <vscale x 8 x i32>
+; CHECK-TF: %[[LOAD1:.*]] = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32
+; CHECK-TF: %[[LOAD2:.*]] = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32
+; CHECK-TF: %[[ICMP1:.*]] = icmp ugt <vscale x 8 x i32> %[[LOAD1]]
+; CHECK-TF: %[[ICMP2:.*]] = icmp ugt <vscale x 8 x i32> %[[LOAD2]]
+; CHECK-TF: %[[SEL1:.*]] = select <vscale x 8 x i1> %[[ICMP1]], <vscale x 8 x i32> %[[LOAD1]]
+; CHECK-TF: %[[SEL2:.*]] = select <vscale x 8 x i1> %[[ICMP2]], <vscale x 8 x i32> %[[LOAD2]]
+; CHECK-TF: %[[SEL_SEL1:.*]] = select <vscale x 8 x i1> %[[ACTIVE_LANE_MASK1]], <vscale x 8 x i32> %[[SEL1]], <vscale x 8 x i32> %[[VEC_PHI1]]
+; CHECK-TF: %[[SEL_SEL2:.*]] = select <vscale x 8 x i1> %[[ACTIVE_LANE_MASK2]], <vscale x 8 x i32> %[[SEL2]], <vscale x 8 x i32> %[[VEC_PHI2]]
+; CHECK-TF: middle.block:
+; CHECK-TF: %[[ICMP:.*]] = icmp ugt <vscale x 8 x i32> %[[SEL_SEL1]], %[[SEL_SEL2]]
+; CHECK-TF-NEXT: %[[SEL:.*]] = select <vscale x 8 x i1> %[[ICMP]], <vscale x 8 x i32> %[[SEL_SEL1]], <vscale x 8 x i32> %[[SEL_SEL2]]
+; CHECK-TF-NEXT: call i32 @llvm.vector.reduce.umax.nxv8i32(<vscale x 8 x i32> %[[SEL]])
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.body
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
   %sum.010 = phi i32 [ 2, %entry ], [ %.sroa.speculated, %for.body ]
   %arrayidx = getelementptr inbounds i32, i32* %a, i64 %iv
   %0 = load i32, i32* %arrayidx, align 4

 ; CHECK: vector.body:
 ; CHECK: %[[LOAD1:.*]] = load <vscale x 8 x float>
 ; CHECK: %[[LOAD2:.*]] = load <vscale x 8 x float>
 ; CHECK: %[[ADD1:.*]] = fadd fast <vscale x 8 x float> %[[LOAD1]]
 ; CHECK: %[[ADD2:.*]] = fadd fast <vscale x 8 x float> %[[LOAD2]]
 ; CHECK: middle.block:
 ; CHECK: %[[ADD:.*]] = fadd fast <vscale x 8 x float> %[[ADD2]], %[[ADD1]]
 ; CHECK-NEXT: call fast float @llvm.vector.reduce.fadd.nxv8f32(float -0.000000e+00, <vscale x 8 x float> %[[ADD]])
+
+; CHECK-TF-LABEL: @fadd_fast
+; CHECK-TF: vector.body:
+; CHECK-TF: %[[ACTIVE_LANE_MASK1:.*]] = phi <vscale x 8 x i1>
+; CHECK-TF: %[[ACTIVE_LANE_MASK2:.*]] = phi <vscale x 8 x i1>
+; CHECK-TF: %[[VEC_PHI1:.*]] = phi <vscale x 8 x float>
+; CHECK-TF: %[[VEC_PHI2:.*]] = phi <vscale x 8 x float>
+; CHECK-TF: %[[LOAD1:.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32
+; CHECK-TF: %[[LOAD2:.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32
+; CHECK-TF: %[[ADD1:.*]] = fadd fast <vscale x 8 x float> %[[LOAD1]]
+; CHECK-TF: %[[ADD2:.*]] = fadd fast <vscale x 8 x float> %[[LOAD2]]
+; CHECK-TF: %[[SEL_ADD1:.*]] = select fast <vscale x 8 x i1> %[[ACTIVE_LANE_MASK1]], <vscale x 8 x float> %[[ADD1]], <vscale x 8 x float> %[[VEC_PHI1]]
+; CHECK-TF: %[[SEL_ADD2:.*]] = select fast <vscale x 8 x i1> %[[ACTIVE_LANE_MASK2]], <vscale x 8 x float> %[[ADD2]], <vscale x 8 x float> %[[VEC_PHI2]]
+; CHECK-TF: middle.block:
+; CHECK-TF: %[[ADD:.*]] = fadd fast <vscale x 8 x float> %[[SEL_ADD2]], %[[SEL_ADD1]]
+; CHECK-TF-NEXT: call fast float @llvm.vector.reduce.fadd.nxv8f32(float -0.000000e+00, <vscale x 8 x float> %[[ADD]])
 entry:
   br label %for.body
 
 for.body:
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
   %sum.07 = phi float [ 0.000000e+00, %entry ], [ %add, %for.body ]
   %arrayidx = getelementptr inbounds float, float* %a, i64 %iv
   %0 = load float, float* %arrayidx, align 4

 ; CHECK: vector.body:
 ; CHECK: %[[LOAD1:.*]] = load <8 x bfloat>
 ; CHECK: %[[LOAD2:.*]] = load <8 x bfloat>
 ; CHECK: %[[FADD1:.*]] = fadd fast <8 x bfloat> %[[LOAD1]]
 ; CHECK: %[[FADD2:.*]] = fadd fast <8 x bfloat> %[[LOAD2]]
 ; CHECK: middle.block:
 ; CHECK: %[[RDX:.*]] = fadd fast <8 x bfloat> %[[FADD2]], %[[FADD1]]
 ; CHECK: call fast bfloat @llvm.vector.reduce.fadd.v8bf16(bfloat 0xR8000, <8 x bfloat> %[[RDX]])
+
+; CHECK-TF-LABEL: @fadd_fast_bfloat
+; CHECK-TF: vector.body:
+; CHECK-TF: %[[ACTIVE_LANE_MASK1:.*]] = phi <8 x i1>
+; CHECK-TF: %[[ACTIVE_LANE_MASK2:.*]] = phi <8 x i1>
+; CHECK-TF: %[[VEC_PHI1:.*]] = phi <8 x bfloat>
+; CHECK-TF: %[[VEC_PHI2:.*]] = phi <8 x bfloat>
+; CHECK-TF: %[[LOAD1:.*]] = call <8 x bfloat> @llvm.masked.load.v8bf16
+; CHECK-TF: %[[LOAD2:.*]] = call <8 x bfloat> @llvm.masked.load.v8bf16
+; CHECK-TF: %[[ADD1:.*]] = fadd fast <8 x bfloat> %[[LOAD1]]
+; CHECK-TF: %[[ADD2:.*]] = fadd fast <8 x bfloat> %[[LOAD2]]
+; CHECK-TF: %[[SEL_ADD1:.*]] = select fast <8 x i1> %[[ACTIVE_LANE_MASK1]], <8 x bfloat> %[[ADD1]], <8 x bfloat> %[[VEC_PHI1]]
+; CHECK-TF: %[[SEL_ADD2:.*]] = select fast <8 x i1> %[[ACTIVE_LANE_MASK2]], <8 x bfloat> %[[ADD2]], <8 x bfloat> %[[VEC_PHI2]]
+; CHECK-TF: middle.block:
+; CHECK-TF: %[[ADD:.*]] = fadd fast <8 x bfloat> %[[SEL_ADD2]], %[[SEL_ADD1]]
+; CHECK-TF-NEXT: call fast bfloat @llvm.vector.reduce.fadd.v8bf16(bfloat 0xR8000, <8 x bfloat> %[[ADD]])
 entry:
   br label %for.body
 
 for.body:
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
   %sum.07 = phi bfloat [ 0.000000e+00, %entry ], [ %add, %for.body ]
   %arrayidx = getelementptr inbounds bfloat, bfloat* %a, i64 %iv
   %0 = load bfloat, bfloat* %arrayidx, align 4

 ; CHECK: %[[FCMP1:.*]] = fcmp fast olt <vscale x 8 x float> %[[LOAD1]]
 ; CHECK: %[[FCMP2:.*]] = fcmp fast olt <vscale x 8 x float> %[[LOAD2]]
 ; CHECK: %[[SEL1:.*]] = select <vscale x 8 x i1> %[[FCMP1]], <vscale x 8 x float> %[[LOAD1]]
 ; CHECK: %[[SEL2:.*]] = select <vscale x 8 x i1> %[[FCMP2]], <vscale x 8 x float> %[[LOAD2]]
 ; CHECK: middle.block:
 ; CHECK: %[[FCMP:.*]] = fcmp fast olt <vscale x 8 x float> %[[SEL1]], %[[SEL2]]
 ; CHECK-NEXT: %[[SEL:.*]] = select fast <vscale x 8 x i1> %[[FCMP]], <vscale x 8 x float> %[[SEL1]], <vscale x 8 x float> %[[SEL2]]
 ; CHECK-NEXT: call fast float @llvm.vector.reduce.fmin.nxv8f32(<vscale x 8 x float> %[[SEL]])
+
+; CHECK-TF-LABEL: @fmin_fast
+; CHECK-TF: vector.body:
+; CHECK-TF: %[[ACTIVE_LANE_MASK1:.*]] = phi <vscale x 8 x i1>
+; CHECK-TF: %[[ACTIVE_LANE_MASK2:.*]] = phi <vscale x 8 x i1>
+; CHECK-TF: %[[VEC_PHI1:.*]] = phi <vscale x 8 x float>
+; CHECK-TF: %[[VEC_PHI2:.*]] = phi <vscale x 8 x float>
+; CHECK-TF: %[[LOAD1:.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32
+; CHECK-TF: %[[LOAD2:.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32
+; CHECK-TF: %[[FCMP1:.*]] = fcmp fast olt <vscale x 8 x float> %[[LOAD1]]
+; CHECK-TF: %[[FCMP2:.*]] = fcmp fast olt <vscale x 8 x float> %[[LOAD2]]
+; CHECK-TF: %[[SEL1:.*]] = select <vscale x 8 x i1> %[[ICMP1]], <vscale x 8 x float> %[[LOAD1]]
+; CHECK-TF: %[[SEL2:.*]] = select <vscale x 8 x i1> %[[ICMP2]], <vscale x 8 x float> %[[LOAD2]]
+; CHECK-TF: %[[SEL_SEL1:.*]] = select fast <vscale x 8 x i1> %[[ACTIVE_LANE_MASK1]], <vscale x 8 x float> %[[SEL1]], <vscale x 8 x float> %[[VEC_PHI1]]
+; CHECK-TF: %[[SEL_SEL2:.*]] = select fast <vscale x 8 x i1> %[[ACTIVE_LANE_MASK2]], <vscale x 8 x float> %[[SEL2]], <vscale x 8 x float> %[[VEC_PHI2]]
+; CHECK-TF: middle.block:
+; CHECK-TF: %[[FCMP:.*]] = fcmp fast olt <vscale x 8 x float> %[[SEL_SEL1]], %[[SEL_SEL2]]
+; CHECK-TF-NEXT: %[[SEL:.*]] = select fast <vscale x 8 x i1> %[[FCMP]], <vscale x 8 x float> %[[SEL_SEL1]], <vscale x 8 x float> %[[SEL_SEL2]]
+; CHECK-TF-NEXT: call fast float @llvm.vector.reduce.fmin.nxv8f32(<vscale x 8 x float> %[[SEL]])
 entry:
   br label %for.body
 
 for.body:
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
   %sum.07 = phi float [ 0.000000e+00, %entry ], [ %.sroa.speculated, %for.body ]
   %arrayidx = getelementptr inbounds float, float* %a, i64 %iv
   %0 = load float, float* %arrayidx, align 4

 ; CHECK: %[[FCMP1:.*]] = fcmp fast ogt <vscale x 8 x float> %[[LOAD1]]
 ; CHECK: %[[FCMP2:.*]] = fcmp fast ogt <vscale x 8 x float> %[[LOAD2]]
 ; CHECK: %[[SEL1:.*]] = select <vscale x 8 x i1> %[[FCMP1]], <vscale x 8 x float> %[[LOAD1]]
 ; CHECK: %[[SEL2:.*]] = select <vscale x 8 x i1> %[[FCMP2]], <vscale x 8 x float> %[[LOAD2]]
 ; CHECK: middle.block:
 ; CHECK: %[[FCMP:.*]] = fcmp fast ogt <vscale x 8 x float> %[[SEL1]], %[[SEL2]]
 ; CHECK-NEXT: %[[SEL:.*]] = select fast <vscale x 8 x i1> %[[FCMP]], <vscale x 8 x float> %[[SEL1]], <vscale x 8 x float> %[[SEL2]]
 ; CHECK-NEXT: call fast float @llvm.vector.reduce.fmax.nxv8f32(<vscale x 8 x float> %[[SEL]])
+
+; CHECK-TF-LABEL: @fmax_fast
+; CHECK-TF: vector.body:
+; CHECK-TF: %[[ACTIVE_LANE_MASK1:.*]] = phi <vscale x 8 x i1>
+; CHECK-TF: %[[ACTIVE_LANE_MASK2:.*]] = phi <vscale x 8 x i1>
+; CHECK-TF: %[[VEC_PHI1:.*]] = phi <vscale x 8 x float>
+; CHECK-TF: %[[VEC_PHI2:.*]] = phi <vscale x 8 x float>
+; CHECK-TF: %[[LOAD1:.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32
+; CHECK-TF: %[[LOAD2:.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32
+; CHECK-TF: %[[FCMP1:.*]] = fcmp fast ogt <vscale x 8 x float> %[[LOAD1]]
+; CHECK-TF: %[[FCMP2:.*]] = fcmp fast ogt <vscale x 8 x float> %[[LOAD2]]
+; CHECK-TF: %[[SEL1:.*]] = select <vscale x 8 x i1> %[[ICMP1]], <vscale x 8 x float> %[[LOAD1]]
+; CHECK-TF: %[[SEL2:.*]] = select <vscale x 8 x i1> %[[ICMP2]], <vscale x 8 x float> %[[LOAD2]]
+; CHECK-TF: %[[SEL_SEL1:.*]] = select fast <vscale x 8 x i1> %[[ACTIVE_LANE_MASK1]], <vscale x 8 x float> %[[SEL1]], <vscale x 8 x float> %[[VEC_PHI1]]
+; CHECK-TF: %[[SEL_SEL2:.*]] = select fast <vscale x 8 x i1> %[[ACTIVE_LANE_MASK2]], <vscale x 8 x float> %[[SEL2]], <vscale x 8 x float> %[[VEC_PHI2]]
+; CHECK-TF: middle.block:
+; CHECK-TF: %[[FCMP:.*]] = fcmp fast ogt <vscale x 8 x float> %[[SEL_SEL1]], %[[SEL_SEL2]]
+; CHECK-TF-NEXT: %[[SEL:.*]] = select fast <vscale x 8 x i1> %[[FCMP]], <vscale x 8 x float> %[[SEL_SEL1]], <vscale x 8 x float> %[[SEL_SEL2]]
+; CHECK-TF-NEXT: call fast float @llvm.vector.reduce.fmax.nxv8f32(<vscale x 8 x float> %[[SEL]])
 entry:
   br label %for.body
 
 for.body:
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
   %sum.07 = phi float [ 0.000000e+00, %entry ], [ %.sroa.speculated, %for.body ]
   %arrayidx = getelementptr inbounds float, float* %a, i64 %iv
   %0 = load float, float* %arrayidx, align 4

 ; CHECK: %[[LOAD1:.*]] = load <vscale x 4 x i32>
 ; CHECK: %[[LOAD2:.*]] = load <vscale x 4 x i32>
 ; CHECK: %[[ADD1:.*]] = add <vscale x 4 x i32> %{{.*}}, %[[LOAD1]]
 ; CHECK: %[[ADD2:.*]] = add <vscale x 4 x i32> %{{.*}}, %[[LOAD2]]
 ; CHECK: middle.block:
 ; CHECK: %[[ADD:.*]] = add <vscale x 4 x i32> %[[ADD2]], %[[ADD1]]
 ; CHECK-NEXT: %[[SUM:.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %[[ADD]])
 ; CHECK-NEXT: store i32 %[[SUM]], i32* %gep.dst, align 4
+
+; CHECK-TF-LABEL: @invariant_store
+; CHECK-TF: vector.body:
+; CHECK-TF: %[[ACTIVE_LANE_MASK1:.*]] = phi <vscale x 4 x i1>
+; CHECK-TF: %[[LOAD1:.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32
+; CHECK-TF: %[[ADD1:.*]] = add <vscale x 4 x i32> %{{.*}}, %[[LOAD1]]
+; CHECK-TF-NEXT: %[[SEL:.*]] = select <vscale x 4 x i1> %[[ACTIVE_LANE_MASK1]], <vscale x 4 x i32> %[[ADD1]], <vscale x 4 x i32>
+; CHECK-TF: middle.block:
+; CHECK-TF-NEXT: %[[SUM:.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %[[SEL]])
+; CHECK-TF-NEXT: store i32 %[[SUM]], i32* %gep.dst, align 4
+
 entry:
   %gep.dst = getelementptr inbounds i32, i32* %dst, i64 42
   store i32 0, i32* %gep.dst, align 4
   br label %for.body
 for.body:
   %sum = phi i32 [ 0, %entry ], [ %add, %for.body ]
   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
   %gep.src = getelementptr inbounds i32, i32* %src, i64 %indvars.iv

 ; CHECK: vector.body:
 ; CHECK: %[[LOAD1:.*]] = load <4 x i32>
 ; CHECK: %[[LOAD2:.*]] = load <4 x i32>
 ; CHECK: %[[MUL1:.*]] = mul <4 x i32> %[[LOAD1]]
 ; CHECK: %[[MUL2:.*]] = mul <4 x i32> %[[LOAD2]]
 ; CHECK: middle.block:
 ; CHECK: %[[RDX:.*]] = mul <4 x i32> %[[MUL2]], %[[MUL1]]
 ; CHECK: call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> %[[RDX]])
+
+; CHECK-TF-LABEL: @mul
+; CHECK-TF: vector.body:
+; CHECK-TF: %[[ACTIVE_LANE_MASK1:.*]] = phi <2 x i1>
+; CHECK-TF: %[[ACTIVE_LANE_MASK2:.*]] = phi <2 x i1>
+; CHECK-TF: %[[VEC_PHI1:.*]] = phi <2 x i32>
+; CHECK-TF: %[[VEC_PHI2:.*]] = phi <2 x i32>
+; CHECK-TF: %[[LOAD1:.*]] = call <2 x i32> @llvm.masked.load.v2i32
+; CHECK-TF: %[[LOAD2:.*]] = call <2 x i32> @llvm.masked.load.v2i32
+; CHECK-TF: %[[MUL1:.*]] = mul <2 x i32> %[[LOAD1]]
+; CHECK-TF: %[[MUL2:.*]] = mul <2 x i32> %[[LOAD2]]
+; CHECK-TF: %[[SEL_MUL1:.*]] = select <2 x i1> %[[ACTIVE_LANE_MASK1]], <2 x i32> %[[MUL1]], <2 x i32> %[[VEC_PHI1]]
+; CHECK-TF: %[[SEL_MUL2:.*]] = select <2 x i1> %[[ACTIVE_LANE_MASK2]], <2 x i32> %[[MUL2]], <2 x i32> %[[VEC_PHI2]]
+; CHECK-TF: middle.block:
+; CHECK-TF: %[[RDX:.*]] = mul <2 x i32> %[[SEL_MUL2]], %[[SEL_MUL1]]
+; CHECK-TF-NEXT: call i32 @llvm.vector.reduce.mul.v2i32(<2 x i32> %[[RDX]])
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.body
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
   %sum.07 = phi i32 [ 2, %entry ], [ %mul, %for.body ]
   %arrayidx = getelementptr inbounds i32, i32* %a, i64 %iv
   %0 = load i32, i32* %arrayidx, align 4

 ; CHECK: %[[LOAD4:.*]] = load <4 x i32>
 ; CHECK: %[[ADD1:.*]] = add nsw <4 x i32> %[[LOAD3]], %[[LOAD1]]
 ; CHECK: %[[ADD2:.*]] = add nsw <4 x i32> %[[LOAD4]], %[[LOAD2]]
 ; CHECK: %[[MUL1:.*]] = mul <4 x i32> %[[LOAD3]]
 ; CHECK: %[[MUL2:.*]] = mul <4 x i32> %[[LOAD4]]
 ; CHECK: middle.block:
 ; CHECK: %[[RDX:.*]] = mul <4 x i32> %[[MUL2]], %[[MUL1]]
 ; CHECK: call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> %[[RDX]])
+
+; CHECK-TF-LABEL: @memory_dependence
+; CHECK-TF: vector.body:
+; CHECK-TF: %[[ACTIVE_LANE_MASK1:.*]] = phi <2 x i1>
+; CHECK-TF: %[[ACTIVE_LANE_MASK2:.*]] = phi <2 x i1>
+; CHECK-TF: %[[VEC_PHI1:.*]] = phi <2 x i32>
+; CHECK-TF: %[[VEC_PHI2:.*]] = phi <2 x i32>
+; CHECK-TF: %[[LOAD1:.*]] = call <2 x i32> @llvm.masked.load.v2i32
+; CHECK-TF: %[[LOAD2:.*]] = call <2 x i32> @llvm.masked.load.v2i32
+; CHECK-TF: %[[LOAD3:.*]] = call <2 x i32> @llvm.masked.load.v2i32
+; CHECK-TF: %[[LOAD4:.*]] = call <2 x i32> @llvm.masked.load.v2i32
+; CHECK-TF: %[[ADD1:.*]] = add nsw <2 x i32> %[[LOAD3]], %[[LOAD1]]
+; CHECK-TF: %[[ADD2:.*]] = add nsw <2 x i32> %[[LOAD4]], %[[LOAD2]]
+; CHECK-TF: %[[MUL1:.*]] = mul <2 x i32> %[[LOAD3]]
+; CHECK-TF: %[[MUL2:.*]] = mul <2 x i32> %[[LOAD4]]
+; CHECK-TF: %[[SEL_MUL1:.*]] = select <2 x i1> %[[ACTIVE_LANE_MASK1]], <2 x i32> %[[MUL1]], <2 x i32> %[[VEC_PHI1]]
+; CHECK-TF: %[[SEL_MUL2:.*]] = select <2 x i1> %[[ACTIVE_LANE_MASK2]], <2 x i32> %[[MUL2]], <2 x i32> %[[VEC_PHI2]]
+; CHECK-TF: middle.block:
+; CHECK-TF: %[[RDX:.*]] = mul <2 x i32> %[[SEL_MUL2]], %[[SEL_MUL1]]
+; CHECK-TF-NEXT: call i32 @llvm.vector.reduce.mul.v2i32(<2 x i32> %[[RDX]])
 entry:
   br label %for.body
 
 for.body:
   %i = phi i64 [ %inc, %for.body ], [ 0, %entry ]
   %sum = phi i32 [ %mul, %for.body ], [ 2, %entry ]
   %arrayidx = getelementptr inbounds i32, i32* %a, i64 %i
   %0 = load i32, i32* %arrayidx, align 4

llvm/test/Transforms/LoopVectorize/AArch64/sve-basic-vec.ll

 ; RUN: opt -loop-vectorize -dce -instcombine -mtriple aarch64-linux-gnu -mattr=+sve \
 ; RUN:   -prefer-predicate-over-epilogue=scalar-epilogue < %s -S | FileCheck %s
+; RUN: opt -loop-vectorize -dce -instcombine -mtriple aarch64-linux-gnu -mattr=+sve \
+; RUN:   -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue < %s -S | FileCheck %s --check-prefix=CHECK-TF
 
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64-unknown-linux-gnu"
 
 define void @cmpsel_i32(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i64 %n) {
 ; CHECK-LABEL: @cmpsel_i32(
 ; CHECK-NEXT:  entry:
 ; CHECK:       vector.body:
 ; CHECK:         [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, <vscale x 4 x i32>* {{.*}}, align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <vscale x 4 x i32> [[WIDE_LOAD]], zeroinitializer
 ; CHECK-NEXT:    [[TMP2:%.*]] = select <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i32> shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 2, i32 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer), <vscale x 4 x i32> shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 10, i32 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
 ; CHECK:         store <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32>* {{.*}}, align 4
 ;
+; CHECK-TF-LABEL: @cmpsel_i32(
+; CHECK-TF-NEXT:  entry:
+; CHECK-TF:       vector.body:
+; CHECK-TF:         [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32
+; CHECK-TF-NEXT:    [[TMP1:%.*]] = icmp eq <vscale x 4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-TF-NEXT:    [[TMP2:%.*]] = select <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i32> shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 2, i32 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer), <vscale x 4 x i32> shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 10, i32 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-TF:         call void @llvm.masked.store.nxv4i32.p0nxv4i32(<vscale x 4 x i32> [[TMP2]]
+;
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.body
   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
   %arrayidx = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
   %0 = load i32, i32* %arrayidx, align 4
   %tobool.not = icmp eq i32 %0, 0

 ; CHECK-LABEL: @cmpsel_f32(
 ; CHECK-NEXT:  entry:
 ; CHECK:       vector.body:
 ; CHECK:         [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, <vscale x 4 x float>* {{.*}}, align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ogt <vscale x 4 x float> [[WIDE_LOAD]], shufflevector (<vscale x 4 x float> insertelement (<vscale x 4 x float> poison, float 3.000000e+00, i32 0), <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer)
 ; CHECK-NEXT:    [[TMP2:%.*]] = select <vscale x 4 x i1> [[TMP1]], <vscale x 4 x float> shufflevector (<vscale x 4 x float> insertelement (<vscale x 4 x float> poison, float 1.000000e+01, i32 0), <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer), <vscale x 4 x float> shufflevector (<vscale x 4 x float> insertelement (<vscale x 4 x float> poison, float 2.000000e+00, i32 0), <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer)
 ; CHECK:         store <vscale x 4 x float> [[TMP2]], <vscale x 4 x float>* {{.*}}, align 4
 
+; CHECK-TF-LABEL: @cmpsel_f32(
+; CHECK-TF-NEXT:  entry:
+; CHECK-TF:       vector.body:
+; CHECK-TF:         [[WIDE_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32
+; CHECK-TF-NEXT:    [[TMP1:%.*]] = fcmp ogt <vscale x 4 x float> [[WIDE_LOAD]], shufflevector (<vscale x 4 x float> insertelement (<vscale x 4 x float> poison, float 3.000000e+00, i32 0), <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-TF-NEXT:    [[TMP2:%.*]] = select <vscale x 4 x i1> [[TMP1]], <vscale x 4 x float> shufflevector (<vscale x 4 x float> insertelement (<vscale x 4 x float> poison, float 1.000000e+01, i32 0), <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer), <vscale x 4 x float> shufflevector (<vscale x 4 x float> insertelement (<vscale x 4 x float> poison, float 2.000000e+00, i32 0), <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-TF:         call void @llvm.masked.store.nxv4f32.p0nxv4f32(<vscale x 4 x float> [[TMP2]]
+;
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.body
   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
   %arrayidx = getelementptr inbounds float, float* %b, i64 %indvars.iv
   %0 = load float, float* %arrayidx, align 4
   %cmp1 = fcmp ogt float %0, 3.000000e+00

 define void @fneg_f32(float* noalias nocapture %a, float* noalias nocapture readonly %b, i64 %n) {
 ; CHECK-LABEL: @fneg_f32(
 ; CHECK-NEXT:  entry:
 ; CHECK:       vector.body:
 ; CHECK:         [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, <vscale x 4 x float>* {{.*}}, align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = fneg <vscale x 4 x float> [[WIDE_LOAD]]
 ; CHECK:         store <vscale x 4 x float> [[TMP1]], <vscale x 4 x float>* {{.*}}, align 4
 
+; CHECK-TF-LABEL: @fneg_f32(
+; CHECK-TF-NEXT:  entry:
+; CHECK-TF:       vector.body:
+; CHECK-TF:         [[WIDE_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32
+; CHECK-TF-NEXT:    [[TMP1:%.*]] = fneg <vscale x 4 x float> [[WIDE_LOAD]]
+; CHECK-TF:         call void @llvm.masked.store.nxv4f32.p0nxv4f32(<vscale x 4 x float> [[TMP1]]
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.body
   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
   %arrayidx = getelementptr inbounds float, float* %b, i64 %indvars.iv
   %0 = load float, float* %arrayidx, align 4
   %fneg = fneg float %0

llvm/test/Transforms/LoopVectorize/AArch64/sve-cond-inv-loads.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -loop-vectorize -dce -instcombine -mtriple aarch64-linux-gnu -mattr=+sve \
 ; RUN:   -prefer-predicate-over-epilogue=scalar-epilogue -S %s -o - | FileCheck %s
+; RUN: opt -loop-vectorize -dce -instcombine -mtriple aarch64-linux-gnu -mattr=+sve \
+; RUN:   -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -S %s -o - | FileCheck %s --check-prefix=CHECK-TF
 
 define void @cond_inv_load_i32i32i16(i32* noalias nocapture %a, i32* noalias nocapture readonly %cond, i16* noalias nocapture readonly %inv, i64 %n) #0 {
 ; CHECK-LABEL: @cond_inv_load_i32i32i16(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ugt i64 [[TMP1]], [[N:%.*]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]

 ; CHECK-NEXT:    br label [[FOR_INC]]
 ; CHECK:       for.inc:
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i64 [[I_07]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
+; CHECK-TF-LABEL: @cond_inv_load_i32i32i16(
+; CHECK-TF-NEXT:  entry:
+; CHECK-TF-NEXT:    [[TMP0:%.*]] = xor i64 [[N:%.*]], -1
+; CHECK-TF-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP1]], 2
+; CHECK-TF-NEXT:    [[TMP3:%.*]] = icmp ugt i64 [[TMP2]], [[TMP0]]
+; CHECK-TF-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-TF:       vector.ph:
+; CHECK-TF-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2
+; CHECK-TF-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 2
+; CHECK-TF-NEXT:    [[TMP8:%.*]] = add nsw i64 [[TMP7]], -1
+; CHECK-TF-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP8]], [[N]]
+; CHECK-TF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
+; CHECK-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i16*> poison, i16* [[INV:%.*]], i64 0
+; CHECK-TF-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i16*> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i16*> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-TF-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-TF:       vector.body:
+; CHECK-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[COND:%.*]], i64 [[INDEX]]
+; CHECK-TF-NEXT:    [[TMP10:%.*]] = bitcast i32* [[TMP9]] to <vscale x 4 x i32>*
+; CHECK-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>* [[TMP10]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-TF-NEXT:    [[TMP11:%.*]] = icmp ne <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], zeroinitializer
+; CHECK-TF-NEXT:    [[TMP12:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i1> [[TMP11]], <vscale x 4 x i1> zeroinitializer
+; CHECK-TF-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i16> @llvm.masked.gather.nxv4i16.nxv4p0i16(<vscale x 4 x i16*> [[BROADCAST_SPLAT]], i32 2, <vscale x 4 x i1> [[TMP12]], <vscale x 4 x i16> undef)
+; CHECK-TF-NEXT:    [[TMP13:%.*]] = sext <vscale x 4 x i16> [[WIDE_MASKED_GATHER]] to <vscale x 4 x i32>
+; CHECK-TF-NEXT:    [[TMP14:%.*]] = getelementptr i32, i32* [[A:%.*]], i64 [[INDEX]]
+; CHECK-TF-NEXT:    [[TMP15:%.*]] = bitcast i32* [[TMP14]] to <vscale x 4 x i32>*
+; CHECK-TF-NEXT:    call void @llvm.masked.store.nxv4i32.p0nxv4i32(<vscale x 4 x i32> [[TMP13]], <vscale x 4 x i32>* [[TMP15]], i32 4, <vscale x 4 x i1> [[TMP12]])
+; CHECK-TF-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP17:%.*]] = shl nuw nsw i64 [[TMP16]], 2
+; CHECK-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP17]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NEXT:    [[TMP18:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NEXT:    br i1 [[TMP18]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF:       middle.block:
+; CHECK-TF-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-TF:       scalar.ph:
+; CHECK-TF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-TF-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-TF:       for.body:
+; CHECK-TF-NEXT:    [[I_07:%.*]] = phi i64 [ [[INC:%.*]], [[FOR_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-TF-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[COND]], i64 [[I_07]]
+; CHECK-TF-NEXT:    [[TMP19:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-TF-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[TMP19]], 0
+; CHECK-TF-NEXT:    br i1 [[TOBOOL_NOT]], label [[FOR_INC]], label [[IF_THEN:%.*]]
+; CHECK-TF:       if.then:
+; CHECK-TF-NEXT:    [[TMP20:%.*]] = load i16, i16* [[INV]], align 2
+; CHECK-TF-NEXT:    [[CONV:%.*]] = sext i16 [[TMP20]] to i32
+; CHECK-TF-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[I_07]]
+; CHECK-TF-NEXT:    store i32 [[CONV]], i32* [[ARRAYIDX1]], align 4
+; CHECK-TF-NEXT:    br label [[FOR_INC]]
+; CHECK-TF:       for.inc:
+; CHECK-TF-NEXT:    [[INC]] = add nuw nsw i64 [[I_07]], 1
+; CHECK-TF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-TF-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-TF:       exit:
+; CHECK-TF-NEXT:    ret void
+;
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.inc
   %i.07 = phi i64 [ %inc, %for.inc ], [ 0, %entry ]
   %arrayidx = getelementptr inbounds i32, i32* %cond, i64 %i.07
   %0 = load i32, i32* %arrayidx, align 4
   %tobool.not = icmp eq i32 %0, 0

 ; CHECK-NEXT:    br label [[FOR_INC]]
 ; CHECK:       for.inc:
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i64 [[I_08]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
+; CHECK-TF-LABEL: @cond_inv_load_f64f64f64(
+; CHECK-TF-NEXT:  entry:
+; CHECK-TF-NEXT:    [[TMP0:%.*]] = xor i64 [[N:%.*]], -1
+; CHECK-TF-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP1]], 2
+; CHECK-TF-NEXT:    [[TMP3:%.*]] = icmp ugt i64 [[TMP2]], [[TMP0]]
+; CHECK-TF-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-TF:       vector.ph:
+; CHECK-TF-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2
+; CHECK-TF-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 2
+; CHECK-TF-NEXT:    [[TMP8:%.*]] = add nsw i64 [[TMP7]], -1
+; CHECK-TF-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP8]], [[N]]
+; CHECK-TF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
+; CHECK-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x double*> poison, double* [[INV:%.*]], i64 0
+; CHECK-TF-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x double*> [[BROADCAST_SPLATINSERT]], <vscale x 4 x double*> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-TF-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-TF:       vector.body:
+; CHECK-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[TMP9:%.*]] = getelementptr inbounds double, double* [[COND:%.*]], i64 [[INDEX]]
+; CHECK-TF-NEXT:    [[TMP10:%.*]] = bitcast double* [[TMP9]] to <vscale x 4 x double>*
+; CHECK-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x double> @llvm.masked.load.nxv4f64.p0nxv4f64(<vscale x 4 x double>* [[TMP10]], i32 8, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x double> poison)
+; CHECK-TF-NEXT:    [[TMP11:%.*]] = fcmp ogt <vscale x 4 x double> [[WIDE_MASKED_LOAD]], shufflevector (<vscale x 4 x double> insertelement (<vscale x 4 x double> poison, double 4.000000e-01, i32 0), <vscale x 4 x double> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-TF-NEXT:    [[TMP12:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i1> [[TMP11]], <vscale x 4 x i1> zeroinitializer
+; CHECK-TF-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x double> @llvm.masked.gather.nxv4f64.nxv4p0f64(<vscale x 4 x double*> [[BROADCAST_SPLAT]], i32 8, <vscale x 4 x i1> [[TMP12]], <vscale x 4 x double> undef)
+; CHECK-TF-NEXT:    [[TMP13:%.*]] = getelementptr double, double* [[A:%.*]], i64 [[INDEX]]
+; CHECK-TF-NEXT:    [[TMP14:%.*]] = bitcast double* [[TMP13]] to <vscale x 4 x double>*
+; CHECK-TF-NEXT:    call void @llvm.masked.store.nxv4f64.p0nxv4f64(<vscale x 4 x double> [[WIDE_MASKED_GATHER]], <vscale x 4 x double>* [[TMP14]], i32 8, <vscale x 4 x i1> [[TMP12]])
+; CHECK-TF-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP16:%.*]] = shl nuw nsw i64 [[TMP15]], 2
+; CHECK-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP16]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NEXT:    [[TMP17:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NEXT:    br i1 [[TMP17]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-TF:       middle.block:
+; CHECK-TF-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-TF:       scalar.ph:
+; CHECK-TF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-TF-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-TF:       for.body:
+; CHECK-TF-NEXT:    [[I_08:%.*]] = phi i64 [ [[INC:%.*]], [[FOR_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-TF-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, double* [[COND]], i64 [[I_08]]
+; CHECK-TF-NEXT:    [[TMP18:%.*]] = load double, double* [[ARRAYIDX]], align 8
+; CHECK-TF-NEXT:    [[CMP1:%.*]] = fcmp ogt double [[TMP18]], 4.000000e-01
+; CHECK-TF-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK-TF:       if.then:
+; CHECK-TF-NEXT:    [[TMP19:%.*]] = load double, double* [[INV]], align 8
+; CHECK-TF-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[I_08]]
+; CHECK-TF-NEXT:    store double [[TMP19]], double* [[ARRAYIDX2]], align 8
+; CHECK-TF-NEXT:    br label [[FOR_INC]]
+; CHECK-TF:       for.inc:
+; CHECK-TF-NEXT:    [[INC]] = add nuw nsw i64 [[I_08]], 1
+; CHECK-TF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-TF-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-TF:       exit:
+; CHECK-TF-NEXT:    ret void
+;
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.inc
   %i.08 = phi i64 [ %inc, %for.inc ], [ 0, %entry ]
   %arrayidx = getelementptr inbounds double, double* %cond, i64 %i.08
   %0 = load double, double* %arrayidx, align 8
   %cmp1 = fcmp ogt double %0, 4.000000e-01

 ; CHECK-NEXT:    br label [[FOR_INC]]
 ; CHECK:       for.inc:
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
 ;
+; CHECK-TF-LABEL: @invariant_load_cond(
+; CHECK-TF-NEXT:  entry:
+; CHECK-TF-NEXT:    [[TMP0:%.*]] = xor i64 [[N:%.*]], -1
+; CHECK-TF-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP1]], 2
+; CHECK-TF-NEXT:    [[TMP3:%.*]] = icmp ugt i64 [[TMP2]], [[TMP0]]
+; CHECK-TF-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-TF:       vector.ph:
+; CHECK-TF-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2
+; CHECK-TF-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 2
+; CHECK-TF-NEXT:    [[TMP8:%.*]] = add nsw i64 [[TMP7]], -1
+; CHECK-TF-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP8]], [[N]]
+; CHECK-TF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
+; CHECK-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-TF:       vector.body:
+; CHECK-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 42
+; CHECK-TF-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i32*> poison, i32* [[TMP9]], i64 0
+; CHECK-TF-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i32*> [[DOTSPLATINSERT]], <vscale x 4 x i32*> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-TF-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[COND:%.*]], i64 [[INDEX]]
+; CHECK-TF-NEXT:    [[TMP11:%.*]] = bitcast i32* [[TMP10]] to <vscale x 4 x i32>*
+; CHECK-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>* [[TMP11]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-TF-NEXT:    [[TMP12:%.*]] = icmp ne <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], zeroinitializer
+; CHECK-TF-NEXT:    [[TMP13:%.*]] = getelementptr i32, i32* [[B]], i64 [[INDEX]]
+; CHECK-TF-NEXT:    [[TMP14:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i1> [[TMP12]], <vscale x 4 x i1> zeroinitializer
+; CHECK-TF-NEXT:    [[TMP15:%.*]] = bitcast i32* [[TMP13]] to <vscale x 4 x i32>*
+; CHECK-TF-NEXT:    [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>* [[TMP15]], i32 4, <vscale x 4 x i1> [[TMP14]], <vscale x 4 x i32> poison)
+; CHECK-TF-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> [[DOTSPLAT]], i32 4, <vscale x 4 x i1> [[TMP14]], <vscale x 4 x i32> undef)
+; CHECK-TF-NEXT:    [[TMP16:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_LOAD1]]
+; CHECK-TF-NEXT:    [[TMP17:%.*]] = getelementptr i32, i32* [[A:%.*]], i64 [[INDEX]]
+; CHECK-TF-NEXT:    [[TMP18:%.*]] = bitcast i32* [[TMP17]] to <vscale x 4 x i32>*
+; CHECK-TF-NEXT:    call void @llvm.masked.store.nxv4i32.p0nxv4i32(<vscale x 4 x i32> [[TMP16]], <vscale x 4 x i32>* [[TMP18]], i32 4, <vscale x 4 x i1> [[TMP14]])
+; CHECK-TF-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP20:%.*]] = shl nuw nsw i64 [[TMP19]], 2
+; CHECK-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP20]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NEXT:    [[TMP21:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NEXT:    br i1 [[TMP21]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-TF:       middle.block:
+; CHECK-TF-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK-TF:       scalar.ph:
+; CHECK-TF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-TF-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-TF:       for.body:
+; CHECK-TF-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-TF-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[COND]], i64 [[IV]]
+; CHECK-TF-NEXT:    [[TMP22:%.*]] = load i32, i32* [[ARRAYIDX2]], align 4
+; CHECK-TF-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[TMP22]], 0
+; CHECK-TF-NEXT:    br i1 [[TOBOOL_NOT]], label [[FOR_INC]], label [[IF_THEN:%.*]]
+; CHECK-TF:       if.then:
+; CHECK-TF-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 42
+; CHECK-TF-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[IV]]
+; CHECK-TF-NEXT:    [[TMP23:%.*]] = load i32, i32* [[ARRAYIDX3]], align 4
+; CHECK-TF-NEXT:    [[TMP24:%.*]] = load i32, i32* [[ARRAYIDX1]], align 4
+; CHECK-TF-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP24]], [[TMP23]]
+; CHECK-TF-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[IV]]
+; CHECK-TF-NEXT:    store i32 [[ADD]], i32* [[ARRAYIDX4]], align 4
+; CHECK-TF-NEXT:    br label [[FOR_INC]]
+; CHECK-TF:       for.inc:
+; CHECK-TF-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-TF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-TF-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-TF:       for.end:
+; CHECK-TF-NEXT:    ret void
+;
 entry:
   br label %for.body
 
 for.body:
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
   %arrayidx1 = getelementptr inbounds i32, i32* %b, i64 42
   %arrayidx2 = getelementptr inbounds i32, i32* %cond, i64 %iv
   %0 = load i32, i32* %arrayidx2, align 4

llvm/test/Transforms/LoopVectorize/AArch64/sve-fneg.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -loop-vectorize -dce -mtriple aarch64-linux-gnu -mattr=+sve \
 ; RUN:   -prefer-predicate-over-epilogue=scalar-epilogue < %s -S | FileCheck %s
+; RUN: opt -loop-vectorize -dce -mtriple aarch64-linux-gnu -mattr=+sve \
+; RUN:   -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue < %s -S | FileCheck %s --check-prefix=CHECK-TF
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64-unknown-linux-gnu"
 
 ; This should be vscale x 8 vectorized, maybe with some interleaving.
 
 define void @fneg(ptr nocapture noundef writeonly %d, ptr nocapture noundef readonly %s, i32 noundef %n) {
 ; CHECK-LABEL: @fneg(

 ; CHECK-NEXT:    [[TMP24:%.*]] = load half, ptr [[ARRAYIDX]], align 2
 ; CHECK-NEXT:    [[FNEG:%.*]] = fneg half [[TMP24]]
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds half, ptr [[D]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    store half [[FNEG]], ptr [[ARRAYIDX2]], align 2
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
 ;
+; CHECK-TF-LABEL: @fneg(
+; CHECK-TF-NEXT:  entry:
+; CHECK-TF-NEXT:    [[S2:%.*]] = ptrtoint ptr [[S:%.*]] to i64
+; CHECK-TF-NEXT:    [[D1:%.*]] = ptrtoint ptr [[D:%.*]] to i64
+; CHECK-TF-NEXT:    [[CMP6:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-TF-NEXT:    br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK-TF:       for.body.preheader:
+; CHECK-TF-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64
+; CHECK-TF-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[WIDE_TRIP_COUNT]]
+; CHECK-TF-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 8
+; CHECK-TF-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP2]])
+; CHECK-TF-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
+; CHECK-TF-NEXT:    br i1 [[TMP4]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; CHECK-TF:       vector.memcheck:
+; CHECK-TF-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
+; CHECK-TF-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 2
+; CHECK-TF-NEXT:    [[TMP8:%.*]] = sub i64 [[D1]], [[S2]]
+; CHECK-TF-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP7]]
+; CHECK-TF-NEXT:    br i1 [[DIFF_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; CHECK-TF:       vector.ph:
+; CHECK-TF-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 8
+; CHECK-TF-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 8
+; CHECK-TF-NEXT:    [[TMP13:%.*]] = sub i64 [[TMP12]], 1
+; CHECK-TF-NEXT:    [[N_RND_UP:%.*]] = add i64 [[WIDE_TRIP_COUNT]], [[TMP13]]
+; CHECK-TF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP10]]
+; CHECK-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[WIDE_TRIP_COUNT]])
+; CHECK-TF-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-TF:       vector.body:
+; CHECK-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[TMP14:%.*]] = add i64 [[INDEX]], 0
+; CHECK-TF-NEXT:    [[TMP15:%.*]] = getelementptr inbounds half, ptr [[S]], i64 [[TMP14]]
+; CHECK-TF-NEXT:    [[TMP16:%.*]] = getelementptr inbounds half, ptr [[TMP15]], i32 0
+; CHECK-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x half> @llvm.masked.load.nxv8f16.p0(ptr [[TMP16]], i32 2, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x half> poison)
+; CHECK-TF-NEXT:    [[TMP17:%.*]] = fneg <vscale x 8 x half> [[WIDE_MASKED_LOAD]]
+; CHECK-TF-NEXT:    [[TMP18:%.*]] = getelementptr inbounds half, ptr [[D]], i64 [[TMP14]]
+; CHECK-TF-NEXT:    [[TMP19:%.*]] = getelementptr inbounds half, ptr [[TMP18]], i32 0
+; CHECK-TF-NEXT:    call void @llvm.masked.store.nxv8f16.p0(<vscale x 8 x half> [[TMP17]], ptr [[TMP19]], i32 2, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP21:%.*]] = mul i64 [[TMP20]], 8
+; CHECK-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP21]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT]], i64 [[WIDE_TRIP_COUNT]])
+; CHECK-TF-NEXT:    [[TMP22:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i32 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer)
+; CHECK-TF-NEXT:    [[TMP23:%.*]] = extractelement <vscale x 8 x i1> [[TMP22]], i32 0
+; CHECK-TF-NEXT:    br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF:       middle.block:
+; CHECK-TF-NEXT:    br i1 true, label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-TF:       scalar.ph:
+; CHECK-TF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-TF-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-TF:       for.cond.cleanup.loopexit:
+; CHECK-TF-NEXT:    br label [[FOR_COND_CLEANUP]]
+; CHECK-TF:       for.cond.cleanup:
+; CHECK-TF-NEXT:    ret void
+; CHECK-TF:       for.body:
+; CHECK-TF-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-TF-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds half, ptr [[S]], i64 [[INDVARS_IV]]
+; CHECK-TF-NEXT:    [[TMP24:%.*]] = load half, ptr [[ARRAYIDX]], align 2
+; CHECK-TF-NEXT:    [[FNEG:%.*]] = fneg half [[TMP24]]
+; CHECK-TF-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds half, ptr [[D]], i64 [[INDVARS_IV]]
+; CHECK-TF-NEXT:    store half [[FNEG]], ptr [[ARRAYIDX2]], align 2
+; CHECK-TF-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-TF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-TF-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
+;
 entry:
   %cmp6 = icmp sgt i32 %n, 0
   br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
 
 for.body.preheader:                               ; preds = %entry
   %wide.trip.count = zext i32 %n to i64
   br label %for.body
 

llvm/test/Transforms/LoopVectorize/AArch64/sve-gather-scatter-cost.ll

 ; REQUIRES: asserts
 ; RUN: opt -loop-vectorize -mcpu=neoverse-v1 -disable-output %s -debug \
 ; RUN:   -prefer-predicate-over-epilogue=scalar-epilogue 2>&1 | FileCheck %s
+; RUN: opt -loop-vectorize -mcpu=neoverse-v1 -disable-output %s -debug \
+; RUN:   -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue 2>&1 | FileCheck %s --check-prefix=CHECK-TF
 
 target triple="aarch64--linux-gnu"
 
 ; CHECK: LV: Checking a loop in 'gather_nxv4i32_loaded_index'
 ; CHECK: LV: Found an estimated cost of 81 for VF vscale x 4 For instruction:   %1 = load float, float* %arrayidx3, align 4
+; CHECK-TF: LV: Checking a loop in 'gather_nxv4i32_loaded_index'
+; CHECK-TF: LV: Found an estimated cost of 81 for VF vscale x 4 For instruction:   %1 = load float, float* %arrayidx3, align 4
 define void @gather_nxv4i32_loaded_index(float* noalias nocapture readonly %a, i64* noalias nocapture readonly %b, float* noalias nocapture %c, i64 %n) #0 {
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.body
   %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
   %arrayidx = getelementptr inbounds i64, i64* %b, i64 %indvars.iv
   %0 = load i64, i64* %arrayidx, align 8
   %arrayidx3 = getelementptr inbounds float, float* %a, i64 %0
   %1 = load float, float* %arrayidx3, align 4
   %arrayidx5 = getelementptr inbounds float, float* %c, i64 %indvars.iv
   store float %1, float* %arrayidx5, align 4
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %exitcond.not = icmp eq i64 %indvars.iv.next, %n
   br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !llvm.loop !0
 
 for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
   ret void
 }
 
 ; CHECK: LV: Checking a loop in 'scatter_nxv4i32_loaded_index'
 ; CHECK: LV: Found an estimated cost of 81 for VF vscale x 4 For instruction:   store float %1, float* %arrayidx5, align 4
+; CHECK-TF: LV: Checking a loop in 'scatter_nxv4i32_loaded_index'
+; CHECK-TF: LV: Found an estimated cost of 81 for VF vscale x 4 For instruction:   store float %1, float* %arrayidx5, align 4
 define void @scatter_nxv4i32_loaded_index(float* noalias nocapture readonly %a, i64* noalias nocapture readonly %b, float* noalias nocapture %c, i64 %n) #0 {
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.body
   %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
   %arrayidx = getelementptr inbounds i64, i64* %b, i64 %indvars.iv
   %0 = load i64, i64* %arrayidx, align 8
   %arrayidx3 = getelementptr inbounds float, float* %a, i64 %indvars.iv
   %1 = load float, float* %arrayidx3, align 4
   %arrayidx5 = getelementptr inbounds float, float* %c, i64 %0
   store float %1, float* %arrayidx5, align 4
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %exitcond.not = icmp eq i64 %indvars.iv.next, %n
   br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !llvm.loop !0
 
 for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
   ret void
 }
 
 ; NOTE: For runtime-determined strides the vectoriser versions the loop and adds SCEV checks
 ; to ensure the stride value is always 1. Therefore, it can assume a contiguous load and a cost of 1.
 ; CHECK: LV: Checking a loop in 'gather_nxv4i32_unknown_stride'
 ; CHECK: LV: Found an estimated cost of 1 for VF vscale x 4 For instruction:   %0 = load float, float* %arrayidx, align 4
+; CHECK-TF: LV: Checking a loop in 'gather_nxv4i32_unknown_stride'
+; CHECK-TF: LV: Found an estimated cost of 2 for VF vscale x 4 For instruction:   %0 = load float, float* %arrayidx, align 4
 define void @gather_nxv4i32_unknown_stride(float* noalias nocapture readonly %a, float* noalias nocapture %b, i64 %stride, i64 %n) #0 {
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.body
   %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
   %indvars.iv.stride2 = mul i64 %indvars.iv, %stride
   %arrayidx = getelementptr inbounds float, float* %b, i64 %indvars.iv.stride2
   %0 = load float, float* %arrayidx, align 4
   %arrayidx2 = getelementptr inbounds float, float* %a, i64 %indvars.iv
   store float %0, float* %arrayidx2, align 4
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %exitcond.not = icmp eq i64 %indvars.iv.next, %n
   br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !llvm.loop !0
 
 for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
   ret void
 }
 
 ; NOTE: For runtime-determined strides the vectoriser versions the loop and adds SCEV checks
 ; to ensure the stride value is always 1. Therefore, it can assume a contiguous load and cost is 1.
 ; CHECK: LV: Checking a loop in 'scatter_nxv4i32_unknown_stride'
 ; CHECK: LV: Found an estimated cost of 1 for VF vscale x 4 For instruction:   store float %0, float* %arrayidx2, align 4
+; CHECK-TF: LV: Checking a loop in 'scatter_nxv4i32_unknown_stride'
+; CHECK-TF: LV: Found an estimated cost of 2 for VF vscale x 4 For instruction:   store float %0, float* %arrayidx2, align 4
 define void @scatter_nxv4i32_unknown_stride(float* noalias nocapture readonly %a, float* noalias nocapture %b, i64 %stride, i64 %n) #0 {
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.body
   %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
   %indvars.iv.stride2 = mul i64 %indvars.iv, %stride
   %arrayidx = getelementptr inbounds float, float* %b, i64 %indvars.iv
   %0 = load float, float* %arrayidx, align 4
   %arrayidx2 = getelementptr inbounds float, float* %a, i64 %indvars.iv.stride2
   store float %0, float* %arrayidx2, align 4
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %exitcond.not = icmp eq i64 %indvars.iv.next, %n
   br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !llvm.loop !0
 
 for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
   ret void
 }
 
 ; CHECK: LV: Checking a loop in 'gather_nxv4i32_stride2'
 ; CHECK: LV: Found an estimated cost of 81 for VF vscale x 4 For instruction:   %0 = load float, float* %arrayidx, align 4
+; CHECK-TF: LV: Checking a loop in 'gather_nxv4i32_stride2'
+; CHECK-TF: LV: Found an estimated cost of 81 for VF vscale x 4 For instruction:   %0 = load float, float* %arrayidx, align 4
 define void @gather_nxv4i32_stride2(float* noalias nocapture readonly %a, float* noalias nocapture readonly %b, i64 %n) #0 {
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.body
   %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
   %indvars.iv.stride2 = mul i64 %indvars.iv, 2
   %arrayidx = getelementptr inbounds float, float* %b, i64 %indvars.iv.stride2

 
 for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
   ret void
 }
 
 
 ; CHECK: LV: Checking a loop in 'gather_nxv4i32_stride64'
 ; CHECK: LV: Found an estimated cost of 81 for VF vscale x 4 For instruction:   %0 = load float, float* %arrayidx, align 4
+; CHECK-TF: LV: Checking a loop in 'gather_nxv4i32_stride64'
+; CHECK-TF: LV: Found an estimated cost of 81 for VF vscale x 4 For instruction:   %0 = load float, float* %arrayidx, align 4
 define void @gather_nxv4i32_stride64(float* noalias nocapture readonly %a, float* noalias nocapture readonly %b, i64 %n) #0 {
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.body
   %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
   %indvars.iv.stride2 = mul i64 %indvars.iv, 64
   %arrayidx = getelementptr inbounds float, float* %b, i64 %indvars.iv.stride2
   %0 = load float, float* %arrayidx, align 4
   %arrayidx2 = getelementptr inbounds float, float* %a, i64 %indvars.iv
   store float %0, float* %arrayidx2, align 4
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %exitcond.not = icmp eq i64 %indvars.iv.next, %n
   br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !llvm.loop !0
 
 for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
   ret void
 }
 
 ; CHECK: LV: Checking a loop in 'scatter_nxv4i32_stride64'
 ; CHECK: LV: Found an estimated cost of 81 for VF vscale x 4 For instruction:   store float %0, float* %arrayidx2, align 4
+; CHECK-TF: LV: Checking a loop in 'scatter_nxv4i32_stride64'
+; CHECK-TF: LV: Found an estimated cost of 81 for VF vscale x 4 For instruction:   store float %0, float* %arrayidx2, align 4
 define void @scatter_nxv4i32_stride64(float* noalias nocapture readonly %a, float* noalias nocapture readonly %b, i64 %n) #0 {
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.body
   %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
   %indvars.iv.stride2 = mul i64 %indvars.iv, 64
   %arrayidx = getelementptr inbounds float, float* %b, i64 %indvars.iv

llvm/test/Transforms/LoopVectorize/AArch64/sve-gather-scatter.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -loop-vectorize -dce -instcombine -mtriple aarch64-linux-gnu -mattr=+sve \
 ; RUN:   -prefer-predicate-over-epilogue=scalar-epilogue -S %s -force-target-instruction-cost=1 -o - | FileCheck %s
+; RUN: opt -loop-vectorize -dce -instcombine -mtriple aarch64-linux-gnu -mattr=+sve \
+; RUN:   -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -S %s \
+; RUN:   -force-target-instruction-cost=1 -o - | FileCheck %s --check-prefix=CHECK-TF
 
 define void @gather_nxv4i32_ind64(float* noalias nocapture readonly %a, i64* noalias nocapture readonly %b, float* noalias nocapture %c, i64 %n) #0 {
 ; CHECK-LABEL: @gather_nxv4i32_ind64(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ugt i64 [[TMP1]], [[N:%.*]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]

 ; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, float* [[C]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    store float [[TMP13]], float* [[ARRAYIDX5]], align 4
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
 ; CHECK-NEXT:    ret void
 ;
+; CHECK-TF-LABEL: @gather_nxv4i32_ind64(
+; CHECK-TF-NEXT:  entry:
+; CHECK-TF-NEXT:    [[TMP0:%.*]] = xor i64 [[N:%.*]], -1
+; CHECK-TF-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP1]], 2
+; CHECK-TF-NEXT:    [[TMP3:%.*]] = icmp ugt i64 [[TMP2]], [[TMP0]]
+; CHECK-TF-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-TF:       vector.ph:
+; CHECK-TF-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2
+; CHECK-TF-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 2
+; CHECK-TF-NEXT:    [[TMP8:%.*]] = add nsw i64 [[TMP7]], -1
+; CHECK-TF-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP8]], [[N]]
+; CHECK-TF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
+; CHECK-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-TF:       vector.body:
+; CHECK-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, i64* [[B:%.*]], i64 [[INDEX]]
+; CHECK-TF-NEXT:    [[TMP10:%.*]] = bitcast i64* [[TMP9]] to <vscale x 4 x i64>*
+; CHECK-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i64> @llvm.masked.load.nxv4i64.p0nxv4i64(<vscale x 4 x i64>* [[TMP10]], i32 8, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i64> poison)
+; CHECK-TF-NEXT:    [[TMP11:%.*]] = getelementptr inbounds float, float* [[A:%.*]], <vscale x 4 x i64> [[WIDE_MASKED_LOAD]]
+; CHECK-TF-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0f32(<vscale x 4 x float*> [[TMP11]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> undef)
+; CHECK-TF-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, float* [[C:%.*]], i64 [[INDEX]]
+; CHECK-TF-NEXT:    [[TMP13:%.*]] = bitcast float* [[TMP12]] to <vscale x 4 x float>*
+; CHECK-TF-NEXT:    call void @llvm.masked.store.nxv4f32.p0nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_GATHER]], <vscale x 4 x float>* [[TMP13]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP15:%.*]] = shl nuw nsw i64 [[TMP14]], 2
+; CHECK-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP15]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NEXT:    [[TMP16:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NEXT:    br i1 [[TMP16]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF:       middle.block:
+; CHECK-TF-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK-TF:       scalar.ph:
+; CHECK-TF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-TF-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-TF:       for.body:
+; CHECK-TF-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-TF-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, i64* [[B]], i64 [[INDVARS_IV]]
+; CHECK-TF-NEXT:    [[TMP17:%.*]] = load i64, i64* [[ARRAYIDX]], align 8
+; CHECK-TF-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[TMP17]]
+; CHECK-TF-NEXT:    [[TMP18:%.*]] = load float, float* [[ARRAYIDX3]], align 4
+; CHECK-TF-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, float* [[C]], i64 [[INDVARS_IV]]
+; CHECK-TF-NEXT:    store float [[TMP18]], float* [[ARRAYIDX5]], align 4
+; CHECK-TF-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-TF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
+; CHECK-TF-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-TF:       for.cond.cleanup:
+; CHECK-TF-NEXT:    ret void
+;
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.body
   %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
   %arrayidx = getelementptr inbounds i64, i64* %b, i64 %indvars.iv
   %0 = load i64, i64* %arrayidx, align 8
   %arrayidx3 = getelementptr inbounds float, float* %a, i64 %0

 ; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[IDXPROM4]]
 ; CHECK-NEXT:    store float [[TMP13]], float* [[ARRAYIDX5]], align 4
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
 ; CHECK-NEXT:    ret void
 ;
+; CHECK-TF-LABEL: @scatter_nxv4i32_ind32(
+; CHECK-TF-NEXT:  entry:
+; CHECK-TF-NEXT:    [[TMP0:%.*]] = xor i64 [[N:%.*]], -1
+; CHECK-TF-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP1]], 2
+; CHECK-TF-NEXT:    [[TMP3:%.*]] = icmp ugt i64 [[TMP2]], [[TMP0]]
+; CHECK-TF-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-TF:       vector.ph:
+; CHECK-TF-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2
+; CHECK-TF-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 2
+; CHECK-TF-NEXT:    [[TMP8:%.*]] = add nsw i64 [[TMP7]], -1
+; CHECK-TF-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP8]], [[N]]
+; CHECK-TF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
+; CHECK-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-TF:       vector.body:
+; CHECK-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, float* [[C:%.*]], i64 [[INDEX]]
+; CHECK-TF-NEXT:    [[TMP10:%.*]] = bitcast float* [[TMP9]] to <vscale x 4 x float>*
+; CHECK-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0nxv4f32(<vscale x 4 x float>* [[TMP10]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)
+; CHECK-TF-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDEX]]
+; CHECK-TF-NEXT:    [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <vscale x 4 x i32>*
+; CHECK-TF-NEXT:    [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>* [[TMP12]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-TF-NEXT:    [[TMP13:%.*]] = sext <vscale x 4 x i32> [[WIDE_MASKED_LOAD1]] to <vscale x 4 x i64>
+; CHECK-TF-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, float* [[A:%.*]], <vscale x 4 x i64> [[TMP13]]
+; CHECK-TF-NEXT:    call void @llvm.masked.scatter.nxv4f32.nxv4p0f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x float*> [[TMP14]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP16:%.*]] = shl nuw nsw i64 [[TMP15]], 2
+; CHECK-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP16]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NEXT:    [[TMP17:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NEXT:    br i1 [[TMP17]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-TF:       middle.block:
+; CHECK-TF-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK-TF:       scalar.ph:
+; CHECK-TF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-TF-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-TF:       for.body:
+; CHECK-TF-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-TF-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[C]], i64 [[INDVARS_IV]]
+; CHECK-TF-NEXT:    [[TMP18:%.*]] = load float, float* [[ARRAYIDX]], align 4
+; CHECK-TF-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]]
+; CHECK-TF-NEXT:    [[TMP19:%.*]] = load i32, i32* [[ARRAYIDX3]], align 4
+; CHECK-TF-NEXT:    [[IDXPROM4:%.*]] = sext i32 [[TMP19]] to i64
+; CHECK-TF-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[IDXPROM4]]
+; CHECK-TF-NEXT:    store float [[TMP18]], float* [[ARRAYIDX5]], align 4
+; CHECK-TF-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-TF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
+; CHECK-TF-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-TF:       for.cond.cleanup:
+; CHECK-TF-NEXT:    ret void
+;
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.body
   %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
   %arrayidx = getelementptr inbounds float, float* %c, i64 %indvars.iv
   %0 = load float, float* %arrayidx, align 4
   %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv

 ; CHECK-NEXT:    br label [[FOR_INC]]
 ; CHECK:       for.inc:
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
 ; CHECK-NEXT:    ret void
 ;
+; CHECK-TF-LABEL: @scatter_inv_nxv4i32(
+; CHECK-TF-NEXT:  entry:
+; CHECK-TF-NEXT:    [[TMP0:%.*]] = xor i64 [[N:%.*]], -1
+; CHECK-TF-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP1]], 2
+; CHECK-TF-NEXT:    [[TMP3:%.*]] = icmp ugt i64 [[TMP2]], [[TMP0]]
+; CHECK-TF-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-TF:       vector.ph:
+; CHECK-TF-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2
+; CHECK-TF-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 2
+; CHECK-TF-NEXT:    [[TMP8:%.*]] = add nsw i64 [[TMP7]], -1
+; CHECK-TF-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP8]], [[N]]
+; CHECK-TF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
+; CHECK-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32*> poison, i32* [[INV:%.*]], i64 0
+; CHECK-TF-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32*> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32*> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-TF-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-TF:       vector.body:
+; CHECK-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDEX]]
+; CHECK-TF-NEXT:    [[TMP10:%.*]] = bitcast i32* [[TMP9]] to <vscale x 4 x i32>*
+; CHECK-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>* [[TMP10]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-TF-NEXT:    [[TMP11:%.*]] = icmp ne <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], zeroinitializer
+; CHECK-TF-NEXT:    [[TMP12:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i1> [[TMP11]], <vscale x 4 x i1> zeroinitializer
+; CHECK-TF-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0i32(<vscale x 4 x i32> shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 3, i32 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer), <vscale x 4 x i32*> [[BROADCAST_SPLAT]], i32 4, <vscale x 4 x i1> [[TMP12]])
+; CHECK-TF-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP14:%.*]] = shl nuw nsw i64 [[TMP13]], 2
+; CHECK-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP14]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NEXT:    [[TMP15:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NEXT:    br i1 [[TMP15]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-TF:       middle.block:
+; CHECK-TF-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK-TF:       scalar.ph:
+; CHECK-TF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-TF-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-TF:       for.body:
+; CHECK-TF-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-TF-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]]
+; CHECK-TF-NEXT:    [[TMP16:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-TF-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[TMP16]], 0
+; CHECK-TF-NEXT:    br i1 [[TOBOOL_NOT]], label [[FOR_INC]], label [[IF_THEN:%.*]]
+; CHECK-TF:       if.then:
+; CHECK-TF-NEXT:    store i32 3, i32* [[INV]], align 4
+; CHECK-TF-NEXT:    br label [[FOR_INC]]
+; CHECK-TF:       for.inc:
+; CHECK-TF-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-TF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
+; CHECK-TF-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-TF:       for.cond.cleanup:
+; CHECK-TF-NEXT:    ret void
+;
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.inc
   %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %entry ]
   %arrayidx = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
   %0 = load i32, i32* %arrayidx, align 4
   %tobool.not = icmp eq i32 %0, 0

 ; CHECK-NEXT:    br label [[FOR_INC]]
 ; CHECK:       for.inc:
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
 ; CHECK-NEXT:    ret void
 ;
+; CHECK-TF-LABEL: @gather_inv_nxv4i32(
+; CHECK-TF-NEXT:  entry:
+; CHECK-TF-NEXT:    [[TMP0:%.*]] = xor i64 [[N:%.*]], -1
+; CHECK-TF-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP1]], 2
+; CHECK-TF-NEXT:    [[TMP3:%.*]] = icmp ugt i64 [[TMP2]], [[TMP0]]
+; CHECK-TF-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-TF:       vector.ph:
+; CHECK-TF-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2
+; CHECK-TF-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 2
+; CHECK-TF-NEXT:    [[TMP8:%.*]] = add nsw i64 [[TMP7]], -1
+; CHECK-TF-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP8]], [[N]]
+; CHECK-TF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
+; CHECK-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32*> poison, i32* [[INV:%.*]], i64 0
+; CHECK-TF-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32*> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32*> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-TF-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-TF:       vector.body:
+; CHECK-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[TMP9:%.*]] = getelementptr i32, i32* [[A:%.*]], i64 [[INDEX]]
+; CHECK-TF-NEXT:    [[TMP10:%.*]] = bitcast i32* [[TMP9]] to <vscale x 4 x i32>*
+; CHECK-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>* [[TMP10]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-TF-NEXT:    [[TMP11:%.*]] = icmp sgt <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 3, i32 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-TF-NEXT:    [[TMP12:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i1> [[TMP11]], <vscale x 4 x i1> zeroinitializer
+; CHECK-TF-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> [[BROADCAST_SPLAT]], i32 4, <vscale x 4 x i1> [[TMP12]], <vscale x 4 x i32> undef)
+; CHECK-TF-NEXT:    [[TMP13:%.*]] = bitcast i32* [[TMP9]] to <vscale x 4 x i32>*
+; CHECK-TF-NEXT:    call void @llvm.masked.store.nxv4i32.p0nxv4i32(<vscale x 4 x i32> [[WIDE_MASKED_GATHER]], <vscale x 4 x i32>* [[TMP13]], i32 4, <vscale x 4 x i1> [[TMP12]])
+; CHECK-TF-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP15:%.*]] = shl nuw nsw i64 [[TMP14]], 2
+; CHECK-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP15]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NEXT:    [[TMP16:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NEXT:    br i1 [[TMP16]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-TF:       middle.block:
+; CHECK-TF-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK-TF:       scalar.ph:
+; CHECK-TF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-TF-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-TF:       for.body:
+; CHECK-TF-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-TF-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]]
+; CHECK-TF-NEXT:    [[TMP17:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-TF-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[TMP17]], 3
+; CHECK-TF-NEXT:    br i1 [[CMP2]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK-TF:       if.then:
+; CHECK-TF-NEXT:    [[TMP18:%.*]] = load i32, i32* [[INV]], align 4
+; CHECK-TF-NEXT:    store i32 [[TMP18]], i32* [[ARRAYIDX]], align 4
+; CHECK-TF-NEXT:    br label [[FOR_INC]]
+; CHECK-TF:       for.inc:
+; CHECK-TF-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-TF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
+; CHECK-TF-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-TF:       for.cond.cleanup:
+; CHECK-TF-NEXT:    ret void
+;
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.inc
   %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %entry ]
   %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
   %0 = load i32, i32* %arrayidx, align 4
   %cmp2 = icmp sgt i32 %0, 3

 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    store float [[TMP21]], float* [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
 ; CHECK-NEXT:    ret void
 ;
+; CHECK-TF-LABEL: @gather_nxv4i32_ind64_stride2(
+; CHECK-TF-NEXT:  entry:
+; CHECK-TF-NEXT:    [[TMP0:%.*]] = xor i64 [[N:%.*]], -1
+; CHECK-TF-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP1]], 2
+; CHECK-TF-NEXT:    [[TMP3:%.*]] = icmp ugt i64 [[TMP2]], [[TMP0]]
+; CHECK-TF-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-TF:       vector.ph:
+; CHECK-TF-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2
+; CHECK-TF-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 2
+; CHECK-TF-NEXT:    [[TMP8:%.*]] = add nsw i64 [[TMP7]], -1
+; CHECK-TF-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP8]], [[N]]
+; CHECK-TF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
+; CHECK-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NEXT:    [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
+; CHECK-TF-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP11:%.*]] = shl nuw nsw i64 [[TMP10]], 2
+; CHECK-TF-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP11]], i64 0
+; CHECK-TF-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-TF-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-TF:       vector.body:
+; CHECK-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP9]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[TMP12:%.*]] = shl <vscale x 4 x i64> [[VEC_IND]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i32 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-TF-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, float* [[B:%.*]], <vscale x 4 x i64> [[TMP12]]
+; CHECK-TF-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0f32(<vscale x 4 x float*> [[TMP13]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> undef)
+; CHECK-TF-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, float* [[A:%.*]], i64 [[INDEX]]
+; CHECK-TF-NEXT:    [[TMP15:%.*]] = bitcast float* [[TMP14]] to <vscale x 4 x float>*
+; CHECK-TF-NEXT:    call void @llvm.masked.store.nxv4f32.p0nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_GATHER]], <vscale x 4 x float>* [[TMP15]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP17:%.*]] = shl nuw nsw i64 [[TMP16]], 2
+; CHECK-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP17]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
+; CHECK-TF-NEXT:    [[TMP18:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NEXT:    br i1 [[TMP18]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-TF:       middle.block:
+; CHECK-TF-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK-TF:       scalar.ph:
+; CHECK-TF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-TF-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-TF:       for.body:
+; CHECK-TF-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-TF-NEXT:    [[INDVARS_IV_STRIDE2:%.*]] = shl i64 [[INDVARS_IV]], 1
+; CHECK-TF-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[INDVARS_IV_STRIDE2]]
+; CHECK-TF-NEXT:    [[TMP19:%.*]] = load float, float* [[ARRAYIDX]], align 4
+; CHECK-TF-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDVARS_IV]]
+; CHECK-TF-NEXT:    store float [[TMP19]], float* [[ARRAYIDX2]], align 4
+; CHECK-TF-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-TF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
+; CHECK-TF-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-TF:       for.cond.cleanup:
+; CHECK-TF-NEXT:    ret void
+;
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.body
   %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
   %indvars.iv.stride2 = mul i64 %indvars.iv, 2
   %arrayidx = getelementptr inbounds float, float* %b, i64 %indvars.iv.stride2
   %0 = load float, float* %arrayidx, align 4

llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions-unusual-types.ll

 ; REQUIRES: asserts
 ; RUN: opt -loop-vectorize -S < %s -debug -prefer-predicate-over-epilogue=scalar-epilogue 2>%t | FileCheck %s
 ; RUN: cat %t | FileCheck %s --check-prefix=DEBUG
+; RUN: opt -loop-vectorize -S < %s -debug -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue 2>%t | FileCheck %s
+; RUN: cat %t | FileCheck %s --check-prefix=DEBUG
 
 target triple = "aarch64-unknown-linux-gnu"
 
 ; DEBUG: Found an estimated cost of Invalid for VF vscale x 1 For instruction:   %indvars.iv1294 = phi i7 [ %indvars.iv.next1295, %for.body ], [ 0, %entry ]
 ; DEBUG: Found an estimated cost of Invalid for VF vscale x 1 For instruction:   %addi7 = add i7 %indvars.iv1294, 0
 ; DEBUG: Found an estimated cost of Invalid for VF vscale x 1 For instruction:   %indvars.iv.next1295 = add i7 %indvars.iv1294, 1
 
 define void @induction_i7(i64* %dst) #0 {
 ; CHECK-LABEL: @induction_i7(
 ; CHECK:       vector.ph:
 ; CHECK:         [[TMP4:%.*]] = call <vscale x 2 x i8> @llvm.experimental.stepvector.nxv2i8()
 ; CHECK:         [[TMP5:%.*]] = trunc <vscale x 2 x i8> [[TMP4]] to <vscale x 2 x i7>
-; CHECK-NEXT:    [[TMP6:%.*]] = add <vscale x 2 x i7> [[TMP5]], zeroinitializer
+; CHECK:         [[TMP6:%.*]] = add <vscale x 2 x i7> [[TMP5]], zeroinitializer
 ; CHECK-NEXT:    [[TMP7:%.*]] = mul <vscale x 2 x i7> [[TMP6]], shufflevector (<vscale x 2 x i7> insertelement (<vscale x 2 x i7> poison, i7 1, i32 0), <vscale x 2 x i7> poison, <vscale x 2 x i32> zeroinitializer)
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i7> zeroinitializer, [[TMP7]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i7> [ [[INDUCTION]], %vector.ph ], [ [[VEC_IND_NEXT:%.*]], %vector.body ]
+; CHECK:         [[VEC_IND:%.*]] = phi <vscale x 2 x i7> [ [[INDUCTION]], %vector.ph ], [ [[VEC_IND_NEXT:%.*]], %vector.body ]
 ; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 0
 ; CHECK-NEXT:    [[TMP11:%.*]] = add <vscale x 2 x i7> [[VEC_IND]], zeroinitializer
 ; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, i64* [[DST:%.*]], i64 [[TMP10]]
 ; CHECK-NEXT:    [[EXT:%.+]]  = zext <vscale x 2 x i7> [[TMP11]] to <vscale x 2 x i64>
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, i64* [[TMP12]], i32 0
 ; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i64* [[TMP13]] to <vscale x 2 x i64>*
-; CHECK-NEXT:    store <vscale x 2 x i64> [[EXT]], <vscale x 2 x i64>* [[TMP14]], align 8
+; CHECK-NEXT:    store{{.*}}<vscale x 2 x i64> [[EXT]], <vscale x 2 x i64>* [[TMP14]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 2
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP16]]
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i7> [[VEC_IND]], 
+; CHECK-NEXT:    [[INDEX_NEXT]] = add {{.*}}i64 [[INDEX]], [[TMP16]]
+; CHECK:         [[VEC_IND_NEXT]] = add <vscale x 2 x i7> [[VEC_IND]],
 ;
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %for.body, %entry
   %indvars.iv1294 = phi i7 [ %indvars.iv.next1295, %for.body ], [ 0, %entry ]
   %indvars.iv1286 = phi i64 [ %indvars.iv.next1287, %for.body ], [ 0, %entry ]
   %addi7 = add i7 %indvars.iv1294, 0

 ; CHECK:       vector.ph:
 ; CHECK:         [[TMP4:%.*]] = call <vscale x 2 x i8> @llvm.experimental.stepvector.nxv2i8()
 ; CHECK:         [[TMP5:%.*]] = trunc <vscale x 2 x i8> [[TMP4]] to <vscale x 2 x i3>
 ; CHECK-NEXT:    [[TMP6:%.*]] = add <vscale x 2 x i3> [[TMP5]], zeroinitializer
 ; CHECK-NEXT:    [[TMP7:%.*]] = mul <vscale x 2 x i3> [[TMP6]], shufflevector (<vscale x 2 x i3> insertelement (<vscale x 2 x i3> poison, i3 1, i32 0), <vscale x 2 x i3> poison, <vscale x 2 x i32> zeroinitializer)
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i3> zeroinitializer, [[TMP7]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i3> [ [[INDUCTION]], %vector.ph ], [ [[VEC_IND_NEXT:%.*]], %vector.body ]
+; CHECK:         [[VEC_IND:%.*]] = phi <vscale x 2 x i3> [ [[INDUCTION]], %vector.ph ], [ [[VEC_IND_NEXT:%.*]], %vector.body ]
 ; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 0
 ; CHECK-NEXT:    [[TMP10:%.*]] = zext <vscale x 2 x i3> [[VEC_IND]] to <vscale x 2 x i64>
 ; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, i64* [[DST:%.*]], i64 [[TMP9]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, i64* [[TMP12]], i32 0
 ; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i64* [[TMP13]] to <vscale x 2 x i64>*
-; CHECK-NEXT:    store <vscale x 2 x i64> [[TMP10]], <vscale x 2 x i64>* [[TMP14]], align 8
+; CHECK-NEXT:    store{{.*}}<vscale x 2 x i64> [[TMP10]], <vscale x 2 x i64>* [[TMP14]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 2
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP16]]
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i3> [[VEC_IND]], 
+; CHECK-NEXT:    [[INDEX_NEXT]] = add {{.*}}i64 [[INDEX]], [[TMP16]]
+; CHECK:         [[VEC_IND_NEXT]] = add <vscale x 2 x i3> [[VEC_IND]],
 ;
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %for.body, %entry
   %indvars.iv1294 = phi i3 [ %indvars.iv.next1295, %for.body ], [ 0, %entry ]
   %indvars.iv1286 = phi i64 [ %indvars.iv.next1287, %for.body ], [ 0, %entry ]
   %zexti3 = zext i3 %indvars.iv1294 to i64

llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -loop-vectorize -force-target-instruction-cost=1 -dce -instcombine \
 ; RUN:   -prefer-predicate-over-epilogue=scalar-epilogue < %s -S | FileCheck %s
+; RUN: opt -loop-vectorize -force-target-instruction-cost=1 -dce -instcombine \
+; RUN:   -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue < %s -S | FileCheck %s --check-prefix=CHECK-TF
 
 target triple = "aarch64-linux-gnu"
 
 ; Test a case where the vectorised induction variable is used to
 ; generate a mask:
 ;   for (long long i = 0; i < n; i++) {
 ;     if (i & 0x1)
 ;       a[i] = b[i];

 ; CHECK-NEXT:    br label [[FOR_INC]]
 ; CHECK:       for.inc:
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i64 [[I_08]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
+; CHECK-TF-LABEL: @cond_ind64(
+; CHECK-TF-NEXT:  entry:
+; CHECK-TF-NEXT:    [[TMP0:%.*]] = xor i64 [[N:%.*]], -1
+; CHECK-TF-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP2:%.*]] = shl i64 [[TMP1]], 2
+; CHECK-TF-NEXT:    [[TMP3:%.*]] = icmp ugt i64 [[TMP2]], [[TMP0]]
+; CHECK-TF-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-TF:       vector.ph:
+; CHECK-TF-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP4]], 2
+; CHECK-TF-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP6]], 2
+; CHECK-TF-NEXT:    [[TMP8:%.*]] = add i64 [[TMP7]], -1
+; CHECK-TF-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP8]], [[N]]
+; CHECK-TF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
+; CHECK-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NEXT:    [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
+; CHECK-TF-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP11:%.*]] = shl i64 [[TMP10]], 2
+; CHECK-TF-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP11]], i64 0
+; CHECK-TF-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-TF-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-TF:       vector.body:
+; CHECK-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP9]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[TMP12:%.*]] = trunc <vscale x 4 x i64> [[VEC_IND]] to <vscale x 4 x i1>
+; CHECK-TF-NEXT:    [[TMP13:%.*]] = getelementptr i32, i32* [[B:%.*]], i64 [[INDEX]]
+; CHECK-TF-NEXT:    [[TMP14:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i1> [[TMP12]], <vscale x 4 x i1> zeroinitializer
+; CHECK-TF-NEXT:    [[TMP15:%.*]] = bitcast i32* [[TMP13]] to <vscale x 4 x i32>*
+; CHECK-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>* [[TMP15]], i32 4, <vscale x 4 x i1> [[TMP14]], <vscale x 4 x i32> poison)
+; CHECK-TF-NEXT:    [[TMP16:%.*]] = getelementptr i32, i32* [[A:%.*]], i64 [[INDEX]]
+; CHECK-TF-NEXT:    [[TMP17:%.*]] = bitcast i32* [[TMP16]] to <vscale x 4 x i32>*
+; CHECK-TF-NEXT:    call void @llvm.masked.store.nxv4i32.p0nxv4i32(<vscale x 4 x i32> [[WIDE_MASKED_LOAD]], <vscale x 4 x i32>* [[TMP17]], i32 4, <vscale x 4 x i1> [[TMP14]])
+; CHECK-TF-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP19:%.*]] = shl i64 [[TMP18]], 2
+; CHECK-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP19]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
+; CHECK-TF-NEXT:    [[TMP20:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NEXT:    br i1 [[TMP20]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF:       middle.block:
+; CHECK-TF-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-TF:       scalar.ph:
+; CHECK-TF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-TF-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-TF:       for.body:
+; CHECK-TF-NEXT:    [[I_08:%.*]] = phi i64 [ [[INC:%.*]], [[FOR_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-TF-NEXT:    [[AND:%.*]] = and i64 [[I_08]], 1
+; CHECK-TF-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i64 [[AND]], 0
+; CHECK-TF-NEXT:    br i1 [[TOBOOL_NOT]], label [[FOR_INC]], label [[IF_THEN:%.*]]
+; CHECK-TF:       if.then:
+; CHECK-TF-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[I_08]]
+; CHECK-TF-NEXT:    [[TMP21:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-TF-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[I_08]]
+; CHECK-TF-NEXT:    store i32 [[TMP21]], i32* [[ARRAYIDX1]], align 4
+; CHECK-TF-NEXT:    br label [[FOR_INC]]
+; CHECK-TF:       for.inc:
+; CHECK-TF-NEXT:    [[INC]] = add nuw nsw i64 [[I_08]], 1
+; CHECK-TF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-TF-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-TF:       exit:
+; CHECK-TF-NEXT:    ret void
+;
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.inc
   %i.08 = phi i64 [ %inc, %for.inc ], [ 0, %entry ]
   %and = and i64 %i.08, 1
   %tobool.not = icmp eq i64 %and, 0
   br i1 %tobool.not, label %for.inc, label %if.then

llvm/test/Transforms/LoopVectorize/AArch64/sve-inv-loads.ll

 ; RUN: opt -S -loop-vectorize -mattr=+sve -mtriple aarch64-linux-gnu \
 ; RUN:   -prefer-predicate-over-epilogue=scalar-epilogue < %s | FileCheck %s
+; RUN: opt -S -loop-vectorize -mattr=+sve -mtriple aarch64-linux-gnu \
+; RUN:   -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue < %s | FileCheck %s --check-prefix=CHECK-TF
 
 define void @invariant_load(i64 %n, i32* noalias nocapture %a, i32* nocapture readonly %b) {
 ; CHECK-LABEL: @invariant_load
 ; CHECK: vector.body:
 ; CHECK: %[[GEP:.*]] = getelementptr inbounds i32, i32* %b, i64 42
 ; CHECK-NEXT: %[[INVLOAD:.*]] = load i32, i32* %[[GEP]]
 ; CHECK-NEXT: %[[SPLATINS:.*]] = insertelement <vscale x 4 x i32> poison, i32 %[[INVLOAD]], i32 0
 ; CHECK-NEXT: %[[SPLAT:.*]] = shufflevector <vscale x 4 x i32> %[[SPLATINS]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK: %[[LOAD:.*]] = load <vscale x 4 x i32>, <vscale x 4 x i32>*
 ; CHECK-NEXT: %[[ADD:.*]] = add nsw <vscale x 4 x i32> %[[SPLAT]], %[[LOAD]]
 ; CHECK: store <vscale x 4 x i32> %[[ADD]], <vscale x 4 x i32>*
+
+; CHECK-TF-LABEL: @invariant_load
+; CHECK-TF: vector.body:
+; CHECK-TF: %[[GEP:.*]] = getelementptr inbounds i32, i32* %b, i64 42
+; CHECK-TF-NEXT: %[[INVLOAD:.*]] = load i32, i32* %[[GEP]]
+; CHECK-TF-NEXT: %[[SPLATINS:.*]] = insertelement <vscale x 4 x i32> poison, i32 %[[INVLOAD]], i32 0
+; CHECK-TF-NEXT: %[[SPLAT:.*]] = shufflevector <vscale x 4 x i32> %[[SPLATINS]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-TF: %[[LOAD:.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>*
+; CHECK-TF-NEXT: %[[ADD:.*]] = add nsw <vscale x 4 x i32> %[[SPLAT]], %[[LOAD]]
+; CHECK-TF: call void @llvm.masked.store.nxv4i32.p0nxv4i32(<vscale x 4 x i32> %[[ADD]]
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %for.body.lr.ph, %for.body
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
   %arrayidx = getelementptr inbounds i32, i32* %b, i64 42
   %0 = load i32, i32* %arrayidx, align 4
   %arrayidx1 = getelementptr inbounds i32, i32* %b, i64 %iv

llvm/test/Transforms/LoopVectorize/AArch64/sve-inv-store.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -loop-vectorize -S -prefer-predicate-over-epilogue=scalar-epilogue < %s | FileCheck %s
+; RUN: opt -loop-vectorize -S -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue < %s | FileCheck %s --check-prefix=CHECK-TF
 
 target triple = "aarch64-unknown-linux-gnu"
 
 define void @inv_store_i16(i16* noalias %dst, i16* noalias readonly %src, i64 %N) #0 {
 ; CHECK-LABEL: @inv_store_i16(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4

 ; CHECK-NEXT:    [[LD:%.*]] = load i16, i16* [[ARRAYIDX]], align 2
 ; CHECK-NEXT:    store i16 [[LD]], i16* [[DST]], align 2
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_INC24]], label [[FOR_BODY14]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       for.inc24:
 ; CHECK-NEXT:    ret void
 ;
+
+; CHECK-TF-LABEL: @inv_store_i16(
+; CHECK-TF:       vector.ph:
+; CHECK-TF:         %[[TMP1:.*]] = insertelement <vscale x 4 x i16*> poison, i16* %dst, i32 0
+; CHECK-TF-NEXT:    %[[SPLAT_PTRS:.*]] = shufflevector <vscale x 4 x i16*> %[[TMP1]], <vscale x 4 x i16*> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-TF:       vector.body:
+; CHECK-TF:         %[[VECLOAD:.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0nxv4i16(
+; CHECK-TF-NEXT:    call void @llvm.masked.scatter.nxv4i16.nxv4p0i16(<vscale x 4 x i16> %[[VECLOAD]]
 entry:
   br label %for.body14
 
 for.body14:                                       ; preds = %for.body14, %entry
   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body14 ]
   %arrayidx = getelementptr inbounds i16, i16* %src, i64 %indvars.iv
   %ld = load i16, i16* %arrayidx
   store i16 %ld, i16* %dst, align 2

 ; CHECK-NEXT:    br label [[FOR_INC]]
 ; CHECK:       for.inc:
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i64 [[I_09]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
 ;
+
+; CHECK-TF-LABEL: @cond_inv_store_i32(
+; CHECK-TF:       vector.ph:
+; CHECK-TF:         %[[TMP1:.*]] = insertelement <vscale x 4 x i32*> poison, i32* %dst, i32 0
+; CHECK-TF-NEXT:    %[[SPLAT_PTRS:.*]] = shufflevector <vscale x 4 x i32*> %[[TMP1]], <vscale x 4 x i32*> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-TF:       vector.body:
+; CHECK-TF:         %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
+; CHECK-TF:         %[[VECLOAD:.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(
+; CHECK-TF-NEXT:    %[[MASK:.*]] = icmp sgt <vscale x 4 x i32> %[[VECLOAD]], zeroinitializer
+; CHECK-TF:         %[[SEL:.*]] = select <vscale x 4 x i1> %[[ACTIVE_LANE_MASK]], <vscale x 4 x i1> %[[MASK]], <vscale x 4 x i1> zeroinitializer
+; CHECK-TF-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0i32(<vscale x 4 x i32> %[[VECLOAD]], <vscale x 4 x i32*> %[[SPLAT_PTRS]], i32 4, <vscale x 4 x i1> %[[SEL]])
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.inc
   %i.09 = phi i64 [ %inc, %for.inc ], [ 0, %entry ]
   %arrayidx = getelementptr inbounds i32, i32* %src, i64 %i.09
   %0 = load i32, i32* %arrayidx, align 4
   %cmp1 = icmp sgt i32 %0, 0

llvm/test/Transforms/LoopVectorize/AArch64/sve-large-strides.ll

 ; RUN: opt -mtriple aarch64-linux-gnu -mattr=+sve -loop-vectorize -dce -instcombine -S \
 ; RUN:   -prefer-predicate-over-epilogue=scalar-epilogue <%s | FileCheck %s
+; RUN: opt -mtriple aarch64-linux-gnu -mattr=+sve -loop-vectorize -dce -instcombine -S \
+; RUN:   -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue <%s | FileCheck %s --check-prefix=CHECK-TF
 
 define void @stride7_i32(i32* noalias nocapture %dst, i64 %n) #0 {
 ; CHECK-LABEL: @stride7_i32(
 ; CHECK:      vector.body
 ; CHECK:        %[[VEC_IND:.*]] = phi <vscale x 4 x i64> [ %{{.*}}, %vector.ph ], [ %{{.*}}, %vector.body ]
 ; CHECK-NEXT:   %[[PTR_INDICES:.*]] = mul nuw nsw <vscale x 4 x i64> %[[VEC_IND]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 7, i32 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
 ; CHECK-NEXT:   %[[PTRS:.*]] = getelementptr inbounds i32, i32* %dst, <vscale x 4 x i64> %[[PTR_INDICES]]
 ; CHECK-NEXT:   %[[GLOAD:.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> %[[PTRS]]
 ; CHECK-NEXT:   %[[VALS:.*]] = add nsw <vscale x 4 x i32> %[[GLOAD]],
 ; CHECK-NEXT:   call void @llvm.masked.scatter.nxv4i32.nxv4p0i32(<vscale x 4 x i32> %[[VALS]], <vscale x 4 x i32*> %[[PTRS]]
+
+; CHECK-TF-LABEL: @stride7_i32(
+; CHECK-TF:      vector.body
+; CHECK-TF:        %[[VEC_IND:.*]] = phi <vscale x 4 x i64> [ %{{.*}}, %vector.ph ], [ %{{.*}}, %vector.body ]
+; CHECK-TF-NEXT:   %[[PTR_INDICES:.*]] = mul nuw nsw <vscale x 4 x i64> %[[VEC_IND]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 7, i32 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-TF-NEXT:   %[[PTRS:.*]] = getelementptr inbounds i32, i32* %dst, <vscale x 4 x i64> %[[PTR_INDICES]]
+; CHECK-TF-NEXT:   %[[GLOAD:.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> %[[PTRS]]
+; CHECK-TF-NEXT:   %[[VALS:.*]] = add nsw <vscale x 4 x i32> %[[GLOAD]],
+; CHECK-TF-NEXT:   call void @llvm.masked.scatter.nxv4i32.nxv4p0i32(<vscale x 4 x i32> %[[VALS]], <vscale x 4 x i32*> %[[PTRS]]
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.body
   %i.05 = phi i64 [ %inc, %for.body ], [ 0, %entry ]
   %mul = mul nuw nsw i64 %i.05, 7
   %arrayidx = getelementptr inbounds i32, i32* %dst, i64 %mul
   %0 = load i32, i32* %arrayidx, align 4

 ; CHECK-LABEL: @stride7_f64(
 ; CHECK:      vector.body
 ; CHECK:        %[[VEC_IND:.*]] = phi <vscale x 2 x i64> [ %{{.*}}, %vector.ph ], [ %{{.*}}, %vector.body ]
 ; CHECK-NEXT:   %[[PTR_INDICES:.*]] = mul nuw nsw <vscale x 2 x i64> %[[VEC_IND]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 7, i32 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
 ; CHECK-NEXT:   %[[PTRS:.*]] = getelementptr inbounds double, double* %dst, <vscale x 2 x i64> %[[PTR_INDICES]]
 ; CHECK-NEXT:   %[[GLOAD:.*]] = call <vscale x 2 x double> @llvm.masked.gather.nxv2f64.nxv2p0f64(<vscale x 2 x double*> %[[PTRS]],
 ; CHECK-NEXT:   %[[VALS:.*]] = fadd <vscale x 2 x double> %[[GLOAD]],
 ; CHECK-NEXT:  call void @llvm.masked.scatter.nxv2f64.nxv2p0f64(<vscale x 2 x double> %[[VALS]], <vscale x 2 x double*> %[[PTRS]],
+
+; CHECK-TF-LABEL: @stride7_f64(
+; CHECK-TF:      vector.body
+; CHECK-TF:        %[[VEC_IND:.*]] = phi <vscale x 2 x i64> [ %{{.*}}, %vector.ph ], [ %{{.*}}, %vector.body ]
+; CHECK-TF-NEXT:   %[[PTR_INDICES:.*]] = mul nuw nsw <vscale x 2 x i64> %[[VEC_IND]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 7, i32 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
+; CHECK-TF-NEXT:   %[[PTRS:.*]] = getelementptr inbounds double, double* %dst, <vscale x 2 x i64> %[[PTR_INDICES]]
+; CHECK-TF-NEXT:   %[[GLOAD:.*]] = call <vscale x 2 x double> @llvm.masked.gather.nxv2f64.nxv2p0f64(<vscale x 2 x double*> %[[PTRS]],
+; CHECK-TF-NEXT:   %[[VALS:.*]] = fadd <vscale x 2 x double> %[[GLOAD]],
+; CHECK-TF-NEXT:  call void @llvm.masked.scatter.nxv2f64.nxv2p0f64(<vscale x 2 x double> %[[VALS]], <vscale x 2 x double*> %[[PTRS]],
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.body
   %i.05 = phi i64 [ %inc, %for.body ], [ 0, %entry ]
   %mul = mul nuw nsw i64 %i.05, 7
   %arrayidx = getelementptr inbounds double, double* %dst, i64 %mul
   %0 = load double, double* %arrayidx, align 8

 define void @cond_stride7_f64(double* noalias nocapture %dst, i64* noalias nocapture readonly %cond, i64 %n) #0 {
 ; CHECK-LABEL: @cond_stride7_f64(
 ; CHECK:      vector.body
 ; CHECK:        %[[MASK:.*]] = icmp ne <vscale x 2 x i64>
 ; CHECK:        %[[PTRS:.*]] = getelementptr inbounds double, double* %dst, <vscale x 2 x i64> %{{.*}}
 ; CHECK-NEXT:   %[[GLOAD:.*]] = call <vscale x 2 x double> @llvm.masked.gather.nxv2f64.nxv2p0f64(<vscale x 2 x double*> %[[PTRS]], i32 8, <vscale x 2 x i1> %[[MASK]]
 ; CHECK-NEXT:   %[[VALS:.*]] = fadd <vscale x 2 x double> %[[GLOAD]],
 ; CHECK-NEXT:  call void @llvm.masked.scatter.nxv2f64.nxv2p0f64(<vscale x 2 x double> %[[VALS]], <vscale x 2 x double*> %[[PTRS]], i32 8, <vscale x 2 x i1> %[[MASK]])
+
+; CHECK-TF-LABEL: @cond_stride7_f64(
+; CHECK-TF:      vector.body
+; CHECK-TF:        %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 2 x i1>
+; CHECK-TF:        %[[MASK:.*]] = icmp ne <vscale x 2 x i64>
+; CHECK-TF:        %[[PTRS:.*]] = getelementptr inbounds double, double* %dst, <vscale x 2 x i64> %{{.*}}
+; CHECK-TF-NEXT:   %[[SEL:.*]] = select <vscale x 2 x i1> %[[ACTIVE_LANE_MASK]], <vscale x 2 x i1> %[[MASK]], <vscale x 2 x i1> zeroinitializer
+; CHECK-TF-NEXT:   %[[GLOAD:.*]] = call <vscale x 2 x double> @llvm.masked.gather.nxv2f64.nxv2p0f64(<vscale x 2 x double*> %[[PTRS]], i32 8, <vscale x 2 x i1> %[[SEL]]
+; CHECK-TF-NEXT:   %[[VALS:.*]] = fadd <vscale x 2 x double> %[[GLOAD]],
+; CHECK-TF-NEXT:  call void @llvm.masked.scatter.nxv2f64.nxv2p0f64(<vscale x 2 x double> %[[VALS]], <vscale x 2 x double*> %[[PTRS]], i32 8, <vscale x 2 x i1> %[[SEL]])
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.inc
   %i.07 = phi i64 [ %inc, %for.inc ], [ 0, %entry ]
   %arrayidx = getelementptr inbounds i64, i64* %cond, i64 %i.07
   %0 = load i64, i64* %arrayidx, align 8
   %tobool.not = icmp eq i64 %0, 0

llvm/test/Transforms/LoopVectorize/AArch64/sve-masked-loadstore.ll

 ; RUN: opt -loop-vectorize -dce -instcombine -mtriple aarch64-linux-gnu -mattr=+sve \
 ; RUN:   -prefer-predicate-over-epilogue=scalar-epilogue -S %s -o - | FileCheck %s
+; RUN: opt -loop-vectorize -dce -instcombine -mtriple aarch64-linux-gnu -mattr=+sve \
+; RUN:   -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -S %s -o - | FileCheck %s --check-prefix=CHECK-TF
 
 define void @mloadstore_f32(float* noalias nocapture %a, float* noalias nocapture readonly %b, i64 %n) {
 ; CHECK-LABEL: @mloadstore_f32
 ; CHECK: vector.body:
 ; CHECK:       %[[LOAD1:.*]] = load <vscale x 4 x float>, <vscale x 4 x float>*
 ; CHECK-NEXT:  %[[MASK:.*]] = fcmp ogt <vscale x 4 x float> %[[LOAD1]],
 ; CHECK-NEXT:  %[[GEPA:.*]] = getelementptr float, float* %a,
 ; CHECK-NEXT:  %[[MLOAD_PTRS:.*]] = bitcast float* %[[GEPA]] to <vscale x 4 x float>*
 ; CHECK-NEXT:  %[[LOAD2:.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0nxv4f32(<vscale x 4 x float>* %[[MLOAD_PTRS]], i32 4, <vscale x 4 x i1> %[[MASK]]
 ; CHECK-NEXT:  %[[FADD:.*]] = fadd <vscale x 4 x float> %[[LOAD1]], %[[LOAD2]]
 ; CHECK-NEXT:  %[[MSTORE_PTRS:.*]] = bitcast float* %[[GEPA]] to <vscale x 4 x float>*
 ; CHECK-NEXT:  call void @llvm.masked.store.nxv4f32.p0nxv4f32(<vscale x 4 x float> %[[FADD]], <vscale x 4 x float>* %[[MSTORE_PTRS]], i32 4, <vscale x 4 x i1> %[[MASK]])
+
+; CHECK-TF-LABEL: @mloadstore_f32
+; CHECK-TF: vector.body:
+; CHECK-TF:       %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
+; CHECK-TF:       %[[LOAD1:.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32
+; CHECK-TF-NEXT:  %[[MASK:.*]] = fcmp ogt <vscale x 4 x float> %[[LOAD1]],
+; CHECK-TF-NEXT:  %[[GEPA:.*]] = getelementptr float, float* %a,
+; CHECK-TF-NEXT:  %[[SEL:.*]] = select <vscale x 4 x i1> %[[ACTIVE_LANE_MASK]], <vscale x 4 x i1> %[[MASK]], <vscale x 4 x i1> zeroinitializer
+; CHECK-TF-NEXT:  %[[MLOAD_PTRS:.*]] = bitcast float* %[[GEPA]] to <vscale x 4 x float>*
+; CHECK-TF-NEXT:  %[[LOAD2:.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0nxv4f32(<vscale x 4 x float>* %[[MLOAD_PTRS]], i32 4, <vscale x 4 x i1> %[[SEL]]
+; CHECK-TF-NEXT:  %[[FADD:.*]] = fadd <vscale x 4 x float> %[[LOAD1]], %[[LOAD2]]
+; CHECK-TF-NEXT:  %[[MSTORE_PTRS:.*]] = bitcast float* %[[GEPA]] to <vscale x 4 x float>*
+; CHECK-TF-NEXT:  call void @llvm.masked.store.nxv4f32.p0nxv4f32(<vscale x 4 x float> %[[FADD]], <vscale x 4 x float>* %[[MSTORE_PTRS]], i32 4, <vscale x 4 x i1> %[[SEL]])
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.inc
   %i.011 = phi i64 [ %inc, %for.inc ], [ 0, %entry ]
   %arrayidx = getelementptr inbounds float, float* %b, i64 %i.011
   %0 = load float, float* %arrayidx, align 4
   %cmp1 = fcmp ogt float %0, 0.000000e+00

 ; CHECK:       %[[LOAD1:.*]] = load <vscale x 4 x i32>, <vscale x 4 x i32>*
 ; CHECK-NEXT:  %[[MASK:.*]] = icmp ne <vscale x 4 x i32> %[[LOAD1]],
 ; CHECK-NEXT:  %[[GEPA:.*]] = getelementptr i32, i32* %a,
 ; CHECK-NEXT:  %[[MLOAD_PTRS:.*]] = bitcast i32* %[[GEPA]] to <vscale x 4 x i32>*
 ; CHECK-NEXT:  %[[LOAD2:.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>* %[[MLOAD_PTRS]], i32 4, <vscale x 4 x i1> %[[MASK]]
 ; CHECK-NEXT:  %[[FADD:.*]] = add <vscale x 4 x i32> %[[LOAD1]], %[[LOAD2]]
 ; CHECK-NEXT:  %[[MSTORE_PTRS:.*]] = bitcast i32* %[[GEPA]] to <vscale x 4 x i32>*
 ; CHECK-NEXT:  call void @llvm.masked.store.nxv4i32.p0nxv4i32(<vscale x 4 x i32> %[[FADD]], <vscale x 4 x i32>* %[[MSTORE_PTRS]], i32 4, <vscale x 4 x i1> %[[MASK]])
+
+; CHECK-TF-LABEL: @mloadstore_i32
+; CHECK-TF: vector.body:
+; CHECK-TF:       %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
+; CHECK-TF:       %[[LOAD1:.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32
+; CHECK-TF-NEXT:  %[[MASK:.*]] = icmp ne <vscale x 4 x i32> %[[LOAD1]],
+; CHECK-TF-NEXT:  %[[GEPA:.*]] = getelementptr i32, i32* %a,
+; CHECK-TF-NEXT:  %[[SEL:.*]] = select <vscale x 4 x i1> %[[ACTIVE_LANE_MASK]], <vscale x 4 x i1> %[[MASK]], <vscale x 4 x i1> zeroinitializer
+; CHECK-TF-NEXT:  %[[MLOAD_PTRS:.*]] = bitcast i32* %[[GEPA]] to <vscale x 4 x i32>*
+; CHECK-TF-NEXT:  %[[LOAD2:.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>* %[[MLOAD_PTRS]], i32 4, <vscale x 4 x i1> %[[SEL]]
+; CHECK-TF-NEXT:  %[[FADD:.*]] = add <vscale x 4 x i32> %[[LOAD1]], %[[LOAD2]]
+; CHECK-TF-NEXT:  %[[MSTORE_PTRS:.*]] = bitcast i32* %[[GEPA]] to <vscale x 4 x i32>*
+; CHECK-TF-NEXT:  call void @llvm.masked.store.nxv4i32.p0nxv4i32(<vscale x 4 x i32> %[[FADD]], <vscale x 4 x i32>* %[[MSTORE_PTRS]], i32 4, <vscale x 4 x i1> %[[SEL]])
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.inc
   %i.011 = phi i64 [ %inc, %for.inc ], [ 0, %entry ]
   %arrayidx = getelementptr inbounds i32, i32* %b, i64 %i.011
   %0 = load i32, i32* %arrayidx, align 4
   %cmp1 = icmp ne i32 %0, 0

llvm/test/Transforms/LoopVectorize/AArch64/sve-select-cmp.ll

 ; RUN: opt -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -S \
 ; RUN:   -prefer-predicate-over-epilogue=scalar-epilogue < %s | FileCheck %s --check-prefix=CHECK-VF4IC1
 ; RUN: opt -loop-vectorize -force-vector-interleave=4 -force-vector-width=4 -S \
 ; RUN:   -prefer-predicate-over-epilogue=scalar-epilogue < %s | FileCheck %s --check-prefix=CHECK-VF4IC4
+; RUN: opt -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -S \
+; RUN:   -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue < %s | FileCheck %s --check-prefix=CHECK-TF-VF4IC1
 
 target triple = "aarch64-linux-gnu"
 
 define i32 @select_const_i32_from_icmp(i32* nocapture readonly %v, i64 %n) #0 {
 ; CHECK-VF4IC1-LABEL: @select_const_i32_from_icmp
 ; CHECK-VF4IC1:      vector.body:
 ; CHECK-VF4IC1:        [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 3, i32 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer), %vector.ph ], [ [[VEC_SEL:%.*]], %vector.body ]
 ; CHECK-VF4IC1:        [[VEC_LOAD:%.*]] = load <vscale x 4 x i32>

 ; CHECK-VF4IC4-NEXT:   [[VEC_SEL5:%.*]] = select <vscale x 4 x i1> [[VEC_ICMP5]], <vscale x 4 x i32> [[VEC_SEL1]], <vscale x 4 x i32> [[VEC_SEL2]]
 ; CHECK-VF4IC4-NEXT:   [[VEC_ICMP6:%.*]] = icmp ne <vscale x 4 x i32> [[VEC_SEL5]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 3, i32 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
 ; CHECK-VF4IC4-NEXT:   [[VEC_SEL6:%.*]] = select <vscale x 4 x i1> [[VEC_ICMP6]], <vscale x 4 x i32> [[VEC_SEL5]], <vscale x 4 x i32> [[VEC_SEL3]]
 ; CHECK-VF4IC4-NEXT:   [[VEC_ICMP7:%.*]] = icmp ne <vscale x 4 x i32> [[VEC_SEL6]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 3, i32 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
 ; CHECK-VF4IC4-NEXT:   [[VEC_SEL7:%.*]] = select <vscale x 4 x i1> [[VEC_ICMP7]], <vscale x 4 x i32> [[VEC_SEL6]], <vscale x 4 x i32> [[VEC_SEL4]]
 ; CHECK-VF4IC4-NEXT:   [[FIN_ICMP:%.*]] = icmp ne <vscale x 4 x i32> [[VEC_SEL7]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 3, i32 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
 ; CHECK-VF4IC4-NEXT:   [[OR_RDX:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[FIN_ICMP]])
 ; CHECK-VF4IC4-NEXT:   {{.*}} = select i1 [[OR_RDX]], i32 7, i32 3
+
+; CHECK-TF-VF4IC1-LABEL: @select_const_i32_from_icmp
+; CHECK-TF-VF4IC1:      vector.body:
+; CHECK-TF-VF4IC1:        [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 3, i32 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer), %vector.ph ], [ [[VEC_SEL:%.*]], %vector.body ]
+; CHECK-TF-VF4IC1:        [[VEC_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32
+; CHECK-TF-VF4IC1-NEXT:   [[VEC_ICMP:%.*]] = icmp eq <vscale x 4 x i32> [[VEC_LOAD]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 3, i32 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-TF-VF4IC1-NEXT:   [[VEC_SEL1:%.*]] = select <vscale x 4 x i1> [[VEC_ICMP]], <vscale x 4 x i32> [[VEC_PHI]], <vscale x 4 x i32> shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 7, i32 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-TF-VF4IC1-NEXT:   [[VEC_SEL2:%.*]] = select <vscale x 4 x i1> %active.lane.mask, <vscale x 4 x i32> [[VEC_SEL1]], <vscale x 4 x i32> [[VEC_PHI]]
+; CHECK-TF-VF4IC1:      middle.block:
+; CHECK-TF-VF4IC1-NEXT:   [[FIN_ICMP:%.*]] = icmp ne <vscale x 4 x i32> [[VEC_SEL2]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 3, i32 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-TF-VF4IC1-NEXT:   [[OR_RDX:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[FIN_ICMP]])
+; CHECK-TF-VF4IC1-NEXT:   {{.*}} = select i1 [[OR_RDX]], i32 7, i32 3
 entry:
   br label %for.body
 
 for.body:                                      ; preds = %entry, %for.body
   %0 = phi i64 [ 0, %entry ], [ %6, %for.body ]
   %1 = phi i32 [ 3, %entry ], [ %5, %for.body ]
   %2 = getelementptr inbounds i32, i32* %v, i64 %0
   %3 = load i32, i32* %2, align 4

 ; CHECK-VF4IC1-NEXT:   [[FIN_INS:%.*]] = insertelement <vscale x 4 x i32> poison, i32 %a, i32 0
 ; CHECK-VF4IC1-NEXT:   [[FIN_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[FIN_INS]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-VF4IC1-NEXT:   [[FIN_CMP:%.*]] = icmp ne <vscale x 4 x i32> [[VEC_SEL]], [[FIN_SPLAT]]
 ; CHECK-VF4IC1-NEXT:   [[OR_RDX:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[FIN_CMP]])
 ; CHECK-VF4IC1-NEXT:   {{.*}} = select i1 [[OR_RDX]], i32 %b, i32 %a
 
 ; CHECK-VF4IC4-LABEL: @select_i32_from_icmp
 ; CHECK-VF4IC4:      vector.body:
+
+; CHECK-TF-VF4IC1-LABEL: @select_i32_from_icmp
+; CHECK-TF-VF4IC1:      vector.body:
 entry:
   br label %for.body
 
 for.body:                                      ; preds = %entry, %for.body
   %0 = phi i64 [ 0, %entry ], [ %6, %for.body ]
   %1 = phi i32 [ %a, %entry ], [ %5, %for.body ]
   %2 = getelementptr inbounds i32, i32* %v, i64 %0
   %3 = load i32, i32* %2, align 4

 ; CHECK-VF4IC1-NEXT:   [[VEC_SEL]] = select <vscale x 4 x i1> [[VEC_ICMP]], <vscale x 4 x i32> [[VEC_PHI]], <vscale x 4 x i32> shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i32 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
 ; CHECK-VF4IC1:      middle.block:
 ; CHECK-VF4IC1-NEXT:   [[FIN_ICMP:%.*]] = icmp ne <vscale x 4 x i32> [[VEC_SEL]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 2, i32 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
 ; CHECK-VF4IC1-NEXT:   [[OR_RDX:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[FIN_ICMP]])
 ; CHECK-VF4IC1-NEXT:   {{.*}} = select i1 [[OR_RDX]], i32 1, i32 2
 
 ; CHECK-VF4IC4-LABEL: @select_const_i32_from_fcmp
 ; CHECK-VF4IC4:      vector.body:
+
+; CHECK-TF-VF4IC1-LABEL: @select_const_i32_from_fcmp
+; CHECK-TF-VF4IC1:      vector.body:
 entry:
   br label %for.body
 
 for.body:                                      ; preds = %entry, %for.body
   %0 = phi i64 [ 0, %entry ], [ %6, %for.body ]
   %1 = phi i32 [ 2, %entry ], [ %5, %for.body ]
   %2 = getelementptr inbounds float, float* %v, i64 %0
   %3 = load float, float* %2, align 4
   %4 = fcmp fast ueq float %3, 3.0
   %5 = select i1 %4, i32 %1, i32 1
   %6 = add nuw nsw i64 %0, 1
   %7 = icmp eq i64 %6, %n
   br i1 %7, label %exit, label %for.body, !llvm.loop !0
 
 exit:                                     ; preds = %for.body
   ret i32 %5
 }
 
 define float @select_const_f32_from_icmp(i32* nocapture readonly %v, i64 %n) #0 {
 ; CHECK-VF4IC1-LABEL: @select_const_f32_from_icmp
 ; CHECK-VF4IC1-NOT: vector.body
 ; CHECK-VF4IC4-LABEL: @select_const_f32_from_icmp
 ; CHECK-VF4IC4-NOT: vector.body
+; CHECK-TF-VF4IC1-LABEL: @select_const_f32_from_icmp
+; CHECK-TF-VF4IC1-NOT: vector.body
 entry:
   br label %for.body
 
 for.body:                                      ; preds = %entry, %for.body
   %0 = phi i64 [ 0, %entry ], [ %6, %for.body ]
   %1 = phi fast float [ 3.0, %entry ], [ %5, %for.body ]
   %2 = getelementptr inbounds i32, i32* %v, i64 %0
   %3 = load i32, i32* %2, align 4

 ; CHECK-VF4IC1:        [[VEC_SEL:%.*]] = select <vscale x 4 x i1> [[MASK]], <vscale x 4 x i32> [[VEC_SEL_TMP]], <vscale x 4 x i32> [[VEC_PHI]]
 ; CHECK-VF4IC1:      middle.block:
 ; CHECK-VF4IC1-NEXT:   [[FIN_ICMP:%.*]] = icmp ne <vscale x 4 x i32> [[VEC_SEL]], zeroinitializer
 ; CHECK-VF4IC1-NEXT:   [[OR_RDX:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[FIN_ICMP]])
 ; CHECK-VF4IC1-NEXT:   {{.*}} = select i1 [[OR_RDX]], i32 1, i32 0
 
 ; CHECK-VF4IC4-LABEL: @pred_select_const_i32_from_icmp
 ; CHECK-VF4IC4:      vector.body:
+
+; CHECK-TF-VF4IC1-LABEL: @pred_select_const_i32_from_icmp
+; CHECK-TF-VF4IC1:      vector.body:
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.inc
   %i.013 = phi i64 [ %inc, %for.inc ], [ 0, %entry ]
   %r.012 = phi i32 [ %r.1, %for.inc ], [ 0, %entry ]
   %arrayidx = getelementptr inbounds i32, i32* %src1, i64 %i.013
   %0 = load i32, i32* %arrayidx, align 4

llvm/test/Transforms/LoopVectorize/AArch64/sve-strict-fadd-cost.ll

 ; REQUIRES: asserts
 ; RUN: opt < %s -loop-vectorize -debug -disable-output -force-ordered-reductions=true -hints-allow-reordering=false \
 ; RUN:   -prefer-predicate-over-epilogue=scalar-epilogue -force-vector-interleave=1 -S 2>&1 | FileCheck %s
 ; RUN: opt < %s -loop-vectorize -debug -disable-output -force-ordered-reductions=true -hints-allow-reordering=false \
 ; RUN:   -prefer-predicate-over-epilogue=scalar-epilogue -force-vector-interleave=1 \
 ; RUN:   -mcpu=neoverse-n2 -S 2>&1 | FileCheck %s --check-prefix=CHECK-CPU-NEOVERSE-N2
+; RUN: opt < %s -loop-vectorize -debug -disable-output -force-ordered-reductions=true -hints-allow-reordering=false \
+; RUN:   -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -force-vector-interleave=1 \
+; RUN:   -S 2>&1 | FileCheck %s --check-prefix=CHECK-TF
 
 target triple="aarch64-unknown-linux-gnu"
 
 ; CHECK: Found an estimated cost of 8 for VF vscale x 2 For instruction:   %add = fadd float %0, %sum.07
 ; CHECK: Found an estimated cost of 16 for VF vscale x 4 For instruction:   %add = fadd float %0, %sum.07
 ; CHECK-CPU-NEOVERSE-N2: Found an estimated cost of 4 for VF vscale x 2 For instruction:   %add = fadd float %0, %sum.07
 ; CHECK-CPU-NEOVERSE-N2: Found an estimated cost of 8 for VF vscale x 4 For instruction:   %add = fadd float %0, %sum.07
+; CHECK-TF: Found an estimated cost of 8 for VF vscale x 2 For instruction:   %add = fadd float %0, %sum.07
+; CHECK-TF: Found an estimated cost of 16 for VF vscale x 4 For instruction:   %add = fadd float %0, %sum.07
 
 define float @fadd_strict32(float* noalias nocapture readonly %a, i64 %n) #0 {
 entry:
   br label %for.body
 
 for.body:
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
   %sum.07 = phi float [ 0.000000e+00, %entry ], [ %add, %for.body ]
   %arrayidx = getelementptr inbounds float, float* %a, i64 %iv
   %0 = load float, float* %arrayidx, align 4
   %add = fadd float %0, %sum.07
   %iv.next = add nuw nsw i64 %iv, 1
   %exitcond.not = icmp eq i64 %iv.next, %n
   br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0
 
 for.end:
   ret float %add
 }
 
 
 ; CHECK: Found an estimated cost of 8 for VF vscale x 2 For instruction:   %add = fadd double %0, %sum.07
 ; CHECK-CPU-NEOVERSE-N2: Found an estimated cost of 4 for VF vscale x 2 For instruction:   %add = fadd double %0, %sum.07
+; CHECK-TF: Found an estimated cost of 8 for VF vscale x 2 For instruction:   %add = fadd double %0, %sum.07
 
 define double @fadd_strict64(double* noalias nocapture readonly %a, i64 %n) #0 {
 entry:
   br label %for.body
 
 for.body:
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
   %sum.07 = phi double [ 0.000000e+00, %entry ], [ %add, %for.body ]

llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse-mask4.ll

 ; This is the loop in c++ being vectorize in this file with
 ; experimental.vector.reverse
 
 ;#pragma clang loop vectorize_width(4, scalable)
 ;  for (long int i = N - 1; i >= 0; i--)
 ;  {
 ;    if (cond[i])
 ;      a[i] += 1;
 ;  }
 
 ; The test checks if the mask is being correctly created, reverted and used
 
 ; RUN: opt -loop-vectorize -dce -instcombine -mtriple aarch64-linux-gnu -S \
 ; RUN:   -prefer-predicate-over-epilogue=scalar-epilogue < %s | FileCheck %s
+; RUN: opt -loop-vectorize -dce -instcombine -mtriple aarch64-linux-gnu -S \
+; RUN:   -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue < %s | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64-unknown-linux-gnu"
 
 define void @vector_reverse_mask_nxv4i1(double* %a, double* %cond, i64 %N) #0 {
 ; CHECK-LABEL: vector.body:
+; CHECK: %{{.*}} = fcmp une <vscale x 4 x double>
 ; CHECK: %[[REVERSE6:.*]] = call <vscale x 4 x i1> @llvm.experimental.vector.reverse.nxv4i1(<vscale x 4 x i1> %{{.*}})
 ; CHECK: %[[WIDEMSKLOAD:.*]] = call <vscale x 4 x double> @llvm.masked.load.nxv4f64.p0nxv4f64(<vscale x 4 x double>* %{{.*}}, i32 8, <vscale x 4 x i1> %[[REVERSE6]], <vscale x 4 x double> poison)
 ; CHECK-NEXT: %[[FADD:.*]] = fadd <vscale x 4 x double> %[[WIDEMSKLOAD]]
 ; CHECK:  %[[REVERSE9:.*]] = call <vscale x 4 x i1> @llvm.experimental.vector.reverse.nxv4i1(<vscale x 4 x i1> %{{.*}})
 ; CHECK: call void @llvm.masked.store.nxv4f64.p0nxv4f64(<vscale x 4 x double> %[[FADD]], <vscale x 4 x double>* %{{.*}}, i32 8, <vscale x 4 x i1> %[[REVERSE9]]
 
 entry:
   %cmp7 = icmp sgt i64 %N, 0

llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; This is the loop in c++ being vectorize in this file with
 ;experimental.vector.reverse
 ;  #pragma clang loop vectorize_width(8, scalable)
 ;  for (int i = N-1; i >= 0; --i)
 ;    a[i] = b[i] + 1.0;
 
 ; RUN: opt -loop-vectorize -dce -instcombine -mtriple aarch64-linux-gnu -S \
 ; RUN:   -prefer-predicate-over-epilogue=scalar-epilogue < %s | FileCheck %s
+; RUN: opt -loop-vectorize -dce -instcombine -mtriple aarch64-linux-gnu -S \
+; RUN:   -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue < %s | FileCheck %s --check-prefix=CHECK-TF
 
 define void @vector_reverse_f64(i64 %N, double* noalias %a, double* noalias %b) #0{
 ; CHECK-LABEL: @vector_reverse_f64(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP7:%.*]] = icmp sgt i64 [[N:%.*]], 0
 ; CHECK-NEXT:    br i1 [[CMP7]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
 ; CHECK:       for.body.preheader:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()

 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[I_08]]
 ; CHECK-NEXT:    [[TMP22:%.*]] = load double, double* [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[ADD:%.*]] = fadd double [[TMP22]], 1.000000e+00
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[I_08]]
 ; CHECK-NEXT:    store double [[ADD]], double* [[ARRAYIDX1]], align 8
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[I_08_IN]], 1
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP_LOOPEXIT]], !llvm.loop [[LOOP3:![0-9]+]]
 ;
+; CHECK-TF-LABEL: @vector_reverse_f64(
+; CHECK-TF-NEXT:  entry:
+; CHECK-TF-NEXT:    [[CMP7:%.*]] = icmp sgt i64 [[N:%.*]], 0
+; CHECK-TF-NEXT:    br i1 [[CMP7]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK-TF:       for.body.preheader:
+; CHECK-TF-NEXT:    [[TMP0:%.*]] = xor i64 [[N]], -1
+; CHECK-TF-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP2:%.*]] = shl i64 [[TMP1]], 3
+; CHECK-TF-NEXT:    [[TMP3:%.*]] = icmp ugt i64 [[TMP2]], [[TMP0]]
+; CHECK-TF-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-TF:       vector.ph:
+; CHECK-TF-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP4]], 3
+; CHECK-TF-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP6]], 3
+; CHECK-TF-NEXT:    [[TMP8:%.*]] = add i64 [[TMP7]], -1
+; CHECK-TF-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP8]], [[N]]
+; CHECK-TF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
+; CHECK-TF-NEXT:    [[N_VEC_NEG:%.*]] = sub i64 [[N_MOD_VF]], [[N_RND_UP]]
+; CHECK-TF-NEXT:    [[IND_END:%.*]] = add i64 [[N_VEC_NEG]], [[N]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-TF:       vector.body:
+; CHECK-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[TMP9:%.*]] = xor i64 [[INDEX]], -1
+; CHECK-TF-NEXT:    [[TMP10:%.*]] = add i64 [[TMP9]], [[N]]
+; CHECK-TF-NEXT:    [[TMP11:%.*]] = getelementptr inbounds double, double* [[B:%.*]], i64 [[TMP10]]
+; CHECK-TF-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-NEXT:    [[DOTNEG:%.*]] = mul i32 [[TMP12]], -8
+; CHECK-TF-NEXT:    [[TMP13:%.*]] = or i32 [[DOTNEG]], 1
+; CHECK-TF-NEXT:    [[TMP14:%.*]] = sext i32 [[TMP13]] to i64
+; CHECK-TF-NEXT:    [[TMP15:%.*]] = getelementptr inbounds double, double* [[TMP11]], i64 [[TMP14]]
+; CHECK-TF-NEXT:    [[REVERSE:%.*]] = call <vscale x 8 x i1> @llvm.experimental.vector.reverse.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NEXT:    [[TMP16:%.*]] = bitcast double* [[TMP15]] to <vscale x 8 x double>*
+; CHECK-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x double> @llvm.masked.load.nxv8f64.p0nxv8f64(<vscale x 8 x double>* nonnull [[TMP16]], i32 8, <vscale x 8 x i1> [[REVERSE]], <vscale x 8 x double> poison)
+; CHECK-TF-NEXT:    [[TMP17:%.*]] = getelementptr inbounds double, double* [[A:%.*]], i64 [[TMP10]]
+; CHECK-TF-NEXT:    [[TMP18:%.*]] = fadd <vscale x 8 x double> [[WIDE_MASKED_LOAD]], shufflevector (<vscale x 8 x double> insertelement (<vscale x 8 x double> poison, double 1.000000e+00, i32 0), <vscale x 8 x double> poison, <vscale x 8 x i32> zeroinitializer)
+; CHECK-TF-NEXT:    [[TMP19:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-NEXT:    [[DOTNEG4:%.*]] = mul i32 [[TMP19]], -8
+; CHECK-TF-NEXT:    [[TMP20:%.*]] = or i32 [[DOTNEG4]], 1
+; CHECK-TF-NEXT:    [[TMP21:%.*]] = sext i32 [[TMP20]] to i64
+; CHECK-TF-NEXT:    [[TMP22:%.*]] = getelementptr inbounds double, double* [[TMP17]], i64 [[TMP21]]
+; CHECK-TF-NEXT:    [[REVERSE3:%.*]] = call <vscale x 8 x i1> @llvm.experimental.vector.reverse.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NEXT:    [[TMP23:%.*]] = bitcast double* [[TMP22]] to <vscale x 8 x double>*
+; CHECK-TF-NEXT:    call void @llvm.masked.store.nxv8f64.p0nxv8f64(<vscale x 8 x double> [[TMP18]], <vscale x 8 x double>* [[TMP23]], i32 8, <vscale x 8 x i1> [[REVERSE3]])
+; CHECK-TF-NEXT:    [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP25:%.*]] = shl i64 [[TMP24]], 3
+; CHECK-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP25]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NEXT:    [[TMP26:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NEXT:    br i1 [[TMP26]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF:       middle.block:
+; CHECK-TF-NEXT:    br i1 true, label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-TF:       scalar.ph:
+; CHECK-TF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[N]], [[FOR_BODY_PREHEADER]] ]
+; CHECK-TF-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-TF:       for.cond.cleanup.loopexit:
+; CHECK-TF-NEXT:    br label [[FOR_COND_CLEANUP]]
+; CHECK-TF:       for.cond.cleanup:
+; CHECK-TF-NEXT:    ret void
+; CHECK-TF:       for.body:
+; CHECK-TF-NEXT:    [[I_08_IN:%.*]] = phi i64 [ [[I_08:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-TF-NEXT:    [[I_08]] = add nsw i64 [[I_08_IN]], -1
+; CHECK-TF-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[I_08]]
+; CHECK-TF-NEXT:    [[TMP27:%.*]] = load double, double* [[ARRAYIDX]], align 8
+; CHECK-TF-NEXT:    [[ADD:%.*]] = fadd double [[TMP27]], 1.000000e+00
+; CHECK-TF-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[I_08]]
+; CHECK-TF-NEXT:    store double [[ADD]], double* [[ARRAYIDX1]], align 8
+; CHECK-TF-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[I_08_IN]], 1
+; CHECK-TF-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP_LOOPEXIT]], !llvm.loop [[LOOP3:![0-9]+]]
+;
 entry:
   %cmp7 = icmp sgt i64 %N, 0
   br i1 %cmp7, label %for.body, label %for.cond.cleanup
 
 for.cond.cleanup:                                 ; preds = %for.body
   ret void
 
 for.body:                                         ; preds = %entry, %for.body

 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, i64* [[B]], i64 [[I_09]]
 ; CHECK-NEXT:    [[TMP28:%.*]] = load i64, i64* [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[TMP28]], 1
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i64, i64* [[A]], i64 [[I_09]]
 ; CHECK-NEXT:    store i64 [[ADD]], i64* [[ARRAYIDX2]], align 8
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[I_09_IN]], 1
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP_LOOPEXIT]], !llvm.loop [[LOOP6:![0-9]+]]
 ;
+; CHECK-TF-LABEL: @vector_reverse_i64(
+; CHECK-TF-NEXT:  entry:
+; CHECK-TF-NEXT:    [[A2:%.*]] = ptrtoint i64* [[A:%.*]] to i64
+; CHECK-TF-NEXT:    [[B1:%.*]] = ptrtoint i64* [[B:%.*]] to i64
+; CHECK-TF-NEXT:    [[CMP8:%.*]] = icmp sgt i64 [[N:%.*]], 0
+; CHECK-TF-NEXT:    br i1 [[CMP8]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK-TF:       for.body.preheader:
+; CHECK-TF-NEXT:    [[TMP0:%.*]] = xor i64 [[N]], -1
+; CHECK-TF-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP2:%.*]] = shl i64 [[TMP1]], 3
+; CHECK-TF-NEXT:    [[TMP3:%.*]] = icmp ugt i64 [[TMP2]], [[TMP0]]
+; CHECK-TF-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; CHECK-TF:       vector.memcheck:
+; CHECK-TF-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP4]], 6
+; CHECK-TF-NEXT:    [[TMP6:%.*]] = shl i64 [[N]], 3
+; CHECK-TF-NEXT:    [[TMP7:%.*]] = add i64 [[TMP6]], [[B1]]
+; CHECK-TF-NEXT:    [[TMP8:%.*]] = add i64 [[TMP6]], [[A2]]
+; CHECK-TF-NEXT:    [[TMP9:%.*]] = sub i64 [[TMP7]], [[TMP8]]
+; CHECK-TF-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP9]], [[TMP5]]
+; CHECK-TF-NEXT:    br i1 [[DIFF_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; CHECK-TF:       vector.ph:
+; CHECK-TF-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP11:%.*]] = shl i64 [[TMP10]], 3
+; CHECK-TF-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP13:%.*]] = shl i64 [[TMP12]], 3
+; CHECK-TF-NEXT:    [[TMP14:%.*]] = add i64 [[TMP13]], -1
+; CHECK-TF-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP14]], [[N]]
+; CHECK-TF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP11]]
+; CHECK-TF-NEXT:    [[N_VEC_NEG:%.*]] = sub i64 [[N_MOD_VF]], [[N_RND_UP]]
+; CHECK-TF-NEXT:    [[IND_END:%.*]] = add i64 [[N_VEC_NEG]], [[N]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-TF:       vector.body:
+; CHECK-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[TMP15:%.*]] = xor i64 [[INDEX]], -1
+; CHECK-TF-NEXT:    [[TMP16:%.*]] = add i64 [[TMP15]], [[N]]
+; CHECK-TF-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i64, i64* [[B]], i64 [[TMP16]]
+; CHECK-TF-NEXT:    [[TMP18:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-NEXT:    [[DOTNEG:%.*]] = mul i32 [[TMP18]], -8
+; CHECK-TF-NEXT:    [[TMP19:%.*]] = or i32 [[DOTNEG]], 1
+; CHECK-TF-NEXT:    [[TMP20:%.*]] = sext i32 [[TMP19]] to i64
+; CHECK-TF-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i64, i64* [[TMP17]], i64 [[TMP20]]
+; CHECK-TF-NEXT:    [[REVERSE:%.*]] = call <vscale x 8 x i1> @llvm.experimental.vector.reverse.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NEXT:    [[TMP22:%.*]] = bitcast i64* [[TMP21]] to <vscale x 8 x i64>*
+; CHECK-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i64> @llvm.masked.load.nxv8i64.p0nxv8i64(<vscale x 8 x i64>* nonnull [[TMP22]], i32 8, <vscale x 8 x i1> [[REVERSE]], <vscale x 8 x i64> poison)
+; CHECK-TF-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i64, i64* [[A]], i64 [[TMP16]]
+; CHECK-TF-NEXT:    [[TMP24:%.*]] = add <vscale x 8 x i64> [[WIDE_MASKED_LOAD]], shufflevector (<vscale x 8 x i64> insertelement (<vscale x 8 x i64> poison, i64 1, i32 0), <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer)
+; CHECK-TF-NEXT:    [[TMP25:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-NEXT:    [[DOTNEG6:%.*]] = mul i32 [[TMP25]], -8
+; CHECK-TF-NEXT:    [[TMP26:%.*]] = or i32 [[DOTNEG6]], 1
+; CHECK-TF-NEXT:    [[TMP27:%.*]] = sext i32 [[TMP26]] to i64
+; CHECK-TF-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i64, i64* [[TMP23]], i64 [[TMP27]]
+; CHECK-TF-NEXT:    [[REVERSE5:%.*]] = call <vscale x 8 x i1> @llvm.experimental.vector.reverse.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NEXT:    [[TMP29:%.*]] = bitcast i64* [[TMP28]] to <vscale x 8 x i64>*
+; CHECK-TF-NEXT:    call void @llvm.masked.store.nxv8i64.p0nxv8i64(<vscale x 8 x i64> [[TMP24]], <vscale x 8 x i64>* [[TMP29]], i32 8, <vscale x 8 x i1> [[REVERSE5]])
+; CHECK-TF-NEXT:    [[TMP30:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP31:%.*]] = shl i64 [[TMP30]], 3
+; CHECK-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP31]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NEXT:    [[TMP32:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NEXT:    br i1 [[TMP32]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-TF:       middle.block:
+; CHECK-TF-NEXT:    br i1 true, label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-TF:       scalar.ph:
+; CHECK-TF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[N]], [[FOR_BODY_PREHEADER]] ], [ [[N]], [[VECTOR_MEMCHECK]] ]
+; CHECK-TF-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-TF:       for.cond.cleanup.loopexit:
+; CHECK-TF-NEXT:    br label [[FOR_COND_CLEANUP]]
+; CHECK-TF:       for.cond.cleanup:
+; CHECK-TF-NEXT:    ret void
+; CHECK-TF:       for.body:
+; CHECK-TF-NEXT:    [[I_09_IN:%.*]] = phi i64 [ [[I_09:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-TF-NEXT:    [[I_09]] = add nsw i64 [[I_09_IN]], -1
+; CHECK-TF-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, i64* [[B]], i64 [[I_09]]
+; CHECK-TF-NEXT:    [[TMP33:%.*]] = load i64, i64* [[ARRAYIDX]], align 8
+; CHECK-TF-NEXT:    [[ADD:%.*]] = add i64 [[TMP33]], 1
+; CHECK-TF-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i64, i64* [[A]], i64 [[I_09]]
+; CHECK-TF-NEXT:    store i64 [[ADD]], i64* [[ARRAYIDX2]], align 8
+; CHECK-TF-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[I_09_IN]], 1
+; CHECK-TF-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP_LOOPEXIT]], !llvm.loop [[LOOP6:![0-9]+]]
+;
 entry:
   %cmp8 = icmp sgt i64 %N, 0
   br i1 %cmp8, label %for.body, label %for.cond.cleanup
 
 for.cond.cleanup:                                 ; preds = %for.body
   ret void
 
 for.body:                                         ; preds = %entry, %for.body

llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; REQUIRES: asserts
 ; RUN: opt -loop-vectorize -S -mtriple=aarch64 -mattr=+sve -debug-only=loop-vectorize \
 ; RUN:   -prefer-predicate-over-epilogue=scalar-epilogue < %s 2>&1 | FileCheck %s
+; RUN: opt -loop-vectorize -S -mtriple=aarch64 -mattr=+sve \
+; RUN:   -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue < %s 2>&1 | FileCheck %s --check-prefix=CHECK-TF
 
 target triple = "aarch64-unknown-linux-gnu"
 
 ; CHECK-LABEL:  LV: Checking a loop in 'pointer_induction_used_as_vector'
 ; CHECK-NOT:    LV: Found {{.*}} scalar instruction:   %ptr.iv.2.next = getelementptr inbounds i8, i8* %ptr.iv.2, i64 1
 ;
 ; CHECK:        VPlan 'Initial VPlan for VF={vscale x 2},UF>=1' {
 ; CHECK-NEXT:   Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count

 ; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[LV]], 1
 ; CHECK-NEXT:    store i8 [[ADD]], i8* [[PTR_IV_2]], align 1
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw i64 [[IV]], 1
 ; CHECK-NEXT:    [[C:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[C]], label [[LOOP_BODY]], label [[EXIT]], !llvm.loop [[LOOP2:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
+; CHECK-TF-LABEL: @pointer_induction_used_as_vector(
+; CHECK-TF-NEXT:  entry:
+; CHECK-TF-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
+; CHECK-TF-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 2
+; CHECK-TF-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; CHECK-TF-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-TF:       vector.ph:
+; CHECK-TF-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; CHECK-TF-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 2
+; CHECK-TF-NEXT:    [[TMP8:%.*]] = sub i64 [[TMP7]], 1
+; CHECK-TF-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP8]]
+; CHECK-TF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
+; CHECK-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-TF-NEXT:    [[IND_END:%.*]] = getelementptr i8*, i8** [[START_1:%.*]], i64 [[N_VEC]]
+; CHECK-TF-NEXT:    [[IND_END2:%.*]] = getelementptr i8, i8* [[START_2:%.*]], i64 [[N_VEC]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-TF:       vector.body:
+; CHECK-TF-NEXT:    [[POINTER_PHI:%.*]] = phi i8* [ [[START_2]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 0
+; CHECK-TF-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8*, i8** [[START_1]], i64 [[TMP9]]
+; CHECK-TF-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 2
+; CHECK-TF-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 1
+; CHECK-TF-NEXT:    [[TMP13:%.*]] = mul i64 1, [[TMP12]]
+; CHECK-TF-NEXT:    [[TMP14:%.*]] = mul i64 [[TMP11]], 0
+; CHECK-TF-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP14]], i32 0
+; CHECK-TF-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-TF-NEXT:    [[TMP15:%.*]] = call <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
+; CHECK-TF-NEXT:    [[TMP16:%.*]] = add <vscale x 2 x i64> [[DOTSPLAT]], [[TMP15]]
+; CHECK-TF-NEXT:    [[VECTOR_GEP:%.*]] = mul <vscale x 2 x i64> [[TMP16]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i32 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
+; CHECK-TF-NEXT:    [[TMP17:%.*]] = getelementptr i8, i8* [[POINTER_PHI]], <vscale x 2 x i64> [[VECTOR_GEP]]
+; CHECK-TF-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, <vscale x 2 x i8*> [[TMP17]], i64 1
+; CHECK-TF-NEXT:    [[TMP19:%.*]] = getelementptr i8*, i8** [[NEXT_GEP]], i32 0
+; CHECK-TF-NEXT:    [[TMP20:%.*]] = bitcast i8** [[TMP19]] to <vscale x 2 x i8*>*
+; CHECK-TF-NEXT:    call void @llvm.masked.store.nxv2p0i8.p0nxv2p0i8(<vscale x 2 x i8*> [[TMP18]], <vscale x 2 x i8*>* [[TMP20]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NEXT:    [[TMP21:%.*]] = extractelement <vscale x 2 x i8*> [[TMP17]], i32 0
+; CHECK-TF-NEXT:    [[TMP22:%.*]] = getelementptr i8, i8* [[TMP21]], i32 0
+; CHECK-TF-NEXT:    [[TMP23:%.*]] = bitcast i8* [[TMP22]] to <vscale x 2 x i8>*
+; CHECK-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i8> @llvm.masked.load.nxv2i8.p0nxv2i8(<vscale x 2 x i8>* [[TMP23]], i32 1, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i8> poison)
+; CHECK-TF-NEXT:    [[TMP24:%.*]] = add <vscale x 2 x i8> [[WIDE_MASKED_LOAD]], shufflevector (<vscale x 2 x i8> insertelement (<vscale x 2 x i8> poison, i8 1, i32 0), <vscale x 2 x i8> poison, <vscale x 2 x i32> zeroinitializer)
+; CHECK-TF-NEXT:    [[TMP25:%.*]] = bitcast i8* [[TMP22]] to <vscale x 2 x i8>*
+; CHECK-TF-NEXT:    call void @llvm.masked.store.nxv2i8.p0nxv2i8(<vscale x 2 x i8> [[TMP24]], <vscale x 2 x i8>* [[TMP25]], i32 1, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NEXT:    [[TMP26:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP27:%.*]] = mul i64 [[TMP26]], 2
+; CHECK-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP27]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NEXT:    [[TMP28:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i32 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)
+; CHECK-TF-NEXT:    [[PTR_IND]] = getelementptr i8, i8* [[POINTER_PHI]], i64 [[TMP13]]
+; CHECK-TF-NEXT:    [[TMP29:%.*]] = extractelement <vscale x 2 x i1> [[TMP28]], i32 0
+; CHECK-TF-NEXT:    br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF:       middle.block:
+; CHECK-TF-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-TF:       scalar.ph:
+; CHECK-TF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-TF-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i8** [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[START_1]], [[ENTRY]] ]
+; CHECK-TF-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i8* [ [[IND_END2]], [[MIDDLE_BLOCK]] ], [ [[START_2]], [[ENTRY]] ]
+; CHECK-TF-NEXT:    br label [[LOOP_BODY:%.*]]
+; CHECK-TF:       loop.body:
+; CHECK-TF-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_BODY]] ]
+; CHECK-TF-NEXT:    [[PTR_IV_1:%.*]] = phi i8** [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[PTR_IV_1_NEXT:%.*]], [[LOOP_BODY]] ]
+; CHECK-TF-NEXT:    [[PTR_IV_2:%.*]] = phi i8* [ [[BC_RESUME_VAL3]], [[SCALAR_PH]] ], [ [[PTR_IV_2_NEXT:%.*]], [[LOOP_BODY]] ]
+; CHECK-TF-NEXT:    [[PTR_IV_1_NEXT]] = getelementptr inbounds i8*, i8** [[PTR_IV_1]], i64 1
+; CHECK-TF-NEXT:    [[PTR_IV_2_NEXT]] = getelementptr inbounds i8, i8* [[PTR_IV_2]], i64 1
+; CHECK-TF-NEXT:    store i8* [[PTR_IV_2_NEXT]], i8** [[PTR_IV_1]], align 8
+; CHECK-TF-NEXT:    [[LV:%.*]] = load i8, i8* [[PTR_IV_2]], align 1
+; CHECK-TF-NEXT:    [[ADD:%.*]] = add i8 [[LV]], 1
+; CHECK-TF-NEXT:    store i8 [[ADD]], i8* [[PTR_IV_2]], align 1
+; CHECK-TF-NEXT:    [[IV_NEXT]] = add nuw i64 [[IV]], 1
+; CHECK-TF-NEXT:    [[C:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
+; CHECK-TF-NEXT:    br i1 [[C]], label [[LOOP_BODY]], label [[EXIT]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK-TF:       exit:
+; CHECK-TF-NEXT:    ret void
+;
 
 
 entry:
   br label %loop.body
 
 loop.body:                                    ; preds = %loop.body, %entry
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.body ]
   %ptr.iv.1 = phi i8** [ %start.1, %entry ], [ %ptr.iv.1.next, %loop.body ]

 ; CHECK-NEXT:    store i8 [[ADD]], i8* [[PTR_PHI]], align 1
 ; CHECK-NEXT:    [[PTR_PHI_NEXT]] = getelementptr inbounds i8, i8* [[PTR_PHI]], i64 1
 ; CHECK-NEXT:    [[CMP_I_NOT:%.*]] = icmp eq i8* [[PTR_PHI_NEXT]], [[START]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[INDEX]], [[N]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[END]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       end:
 ; CHECK-NEXT:    ret void
 ;
+; CHECK-TF-LABEL: @pointer_induction(
+; CHECK-TF-NEXT:  entry:
+; CHECK-TF-NEXT:    [[TMP0:%.*]] = add i64 [[N:%.*]], 1
+; CHECK-TF-NEXT:    [[TMP1:%.*]] = sub i64 -1, [[TMP0]]
+; CHECK-TF-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; CHECK-TF-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP1]], [[TMP3]]
+; CHECK-TF-NEXT:    br i1 [[TMP4]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-TF:       vector.ph:
+; CHECK-TF-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; CHECK-TF-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 2
+; CHECK-TF-NEXT:    [[TMP9:%.*]] = sub i64 [[TMP8]], 1
+; CHECK-TF-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], [[TMP9]]
+; CHECK-TF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
+; CHECK-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-TF-NEXT:    [[IND_END:%.*]] = getelementptr i8, i8* [[START:%.*]], i64 [[N_VEC]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[TMP0]])
+; CHECK-TF-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-TF:       vector.body:
+; CHECK-TF-NEXT:    [[INDEX2:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX2]], 0
+; CHECK-TF-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, i8* [[START]], i64 [[TMP10]]
+; CHECK-TF-NEXT:    [[TMP11:%.*]] = getelementptr i8, i8* [[NEXT_GEP]], i32 0
+; CHECK-TF-NEXT:    [[TMP12:%.*]] = bitcast i8* [[TMP11]] to <vscale x 2 x i8>*
+; CHECK-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i8> @llvm.masked.load.nxv2i8.p0nxv2i8(<vscale x 2 x i8>* [[TMP12]], i32 1, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i8> poison)
+; CHECK-TF-NEXT:    [[TMP13:%.*]] = add <vscale x 2 x i8> [[WIDE_MASKED_LOAD]], shufflevector (<vscale x 2 x i8> insertelement (<vscale x 2 x i8> poison, i8 1, i32 0), <vscale x 2 x i8> poison, <vscale x 2 x i32> zeroinitializer)
+; CHECK-TF-NEXT:    [[TMP14:%.*]] = bitcast i8* [[TMP11]] to <vscale x 2 x i8>*
+; CHECK-TF-NEXT:    call void @llvm.masked.store.nxv2i8.p0nxv2i8(<vscale x 2 x i8> [[TMP13]], <vscale x 2 x i8>* [[TMP14]], i32 1, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 2
+; CHECK-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX2]], [[TMP16]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
+; CHECK-TF-NEXT:    [[TMP17:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i32 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)
+; CHECK-TF-NEXT:    [[TMP18:%.*]] = extractelement <vscale x 2 x i1> [[TMP17]], i32 0
+; CHECK-TF-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-TF:       middle.block:
+; CHECK-TF-NEXT:    br i1 true, label [[END:%.*]], label [[SCALAR_PH]]
+; CHECK-TF:       scalar.ph:
+; CHECK-TF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8* [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[START]], [[ENTRY:%.*]] ]
+; CHECK-TF-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-TF-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-TF:       for.body:
+; CHECK-TF-NEXT:    [[PTR_PHI:%.*]] = phi i8* [ [[PTR_PHI_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
+; CHECK-TF-NEXT:    [[INDEX_NXT]] = add i64 [[INDEX]], 1
+; CHECK-TF-NEXT:    [[TMP19:%.*]] = load i8, i8* [[PTR_PHI]], align 1
+; CHECK-TF-NEXT:    [[ADD:%.*]] = add i8 [[TMP19]], 1
+; CHECK-TF-NEXT:    store i8 [[ADD]], i8* [[PTR_PHI]], align 1
+; CHECK-TF-NEXT:    [[PTR_PHI_NEXT]] = getelementptr inbounds i8, i8* [[PTR_PHI]], i64 1
+; CHECK-TF-NEXT:    [[CMP_I_NOT:%.*]] = icmp eq i8* [[PTR_PHI_NEXT]], [[START]]
+; CHECK-TF-NEXT:    [[CMP:%.*]] = icmp ult i64 [[INDEX]], [[N]]
+; CHECK-TF-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[END]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-TF:       end:
+; CHECK-TF-NEXT:    ret void
+;
 entry:
   br label %for.body
 
 for.body:
   %ptr.phi = phi i8* [ %ptr.phi.next, %for.body ], [ %start, %entry ]
   %index = phi i64 [ %index_nxt, %for.body ], [ 0, %entry ]
   %index_nxt = add i64 %index, 1
   %0 = load i8, i8* %ptr.phi, align 1

llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-phi.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -mtriple aarch64-linux-gnu -mattr=+sve -loop-vectorize -dce -instcombine -S \
 ; RUN:   -prefer-predicate-over-epilogue=scalar-epilogue < %s | FileCheck %s
+; RUN: opt -mtriple aarch64-linux-gnu -mattr=+sve -loop-vectorize -dce -instcombine -S \
+; RUN:   -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue < %s | FileCheck %s --check-prefix=CHECK-TF
 
 ; Ensure that we can vectorize loops such as:
 ;   int *ptr = c;
 ;   for (long long i = 0; i < n; i++) {
 ;     int X1 = *ptr++;
 ;     int X2 = *ptr++;
 ;     a[i] = X1 + 1;
 ;     b[i] = X2 + 1;

 ; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[I_013]]
 ; CHECK-NEXT:    store i32 [[ADD2]], i32* [[ARRAYIDX3]], align 4
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i64 [[I_013]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       for.exit:
 ; CHECK-NEXT:    ret void
 ;
+; CHECK-TF-LABEL: @widen_ptr_phi_unrolled(
+; CHECK-TF-NEXT:  entry:
+; CHECK-TF-NEXT:    [[TMP0:%.*]] = xor i64 [[N:%.*]], -1
+; CHECK-TF-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP1]], 3
+; CHECK-TF-NEXT:    [[TMP3:%.*]] = icmp ugt i64 [[TMP2]], [[TMP0]]
+; CHECK-TF-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-TF:       vector.ph:
+; CHECK-TF-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 3
+; CHECK-TF-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 3
+; CHECK-TF-NEXT:    [[TMP8:%.*]] = add nsw i64 [[TMP7]], -1
+; CHECK-TF-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP8]], [[N]]
+; CHECK-TF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
+; CHECK-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-TF-NEXT:    [[TMP9:%.*]] = shl i64 [[N_VEC]], 1
+; CHECK-TF-NEXT:    [[IND_END:%.*]] = getelementptr i32, i32* [[C:%.*]], i64 [[TMP9]]
+; CHECK-TF-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP11:%.*]] = shl nuw nsw i64 [[TMP10]], 2
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY2:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP11]], i64 [[N]])
+; CHECK-TF-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-TF:       vector.body:
+; CHECK-TF-NEXT:    [[POINTER_PHI:%.*]] = phi i32* [ [[C]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK3:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY2]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT10:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP13:%.*]] = shl nuw nsw i64 [[TMP12]], 2
+; CHECK-TF-NEXT:    [[TMP14:%.*]] = shl nuw nsw i64 [[TMP12]], 4
+; CHECK-TF-NEXT:    [[TMP15:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
+; CHECK-TF-NEXT:    [[VECTOR_GEP:%.*]] = shl <vscale x 4 x i64> [[TMP15]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i32 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-TF-NEXT:    [[TMP16:%.*]] = getelementptr i32, i32* [[POINTER_PHI]], <vscale x 4 x i64> [[VECTOR_GEP]]
+; CHECK-TF-NEXT:    [[DOTSPLATINSERT4:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP13]], i64 0
+; CHECK-TF-NEXT:    [[DOTSPLAT5:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT4]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-TF-NEXT:    [[TMP17:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
+; CHECK-TF-NEXT:    [[TMP18:%.*]] = add <vscale x 4 x i64> [[DOTSPLAT5]], [[TMP17]]
+; CHECK-TF-NEXT:    [[VECTOR_GEP6:%.*]] = shl <vscale x 4 x i64> [[TMP18]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i32 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-TF-NEXT:    [[TMP19:%.*]] = getelementptr i32, i32* [[POINTER_PHI]], <vscale x 4 x i64> [[VECTOR_GEP6]]
+; CHECK-TF-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i32, <vscale x 4 x i32*> [[TMP16]], i64 1
+; CHECK-TF-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, <vscale x 4 x i32*> [[TMP19]], i64 1
+; CHECK-TF-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> [[TMP16]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> undef)
+; CHECK-TF-NEXT:    [[WIDE_MASKED_GATHER7:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> [[TMP19]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK3]], <vscale x 4 x i32> undef)
+; CHECK-TF-NEXT:    [[WIDE_MASKED_GATHER8:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> [[TMP20]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> undef)
+; CHECK-TF-NEXT:    [[WIDE_MASKED_GATHER9:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> [[TMP21]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK3]], <vscale x 4 x i32> undef)
+; CHECK-TF-NEXT:    [[TMP22:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i32 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-TF-NEXT:    [[TMP23:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_GATHER7]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i32 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-TF-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDEX]]
+; CHECK-TF-NEXT:    [[TMP25:%.*]] = bitcast i32* [[TMP24]] to <vscale x 4 x i32>*
+; CHECK-TF-NEXT:    call void @llvm.masked.store.nxv4i32.p0nxv4i32(<vscale x 4 x i32> [[TMP22]], <vscale x 4 x i32>* [[TMP25]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NEXT:    [[TMP26:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-NEXT:    [[TMP27:%.*]] = shl nuw nsw i32 [[TMP26]], 2
+; CHECK-TF-NEXT:    [[TMP28:%.*]] = zext i32 [[TMP27]] to i64
+; CHECK-TF-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i32, i32* [[TMP24]], i64 [[TMP28]]
+; CHECK-TF-NEXT:    [[TMP30:%.*]] = bitcast i32* [[TMP29]] to <vscale x 4 x i32>*
+; CHECK-TF-NEXT:    call void @llvm.masked.store.nxv4i32.p0nxv4i32(<vscale x 4 x i32> [[TMP23]], <vscale x 4 x i32>* [[TMP30]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK3]])
+; CHECK-TF-NEXT:    [[TMP31:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_GATHER8]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i32 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-TF-NEXT:    [[TMP32:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_GATHER9]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i32 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-TF-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDEX]]
+; CHECK-TF-NEXT:    [[TMP34:%.*]] = bitcast i32* [[TMP33]] to <vscale x 4 x i32>*
+; CHECK-TF-NEXT:    call void @llvm.masked.store.nxv4i32.p0nxv4i32(<vscale x 4 x i32> [[TMP31]], <vscale x 4 x i32>* [[TMP34]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NEXT:    [[TMP35:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-NEXT:    [[TMP36:%.*]] = shl nuw nsw i32 [[TMP35]], 2
+; CHECK-TF-NEXT:    [[TMP37:%.*]] = zext i32 [[TMP36]] to i64
+; CHECK-TF-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i32, i32* [[TMP33]], i64 [[TMP37]]
+; CHECK-TF-NEXT:    [[TMP39:%.*]] = bitcast i32* [[TMP38]] to <vscale x 4 x i32>*
+; CHECK-TF-NEXT:    call void @llvm.masked.store.nxv4i32.p0nxv4i32(<vscale x 4 x i32> [[TMP32]], <vscale x 4 x i32>* [[TMP39]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK3]])
+; CHECK-TF-NEXT:    [[TMP40:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP41:%.*]] = shl nuw nsw i64 [[TMP40]], 3
+; CHECK-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP41]]
+; CHECK-TF-NEXT:    [[TMP42:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP43:%.*]] = shl nuw nsw i64 [[TMP42]], 2
+; CHECK-TF-NEXT:    [[TMP44:%.*]] = add i64 [[INDEX_NEXT]], [[TMP43]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT10]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP44]], i64 [[N]])
+; CHECK-TF-NEXT:    [[PTR_IND]] = getelementptr i32, i32* [[POINTER_PHI]], i64 [[TMP14]]
+; CHECK-TF-NEXT:    [[TMP45:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NEXT:    br i1 [[TMP45]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF:       middle.block:
+; CHECK-TF-NEXT:    br i1 true, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-TF:       scalar.ph:
+; CHECK-TF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32* [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[C]], [[ENTRY:%.*]] ]
+; CHECK-TF-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-TF-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-TF:       for.body:
+; CHECK-TF-NEXT:    [[PTR_014:%.*]] = phi i32* [ [[INCDEC_PTR1:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-TF-NEXT:    [[I_013:%.*]] = phi i64 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
+; CHECK-TF-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, i32* [[PTR_014]], i64 1
+; CHECK-TF-NEXT:    [[TMP46:%.*]] = load i32, i32* [[PTR_014]], align 4
+; CHECK-TF-NEXT:    [[INCDEC_PTR1]] = getelementptr inbounds i32, i32* [[PTR_014]], i64 2
+; CHECK-TF-NEXT:    [[TMP47:%.*]] = load i32, i32* [[INCDEC_PTR]], align 4
+; CHECK-TF-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP46]], 1
+; CHECK-TF-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[I_013]]
+; CHECK-TF-NEXT:    store i32 [[ADD]], i32* [[ARRAYIDX]], align 4
+; CHECK-TF-NEXT:    [[ADD2:%.*]] = add nsw i32 [[TMP47]], 1
+; CHECK-TF-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[I_013]]
+; CHECK-TF-NEXT:    store i32 [[ADD2]], i32* [[ARRAYIDX3]], align 4
+; CHECK-TF-NEXT:    [[INC]] = add nuw nsw i64 [[I_013]], 1
+; CHECK-TF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-TF-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-TF:       for.exit:
+; CHECK-TF-NEXT:    ret void
+;
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.body
   %ptr.014 = phi i32* [ %incdec.ptr1, %for.body ], [ %c, %entry ]
   %i.013 = phi i64 [ %inc, %for.body ], [ 0, %entry ]
   %incdec.ptr = getelementptr inbounds i32, i32* %ptr.014, i64 1
   %0 = load i32, i32* %ptr.014, align 4

 ; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i32, i32* [[D_09]], i64 1
 ; CHECK-NEXT:    [[INCDEC_PTR1]] = getelementptr inbounds i32, i32* [[S_010]], i64 1
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i64 [[I_011]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
 ; CHECK-NEXT:    ret void
 ;
+; CHECK-TF-LABEL: @widen_2ptrs_phi_unrolled(
+; CHECK-TF-NEXT:  entry:
+; CHECK-TF-NEXT:    [[TMP0:%.*]] = xor i64 [[N:%.*]], -1
+; CHECK-TF-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP1]], 3
+; CHECK-TF-NEXT:    [[TMP3:%.*]] = icmp ugt i64 [[TMP2]], [[TMP0]]
+; CHECK-TF-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-TF:       vector.ph:
+; CHECK-TF-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 3
+; CHECK-TF-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 3
+; CHECK-TF-NEXT:    [[TMP8:%.*]] = add nsw i64 [[TMP7]], -1
+; CHECK-TF-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP8]], [[N]]
+; CHECK-TF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
+; CHECK-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-TF-NEXT:    [[IND_END:%.*]] = getelementptr i32, i32* [[SRC:%.*]], i64 [[N_VEC]]
+; CHECK-TF-NEXT:    [[IND_END2:%.*]] = getelementptr i32, i32* [[DST:%.*]], i64 [[N_VEC]]
+; CHECK-TF-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP10:%.*]] = shl nuw nsw i64 [[TMP9]], 2
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY4:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP10]], i64 [[N]])
+; CHECK-TF-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-TF:       vector.body:
+; CHECK-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK5:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY4]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT10:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i32, i32* [[SRC]], i64 [[INDEX]]
+; CHECK-TF-NEXT:    [[NEXT_GEP7:%.*]] = getelementptr i32, i32* [[DST]], i64 [[INDEX]]
+; CHECK-TF-NEXT:    [[TMP11:%.*]] = bitcast i32* [[NEXT_GEP]] to <vscale x 4 x i32>*
+; CHECK-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>* [[TMP11]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-TF-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-NEXT:    [[TMP13:%.*]] = shl nuw nsw i32 [[TMP12]], 2
+; CHECK-TF-NEXT:    [[TMP14:%.*]] = zext i32 [[TMP13]] to i64
+; CHECK-TF-NEXT:    [[TMP15:%.*]] = getelementptr i32, i32* [[NEXT_GEP]], i64 [[TMP14]]
+; CHECK-TF-NEXT:    [[TMP16:%.*]] = bitcast i32* [[TMP15]] to <vscale x 4 x i32>*
+; CHECK-TF-NEXT:    [[WIDE_MASKED_LOAD9:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>* [[TMP16]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK5]], <vscale x 4 x i32> poison)
+; CHECK-TF-NEXT:    [[TMP17:%.*]] = shl nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i32 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-TF-NEXT:    [[TMP18:%.*]] = shl nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD9]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i32 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-TF-NEXT:    [[TMP19:%.*]] = bitcast i32* [[NEXT_GEP7]] to <vscale x 4 x i32>*
+; CHECK-TF-NEXT:    call void @llvm.masked.store.nxv4i32.p0nxv4i32(<vscale x 4 x i32> [[TMP17]], <vscale x 4 x i32>* [[TMP19]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NEXT:    [[TMP20:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-NEXT:    [[TMP21:%.*]] = shl nuw nsw i32 [[TMP20]], 2
+; CHECK-TF-NEXT:    [[TMP22:%.*]] = zext i32 [[TMP21]] to i64
+; CHECK-TF-NEXT:    [[TMP23:%.*]] = getelementptr i32, i32* [[NEXT_GEP7]], i64 [[TMP22]]
+; CHECK-TF-NEXT:    [[TMP24:%.*]] = bitcast i32* [[TMP23]] to <vscale x 4 x i32>*
+; CHECK-TF-NEXT:    call void @llvm.masked.store.nxv4i32.p0nxv4i32(<vscale x 4 x i32> [[TMP18]], <vscale x 4 x i32>* [[TMP24]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK5]])
+; CHECK-TF-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP26:%.*]] = shl nuw nsw i64 [[TMP25]], 3
+; CHECK-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP26]]
+; CHECK-TF-NEXT:    [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP28:%.*]] = shl nuw nsw i64 [[TMP27]], 2
+; CHECK-TF-NEXT:    [[TMP29:%.*]] = add i64 [[INDEX_NEXT]], [[TMP28]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT10]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP29]], i64 [[N]])
+; CHECK-TF-NEXT:    [[TMP30:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NEXT:    br i1 [[TMP30]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-TF:       middle.block:
+; CHECK-TF-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK-TF:       scalar.ph:
+; CHECK-TF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-TF-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32* [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[SRC]], [[ENTRY]] ]
+; CHECK-TF-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i32* [ [[IND_END2]], [[MIDDLE_BLOCK]] ], [ [[DST]], [[ENTRY]] ]
+; CHECK-TF-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-TF:       for.body:
+; CHECK-TF-NEXT:    [[I_011:%.*]] = phi i64 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-TF-NEXT:    [[S_010:%.*]] = phi i32* [ [[INCDEC_PTR1:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
+; CHECK-TF-NEXT:    [[D_09:%.*]] = phi i32* [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL3]], [[SCALAR_PH]] ]
+; CHECK-TF-NEXT:    [[TMP31:%.*]] = load i32, i32* [[S_010]], align 4
+; CHECK-TF-NEXT:    [[MUL:%.*]] = shl nsw i32 [[TMP31]], 1
+; CHECK-TF-NEXT:    store i32 [[MUL]], i32* [[D_09]], align 4
+; CHECK-TF-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i32, i32* [[D_09]], i64 1
+; CHECK-TF-NEXT:    [[INCDEC_PTR1]] = getelementptr inbounds i32, i32* [[S_010]], i64 1
+; CHECK-TF-NEXT:    [[INC]] = add nuw nsw i64 [[I_011]], 1
+; CHECK-TF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-TF-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-TF:       for.cond.cleanup:
+; CHECK-TF-NEXT:    ret void
+;
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.body
   %i.011 = phi i64 [ %inc, %for.body ], [ 0, %entry ]
   %S.010 = phi i32* [ %incdec.ptr1, %for.body ], [ %src, %entry ]
   %D.09 = phi i32* [ %incdec.ptr, %for.body ], [ %dst, %entry ]
   %0 = load i32, i32* %S.010, align 4

 ; CHECK-NEXT:    [[VAR4]] = getelementptr inbounds i32*, i32** [[Q]], i64 1
 ; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
 ; CHECK-NEXT:    [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    [[VAR5:%.*]] = phi i32 [ [[VAR2]], [[FOR_BODY]] ], [ [[TMP14]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[VAR5]]
 ;
+; CHECK-TF-LABEL: @pointer_iv_mixed(
+; CHECK-TF-NEXT:  entry:
+; CHECK-TF-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1)
+; CHECK-TF-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-TF:       vector.ph:
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[SMAX]])
+; CHECK-TF-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-TF:       vector.body:
+; CHECK-TF-NEXT:    [[POINTER_PHI:%.*]] = phi i32* [ [[A:%.*]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 1
+; CHECK-TF-NEXT:    [[TMP2:%.*]] = call <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
+; CHECK-TF-NEXT:    [[TMP3:%.*]] = getelementptr i32, i32* [[POINTER_PHI]], <vscale x 2 x i64> [[TMP2]]
+; CHECK-TF-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i32*, i32** [[B:%.*]], i64 [[INDEX]]
+; CHECK-TF-NEXT:    [[BC:%.*]] = bitcast <vscale x 2 x i32*> [[TMP3]] to <vscale x 2 x <vscale x 2 x i32>*>
+; CHECK-TF-NEXT:    [[TMP4:%.*]] = extractelement <vscale x 2 x <vscale x 2 x i32>*> [[BC]], i64 0
+; CHECK-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i32> @llvm.masked.load.nxv2i32.p0nxv2i32(<vscale x 2 x i32>* [[TMP4]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i32> zeroinitializer)
+; CHECK-TF-NEXT:    [[TMP5:%.*]] = bitcast i32** [[NEXT_GEP]] to <vscale x 2 x i32*>*
+; CHECK-TF-NEXT:    call void @llvm.masked.store.nxv2p0i32.p0nxv2p0i32(<vscale x 2 x i32*> [[TMP3]], <vscale x 2 x i32*>* [[TMP5]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NEXT:    [[TMP6]] = add <vscale x 2 x i32> [[VEC_PHI]], [[WIDE_MASKED_LOAD]]
+; CHECK-TF-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP8:%.*]] = shl nuw nsw i64 [[TMP7]], 1
+; CHECK-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 [[SMAX]])
+; CHECK-TF-NEXT:    [[PTR_IND]] = getelementptr i32, i32* [[POINTER_PHI]], i64 [[TMP1]]
+; CHECK-TF-NEXT:    [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NEXT:    br i1 [[TMP9]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-TF:       middle.block:
+; CHECK-TF-NEXT:    [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32> [[TMP6]])
+; CHECK-TF-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK-TF:       scalar.ph:
+; CHECK-TF-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-TF:       for.body:
+; CHECK-TF-NEXT:    br i1 poison, label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-TF:       for.end:
+; CHECK-TF-NEXT:    [[VAR5:%.*]] = phi i32 [ poison, [[FOR_BODY]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ]
+; CHECK-TF-NEXT:    ret i32 [[VAR5]]
+;
 entry:
   br label %for.body
 
 for.body:
   %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
   %p = phi i32* [ %var3, %for.body ], [ %a, %entry ]
   %q = phi i32** [ %var4, %for.body ], [ %b, %entry ]
   %var0 = phi i32 [ %var2, %for.body ], [ 0, %entry ]

 ; CHECK:       if.end:
 ; CHECK-NEXT:    [[INCDEC_IV_PTR]] = getelementptr inbounds i16, i16* [[IV_PTR]], i64 1
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp ult i64 [[IV]], 1023
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
 ;
+; CHECK-TF-LABEL: @phi_used_in_vector_compare_and_scalar_indvar_update_and_store(
+; CHECK-TF-NEXT:  entry:
+; CHECK-TF-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-TF:       vector.ph:
+; CHECK-TF-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 1
+; CHECK-TF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
+; CHECK-TF-NEXT:    [[N_VEC:%.*]] = sub nuw nsw i64 1024, [[N_MOD_VF]]
+; CHECK-TF-NEXT:    [[IND_END:%.*]] = getelementptr i16, i16* [[PTR:%.*]], i64 [[N_VEC]]
+; CHECK-TF-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-TF:       vector.body:
+; CHECK-TF-NEXT:    [[POINTER_PHI:%.*]] = phi i16* [ [[PTR]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 1
+; CHECK-TF-NEXT:    [[TMP4:%.*]] = call <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
+; CHECK-TF-NEXT:    [[TMP5:%.*]] = getelementptr i16, i16* [[POINTER_PHI]], <vscale x 2 x i64> [[TMP4]]
+; CHECK-TF-NEXT:    [[TMP6:%.*]] = icmp ne <vscale x 2 x i16*> [[TMP5]], zeroinitializer
+; CHECK-TF-NEXT:    [[BC:%.*]] = bitcast <vscale x 2 x i16*> [[TMP5]] to <vscale x 2 x <vscale x 2 x i16>*>
+; CHECK-TF-NEXT:    [[TMP7:%.*]] = extractelement <vscale x 2 x <vscale x 2 x i16>*> [[BC]], i64 0
+; CHECK-TF-NEXT:    call void @llvm.masked.store.nxv2i16.p0nxv2i16(<vscale x 2 x i16> zeroinitializer, <vscale x 2 x i16>* [[TMP7]], i32 2, <vscale x 2 x i1> [[TMP6]])
+; CHECK-TF-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP9:%.*]] = shl nuw nsw i64 [[TMP8]], 1
+; CHECK-TF-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]
+; CHECK-TF-NEXT:    [[PTR_IND]] = getelementptr i16, i16* [[POINTER_PHI]], i64 [[TMP3]]
+; CHECK-TF-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-TF:       middle.block:
+; CHECK-TF-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; CHECK-TF-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK-TF:       scalar.ph:
+; CHECK-TF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-TF-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i16* [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[PTR]], [[ENTRY]] ]
+; CHECK-TF-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-TF:       for.body:
+; CHECK-TF-NEXT:    [[IV:%.*]] = phi i64 [ [[INC:%.*]], [[IF_END:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-TF-NEXT:    [[IV_PTR:%.*]] = phi i16* [ [[INCDEC_IV_PTR:%.*]], [[IF_END]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
+; CHECK-TF-NEXT:    [[CMP_I_NOT:%.*]] = icmp eq i16* [[IV_PTR]], null
+; CHECK-TF-NEXT:    br i1 [[CMP_I_NOT]], label [[IF_END]], label [[IF_END_SINK_SPLIT:%.*]]
+; CHECK-TF:       if.end.sink.split:
+; CHECK-TF-NEXT:    store i16 0, i16* [[IV_PTR]], align 2
+; CHECK-TF-NEXT:    br label [[IF_END]]
+; CHECK-TF:       if.end:
+; CHECK-TF-NEXT:    [[INCDEC_IV_PTR]] = getelementptr inbounds i16, i16* [[IV_PTR]], i64 1
+; CHECK-TF-NEXT:    [[INC]] = add nuw nsw i64 [[IV]], 1
+; CHECK-TF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp ult i64 [[IV]], 1023
+; CHECK-TF-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-TF:       for.end:
+; CHECK-TF-NEXT:    ret void
+;
 entry:
   br label %for.body
 
 for.body:                                      ; preds = %if.end, %entry
   %iv = phi i64 [ %inc, %if.end ], [ 0, %entry ]
   %iv.ptr = phi i16* [ %incdec.iv.ptr, %if.end ], [ %ptr, %entry ]
   %cmp.i = icmp ne i16* %iv.ptr, null
   br i1 %cmp.i, label %if.end.sink.split, label %if.end

llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse-mask4.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; This is the loop in c++ being vectorize in this file with
 ; shuffle reverse
 
 ;#pragma clang loop vectorize_width(4, fixed)
 ;  for (long int i = N - 1; i >= 0; i--)
 ;  {
 ;    if (cond[i])
 ;      a[i] += 1;
 ;  }
 
 ; The test checks if the mask is being correctly created, reverted  and used
 
 ; RUN: opt -loop-vectorize -dce -instcombine -mtriple aarch64-linux-gnu -S \
 ; RUN:   -prefer-predicate-over-epilogue=scalar-epilogue < %s | FileCheck %s
+; RUN: opt -loop-vectorize -dce -instcombine -mtriple aarch64-linux-gnu -S \
+; RUN:   -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue < %s | FileCheck %s --check-prefix=CHECK-TF
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64-unknown-linux-gnu"
 
 define void @vector_reverse_mask_v4i1(double* noalias %a, double* noalias %cond, i64 %N) #0 {
 ; CHECK-LABEL: @vector_reverse_mask_v4i1(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP7:%.*]] = icmp sgt i64 [[N:%.*]], 0

 ; CHECK-NEXT:    [[TMP22:%.*]] = load double, double* [[ARRAYIDX1]], align 8
 ; CHECK-NEXT:    [[ADD:%.*]] = fadd double [[TMP22]], 1.000000e+00
 ; CHECK-NEXT:    store double [[ADD]], double* [[ARRAYIDX1]], align 8
 ; CHECK-NEXT:    br label [[FOR_INC]]
 ; CHECK:       for.inc:
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[I_08_IN]], 1
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP_LOOPEXIT]], !llvm.loop [[LOOP3:![0-9]+]]
 ;
+; CHECK-TF-LABEL: @vector_reverse_mask_v4i1(
+; CHECK-TF-NEXT:  entry:
+; CHECK-TF-NEXT:    [[CMP7:%.*]] = icmp sgt i64 [[N:%.*]], 0
+; CHECK-TF-NEXT:    br i1 [[CMP7]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK-TF:       for.body.preheader:
+; CHECK-TF-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-TF:       vector.ph:
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 4, i64 [[N]])
+; CHECK-TF-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-TF:       vector.body:
+; CHECK-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK2:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT17:%.*]], [[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[TMP0:%.*]] = xor i64 [[INDEX]], -1
+; CHECK-TF-NEXT:    [[TMP1:%.*]] = add i64 [[TMP0]], [[N]]
+; CHECK-TF-NEXT:    [[TMP2:%.*]] = getelementptr inbounds double, double* [[COND:%.*]], i64 [[TMP1]]
+; CHECK-TF-NEXT:    [[TMP3:%.*]] = getelementptr inbounds double, double* [[TMP2]], i64 -3
+; CHECK-TF-NEXT:    [[REVERSE:%.*]] = shufflevector <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i1> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-TF-NEXT:    [[TMP4:%.*]] = bitcast double* [[TMP3]] to <4 x double>*
+; CHECK-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* nonnull [[TMP4]], i32 8, <4 x i1> [[REVERSE]], <4 x double> poison)
+; CHECK-TF-NEXT:    [[REVERSE3:%.*]] = shufflevector <4 x double> [[WIDE_MASKED_LOAD]], <4 x double> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-TF-NEXT:    [[TMP5:%.*]] = getelementptr inbounds double, double* [[TMP2]], i64 -4
+; CHECK-TF-NEXT:    [[TMP6:%.*]] = getelementptr inbounds double, double* [[TMP5]], i64 -3
+; CHECK-TF-NEXT:    [[REVERSE4:%.*]] = shufflevector <4 x i1> [[ACTIVE_LANE_MASK2]], <4 x i1> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-TF-NEXT:    [[TMP7:%.*]] = bitcast double* [[TMP6]] to <4 x double>*
+; CHECK-TF-NEXT:    [[WIDE_MASKED_LOAD5:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* nonnull [[TMP7]], i32 8, <4 x i1> [[REVERSE4]], <4 x double> poison)
+; CHECK-TF-NEXT:    [[REVERSE6:%.*]] = shufflevector <4 x double> [[WIDE_MASKED_LOAD5]], <4 x double> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-TF-NEXT:    [[TMP8:%.*]] = fcmp une <4 x double> [[REVERSE3]], zeroinitializer
+; CHECK-TF-NEXT:    [[TMP9:%.*]] = fcmp une <4 x double> [[REVERSE6]], zeroinitializer
+; CHECK-TF-NEXT:    [[TMP10:%.*]] = getelementptr double, double* [[A:%.*]], i64 [[TMP1]]
+; CHECK-TF-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i1> [[TMP8]], <4 x i1> zeroinitializer
+; CHECK-TF-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[ACTIVE_LANE_MASK2]], <4 x i1> [[TMP9]], <4 x i1> zeroinitializer
+; CHECK-TF-NEXT:    [[TMP13:%.*]] = getelementptr double, double* [[TMP10]], i64 -3
+; CHECK-TF-NEXT:    [[REVERSE7:%.*]] = shufflevector <4 x i1> [[TMP11]], <4 x i1> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-TF-NEXT:    [[TMP14:%.*]] = bitcast double* [[TMP13]] to <4 x double>*
+; CHECK-TF-NEXT:    [[WIDE_MASKED_LOAD8:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* [[TMP14]], i32 8, <4 x i1> [[REVERSE7]], <4 x double> poison)
+; CHECK-TF-NEXT:    [[TMP15:%.*]] = getelementptr double, double* [[TMP10]], i64 -4
+; CHECK-TF-NEXT:    [[TMP16:%.*]] = getelementptr double, double* [[TMP15]], i64 -3
+; CHECK-TF-NEXT:    [[REVERSE10:%.*]] = shufflevector <4 x i1> [[TMP12]], <4 x i1> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-TF-NEXT:    [[TMP17:%.*]] = bitcast double* [[TMP16]] to <4 x double>*
+; CHECK-TF-NEXT:    [[WIDE_MASKED_LOAD11:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* [[TMP17]], i32 8, <4 x i1> [[REVERSE10]], <4 x double> poison)
+; CHECK-TF-NEXT:    [[TMP18:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD8]], <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
+; CHECK-TF-NEXT:    [[TMP19:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD11]], <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
+; CHECK-TF-NEXT:    [[TMP20:%.*]] = bitcast double* [[TMP13]] to <4 x double>*
+; CHECK-TF-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> [[TMP18]], <4 x double>* [[TMP20]], i32 8, <4 x i1> [[REVERSE7]])
+; CHECK-TF-NEXT:    [[TMP21:%.*]] = bitcast double* [[TMP16]] to <4 x double>*
+; CHECK-TF-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> [[TMP19]], <4 x double>* [[TMP21]], i32 8, <4 x i1> [[REVERSE10]])
+; CHECK-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; CHECK-TF-NEXT:    [[TMP22:%.*]] = add i64 [[INDEX]], 12
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT17]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[TMP22]], i64 [[N]])
+; CHECK-TF-NEXT:    [[TMP23:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NEXT:    br i1 [[TMP23]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF:       middle.block:
+; CHECK-TF-NEXT:    br i1 true, label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-TF:       scalar.ph:
+; CHECK-TF-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-TF:       for.cond.cleanup.loopexit:
+; CHECK-TF-NEXT:    br label [[FOR_COND_CLEANUP]]
+; CHECK-TF:       for.cond.cleanup:
+; CHECK-TF-NEXT:    ret void
+; CHECK-TF:       for.body:
+; CHECK-TF-NEXT:    br i1 poison, label [[IF_THEN:%.*]], label [[FOR_INC:%.*]]
+; CHECK-TF:       if.then:
+; CHECK-TF-NEXT:    br label [[FOR_INC]]
+; CHECK-TF:       for.inc:
+; CHECK-TF-NEXT:    br i1 poison, label [[FOR_BODY]], label [[FOR_COND_CLEANUP_LOOPEXIT]], !llvm.loop [[LOOP3:![0-9]+]]
+;
 
 entry:
   %cmp7 = icmp sgt i64 %N, 0
   br i1 %cmp7, label %for.body, label %for.cond.cleanup
 
 for.cond.cleanup:                                 ; preds = %for.cond.cleanup, %entry
   ret void
 

llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll

 ; Test VLA for reverse with fixed size vector
 ; This is the loop in c++ being vectorize in this file with
 ; shuffle reverse
 ;  #pragma clang loop vectorize_width(8, fixed)
 ;  for (int i = N-1; i >= 0; --i)
 ;    a[i] = b[i] + 1.0;
 
 ; RUN: opt -loop-vectorize -dce  -mtriple aarch64-linux-gnu -S \
 ; RUN:   -prefer-predicate-over-epilogue=scalar-epilogue < %s | FileCheck %s
+; RUN: opt -loop-vectorize -dce  -mtriple aarch64-linux-gnu -S \
+; RUN:   -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue < %s | FileCheck %s
 
 define void @vector_reverse_f64(i64 %N, double* %a, double* %b) #0 {
 ; CHECK-LABEL: vector_reverse_f64
 ; CHECK-LABEL: vector.body
 ; CHECK: %[[GEP:.*]] = getelementptr inbounds double, double* %{{.*}}, i32 0
 ; CHECK-NEXT: %[[GEP1:.*]] = getelementptr inbounds double, double* %[[GEP]], i32 -7
-; CHECK-NEXT: %[[CAST:.*]] = bitcast double* %[[GEP1]] to <8 x double>*
-; CHECK-NEXT: %[[WIDE:.*]] = load <8 x double>, <8 x double>* %[[CAST]], align 8
+; CHECK:      %[[CAST:.*]] = bitcast double* %[[GEP1]] to <8 x double>*
+; CHECK-NEXT: %[[WIDE:.*]] = {{.*}}load{{.*}}<8 x double>* %[[CAST]]
 ; CHECK-NEXT: %[[REVERSE:.*]] = shufflevector <8 x double> %[[WIDE]], <8 x double> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT: %[[FADD:.*]] = fadd <8 x double> %[[REVERSE]]
 ; CHECK-NEXT: %[[GEP2:.*]] = getelementptr inbounds double, double* {{.*}}, i64 {{.*}}
 ; CHECK-NEXT: %[[REVERSE6:.*]] = shufflevector <8 x double> %[[FADD]], <8 x double> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT: %[[GEP3:.*]] = getelementptr inbounds double, double* %[[GEP2]], i32 0
 ; CHECK-NEXT: %[[GEP4:.*]] = getelementptr inbounds double, double* %[[GEP3]], i32 -7
 ; CHECK-NEXT: %[[CAST:.*]] = bitcast double* %[[GEP4]] to <8 x double>*
-; CHECK-NEXT:  store <8 x double> %[[REVERSE6]], <8 x double>* %[[CAST]], align 8
+; CHECK-NEXT:  store{{.*}}<8 x double> %[[REVERSE6]], <8 x double>* %[[CAST]]
 
 entry:
   %cmp7 = icmp sgt i64 %N, 0
   br i1 %cmp7, label %for.body, label %for.cond.cleanup
 
 for.cond.cleanup:                                 ; preds = %for.cond.cleanup, %entry
   ret void
 
for.body:                                         ; preds = %entry, %for.body
   br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !0
 }
 
 define void @vector_reverse_i64(i64 %N, i64* %a, i64* %b) #0 {
 ; CHECK-LABEL: vector_reverse_i64
 ; CHECK-LABEL: vector.body
 ; CHECK: %[[GEP:.*]] = getelementptr inbounds i64, i64* %{{.*}}, i32 0
 ; CHECK-NEXT: %[[GEP1:.*]] = getelementptr inbounds i64, i64* %[[GEP]], i32 -7
-; CHECK-NEXT: %[[CAST:.*]] = bitcast i64* %[[GEP1]] to <8 x i64>*
-; CHECK-NEXT: %[[WIDE:.*]] = load <8 x i64>, <8 x i64>* %[[CAST]], align 8
+; CHECK:      %[[CAST:.*]] = bitcast i64* %[[GEP1]] to <8 x i64>*
+; CHECK-NEXT: %[[WIDE:.*]] = {{.*}}load{{.*}}<8 x i64>* %[[CAST]]
 ; CHECK-NEXT: %[[REVERSE:.*]] = shufflevector <8 x i64> %[[WIDE]], <8 x i64> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT: %[[FADD:.*]] = add <8 x i64> %[[REVERSE]]
 ; CHECK-NEXT: %[[GEP2:.*]] = getelementptr inbounds i64, i64* {{.*}}, i64 {{.*}}
 ; CHECK-NEXT: %[[REVERSE6:.*]] = shufflevector <8 x i64> %[[FADD]], <8 x i64> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT: %[[GEP3:.*]] = getelementptr inbounds i64, i64* %[[GEP2]], i32 0
 ; CHECK-NEXT: %[[GEP4:.*]] = getelementptr inbounds i64, i64* %[[GEP3]], i32 -7
 ; CHECK-NEXT: %[[CAST1:.*]] = bitcast i64* %[[GEP4]] to <8 x i64>*
-; CHECK-NEXT:  store <8 x i64> %[[REVERSE6]], <8 x i64>* %[[CAST1]], align 8
+; CHECK-NEXT:  store{{.*}}<8 x i64> %[[REVERSE6]], <8 x i64>* %[[CAST1]]
 
 entry:
   %cmp8 = icmp sgt i64 %N, 0
   br i1 %cmp8, label %for.body, label %for.cond.cleanup
 
 for.cond.cleanup:                                 ; preds = %for.cond.cleanup, %entry
   ret void