[ARM] A predicate cast of a predicate cast is a predicate cast

[dmgreen]

Summary

The adds some very basic folding of PREDICATE_CASTS, removing cases when they are chained together. These would already be removed eventually, as these are lowered to copies. This just allows it to happen earlier, which can help during lowering.

[samparker]

Could you say what it is generally helping? For the first few test cases, the main change that I see is push and pop instructions being introduced...

[dmgreen]

Yeah, it's a shame about the masked ld-st tests. They appear to be in a different order, causing the extra register usage. Seems like more of a scheduling problem. The tests in mve-pred-bitcast.ll and mve-pred-loadstore.ll are both smaller (and the masked load-store tests will eventually generate narrow/widen masked loads/stores.)

This seems like a fairly obvious simplification to me. It should in certain situations allow further simplification, so should in general be an improvement.

[samparker]

Good point, the copies probably would do a good job of constraining the scheduler!

llvm/lib/Target/ARM/ARMISelLowering.cpp

12641

Shouldn't this be casting to VT?

llvm/lib/Target/ARM/ARMISelLowering.cpp

for (unsigned Idx = 0 ; Idx < NumElts; ++Idx) {
     Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VecVT, Vec, V, LaneIdx);
   }
   Vec = DAG.getNode(ISD::BITCAST, dl, VT, Vec);
   // Make the DAGCombiner fold the bitcasts.
   DCI.AddToWorklist(Vec.getNode());
   return Vec;
 }
 
+static SDValue
+PerformPREDICATE_CASTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
+  EVT VT = N->getValueType(0);
+  SDValue Op = N->getOperand(0);
+  SDLoc dl(N);
+
+  // PREDICATE_CAST(PREDICATE_CAST(x)) == PREDICATE_CAST(x)
+  if (Op->getOpcode() == ARMISD::PREDICATE_CAST) {
+    // If the valuetypes are the same, we can remove the cast entirely.
+    if (Op->getOperand(0).getValueType() == VT)
+      return Op->getOperand(0);
+    return DCI.DAG.getNode(ARMISD::PREDICATE_CAST, dl,
+                           Op->getOperand(0).getValueType(), Op->getOperand(0));
+  }
+
+  return SDValue();
+}
+
 /// PerformInsertEltCombine - Target-specific dag combine xforms for
 /// ISD::INSERT_VECTOR_ELT.
 static SDValue PerformInsertEltCombine(SDNode *N,
                                        TargetLowering::DAGCombinerInfo &DCI) {
   // Bitcast an i64 load inserted into a vector to f64.
   // Otherwise, the i64 value will be legalized to a pair of i32 values.
   EVT VT = N->getValueType(0);
   SDNode *Elt = N->getOperand(1).getNode();
SDValue ARMTargetLowering::PerformDAGCombine(SDNode *N,
   case ISD::LOAD:       return PerformLOADCombine(N, DCI);
   case ARMISD::VLD1DUP:
   case ARMISD::VLD2DUP:
   case ARMISD::VLD3DUP:
   case ARMISD::VLD4DUP:
     return PerformVLDCombine(N, DCI);
   case ARMISD::BUILD_VECTOR:
     return PerformARMBUILD_VECTORCombine(N, DCI);
+  case ARMISD::PREDICATE_CAST:
+    return PerformPREDICATE_CASTCombine(N, DCI);
   case ARMISD::SMULWB: {
     unsigned BitWidth = N->getValueType(0).getSizeInBits();
     APInt DemandedMask = APInt::getLowBitsSet(BitWidth, 16);
     if (SimplifyDemandedBits(N->getOperand(1), DemandedMask, DCI))
       return SDValue();
     break;
   }
   case ARMISD::SMULWT: {

llvm/test/CodeGen/Thumb2/mve-masked-ldst.ll

entry:
   %2 = call <4 x i32> @llvm.masked.load.v4i32(<4 x i32>* %src, i32 4, <4 x i1> %1, <4 x i32> undef)
   call void @llvm.masked.store.v4i32(<4 x i32> %2, <4 x i32>* %dest, i32 4, <4 x i1> %1)
   ret void
 }
 
 define void @foo_sext_v4i32_v4i8(<4 x i32> *%dest, <4 x i32> *%mask, <4 x i8> *%src) {
 ; CHECK-LABEL: foo_sext_v4i32_v4i8:
 ; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    .pad #4
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    vcmp.s32 gt, q0, zr
 ; CHECK-NEXT:    @ implicit-def: $q0
-; CHECK-NEXT:    vmrs r12, p0
-; CHECK-NEXT:    and r1, r12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #0, #1
-; CHECK-NEXT:    ubfx r1, r12, #4, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #1, #1
-; CHECK-NEXT:    ubfx r1, r12, #8, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #2, #1
-; CHECK-NEXT:    ubfx r1, r12, #12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #3, #1
-; CHECK-NEXT:    and r1, r3, #15
+; CHECK-NEXT:    vmrs lr, p0
+; CHECK-NEXT:    and r1, lr, #1
+; CHECK-NEXT:    ubfx r3, lr, #4, #1
+; CHECK-NEXT:    rsb.w r12, r1, #0
+; CHECK-NEXT:    movs r1, #0
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r1, r12, #0, #1
+; CHECK-NEXT:    bfi r1, r3, #1, #1
+; CHECK-NEXT:    ubfx r3, lr, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r1, r3, #2, #1
+; CHECK-NEXT:    ubfx r3, lr, #12, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r1, r3, #3, #1
 ; CHECK-NEXT:    lsls r3, r1, #31
 ; CHECK-NEXT:    itt ne
 ; CHECK-NEXT:    ldrbne r3, [r2]
 ; CHECK-NEXT:    vmovne.32 q0[0], r3
 ; CHECK-NEXT:    lsls r3, r1, #30
 ; CHECK-NEXT:    itt mi
 ; CHECK-NEXT:    ldrbmi r3, [r2, #1]
 ; CHECK-NEXT:    vmovmi.32 q0[1], r3
 ; CHECK-NEXT:    lsls r3, r1, #29
 ; CHECK-NEXT:    itt mi
 ; CHECK-NEXT:    ldrbmi r3, [r2, #2]
 ; CHECK-NEXT:    vmovmi.32 q0[2], r3
 ; CHECK-NEXT:    lsls r1, r1, #28
 ; CHECK-NEXT:    itt mi
 ; CHECK-NEXT:    ldrbmi r1, [r2, #3]
 ; CHECK-NEXT:    vmovmi.32 q0[3], r1
 ; CHECK-NEXT:    vmovlb.s8 q0, q0
 ; CHECK-NEXT:    vmovlb.s16 q0, q0
 ; CHECK-NEXT:    vpst
 ; CHECK-NEXT:    vstrwt.32 q0, [r0]
 ; CHECK-NEXT:    add sp, #4
-; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    pop {r7, pc}
 entry:
   %0 = load <4 x i32>, <4 x i32>* %mask, align 4
   %1 = icmp sgt <4 x i32> %0, zeroinitializer
   %2 = call <4 x i8> @llvm.masked.load.v4i8(<4 x i8>* %src, i32 1, <4 x i1> %1, <4 x i8> undef)
   %3 = sext <4 x i8> %2 to <4 x i32>
   call void @llvm.masked.store.v4i32(<4 x i32> %3, <4 x i32>* %dest, i32 4, <4 x i1> %1)
   ret void
 }
 
 define void @foo_sext_v4i32_v4i16(<4 x i32> *%dest, <4 x i32> *%mask, <4 x i16> *%src) {
 ; CHECK-LABEL: foo_sext_v4i32_v4i16:
 ; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    .pad #4
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    vcmp.s32 gt, q0, zr
 ; CHECK-NEXT:    @ implicit-def: $q0
-; CHECK-NEXT:    vmrs r12, p0
-; CHECK-NEXT:    and r1, r12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #0, #1
-; CHECK-NEXT:    ubfx r1, r12, #4, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #1, #1
-; CHECK-NEXT:    ubfx r1, r12, #8, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #2, #1
-; CHECK-NEXT:    ubfx r1, r12, #12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #3, #1
-; CHECK-NEXT:    and r1, r3, #15
+; CHECK-NEXT:    vmrs lr, p0
+; CHECK-NEXT:    and r1, lr, #1
+; CHECK-NEXT:    ubfx r3, lr, #4, #1
+; CHECK-NEXT:    rsb.w r12, r1, #0
+; CHECK-NEXT:    movs r1, #0
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r1, r12, #0, #1
+; CHECK-NEXT:    bfi r1, r3, #1, #1
+; CHECK-NEXT:    ubfx r3, lr, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r1, r3, #2, #1
+; CHECK-NEXT:    ubfx r3, lr, #12, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r1, r3, #3, #1
 ; CHECK-NEXT:    lsls r3, r1, #31
 ; CHECK-NEXT:    itt ne
 ; CHECK-NEXT:    ldrhne r3, [r2]
 ; CHECK-NEXT:    vmovne.32 q0[0], r3
 ; CHECK-NEXT:    lsls r3, r1, #30
 ; CHECK-NEXT:    itt mi
 ; CHECK-NEXT:    ldrhmi r3, [r2, #2]
 ; CHECK-NEXT:    vmovmi.32 q0[1], r3
 ; CHECK-NEXT:    lsls r3, r1, #29
 ; CHECK-NEXT:    itt mi
 ; CHECK-NEXT:    ldrhmi r3, [r2, #4]
 ; CHECK-NEXT:    vmovmi.32 q0[2], r3
 ; CHECK-NEXT:    lsls r1, r1, #28
 ; CHECK-NEXT:    itt mi
 ; CHECK-NEXT:    ldrhmi r1, [r2, #6]
 ; CHECK-NEXT:    vmovmi.32 q0[3], r1
 ; CHECK-NEXT:    vmovlb.s16 q0, q0
 ; CHECK-NEXT:    vpst
 ; CHECK-NEXT:    vstrwt.32 q0, [r0]
 ; CHECK-NEXT:    add sp, #4
-; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    pop {r7, pc}
 entry:
   %0 = load <4 x i32>, <4 x i32>* %mask, align 4
   %1 = icmp sgt <4 x i32> %0, zeroinitializer
   %2 = call <4 x i16> @llvm.masked.load.v4i16(<4 x i16>* %src, i32 2, <4 x i1> %1, <4 x i16> undef)
   %3 = sext <4 x i16> %2 to <4 x i32>
   call void @llvm.masked.store.v4i32(<4 x i32> %3, <4 x i32>* %dest, i32 4, <4 x i1> %1)
   ret void
 }
 
 define void @foo_zext_v4i32_v4i8(<4 x i32> *%dest, <4 x i32> *%mask, <4 x i8> *%src) {
 ; CHECK-LABEL: foo_zext_v4i32_v4i8:
 ; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    .pad #4
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    vmov.i32 q1, #0xff
 ; CHECK-NEXT:    vcmp.s32 gt, q0, zr
 ; CHECK-NEXT:    @ implicit-def: $q0
-; CHECK-NEXT:    vmrs r12, p0
-; CHECK-NEXT:    and r1, r12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #0, #1
-; CHECK-NEXT:    ubfx r1, r12, #4, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #1, #1
-; CHECK-NEXT:    ubfx r1, r12, #8, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #2, #1
-; CHECK-NEXT:    ubfx r1, r12, #12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #3, #1
-; CHECK-NEXT:    and r1, r3, #15
+; CHECK-NEXT:    vmrs lr, p0
+; CHECK-NEXT:    and r1, lr, #1
+; CHECK-NEXT:    ubfx r3, lr, #4, #1
+; CHECK-NEXT:    rsb.w r12, r1, #0
+; CHECK-NEXT:    movs r1, #0
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r1, r12, #0, #1
+; CHECK-NEXT:    bfi r1, r3, #1, #1
+; CHECK-NEXT:    ubfx r3, lr, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r1, r3, #2, #1
+; CHECK-NEXT:    ubfx r3, lr, #12, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r1, r3, #3, #1
 ; CHECK-NEXT:    lsls r3, r1, #31
 ; CHECK-NEXT:    itt ne
 ; CHECK-NEXT:    ldrbne r3, [r2]
 ; CHECK-NEXT:    vmovne.32 q0[0], r3
 ; CHECK-NEXT:    lsls r3, r1, #30
 ; CHECK-NEXT:    itt mi
 ; CHECK-NEXT:    ldrbmi r3, [r2, #1]
 ; CHECK-NEXT:    vmovmi.32 q0[1], r3
 ; CHECK-NEXT:    lsls r3, r1, #29
 ; CHECK-NEXT:    itt mi
 ; CHECK-NEXT:    ldrbmi r3, [r2, #2]
 ; CHECK-NEXT:    vmovmi.32 q0[2], r3
 ; CHECK-NEXT:    lsls r1, r1, #28
 ; CHECK-NEXT:    itt mi
 ; CHECK-NEXT:    ldrbmi r1, [r2, #3]
 ; CHECK-NEXT:    vmovmi.32 q0[3], r1
 ; CHECK-NEXT:    vand q0, q0, q1
 ; CHECK-NEXT:    vpst
 ; CHECK-NEXT:    vstrwt.32 q0, [r0]
 ; CHECK-NEXT:    add sp, #4
-; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    pop {r7, pc}
 entry:
   %0 = load <4 x i32>, <4 x i32>* %mask, align 4
   %1 = icmp sgt <4 x i32> %0, zeroinitializer
   %2 = call <4 x i8> @llvm.masked.load.v4i8(<4 x i8>* %src, i32 1, <4 x i1> %1, <4 x i8> undef)
   %3 = zext <4 x i8> %2 to <4 x i32>
   call void @llvm.masked.store.v4i32(<4 x i32> %3, <4 x i32>* %dest, i32 4, <4 x i1> %1)
   ret void
 }
 
 define void @foo_zext_v4i32_v4i16(<4 x i32> *%dest, <4 x i32> *%mask, <4 x i16> *%src) {
 ; CHECK-LABEL: foo_zext_v4i32_v4i16:
 ; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    .pad #4
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    vcmp.s32 gt, q0, zr
 ; CHECK-NEXT:    @ implicit-def: $q0
-; CHECK-NEXT:    vmrs r12, p0
-; CHECK-NEXT:    and r1, r12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #0, #1
-; CHECK-NEXT:    ubfx r1, r12, #4, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #1, #1
-; CHECK-NEXT:    ubfx r1, r12, #8, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #2, #1
-; CHECK-NEXT:    ubfx r1, r12, #12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #3, #1
-; CHECK-NEXT:    and r1, r3, #15
+; CHECK-NEXT:    vmrs lr, p0
+; CHECK-NEXT:    and r1, lr, #1
+; CHECK-NEXT:    ubfx r3, lr, #4, #1
+; CHECK-NEXT:    rsb.w r12, r1, #0
+; CHECK-NEXT:    movs r1, #0
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r1, r12, #0, #1
+; CHECK-NEXT:    bfi r1, r3, #1, #1
+; CHECK-NEXT:    ubfx r3, lr, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r1, r3, #2, #1
+; CHECK-NEXT:    ubfx r3, lr, #12, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r1, r3, #3, #1
 ; CHECK-NEXT:    lsls r3, r1, #31
 ; CHECK-NEXT:    itt ne
 ; CHECK-NEXT:    ldrhne r3, [r2]
 ; CHECK-NEXT:    vmovne.32 q0[0], r3
 ; CHECK-NEXT:    lsls r3, r1, #30
 ; CHECK-NEXT:    itt mi
 ; CHECK-NEXT:    ldrhmi r3, [r2, #2]
 ; CHECK-NEXT:    vmovmi.32 q0[1], r3
 ; CHECK-NEXT:    lsls r3, r1, #29
 ; CHECK-NEXT:    itt mi
 ; CHECK-NEXT:    ldrhmi r3, [r2, #4]
 ; CHECK-NEXT:    vmovmi.32 q0[2], r3
 ; CHECK-NEXT:    lsls r1, r1, #28
 ; CHECK-NEXT:    itt mi
 ; CHECK-NEXT:    ldrhmi r1, [r2, #6]
 ; CHECK-NEXT:    vmovmi.32 q0[3], r1
 ; CHECK-NEXT:    vmovlb.u16 q0, q0
 ; CHECK-NEXT:    vpst
 ; CHECK-NEXT:    vstrwt.32 q0, [r0]
 ; CHECK-NEXT:    add sp, #4
-; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    pop {r7, pc}
 entry:
   %0 = load <4 x i32>, <4 x i32>* %mask, align 4
   %1 = icmp sgt <4 x i32> %0, zeroinitializer
   %2 = call <4 x i16> @llvm.masked.load.v4i16(<4 x i16>* %src, i32 2, <4 x i1> %1, <4 x i16> undef)
   %3 = zext <4 x i16> %2 to <4 x i32>
   call void @llvm.masked.store.v4i32(<4 x i32> %3, <4 x i32>* %dest, i32 4, <4 x i1> %1)
   ret void
 }
entry:
   %2 = call <8 x i16> @llvm.masked.load.v8i16(<8 x i16>* %src, i32 2, <8 x i1> %1, <8 x i16> undef)
   call void @llvm.masked.store.v8i16(<8 x i16> %2, <8 x i16>* %dest, i32 2, <8 x i1> %1)
   ret void
 }
 
 define void @foo_sext_v8i16_v8i8(<8 x i16> *%dest, <8 x i16> *%mask, <8 x i8> *%src) {
 ; CHECK-LABEL: foo_sext_v8i16_v8i8:
 ; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    .pad #8
 ; CHECK-NEXT:    sub sp, #8
 ; CHECK-NEXT:    vldrh.u16 q0, [r1]
-; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    vcmp.s16 gt, q0, zr
 ; CHECK-NEXT:    @ implicit-def: $q0
-; CHECK-NEXT:    vmrs r12, p0
-; CHECK-NEXT:    and r1, r12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #0, #1
-; CHECK-NEXT:    ubfx r1, r12, #2, #1
+; CHECK-NEXT:    vmrs lr, p0
+; CHECK-NEXT:    and r3, lr, #1
+; CHECK-NEXT:    ubfx r1, lr, #2, #1
+; CHECK-NEXT:    rsb.w r12, r3, #0
+; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    rsbs r1, r1, #0
+; CHECK-NEXT:    bfi r3, r12, #0, #1
 ; CHECK-NEXT:    bfi r3, r1, #1, #1
-; CHECK-NEXT:    ubfx r1, r12, #4, #1
+; CHECK-NEXT:    ubfx r1, lr, #4, #1
 ; CHECK-NEXT:    rsbs r1, r1, #0
 ; CHECK-NEXT:    bfi r3, r1, #2, #1
-; CHECK-NEXT:    ubfx r1, r12, #6, #1
+; CHECK-NEXT:    ubfx r1, lr, #6, #1
 ; CHECK-NEXT:    rsbs r1, r1, #0
 ; CHECK-NEXT:    bfi r3, r1, #3, #1
-; CHECK-NEXT:    ubfx r1, r12, #8, #1
+; CHECK-NEXT:    ubfx r1, lr, #8, #1
 ; CHECK-NEXT:    rsbs r1, r1, #0
 ; CHECK-NEXT:    bfi r3, r1, #4, #1
-; CHECK-NEXT:    ubfx r1, r12, #10, #1
+; CHECK-NEXT:    ubfx r1, lr, #10, #1
 ; CHECK-NEXT:    rsbs r1, r1, #0
 ; CHECK-NEXT:    bfi r3, r1, #5, #1
-; CHECK-NEXT:    ubfx r1, r12, #12, #1
+; CHECK-NEXT:    ubfx r1, lr, #12, #1
 ; CHECK-NEXT:    rsbs r1, r1, #0
 ; CHECK-NEXT:    bfi r3, r1, #6, #1
-; CHECK-NEXT:    ubfx r1, r12, #14, #1
+; CHECK-NEXT:    ubfx r1, lr, #14, #1
 ; CHECK-NEXT:    rsbs r1, r1, #0
 ; CHECK-NEXT:    bfi r3, r1, #7, #1
 ; CHECK-NEXT:    uxtb r1, r3
 ; CHECK-NEXT:    lsls r3, r3, #31
 ; CHECK-NEXT:    itt ne
 ; CHECK-NEXT:    ldrbne r3, [r2]
 ; CHECK-NEXT:    vmovne.16 q0[0], r3
 ; CHECK-NEXT:    lsls r3, r1, #30

 ; CHECK-NEXT:    lsls r1, r1, #24
 ; CHECK-NEXT:    itt mi
 ; CHECK-NEXT:    ldrbmi r1, [r2, #7]
 ; CHECK-NEXT:    vmovmi.16 q0[7], r1
 ; CHECK-NEXT:    vmovlb.s8 q0, q0
 ; CHECK-NEXT:    vpst
 ; CHECK-NEXT:    vstrht.16 q0, [r0]
 ; CHECK-NEXT:    add sp, #8
-; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    pop {r7, pc}
 entry:
   %0 = load <8 x i16>, <8 x i16>* %mask, align 2
   %1 = icmp sgt <8 x i16> %0, zeroinitializer
   %2 = call <8 x i8> @llvm.masked.load.v8i8(<8 x i8>* %src, i32 1, <8 x i1> %1, <8 x i8> undef)
   %3 = sext <8 x i8> %2 to <8 x i16>
   call void @llvm.masked.store.v8i16(<8 x i16> %3, <8 x i16>* %dest, i32 2, <8 x i1> %1)
   ret void
 }
 
 define void @foo_zext_v8i16_v8i8(<8 x i16> *%dest, <8 x i16> *%mask, <8 x i8> *%src) {
 ; CHECK-LABEL: foo_zext_v8i16_v8i8:
 ; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    .pad #8
 ; CHECK-NEXT:    sub sp, #8
 ; CHECK-NEXT:    vldrh.u16 q0, [r1]
-; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    vcmp.s16 gt, q0, zr
 ; CHECK-NEXT:    @ implicit-def: $q0
-; CHECK-NEXT:    vmrs r12, p0
-; CHECK-NEXT:    and r1, r12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #0, #1
-; CHECK-NEXT:    ubfx r1, r12, #2, #1
+; CHECK-NEXT:    vmrs lr, p0
+; CHECK-NEXT:    and r3, lr, #1
+; CHECK-NEXT:    ubfx r1, lr, #2, #1
+; CHECK-NEXT:    rsb.w r12, r3, #0
+; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    rsbs r1, r1, #0
+; CHECK-NEXT:    bfi r3, r12, #0, #1
 ; CHECK-NEXT:    bfi r3, r1, #1, #1
-; CHECK-NEXT:    ubfx r1, r12, #4, #1
+; CHECK-NEXT:    ubfx r1, lr, #4, #1
 ; CHECK-NEXT:    rsbs r1, r1, #0
 ; CHECK-NEXT:    bfi r3, r1, #2, #1
-; CHECK-NEXT:    ubfx r1, r12, #6, #1
+; CHECK-NEXT:    ubfx r1, lr, #6, #1
 ; CHECK-NEXT:    rsbs r1, r1, #0
 ; CHECK-NEXT:    bfi r3, r1, #3, #1
-; CHECK-NEXT:    ubfx r1, r12, #8, #1
+; CHECK-NEXT:    ubfx r1, lr, #8, #1
 ; CHECK-NEXT:    rsbs r1, r1, #0
 ; CHECK-NEXT:    bfi r3, r1, #4, #1
-; CHECK-NEXT:    ubfx r1, r12, #10, #1
+; CHECK-NEXT:    ubfx r1, lr, #10, #1
 ; CHECK-NEXT:    rsbs r1, r1, #0
 ; CHECK-NEXT:    bfi r3, r1, #5, #1
-; CHECK-NEXT:    ubfx r1, r12, #12, #1
+; CHECK-NEXT:    ubfx r1, lr, #12, #1
 ; CHECK-NEXT:    rsbs r1, r1, #0
 ; CHECK-NEXT:    bfi r3, r1, #6, #1
-; CHECK-NEXT:    ubfx r1, r12, #14, #1
+; CHECK-NEXT:    ubfx r1, lr, #14, #1
 ; CHECK-NEXT:    rsbs r1, r1, #0
 ; CHECK-NEXT:    bfi r3, r1, #7, #1
 ; CHECK-NEXT:    uxtb r1, r3
 ; CHECK-NEXT:    lsls r3, r3, #31
 ; CHECK-NEXT:    itt ne
 ; CHECK-NEXT:    ldrbne r3, [r2]
 ; CHECK-NEXT:    vmovne.16 q0[0], r3
 ; CHECK-NEXT:    lsls r3, r1, #30

 ; CHECK-NEXT:    lsls r1, r1, #24
 ; CHECK-NEXT:    itt mi
 ; CHECK-NEXT:    ldrbmi r1, [r2, #7]
 ; CHECK-NEXT:    vmovmi.16 q0[7], r1
 ; CHECK-NEXT:    vmovlb.u8 q0, q0
 ; CHECK-NEXT:    vpst
 ; CHECK-NEXT:    vstrht.16 q0, [r0]
 ; CHECK-NEXT:    add sp, #8
-; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    pop {r7, pc}
 entry:
   %0 = load <8 x i16>, <8 x i16>* %mask, align 2
   %1 = icmp sgt <8 x i16> %0, zeroinitializer
   %2 = call <8 x i8> @llvm.masked.load.v8i8(<8 x i8>* %src, i32 1, <8 x i1> %1, <8 x i8> undef)
   %3 = zext <8 x i8> %2 to <8 x i16>
   call void @llvm.masked.store.v8i16(<8 x i16> %3, <8 x i16>* %dest, i32 2, <8 x i1> %1)
   ret void
 }

 }
 
 define void @foo_trunc_v8i8_v8i16(<8 x i8> *%dest, <8 x i16> *%mask, <8 x i16> *%src) {
 ; CHECK-LABEL: foo_trunc_v8i8_v8i16:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .pad #8
 ; CHECK-NEXT:    sub sp, #8
 ; CHECK-NEXT:    vldrh.u16 q0, [r1]
-; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    vcmp.s16 gt, q0, zr
 ; CHECK-NEXT:    vpst
 ; CHECK-NEXT:    vldrht.u16 q0, [r2]
 ; CHECK-NEXT:    vmrs r1, p0
 ; CHECK-NEXT:    and r2, r1, #1
-; CHECK-NEXT:    rsbs r2, r2, #0
-; CHECK-NEXT:    bfi r3, r2, #0, #1
-; CHECK-NEXT:    ubfx r2, r1, #2, #1
-; CHECK-NEXT:    rsbs r2, r2, #0
-; CHECK-NEXT:    bfi r3, r2, #1, #1
-; CHECK-NEXT:    ubfx r2, r1, #4, #1
-; CHECK-NEXT:    rsbs r2, r2, #0
-; CHECK-NEXT:    bfi r3, r2, #2, #1
-; CHECK-NEXT:    ubfx r2, r1, #6, #1
-; CHECK-NEXT:    rsbs r2, r2, #0
-; CHECK-NEXT:    bfi r3, r2, #3, #1
-; CHECK-NEXT:    ubfx r2, r1, #8, #1
-; CHECK-NEXT:    rsbs r2, r2, #0
-; CHECK-NEXT:    bfi r3, r2, #4, #1
-; CHECK-NEXT:    ubfx r2, r1, #10, #1
-; CHECK-NEXT:    rsbs r2, r2, #0
-; CHECK-NEXT:    bfi r3, r2, #5, #1
-; CHECK-NEXT:    ubfx r2, r1, #12, #1
+; CHECK-NEXT:    rsbs r3, r2, #0
+; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    bfi r2, r3, #0, #1
+; CHECK-NEXT:    ubfx r3, r1, #2, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r2, r3, #1, #1
+; CHECK-NEXT:    ubfx r3, r1, #4, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r2, r3, #2, #1
+; CHECK-NEXT:    ubfx r3, r1, #6, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r2, r3, #3, #1
+; CHECK-NEXT:    ubfx r3, r1, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r2, r3, #4, #1
+; CHECK-NEXT:    ubfx r3, r1, #10, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r2, r3, #5, #1
+; CHECK-NEXT:    ubfx r3, r1, #12, #1
 ; CHECK-NEXT:    ubfx r1, r1, #14, #1
-; CHECK-NEXT:    rsbs r2, r2, #0
-; CHECK-NEXT:    bfi r3, r2, #6, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r2, r3, #6, #1
 ; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #7, #1
-; CHECK-NEXT:    lsls r2, r3, #31
-; CHECK-NEXT:    uxtb r1, r3
+; CHECK-NEXT:    bfi r2, r1, #7, #1
+; CHECK-NEXT:    uxtb r1, r2
+; CHECK-NEXT:    lsls r2, r2, #31
 ; CHECK-NEXT:    itt ne
 ; CHECK-NEXT:    vmovne.u16 r2, q0[0]
 ; CHECK-NEXT:    strbne r2, [r0]
 ; CHECK-NEXT:    lsls r2, r1, #30
 ; CHECK-NEXT:    itt mi
 ; CHECK-NEXT:    vmovmi.u16 r2, q0[1]
 ; CHECK-NEXT:    strbmi r2, [r0, #1]
 ; CHECK-NEXT:    lsls r2, r1, #29

 }
 
 define void @foo_trunc_v4i8_v4i32(<4 x i8> *%dest, <4 x i32> *%mask, <4 x i32> *%src) {
 ; CHECK-LABEL: foo_trunc_v4i8_v4i32:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .pad #4
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    vcmp.s32 gt, q0, zr
 ; CHECK-NEXT:    vpst
 ; CHECK-NEXT:    vldrwt.u32 q0, [r2]
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    and r2, r1, #1
-; CHECK-NEXT:    rsbs r2, r2, #0
-; CHECK-NEXT:    bfi r3, r2, #0, #1
-; CHECK-NEXT:    ubfx r2, r1, #4, #1
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    and r1, r2, #1
+; CHECK-NEXT:    rsbs r3, r1, #0
+; CHECK-NEXT:    movs r1, #0
+; CHECK-NEXT:    bfi r1, r3, #0, #1
+; CHECK-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r1, r3, #1, #1
+; CHECK-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-NEXT:    rsbs r2, r2, #0
-; CHECK-NEXT:    bfi r3, r2, #1, #1
-; CHECK-NEXT:    ubfx r2, r1, #8, #1
-; CHECK-NEXT:    ubfx r1, r1, #12, #1
-; CHECK-NEXT:    rsbs r2, r2, #0
-; CHECK-NEXT:    bfi r3, r2, #2, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #3, #1
-; CHECK-NEXT:    and r1, r3, #15
+; CHECK-NEXT:    bfi r1, r2, #3, #1
 ; CHECK-NEXT:    lsls r2, r1, #31
 ; CHECK-NEXT:    itt ne
 ; CHECK-NEXT:    vmovne r2, s0
 ; CHECK-NEXT:    strbne r2, [r0]
 ; CHECK-NEXT:    lsls r2, r1, #30
 ; CHECK-NEXT:    itt mi
 ; CHECK-NEXT:    vmovmi r2, s1
 ; CHECK-NEXT:    strbmi r2, [r0, #1]

 }
 
 define void @foo_trunc_v4i16_v4i32(<4 x i16> *%dest, <4 x i32> *%mask, <4 x i32> *%src) {
 ; CHECK-LABEL: foo_trunc_v4i16_v4i32:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .pad #4
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    vcmp.s32 gt, q0, zr
 ; CHECK-NEXT:    vpst
 ; CHECK-NEXT:    vldrwt.u32 q0, [r2]
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    and r2, r1, #1
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    and r1, r2, #1
+; CHECK-NEXT:    rsbs r3, r1, #0
+; CHECK-NEXT:    movs r1, #0
+; CHECK-NEXT:    bfi r1, r3, #0, #1
+; CHECK-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r1, r3, #1, #1
+; CHECK-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-NEXT:    rsbs r2, r2, #0
-; CHECK-NEXT:    bfi r3, r2, #0, #1
-; CHECK-NEXT:    ubfx r2, r1, #4, #1
-; CHECK-NEXT:    rsbs r2, r2, #0
-; CHECK-NEXT:    bfi r3, r2, #1, #1
-; CHECK-NEXT:    ubfx r2, r1, #8, #1
-; CHECK-NEXT:    ubfx r1, r1, #12, #1
-; CHECK-NEXT:    rsbs r2, r2, #0
-; CHECK-NEXT:    bfi r3, r2, #2, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #3, #1
-; CHECK-NEXT:    and r1, r3, #15
+; CHECK-NEXT:    bfi r1, r2, #3, #1
 ; CHECK-NEXT:    lsls r2, r1, #31
 ; CHECK-NEXT:    itt ne
 ; CHECK-NEXT:    vmovne r2, s0
 ; CHECK-NEXT:    strhne r2, [r0]
 ; CHECK-NEXT:    lsls r2, r1, #30
 ; CHECK-NEXT:    itt mi
 ; CHECK-NEXT:    vmovmi r2, s1
 ; CHECK-NEXT:    strhmi r2, [r0, #2]

llvm/test/CodeGen/Thumb2/mve-pred-bitcast.ll

 
 
 define arm_aapcs_vfpcc i4 @bitcast_from_v4i1(<4 x i32> %a) {
 ; CHECK-LE-LABEL: bitcast_from_v4i1:
 ; CHECK-LE:       @ %bb.0: @ %entry
 ; CHECK-LE-NEXT:    .pad #4
 ; CHECK-LE-NEXT:    sub sp, #4
 ; CHECK-LE-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-LE-NEXT:    movs r0, #0
 ; CHECK-LE-NEXT:    vmrs r1, p0
-; CHECK-LE-NEXT:    and r2, r1, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    and r0, r1, #1
+; CHECK-LE-NEXT:    rsbs r2, r0, #0
+; CHECK-LE-NEXT:    movs r0, #0
 ; CHECK-LE-NEXT:    bfi r0, r2, #0, #1
 ; CHECK-LE-NEXT:    ubfx r2, r1, #4, #1
 ; CHECK-LE-NEXT:    rsbs r2, r2, #0
 ; CHECK-LE-NEXT:    bfi r0, r2, #1, #1
 ; CHECK-LE-NEXT:    ubfx r2, r1, #8, #1
 ; CHECK-LE-NEXT:    ubfx r1, r1, #12, #1
 ; CHECK-LE-NEXT:    rsbs r2, r2, #0
 ; CHECK-LE-NEXT:    bfi r0, r2, #2, #1
 ; CHECK-LE-NEXT:    rsbs r1, r1, #0
 ; CHECK-LE-NEXT:    bfi r0, r1, #3, #1
-; CHECK-LE-NEXT:    and r0, r0, #15
 ; CHECK-LE-NEXT:    add sp, #4
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: bitcast_from_v4i1:
 ; CHECK-BE:       @ %bb.0: @ %entry
 ; CHECK-BE-NEXT:    .pad #4
 ; CHECK-BE-NEXT:    sub sp, #4
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
-; CHECK-BE-NEXT:    movs r3, #0
 ; CHECK-BE-NEXT:    vcmp.i32 eq, q1, zr
-; CHECK-BE-NEXT:    vmrs r0, p0
-; CHECK-BE-NEXT:    and r2, r0, #1
-; CHECK-BE-NEXT:    ubfx r1, r0, #4, #1
+; CHECK-BE-NEXT:    vmrs r1, p0
+; CHECK-BE-NEXT:    and r0, r1, #1
+; CHECK-BE-NEXT:    rsbs r2, r0, #0
+; CHECK-BE-NEXT:    movs r0, #0
+; CHECK-BE-NEXT:    bfi r0, r2, #0, #1
+; CHECK-BE-NEXT:    ubfx r2, r1, #4, #1
 ; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r0, r2, #1, #1
+; CHECK-BE-NEXT:    ubfx r2, r1, #8, #1
+; CHECK-BE-NEXT:    ubfx r1, r1, #12, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r0, r2, #2, #1
 ; CHECK-BE-NEXT:    rsbs r1, r1, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #0, #1
-; CHECK-BE-NEXT:    bfi r3, r1, #1, #1
-; CHECK-BE-NEXT:    ubfx r1, r0, #8, #1
-; CHECK-BE-NEXT:    ubfx r0, r0, #12, #1
-; CHECK-BE-NEXT:    rsbs r1, r1, #0
-; CHECK-BE-NEXT:    bfi r3, r1, #2, #1
-; CHECK-BE-NEXT:    rsbs r0, r0, #0
-; CHECK-BE-NEXT:    bfi r3, r0, #3, #1
-; CHECK-BE-NEXT:    and r0, r3, #15
+; CHECK-BE-NEXT:    bfi r0, r1, #3, #1
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp eq <4 x i32> %a, zeroinitializer
   %b = bitcast <4 x i1> %c to i4
   ret i4 %b
 }
 
 define arm_aapcs_vfpcc i8 @bitcast_from_v8i1(<8 x i16> %a) {
 ; CHECK-LE-LABEL: bitcast_from_v8i1:
 ; CHECK-LE:       @ %bb.0: @ %entry
 ; CHECK-LE-NEXT:    .pad #8
 ; CHECK-LE-NEXT:    sub sp, #8
 ; CHECK-LE-NEXT:    vcmp.i16 eq, q0, zr
-; CHECK-LE-NEXT:    movs r0, #0
 ; CHECK-LE-NEXT:    vmrs r1, p0
-; CHECK-LE-NEXT:    and r2, r1, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    and r0, r1, #1
+; CHECK-LE-NEXT:    rsbs r2, r0, #0
+; CHECK-LE-NEXT:    movs r0, #0
 ; CHECK-LE-NEXT:    bfi r0, r2, #0, #1
 ; CHECK-LE-NEXT:    ubfx r2, r1, #2, #1
 ; CHECK-LE-NEXT:    rsbs r2, r2, #0
 ; CHECK-LE-NEXT:    bfi r0, r2, #1, #1
 ; CHECK-LE-NEXT:    ubfx r2, r1, #4, #1
 ; CHECK-LE-NEXT:    rsbs r2, r2, #0
 ; CHECK-LE-NEXT:    bfi r0, r2, #2, #1
 ; CHECK-LE-NEXT:    ubfx r2, r1, #6, #1

 ;
 ; CHECK-BE-LABEL: bitcast_from_v8i1:
 ; CHECK-BE:       @ %bb.0: @ %entry
 ; CHECK-BE-NEXT:    .pad #8
 ; CHECK-BE-NEXT:    sub sp, #8
 ; CHECK-BE-NEXT:    vrev64.16 q1, q0
 ; CHECK-BE-NEXT:    vcmp.i16 eq, q1, zr
 ; CHECK-BE-NEXT:    vmrs r1, p0
-; CHECK-BE-NEXT:    ubfx r0, r1, #2, #1
-; CHECK-BE-NEXT:    rsbs r2, r0, #0
 ; CHECK-BE-NEXT:    and r0, r1, #1
-; CHECK-BE-NEXT:    rsbs r3, r0, #0
+; CHECK-BE-NEXT:    rsbs r2, r0, #0
 ; CHECK-BE-NEXT:    movs r0, #0
-; CHECK-BE-NEXT:    bfi r0, r3, #0, #1
+; CHECK-BE-NEXT:    bfi r0, r2, #0, #1
+; CHECK-BE-NEXT:    ubfx r2, r1, #2, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
 ; CHECK-BE-NEXT:    bfi r0, r2, #1, #1
 ; CHECK-BE-NEXT:    ubfx r2, r1, #4, #1
 ; CHECK-BE-NEXT:    rsbs r2, r2, #0
 ; CHECK-BE-NEXT:    bfi r0, r2, #2, #1
 ; CHECK-BE-NEXT:    ubfx r2, r1, #6, #1
 ; CHECK-BE-NEXT:    rsbs r2, r2, #0
 ; CHECK-BE-NEXT:    bfi r0, r2, #3, #1
 ; CHECK-BE-NEXT:    ubfx r2, r1, #8, #1

llvm/test/CodeGen/Thumb2/mve-pred-loadstore.ll

entry:
   ret <2 x i64> %s
 }
 
 
 define arm_aapcs_vfpcc void @store_v4i1(<4 x i1> *%dst, <4 x i32> %a) {
 ; CHECK-LE-LABEL: store_v4i1:
 ; CHECK-LE:       @ %bb.0: @ %entry
 ; CHECK-LE-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-LE-NEXT:    movs r1, #0
-; CHECK-LE-NEXT:    vmrs r2, p0
-; CHECK-LE-NEXT:    and r3, r2, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r2, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r2, #8, #1
-; CHECK-LE-NEXT:    ubfx r2, r2, #12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #2, #1
+; CHECK-LE-NEXT:    movs r3, #0
+; CHECK-LE-NEXT:    vmrs r1, p0
+; CHECK-LE-NEXT:    and r2, r1, #1
 ; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-LE-NEXT:    and r1, r1, #15
-; CHECK-LE-NEXT:    strb r1, [r0]
+; CHECK-LE-NEXT:    bfi r3, r2, #0, #1
+; CHECK-LE-NEXT:    ubfx r2, r1, #4, #1
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    bfi r3, r2, #1, #1
+; CHECK-LE-NEXT:    ubfx r2, r1, #8, #1
+; CHECK-LE-NEXT:    ubfx r1, r1, #12, #1
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    bfi r3, r2, #2, #1
+; CHECK-LE-NEXT:    rsbs r1, r1, #0
+; CHECK-LE-NEXT:    bfi r3, r1, #3, #1
+; CHECK-LE-NEXT:    strb r3, [r0]
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: store_v4i1:
 ; CHECK-BE:       @ %bb.0: @ %entry
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    movs r3, #0
 ; CHECK-BE-NEXT:    vcmp.i32 eq, q1, zr
 ; CHECK-BE-NEXT:    vmrs r1, p0
-; CHECK-BE-NEXT:    and r3, r1, #1
+; CHECK-BE-NEXT:    and r2, r1, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r3, r2, #0, #1
 ; CHECK-BE-NEXT:    ubfx r2, r1, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    rsb.w r12, r2, #0
-; CHECK-BE-NEXT:    movs r2, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #8, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r3, r2, #1, #1
+; CHECK-BE-NEXT:    ubfx r2, r1, #8, #1
 ; CHECK-BE-NEXT:    ubfx r1, r1, #12, #1
-; CHECK-BE-NEXT:    bfi r2, r12, #1, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r3, r2, #2, #1
 ; CHECK-BE-NEXT:    rsbs r1, r1, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-BE-NEXT:    bfi r2, r1, #3, #1
-; CHECK-BE-NEXT:    and r1, r2, #15
-; CHECK-BE-NEXT:    strb r1, [r0]
+; CHECK-BE-NEXT:    bfi r3, r1, #3, #1
+; CHECK-BE-NEXT:    strb r3, [r0]
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp eq <4 x i32> %a, zeroinitializer
   store <4 x i1> %c, <4 x i1>* %dst
   ret void
 }
 
 define arm_aapcs_vfpcc void @store_v8i1(<8 x i1> *%dst, <8 x i16> %a) {
 ; CHECK-LE-LABEL: store_v8i1:
 ; CHECK-LE:       @ %bb.0: @ %entry
 ; CHECK-LE-NEXT:    vcmp.i16 eq, q0, zr
-; CHECK-LE-NEXT:    movs r1, #0
 ; CHECK-LE-NEXT:    vmrs r2, p0
-; CHECK-LE-NEXT:    and r3, r2, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    and r1, r2, #1
+; CHECK-LE-NEXT:    rsbs r3, r1, #0
+; CHECK-LE-NEXT:    movs r1, #0
 ; CHECK-LE-NEXT:    bfi r1, r3, #0, #1
 ; CHECK-LE-NEXT:    ubfx r3, r2, #2, #1
 ; CHECK-LE-NEXT:    rsbs r3, r3, #0
 ; CHECK-LE-NEXT:    bfi r1, r3, #1, #1
 ; CHECK-LE-NEXT:    ubfx r3, r2, #4, #1
 ; CHECK-LE-NEXT:    rsbs r3, r3, #0
 ; CHECK-LE-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-LE-NEXT:    ubfx r3, r2, #6, #1

 ; CHECK-LE-NEXT:    strb r1, [r0]
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: store_v8i1:
 ; CHECK-BE:       @ %bb.0: @ %entry
 ; CHECK-BE-NEXT:    vrev64.16 q1, q0
 ; CHECK-BE-NEXT:    vcmp.i16 eq, q1, zr
 ; CHECK-BE-NEXT:    vmrs r2, p0
-; CHECK-BE-NEXT:    ubfx r1, r2, #2, #1
-; CHECK-BE-NEXT:    rsb.w r12, r1, #0
 ; CHECK-BE-NEXT:    and r1, r2, #1
 ; CHECK-BE-NEXT:    rsbs r3, r1, #0
 ; CHECK-BE-NEXT:    movs r1, #0
 ; CHECK-BE-NEXT:    bfi r1, r3, #0, #1
+; CHECK-BE-NEXT:    ubfx r3, r2, #2, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
 ; CHECK-BE-NEXT:    ubfx r3, r2, #4, #1
-; CHECK-BE-NEXT:    bfi r1, r12, #1, #1
 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-BE-NEXT:    ubfx r3, r2, #6, #1
 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r1, r3, #3, #1
 ; CHECK-BE-NEXT:    ubfx r3, r2, #8, #1
 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r1, r3, #4, #1