Expand masked mem intrinsics correctly wrt big-endian

[markus]

Summary

Need to take endianness into account when doing vector to scalar casts such as

%bc = bitcast <8 x i1> %v to i8

Companion review for https://reviews.llvm.org/D94867
Upload in response to https://lists.llvm.org/pipermail/llvm-dev/2021-January/147862.html
Attempting to document the actual memory layout rules for vectors in https://reviews.llvm.org/D94964

[dmgreen]

Thanks for the patch. There is a patch to make MVE consistent with the rest of MVE in D94867. This will need rebasing on top of that, with update tests to make the two consistent again.

llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp

332–333

It's a shame we can't just use extract elements - that would simplify this a lot!

[markus]

In D94765#2503378, @dmgreen wrote:

Thanks for the patch. There is a patch to make MVE consistent with the rest of MVE in D94867. This will need rebasing on top of that, with update tests to make the two consistent again.

Thank you for looking into this.

llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp

332–333

It's a shame we can't just use extract elements - that would simplify this a lot!

I fully agree.

[lebedev.ri]

Tests missing.

[bjope]

Add tests using big endian datalayout.

Also updated test checks in the thumb codegen test cases that is impacted by
this change. Not sure if this patch should be parent or child to D94867 (as
both patches need to land at the same time to not break big-endian ARM.

[dmgreen]

Yeah thanks. I've added a link

[dmgreen]

OK. Thanks for the commit. This LGTM.

Are you happy for me to commit this and D94867 together, to keep them in-sync?

[bjope]

In D94765#2548276, @dmgreen wrote:

OK. Thanks for the commit. This LGTM.

Are you happy for me to commit this and D94867 together, to keep them in-sync?

(answering on behalf of Markus who will be out-of-office for a couple of weeks)

That sounds great! Thanks for all the help with this problem.

llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp

 //===- ScalarizeMaskedMemIntrin.cpp - Scalarize unsupported masked mem ----===//
 //                                    instrinsics
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
for (unsigned i = 0; i != NumElts; ++i) {
     Constant *CElt = C->getAggregateElement(i);
     if (!CElt || !isa<ConstantInt>(CElt))
       return false;
   }
 
   return true;
 }
 
+static unsigned adjustForEndian(const DataLayout &DL, unsigned VectorWidth,
+                                unsigned Idx) {
+  return DL.isBigEndian() ? VectorWidth - 1 - Idx : Idx;
+}
+
 // Translate a masked load intrinsic like
 // <16 x i32 > @llvm.masked.load( <16 x i32>* %addr, i32 align,
 //                               <16 x i1> %mask, <16 x i32> %passthru)
 // to a chain of basic blocks, with loading element one-by-one if
 // the appropriate mask bit is set
 //
 //  %1 = bitcast i8* %addr to i32*
 //  %2 = extractelement <16 x i1> %mask, i32 0

 //  %9 = insertelement <16 x i32> %res.phi.else, i32 %8, i32 1
 //  br label %else2
 //
 // else2:                                          ; preds = %else, %cond.load1
 //  %res.phi.else3 = phi <16 x i32> [ %9, %cond.load1 ], [ %res.phi.else, %else ]
 //  %10 = extractelement <16 x i1> %mask, i32 2
 //  br i1 %10, label %cond.load4, label %else5
 //
-static void scalarizeMaskedLoad(CallInst *CI, DomTreeUpdater *DTU,
-                                bool &ModifiedDT) {
+static void scalarizeMaskedLoad(const DataLayout &DL, CallInst *CI,
+                                DomTreeUpdater *DTU, bool &ModifiedDT) {
   Value *Ptr = CI->getArgOperand(0);
   Value *Alignment = CI->getArgOperand(1);
   Value *Mask = CI->getArgOperand(2);
   Value *Src0 = CI->getArgOperand(3);
 
   const Align AlignVal = cast<ConstantInt>(Alignment)->getAlignValue();
   VectorType *VecType = cast<FixedVectorType>(CI->getType());
 
for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
     //
     //  %res.phi.else3 = phi <16 x i32> [ %11, %cond.load1 ], [ %res.phi.else, %else ]
     //  %mask_1 = and i16 %scalar_mask, i32 1 << Idx
     //  %cond = icmp ne i16 %mask_1, 0
     //  br i1 %mask_1, label %cond.load, label %else
     //
     Value *Predicate;
     if (VectorWidth != 1) {
-      Value *Mask = Builder.getInt(APInt::getOneBitSet(VectorWidth, Idx));
+      Value *Mask = Builder.getInt(APInt::getOneBitSet(
+          VectorWidth, adjustForEndian(DL, VectorWidth, Idx)));
       Predicate = Builder.CreateICmpNE(Builder.CreateAnd(SclrMask, Mask),
                                        Builder.getIntN(VectorWidth, 0));
     } else {
       Predicate = Builder.CreateExtractElement(Mask, Idx);
     }
 
     // Create "cond" block
     //

 //   br i1 %5, label %cond.store1, label %else2
 //
 // cond.store1:                                      ; preds = %else
 //   %6 = extractelement <16 x i32> %val, i32 1
 //   %7 = getelementptr i32* %1, i32 1
 //   store i32 %6, i32* %7
 //   br label %else2
 //   . . .
-static void scalarizeMaskedStore(CallInst *CI, DomTreeUpdater *DTU,
-                                 bool &ModifiedDT) {
+static void scalarizeMaskedStore(const DataLayout &DL, CallInst *CI,
+                                 DomTreeUpdater *DTU, bool &ModifiedDT) {
   Value *Src = CI->getArgOperand(0);
   Value *Ptr = CI->getArgOperand(1);
   Value *Alignment = CI->getArgOperand(2);
   Value *Mask = CI->getArgOperand(3);
 
   const Align AlignVal = cast<ConstantInt>(Alignment)->getAlignValue();
   auto *VecType = cast<VectorType>(Src->getType());
 
for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
       Value *OneElt = Builder.CreateExtractElement(Src, Idx);
       Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, FirstEltPtr, Idx);
       Builder.CreateAlignedStore(OneElt, Gep, AdjustedAlignVal);
     }
     CI->eraseFromParent();
     return;
   }
 
   // If the mask is not v1i1, use scalar bit test operations. This generates
   // better results on X86 at least.
   Value *SclrMask;
   if (VectorWidth != 1) {
     Type *SclrMaskTy = Builder.getIntNTy(VectorWidth);
     SclrMask = Builder.CreateBitCast(Mask, SclrMaskTy, "scalar_mask");
   }
 
   for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
     // Fill the "else" block, created in the previous iteration
     //
     //  %mask_1 = and i16 %scalar_mask, i32 1 << Idx
     //  %cond = icmp ne i16 %mask_1, 0
     //  br i1 %mask_1, label %cond.store, label %else
     //
     Value *Predicate;
     if (VectorWidth != 1) {
-      Value *Mask = Builder.getInt(APInt::getOneBitSet(VectorWidth, Idx));
+      Value *Mask = Builder.getInt(APInt::getOneBitSet(
+          VectorWidth, adjustForEndian(DL, VectorWidth, Idx)));
       Predicate = Builder.CreateICmpNE(Builder.CreateAnd(SclrMask, Mask),
                                        Builder.getIntN(VectorWidth, 0));
     } else {
       Predicate = Builder.CreateExtractElement(Mask, Idx);
     }
 
     // Create "cond" block
     //

 // cond.load1:
 // %Ptr1 = extractelement <16 x i32*> %Ptrs, i32 1
 // %Load1 = load i32, i32* %Ptr1, align 4
 // %Res1 = insertelement <16 x i32> %res.phi.else, i32 %Load1, i32 1
 // br label %else2
 // . . .
 // %Result = select <16 x i1> %Mask, <16 x i32> %res.phi.select, <16 x i32> %Src
 // ret <16 x i32> %Result
-static void scalarizeMaskedGather(CallInst *CI, DomTreeUpdater *DTU,
-                                  bool &ModifiedDT) {
+static void scalarizeMaskedGather(const DataLayout &DL, CallInst *CI,
+                                  DomTreeUpdater *DTU, bool &ModifiedDT) {
   Value *Ptrs = CI->getArgOperand(0);
   Value *Alignment = CI->getArgOperand(1);
   Value *Mask = CI->getArgOperand(2);
   Value *Src0 = CI->getArgOperand(3);
 
   auto *VecType = cast<FixedVectorType>(CI->getType());
   Type *EltTy = VecType->getElementType();
 
for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
     //
     //  %Mask1 = and i16 %scalar_mask, i32 1 << Idx
     //  %cond = icmp ne i16 %mask_1, 0
     //  br i1 %Mask1, label %cond.load, label %else
     //
 
     Value *Predicate;
     if (VectorWidth != 1) {
-      Value *Mask = Builder.getInt(APInt::getOneBitSet(VectorWidth, Idx));
+      Value *Mask = Builder.getInt(APInt::getOneBitSet(
+          VectorWidth, adjustForEndian(DL, VectorWidth, Idx)));
       Predicate = Builder.CreateICmpNE(Builder.CreateAnd(SclrMask, Mask),
                                        Builder.getIntN(VectorWidth, 0));
     } else {
       Predicate = Builder.CreateExtractElement(Mask, Idx, "Mask" + Twine(Idx));
     }
 
     // Create "cond" block
     //

 // br i1 %Mask1, label %cond.store1, label %else2
 //
 // cond.store1:
 // %Elt1 = extractelement <16 x i32> %Src, i32 1
 // %Ptr1 = extractelement <16 x i32*> %Ptrs, i32 1
 // store i32 %Elt1, i32* %Ptr1, align 4
 // br label %else2
 //   . . .
-static void scalarizeMaskedScatter(CallInst *CI, DomTreeUpdater *DTU,
-                                   bool &ModifiedDT) {
+static void scalarizeMaskedScatter(const DataLayout &DL, CallInst *CI,
+                                   DomTreeUpdater *DTU, bool &ModifiedDT) {
   Value *Src = CI->getArgOperand(0);
   Value *Ptrs = CI->getArgOperand(1);
   Value *Alignment = CI->getArgOperand(2);
   Value *Mask = CI->getArgOperand(3);
 
   auto *SrcFVTy = cast<FixedVectorType>(Src->getType());
 
   assert(
for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
     // Fill the "else" block, created in the previous iteration
     //
     //  %Mask1 = and i16 %scalar_mask, i32 1 << Idx
     //  %cond = icmp ne i16 %mask_1, 0
     //  br i1 %Mask1, label %cond.store, label %else
     //
     Value *Predicate;
     if (VectorWidth != 1) {
-      Value *Mask = Builder.getInt(APInt::getOneBitSet(VectorWidth, Idx));
+      Value *Mask = Builder.getInt(APInt::getOneBitSet(
+          VectorWidth, adjustForEndian(DL, VectorWidth, Idx)));
       Predicate = Builder.CreateICmpNE(Builder.CreateAnd(SclrMask, Mask),
                                        Builder.getIntN(VectorWidth, 0));
     } else {
       Predicate = Builder.CreateExtractElement(Mask, Idx, "Mask" + Twine(Idx));
     }
 
     // Create "cond" block
     //
for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
 
     Builder.SetInsertPoint(NewIfBlock, NewIfBlock->begin());
   }
   CI->eraseFromParent();
 
   ModifiedDT = true;
 }
 
-static void scalarizeMaskedExpandLoad(CallInst *CI, DomTreeUpdater *DTU,
-                                      bool &ModifiedDT) {
+static void scalarizeMaskedExpandLoad(const DataLayout &DL, CallInst *CI,
+                                      DomTreeUpdater *DTU, bool &ModifiedDT) {
   Value *Ptr = CI->getArgOperand(0);
   Value *Mask = CI->getArgOperand(1);
   Value *PassThru = CI->getArgOperand(2);
 
   auto *VecType = cast<FixedVectorType>(CI->getType());
 
   Type *EltTy = VecType->getElementType();
 
for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
     //
     //  %res.phi.else3 = phi <16 x i32> [ %11, %cond.load1 ], [ %res.phi.else, %else ]
     //  %mask_1 = extractelement <16 x i1> %mask, i32 Idx
     //  br i1 %mask_1, label %cond.load, label %else
     //
 
     Value *Predicate;
     if (VectorWidth != 1) {
-      Value *Mask = Builder.getInt(APInt::getOneBitSet(VectorWidth, Idx));
+      Value *Mask = Builder.getInt(APInt::getOneBitSet(
+          VectorWidth, adjustForEndian(DL, VectorWidth, Idx)));
       Predicate = Builder.CreateICmpNE(Builder.CreateAnd(SclrMask, Mask),
                                        Builder.getIntN(VectorWidth, 0));
     } else {
       Predicate = Builder.CreateExtractElement(Mask, Idx, "Mask" + Twine(Idx));
     }
 
     // Create "cond" block
     //
static void scalarizeMaskedExpandLoad(const DataLayout &DL, CallInst *CI,
   }
 
   CI->replaceAllUsesWith(VResult);
   CI->eraseFromParent();
 
   ModifiedDT = true;
 }
 
-static void scalarizeMaskedCompressStore(CallInst *CI, DomTreeUpdater *DTU,
+static void scalarizeMaskedCompressStore(const DataLayout &DL, CallInst *CI,
+                                         DomTreeUpdater *DTU,
                                          bool &ModifiedDT) {
   Value *Src = CI->getArgOperand(0);
   Value *Ptr = CI->getArgOperand(1);
   Value *Mask = CI->getArgOperand(2);
 
   auto *VecType = cast<FixedVectorType>(Src->getType());
 
   IRBuilder<> Builder(CI->getContext());
static void scalarizeMaskedCompressStore(const DataLayout &DL, CallInst *CI,
   for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
     // Fill the "else" block, created in the previous iteration
     //
     //  %mask_1 = extractelement <16 x i1> %mask, i32 Idx
     //  br i1 %mask_1, label %cond.store, label %else
     //
     Value *Predicate;
     if (VectorWidth != 1) {
-      Value *Mask = Builder.getInt(APInt::getOneBitSet(VectorWidth, Idx));
+      Value *Mask = Builder.getInt(APInt::getOneBitSet(
+          VectorWidth, adjustForEndian(DL, VectorWidth, Idx)));
       Predicate = Builder.CreateICmpNE(Builder.CreateAnd(SclrMask, Mask),
                                        Builder.getIntN(VectorWidth, 0));
     } else {
       Predicate = Builder.CreateExtractElement(Mask, Idx, "Mask" + Twine(Idx));
     }
 
     // Create "cond" block
     //
if (II) {
     default:
       break;
     case Intrinsic::masked_load:
       // Scalarize unsupported vector masked load
       if (TTI.isLegalMaskedLoad(
               CI->getType(),
               cast<ConstantInt>(CI->getArgOperand(1))->getAlignValue()))
         return false;
-      scalarizeMaskedLoad(CI, DTU, ModifiedDT);
+      scalarizeMaskedLoad(DL, CI, DTU, ModifiedDT);
       return true;
     case Intrinsic::masked_store:
       if (TTI.isLegalMaskedStore(
               CI->getArgOperand(0)->getType(),
               cast<ConstantInt>(CI->getArgOperand(2))->getAlignValue()))
         return false;
-      scalarizeMaskedStore(CI, DTU, ModifiedDT);
+      scalarizeMaskedStore(DL, CI, DTU, ModifiedDT);
       return true;
     case Intrinsic::masked_gather: {
       unsigned AlignmentInt =
           cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
       Type *LoadTy = CI->getType();
       Align Alignment =
           DL.getValueOrABITypeAlignment(MaybeAlign(AlignmentInt), LoadTy);
       if (TTI.isLegalMaskedGather(LoadTy, Alignment))
         return false;
-      scalarizeMaskedGather(CI, DTU, ModifiedDT);
+      scalarizeMaskedGather(DL, CI, DTU, ModifiedDT);
       return true;
     }
     case Intrinsic::masked_scatter: {
       unsigned AlignmentInt =
           cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
       Type *StoreTy = CI->getArgOperand(0)->getType();
       Align Alignment =
           DL.getValueOrABITypeAlignment(MaybeAlign(AlignmentInt), StoreTy);
       if (TTI.isLegalMaskedScatter(StoreTy, Alignment))
         return false;
-      scalarizeMaskedScatter(CI, DTU, ModifiedDT);
+      scalarizeMaskedScatter(DL, CI, DTU, ModifiedDT);
       return true;
     }
     case Intrinsic::masked_expandload:
       if (TTI.isLegalMaskedExpandLoad(CI->getType()))
         return false;
-      scalarizeMaskedExpandLoad(CI, DTU, ModifiedDT);
+      scalarizeMaskedExpandLoad(DL, CI, DTU, ModifiedDT);
       return true;
     case Intrinsic::masked_compressstore:
       if (TTI.isLegalMaskedCompressStore(CI->getArgOperand(0)->getType()))
         return false;
-      scalarizeMaskedCompressStore(CI, DTU, ModifiedDT);
+      scalarizeMaskedCompressStore(DL, CI, DTU, ModifiedDT);
       return true;
     }
   }
 
   return false;
 }

llvm/test/CodeGen/Thumb2/mve-masked-ldst.ll

 ; CHECK-BE-NEXT:    it lt
 ; CHECK-BE-NEXT:    movlt r3, #1
 ; CHECK-BE-NEXT:    cmp r3, #0
 ; CHECK-BE-NEXT:    it ne
 ; CHECK-BE-NEXT:    mvnne r3, #1
 ; CHECK-BE-NEXT:    bfi r3, lr, #0, #1
 ; CHECK-BE-NEXT:    @ implicit-def: $q2
 ; CHECK-BE-NEXT:    and r1, r3, #3
-; CHECK-BE-NEXT:    lsls r3, r3, #31
-; CHECK-BE-NEXT:    beq .LBB5_2
+; CHECK-BE-NEXT:    lsls r3, r3, #30
+; CHECK-BE-NEXT:    bpl .LBB5_2
 ; CHECK-BE-NEXT:  @ %bb.1: @ %cond.load
 ; CHECK-BE-NEXT:    ldr r3, [r2]
 ; CHECK-BE-NEXT:    vmov.32 q1[1], r3
 ; CHECK-BE-NEXT:    vrev64.32 q2, q1
 ; CHECK-BE-NEXT:  .LBB5_2: @ %else
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
-; CHECK-BE-NEXT:    lsls r1, r1, #30
-; CHECK-BE-NEXT:    bpl .LBB5_4
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    beq .LBB5_4
 ; CHECK-BE-NEXT:  @ %bb.3: @ %cond.load1
 ; CHECK-BE-NEXT:    ldr r1, [r2, #4]
 ; CHECK-BE-NEXT:    vrev64.32 q0, q2
 ; CHECK-BE-NEXT:    vmov.32 q0[3], r1
 ; CHECK-BE-NEXT:    vrev64.32 q2, q0
 ; CHECK-BE-NEXT:  .LBB5_4: @ %else2
 ; CHECK-BE-NEXT:    vrev64.32 q0, q2
 ; CHECK-BE-NEXT:    vrev64.32 q2, q1

 ; CHECK-BE-NEXT:    sbcs.w r2, r4, r2, asr #31
 ; CHECK-BE-NEXT:    it lt
 ; CHECK-BE-NEXT:    movlt r4, #1
 ; CHECK-BE-NEXT:    cmp r4, #0
 ; CHECK-BE-NEXT:    it ne
 ; CHECK-BE-NEXT:    mvnne r4, #1
 ; CHECK-BE-NEXT:    bfi r4, r1, #0, #1
 ; CHECK-BE-NEXT:    and r1, r4, #3
-; CHECK-BE-NEXT:    lsls r2, r4, #31
-; CHECK-BE-NEXT:    it ne
-; CHECK-BE-NEXT:    vstrne d0, [r0]
-; CHECK-BE-NEXT:    lsls r1, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r4, #30
 ; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vstrmi d1, [r0, #8]
+; CHECK-BE-NEXT:    vstrmi d0, [r0]
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    it ne
+; CHECK-BE-NEXT:    vstrne d1, [r0, #8]
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    pop {r4, r5, r7, pc}
 entry:
   %0 = load <2 x i32>, <2 x i32>* %mask, align 4
   %1 = icmp sgt <2 x i32> %0, zeroinitializer
   %2 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %src, i32 4, <2 x i1> %1, <2 x i32> undef)
   %3 = sext <2 x i32> %2 to <2 x i64>
   call void @llvm.masked.store.v2i64.p0v2i64(<2 x i64> %3, <2 x i64>* %dest, i32 8, <2 x i1> %1)

 ; CHECK-BE-NEXT:    it lt
 ; CHECK-BE-NEXT:    movlt r3, #1
 ; CHECK-BE-NEXT:    cmp r3, #0
 ; CHECK-BE-NEXT:    it ne
 ; CHECK-BE-NEXT:    mvnne r3, #1
 ; CHECK-BE-NEXT:    bfi r3, lr, #0, #1
 ; CHECK-BE-NEXT:    @ implicit-def: $q2
 ; CHECK-BE-NEXT:    and r1, r3, #3
-; CHECK-BE-NEXT:    lsls r3, r3, #31
-; CHECK-BE-NEXT:    beq .LBB6_2
+; CHECK-BE-NEXT:    lsls r3, r3, #30
+; CHECK-BE-NEXT:    bpl .LBB6_2
 ; CHECK-BE-NEXT:  @ %bb.1: @ %cond.load
 ; CHECK-BE-NEXT:    ldr r3, [r2]
 ; CHECK-BE-NEXT:    vmov.32 q1[1], r3
 ; CHECK-BE-NEXT:    vrev64.32 q2, q1
 ; CHECK-BE-NEXT:  .LBB6_2: @ %else
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
-; CHECK-BE-NEXT:    lsls r1, r1, #30
-; CHECK-BE-NEXT:    bpl .LBB6_4
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    beq .LBB6_4
 ; CHECK-BE-NEXT:  @ %bb.3: @ %cond.load1
 ; CHECK-BE-NEXT:    ldr r1, [r2, #4]
 ; CHECK-BE-NEXT:    vrev64.32 q0, q2
 ; CHECK-BE-NEXT:    vmov.32 q0[3], r1
 ; CHECK-BE-NEXT:    vrev64.32 q2, q0
 ; CHECK-BE-NEXT:  .LBB6_4: @ %else2
 ; CHECK-BE-NEXT:    vrev64.32 q0, q2
 ; CHECK-BE-NEXT:    vrev64.32 q2, q1

 ; CHECK-BE-NEXT:    sbcs.w r2, r4, r2, asr #31
 ; CHECK-BE-NEXT:    it lt
 ; CHECK-BE-NEXT:    movlt r4, #1
 ; CHECK-BE-NEXT:    cmp r4, #0
 ; CHECK-BE-NEXT:    it ne
 ; CHECK-BE-NEXT:    mvnne r4, #1
 ; CHECK-BE-NEXT:    bfi r4, r1, #0, #1
 ; CHECK-BE-NEXT:    and r1, r4, #3
-; CHECK-BE-NEXT:    lsls r2, r4, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    vmovne r2, r3, d0
-; CHECK-BE-NEXT:    strdne r3, r2, [r0]
-; CHECK-BE-NEXT:    lsls r1, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r4, #30
 ; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi r1, r2, d1
-; CHECK-BE-NEXT:    strdmi r2, r1, [r0, #8]
+; CHECK-BE-NEXT:    vmovmi r2, r3, d0
+; CHECK-BE-NEXT:    strdmi r3, r2, [r0]
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    vmovne r1, r2, d1
+; CHECK-BE-NEXT:    strdne r2, r1, [r0, #8]
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    pop {r4, r5, r7, pc}
 entry:
   %0 = load <2 x i32>, <2 x i32>* %mask, align 4
   %1 = icmp sgt <2 x i32> %0, zeroinitializer
   %2 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %src, i32 2, <2 x i1> %1, <2 x i32> undef)
   %3 = sext <2 x i32> %2 to <2 x i64>
   call void @llvm.masked.store.v2i64.p0v2i64(<2 x i64> %3, <2 x i64>* %dest, i32 4, <2 x i1> %1)

 ; CHECK-BE-NEXT:    it lt
 ; CHECK-BE-NEXT:    movlt r3, #1
 ; CHECK-BE-NEXT:    cmp r3, #0
 ; CHECK-BE-NEXT:    it ne
 ; CHECK-BE-NEXT:    mvnne r3, #1
 ; CHECK-BE-NEXT:    bfi r3, lr, #0, #1
 ; CHECK-BE-NEXT:    @ implicit-def: $q1
 ; CHECK-BE-NEXT:    and r1, r3, #3
-; CHECK-BE-NEXT:    lsls r3, r3, #31
-; CHECK-BE-NEXT:    beq .LBB7_2
+; CHECK-BE-NEXT:    lsls r3, r3, #30
+; CHECK-BE-NEXT:    bpl .LBB7_2
 ; CHECK-BE-NEXT:  @ %bb.1: @ %cond.load
 ; CHECK-BE-NEXT:    ldr r3, [r2]
 ; CHECK-BE-NEXT:    vmov.32 q2[1], r3
 ; CHECK-BE-NEXT:    vrev64.32 q1, q2
 ; CHECK-BE-NEXT:  .LBB7_2: @ %else
 ; CHECK-BE-NEXT:    vrev64.32 q2, q0
-; CHECK-BE-NEXT:    lsls r1, r1, #30
-; CHECK-BE-NEXT:    bpl .LBB7_4
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    beq .LBB7_4
 ; CHECK-BE-NEXT:  @ %bb.3: @ %cond.load1
 ; CHECK-BE-NEXT:    ldr r1, [r2, #4]
 ; CHECK-BE-NEXT:    vrev64.32 q0, q1
 ; CHECK-BE-NEXT:    vmov.32 q0[3], r1
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
 ; CHECK-BE-NEXT:  .LBB7_4: @ %else2
 ; CHECK-BE-NEXT:    vrev64.32 q3, q2
 ; CHECK-BE-NEXT:    movs r2, #0

 ; CHECK-BE-NEXT:    sbcs.w r1, r2, r3, asr #31
 ; CHECK-BE-NEXT:    it lt
 ; CHECK-BE-NEXT:    movlt r2, #1
 ; CHECK-BE-NEXT:    cmp r2, #0
 ; CHECK-BE-NEXT:    it ne
 ; CHECK-BE-NEXT:    mvnne r2, #1
 ; CHECK-BE-NEXT:    bfi r2, r12, #0, #1
 ; CHECK-BE-NEXT:    and r1, r2, #3
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    it ne
-; CHECK-BE-NEXT:    vstrne d0, [r0]
-; CHECK-BE-NEXT:    lsls r1, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r2, #30
 ; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vstrmi d1, [r0, #8]
+; CHECK-BE-NEXT:    vstrmi d0, [r0]
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    it ne
+; CHECK-BE-NEXT:    vstrne d1, [r0, #8]
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    pop {r7, pc}
 entry:
   %0 = load <2 x i32>, <2 x i32>* %mask, align 4
   %1 = icmp sgt <2 x i32> %0, zeroinitializer
   %2 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %src, i32 4, <2 x i1> %1, <2 x i32> undef)
   %3 = zext <2 x i32> %2 to <2 x i64>
   call void @llvm.masked.store.v2i64.p0v2i64(<2 x i64> %3, <2 x i64>* %dest, i32 8, <2 x i1> %1)

 ; CHECK-BE-NEXT:    it lt
 ; CHECK-BE-NEXT:    movlt r3, #1
 ; CHECK-BE-NEXT:    cmp r3, #0
 ; CHECK-BE-NEXT:    it ne
 ; CHECK-BE-NEXT:    mvnne r3, #1
 ; CHECK-BE-NEXT:    bfi r3, lr, #0, #1
 ; CHECK-BE-NEXT:    @ implicit-def: $q1
 ; CHECK-BE-NEXT:    and r1, r3, #3
-; CHECK-BE-NEXT:    lsls r3, r3, #31
-; CHECK-BE-NEXT:    beq .LBB8_2
+; CHECK-BE-NEXT:    lsls r3, r3, #30
+; CHECK-BE-NEXT:    bpl .LBB8_2
 ; CHECK-BE-NEXT:  @ %bb.1: @ %cond.load
 ; CHECK-BE-NEXT:    ldr r3, [r2]
 ; CHECK-BE-NEXT:    vmov.32 q2[1], r3
 ; CHECK-BE-NEXT:    vrev64.32 q1, q2
 ; CHECK-BE-NEXT:  .LBB8_2: @ %else
 ; CHECK-BE-NEXT:    vrev64.32 q2, q0
-; CHECK-BE-NEXT:    lsls r1, r1, #30
-; CHECK-BE-NEXT:    bpl .LBB8_4
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    beq .LBB8_4
 ; CHECK-BE-NEXT:  @ %bb.3: @ %cond.load1
 ; CHECK-BE-NEXT:    ldr r1, [r2, #4]
 ; CHECK-BE-NEXT:    vrev64.32 q0, q1
 ; CHECK-BE-NEXT:    vmov.32 q0[3], r1
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
 ; CHECK-BE-NEXT:  .LBB8_4: @ %else2
 ; CHECK-BE-NEXT:    vrev64.32 q3, q2
 ; CHECK-BE-NEXT:    movs r2, #0

 ; CHECK-BE-NEXT:    sbcs.w r1, r2, r3, asr #31
 ; CHECK-BE-NEXT:    it lt
 ; CHECK-BE-NEXT:    movlt r2, #1
 ; CHECK-BE-NEXT:    cmp r2, #0
 ; CHECK-BE-NEXT:    it ne
 ; CHECK-BE-NEXT:    mvnne r2, #1
 ; CHECK-BE-NEXT:    bfi r2, r12, #0, #1
 ; CHECK-BE-NEXT:    and r1, r2, #3
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    vmovne r2, r3, d0
-; CHECK-BE-NEXT:    strdne r3, r2, [r0]
-; CHECK-BE-NEXT:    lsls r1, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r2, #30
 ; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi r1, r2, d1
-; CHECK-BE-NEXT:    strdmi r2, r1, [r0, #8]
+; CHECK-BE-NEXT:    vmovmi r2, r3, d0
+; CHECK-BE-NEXT:    strdmi r3, r2, [r0]
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    vmovne r1, r2, d1
+; CHECK-BE-NEXT:    strdne r2, r1, [r0, #8]
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    pop {r7, pc}
 entry:
   %0 = load <2 x i32>, <2 x i32>* %mask, align 4
   %1 = icmp sgt <2 x i32> %0, zeroinitializer
   %2 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %src, i32 2, <2 x i1> %1, <2 x i32> undef)
   %3 = zext <2 x i32> %2 to <2 x i64>
   call void @llvm.masked.store.v2i64.p0v2i64(<2 x i64> %3, <2 x i64>* %dest, i32 4, <2 x i1> %1)
entry:
   %0 = load <8 x i16>, <8 x i16>* %mask, align 2
   %1 = icmp sgt <8 x i16> %0, zeroinitializer
   %2 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %src, i32 2, <8 x i1> %1, <8 x half> undef)
   call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %2, <8 x half>* %dest, i32 2, <8 x i1> %1)
   ret void
 }
 
 define void @foo_v4f32_v4f16(<4 x float> *%dest, <4 x i16> *%mask, <4 x half> *%src) {
-; CHECK-LABEL: foo_v4f32_v4f16:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r7, lr}
-; CHECK-NEXT:    push {r7, lr}
-; CHECK-NEXT:    .pad #8
-; CHECK-NEXT:    sub sp, #8
-; CHECK-NEXT:    vldrh.s32 q0, [r1]
-; CHECK-NEXT:    mov.w lr, #0
-; CHECK-NEXT:    @ implicit-def: $q1
-; CHECK-NEXT:    vcmp.s32 gt, q0, zr
-; CHECK-NEXT:    vmrs r3, p0
-; CHECK-NEXT:    and r1, r3, #1
-; CHECK-NEXT:    rsb.w r12, r1, #0
-; CHECK-NEXT:    ubfx r1, r3, #4, #1
-; CHECK-NEXT:    bfi lr, r12, #0, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi lr, r1, #1, #1
-; CHECK-NEXT:    ubfx r1, r3, #8, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi lr, r1, #2, #1
-; CHECK-NEXT:    ubfx r1, r3, #12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi lr, r1, #3, #1
-; CHECK-NEXT:    lsls.w r1, lr, #31
-; CHECK-NEXT:    beq .LBB18_2
-; CHECK-NEXT:  @ %bb.1: @ %cond.load
-; CHECK-NEXT:    vldr.16 s4, [r2]
-; CHECK-NEXT:  .LBB18_2: @ %else
-; CHECK-NEXT:    lsls.w r1, lr, #30
-; CHECK-NEXT:    bpl .LBB18_6
-; CHECK-NEXT:  @ %bb.3: @ %cond.load1
-; CHECK-NEXT:    vldr.16 s0, [r2, #2]
-; CHECK-NEXT:    vmov r3, s4
-; CHECK-NEXT:    vmovx.f16 s4, s5
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vmov.16 q0[0], r3
-; CHECK-NEXT:    vmov.16 q0[1], r1
-; CHECK-NEXT:    vmov r1, s5
-; CHECK-NEXT:    vmov.16 q0[2], r1
-; CHECK-NEXT:    vmov r1, s4
-; CHECK-NEXT:    vmov.16 q0[3], r1
-; CHECK-NEXT:    lsls.w r1, lr, #29
-; CHECK-NEXT:    bmi .LBB18_7
-; CHECK-NEXT:  .LBB18_4:
-; CHECK-NEXT:    vmov q1, q0
-; CHECK-NEXT:    lsls.w r1, lr, #28
-; CHECK-NEXT:    bmi .LBB18_8
-; CHECK-NEXT:  .LBB18_5:
-; CHECK-NEXT:    vmov q2, q1
-; CHECK-NEXT:    b .LBB18_9
-; CHECK-NEXT:  .LBB18_6:
-; CHECK-NEXT:    vmov q0, q1
-; CHECK-NEXT:    lsls.w r1, lr, #29
-; CHECK-NEXT:    bpl .LBB18_4
-; CHECK-NEXT:  .LBB18_7: @ %cond.load4
-; CHECK-NEXT:    vmovx.f16 s4, s0
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vldr.16 s8, [r2, #4]
-; CHECK-NEXT:    vmov r3, s4
-; CHECK-NEXT:    vmov.16 q1[0], r1
-; CHECK-NEXT:    vmovx.f16 s0, s1
-; CHECK-NEXT:    vmov.16 q1[1], r3
-; CHECK-NEXT:    vmov r1, s8
-; CHECK-NEXT:    vmov.16 q1[2], r1
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vmov.16 q1[3], r1
-; CHECK-NEXT:    lsls.w r1, lr, #28
-; CHECK-NEXT:    bpl .LBB18_5
-; CHECK-NEXT:  .LBB18_8: @ %cond.load7
-; CHECK-NEXT:    vmovx.f16 s0, s4
-; CHECK-NEXT:    vmov r3, s4
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vmov.16 q2[0], r3
-; CHECK-NEXT:    vldr.16 s0, [r2, #6]
-; CHECK-NEXT:    vmov.16 q2[1], r1
-; CHECK-NEXT:    vmov r1, s5
-; CHECK-NEXT:    vmov.16 q2[2], r1
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vmov.16 q2[3], r1
-; CHECK-NEXT:  .LBB18_9: @ %else8
-; CHECK-NEXT:    vmrs r2, p0
-; CHECK-NEXT:    movs r1, #0
-; CHECK-NEXT:    vcvtt.f32.f16 s3, s9
-; CHECK-NEXT:    vcvtb.f32.f16 s2, s9
-; CHECK-NEXT:    vcvtt.f32.f16 s1, s8
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s8
-; CHECK-NEXT:    and r3, r2, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r1, r3, #0, #1
-; CHECK-NEXT:    ubfx r3, r2, #4, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r1, r3, #1, #1
-; CHECK-NEXT:    ubfx r3, r2, #8, #1
-; CHECK-NEXT:    ubfx r2, r2, #12, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r1, r3, #2, #1
-; CHECK-NEXT:    rsbs r2, r2, #0
-; CHECK-NEXT:    bfi r1, r2, #3, #1
-; CHECK-NEXT:    lsls r2, r1, #31
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    vmovne r2, s0
-; CHECK-NEXT:    strne r2, [r0]
-; CHECK-NEXT:    lsls r2, r1, #30
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi r2, s1
-; CHECK-NEXT:    strmi r2, [r0, #4]
-; CHECK-NEXT:    lsls r2, r1, #29
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi r2, s2
-; CHECK-NEXT:    strmi r2, [r0, #8]
-; CHECK-NEXT:    lsls r1, r1, #28
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi r1, s3
-; CHECK-NEXT:    strmi r1, [r0, #12]
-; CHECK-NEXT:    add sp, #8
-; CHECK-NEXT:    pop {r7, pc}
+; CHECK-LE-LABEL: foo_v4f32_v4f16:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .save {r7, lr}
+; CHECK-LE-NEXT:    push {r7, lr}
+; CHECK-LE-NEXT:    .pad #8
+; CHECK-LE-NEXT:    sub sp, #8
+; CHECK-LE-NEXT:    vldrh.s32 q0, [r1]
+; CHECK-LE-NEXT:    mov.w lr, #0
+; CHECK-LE-NEXT:    @ implicit-def: $q1
+; CHECK-LE-NEXT:    vcmp.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vmrs r3, p0
+; CHECK-LE-NEXT:    and r1, r3, #1
+; CHECK-LE-NEXT:    rsb.w r12, r1, #0
+; CHECK-LE-NEXT:    ubfx r1, r3, #4, #1
+; CHECK-LE-NEXT:    bfi lr, r12, #0, #1
+; CHECK-LE-NEXT:    rsbs r1, r1, #0
+; CHECK-LE-NEXT:    bfi lr, r1, #1, #1
+; CHECK-LE-NEXT:    ubfx r1, r3, #8, #1
+; CHECK-LE-NEXT:    rsbs r1, r1, #0
+; CHECK-LE-NEXT:    bfi lr, r1, #2, #1
+; CHECK-LE-NEXT:    ubfx r1, r3, #12, #1
+; CHECK-LE-NEXT:    rsbs r1, r1, #0
+; CHECK-LE-NEXT:    bfi lr, r1, #3, #1
+; CHECK-LE-NEXT:    lsls.w r1, lr, #31
+; CHECK-LE-NEXT:    beq .LBB18_2
+; CHECK-LE-NEXT:  @ %bb.1: @ %cond.load
+; CHECK-LE-NEXT:    vldr.16 s4, [r2]
+; CHECK-LE-NEXT:  .LBB18_2: @ %else
+; CHECK-LE-NEXT:    lsls.w r1, lr, #30
+; CHECK-LE-NEXT:    bpl .LBB18_6
+; CHECK-LE-NEXT:  @ %bb.3: @ %cond.load1
+; CHECK-LE-NEXT:    vldr.16 s0, [r2, #2]
+; CHECK-LE-NEXT:    vmov r3, s4
+; CHECK-LE-NEXT:    vmovx.f16 s4, s5
+; CHECK-LE-NEXT:    vmov r1, s0
+; CHECK-LE-NEXT:    vmov.16 q0[0], r3
+; CHECK-LE-NEXT:    vmov.16 q0[1], r1
+; CHECK-LE-NEXT:    vmov r1, s5
+; CHECK-LE-NEXT:    vmov.16 q0[2], r1
+; CHECK-LE-NEXT:    vmov r1, s4
+; CHECK-LE-NEXT:    vmov.16 q0[3], r1
+; CHECK-LE-NEXT:    lsls.w r1, lr, #29
+; CHECK-LE-NEXT:    bmi .LBB18_7
+; CHECK-LE-NEXT:  .LBB18_4:
+; CHECK-LE-NEXT:    vmov q1, q0
+; CHECK-LE-NEXT:    lsls.w r1, lr, #28
+; CHECK-LE-NEXT:    bmi .LBB18_8
+; CHECK-LE-NEXT:  .LBB18_5:
+; CHECK-LE-NEXT:    vmov q2, q1
+; CHECK-LE-NEXT:    b .LBB18_9
+; CHECK-LE-NEXT:  .LBB18_6:
+; CHECK-LE-NEXT:    vmov q0, q1
+; CHECK-LE-NEXT:    lsls.w r1, lr, #29
+; CHECK-LE-NEXT:    bpl .LBB18_4
+; CHECK-LE-NEXT:  .LBB18_7: @ %cond.load4
+; CHECK-LE-NEXT:    vmovx.f16 s4, s0
+; CHECK-LE-NEXT:    vmov r1, s0
+; CHECK-LE-NEXT:    vldr.16 s8, [r2, #4]
+; CHECK-LE-NEXT:    vmov r3, s4
+; CHECK-LE-NEXT:    vmov.16 q1[0], r1
+; CHECK-LE-NEXT:    vmovx.f16 s0, s1
+; CHECK-LE-NEXT:    vmov.16 q1[1], r3
+; CHECK-LE-NEXT:    vmov r1, s8
+; CHECK-LE-NEXT:    vmov.16 q1[2], r1
+; CHECK-LE-NEXT:    vmov r1, s0
+; CHECK-LE-NEXT:    vmov.16 q1[3], r1
+; CHECK-LE-NEXT:    lsls.w r1, lr, #28
+; CHECK-LE-NEXT:    bpl .LBB18_5
+; CHECK-LE-NEXT:  .LBB18_8: @ %cond.load7
+; CHECK-LE-NEXT:    vmovx.f16 s0, s4
+; CHECK-LE-NEXT:    vmov r3, s4
+; CHECK-LE-NEXT:    vmov r1, s0
+; CHECK-LE-NEXT:    vmov.16 q2[0], r3
+; CHECK-LE-NEXT:    vldr.16 s0, [r2, #6]
+; CHECK-LE-NEXT:    vmov.16 q2[1], r1
+; CHECK-LE-NEXT:    vmov r1, s5
+; CHECK-LE-NEXT:    vmov.16 q2[2], r1
+; CHECK-LE-NEXT:    vmov r1, s0
+; CHECK-LE-NEXT:    vmov.16 q2[3], r1
+; CHECK-LE-NEXT:  .LBB18_9: @ %else8
+; CHECK-LE-NEXT:    vmrs r2, p0
+; CHECK-LE-NEXT:    movs r1, #0
+; CHECK-LE-NEXT:    vcvtt.f32.f16 s3, s9
+; CHECK-LE-NEXT:    vcvtb.f32.f16 s2, s9
+; CHECK-LE-NEXT:    vcvtt.f32.f16 s1, s8
+; CHECK-LE-NEXT:    vcvtb.f32.f16 s0, s8
+; CHECK-LE-NEXT:    and r3, r2, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #0, #1
+; CHECK-LE-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #1, #1
+; CHECK-LE-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-LE-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #2, #1
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    bfi r1, r2, #3, #1
+; CHECK-LE-NEXT:    lsls r2, r1, #31
+; CHECK-LE-NEXT:    itt ne
+; CHECK-LE-NEXT:    vmovne r2, s0
+; CHECK-LE-NEXT:    strne r2, [r0]
+; CHECK-LE-NEXT:    lsls r2, r1, #30
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    vmovmi r2, s1
+; CHECK-LE-NEXT:    strmi r2, [r0, #4]
+; CHECK-LE-NEXT:    lsls r2, r1, #29
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    vmovmi r2, s2
+; CHECK-LE-NEXT:    strmi r2, [r0, #8]
+; CHECK-LE-NEXT:    lsls r1, r1, #28
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    vmovmi r1, s3
+; CHECK-LE-NEXT:    strmi r1, [r0, #12]
+; CHECK-LE-NEXT:    add sp, #8
+; CHECK-LE-NEXT:    pop {r7, pc}
+;
+; CHECK-BE-LABEL: foo_v4f32_v4f16:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .save {r7, lr}
+; CHECK-BE-NEXT:    push {r7, lr}
+; CHECK-BE-NEXT:    .pad #8
+; CHECK-BE-NEXT:    sub sp, #8
+; CHECK-BE-NEXT:    vldrh.s32 q0, [r1]
+; CHECK-BE-NEXT:    mov.w lr, #0
+; CHECK-BE-NEXT:    @ implicit-def: $q1
+; CHECK-BE-NEXT:    vcmp.s32 gt, q0, zr
+; CHECK-BE-NEXT:    vmrs r3, p0
+; CHECK-BE-NEXT:    and r1, r3, #1
+; CHECK-BE-NEXT:    rsb.w r12, r1, #0
+; CHECK-BE-NEXT:    ubfx r1, r3, #4, #1
+; CHECK-BE-NEXT:    bfi lr, r12, #0, #1
+; CHECK-BE-NEXT:    rsbs r1, r1, #0
+; CHECK-BE-NEXT:    bfi lr, r1, #1, #1
+; CHECK-BE-NEXT:    ubfx r1, r3, #8, #1
+; CHECK-BE-NEXT:    rsbs r1, r1, #0
+; CHECK-BE-NEXT:    bfi lr, r1, #2, #1
+; CHECK-BE-NEXT:    ubfx r1, r3, #12, #1
+; CHECK-BE-NEXT:    rsbs r1, r1, #0
+; CHECK-BE-NEXT:    bfi lr, r1, #3, #1
+; CHECK-BE-NEXT:    lsls.w r1, lr, #28
+; CHECK-BE-NEXT:    bpl .LBB18_2
+; CHECK-BE-NEXT:  @ %bb.1: @ %cond.load
+; CHECK-BE-NEXT:    vldr.16 s4, [r2]
+; CHECK-BE-NEXT:  .LBB18_2: @ %else
+; CHECK-BE-NEXT:    lsls.w r1, lr, #29
+; CHECK-BE-NEXT:    bpl .LBB18_6
+; CHECK-BE-NEXT:  @ %bb.3: @ %cond.load1
+; CHECK-BE-NEXT:    vldr.16 s0, [r2, #2]
+; CHECK-BE-NEXT:    vmov r3, s4
+; CHECK-BE-NEXT:    vmovx.f16 s4, s5
+; CHECK-BE-NEXT:    vmov r1, s0
+; CHECK-BE-NEXT:    vmov.16 q0[0], r3
+; CHECK-BE-NEXT:    vmov.16 q0[1], r1
+; CHECK-BE-NEXT:    vmov r1, s5
+; CHECK-BE-NEXT:    vmov.16 q0[2], r1
+; CHECK-BE-NEXT:    vmov r1, s4
+; CHECK-BE-NEXT:    vmov.16 q0[3], r1
+; CHECK-BE-NEXT:    lsls.w r1, lr, #30
+; CHECK-BE-NEXT:    bmi .LBB18_7
+; CHECK-BE-NEXT:  .LBB18_4:
+; CHECK-BE-NEXT:    vmov q1, q0
+; CHECK-BE-NEXT:    lsls.w r1, lr, #31
+; CHECK-BE-NEXT:    bne .LBB18_8
+; CHECK-BE-NEXT:  .LBB18_5:
+; CHECK-BE-NEXT:    vmov q2, q1
+; CHECK-BE-NEXT:    b .LBB18_9
+; CHECK-BE-NEXT:  .LBB18_6:
+; CHECK-BE-NEXT:    vmov q0, q1
+; CHECK-BE-NEXT:    lsls.w r1, lr, #30
+; CHECK-BE-NEXT:    bpl .LBB18_4
+; CHECK-BE-NEXT:  .LBB18_7: @ %cond.load4
+; CHECK-BE-NEXT:    vmovx.f16 s4, s0
+; CHECK-BE-NEXT:    vmov r1, s0
+; CHECK-BE-NEXT:    vldr.16 s8, [r2, #4]
+; CHECK-BE-NEXT:    vmov r3, s4
+; CHECK-BE-NEXT:    vmov.16 q1[0], r1
+; CHECK-BE-NEXT:    vmovx.f16 s0, s1
+; CHECK-BE-NEXT:    vmov.16 q1[1], r3
+; CHECK-BE-NEXT:    vmov r1, s8
+; CHECK-BE-NEXT:    vmov.16 q1[2], r1
+; CHECK-BE-NEXT:    vmov r1, s0
+; CHECK-BE-NEXT:    vmov.16 q1[3], r1
+; CHECK-BE-NEXT:    lsls.w r1, lr, #31
+; CHECK-BE-NEXT:    beq .LBB18_5
+; CHECK-BE-NEXT:  .LBB18_8: @ %cond.load7
+; CHECK-BE-NEXT:    vmovx.f16 s0, s4
+; CHECK-BE-NEXT:    vmov r3, s4
+; CHECK-BE-NEXT:    vmov r1, s0
+; CHECK-BE-NEXT:    vmov.16 q2[0], r3
+; CHECK-BE-NEXT:    vldr.16 s0, [r2, #6]
+; CHECK-BE-NEXT:    vmov.16 q2[1], r1
+; CHECK-BE-NEXT:    vmov r1, s5
+; CHECK-BE-NEXT:    vmov.16 q2[2], r1
+; CHECK-BE-NEXT:    vmov r1, s0
+; CHECK-BE-NEXT:    vmov.16 q2[3], r1
+; CHECK-BE-NEXT:  .LBB18_9: @ %else8
+; CHECK-BE-NEXT:    vmrs r2, p0
+; CHECK-BE-NEXT:    movs r1, #0
+; CHECK-BE-NEXT:    vcvtt.f32.f16 s3, s9
+; CHECK-BE-NEXT:    vcvtb.f32.f16 s2, s9
+; CHECK-BE-NEXT:    vcvtt.f32.f16 s1, s8
+; CHECK-BE-NEXT:    vcvtb.f32.f16 s0, s8
+; CHECK-BE-NEXT:    and r3, r2, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #0, #1
+; CHECK-BE-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
+; CHECK-BE-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-BE-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
+; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    vmovmi r2, s0
+; CHECK-BE-NEXT:    strmi r2, [r0]
+; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    vmovmi r2, s1
+; CHECK-BE-NEXT:    strmi r2, [r0, #4]
+; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    vmovmi r2, s2
+; CHECK-BE-NEXT:    strmi r2, [r0, #8]
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    vmovne r1, s3
+; CHECK-BE-NEXT:    strne r1, [r0, #12]
+; CHECK-BE-NEXT:    add sp, #8
+; CHECK-BE-NEXT:    pop {r7, pc}
 entry:
   %0 = load <4 x i16>, <4 x i16>* %mask, align 2
   %1 = icmp sgt <4 x i16> %0, zeroinitializer
   %2 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* %src, i32 2, <4 x i1> %1, <4 x half> undef)
   %3 = fpext <4 x half> %2 to <4 x float>
   call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %3, <4 x float>* %dest, i32 2, <4 x i1> %1)
   ret void
 }
 
 define void @foo_v4f32_v4f16_unaligned(<4 x float> *%dest, <4 x i16> *%mask, <4 x half> *%src) {
-; CHECK-LABEL: foo_v4f32_v4f16_unaligned:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r7, lr}
-; CHECK-NEXT:    push {r7, lr}
-; CHECK-NEXT:    .pad #8
-; CHECK-NEXT:    sub sp, #8
-; CHECK-NEXT:    vldrh.s32 q0, [r1]
-; CHECK-NEXT:    mov.w lr, #0
-; CHECK-NEXT:    @ implicit-def: $q1
-; CHECK-NEXT:    vcmp.s32 gt, q0, zr
-; CHECK-NEXT:    vmrs r3, p0
-; CHECK-NEXT:    and r1, r3, #1
-; CHECK-NEXT:    rsb.w r12, r1, #0
-; CHECK-NEXT:    ubfx r1, r3, #4, #1
-; CHECK-NEXT:    bfi lr, r12, #0, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi lr, r1, #1, #1
-; CHECK-NEXT:    ubfx r1, r3, #8, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi lr, r1, #2, #1
-; CHECK-NEXT:    ubfx r1, r3, #12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi lr, r1, #3, #1
-; CHECK-NEXT:    lsls.w r1, lr, #31
-; CHECK-NEXT:    beq .LBB19_2
-; CHECK-NEXT:  @ %bb.1: @ %cond.load
-; CHECK-NEXT:    vldr.16 s4, [r2]
-; CHECK-NEXT:  .LBB19_2: @ %else
-; CHECK-NEXT:    lsls.w r1, lr, #30
-; CHECK-NEXT:    bpl .LBB19_6
-; CHECK-NEXT:  @ %bb.3: @ %cond.load1
-; CHECK-NEXT:    vldr.16 s0, [r2, #2]
-; CHECK-NEXT:    vmov r3, s4
-; CHECK-NEXT:    vmovx.f16 s4, s5
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vmov.16 q0[0], r3
-; CHECK-NEXT:    vmov.16 q0[1], r1
-; CHECK-NEXT:    vmov r1, s5
-; CHECK-NEXT:    vmov.16 q0[2], r1
-; CHECK-NEXT:    vmov r1, s4
-; CHECK-NEXT:    vmov.16 q0[3], r1
-; CHECK-NEXT:    lsls.w r1, lr, #29
-; CHECK-NEXT:    bmi .LBB19_7
-; CHECK-NEXT:  .LBB19_4:
-; CHECK-NEXT:    vmov q1, q0
-; CHECK-NEXT:    lsls.w r1, lr, #28
-; CHECK-NEXT:    bmi .LBB19_8
-; CHECK-NEXT:  .LBB19_5:
-; CHECK-NEXT:    vmov q2, q1
-; CHECK-NEXT:    b .LBB19_9
-; CHECK-NEXT:  .LBB19_6:
-; CHECK-NEXT:    vmov q0, q1
-; CHECK-NEXT:    lsls.w r1, lr, #29
-; CHECK-NEXT:    bpl .LBB19_4
-; CHECK-NEXT:  .LBB19_7: @ %cond.load4
-; CHECK-NEXT:    vmovx.f16 s4, s0
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vldr.16 s8, [r2, #4]
-; CHECK-NEXT:    vmov r3, s4
-; CHECK-NEXT:    vmov.16 q1[0], r1
-; CHECK-NEXT:    vmovx.f16 s0, s1
-; CHECK-NEXT:    vmov.16 q1[1], r3
-; CHECK-NEXT:    vmov r1, s8
-; CHECK-NEXT:    vmov.16 q1[2], r1
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vmov.16 q1[3], r1
-; CHECK-NEXT:    lsls.w r1, lr, #28
-; CHECK-NEXT:    bpl .LBB19_5
-; CHECK-NEXT:  .LBB19_8: @ %cond.load7
-; CHECK-NEXT:    vmovx.f16 s0, s4
-; CHECK-NEXT:    vmov r3, s4
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vmov.16 q2[0], r3
-; CHECK-NEXT:    vldr.16 s0, [r2, #6]
-; CHECK-NEXT:    vmov.16 q2[1], r1
-; CHECK-NEXT:    vmov r1, s5
-; CHECK-NEXT:    vmov.16 q2[2], r1
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vmov.16 q2[3], r1
-; CHECK-NEXT:  .LBB19_9: @ %else8
-; CHECK-NEXT:    vmrs r2, p0
-; CHECK-NEXT:    movs r1, #0
-; CHECK-NEXT:    vcvtt.f32.f16 s3, s9
-; CHECK-NEXT:    vcvtb.f32.f16 s2, s9
-; CHECK-NEXT:    vcvtt.f32.f16 s1, s8
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s8
-; CHECK-NEXT:    and r3, r2, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r1, r3, #0, #1
-; CHECK-NEXT:    ubfx r3, r2, #4, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r1, r3, #1, #1
-; CHECK-NEXT:    ubfx r3, r2, #8, #1
-; CHECK-NEXT:    ubfx r2, r2, #12, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r1, r3, #2, #1
-; CHECK-NEXT:    rsbs r2, r2, #0
-; CHECK-NEXT:    bfi r1, r2, #3, #1
-; CHECK-NEXT:    lsls r2, r1, #31
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    vmovne r2, s0
-; CHECK-NEXT:    strne r2, [r0]
-; CHECK-NEXT:    lsls r2, r1, #30
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi r2, s1
-; CHECK-NEXT:    strmi r2, [r0, #4]
-; CHECK-NEXT:    lsls r2, r1, #29
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi r2, s2
-; CHECK-NEXT:    strmi r2, [r0, #8]
-; CHECK-NEXT:    lsls r1, r1, #28
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi r1, s3
-; CHECK-NEXT:    strmi r1, [r0, #12]
-; CHECK-NEXT:    add sp, #8
-; CHECK-NEXT:    pop {r7, pc}
+; CHECK-LE-LABEL: foo_v4f32_v4f16_unaligned:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    .save {r7, lr}
+; CHECK-LE-NEXT:    push {r7, lr}
+; CHECK-LE-NEXT:    .pad #8
+; CHECK-LE-NEXT:    sub sp, #8
+; CHECK-LE-NEXT:    vldrh.s32 q0, [r1]
+; CHECK-LE-NEXT:    mov.w lr, #0
+; CHECK-LE-NEXT:    @ implicit-def: $q1
+; CHECK-LE-NEXT:    vcmp.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vmrs r3, p0
+; CHECK-LE-NEXT:    and r1, r3, #1
+; CHECK-LE-NEXT:    rsb.w r12, r1, #0
+; CHECK-LE-NEXT:    ubfx r1, r3, #4, #1
+; CHECK-LE-NEXT:    bfi lr, r12, #0, #1
+; CHECK-LE-NEXT:    rsbs r1, r1, #0
+; CHECK-LE-NEXT:    bfi lr, r1, #1, #1
+; CHECK-LE-NEXT:    ubfx r1, r3, #8, #1
+; CHECK-LE-NEXT:    rsbs r1, r1, #0
+; CHECK-LE-NEXT:    bfi lr, r1, #2, #1
+; CHECK-LE-NEXT:    ubfx r1, r3, #12, #1
+; CHECK-LE-NEXT:    rsbs r1, r1, #0
+; CHECK-LE-NEXT:    bfi lr, r1, #3, #1
+; CHECK-LE-NEXT:    lsls.w r1, lr, #31
+; CHECK-LE-NEXT:    beq .LBB19_2
+; CHECK-LE-NEXT:  @ %bb.1: @ %cond.load
+; CHECK-LE-NEXT:    vldr.16 s4, [r2]
+; CHECK-LE-NEXT:  .LBB19_2: @ %else
+; CHECK-LE-NEXT:    lsls.w r1, lr, #30
+; CHECK-LE-NEXT:    bpl .LBB19_6
+; CHECK-LE-NEXT:  @ %bb.3: @ %cond.load1
+; CHECK-LE-NEXT:    vldr.16 s0, [r2, #2]
+; CHECK-LE-NEXT:    vmov r3, s4
+; CHECK-LE-NEXT:    vmovx.f16 s4, s5
+; CHECK-LE-NEXT:    vmov r1, s0
+; CHECK-LE-NEXT:    vmov.16 q0[0], r3
+; CHECK-LE-NEXT:    vmov.16 q0[1], r1
+; CHECK-LE-NEXT:    vmov r1, s5
+; CHECK-LE-NEXT:    vmov.16 q0[2], r1
+; CHECK-LE-NEXT:    vmov r1, s4
+; CHECK-LE-NEXT:    vmov.16 q0[3], r1
+; CHECK-LE-NEXT:    lsls.w r1, lr, #29
+; CHECK-LE-NEXT:    bmi .LBB19_7
+; CHECK-LE-NEXT:  .LBB19_4:
+; CHECK-LE-NEXT:    vmov q1, q0
+; CHECK-LE-NEXT:    lsls.w r1, lr, #28
+; CHECK-LE-NEXT:    bmi .LBB19_8
+; CHECK-LE-NEXT:  .LBB19_5:
+; CHECK-LE-NEXT:    vmov q2, q1
+; CHECK-LE-NEXT:    b .LBB19_9
+; CHECK-LE-NEXT:  .LBB19_6:
+; CHECK-LE-NEXT:    vmov q0, q1
+; CHECK-LE-NEXT:    lsls.w r1, lr, #29
+; CHECK-LE-NEXT:    bpl .LBB19_4
+; CHECK-LE-NEXT:  .LBB19_7: @ %cond.load4
+; CHECK-LE-NEXT:    vmovx.f16 s4, s0
+; CHECK-LE-NEXT:    vmov r1, s0
+; CHECK-LE-NEXT:    vldr.16 s8, [r2, #4]
+; CHECK-LE-NEXT:    vmov r3, s4
+; CHECK-LE-NEXT:    vmov.16 q1[0], r1
+; CHECK-LE-NEXT:    vmovx.f16 s0, s1
+; CHECK-LE-NEXT:    vmov.16 q1[1], r3
+; CHECK-LE-NEXT:    vmov r1, s8
+; CHECK-LE-NEXT:    vmov.16 q1[2], r1
+; CHECK-LE-NEXT:    vmov r1, s0
+; CHECK-LE-NEXT:    vmov.16 q1[3], r1
+; CHECK-LE-NEXT:    lsls.w r1, lr, #28
+; CHECK-LE-NEXT:    bpl .LBB19_5
+; CHECK-LE-NEXT:  .LBB19_8: @ %cond.load7
+; CHECK-LE-NEXT:    vmovx.f16 s0, s4
+; CHECK-LE-NEXT:    vmov r3, s4
+; CHECK-LE-NEXT:    vmov r1, s0
+; CHECK-LE-NEXT:    vmov.16 q2[0], r3
+; CHECK-LE-NEXT:    vldr.16 s0, [r2, #6]
+; CHECK-LE-NEXT:    vmov.16 q2[1], r1
+; CHECK-LE-NEXT:    vmov r1, s5
+; CHECK-LE-NEXT:    vmov.16 q2[2], r1
+; CHECK-LE-NEXT:    vmov r1, s0
+; CHECK-LE-NEXT:    vmov.16 q2[3], r1
+; CHECK-LE-NEXT:  .LBB19_9: @ %else8
+; CHECK-LE-NEXT:    vmrs r2, p0
+; CHECK-LE-NEXT:    movs r1, #0
+; CHECK-LE-NEXT:    vcvtt.f32.f16 s3, s9
+; CHECK-LE-NEXT:    vcvtb.f32.f16 s2, s9
+; CHECK-LE-NEXT:    vcvtt.f32.f16 s1, s8
+; CHECK-LE-NEXT:    vcvtb.f32.f16 s0, s8
+; CHECK-LE-NEXT:    and r3, r2, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #0, #1
+; CHECK-LE-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #1, #1
+; CHECK-LE-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-LE-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    bfi r1, r3, #2, #1
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    bfi r1, r2, #3, #1
+; CHECK-LE-NEXT:    lsls r2, r1, #31
+; CHECK-LE-NEXT:    itt ne
+; CHECK-LE-NEXT:    vmovne r2, s0
+; CHECK-LE-NEXT:    strne r2, [r0]
+; CHECK-LE-NEXT:    lsls r2, r1, #30
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    vmovmi r2, s1
+; CHECK-LE-NEXT:    strmi r2, [r0, #4]
+; CHECK-LE-NEXT:    lsls r2, r1, #29
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    vmovmi r2, s2
+; CHECK-LE-NEXT:    strmi r2, [r0, #8]
+; CHECK-LE-NEXT:    lsls r1, r1, #28
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    vmovmi r1, s3
+; CHECK-LE-NEXT:    strmi r1, [r0, #12]
+; CHECK-LE-NEXT:    add sp, #8
+; CHECK-LE-NEXT:    pop {r7, pc}
+;
+; CHECK-BE-LABEL: foo_v4f32_v4f16_unaligned:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    .save {r7, lr}
+; CHECK-BE-NEXT:    push {r7, lr}
+; CHECK-BE-NEXT:    .pad #8
+; CHECK-BE-NEXT:    sub sp, #8
+; CHECK-BE-NEXT:    vldrh.s32 q0, [r1]
+; CHECK-BE-NEXT:    mov.w lr, #0
+; CHECK-BE-NEXT:    @ implicit-def: $q1
+; CHECK-BE-NEXT:    vcmp.s32 gt, q0, zr
+; CHECK-BE-NEXT:    vmrs r3, p0
+; CHECK-BE-NEXT:    and r1, r3, #1
+; CHECK-BE-NEXT:    rsb.w r12, r1, #0
+; CHECK-BE-NEXT:    ubfx r1, r3, #4, #1
+; CHECK-BE-NEXT:    bfi lr, r12, #0, #1
+; CHECK-BE-NEXT:    rsbs r1, r1, #0
+; CHECK-BE-NEXT:    bfi lr, r1, #1, #1
+; CHECK-BE-NEXT:    ubfx r1, r3, #8, #1
+; CHECK-BE-NEXT:    rsbs r1, r1, #0
+; CHECK-BE-NEXT:    bfi lr, r1, #2, #1
+; CHECK-BE-NEXT:    ubfx r1, r3, #12, #1
+; CHECK-BE-NEXT:    rsbs r1, r1, #0
+; CHECK-BE-NEXT:    bfi lr, r1, #3, #1
+; CHECK-BE-NEXT:    lsls.w r1, lr, #28
+; CHECK-BE-NEXT:    bpl .LBB19_2
+; CHECK-BE-NEXT:  @ %bb.1: @ %cond.load
+; CHECK-BE-NEXT:    vldr.16 s4, [r2]
+; CHECK-BE-NEXT:  .LBB19_2: @ %else
+; CHECK-BE-NEXT:    lsls.w r1, lr, #29
+; CHECK-BE-NEXT:    bpl .LBB19_6
+; CHECK-BE-NEXT:  @ %bb.3: @ %cond.load1
+; CHECK-BE-NEXT:    vldr.16 s0, [r2, #2]
+; CHECK-BE-NEXT:    vmov r3, s4
+; CHECK-BE-NEXT:    vmovx.f16 s4, s5
+; CHECK-BE-NEXT:    vmov r1, s0
+; CHECK-BE-NEXT:    vmov.16 q0[0], r3
+; CHECK-BE-NEXT:    vmov.16 q0[1], r1
+; CHECK-BE-NEXT:    vmov r1, s5
+; CHECK-BE-NEXT:    vmov.16 q0[2], r1
+; CHECK-BE-NEXT:    vmov r1, s4
+; CHECK-BE-NEXT:    vmov.16 q0[3], r1
+; CHECK-BE-NEXT:    lsls.w r1, lr, #30
+; CHECK-BE-NEXT:    bmi .LBB19_7
+; CHECK-BE-NEXT:  .LBB19_4:
+; CHECK-BE-NEXT:    vmov q1, q0
+; CHECK-BE-NEXT:    lsls.w r1, lr, #31
+; CHECK-BE-NEXT:    bne .LBB19_8
+; CHECK-BE-NEXT:  .LBB19_5:
+; CHECK-BE-NEXT:    vmov q2, q1
+; CHECK-BE-NEXT:    b .LBB19_9
+; CHECK-BE-NEXT:  .LBB19_6:
+; CHECK-BE-NEXT:    vmov q0, q1
+; CHECK-BE-NEXT:    lsls.w r1, lr, #30
+; CHECK-BE-NEXT:    bpl .LBB19_4
+; CHECK-BE-NEXT:  .LBB19_7: @ %cond.load4
+; CHECK-BE-NEXT:    vmovx.f16 s4, s0
+; CHECK-BE-NEXT:    vmov r1, s0
+; CHECK-BE-NEXT:    vldr.16 s8, [r2, #4]
+; CHECK-BE-NEXT:    vmov r3, s4
+; CHECK-BE-NEXT:    vmov.16 q1[0], r1
+; CHECK-BE-NEXT:    vmovx.f16 s0, s1
+; CHECK-BE-NEXT:    vmov.16 q1[1], r3
+; CHECK-BE-NEXT:    vmov r1, s8
+; CHECK-BE-NEXT:    vmov.16 q1[2], r1
+; CHECK-BE-NEXT:    vmov r1, s0
+; CHECK-BE-NEXT:    vmov.16 q1[3], r1
+; CHECK-BE-NEXT:    lsls.w r1, lr, #31
+; CHECK-BE-NEXT:    beq .LBB19_5
+; CHECK-BE-NEXT:  .LBB19_8: @ %cond.load7
+; CHECK-BE-NEXT:    vmovx.f16 s0, s4
+; CHECK-BE-NEXT:    vmov r3, s4
+; CHECK-BE-NEXT:    vmov r1, s0
+; CHECK-BE-NEXT:    vmov.16 q2[0], r3
+; CHECK-BE-NEXT:    vldr.16 s0, [r2, #6]
+; CHECK-BE-NEXT:    vmov.16 q2[1], r1
+; CHECK-BE-NEXT:    vmov r1, s5
+; CHECK-BE-NEXT:    vmov.16 q2[2], r1
+; CHECK-BE-NEXT:    vmov r1, s0
+; CHECK-BE-NEXT:    vmov.16 q2[3], r1
+; CHECK-BE-NEXT:  .LBB19_9: @ %else8
+; CHECK-BE-NEXT:    vmrs r2, p0
+; CHECK-BE-NEXT:    movs r1, #0
+; CHECK-BE-NEXT:    vcvtt.f32.f16 s3, s9
+; CHECK-BE-NEXT:    vcvtb.f32.f16 s2, s9
+; CHECK-BE-NEXT:    vcvtt.f32.f16 s1, s8
+; CHECK-BE-NEXT:    vcvtb.f32.f16 s0, s8
+; CHECK-BE-NEXT:    and r3, r2, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #0, #1
+; CHECK-BE-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
+; CHECK-BE-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-BE-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
+; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    vmovmi r2, s0
+; CHECK-BE-NEXT:    strmi r2, [r0]
+; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    vmovmi r2, s1
+; CHECK-BE-NEXT:    strmi r2, [r0, #4]
+; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    vmovmi r2, s2
+; CHECK-BE-NEXT:    strmi r2, [r0, #8]
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    vmovne r1, s3
+; CHECK-BE-NEXT:    strne r1, [r0, #12]
+; CHECK-BE-NEXT:    add sp, #8
+; CHECK-BE-NEXT:    pop {r7, pc}
 entry:
   %0 = load <4 x i16>, <4 x i16>* %mask, align 2
   %1 = icmp sgt <4 x i16> %0, zeroinitializer
   %2 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* %src, i32 2, <4 x i1> %1, <4 x half> undef)
   %3 = fpext <4 x half> %2 to <4 x float>
   call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %3, <4 x float>* %dest, i32 1, <4 x i1> %1)
   ret void
 }

llvm/test/CodeGen/Thumb2/mve-masked-load.ll

 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
 ; CHECK-BE-NEXT:    ubfx r3, r2, #8, #1
 ; CHECK-BE-NEXT:    ubfx r2, r2, #12, #1
 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-BE-NEXT:    rsbs r2, r2, #0
 ; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    ldrne r2, [r0]
-; CHECK-BE-NEXT:    vmovne.32 q1[0], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    ldrmi r2, [r0]
+; CHECK-BE-NEXT:    vmovmi.32 q1[0], r2
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    ldrmi r2, [r0, #4]
 ; CHECK-BE-NEXT:    vmovmi.32 q1[1], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    ldrmi r2, [r0, #8]
 ; CHECK-BE-NEXT:    vmovmi.32 q1[2], r2
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrmi r0, [r0, #12]
-; CHECK-BE-NEXT:    vmovmi.32 q1[3], r0
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    ldrne r0, [r0, #12]
+; CHECK-BE-NEXT:    vmovne.32 q1[3], r0
 ; CHECK-BE-NEXT:    vrev64.32 q0, q1
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp sgt <4 x i32> %a, zeroinitializer
   %l = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %dest, i32 1, <4 x i1> %c, <4 x i32> undef)
   ret <4 x i32> %l
 }

 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
 ; CHECK-BE-NEXT:    ubfx r3, r2, #8, #1
 ; CHECK-BE-NEXT:    ubfx r2, r2, #12, #1
 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-BE-NEXT:    rsbs r2, r2, #0
 ; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    ldrhne r2, [r0]
-; CHECK-BE-NEXT:    vmovne.32 q0[0], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    ldrhmi r2, [r0]
+; CHECK-BE-NEXT:    vmovmi.32 q0[0], r2
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    ldrhmi r2, [r0, #2]
 ; CHECK-BE-NEXT:    vmovmi.32 q0[1], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    ldrhmi r2, [r0, #4]
 ; CHECK-BE-NEXT:    vmovmi.32 q0[2], r2
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r0, [r0, #6]
-; CHECK-BE-NEXT:    vmovmi.32 q0[3], r0
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    ldrhne r0, [r0, #6]
+; CHECK-BE-NEXT:    vmovne.32 q0[3], r0
 ; CHECK-BE-NEXT:    vmovlb.s16 q1, q0
 ; CHECK-BE-NEXT:    vrev64.32 q0, q1
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp sgt <4 x i32> %a, zeroinitializer
   %l = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %dest, i32 1, <4 x i1> %c, <4 x i16> undef)
   %ext = sext <4 x i16> %l to <4 x i32>

 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
 ; CHECK-BE-NEXT:    ubfx r3, r2, #8, #1
 ; CHECK-BE-NEXT:    ubfx r2, r2, #12, #1
 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-BE-NEXT:    rsbs r2, r2, #0
 ; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    ldrhne r2, [r0]
-; CHECK-BE-NEXT:    vmovne.32 q0[0], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    ldrhmi r2, [r0]
+; CHECK-BE-NEXT:    vmovmi.32 q0[0], r2
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    ldrhmi r2, [r0, #2]
 ; CHECK-BE-NEXT:    vmovmi.32 q0[1], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    ldrhmi r2, [r0, #4]
 ; CHECK-BE-NEXT:    vmovmi.32 q0[2], r2
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r0, [r0, #6]
-; CHECK-BE-NEXT:    vmovmi.32 q0[3], r0
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    ldrhne r0, [r0, #6]
+; CHECK-BE-NEXT:    vmovne.32 q0[3], r0
 ; CHECK-BE-NEXT:    vmovlb.s16 q1, q0
 ; CHECK-BE-NEXT:    vrev64.32 q0, q1
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp sgt <4 x i32> %a, zeroinitializer
   %l = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %dest, i32 1, <4 x i1> %c, <4 x i16> undef)
   %sext = sext <4 x i16> %l to <4 x i32>

 ; CHECK-BE-NEXT:    bfi r2, r3, #5, #1
 ; CHECK-BE-NEXT:    ubfx r3, r1, #12, #1
 ; CHECK-BE-NEXT:    ubfx r1, r1, #14, #1
 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r2, r3, #6, #1
 ; CHECK-BE-NEXT:    rsbs r1, r1, #0
 ; CHECK-BE-NEXT:    bfi r2, r1, #7, #1
 ; CHECK-BE-NEXT:    uxtb r1, r2
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    ldrhne r2, [r0]
-; CHECK-BE-NEXT:    vmovne.16 q1[0], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r2, #24
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    ldrhmi r2, [r0]
+; CHECK-BE-NEXT:    vmovmi.16 q1[0], r2
+; CHECK-BE-NEXT:    lsls r2, r1, #25
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    ldrhmi r2, [r0, #2]
 ; CHECK-BE-NEXT:    vmovmi.16 q1[1], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #26
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    ldrhmi r2, [r0, #4]
 ; CHECK-BE-NEXT:    vmovmi.16 q1[2], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    lsls r2, r1, #27
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    ldrhmi r2, [r0, #6]
 ; CHECK-BE-NEXT:    vmovmi.16 q1[3], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #27
+; CHECK-BE-NEXT:    lsls r2, r1, #28
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    ldrhmi r2, [r0, #8]
 ; CHECK-BE-NEXT:    vmovmi.16 q1[4], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #26
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    ldrhmi r2, [r0, #10]
 ; CHECK-BE-NEXT:    vmovmi.16 q1[5], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #25
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    ldrhmi r2, [r0, #12]
 ; CHECK-BE-NEXT:    vmovmi.16 q1[6], r2
-; CHECK-BE-NEXT:    lsls r1, r1, #24
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r0, [r0, #14]
-; CHECK-BE-NEXT:    vmovmi.16 q1[7], r0
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    ldrhne r0, [r0, #14]
+; CHECK-BE-NEXT:    vmovne.16 q1[7], r0
 ; CHECK-BE-NEXT:    vrev64.16 q0, q1
 ; CHECK-BE-NEXT:    add sp, #8
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp sgt <8 x i16> %a, zeroinitializer
   %l = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %dest, i32 1, <8 x i1> %c, <8 x i16> undef)
   ret <8 x i16> %l
 }

 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
 ; CHECK-BE-NEXT:    ubfx r3, r2, #8, #1
 ; CHECK-BE-NEXT:    ubfx r2, r2, #12, #1
 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-BE-NEXT:    rsbs r2, r2, #0
 ; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    ldrne r2, [r0]
-; CHECK-BE-NEXT:    vmovne s4, r2
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    ldrmi r2, [r0]
+; CHECK-BE-NEXT:    vmovmi s4, r2
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    ldrmi r2, [r0, #4]
 ; CHECK-BE-NEXT:    vmovmi s5, r2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    ldrmi r2, [r0, #8]
 ; CHECK-BE-NEXT:    vmovmi s6, r2
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrmi r0, [r0, #12]
-; CHECK-BE-NEXT:    vmovmi s7, r0
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    ldrne r0, [r0, #12]
+; CHECK-BE-NEXT:    vmovne s7, r0
 ; CHECK-BE-NEXT:    vrev64.32 q0, q1
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp sgt <4 x i32> %a, zeroinitializer
   %l = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %dest, i32 1, <4 x i1> %c, <4 x float> undef)
   ret <4 x float> %l
 }

 ; CHECK-BE-NEXT:    bfi r2, r3, #5, #1
 ; CHECK-BE-NEXT:    ubfx r3, r1, #12, #1
 ; CHECK-BE-NEXT:    ubfx r1, r1, #14, #1
 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r2, r3, #6, #1
 ; CHECK-BE-NEXT:    rsbs r1, r1, #0
 ; CHECK-BE-NEXT:    bfi r2, r1, #7, #1
 ; CHECK-BE-NEXT:    uxtb r1, r2
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    bne .LBB45_10
+; CHECK-BE-NEXT:    lsls r2, r2, #24
+; CHECK-BE-NEXT:    bmi .LBB45_10
 ; CHECK-BE-NEXT:  @ %bb.1: @ %else
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #25
 ; CHECK-BE-NEXT:    bmi .LBB45_11
 ; CHECK-BE-NEXT:  .LBB45_2: @ %else2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #26
 ; CHECK-BE-NEXT:    bmi .LBB45_12
 ; CHECK-BE-NEXT:  .LBB45_3: @ %else5
-; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    lsls r2, r1, #27
 ; CHECK-BE-NEXT:    bmi .LBB45_13
 ; CHECK-BE-NEXT:  .LBB45_4: @ %else8
-; CHECK-BE-NEXT:    lsls r2, r1, #27
+; CHECK-BE-NEXT:    lsls r2, r1, #28
 ; CHECK-BE-NEXT:    bmi .LBB45_14
 ; CHECK-BE-NEXT:  .LBB45_5: @ %else11
-; CHECK-BE-NEXT:    lsls r2, r1, #26
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    bmi .LBB45_15
 ; CHECK-BE-NEXT:  .LBB45_6: @ %else14
-; CHECK-BE-NEXT:    lsls r2, r1, #25
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    bmi .LBB45_16
 ; CHECK-BE-NEXT:  .LBB45_7: @ %else17
-; CHECK-BE-NEXT:    lsls r1, r1, #24
-; CHECK-BE-NEXT:    bpl .LBB45_9
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    beq .LBB45_9
 ; CHECK-BE-NEXT:  .LBB45_8: @ %cond.load19
 ; CHECK-BE-NEXT:    ldrh r0, [r0, #14]
 ; CHECK-BE-NEXT:    strh.w r0, [sp]
 ; CHECK-BE-NEXT:    vldr.16 s0, [sp]
 ; CHECK-BE-NEXT:    vmov r0, s0
 ; CHECK-BE-NEXT:    vmov.16 q1[7], r0
 ; CHECK-BE-NEXT:  .LBB45_9: @ %else20
 ; CHECK-BE-NEXT:    vrev64.16 q0, q1
 ; CHECK-BE-NEXT:    add sp, #40
 ; CHECK-BE-NEXT:    bx lr
 ; CHECK-BE-NEXT:  .LBB45_10: @ %cond.load
 ; CHECK-BE-NEXT:    ldrh r2, [r0]
 ; CHECK-BE-NEXT:    strh.w r2, [sp, #28]
 ; CHECK-BE-NEXT:    vldr.16 s4, [sp, #28]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #25
 ; CHECK-BE-NEXT:    bpl .LBB45_2
 ; CHECK-BE-NEXT:  .LBB45_11: @ %cond.load1
 ; CHECK-BE-NEXT:    ldrh r2, [r0, #2]
 ; CHECK-BE-NEXT:    strh.w r2, [sp, #24]
 ; CHECK-BE-NEXT:    vldr.16 s0, [sp, #24]
 ; CHECK-BE-NEXT:    vmov r2, s0
 ; CHECK-BE-NEXT:    vmov.16 q1[1], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #26
 ; CHECK-BE-NEXT:    bpl .LBB45_3
 ; CHECK-BE-NEXT:  .LBB45_12: @ %cond.load4
 ; CHECK-BE-NEXT:    ldrh r2, [r0, #4]
 ; CHECK-BE-NEXT:    strh.w r2, [sp, #20]
 ; CHECK-BE-NEXT:    vldr.16 s0, [sp, #20]
 ; CHECK-BE-NEXT:    vmov r2, s0
 ; CHECK-BE-NEXT:    vmov.16 q1[2], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    lsls r2, r1, #27
 ; CHECK-BE-NEXT:    bpl .LBB45_4
 ; CHECK-BE-NEXT:  .LBB45_13: @ %cond.load7
 ; CHECK-BE-NEXT:    ldrh r2, [r0, #6]
 ; CHECK-BE-NEXT:    strh.w r2, [sp, #16]
 ; CHECK-BE-NEXT:    vldr.16 s0, [sp, #16]
 ; CHECK-BE-NEXT:    vmov r2, s0
 ; CHECK-BE-NEXT:    vmov.16 q1[3], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #27
+; CHECK-BE-NEXT:    lsls r2, r1, #28
 ; CHECK-BE-NEXT:    bpl .LBB45_5
 ; CHECK-BE-NEXT:  .LBB45_14: @ %cond.load10
 ; CHECK-BE-NEXT:    ldrh r2, [r0, #8]
 ; CHECK-BE-NEXT:    strh.w r2, [sp, #12]
 ; CHECK-BE-NEXT:    vldr.16 s0, [sp, #12]
 ; CHECK-BE-NEXT:    vmov r2, s0
 ; CHECK-BE-NEXT:    vmov.16 q1[4], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #26
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    bpl .LBB45_6
 ; CHECK-BE-NEXT:  .LBB45_15: @ %cond.load13
 ; CHECK-BE-NEXT:    ldrh r2, [r0, #10]
 ; CHECK-BE-NEXT:    strh.w r2, [sp, #8]
 ; CHECK-BE-NEXT:    vldr.16 s0, [sp, #8]
 ; CHECK-BE-NEXT:    vmov r2, s0
 ; CHECK-BE-NEXT:    vmov.16 q1[5], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #25
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    bpl .LBB45_7
 ; CHECK-BE-NEXT:  .LBB45_16: @ %cond.load16
 ; CHECK-BE-NEXT:    ldrh r2, [r0, #12]
 ; CHECK-BE-NEXT:    strh.w r2, [sp, #4]
 ; CHECK-BE-NEXT:    vldr.16 s0, [sp, #4]
 ; CHECK-BE-NEXT:    vmov r2, s0
 ; CHECK-BE-NEXT:    vmov.16 q1[6], r2
-; CHECK-BE-NEXT:    lsls r1, r1, #24
-; CHECK-BE-NEXT:    bmi .LBB45_8
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    bne .LBB45_8
 ; CHECK-BE-NEXT:    b .LBB45_9
 entry:
   %c = icmp sgt <8 x i16> %a, zeroinitializer
   %l = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %dest, i32 1, <8 x i1> %c, <8 x half> undef)
   ret <8 x half> %l
 }
 
 define arm_aapcs_vfpcc <8 x half> @masked_v8f16_align4_other(<8 x half> *%dest, <8 x i16> %a, <8 x half> %b) {

 ; CHECK-BE-NEXT:    sbcs.w r3, r2, r12
 ; CHECK-BE-NEXT:    it lt
 ; CHECK-BE-NEXT:    movlt r2, #1
 ; CHECK-BE-NEXT:    cmp r2, #0
 ; CHECK-BE-NEXT:    it ne
 ; CHECK-BE-NEXT:    mvnne r2, #1
 ; CHECK-BE-NEXT:    bfi r2, r1, #0, #1
 ; CHECK-BE-NEXT:    and r1, r2, #3
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    beq .LBB49_2
+; CHECK-BE-NEXT:    lsls r2, r2, #30
+; CHECK-BE-NEXT:    bpl .LBB49_2
 ; CHECK-BE-NEXT:  @ %bb.1: @ %cond.load
 ; CHECK-BE-NEXT:    vldr d1, .LCPI49_0
 ; CHECK-BE-NEXT:    vldr d0, [r0]
 ; CHECK-BE-NEXT:    b .LBB49_3
 ; CHECK-BE-NEXT:  .LBB49_2:
 ; CHECK-BE-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-BE-NEXT:  .LBB49_3: @ %else
-; CHECK-BE-NEXT:    lsls r1, r1, #30
-; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vldrmi d1, [r0, #8]
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    it ne
+; CHECK-BE-NEXT:    vldrne d1, [r0, #8]
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 ; CHECK-BE-NEXT:    .p2align 3
 ; CHECK-BE-NEXT:  @ %bb.4:
 ; CHECK-BE-NEXT:  .LCPI49_0:
 ; CHECK-BE-NEXT:    .long 0 @ double 0
 ; CHECK-BE-NEXT:    .long 0
 entry:

 ; CHECK-BE-NEXT:    sbcs.w r3, r2, r12
 ; CHECK-BE-NEXT:    it lt
 ; CHECK-BE-NEXT:    movlt r2, #1
 ; CHECK-BE-NEXT:    cmp r2, #0
 ; CHECK-BE-NEXT:    it ne
 ; CHECK-BE-NEXT:    mvnne r2, #1
 ; CHECK-BE-NEXT:    bfi r2, r1, #0, #1
 ; CHECK-BE-NEXT:    and r1, r2, #3
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    beq .LBB50_2
+; CHECK-BE-NEXT:    lsls r2, r2, #30
+; CHECK-BE-NEXT:    bpl .LBB50_2
 ; CHECK-BE-NEXT:  @ %bb.1: @ %cond.load
 ; CHECK-BE-NEXT:    vldr d1, .LCPI50_0
 ; CHECK-BE-NEXT:    vldr d0, [r0]
 ; CHECK-BE-NEXT:    b .LBB50_3
 ; CHECK-BE-NEXT:  .LBB50_2:
 ; CHECK-BE-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-BE-NEXT:  .LBB50_3: @ %else
-; CHECK-BE-NEXT:    lsls r1, r1, #30
-; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vldrmi d1, [r0, #8]
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    it ne
+; CHECK-BE-NEXT:    vldrne d1, [r0, #8]
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 ; CHECK-BE-NEXT:    .p2align 3
 ; CHECK-BE-NEXT:  @ %bb.4:
 ; CHECK-BE-NEXT:  .LCPI50_0:
 ; CHECK-BE-NEXT:    .long 0 @ double 0
 ; CHECK-BE-NEXT:    .long 0
 entry:

 ; CHECK-BE-NEXT:    rsbs r2, r2, #0
 ; CHECK-BE-NEXT:    bfi r1, r2, #1, #1
 ; CHECK-BE-NEXT:    ubfx r2, r3, #8, #1
 ; CHECK-BE-NEXT:    rsbs r2, r2, #0
 ; CHECK-BE-NEXT:    bfi r1, r2, #2, #1
 ; CHECK-BE-NEXT:    ubfx r2, r3, #12, #1
 ; CHECK-BE-NEXT:    rsbs r2, r2, #0
 ; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    beq .LBB52_2
+; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    bpl .LBB52_2
 ; CHECK-BE-NEXT:  @ %bb.1: @ %cond.load
 ; CHECK-BE-NEXT:    ldrh r2, [r0]
 ; CHECK-BE-NEXT:    vdup.32 q1, r12
 ; CHECK-BE-NEXT:    vmov.32 q1[0], r2
 ; CHECK-BE-NEXT:    b .LBB52_3
 ; CHECK-BE-NEXT:  .LBB52_2:
 ; CHECK-BE-NEXT:    vmov.i32 q1, #0x0
 ; CHECK-BE-NEXT:  .LBB52_3: @ %else
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    ldrhmi r2, [r0, #2]
 ; CHECK-BE-NEXT:    vmovmi.32 q1[1], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    ldrhmi r2, [r0, #4]
 ; CHECK-BE-NEXT:    vmovmi.32 q1[2], r2
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r0, [r0, #6]
-; CHECK-BE-NEXT:    vmovmi.32 q1[3], r0
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    ldrhne r0, [r0, #6]
+; CHECK-BE-NEXT:    vmovne.32 q1[3], r0
 ; CHECK-BE-NEXT:    vrev64.32 q0, q1
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp sgt <4 x i32> %a, zeroinitializer
   %l = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %dest, i32 1, <4 x i1> %c, <4 x i16> zeroinitializer)
   ret <4 x i16> %l
 }

llvm/test/CodeGen/Thumb2/mve-masked-store.ll

 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
 ; CHECK-BE-NEXT:    ubfx r3, r2, #8, #1
 ; CHECK-BE-NEXT:    ubfx r2, r2, #12, #1
 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-BE-NEXT:    rsbs r2, r2, #0
 ; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    vmovne r2, s4
-; CHECK-BE-NEXT:    strne r2, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    vmovmi r2, s4
+; CHECK-BE-NEXT:    strmi r2, [r0]
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    vmovmi r2, s5
 ; CHECK-BE-NEXT:    strmi r2, [r0, #4]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    vmovmi r2, s6
 ; CHECK-BE-NEXT:    strmi r2, [r0, #8]
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi r1, s7
-; CHECK-BE-NEXT:    strmi r1, [r0, #12]
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    vmovne r1, s7
+; CHECK-BE-NEXT:    strne r1, [r0, #12]
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp sgt <4 x i32> %a, zeroinitializer
   call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %a, <4 x i32>* %dest, i32 1, <4 x i1> %c)
   ret void
 }
 

 ; CHECK-BE-NEXT:    bfi r2, r3, #5, #1
 ; CHECK-BE-NEXT:    ubfx r3, r1, #12, #1
 ; CHECK-BE-NEXT:    ubfx r1, r1, #14, #1
 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r2, r3, #6, #1
 ; CHECK-BE-NEXT:    rsbs r1, r1, #0
 ; CHECK-BE-NEXT:    bfi r2, r1, #7, #1
 ; CHECK-BE-NEXT:    uxtb r1, r2
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    vmovne.u16 r2, q1[0]
-; CHECK-BE-NEXT:    strhne r2, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r2, #24
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    vmovmi.u16 r2, q1[0]
+; CHECK-BE-NEXT:    strhmi r2, [r0]
+; CHECK-BE-NEXT:    lsls r2, r1, #25
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    vmovmi.u16 r2, q1[1]
 ; CHECK-BE-NEXT:    strhmi r2, [r0, #2]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #26
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    vmovmi.u16 r2, q1[2]
 ; CHECK-BE-NEXT:    strhmi r2, [r0, #4]
-; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    lsls r2, r1, #27
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    vmovmi.u16 r2, q1[3]
 ; CHECK-BE-NEXT:    strhmi r2, [r0, #6]
-; CHECK-BE-NEXT:    lsls r2, r1, #27
+; CHECK-BE-NEXT:    lsls r2, r1, #28
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    vmovmi.u16 r2, q1[4]
 ; CHECK-BE-NEXT:    strhmi r2, [r0, #8]
-; CHECK-BE-NEXT:    lsls r2, r1, #26
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    vmovmi.u16 r2, q1[5]
 ; CHECK-BE-NEXT:    strhmi r2, [r0, #10]
-; CHECK-BE-NEXT:    lsls r2, r1, #25
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    vmovmi.u16 r2, q1[6]
 ; CHECK-BE-NEXT:    strhmi r2, [r0, #12]
-; CHECK-BE-NEXT:    lsls r1, r1, #24
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u16 r1, q1[7]
-; CHECK-BE-NEXT:    strhmi r1, [r0, #14]
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    vmovne.u16 r1, q1[7]
+; CHECK-BE-NEXT:    strhne r1, [r0, #14]
 ; CHECK-BE-NEXT:    add sp, #8
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp sgt <8 x i16> %a, zeroinitializer
   call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %a, <8 x i16>* %dest, i32 1, <8 x i1> %c)
   ret void
 }
 

 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
 ; CHECK-BE-NEXT:    ubfx r3, r2, #8, #1
 ; CHECK-BE-NEXT:    ubfx r2, r2, #12, #1
 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-BE-NEXT:    rsbs r2, r2, #0
 ; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    vmovne r2, s4
-; CHECK-BE-NEXT:    strne r2, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    vmovmi r2, s4
+; CHECK-BE-NEXT:    strmi r2, [r0]
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    vmovmi r2, s5
 ; CHECK-BE-NEXT:    strmi r2, [r0, #4]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    vmovmi r2, s6
 ; CHECK-BE-NEXT:    strmi r2, [r0, #8]
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi r1, s7
-; CHECK-BE-NEXT:    strmi r1, [r0, #12]
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    vmovne r1, s7
+; CHECK-BE-NEXT:    strne r1, [r0, #12]
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp ugt <4 x i32> %b, zeroinitializer
   call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %a, <4 x float>* %dest, i32 1, <4 x i1> %c)
   ret void
 }
 

 ; CHECK-BE-NEXT:    bfi r2, r3, #5, #1
 ; CHECK-BE-NEXT:    ubfx r3, r1, #12, #1
 ; CHECK-BE-NEXT:    ubfx r1, r1, #14, #1
 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r2, r3, #6, #1
 ; CHECK-BE-NEXT:    rsbs r1, r1, #0
 ; CHECK-BE-NEXT:    bfi r2, r1, #7, #1
 ; CHECK-BE-NEXT:    uxtb r1, r2
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    bne .LBB16_9
+; CHECK-BE-NEXT:    lsls r2, r2, #24
+; CHECK-BE-NEXT:    bmi .LBB16_9
 ; CHECK-BE-NEXT:  @ %bb.1: @ %else
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #25
 ; CHECK-BE-NEXT:    bmi .LBB16_10
 ; CHECK-BE-NEXT:  .LBB16_2: @ %else2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #26
 ; CHECK-BE-NEXT:    bmi .LBB16_11
 ; CHECK-BE-NEXT:  .LBB16_3: @ %else4
-; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    lsls r2, r1, #27
 ; CHECK-BE-NEXT:    bmi .LBB16_12
 ; CHECK-BE-NEXT:  .LBB16_4: @ %else6
-; CHECK-BE-NEXT:    lsls r2, r1, #27
+; CHECK-BE-NEXT:    lsls r2, r1, #28
 ; CHECK-BE-NEXT:    bmi .LBB16_13
 ; CHECK-BE-NEXT:  .LBB16_5: @ %else8
-; CHECK-BE-NEXT:    lsls r2, r1, #26
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    bmi .LBB16_14
 ; CHECK-BE-NEXT:  .LBB16_6: @ %else10
-; CHECK-BE-NEXT:    lsls r2, r1, #25
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    bmi .LBB16_15
 ; CHECK-BE-NEXT:  .LBB16_7: @ %else12
-; CHECK-BE-NEXT:    lsls r1, r1, #24
-; CHECK-BE-NEXT:    bmi .LBB16_16
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    bne .LBB16_16
 ; CHECK-BE-NEXT:  .LBB16_8: @ %else14
 ; CHECK-BE-NEXT:    add sp, #40
 ; CHECK-BE-NEXT:    bx lr
 ; CHECK-BE-NEXT:  .LBB16_9: @ %cond.store
 ; CHECK-BE-NEXT:    vstr.16 s4, [sp, #28]
 ; CHECK-BE-NEXT:    ldrh.w r2, [sp, #28]
 ; CHECK-BE-NEXT:    strh r2, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #25
 ; CHECK-BE-NEXT:    bpl .LBB16_2
 ; CHECK-BE-NEXT:  .LBB16_10: @ %cond.store1
 ; CHECK-BE-NEXT:    vmovx.f16 s0, s4
 ; CHECK-BE-NEXT:    vstr.16 s0, [sp, #24]
 ; CHECK-BE-NEXT:    ldrh.w r2, [sp, #24]
 ; CHECK-BE-NEXT:    strh r2, [r0, #2]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #26
 ; CHECK-BE-NEXT:    bpl .LBB16_3
 ; CHECK-BE-NEXT:  .LBB16_11: @ %cond.store3
 ; CHECK-BE-NEXT:    vstr.16 s5, [sp, #20]
 ; CHECK-BE-NEXT:    ldrh.w r2, [sp, #20]
 ; CHECK-BE-NEXT:    strh r2, [r0, #4]
-; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    lsls r2, r1, #27
 ; CHECK-BE-NEXT:    bpl .LBB16_4
 ; CHECK-BE-NEXT:  .LBB16_12: @ %cond.store5
 ; CHECK-BE-NEXT:    vmovx.f16 s0, s5
 ; CHECK-BE-NEXT:    vstr.16 s0, [sp, #16]
 ; CHECK-BE-NEXT:    ldrh.w r2, [sp, #16]
 ; CHECK-BE-NEXT:    strh r2, [r0, #6]
-; CHECK-BE-NEXT:    lsls r2, r1, #27
+; CHECK-BE-NEXT:    lsls r2, r1, #28
 ; CHECK-BE-NEXT:    bpl .LBB16_5
 ; CHECK-BE-NEXT:  .LBB16_13: @ %cond.store7
 ; CHECK-BE-NEXT:    vstr.16 s6, [sp, #12]
 ; CHECK-BE-NEXT:    ldrh.w r2, [sp, #12]
 ; CHECK-BE-NEXT:    strh r2, [r0, #8]
-; CHECK-BE-NEXT:    lsls r2, r1, #26
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    bpl .LBB16_6
 ; CHECK-BE-NEXT:  .LBB16_14: @ %cond.store9
 ; CHECK-BE-NEXT:    vmovx.f16 s0, s6
 ; CHECK-BE-NEXT:    vstr.16 s0, [sp, #8]
 ; CHECK-BE-NEXT:    ldrh.w r2, [sp, #8]
 ; CHECK-BE-NEXT:    strh r2, [r0, #10]
-; CHECK-BE-NEXT:    lsls r2, r1, #25
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    bpl .LBB16_7
 ; CHECK-BE-NEXT:  .LBB16_15: @ %cond.store11
 ; CHECK-BE-NEXT:    vstr.16 s7, [sp, #4]
 ; CHECK-BE-NEXT:    ldrh.w r2, [sp, #4]
 ; CHECK-BE-NEXT:    strh r2, [r0, #12]
-; CHECK-BE-NEXT:    lsls r1, r1, #24
-; CHECK-BE-NEXT:    bpl .LBB16_8
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    beq .LBB16_8
 ; CHECK-BE-NEXT:  .LBB16_16: @ %cond.store13
 ; CHECK-BE-NEXT:    vmovx.f16 s0, s7
 ; CHECK-BE-NEXT:    vstr.16 s0, [sp]
 ; CHECK-BE-NEXT:    ldrh.w r1, [sp]
 ; CHECK-BE-NEXT:    strh r1, [r0, #14]
 ; CHECK-BE-NEXT:    add sp, #40
 ; CHECK-BE-NEXT:    bx lr
 entry:

 ; CHECK-BE-NEXT:    sbcs.w r2, r3, r12
 ; CHECK-BE-NEXT:    it lt
 ; CHECK-BE-NEXT:    movlt r3, #1
 ; CHECK-BE-NEXT:    cmp r3, #0
 ; CHECK-BE-NEXT:    it ne
 ; CHECK-BE-NEXT:    mvnne r3, #1
 ; CHECK-BE-NEXT:    bfi r3, r1, #0, #1
 ; CHECK-BE-NEXT:    and r1, r3, #3
-; CHECK-BE-NEXT:    lsls r2, r3, #31
-; CHECK-BE-NEXT:    it ne
-; CHECK-BE-NEXT:    vstrne d0, [r0]
-; CHECK-BE-NEXT:    lsls r1, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r3, #30
 ; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vstrmi d1, [r0, #8]
+; CHECK-BE-NEXT:    vstrmi d0, [r0]
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    it ne
+; CHECK-BE-NEXT:    vstrne d1, [r0, #8]
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp sgt <2 x i64> %a, zeroinitializer
   call void @llvm.masked.store.v2i64.p0v2i64(<2 x i64> %a, <2 x i64>* %dest, i32 8, <2 x i1> %c)
   ret void
 }
 

 ; CHECK-BE-NEXT:    sbcs.w r2, r3, r12
 ; CHECK-BE-NEXT:    it lt
 ; CHECK-BE-NEXT:    movlt r3, #1
 ; CHECK-BE-NEXT:    cmp r3, #0
 ; CHECK-BE-NEXT:    it ne
 ; CHECK-BE-NEXT:    mvnne r3, #1
 ; CHECK-BE-NEXT:    bfi r3, r1, #0, #1
 ; CHECK-BE-NEXT:    and r1, r3, #3
-; CHECK-BE-NEXT:    lsls r2, r3, #31
-; CHECK-BE-NEXT:    it ne
-; CHECK-BE-NEXT:    vstrne d0, [r0]
-; CHECK-BE-NEXT:    lsls r1, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r3, #30
 ; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vstrmi d1, [r0, #8]
+; CHECK-BE-NEXT:    vstrmi d0, [r0]
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    it ne
+; CHECK-BE-NEXT:    vstrne d1, [r0, #8]
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp sgt <2 x i64> %b, zeroinitializer
   call void @llvm.masked.store.v2f64.p0v2f64(<2 x double> %a, <2 x double>* %dest, i32 8, <2 x i1> %c)
   ret void
 }
 

 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
 ; CHECK-BE-NEXT:    ubfx r3, r2, #8, #1
 ; CHECK-BE-NEXT:    ubfx r2, r2, #12, #1
 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-BE-NEXT:    rsbs r2, r2, #0
 ; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    vmovne r2, s4
-; CHECK-BE-NEXT:    strhne r2, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    vmovmi r2, s4
+; CHECK-BE-NEXT:    strhmi r2, [r0]
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    vmovmi r2, s5
 ; CHECK-BE-NEXT:    strhmi r2, [r0, #2]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    vmovmi r2, s6
 ; CHECK-BE-NEXT:    strhmi r2, [r0, #4]
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi r1, s7
-; CHECK-BE-NEXT:    strhmi r1, [r0, #6]
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    vmovne r1, s7
+; CHECK-BE-NEXT:    strhne r1, [r0, #6]
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp sgt <4 x i32> %a, zeroinitializer
   %trunc = trunc <4 x i32> %a to <4 x i16>
   call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %trunc, <4 x i16>* %dest, i32 1, <4 x i1> %c)
   ret void
 }

 ; CHECK-BE-NEXT:    and r2, r2, #1
 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-BE-NEXT:    rsbs r2, r2, #0
 ; CHECK-BE-NEXT:    vcvtt.f16.f32 s0, s5
 ; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
 ; CHECK-BE-NEXT:    vcvtb.f16.f32 s1, s6
 ; CHECK-BE-NEXT:    vcvtt.f16.f32 s1, s7
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    bne .LBB25_5
+; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    bmi .LBB25_5
 ; CHECK-BE-NEXT:  @ %bb.1: @ %else
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    bmi .LBB25_6
 ; CHECK-BE-NEXT:  .LBB25_2: @ %else2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    bmi .LBB25_7
 ; CHECK-BE-NEXT:  .LBB25_3: @ %else4
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    bmi .LBB25_8
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    bne .LBB25_8
 ; CHECK-BE-NEXT:  .LBB25_4: @ %else6
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 ; CHECK-BE-NEXT:  .LBB25_5: @ %cond.store
 ; CHECK-BE-NEXT:    vstr.16 s0, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    bpl .LBB25_2
 ; CHECK-BE-NEXT:  .LBB25_6: @ %cond.store1
 ; CHECK-BE-NEXT:    vmovx.f16 s4, s0
 ; CHECK-BE-NEXT:    vstr.16 s4, [r0, #2]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    bpl .LBB25_3
 ; CHECK-BE-NEXT:  .LBB25_7: @ %cond.store3
 ; CHECK-BE-NEXT:    vstr.16 s1, [r0, #4]
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    bpl .LBB25_4
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    beq .LBB25_4
 ; CHECK-BE-NEXT:  .LBB25_8: @ %cond.store5
 ; CHECK-BE-NEXT:    vmovx.f16 s0, s1
 ; CHECK-BE-NEXT:    vstr.16 s0, [r0, #6]
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = fcmp ogt <4 x float> %a, zeroinitializer
   %trunc = fptrunc <4 x float> %a to <4 x half>

 ; CHECK-BE-NEXT:    and r2, r2, #1
 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-BE-NEXT:    rsbs r2, r2, #0
 ; CHECK-BE-NEXT:    vcvtt.f16.f32 s0, s5
 ; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
 ; CHECK-BE-NEXT:    vcvtb.f16.f32 s1, s6
 ; CHECK-BE-NEXT:    vcvtt.f16.f32 s1, s7
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    bne .LBB26_5
+; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    bmi .LBB26_5
 ; CHECK-BE-NEXT:  @ %bb.1: @ %else
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    bmi .LBB26_6
 ; CHECK-BE-NEXT:  .LBB26_2: @ %else2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    bmi .LBB26_7
 ; CHECK-BE-NEXT:  .LBB26_3: @ %else4
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    bmi .LBB26_8
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    bne .LBB26_8
 ; CHECK-BE-NEXT:  .LBB26_4: @ %else6
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 ; CHECK-BE-NEXT:  .LBB26_5: @ %cond.store
 ; CHECK-BE-NEXT:    vstr.16 s0, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    bpl .LBB26_2
 ; CHECK-BE-NEXT:  .LBB26_6: @ %cond.store1
 ; CHECK-BE-NEXT:    vmovx.f16 s4, s0
 ; CHECK-BE-NEXT:    vstr.16 s4, [r0, #2]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    bpl .LBB26_3
 ; CHECK-BE-NEXT:  .LBB26_7: @ %cond.store3
 ; CHECK-BE-NEXT:    vstr.16 s1, [r0, #4]
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    bpl .LBB26_4
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    beq .LBB26_4
 ; CHECK-BE-NEXT:  .LBB26_8: @ %cond.store5
 ; CHECK-BE-NEXT:    vmovx.f16 s0, s1
 ; CHECK-BE-NEXT:    vstr.16 s0, [r0, #6]
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = fcmp ogt <4 x float> %a, zeroinitializer
   %trunc = fptrunc <4 x float> %a to <4 x half>

 ; CHECK-BE-NEXT:    and r2, r2, #1
 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-BE-NEXT:    rsbs r2, r2, #0
 ; CHECK-BE-NEXT:    vcvtt.f16.f32 s0, s5
 ; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
 ; CHECK-BE-NEXT:    vcvtb.f16.f32 s1, s6
 ; CHECK-BE-NEXT:    vcvtt.f16.f32 s1, s7
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    bne .LBB27_5
+; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    bmi .LBB27_5
 ; CHECK-BE-NEXT:  @ %bb.1: @ %else
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    bmi .LBB27_6
 ; CHECK-BE-NEXT:  .LBB27_2: @ %else2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    bmi .LBB27_7
 ; CHECK-BE-NEXT:  .LBB27_3: @ %else4
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    bmi .LBB27_8
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    bne .LBB27_8
 ; CHECK-BE-NEXT:  .LBB27_4: @ %else6
 ; CHECK-BE-NEXT:    add sp, #20
 ; CHECK-BE-NEXT:    bx lr
 ; CHECK-BE-NEXT:  .LBB27_5: @ %cond.store
 ; CHECK-BE-NEXT:    vstr.16 s0, [sp, #12]
 ; CHECK-BE-NEXT:    ldrh.w r2, [sp, #12]
 ; CHECK-BE-NEXT:    strh r2, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    bpl .LBB27_2
 ; CHECK-BE-NEXT:  .LBB27_6: @ %cond.store1
 ; CHECK-BE-NEXT:    vmovx.f16 s4, s0
 ; CHECK-BE-NEXT:    vstr.16 s4, [sp, #8]
 ; CHECK-BE-NEXT:    ldrh.w r2, [sp, #8]
 ; CHECK-BE-NEXT:    strh r2, [r0, #2]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    bpl .LBB27_3
 ; CHECK-BE-NEXT:  .LBB27_7: @ %cond.store3
 ; CHECK-BE-NEXT:    vstr.16 s1, [sp, #4]
 ; CHECK-BE-NEXT:    ldrh.w r2, [sp, #4]
 ; CHECK-BE-NEXT:    strh r2, [r0, #4]
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    bpl .LBB27_4
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    beq .LBB27_4
 ; CHECK-BE-NEXT:  .LBB27_8: @ %cond.store5
 ; CHECK-BE-NEXT:    vmovx.f16 s0, s1
 ; CHECK-BE-NEXT:    vstr.16 s0, [sp]
 ; CHECK-BE-NEXT:    ldrh.w r1, [sp]
 ; CHECK-BE-NEXT:    strh r1, [r0, #6]
 ; CHECK-BE-NEXT:    add sp, #20
 ; CHECK-BE-NEXT:    bx lr
 entry:

llvm/test/Transforms/ScalarizeMaskedMemIntrin/AArch64/expand-masked-load.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S %s -scalarize-masked-mem-intrin -mtriple=aarch64-linux-gnu | FileCheck %s
-; RUN: opt -S %s -scalarize-masked-mem-intrin -mtriple=aarch64-linux-gnu -mattr=+sve | FileCheck %s
+; RUN: opt -S %s -scalarize-masked-mem-intrin -mtriple=aarch64-linux-gnu | FileCheck -check-prefixes=CHECK,CHECK-LE %s
+; RUN: opt -S %s -scalarize-masked-mem-intrin -mtriple=aarch64-linux-gnu -mattr=+sve  | FileCheck -check-prefixes=CHECK,CHECK-LE %s
+; RUN: opt -S %s -scalarize-masked-mem-intrin -mtriple=aarch64_be-linux-gnu -data-layout="E-m:o-i64:64-i128:128-n32:64-S128" | FileCheck -check-prefixes=CHECK,CHECK-BE %s
 
 define <2 x i64> @scalarize_v2i64(<2 x i64>* %p, <2 x i1> %mask, <2 x i64> %passthru) {
-; CHECK-LABEL: @scalarize_v2i64(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64>* [[P:%.*]] to i64*
-; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
-; CHECK-NEXT:    [[TMP2:%.*]] = and i2 [[SCALAR_MASK]], 1
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne i2 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[TMP3]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
-; CHECK:       cond.load:
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 0
-; CHECK-NEXT:    [[TMP5:%.*]] = load i64, i64* [[TMP4]], align 8
-; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP5]], i64 0
-; CHECK-NEXT:    br label [[ELSE]]
-; CHECK:       else:
-; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[TMP6]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
-; CHECK-NEXT:    [[TMP7:%.*]] = and i2 [[SCALAR_MASK]], -2
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne i2 [[TMP7]], 0
-; CHECK-NEXT:    br i1 [[TMP8]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
-; CHECK:       cond.load1:
-; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 1
-; CHECK-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP9]], align 8
-; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP10]], i64 1
-; CHECK-NEXT:    br label [[ELSE2]]
-; CHECK:       else2:
-; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP11]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
-; CHECK-NEXT:    ret <2 x i64> [[RES_PHI_ELSE3]]
+; CHECK-LE-LABEL: @scalarize_v2i64(
+; CHECK-LE-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64>* [[P:%.*]] to i64*
+; CHECK-LE-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-LE-NEXT:    [[TMP2:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-LE-NEXT:    [[TMP3:%.*]] = icmp ne i2 [[TMP2]], 0
+; CHECK-LE-NEXT:    br i1 [[TMP3]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
+; CHECK-LE:       cond.load:
+; CHECK-LE-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 0
+; CHECK-LE-NEXT:    [[TMP5:%.*]] = load i64, i64* [[TMP4]], align 8
+; CHECK-LE-NEXT:    [[TMP6:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP5]], i64 0
+; CHECK-LE-NEXT:    br label [[ELSE]]
+; CHECK-LE:       else:
+; CHECK-LE-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[TMP6]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-LE-NEXT:    [[TMP7:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-LE-NEXT:    [[TMP8:%.*]] = icmp ne i2 [[TMP7]], 0
+; CHECK-LE-NEXT:    br i1 [[TMP8]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
+; CHECK-LE:       cond.load1:
+; CHECK-LE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 1
+; CHECK-LE-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP9]], align 8
+; CHECK-LE-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP10]], i64 1
+; CHECK-LE-NEXT:    br label [[ELSE2]]
+; CHECK-LE:       else2:
+; CHECK-LE-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP11]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
+; CHECK-LE-NEXT:    ret <2 x i64> [[RES_PHI_ELSE3]]
+;
+; CHECK-BE-LABEL: @scalarize_v2i64(
+; CHECK-BE-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64>* [[P:%.*]] to i64*
+; CHECK-BE-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-BE-NEXT:    [[TMP2:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-BE-NEXT:    [[TMP3:%.*]] = icmp ne i2 [[TMP2]], 0
+; CHECK-BE-NEXT:    br i1 [[TMP3]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
+; CHECK-BE:       cond.load:
+; CHECK-BE-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 0
+; CHECK-BE-NEXT:    [[TMP5:%.*]] = load i64, i64* [[TMP4]], align 8
+; CHECK-BE-NEXT:    [[TMP6:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP5]], i64 0
+; CHECK-BE-NEXT:    br label [[ELSE]]
+; CHECK-BE:       else:
+; CHECK-BE-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[TMP6]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-BE-NEXT:    [[TMP7:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-BE-NEXT:    [[TMP8:%.*]] = icmp ne i2 [[TMP7]], 0
+; CHECK-BE-NEXT:    br i1 [[TMP8]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
+; CHECK-BE:       cond.load1:
+; CHECK-BE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 1
+; CHECK-BE-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP9]], align 8
+; CHECK-BE-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP10]], i64 1
+; CHECK-BE-NEXT:    br label [[ELSE2]]
+; CHECK-BE:       else2:
+; CHECK-BE-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP11]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
+; CHECK-BE-NEXT:    ret <2 x i64> [[RES_PHI_ELSE3]]
 ;
   %ret = call <2 x i64> @llvm.masked.load.v2i64.p0v2i64(<2 x i64>* %p, i32 128, <2 x i1> %mask, <2 x i64> %passthru)
   ret <2 x i64> %ret
 }
 
 define <2 x i64> @scalarize_v2i64_ones_mask(<2 x i64>* %p, <2 x i64> %passthru) {
 ; CHECK-LABEL: @scalarize_v2i64_ones_mask(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* [[P:%.*]], align 8

 ; CHECK-NEXT:    ret <2 x i64> [[TMP4]]
 ;
   %ret = call <2 x i64> @llvm.masked.load.v2i64.p0v2i64(<2 x i64>* %p, i32 8, <2 x i1> <i1 false, i1 true>, <2 x i64> %passthru)
   ret <2 x i64> %ret
 }
 
 ; This use a byte sized but non power of 2 element size. This used to crash due to bad alignment calculation.
 define <2 x i24> @scalarize_v2i24(<2 x i24>* %p, <2 x i1> %mask, <2 x i24> %passthru) {
-; CHECK-LABEL: @scalarize_v2i24(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i24>* [[P:%.*]] to i24*
-; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
-; CHECK-NEXT:    [[TMP2:%.*]] = and i2 [[SCALAR_MASK]], 1
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne i2 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[TMP3]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
-; CHECK:       cond.load:
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i24, i24* [[TMP1]], i32 0
-; CHECK-NEXT:    [[TMP5:%.*]] = load i24, i24* [[TMP4]], align 1
-; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i24> [[PASSTHRU:%.*]], i24 [[TMP5]], i64 0
-; CHECK-NEXT:    br label [[ELSE]]
-; CHECK:       else:
-; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i24> [ [[TMP6]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
-; CHECK-NEXT:    [[TMP7:%.*]] = and i2 [[SCALAR_MASK]], -2
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne i2 [[TMP7]], 0
-; CHECK-NEXT:    br i1 [[TMP8]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
-; CHECK:       cond.load1:
-; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i24, i24* [[TMP1]], i32 1
-; CHECK-NEXT:    [[TMP10:%.*]] = load i24, i24* [[TMP9]], align 1
-; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i24> [[RES_PHI_ELSE]], i24 [[TMP10]], i64 1
-; CHECK-NEXT:    br label [[ELSE2]]
-; CHECK:       else2:
-; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i24> [ [[TMP11]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
-; CHECK-NEXT:    ret <2 x i24> [[RES_PHI_ELSE3]]
+; CHECK-LE-LABEL: @scalarize_v2i24(
+; CHECK-LE-NEXT:    [[TMP1:%.*]] = bitcast <2 x i24>* [[P:%.*]] to i24*
+; CHECK-LE-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-LE-NEXT:    [[TMP2:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-LE-NEXT:    [[TMP3:%.*]] = icmp ne i2 [[TMP2]], 0
+; CHECK-LE-NEXT:    br i1 [[TMP3]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
+; CHECK-LE:       cond.load:
+; CHECK-LE-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i24, i24* [[TMP1]], i32 0
+; CHECK-LE-NEXT:    [[TMP5:%.*]] = load i24, i24* [[TMP4]], align 1
+; CHECK-LE-NEXT:    [[TMP6:%.*]] = insertelement <2 x i24> [[PASSTHRU:%.*]], i24 [[TMP5]], i64 0
+; CHECK-LE-NEXT:    br label [[ELSE]]
+; CHECK-LE:       else:
+; CHECK-LE-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i24> [ [[TMP6]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-LE-NEXT:    [[TMP7:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-LE-NEXT:    [[TMP8:%.*]] = icmp ne i2 [[TMP7]], 0
+; CHECK-LE-NEXT:    br i1 [[TMP8]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
+; CHECK-LE:       cond.load1:
+; CHECK-LE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i24, i24* [[TMP1]], i32 1
+; CHECK-LE-NEXT:    [[TMP10:%.*]] = load i24, i24* [[TMP9]], align 1
+; CHECK-LE-NEXT:    [[TMP11:%.*]] = insertelement <2 x i24> [[RES_PHI_ELSE]], i24 [[TMP10]], i64 1
+; CHECK-LE-NEXT:    br label [[ELSE2]]
+; CHECK-LE:       else2:
+; CHECK-LE-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i24> [ [[TMP11]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
+; CHECK-LE-NEXT:    ret <2 x i24> [[RES_PHI_ELSE3]]
+;
+; CHECK-BE-LABEL: @scalarize_v2i24(
+; CHECK-BE-NEXT:    [[TMP1:%.*]] = bitcast <2 x i24>* [[P:%.*]] to i24*
+; CHECK-BE-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-BE-NEXT:    [[TMP2:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-BE-NEXT:    [[TMP3:%.*]] = icmp ne i2 [[TMP2]], 0
+; CHECK-BE-NEXT:    br i1 [[TMP3]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
+; CHECK-BE:       cond.load:
+; CHECK-BE-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i24, i24* [[TMP1]], i32 0
+; CHECK-BE-NEXT:    [[TMP5:%.*]] = load i24, i24* [[TMP4]], align 1
+; CHECK-BE-NEXT:    [[TMP6:%.*]] = insertelement <2 x i24> [[PASSTHRU:%.*]], i24 [[TMP5]], i64 0
+; CHECK-BE-NEXT:    br label [[ELSE]]
+; CHECK-BE:       else:
+; CHECK-BE-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i24> [ [[TMP6]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-BE-NEXT:    [[TMP7:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-BE-NEXT:    [[TMP8:%.*]] = icmp ne i2 [[TMP7]], 0
+; CHECK-BE-NEXT:    br i1 [[TMP8]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
+; CHECK-BE:       cond.load1:
+; CHECK-BE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i24, i24* [[TMP1]], i32 1
+; CHECK-BE-NEXT:    [[TMP10:%.*]] = load i24, i24* [[TMP9]], align 1
+; CHECK-BE-NEXT:    [[TMP11:%.*]] = insertelement <2 x i24> [[RES_PHI_ELSE]], i24 [[TMP10]], i64 1
+; CHECK-BE-NEXT:    br label [[ELSE2]]
+; CHECK-BE:       else2:
+; CHECK-BE-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i24> [ [[TMP11]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
+; CHECK-BE-NEXT:    ret <2 x i24> [[RES_PHI_ELSE3]]
 ;
   %ret = call <2 x i24> @llvm.masked.load.v2i24.p0v2i24(<2 x i24>* %p, i32 8, <2 x i1> %mask, <2 x i24> %passthru)
   ret <2 x i24> %ret
 }
 
 ; This use a byte sized but non power of 2 element size. This used to crash due to bad alignment calculation.
 define <2 x i48> @scalarize_v2i48(<2 x i48>* %p, <2 x i1> %mask, <2 x i48> %passthru) {
-; CHECK-LABEL: @scalarize_v2i48(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i48>* [[P:%.*]] to i48*
-; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
-; CHECK-NEXT:    [[TMP2:%.*]] = and i2 [[SCALAR_MASK]], 1
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne i2 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[TMP3]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
-; CHECK:       cond.load:
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i48, i48* [[TMP1]], i32 0
-; CHECK-NEXT:    [[TMP5:%.*]] = load i48, i48* [[TMP4]], align 2
-; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i48> [[PASSTHRU:%.*]], i48 [[TMP5]], i64 0
-; CHECK-NEXT:    br label [[ELSE]]
-; CHECK:       else:
-; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i48> [ [[TMP6]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
-; CHECK-NEXT:    [[TMP7:%.*]] = and i2 [[SCALAR_MASK]], -2
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne i2 [[TMP7]], 0
-; CHECK-NEXT:    br i1 [[TMP8]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
-; CHECK:       cond.load1:
-; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i48, i48* [[TMP1]], i32 1
-; CHECK-NEXT:    [[TMP10:%.*]] = load i48, i48* [[TMP9]], align 2
-; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i48> [[RES_PHI_ELSE]], i48 [[TMP10]], i64 1
-; CHECK-NEXT:    br label [[ELSE2]]
-; CHECK:       else2:
-; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i48> [ [[TMP11]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
-; CHECK-NEXT:    ret <2 x i48> [[RES_PHI_ELSE3]]
+; CHECK-LE-LABEL: @scalarize_v2i48(
+; CHECK-LE-NEXT:    [[TMP1:%.*]] = bitcast <2 x i48>* [[P:%.*]] to i48*
+; CHECK-LE-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-LE-NEXT:    [[TMP2:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-LE-NEXT:    [[TMP3:%.*]] = icmp ne i2 [[TMP2]], 0
+; CHECK-LE-NEXT:    br i1 [[TMP3]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
+; CHECK-LE:       cond.load:
+; CHECK-LE-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i48, i48* [[TMP1]], i32 0
+; CHECK-LE-NEXT:    [[TMP5:%.*]] = load i48, i48* [[TMP4]], align 2
+; CHECK-LE-NEXT:    [[TMP6:%.*]] = insertelement <2 x i48> [[PASSTHRU:%.*]], i48 [[TMP5]], i64 0
+; CHECK-LE-NEXT:    br label [[ELSE]]
+; CHECK-LE:       else:
+; CHECK-LE-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i48> [ [[TMP6]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-LE-NEXT:    [[TMP7:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-LE-NEXT:    [[TMP8:%.*]] = icmp ne i2 [[TMP7]], 0
+; CHECK-LE-NEXT:    br i1 [[TMP8]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
+; CHECK-LE:       cond.load1:
+; CHECK-LE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i48, i48* [[TMP1]], i32 1
+; CHECK-LE-NEXT:    [[TMP10:%.*]] = load i48, i48* [[TMP9]], align 2
+; CHECK-LE-NEXT:    [[TMP11:%.*]] = insertelement <2 x i48> [[RES_PHI_ELSE]], i48 [[TMP10]], i64 1
+; CHECK-LE-NEXT:    br label [[ELSE2]]
+; CHECK-LE:       else2:
+; CHECK-LE-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i48> [ [[TMP11]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
+; CHECK-LE-NEXT:    ret <2 x i48> [[RES_PHI_ELSE3]]
+;
+; CHECK-BE-LABEL: @scalarize_v2i48(
+; CHECK-BE-NEXT:    [[TMP1:%.*]] = bitcast <2 x i48>* [[P:%.*]] to i48*
+; CHECK-BE-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-BE-NEXT:    [[TMP2:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-BE-NEXT:    [[TMP3:%.*]] = icmp ne i2 [[TMP2]], 0
+; CHECK-BE-NEXT:    br i1 [[TMP3]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
+; CHECK-BE:       cond.load:
+; CHECK-BE-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i48, i48* [[TMP1]], i32 0
+; CHECK-BE-NEXT:    [[TMP5:%.*]] = load i48, i48* [[TMP4]], align 2
+; CHECK-BE-NEXT:    [[TMP6:%.*]] = insertelement <2 x i48> [[PASSTHRU:%.*]], i48 [[TMP5]], i64 0
+; CHECK-BE-NEXT:    br label [[ELSE]]
+; CHECK-BE:       else:
+; CHECK-BE-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i48> [ [[TMP6]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-BE-NEXT:    [[TMP7:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-BE-NEXT:    [[TMP8:%.*]] = icmp ne i2 [[TMP7]], 0
+; CHECK-BE-NEXT:    br i1 [[TMP8]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
+; CHECK-BE:       cond.load1:
+; CHECK-BE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i48, i48* [[TMP1]], i32 1
+; CHECK-BE-NEXT:    [[TMP10:%.*]] = load i48, i48* [[TMP9]], align 2
+; CHECK-BE-NEXT:    [[TMP11:%.*]] = insertelement <2 x i48> [[RES_PHI_ELSE]], i48 [[TMP10]], i64 1
+; CHECK-BE-NEXT:    br label [[ELSE2]]
+; CHECK-BE:       else2:
+; CHECK-BE-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i48> [ [[TMP11]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
+; CHECK-BE-NEXT:    ret <2 x i48> [[RES_PHI_ELSE3]]
 ;
   %ret = call <2 x i48> @llvm.masked.load.v2i48.p0v2i48(<2 x i48>* %p, i32 16, <2 x i1> %mask, <2 x i48> %passthru)
   ret <2 x i48> %ret
 }
 
 declare <2 x i24> @llvm.masked.load.v2i24.p0v2i24(<2 x i24>*, i32, <2 x i1>, <2 x i24>)
 declare <2 x i48> @llvm.masked.load.v2i48.p0v2i48(<2 x i48>*, i32, <2 x i1>, <2 x i48>)
 declare <2 x i64> @llvm.masked.load.v2i64.p0v2i64(<2 x i64>*, i32, <2 x i1>, <2 x i64>)

llvm/test/Transforms/ScalarizeMaskedMemIntrin/AArch64/expand-masked-store.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S %s -scalarize-masked-mem-intrin -mtriple=aarch64-linux-gnu | FileCheck %s
-; RUN: opt -S %s -scalarize-masked-mem-intrin -mtriple=aarch64-linux-gnu -mattr=+sve | FileCheck %s
+; RUN: opt -S %s -scalarize-masked-mem-intrin -mtriple=aarch64-linux-gnu | FileCheck -check-prefixes=CHECK,CHECK-LE %s
+; RUN: opt -S %s -scalarize-masked-mem-intrin -mtriple=aarch64-linux-gnu -mattr=+sve | FileCheck -check-prefixes=CHECK,CHECK-LE %s
+; RUN: opt -S %s -scalarize-masked-mem-intrin -mtriple=aarch64_be-linux-gnu -data-layout="E-m:o-i64:64-i128:128-n32:64-S128" | FileCheck -check-prefixes=CHECK,CHECK-BE %s
 
 define void @scalarize_v2i64(<2 x i64>* %p, <2 x i1> %mask, <2 x i64> %data) {
-; CHECK-LABEL: @scalarize_v2i64(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64>* [[P:%.*]] to i64*
-; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
-; CHECK-NEXT:    [[TMP2:%.*]] = and i2 [[SCALAR_MASK]], 1
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne i2 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[TMP3]], label [[COND_STORE:%.*]], label [[ELSE:%.*]]
-; CHECK:       cond.store:
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[DATA:%.*]], i64 0
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 0
-; CHECK-NEXT:    store i64 [[TMP4]], i64* [[TMP5]], align 8
-; CHECK-NEXT:    br label [[ELSE]]
-; CHECK:       else:
-; CHECK-NEXT:    [[TMP6:%.*]] = and i2 [[SCALAR_MASK]], -2
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i2 [[TMP6]], 0
-; CHECK-NEXT:    br i1 [[TMP7]], label [[COND_STORE1:%.*]], label [[ELSE2:%.*]]
-; CHECK:       cond.store1:
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i64> [[DATA]], i64 1
-; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 1
-; CHECK-NEXT:    store i64 [[TMP8]], i64* [[TMP9]], align 8
-; CHECK-NEXT:    br label [[ELSE2]]
-; CHECK:       else2:
-; CHECK-NEXT:    ret void
+; CHECK-LE-LABEL: @scalarize_v2i64(
+; CHECK-LE-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64>* [[P:%.*]] to i64*
+; CHECK-LE-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-LE-NEXT:    [[TMP2:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-LE-NEXT:    [[TMP3:%.*]] = icmp ne i2 [[TMP2]], 0
+; CHECK-LE-NEXT:    br i1 [[TMP3]], label [[COND_STORE:%.*]], label [[ELSE:%.*]]
+; CHECK-LE:       cond.store:
+; CHECK-LE-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[DATA:%.*]], i64 0
+; CHECK-LE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 0
+; CHECK-LE-NEXT:    store i64 [[TMP4]], i64* [[TMP5]], align 8
+; CHECK-LE-NEXT:    br label [[ELSE]]
+; CHECK-LE:       else:
+; CHECK-LE-NEXT:    [[TMP6:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-LE-NEXT:    [[TMP7:%.*]] = icmp ne i2 [[TMP6]], 0
+; CHECK-LE-NEXT:    br i1 [[TMP7]], label [[COND_STORE1:%.*]], label [[ELSE2:%.*]]
+; CHECK-LE:       cond.store1:
+; CHECK-LE-NEXT:    [[TMP8:%.*]] = extractelement <2 x i64> [[DATA]], i64 1
+; CHECK-LE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 1
+; CHECK-LE-NEXT:    store i64 [[TMP8]], i64* [[TMP9]], align 8
+; CHECK-LE-NEXT:    br label [[ELSE2]]
+; CHECK-LE:       else2:
+; CHECK-LE-NEXT:    ret void
+;
+; CHECK-BE-LABEL: @scalarize_v2i64(
+; CHECK-BE-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64>* [[P:%.*]] to i64*
+; CHECK-BE-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-BE-NEXT:    [[TMP2:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-BE-NEXT:    [[TMP3:%.*]] = icmp ne i2 [[TMP2]], 0
+; CHECK-BE-NEXT:    br i1 [[TMP3]], label [[COND_STORE:%.*]], label [[ELSE:%.*]]
+; CHECK-BE:       cond.store:
+; CHECK-BE-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[DATA:%.*]], i64 0
+; CHECK-BE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 0
+; CHECK-BE-NEXT:    store i64 [[TMP4]], i64* [[TMP5]], align 8
+; CHECK-BE-NEXT:    br label [[ELSE]]
+; CHECK-BE:       else:
+; CHECK-BE-NEXT:    [[TMP6:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-BE-NEXT:    [[TMP7:%.*]] = icmp ne i2 [[TMP6]], 0
+; CHECK-BE-NEXT:    br i1 [[TMP7]], label [[COND_STORE1:%.*]], label [[ELSE2:%.*]]
+; CHECK-BE:       cond.store1:
+; CHECK-BE-NEXT:    [[TMP8:%.*]] = extractelement <2 x i64> [[DATA]], i64 1
+; CHECK-BE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 1
+; CHECK-BE-NEXT:    store i64 [[TMP8]], i64* [[TMP9]], align 8
+; CHECK-BE-NEXT:    br label [[ELSE2]]
+; CHECK-BE:       else2:
+; CHECK-BE-NEXT:    ret void
 ;
   call void @llvm.masked.store.v2i64.p0v2i64(<2 x i64> %data, <2 x i64>* %p, i32 128, <2 x i1> %mask)
   ret void
 }
 
 define void @scalarize_v2i64_ones_mask(<2 x i64>* %p, <2 x i64> %data) {
 ; CHECK-LABEL: @scalarize_v2i64_ones_mask(
 ; CHECK-NEXT:    store <2 x i64> [[DATA:%.*]], <2 x i64>* [[P:%.*]], align 8