[AArch64] Sink to umull if we know tops bits are zero.

[dmgreen]

Summary

This is an extension to the code for sinking splats to multiplies, where if we can detect that the top bits are known-zero we can treat the instruction like a zext. The existing code was also adjusted in the process to make it more precise about only sink if both operands are zext or sext.

[samtebbs]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

14092

Would an early return within the loop in the case of NumSExts > 2 && NumZExts > 2 make sense?

[dmgreen]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

14092

I'm working under the assumption that a mul can only have 2 operands. The loop is just checking both of them.

[samtebbs]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

14092

I see, cool. Semantically an early return and not is the same thing when assuming 2 operands so LGTM.

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/MemoryLocation.h"
 #include "llvm/Analysis/ObjCARCUtil.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/Analysis/ValueTracking.h"
 #include "llvm/Analysis/VectorUtils.h"
 #include "llvm/CodeGen/Analysis.h"
 #include "llvm/CodeGen/CallingConvLower.h"
 #include "llvm/CodeGen/ISDOpcodes.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstr.h"
case Instruction::Add: {
     }
 
     Ops.push_back(&I->getOperandUse(0));
     Ops.push_back(&I->getOperandUse(1));
 
     return true;
   }
   case Instruction::Mul: {
-    bool IsProfitable = false;
+    int NumZExts = 0, NumSExts = 0;
     for (auto &Op : I->operands()) {
       // Make sure we are not already sinking this operand
       if (any_of(Ops, [&](Use *U) { return U->get() == Op; }))
         continue;
 
+      if (match(&Op, m_SExt(m_Value()))) {
+        NumSExts++;
+        continue;
+      } else if (match(&Op, m_ZExt(m_Value()))) {
+        NumZExts++;
+        continue;
+      }
+
       ShuffleVectorInst *Shuffle = dyn_cast<ShuffleVectorInst>(Op);
 
       // If the Shuffle is a splat and the operand is a zext/sext, sinking the
       // operand and the s/zext can help create indexed s/umull. This is
       // especially useful to prevent i64 mul being scalarized.
       if (Shuffle && isSplatShuffle(Shuffle) &&
           match(Shuffle->getOperand(0), m_ZExtOrSExt(m_Value()))) {
         Ops.push_back(&Shuffle->getOperandUse(0));
         Ops.push_back(&Op);
-        IsProfitable = true;
+        if (match(Shuffle->getOperand(0), m_SExt(m_Value())))
+          NumSExts++;
+        else
+          NumZExts++;
         continue;
       }
 
-      if (!Shuffle || !Shuffle->isZeroEltSplat())
+      if (!Shuffle)
         continue;
 
       Value *ShuffleOperand = Shuffle->getOperand(0);
       InsertElementInst *Insert = dyn_cast<InsertElementInst>(ShuffleOperand);
       if (!Insert)
         continue;
 
       Instruction *OperandInstr = dyn_cast<Instruction>(Insert->getOperand(1));
       if (!OperandInstr)
         continue;
 
       ConstantInt *ElementConstant =
           dyn_cast<ConstantInt>(Insert->getOperand(2));
       // Check that the insertelement is inserting into element 0
       if (!ElementConstant || ElementConstant->getZExtValue() != 0)
         continue;
 
       unsigned Opcode = OperandInstr->getOpcode();
-      if (Opcode != Instruction::SExt && Opcode != Instruction::ZExt)
+      if (Opcode == Instruction::SExt)
+        NumSExts++;
+      else if (Opcode == Instruction::ZExt)
+        NumZExts++;
+      else {
+        // If we find that the top bits are known 0, then we can sink and allow
+        // the backend to generate a umull.
+        unsigned Bitwidth = I->getType()->getScalarSizeInBits();
+        APInt UpperMask = APInt::getHighBitsSet(Bitwidth, Bitwidth / 2);
+        const DataLayout &DL = I->getFunction()->getParent()->getDataLayout();
+        if (!MaskedValueIsZero(OperandInstr, UpperMask, DL))
           continue;
+        NumZExts++;
+      }
 
       Ops.push_back(&Shuffle->getOperandUse(0));
       Ops.push_back(&Op);
-      IsProfitable = true;
     }
 
-    return IsProfitable;
+    // Is it profitable to sink if we found two of the same type of extends.
+    return !Ops.empty() && (NumSExts == 2 || NumZExts == 2);
   }
   default:
     return false;
   }
   return false;
 }
 
 static void createTblShuffleForZExt(ZExtInst *ZExt, bool IsLittleEndian) {

llvm/test/CodeGen/AArch64/aarch64-matrix-umull-smull.ll

 ; CHECK-NEXT:  // %bb.2:
 ; CHECK-NEXT:    mov x11, xzr
 ; CHECK-NEXT:    mov w8, wzr
 ; CHECK-NEXT:    b .LBB5_7
 ; CHECK-NEXT:  .LBB5_3:
 ; CHECK-NEXT:    mov w0, wzr
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB5_4: // %vector.ph
-; CHECK-NEXT:    dup v2.8b, w9
 ; CHECK-NEXT:    and x11, x10, #0xfffffff0
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
 ; CHECK-NEXT:    add x8, x0, #8
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
 ; CHECK-NEXT:    mov x12, x11
-; CHECK-NEXT:    sshll v2.8h, v2.8b, #0
+; CHECK-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-NEXT:    dup v2.8h, w9
 ; CHECK-NEXT:  .LBB5_5: // %vector.body
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldp d3, d4, [x8, #-8]
 ; CHECK-NEXT:    subs x12, x12, #16
 ; CHECK-NEXT:    add x8, x8, #16
 ; CHECK-NEXT:    ushll v3.8h, v3.8b, #0
 ; CHECK-NEXT:    ushll v4.8h, v4.8b, #0
 ; CHECK-NEXT:    mla v0.8h, v2.8h, v3.8h
exit:
   ret void
 }
 
 define void @matrix_mul_unsigned_and(i32 %N, i32* nocapture %C, i16* nocapture readonly %A, i32 %val) {
 ; CHECK-LABEL: matrix_mul_unsigned_and:
 ; CHECK:       // %bb.0: // %vector.header
 ; CHECK-NEXT:    and w8, w3, #0xffff
 ; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
-; CHECK-NEXT:    dup v0.4s, w8
+; CHECK-NEXT:    dup v0.4h, w8
 ; CHECK-NEXT:    and x8, x0, #0xfffffff8
-; CHECK-NEXT:    xtn v0.4h, v0.4s
 ; CHECK-NEXT:  .LBB10_1: // %vector.body
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    add x9, x2, w0, uxtw #1
 ; CHECK-NEXT:    subs x8, x8, #8
 ; CHECK-NEXT:    ldp d1, d2, [x9]
 ; CHECK-NEXT:    add x9, x1, w0, uxtw #2
 ; CHECK-NEXT:    add w0, w0, #8
 ; CHECK-NEXT:    umull v1.4s, v0.4h, v1.4h

 
 for.end12:                                        ; preds = %vector.body
   ret void
 }
 
 define void @matrix_mul_unsigned_and_double(i32 %N, i32* nocapture %C, i16* nocapture readonly %A, i32 %val) {
 ; CHECK-LABEL: matrix_mul_unsigned_and_double:
 ; CHECK:       // %bb.0: // %vector.header
-; CHECK-NEXT:    and w8, w3, #0xffff
+; CHECK-NEXT:    and w9, w3, #0xffff
 ; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
-; CHECK-NEXT:    dup v0.4s, w8
 ; CHECK-NEXT:    and x8, x0, #0xfffffff0
-; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    dup v0.8h, w9
 ; CHECK-NEXT:  .LBB11_1: // %vector.body
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    add x9, x2, w0, uxtw #1
+; CHECK-NEXT:    add x10, x1, w0, uxtw #2
 ; CHECK-NEXT:    subs x8, x8, #16
+; CHECK-NEXT:    add w0, w0, #16
 ; CHECK-NEXT:    ldr q1, [x9]
 ; CHECK-NEXT:    ldur q2, [x9, #8]
-; CHECK-NEXT:    add x9, x1, w0, uxtw #2
-; CHECK-NEXT:    add w0, w0, #16
-; CHECK-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT:    ext v4.16b, v2.16b, v2.16b, #8
+; CHECK-NEXT:    umull2 v3.4s, v0.8h, v1.8h
 ; CHECK-NEXT:    umull v1.4s, v0.4h, v1.4h
+; CHECK-NEXT:    umull2 v4.4s, v0.8h, v2.8h
 ; CHECK-NEXT:    umull v2.4s, v0.4h, v2.4h
-; CHECK-NEXT:    umull v3.4s, v0.4h, v3.4h
-; CHECK-NEXT:    umull v4.4s, v0.4h, v4.4h
-; CHECK-NEXT:    stp q1, q3, [x9]
-; CHECK-NEXT:    stp q2, q4, [x9, #32]
+; CHECK-NEXT:    stp q1, q3, [x10]
+; CHECK-NEXT:    stp q2, q4, [x10, #32]
 ; CHECK-NEXT:    b.ne .LBB11_1
 ; CHECK-NEXT:  // %bb.2: // %for.end12
 ; CHECK-NEXT:    ret
 vector.header:
   %conv4 = and i32 %val, 65535
   %wide.trip.count = zext i32 %N to i64
   %0 = add nsw i64 %wide.trip.count, -1
   %min.iters.check = icmp ult i32 %N, 16

llvm/test/CodeGen/AArch64/sve-loopforms.ll

 ; CHECK-NEXT:    sub w14, w14, w10
 ; CHECK-NEXT:    add w15, w15, w4, asr #3
 ; CHECK-NEXT:    sxtw x13, w1
 ; CHECK-NEXT:    sxtw x18, w15
 ; CHECK-NEXT:    mov w15, w6
 ; CHECK-NEXT:    add x17, x11, x18
 ; CHECK-NEXT:    add x3, x18, x15
 ; CHECK-NEXT:    add x1, x17, x15
-; CHECK-NEXT:    dup v2.8h, w12
-; CHECK-NEXT:    dup v3.8h, w9
 ; CHECK-NEXT:    and x4, x15, #0xfffffff8
+; CHECK-NEXT:    dup v2.8b, w12
+; CHECK-NEXT:    dup v3.8b, w9
 ; CHECK-NEXT:    mov x8, xzr
 ; CHECK-NEXT:    add x16, x2, x18
 ; CHECK-NEXT:    add x18, x1, #1
 ; CHECK-NEXT:    add x1, x3, #1
 ; CHECK-NEXT:    and x3, x15, #0xfffffff0
 ; CHECK-NEXT:    neg x5, x4
 ; CHECK-NEXT:    add x19, x11, #1
 ; CHECK-NEXT:    neg x20, x15
 ; CHECK-NEXT:    mov x21, x0
 ; CHECK-NEXT:    dup v0.8h, w10
 ; CHECK-NEXT:    dup v1.8h, w14
-; CHECK-NEXT:    xtn v2.8b, v2.8h
-; CHECK-NEXT:    xtn v3.8b, v3.8h
+; CHECK-NEXT:    dup v4.16b, w12
+; CHECK-NEXT:    dup v5.16b, w9
 ; CHECK-NEXT:    b .LBB0_4
 ; CHECK-NEXT:  .LBB0_3: // %for.cond10.for.cond.cleanup12_crit_edge.us
 ; CHECK-NEXT:    // in Loop: Header=BB0_4 Depth=1
 ; CHECK-NEXT:    add x16, x16, x11
 ; CHECK-NEXT:    add x21, x21, x13
 ; CHECK-NEXT:    add x8, x8, #1
 ; CHECK-NEXT:    cmp w8, w7
 ; CHECK-NEXT:    b.eq .LBB0_19

 ; CHECK-NEXT:    add x22, x24, #1
 ; CHECK-NEXT:    add x23, x19, x24
 ; CHECK-NEXT:  .LBB0_13: // %vec.epilog.vector.body
 ; CHECK-NEXT:    // Parent Loop BB0_4 Depth=1
 ; CHECK-NEXT:    // => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    add x24, x16, x23
 ; CHECK-NEXT:    add x25, x16, x22
 ; CHECK-NEXT:    add x23, x23, #8
-; CHECK-NEXT:    ldr d4, [x24]
-; CHECK-NEXT:    ldr d5, [x25]
-; CHECK-NEXT:    ldur d6, [x24, #-1]
+; CHECK-NEXT:    ldr d6, [x24]
+; CHECK-NEXT:    ldr d7, [x25]
+; CHECK-NEXT:    ldur d16, [x24, #-1]
 ; CHECK-NEXT:    add x24, x21, x22
-; CHECK-NEXT:    umull v4.8h, v3.8b, v4.8b
-; CHECK-NEXT:    ldur d7, [x25, #-1]
-; CHECK-NEXT:    umull v5.8h, v3.8b, v5.8b
+; CHECK-NEXT:    umull v6.8h, v3.8b, v6.8b
+; CHECK-NEXT:    ldur d17, [x25, #-1]
+; CHECK-NEXT:    umull v7.8h, v3.8b, v7.8b
 ; CHECK-NEXT:    add x22, x22, #8
 ; CHECK-NEXT:    add x25, x5, x22
 ; CHECK-NEXT:    cmp x25, #1
-; CHECK-NEXT:    umlal v4.8h, v2.8b, v6.8b
-; CHECK-NEXT:    umlal v5.8h, v2.8b, v7.8b
-; CHECK-NEXT:    mul v4.8h, v4.8h, v0.8h
-; CHECK-NEXT:    mla v4.8h, v5.8h, v1.8h
-; CHECK-NEXT:    rshrn v4.8b, v4.8h, #6
-; CHECK-NEXT:    stur d4, [x24, #-1]
+; CHECK-NEXT:    umlal v6.8h, v2.8b, v16.8b
+; CHECK-NEXT:    umlal v7.8h, v2.8b, v17.8b
+; CHECK-NEXT:    mul v6.8h, v6.8h, v0.8h
+; CHECK-NEXT:    mla v6.8h, v7.8h, v1.8h
+; CHECK-NEXT:    rshrn v6.8b, v6.8h, #6
+; CHECK-NEXT:    stur d6, [x24, #-1]
 ; CHECK-NEXT:    b.ne .LBB0_13
 ; CHECK-NEXT:  // %bb.14: // %vec.epilog.middle.block
 ; CHECK-NEXT:    // in Loop: Header=BB0_4 Depth=1
 ; CHECK-NEXT:    mov x23, x4
 ; CHECK-NEXT:    cmp x4, x15
 ; CHECK-NEXT:    b.eq .LBB0_3
 ; CHECK-NEXT:    b .LBB0_6
 ; CHECK-NEXT:  .LBB0_15: // %vector.body.preheader
 ; CHECK-NEXT:    // in Loop: Header=BB0_4 Depth=1
 ; CHECK-NEXT:    mov x22, xzr
 ; CHECK-NEXT:    add x23, x16, x11
 ; CHECK-NEXT:  .LBB0_16: // %vector.body
 ; CHECK-NEXT:    // Parent Loop BB0_4 Depth=1
 ; CHECK-NEXT:    // => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    add x24, x23, x22
-; CHECK-NEXT:    add x25, x16, x22
-; CHECK-NEXT:    ldr q4, [x23, x22]
-; CHECK-NEXT:    ldur q5, [x24, #1]
-; CHECK-NEXT:    ldur q7, [x25, #1]
-; CHECK-NEXT:    ext v16.16b, v4.16b, v4.16b, #8
-; CHECK-NEXT:    ldr q6, [x25]
-; CHECK-NEXT:    ext v17.16b, v5.16b, v5.16b, #8
-; CHECK-NEXT:    umull v5.8h, v3.8b, v5.8b
-; CHECK-NEXT:    ext v19.16b, v7.16b, v7.16b, #8
-; CHECK-NEXT:    umull v7.8h, v3.8b, v7.8b
-; CHECK-NEXT:    umull v17.8h, v3.8b, v17.8b
-; CHECK-NEXT:    umlal v5.8h, v2.8b, v4.8b
-; CHECK-NEXT:    ext v18.16b, v6.16b, v6.16b, #8
-; CHECK-NEXT:    umull v4.8h, v3.8b, v19.8b
-; CHECK-NEXT:    umlal v17.8h, v2.8b, v16.8b
-; CHECK-NEXT:    umlal v7.8h, v2.8b, v6.8b
-; CHECK-NEXT:    mul v5.8h, v5.8h, v0.8h
-; CHECK-NEXT:    umlal v4.8h, v2.8b, v18.8b
-; CHECK-NEXT:    mul v6.8h, v17.8h, v0.8h
-; CHECK-NEXT:    mla v5.8h, v7.8h, v1.8h
-; CHECK-NEXT:    mla v6.8h, v4.8h, v1.8h
-; CHECK-NEXT:    rshrn v4.8b, v5.8h, #6
-; CHECK-NEXT:    rshrn2 v4.16b, v6.8h, #6
-; CHECK-NEXT:    str q4, [x21, x22]
+; CHECK-NEXT:    ldr q7, [x23, x22]
+; CHECK-NEXT:    ldur q6, [x24, #1]
+; CHECK-NEXT:    add x24, x16, x22
+; CHECK-NEXT:    umull v16.8h, v5.8b, v6.8b
+; CHECK-NEXT:    ldur q18, [x24, #1]
+; CHECK-NEXT:    umull2 v6.8h, v5.16b, v6.16b
+; CHECK-NEXT:    ldr q17, [x24]
+; CHECK-NEXT:    umull v19.8h, v5.8b, v18.8b
+; CHECK-NEXT:    umlal v16.8h, v4.8b, v7.8b
+; CHECK-NEXT:    umull2 v18.8h, v5.16b, v18.16b
+; CHECK-NEXT:    umlal2 v6.8h, v4.16b, v7.16b
+; CHECK-NEXT:    umlal v19.8h, v4.8b, v17.8b
+; CHECK-NEXT:    mul v7.8h, v16.8h, v0.8h
+; CHECK-NEXT:    umlal2 v18.8h, v4.16b, v17.16b
+; CHECK-NEXT:    mul v6.8h, v6.8h, v0.8h
+; CHECK-NEXT:    mla v7.8h, v19.8h, v1.8h
+; CHECK-NEXT:    mla v6.8h, v18.8h, v1.8h
+; CHECK-NEXT:    rshrn v7.8b, v7.8h, #6
+; CHECK-NEXT:    rshrn2 v7.16b, v6.8h, #6
+; CHECK-NEXT:    str q7, [x21, x22]
 ; CHECK-NEXT:    add x22, x22, #16
 ; CHECK-NEXT:    cmp x3, x22
 ; CHECK-NEXT:    b.ne .LBB0_16
 ; CHECK-NEXT:  // %bb.17: // %middle.block
 ; CHECK-NEXT:    // in Loop: Header=BB0_4 Depth=1
 ; CHECK-NEXT:    cmp x3, x15
 ; CHECK-NEXT:    b.eq .LBB0_3
 ; CHECK-NEXT:  // %bb.18: // %vec.epilog.iter.check

 ; CHECK-NEXT:    add x1, x16, x13
 ; CHECK-NEXT:    mov x8, xzr
 ; CHECK-NEXT:    add x17, x2, x18
 ; CHECK-NEXT:    add x18, x1, #1
 ; CHECK-NEXT:    add x1, x3, #1
 ; CHECK-NEXT:    add x4, x2, x16
 ; CHECK-NEXT:    neg x5, x13
 ; CHECK-NEXT:    add x6, x11, #1
-; CHECK-NEXT:    mov w19, #1
-; CHECK-NEXT:    mov x20, x0
+; CHECK-NEXT:    mov w20, #1
+; CHECK-NEXT:    mov x21, x0
 ; CHECK-NEXT:    cnth x3
-; CHECK-NEXT:    mov z0.h, w14
-; CHECK-NEXT:    mov z1.h, w9
-; CHECK-NEXT:    mov z2.h, w10
-; CHECK-NEXT:    mov z3.h, w15
-; CHECK-NEXT:    rdvl x21, #1
+; CHECK-NEXT:    mov z0.h, w10
+; CHECK-NEXT:    mov z1.h, w15
+; CHECK-NEXT:    rdvl x19, #1
 ; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    mov z2.h, w14
+; CHECK-NEXT:    mov z3.h, w9
 ; CHECK-NEXT:    ptrue p1.b
 ; CHECK-NEXT:    b .LBB1_4
 ; CHECK-NEXT:  .LBB1_3: // %for.cond10.for.cond.cleanup12_crit_edge.us
 ; CHECK-NEXT:    // in Loop: Header=BB1_4 Depth=1
 ; CHECK-NEXT:    add x17, x17, x11
-; CHECK-NEXT:    add x20, x20, x12
+; CHECK-NEXT:    add x21, x21, x12
 ; CHECK-NEXT:    add x8, x8, #1
 ; CHECK-NEXT:    add x4, x4, x11
 ; CHECK-NEXT:    cmp w8, w7
 ; CHECK-NEXT:    b.eq .LBB1_19
 ; CHECK-NEXT:  .LBB1_4: // %iter.check
 ; CHECK-NEXT:    // =>This Loop Header: Depth=1
 ; CHECK-NEXT:    // Child Loop BB1_13 Depth 2
 ; CHECK-NEXT:    // Child Loop BB1_17 Depth 2

 ; CHECK-NEXT:    ldrb w24, [x24]
 ; CHECK-NEXT:    mul w26, w14, w26
 ; CHECK-NEXT:    ldrb w25, [x25]
 ; CHECK-NEXT:    mul w27, w14, w27
 ; CHECK-NEXT:    madd w24, w9, w24, w26
 ; CHECK-NEXT:    madd w25, w9, w25, w27
 ; CHECK-NEXT:    mul w24, w24, w10
 ; CHECK-NEXT:    madd w24, w25, w15, w24
-; CHECK-NEXT:    add x25, x20, x23
+; CHECK-NEXT:    add x25, x21, x23
 ; CHECK-NEXT:    add x23, x23, #1
 ; CHECK-NEXT:    add w24, w24, #32
 ; CHECK-NEXT:    add x26, x5, x23
 ; CHECK-NEXT:    lsr w24, w24, #6
 ; CHECK-NEXT:    cmp x26, #1
 ; CHECK-NEXT:    sturb w24, [x25, #-1]
 ; CHECK-NEXT:    b.ne .LBB1_7
 ; CHECK-NEXT:    b .LBB1_3
 ; CHECK-NEXT:  .LBB1_8: // %vector.memcheck
 ; CHECK-NEXT:    // in Loop: Header=BB1_4 Depth=1
 ; CHECK-NEXT:    mul x22, x8, x11
 ; CHECK-NEXT:    madd x24, x8, x12, x13
 ; CHECK-NEXT:    add x23, x1, x22
 ; CHECK-NEXT:    add x23, x2, x23
 ; CHECK-NEXT:    add x24, x0, x24
-; CHECK-NEXT:    cmp x20, x23
+; CHECK-NEXT:    cmp x21, x23
 ; CHECK-NEXT:    add x23, x18, x22
 ; CHECK-NEXT:    ccmp x17, x24, #2, lo
 ; CHECK-NEXT:    add x22, x16, x22
 ; CHECK-NEXT:    add x22, x2, x22
 ; CHECK-NEXT:    add x25, x2, x23
 ; CHECK-NEXT:    cset w23, lo
 ; CHECK-NEXT:    cmp x22, x24
-; CHECK-NEXT:    ccmp x20, x25, #2, lo
+; CHECK-NEXT:    ccmp x21, x25, #2, lo
 ; CHECK-NEXT:    mov x22, xzr
 ; CHECK-NEXT:    b.lo .LBB1_6
 ; CHECK-NEXT:  // %bb.9: // %vector.memcheck
 ; CHECK-NEXT:    // in Loop: Header=BB1_4 Depth=1
 ; CHECK-NEXT:    tbnz w23, #0, .LBB1_6
 ; CHECK-NEXT:  // %bb.10: // %vector.main.loop.iter.check
 ; CHECK-NEXT:    // in Loop: Header=BB1_4 Depth=1
-; CHECK-NEXT:    cmp x21, x13
+; CHECK-NEXT:    cmp x19, x13
 ; CHECK-NEXT:    b.ls .LBB1_12
 ; CHECK-NEXT:  // %bb.11: // in Loop: Header=BB1_4 Depth=1
 ; CHECK-NEXT:    mov x22, xzr
 ; CHECK-NEXT:    b .LBB1_16
 ; CHECK-NEXT:  .LBB1_12: // %vector.ph
 ; CHECK-NEXT:    // in Loop: Header=BB1_4 Depth=1
-; CHECK-NEXT:    udiv x22, x13, x21
+; CHECK-NEXT:    udiv x22, x13, x19
 ; CHECK-NEXT:    mov x24, xzr
-; CHECK-NEXT:    mul x22, x22, x21
+; CHECK-NEXT:    mul x22, x22, x19
 ; CHECK-NEXT:    sub x23, x13, x22
 ; CHECK-NEXT:  .LBB1_13: // %vector.body
 ; CHECK-NEXT:    // Parent Loop BB1_4 Depth=1
 ; CHECK-NEXT:    // => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    add x25, x17, x24
 ; CHECK-NEXT:    add x27, x4, x24
 ; CHECK-NEXT:    add x26, x25, #1
 ; CHECK-NEXT:    add x28, x27, #1
 ; CHECK-NEXT:    ld1b { z4.h }, p0/z, [x17, x24]
 ; CHECK-NEXT:    ld1b { z5.h }, p0/z, [x25, #1, mul vl]
-; CHECK-NEXT:    ld1b { z6.h }, p0/z, [x25, x19]
+; CHECK-NEXT:    ld1b { z6.h }, p0/z, [x25, x20]
 ; CHECK-NEXT:    ld1b { z7.h }, p0/z, [x26, #1, mul vl]
 ; CHECK-NEXT:    ld1b { z16.h }, p0/z, [x4, x24]
 ; CHECK-NEXT:    ld1b { z17.h }, p0/z, [x27, #1, mul vl]
 ; CHECK-NEXT:    ld1b { z18.h }, p0/z, [x28, #1, mul vl]
-; CHECK-NEXT:    ld1b { z19.h }, p0/z, [x27, x19]
-; CHECK-NEXT:    mul z6.h, z1.h, z6.h
-; CHECK-NEXT:    mul z7.h, z1.h, z7.h
-; CHECK-NEXT:    mla z6.h, p0/m, z0.h, z4.h
-; CHECK-NEXT:    mul z18.h, z1.h, z18.h
-; CHECK-NEXT:    mla z7.h, p0/m, z0.h, z5.h
-; CHECK-NEXT:    mul z19.h, z1.h, z19.h
-; CHECK-NEXT:    mla z18.h, p0/m, z0.h, z17.h
-; CHECK-NEXT:    mla z19.h, p0/m, z0.h, z16.h
-; CHECK-NEXT:    mul z4.h, z7.h, z3.h
-; CHECK-NEXT:    mul z5.h, z6.h, z3.h
-; CHECK-NEXT:    mla z4.h, p0/m, z18.h, z2.h
-; CHECK-NEXT:    mla z5.h, p0/m, z19.h, z2.h
+; CHECK-NEXT:    ld1b { z19.h }, p0/z, [x27, x20]
+; CHECK-NEXT:    mul z6.h, z3.h, z6.h
+; CHECK-NEXT:    mul z7.h, z3.h, z7.h
+; CHECK-NEXT:    mla z6.h, p0/m, z2.h, z4.h
+; CHECK-NEXT:    mul z18.h, z3.h, z18.h
+; CHECK-NEXT:    mla z7.h, p0/m, z2.h, z5.h
+; CHECK-NEXT:    mul z19.h, z3.h, z19.h
+; CHECK-NEXT:    mla z18.h, p0/m, z2.h, z17.h
+; CHECK-NEXT:    mla z19.h, p0/m, z2.h, z16.h
+; CHECK-NEXT:    mul z4.h, z7.h, z1.h
+; CHECK-NEXT:    mul z5.h, z6.h, z1.h
+; CHECK-NEXT:    mla z4.h, p0/m, z18.h, z0.h
+; CHECK-NEXT:    mla z5.h, p0/m, z19.h, z0.h
 ; CHECK-NEXT:    add z4.h, z4.h, #32 // =0x20
 ; CHECK-NEXT:    add z5.h, z5.h, #32 // =0x20
 ; CHECK-NEXT:    lsr z4.h, z4.h, #6
 ; CHECK-NEXT:    lsr z5.h, z5.h, #6
 ; CHECK-NEXT:    uzp1 z4.b, z5.b, z4.b
-; CHECK-NEXT:    st1b { z4.b }, p1, [x20, x24]
-; CHECK-NEXT:    add x24, x24, x21
+; CHECK-NEXT:    st1b { z4.b }, p1, [x21, x24]
+; CHECK-NEXT:    add x24, x24, x19
 ; CHECK-NEXT:    cmp x22, x24
 ; CHECK-NEXT:    b.ne .LBB1_13
 ; CHECK-NEXT:  // %bb.14: // %middle.block
 ; CHECK-NEXT:    // in Loop: Header=BB1_4 Depth=1
 ; CHECK-NEXT:    cbz x23, .LBB1_3
 ; CHECK-NEXT:  // %bb.15: // %vec.epilog.iter.check
 ; CHECK-NEXT:    // in Loop: Header=BB1_4 Depth=1
 ; CHECK-NEXT:    cmp x23, x3
 ; CHECK-NEXT:    b.lo .LBB1_6
 ; CHECK-NEXT:  .LBB1_16: // %vec.epilog.ph
 ; CHECK-NEXT:    // in Loop: Header=BB1_4 Depth=1
 ; CHECK-NEXT:    udiv x23, x13, x3
 ; CHECK-NEXT:    mov x24, x22
 ; CHECK-NEXT:    mul x22, x23, x3
 ; CHECK-NEXT:    sub x23, x13, x22
 ; CHECK-NEXT:  .LBB1_17: // %vec.epilog.vector.body
 ; CHECK-NEXT:    // Parent Loop BB1_4 Depth=1
 ; CHECK-NEXT:    // => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    add x25, x17, x24
 ; CHECK-NEXT:    add x26, x4, x24
 ; CHECK-NEXT:    ld1b { z5.h }, p0/z, [x17, x24]
 ; CHECK-NEXT:    ld1b { z7.h }, p0/z, [x4, x24]
-; CHECK-NEXT:    ld1b { z4.h }, p0/z, [x25, x19]
-; CHECK-NEXT:    ld1b { z6.h }, p0/z, [x26, x19]
-; CHECK-NEXT:    mul z4.h, z1.h, z4.h
-; CHECK-NEXT:    mul z6.h, z1.h, z6.h
-; CHECK-NEXT:    mla z4.h, p0/m, z0.h, z5.h
-; CHECK-NEXT:    mla z6.h, p0/m, z0.h, z7.h
-; CHECK-NEXT:    mul z4.h, z4.h, z3.h
-; CHECK-NEXT:    mla z4.h, p0/m, z6.h, z2.h
+; CHECK-NEXT:    ld1b { z4.h }, p0/z, [x25, x20]
+; CHECK-NEXT:    ld1b { z6.h }, p0/z, [x26, x20]
+; CHECK-NEXT:    mul z4.h, z3.h, z4.h
+; CHECK-NEXT:    mul z6.h, z3.h, z6.h
+; CHECK-NEXT:    mla z4.h, p0/m, z2.h, z5.h
+; CHECK-NEXT:    mla z6.h, p0/m, z2.h, z7.h
+; CHECK-NEXT:    mul z4.h, z4.h, z1.h
+; CHECK-NEXT:    mla z4.h, p0/m, z6.h, z0.h
 ; CHECK-NEXT:    add z4.h, z4.h, #32 // =0x20
 ; CHECK-NEXT:    lsr z4.h, z4.h, #6
-; CHECK-NEXT:    st1b { z4.h }, p0, [x20, x24]
+; CHECK-NEXT:    st1b { z4.h }, p0, [x21, x24]
 ; CHECK-NEXT:    add x24, x24, x3
 ; CHECK-NEXT:    cmp x22, x24
 ; CHECK-NEXT:    b.ne .LBB1_17
 ; CHECK-NEXT:  // %bb.18: // %vec.epilog.middle.block
 ; CHECK-NEXT:    // in Loop: Header=BB1_4 Depth=1
 ; CHECK-NEXT:    cbnz x23, .LBB1_6
 ; CHECK-NEXT:    b .LBB1_3
 ; CHECK-NEXT:  .LBB1_19: // %for.cond.cleanup

 ; CHECK-NEXT:    mov x8, xzr
 ; CHECK-NEXT:    add x4, x2, x17
 ; CHECK-NEXT:    add x17, x18, #1
 ; CHECK-NEXT:    add x18, x1, #1
 ; CHECK-NEXT:    add x3, x2, x16
 ; CHECK-NEXT:    mov w5, #1
 ; CHECK-NEXT:    mov x6, x0
 ; CHECK-NEXT:    rdvl x1, #1
-; CHECK-NEXT:    mov z0.h, w12
-; CHECK-NEXT:    mov z1.h, w9
-; CHECK-NEXT:    mov z2.h, w10
-; CHECK-NEXT:    mov z3.h, w14
+; CHECK-NEXT:    mov z0.h, w10
+; CHECK-NEXT:    mov z1.h, w14
+; CHECK-NEXT:    mov z2.h, w12
 ; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    mov z3.h, w9
 ; CHECK-NEXT:    ptrue p1.b
 ; CHECK-NEXT:    b .LBB2_4
 ; CHECK-NEXT:  .LBB2_3: // %for.cond10.for.cond.cleanup12_crit_edge.us
 ; CHECK-NEXT:    // in Loop: Header=BB2_4 Depth=1
 ; CHECK-NEXT:    add x6, x6, x13
 ; CHECK-NEXT:    add x8, x8, #1
 ; CHECK-NEXT:    add x3, x3, x11
 ; CHECK-NEXT:    mov x4, x19

 ; CHECK-NEXT:    ld1b { z4.h }, p0/z, [x4, x21]
 ; CHECK-NEXT:    ld1b { z5.h }, p0/z, [x23, #1, mul vl]
 ; CHECK-NEXT:    ld1b { z6.h }, p0/z, [x23, x5]
 ; CHECK-NEXT:    ld1b { z7.h }, p0/z, [x24, #1, mul vl]
 ; CHECK-NEXT:    ld1b { z16.h }, p0/z, [x3, x21]
 ; CHECK-NEXT:    ld1b { z17.h }, p0/z, [x25, #1, mul vl]
 ; CHECK-NEXT:    ld1b { z18.h }, p0/z, [x26, #1, mul vl]
 ; CHECK-NEXT:    ld1b { z19.h }, p0/z, [x25, x5]
-; CHECK-NEXT:    mul z6.h, z1.h, z6.h
-; CHECK-NEXT:    mul z7.h, z1.h, z7.h
-; CHECK-NEXT:    mla z6.h, p0/m, z0.h, z4.h
-; CHECK-NEXT:    mul z18.h, z1.h, z18.h
-; CHECK-NEXT:    mla z7.h, p0/m, z0.h, z5.h
-; CHECK-NEXT:    mul z19.h, z1.h, z19.h
-; CHECK-NEXT:    mla z18.h, p0/m, z0.h, z17.h
-; CHECK-NEXT:    mla z19.h, p0/m, z0.h, z16.h
-; CHECK-NEXT:    mul z4.h, z7.h, z3.h
-; CHECK-NEXT:    mul z5.h, z6.h, z3.h
-; CHECK-NEXT:    mla z4.h, p0/m, z18.h, z2.h
-; CHECK-NEXT:    mla z5.h, p0/m, z19.h, z2.h
+; CHECK-NEXT:    mul z6.h, z3.h, z6.h
+; CHECK-NEXT:    mul z7.h, z3.h, z7.h
+; CHECK-NEXT:    mla z6.h, p0/m, z2.h, z4.h
+; CHECK-NEXT:    mul z18.h, z3.h, z18.h
+; CHECK-NEXT:    mla z7.h, p0/m, z2.h, z5.h
+; CHECK-NEXT:    mul z19.h, z3.h, z19.h
+; CHECK-NEXT:    mla z18.h, p0/m, z2.h, z17.h
+; CHECK-NEXT:    mla z19.h, p0/m, z2.h, z16.h
+; CHECK-NEXT:    mul z4.h, z7.h, z1.h
+; CHECK-NEXT:    mul z5.h, z6.h, z1.h
+; CHECK-NEXT:    mla z4.h, p0/m, z18.h, z0.h
+; CHECK-NEXT:    mla z5.h, p0/m, z19.h, z0.h
 ; CHECK-NEXT:    add z4.h, z4.h, #32 // =0x20
 ; CHECK-NEXT:    add z5.h, z5.h, #32 // =0x20
 ; CHECK-NEXT:    lsr z4.h, z4.h, #6
 ; CHECK-NEXT:    lsr z5.h, z5.h, #6
 ; CHECK-NEXT:    uzp1 z4.b, z5.b, z4.b
 ; CHECK-NEXT:    st1b { z4.b }, p1, [x6, x21]
 ; CHECK-NEXT:    add x21, x21, x1
 ; CHECK-NEXT:    cmp x20, x21

 ; CHECK-NEXT:    uunpklo z16.h, z4.b
 ; CHECK-NEXT:    uunpkhi z4.h, z4.b
 ; CHECK-NEXT:    uunpkhi z17.h, z5.b
 ; CHECK-NEXT:    uunpklo z5.h, z5.b
 ; CHECK-NEXT:    uunpkhi z18.h, z6.b
 ; CHECK-NEXT:    uunpklo z6.h, z6.b
 ; CHECK-NEXT:    uunpklo z19.h, z7.b
 ; CHECK-NEXT:    uunpkhi z7.h, z7.b
-; CHECK-NEXT:    mul z6.h, z1.h, z6.h
-; CHECK-NEXT:    mul z18.h, z1.h, z18.h
-; CHECK-NEXT:    mul z7.h, z1.h, z7.h
-; CHECK-NEXT:    mul z19.h, z1.h, z19.h
-; CHECK-NEXT:    mla z18.h, p0/m, z0.h, z4.h
-; CHECK-NEXT:    mla z6.h, p0/m, z0.h, z16.h
-; CHECK-NEXT:    mla z19.h, p0/m, z0.h, z5.h
-; CHECK-NEXT:    mla z7.h, p0/m, z0.h, z17.h
-; CHECK-NEXT:    mul z4.h, z6.h, z3.h
-; CHECK-NEXT:    mul z5.h, z18.h, z3.h
-; CHECK-NEXT:    mla z5.h, p0/m, z7.h, z2.h
-; CHECK-NEXT:    mla z4.h, p0/m, z19.h, z2.h
+; CHECK-NEXT:    mul z6.h, z3.h, z6.h
+; CHECK-NEXT:    mul z18.h, z3.h, z18.h
+; CHECK-NEXT:    mul z7.h, z3.h, z7.h
+; CHECK-NEXT:    mul z19.h, z3.h, z19.h
+; CHECK-NEXT:    mla z18.h, p0/m, z2.h, z4.h
+; CHECK-NEXT:    mla z6.h, p0/m, z2.h, z16.h
+; CHECK-NEXT:    mla z19.h, p0/m, z2.h, z5.h
+; CHECK-NEXT:    mla z7.h, p0/m, z2.h, z17.h
+; CHECK-NEXT:    mul z4.h, z6.h, z1.h
+; CHECK-NEXT:    mul z5.h, z18.h, z1.h
+; CHECK-NEXT:    mla z5.h, p0/m, z7.h, z0.h
+; CHECK-NEXT:    mla z4.h, p0/m, z19.h, z0.h
 ; CHECK-NEXT:    add z4.h, z4.h, #32 // =0x20
 ; CHECK-NEXT:    add z5.h, z5.h, #32 // =0x20
 ; CHECK-NEXT:    lsr z5.h, z5.h, #6
 ; CHECK-NEXT:    lsr z4.h, z4.h, #6
 ; CHECK-NEXT:    uzp1 z4.b, z4.b, z5.b
 ; CHECK-NEXT:    st1b { z4.b }, p2, [x6, x20]
 ; CHECK-NEXT:    add x20, x20, x1
 ; CHECK-NEXT:    whilelo p2.b, x20, x15

 ; CHECK-NEXT:    add w16, w14, w4, asr #3
 ; CHECK-NEXT:    sub w14, w15, w9
 ; CHECK-NEXT:    sxtw x17, w16
 ; CHECK-NEXT:    sub w15, w15, w10
 ; CHECK-NEXT:    add x16, x11, x17
 ; CHECK-NEXT:    add x1, x17, x13
 ; CHECK-NEXT:    add x18, x16, x13
 ; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    add x3, x2, x17
+; CHECK-NEXT:    add x4, x2, x17
 ; CHECK-NEXT:    add x17, x18, #1
 ; CHECK-NEXT:    add x18, x1, #1
 ; CHECK-NEXT:    add x1, x2, x16
-; CHECK-NEXT:    mov w4, #1
+; CHECK-NEXT:    mov w3, #1
 ; CHECK-NEXT:    mov x5, x0
-; CHECK-NEXT:    mov z0.h, w14
-; CHECK-NEXT:    mov z1.h, w9
-; CHECK-NEXT:    mov z2.h, w10
-; CHECK-NEXT:    mov z3.h, w15
+; CHECK-NEXT:    mov z0.h, w10
+; CHECK-NEXT:    mov z1.h, w15
 ; CHECK-NEXT:    rdvl x6, #1
+; CHECK-NEXT:    mov z2.h, w14
 ; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    mov z3.h, w9
 ; CHECK-NEXT:    b .LBB3_4
 ; CHECK-NEXT:  .LBB3_3: // %for.cond10.for.cond.cleanup12_crit_edge.us
 ; CHECK-NEXT:    // in Loop: Header=BB3_4 Depth=1
 ; CHECK-NEXT:    add x5, x5, x12
 ; CHECK-NEXT:    add x8, x8, #1
 ; CHECK-NEXT:    add x1, x1, x11
-; CHECK-NEXT:    mov x3, x19
+; CHECK-NEXT:    mov x4, x19
 ; CHECK-NEXT:    cmp w8, w7
 ; CHECK-NEXT:    b.eq .LBB3_10
 ; CHECK-NEXT:  .LBB3_4: // %for.cond10.preheader.us
 ; CHECK-NEXT:    // =>This Loop Header: Depth=1
 ; CHECK-NEXT:    // Child Loop BB3_7 Depth 2
 ; CHECK-NEXT:    // Child Loop BB3_9 Depth 2
 ; CHECK-NEXT:    mul x19, x8, x11
 ; CHECK-NEXT:    madd x21, x8, x12, x13
 ; CHECK-NEXT:    add x20, x18, x19
 ; CHECK-NEXT:    add x20, x2, x20
 ; CHECK-NEXT:    add x21, x0, x21
 ; CHECK-NEXT:    cmp x5, x20
 ; CHECK-NEXT:    add x20, x17, x19
-; CHECK-NEXT:    ccmp x3, x21, #2, lo
+; CHECK-NEXT:    ccmp x4, x21, #2, lo
 ; CHECK-NEXT:    add x19, x16, x19
 ; CHECK-NEXT:    add x19, x2, x19
 ; CHECK-NEXT:    add x22, x2, x20
 ; CHECK-NEXT:    cset w20, lo
 ; CHECK-NEXT:    cmp x19, x21
 ; CHECK-NEXT:    ccmp x5, x22, #2, lo
-; CHECK-NEXT:    add x19, x3, x11
+; CHECK-NEXT:    add x19, x4, x11
 ; CHECK-NEXT:    b.lo .LBB3_8
 ; CHECK-NEXT:  // %bb.5: // %for.cond10.preheader.us
 ; CHECK-NEXT:    // in Loop: Header=BB3_4 Depth=1
 ; CHECK-NEXT:    cbnz w20, .LBB3_8
 ; CHECK-NEXT:  // %bb.6: // %vector.ph
 ; CHECK-NEXT:    // in Loop: Header=BB3_4 Depth=1
 ; CHECK-NEXT:    mov x20, xzr
 ; CHECK-NEXT:    whilelo p1.b, xzr, x13
 ; CHECK-NEXT:  .LBB3_7: // %vector.body
 ; CHECK-NEXT:    // Parent Loop BB3_4 Depth=1
 ; CHECK-NEXT:    // => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    add x21, x3, x20
+; CHECK-NEXT:    add x21, x4, x20
 ; CHECK-NEXT:    add x22, x1, x20
-; CHECK-NEXT:    ld1b { z4.b }, p1/z, [x3, x20]
+; CHECK-NEXT:    ld1b { z4.b }, p1/z, [x4, x20]
 ; CHECK-NEXT:    ld1b { z5.b }, p1/z, [x1, x20]
-; CHECK-NEXT:    ld1b { z6.b }, p1/z, [x21, x4]
-; CHECK-NEXT:    ld1b { z7.b }, p1/z, [x22, x4]
+; CHECK-NEXT:    ld1b { z6.b }, p1/z, [x21, x3]
+; CHECK-NEXT:    ld1b { z7.b }, p1/z, [x22, x3]
 ; CHECK-NEXT:    uunpklo z16.h, z4.b
 ; CHECK-NEXT:    uunpkhi z4.h, z4.b
 ; CHECK-NEXT:    uunpkhi z17.h, z5.b
 ; CHECK-NEXT:    uunpklo z5.h, z5.b
 ; CHECK-NEXT:    uunpkhi z18.h, z6.b
 ; CHECK-NEXT:    uunpklo z6.h, z6.b
 ; CHECK-NEXT:    uunpklo z19.h, z7.b
 ; CHECK-NEXT:    uunpkhi z7.h, z7.b
-; CHECK-NEXT:    mul z6.h, z1.h, z6.h
-; CHECK-NEXT:    mul z18.h, z1.h, z18.h
-; CHECK-NEXT:    mul z7.h, z1.h, z7.h
-; CHECK-NEXT:    mul z19.h, z1.h, z19.h
-; CHECK-NEXT:    mla z18.h, p0/m, z0.h, z4.h
-; CHECK-NEXT:    mla z6.h, p0/m, z0.h, z16.h
-; CHECK-NEXT:    mla z19.h, p0/m, z0.h, z5.h
-; CHECK-NEXT:    mla z7.h, p0/m, z0.h, z17.h
-; CHECK-NEXT:    mul z4.h, z6.h, z3.h
-; CHECK-NEXT:    mul z5.h, z18.h, z3.h
-; CHECK-NEXT:    mla z5.h, p0/m, z7.h, z2.h
-; CHECK-NEXT:    mla z4.h, p0/m, z19.h, z2.h
+; CHECK-NEXT:    mul z6.h, z3.h, z6.h
+; CHECK-NEXT:    mul z18.h, z3.h, z18.h
+; CHECK-NEXT:    mul z7.h, z3.h, z7.h
+; CHECK-NEXT:    mul z19.h, z3.h, z19.h
+; CHECK-NEXT:    mla z18.h, p0/m, z2.h, z4.h
+; CHECK-NEXT:    mla z6.h, p0/m, z2.h, z16.h
+; CHECK-NEXT:    mla z19.h, p0/m, z2.h, z5.h
+; CHECK-NEXT:    mla z7.h, p0/m, z2.h, z17.h
+; CHECK-NEXT:    mul z4.h, z6.h, z1.h
+; CHECK-NEXT:    mul z5.h, z18.h, z1.h
+; CHECK-NEXT:    mla z5.h, p0/m, z7.h, z0.h
+; CHECK-NEXT:    mla z4.h, p0/m, z19.h, z0.h
 ; CHECK-NEXT:    add z4.h, z4.h, #32 // =0x20
 ; CHECK-NEXT:    add z5.h, z5.h, #32 // =0x20
 ; CHECK-NEXT:    lsr z5.h, z5.h, #6
 ; CHECK-NEXT:    lsr z4.h, z4.h, #6
 ; CHECK-NEXT:    uzp1 z4.b, z4.b, z5.b
 ; CHECK-NEXT:    st1b { z4.b }, p1, [x5, x20]
 ; CHECK-NEXT:    add x20, x20, x6
 ; CHECK-NEXT:    whilelo p1.b, x20, x13
 ; CHECK-NEXT:    b.mi .LBB3_7
 ; CHECK-NEXT:    b .LBB3_3
 ; CHECK-NEXT:  .LBB3_8: // %for.body13.us.preheader
 ; CHECK-NEXT:    // in Loop: Header=BB3_4 Depth=1
 ; CHECK-NEXT:    mov x20, xzr
 ; CHECK-NEXT:  .LBB3_9: // %for.body13.us
 ; CHECK-NEXT:    // Parent Loop BB3_4 Depth=1
 ; CHECK-NEXT:    // => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    add x21, x3, x20
+; CHECK-NEXT:    add x21, x4, x20
 ; CHECK-NEXT:    ldrb w22, [x19, x20]
 ; CHECK-NEXT:    add x23, x19, x20
 ; CHECK-NEXT:    ldrb w24, [x21]
 ; CHECK-NEXT:    ldrb w23, [x23, #1]
 ; CHECK-NEXT:    mul w22, w14, w22
 ; CHECK-NEXT:    ldrb w21, [x21, #1]
 ; CHECK-NEXT:    mul w24, w14, w24
 ; CHECK-NEXT:    madd w22, w9, w23, w22

llvm/test/Transforms/CodeGenPrepare/AArch64/sink-free-instructions.ll

 declare <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8>, <8 x i8>)
 
 ; The insertelement should be inserted before shufflevector, otherwise 'does not dominate all uses' error will occur.
 define <4 x i32> @sink_insertelement(i16 %e, i8 %f) {
 ; CHECK-LABEL: @sink_insertelement(
 ; CHECK-NEXT:  for.cond4.preheader.lr.ph:
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[F:%.*]], 0
 ; CHECK-NEXT:    [[CONV25:%.*]] = sext i16 [[E:%.*]] to i32
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT143:%.*]] = insertelement <4 x i32> poison, i32 [[CONV25]], i32 0
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_COND4_PREHEADER_US_PREHEADER:%.*]], label [[FOR_COND4_PREHEADER_PREHEADER:%.*]]
 ; CHECK:       for.cond4.preheader.us.preheader:
-; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[CONV25]], i32 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT144:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP1:%.*]] = mul <4 x i32> zeroinitializer, [[BROADCAST_SPLAT144]]
-; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+; CHECK-NEXT:    [[BROADCAST_SPLAT144:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT143]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = mul <4 x i32> zeroinitializer, [[BROADCAST_SPLAT144]]
+; CHECK-NEXT:    ret <4 x i32> [[TMP0]]
 ; CHECK:       for.cond4.preheader.preheader:
 ; CHECK-NEXT:    ret <4 x i32> zeroinitializer
 ;
 for.cond4.preheader.lr.ph:
   %cmp = icmp slt i8 %f, 0
   %conv25 = sext i16 %e to i32
   %broadcast.splatinsert143 = insertelement <4 x i32> poison, i32 %conv25, i32 0
   br i1 %cmp, label %for.cond4.preheader.us.preheader, label %for.cond4.preheader.preheader