[DAG][AArch64][SVE] Fix VLS mulh code generation

[DavidTruby]

Summary

Introducing lowering for extending loads regressed an optimisation in
VLS codegen. Since the extend is merged into the load before the mulh is
generated, a standard mul followed by a lsr and truncating store is used
instead. This patch causes the mulh to be generated before the extends
are merged into the loads.

[paulwalker-arm]

Whilst not wrong this looks like something that should be canonicalised earlier. Is it possible to add this logic to SelectionDAG::getVectorShuffle() and utilise SelectionDAG::getConstant()? I ask because there are several functions of isConstOrConstSplat's ilk and we don't really want to have to add similar support to all of them.

[DavidTruby]

In D116737#3225594, @paulwalker-arm wrote:

Whilst not wrong this looks like something that should be canonicalised earlier. Is it possible to add this logic to SelectionDAG::getVectorShuffle() and utilise SelectionDAG::getConstant()? I ask because there are several functions of isConstOrConstSplat's ilk and we don't really want to have to add similar support to all of them.

I'm not 100% sure what you mean here so I thought I ought to check; do you mean that SelectionDAG::getVectorShuffle() should return a BUILD_VECTOR generated using SelectionDAG::getConstant() in cases like this where the vector shuffle is a constant splat?

[paulwalker-arm]

Yes, that was what I was thinking, because it seemed odd that such a constant representation exists by this point in the pipeline. However, I can see that it's coming from the test itself and so I've changed my mind and am now wondering if there actually a problem that needs solving here?

The tests are using a non-standard immediate representation (aka "fixed vector shuffle of an inserted constant") that LLVM would have converted to a constant vector (aka <i16 8, i16 8,....) long before it gets to the code generator. This is backed up by running the test through clang, which results in smul when expected.

This makes me think that to protect the MULS/MULU functionality, you just need to update the test itself to remove the non-standard representation.

[DavidTruby]

I changed the tests as suggested above in rG0af1808f9b99 and this issue does go away as suggested so I am closing this revision.

llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp

if (CN && (UndefElements.none() || AllowUndefs)) {
       EVT CVT = CN->getValueType(0);
       EVT NSVT = N.getValueType().getScalarType();
       assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension");
       if (AllowTruncation || (CVT == NSVT))
         return CN;
     }
   }
 
+  if (ShuffleVectorSDNode *SV = dyn_cast<ShuffleVectorSDNode>(N)) {
+    if (!SV->isSplat())
+      return nullptr;
+
+    if (SV->getOperand(0).getOpcode() != ISD::INSERT_VECTOR_ELT)
+      return nullptr;
+    
+    SDValue IVE = SV->getOperand(0);
+    if (IVE->getConstantOperandVal(2) != SV->getSplatIndex())
+      return nullptr;
+
+    ConstantSDNode *CN = dyn_cast<ConstantSDNode>(IVE->getOperand(1));
+    return CN;
+  }
+
   return nullptr;
 }
 
 ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
                                           bool AllowUndefs,
                                           bool AllowTruncation) {
   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N))
     return CN;

llvm/test/CodeGen/AArch64/sve-fixed-length-int-mulh.ll

 ; RUN: llc -aarch64-sve-vector-bits-min=128  < %s | FileCheck %s -D#VBYTES=16  -check-prefix=NO_SVE
-; RUN: llc -aarch64-sve-vector-bits-min=256  < %s | FileCheck %s -D#VBYTES=32  -check-prefixes=CHECK,VBITS_EQ_256
+; RUN: llc -aarch64-sve-vector-bits-min=256  < %s | FileCheck %s -D#VBYTES=32  -check-prefixes=CHECK,VBITS_GE_256
 ; RUN: llc -aarch64-sve-vector-bits-min=384  < %s | FileCheck %s -D#VBYTES=32  -check-prefixes=CHECK
-; RUN: llc -aarch64-sve-vector-bits-min=512  < %s | FileCheck %s -D#VBYTES=64  -check-prefixes=CHECK,VBITS_GE_512,VBITS_EQ_512
+; RUN: llc -aarch64-sve-vector-bits-min=512  < %s | FileCheck %s -D#VBYTES=64  -check-prefixes=CHECK,VBITS_GE_512
 ; RUN: llc -aarch64-sve-vector-bits-min=640  < %s | FileCheck %s -D#VBYTES=64  -check-prefixes=CHECK,VBITS_GE_512
 ; RUN: llc -aarch64-sve-vector-bits-min=768  < %s | FileCheck %s -D#VBYTES=64  -check-prefixes=CHECK,VBITS_GE_512
 ; RUN: llc -aarch64-sve-vector-bits-min=896  < %s | FileCheck %s -D#VBYTES=64  -check-prefixes=CHECK,VBITS_GE_512
-; RUN: llc -aarch64-sve-vector-bits-min=1024 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024,VBITS_EQ_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1024 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
 ; RUN: llc -aarch64-sve-vector-bits-min=1152 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
 ; RUN: llc -aarch64-sve-vector-bits-min=1280 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
 ; RUN: llc -aarch64-sve-vector-bits-min=1408 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
 ; RUN: llc -aarch64-sve-vector-bits-min=1536 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
 ; RUN: llc -aarch64-sve-vector-bits-min=1664 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
 ; RUN: llc -aarch64-sve-vector-bits-min=1792 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
 ; RUN: llc -aarch64-sve-vector-bits-min=1920 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
 ; RUN: llc -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -D#VBYTES=256 -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024,VBITS_GE_2048
; CHECK-NEXT:    ret
   %2 = sext <16 x i8> %op2 to <16 x i16>
   %mul = mul <16 x i16> %1, %2
   %shr = lshr <16 x i16> %mul, %splat
   %res = trunc <16 x i16> %shr to <16 x i8>
   ret <16 x i8> %res
 }
 
 define void @smulh_v32i8(<32 x i8>* %a, <32 x i8>* %b) #0 {
-; VBITS_EQ_256-LABEL: smulh_v32i8:
-; VBITS_EQ_256:       // %bb.0:
-; VBITS_EQ_256-NEXT:    ptrue p0.b, vl32
-; VBITS_EQ_256-NEXT:    ld1b { z0.b }, p0/z, [x0]
-; VBITS_EQ_256-NEXT:    ld1b { z1.b }, p0/z, [x1]
-; VBITS_EQ_256-NEXT:    smulh z0.b, p0/m, z0.b, z1.b
-; VBITS_EQ_256-NEXT:    st1b { z0.b }, p0, [x0]
-; VBITS_EQ_256-NEXT:    ret
-;
-; VBITS_GE_512-LABEL: smulh_v32i8:
-; VBITS_GE_512:       // %bb.0:
-; VBITS_GE_512-NEXT:    ptrue p0.h, vl32
-; VBITS_GE_512-NEXT:    ld1sb { z0.h }, p0/z, [x0]
-; VBITS_GE_512-NEXT:    ld1sb { z1.h }, p0/z, [x1]
-; VBITS_GE_512-NEXT:    mul z0.h, p0/m, z0.h, z1.h
-; VBITS_GE_512-NEXT:    lsr z0.h, p0/m, z0.h, #8
-; VBITS_GE_512-NEXT:    st1b { z0.h }, p0, [x0]
-; VBITS_GE_512-NEXT:    ret
-
+; VBITS_GE_256-LABEL: smulh_v32i8:
+; VBITS_GE_256:       // %bb.0:
+; VBITS_GE_256-NEXT:    ptrue p0.b, vl32
+; VBITS_GE_256-NEXT:    ld1b { z0.b }, p0/z, [x0]
+; VBITS_GE_256-NEXT:    ld1b { z1.b }, p0/z, [x1]
+; VBITS_GE_256-NEXT:    smulh z0.b, p0/m, z0.b, z1.b
+; VBITS_GE_256-NEXT:    st1b { z0.b }, p0, [x0]
+; VBITS_GE_256-NEXT:    ret
   %op1 = load <32 x i8>, <32 x i8>* %a
   %op2 = load <32 x i8>, <32 x i8>* %b
   %insert = insertelement <32 x i16> undef, i16 8, i64 0
   %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer
   %1 = sext <32 x i8> %op1 to <32 x i16>
   %2 = sext <32 x i8> %op2 to <32 x i16>
   %mul = mul <32 x i16> %1, %2
   %shr = lshr <32 x i16> %mul, %splat
   %res = trunc <32 x i16> %shr to <32 x i8>
   store <32 x i8> %res, <32 x i8>* %a
   ret void
 }
 
 define void @smulh_v64i8(<64 x i8>* %a, <64 x i8>* %b) #0 {
-; VBITS_EQ_512-LABEL: smulh_v64i8:
-; VBITS_EQ_512:       // %bb.0:
-; VBITS_EQ_512-NEXT:    ptrue p0.b, vl64
-; VBITS_EQ_512-NEXT:    ld1b { z0.b }, p0/z, [x0]
-; VBITS_EQ_512-NEXT:    ld1b { z1.b }, p0/z, [x1]
-; VBITS_EQ_512-NEXT:    smulh z0.b, p0/m, z0.b, z1.b
-; VBITS_EQ_512-NEXT:    st1b { z0.b }, p0, [x0]
-; VBITS_EQ_512-NEXT:    ret
-;
-; VBITS_GE_1024-LABEL: smulh_v64i8:
-; VBITS_GE_1024:       // %bb.0:
-; VBITS_GE_1024-NEXT:    ptrue p0.h, vl64
-; VBITS_GE_1024-NEXT:    ld1sb { z0.h }, p0/z, [x0]
-; VBITS_GE_1024-NEXT:    ld1sb { z1.h }, p0/z, [x1]
-; VBITS_GE_1024-NEXT:    mul z0.h, p0/m, z0.h, z1.h
-; VBITS_GE_1024-NEXT:    lsr z0.h, p0/m, z0.h, #8
-; VBITS_GE_1024-NEXT:    st1b { z0.h }, p0, [x0]
-; VBITS_GE_1024-NEXT:    ret
-
+; VBITS_GE_512-LABEL: smulh_v64i8:
+; VBITS_GE_512:       // %bb.0:
+; VBITS_GE_512-NEXT:    ptrue p0.b, vl64
+; VBITS_GE_512-NEXT:    ld1b { z0.b }, p0/z, [x0]
+; VBITS_GE_512-NEXT:    ld1b { z1.b }, p0/z, [x1]
+; VBITS_GE_512-NEXT:    smulh z0.b, p0/m, z0.b, z1.b
+; VBITS_GE_512-NEXT:    st1b { z0.b }, p0, [x0]
+; VBITS_GE_512-NEXT:    ret
   %op1 = load <64 x i8>, <64 x i8>* %a
   %op2 = load <64 x i8>, <64 x i8>* %b
   %insert = insertelement <64 x i16> undef, i16 8, i64 0
   %splat = shufflevector <64 x i16> %insert, <64 x i16> undef, <64 x i32> zeroinitializer
   %1 = sext <64 x i8> %op1 to <64 x i16>
   %2 = sext <64 x i8> %op2 to <64 x i16>
   %mul = mul <64 x i16> %1, %2
   %shr = lshr <64 x i16> %mul, %splat
   %res = trunc <64 x i16> %shr to <64 x i8>
   store <64 x i8> %res, <64 x i8>* %a
   ret void
 }
 
 define void @smulh_v128i8(<128 x i8>* %a, <128 x i8>* %b) #0 {
-; VBITS_EQ_1024-LABEL: smulh_v128i8:
-; VBITS_EQ_1024:       // %bb.0:
-; VBITS_EQ_1024-NEXT:    ptrue p0.b, vl128
-; VBITS_EQ_1024-NEXT:    ld1b { z0.b }, p0/z, [x0]
-; VBITS_EQ_1024-NEXT:    ld1b { z1.b }, p0/z, [x1]
-; VBITS_EQ_1024-NEXT:    smulh z0.b, p0/m, z0.b, z1.b
-; VBITS_EQ_1024-NEXT:    st1b { z0.b }, p0, [x0]
-; VBITS_EQ_1024-NEXT:    ret
-;
-; VBITS_GE_2048-LABEL: smulh_v128i8:
-; VBITS_GE_2048:       // %bb.0:
-; VBITS_GE_2048-NEXT:    ptrue p0.h, vl128
-; VBITS_GE_2048-NEXT:    ld1sb { z0.h }, p0/z, [x0]
-; VBITS_GE_2048-NEXT:    ld1sb { z1.h }, p0/z, [x1]
-; VBITS_GE_2048-NEXT:    mul z0.h, p0/m, z0.h, z1.h
-; VBITS_GE_2048-NEXT:    lsr z0.h, p0/m, z0.h, #8
-; VBITS_GE_2048-NEXT:    st1b { z0.h }, p0, [x0]
-; VBITS_GE_2048-NEXT:    ret
-
+; VBITS_GE_1024-LABEL: smulh_v128i8:
+; VBITS_GE_1024:       // %bb.0:
+; VBITS_GE_1024-NEXT:    ptrue p0.b, vl128
+; VBITS_GE_1024-NEXT:    ld1b { z0.b }, p0/z, [x0]
+; VBITS_GE_1024-NEXT:    ld1b { z1.b }, p0/z, [x1]
+; VBITS_GE_1024-NEXT:    smulh z0.b, p0/m, z0.b, z1.b
+; VBITS_GE_1024-NEXT:    st1b { z0.b }, p0, [x0]
+; VBITS_GE_1024-NEXT:    ret
   %op1 = load <128 x i8>, <128 x i8>* %a
   %op2 = load <128 x i8>, <128 x i8>* %b
   %insert = insertelement <128 x i16> undef, i16 8, i64 0
   %splat = shufflevector <128 x i16> %insert, <128 x i16> undef, <128 x i32> zeroinitializer
   %1 = sext <128 x i8> %op1 to <128 x i16>
   %2 = sext <128 x i8> %op2 to <128 x i16>
   %mul = mul <128 x i16> %1, %2
   %shr = lshr <128 x i16> %mul, %splat
; CHECK-NEXT:    ret
   %2 = sext <8 x i16> %op2 to <8 x i32>
   %mul = mul <8 x i32> %1, %2
   %shr = lshr <8 x i32> %mul, %splat
   %res = trunc <8 x i32> %shr to <8 x i16>
   ret <8 x i16> %res
 }
 
 define void @smulh_v16i16(<16 x i16>* %a, <16 x i16>* %b) #0 {
-; VBITS_EQ_256-LABEL: smulh_v16i16:
-; VBITS_EQ_256:       // %bb.0:
-; VBITS_EQ_256-NEXT:    ptrue p0.h, vl16
-; VBITS_EQ_256-NEXT:    ld1h { z0.h }, p0/z, [x0]
-; VBITS_EQ_256-NEXT:    ld1h { z1.h }, p0/z, [x1]
-; VBITS_EQ_256-NEXT:    smulh z0.h, p0/m, z0.h, z1.h
-; VBITS_EQ_256-NEXT:    st1h { z0.h }, p0, [x0]
-; VBITS_EQ_256-NEXT:    ret
-;
-; VBITS_GE_512-LABEL: smulh_v16i16:
-; VBITS_GE_512:       // %bb.0:
-; VBITS_GE_512-NEXT:    ptrue p0.s, vl16
-; VBITS_GE_512-NEXT:    ld1sh { z0.s }, p0/z, [x0]
-; VBITS_GE_512-NEXT:    ld1sh { z1.s }, p0/z, [x1]
-; VBITS_GE_512-NEXT:    mul z0.s, p0/m, z0.s, z1.s
-; VBITS_GE_512-NEXT:    lsr z0.s, p0/m, z0.s, #16
-; VBITS_GE_512-NEXT:    st1h { z0.s }, p0, [x0]
-; VBITS_GE_512-NEXT:    ret
-
+; VBITS_GE_256-LABEL: smulh_v16i16:
+; VBITS_GE_256:       // %bb.0:
+; VBITS_GE_256-NEXT:    ptrue p0.h, vl16
+; VBITS_GE_256-NEXT:    ld1h { z0.h }, p0/z, [x0]
+; VBITS_GE_256-NEXT:    ld1h { z1.h }, p0/z, [x1]
+; VBITS_GE_256-NEXT:    smulh z0.h, p0/m, z0.h, z1.h
+; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x0]
+; VBITS_GE_256-NEXT:    ret
   %op1 = load <16 x i16>, <16 x i16>* %a
   %op2 = load <16 x i16>, <16 x i16>* %b
   %insert = insertelement <16 x i32> undef, i32 16, i64 0
   %splat = shufflevector <16 x i32> %insert, <16 x i32> undef, <16 x i32> zeroinitializer
   %1 = sext <16 x i16> %op1 to <16 x i32>
   %2 = sext <16 x i16> %op2 to <16 x i32>
   %mul = mul <16 x i32> %1, %2
   %shr = lshr <16 x i32> %mul, %splat
   %res = trunc <16 x i32> %shr to <16 x i16>
   store <16 x i16> %res, <16 x i16>* %a
   ret void
 }
 
 define void @smulh_v32i16(<32 x i16>* %a, <32 x i16>* %b) #0 {
-; VBITS_EQ_512-LABEL: smulh_v32i16:
-; VBITS_EQ_512:       // %bb.0:
-; VBITS_EQ_512-NEXT:    ptrue p0.h, vl32
-; VBITS_EQ_512-NEXT:    ld1h { z0.h }, p0/z, [x0]
-; VBITS_EQ_512-NEXT:    ld1h { z1.h }, p0/z, [x1]
-; VBITS_EQ_512-NEXT:    smulh z0.h, p0/m, z0.h, z1.h
-; VBITS_EQ_512-NEXT:    st1h { z0.h }, p0, [x0]
-; VBITS_EQ_512-NEXT:    ret
-;
-; VBITS_GE_1024-LABEL: smulh_v32i16:
-; VBITS_GE_1024:       // %bb.0:
-; VBITS_GE_1024-NEXT:    ptrue p0.s, vl32
-; VBITS_GE_1024-NEXT:    ld1sh { z0.s }, p0/z, [x0]
-; VBITS_GE_1024-NEXT:    ld1sh { z1.s }, p0/z, [x1]
-; VBITS_GE_1024-NEXT:    mul z0.s, p0/m, z0.s, z1.s
-; VBITS_GE_1024-NEXT:    lsr z0.s, p0/m, z0.s, #16
-; VBITS_GE_1024-NEXT:    st1h { z0.s }, p0, [x0]
-; VBITS_GE_1024-NEXT:    ret
-
+; VBITS_GE_512-LABEL: smulh_v32i16:
+; VBITS_GE_512:       // %bb.0:
+; VBITS_GE_512-NEXT:    ptrue p0.h, vl32
+; VBITS_GE_512-NEXT:    ld1h { z0.h }, p0/z, [x0]
+; VBITS_GE_512-NEXT:    ld1h { z1.h }, p0/z, [x1]
+; VBITS_GE_512-NEXT:    smulh z0.h, p0/m, z0.h, z1.h
+; VBITS_GE_512-NEXT:    st1h { z0.h }, p0, [x0]
+; VBITS_GE_512-NEXT:    ret
   %op1 = load <32 x i16>, <32 x i16>* %a
   %op2 = load <32 x i16>, <32 x i16>* %b
   %insert = insertelement <32 x i32> undef, i32 16, i64 0
   %splat = shufflevector <32 x i32> %insert, <32 x i32> undef, <32 x i32> zeroinitializer
   %1 = sext <32 x i16> %op1 to <32 x i32>
   %2 = sext <32 x i16> %op2 to <32 x i32>
   %mul = mul <32 x i32> %1, %2
   %shr = lshr <32 x i32> %mul, %splat
   %res = trunc <32 x i32> %shr to <32 x i16>
   store <32 x i16> %res, <32 x i16>* %a
   ret void
 }
 
 define void @smulh_v64i16(<64 x i16>* %a, <64 x i16>* %b) #0 {
-; VBITS_EQ_1024-LABEL: smulh_v64i16:
-; VBITS_EQ_1024:       // %bb.0:
-; VBITS_EQ_1024-NEXT:    ptrue p0.h, vl64
-; VBITS_EQ_1024-NEXT:    ld1h { z0.h }, p0/z, [x0]
-; VBITS_EQ_1024-NEXT:    ld1h { z1.h }, p0/z, [x1]
-; VBITS_EQ_1024-NEXT:    smulh z0.h, p0/m, z0.h, z1.h
-; VBITS_EQ_1024-NEXT:    st1h { z0.h }, p0, [x0]
-; VBITS_EQ_1024-NEXT:    ret
-;
-; VBITS_GE_2048-LABEL: smulh_v64i16:
-; VBITS_GE_2048:       // %bb.0:
-; VBITS_GE_2048-NEXT:    ptrue p0.s, vl64
-; VBITS_GE_2048-NEXT:    ld1sh { z0.s }, p0/z, [x0]
-; VBITS_GE_2048-NEXT:    ld1sh { z1.s }, p0/z, [x1]
-; VBITS_GE_2048-NEXT:    mul z0.s, p0/m, z0.s, z1.s
-; VBITS_GE_2048-NEXT:    lsr z0.s, p0/m, z0.s, #16
-; VBITS_GE_2048-NEXT:    st1h { z0.s }, p0, [x0]
-; VBITS_GE_2048-NEXT:    ret
-
+; VBITS_GE_1024-LABEL: smulh_v64i16:
+; VBITS_GE_1024:       // %bb.0:
+; VBITS_GE_1024-NEXT:    ptrue p0.h, vl64
+; VBITS_GE_1024-NEXT:    ld1h { z0.h }, p0/z, [x0]
+; VBITS_GE_1024-NEXT:    ld1h { z1.h }, p0/z, [x1]
+; VBITS_GE_1024-NEXT:    smulh z0.h, p0/m, z0.h, z1.h
+; VBITS_GE_1024-NEXT:    st1h { z0.h }, p0, [x0]
+; VBITS_GE_1024-NEXT:    ret
   %op1 = load <64 x i16>, <64 x i16>* %a
   %op2 = load <64 x i16>, <64 x i16>* %b
   %insert = insertelement <64 x i32> undef, i32 16, i64 0
   %splat = shufflevector <64 x i32> %insert, <64 x i32> undef, <64 x i32> zeroinitializer
   %1 = sext <64 x i16> %op1 to <64 x i32>
   %2 = sext <64 x i16> %op2 to <64 x i32>
   %mul = mul <64 x i32> %1, %2
   %shr = lshr <64 x i32> %mul, %splat
; CHECK-NEXT:    ret
   %2 = sext <4 x i32> %op2 to <4 x i64>
   %mul = mul <4 x i64> %1, %2
   %shr = lshr <4 x i64> %mul, %splat
   %res = trunc <4 x i64> %shr to <4 x i32>
   ret <4 x i32> %res
 }
 
 define void @smulh_v8i32(<8 x i32>* %a, <8 x i32>* %b) #0 {
-; VBITS_EQ_256-LABEL: smulh_v8i32:
-; VBITS_EQ_256:       // %bb.0:
-; VBITS_EQ_256-NEXT:    ptrue p0.s, vl8
-; VBITS_EQ_256-NEXT:    ld1w { z0.s }, p0/z, [x0]
-; VBITS_EQ_256-NEXT:    ld1w { z1.s }, p0/z, [x1]
-; VBITS_EQ_256-NEXT:    smulh z0.s, p0/m, z0.s, z1.s
-; VBITS_EQ_256-NEXT:    st1w { z0.s }, p0, [x0]
-; VBITS_EQ_256-NEXT:    ret
-;
-; VBITS_GE_512-LABEL: smulh_v8i32:
-; VBITS_GE_512:       // %bb.0:
-; VBITS_GE_512-NEXT:    ptrue p0.d, vl8
-; VBITS_GE_512-NEXT:    ld1sw { z0.d }, p0/z, [x0]
-; VBITS_GE_512-NEXT:    ld1sw { z1.d }, p0/z, [x1]
-; VBITS_GE_512-NEXT:    mul z0.d, p0/m, z0.d, z1.d
-; VBITS_GE_512-NEXT:    lsr z0.d, p0/m, z0.d, #32
-; VBITS_GE_512-NEXT:    st1w { z0.d }, p0, [x0]
-; VBITS_GE_512-NEXT:    ret
-
+; VBITS_GE_256-LABEL: smulh_v8i32:
+; VBITS_GE_256:       // %bb.0:
+; VBITS_GE_256-NEXT:    ptrue p0.s, vl8
+; VBITS_GE_256-NEXT:    ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_256-NEXT:    ld1w { z1.s }, p0/z, [x1]
+; VBITS_GE_256-NEXT:    smulh z0.s, p0/m, z0.s, z1.s
+; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x0]
+; VBITS_GE_256-NEXT:    ret
   %op1 = load <8 x i32>, <8 x i32>* %a
   %op2 = load <8 x i32>, <8 x i32>* %b
   %insert = insertelement <8 x i64> undef, i64 32, i64 0
   %splat = shufflevector <8 x i64> %insert, <8 x i64> undef, <8 x i32> zeroinitializer
   %1 = sext <8 x i32> %op1 to <8 x i64>
   %2 = sext <8 x i32> %op2 to <8 x i64>
   %mul = mul <8 x i64> %1, %2
   %shr = lshr <8 x i64> %mul, %splat
   %res = trunc <8 x i64> %shr to <8 x i32>
   store <8 x i32> %res, <8 x i32>* %a
   ret void
 }
 
 define void @smulh_v16i32(<16 x i32>* %a, <16 x i32>* %b) #0 {
-; VBITS_EQ_512-LABEL: smulh_v16i32:
-; VBITS_EQ_512:       // %bb.0:
-; VBITS_EQ_512-NEXT:    ptrue p0.s, vl16
-; VBITS_EQ_512-NEXT:    ld1w { z0.s }, p0/z, [x0]
-; VBITS_EQ_512-NEXT:    ld1w { z1.s }, p0/z, [x1]
-; VBITS_EQ_512-NEXT:    smulh z0.s, p0/m, z0.s, z1.s
-; VBITS_EQ_512-NEXT:    st1w { z0.s }, p0, [x0]
-; VBITS_EQ_512-NEXT:    ret
-;
-; VBITS_GE_1024-LABEL: smulh_v16i32:
-; VBITS_GE_1024:       // %bb.0:
-; VBITS_GE_1024-NEXT:    ptrue p0.d, vl16
-; VBITS_GE_1024-NEXT:    ld1sw { z0.d }, p0/z, [x0]
-; VBITS_GE_1024-NEXT:    ld1sw { z1.d }, p0/z, [x1]
-; VBITS_GE_1024-NEXT:    mul z0.d, p0/m, z0.d, z1.d
-; VBITS_GE_1024-NEXT:    lsr z0.d, p0/m, z0.d, #32
-; VBITS_GE_1024-NEXT:    st1w { z0.d }, p0, [x0]
-; VBITS_GE_1024-NEXT:    ret
-
+; VBITS_GE_512-LABEL: smulh_v16i32:
+; VBITS_GE_512:       // %bb.0:
+; VBITS_GE_512-NEXT:    ptrue p0.s, vl16
+; VBITS_GE_512-NEXT:    ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_512-NEXT:    ld1w { z1.s }, p0/z, [x1]
+; VBITS_GE_512-NEXT:    smulh z0.s, p0/m, z0.s, z1.s
+; VBITS_GE_512-NEXT:    st1w { z0.s }, p0, [x0]
+; VBITS_GE_512-NEXT:    ret
   %op1 = load <16 x i32>, <16 x i32>* %a
   %op2 = load <16 x i32>, <16 x i32>* %b
   %insert = insertelement <16 x i64> undef, i64 32, i64 0
   %splat = shufflevector <16 x i64> %insert, <16 x i64> undef, <16 x i32> zeroinitializer
   %1 = sext <16 x i32> %op1 to <16 x i64>
   %2 = sext <16 x i32> %op2 to <16 x i64>
   %mul = mul <16 x i64> %1, %2
   %shr = lshr <16 x i64> %mul, %splat
   %res = trunc <16 x i64> %shr to <16 x i32>
   store <16 x i32> %res, <16 x i32>* %a
   ret void
 }
 
 define void @smulh_v32i32(<32 x i32>* %a, <32 x i32>* %b) #0 {
-; VBITS_EQ_1024-LABEL: smulh_v32i32:
-; VBITS_EQ_1024:       // %bb.0:
-; VBITS_EQ_1024-NEXT:    ptrue p0.s, vl32
-; VBITS_EQ_1024-NEXT:    ld1w { z0.s }, p0/z, [x0]
-; VBITS_EQ_1024-NEXT:    ld1w { z1.s }, p0/z, [x1]
-; VBITS_EQ_1024-NEXT:    smulh z0.s, p0/m, z0.s, z1.s
-; VBITS_EQ_1024-NEXT:    st1w { z0.s }, p0, [x0]
-; VBITS_EQ_1024-NEXT:    ret
-;
-; VBITS_GE_2048-LABEL: smulh_v32i32:
-; VBITS_GE_2048:       // %bb.0:
-; VBITS_GE_2048-NEXT:    ptrue p0.d, vl32
-; VBITS_GE_2048-NEXT:    ld1sw { z0.d }, p0/z, [x0]
-; VBITS_GE_2048-NEXT:    ld1sw { z1.d }, p0/z, [x1]
-; VBITS_GE_2048-NEXT:    mul z0.d, p0/m, z0.d, z1.d
-; VBITS_GE_2048-NEXT:    lsr z0.d, p0/m, z0.d, #32
-; VBITS_GE_2048-NEXT:    st1w { z0.d }, p0, [x0]
-; VBITS_GE_2048-NEXT:    ret
-
+; VBITS_GE_1024-LABEL: smulh_v32i32:
+; VBITS_GE_1024:       // %bb.0:
+; VBITS_GE_1024-NEXT:    ptrue p0.s, vl32
+; VBITS_GE_1024-NEXT:    ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_1024-NEXT:    ld1w { z1.s }, p0/z, [x1]
+; VBITS_GE_1024-NEXT:    smulh z0.s, p0/m, z0.s, z1.s
+; VBITS_GE_1024-NEXT:    st1w { z0.s }, p0, [x0]
+; VBITS_GE_1024-NEXT:    ret
   %op1 = load <32 x i32>, <32 x i32>* %a
   %op2 = load <32 x i32>, <32 x i32>* %b
   %insert = insertelement <32 x i64> undef, i64 32, i64 0
   %splat = shufflevector <32 x i64> %insert, <32 x i64> undef, <32 x i32> zeroinitializer
   %1 = sext <32 x i32> %op1 to <32 x i64>
   %2 = sext <32 x i32> %op2 to <32 x i64>
   %mul = mul <32 x i64> %1, %2
   %shr = lshr <32 x i64> %mul, %splat
; CHECK-NEXT:    ret
   %2 = zext <16 x i8> %op2 to <16 x i16>
   %mul = mul <16 x i16> %1, %2
   %shr = lshr <16 x i16> %mul, %splat
   %res = trunc <16 x i16> %shr to <16 x i8>
   ret <16 x i8> %res
 }
 
 define void @umulh_v32i8(<32 x i8>* %a, <32 x i8>* %b) #0 {
-; VBITS_EQ_256-LABEL: umulh_v32i8:
-; VBITS_EQ_256:       // %bb.0:
-; VBITS_EQ_256-NEXT:    ptrue p0.b, vl32
-; VBITS_EQ_256-NEXT:    ld1b { z0.b }, p0/z, [x0]
-; VBITS_EQ_256-NEXT:    ld1b { z1.b }, p0/z, [x1]
-; VBITS_EQ_256-NEXT:    umulh z0.b, p0/m, z0.b, z1.b
-; VBITS_EQ_256-NEXT:    st1b { z0.b }, p0, [x0]
-; VBITS_EQ_256-NEXT:    ret
-;
-; VBITS_GE_512-LABEL: umulh_v32i8:
-; VBITS_GE_512:       // %bb.0:
-; VBITS_GE_512-NEXT:    ptrue p0.h, vl32
-; VBITS_GE_512-NEXT:    ld1b { z0.h }, p0/z, [x0]
-; VBITS_GE_512-NEXT:    ld1b { z1.h }, p0/z, [x1]
-; VBITS_GE_512-NEXT:    mul z0.h, p0/m, z0.h, z1.h
-; VBITS_GE_512-NEXT:    lsr z0.h, p0/m, z0.h, #8
-; VBITS_GE_512-NEXT:    st1b { z0.h }, p0, [x0]
-; VBITS_GE_512-NEXT:    ret
-
+; VBITS_GE_256-LABEL: umulh_v32i8:
+; VBITS_GE_256:       // %bb.0:
+; VBITS_GE_256-NEXT:    ptrue p0.b, vl32
+; VBITS_GE_256-NEXT:    ld1b { z0.b }, p0/z, [x0]
+; VBITS_GE_256-NEXT:    ld1b { z1.b }, p0/z, [x1]
+; VBITS_GE_256-NEXT:    umulh z0.b, p0/m, z0.b, z1.b
+; VBITS_GE_256-NEXT:    st1b { z0.b }, p0, [x0]
+; VBITS_GE_256-NEXT:    ret
   %op1 = load <32 x i8>, <32 x i8>* %a
   %op2 = load <32 x i8>, <32 x i8>* %b
   %insert = insertelement <32 x i16> undef, i16 8, i64 0
   %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer
   %1 = zext <32 x i8> %op1 to <32 x i16>
   %2 = zext <32 x i8> %op2 to <32 x i16>
   %mul = mul <32 x i16> %1, %2
   %shr = lshr <32 x i16> %mul, %splat
   %res = trunc <32 x i16> %shr to <32 x i8>
   store <32 x i8> %res, <32 x i8>* %a
   ret void
 }
 
 define void @umulh_v64i8(<64 x i8>* %a, <64 x i8>* %b) #0 {
-; VBITS_EQ_512-LABEL: umulh_v64i8:
-; VBITS_EQ_512:       // %bb.0:
-; VBITS_EQ_512-NEXT:    ptrue p0.b, vl64
-; VBITS_EQ_512-NEXT:    ld1b { z0.b }, p0/z, [x0]
-; VBITS_EQ_512-NEXT:    ld1b { z1.b }, p0/z, [x1]
-; VBITS_EQ_512-NEXT:    umulh z0.b, p0/m, z0.b, z1.b
-; VBITS_EQ_512-NEXT:    st1b { z0.b }, p0, [x0]
-; VBITS_EQ_512-NEXT:    ret
-;
-; VBITS_GE_1024-LABEL: umulh_v64i8:
-; VBITS_GE_1024:       // %bb.0:
-; VBITS_GE_1024-NEXT:    ptrue p0.h, vl64
-; VBITS_GE_1024-NEXT:    ld1b { z0.h }, p0/z, [x0]
-; VBITS_GE_1024-NEXT:    ld1b { z1.h }, p0/z, [x1]
-; VBITS_GE_1024-NEXT:    mul z0.h, p0/m, z0.h, z1.h
-; VBITS_GE_1024-NEXT:    lsr z0.h, p0/m, z0.h, #8
-; VBITS_GE_1024-NEXT:    st1b { z0.h }, p0, [x0]
-; VBITS_GE_1024-NEXT:    ret
-
+; VBITS_GE_512-LABEL: umulh_v64i8:
+; VBITS_GE_512:       // %bb.0:
+; VBITS_GE_512-NEXT:    ptrue p0.b, vl64
+; VBITS_GE_512-NEXT:    ld1b { z0.b }, p0/z, [x0]
+; VBITS_GE_512-NEXT:    ld1b { z1.b }, p0/z, [x1]
+; VBITS_GE_512-NEXT:    umulh z0.b, p0/m, z0.b, z1.b
+; VBITS_GE_512-NEXT:    st1b { z0.b }, p0, [x0]
+; VBITS_GE_512-NEXT:    ret
   %op1 = load <64 x i8>, <64 x i8>* %a
   %op2 = load <64 x i8>, <64 x i8>* %b
   %insert = insertelement <64 x i16> undef, i16 8, i64 0
   %splat = shufflevector <64 x i16> %insert, <64 x i16> undef, <64 x i32> zeroinitializer
   %1 = zext <64 x i8> %op1 to <64 x i16>
   %2 = zext <64 x i8> %op2 to <64 x i16>
   %mul = mul <64 x i16> %1, %2
   %shr = lshr <64 x i16> %mul, %splat
   %res = trunc <64 x i16> %shr to <64 x i8>
   store <64 x i8> %res, <64 x i8>* %a
   ret void
 }
 
 define void @umulh_v128i8(<128 x i8>* %a, <128 x i8>* %b) #0 {
-; VBITS_EQ_1024-LABEL: umulh_v128i8:
-; VBITS_EQ_1024:       // %bb.0:
-; VBITS_EQ_1024-NEXT:    ptrue p0.b, vl128
-; VBITS_EQ_1024-NEXT:    ld1b { z0.b }, p0/z, [x0]
-; VBITS_EQ_1024-NEXT:    ld1b { z1.b }, p0/z, [x1]
-; VBITS_EQ_1024-NEXT:    umulh z0.b, p0/m, z0.b, z1.b
-; VBITS_EQ_1024-NEXT:    st1b { z0.b }, p0, [x0]
-; VBITS_EQ_1024-NEXT:    ret
-;
-; VBITS_GE_2048-LABEL: umulh_v128i8:
-; VBITS_GE_2048:       // %bb.0:
-; VBITS_GE_2048-NEXT:    ptrue p0.h, vl128
-; VBITS_GE_2048-NEXT:    ld1b { z0.h }, p0/z, [x0]
-; VBITS_GE_2048-NEXT:    ld1b { z1.h }, p0/z, [x1]
-; VBITS_GE_2048-NEXT:    mul z0.h, p0/m, z0.h, z1.h
-; VBITS_GE_2048-NEXT:    lsr z0.h, p0/m, z0.h, #8
-; VBITS_GE_2048-NEXT:    st1b { z0.h }, p0, [x0]
-; VBITS_GE_2048-NEXT:    ret
-
+; VBITS_GE_1024-LABEL: umulh_v128i8:
+; VBITS_GE_1024:       // %bb.0:
+; VBITS_GE_1024-NEXT:    ptrue p0.b, vl128
+; VBITS_GE_1024-NEXT:    ld1b { z0.b }, p0/z, [x0]
+; VBITS_GE_1024-NEXT:    ld1b { z1.b }, p0/z, [x1]
+; VBITS_GE_1024-NEXT:    umulh z0.b, p0/m, z0.b, z1.b
+; VBITS_GE_1024-NEXT:    st1b { z0.b }, p0, [x0]
+; VBITS_GE_1024-NEXT:    ret
   %op1 = load <128 x i8>, <128 x i8>* %a
   %op2 = load <128 x i8>, <128 x i8>* %b
   %insert = insertelement <128 x i16> undef, i16 8, i64 0
   %splat = shufflevector <128 x i16> %insert, <128 x i16> undef, <128 x i32> zeroinitializer
   %1 = zext <128 x i8> %op1 to <128 x i16>
   %2 = zext <128 x i8> %op2 to <128 x i16>
   %mul = mul <128 x i16> %1, %2
   %shr = lshr <128 x i16> %mul, %splat
; CHECK-NEXT:    ret
   %2 = zext <8 x i16> %op2 to <8 x i32>
   %mul = mul <8 x i32> %1, %2
   %shr = lshr <8 x i32> %mul, %splat
   %res = trunc <8 x i32> %shr to <8 x i16>
   ret <8 x i16> %res
 }
 
 define void @umulh_v16i16(<16 x i16>* %a, <16 x i16>* %b) #0 {
-; VBITS_EQ_256-LABEL: umulh_v16i16:
-; VBITS_EQ_256:       // %bb.0:
-; VBITS_EQ_256-NEXT:    ptrue p0.h, vl16
-; VBITS_EQ_256-NEXT:    ld1h { z0.h }, p0/z, [x0]
-; VBITS_EQ_256-NEXT:    ld1h { z1.h }, p0/z, [x1]
-; VBITS_EQ_256-NEXT:    umulh z0.h, p0/m, z0.h, z1.h
-; VBITS_EQ_256-NEXT:    st1h { z0.h }, p0, [x0]
-; VBITS_EQ_256-NEXT:    ret
-;
-; VBITS_GE_512-LABEL: umulh_v16i16:
-; VBITS_GE_512:       // %bb.0:
-; VBITS_GE_512-NEXT:    ptrue p0.s, vl16
-; VBITS_GE_512-NEXT:    ld1h { z0.s }, p0/z, [x0]
-; VBITS_GE_512-NEXT:    ld1h { z1.s }, p0/z, [x1]
-; VBITS_GE_512-NEXT:    mul z0.s, p0/m, z0.s, z1.s
-; VBITS_GE_512-NEXT:    lsr z0.s, p0/m, z0.s, #16
-; VBITS_GE_512-NEXT:    st1h { z0.s }, p0, [x0]
-; VBITS_GE_512-NEXT:    ret
-
+; VBITS_GE_256-LABEL: umulh_v16i16:
+; VBITS_GE_256:       // %bb.0:
+; VBITS_GE_256-NEXT:    ptrue p0.h, vl16
+; VBITS_GE_256-NEXT:    ld1h { z0.h }, p0/z, [x0]
+; VBITS_GE_256-NEXT:    ld1h { z1.h }, p0/z, [x1]
+; VBITS_GE_256-NEXT:    umulh z0.h, p0/m, z0.h, z1.h
+; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x0]
+; VBITS_GE_256-NEXT:    ret
   %op1 = load <16 x i16>, <16 x i16>* %a
   %op2 = load <16 x i16>, <16 x i16>* %b
   %insert = insertelement <16 x i32> undef, i32 16, i64 0
   %splat = shufflevector <16 x i32> %insert, <16 x i32> undef, <16 x i32> zeroinitializer
   %1 = zext <16 x i16> %op1 to <16 x i32>
   %2 = zext <16 x i16> %op2 to <16 x i32>
   %mul = mul <16 x i32> %1, %2
   %shr = lshr <16 x i32> %mul, %splat
   %res = trunc <16 x i32> %shr to <16 x i16>
   store <16 x i16> %res, <16 x i16>* %a
   ret void
 }
 
 define void @umulh_v32i16(<32 x i16>* %a, <32 x i16>* %b) #0 {
-; VBITS_EQ_512-LABEL: umulh_v32i16:
-; VBITS_EQ_512:       // %bb.0:
-; VBITS_EQ_512-NEXT:    ptrue p0.h, vl32
-; VBITS_EQ_512-NEXT:    ld1h { z0.h }, p0/z, [x0]
-; VBITS_EQ_512-NEXT:    ld1h { z1.h }, p0/z, [x1]
-; VBITS_EQ_512-NEXT:    umulh z0.h, p0/m, z0.h, z1.h
-; VBITS_EQ_512-NEXT:    st1h { z0.h }, p0, [x0]
-; VBITS_EQ_512-NEXT:    ret
-;
-; VBITS_GE_1024-LABEL: umulh_v32i16:
-; VBITS_GE_1024:       // %bb.0:
-; VBITS_GE_1024-NEXT:    ptrue p0.s, vl32
-; VBITS_GE_1024-NEXT:    ld1h { z0.s }, p0/z, [x0]
-; VBITS_GE_1024-NEXT:    ld1h { z1.s }, p0/z, [x1]
-; VBITS_GE_1024-NEXT:    mul z0.s, p0/m, z0.s, z1.s
-; VBITS_GE_1024-NEXT:    lsr z0.s, p0/m, z0.s, #16
-; VBITS_GE_1024-NEXT:    st1h { z0.s }, p0, [x0]
-; VBITS_GE_1024-NEXT:    ret
-
+; VBITS_GE_512-LABEL: umulh_v32i16:
+; VBITS_GE_512:       // %bb.0:
+; VBITS_GE_512-NEXT:    ptrue p0.h, vl32
+; VBITS_GE_512-NEXT:    ld1h { z0.h }, p0/z, [x0]
+; VBITS_GE_512-NEXT:    ld1h { z1.h }, p0/z, [x1]
+; VBITS_GE_512-NEXT:    umulh z0.h, p0/m, z0.h, z1.h
+; VBITS_GE_512-NEXT:    st1h { z0.h }, p0, [x0]
+; VBITS_GE_512-NEXT:    ret
   %op1 = load <32 x i16>, <32 x i16>* %a
   %op2 = load <32 x i16>, <32 x i16>* %b
   %insert = insertelement <32 x i32> undef, i32 16, i64 0
   %splat = shufflevector <32 x i32> %insert, <32 x i32> undef, <32 x i32> zeroinitializer
   %1 = zext <32 x i16> %op1 to <32 x i32>
   %2 = zext <32 x i16> %op2 to <32 x i32>
   %mul = mul <32 x i32> %1, %2
   %shr = lshr <32 x i32> %mul, %splat
   %res = trunc <32 x i32> %shr to <32 x i16>
   store <32 x i16> %res, <32 x i16>* %a
   ret void
 }
 
 define void @umulh_v64i16(<64 x i16>* %a, <64 x i16>* %b) #0 {
-; VBITS_EQ_1024-LABEL: umulh_v64i16:
-; VBITS_EQ_1024:       // %bb.0:
-; VBITS_EQ_1024-NEXT:    ptrue p0.h, vl64
-; VBITS_EQ_1024-NEXT:    ld1h { z0.h }, p0/z, [x0]
-; VBITS_EQ_1024-NEXT:    ld1h { z1.h }, p0/z, [x1]
-; VBITS_EQ_1024-NEXT:    umulh z0.h, p0/m, z0.h, z1.h
-; VBITS_EQ_1024-NEXT:    st1h { z0.h }, p0, [x0]
-; VBITS_EQ_1024-NEXT:    ret
-;
-; VBITS_GE_2048-LABEL: umulh_v64i16:
-; VBITS_GE_2048:       // %bb.0:
-; VBITS_GE_2048-NEXT:    ptrue p0.s, vl64
-; VBITS_GE_2048-NEXT:    ld1h { z0.s }, p0/z, [x0]
-; VBITS_GE_2048-NEXT:    ld1h { z1.s }, p0/z, [x1]
-; VBITS_GE_2048-NEXT:    mul z0.s, p0/m, z0.s, z1.s
-; VBITS_GE_2048-NEXT:    lsr z0.s, p0/m, z0.s, #16
-; VBITS_GE_2048-NEXT:    st1h { z0.s }, p0, [x0]
-; VBITS_GE_2048-NEXT:    ret
-
+; VBITS_GE_1024-LABEL: umulh_v64i16:
+; VBITS_GE_1024:       // %bb.0:
+; VBITS_GE_1024-NEXT:    ptrue p0.h, vl64
+; VBITS_GE_1024-NEXT:    ld1h { z0.h }, p0/z, [x0]
+; VBITS_GE_1024-NEXT:    ld1h { z1.h }, p0/z, [x1]
+; VBITS_GE_1024-NEXT:    umulh z0.h, p0/m, z0.h, z1.h
+; VBITS_GE_1024-NEXT:    st1h { z0.h }, p0, [x0]
+; VBITS_GE_1024-NEXT:    ret
   %op1 = load <64 x i16>, <64 x i16>* %a
   %op2 = load <64 x i16>, <64 x i16>* %b
   %insert = insertelement <64 x i32> undef, i32 16, i64 0
   %splat = shufflevector <64 x i32> %insert, <64 x i32> undef, <64 x i32> zeroinitializer
   %1 = zext <64 x i16> %op1 to <64 x i32>
   %2 = zext <64 x i16> %op2 to <64 x i32>
   %mul = mul <64 x i32> %1, %2
   %shr = lshr <64 x i32> %mul, %splat
; CHECK-NEXT:    ret
   %2 = zext <4 x i32> %op2 to <4 x i64>
   %mul = mul <4 x i64> %1, %2
   %shr = lshr <4 x i64> %mul, %splat
   %res = trunc <4 x i64> %shr to <4 x i32>
   ret <4 x i32> %res
 }
 
 define void @umulh_v8i32(<8 x i32>* %a, <8 x i32>* %b) #0 {
-; VBITS_EQ_256-LABEL: umulh_v8i32:
-; VBITS_EQ_256:       // %bb.0:
-; VBITS_EQ_256-NEXT:    ptrue p0.s, vl8
-; VBITS_EQ_256-NEXT:    ld1w { z0.s }, p0/z, [x0]
-; VBITS_EQ_256-NEXT:    ld1w { z1.s }, p0/z, [x1]
-; VBITS_EQ_256-NEXT:    umulh z0.s, p0/m, z0.s, z1.s
-; VBITS_EQ_256-NEXT:    st1w { z0.s }, p0, [x0]
-; VBITS_EQ_256-NEXT:    ret
-;
-; VBITS_GE_512-LABEL: umulh_v8i32:
-; VBITS_GE_512:       // %bb.0:
-; VBITS_GE_512-NEXT:    ptrue p0.d, vl8
-; VBITS_GE_512-NEXT:    ld1w { z0.d }, p0/z, [x0]
-; VBITS_GE_512-NEXT:    ld1w { z1.d }, p0/z, [x1]
-; VBITS_GE_512-NEXT:    mul z0.d, p0/m, z0.d, z1.d
-; VBITS_GE_512-NEXT:    lsr z0.d, p0/m, z0.d, #32
-; VBITS_GE_512-NEXT:    st1w { z0.d }, p0, [x0]
-; VBITS_GE_512-NEXT:    ret
-
+; VBITS_GE_256-LABEL: umulh_v8i32:
+; VBITS_GE_256:       // %bb.0:
+; VBITS_GE_256-NEXT:    ptrue p0.s, vl8
+; VBITS_GE_256-NEXT:    ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_256-NEXT:    ld1w { z1.s }, p0/z, [x1]
+; VBITS_GE_256-NEXT:    umulh z0.s, p0/m, z0.s, z1.s
+; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x0]
+; VBITS_GE_256-NEXT:    ret
   %op1 = load <8 x i32>, <8 x i32>* %a
   %op2 = load <8 x i32>, <8 x i32>* %b
   %insert = insertelement <8 x i64> undef, i64 32, i64 0
   %splat = shufflevector <8 x i64> %insert, <8 x i64> undef, <8 x i32> zeroinitializer
   %1 = zext <8 x i32> %op1 to <8 x i64>
   %2 = zext <8 x i32> %op2 to <8 x i64>
   %mul = mul <8 x i64> %1, %2
   %shr = lshr <8 x i64> %mul, %splat
   %res = trunc <8 x i64> %shr to <8 x i32>
   store <8 x i32> %res, <8 x i32>* %a
   ret void
 }
 
 define void @umulh_v16i32(<16 x i32>* %a, <16 x i32>* %b) #0 {
-; VBITS_EQ_512-LABEL: umulh_v16i32:
-; VBITS_EQ_512:       // %bb.0:
-; VBITS_EQ_512-NEXT:    ptrue p0.s, vl16
-; VBITS_EQ_512-NEXT:    ld1w { z0.s }, p0/z, [x0]
-; VBITS_EQ_512-NEXT:    ld1w { z1.s }, p0/z, [x1]
-; VBITS_EQ_512-NEXT:    umulh z0.s, p0/m, z0.s, z1.s
-; VBITS_EQ_512-NEXT:    st1w { z0.s }, p0, [x0]
-; VBITS_EQ_512-NEXT:    ret
-;
-; VBITS_GE_1024-LABEL: umulh_v16i32:
-; VBITS_GE_1024:       // %bb.0:
-; VBITS_GE_1024-NEXT:    ptrue p0.d, vl16
-; VBITS_GE_1024-NEXT:    ld1w { z0.d }, p0/z, [x0]
-; VBITS_GE_1024-NEXT:    ld1w { z1.d }, p0/z, [x1]
-; VBITS_GE_1024-NEXT:    mul z0.d, p0/m, z0.d, z1.d
-; VBITS_GE_1024-NEXT:    lsr z0.d, p0/m, z0.d, #32
-; VBITS_GE_1024-NEXT:    st1w { z0.d }, p0, [x0]
-; VBITS_GE_1024-NEXT:    ret
-
+; VBITS_GE_512-LABEL: umulh_v16i32:
+; VBITS_GE_512:       // %bb.0:
+; VBITS_GE_512-NEXT:    ptrue p0.s, vl16
+; VBITS_GE_512-NEXT:    ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_512-NEXT:    ld1w { z1.s }, p0/z, [x1]
+; VBITS_GE_512-NEXT:    umulh z0.s, p0/m, z0.s, z1.s
+; VBITS_GE_512-NEXT:    st1w { z0.s }, p0, [x0]
+; VBITS_GE_512-NEXT:    ret
   %op1 = load <16 x i32>, <16 x i32>* %a
   %op2 = load <16 x i32>, <16 x i32>* %b
   %insert = insertelement <16 x i64> undef, i64 32, i64 0
   %splat = shufflevector <16 x i64> %insert, <16 x i64> undef, <16 x i32> zeroinitializer
   %1 = zext <16 x i32> %op1 to <16 x i64>
   %2 = zext <16 x i32> %op2 to <16 x i64>
   %mul = mul <16 x i64> %1, %2
   %shr = lshr <16 x i64> %mul, %splat
   %res = trunc <16 x i64> %shr to <16 x i32>
   store <16 x i32> %res, <16 x i32>* %a
   ret void
 }
 
 define void @umulh_v32i32(<32 x i32>* %a, <32 x i32>* %b) #0 {
-; VBITS_EQ_1024-LABEL: umulh_v32i32:
-; VBITS_EQ_1024:       // %bb.0:
-; VBITS_EQ_1024-NEXT:    ptrue p0.s, vl32
-; VBITS_EQ_1024-NEXT:    ld1w { z0.s }, p0/z, [x0]
-; VBITS_EQ_1024-NEXT:    ld1w { z1.s }, p0/z, [x1]
-; VBITS_EQ_1024-NEXT:    umulh z0.s, p0/m, z0.s, z1.s
-; VBITS_EQ_1024-NEXT:    st1w { z0.s }, p0, [x0]
-; VBITS_EQ_1024-NEXT:    ret
-;
-; VBITS_GE_2048-LABEL: umulh_v32i32:
-; VBITS_GE_2048:       // %bb.0:
-; VBITS_GE_2048-NEXT:    ptrue p0.d, vl32
-; VBITS_GE_2048-NEXT:    ld1w { z0.d }, p0/z, [x0]
-; VBITS_GE_2048-NEXT:    ld1w { z1.d }, p0/z, [x1]
-; VBITS_GE_2048-NEXT:    mul z0.d, p0/m, z0.d, z1.d
-; VBITS_GE_2048-NEXT:    lsr z0.d, p0/m, z0.d, #32
-; VBITS_GE_2048-NEXT:    st1w { z0.d }, p0, [x0]
-; VBITS_GE_2048-NEXT:    ret
-
+; VBITS_GE_1024-LABEL: umulh_v32i32:
+; VBITS_GE_1024:       // %bb.0:
+; VBITS_GE_1024-NEXT:    ptrue p0.s, vl32
+; VBITS_GE_1024-NEXT:    ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_1024-NEXT:    ld1w { z1.s }, p0/z, [x1]
+; VBITS_GE_1024-NEXT:    umulh z0.s, p0/m, z0.s, z1.s
+; VBITS_GE_1024-NEXT:    st1w { z0.s }, p0, [x0]
+; VBITS_GE_1024-NEXT:    ret
   %op1 = load <32 x i32>, <32 x i32>* %a
   %op2 = load <32 x i32>, <32 x i32>* %b
   %insert = insertelement <32 x i64> undef, i64 32, i64 0
   %splat = shufflevector <32 x i64> %insert, <32 x i64> undef, <32 x i32> zeroinitializer
   %1 = zext <32 x i32> %op1 to <32 x i64>
   %2 = zext <32 x i32> %op2 to <32 x i64>
   %mul = mul <32 x i64> %1, %2
   %shr = lshr <32 x i64> %mul, %splat