[llvm][sve] Lowering for VLS masked extending loads

[DavidTruby]

Summary

This extends the custom lowering for extending loads on
fixed length vectors in SVE to support masked extending loads.

The existing tests for correct behaviour of masked extending loads
exhibit bad code generation due to the legalistaion of i1 vectors.
They have been left as-is and new tests have been added that do not
exhibit this behaviour.

[paulwalker-arm]

llvm/test/CodeGen/AArch64/sve-fixed-length-masked-loads.ll

656–657

Why is p0 hardcoded? Should this be [[PG0]]?

Also I think these should be VBITS_GE_512-DAG as the order of the loads can be switched,

658

This sets PG but no following instructions use it.

659–660

PG3 and PG2 are not set within this function.

[DavidTruby]

Corrected tests to use the right PG values

[paulwalker-arm]

llvm/test/CodeGen/AArch64/sve-fixed-length-masked-loads.ll

664

Is this necessary? As in, why not just change %ap to have the same types as %bp.

As an extension to this, once %a is a <32 x i16> do you need %b? I ask because the computation of %mask could be %mask = icmp eq <32 x i16> %a, zeroinitializer. This latter point probably goes for the whole file to be honest.

[DavidTruby]

Modified new tests to use zeroinitializer rather than another loaded
pointer.

I will change all the remaining tests in this file and the equivalent
truncating store file to use this pattern instead if this is approved,
but I don't want to clutter this review or commit with unrelated test
changes so I'll do it as a separate commit.

llvm/test/CodeGen/AArch64/sve-fixed-length-masked-loads.ll

664

%ap needs to have the type <32 x i8> for us to test the masked extending load a few lines later; otherwise we're just performing a normal masked load which is already covered by other tests in this file.

[paulwalker-arm]

llvm/test/CodeGen/AArch64/sve-fixed-length-masked-loads.ll

664

Sure I get that but this test is testing a load that is sign extended, which corresponds to the last three lines of the test. The lines above the call to llvm.masked.load are just setting up the test and thus it's preferred to keep them as simple as possible.

Before the most recent change the function took the parameter <32 x i16>* %bp which can be used directly by the compare used to generate the mask and thus the zext can be removed.

[DavidTruby]

Update tests to remove extra argument

[DavidTruby]

@paulwalker-arm sorry I think I misunderstood your comments before,
I believe the form I've changed the tests to now should be more what
you were expecting?

[efriedma]

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

10879

Do we need a check here to make sure we don't create illegal masked loads after legalization?

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

static SDValue tryToFoldExtOfMaskedLoad(SelectionDAG &DAG,
                                         ISD::NodeType ExtOpc) {
   if (!N0.hasOneUse())
     return SDValue();
 
   MaskedLoadSDNode *Ld = dyn_cast<MaskedLoadSDNode>(N0);
   if (!Ld || Ld->getExtensionType() != ISD::NON_EXTLOAD)
     return SDValue();
 
-  if (!TLI.isLoadExtLegal(ExtLoadType, VT, Ld->getValueType(0)))
+  if (!TLI.isLoadExtLegalOrCustom(ExtLoadType, VT, Ld->getValueType(0)))
     return SDValue();
 
   if (!TLI.isVectorLoadExtDesirable(SDValue(N, 0)))
     return SDValue();
 
   SDLoc dl(Ld);
   SDValue PassThru = DAG.getNode(ExtOpc, dl, VT, Ld->getPassThru());
   SDValue NewLoad = DAG.getMaskedLoad(

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, DL, CmpVT,
                      {Pg, Op1, Op2, DAG.getCondCode(ISD::SETNE)});
 }
 
 // Convert all fixed length vector loads larger than NEON to masked_loads.
 SDValue AArch64TargetLowering::LowerFixedLengthVectorMLoadToSVE(
     SDValue Op, SelectionDAG &DAG) const {
   auto Load = cast<MaskedLoadSDNode>(Op);
 
-  if (Load->getExtensionType() != ISD::LoadExtType::NON_EXTLOAD)
-    return SDValue();
-
   SDLoc DL(Op);
   EVT VT = Op.getValueType();
   EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
 
   SDValue Mask = convertFixedMaskToScalableVector(Load->getMask(), DAG);
 
   SDValue PassThru;
   bool IsPassThruZeroOrUndef = false;

llvm/test/CodeGen/AArch64/sve-fixed-length-masked-loads.ll

; VBITS_GE_512-NEXT: ret
   %mask = fcmp oeq <8 x double> %a, %b
   %load = call <8 x double> @llvm.masked.load.v8f64(<8 x double>* %ap, i32 8, <8 x i1> %mask, <8 x double> %b)
   ret <8 x double> %load
 }
 
 define <32 x i16> @masked_load_sext_v32i8i16(<32 x i8>* %ap, <32 x i8>* %bp) #0 {
 ; CHECK-LABEL: masked_load_sext_v32i8i16:
 ; VBITS_GE_512: ptrue [[PG0:p[0-9]+]].b, vl32
-; VBITS_GE_512-NEXT: ld1b { [[Z0:z[0-9]+]].b }, p0/z, [x0]
-; VBITS_GE_512-NEXT: ld1b { [[Z1:z[0-9]+]].b }, p0/z, [x1]
-; VBITS_GE_512-NEXT: cmpeq [[PG1:p[0-9]+]].b, [[PG0]]/z, [[Z0]].b, [[Z1]].b
-; VBITS_GE_512-NEXT: ld1b { [[Z0]].b }, [[PG1]]/z, [x{{[0-9]+}}]
-; VBITS_GE_512-NEXT: ptrue [[PG2:p[0-9]+]].h, vl32
+; VBITS_GE_512-NEXT: ld1b { [[Z0:z[0-9]+]].b }, p0/z, [x1]
+; VBITS_GE_512-NEXT: cmpeq [[PG1:p[0-9]+]].b, [[PG0]]/z, [[Z0]].b, #0
+; VBITS_GE_512-NEXT: mov [[Z0]].b, [[PG0]]/z, #-1
 ; VBITS_GE_512-NEXT: sunpklo [[Z0]].h, [[Z0]].b
-; VBITS_GE_512-NEXT: st1h { [[Z0]].h }, [[PG1]], [x8]
+; VBITS_GE_512-NEXT: ptrue [[PG2:p[0-9]+]].h, vl32
+; VBITS_GE_512-NEXT: cmpne [[PG3:p[0-9]+]].h, [[PG2]]/z, [[Z0]].h, #0
+; VBITS_GE_512-NEXT: ld1sb { [[Z0]].h }, [[PG3]]/z, [x0]
+; VBITS_GE_512-NEXT: st1h { [[Z0]].h }, [[PG2]], [x8]
 ; VBITS_GE_512-NEXT: ret
-  %a = load <32 x i8>, <32 x i8>* %ap
   %b = load <32 x i8>, <32 x i8>* %bp
-  %mask = icmp eq <32 x i8> %a, %b
+  %mask = icmp eq <32 x i8> %b, zeroinitializer
   %load = call <32 x i8> @llvm.masked.load.v32i8(<32 x i8>* %ap, i32 8, <32 x i1> %mask, <32 x i8> undef)
   %ext = sext <32 x i8> %load to <32 x i16>
   ret <32 x i16> %ext
 }
 
 define <16 x i32> @masked_load_sext_v16i8i32(<16 x i8>* %ap, <16 x i8>* %bp) #0 {
 ; CHECK-LABEL: masked_load_sext_v16i8i32:
-; VBITS_GE_512: ldr q0, [x0]
-; VBITS_GE_512-NEXT: ldr q1, [x1]
-; VBITS_GE_512-NEXT: ptrue [[PG0:p[0-9]+]].b, vl16
-; VBITS_GE_512-NEXT: cmeq v[[V:[0-9]+]].16b, v{{[0-9]+}}.16b, v{{[0-9]+}}.16b
-; VBITS_GE_512-NEXT: cmpne [[PG2:p[0-9]+]].b, [[PG0]]/z, [[Z0]].b, #0
-; VBITS_GE_512-NEXT: ld1b { [[Z0]].b }, [[PG2]]/z, [x{{[0-9]+}}]
-; VBITS_GE_512-NEXT: ptrue [[PG2:p[0-9]+]].s, vl16
+; VBITS_GE_512: ldr q0, [x1]
+; VBITS_GE_512-NEXT: ptrue [[PG0:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: cmeq v[[V:[0-9]+]].16b, v{{[0-9]+}}.16b, #0
 ; VBITS_GE_512-NEXT: sunpklo [[Z0]].h, [[Z0]].b
 ; VBITS_GE_512-NEXT: sunpklo [[Z0]].s, [[Z0]].h
-; VBITS_GE_512-NEXT: st1w { [[Z0]].s }, [[PG2]], [x8]
+; VBITS_GE_512-NEXT: cmpne [[PG1:p[0-9]+]].s, [[PG0]]/z, [[Z0]].s, #0
+; VBITS_GE_512-NEXT: ld1sb { [[Z0]].s }, [[PG1]]/z, [x{{[0-9]+}}]
+; VBITS_GE_512-NEXT: st1w { [[Z0]].s }, [[PG0]], [x8]
 ; VBITS_GE_512: ret
-  %a = load <16 x i8>, <16 x i8>* %ap
   %b = load <16 x i8>, <16 x i8>* %bp
-  %mask = icmp eq <16 x i8> %a, %b
+  %mask = icmp eq <16 x i8> %b, zeroinitializer
   %load = call <16 x i8> @llvm.masked.load.v16i8(<16 x i8>* %ap, i32 8, <16 x i1> %mask, <16 x i8> undef)
   %ext = sext <16 x i8> %load to <16 x i32>
   ret <16 x i32> %ext
 }
 
 define <8 x i64> @masked_load_sext_v8i8i64(<8 x i8>* %ap, <8 x i8>* %bp) #0 {
 ; CHECK-LABEL: masked_load_sext_v8i8i64:
-; VBITS_GE_512: ldr d0, [x0]
-; VBITS_GE_512-NEXT: ldr d1, [x1]
-; VBITS_GE_512-NEXT: ptrue [[PG0:p[0-9]+]].b, vl8
-; VBITS_GE_512-NEXT: cmeq v[[V:[0-9]+]].8b, v{{[0-9]+}}.8b, v{{[0-9]+}}.8b
-; VBITS_GE_512-NEXT: cmpne p[[PG:[0-9]+]].b, p0/z, z[[V]].b, #0
-; VBITS_GE_512-NEXT: ld1b { [[Z0]].b }, p[[PG]]/z, [x{{[0-9]+}}]
-; VBITS_GE_512-NEXT: ptrue [[PG2:p[0-9]+]].d, vl8
+; VBITS_GE_512: ldr d0, [x1]
+; VBITS_GE_512-NEXT: ptrue [[PG0:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: cmeq v[[V:[0-9]+]].8b, v{{[0-9]+}}.8b, #0
 ; VBITS_GE_512-NEXT: sunpklo [[Z0]].h, [[Z0]].b
 ; VBITS_GE_512-NEXT: sunpklo [[Z0]].s, [[Z0]].h
 ; VBITS_GE_512-NEXT: sunpklo [[Z0]].d, [[Z0]].s
-; VBITS_GE_512-NEXT: st1d { [[Z0]].d }, [[PG2]], [x8]
+; VBITS_GE_512-NEXT: cmpne p[[PG:[0-9]+]].d, p0/z, z[[V]].d, #0
+; VBITS_GE_512-NEXT: ld1sb { [[Z0]].d }, p[[PG]]/z, [x{{[0-9]+}}]
+; VBITS_GE_512-NEXT: st1d { [[Z0]].d }, [[PG0]], [x8]
 ; VBITS_GE_512-NEXT: ret
-  %a = load <8 x i8>, <8 x i8>* %ap
   %b = load <8 x i8>, <8 x i8>* %bp
-  %mask = icmp eq <8 x i8> %a, %b
+  %mask = icmp eq <8 x i8> %b, zeroinitializer
   %load = call <8 x i8> @llvm.masked.load.v8i8(<8 x i8>* %ap, i32 8, <8 x i1> %mask, <8 x i8> undef)
   %ext = sext <8 x i8> %load to <8 x i64>
   ret <8 x i64> %ext
 }
 
 define <16 x i32> @masked_load_sext_v16i16i32(<16 x i16>* %ap, <16 x i16>* %bp) #0 {
 ; CHECK-LABEL: masked_load_sext_v16i16i32:
 ; VBITS_GE_512: ptrue [[PG0:p[0-9]+]].h, vl16
-; VBITS_GE_512-NEXT: ld1h { [[Z0:z[0-9]+]].h }, p0/z, [x0]
-; VBITS_GE_512-NEXT: ld1h { [[Z1:z[0-9]+]].h }, p0/z, [x1]
-; VBITS_GE_512-NEXT: cmpeq [[PG1:p[0-9]+]].h, [[PG0]]/z, [[Z0]].h, [[Z1]].h
-; VBITS_GE_512-NEXT: ld1h { [[Z0]].h }, [[PG1]]/z, [x{{[0-9]+}}]
-; VBITS_GE_512-NEXT: ptrue [[PG2:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: ld1h { [[Z0:z[0-9]+]].h }, p0/z, [x1]
+; VBITS_GE_512-NEXT: cmpeq [[PG1:p[0-9]+]].h, [[PG0]]/z, [[Z0]].h, #0
+; VBITS_GE_512-NEXT: mov [[Z0]].h, [[PG1]]/z, #-1
 ; VBITS_GE_512-NEXT: sunpklo [[Z0]].s, [[Z0]].h
-; VBITS_GE_512-NEXT: st1w { [[Z0]].s }, [[PG1]], [x8]
+; VBITS_GE_512-NEXT: ptrue [[PG2:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: cmpne [[PG3:p[0-9]+]].s, [[PG2]]/z, [[Z0]].s, #0
+; VBITS_GE_512-NEXT: ld1sh { [[Z0]].s }, [[PG3]]/z, [x{{[0-9]+}}]
+; VBITS_GE_512-NEXT: st1w { [[Z0]].s }, [[PG2]], [x8]
 ; VBITS_GE_512-NEXT: ret
-  %a = load <16 x i16>, <16 x i16>* %ap
   %b = load <16 x i16>, <16 x i16>* %bp
-  %mask = icmp eq <16 x i16> %a, %b
+  %mask = icmp eq <16 x i16> %b, zeroinitializer
   %load = call <16 x i16> @llvm.masked.load.v16i16(<16 x i16>* %ap, i32 8, <16 x i1> %mask, <16 x i16> undef)
   %ext = sext <16 x i16> %load to <16 x i32>
   ret <16 x i32> %ext
 }
 
 define <8 x i64> @masked_load_sext_v8i16i64(<8 x i16>* %ap, <8 x i16>* %bp) #0 {
 ; CHECK-LABEL: masked_load_sext_v8i16i64:
-; VBITS_GE_512: ldr q0, [x0]
-; VBITS_GE_512-NEXT: ldr q1, [x1]
-; VBITS_GE_512-NEXT: ptrue [[PG0:p[0-9]+]].h, vl8
-; VBITS_GE_512-NEXT: cmeq v[[V:[0-9]+]].8h, v{{[0-9]+}}.8h, v{{[0-9]+}}.8h
-; VBITS_GE_512-NEXT: cmpne p[[PG:[0-9]+]].h, p0/z, z[[V]].h, #0
-; VBITS_GE_512-NEXT: ld1h { [[Z0]].h }, p[[PG]]/z, [x{{[0-9]+}}]
-; VBITS_GE_512-NEXT: ptrue [[PG2:p[0-9]+]].d, vl8
+; VBITS_GE_512: ldr q0, [x1]
+; VBITS_GE_512-NEXT: ptrue [[PG0:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: cmeq v[[V:[0-9]+]].8h, v{{[0-9]+}}.8h, #0
 ; VBITS_GE_512-NEXT: sunpklo [[Z0]].s, [[Z0]].h
 ; VBITS_GE_512-NEXT: sunpklo [[Z0]].d, [[Z0]].s
-; VBITS_GE_512-NEXT: st1d { [[Z0]].d }, [[PG2]], [x8]
+; VBITS_GE_512-NEXT: cmpne p[[PG:[0-9]+]].d, p0/z, z[[V]].d, #0
+; VBITS_GE_512-NEXT: ld1sh { [[Z0]].d }, p[[PG]]/z, [x{{[0-9]+}}]
+; VBITS_GE_512-NEXT: st1d { [[Z0]].d }, [[PG0]], [x8]
 ; VBITS_GE_512-NEXT: ret
-  %a = load <8 x i16>, <8 x i16>* %ap
   %b = load <8 x i16>, <8 x i16>* %bp
-  %mask = icmp eq <8 x i16> %a, %b
+  %mask = icmp eq <8 x i16> %b, zeroinitializer
   %load = call <8 x i16> @llvm.masked.load.v8i16(<8 x i16>* %ap, i32 8, <8 x i1> %mask, <8 x i16> undef)
   %ext = sext <8 x i16> %load to <8 x i64>
   ret <8 x i64> %ext
 }
 
 define <8 x i64> @masked_load_sext_v8i32i64(<8 x i32>* %ap, <8 x i32>* %bp) #0 {
 ; CHECK-LABEL: masked_load_sext_v8i32i64:
 ; VBITS_GE_512: ptrue [[PG0:p[0-9]+]].s, vl8
-; VBITS_GE_512-NEXT: ld1w { [[Z0:z[0-9]+]].s }, p0/z, [x0]
-; VBITS_GE_512-NEXT: ld1w { [[Z1:z[0-9]+]].s }, p0/z, [x1]
-; VBITS_GE_512-NEXT: cmpeq [[PG1:p[0-9]+]].s, [[PG0]]/z, [[Z0]].s, [[Z1]].s
-; VBITS_GE_512-NEXT: ld1w { [[Z0]].s }, [[PG1]]/z, [x{{[0-9]+}}]
-; VBITS_GE_512-NEXT: ptrue [[PG2:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: ld1w { [[Z0:z[0-9]+]].s }, p0/z, [x1]
+; VBITS_GE_512-NEXT: cmpeq [[PG1:p[0-9]+]].s, [[PG0]]/z, [[Z0]].s, #0
+; VBITS_GE_512-NEXT: mov [[Z0]].s, [[PG0]]/z, #-1
 ; VBITS_GE_512-NEXT: sunpklo [[Z0]].d, [[Z0]].s
-; VBITS_GE_512-NEXT: st1d { [[Z0]].d }, [[PG1]], [x8]
+; VBITS_GE_512-NEXT: ptrue [[PG2:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: cmpne [[PG3:p[0-9]+]].d, [[PG2]]/z, [[Z0]].d, #0
+; VBITS_GE_512-NEXT: ld1sw { [[Z0]].d }, [[PG3]]/z, [x{{[0-9]+}}]
+; VBITS_GE_512-NEXT: st1d { [[Z0]].d }, [[PG2]], [x8]
 ; VBITS_GE_512-NEXT: ret
-  %a = load <8 x i32>, <8 x i32>* %ap
   %b = load <8 x i32>, <8 x i32>* %bp
-  %mask = icmp eq <8 x i32> %a, %b
+  %mask = icmp eq <8 x i32> %b, zeroinitializer
   %load = call <8 x i32> @llvm.masked.load.v8i32(<8 x i32>* %ap, i32 8, <8 x i1> %mask, <8 x i32> undef)
   %ext = sext <8 x i32> %load to <8 x i64>
   ret <8 x i64> %ext
 }
 
 define <32 x i16> @masked_load_zext_v32i8i16(<32 x i8>* %ap, <32 x i8>* %bp) #0 {
 ; CHECK-LABEL: masked_load_zext_v32i8i16:
 ; VBITS_GE_512: ptrue [[PG0:p[0-9]+]].b, vl32
-; VBITS_GE_512-NEXT: ld1b { [[Z0:z[0-9]+]].b }, p0/z, [x0]
-; VBITS_GE_512-NEXT: ld1b { [[Z1:z[0-9]+]].b }, p0/z, [x1]
-; VBITS_GE_512-NEXT: cmpeq [[PG1:p[0-9]+]].b, [[PG0]]/z, [[Z0]].b, [[Z1]].b
-; VBITS_GE_512-NEXT: ld1b { [[Z0]].b }, [[PG1]]/z, [x{{[0-9]+}}]
+; VBITS_GE_512-NEXT: ld1b { [[Z0:z[0-9]+]].b }, p0/z, [x1]
+; VBITS_GE_512-NEXT: cmpeq [[PG1:p[0-9]+]].b, [[PG0]]/z, [[Z0]].b, #0
+; VBITS_GE_512-NEXT: mov [[Z0]].b, [[PG0]]/z, #-1
+; VBITS_GE_512-NEXT: sunpklo [[Z0]].h, [[Z0]].b
 ; VBITS_GE_512-NEXT: ptrue [[PG2:p[0-9]+]].h, vl32
-; VBITS_GE_512-NEXT: uunpklo [[Z0]].h, [[Z0]].b
-; VBITS_GE_512-NEXT: st1h { [[Z0]].h }, [[PG1]], [x8]
+; VBITS_GE_512-NEXT: cmpne [[PG3:p[0-9]+]].h, [[PG2]]/z, [[Z0]].h, #0
+; VBITS_GE_512-NEXT: ld1b { [[Z0]].h }, [[PG3]]/z, [x0]
+; VBITS_GE_512-NEXT: st1h { [[Z0]].h }, [[PG2]], [x8]
 ; VBITS_GE_512-NEXT: ret
-  %a = load <32 x i8>, <32 x i8>* %ap
   %b = load <32 x i8>, <32 x i8>* %bp
-  %mask = icmp eq <32 x i8> %a, %b
+  %mask = icmp eq <32 x i8> %b, zeroinitializer
   %load = call <32 x i8> @llvm.masked.load.v32i8(<32 x i8>* %ap, i32 8, <32 x i1> %mask, <32 x i8> undef)
   %ext = zext <32 x i8> %load to <32 x i16>
   ret <32 x i16> %ext
 }
 
 define <16 x i32> @masked_load_zext_v16i8i32(<16 x i8>* %ap, <16 x i8>* %bp) #0 {
 ; CHECK-LABEL: masked_load_zext_v16i8i32:
-; VBITS_GE_512: ldr q0, [x0]
-; VBITS_GE_512-NEXT: ldr q1, [x1]
-; VBITS_GE_512-NEXT: ptrue [[PG0:p[0-9]+]].b, vl16
-; VBITS_GE_512-NEXT: cmeq v[[V:[0-9]+]].16b, v{{[0-9]+}}.16b, v{{[0-9]+}}.16b
-; VBITS_GE_512-NEXT: cmpne [[PG2:p[0-9]+]].b, [[PG0]]/z, [[Z0]].b, #0
-; VBITS_GE_512-NEXT: ld1b { [[Z0]].b }, [[PG2]]/z, [x{{[0-9]+}}]
-; VBITS_GE_512-NEXT: ptrue [[PG2:p[0-9]+]].s, vl16
-; VBITS_GE_512-NEXT: uunpklo [[Z0]].h, [[Z0]].b
-; VBITS_GE_512-NEXT: uunpklo [[Z0]].s, [[Z0]].h
-; VBITS_GE_512-NEXT: st1w { [[Z0]].s }, [[PG2]], [x8]
+; VBITS_GE_512: ldr q0, [x1]
+; VBITS_GE_512-NEXT: ptrue [[PG0:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: cmeq v[[V:[0-9]+]].16b, v{{[0-9]+}}.16b, #0
+; VBITS_GE_512-NEXT: sunpklo [[Z0]].h, [[Z0]].b
+; VBITS_GE_512-NEXT: sunpklo [[Z0]].s, [[Z0]].h
+; VBITS_GE_512-NEXT: cmpne [[PG1:p[0-9]+]].s, [[PG0]]/z, [[Z0]].s, #0
+; VBITS_GE_512-NEXT: ld1b { [[Z0]].s }, [[PG1]]/z, [x{{[0-9]+}}]
+; VBITS_GE_512-NEXT: st1w { [[Z0]].s }, [[PG0]], [x8]
 ; VBITS_GE_512: ret
-  %a = load <16 x i8>, <16 x i8>* %ap
   %b = load <16 x i8>, <16 x i8>* %bp
-  %mask = icmp eq <16 x i8> %a, %b
+  %mask = icmp eq <16 x i8> %b, zeroinitializer
   %load = call <16 x i8> @llvm.masked.load.v16i8(<16 x i8>* %ap, i32 8, <16 x i1> %mask, <16 x i8> undef)
   %ext = zext <16 x i8> %load to <16 x i32>
   ret <16 x i32> %ext
 }
 
 define <8 x i64> @masked_load_zext_v8i8i64(<8 x i8>* %ap, <8 x i8>* %bp) #0 {
 ; CHECK-LABEL: masked_load_zext_v8i8i64:
-; VBITS_GE_512: ldr d0, [x0]
-; VBITS_GE_512-NEXT: ldr d1, [x1]
-; VBITS_GE_512-NEXT: ptrue [[PG0:p[0-9]+]].b, vl8
-; VBITS_GE_512-NEXT: cmeq v[[V:[0-9]+]].8b, v{{[0-9]+}}.8b, v{{[0-9]+}}.8b
-; VBITS_GE_512-NEXT: cmpne p[[PG:[0-9]+]].b, p0/z, z[[V]].b, #0
-; VBITS_GE_512-NEXT: ld1b { [[Z0]].b }, p[[PG]]/z, [x{{[0-9]+}}]
-; VBITS_GE_512-NEXT: ptrue [[PG2:p[0-9]+]].d, vl8
-; VBITS_GE_512-NEXT: uunpklo [[Z0]].h, [[Z0]].b
-; VBITS_GE_512-NEXT: uunpklo [[Z0]].s, [[Z0]].h
-; VBITS_GE_512-NEXT: uunpklo [[Z0]].d, [[Z0]].s
-; VBITS_GE_512-NEXT: st1d { [[Z0]].d }, [[PG2]], [x8]
+; VBITS_GE_512: ldr d0, [x1]
+; VBITS_GE_512-NEXT: ptrue [[PG0:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: cmeq v[[V:[0-9]+]].8b, v{{[0-9]+}}.8b, #0
+; VBITS_GE_512-NEXT: sunpklo [[Z0]].h, [[Z0]].b
+; VBITS_GE_512-NEXT: sunpklo [[Z0]].s, [[Z0]].h
+; VBITS_GE_512-NEXT: sunpklo [[Z0]].d, [[Z0]].s
+; VBITS_GE_512-NEXT: cmpne p[[PG:[0-9]+]].d, p0/z, z[[V]].d, #0
+; VBITS_GE_512-NEXT: ld1b { [[Z0]].d }, p[[PG]]/z, [x{{[0-9]+}}]
+; VBITS_GE_512-NEXT: st1d { [[Z0]].d }, [[PG0]], [x8]
 ; VBITS_GE_512-NEXT: ret
-  %a = load <8 x i8>, <8 x i8>* %ap
   %b = load <8 x i8>, <8 x i8>* %bp
-  %mask = icmp eq <8 x i8> %a, %b
+  %mask = icmp eq <8 x i8> %b, zeroinitializer
   %load = call <8 x i8> @llvm.masked.load.v8i8(<8 x i8>* %ap, i32 8, <8 x i1> %mask, <8 x i8> undef)
   %ext = zext <8 x i8> %load to <8 x i64>
   ret <8 x i64> %ext
 }
 
 define <16 x i32> @masked_load_zext_v16i16i32(<16 x i16>* %ap, <16 x i16>* %bp) #0 {
 ; CHECK-LABEL: masked_load_zext_v16i16i32:
 ; VBITS_GE_512: ptrue [[PG0:p[0-9]+]].h, vl16
-; VBITS_GE_512-NEXT: ld1h { [[Z0:z[0-9]+]].h }, p0/z, [x0]
-; VBITS_GE_512-NEXT: ld1h { [[Z1:z[0-9]+]].h }, p0/z, [x1]
-; VBITS_GE_512-NEXT: cmpeq [[PG1:p[0-9]+]].h, [[PG0]]/z, [[Z0]].h, [[Z1]].h
-; VBITS_GE_512-NEXT: ld1h { [[Z0]].h }, [[PG1]]/z, [x{{[0-9]+}}]
+; VBITS_GE_512-NEXT: ld1h { [[Z0:z[0-9]+]].h }, p0/z, [x1]
+; VBITS_GE_512-NEXT: cmpeq [[PG1:p[0-9]+]].h, [[PG0]]/z, [[Z0]].h, #0
+; VBITS_GE_512-NEXT: mov [[Z0]].h, [[PG1]]/z, #-1
+; VBITS_GE_512-NEXT: sunpklo [[Z0]].s, [[Z0]].h
 ; VBITS_GE_512-NEXT: ptrue [[PG2:p[0-9]+]].s, vl16
-; VBITS_GE_512-NEXT: uunpklo [[Z0]].s, [[Z0]].h
-; VBITS_GE_512-NEXT: st1w { [[Z0]].s }, [[PG1]], [x8]
+; VBITS_GE_512-NEXT: cmpne [[PG3:p[0-9]+]].s, [[PG2]]/z, [[Z0]].s, #0
+; VBITS_GE_512-NEXT: ld1h { [[Z0]].s }, [[PG3]]/z, [x{{[0-9]+}}]
+; VBITS_GE_512-NEXT: st1w { [[Z0]].s }, [[PG2]], [x8]
 ; VBITS_GE_512-NEXT: ret
-  %a = load <16 x i16>, <16 x i16>* %ap
   %b = load <16 x i16>, <16 x i16>* %bp
-  %mask = icmp eq <16 x i16> %a, %b
+  %mask = icmp eq <16 x i16> %b, zeroinitializer
   %load = call <16 x i16> @llvm.masked.load.v16i16(<16 x i16>* %ap, i32 8, <16 x i1> %mask, <16 x i16> undef)
   %ext = zext <16 x i16> %load to <16 x i32>
   ret <16 x i32> %ext
 }
 
 define <8 x i64> @masked_load_zext_v8i16i64(<8 x i16>* %ap, <8 x i16>* %bp) #0 {
 ; CHECK-LABEL: masked_load_zext_v8i16i64:
-; VBITS_GE_512: ldr q0, [x0]
-; VBITS_GE_512-NEXT: ldr q1, [x1]
-; VBITS_GE_512-NEXT: ptrue [[PG0:p[0-9]+]].h, vl8
-; VBITS_GE_512-NEXT: cmeq v[[V:[0-9]+]].8h, v{{[0-9]+}}.8h, v{{[0-9]+}}.8h
-; VBITS_GE_512-NEXT: cmpne p[[PG:[0-9]+]].h, p0/z, z[[V]].h, #0
-; VBITS_GE_512-NEXT: ld1h { [[Z0]].h }, p[[PG]]/z, [x{{[0-9]+}}]
-; VBITS_GE_512-NEXT: ptrue [[PG2:p[0-9]+]].d, vl8
-; VBITS_GE_512-NEXT: uunpklo [[Z0]].s, [[Z0]].h
-; VBITS_GE_512-NEXT: uunpklo [[Z0]].d, [[Z0]].s
-; VBITS_GE_512-NEXT: st1d { [[Z0]].d }, [[PG2]], [x8]
+; VBITS_GE_512: ldr q0, [x1]
+; VBITS_GE_512-NEXT: ptrue [[PG0:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: cmeq v[[V:[0-9]+]].8h, v{{[0-9]+}}.8h, #0
+; VBITS_GE_512-NEXT: sunpklo [[Z0]].s, [[Z0]].h
+; VBITS_GE_512-NEXT: sunpklo [[Z0]].d, [[Z0]].s
+; VBITS_GE_512-NEXT: cmpne p[[PG:[0-9]+]].d, p0/z, z[[V]].d, #0
+; VBITS_GE_512-NEXT: ld1h { [[Z0]].d }, p[[PG]]/z, [x{{[0-9]+}}]
+; VBITS_GE_512-NEXT: st1d { [[Z0]].d }, [[PG0]], [x8]
 ; VBITS_GE_512-NEXT: ret
-  %a = load <8 x i16>, <8 x i16>* %ap
   %b = load <8 x i16>, <8 x i16>* %bp
-  %mask = icmp eq <8 x i16> %a, %b
+  %mask = icmp eq <8 x i16> %b, zeroinitializer
   %load = call <8 x i16> @llvm.masked.load.v8i16(<8 x i16>* %ap, i32 8, <8 x i1> %mask, <8 x i16> undef)
   %ext = zext <8 x i16> %load to <8 x i64>
   ret <8 x i64> %ext
 }
 
 define <8 x i64> @masked_load_zext_v8i32i64(<8 x i32>* %ap, <8 x i32>* %bp) #0 {
 ; CHECK-LABEL: masked_load_zext_v8i32i64:
 ; VBITS_GE_512: ptrue [[PG0:p[0-9]+]].s, vl8
-; VBITS_GE_512-NEXT: ld1w { [[Z0:z[0-9]+]].s }, p0/z, [x0]
-; VBITS_GE_512-NEXT: ld1w { [[Z1:z[0-9]+]].s }, p0/z, [x1]
-; VBITS_GE_512-NEXT: cmpeq [[PG1:p[0-9]+]].s, [[PG0]]/z, [[Z0]].s, [[Z1]].s
-; VBITS_GE_512-NEXT: ld1w { [[Z0]].s }, [[PG1]]/z, [x{{[0-9]+}}]
+; VBITS_GE_512-NEXT: ld1w { [[Z0:z[0-9]+]].s }, p0/z, [x1]
+; VBITS_GE_512-NEXT: cmpeq [[PG1:p[0-9]+]].s, [[PG0]]/z, [[Z0]].s, #0
+; VBITS_GE_512-NEXT: mov [[Z0]].s, [[PG0]]/z, #-1
+; VBITS_GE_512-NEXT: sunpklo [[Z0]].d, [[Z0]].s
 ; VBITS_GE_512-NEXT: ptrue [[PG2:p[0-9]+]].d, vl8
-; VBITS_GE_512-NEXT: uunpklo [[Z0]].d, [[Z0]].s
-; VBITS_GE_512-NEXT: st1d { [[Z0]].d }, [[PG1]], [x8]
-; VBITS_GE_512-NEXT: ret
-  %a = load <8 x i32>, <8 x i32>* %ap
+; VBITS_GE_512-NEXT: cmpne [[PG3:p[0-9]+]].d, [[PG2]]/z, [[Z0]].d, #0
+; VBITS_GE_512-NEXT: ld1w { [[Z0]].d }, [[PG3]]/z, [x{{[0-9]+}}]
+; VBITS_GE_512-NEXT: st1d { [[Z0]].d }, [[PG2]], [x8]
   %b = load <8 x i32>, <8 x i32>* %bp
-  %mask = icmp eq <8 x i32> %a, %b
+  %mask = icmp eq <8 x i32> %b, zeroinitializer
+  %load = call <8 x i32> @llvm.masked.load.v8i32(<8 x i32>* %ap, i32 8, <8 x i1> %mask, <8 x i32> undef)
+  %ext = zext <8 x i32> %load to <8 x i64>
+  ret <8 x i64> %ext
+}
+
+define <32 x i16> @masked_load_sext_v32i8i16_m16(<32 x i8>* %ap, <32 x i16>* %bp) #0 {
+; CHECK-LABEL: masked_load_sext_v32i8i16_m16:
+; VBITS_GE_512: ptrue [[PG0:p[0-9]+]].h, vl32
+; VBITS_GE_512-NEXT: ld1h { [[Z0:z[0-9]+]].h }, [[PG0]]/z, [x1]
+; VBITS_GE_512-NEXT: cmpeq [[PG1:p[0-9]+]].h, [[PG0]]/z, [[Z0]].h, #0
+; VBITS_GE_512-NEXT: ld1sb { [[Z0]].h }, [[PG1]]/z, [x0]
+; VBITS_GE_512-NEXT: st1h { [[Z0]].h }, [[PG0]], [x8]
+; VBITS_GE_512-NEXT: ret
+  %b = load <32 x i16>, <32 x i16>* %bp
+  %mask = icmp eq <32 x i16> %b, zeroinitializer
+  %load = call <32 x i8> @llvm.masked.load.v32i8(<32 x i8>* %ap, i32 8, <32 x i1> %mask, <32 x i8> undef)
+  %ext = sext <32 x i8> %load to <32 x i16>
+  ret <32 x i16> %ext
+}
+
+define <16 x i32> @masked_load_sext_v16i8i32_m32(<16 x i8>* %ap, <16 x i32>* %bp) #0 {
+; CHECK-LABEL: masked_load_sext_v16i8i32_m32:
+; VBITS_GE_512: ptrue [[PG0:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: ld1w { [[Z0:z[0-9]+]].s }, [[PG0]]/z, [x1]
+; VBITS_GE_512-NEXT: cmpeq [[PG1:p[0-9]+]].s, [[PG0]]/z, [[Z0]].s, #0
+; VBITS_GE_512-NEXT: ld1sb { [[Z0]].s }, [[PG1]]/z, [x0]
+; VBITS_GE_512-NEXT: st1w { [[Z0]].s }, [[PG0]], [x8]
+; VBITS_GE_512: ret
+  %b = load <16 x i32>, <16 x i32>* %bp
+  %mask = icmp eq <16 x i32> %b, zeroinitializer
+  %load = call <16 x i8> @llvm.masked.load.v16i8(<16 x i8>* %ap, i32 8, <16 x i1> %mask, <16 x i8> undef)
+  %ext = sext <16 x i8> %load to <16 x i32>
+  ret <16 x i32> %ext
+}
+
+define <8 x i64> @masked_load_sext_v8i8i64_m64(<8 x i8>* %ap, <8 x i64>* %bp) #0 {
+; CHECK-LABEL: masked_load_sext_v8i8i64_m64:
+; VBITS_GE_512: ptrue [[PG0:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: ld1d { [[Z0:z[0-9]+]].d }, [[PG0]]/z, [x1]
+; VBITS_GE_512-NEXT: cmpeq [[PG1:p[0-9]+]].d, [[PG0]]/z, [[Z0]].d, #0
+; VBITS_GE_512-NEXT: ld1sb { [[Z0]].d }, [[PG1]]/z, [x0]
+; VBITS_GE_512-NEXT: st1d { [[Z0]].d }, [[PG0]], [x8]
+; VBITS_GE_512-NEXT: ret
+  %b = load <8 x i64>, <8 x i64>* %bp
+  %mask = icmp eq <8 x i64> %b, zeroinitializer
+  %load = call <8 x i8> @llvm.masked.load.v8i8(<8 x i8>* %ap, i32 8, <8 x i1> %mask, <8 x i8> undef)
+  %ext = sext <8 x i8> %load to <8 x i64>
+  ret <8 x i64> %ext
+}
+
+define <16 x i32> @masked_load_sext_v16i16i32_m32(<16 x i16>* %ap, <16 x i32>* %bp) #0 {
+; CHECK-LABEL: masked_load_sext_v16i16i32_m32:
+; VBITS_GE_512: ptrue [[PG0:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: ld1w { [[Z0:z[0-9]+]].s }, [[PG0]]/z, [x1]
+; VBITS_GE_512-NEXT: cmpeq [[PG1:p[0-9]+]].s, [[PG0]]/z, [[Z0]].s, #0
+; VBITS_GE_512-NEXT: ld1sh { [[Z0]].s }, [[PG1]]/z, [x0]
+; VBITS_GE_512-NEXT: st1w { [[Z0]].s }, [[PG0]], [x8]
+; VBITS_GE_512-NEXT: ret
+  %b = load <16 x i32>, <16 x i32>* %bp
+  %mask = icmp eq <16 x i32> %b, zeroinitializer
+  %load = call <16 x i16> @llvm.masked.load.v16i16(<16 x i16>* %ap, i32 8, <16 x i1> %mask, <16 x i16> undef)
+  %ext = sext <16 x i16> %load to <16 x i32>
+  ret <16 x i32> %ext
+}
+
+define <8 x i64> @masked_load_sext_v8i16i64_m64(<8 x i16>* %ap, <8 x i64>* %bp) #0 {
+; CHECK-LABEL: masked_load_sext_v8i16i64_m64:
+; VBITS_GE_512: ptrue [[PG0:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: ld1d { [[Z0:z[0-9]+]].d }, [[PG0]]/z, [x1]
+; VBITS_GE_512-NEXT: cmpeq [[PG1:p[0-9]+]].d, [[PG0]]/z, [[Z0]].d, #0
+; VBITS_GE_512-NEXT: ld1sh { [[Z0]].d }, [[PG1]]/z, [x0]
+; VBITS_GE_512-NEXT: st1d { [[Z0]].d }, [[PG0]], [x8]
+; VBITS_GE_512-NEXT: ret
+  %b = load <8 x i64>, <8 x i64>* %bp
+  %mask = icmp eq <8 x i64> %b, zeroinitializer
+  %load = call <8 x i16> @llvm.masked.load.v8i16(<8 x i16>* %ap, i32 8, <8 x i1> %mask, <8 x i16> undef)
+  %ext = sext <8 x i16> %load to <8 x i64>
+  ret <8 x i64> %ext
+}
+
+define <8 x i64> @masked_load_sext_v8i32i64_m64(<8 x i32>* %ap, <8 x i64>* %bp) #0 {
+; CHECK-LABEL: masked_load_sext_v8i32i64_m64:
+; VBITS_GE_512: ptrue [[PG0:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: ld1d { [[Z0:z[0-9]+]].d }, [[PG0]]/z, [x1]
+; VBITS_GE_512-NEXT: cmpeq [[PG1:p[0-9]+]].d, [[PG0]]/z, [[Z0]].d, #0
+; VBITS_GE_512-NEXT: ld1sw { [[Z0]].d }, [[PG1]]/z, [x0]
+; VBITS_GE_512-NEXT: st1d { [[Z0]].d }, [[PG0]], [x8]
+; VBITS_GE_512-NEXT: ret
+  %b = load <8 x i64>, <8 x i64>* %bp
+  %mask = icmp eq <8 x i64> %b, zeroinitializer
+  %load = call <8 x i32> @llvm.masked.load.v8i32(<8 x i32>* %ap, i32 8, <8 x i1> %mask, <8 x i32> undef)
+  %ext = sext <8 x i32> %load to <8 x i64>
+  ret <8 x i64> %ext
+}
+
+define <32 x i16> @masked_load_zext_v32i8i16_m16(<32 x i8>* %ap, <32 x i16>* %bp) #0 {
+; CHECK-LABEL: masked_load_zext_v32i8i16_m16:
+; VBITS_GE_512: ptrue [[PG0:p[0-9]+]].h, vl32
+; VBITS_GE_512-NEXT: ld1h { [[Z0:z[0-9]+]].h }, [[PG0]]/z, [x1]
+; VBITS_GE_512-NEXT: cmpeq [[PG1:p[0-9]+]].h, [[PG0]]/z, [[Z0]].h, #0
+; VBITS_GE_512-NEXT: ld1b { [[Z0]].h }, [[PG1]]/z, [x0]
+; VBITS_GE_512-NEXT: st1h { [[Z0]].h }, [[PG0]], [x8]
+; VBITS_GE_512-NEXT: ret
+  %b = load <32 x i16>, <32 x i16>* %bp
+  %mask = icmp eq <32 x i16> %b, zeroinitializer
+  %load = call <32 x i8> @llvm.masked.load.v32i8(<32 x i8>* %ap, i32 8, <32 x i1> %mask, <32 x i8> undef)
+  %ext = zext <32 x i8> %load to <32 x i16>
+  ret <32 x i16> %ext
+}
+
+define <16 x i32> @masked_load_zext_v16i8i32_m32(<16 x i8>* %ap, <16 x i32>* %bp) #0 {
+; CHECK-LABEL: masked_load_zext_v16i8i32_m32:
+; VBITS_GE_512: ptrue [[PG0:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: ld1w { [[Z0:z[0-9]+]].s }, [[PG0]]/z, [x1]
+; VBITS_GE_512-NEXT: cmpeq [[PG1:p[0-9]+]].s, [[PG0]]/z, [[Z0]].s, #0
+; VBITS_GE_512-NEXT: ld1b { [[Z0]].s }, [[PG1]]/z, [x0]
+; VBITS_GE_512-NEXT: st1w { [[Z0]].s }, [[PG0]], [x8]
+; VBITS_GE_512-NEXT: ret
+  %b = load <16 x i32>, <16 x i32>* %bp
+  %mask = icmp eq <16 x i32> %b, zeroinitializer
+  %load = call <16 x i8> @llvm.masked.load.v16i8(<16 x i8>* %ap, i32 8, <16 x i1> %mask, <16 x i8> undef)
+  %ext = zext <16 x i8> %load to <16 x i32>
+  ret <16 x i32> %ext
+}
+
+define <8 x i64> @masked_load_zext_v8i8i64_m64(<8 x i8>* %ap, <8 x i64>* %bp) #0 {
+; CHECK-LABEL: masked_load_zext_v8i8i64_m64:
+; VBITS_GE_512: ptrue [[PG0:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: ld1d { [[Z0:z[0-9]+]].d }, [[PG0]]/z, [x1]
+; VBITS_GE_512-NEXT: cmpeq [[PG1:p[0-9]+]].d, [[PG0]]/z, [[Z0]].d, #0
+; VBITS_GE_512-NEXT: ld1b { [[Z0]].d }, [[PG1]]/z, [x0]
+; VBITS_GE_512-NEXT: st1d { [[Z0]].d }, [[PG0]], [x8]
+; VBITS_GE_512-NEXT: ret
+  %b = load <8 x i64>, <8 x i64>* %bp
+  %mask = icmp eq <8 x i64> %b, zeroinitializer
+  %load = call <8 x i8> @llvm.masked.load.v8i8(<8 x i8>* %ap, i32 8, <8 x i1> %mask, <8 x i8> undef)
+  %ext = zext <8 x i8> %load to <8 x i64>
+  ret <8 x i64> %ext
+}
+
+define <16 x i32> @masked_load_zext_v16i16i32_m32(<16 x i16>* %ap, <16 x i32>* %bp) #0 {
+; CHECK-LABEL: masked_load_zext_v16i16i32_m32:
+; VBITS_GE_512: ptrue [[PG0:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: ld1w { [[Z0:z[0-9]+]].s }, [[PG0]]/z, [x1]
+; VBITS_GE_512-NEXT: cmpeq [[PG1:p[0-9]+]].s, [[PG0]]/z, [[Z0]].s, #0
+; VBITS_GE_512-NEXT: ld1h { [[Z0]].s }, [[PG1]]/z, [x0]
+; VBITS_GE_512-NEXT: st1w { [[Z0]].s }, [[PG0]], [x8]
+; VBITS_GE_512-NEXT: ret
+  %b = load <16 x i32>, <16 x i32>* %bp
+  %mask = icmp eq <16 x i32> %b, zeroinitializer
+  %load = call <16 x i16> @llvm.masked.load.v16i16(<16 x i16>* %ap, i32 8, <16 x i1> %mask, <16 x i16> undef)
+  %ext = zext <16 x i16> %load to <16 x i32>
+  ret <16 x i32> %ext
+}
+
+define <8 x i64> @masked_load_zext_v8i16i64_m64(<8 x i16>* %ap, <8 x i64>* %bp) #0 {
+; CHECK-LABEL: masked_load_zext_v8i16i64_m64:
+; VBITS_GE_512: ptrue [[PG0:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: ld1d { [[Z0:z[0-9]+]].d }, [[PG0]]/z, [x1]
+; VBITS_GE_512-NEXT: cmpeq [[PG1:p[0-9]+]].d, [[PG0]]/z, [[Z0]].d, #0
+; VBITS_GE_512-NEXT: ld1h { [[Z0]].d }, [[PG1]]/z, [x0]
+; VBITS_GE_512-NEXT: st1d { [[Z0]].d }, [[PG0]], [x8]
+; VBITS_GE_512-NEXT: ret
+  %b = load <8 x i64>, <8 x i64>* %bp
+  %mask = icmp eq <8 x i64> %b, zeroinitializer
+  %load = call <8 x i16> @llvm.masked.load.v8i16(<8 x i16>* %ap, i32 8, <8 x i1> %mask, <8 x i16> undef)
+  %ext = zext <8 x i16> %load to <8 x i64>
+  ret <8 x i64> %ext
+}
+
+define <8 x i64> @masked_load_zext_v8i32i64_m64(<8 x i32>* %ap, <8 x i64>* %bp) #0 {
+; CHECK-LABEL: masked_load_zext_v8i32i64_m64:
+; VBITS_GE_512: ptrue [[PG0:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: ld1d { [[Z0:z[0-9]+]].d }, [[PG0]]/z, [x1]
+; VBITS_GE_512-NEXT: cmpeq [[PG1:p[0-9]+]].d, [[PG0]]/z, [[Z0]].d, #0
+; VBITS_GE_512-NEXT: ld1w { [[Z0]].d }, [[PG1]]/z, [x0]
+; VBITS_GE_512-NEXT: st1d { [[Z0]].d }, [[PG0]], [x8]
+; VBITS_GE_512-NEXT: ret
+  %b = load <8 x i64>, <8 x i64>* %bp
+  %mask = icmp eq <8 x i64> %b, zeroinitializer
   %load = call <8 x i32> @llvm.masked.load.v8i32(<8 x i32>* %ap, i32 8, <8 x i1> %mask, <8 x i32> undef)
   %ext = zext <8 x i32> %load to <8 x i64>
   ret <8 x i64> %ext
 }
 
 declare <2 x half> @llvm.masked.load.v2f16(<2 x half>*, i32, <2 x i1>, <2 x half>)
 declare <2 x float> @llvm.masked.load.v2f32(<2 x float>*, i32, <2 x i1>, <2 x float>)
 declare <4 x float> @llvm.masked.load.v4f32(<4 x float>*, i32, <4 x i1>, <4 x float>)
 declare <8 x float> @llvm.masked.load.v8f32(<8 x float>*, i32, <8 x i1>, <8 x float>)
 declare <16 x float> @llvm.masked.load.v16f32(<16 x float>*, i32, <16 x i1>, <16 x float>)
 declare <32 x float> @llvm.masked.load.v32f32(<32 x float>*, i32, <32 x i1>, <32 x float>)
 declare <64 x float> @llvm.masked.load.v64f32(<64 x float>*, i32, <64 x i1>, <64 x float>)
 
 declare <64 x i8> @llvm.masked.load.v64i8(<64 x i8>*, i32, <64 x i1>, <64 x i8>)
 declare <32 x i8> @llvm.masked.load.v32i8(<32 x i8>*, i32, <32 x i1>, <32 x i8>)
 declare <16 x i8> @llvm.masked.load.v16i8(<16 x i8>*, i32, <16 x i1>, <16 x i8>)
 declare <16 x i16> @llvm.masked.load.v16i16(<16 x i16>*, i32, <16 x i1>, <16 x i16>)
 declare <8 x i8> @llvm.masked.load.v8i8(<8 x i8>*, i32, <8 x i1>, <8 x i8>)
 declare <8 x i16> @llvm.masked.load.v8i16(<8 x i16>*, i32, <8 x i1>, <8 x i16>)
 declare <8 x i32> @llvm.masked.load.v8i32(<8 x i32>*, i32, <8 x i1>, <8 x i32>)
 declare <32 x i16> @llvm.masked.load.v32i16(<32 x i16>*, i32, <32 x i1>, <32 x i16>)
 declare <16 x i32> @llvm.masked.load.v16i32(<16 x i32>*, i32, <16 x i1>, <16 x i32>)
 declare <8 x i64> @llvm.masked.load.v8i64(<8 x i64>*, i32, <8 x i1>, <8 x i64>)
 declare <8 x double> @llvm.masked.load.v8f64(<8 x double>*, i32, <8 x i1>, <8 x double>)
 
 attributes #0 = { "target-features"="+sve" }