[AArch64][CodeGen][SME] Allow vectors large than hardware support to use SVE's load zero/sign-extend for fixed vectors

[dtemirbulatov]

Summary

Proposed change improved some tests performance by allowing to fold Load + Extend to SVE's Load zero/sign extend for fixed vectors.

[david-arm]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

5230

Hi @dtemirbulatov, I personally feel this patch should be only trying to fix one thing at a time, but it seems to be trying to change both SVE fixed-length behaviour as well as streaming SVE behaviour. Perhaps it's better to focus on just the streaming SVE case for now, in which case I think you can more simply write this as:

bool OverrideNEON = Subtarget->useSVEForFixedLengthVectors() || Subtarget->forceStreamingCompatibleSVE();
return ExtVal.getValueType().isScalableVector() ||
       useSVEForFixedLengthVectorVT(
           ExtVal.getValueType(), OverrideNEON);

5233

Just for reference, I'm not sure if this is really what you want for the SVE fixed-length case (no streaming SVE) because getMinSVEVectorSizeInBits() could return 256 or greater. In this case it has nothing to do with NEON-sized vectors, which are 64-bit or 128-bit. In any case, I feel like the normal SVE fixed-length case should be done in a different patch.

llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-mulh.ll

171 ↗

(On Diff #510791)

Hi @dtemirbulatov, this looks like a regression to me. We've gone from 20 instructions to 29, which doesn't seem right. Can you investigate why this is happening?

248 ↗

(On Diff #510791)

Again, this has gotten worse - from 20 -> 29 instructions.

[sdesmalen]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

5230

bool OverrideNEON = Subtarget->useSVEForFixedLengthVectors() || Subtarget->forceStreamingCompatibleSVE();

@david-arm just pointing out that if forceStreamingCompatibleSVE() is true, then useSVEForFixedLengthVectors() is also true, so your suggestion is actually equivalent to the original code.

5230–5232

When I change this code to:

return ExtVal.getValueType().isScalableVector() || Subtarget->useSVEForFixedLengthVectors();

all the tests still pass. That suggests that the test for 'ExtVal.getSizeInBits() > getMinSVEVectorSizeInBits()' is not used.

[dtemirbulatov]

Rebasing after rGe6096871fdd4 commit.

[sdesmalen]

LGTM

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

if (IndexVT.getScalarSizeInBits() < DataVT.getScalarSizeInBits())
     return false;
 
   // Scalable vectors with "vscale * 2" or fewer elements sit within a 64-bit
   // element container type, which would violate the previous clause.
   return DataVT.isFixedLengthVector() || DataVT.getVectorMinNumElements() > 2;
 }
 
 bool AArch64TargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
   return ExtVal.getValueType().isScalableVector() ||
-         useSVEForFixedLengthVectorVT(
-             ExtVal.getValueType(),
-             /*OverrideNEON=*/Subtarget->useSVEForFixedLengthVectors());
+         Subtarget->useSVEForFixedLengthVectors();
 }
 
 unsigned getGatherVecOpcode(bool IsScaled, bool IsSigned, bool NeedsExtend) {
   std::map<std::tuple<bool, bool, bool>, unsigned> AddrModes = {
       {std::make_tuple(/*Scaled*/ false, /*Signed*/ false, /*Extend*/ false),
        AArch64ISD::GLD1_MERGE_ZERO},
       {std::make_tuple(/*Scaled*/ false, /*Signed*/ false, /*Extend*/ true),
        AArch64ISD::GLD1_UXTW_MERGE_ZERO},
       {std::make_tuple(/*Scaled*/ false, /*Signed*/ true, /*Extend*/ false),
        AArch64ISD::GLD1_MERGE_ZERO},

llvm/test/CodeGen/AArch64/sve-fixed-length-ext-loads.ll

; CHECK-NEXT:    ret
   %a = load <32 x i16>, ptr %ap
   %val = zext <32 x i16> %a to <32 x i32>
   ret <32 x i32> %val
 }
 
 define <64 x i32> @load_zext_v64i16i32(ptr %ap) #0 {
 ; VBITS_GE_1024-LABEL: load_zext_v64i16i32:
 ; VBITS_GE_1024:       // %bb.0:
-; VBITS_GE_1024-NEXT:    ptrue p0.h, vl64
-; VBITS_GE_1024-NEXT:    mov x9, #32
-; VBITS_GE_1024-NEXT:    ld1h { z0.h }, p0/z, [x0]
+; VBITS_GE_1024-NEXT:    mov x9, #32 // =0x20
 ; VBITS_GE_1024-NEXT:    ptrue p0.s, vl32
-; VBITS_GE_1024-NEXT:    uunpklo z1.s, z0.h
-; VBITS_GE_1024-NEXT:    ext z0.b, z0.b, z0.b, #64
-; VBITS_GE_1024-NEXT:    uunpklo z0.s, z0.h
+; VBITS_GE_1024-NEXT:    ld1h { z0.s }, p0/z, [x0, x9, lsl #1]
+; VBITS_GE_1024-NEXT:    ld1h { z1.s }, p0/z, [x0]
 ; VBITS_GE_1024-NEXT:    st1w { z0.s }, p0, [x8, x9, lsl #2]
 ; VBITS_GE_1024-NEXT:    st1w { z1.s }, p0, [x8]
 ; VBITS_GE_1024-NEXT:    ret
 ;
 ; VBITS_GE_2048-LABEL: load_zext_v64i16i32:
 ; VBITS_GE_2048:       // %bb.0:
 ; VBITS_GE_2048-NEXT:    ptrue p0.s, vl64
 ; VBITS_GE_2048-NEXT:    ld1h { z0.s }, p0/z, [x0]
; CHECK-NEXT:    ret
   %a = load <32 x i16>, ptr %ap
   %val = sext <32 x i16> %a to <32 x i32>
   ret <32 x i32> %val
 }
 
 define <64 x i32> @load_sext_v64i16i32(ptr %ap) #0 {
 ; VBITS_GE_1024-LABEL: load_sext_v64i16i32:
 ; VBITS_GE_1024:       // %bb.0:
-; VBITS_GE_1024-NEXT:    ptrue p0.h, vl64
-; VBITS_GE_1024-NEXT:    mov x9, #32
-; VBITS_GE_1024-NEXT:    ld1h { z0.h }, p0/z, [x0]
+; VBITS_GE_1024-NEXT:    mov x9, #32 // =0x20
 ; VBITS_GE_1024-NEXT:    ptrue p0.s, vl32
-; VBITS_GE_1024-NEXT:    sunpklo z1.s, z0.h
-; VBITS_GE_1024-NEXT:    ext z0.b, z0.b, z0.b, #64
-; VBITS_GE_1024-NEXT:    sunpklo z0.s, z0.h
+; VBITS_GE_1024-NEXT:    ld1sh { z0.s }, p0/z, [x0, x9, lsl #1]
+; VBITS_GE_1024-NEXT:    ld1sh { z1.s }, p0/z, [x0]
 ; VBITS_GE_1024-NEXT:    st1w { z0.s }, p0, [x8, x9, lsl #2]
 ; VBITS_GE_1024-NEXT:    st1w { z1.s }, p0, [x8]
 ; VBITS_GE_1024-NEXT:    ret
 ;
 ; VBITS_GE_2048-LABEL: load_sext_v64i16i32:
 ; VBITS_GE_2048:       // %bb.0:
 ; VBITS_GE_2048-NEXT:    ptrue p0.s, vl64
 ; VBITS_GE_2048-NEXT:    ld1sh { z0.s }, p0/z, [x0]
 ; VBITS_GE_2048-NEXT:    st1w { z0.s }, p0, [x8]
 ; VBITS_GE_2048-NEXT:    ret
   %a = load <64 x i16>, ptr %ap
   %val = sext <64 x i16> %a to <64 x i32>
   ret <64 x i32> %val
 }
 
 define <32 x i64> @load_zext_v32i8i64(ptr %ap) #0 {
 ; VBITS_GE_1024-LABEL: load_zext_v32i8i64:
 ; VBITS_GE_1024:       // %bb.0:
-; VBITS_GE_1024-NEXT:    ptrue p0.b, vl32
-; VBITS_GE_1024-NEXT:    mov x9, #16
-; VBITS_GE_1024-NEXT:    ld1b { z0.b }, p0/z, [x0]
+; VBITS_GE_1024-NEXT:    mov w9, #16 // =0x10
 ; VBITS_GE_1024-NEXT:    ptrue p0.d, vl16
-; VBITS_GE_1024-NEXT:    uunpklo z1.h, z0.b
-; VBITS_GE_1024-NEXT:    ext z0.b, z0.b, z0.b, #16
-; VBITS_GE_1024-NEXT:    uunpklo z0.h, z0.b
-; VBITS_GE_1024-NEXT:    uunpklo z1.s, z1.h
-; VBITS_GE_1024-NEXT:    uunpklo z0.s, z0.h
-; VBITS_GE_1024-NEXT:    uunpklo z1.d, z1.s
-; VBITS_GE_1024-NEXT:    uunpklo z0.d, z0.s
+; VBITS_GE_1024-NEXT:    ld1b { z0.d }, p0/z, [x0, x9]
+; VBITS_GE_1024-NEXT:    ld1b { z1.d }, p0/z, [x0]
+; VBITS_GE_1024-NEXT:    mov x9, #16 // =0x10
 ; VBITS_GE_1024-NEXT:    st1d { z0.d }, p0, [x8, x9, lsl #3]
 ; VBITS_GE_1024-NEXT:    st1d { z1.d }, p0, [x8]
 ; VBITS_GE_1024-NEXT:    ret
 ;
 ; VBITS_GE_2048-LABEL: load_zext_v32i8i64:
 ; VBITS_GE_2048:       // %bb.0:
 ; VBITS_GE_2048-NEXT:    ptrue p0.d, vl32
 ; VBITS_GE_2048-NEXT:    ld1b { z0.d }, p0/z, [x0]
 ; VBITS_GE_2048-NEXT:    st1d { z0.d }, p0, [x8]
 ; VBITS_GE_2048-NEXT:    ret
   %a = load <32 x i8>, ptr %ap
   %val = zext <32 x i8> %a to <32 x i64>
   ret <32 x i64> %val
 }
 
 define <32 x i64> @load_sext_v32i8i64(ptr %ap) #0 {
 ; VBITS_GE_1024-LABEL: load_sext_v32i8i64:
 ; VBITS_GE_1024:       // %bb.0:
-; VBITS_GE_1024-NEXT:    ptrue p0.b, vl32
-; VBITS_GE_1024-NEXT:    mov x9, #16
-; VBITS_GE_1024-NEXT:    ld1b { z0.b }, p0/z, [x0]
+; VBITS_GE_1024-NEXT:    mov w9, #16 // =0x10
 ; VBITS_GE_1024-NEXT:    ptrue p0.d, vl16
-; VBITS_GE_1024-NEXT:    sunpklo z1.h, z0.b
-; VBITS_GE_1024-NEXT:    ext z0.b, z0.b, z0.b, #16
-; VBITS_GE_1024-NEXT:    sunpklo z0.h, z0.b
-; VBITS_GE_1024-NEXT:    sunpklo z1.s, z1.h
-; VBITS_GE_1024-NEXT:    sunpklo z0.s, z0.h
-; VBITS_GE_1024-NEXT:    sunpklo z1.d, z1.s
-; VBITS_GE_1024-NEXT:    sunpklo z0.d, z0.s
+; VBITS_GE_1024-NEXT:    ld1sb { z0.d }, p0/z, [x0, x9]
+; VBITS_GE_1024-NEXT:    ld1sb { z1.d }, p0/z, [x0]
+; VBITS_GE_1024-NEXT:    mov x9, #16 // =0x10
 ; VBITS_GE_1024-NEXT:    st1d { z0.d }, p0, [x8, x9, lsl #3]
 ; VBITS_GE_1024-NEXT:    st1d { z1.d }, p0, [x8]
 ; VBITS_GE_1024-NEXT:    ret
 ;
 ; VBITS_GE_2048-LABEL: load_sext_v32i8i64:
 ; VBITS_GE_2048:       // %bb.0:
 ; VBITS_GE_2048-NEXT:    ptrue p0.d, vl32
 ; VBITS_GE_2048-NEXT:    ld1sb { z0.d }, p0/z, [x0]
 ; VBITS_GE_2048-NEXT:    st1d { z0.d }, p0, [x8]
 ; VBITS_GE_2048-NEXT:    ret
   %a = load <32 x i8>, ptr %ap
   %val = sext <32 x i8> %a to <32 x i64>
   ret <32 x i64> %val
 }
 
 define <32 x i64> @load_zext_v32i16i64(ptr %ap) #0 {
 ; VBITS_GE_1024-LABEL: load_zext_v32i16i64:
 ; VBITS_GE_1024:       // %bb.0:
-; VBITS_GE_1024-NEXT:    ptrue p0.h, vl32
-; VBITS_GE_1024-NEXT:    mov x9, #16
-; VBITS_GE_1024-NEXT:    ld1h { z0.h }, p0/z, [x0]
+; VBITS_GE_1024-NEXT:    mov x9, #16 // =0x10
 ; VBITS_GE_1024-NEXT:    ptrue p0.d, vl16
-; VBITS_GE_1024-NEXT:    uunpklo z1.s, z0.h
-; VBITS_GE_1024-NEXT:    ext z0.b, z0.b, z0.b, #32
-; VBITS_GE_1024-NEXT:    uunpklo z0.s, z0.h
-; VBITS_GE_1024-NEXT:    uunpklo z1.d, z1.s
-; VBITS_GE_1024-NEXT:    uunpklo z0.d, z0.s
+; VBITS_GE_1024-NEXT:    ld1h { z0.d }, p0/z, [x0, x9, lsl #1]
+; VBITS_GE_1024-NEXT:    ld1h { z1.d }, p0/z, [x0]
 ; VBITS_GE_1024-NEXT:    st1d { z0.d }, p0, [x8, x9, lsl #3]
 ; VBITS_GE_1024-NEXT:    st1d { z1.d }, p0, [x8]
 ; VBITS_GE_1024-NEXT:    ret
 ;
 ; VBITS_GE_2048-LABEL: load_zext_v32i16i64:
 ; VBITS_GE_2048:       // %bb.0:
 ; VBITS_GE_2048-NEXT:    ptrue p0.d, vl32
 ; VBITS_GE_2048-NEXT:    ld1h { z0.d }, p0/z, [x0]
 ; VBITS_GE_2048-NEXT:    st1d { z0.d }, p0, [x8]
 ; VBITS_GE_2048-NEXT:    ret
   %a = load <32 x i16>, ptr %ap
   %val = zext <32 x i16> %a to <32 x i64>
   ret <32 x i64> %val
 }
 
 define <32 x i64> @load_sext_v32i16i64(ptr %ap) #0 {
 ; VBITS_GE_1024-LABEL: load_sext_v32i16i64:
 ; VBITS_GE_1024:       // %bb.0:
-; VBITS_GE_1024-NEXT:    ptrue p0.h, vl32
-; VBITS_GE_1024-NEXT:    mov x9, #16
-; VBITS_GE_1024-NEXT:    ld1h { z0.h }, p0/z, [x0]
+; VBITS_GE_1024-NEXT:    mov x9, #16 // =0x10
 ; VBITS_GE_1024-NEXT:    ptrue p0.d, vl16
-; VBITS_GE_1024-NEXT:    sunpklo z1.s, z0.h
-; VBITS_GE_1024-NEXT:    ext z0.b, z0.b, z0.b, #32
-; VBITS_GE_1024-NEXT:    sunpklo z0.s, z0.h
-; VBITS_GE_1024-NEXT:    sunpklo z1.d, z1.s
-; VBITS_GE_1024-NEXT:    sunpklo z0.d, z0.s
+; VBITS_GE_1024-NEXT:    ld1sh { z0.d }, p0/z, [x0, x9, lsl #1]
+; VBITS_GE_1024-NEXT:    ld1sh { z1.d }, p0/z, [x0]
 ; VBITS_GE_1024-NEXT:    st1d { z0.d }, p0, [x8, x9, lsl #3]
 ; VBITS_GE_1024-NEXT:    st1d { z1.d }, p0, [x8]
 ; VBITS_GE_1024-NEXT:    ret
 ;
 ; VBITS_GE_2048-LABEL: load_sext_v32i16i64:
 ; VBITS_GE_2048:       // %bb.0:
 ; VBITS_GE_2048-NEXT:    ptrue p0.d, vl32
 ; VBITS_GE_2048-NEXT:    ld1sh { z0.d }, p0/z, [x0]
 ; VBITS_GE_2048-NEXT:    st1d { z0.d }, p0, [x8]
 ; VBITS_GE_2048-NEXT:    ret
   %a = load <32 x i16>, ptr %ap
   %val = sext <32 x i16> %a to <32 x i64>
   ret <32 x i64> %val
 }
 
 define <32 x i64> @load_zext_v32i32i64(ptr %ap) #0 {
 ; VBITS_GE_1024-LABEL: load_zext_v32i32i64:
 ; VBITS_GE_1024:       // %bb.0:
-; VBITS_GE_1024-NEXT:    ptrue p0.s, vl32
-; VBITS_GE_1024-NEXT:    mov x9, #16
-; VBITS_GE_1024-NEXT:    ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_1024-NEXT:    mov x9, #16 // =0x10
 ; VBITS_GE_1024-NEXT:    ptrue p0.d, vl16
-; VBITS_GE_1024-NEXT:    uunpklo z1.d, z0.s
-; VBITS_GE_1024-NEXT:    ext z0.b, z0.b, z0.b, #64
-; VBITS_GE_1024-NEXT:    uunpklo z0.d, z0.s
+; VBITS_GE_1024-NEXT:    ld1w { z0.d }, p0/z, [x0, x9, lsl #2]
+; VBITS_GE_1024-NEXT:    ld1w { z1.d }, p0/z, [x0]
 ; VBITS_GE_1024-NEXT:    st1d { z0.d }, p0, [x8, x9, lsl #3]
 ; VBITS_GE_1024-NEXT:    st1d { z1.d }, p0, [x8]
 ; VBITS_GE_1024-NEXT:    ret
 ;
 ; VBITS_GE_2048-LABEL: load_zext_v32i32i64:
 ; VBITS_GE_2048:       // %bb.0:
 ; VBITS_GE_2048-NEXT:    ptrue p0.d, vl32
 ; VBITS_GE_2048-NEXT:    ld1w { z0.d }, p0/z, [x0]
 ; VBITS_GE_2048-NEXT:    st1d { z0.d }, p0, [x8]
 ; VBITS_GE_2048-NEXT:    ret
   %a = load <32 x i32>, ptr %ap
   %val = zext <32 x i32> %a to <32 x i64>
   ret <32 x i64> %val
 }
 
 define <32 x i64> @load_sext_v32i32i64(ptr %ap) #0 {
 ; VBITS_GE_1024-LABEL: load_sext_v32i32i64:
 ; VBITS_GE_1024:       // %bb.0:
-; VBITS_GE_1024-NEXT:    ptrue p0.s, vl32
-; VBITS_GE_1024-NEXT:    mov x9, #16
-; VBITS_GE_1024-NEXT:    ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_1024-NEXT:    mov x9, #16 // =0x10
 ; VBITS_GE_1024-NEXT:    ptrue p0.d, vl16
-; VBITS_GE_1024-NEXT:    sunpklo z1.d, z0.s
-; VBITS_GE_1024-NEXT:    ext z0.b, z0.b, z0.b, #64
-; VBITS_GE_1024-NEXT:    sunpklo z0.d, z0.s
+; VBITS_GE_1024-NEXT:    ld1sw { z0.d }, p0/z, [x0, x9, lsl #2]
+; VBITS_GE_1024-NEXT:    ld1sw { z1.d }, p0/z, [x0]
 ; VBITS_GE_1024-NEXT:    st1d { z0.d }, p0, [x8, x9, lsl #3]
 ; VBITS_GE_1024-NEXT:    st1d { z1.d }, p0, [x8]
 ; VBITS_GE_1024-NEXT:    ret
 ;
 ; VBITS_GE_2048-LABEL: load_sext_v32i32i64:
 ; VBITS_GE_2048:       // %bb.0:
 ; VBITS_GE_2048-NEXT:    ptrue p0.d, vl32
 ; VBITS_GE_2048-NEXT:    ld1sw { z0.d }, p0/z, [x0]
 ; VBITS_GE_2048-NEXT:    st1d { z0.d }, p0, [x8]
 ; VBITS_GE_2048-NEXT:    ret
   %a = load <32 x i32>, ptr %ap
   %val = sext <32 x i32> %a to <32 x i64>
   ret <32 x i64> %val
 }
 
 attributes #0 = { "target-features"="+sve" }

llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ext-loads.ll

; CHECK-NEXT:    ret
   %a = load <2 x i64>, ptr %ap
   %val = zext <2 x i64> %a to <2 x i256>
   ret <2 x i256> %val
 }
 
 define <16 x i32> @load_sext_v16i8i32(ptr %ap)  #0 {
 ; CHECK-LABEL: load_sext_v16i8i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldr q1, [x0]
-; CHECK-NEXT:    sunpklo z3.h, z1.b
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
-; CHECK-NEXT:    sunpklo z4.h, z1.b
-; CHECK-NEXT:    sunpklo z0.s, z3.h
-; CHECK-NEXT:    ext z3.b, z3.b, z3.b, #8
-; CHECK-NEXT:    sunpklo z2.s, z4.h
-; CHECK-NEXT:    ext z4.b, z4.b, z4.b, #8
-; CHECK-NEXT:    sunpklo z1.s, z3.h
-; CHECK-NEXT:    sunpklo z3.s, z4.h
+; CHECK-NEXT:    mov w8, #4 // =0x4
+; CHECK-NEXT:    mov w9, #8 // =0x8
+; CHECK-NEXT:    mov w10, #12 // =0xc
+; CHECK-NEXT:    ptrue p0.s, vl4
+; CHECK-NEXT:    ld1sb { z1.s }, p0/z, [x0, x8]
+; CHECK-NEXT:    ld1sb { z2.s }, p0/z, [x0, x9]
+; CHECK-NEXT:    ld1sb { z3.s }, p0/z, [x0, x10]
+; CHECK-NEXT:    ld1sb { z0.s }, p0/z, [x0]
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $z1
 ; CHECK-NEXT:    // kill: def $q2 killed $q2 killed $z2
 ; CHECK-NEXT:    // kill: def $q3 killed $q3 killed $z3
 ; CHECK-NEXT:    ret
   %a = load <16 x i8>, ptr %ap
   %val = sext <16 x i8> %a to <16 x i32>
   ret <16 x i32> %val
 }
 
 define <8 x i32> @load_sext_v8i16i32(ptr %ap)  #0 {
 ; CHECK-LABEL: load_sext_v8i16i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldr q1, [x0]
-; CHECK-NEXT:    sunpklo z0.s, z1.h
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
-; CHECK-NEXT:    sunpklo z1.s, z1.h
+; CHECK-NEXT:    mov x8, #4 // =0x4
+; CHECK-NEXT:    ptrue p0.s, vl4
+; CHECK-NEXT:    ld1sh { z1.s }, p0/z, [x0, x8, lsl #1]
+; CHECK-NEXT:    ld1sh { z0.s }, p0/z, [x0]
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $z1
 ; CHECK-NEXT:    ret
   %a = load <8 x i16>, ptr %ap
   %val = sext <8 x i16> %a to <8 x i32>
   ret <8 x i32> %val
 }
 
; CHECK-NEXT:    ret
   %a = load <2 x i64>, ptr %ap
   %val = sext <2 x i64> %a to <2 x i256>
   ret <2 x i256> %val
 }
 
 define <16 x i64> @load_zext_v16i16i64(ptr %ap)  #0 {
 ; CHECK-LABEL: load_zext_v16i16i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q1, q2, [x0]
-; CHECK-NEXT:    uunpklo z3.s, z1.h
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
-; CHECK-NEXT:    uunpklo z7.s, z1.h
-; CHECK-NEXT:    uunpklo z0.d, z3.s
-; CHECK-NEXT:    uunpklo z5.s, z2.h
-; CHECK-NEXT:    ext z2.b, z2.b, z2.b, #8
-; CHECK-NEXT:    uunpklo z16.s, z2.h
-; CHECK-NEXT:    uunpklo z4.d, z5.s
-; CHECK-NEXT:    ext z3.b, z3.b, z3.b, #8
-; CHECK-NEXT:    ext z5.b, z5.b, z5.b, #8
-; CHECK-NEXT:    uunpklo z2.d, z7.s
-; CHECK-NEXT:    uunpklo z6.d, z16.s
-; CHECK-NEXT:    ext z7.b, z7.b, z7.b, #8
-; CHECK-NEXT:    ext z16.b, z16.b, z16.b, #8
-; CHECK-NEXT:    uunpklo z1.d, z3.s
-; CHECK-NEXT:    uunpklo z5.d, z5.s
-; CHECK-NEXT:    uunpklo z3.d, z7.s
-; CHECK-NEXT:    uunpklo z7.d, z16.s
+; CHECK-NEXT:    mov x8, #2 // =0x2
+; CHECK-NEXT:    mov x9, #4 // =0x4
+; CHECK-NEXT:    ptrue p0.d, vl2
+; CHECK-NEXT:    mov x10, #6 // =0x6
+; CHECK-NEXT:    mov x11, #8 // =0x8
+; CHECK-NEXT:    mov x12, #10 // =0xa
+; CHECK-NEXT:    ld1h { z1.d }, p0/z, [x0, x8, lsl #1]
+; CHECK-NEXT:    mov x8, #12 // =0xc
+; CHECK-NEXT:    ld1h { z2.d }, p0/z, [x0, x9, lsl #1]
+; CHECK-NEXT:    mov x9, #14 // =0xe
+; CHECK-NEXT:    ld1h { z3.d }, p0/z, [x0, x10, lsl #1]
+; CHECK-NEXT:    ld1h { z4.d }, p0/z, [x0, x11, lsl #1]
+; CHECK-NEXT:    ld1h { z5.d }, p0/z, [x0, x12, lsl #1]
+; CHECK-NEXT:    ld1h { z6.d }, p0/z, [x0, x8, lsl #1]
+; CHECK-NEXT:    ld1h { z7.d }, p0/z, [x0, x9, lsl #1]
+; CHECK-NEXT:    ld1h { z0.d }, p0/z, [x0]
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $z1
 ; CHECK-NEXT:    // kill: def $q2 killed $q2 killed $z2
 ; CHECK-NEXT:    // kill: def $q3 killed $q3 killed $z3
 ; CHECK-NEXT:    // kill: def $q4 killed $q4 killed $z4
 ; CHECK-NEXT:    // kill: def $q5 killed $q5 killed $z5
 ; CHECK-NEXT:    // kill: def $q6 killed $q6 killed $z6
 ; CHECK-NEXT:    // kill: def $q7 killed $q7 killed $z7
 ; CHECK-NEXT:    ret
   %a = load <16 x i16>, ptr %ap
   %val = zext <16 x i16> %a to <16 x i64>
   ret <16 x i64> %val
 }
 
 attributes #0 = { "target-features"="+sve" }