[SVE] Implement fixed-width ZEXT lowering

[cameron.mcinally]

Summary

Looks like this patch is already outdated by D85546, but thought I'd offer it anyway in case the framework helps...

This patch is an attempt to lower fixed-width ZEXT by bitcasting the input to the result type, then AND'ing with an immediate that matches the input type.

[paulwalker-arm]

It does seem like we're risking treading on each others toes :) Perhaps worth syncing up so we don't duplicate effort. I had been trying to kill off D71767 but instead I'll updated it (probably tomorrow) to show work I've already got downstream where I just need to write tests. I doubt I'll get chance to cover additional nodes over the next few weeks so everything else should be fair game.

llvm/test/CodeGen/AArch64/sve-fixed-length-zext.ll

78–85

The test's output shows this is not really a zero extend.

The load will read 32 bytes into consecutive byte lanes. The and will zero the odd numbered bytes (i.e. zero-extend the even lanes). The store will write those 16 zero-extend even numbered bytes along with 16 more zeros that result from the load zeroing byte 32 onward.

I'm afraid to say that currently the extends are not going to be a cheap operation being they are effectively the reverse of the truncate operation. Truncates use a uzp1 sequence, so the extends require an upklo sequence.

I've got a patch already to do this that I'll add you to.

[cameron.mcinally]

llvm/test/CodeGen/AArch64/sve-fixed-length-zext.ll

78–85

Ah, they're packed. I got mixed up. Was thinking of it as:

<n x 2 x i64> res = zext(<n x 2 x i32> x)

But, yeah, I see the problem with that now. Thanks.

llvm/lib/Target/AArch64/AArch64ISelLowering.h

private:
   SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVectorOR(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVSCALE(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerZERO_EXTEND(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVECREDUCE(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerATOMIC_LOAD_SUB(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerATOMIC_LOAD_AND(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerWindowsDYNAMIC_STACKALLOC(SDValue Op, SDValue Chain,
                                          SDValue &Size,
                                          SelectionDAG &DAG) const;
   SDValue LowerSVEStructLoad(unsigned Intrinsic, ArrayRef<SDValue> LoadOps,
                              EVT VT, SelectionDAG &DAG, const SDLoc &DL) const;
 
   SDValue LowerFixedLengthVectorLoadToSVE(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFixedLengthVectorStoreToSVE(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFixedLengthVectorTruncateToSVE(SDValue Op,
                                               SelectionDAG &DAG) const;
+  SDValue LowerFixedLengthVectorZeroExtendToSVE(SDValue Op,
+                                                SelectionDAG &DAG) const;
 
   SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
                         SmallVectorImpl<SDNode *> &Created) const override;
   SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
                           int &ExtraSteps, bool &UseOneConst,
                           bool Reciprocal) const override;
   SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
                            int &ExtraSteps) const override;

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

void AArch64TargetLowering::addTypeForFixedLengthSVE(MVT VT) {
   setOperationAction(ISD::FADD, VT, Custom);
   setOperationAction(ISD::FDIV, VT, Custom);
   setOperationAction(ISD::FMA, VT, Custom);
   setOperationAction(ISD::FMUL, VT, Custom);
   setOperationAction(ISD::FSUB, VT, Custom);
   setOperationAction(ISD::LOAD, VT, Custom);
   setOperationAction(ISD::STORE, VT, Custom);
   setOperationAction(ISD::TRUNCATE, VT, Custom);
+  setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
 }
 
 void AArch64TargetLowering::addDRTypeForNEON(MVT VT) {
   addRegisterClass(VT, &AArch64::FPR64RegClass);
   addTypeForNEON(VT, MVT::v2i32);
 }
 
 void AArch64TargetLowering::addQRTypeForNEON(MVT VT) {
SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
   case ISD::ATOMIC_LOAD_AND:
     return LowerATOMIC_LOAD_AND(Op, DAG);
   case ISD::DYNAMIC_STACKALLOC:
     return LowerDYNAMIC_STACKALLOC(Op, DAG);
   case ISD::VSCALE:
     return LowerVSCALE(Op, DAG);
   case ISD::TRUNCATE:
     return LowerTRUNCATE(Op, DAG);
+  case ISD::ZERO_EXTEND:
+    return LowerZERO_EXTEND(Op, DAG);
   case ISD::LOAD:
     if (useSVEForFixedLengthVectorVT(Op.getValueType()))
       return LowerFixedLengthVectorLoadToSVE(Op, DAG);
     llvm_unreachable("Unexpected request to lower ISD::LOAD");
   case ISD::ADD:
     if (useSVEForFixedLengthVectorVT(Op.getValueType()))
       return LowerToPredicatedOp(Op, DAG, AArch64ISD::ADD_PRED);
     llvm_unreachable("Unexpected request to lower ISD::ADD");
if (!VT.isVector() || VT.isScalableVector())
     return SDValue();
 
   if (useSVEForFixedLengthVectorVT(Op.getOperand(0).getValueType()))
     return LowerFixedLengthVectorTruncateToSVE(Op, DAG);
 
   return SDValue();
 }
 
+SDValue AArch64TargetLowering::LowerZERO_EXTEND(SDValue Op,
+                                                SelectionDAG &DAG) const {
+  if (useSVEForFixedLengthVectorVT(Op.getOperand(0).getValueType()))
+    return LowerFixedLengthVectorZeroExtendToSVE(Op, DAG);
+
+  return SDValue();
+}
+
 SDValue AArch64TargetLowering::LowerVectorSRA_SRL_SHL(SDValue Op,
                                                       SelectionDAG &DAG) const {
   EVT VT = Op.getValueType();
   SDLoc DL(Op);
   int64_t Cnt;
 
   if (!Op.getOperand(1).getValueType().isVector())
     return Op;
case MVT::nxv8i16:
     Val = DAG.getNode(AArch64ISD::UZP1, DL, MVT::nxv16i8, Val, Val);
     assert(VT.getVectorElementType() == MVT::i8 && "Unexpected element type!");
     break;
   }
 
   return convertFromScalableVector(DAG, VT, Val);
 }
 
+SDValue AArch64TargetLowering::LowerFixedLengthVectorZeroExtendToSVE(
+    SDValue Op, SelectionDAG &DAG) const {
+  EVT VT = Op.getValueType();
+  assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
+
+  SDLoc DL(Op);
+  SDValue Val = Op.getOperand(0);
+  EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
+  EVT OpContainerVT = getContainerForFixedLengthVector(DAG, Val.getValueType());
+  Val = convertToScalableVector(DAG, ContainerVT, Val);
+
+  SDValue Imm;
+  switch (OpContainerVT.getVectorElementType().getSimpleVT().SimpleTy) {
+  default:
+    llvm_unreachable("unimplemented container type");
+  case MVT::i8:
+    Imm = DAG.getConstant(0xFF, DL, ContainerVT);
+    break;
+  case MVT::i16:
+    Imm = DAG.getConstant(0xFFFF, DL, ContainerVT);
+    break;
+  case MVT::i32:
+    Imm = DAG.getConstant(0xFFFFFFFF, DL, ContainerVT);
+    break;
+  }
+
+  Val = DAG.getNode(ISD::BITCAST, DL, ContainerVT, Val);
+  Val = DAG.getNode(ISD::AND, DL, ContainerVT, Val, Imm);
+
+  return convertFromScalableVector(DAG, VT, Val);
+}
+
 SDValue AArch64TargetLowering::LowerToPredicatedOp(SDValue Op,
                                                    SelectionDAG &DAG,
                                                    unsigned NewOp) const {
   EVT VT = Op.getValueType();
   SDLoc DL(Op);
   auto Pg = getPredicateForVector(DAG, DL, VT);
 
   if (useSVEForFixedLengthVectorVT(VT)) {

llvm/test/CodeGen/AArch64/sve-fixed-length-zext.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -aarch64-sve-vector-bits-min=128 -asm-verbose=0 < %s | FileCheck %s -check-prefix=NO_SVE
+; RUN: llc -aarch64-sve-vector-bits-min=256 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK
+; RUN: llc -aarch64-sve-vector-bits-min=384 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK
+; RUN: llc -aarch64-sve-vector-bits-min=512 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=640 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=768 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=896 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=1024 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1152 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1280 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1408 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1536 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1664 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1792 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1920 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=2048 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024,VBITS_GE_2048
+
+; Don't use SVE when its registers are no bigger than NEON.
+; NO_SVE-NOT: z{0-9}
+
+target triple = "aarch64-unknown-linux-gnu"
+
+; zero_extend vXi8 -> vXi16
+
+; NOTE: Input is NEON vector - Output is Scalable vector.
+define <16 x i16> @zext_v16i8_v16i16(<16 x i8>* %in) #0 {
+; CHECK-LABEL: zext_v16i8_v16i16:
+; CHECK:    stp x29, x30, [sp, #-16]!
+; CHECK:    sub x9, sp, #48
+; CHECK:    mov x29, sp
+; CHECK:    and sp, x9, #0xffffffffffffffe0
+; CHECK:    ldr q0, [x0]
+; CHECK:    ptrue [[PG:p[0-9]+]].h, vl16
+; CHECK:    umov w9, v0.b[15]
+; CHECK:    strh w9, [sp, #30]
+; CHECK:    umov w9, v0.b[14]
+; CHECK:    strh w9, [sp, #28]
+; CHECK:    umov w9, v0.b[13]
+; CHECK:    strh w9, [sp, #26]
+; CHECK:    umov w9, v0.b[12]
+; CHECK:    strh w9, [sp, #24]
+; CHECK:    umov w9, v0.b[11]
+; CHECK:    strh w9, [sp, #22]
+; CHECK:    umov w9, v0.b[10]
+; CHECK:    strh w9, [sp, #20]
+; CHECK:    umov w9, v0.b[9]
+; CHECK:    strh w9, [sp, #18]
+; CHECK:    umov w9, v0.b[8]
+; CHECK:    strh w9, [sp, #16]
+; CHECK:    umov w9, v0.b[7]
+; CHECK:    strh w9, [sp, #14]
+; CHECK:    umov w9, v0.b[6]
+; CHECK:    strh w9, [sp, #12]
+; CHECK:    umov w9, v0.b[5]
+; CHECK:    strh w9, [sp, #10]
+; CHECK:    umov w9, v0.b[4]
+; CHECK:    strh w9, [sp, #8]
+; CHECK:    umov w9, v0.b[3]
+; CHECK:    strh w9, [sp, #6]
+; CHECK:    umov w9, v0.b[2]
+; CHECK:    strh w9, [sp, #4]
+; CHECK:    umov w9, v0.b[1]
+; CHECK:    strh w9, [sp, #2]
+; CHECK:    umov w9, v0.b[0]
+; CHECK:    strh w9, [sp]
+; CHECK:    mov x9, sp
+; CHECK:    ld1h { [[ZPR:z[0-9]+]].h }, [[PG]]/z, [x9]
+; CHECK:    st1h { [[ZPR]].h }, [[PG]], [x8]
+; CHECK:    mov sp, x29
+; CHECK:    ldp x29, x30, [sp], #16
+; CHECK:    ret
+  %a = load <16 x i8>, <16 x i8>* %in
+  %b = zext <16 x i8> %a to <16 x i16>
+  ret <16 x i16> %b
+}
+
+define <32 x i16> @zext_v32i8_v32i16(<32 x i8>* %in) #0 {
+; VBITS_GE_512-LABEL: zext_v32i8_v32i16:
+; VBITS_GE_512:    ptrue [[PG0:p[0-9]+]].b, vl32
+; VBITS_GE_512-NEXT:    ld1b { [[ZPR:z[0-9]+]].b }, [[PG0:p[0-9]+]]/z, [x0]
+; VBITS_GE_512-NEXT:    ptrue [[PG1:p[0-9]+]].h, vl32
+; VBITS_GE_512-NEXT:    and [[ZPR]].h, [[ZPR]].h, #0xff
+; VBITS_GE_512-NEXT:    st1h { [[ZPR]].h }, [[PG1]], [x8]
+; VBITS_GE_512-NEXT:    ret
+  %a = load <32 x i8>, <32 x i8>* %in
+  %b = zext <32 x i8> %a to <32 x i16>
+  ret <32 x i16> %b
+}
+
+define <64 x i16> @zext_v64i8_v64i16(<64 x i8>* %in) #0 {
+; VBITS_GE_1024-LABEL: zext_v64i8_v64i16:
+; VBITS_GE_1024:    ptrue [[PG0:p[0-9]+]].b, vl64
+; VBITS_GE_1024-NEXT:    ld1b { [[ZPR:z[0-9]+]].b }, [[PG0:p[0-9]+]]/z, [x0]
+; VBITS_GE_1024-NEXT:    ptrue [[PG1:p[0-9]+]].h, vl64
+; VBITS_GE_1024-NEXT:    and [[ZPR]].h, [[ZPR]].h, #0xff
+; VBITS_GE_1024-NEXT:    st1h { [[ZPR]].h }, [[PG1]], [x8]
+; VBITS_GE_1024-NEXT:    ret
+  %a = load <64 x i8>, <64 x i8>* %in
+  %b = zext <64 x i8> %a to <64 x i16>
+  ret <64 x i16> %b
+}
+
+; zero_extend vXi8 -> vXi32
+
+; NOTE: Input is NEON vector - Output is Scalable vector.
+define <8 x i32> @zext_v8i8_v8i32(<8 x i8>* %in) #0 {
+; CHECK-LABEL: zext_v8i8_v8i32:
+; CHECK:    stp x29, x30, [sp, #-16]!
+; CHECK:    sub x9, sp, #48
+; CHECK:    mov x29, sp
+; CHECK:    and sp, x9, #0xffffffffffffffe0
+; CHECK:    ldr d0, [x0]
+; CHECK:    ptrue [[PG:p[0-9]+]].s, vl8
+; CHECK:    umov w9, v0.b[7]
+; CHECK:    umov w10, v0.b[6]
+; CHECK:    umov w13, v0.b[3]
+; CHECK:    stp w10, w9, [sp, #24]
+; CHECK:    umov w9, v0.b[2]
+; CHECK:    umov w11, v0.b[5]
+; CHECK:    umov w12, v0.b[4]
+; CHECK:    umov w10, v0.b[1]
+; CHECK:    stp w9, w13, [sp, #8]
+; CHECK:    umov w9, v0.b[0]
+; CHECK:    stp w12, w11, [sp, #16]
+; CHECK:    stp w9, w10, [sp]
+; CHECK:    mov x9, sp
+; CHECK:    ld1w { [[ZPR:z[0-9]+]].s }, [[PG]]/z, [x9]
+; CHECK:    st1w { [[ZPR]].s }, [[PG]], [x8]
+; CHECK:    mov sp, x29
+; CHECK:    ldp x29, x30, [sp], #16
+; CHECK:    ret
+  %a = load <8 x i8>, <8 x i8>* %in
+  %b = zext <8 x i8> %a to <8 x i32>
+  ret <8 x i32> %b
+}
+
+; NOTE: Input is NEON vector - Output is Scalable vector.
+define <16 x i32> @zext_v16i8_v16i32(<16 x i8>* %in) #0 {
+; VBITS_GE_512-LABEL: zext_v16i8_v16i32:
+; VBITS_GE_512:    stp x29, x30, [sp, #-16]!
+; VBITS_GE_512:    sub x9, sp, #112
+; VBITS_GE_512:    mov x29, sp
+; VBITS_GE_512:    and sp, x9, #0xffffffffffffffc0
+; VBITS_GE_512:    ldr q0, [x0]
+; VBITS_GE_512:    ptrue [[PG:p[0-9]+]].s, vl16
+; VBITS_GE_512:    umov w9, v0.b[15]
+; VBITS_GE_512:    umov w10, v0.b[14]
+; VBITS_GE_512:    umov w11, v0.b[13]
+; VBITS_GE_512:    umov w12, v0.b[12]
+; VBITS_GE_512:    umov w17, v0.b[7]
+; VBITS_GE_512:    stp w10, w9, [sp, #56]
+; VBITS_GE_512:    umov w9, v0.b[6]
+; VBITS_GE_512:    stp w12, w11, [sp, #48]
+; VBITS_GE_512:    umov w10, v0.b[5]
+; VBITS_GE_512:    umov w11, v0.b[4]
+; VBITS_GE_512:    umov w12, v0.b[3]
+; VBITS_GE_512:    stp w9, w17, [sp, #24]
+; VBITS_GE_512:    umov w9, v0.b[2]
+; VBITS_GE_512:    umov w13, v0.b[11]
+; VBITS_GE_512:    umov w14, v0.b[10]
+; VBITS_GE_512:    umov w15, v0.b[9]
+; VBITS_GE_512:    umov w16, v0.b[8]
+; VBITS_GE_512:    stp w11, w10, [sp, #16]
+; VBITS_GE_512:    umov w10, v0.b[1]
+; VBITS_GE_512:    stp w9, w12, [sp, #8]
+; VBITS_GE_512:    umov w9, v0.b[0]
+; VBITS_GE_512:    stp w14, w13, [sp, #40]
+; VBITS_GE_512:    stp w16, w15, [sp, #32]
+; VBITS_GE_512:    stp w9, w10, [sp]
+; VBITS_GE_512:    mov x9, sp
+; VBITS_GE_512:    ld1w { [[ZPR:z[0-9]+]].s }, [[PG]]/z, [x9]
+; VBITS_GE_512:    st1w { [[ZPR]].s }, [[PG]], [x8]
+; VBITS_GE_512:    mov sp, x29
+; VBITS_GE_512:    ldp x29, x30, [sp], #16
+; VBITS_GE_512:    ret
+  %a = load <16 x i8>, <16 x i8>* %in
+  %b = zext <16 x i8> %a to <16 x i32>
+  ret <16 x i32> %b
+}
+
+define <32 x i32> @zext_v32i8_v32i32(<32 x i8>* %in) #0 {
+; VBITS_GE_1024-LABEL: zext_v32i8_v32i32:
+; VBITS_GE_1024:    ptrue [[PG0:p[0-9]+]].b, vl32
+; VBITS_GE_1024-NEXT:    ld1b { [[ZPR:z[0-9]+]].b }, [[PG0]]/z, [x0]
+; VBITS_GE_1024-NEXT:    ptrue [[PG1:p[0-9]+]].s, vl32
+; VBITS_GE_1024-NEXT:    and [[ZPR]].s, [[ZPR]].s, #0xff
+; VBITS_GE_1024-NEXT:    st1w { [[ZPR]].s }, [[PG1]], [x8]
+; VBITS_GE_1024-NEXT:    ret
+  %a = load <32 x i8>, <32 x i8>* %in
+  %b = zext <32 x i8> %a to <32 x i32>
+  ret <32 x i32> %b
+}
+
+define <64 x i32> @zext_v64i8_v64i32(<64 x i8>* %in) #0 {
+; VBITS_GE_2048-LABEL: zext_v64i8_v64i32:
+; VBITS_GE_2048:    ptrue [[PG0:p[0-9]+]].b, vl64
+; VBITS_GE_2048-NEXT:    ld1b { [[ZPR:z[0-9]+]].b }, [[PG0]]/z, [x0]
+; VBITS_GE_2048-NEXT:    ptrue [[PG1:p[0-9]+]].s, vl64
+; VBITS_GE_2048-NEXT:    and [[ZPR]].s, [[ZPR]].s, #0xff
+; VBITS_GE_2048-NEXT:    st1w { [[ZPR]].s }, [[PG1]], [x8]
+; VBITS_GE_2048-NEXT:    ret
+  %a = load <64 x i8>, <64 x i8>* %in
+  %b = zext <64 x i8> %a to <64 x i32>
+  ret <64 x i32> %b
+}
+
+; zero_extend vXi16 -> vXi32
+
+; NOTE: Input is NEON vector - Output is Scalable vector.
+define <8 x i32> @zext_v8i16_v8i32(<8 x i16>* %in) #0 {
+; CHECK-LABEL: zext_v8i16_v8i32:
+; CHECK:    stp x29, x30, [sp, #-16]!
+; CHECK:    sub x9, sp, #48
+; CHECK:    mov x29, sp
+; CHECK:    and sp, x9, #0xffffffffffffffe0
+; CHECK:    ldr q0, [x0]
+; CHECK:    ptrue [[PG:p[0-9]+]].s, vl8
+; CHECK:    umov w9, v0.h[7]
+; CHECK:    umov w10, v0.h[6]
+; CHECK:    umov w13, v0.h[3]
+; CHECK:    stp w10, w9, [sp, #24]
+; CHECK:    umov w9, v0.h[2]
+; CHECK:    umov w11, v0.h[5]
+; CHECK:    umov w12, v0.h[4]
+; CHECK:    umov w10, v0.h[1]
+; CHECK:    stp w9, w13, [sp, #8]
+; CHECK:    umov w9, v0.h[0]
+; CHECK:    stp w12, w11, [sp, #16]
+; CHECK:    stp w9, w10, [sp]
+; CHECK:    mov x9, sp
+; CHECK:    ld1w { [[ZPR:z[0-9]+]].s }, [[PG]]/z, [x9]
+; CHECK:    st1w { [[ZPR]].s }, [[PG]], [x8]
+; CHECK:    mov sp, x29
+; CHECK:    ldp x29, x30, [sp], #16
+; CHECK:    ret
+  %a = load <8 x i16>, <8 x i16>* %in
+  %b = zext <8 x i16> %a to <8 x i32>
+  ret <8 x i32> %b
+}
+
+define <16 x i32> @zext_v16i16_v16i32(<16 x i16>* %in) #0 {
+; VBITS_GE_512-LABEL: zext_v16i16_v16i32:
+; VBITS_GE_512:    ptrue [[PG0:p[0-9]+]].h, vl16
+; VBITS_GE_512-NEXT:    ld1h { [[ZPR:z[0-9]+]].h }, [[PG0]]/z, [x0]
+; VBITS_GE_512-NEXT:    ptrue [[PG1:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT:    and [[ZPR]].s, [[ZPR]].s, #0xffff
+; VBITS_GE_512-NEXT:    st1w { [[ZPR]].s }, [[PG1]], [x8]
+; VBITS_GE_512-NEXT:    ret
+  %a = load <16 x i16>, <16 x i16>* %in
+  %b = zext <16 x i16> %a to <16 x i32>
+  ret <16 x i32> %b
+}
+
+define <32 x i32> @zext_v32i16_v32i32(<32 x i16>* %in) #0 {
+; VBITS_GE_1024-LABEL: zext_v32i16_v32i32:
+; VBITS_GE_1024:    ptrue [[PG0:p[0-9]+]].h, vl32
+; VBITS_GE_1024-NEXT:    ld1h { [[ZPR:z[0-9]+]].h }, [[PG0]]/z, [x0]
+; VBITS_GE_1024-NEXT:    ptrue [[PG1:p[0-9]+]].s, vl32
+; VBITS_GE_1024-NEXT:    and [[ZPR]].s, [[ZPR]].s, #0xffff
+; VBITS_GE_1024-NEXT:    st1w { [[ZPR]].s }, [[PG1]], [x8]
+; VBITS_GE_1024-NEXT:    ret
+  %a = load <32 x i16>, <32 x i16>* %in
+  %b = zext <32 x i16> %a to <32 x i32>
+  ret <32 x i32> %b
+}
+
+define <64 x i32> @zext_v64i16_v64i32(<64 x i16>* %in) #0 {
+; VBITS_GE_2048-LABEL: zext_v64i16_v64i32:
+; VBITS_GE_2048:    ptrue [[PG0:p[0-9]+]].h, vl64
+; VBITS_GE_2048-NEXT:    ld1h { [[ZPR:z[0-9]+]].h }, [[PG0]]/z, [x0]
+; VBITS_GE_2048-NEXT:    ptrue [[PG1:p[0-9]+]].s, vl64
+; VBITS_GE_2048-NEXT:    and [[ZPR]].s, [[ZPR]].s, #0xffff
+; VBITS_GE_2048-NEXT:    st1w { [[ZPR]].s }, [[PG1]], [x8]
+; VBITS_GE_2048-NEXT:    ret
+  %a = load <64 x i16>, <64 x i16>* %in
+  %b = zext <64 x i16> %a to <64 x i32>
+  ret <64 x i32> %b
+}
+
+; zero_extend vXi8 -> vXi64
+
+; NOTE: Input is NEON vector - Output is Scalable vector.
+define <4 x i64> @zext_v4i8_v4i64(<4 x i8>* %in) #0 {
+; CHECK-LABEL: zext_v4i8_v4i64:
+; CHECK:    stp x29, x30, [sp, #-16]!
+; CHECK:    sub x9, sp, #48
+; CHECK:    mov x29, sp
+; CHECK:    and sp, x9, #0xffffffffffffffe0
+; CHECK:    ldrb w9, [x0, #3]
+; CHECK:    ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK:    str x9, [sp, #24]
+; CHECK:    ldrb w9, [x0, #2]
+; CHECK:    str x9, [sp, #16]
+; CHECK:    ldrb w9, [x0, #1]
+; CHECK:    str x9, [sp, #8]
+; CHECK:    ldrb w9, [x0]
+; CHECK:    str x9, [sp]
+; CHECK:    mov x9, sp
+; CHECK:    ld1d { [[ZPR:z[0-9]+]].d }, [[PG]]/z, [x9]
+; CHECK:    st1d { [[ZPR]].d }, [[PG]], [x8]
+; CHECK:    mov sp, x29
+; CHECK:    ldp x29, x30, [sp], #16
+; CHECK:    ret
+  %a = load <4 x i8>, <4 x i8>* %in
+  %b = zext <4 x i8> %a to <4 x i64>
+  ret <4 x i64> %b
+}
+
+; NOTE: Input is NEON vector - Output is Scalable vector.
+define <8 x i64> @zext_v8i8_v8i64(<8 x i8>* %in) #0 {
+; VBITS_GE_512-LABEL: zext_v8i8_v8i64:
+; VBITS_GE_512:    stp x29, x30, [sp, #-16]!
+; VBITS_GE_512:    sub x9, sp, #112
+; VBITS_GE_512:    mov x29, sp
+; VBITS_GE_512:    and sp, x9, #0xffffffffffffffc0
+; VBITS_GE_512:    ldr d0, [x0]
+; VBITS_GE_512:    ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512:    umov w9, v0.b[7]
+; VBITS_GE_512:    umov w10, v0.b[6]
+; VBITS_GE_512:    umov w11, v0.b[5]
+; VBITS_GE_512:    umov w12, v0.b[4]
+; VBITS_GE_512:    and x9, x9, #0xff
+; VBITS_GE_512:    and x10, x10, #0xff
+; VBITS_GE_512:    umov w13, v0.b[3]
+; VBITS_GE_512:    stp x10, x9, [sp, #48]
+; VBITS_GE_512:    umov w9, v0.b[2]
+; VBITS_GE_512:    and x10, x11, #0xff
+; VBITS_GE_512:    and x11, x12, #0xff
+; VBITS_GE_512:    stp x11, x10, [sp, #32]
+; VBITS_GE_512:    and x11, x13, #0xff
+; VBITS_GE_512:    and x9, x9, #0xff
+; VBITS_GE_512:    umov w10, v0.b[1]
+; VBITS_GE_512:    stp x9, x11, [sp, #16]
+; VBITS_GE_512:    umov w9, v0.b[0]
+; VBITS_GE_512:    and x10, x10, #0xff
+; VBITS_GE_512:    and x9, x9, #0xff
+; VBITS_GE_512:    stp x9, x10, [sp]
+; VBITS_GE_512:    mov x9, sp
+; VBITS_GE_512:    ld1d { [[ZPR:z[0-9]+]].d }, [[PG]]/z, [x9]
+; VBITS_GE_512:    st1d { [[ZPR]].d }, [[PG]], [x8]
+; VBITS_GE_512:    mov sp, x29
+; VBITS_GE_512:    ldp x29, x30, [sp], #16
+; VBITS_GE_512:    ret
+  %a = load <8 x i8>, <8 x i8>* %in
+  %b = zext <8 x i8> %a to <8 x i64>
+  ret <8 x i64> %b
+}
+
+; NOTE: Input is NEON vector - Output is Scalable vector.
+define <16 x i64> @zext_v16i8_v16i64(<16 x i8>* %in) #0 {
+; VBITS_GE_1024-LABEL: zext_v16i8_v16i64:
+; VBITS_GE_1024:    stp x29, x30, [sp, #-16]!
+; VBITS_GE_1024:    sub x9, sp, #240
+; VBITS_GE_1024:    mov x29, sp
+; VBITS_GE_1024:    and sp, x9, #0xffffffffffffff80
+; VBITS_GE_1024:    ldr q0, [x0]
+; VBITS_GE_1024:    ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024:    umov w9, v0.b[15]
+; VBITS_GE_1024:    umov w10, v0.b[14]
+; VBITS_GE_1024:    umov w11, v0.b[13]
+; VBITS_GE_1024:    umov w12, v0.b[12]
+; VBITS_GE_1024:    and x9, x9, #0xff
+; VBITS_GE_1024:    and x10, x10, #0xff
+; VBITS_GE_1024:    umov w13, v0.b[11]
+; VBITS_GE_1024:    umov w14, v0.b[10]
+; VBITS_GE_1024:    stp x10, x9, [sp, #112]
+; VBITS_GE_1024:    and x10, x11, #0xff
+; VBITS_GE_1024:    and x11, x12, #0xff
+; VBITS_GE_1024:    umov w15, v0.b[9]
+; VBITS_GE_1024:    umov w16, v0.b[8]
+; VBITS_GE_1024:    stp x11, x10, [sp, #96]
+; VBITS_GE_1024:    and x11, x13, #0xff
+; VBITS_GE_1024:    and x12, x14, #0xff
+; VBITS_GE_1024:    umov w17, v0.b[7]
+; VBITS_GE_1024:    umov w9, v0.b[6]
+; VBITS_GE_1024:    stp x12, x11, [sp, #80]
+; VBITS_GE_1024:    and x12, x15, #0xff
+; VBITS_GE_1024:    and x13, x16, #0xff
+; VBITS_GE_1024:    umov w10, v0.b[5]
+; VBITS_GE_1024:    umov w11, v0.b[4]
+; VBITS_GE_1024:    stp x13, x12, [sp, #64]
+; VBITS_GE_1024:    and x13, x17, #0xff
+; VBITS_GE_1024:    and x9, x9, #0xff
+; VBITS_GE_1024:    umov w12, v0.b[3]
+; VBITS_GE_1024:    stp x9, x13, [sp, #48]
+; VBITS_GE_1024:    umov w9, v0.b[2]
+; VBITS_GE_1024:    and x10, x10, #0xff
+; VBITS_GE_1024:    and x11, x11, #0xff
+; VBITS_GE_1024:    stp x11, x10, [sp, #32]
+; VBITS_GE_1024:    and x11, x12, #0xff
+; VBITS_GE_1024:    and x9, x9, #0xff
+; VBITS_GE_1024:    umov w10, v0.b[1]
+; VBITS_GE_1024:    stp x9, x11, [sp, #16]
+; VBITS_GE_1024:    umov w9, v0.b[0]
+; VBITS_GE_1024:    and x10, x10, #0xff
+; VBITS_GE_1024:    and x9, x9, #0xff
+; VBITS_GE_1024:    stp x9, x10, [sp]
+; VBITS_GE_1024:    mov x9, sp
+; VBITS_GE_1024:    ld1d { [[ZPR:z[0-9]+]].d }, [[PG]]/z, [x9]
+; VBITS_GE_1024:    st1d { [[ZPR]].d }, [[PG]], [x8]
+; VBITS_GE_1024:    mov sp, x29
+; VBITS_GE_1024:    ldp x29, x30, [sp], #16
+; VBITS_GE_1024:    ret
+  %a = load <16 x i8>, <16 x i8>* %in
+  %b = zext <16 x i8> %a to <16 x i64>
+  ret <16 x i64> %b
+}
+
+define <32 x i64> @zext_v32i8_v32i64(<32 x i8>* %in) #0 {
+; VBITS_GE_2048-LABEL: zext_v32i8_v32i64:
+; VBITS_GE_2048:    ptrue [[PG0:p[0-9]+]].b, vl32
+; VBITS_GE_2048-NEXT:    ld1b { [[ZPR:z[0-9]+]].b }, [[PG0]]/z, [x0]
+; VBITS_GE_2048-NEXT:    ptrue [[PG1:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT:    and [[ZPR]].d, [[ZPR]].d, #0xff
+; VBITS_GE_2048-NEXT:    st1d { [[ZPR]].d }, [[PG1]], [x8]
+; VBITS_GE_2048-NEXT:    ret
+  %a = load <32 x i8>, <32 x i8>* %in
+  %b = zext <32 x i8> %a to <32 x i64>
+  ret <32 x i64> %b
+}
+
+; zero_extend vXi16 -> vXi64
+
+; NOTE: Input is NEON vector - Output is Scalable vector.
+define <4 x i64> @zext_v4i16_v4i64(<4 x i16>* %in) #0 {
+; CHECK-LABEL: zext_v4i16_v4i64:
+; CHECK:    stp x29, x30, [sp, #-16]!
+; CHECK:    sub x9, sp, #48
+; CHECK:    mov x29, sp
+; CHECK:    and sp, x9, #0xffffffffffffffe0
+; CHECK:    ldr d0, [x0]
+; CHECK:    ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK:    umov w9, v0.h[3]
+; CHECK:    umov w10, v0.h[2]
+; CHECK:    and x9, x9, #0xffff
+; CHECK:    and x10, x10, #0xffff
+; CHECK:    umov w11, v0.h[1]
+; CHECK:    stp x10, x9, [sp, #16]
+; CHECK:    umov w9, v0.h[0]
+; CHECK:    and x10, x11, #0xffff
+; CHECK:    and x9, x9, #0xffff
+; CHECK:    stp x9, x10, [sp]
+; CHECK:    mov x9, sp
+; CHECK:    ld1d { [[ZPR:z[0-9]+]].d }, [[PG]]/z, [x9]
+; CHECK:    st1d { [[ZPR]].d }, [[PG]], [x8]
+; CHECK:    mov sp, x29
+; CHECK:    ldp x29, x30, [sp], #16
+; CHECK:    ret
+  %a = load <4 x i16>, <4 x i16>* %in
+  %b = zext <4 x i16> %a to <4 x i64>
+  ret <4 x i64> %b
+}
+
+; NOTE: Input is NEON vector - Output is Scalable vector.
+define <8 x i64> @zext_v8i16_v8i64(<8 x i16>* %in) #0 {
+; VBITS_GE_512-LABEL: zext_v8i16_v8i64:
+; VBITS_GE_512:    stp x29, x30, [sp, #-16]!
+; VBITS_GE_512:    sub x9, sp, #112
+; VBITS_GE_512:    mov x29, sp
+; VBITS_GE_512:    and sp, x9, #0xffffffffffffffc0
+; VBITS_GE_512:    ldr q0, [x0]
+; VBITS_GE_512:    ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512:    umov w9, v0.h[7]
+; VBITS_GE_512:    umov w10, v0.h[6]
+; VBITS_GE_512:    umov w11, v0.h[5]
+; VBITS_GE_512:    umov w12, v0.h[4]
+; VBITS_GE_512:    and x9, x9, #0xffff
+; VBITS_GE_512:    and x10, x10, #0xffff
+; VBITS_GE_512:    umov w13, v0.h[3]
+; VBITS_GE_512:    stp x10, x9, [sp, #48]
+; VBITS_GE_512:    umov w9, v0.h[2]
+; VBITS_GE_512:    and x10, x11, #0xffff
+; VBITS_GE_512:    and x11, x12, #0xffff
+; VBITS_GE_512:    stp x11, x10, [sp, #32]
+; VBITS_GE_512:    and x11, x13, #0xffff
+; VBITS_GE_512:    and x9, x9, #0xffff
+; VBITS_GE_512:    umov w10, v0.h[1]
+; VBITS_GE_512:    stp x9, x11, [sp, #16]
+; VBITS_GE_512:    umov w9, v0.h[0]
+; VBITS_GE_512:    and x10, x10, #0xffff
+; VBITS_GE_512:    and x9, x9, #0xffff
+; VBITS_GE_512:    stp x9, x10, [sp]
+; VBITS_GE_512:    mov x9, sp
+; VBITS_GE_512:    ld1d { [[ZPR:z[0-9]+]].d }, [[PG]]/z, [x9]
+; VBITS_GE_512:    st1d { [[ZPR]].d }, [[PG]], [x8]
+; VBITS_GE_512:    mov sp, x29
+; VBITS_GE_512:    ldp x29, x30, [sp], #16
+; VBITS_GE_512:    ret
+  %a = load <8 x i16>, <8 x i16>* %in
+  %b = zext <8 x i16> %a to <8 x i64>
+  ret <8 x i64> %b
+}
+
+define <16 x i64> @zext_v16i16_v16i64(<16 x i16>* %in) #0 {
+; VBITS_GE_1024-LABEL: zext_v16i16_v16i64:
+; VBITS_GE_1024:    ptrue [[PG0:p[0-9]+]].h, vl16
+; VBITS_GE_1024-NEXT:    ld1h { [[ZPR:z[0-9]+]].h }, [[PG0]]/z, [x0]
+; VBITS_GE_1024-NEXT:    ptrue [[PG1:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT:    and [[ZPR]].d, [[ZPR]].d, #0xffff
+; VBITS_GE_1024-NEXT:    st1d { [[ZPR]].d }, [[PG1]], [x8]
+; VBITS_GE_1024-NEXT:    ret
+  %a = load <16 x i16>, <16 x i16>* %in
+  %b = zext <16 x i16> %a to <16 x i64>
+  ret <16 x i64> %b
+}
+
+define <32 x i64> @zext_v32i16_v32i64(<32 x i16>* %in) #0 {
+; VBITS_GE_2048-LABEL: zext_v32i16_v32i64:
+; VBITS_GE_2048:    ptrue [[PG0:p[0-9]+]].h, vl32
+; VBITS_GE_2048-NEXT:    ld1h { [[ZPR:z[0-9]+]].h }, [[PG0]]/z, [x0]
+; VBITS_GE_2048-NEXT:    ptrue [[PG1:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT:    and [[ZPR]].d, [[ZPR]].d, #0xffff
+; VBITS_GE_2048-NEXT:    st1d { [[ZPR]].d }, [[PG1]], [x8]
+; VBITS_GE_2048-NEXT:    ret
+  %a = load <32 x i16>, <32 x i16>* %in
+  %b = zext <32 x i16> %a to <32 x i64>
+  ret <32 x i64> %b
+}
+
+; zero_extend vXi32 -> vXi64
+
+; NOTE: Input is NEON vector - Output is Scalable vector.
+define <4 x i64> @zext_v4i32_v4i64(<4 x i32>* %in) #0 {
+; CHECK-LABEL: zext_v4i32_v4i64:
+; CHECK:    stp x29, x30, [sp, #-16]!
+; CHECK:    sub x9, sp, #80
+; CHECK:    mov x29, sp
+; CHECK:    and sp, x9, #0xffffffffffffffe0
+; CHECK:    ldr q0, [x0]
+; CHECK:    mov x9, sp
+; CHECK:    orr x10, x9, #0x18
+; CHECK:    str wzr, [sp, #28]
+; CHECK:    st1 { v0.s }[3], [x10]
+; CHECK:    orr x10, x9, #0x10
+; CHECK:    st1 { v0.s }[2], [x10]
+; CHECK:    orr x10, x9, #0x8
+; CHECK:    str wzr, [sp, #20]
+; CHECK:    str wzr, [sp, #12]
+; CHECK:    str wzr, [sp, #4]
+; CHECK:    ptrue [[PG:p[0-9]+]].s, vl8
+; CHECK:    st1 { v0.s }[1], [x10]
+; CHECK:    str s0, [sp]
+; CHECK:    ld1w { [[ZPR0:z[0-9]+]].s }, [[PG]]/z, [x9]
+; CHECK:    add x9, sp, #32
+; CHECK:    st1w { [[ZPR0]].s }, [[PG]], [x9]
+; CHECK:    ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK:    ld1d { [[ZPR1:z[0-9]+]].d }, [[PG]]/z, [x9]
+; CHECK:    st1d { [[ZPR1]].d }, [[PG]], [x8]
+; CHECK:    mov sp, x29
+; CHECK:    ldp x29, x30, [sp], #16
+; CHECK:    ret
+  %a = load <4 x i32>, <4 x i32>* %in
+  %b = zext <4 x i32> %a to <4 x i64>
+  ret <4 x i64> %b
+}
+
+define <8 x i64> @zext_v8i32_v8i64(<8 x i32>* %in) #0 {
+; VBITS_GE_512-LABEL: zext_v8i32_v8i64:
+; VBITS_GE_512:    ptrue [[PG0:p[0-9]+]].s, vl8
+; VBITS_GE_512-NEXT:    ld1w { [[ZPR:z[0-9]+]].s }, [[PG0]]/z, [x0]
+; VBITS_GE_512-NEXT:    ptrue [[PG1:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT:    and [[ZPR]].d, [[ZPR]].d, #0xffffffff
+; VBITS_GE_512-NEXT:    st1d { [[ZPR]].d }, [[PG1]], [x8]
+; VBITS_GE_512-NEXT:    ret
+  %a = load <8 x i32>, <8 x i32>* %in
+  %b = zext <8 x i32> %a to <8 x i64>
+  ret <8 x i64> %b
+}
+
+define <16 x i64> @zext_v16i32_v16i64(<16 x i32>* %in) #0 {
+; VBITS_GE_1024-LABEL: zext_v16i32_v16i64:
+; VBITS_GE_1024:    ptrue [[PG0:p[0-9]+]].s, vl16
+; VBITS_GE_1024-NEXT:    ld1w { [[ZPR:z[0-9]+]].s }, [[PG0]]/z, [x0]
+; VBITS_GE_1024-NEXT:    ptrue [[PG1:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT:    and [[ZPR]].d, [[ZPR]].d, #0xffffffff
+; VBITS_GE_1024-NEXT:    st1d { [[ZPR]].d }, [[PG1]], [x8]
+; VBITS_GE_1024-NEXT:    ret
+  %a = load <16 x i32>, <16 x i32>* %in
+  %b = zext <16 x i32> %a to <16 x i64>
+  ret <16 x i64> %b
+}
+
+define <32 x i64> @zext_v32i32_v32i64(<32 x i32>* %in) #0 {
+; VBITS_GE_2048-LABEL: zext_v32i32_v32i64:
+; VBITS_GE_2048:      ptrue [[PG0:p[0-9]+]].s, vl32
+; VBITS_GE_2048-NEXT: ld1w { [[ZPR:z[0-9]+]].s }, [[PG0]]/z, [x0]
+; VBITS_GE_2048-NEXT: ptrue [[PG1:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: and [[ZPR]].d, [[ZPR]].d, #0xffffffff
+; VBITS_GE_2048-NEXT: st1d { [[ZPR]].d }, [[PG1]], [x8]
+; VBITS_GE_2048-NEXT: ret
+  %a = load <32 x i32>, <32 x i32>* %in
+  %b = zext <32 x i32> %a to <32 x i64>
+  ret <32 x i64> %b
+}
+
+attributes #0 = { "target-features"="+sve" }