[SVE] Lower fixed length integer extend operations.

[paulwalker-arm]

Slightly more complicated than expected, hence why I broke out D85546. I needed to custom lower ANY_EXTEND and SIGNED_EXTEND_INREG for the test named sext_v4i8_v4i64.

[paulwalker-arm]

Sorry I pushed too soon, I've yet to add the zext tests.

[paulwalker-arm]

Added zext tests.

[efriedma]

llvm/test/CodeGen/AArch64/sve-fixed-length-int-extends.ll

99

Maybe worth adding a couple CHECK lines for sext_v16i8_v16i32 if the fixed vector width is 256, to make sure we do something sane if we need to legalize the result type?

[paulwalker-arm]

Added VBITS_EQ_256 check lines for sext_v16i8_v16i32 and zext_v16i8_v16i32.

[efriedma]

LGTM

llvm/lib/Target/AArch64/AArch64ISelLowering.h

private:
   SDValue LowerATOMIC_LOAD_AND(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerWindowsDYNAMIC_STACKALLOC(SDValue Op, SDValue Chain,
                                          SDValue &Size,
                                          SelectionDAG &DAG) const;
   SDValue LowerSVEStructLoad(unsigned Intrinsic, ArrayRef<SDValue> LoadOps,
                              EVT VT, SelectionDAG &DAG, const SDLoc &DL) const;
 
+  SDValue LowerFixedLengthVectorIntExtendToSVE(SDValue Op,
+                                               SelectionDAG &DAG) const;
   SDValue LowerFixedLengthVectorLoadToSVE(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFixedLengthVectorStoreToSVE(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFixedLengthVectorTruncateToSVE(SDValue Op,
                                               SelectionDAG &DAG) const;
 
   SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
                         SmallVectorImpl<SDNode *> &Created) const override;
   SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

 /// same register class, but only nxv8f16 can be treated as a packed vector.
 static inline bool isPackedVectorType(EVT VT, SelectionDAG &DAG) {
   assert(VT.isVector() && DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
          "Expected legal vector type!");
   return VT.isFixedLengthVector() ||
          VT.getSizeInBits().getKnownMinSize() == AArch64::SVEBitsPerBlock;
 }
 
+// Returns true for ####_MERGE_PASSTHRU opcodes, whose operands have a leading
+// predicate and end with a passthru value matching the result type.
+static bool isMergePassthruOpcode(unsigned Opc) {
+  switch (Opc) {
+  default:
+    return false;
+  case AArch64ISD::DUP_MERGE_PASSTHRU:
+  case AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU:
+  case AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU:
+    return true;
+  }
+}
+
 AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
                                              const AArch64Subtarget &STI)
     : TargetLowering(TM), Subtarget(&STI) {
   // AArch64 doesn't have comparisons which set GPRs or setcc instructions, so
   // we have to make something up. Arbitrarily, choose ZeroOrOne.
   setBooleanContents(ZeroOrOneBooleanContent);
   // When comparing vectors the result sets the different elements in the
   // vector to all-one or all-zero.
for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
     setOperationAction(Op, VT, Expand);
 
   // We use EXTRACT_SUBVECTOR to "cast" a scalable vector to a fixed length one.
   setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
 
   // Lower fixed length vector operations to scalable equivalents.
   setOperationAction(ISD::ADD, VT, Custom);
   setOperationAction(ISD::AND, VT, Custom);
+  setOperationAction(ISD::ANY_EXTEND, VT, Custom);
   setOperationAction(ISD::FADD, VT, Custom);
   setOperationAction(ISD::FDIV, VT, Custom);
   setOperationAction(ISD::FMA, VT, Custom);
   setOperationAction(ISD::FMAXNUM, VT, Custom);
   setOperationAction(ISD::FMINNUM, VT, Custom);
   setOperationAction(ISD::FMUL, VT, Custom);
   setOperationAction(ISD::FSUB, VT, Custom);
   setOperationAction(ISD::LOAD, VT, Custom);
   setOperationAction(ISD::MUL, VT, Custom);
   setOperationAction(ISD::OR, VT, Custom);
+  setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
+  setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Custom);
   setOperationAction(ISD::STORE, VT, Custom);
   setOperationAction(ISD::SUB, VT, Custom);
   setOperationAction(ISD::TRUNCATE, VT, Custom);
   setOperationAction(ISD::XOR, VT, Custom);
+  setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
 }
 
 void AArch64TargetLowering::addDRTypeForNEON(MVT VT) {
   addRegisterClass(VT, &AArch64::FPR64RegClass);
   addTypeForNEON(VT, MVT::v2i32);
 }
 
 void AArch64TargetLowering::addQRTypeForNEON(MVT VT) {
SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
   case ISD::ATOMIC_LOAD_SUB:
     return LowerATOMIC_LOAD_SUB(Op, DAG);
   case ISD::ATOMIC_LOAD_AND:
     return LowerATOMIC_LOAD_AND(Op, DAG);
   case ISD::DYNAMIC_STACKALLOC:
     return LowerDYNAMIC_STACKALLOC(Op, DAG);
   case ISD::VSCALE:
     return LowerVSCALE(Op, DAG);
+  case ISD::ANY_EXTEND:
+  case ISD::SIGN_EXTEND:
+  case ISD::ZERO_EXTEND:
+    return LowerFixedLengthVectorIntExtendToSVE(Op, DAG);
+  case ISD::SIGN_EXTEND_INREG:
+    return LowerToPredicatedOp(Op, DAG,
+                               AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU);
   case ISD::TRUNCATE:
     return LowerTRUNCATE(Op, DAG);
   case ISD::LOAD:
     if (useSVEForFixedLengthVectorVT(Op.getValueType()))
       return LowerFixedLengthVectorLoadToSVE(Op, DAG);
     llvm_unreachable("Unexpected request to lower ISD::LOAD");
   case ISD::ADD:
     return LowerToPredicatedOp(Op, DAG, AArch64ISD::ADD_PRED);
SDValue AArch64TargetLowering::LowerFixedLengthVectorStoreToSVE(
   auto NewValue = convertToScalableVector(DAG, ContainerVT, Store->getValue());
   return DAG.getMaskedStore(
       Store->getChain(), DL, NewValue, Store->getBasePtr(), Store->getOffset(),
       getPredicateForFixedLengthVector(DAG, DL, VT), Store->getMemoryVT(),
       Store->getMemOperand(), Store->getAddressingMode(),
       Store->isTruncatingStore());
 }
 
+SDValue AArch64TargetLowering::LowerFixedLengthVectorIntExtendToSVE(
+    SDValue Op, SelectionDAG &DAG) const {
+  EVT VT = Op.getValueType();
+  assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
+
+  SDLoc DL(Op);
+  SDValue Val = Op.getOperand(0);
+  EVT ContainerVT = getContainerForFixedLengthVector(DAG, Val.getValueType());
+  Val = convertToScalableVector(DAG, ContainerVT, Val);
+
+  bool Signed = Op.getOpcode() == ISD::SIGN_EXTEND;
+  unsigned ExtendOpc = Signed ? AArch64ISD::SUNPKLO : AArch64ISD::UUNPKLO;
+
+  // Repeatedly unpack Val until the result is of the desired element type.
+  switch (ContainerVT.getSimpleVT().SimpleTy) {
+  default:
+    llvm_unreachable("unimplemented container type");
+  case MVT::nxv16i8:
+    Val = DAG.getNode(ExtendOpc, DL, MVT::nxv8i16, Val);
+    if (VT.getVectorElementType() == MVT::i16)
+      break;
+    LLVM_FALLTHROUGH;
+  case MVT::nxv8i16:
+    Val = DAG.getNode(ExtendOpc, DL, MVT::nxv4i32, Val);
+    if (VT.getVectorElementType() == MVT::i32)
+      break;
+    LLVM_FALLTHROUGH;
+  case MVT::nxv4i32:
+    Val = DAG.getNode(ExtendOpc, DL, MVT::nxv2i64, Val);
+    assert(VT.getVectorElementType() == MVT::i64 && "Unexpected element type!");
+    break;
+  }
+
+  return convertFromScalableVector(DAG, VT, Val);
+}
+
 SDValue AArch64TargetLowering::LowerFixedLengthVectorTruncateToSVE(
     SDValue Op, SelectionDAG &DAG) const {
   EVT VT = Op.getValueType();
   assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
 
   SDLoc DL(Op);
   SDValue Val = Op.getOperand(0);
   EVT ContainerVT = getContainerForFixedLengthVector(DAG, Val.getValueType());
case MVT::nxv8i16:
     break;
   }
 
   return convertFromScalableVector(DAG, VT, Val);
 }
 
 // Convert vector operation 'Op' to an equivalent predicated operation whereby
 // the original operation's type is used to construct a suitable predicate.
+// NOTE: The results for inactive lanes are undefined.
 SDValue AArch64TargetLowering::LowerToPredicatedOp(SDValue Op,
                                                    SelectionDAG &DAG,
                                                    unsigned NewOp,
                                                    bool OverrideNEON) const {
   EVT VT = Op.getValueType();
   SDLoc DL(Op);
   auto Pg = getPredicateForVector(DAG, DL, VT);
 
   if (useSVEForFixedLengthVectorVT(VT, OverrideNEON)) {
     EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
 
     // Create list of operands by converting existing ones to scalable types.
     SmallVector<SDValue, 4> Operands = {Pg};
     for (const SDValue &V : Op->op_values()) {
       if (isa<CondCodeSDNode>(V)) {
         Operands.push_back(V);
         continue;
       }
 
+      if (const VTSDNode *VTNode = dyn_cast<VTSDNode>(V)) {
+        EVT VTArg = VTNode->getVT().getVectorElementType();
+        EVT NewVTArg = ContainerVT.changeVectorElementType(VTArg);
+        Operands.push_back(DAG.getValueType(NewVTArg));
+        continue;
+      }
+
       assert(useSVEForFixedLengthVectorVT(V.getValueType(), OverrideNEON) &&
              "Only fixed length vectors are supported!");
       Operands.push_back(convertToScalableVector(DAG, ContainerVT, V));
     }
 
+    if (isMergePassthruOpcode(NewOp))
+      Operands.push_back(DAG.getUNDEF(ContainerVT));
+
     auto ScalableRes = DAG.getNode(NewOp, DL, ContainerVT, Operands);
     return convertFromScalableVector(DAG, VT, ScalableRes);
   }
 
   assert(VT.isScalableVector() && "Only expect to lower scalable vector op!");
 
   SmallVector<SDValue, 4> Operands = {Pg};
   for (const SDValue &V : Op->op_values()) {
     assert((isa<CondCodeSDNode>(V) || V.getValueType().isScalableVector()) &&
            "Only scalable vectors are supported!");
     Operands.push_back(V);
   }
 
+  if (isMergePassthruOpcode(NewOp))
+    Operands.push_back(DAG.getUNDEF(VT));
+
   return DAG.getNode(NewOp, DL, VT, Operands);
 }
 
 // If a fixed length vector operation has no side effects when applied to
 // undefined elements, we can safely use scalable vectors to perform the same
 // operation without needing to worry about predication.
 SDValue AArch64TargetLowering::LowerToScalableOp(SDValue Op,
                                                  SelectionDAG &DAG) const {

llvm/test/CodeGen/AArch64/sve-fixed-length-int-extends.ll

+; RUN: llc -aarch64-sve-vector-bits-min=128  -asm-verbose=0 < %s | FileCheck %s -check-prefix=NO_SVE
+; RUN: llc -aarch64-sve-vector-bits-min=256  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_EQ_256
+; RUN: llc -aarch64-sve-vector-bits-min=384  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK
+; RUN: llc -aarch64-sve-vector-bits-min=512  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=640  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=768  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=896  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=1024 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1152 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1280 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1408 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1536 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1664 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1792 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1920 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=2048 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024,VBITS_GE_2048
+
+target triple = "aarch64-unknown-linux-gnu"
+
+; Don't use SVE when its registers are no bigger than NEON.
+; NO_SVE-NOT: z{0-9}
+
+;
+; sext i8 -> i16
+;
+
+define void @sext_v16i8_v16i16(<16 x i8> %a, <16 x i16>* %out) #0 {
+; CHECK-LABEL: sext_v16i8_v16i16:
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl16
+; CHECK-NEXT: sunpklo [[A_HALFS:z[0-9]+]].h, z0.b
+; CHECK-NEXT: st1h { [[A_HALFS]].h }, [[PG]], [x0]
+; CHECK-NEXT: ret
+  %b = sext <16 x i8> %a to <16 x i16>
+  store <16 x i16>%b, <16 x i16>* %out
+  ret void
+}
+
+; NOTE: Extra 'add' is to prevent the extend being combined with the load.
+define void @sext_v32i8_v32i16(<32 x i8>* %in, <32 x i16>* %out) #0 {
+; CHECK-LABEL: sext_v32i8_v32i16:
+; VBITS_GE_512: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
+; VBITS_GE_512-NEXT: sunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_512-NEXT: ptrue [[PG:p[0-9]+]].h, vl32
+; VBITS_GE_512-NEXT: st1h { [[A_HALFS]].h }, [[PG]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %a = load <32 x i8>, <32 x i8>* %in
+  %b = add <32 x i8> %a, %a
+  %c = sext <32 x i8> %b to <32 x i16>
+  store <32 x i16> %c, <32 x i16>* %out
+  ret void
+}
+
+define void @sext_v64i8_v64i16(<64 x i8>* %in, <64 x i16>* %out) #0 {
+; CHECK-LABEL: sext_v64i8_v64i16:
+; VBITS_GE_1024: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
+; VBITS_GE_1024-NEXT: sunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_1024-NEXT: ptrue [[PG:p[0-9]+]].h, vl64
+; VBITS_GE_1024-NEXT: st1h { [[A_HALFS]].h }, [[PG]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %a = load <64 x i8>, <64 x i8>* %in
+  %b = add <64 x i8> %a, %a
+  %c = sext <64 x i8> %b to <64 x i16>
+  store <64 x i16> %c, <64 x i16>* %out
+  ret void
+}
+
+define void @sext_v128i8_v128i16(<128 x i8>* %in, <128 x i16>* %out) #0 {
+; CHECK-LABEL: sext_v128i8_v128i16:
+; VBITS_GE_2048: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
+; VBITS_GE_2048-NEXT: sunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].h, vl128
+; VBITS_GE_2048-NEXT: st1h { [[A_HALFS]].h }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %a = load <128 x i8>, <128 x i8>* %in
+  %b = add <128 x i8> %a, %a
+  %c = sext <128 x i8> %b to <128 x i16>
+  store <128 x i16> %c, <128 x i16>* %out
+  ret void
+}
+
+;
+; sext i8 -> i32
+;
+
+define void @sext_v8i8_v8i32(<8 x i8> %a, <8 x i32>* %out) #0 {
+; CHECK-LABEL: sext_v8i8_v8i32:
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl8
+; CHECK-NEXT: sunpklo [[A_HALFS:z[0-9]+]].h, z0.b
+; CHECK-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; CHECK-NEXT: st1w { [[A_HALFS]].s }, [[PG]], [x0]
+; CHECK-NEXT: ret
+  %b = sext <8 x i8> %a to <8 x i32>
+  store <8 x i32>%b, <8 x i32>* %out
+  ret void
+}
+
+define void @sext_v16i8_v16i32(<16 x i8> %a, <16 x i32>* %out) #0 {
+; CHECK-LABEL: sext_v16i8_v16i32:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: sunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_512-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_512-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x0]
+; VBITS_GE_512-NEXT: ret
+
+; Ensure sensible type legalisation.
+; VBITS_EQ_256: ext v[[A_HI:[0-9]+]].16b, v0.16b, v0.16b, #8
+; VBITS_EQ_256-DAG: sunpklo [[A_HALFS_LO:z[0-9]+]].h, z0.b
+; VBITS_EQ_256-DAG: sunpklo [[A_HALFS_HI:z[0-9]+]].h, z[[A_HI]].b
+; VBITS_EQ_256-DAG: sunpklo [[A_WORDS_LO:z[0-9]+]].s, [[A_HALFS_LO]].h
+; VBITS_EQ_256-DAG: sunpklo [[A_WORDS_HI:z[0-9]+]].s, [[A_HALFS_HI]].h
+; VBITS_EQ_256-DAG: ptrue [[PG:p[0-9]+]].s, vl8
+; VBITS_EQ_256-DAG: add x[[OUT_HI:[0-9]+]], x0, #32
+; VBITS_EQ_256-DAG: st1w { [[A_WORDS_LO]].s }, [[PG]], [x0]
+; VBITS_EQ_256-DAG: st1w { [[A_WORDS_HI]].s }, [[PG]], [x[[OUT_HI]]]
+; VBITS_EQ_256-NEXT: ret
+  %b = sext <16 x i8> %a to <16 x i32>
+  store <16 x i32> %b, <16 x i32>* %out
+  ret void
+}
+
+define void @sext_v32i8_v32i32(<32 x i8>* %in, <32 x i32>* %out) #0 {
+; CHECK-LABEL: sext_v32i8_v32i32:
+; VBITS_GE_1024: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
+; VBITS_GE_1024-NEXT: sunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_1024-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_1024-NEXT: ptrue [[PG:p[0-9]+]].s, vl32
+; VBITS_GE_1024-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %a = load <32 x i8>, <32 x i8>* %in
+  %b = add <32 x i8> %a, %a
+  %c = sext <32 x i8> %b to <32 x i32>
+  store <32 x i32> %c, <32 x i32>* %out
+  ret void
+}
+
+define void @sext_v64i8_v64i32(<64 x i8>* %in, <64 x i32>* %out) #0 {
+; CHECK-LABEL: sext_v64i8_v64i32:
+; VBITS_GE_2048: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
+; VBITS_GE_2048-NEXT: sunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_2048-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].s, vl64
+; VBITS_GE_2048-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %a = load <64 x i8>, <64 x i8>* %in
+  %b = add <64 x i8> %a, %a
+  %c = sext <64 x i8> %b to <64 x i32>
+  store <64 x i32> %c, <64 x i32>* %out
+  ret void
+}
+
+;
+; sext i8 -> i64
+;
+
+; NOTE: v4i8 is an unpacked typed stored within a v4i16 container. The sign
+; extend is a two step process where the container is any_extend'd with the
+; result feeding an inreg sign extend.
+define void @sext_v4i8_v4i64(<4 x i8> %a, <4 x i64>* %out) #0 {
+; CHECK-LABEL: sext_v4i8_v4i64:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: uunpklo [[ANYEXT_W:z[0-9]+]].s, z0.h
+; CHECK-NEXT: uunpklo [[ANYEXT_D:z[0-9]+]].d, [[ANYEXT_W]].s
+; CHECK-NEXT: sxtb [[A_DWORDS:z[0-9]+]].d, [[PG]]/m, [[ANYEXT_D]].d
+; CHECK-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x0]
+; CHECK-NEXT: ret
+  %b = sext <4 x i8> %a to <4 x i64>
+  store <4 x i64>%b, <4 x i64>* %out
+  ret void
+}
+
+define void @sext_v8i8_v8i64(<8 x i8> %a, <8 x i64>* %out) #0 {
+; CHECK-LABEL: sext_v8i8_v8i64:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: sunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_512-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_512-NEXT: sunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_512-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x0]
+; VBITS_GE_512-NEXT: ret
+  %b = sext <8 x i8> %a to <8 x i64>
+  store <8 x i64>%b, <8 x i64>* %out
+  ret void
+}
+
+define void @sext_v16i8_v16i64(<16 x i8> %a, <16 x i64>* %out) #0 {
+; CHECK-LABEL: sext_v16i8_v16i64:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: sunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_1024-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_1024-NEXT: sunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_1024-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x0]
+; VBITS_GE_1024-NEXT: ret
+  %b = sext <16 x i8> %a to <16 x i64>
+  store <16 x i64> %b, <16 x i64>* %out
+  ret void
+}
+
+define void @sext_v32i8_v32i64(<32 x i8>* %in, <32 x i64>* %out) #0 {
+; CHECK-LABEL: sext_v32i8_v32i64:
+; VBITS_GE_2048: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
+; VBITS_GE_2048-NEXT: sunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_2048-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_2048-NEXT: sunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %a = load <32 x i8>, <32 x i8>* %in
+  %b = add <32 x i8> %a, %a
+  %c = sext <32 x i8> %b to <32 x i64>
+  store <32 x i64> %c, <32 x i64>* %out
+  ret void
+}
+
+;
+; sext i16 -> i32
+;
+
+define void @sext_v8i16_v8i32(<8 x i16> %a, <8 x i32>* %out) #0 {
+; CHECK-LABEL: sext_v8i16_v8i32:
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl8
+; CHECK-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, z0.h
+; CHECK-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x0]
+; CHECK-NEXT: ret
+  %b = sext <8 x i16> %a to <8 x i32>
+  store <8 x i32>%b, <8 x i32>* %out
+  ret void
+}
+
+define void @sext_v16i16_v16i32(<16 x i16>* %in, <16 x i32>* %out) #0 {
+; CHECK-LABEL: sext_v16i16_v16i32:
+; VBITS_GE_512: add [[A_HALFS:z[0-9]+]].h, {{p[0-9]+}}/m, {{z[0-9]+}}.h, {{z[0-9]+}}.h
+; VBITS_GE_512-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_512-NEXT: ptrue [[PG:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %a = load <16 x i16>, <16 x i16>* %in
+  %b = add <16 x i16> %a, %a
+  %c = sext <16 x i16> %b to <16 x i32>
+  store <16 x i32> %c, <16 x i32>* %out
+  ret void
+}
+
+define void @sext_v32i16_v32i32(<32 x i16>* %in, <32 x i32>* %out) #0 {
+; CHECK-LABEL: sext_v32i16_v32i32:
+; VBITS_GE_1024: add [[A_HALFS:z[0-9]+]].h, {{p[0-9]+}}/m, {{z[0-9]+}}.h, {{z[0-9]+}}.h
+; VBITS_GE_1024-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_1024-NEXT: ptrue [[PG:p[0-9]+]].s, vl32
+; VBITS_GE_1024-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %a = load <32 x i16>, <32 x i16>* %in
+  %b = add <32 x i16> %a, %a
+  %c = sext <32 x i16> %b to <32 x i32>
+  store <32 x i32> %c, <32 x i32>* %out
+  ret void
+}
+
+define void @sext_v64i16_v64i32(<64 x i16>* %in, <64 x i32>* %out) #0 {
+; CHECK-LABEL: sext_v64i16_v64i32:
+; VBITS_GE_2048: add [[A_HALFS:z[0-9]+]].h, {{p[0-9]+}}/m, {{z[0-9]+}}.h, {{z[0-9]+}}.h
+; VBITS_GE_2048-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].s, vl64
+; VBITS_GE_2048-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %a = load <64 x i16>, <64 x i16>* %in
+  %b = add <64 x i16> %a, %a
+  %c = sext <64 x i16> %b to <64 x i32>
+  store <64 x i32> %c, <64 x i32>* %out
+  ret void
+}
+
+;
+; sext i16 -> i64
+;
+
+define void @sext_v4i16_v4i64(<4 x i16> %a, <4 x i64>* %out) #0 {
+; CHECK-LABEL: sext_v4i16_v4i64:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, z0.h
+; CHECK-NEXT: sunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; CHECK-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x0]
+; CHECK-NEXT: ret
+  %b = sext <4 x i16> %a to <4 x i64>
+  store <4 x i64>%b, <4 x i64>* %out
+  ret void
+}
+
+define void @sext_v8i16_v8i64(<8 x i16> %a, <8 x i64>* %out) #0 {
+; CHECK-LABEL: sext_v8i16_v8i64:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, z0.h
+; VBITS_GE_512-NEXT: sunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_512-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x0]
+; VBITS_GE_512-NEXT: ret
+  %b = sext <8 x i16> %a to <8 x i64>
+  store <8 x i64>%b, <8 x i64>* %out
+  ret void
+}
+
+define void @sext_v16i16_v16i64(<16 x i16>* %in, <16 x i64>* %out) #0 {
+; CHECK-LABEL: sext_v16i16_v16i64:
+; VBITS_GE_1024: add [[A_HALFS:z[0-9]+]].h, {{p[0-9]+}}/m, {{z[0-9]+}}.h, {{z[0-9]+}}.h
+; VBITS_GE_1024-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_1024-NEXT: sunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_1024-NEXT: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %a = load <16 x i16>, <16 x i16>* %in
+  %b = add <16 x i16> %a, %a
+  %c = sext <16 x i16> %b to <16 x i64>
+  store <16 x i64> %c, <16 x i64>* %out
+  ret void
+}
+
+define void @sext_v32i16_v32i64(<32 x i16>* %in, <32 x i64>* %out) #0 {
+; CHECK-LABEL: sext_v32i16_v32i64:
+; VBITS_GE_2048: add [[A_HALFS:z[0-9]+]].h, {{p[0-9]+}}/m, {{z[0-9]+}}.h, {{z[0-9]+}}.h
+; VBITS_GE_2048-NEXT: sunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_2048-NEXT: sunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %a = load <32 x i16>, <32 x i16>* %in
+  %b = add <32 x i16> %a, %a
+  %c = sext <32 x i16> %b to <32 x i64>
+  store <32 x i64> %c, <32 x i64>* %out
+  ret void
+}
+
+;
+; sext i32 -> i64
+;
+
+define void @sext_v4i32_v4i64(<4 x i32> %a, <4 x i64>* %out) #0 {
+; CHECK-LABEL: sext_v4i32_v4i64:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: sunpklo [[A_DWORDS:z[0-9]+]].d, z0.s
+; CHECK-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x0]
+; CHECK-NEXT: ret
+  %b = sext <4 x i32> %a to <4 x i64>
+  store <4 x i64>%b, <4 x i64>* %out
+  ret void
+}
+
+define void @sext_v8i32_v8i64(<8 x i32>* %in, <8 x i64>* %out) #0 {
+; CHECK-LABEL: sext_v8i32_v8i64:
+; VBITS_GE_512: add [[A_WORDS:z[0-9]+]].s, {{p[0-9]+}}/m, {{z[0-9]+}}.s, {{z[0-9]+}}.s
+; VBITS_GE_512-NEXT: sunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_512-NEXT: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %a = load <8 x i32>, <8 x i32>* %in
+  %b = add <8 x i32> %a, %a
+  %c = sext <8 x i32> %b to <8 x i64>
+  store <8 x i64> %c, <8 x i64>* %out
+  ret void
+}
+
+define void @sext_v16i32_v16i64(<16 x i32>* %in, <16 x i64>* %out) #0 {
+; CHECK-LABEL: sext_v16i32_v16i64:
+; VBITS_GE_1024: add [[A_WORDS:z[0-9]+]].s, {{p[0-9]+}}/m, {{z[0-9]+}}.s, {{z[0-9]+}}.s
+; VBITS_GE_1024-NEXT: sunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_1024-NEXT: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %a = load <16 x i32>, <16 x i32>* %in
+  %b = add <16 x i32> %a, %a
+  %c = sext <16 x i32> %b to <16 x i64>
+  store <16 x i64> %c, <16 x i64>* %out
+  ret void
+}
+
+define void @sext_v32i32_v32i64(<32 x i32>* %in, <32 x i64>* %out) #0 {
+; CHECK-LABEL: sext_v32i32_v32i64:
+; VBITS_GE_2048: add [[A_WORDS:z[0-9]+]].s, {{p[0-9]+}}/m, {{z[0-9]+}}.s, {{z[0-9]+}}.s
+; VBITS_GE_2048-NEXT: sunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %a = load <32 x i32>, <32 x i32>* %in
+  %b = add <32 x i32> %a, %a
+  %c = sext <32 x i32> %b to <32 x i64>
+  store <32 x i64> %c, <32 x i64>* %out
+  ret void
+}
+
+;
+; zext i8 -> i16
+;
+
+define void @zext_v16i8_v16i16(<16 x i8> %a, <16 x i16>* %out) #0 {
+; CHECK-LABEL: zext_v16i8_v16i16:
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl16
+; CHECK-NEXT: uunpklo [[A_HALFS:z[0-9]+]].h, z0.b
+; CHECK-NEXT: st1h { [[A_HALFS]].h }, [[PG]], [x0]
+; CHECK-NEXT: ret
+  %b = zext <16 x i8> %a to <16 x i16>
+  store <16 x i16>%b, <16 x i16>* %out
+  ret void
+}
+
+; NOTE: Extra 'add' is to prevent the extend being combined with the load.
+define void @zext_v32i8_v32i16(<32 x i8>* %in, <32 x i16>* %out) #0 {
+; CHECK-LABEL: zext_v32i8_v32i16:
+; VBITS_GE_512: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
+; VBITS_GE_512-NEXT: uunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_512-NEXT: ptrue [[PG:p[0-9]+]].h, vl32
+; VBITS_GE_512-NEXT: st1h { [[A_HALFS]].h }, [[PG]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %a = load <32 x i8>, <32 x i8>* %in
+  %b = add <32 x i8> %a, %a
+  %c = zext <32 x i8> %b to <32 x i16>
+  store <32 x i16> %c, <32 x i16>* %out
+  ret void
+}
+
+define void @zext_v64i8_v64i16(<64 x i8>* %in, <64 x i16>* %out) #0 {
+; CHECK-LABEL: zext_v64i8_v64i16:
+; VBITS_GE_1024: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
+; VBITS_GE_1024-NEXT: uunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_1024-NEXT: ptrue [[PG:p[0-9]+]].h, vl64
+; VBITS_GE_1024-NEXT: st1h { [[A_HALFS]].h }, [[PG]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %a = load <64 x i8>, <64 x i8>* %in
+  %b = add <64 x i8> %a, %a
+  %c = zext <64 x i8> %b to <64 x i16>
+  store <64 x i16> %c, <64 x i16>* %out
+  ret void
+}
+
+define void @zext_v128i8_v128i16(<128 x i8>* %in, <128 x i16>* %out) #0 {
+; CHECK-LABEL: zext_v128i8_v128i16:
+; VBITS_GE_2048: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
+; VBITS_GE_2048-NEXT: uunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].h, vl128
+; VBITS_GE_2048-NEXT: st1h { [[A_HALFS]].h }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %a = load <128 x i8>, <128 x i8>* %in
+  %b = add <128 x i8> %a, %a
+  %c = zext <128 x i8> %b to <128 x i16>
+  store <128 x i16> %c, <128 x i16>* %out
+  ret void
+}
+
+;
+; zext i8 -> i32
+;
+
+define void @zext_v8i8_v8i32(<8 x i8> %a, <8 x i32>* %out) #0 {
+; CHECK-LABEL: zext_v8i8_v8i32:
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl8
+; CHECK-NEXT: uunpklo [[A_HALFS:z[0-9]+]].h, z0.b
+; CHECK-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; CHECK-NEXT: st1w { [[A_HALFS]].s }, [[PG]], [x0]
+; CHECK-NEXT: ret
+  %b = zext <8 x i8> %a to <8 x i32>
+  store <8 x i32>%b, <8 x i32>* %out
+  ret void
+}
+
+define void @zext_v16i8_v16i32(<16 x i8> %a, <16 x i32>* %out) #0 {
+; CHECK-LABEL: zext_v16i8_v16i32:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: uunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_512-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_512-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x0]
+; VBITS_GE_512-NEXT: ret
+
+; Ensure sensible type legalisation.
+; VBITS_EQ_256: ext v[[A_HI:[0-9]+]].16b, v0.16b, v0.16b, #8
+; VBITS_EQ_256-DAG: uunpklo [[A_HALFS_LO:z[0-9]+]].h, z0.b
+; VBITS_EQ_256-DAG: uunpklo [[A_HALFS_HI:z[0-9]+]].h, z[[A_HI]].b
+; VBITS_EQ_256-DAG: uunpklo [[A_WORDS_LO:z[0-9]+]].s, [[A_HALFS_LO]].h
+; VBITS_EQ_256-DAG: uunpklo [[A_WORDS_HI:z[0-9]+]].s, [[A_HALFS_HI]].h
+; VBITS_EQ_256-DAG: ptrue [[PG:p[0-9]+]].s, vl8
+; VBITS_EQ_256-DAG: add x[[OUT_HI:[0-9]+]], x0, #32
+; VBITS_EQ_256-DAG: st1w { [[A_WORDS_LO]].s }, [[PG]], [x0]
+; VBITS_EQ_256-DAG: st1w { [[A_WORDS_HI]].s }, [[PG]], [x[[OUT_HI]]]
+; VBITS_EQ_256-NEXT: ret
+  %b = zext <16 x i8> %a to <16 x i32>
+  store <16 x i32> %b, <16 x i32>* %out
+  ret void
+}
+
+define void @zext_v32i8_v32i32(<32 x i8>* %in, <32 x i32>* %out) #0 {
+; CHECK-LABEL: zext_v32i8_v32i32:
+; VBITS_GE_1024: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
+; VBITS_GE_1024-NEXT: uunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_1024-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_1024-NEXT: ptrue [[PG:p[0-9]+]].s, vl32
+; VBITS_GE_1024-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %a = load <32 x i8>, <32 x i8>* %in
+  %b = add <32 x i8> %a, %a
+  %c = zext <32 x i8> %b to <32 x i32>
+  store <32 x i32> %c, <32 x i32>* %out
+  ret void
+}
+
+define void @zext_v64i8_v64i32(<64 x i8>* %in, <64 x i32>* %out) #0 {
+; CHECK-LABEL: zext_v64i8_v64i32:
+; VBITS_GE_2048: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
+; VBITS_GE_2048-NEXT: uunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_2048-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].s, vl64
+; VBITS_GE_2048-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %a = load <64 x i8>, <64 x i8>* %in
+  %b = add <64 x i8> %a, %a
+  %c = zext <64 x i8> %b to <64 x i32>
+  store <64 x i32> %c, <64 x i32>* %out
+  ret void
+}
+
+;
+; zext i8 -> i64
+;
+
+; NOTE: v4i8 is an unpacked typed stored within a v4i16 container. The zero
+; extend is a two step process where the container is zero_extend_inreg'd with
+; the result feeding a normal zero extend from halfs to doublewords.
+define void @zext_v4i8_v4i64(<4 x i8> %a, <4 x i64>* %out) #0 {
+; CHECK-LABEL: zext_v4i8_v4i64:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: bic v0.4h, #255, lsl #8
+; CHECK-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, z0.h
+; CHECK-NEXT: uunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; CHECK-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x0]
+; CHECK-NEXT: ret
+  %b = zext <4 x i8> %a to <4 x i64>
+  store <4 x i64>%b, <4 x i64>* %out
+  ret void
+}
+
+define void @zext_v8i8_v8i64(<8 x i8> %a, <8 x i64>* %out) #0 {
+; CHECK-LABEL: zext_v8i8_v8i64:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: uunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_512-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_512-NEXT: uunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_512-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x0]
+; VBITS_GE_512-NEXT: ret
+  %b = zext <8 x i8> %a to <8 x i64>
+  store <8 x i64>%b, <8 x i64>* %out
+  ret void
+}
+
+define void @zext_v16i8_v16i64(<16 x i8> %a, <16 x i64>* %out) #0 {
+; CHECK-LABEL: zext_v16i8_v16i64:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: uunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_1024-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_1024-NEXT: uunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_1024-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x0]
+; VBITS_GE_1024-NEXT: ret
+  %b = zext <16 x i8> %a to <16 x i64>
+  store <16 x i64> %b, <16 x i64>* %out
+  ret void
+}
+
+define void @zext_v32i8_v32i64(<32 x i8>* %in, <32 x i64>* %out) #0 {
+; CHECK-LABEL: zext_v32i8_v32i64:
+; VBITS_GE_2048: add [[A_BYTES:z[0-9]+]].b, {{p[0-9]+}}/m, {{z[0-9]+}}.b, {{z[0-9]+}}.b
+; VBITS_GE_2048-NEXT: uunpklo [[A_HALFS:z[0-9]+]].h, [[A_BYTES]].b
+; VBITS_GE_2048-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_2048-NEXT: uunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %a = load <32 x i8>, <32 x i8>* %in
+  %b = add <32 x i8> %a, %a
+  %c = zext <32 x i8> %b to <32 x i64>
+  store <32 x i64> %c, <32 x i64>* %out
+  ret void
+}
+
+;
+; zext i16 -> i32
+;
+
+define void @zext_v8i16_v8i32(<8 x i16> %a, <8 x i32>* %out) #0 {
+; CHECK-LABEL: zext_v8i16_v8i32:
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl8
+; CHECK-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, z0.h
+; CHECK-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x0]
+; CHECK-NEXT: ret
+  %b = zext <8 x i16> %a to <8 x i32>
+  store <8 x i32>%b, <8 x i32>* %out
+  ret void
+}
+
+define void @zext_v16i16_v16i32(<16 x i16>* %in, <16 x i32>* %out) #0 {
+; CHECK-LABEL: zext_v16i16_v16i32:
+; VBITS_GE_512: add [[A_HALFS:z[0-9]+]].h, {{p[0-9]+}}/m, {{z[0-9]+}}.h, {{z[0-9]+}}.h
+; VBITS_GE_512-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_512-NEXT: ptrue [[PG:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %a = load <16 x i16>, <16 x i16>* %in
+  %b = add <16 x i16> %a, %a
+  %c = zext <16 x i16> %b to <16 x i32>
+  store <16 x i32> %c, <16 x i32>* %out
+  ret void
+}
+
+define void @zext_v32i16_v32i32(<32 x i16>* %in, <32 x i32>* %out) #0 {
+; CHECK-LABEL: zext_v32i16_v32i32:
+; VBITS_GE_1024: add [[A_HALFS:z[0-9]+]].h, {{p[0-9]+}}/m, {{z[0-9]+}}.h, {{z[0-9]+}}.h
+; VBITS_GE_1024-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_1024-NEXT: ptrue [[PG:p[0-9]+]].s, vl32
+; VBITS_GE_1024-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %a = load <32 x i16>, <32 x i16>* %in
+  %b = add <32 x i16> %a, %a
+  %c = zext <32 x i16> %b to <32 x i32>
+  store <32 x i32> %c, <32 x i32>* %out
+  ret void
+}
+
+define void @zext_v64i16_v64i32(<64 x i16>* %in, <64 x i32>* %out) #0 {
+; CHECK-LABEL: zext_v64i16_v64i32:
+; VBITS_GE_2048: add [[A_HALFS:z[0-9]+]].h, {{p[0-9]+}}/m, {{z[0-9]+}}.h, {{z[0-9]+}}.h
+; VBITS_GE_2048-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].s, vl64
+; VBITS_GE_2048-NEXT: st1w { [[A_WORDS]].s }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %a = load <64 x i16>, <64 x i16>* %in
+  %b = add <64 x i16> %a, %a
+  %c = zext <64 x i16> %b to <64 x i32>
+  store <64 x i32> %c, <64 x i32>* %out
+  ret void
+}
+
+;
+; zext i16 -> i64
+;
+
+define void @zext_v4i16_v4i64(<4 x i16> %a, <4 x i64>* %out) #0 {
+; CHECK-LABEL: zext_v4i16_v4i64:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, z0.h
+; CHECK-NEXT: uunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; CHECK-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x0]
+; CHECK-NEXT: ret
+  %b = zext <4 x i16> %a to <4 x i64>
+  store <4 x i64>%b, <4 x i64>* %out
+  ret void
+}
+
+define void @zext_v8i16_v8i64(<8 x i16> %a, <8 x i64>* %out) #0 {
+; CHECK-LABEL: zext_v8i16_v8i64:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, z0.h
+; VBITS_GE_512-NEXT: uunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_512-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x0]
+; VBITS_GE_512-NEXT: ret
+  %b = zext <8 x i16> %a to <8 x i64>
+  store <8 x i64>%b, <8 x i64>* %out
+  ret void
+}
+
+define void @zext_v16i16_v16i64(<16 x i16>* %in, <16 x i64>* %out) #0 {
+; CHECK-LABEL: zext_v16i16_v16i64:
+; VBITS_GE_1024: add [[A_HALFS:z[0-9]+]].h, {{p[0-9]+}}/m, {{z[0-9]+}}.h, {{z[0-9]+}}.h
+; VBITS_GE_1024-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_1024-NEXT: uunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_1024-NEXT: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %a = load <16 x i16>, <16 x i16>* %in
+  %b = add <16 x i16> %a, %a
+  %c = zext <16 x i16> %b to <16 x i64>
+  store <16 x i64> %c, <16 x i64>* %out
+  ret void
+}
+
+define void @zext_v32i16_v32i64(<32 x i16>* %in, <32 x i64>* %out) #0 {
+; CHECK-LABEL: zext_v32i16_v32i64:
+; VBITS_GE_2048: add [[A_HALFS:z[0-9]+]].h, {{p[0-9]+}}/m, {{z[0-9]+}}.h, {{z[0-9]+}}.h
+; VBITS_GE_2048-NEXT: uunpklo [[A_WORDS:z[0-9]+]].s, [[A_HALFS]].h
+; VBITS_GE_2048-NEXT: uunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %a = load <32 x i16>, <32 x i16>* %in
+  %b = add <32 x i16> %a, %a
+  %c = zext <32 x i16> %b to <32 x i64>
+  store <32 x i64> %c, <32 x i64>* %out
+  ret void
+}
+
+;
+; zext i32 -> i64
+;
+
+define void @zext_v4i32_v4i64(<4 x i32> %a, <4 x i64>* %out) #0 {
+; CHECK-LABEL: zext_v4i32_v4i64:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: uunpklo [[A_DWORDS:z[0-9]+]].d, z0.s
+; CHECK-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x0]
+; CHECK-NEXT: ret
+  %b = zext <4 x i32> %a to <4 x i64>
+  store <4 x i64>%b, <4 x i64>* %out
+  ret void
+}
+
+define void @zext_v8i32_v8i64(<8 x i32>* %in, <8 x i64>* %out) #0 {
+; CHECK-LABEL: zext_v8i32_v8i64:
+; VBITS_GE_512: add [[A_WORDS:z[0-9]+]].s, {{p[0-9]+}}/m, {{z[0-9]+}}.s, {{z[0-9]+}}.s
+; VBITS_GE_512-NEXT: uunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_512-NEXT: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %a = load <8 x i32>, <8 x i32>* %in
+  %b = add <8 x i32> %a, %a
+  %c = zext <8 x i32> %b to <8 x i64>
+  store <8 x i64> %c, <8 x i64>* %out
+  ret void
+}
+
+define void @zext_v16i32_v16i64(<16 x i32>* %in, <16 x i64>* %out) #0 {
+; CHECK-LABEL: zext_v16i32_v16i64:
+; VBITS_GE_1024: add [[A_WORDS:z[0-9]+]].s, {{p[0-9]+}}/m, {{z[0-9]+}}.s, {{z[0-9]+}}.s
+; VBITS_GE_1024-NEXT: uunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_1024-NEXT: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %a = load <16 x i32>, <16 x i32>* %in
+  %b = add <16 x i32> %a, %a
+  %c = zext <16 x i32> %b to <16 x i64>
+  store <16 x i64> %c, <16 x i64>* %out
+  ret void
+}
+
+define void @zext_v32i32_v32i64(<32 x i32>* %in, <32 x i64>* %out) #0 {
+; CHECK-LABEL: zext_v32i32_v32i64:
+; VBITS_GE_2048: add [[A_WORDS:z[0-9]+]].s, {{p[0-9]+}}/m, {{z[0-9]+}}.s, {{z[0-9]+}}.s
+; VBITS_GE_2048-NEXT: uunpklo [[A_DWORDS:z[0-9]+]].d, [[A_WORDS]].s
+; VBITS_GE_2048-NEXT: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: st1d { [[A_DWORDS]].d }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %a = load <32 x i32>, <32 x i32>* %in
+  %b = add <32 x i32> %a, %a
+  %c = zext <32 x i32> %b to <32 x i64>
+  store <32 x i64> %c, <32 x i64>* %out
+  ret void
+}
+
+attributes #0 = { nounwind "target-features"="+sve" }