[AArch64][SVE] Add fixed length codegen for FP_TO_{S,U}INT/{S,U}INT_TO_FP

[bsmith]

Summary

Depends on D102607

[bsmith]

• Use existing fixed length extend/truncate codegen rather than duplicating it

[bsmith]

• Add tests to check type legalisation

[peterwaller-arm]

LGTM. Optional nit.

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

18091

Nit: suggestion: a name like 'CvtOpcode' would be a small readability improvement in use at DAG.getNode().

llvm/lib/Target/AArch64/AArch64ISelLowering.h

SDValue LowerFixedLengthVectorTruncateToSVE(SDValue Op,
                                               SelectionDAG &DAG) const;
   SDValue LowerFixedLengthExtractVectorElt(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFixedLengthInsertVectorElt(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFixedLengthBitcastToSVE(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFixedLengthConcatVectorsToSVE(SDValue Op,
                                              SelectionDAG &DAG) const;
   SDValue LowerFixedLengthFPExtendToSVE(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFixedLengthFPRoundToSVE(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerFixedLengthIntToFPToSVE(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerFixedLengthFPToIntToSVE(SDValue Op, SelectionDAG &DAG) const;
 
   SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
                         SmallVectorImpl<SDNode *> &Created) const override;
   SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
                           int &ExtraSteps, bool &UseOneConst,
                           bool Reciprocal) const override;
   SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
                            int &ExtraSteps) const override;

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

void AArch64TargetLowering::addTypeForFixedLengthSVE(MVT VT) {
   setOperationAction(ISD::FMAXNUM, VT, Custom);
   setOperationAction(ISD::FMINIMUM, VT, Custom);
   setOperationAction(ISD::FMINNUM, VT, Custom);
   setOperationAction(ISD::FMUL, VT, Custom);
   setOperationAction(ISD::FNEARBYINT, VT, Custom);
   setOperationAction(ISD::FNEG, VT, Custom);
   setOperationAction(ISD::FP_EXTEND, VT, Custom);
   setOperationAction(ISD::FP_ROUND, VT, Custom);
+  setOperationAction(ISD::FP_TO_SINT, VT, Custom);
+  setOperationAction(ISD::FP_TO_UINT, VT, Custom);
   setOperationAction(ISD::FRINT, VT, Custom);
   setOperationAction(ISD::FROUND, VT, Custom);
   setOperationAction(ISD::FROUNDEVEN, VT, Custom);
   setOperationAction(ISD::FSQRT, VT, Custom);
   setOperationAction(ISD::FSUB, VT, Custom);
   setOperationAction(ISD::FTRUNC, VT, Custom);
   setOperationAction(ISD::LOAD, VT, Custom);
   setOperationAction(ISD::MUL, VT, Custom);
   setOperationAction(ISD::MULHS, VT, Custom);
   setOperationAction(ISD::MULHU, VT, Custom);
   setOperationAction(ISD::OR, VT, Custom);
   setOperationAction(ISD::SDIV, VT, Custom);
   setOperationAction(ISD::SELECT, VT, Custom);
   setOperationAction(ISD::SETCC, VT, Custom);
   setOperationAction(ISD::SHL, VT, Custom);
   setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
   setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Custom);
+  setOperationAction(ISD::SINT_TO_FP, VT, Custom);
   setOperationAction(ISD::SMAX, VT, Custom);
   setOperationAction(ISD::SMIN, VT, Custom);
   setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
   setOperationAction(ISD::SRA, VT, Custom);
   setOperationAction(ISD::SRL, VT, Custom);
   setOperationAction(ISD::STORE, VT, Custom);
   setOperationAction(ISD::SUB, VT, Custom);
   setOperationAction(ISD::TRUNCATE, VT, Custom);
   setOperationAction(ISD::UDIV, VT, Custom);
+  setOperationAction(ISD::UINT_TO_FP, VT, Custom);
   setOperationAction(ISD::UMAX, VT, Custom);
   setOperationAction(ISD::UMIN, VT, Custom);
   setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
   setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
   setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
   setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
   setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
   setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
SDValue AArch64TargetLowering::LowerVectorFP_TO_INT(SDValue Op,
 
   if (VT.isScalableVector()) {
     unsigned Opcode = Op.getOpcode() == ISD::FP_TO_UINT
                           ? AArch64ISD::FCVTZU_MERGE_PASSTHRU
                           : AArch64ISD::FCVTZS_MERGE_PASSTHRU;
     return LowerToPredicatedOp(Op, DAG, Opcode);
   }
 
+  if (useSVEForFixedLengthVectorVT(VT) || useSVEForFixedLengthVectorVT(InVT))
+    return LowerFixedLengthFPToIntToSVE(Op, DAG);
+
   unsigned NumElts = InVT.getVectorNumElements();
 
   // f16 conversions are promoted to f32 when full fp16 is not supported.
   if (InVT.getVectorElementType() == MVT::f16 &&
       !Subtarget->hasFullFP16()) {
     MVT NewVT = MVT::getVectorVT(MVT::f32, NumElts);
     SDLoc dl(Op);
     return DAG.getNode(
if (InVT.getVectorElementType() == MVT::i1) {
       return DAG.getNode(Opc, dl, VT, In);
     }
 
     unsigned Opcode = IsSigned ? AArch64ISD::SINT_TO_FP_MERGE_PASSTHRU
                                : AArch64ISD::UINT_TO_FP_MERGE_PASSTHRU;
     return LowerToPredicatedOp(Op, DAG, Opcode);
   }
 
+  if (useSVEForFixedLengthVectorVT(VT) || useSVEForFixedLengthVectorVT(InVT))
+    return LowerFixedLengthIntToFPToSVE(Op, DAG);
+
   uint64_t VTSize = VT.getFixedSizeInBits();
   uint64_t InVTSize = InVT.getFixedSizeInBits();
   if (VTSize < InVTSize) {
     MVT CastVT =
         MVT::getVectorVT(MVT::getFloatingPointVT(InVT.getScalarSizeInBits()),
                          InVT.getVectorNumElements());
     In = DAG.getNode(Opc, dl, CastVT, In);
     return DAG.getNode(ISD::FP_ROUND, dl, VT, In, DAG.getIntPtrConstant(0, dl));
SDValue AArch64TargetLowering::LowerFixedLengthConcatVectorsToSVE(
   SrcOp1 = convertToScalableVector(DAG, ContainerVT, SrcOp1);
   SrcOp2 = convertToScalableVector(DAG, ContainerVT, SrcOp2);
 
   Op = DAG.getNode(AArch64ISD::SPLICE, DL, ContainerVT, Pg, SrcOp1, SrcOp2);
 
   return convertFromScalableVector(DAG, VT, Op);
 }
 
+static SDValue unpackSVEVectorTo(SelectionDAG &DAG, SDLoc DL, EVT VT,
+                                 SDValue Val, bool IsSigned) {
+  EVT SrcIntVT = Val.getValueType().changeVectorElementTypeToInteger();
+  EVT DstIntVT = VT.changeVectorElementTypeToInteger();
+
+  // Just bitcast to expected return type if types are already same size
+  if (SrcIntVT == DstIntVT)
+    return DAG.getNode(ISD::BITCAST, DL, VT, Val);
+
+  Val = DAG.getNode(ISD::BITCAST, DL, SrcIntVT, Val);
+
+  // Repeatedly unpack Val until the result is of the desired element type.
+  switch (SrcIntVT.getSimpleVT().SimpleTy) {
+  default:
+    llvm_unreachable("unimplemented container type");
+  case MVT::nxv8i16:
+    Val = DAG.getNode(IsSigned ? AArch64ISD::SUNPKLO : AArch64ISD::UUNPKLO, DL,
+                      MVT::nxv4i32, Val);
+    if (DstIntVT.getVectorElementType() == MVT::i32)
+      break;
+    LLVM_FALLTHROUGH;
+  case MVT::nxv4i32:
+    Val = DAG.getNode(IsSigned ? AArch64ISD::SUNPKLO : AArch64ISD::UUNPKLO, DL,
+                      MVT::nxv2i64, Val);
+    assert(DstIntVT.getVectorElementType() == MVT::i64 &&
+           "Unexpected element type!");
+    break;
+  }
+
+  return DAG.getNode(ISD::BITCAST, DL, VT, Val);
+}
+
+static SDValue packSVEVectorTo(SelectionDAG &DAG, SDLoc DL, EVT VT,
+                               SDValue Val) {
+  EVT SrcIntVT = Val.getValueType().changeVectorElementTypeToInteger();
+  EVT DstIntVT = VT.changeVectorElementTypeToInteger();
+
+  // Just bitcast to expected return type if types are already same size
+  if (SrcIntVT == DstIntVT)
+    return DAG.getNode(ISD::BITCAST, DL, VT, Val);
+
+  Val = DAG.getNode(ISD::BITCAST, DL, SrcIntVT, Val);
+
+  // Repeatedly truncate Val until the result is of the desired element type.
+  switch (SrcIntVT.getSimpleVT().SimpleTy) {
+  default:
+    llvm_unreachable("unimplemented container type");
+  case MVT::nxv2i64:
+    Val = DAG.getNode(ISD::BITCAST, DL, MVT::nxv4i32, Val);
+    Val = DAG.getNode(AArch64ISD::UZP1, DL, MVT::nxv4i32, Val, Val);
+    if (DstIntVT.getVectorElementType() == MVT::i32)
+      break;
+    LLVM_FALLTHROUGH;
+  case MVT::nxv4i32:
+    Val = DAG.getNode(ISD::BITCAST, DL, MVT::nxv8i16, Val);
+    Val = DAG.getNode(AArch64ISD::UZP1, DL, MVT::nxv8i16, Val, Val);
+    assert(DstIntVT.getVectorElementType() == MVT::i16 &&
+           "Unexpected element type!");
+  }
+
+  return DAG.getNode(ISD::BITCAST, DL, VT, Val);
+}
+
 SDValue
 AArch64TargetLowering::LowerFixedLengthFPExtendToSVE(SDValue Op,
                                                      SelectionDAG &DAG) const {
   EVT VT = Op.getValueType();
   assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
 
   SDLoc DL(Op);
   SDValue Val = Op.getOperand(0);
   SDValue Pg = getPredicateForVector(DAG, DL, VT);
   EVT ContainerDstVT = getContainerForFixedLengthVector(DAG, VT);
   EVT ContainerSrcVT =
       getContainerForFixedLengthVector(DAG, Val.getValueType());
   EVT ExtendVT = ContainerDstVT.changeVectorElementType(
       ContainerSrcVT.getVectorElementType());
 
   Val = convertToScalableVector(DAG, ContainerSrcVT, Val);
-  Val = DAG.getNode(ISD::BITCAST, DL,
-                    ContainerSrcVT.changeVectorElementTypeToInteger(), Val);
-
-  // Repeatedly unpack Val until the result is of the desired element type.
-  switch (Val.getValueType().getSimpleVT().SimpleTy) {
-  default:
-    llvm_unreachable("unimplemented container type");
-  case MVT::nxv8i16:
-    Val = DAG.getNode(AArch64ISD::UUNPKLO, DL, MVT::nxv4i32, Val);
-    if (VT.getVectorElementType() == MVT::f32)
-      break;
-    LLVM_FALLTHROUGH;
-  case MVT::nxv4i32:
-    Val = DAG.getNode(AArch64ISD::UUNPKLO, DL, MVT::nxv2i64, Val);
-    assert(VT.getVectorElementType() == MVT::f64 && "Unexpected element type!");
-    break;
-  }
-
+  Val = unpackSVEVectorTo(DAG, DL, ContainerDstVT, Val, false);
   Val = getSVESafeBitCast(ExtendVT, Val, DAG);
   Val = DAG.getNode(AArch64ISD::FP_EXTEND_MERGE_PASSTHRU, DL, ContainerDstVT,
                     Pg, Val, DAG.getUNDEF(ContainerDstVT));
 
   return convertFromScalableVector(DAG, VT, Val);
 }
 
 SDValue
EVT ContainerSrcVT =
       getContainerForFixedLengthVector(DAG, Val.getValueType());
   EVT RoundVT = ContainerSrcVT.changeVectorElementType(
       ContainerDstVT.getVectorElementType());
   SDValue Pg = getPredicateForVector(DAG, DL, RoundVT);
 
   Val = convertToScalableVector(DAG, ContainerSrcVT, Val);
   Val = DAG.getNode(AArch64ISD::FP_ROUND_MERGE_PASSTHRU, DL, RoundVT, Pg, Val,
                     Op.getOperand(1), DAG.getUNDEF(RoundVT));
-  Val = getSVESafeBitCast(ContainerSrcVT.changeVectorElementTypeToInteger(),
-                          Val, DAG);
+  Val = getSVESafeBitCast(ContainerSrcVT, Val, DAG);
+  Val = packSVEVectorTo(DAG, DL, ContainerDstVT, Val);
 
-  // Repeatedly truncate Val until the result is of the desired element type.
-  switch (Val.getValueType().getSimpleVT().SimpleTy) {
-  default:
-    llvm_unreachable("unimplemented container type");
-  case MVT::nxv2i64:
-    Val = DAG.getNode(ISD::BITCAST, DL, MVT::nxv4i32, Val);
-    Val = DAG.getNode(AArch64ISD::UZP1, DL, MVT::nxv4i32, Val, Val);
-    if (VT.getVectorElementType() == MVT::f32)
-      break;
-    LLVM_FALLTHROUGH;
-  case MVT::nxv4i32:
-    Val = DAG.getNode(ISD::BITCAST, DL, MVT::nxv8i16, Val);
-    Val = DAG.getNode(AArch64ISD::UZP1, DL, MVT::nxv8i16, Val, Val);
-    if (VT.getVectorElementType() == MVT::f16)
-      break;
-  }
+  return convertFromScalableVector(DAG, VT, Val);
 
-  Val = DAG.getNode(ISD::BITCAST, DL, ContainerDstVT, Val);
+SDValue
+AArch64TargetLowering::LowerFixedLengthIntToFPToSVE(SDValue Op,
+                                                    SelectionDAG &DAG) const {
+  EVT VT = Op.getValueType();
+  assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
+
+  bool IsSigned = Op.getOpcode() == ISD::SINT_TO_FP;
+  unsigned Opcode = IsSigned ? AArch64ISD::SINT_TO_FP_MERGE_PASSTHRU
+                             : AArch64ISD::UINT_TO_FP_MERGE_PASSTHRU;
+
+  SDLoc DL(Op);
+  SDValue Val = Op.getOperand(0);
+  EVT ContainerDstVT = getContainerForFixedLengthVector(DAG, VT);
+  EVT ContainerSrcVT =
+      getContainerForFixedLengthVector(DAG, Val.getValueType());
+
+  Val = convertToScalableVector(DAG, ContainerSrcVT, Val);
+
+  if (ContainerSrcVT.getVectorElementType().getSizeInBits() <=
+      ContainerDstVT.getVectorElementType().getSizeInBits()) {
+    SDValue Pg = getPredicateForVector(DAG, DL, VT);
+    Val = unpackSVEVectorTo(DAG, DL,
+                            ContainerDstVT.changeVectorElementTypeToInteger(),
+                            Val, IsSigned);
+    // Safe to use a larger than specified operand since we just unpacked the
+    // data, hence the upper bits are zero.
+    Val = DAG.getNode(Opcode, DL, ContainerDstVT, Pg, Val,
+                      DAG.getUNDEF(ContainerDstVT));
+  } else {
+    EVT CvtVT = ContainerSrcVT.changeVectorElementType(
+        ContainerDstVT.getVectorElementType());
+
+    SDValue Pg = getPredicateForVector(DAG, DL, CvtVT);
+    Val = DAG.getNode(Opcode, DL, CvtVT, Pg, Val, DAG.getUNDEF(CvtVT));
+    Val = getSVESafeBitCast(ContainerSrcVT, Val, DAG);
+    Val = packSVEVectorTo(DAG, DL, ContainerDstVT, Val);
+  }
+
+  return convertFromScalableVector(DAG, VT, Val);
+}
+
+SDValue
+AArch64TargetLowering::LowerFixedLengthFPToIntToSVE(SDValue Op,
+                                                    SelectionDAG &DAG) const {
+  EVT VT = Op.getValueType();
+  assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
+
+  bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT;
+  unsigned Opcode = IsSigned ? AArch64ISD::FCVTZS_MERGE_PASSTHRU
+                             : AArch64ISD::FCVTZU_MERGE_PASSTHRU;
+
+  SDLoc DL(Op);
+  SDValue Val = Op.getOperand(0);
+  EVT ContainerDstVT = getContainerForFixedLengthVector(DAG, VT);
+  EVT ContainerSrcVT =
+      getContainerForFixedLengthVector(DAG, Val.getValueType());
+
+  Val = convertToScalableVector(DAG, ContainerSrcVT, Val);
+
+  if (ContainerSrcVT.getVectorElementType().getSizeInBits() <=
+      ContainerDstVT.getVectorElementType().getSizeInBits()) {
+    SDValue Pg = getPredicateForVector(DAG, DL, VT);
+    Val = unpackSVEVectorTo(DAG, DL, ContainerDstVT, Val, false);
+    Val = getSVESafeBitCast(ContainerDstVT.changeVectorElementType(
+                                ContainerSrcVT.getVectorElementType()),
+                            Val, DAG);
+    Val = DAG.getNode(Opcode, DL, ContainerDstVT, Pg, Val,
+                      DAG.getUNDEF(ContainerDstVT));
+  } else {
+    EVT CvtVT = ContainerSrcVT.changeVectorElementTypeToInteger();
+
+    SDValue Pg = getPredicateForVector(DAG, DL, CvtVT);
+    // Safe to use a larger than specified result since an fp_to_int where the
+    // result doesn't fit into the destination is undefined.
+    Val = DAG.getNode(Opcode, DL, CvtVT, Pg, Val, DAG.getUNDEF(CvtVT));
+    Val = getSVESafeBitCast(ContainerSrcVT, Val, DAG);
+    Val = packSVEVectorTo(DAG, DL, ContainerDstVT, Val);
+  }
 
   return convertFromScalableVector(DAG, VT, Val);
 }
 
 SDValue AArch64TargetLowering::getSVESafeBitCast(EVT VT, SDValue Op,
                                                  SelectionDAG &DAG) const {
   SDLoc DL(Op);
   EVT InVT = Op.getValueType();
SDValue AArch64TargetLowering::getSVESafeBitCast(EVT VT, SDValue Op,
   if (VT != PackedVT)
     Op = DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, VT, Op);
 
   return Op;
 }
 
 bool AArch64TargetLowering::isAllActivePredicate(SDValue N) const {
   return ::isAllActivePredicate(N);
 }

llvm/test/CodeGen/AArch64/sve-fixed-length-fp-converts.ll

-; RUN: llc -aarch64-sve-vector-bits-min=128  -asm-verbose=0 < %s | FileCheck %s -check-prefix=NO_SVE
-; RUN: llc -aarch64-sve-vector-bits-min=256  -asm-verbose=0 < %s | FileCheck %s
-; RUN: llc -aarch64-sve-vector-bits-min=384  -asm-verbose=0 < %s | FileCheck %s
-; RUN: llc -aarch64-sve-vector-bits-min=512  -asm-verbose=0 < %s | FileCheck %s
-; RUN: llc -aarch64-sve-vector-bits-min=640  -asm-verbose=0 < %s | FileCheck %s
-; RUN: llc -aarch64-sve-vector-bits-min=768  -asm-verbose=0 < %s | FileCheck %s
-; RUN: llc -aarch64-sve-vector-bits-min=896  -asm-verbose=0 < %s | FileCheck %s
-; RUN: llc -aarch64-sve-vector-bits-min=1024 -asm-verbose=0 < %s | FileCheck %s
-; RUN: llc -aarch64-sve-vector-bits-min=1152 -asm-verbose=0 < %s | FileCheck %s
-; RUN: llc -aarch64-sve-vector-bits-min=1280 -asm-verbose=0 < %s | FileCheck %s
-; RUN: llc -aarch64-sve-vector-bits-min=1408 -asm-verbose=0 < %s | FileCheck %s
-; RUN: llc -aarch64-sve-vector-bits-min=1536 -asm-verbose=0 < %s | FileCheck %s
-; RUN: llc -aarch64-sve-vector-bits-min=1664 -asm-verbose=0 < %s | FileCheck %s
-; RUN: llc -aarch64-sve-vector-bits-min=1792 -asm-verbose=0 < %s | FileCheck %s
-; RUN: llc -aarch64-sve-vector-bits-min=1920 -asm-verbose=0 < %s | FileCheck %s
-; RUN: llc -aarch64-sve-vector-bits-min=2048 -asm-verbose=0 < %s | FileCheck %s
-
-target triple = "aarch64-unknown-linux-gnu"
-
-; Don't use SVE when its registers are no bigger than NEON.
-; NO_SVE-NOT: z{0-9}
-
-; NOTE: fptrunc operations bigger than NEON are expanded. These tests just
-; ensure we've correctly set the operation action for fixed length vector types
-; that require SVE. They'll be updated to protect their expected code generation
-; when lowering it implemented.
-
-;
-; vector uint_to_fp i8 -> f32
-; AArch64 doesn't have a direct vector->f32 conversion instructions for
-; elements smaller than i32, so make sure inputs are promoted to i32 first.
-;
-
-define void @uitofp_v4i8_v4f32(<4 x i8>* %in, <4 x float>* %out) #0 {
-; CHECK-LABEL: uitofp_v4i8_v4f32:
-; CHECK-COUNT-1: ucvt
-  %vec = load <4 x i8>, <4 x i8>* %in
-  %conv = uitofp <4 x i8> %vec to <4 x float>
-  store <4 x float> %conv, <4 x float>* %out
-  ret void
-}
-
-define void @uitofp_v8i8_v8f32(<8 x i8>* %in, <8 x float>* %out) #0 {
-; CHECK-LABEL: uitofp_v8i8_v8f32:
-; CHECK-COUNT-8: ucvt
-  %vec = load <8 x i8>, <8 x i8>* %in
-  %conv = uitofp <8 x i8> %vec to <8 x float>
-  store <8 x float> %conv, <8 x float>* %out
-  ret void
-}
-
-define void @uitofp_v16i8_v16f32(<16 x i8>* %in, <16 x float>* %out) #0 {
-; CHECK-LABEL: uitofp_v16i8_v16f32:
-; CHECK-COUNT-16: ucvt
-  %vec = load <16 x i8>, <16 x i8>* %in
-  %conv = uitofp <16 x i8> %vec to <16 x float>
-  store <16 x float> %conv, <16 x float>* %out
-  ret void
-}
-
-define void @uitofp_v32i8_v32f32(<32 x i8>* %in, <32 x float>* %out) #0 {
-; CHECK-LABEL: uitofp_v32i8_v32f32:
-; CHECK-COUNT-32: ucvt
-  %vec = load <32 x i8>, <32 x i8>* %in
-  %conv = uitofp <32 x i8> %vec to <32 x float>
-  store <32 x float> %conv, <32 x float>* %out
-  ret void
-}
-
-attributes #0 = { nounwind "target-features"="+sve" }

llvm/test/CodeGen/AArch64/sve-fixed-length-fp-to-int.ll

+; RUN: llc -aarch64-sve-vector-bits-min=128  -asm-verbose=0 < %s | FileCheck %s -check-prefix=NO_SVE
+; RUN: llc -aarch64-sve-vector-bits-min=256  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK
+; RUN: llc -aarch64-sve-vector-bits-min=384  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK
+; RUN: llc -aarch64-sve-vector-bits-min=512  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=640  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=768  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=896  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=1024 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1152 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1280 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1408 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1536 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1664 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1792 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1920 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=2048 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024,VBITS_GE_2048
+
+target triple = "aarch64-unknown-linux-gnu"
+
+; Don't use SVE when its registers are no bigger than NEON.
+; NO_SVE-NOT: ptrue
+
+;
+; FCVTZU H -> H
+;
+
+; Don't use SVE for 64-bit vectors.
+define <4 x i16> @fcvtzu_v4f16_v4i16(<4 x half> %op1) #0 {
+; CHECK-LABEL: fcvtzu_v4f16_v4i16:
+; CHECK: fcvtzu v0.4h, v0.4h
+; CHECK-NEXT: ret
+  %res = fptoui <4 x half> %op1 to <4 x i16>
+  ret <4 x i16> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define void @fcvtzu_v8f16_v8i16(<8 x half>* %a, <8 x i16>* %b) #0 {
+; CHECK-LABEL: fcvtzu_v8f16_v8i16:
+; CHECK: ldr q0, [x0]
+; CHECK-NEXT: fcvtzu v0.8h, v0.8h
+; CHECK-NEXT: str q0, [x1]
+; CHECK-NEXT: ret
+  %op1 = load <8 x half>, <8 x half>* %a
+  %res = fptoui <8 x half> %op1 to <8 x i16>
+  store <8 x i16> %res, <8 x i16>* %b
+  ret void
+}
+
+define void @fcvtzu_v16f16_v16i16(<16 x half>* %a, <16 x i16>* %b) #0 {
+; CHECK-LABEL: fcvtzu_v16f16_v16i16:
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl16
+; CHECK-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK-NEXT: fcvtzu [[RES:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; CHECK-NEXT: st1h { [[RES]].h }, [[PG]], [x1]
+; CHECK-NEXT: ret
+  %op1 = load <16 x half>, <16 x half>* %a
+  %res = fptoui <16 x half> %op1 to <16 x i16>
+  store <16 x i16> %res, <16 x i16>* %b
+  ret void
+}
+
+define void @fcvtzu_v32f16_v32i16(<32 x half>* %a, <32 x i16>* %b) #0 {
+; CHECK-LABEL: fcvtzu_v32f16_v32i16:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].h, vl32
+; VBITS_GE_512-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; VBITS_GE_512-NEXT: fcvtzu [[RES:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; VBITS_GE_512-NEXT: st1h { [[RES]].h }, [[PG]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <32 x half>, <32 x half>* %a
+  %res = fptoui <32 x half> %op1 to <32 x i16>
+  store <32 x i16> %res, <32 x i16>* %b
+  ret void
+}
+
+define void @fcvtzu_v64f16_v64i16(<64 x half>* %a, <64 x i16>* %b) #0 {
+; CHECK-LABEL: fcvtzu_v64f16_v64i16:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].h, vl64
+; VBITS_GE_1024-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; VBITS_GE_1024-NEXT: fcvtzu [[RES:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; VBITS_GE_1024-NEXT: st1h { [[RES]].h }, [[PG]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <64 x half>, <64 x half>* %a
+  %res = fptoui <64 x half> %op1 to <64 x i16>
+  store <64 x i16> %res, <64 x i16>* %b
+  ret void
+}
+
+define void @fcvtzu_v128f16_v128i16(<128 x half>* %a, <128 x i16>* %b) #0 {
+; CHECK-LABEL: fcvtzu_v128f16_v128i16:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].h, vl128
+; VBITS_GE_2048-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; VBITS_GE_2048-NEXT: fcvtzu [[RES:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; VBITS_GE_2048-NEXT: st1h { [[RES]].h }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <128 x half>, <128 x half>* %a
+  %res = fptoui <128 x half> %op1 to <128 x i16>
+  store <128 x i16> %res, <128 x i16>* %b
+  ret void
+}
+
+;
+; FCVTZU H -> S
+;
+
+; Don't use SVE for 64-bit vectors.
+define <2 x i32> @fcvtzu_v2f16_v2i32(<2 x half> %op1) #0 {
+; CHECK-LABEL: fcvtzu_v2f16_v2i32:
+; CHECK: fcvtl v0.4s, v0.4h
+; CHECK-NEXT: fcvtzu v0.4s, v0.4s
+; CHECK-NEXT: ret
+  %res = fptoui <2 x half> %op1 to <2 x i32>
+  ret <2 x i32> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <4 x i32> @fcvtzu_v4f16_v4i32(<4 x half> %op1) #0 {
+; CHECK-LABEL: fcvtzu_v4f16_v4i32:
+; CHECK: fcvtzu v0.4s, v0.4s
+; CHECK-NEXT: ret
+  %res = fptoui <4 x half> %op1 to <4 x i32>
+  ret <4 x i32> %res
+}
+
+define void @fcvtzu_v8f16_v8i32(<8 x half>* %a, <8 x i32>* %b) #0 {
+; CHECK-LABEL: fcvtzu_v8f16_v8i32:
+; CHECK: ldr q[[OP:[0-9]+]], [x0]
+; CHECK-NEXT: ptrue [[PG:p[0-9]+]].s, vl8
+; CHECK-NEXT: uunpklo [[UPK:z[0-9]+]].s, z[[OP]].h
+; CHECK-NEXT: fcvtzu [[RES:z[0-9]+]].s, [[PG]]/m, [[UPK]].h
+; CHECK-NEXT: st1w { [[RES]].s }, [[PG]], [x1]
+; CHECK-NEXT: ret
+  %op1 = load <8 x half>, <8 x half>* %a
+  %res = fptoui <8 x half> %op1 to <8 x i32>
+  store <8 x i32> %res, <8 x i32>* %b
+  ret void
+}
+
+define void @fcvtzu_v16f16_v16i32(<16 x half>* %a, <16 x i32>* %b) #0 {
+; CHECK-LABEL: fcvtzu_v16f16_v16i32:
+; VBITS_GE_512: ptrue [[PG1:p[0-9]+]].h, vl16
+; VBITS_GE_512-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG1]]/z, [x0]
+; VBITS_GE_512-NEXT: ptrue [[PG2:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: uunpklo [[UPK:z[0-9]+]].s, [[OP]].h
+; VBITS_GE_512-NEXT: fcvtzu [[RES:z[0-9]+]].s, [[PG2]]/m, [[UPK]].h
+; VBITS_GE_512-NEXT: st1w { [[RES]].s }, [[PG1]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <16 x half>, <16 x half>* %a
+  %res = fptoui <16 x half> %op1 to <16 x i32>
+  store <16 x i32> %res, <16 x i32>* %b
+  ret void
+}
+
+define void @fcvtzu_v32f16_v32i32(<32 x half>* %a, <32 x i32>* %b) #0 {
+; CHECK-LABEL: fcvtzu_v32f16_v32i32:
+; VBITS_GE_1024: ptrue [[PG1:p[0-9]+]].h, vl32
+; VBITS_GE_1024-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG1]]/z, [x0]
+; VBITS_GE_1024-NEXT: ptrue [[PG2:p[0-9]+]].s, vl32
+; VBITS_GE_1024-NEXT: uunpklo [[UPK:z[0-9]+]].s, [[OP]].h
+; VBITS_GE_1024-NEXT: fcvtzu [[RES:z[0-9]+]].s, [[PG2]]/m, [[UPK]].h
+; VBITS_GE_1024-NEXT: st1w { [[RES]].s }, [[PG1]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <32 x half>, <32 x half>* %a
+  %res = fptoui <32 x half> %op1 to <32 x i32>
+  store <32 x i32> %res, <32 x i32>* %b
+  ret void
+}
+
+define void @fcvtzu_v64f16_v64i32(<64 x half>* %a, <64 x i32>* %b) #0 {
+; CHECK-LABEL: fcvtzu_v64f16_v64i32:
+; VBITS_GE_2048: ptrue [[PG1:p[0-9]+]].h, vl64
+; VBITS_GE_2048-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG1]]/z, [x0]
+; VBITS_GE_2048-NEXT: ptrue [[PG2:p[0-9]+]].s, vl64
+; VBITS_GE_2048-NEXT: uunpklo [[UPK:z[0-9]+]].s, [[OP]].h
+; VBITS_GE_2048-NEXT: fcvtzu [[RES:z[0-9]+]].s, [[PG2]]/m, [[UPK]].h
+; VBITS_GE_2048-NEXT: st1w { [[RES]].s }, [[PG1]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <64 x half>, <64 x half>* %a
+  %res = fptoui <64 x half> %op1 to <64 x i32>
+  store <64 x i32> %res, <64 x i32>* %b
+  ret void
+}
+
+;
+; FCVTZU H -> D
+;
+
+; Don't use SVE for 64-bit vectors.
+define <1 x i64> @fcvtzu_v1f16_v1i64(<1 x half> %op1) #0 {
+; CHECK-LABEL: fcvtzu_v1f16_v1i64:
+; CHECK: fcvtzu x8, h0
+; CHECK-NEXT: fmov d0, x8
+; CHECK-NEXT: ret
+  %res = fptoui <1 x half> %op1 to <1 x i64>
+  ret <1 x i64> %res
+}
+
+; v2f16 is not legal for NEON, so use SVE
+define <2 x i64> @fcvtzu_v2f16_v2i64(<2 x half> %op1) #0 {
+; CHECK-LABEL: fcvtzu_v2f16_v2i64:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: uunpklo [[UPK1:z[0-9]+]].s, z0.h
+; CHECK-NEXT: uunpklo [[UPK2:z[0-9]+]].d, [[UPK1]].s
+; CHECK-NEXT: fcvtzu z0.d, [[PG]]/m, [[UPK2]].h
+; CHECK-NEXT: ret
+  %res = fptoui <2 x half> %op1 to <2 x i64>
+  ret <2 x i64> %res
+}
+
+define void @fcvtzu_v4f16_v4i64(<4 x half>* %a, <4 x i64>* %b) #0 {
+; CHECK-LABEL: fcvtzu_v4f16_v4i64:
+; CHECK: ldr d[[OP:[0-9]+]], [x0]
+; CHECK-NEXT: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: uunpklo [[UPK1:z[0-9]+]].s, z[[OP]].h
+; CHECK-NEXT: uunpklo [[UPK2:z[0-9]+]].d, [[UPK1]].s
+; CHECK-NEXT: fcvtzu [[RES:z[0-9]+]].d, [[PG]]/m, [[UPK2]].h
+; CHECK-NEXT: st1d { [[RES]].d }, [[PG]], [x1]
+; CHECK-NEXT: ret
+  %op1 = load <4 x half>, <4 x half>* %a
+  %res = fptoui <4 x half> %op1 to <4 x i64>
+  store <4 x i64> %res, <4 x i64>* %b
+  ret void
+}
+
+define void @fcvtzu_v8f16_v8i64(<8 x half>* %a, <8 x i64>* %b) #0 {
+; CHECK-LABEL: fcvtzu_v8f16_v8i64:
+; VBITS_GE_512: ldr q[[OP:[0-9]+]], [x0]
+; VBITS_GE_512-NEXT: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: uunpklo [[UPK1:z[0-9]+]].s, z[[OP]].h
+; VBITS_GE_512-NEXT: uunpklo [[UPK2:z[0-9]+]].d, [[UPK1]].s
+; VBITS_GE_512-NEXT: fcvtzu [[RES:z[0-9]+]].d, [[PG]]/m, [[UPK2]].h
+; VBITS_GE_512-NEXT: st1d { [[RES]].d }, [[PG]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <8 x half>, <8 x half>* %a
+  %res = fptoui <8 x half> %op1 to <8 x i64>
+  store <8 x i64> %res, <8 x i64>* %b
+  ret void
+}
+
+define void @fcvtzu_v16f16_v16i64(<16 x half>* %a, <16 x i64>* %b) #0 {
+; CHECK-LABEL: fcvtzu_v16f16_v16i64:
+; VBITS_GE_1024: ptrue [[PG1:p[0-9]+]].h, vl16
+; VBITS_GE_1024-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG1]]/z, [x0]
+; VBITS_GE_1024-NEXT: ptrue [[PG2:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: uunpklo [[UPK1:z[0-9]+]].s, [[OP]].h
+; VBITS_GE_1024-NEXT: uunpklo [[UPK2:z[0-9]+]].d, [[UPK1]].s
+; VBITS_GE_1024-NEXT: fcvtzu [[RES:z[0-9]+]].d, [[PG2]]/m, [[UPK2]].h
+; VBITS_GE_1024-NEXT: st1d { [[RES]].d }, [[PG1]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <16 x half>, <16 x half>* %a
+  %res = fptoui <16 x half> %op1 to <16 x i64>
+  store <16 x i64> %res, <16 x i64>* %b
+  ret void
+}
+
+define void @fcvtzu_v32f16_v32i64(<32 x half>* %a, <32 x i64>* %b) #0 {
+; CHECK-LABEL: fcvtzu_v32f16_v32i64:
+; VBITS_GE_2048: ptrue [[PG1:p[0-9]+]].h, vl32
+; VBITS_GE_2048-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG1]]/z, [x0]
+; VBITS_GE_2048-NEXT: ptrue [[PG2:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: uunpklo [[UPK1:z[0-9]+]].s, [[OP]].h
+; VBITS_GE_2048-NEXT: uunpklo [[UPK2:z[0-9]+]].d, [[UPK]].s
+; VBITS_GE_2048-NEXT: fcvtzu [[RES:z[0-9]+]].d, [[PG2]]/m, [[UPK2]].h
+; VBITS_GE_2048-NEXT: st1d { [[RES]].d }, [[PG1]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <32 x half>, <32 x half>* %a
+  %res = fptoui <32 x half> %op1 to <32 x i64>
+  store <32 x i64> %res, <32 x i64>* %b
+  ret void
+}
+
+;
+; FCVTZU S -> H
+;
+
+; Don't use SVE for 64-bit vectors.
+define <2 x i16> @fcvtzu_v2f32_v2i16(<2 x float> %op1) #0 {
+; CHECK-LABEL: fcvtzu_v2f32_v2i16:
+; CHECK: fcvtzs v0.2s, v0.2s
+; CHECK-NEXT: ret
+  %res = fptoui <2 x float> %op1 to <2 x i16>
+  ret <2 x i16> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <4 x i16> @fcvtzu_v4f32_v4i16(<4 x float> %op1) #0 {
+; CHECK-LABEL: fcvtzu_v4f32_v4i16:
+; CHECK: fcvtzu v0.4s, v0.4s
+; CHECK-NEXT: mov w8, v0.s[1]
+; CHECK-NEXT: mov w9, v0.s[2]
+; CHECK-NEXT: mov w10, v0.s[3]
+; CHECK-NEXT: mov v0.h[1], w8
+; CHECK-NEXT: mov v0.h[2], w9
+; CHECK-NEXT: mov v0.h[3], w10
+; CHECK-NEXT: ret
+  %res = fptoui <4 x float> %op1 to <4 x i16>
+  ret <4 x i16> %res
+}
+
+define <8 x i16> @fcvtzu_v8f32_v8i16(<8 x float>* %a) #0 {
+; CHECK-LABEL: fcvtzu_v8f32_v8i16:
+; CHECK: ptrue [[PG1:p[0-9]+]].s, vl8
+; CHECK-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG1]]/z, [x0]
+; CHECK-NEXT: ptrue [[PG2:p[0-9]+]].s
+; CHECK-NEXT: fcvtzu [[CVT:z[0-9]+]].s, [[PG2]]/m, [[OP]].s
+; CHECK-NEXT: uzp1 z0.h, [[CVT]].h, [[CVT]].h
+; CHECK-NEXT: ret
+  %op1 = load <8 x float>, <8 x float>* %a
+  %res = fptoui <8 x float> %op1 to <8 x i16>
+  ret <8 x i16> %res
+}
+
+define void @fcvtzu_v16f32_v16i16(<16 x float>* %a, <16 x i16>* %b) #0 {
+; CHECK-LABEL: fcvtzu_v16f32_v16i16:
+; VBITS_GE_512: ptrue [[PG1:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG1]]/z, [x0]
+; VBITS_GE_512-NEXT: ptrue [[PG2:p[0-9]+]].s
+; VBITS_GE_512-NEXT: fcvtzu [[CVT:z[0-9]+]].s, [[PG2]]/m, [[OP]].s
+; VBITS_GE_512-NEXT: uzp1 [[RES:z[0-9]+]].h, [[CVT]].h, [[CVT]].h
+; VBITS_GE_512-NEXT: ptrue [[PG3:p[0-9]+]].h, vl16
+; VBITS_GE_512-NEXT: st1h { [[RES]].h }, [[PG3]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <16 x float>, <16 x float>* %a
+  %res = fptoui <16 x float> %op1 to <16 x i16>
+  store <16 x i16> %res, <16 x i16>* %b
+  ret void
+}
+
+define void @fcvtzu_v32f32_v32i16(<32 x float>* %a, <32 x i16>* %b) #0 {
+; CHECK-LABEL: fcvtzu_v32f32_v32i16:
+; VBITS_GE_1024: ptrue [[PG1:p[0-9]+]].s, vl32
+; VBITS_GE_1024-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG1]]/z, [x0]
+; VBITS_GE_1024-NEXT: ptrue [[PG2:p[0-9]+]].s
+; VBITS_GE_1024-NEXT: fcvtzu [[CVT:z[0-9]+]].s, [[PG2]]/m, [[OP]].s
+; VBITS_GE_1024-NEXT: uzp1 [[RES:z[0-9]+]].h, [[CVT]].h, [[CVT]].h
+; VBITS_GE_1024-NEXT: ptrue [[PG3:p[0-9]+]].h, vl32
+; VBITS_GE_1024-NEXT: st1h { [[RES]].h }, [[PG3]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <32 x float>, <32 x float>* %a
+  %res = fptoui <32 x float> %op1 to <32 x i16>
+  store <32 x i16> %res, <32 x i16>* %b
+  ret void
+}
+
+define void @fcvtzu_v64f32_v64i16(<64 x float>* %a, <64 x i16>* %b) #0 {
+; CHECK-LABEL: fcvtzu_v64f32_v64i16:
+; VBITS_GE_2048: ptrue [[PG1:p[0-9]+]].s, vl64
+; VBITS_GE_2048-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG1]]/z, [x0]
+; VBITS_GE_2048-NEXT: ptrue [[PG2:p[0-9]+]].s
+; VBITS_GE_2048-NEXT: fcvtzu [[RES:z[0-9]+]].s, [[PG2]]/m, [[UPK]].s
+; VBITS_GE_2048-NEXT: uzp1 [[RES:z[0-9]+]].h, [[CVT]].h, [[CVT]].h
+; VBITS_GE_2048-NEXT: ptrue [[PG3:p[0-9]+]].h, vl64
+; VBITS_GE_2048-NEXT: st1h { [[RES]].h }, [[PG3]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <64 x float>, <64 x float>* %a
+  %res = fptoui <64 x float> %op1 to <64 x i16>
+  store <64 x i16> %res, <64 x i16>* %b
+  ret void
+}
+
+;
+; FCVTZU S -> S
+;
+
+; Don't use SVE for 64-bit vectors.
+define <2 x i32> @fcvtzu_v2f32_v2i32(<2 x float> %op1) #0 {
+; CHECK-LABEL: fcvtzu_v2f32_v2i32:
+; CHECK: fcvtzu v0.2s, v0.2s
+; CHECK-NEXT: ret
+  %res = fptoui <2 x float> %op1 to <2 x i32>
+  ret <2 x i32> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <4 x i32> @fcvtzu_v4f32_v4i32(<4 x float> %op1) #0 {
+; CHECK-LABEL: fcvtzu_v4f32_v4i32:
+; CHECK: fcvtzu v0.4s, v0.4s
+; CHECK-NEXT: ret
+  %res = fptoui <4 x float> %op1 to <4 x i32>
+  ret <4 x i32> %res
+}
+
+define void @fcvtzu_v8f32_v8i32(<8 x float>* %a, <8 x i32>* %b) #0 {
+; CHECK-LABEL: fcvtzu_v8f32_v8i32:
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl8
+; CHECK-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-NEXT: fcvtzu [[RES:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; CHECK-NEXT: st1w { [[RES]].s }, [[PG]], [x1]
+; CHECK-NEXT: ret
+  %op1 = load <8 x float>, <8 x float>* %a
+  %res = fptoui <8 x float> %op1 to <8 x i32>
+  store <8 x i32> %res, <8 x i32>* %b
+  ret void
+}
+
+define void @fcvtzu_v16f32_v16i32(<16 x float>* %a, <16 x i32>* %b) #0 {
+; CHECK-LABEL: fcvtzu_v16f32_v16i32:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_512-NEXT: fcvtzu [[RES:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; VBITS_GE_512-NEXT: st1w { [[RES]].s }, [[PG]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <16 x float>, <16 x float>* %a
+  %res = fptoui <16 x float> %op1 to <16 x i32>
+  store <16 x i32> %res, <16 x i32>* %b
+  ret void
+}
+
+define void @fcvtzu_v32f32_v32i32(<32 x float>* %a, <32 x i32>* %b) #0 {
+; CHECK-LABEL: fcvtzu_v32f32_v32i32:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].s, vl32
+; VBITS_GE_1024-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_1024-NEXT: fcvtzu [[RES:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; VBITS_GE_1024-NEXT: st1w { [[RES]].s }, [[PG]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <32 x float>, <32 x float>* %a
+  %res = fptoui <32 x float> %op1 to <32 x i32>
+  store <32 x i32> %res, <32 x i32>* %b
+  ret void
+}
+
+define void @fcvtzu_v64f32_v64i32(<64 x float>* %a, <64 x i32>* %b) #0 {
+; CHECK-LABEL: fcvtzu_v64f32_v64i32:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].s, vl64
+; VBITS_GE_2048-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_2048-NEXT: fcvtzu [[RES:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; VBITS_GE_2048-NEXT: st1w { [[RES]].s }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <64 x float>, <64 x float>* %a
+  %res = fptoui <64 x float> %op1 to <64 x i32>
+  store <64 x i32> %res, <64 x i32>* %b
+  ret void
+}
+
+;
+; FCVTZU S -> D
+;
+
+; Don't use SVE for 64-bit vectors.
+define <1 x i64> @fcvtzu_v1f32_v1i64(<1 x float> %op1) #0 {
+; CHECK-LABEL: fcvtzu_v1f32_v1i64:
+; CHECK: fcvtl v0.2d, v0.2s
+; CHECK-NEXT: fcvtzu v0.2d, v0.2d
+; CHECK-NEXT: ret
+  %res = fptoui <1 x float> %op1 to <1 x i64>
+  ret <1 x i64> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <2 x i64> @fcvtzu_v2f32_v2i64(<2 x float> %op1) #0 {
+; CHECK-LABEL: fcvtzu_v2f32_v2i64:
+; CHECK: fcvtl v0.2d, v0.2s
+; CHECK-NEXT: fcvtzu v0.2d, v0.2d
+; CHECK-NEXT: ret
+  %res = fptoui <2 x float> %op1 to <2 x i64>
+  ret <2 x i64> %res
+}
+
+define void @fcvtzu_v4f32_v4i64(<4 x float>* %a, <4 x i64>* %b) #0 {
+; CHECK-LABEL: fcvtzu_v4f32_v4i64:
+; CHECK: ldr q[[OP:[0-9]+]], [x0]
+; CHECK-NEXT: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: uunpklo [[UPK:z[0-9]+]].d, z[[OP]].s
+; CHECK-NEXT: fcvtzu [[RES:z[0-9]+]].d, [[PG]]/m, [[UPK]].s
+; CHECK-NEXT: st1d { [[RES]].d }, [[PG]], [x1]
+; CHECK-NEXT: ret
+  %op1 = load <4 x float>, <4 x float>* %a
+  %res = fptoui <4 x float> %op1 to <4 x i64>
+  store <4 x i64> %res, <4 x i64>* %b
+  ret void
+}
+
+define void @fcvtzu_v8f32_v8i64(<8 x float>* %a, <8 x i64>* %b) #0 {
+; CHECK-LABEL: fcvtzu_v8f32_v8i64:
+; VBITS_GE_512: ptrue [[PG1:p[0-9]+]].s, vl8
+; VBITS_GE_512-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG1]]/z, [x0]
+; VBITS_GE_512-NEXT: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: uunpklo [[UPK:z[0-9]+]].d, [[OP]].s
+; VBITS_GE_512-NEXT: fcvtzu [[RES:z[0-9]+]].d, [[PG1]]/m, [[UPK]].s
+; VBITS_GE_512-NEXT: st1d { [[RES]].d }, [[PG1]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <8 x float>, <8 x float>* %a
+  %res = fptoui <8 x float> %op1 to <8 x i64>
+  store <8 x i64> %res, <8 x i64>* %b
+  ret void
+}
+
+define void @fcvtzu_v16f32_v16i64(<16 x float>* %a, <16 x i64>* %b) #0 {
+; CHECK-LABEL: fcvtzu_v16f32_v16i64:
+; VBITS_GE_1024: ptrue [[PG1:p[0-9]+]].s, vl16
+; VBITS_GE_1024-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG1]]/z, [x0]
+; VBITS_GE_1024-NEXT: ptrue [[PG2:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: uunpklo [[UPK:z[0-9]+]].d, [[OP]].s
+; VBITS_GE_1024-NEXT: fcvtzu [[RES:z[0-9]+]].d, [[PG2]]/m, [[UPK]].s
+; VBITS_GE_1024-NEXT: st1d { [[RES]].d }, [[PG1]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <16 x float>, <16 x float>* %a
+  %res = fptoui <16 x float> %op1 to <16 x i64>
+  store <16 x i64> %res, <16 x i64>* %b
+  ret void
+}
+
+define void @fcvtzu_v32f32_v32i64(<32 x float>* %a, <32 x i64>* %b) #0 {
+; CHECK-LABEL: fcvtzu_v32f32_v32i64:
+; VBITS_GE_2048: ptrue [[PG1:p[0-9]+]].s, vl32
+; VBITS_GE_2048-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG1]]/z, [x0]
+; VBITS_GE_2048-NEXT: ptrue [[PG2:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: uunpklo [[UPK:z[0-9]+]].d, [[OP]].s
+; VBITS_GE_2048-NEXT: fcvtzu [[RES:z[0-9]+]].d, [[PG2]]/m, [[UPK]].s
+; VBITS_GE_2048-NEXT: st1d { [[RES]].d }, [[PG1]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <32 x float>, <32 x float>* %a
+  %res = fptoui <32 x float> %op1 to <32 x i64>
+  store <32 x i64> %res, <32 x i64>* %b
+  ret void
+}
+
+
+;
+; FCVTZU D -> H
+;
+
+; v1f64 is perfered to be widened to v4f64, so use SVE
+define <1 x i16> @fcvtzu_v1f64_v1i16(<1 x double> %op1) #0 {
+; CHECK-LABEL: fcvtzu_v1f64_v1i16:
+; CHECK: ptrue [[PG:p[0-9]+]].d
+; CHECK-NEXT: fcvtzu [[CVT:z[0-9]+]].d, [[PG]]/m, z0.d
+; CHECK-NEXT: uzp1 [[UZP:z[0-9]+]].s, [[CVT]].s, [[CVT]].s
+; CHECK-NEXT: uzp1 z0.h, [[UZP]].h, [[UZP]].h
+; CHECK-NEXT: ret
+  %res = fptoui <1 x double> %op1 to <1 x i16>
+  ret <1 x i16> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <2 x i16> @fcvtzu_v2f64_v2i16(<2 x double> %op1) #0 {
+; CHECK-LABEL: fcvtzu_v2f64_v2i16:
+; CHECK: fcvtzs v0.2d, v0.2d
+; CHECK-NEXT: xtn v0.2s, v0.2d
+; CHECK-NEXT: ret
+  %res = fptoui <2 x double> %op1 to <2 x i16>
+  ret <2 x i16> %res
+}
+
+define <4 x i16> @fcvtzu_v4f64_v4i16(<4 x double>* %a) #0 {
+; CHECK-LABEL: fcvtzu_v4f64_v4i16:
+; CHECK: ptrue [[PG1:p[0-9]+]].d, vl4
+; CHECK-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG1]]/z, [x0]
+; CHECK-NEXT: ptrue [[PG2:p[0-9]+]].d
+; CHECK-NEXT: fcvtzu [[CVT:z[0-9]+]].d, [[PG2]]/m, [[OP]].d
+; CHECK-NEXT: uzp1 [[UZP:z[0-9]+]].s, [[CVT]].s, [[CVT]].s
+; CHECK-NEXT: uzp1 z0.h, [[UZP]].h, [[UZP]].h
+; CHECK-NEXT: ret
+  %op1 = load <4 x double>, <4 x double>* %a
+  %res = fptoui <4 x double> %op1 to <4 x i16>
+  ret <4 x i16> %res
+}
+
+define <8 x i16> @fcvtzu_v8f64_v8i16(<8 x double>* %a) #0 {
+; CHECK-LABEL: fcvtzu_v8f64_v8i16:
+; VBITS_GE_512: ptrue [[PG1:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG1]]/z, [x0]
+; VBITS_GE_512-NEXT: ptrue [[PG2:p[0-9]+]].d
+; VBITS_GE_512-NEXT: fcvtzu [[CVT:z[0-9]+]].d, [[PG2]]/m, [[OP]].d
+; VBITS_GE_512-NEXT: uzp1 [[UZP:z[0-9]+]].s, [[CVT]].s, [[CVT]].s
+; VBITS_GE_512-NEXT: uzp1 z0.h, [[UZP]].h, [[UZP]].h
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <8 x double>, <8 x double>* %a
+  %res = fptoui <8 x double> %op1 to <8 x i16>
+  ret <8 x i16> %res
+}
+
+define void @fcvtzu_v16f64_v16i16(<16 x double>* %a, <16 x i16>* %b) #0 {
+; CHECK-LABEL: fcvtzu_v16f64_v16i16:
+; VBITS_GE_1024: ptrue [[PG1:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG1]]/z, [x0]
+; VBITS_GE_1024-NEXT: ptrue [[PG2:p[0-9]+]].d
+; VBITS_GE_1024-NEXT: fcvtzu [[CVT:z[0-9]+]].d, [[PG2]]/m, [[OP]].d
+; VBITS_GE_1024-NEXT: uzp1 [[UZP:z[0-9]+]].s, [[CVT]].s, [[CVT]].s
+; VBITS_GE_1024-NEXT: uzp1 [[RES:z[0-9]+]].h, [[UZP]].h, [[UZP]].h
+; VBITS_GE_1024-NEXT: ptrue [[PG3:p[0-9]+]].h, vl16
+; VBITS_GE_1024-NEXT: st1h { [[RES]].h }, [[PG3]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <16 x double>, <16 x double>* %a
+  %res = fptoui <16 x double> %op1 to <16 x i16>
+  store <16 x i16> %res, <16 x i16>* %b
+  ret void
+}
+
+define void @fcvtzu_v32f64_v32i16(<32 x double>* %a, <32 x i16>* %b) #0 {
+; CHECK-LABEL: fcvtzu_v32f64_v32i16:
+; VBITS_GE_2048: ptrue [[PG1:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG1]]/z, [x0]
+; VBITS_GE_2048-NEXT: ptrue [[PG2:p[0-9]+]].d
+; VBITS_GE_2048-NEXT: fcvtzu [[CVT:z[0-9]+]].d, [[PG2]]/m, [[OP]].d
+; VBITS_GE_2048-NEXT: uzp1 [[UZP:z[0-9]+]].s, [[CVT]].s, [[CVT]].s
+; VBITS_GE_2048-NEXT: uzp1 [[RES:z[0-9]+]].h, [[UZP]].h, [[UZP]].h
+; VBITS_GE_2048-NEXT: ptrue [[PG3:p[0-9]+]].h, vl32
+; VBITS_GE_2048-NEXT: st1h { [[RES]].h }, [[PG3]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <32 x double>, <32 x double>* %a
+  %res = fptoui <32 x double> %op1 to <32 x i16>
+  store <32 x i16> %res, <32 x i16>* %b
+  ret void
+}
+
+;
+; FCVTZU D -> S
+;
+
+; Don't use SVE for 64-bit vectors.
+define <1 x i32> @fcvtzu_v1f64_v1i32(<1 x double> %op1) #0 {
+; CHECK-LABEL: fcvtzu_v1f64_v1i32:
+; CHECK: fcvtzu v0.2d, v0.2d
+; CHECK-NEXT: xtn v0.2s, v0.2d
+; CHECK-NEXT: ret
+  %res = fptoui <1 x double> %op1 to <1 x i32>
+  ret <1 x i32> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <2 x i32> @fcvtzu_v2f64_v2i32(<2 x double> %op1) #0 {
+; CHECK-LABEL: fcvtzu_v2f64_v2i32:
+; CHECK: fcvtzu v0.2d, v0.2d
+; CHECK-NEXT: xtn v0.2s, v0.2d
+; CHECK-NEXT: ret
+  %res = fptoui <2 x double> %op1 to <2 x i32>
+  ret <2 x i32> %res
+}
+
+define <4 x i32> @fcvtzu_v4f64_v4i32(<4 x double>* %a) #0 {
+; CHECK-LABEL: fcvtzu_v4f64_v4i32:
+; CHECK: ptrue [[PG1:p[0-9]+]].d, vl4
+; CHECK-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG1]]/z, [x0]
+; CHECK-NEXT: ptrue [[PG2:p[0-9]+]].d
+; CHECK-NEXT: fcvtzu [[CVT:z[0-9]+]].d, [[PG2]]/m, [[OP]].d
+; CHECK-NEXT: uzp1 z0.s, [[CVT]].s, [[CVT]].s
+; CHECK-NEXT: ret
+  %op1 = load <4 x double>, <4 x double>* %a
+  %res = fptoui <4 x double> %op1 to <4 x i32>
+  ret <4 x i32> %res
+}
+
+define void @fcvtzu_v8f64_v8i32(<8 x double>* %a, <8 x i32>* %b) #0 {
+; CHECK-LABEL: fcvtzu_v8f64_v8i32:
+; VBITS_GE_512: ptrue [[PG1:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG1]]/z, [x0]
+; VBITS_GE_512-NEXT: ptrue [[PG2:p[0-9]+]].d
+; VBITS_GE_512-NEXT: fcvtzu [[CVT:z[0-9]+]].d, [[PG2]]/m, [[OP]].d
+; VBITS_GE_512-NEXT: uzp1 [[RES:z[0-9]+]].s, [[CVT]].s, [[CVT]].s
+; VBITS_GE_512-NEXT: ptrue [[PG3:p[0-9]+]].s, vl8
+; VBITS_GE_512-NEXT: st1w { [[RES]].s }, [[PG3]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <8 x double>, <8 x double>* %a
+  %res = fptoui <8 x double> %op1 to <8 x i32>
+  store <8 x i32> %res, <8 x i32>* %b
+  ret void
+}
+
+define void @fcvtzu_v16f64_v16i32(<16 x double>* %a, <16 x i32>* %b) #0 {
+; CHECK-LABEL: fcvtzu_v16f64_v16i32:
+; VBITS_GE_1024: ptrue [[PG1:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG1]]/z, [x0]
+; VBITS_GE_1024-NEXT: ptrue [[PG2:p[0-9]+]].d
+; VBITS_GE_1024-NEXT: fcvtzu [[CVT:z[0-9]+]].d, [[PG2]]/m, [[OP]].d
+; VBITS_GE_1024-NEXT: uzp1 [[RES:z[0-9]+]].s, [[CVT]].s, [[CVT]].s
+; VBITS_GE_1024-NEXT: ptrue [[PG3:p[0-9]+]].s, vl16
+; VBITS_GE_1024-NEXT: st1w { [[RES]].s }, [[PG3]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <16 x double>, <16 x double>* %a
+  %res = fptoui <16 x double> %op1 to <16 x i32>
+  store <16 x i32> %res, <16 x i32>* %b
+  ret void
+}
+
+define void @fcvtzu_v32f64_v32i32(<32 x double>* %a, <32 x i32>* %b) #0 {
+; CHECK-LABEL: fcvtzu_v32f64_v32i32:
+; VBITS_GE_2048: ptrue [[PG1:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG1]]/z, [x0]
+; VBITS_GE_2048-NEXT: ptrue [[PG2:p[0-9]+]].d
+; VBITS_GE_2048-NEXT: fcvtzu [[CVT:z[0-9]+]].d, [[PG2]]/m, [[OP]].d
+; VBITS_GE_2048-NEXT: uzp1 [[RES:z[0-9]+]].s, [[CVT]].s, [[CVT]].s
+; VBITS_GE_2048-NEXT: ptrue [[PG3:p[0-9]+]].s, vl32
+; VBITS_GE_2048-NEXT: st1w { [[RES]].s }, [[PG3]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <32 x double>, <32 x double>* %a
+  %res = fptoui <32 x double> %op1 to <32 x i32>
+  store <32 x i32> %res, <32 x i32>* %b
+  ret void
+}
+
+;
+; FCVTZU D -> D
+;
+
+; Don't use SVE for 64-bit vectors.
+define <1 x i64> @fcvtzu_v1f64_v1i64(<1 x double> %op1) #0 {
+; CHECK-LABEL: fcvtzu_v1f64_v1i64:
+; CHECK: fcvtzu x8, d0
+; CHECK: fmov d0, x8
+; CHECK-NEXT: ret
+  %res = fptoui <1 x double> %op1 to <1 x i64>
+  ret <1 x i64> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <2 x i64> @fcvtzu_v2f64_v2i64(<2 x double> %op1) #0 {
+; CHECK-LABEL: fcvtzu_v2f64_v2i64:
+; CHECK: fcvtzu v0.2d, v0.2d
+; CHECK-NEXT: ret
+  %res = fptoui <2 x double> %op1 to <2 x i64>
+  ret <2 x i64> %res
+}
+
+define void @fcvtzu_v4f64_v4i64(<4 x double>* %a, <4 x i64>* %b) #0 {
+; CHECK-LABEL: fcvtzu_v4f64_v4i64:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-NEXT: fcvtzu [[RES:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; CHECK-NEXT: st1d { [[RES]].d }, [[PG]], [x1]
+; CHECK-NEXT: ret
+  %op1 = load <4 x double>, <4 x double>* %a
+  %res = fptoui <4 x double> %op1 to <4 x i64>
+  store <4 x i64> %res, <4 x i64>* %b
+  ret void
+}
+
+define void @fcvtzu_v8f64_v8i64(<8 x double>* %a, <8 x i64>* %b) #0 {
+; CHECK-LABEL: fcvtzu_v8f64_v8i64:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_512-NEXT: fcvtzu [[RES:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; VBITS_GE_512-NEXT: st1d { [[RES]].d }, [[PG]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <8 x double>, <8 x double>* %a
+  %res = fptoui <8 x double> %op1 to <8 x i64>
+  store <8 x i64> %res, <8 x i64>* %b
+  ret void
+}
+
+define void @fcvtzu_v16f64_v16i64(<16 x double>* %a, <16 x i64>* %b) #0 {
+; CHECK-LABEL: fcvtzu_v16f64_v16i64:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_1024-NEXT: fcvtzu [[RES:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; VBITS_GE_1024-NEXT: st1d { [[RES]].d }, [[PG]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <16 x double>, <16 x double>* %a
+  %res = fptoui <16 x double> %op1 to <16 x i64>
+  store <16 x i64> %res, <16 x i64>* %b
+  ret void
+}
+
+define void @fcvtzu_v32f64_v32i64(<32 x double>* %a, <32 x i64>* %b) #0 {
+; CHECK-LABEL: fcvtzu_v32f64_v32i64:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_2048-NEXT: fcvtzu [[RES:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; VBITS_GE_2048-NEXT: st1d { [[RES]].d }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <32 x double>, <32 x double>* %a
+  %res = fptoui <32 x double> %op1 to <32 x i64>
+  store <32 x i64> %res, <32 x i64>* %b
+  ret void
+}
+
+;
+; FCVTZS H -> H
+;
+
+; Don't use SVE for 64-bit vectors.
+define <4 x i16> @fcvtzs_v4f16_v4i16(<4 x half> %op1) #0 {
+; CHECK-LABEL: fcvtzs_v4f16_v4i16:
+; CHECK: fcvtzs v0.4h, v0.4h
+; CHECK-NEXT: ret
+  %res = fptosi <4 x half> %op1 to <4 x i16>
+  ret <4 x i16> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define void @fcvtzs_v8f16_v8i16(<8 x half>* %a, <8 x i16>* %b) #0 {
+; CHECK-LABEL: fcvtzs_v8f16_v8i16:
+; CHECK: ldr q0, [x0]
+; CHECK-NEXT: fcvtzs v0.8h, v0.8h
+; CHECK-NEXT: str q0, [x1]
+; CHECK-NEXT: ret
+  %op1 = load <8 x half>, <8 x half>* %a
+  %res = fptosi <8 x half> %op1 to <8 x i16>
+  store <8 x i16> %res, <8 x i16>* %b
+  ret void
+}
+
+define void @fcvtzs_v16f16_v16i16(<16 x half>* %a, <16 x i16>* %b) #0 {
+; CHECK-LABEL: fcvtzs_v16f16_v16i16:
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl16
+; CHECK-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK-NEXT: fcvtzs [[RES:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; CHECK-NEXT: st1h { [[RES]].h }, [[PG]], [x1]
+; CHECK-NEXT: ret
+  %op1 = load <16 x half>, <16 x half>* %a
+  %res = fptosi <16 x half> %op1 to <16 x i16>
+  store <16 x i16> %res, <16 x i16>* %b
+  ret void
+}
+
+define void @fcvtzs_v32f16_v32i16(<32 x half>* %a, <32 x i16>* %b) #0 {
+; CHECK-LABEL: fcvtzs_v32f16_v32i16:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].h, vl32
+; VBITS_GE_512-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; VBITS_GE_512-NEXT: fcvtzs [[RES:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; VBITS_GE_512-NEXT: st1h { [[RES]].h }, [[PG]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <32 x half>, <32 x half>* %a
+  %res = fptosi <32 x half> %op1 to <32 x i16>
+  store <32 x i16> %res, <32 x i16>* %b
+  ret void
+}
+
+define void @fcvtzs_v64f16_v64i16(<64 x half>* %a, <64 x i16>* %b) #0 {
+; CHECK-LABEL: fcvtzs_v64f16_v64i16:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].h, vl64
+; VBITS_GE_1024-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; VBITS_GE_1024-NEXT: fcvtzs [[RES:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; VBITS_GE_1024-NEXT: st1h { [[RES]].h }, [[PG]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <64 x half>, <64 x half>* %a
+  %res = fptosi <64 x half> %op1 to <64 x i16>
+  store <64 x i16> %res, <64 x i16>* %b
+  ret void
+}
+
+define void @fcvtzs_v128f16_v128i16(<128 x half>* %a, <128 x i16>* %b) #0 {
+; CHECK-LABEL: fcvtzs_v128f16_v128i16:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].h, vl128
+; VBITS_GE_2048-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; VBITS_GE_2048-NEXT: fcvtzs [[RES:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; VBITS_GE_2048-NEXT: st1h { [[RES]].h }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <128 x half>, <128 x half>* %a
+  %res = fptosi <128 x half> %op1 to <128 x i16>
+  store <128 x i16> %res, <128 x i16>* %b
+  ret void
+}
+
+;
+; FCVTZS H -> S
+;
+
+; Don't use SVE for 64-bit vectors.
+define <2 x i32> @fcvtzs_v2f16_v2i32(<2 x half> %op1) #0 {
+; CHECK-LABEL: fcvtzs_v2f16_v2i32:
+; CHECK: fcvtl v0.4s, v0.4h
+; CHECK-NEXT: fcvtzs v0.4s, v0.4s
+; CHECK-NEXT: ret
+  %res = fptosi <2 x half> %op1 to <2 x i32>
+  ret <2 x i32> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <4 x i32> @fcvtzs_v4f16_v4i32(<4 x half> %op1) #0 {
+; CHECK-LABEL: fcvtzs_v4f16_v4i32:
+; CHECK: fcvtzs v0.4s, v0.4s
+; CHECK-NEXT: ret
+  %res = fptosi <4 x half> %op1 to <4 x i32>
+  ret <4 x i32> %res
+}
+
+define void @fcvtzs_v8f16_v8i32(<8 x half>* %a, <8 x i32>* %b) #0 {
+; CHECK-LABEL: fcvtzs_v8f16_v8i32:
+; CHECK: ldr q[[OP:[0-9]+]], [x0]
+; CHECK-NEXT: ptrue [[PG:p[0-9]+]].s, vl8
+; CHECK-NEXT: uunpklo [[UPK:z[0-9]+]].s, z[[OP]].h
+; CHECK-NEXT: fcvtzs [[RES:z[0-9]+]].s, [[PG]]/m, [[UPK]].h
+; CHECK-NEXT: st1w { [[RES]].s }, [[PG]], [x1]
+; CHECK-NEXT: ret
+  %op1 = load <8 x half>, <8 x half>* %a
+  %res = fptosi <8 x half> %op1 to <8 x i32>
+  store <8 x i32> %res, <8 x i32>* %b
+  ret void
+}
+
+define void @fcvtzs_v16f16_v16i32(<16 x half>* %a, <16 x i32>* %b) #0 {
+; CHECK-LABEL: fcvtzs_v16f16_v16i32:
+; VBITS_GE_512: ptrue [[PG1:p[0-9]+]].h, vl16
+; VBITS_GE_512-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG1]]/z, [x0]
+; VBITS_GE_512-NEXT: ptrue [[PG2:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: uunpklo [[UPK:z[0-9]+]].s, [[OP]].h
+; VBITS_GE_512-NEXT: fcvtzs [[RES:z[0-9]+]].s, [[PG2]]/m, [[UPK]].h
+; VBITS_GE_512-NEXT: st1w { [[RES]].s }, [[PG1]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <16 x half>, <16 x half>* %a
+  %res = fptosi <16 x half> %op1 to <16 x i32>
+  store <16 x i32> %res, <16 x i32>* %b
+  ret void
+}
+
+define void @fcvtzs_v32f16_v32i32(<32 x half>* %a, <32 x i32>* %b) #0 {
+; CHECK-LABEL: fcvtzs_v32f16_v32i32:
+; VBITS_GE_1024: ptrue [[PG1:p[0-9]+]].h, vl32
+; VBITS_GE_1024-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG1]]/z, [x0]
+; VBITS_GE_1024-NEXT: ptrue [[PG2:p[0-9]+]].s, vl32
+; VBITS_GE_1024-NEXT: uunpklo [[UPK:z[0-9]+]].s, [[OP]].h
+; VBITS_GE_1024-NEXT: fcvtzs [[RES:z[0-9]+]].s, [[PG2]]/m, [[UPK]].h
+; VBITS_GE_1024-NEXT: st1w { [[RES]].s }, [[PG1]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <32 x half>, <32 x half>* %a
+  %res = fptosi <32 x half> %op1 to <32 x i32>
+  store <32 x i32> %res, <32 x i32>* %b
+  ret void
+}
+
+define void @fcvtzs_v64f16_v64i32(<64 x half>* %a, <64 x i32>* %b) #0 {
+; CHECK-LABEL: fcvtzs_v64f16_v64i32:
+; VBITS_GE_2048: ptrue [[PG1:p[0-9]+]].h, vl64
+; VBITS_GE_2048-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG1]]/z, [x0]
+; VBITS_GE_2048-NEXT: ptrue [[PG2:p[0-9]+]].s, vl64
+; VBITS_GE_2048-NEXT: uunpklo [[UPK:z[0-9]+]].s, [[OP]].h
+; VBITS_GE_2048-NEXT: fcvtzs [[RES:z[0-9]+]].s, [[PG2]]/m, [[UPK]].h
+; VBITS_GE_2048-NEXT: st1w { [[RES]].s }, [[PG1]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <64 x half>, <64 x half>* %a
+  %res = fptosi <64 x half> %op1 to <64 x i32>
+  store <64 x i32> %res, <64 x i32>* %b
+  ret void
+}
+
+;
+; FCVTZS H -> D
+;
+
+; Don't use SVE for 64-bit vectors.
+define <1 x i64> @fcvtzs_v1f16_v1i64(<1 x half> %op1) #0 {
+; CHECK-LABEL: fcvtzs_v1f16_v1i64:
+; CHECK: fcvtzs x8, h0
+; CHECK-NEXT: fmov d0, x8
+; CHECK-NEXT: ret
+  %res = fptosi <1 x half> %op1 to <1 x i64>
+  ret <1 x i64> %res
+}
+
+; v2f16 is not legal for NEON, so use SVE
+define <2 x i64> @fcvtzs_v2f16_v2i64(<2 x half> %op1) #0 {
+; CHECK-LABEL: fcvtzs_v2f16_v2i64:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: uunpklo [[UPK1:z[0-9]+]].s, z0.h
+; CHECK-NEXT: uunpklo [[UPK2:z[0-9]+]].d, [[UPK1]].s
+; CHECK-NEXT: fcvtzs z0.d, [[PG]]/m, [[UPK2]].h
+; CHECK-NEXT: ret
+  %res = fptosi <2 x half> %op1 to <2 x i64>
+  ret <2 x i64> %res
+}
+
+define void @fcvtzs_v4f16_v4i64(<4 x half>* %a, <4 x i64>* %b) #0 {
+; CHECK-LABEL: fcvtzs_v4f16_v4i64:
+; CHECK: ldr d[[OP:[0-9]+]], [x0]
+; CHECK-NEXT: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: uunpklo [[UPK1:z[0-9]+]].s, z[[OP]].h
+; CHECK-NEXT: uunpklo [[UPK2:z[0-9]+]].d, [[UPK1]].s
+; CHECK-NEXT: fcvtzs [[RES:z[0-9]+]].d, [[PG]]/m, [[UPK2]].h
+; CHECK-NEXT: st1d { [[RES]].d }, [[PG]], [x1]
+; CHECK-NEXT: ret
+  %op1 = load <4 x half>, <4 x half>* %a
+  %res = fptosi <4 x half> %op1 to <4 x i64>
+  store <4 x i64> %res, <4 x i64>* %b
+  ret void
+}
+
+define void @fcvtzs_v8f16_v8i64(<8 x half>* %a, <8 x i64>* %b) #0 {
+; CHECK-LABEL: fcvtzs_v8f16_v8i64:
+; VBITS_GE_512: ldr q[[OP:[0-9]+]], [x0]
+; VBITS_GE_512-NEXT: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: uunpklo [[UPK1:z[0-9]+]].s, z[[OP]].h
+; VBITS_GE_512-NEXT: uunpklo [[UPK2:z[0-9]+]].d, [[UPK1]].s
+; VBITS_GE_512-NEXT: fcvtzs [[RES:z[0-9]+]].d, [[PG]]/m, [[UPK2]].h
+; VBITS_GE_512-NEXT: st1d { [[RES]].d }, [[PG]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <8 x half>, <8 x half>* %a
+  %res = fptosi <8 x half> %op1 to <8 x i64>
+  store <8 x i64> %res, <8 x i64>* %b
+  ret void
+}
+
+define void @fcvtzs_v16f16_v16i64(<16 x half>* %a, <16 x i64>* %b) #0 {
+; CHECK-LABEL: fcvtzs_v16f16_v16i64:
+; VBITS_GE_1024: ptrue [[PG1:p[0-9]+]].h, vl16
+; VBITS_GE_1024-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG1]]/z, [x0]
+; VBITS_GE_1024-NEXT: ptrue [[PG2:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: uunpklo [[UPK1:z[0-9]+]].s, [[OP]].h
+; VBITS_GE_1024-NEXT: uunpklo [[UPK2:z[0-9]+]].d, [[UPK1]].s
+; VBITS_GE_1024-NEXT: fcvtzs [[RES:z[0-9]+]].d, [[PG2]]/m, [[UPK2]].h
+; VBITS_GE_1024-NEXT: st1d { [[RES]].d }, [[PG1]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <16 x half>, <16 x half>* %a
+  %res = fptosi <16 x half> %op1 to <16 x i64>
+  store <16 x i64> %res, <16 x i64>* %b
+  ret void
+}
+
+define void @fcvtzs_v32f16_v32i64(<32 x half>* %a, <32 x i64>* %b) #0 {
+; CHECK-LABEL: fcvtzs_v32f16_v32i64:
+; VBITS_GE_2048: ptrue [[PG1:p[0-9]+]].h, vl32
+; VBITS_GE_2048-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG1]]/z, [x0]
+; VBITS_GE_2048-NEXT: ptrue [[PG2:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: uunpklo [[UPK1:z[0-9]+]].s, [[OP]].h
+; VBITS_GE_2048-NEXT: uunpklo [[UPK2:z[0-9]+]].d, [[UPK]].s
+; VBITS_GE_2048-NEXT: fcvtzs [[RES:z[0-9]+]].d, [[PG2]]/m, [[UPK2]].h
+; VBITS_GE_2048-NEXT: st1d { [[RES]].d }, [[PG1]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <32 x half>, <32 x half>* %a
+  %res = fptosi <32 x half> %op1 to <32 x i64>
+  store <32 x i64> %res, <32 x i64>* %b
+  ret void
+}
+
+;
+; FCVTZS S -> H
+;
+
+; Don't use SVE for 64-bit vectors.
+define <2 x i16> @fcvtzs_v2f32_v2i16(<2 x float> %op1) #0 {
+; CHECK-LABEL: fcvtzs_v2f32_v2i16:
+; CHECK: fcvtzs v0.2s, v0.2s
+; CHECK-NEXT: ret
+  %res = fptosi <2 x float> %op1 to <2 x i16>
+  ret <2 x i16> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <4 x i16> @fcvtzs_v4f32_v4i16(<4 x float> %op1) #0 {
+; CHECK-LABEL: fcvtzs_v4f32_v4i16:
+; CHECK: fcvtzs v0.4s, v0.4s
+; CHECK-NEXT: mov w8, v0.s[1]
+; CHECK-NEXT: mov w9, v0.s[2]
+; CHECK-NEXT: mov w10, v0.s[3]
+; CHECK-NEXT: mov v0.h[1], w8
+; CHECK-NEXT: mov v0.h[2], w9
+; CHECK-NEXT: mov v0.h[3], w10
+; CHECK-NEXT: ret
+  %res = fptosi <4 x float> %op1 to <4 x i16>
+  ret <4 x i16> %res
+}
+
+define <8 x i16> @fcvtzs_v8f32_v8i16(<8 x float>* %a) #0 {
+; CHECK-LABEL: fcvtzs_v8f32_v8i16:
+; CHECK: ptrue [[PG1:p[0-9]+]].s, vl8
+; CHECK-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG1]]/z, [x0]
+; CHECK-NEXT: ptrue [[PG2:p[0-9]+]].s
+; CHECK-NEXT: fcvtzs [[CVT:z[0-9]+]].s, [[PG2]]/m, [[OP]].s
+; CHECK-NEXT: uzp1 z0.h, [[CVT]].h, [[CVT]].h
+; CHECK-NEXT: ret
+  %op1 = load <8 x float>, <8 x float>* %a
+  %res = fptosi <8 x float> %op1 to <8 x i16>
+  ret <8 x i16> %res
+}
+
+define void @fcvtzs_v16f32_v16i16(<16 x float>* %a, <16 x i16>* %b) #0 {
+; CHECK-LABEL: fcvtzs_v16f32_v16i16:
+; VBITS_GE_512: ptrue [[PG1:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG1]]/z, [x0]
+; VBITS_GE_512-NEXT: ptrue [[PG2:p[0-9]+]].s
+; VBITS_GE_512-NEXT: fcvtzs [[CVT:z[0-9]+]].s, [[PG2]]/m, [[OP]].s
+; VBITS_GE_512-NEXT: uzp1 [[RES:z[0-9]+]].h, [[CVT]].h, [[CVT]].h
+; VBITS_GE_512-NEXT: ptrue [[PG3:p[0-9]+]].h, vl16
+; VBITS_GE_512-NEXT: st1h { [[RES]].h }, [[PG3]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <16 x float>, <16 x float>* %a
+  %res = fptosi <16 x float> %op1 to <16 x i16>
+  store <16 x i16> %res, <16 x i16>* %b
+  ret void
+}
+
+define void @fcvtzs_v32f32_v32i16(<32 x float>* %a, <32 x i16>* %b) #0 {
+; CHECK-LABEL: fcvtzs_v32f32_v32i16:
+; VBITS_GE_1024: ptrue [[PG1:p[0-9]+]].s, vl32
+; VBITS_GE_1024-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG1]]/z, [x0]
+; VBITS_GE_1024-NEXT: ptrue [[PG2:p[0-9]+]].s
+; VBITS_GE_1024-NEXT: fcvtzs [[CVT:z[0-9]+]].s, [[PG2]]/m, [[OP]].s
+; VBITS_GE_1024-NEXT: uzp1 [[RES:z[0-9]+]].h, [[CVT]].h, [[CVT]].h
+; VBITS_GE_1024-NEXT: ptrue [[PG3:p[0-9]+]].h, vl32
+; VBITS_GE_1024-NEXT: st1h { [[RES]].h }, [[PG3]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <32 x float>, <32 x float>* %a
+  %res = fptosi <32 x float> %op1 to <32 x i16>
+  store <32 x i16> %res, <32 x i16>* %b
+  ret void
+}
+
+define void @fcvtzs_v64f32_v64i16(<64 x float>* %a, <64 x i16>* %b) #0 {
+; CHECK-LABEL: fcvtzs_v64f32_v64i16:
+; VBITS_GE_2048: ptrue [[PG1:p[0-9]+]].s, vl64
+; VBITS_GE_2048-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG1]]/z, [x0]
+; VBITS_GE_2048-NEXT: ptrue [[PG2:p[0-9]+]].s
+; VBITS_GE_2048-NEXT: fcvtzs [[RES:z[0-9]+]].s, [[PG2]]/m, [[UPK]].s
+; VBITS_GE_2048-NEXT: uzp1 [[RES:z[0-9]+]].h, [[CVT]].h, [[CVT]].h
+; VBITS_GE_2048-NEXT: ptrue [[PG3:p[0-9]+]].h, vl64
+; VBITS_GE_2048-NEXT: st1h { [[RES]].h }, [[PG3]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <64 x float>, <64 x float>* %a
+  %res = fptosi <64 x float> %op1 to <64 x i16>
+  store <64 x i16> %res, <64 x i16>* %b
+  ret void
+}
+
+;
+; FCVTZS S -> S
+;
+
+; Don't use SVE for 64-bit vectors.
+define <2 x i32> @fcvtzs_v2f32_v2i32(<2 x float> %op1) #0 {
+; CHECK-LABEL: fcvtzs_v2f32_v2i32:
+; CHECK: fcvtzs v0.2s, v0.2s
+; CHECK-NEXT: ret
+  %res = fptosi <2 x float> %op1 to <2 x i32>
+  ret <2 x i32> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <4 x i32> @fcvtzs_v4f32_v4i32(<4 x float> %op1) #0 {
+; CHECK-LABEL: fcvtzs_v4f32_v4i32:
+; CHECK: fcvtzs v0.4s, v0.4s
+; CHECK-NEXT: ret
+  %res = fptosi <4 x float> %op1 to <4 x i32>
+  ret <4 x i32> %res
+}
+
+define void @fcvtzs_v8f32_v8i32(<8 x float>* %a, <8 x i32>* %b) #0 {
+; CHECK-LABEL: fcvtzs_v8f32_v8i32:
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl8
+; CHECK-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-NEXT: fcvtzs [[RES:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; CHECK-NEXT: st1w { [[RES]].s }, [[PG]], [x1]
+; CHECK-NEXT: ret
+  %op1 = load <8 x float>, <8 x float>* %a
+  %res = fptosi <8 x float> %op1 to <8 x i32>
+  store <8 x i32> %res, <8 x i32>* %b
+  ret void
+}
+
+define void @fcvtzs_v16f32_v16i32(<16 x float>* %a, <16 x i32>* %b) #0 {
+; CHECK-LABEL: fcvtzs_v16f32_v16i32:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_512-NEXT: fcvtzs [[RES:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; VBITS_GE_512-NEXT: st1w { [[RES]].s }, [[PG]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <16 x float>, <16 x float>* %a
+  %res = fptosi <16 x float> %op1 to <16 x i32>
+  store <16 x i32> %res, <16 x i32>* %b
+  ret void
+}
+
+define void @fcvtzs_v32f32_v32i32(<32 x float>* %a, <32 x i32>* %b) #0 {
+; CHECK-LABEL: fcvtzs_v32f32_v32i32:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].s, vl32
+; VBITS_GE_1024-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_1024-NEXT: fcvtzs [[RES:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; VBITS_GE_1024-NEXT: st1w { [[RES]].s }, [[PG]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <32 x float>, <32 x float>* %a
+  %res = fptosi <32 x float> %op1 to <32 x i32>
+  store <32 x i32> %res, <32 x i32>* %b
+  ret void
+}
+
+define void @fcvtzs_v64f32_v64i32(<64 x float>* %a, <64 x i32>* %b) #0 {
+; CHECK-LABEL: fcvtzs_v64f32_v64i32:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].s, vl64
+; VBITS_GE_2048-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_2048-NEXT: fcvtzs [[RES:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; VBITS_GE_2048-NEXT: st1w { [[RES]].s }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <64 x float>, <64 x float>* %a
+  %res = fptosi <64 x float> %op1 to <64 x i32>
+  store <64 x i32> %res, <64 x i32>* %b
+  ret void
+}
+
+;
+; FCVTZS S -> D
+;
+
+; Don't use SVE for 64-bit vectors.
+define <1 x i64> @fcvtzs_v1f32_v1i64(<1 x float> %op1) #0 {
+; CHECK-LABEL: fcvtzs_v1f32_v1i64:
+; CHECK: fcvtl v0.2d, v0.2s
+; CHECK-NEXT: fcvtzs v0.2d, v0.2d
+; CHECK-NEXT: ret
+  %res = fptosi <1 x float> %op1 to <1 x i64>
+  ret <1 x i64> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <2 x i64> @fcvtzs_v2f32_v2i64(<2 x float> %op1) #0 {
+; CHECK-LABEL: fcvtzs_v2f32_v2i64:
+; CHECK: fcvtl v0.2d, v0.2s
+; CHECK-NEXT: fcvtzs v0.2d, v0.2d
+; CHECK-NEXT: ret
+  %res = fptosi <2 x float> %op1 to <2 x i64>
+  ret <2 x i64> %res
+}
+
+define void @fcvtzs_v4f32_v4i64(<4 x float>* %a, <4 x i64>* %b) #0 {
+; CHECK-LABEL: fcvtzs_v4f32_v4i64:
+; CHECK: ldr q[[OP:[0-9]+]], [x0]
+; CHECK-NEXT: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: uunpklo [[UPK:z[0-9]+]].d, z[[OP]].s
+; CHECK-NEXT: fcvtzs [[RES:z[0-9]+]].d, [[PG]]/m, [[UPK]].s
+; CHECK-NEXT: st1d { [[RES]].d }, [[PG]], [x1]
+; CHECK-NEXT: ret
+  %op1 = load <4 x float>, <4 x float>* %a
+  %res = fptosi <4 x float> %op1 to <4 x i64>
+  store <4 x i64> %res, <4 x i64>* %b
+  ret void
+}
+
+define void @fcvtzs_v8f32_v8i64(<8 x float>* %a, <8 x i64>* %b) #0 {
+; CHECK-LABEL: fcvtzs_v8f32_v8i64:
+; VBITS_GE_512: ptrue [[PG1:p[0-9]+]].s, vl8
+; VBITS_GE_512-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG1]]/z, [x0]
+; VBITS_GE_512-NEXT: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: uunpklo [[UPK:z[0-9]+]].d, [[OP]].s
+; VBITS_GE_512-NEXT: fcvtzs [[RES:z[0-9]+]].d, [[PG1]]/m, [[UPK]].s
+; VBITS_GE_512-NEXT: st1d { [[RES]].d }, [[PG1]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <8 x float>, <8 x float>* %a
+  %res = fptosi <8 x float> %op1 to <8 x i64>
+  store <8 x i64> %res, <8 x i64>* %b
+  ret void
+}
+
+define void @fcvtzs_v16f32_v16i64(<16 x float>* %a, <16 x i64>* %b) #0 {
+; CHECK-LABEL: fcvtzs_v16f32_v16i64:
+; VBITS_GE_1024: ptrue [[PG1:p[0-9]+]].s, vl16
+; VBITS_GE_1024-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG1]]/z, [x0]
+; VBITS_GE_1024-NEXT: ptrue [[PG2:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: uunpklo [[UPK:z[0-9]+]].d, [[OP]].s
+; VBITS_GE_1024-NEXT: fcvtzs [[RES:z[0-9]+]].d, [[PG2]]/m, [[UPK]].s
+; VBITS_GE_1024-NEXT: st1d { [[RES]].d }, [[PG1]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <16 x float>, <16 x float>* %a
+  %res = fptosi <16 x float> %op1 to <16 x i64>
+  store <16 x i64> %res, <16 x i64>* %b
+  ret void
+}
+
+define void @fcvtzs_v32f32_v32i64(<32 x float>* %a, <32 x i64>* %b) #0 {
+; CHECK-LABEL: fcvtzs_v32f32_v32i64:
+; VBITS_GE_2048: ptrue [[PG1:p[0-9]+]].s, vl32
+; VBITS_GE_2048-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG1]]/z, [x0]
+; VBITS_GE_2048-NEXT: ptrue [[PG2:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: uunpklo [[UPK:z[0-9]+]].d, [[OP]].s
+; VBITS_GE_2048-NEXT: fcvtzs [[RES:z[0-9]+]].d, [[PG2]]/m, [[UPK]].s
+; VBITS_GE_2048-NEXT: st1d { [[RES]].d }, [[PG1]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <32 x float>, <32 x float>* %a
+  %res = fptosi <32 x float> %op1 to <32 x i64>
+  store <32 x i64> %res, <32 x i64>* %b
+  ret void
+}
+
+
+;
+; FCVTZS D -> H
+;
+
+; v1f64 is perfered to be widened to v4f64, so use SVE
+define <1 x i16> @fcvtzs_v1f64_v1i16(<1 x double> %op1) #0 {
+; CHECK-LABEL: fcvtzs_v1f64_v1i16:
+; CHECK: ptrue [[PG:p[0-9]+]].d
+; CHECK-NEXT: fcvtzs [[CVT:z[0-9]+]].d, [[PG]]/m, z0.d
+; CHECK-NEXT: uzp1 [[UZP:z[0-9]+]].s, [[CVT]].s, [[CVT]].s
+; CHECK-NEXT: uzp1 z0.h, [[UZP]].h, [[UZP]].h
+; CHECK-NEXT: ret
+  %res = fptosi <1 x double> %op1 to <1 x i16>
+  ret <1 x i16> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <2 x i16> @fcvtzs_v2f64_v2i16(<2 x double> %op1) #0 {
+; CHECK-LABEL: fcvtzs_v2f64_v2i16:
+; CHECK: fcvtzs v0.2d, v0.2d
+; CHECK-NEXT: xtn v0.2s, v0.2d
+; CHECK-NEXT: ret
+  %res = fptosi <2 x double> %op1 to <2 x i16>
+  ret <2 x i16> %res
+}
+
+define <4 x i16> @fcvtzs_v4f64_v4i16(<4 x double>* %a) #0 {
+; CHECK-LABEL: fcvtzs_v4f64_v4i16:
+; CHECK: ptrue [[PG1:p[0-9]+]].d, vl4
+; CHECK-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG1]]/z, [x0]
+; CHECK-NEXT: ptrue [[PG2:p[0-9]+]].d
+; CHECK-NEXT: fcvtzs [[CVT:z[0-9]+]].d, [[PG2]]/m, [[OP]].d
+; CHECK-NEXT: uzp1 [[UZP:z[0-9]+]].s, [[CVT]].s, [[CVT]].s
+; CHECK-NEXT: uzp1 z0.h, [[UZP]].h, [[UZP]].h
+; CHECK-NEXT: ret
+  %op1 = load <4 x double>, <4 x double>* %a
+  %res = fptosi <4 x double> %op1 to <4 x i16>
+  ret <4 x i16> %res
+}
+
+define <8 x i16> @fcvtzs_v8f64_v8i16(<8 x double>* %a) #0 {
+; CHECK-LABEL: fcvtzs_v8f64_v8i16:
+; VBITS_GE_512: ptrue [[PG1:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG1]]/z, [x0]
+; VBITS_GE_512-NEXT: ptrue [[PG2:p[0-9]+]].d
+; VBITS_GE_512-NEXT: fcvtzs [[CVT:z[0-9]+]].d, [[PG2]]/m, [[OP]].d
+; VBITS_GE_512-NEXT: uzp1 [[UZP:z[0-9]+]].s, [[CVT]].s, [[CVT]].s
+; VBITS_GE_512-NEXT: uzp1 z0.h, [[UZP]].h, [[UZP]].h
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <8 x double>, <8 x double>* %a
+  %res = fptosi <8 x double> %op1 to <8 x i16>
+  ret <8 x i16> %res
+}
+
+define void @fcvtzs_v16f64_v16i16(<16 x double>* %a, <16 x i16>* %b) #0 {
+; CHECK-LABEL: fcvtzs_v16f64_v16i16:
+; VBITS_GE_1024: ptrue [[PG1:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG1]]/z, [x0]
+; VBITS_GE_1024-NEXT: ptrue [[PG2:p[0-9]+]].d
+; VBITS_GE_1024-NEXT: fcvtzs [[CVT:z[0-9]+]].d, [[PG2]]/m, [[OP]].d
+; VBITS_GE_1024-NEXT: uzp1 [[UZP:z[0-9]+]].s, [[CVT]].s, [[CVT]].s
+; VBITS_GE_1024-NEXT: uzp1 [[RES:z[0-9]+]].h, [[UZP]].h, [[UZP]].h
+; VBITS_GE_1024-NEXT: ptrue [[PG3:p[0-9]+]].h, vl16
+; VBITS_GE_1024-NEXT: st1h { [[RES]].h }, [[PG3]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <16 x double>, <16 x double>* %a
+  %res = fptosi <16 x double> %op1 to <16 x i16>
+  store <16 x i16> %res, <16 x i16>* %b
+  ret void
+}
+
+define void @fcvtzs_v32f64_v32i16(<32 x double>* %a, <32 x i16>* %b) #0 {
+; CHECK-LABEL: fcvtzs_v32f64_v32i16:
+; VBITS_GE_2048: ptrue [[PG1:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG1]]/z, [x0]
+; VBITS_GE_2048-NEXT: ptrue [[PG2:p[0-9]+]].d
+; VBITS_GE_2048-NEXT: fcvtzs [[CVT:z[0-9]+]].d, [[PG2]]/m, [[OP]].d
+; VBITS_GE_2048-NEXT: uzp1 [[UZP:z[0-9]+]].s, [[CVT]].s, [[CVT]].s
+; VBITS_GE_2048-NEXT: uzp1 [[RES:z[0-9]+]].h, [[UZP]].h, [[UZP]].h
+; VBITS_GE_2048-NEXT: ptrue [[PG3:p[0-9]+]].h, vl32
+; VBITS_GE_2048-NEXT: st1h { [[RES]].h }, [[PG3]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <32 x double>, <32 x double>* %a
+  %res = fptosi <32 x double> %op1 to <32 x i16>
+  store <32 x i16> %res, <32 x i16>* %b
+  ret void
+}
+
+;
+; FCVTZS D -> S
+;
+
+; Don't use SVE for 64-bit vectors.
+define <1 x i32> @fcvtzs_v1f64_v1i32(<1 x double> %op1) #0 {
+; CHECK-LABEL: fcvtzs_v1f64_v1i32:
+; CHECK: fcvtzs v0.2d, v0.2d
+; CHECK-NEXT: xtn v0.2s, v0.2d
+; CHECK-NEXT: ret
+  %res = fptosi <1 x double> %op1 to <1 x i32>
+  ret <1 x i32> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <2 x i32> @fcvtzs_v2f64_v2i32(<2 x double> %op1) #0 {
+; CHECK-LABEL: fcvtzs_v2f64_v2i32:
+; CHECK: fcvtzs v0.2d, v0.2d
+; CHECK-NEXT: xtn v0.2s, v0.2d
+; CHECK-NEXT: ret
+  %res = fptosi <2 x double> %op1 to <2 x i32>
+  ret <2 x i32> %res
+}
+
+define <4 x i32> @fcvtzs_v4f64_v4i32(<4 x double>* %a) #0 {
+; CHECK-LABEL: fcvtzs_v4f64_v4i32:
+; CHECK: ptrue [[PG1:p[0-9]+]].d, vl4
+; CHECK-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG1]]/z, [x0]
+; CHECK-NEXT: ptrue [[PG2:p[0-9]+]].d
+; CHECK-NEXT: fcvtzs [[CVT:z[0-9]+]].d, [[PG2]]/m, [[OP]].d
+; CHECK-NEXT: uzp1 z0.s, [[CVT]].s, [[CVT]].s
+; CHECK-NEXT: ret
+  %op1 = load <4 x double>, <4 x double>* %a
+  %res = fptosi <4 x double> %op1 to <4 x i32>
+  ret <4 x i32> %res
+}
+
+define void @fcvtzs_v8f64_v8i32(<8 x double>* %a, <8 x i32>* %b) #0 {
+; CHECK-LABEL: fcvtzs_v8f64_v8i32:
+; VBITS_GE_512: ptrue [[PG1:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG1]]/z, [x0]
+; VBITS_GE_512-NEXT: ptrue [[PG2:p[0-9]+]].d
+; VBITS_GE_512-NEXT: fcvtzs [[CVT:z[0-9]+]].d, [[PG2]]/m, [[OP]].d
+; VBITS_GE_512-NEXT: uzp1 [[RES:z[0-9]+]].s, [[CVT]].s, [[CVT]].s
+; VBITS_GE_512-NEXT: ptrue [[PG3:p[0-9]+]].s, vl8
+; VBITS_GE_512-NEXT: st1w { [[RES]].s }, [[PG3]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <8 x double>, <8 x double>* %a
+  %res = fptosi <8 x double> %op1 to <8 x i32>
+  store <8 x i32> %res, <8 x i32>* %b
+  ret void
+}
+
+define void @fcvtzs_v16f64_v16i32(<16 x double>* %a, <16 x i32>* %b) #0 {
+; CHECK-LABEL: fcvtzs_v16f64_v16i32:
+; VBITS_GE_1024: ptrue [[PG1:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG1]]/z, [x0]
+; VBITS_GE_1024-NEXT: ptrue [[PG2:p[0-9]+]].d
+; VBITS_GE_1024-NEXT: fcvtzs [[CVT:z[0-9]+]].d, [[PG2]]/m, [[OP]].d
+; VBITS_GE_1024-NEXT: uzp1 [[RES:z[0-9]+]].s, [[CVT]].s, [[CVT]].s
+; VBITS_GE_1024-NEXT: ptrue [[PG3:p[0-9]+]].s, vl16
+; VBITS_GE_1024-NEXT: st1w { [[RES]].s }, [[PG3]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <16 x double>, <16 x double>* %a
+  %res = fptosi <16 x double> %op1 to <16 x i32>
+  store <16 x i32> %res, <16 x i32>* %b
+  ret void
+}
+
+define void @fcvtzs_v32f64_v32i32(<32 x double>* %a, <32 x i32>* %b) #0 {
+; CHECK-LABEL: fcvtzs_v32f64_v32i32:
+; VBITS_GE_2048: ptrue [[PG1:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG1]]/z, [x0]
+; VBITS_GE_2048-NEXT: ptrue [[PG2:p[0-9]+]].d
+; VBITS_GE_2048-NEXT: fcvtzs [[CVT:z[0-9]+]].d, [[PG2]]/m, [[OP]].d
+; VBITS_GE_2048-NEXT: uzp1 [[RES:z[0-9]+]].s, [[CVT]].s, [[CVT]].s
+; VBITS_GE_2048-NEXT: ptrue [[PG3:p[0-9]+]].s, vl32
+; VBITS_GE_2048-NEXT: st1w { [[RES]].s }, [[PG3]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <32 x double>, <32 x double>* %a
+  %res = fptosi <32 x double> %op1 to <32 x i32>
+  store <32 x i32> %res, <32 x i32>* %b
+  ret void
+}
+
+;
+; FCVTZS D -> D
+;
+
+; Don't use SVE for 64-bit vectors.
+define <1 x i64> @fcvtzs_v1f64_v1i64(<1 x double> %op1) #0 {
+; CHECK-LABEL: fcvtzs_v1f64_v1i64:
+; CHECK: fcvtzs x8, d0
+; CHECK: fmov d0, x8
+; CHECK-NEXT: ret
+  %res = fptosi <1 x double> %op1 to <1 x i64>
+  ret <1 x i64> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <2 x i64> @fcvtzs_v2f64_v2i64(<2 x double> %op1) #0 {
+; CHECK-LABEL: fcvtzs_v2f64_v2i64:
+; CHECK: fcvtzs v0.2d, v0.2d
+; CHECK-NEXT: ret
+  %res = fptosi <2 x double> %op1 to <2 x i64>
+  ret <2 x i64> %res
+}
+
+define void @fcvtzs_v4f64_v4i64(<4 x double>* %a, <4 x i64>* %b) #0 {
+; CHECK-LABEL: fcvtzs_v4f64_v4i64:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-NEXT: fcvtzs [[RES:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; CHECK-NEXT: st1d { [[RES]].d }, [[PG]], [x1]
+; CHECK-NEXT: ret
+  %op1 = load <4 x double>, <4 x double>* %a
+  %res = fptosi <4 x double> %op1 to <4 x i64>
+  store <4 x i64> %res, <4 x i64>* %b
+  ret void
+}
+
+define void @fcvtzs_v8f64_v8i64(<8 x double>* %a, <8 x i64>* %b) #0 {
+; CHECK-LABEL: fcvtzs_v8f64_v8i64:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_512-NEXT: fcvtzs [[RES:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; VBITS_GE_512-NEXT: st1d { [[RES]].d }, [[PG]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <8 x double>, <8 x double>* %a
+  %res = fptosi <8 x double> %op1 to <8 x i64>
+  store <8 x i64> %res, <8 x i64>* %b
+  ret void
+}
+
+define void @fcvtzs_v16f64_v16i64(<16 x double>* %a, <16 x i64>* %b) #0 {
+; CHECK-LABEL: fcvtzs_v16f64_v16i64:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_1024-NEXT: fcvtzs [[RES:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; VBITS_GE_1024-NEXT: st1d { [[RES]].d }, [[PG]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <16 x double>, <16 x double>* %a
+  %res = fptosi <16 x double> %op1 to <16 x i64>
+  store <16 x i64> %res, <16 x i64>* %b
+  ret void
+}
+
+define void @fcvtzs_v32f64_v32i64(<32 x double>* %a, <32 x i64>* %b) #0 {
+; CHECK-LABEL: fcvtzs_v32f64_v32i64:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_2048-NEXT: fcvtzs [[RES:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; VBITS_GE_2048-NEXT: st1d { [[RES]].d }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <32 x double>, <32 x double>* %a
+  %res = fptosi <32 x double> %op1 to <32 x i64>
+  store <32 x i64> %res, <32 x i64>* %b
+  ret void
+}
+
+attributes #0 = { "target-features"="+sve" }

llvm/test/CodeGen/AArch64/sve-fixed-length-int-to-fp.ll

+; RUN: llc -aarch64-sve-vector-bits-min=128  -asm-verbose=0 < %s | FileCheck %s -check-prefix=NO_SVE
+; RUN: llc -aarch64-sve-vector-bits-min=256  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK
+; RUN: llc -aarch64-sve-vector-bits-min=384  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK
+; RUN: llc -aarch64-sve-vector-bits-min=512  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=640  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=768  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=896  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=1024 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1152 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1280 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1408 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1536 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1664 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1792 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1920 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=2048 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024,VBITS_GE_2048
+
+target triple = "aarch64-unknown-linux-gnu"
+
+; Don't use SVE when its registers are no bigger than NEON.
+; NO_SVE-NOT: ptrue
+
+;
+; UCVTF H -> H
+;
+
+; Don't use SVE for 64-bit vectors.
+define <4 x half> @ucvtf_v4i16_v4f16(<4 x i16> %op1) #0 {
+; CHECK-LABEL: ucvtf_v4i16_v4f16:
+; CHECK: ucvtf v0.4h, v0.4h
+; CHECK-NEXT: ret
+  %res = uitofp <4 x i16> %op1 to <4 x half>
+  ret <4 x half> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define void @ucvtf_v8i16_v8f16(<8 x i16>* %a, <8 x half>* %b) #0 {
+; CHECK-LABEL: ucvtf_v8i16_v8f16:
+; CHECK: ldr q0, [x0]
+; CHECK-NEXT: ucvtf v0.8h, v0.8h
+; CHECK-NEXT: str q0, [x1]
+; CHECK-NEXT: ret
+  %op1 = load <8 x i16>, <8 x i16>* %a
+  %res = uitofp <8 x i16> %op1 to <8 x half>
+  store <8 x half> %res, <8 x half>* %b
+  ret void
+}
+
+define void @ucvtf_v16i16_v16f16(<16 x i16>* %a, <16 x half>* %b) #0 {
+; CHECK-LABEL: ucvtf_v16i16_v16f16:
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl16
+; CHECK-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK-NEXT: ucvtf [[RES:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; CHECK-NEXT: st1h { [[RES]].h }, [[PG]], [x1]
+; CHECK-NEXT: ret
+  %op1 = load <16 x i16>, <16 x i16>* %a
+  %res = uitofp <16 x i16> %op1 to <16 x half>
+  store <16 x half> %res, <16 x half>* %b
+  ret void
+}
+
+define void @ucvtf_v32i16_v32f16(<32 x i16>* %a, <32 x half>* %b) #0 {
+; CHECK-LABEL: ucvtf_v32i16_v32f16:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].h, vl32
+; VBITS_GE_512-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; VBITS_GE_512-NEXT: ucvtf [[RES:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; VBITS_GE_512-NEXT: st1h { [[RES]].h }, [[PG]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <32 x i16>, <32 x i16>* %a
+  %res = uitofp <32 x i16> %op1 to <32 x half>
+  store <32 x half> %res, <32 x half>* %b
+  ret void
+}
+
+define void @ucvtf_v64i16_v64f16(<64 x i16>* %a, <64 x half>* %b) #0 {
+; CHECK-LABEL: ucvtf_v64i16_v64f16:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].h, vl64
+; VBITS_GE_1024-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; VBITS_GE_1024-NEXT: ucvtf [[RES:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; VBITS_GE_1024-NEXT: st1h { [[RES]].h }, [[PG]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <64 x i16>, <64 x i16>* %a
+  %res = uitofp <64 x i16> %op1 to <64 x half>
+  store <64 x half> %res, <64 x half>* %b
+  ret void
+}
+
+define void @ucvtf_v128i16_v128f16(<128 x i16>* %a, <128 x half>* %b) #0 {
+; CHECK-LABEL: ucvtf_v128i16_v128f16:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].h, vl128
+; VBITS_GE_2048-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; VBITS_GE_2048-NEXT: ucvtf [[RES:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; VBITS_GE_2048-NEXT: st1h { [[RES]].h }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <128 x i16>, <128 x i16>* %a
+  %res = uitofp <128 x i16> %op1 to <128 x half>
+  store <128 x half> %res, <128 x half>* %b
+  ret void
+}
+
+;
+; UCVTF H -> S
+;
+
+; Don't use SVE for 64-bit vectors.
+define <2 x float> @ucvtf_v2i16_v2f32(<2 x i16> %op1) #0 {
+; CHECK-LABEL: ucvtf_v2i16_v2f32:
+; CHECK: movi d1, #0x00ffff0000ffff
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: ucvtf v0.2s, v0.2s
+; CHECK-NEXT: ret
+  %res = uitofp <2 x i16> %op1 to <2 x float>
+  ret <2 x float> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <4 x float> @ucvtf_v4i16_v4f32(<4 x i16> %op1) #0 {
+; CHECK-LABEL: ucvtf_v4i16_v4f32:
+; CHECK: ucvtf v0.4s, v0.4s
+; CHECK-NEXT: ret
+  %res = uitofp <4 x i16> %op1 to <4 x float>
+  ret <4 x float> %res
+}
+
+define void @ucvtf_v8i16_v8f32(<8 x i16>* %a, <8 x float>* %b) #0 {
+; CHECK-LABEL: ucvtf_v8i16_v8f32:
+; CHECK: ldr q[[OP:[0-9]+]], [x0]
+; CHECK-NEXT: ptrue [[PG:p[0-9]+]].s, vl8
+; CHECK-NEXT: uunpklo [[UPK:z[0-9]+]].s, z[[OP]].h
+; CHECK-NEXT: ucvtf [[RES:z[0-9]+]].s, [[PG]]/m, [[UPK]].s
+; CHECK-NEXT: st1w { [[RES]].s }, [[PG]], [x1]
+; CHECK-NEXT: ret
+  %op1 = load <8 x i16>, <8 x i16>* %a
+  %res = uitofp <8 x i16> %op1 to <8 x float>
+  store <8 x float> %res, <8 x float>* %b
+  ret void
+}
+
+define void @ucvtf_v16i16_v16f32(<16 x i16>* %a, <16 x float>* %b) #0 {
+; CHECK-LABEL: ucvtf_v16i16_v16f32:
+; VBITS_GE_512: ptrue [[PG1:p[0-9]+]].h, vl16
+; VBITS_GE_512-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG1]]/z, [x0]
+; VBITS_GE_512-NEXT: ptrue [[PG2:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: uunpklo [[UPK:z[0-9]+]].s, [[OP]].h
+; VBITS_GE_512-NEXT: ucvtf [[RES:z[0-9]+]].s, [[PG2]]/m, [[UPK]].s
+; VBITS_GE_512-NEXT: st1w { [[RES]].s }, [[PG1]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <16 x i16>, <16 x i16>* %a
+  %res = uitofp <16 x i16> %op1 to <16 x float>
+  store <16 x float> %res, <16 x float>* %b
+  ret void
+}
+
+define void @ucvtf_v32i16_v32f32(<32 x i16>* %a, <32 x float>* %b) #0 {
+; CHECK-LABEL: ucvtf_v32i16_v32f32:
+; VBITS_GE_1024: ptrue [[PG1:p[0-9]+]].h, vl32
+; VBITS_GE_1024-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG1]]/z, [x0]
+; VBITS_GE_1024-NEXT: ptrue [[PG2:p[0-9]+]].s, vl32
+; VBITS_GE_1024-NEXT: uunpklo [[UPK:z[0-9]+]].s, [[OP]].h
+; VBITS_GE_1024-NEXT: ucvtf [[RES:z[0-9]+]].s, [[PG2]]/m, [[UPK]].s
+; VBITS_GE_1024-NEXT: st1w { [[RES]].s }, [[PG1]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <32 x i16>, <32 x i16>* %a
+  %res = uitofp <32 x i16> %op1 to <32 x float>
+  store <32 x float> %res, <32 x float>* %b
+  ret void
+}
+
+define void @ucvtf_v64i16_v64f32(<64 x i16>* %a, <64 x float>* %b) #0 {
+; CHECK-LABEL: ucvtf_v64i16_v64f32:
+; VBITS_GE_2048: ptrue [[PG1:p[0-9]+]].h, vl64
+; VBITS_GE_2048-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG1]]/z, [x0]
+; VBITS_GE_2048-NEXT: ptrue [[PG2:p[0-9]+]].s, vl64
+; VBITS_GE_2048-NEXT: uunpklo [[UPK:z[0-9]+]].s, [[OP]].h
+; VBITS_GE_2048-NEXT: ucvtf [[RES:z[0-9]+]].s, [[PG2]]/m, [[UPK]].s
+; VBITS_GE_2048-NEXT: st1w { [[RES]].s }, [[PG1]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <64 x i16>, <64 x i16>* %a
+  %res = uitofp <64 x i16> %op1 to <64 x float>
+  store <64 x float> %res, <64 x float>* %b
+  ret void
+}
+
+;
+; UCVTF H -> D
+;
+
+; v1i16 is perfered to be widened to v4i16, which pushes the output into SVE types, so use SVE
+define <1 x double> @ucvtf_v1i16_v1f64(<1 x i16> %op1) #0 {
+; CHECK-LABEL: ucvtf_v1i16_v1f64:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: uunpklo [[UPK1:z[0-9]+]].s, z0.h
+; CHECK-NEXT: uunpklo [[UPK2:z[0-9]+]].d, [[UPK1]].s
+; CHECK-NEXT: ucvtf z0.d, [[PG]]/m, [[UPK2]].d
+; CHECK-NEXT: ret
+  %res = uitofp <1 x i16> %op1 to <1 x double>
+  ret <1 x double> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <2 x double> @ucvtf_v2i16_v2f64(<2 x i16> %op1) #0 {
+; CHECK-LABEL: ucvtf_v2i16_v2f64:
+; CHECK: movi d1, #0x00ffff0000ffff
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-NEXT: ret
+  %res = uitofp <2 x i16> %op1 to <2 x double>
+  ret <2 x double> %res
+}
+
+define void @ucvtf_v4i16_v4f64(<4 x i16>* %a, <4 x double>* %b) #0 {
+; CHECK-LABEL: ucvtf_v4i16_v4f64:
+; CHECK: ldr d[[OP:[0-9]+]], [x0]
+; CHECK-NEXT: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: uunpklo [[UPK1:z[0-9]+]].s, z[[OP]].h
+; CHECK-NEXT: uunpklo [[UPK2:z[0-9]+]].d, [[UPK1]].s
+; CHECK-NEXT: ucvtf [[RES:z[0-9]+]].d, [[PG]]/m, [[UPK2]].d
+; CHECK-NEXT: st1d { [[RES]].d }, [[PG]], [x1]
+; CHECK-NEXT: ret
+  %op1 = load <4 x i16>, <4 x i16>* %a
+  %res = uitofp <4 x i16> %op1 to <4 x double>
+  store <4 x double> %res, <4 x double>* %b
+  ret void
+}
+
+define void @ucvtf_v8i16_v8f64(<8 x i16>* %a, <8 x double>* %b) #0 {
+; CHECK-LABEL: ucvtf_v8i16_v8f64:
+; VBITS_GE_512: ldr q[[OP:[0-9]+]], [x0]
+; VBITS_GE_512-NEXT: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: uunpklo [[UPK1:z[0-9]+]].s, z[[OP]].h
+; VBITS_GE_512-NEXT: uunpklo [[UPK2:z[0-9]+]].d, [[UPK1]].s
+; VBITS_GE_512-NEXT: ucvtf [[RES:z[0-9]+]].d, [[PG]]/m, [[UPK2]].d
+; VBITS_GE_512-NEXT: st1d { [[RES]].d }, [[PG]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <8 x i16>, <8 x i16>* %a
+  %res = uitofp <8 x i16> %op1 to <8 x double>
+  store <8 x double> %res, <8 x double>* %b
+  ret void
+}
+
+define void @ucvtf_v16i16_v16f64(<16 x i16>* %a, <16 x double>* %b) #0 {
+; CHECK-LABEL: ucvtf_v16i16_v16f64:
+; VBITS_GE_1024: ptrue [[PG1:p[0-9]+]].h, vl16
+; VBITS_GE_1024-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG1]]/z, [x0]
+; VBITS_GE_1024-NEXT: ptrue [[PG2:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: uunpklo [[UPK1:z[0-9]+]].s, [[OP]].h
+; VBITS_GE_1024-NEXT: uunpklo [[UPK2:z[0-9]+]].d, [[UPK1]].s
+; VBITS_GE_1024-NEXT: ucvtf [[RES:z[0-9]+]].d, [[PG2]]/m, [[UPK2]].d
+; VBITS_GE_1024-NEXT: st1d { [[RES]].d }, [[PG1]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <16 x i16>, <16 x i16>* %a
+  %res = uitofp <16 x i16> %op1 to <16 x double>
+  store <16 x double> %res, <16 x double>* %b
+  ret void
+}
+
+define void @ucvtf_v32i16_v32f64(<32 x i16>* %a, <32 x double>* %b) #0 {
+; CHECK-LABEL: ucvtf_v32i16_v32f64:
+; VBITS_GE_2048: ptrue [[PG1:p[0-9]+]].h, vl32
+; VBITS_GE_2048-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG1]]/z, [x0]
+; VBITS_GE_2048-NEXT: ptrue [[PG2:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: uunpklo [[UPK1:z[0-9]+]].s, [[OP]].h
+; VBITS_GE_2048-NEXT: uunpklo [[UPK2:z[0-9]+]].d, [[UPK]].s
+; VBITS_GE_2048-NEXT: ucvtf [[RES:z[0-9]+]].d, [[PG2]]/m, [[UPK2]].d
+; VBITS_GE_2048-NEXT: st1d { [[RES]].d }, [[PG1]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <32 x i16>, <32 x i16>* %a
+  %res = uitofp <32 x i16> %op1 to <32 x double>
+  store <32 x double> %res, <32 x double>* %b
+  ret void
+}
+
+;
+; UCVTF S -> H
+;
+
+; Don't use SVE for 64-bit vectors.
+define <2 x half> @ucvtf_v2i32_v2f16(<2 x i32> %op1) #0 {
+; CHECK-LABEL: ucvtf_v2i32_v2f16:
+; CHECK: ucvtf v0.4s, v0.4s
+; CHECK-NEXT: fcvtn v0.4h, v0.4s
+; CHECK-NEXT: ret
+  %res = uitofp <2 x i32> %op1 to <2 x half>
+  ret <2 x half> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <4 x half> @ucvtf_v4i32_v4f16(<4 x i32> %op1) #0 {
+; CHECK-LABEL: ucvtf_v4i32_v4f16:
+; CHECK: ucvtf v0.4s, v0.4s
+; CHECK-NEXT: fcvtn v0.4h, v0.4s
+; CHECK-NEXT: ret
+  %res = uitofp <4 x i32> %op1 to <4 x half>
+  ret <4 x half> %res
+}
+
+define <8 x half> @ucvtf_v8i32_v8f16(<8 x i32>* %a) #0 {
+; CHECK-LABEL: ucvtf_v8i32_v8f16:
+; CHECK: ptrue [[PG1:p[0-9]+]].s, vl8
+; CHECK-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG1]]/z, [x0]
+; CHECK-NEXT: ptrue [[PG2:p[0-9]+]].s
+; CHECK-NEXT: ucvtf [[CVT:z[0-9]+]].h, [[PG2]]/m, [[OP]].s
+; CHECK-NEXT: uzp1 z0.h, [[CVT]].h, [[CVT]].h
+; CHECK-NEXT: ret
+  %op1 = load <8 x i32>, <8 x i32>* %a
+  %res = uitofp <8 x i32> %op1 to <8 x half>
+  ret <8 x half> %res
+}
+
+define void @ucvtf_v16i32_v16f16(<16 x i32>* %a, <16 x half>* %b) #0 {
+; CHECK-LABEL: ucvtf_v16i32_v16f16:
+; VBITS_GE_512: ptrue [[PG1:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG1]]/z, [x0]
+; VBITS_GE_512-NEXT: ptrue [[PG2:p[0-9]+]].s
+; VBITS_GE_512-NEXT: ucvtf [[CVT:z[0-9]+]].h, [[PG2]]/m, [[OP]].s
+; VBITS_GE_512-NEXT: uzp1 [[RES:z[0-9]+]].h, [[CVT]].h, [[CVT]].h
+; VBITS_GE_512-NEXT: ptrue [[PG3:p[0-9]+]].h, vl16
+; VBITS_GE_512-NEXT: st1h { [[RES]].h }, [[PG3]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <16 x i32>, <16 x i32>* %a
+  %res = uitofp <16 x i32> %op1 to <16 x half>
+  store <16 x half> %res, <16 x half>* %b
+  ret void
+}
+
+define void @ucvtf_v32i32_v32f16(<32 x i32>* %a, <32 x half>* %b) #0 {
+; CHECK-LABEL: ucvtf_v32i32_v32f16:
+; VBITS_GE_1024: ptrue [[PG1:p[0-9]+]].s, vl32
+; VBITS_GE_1024-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG1]]/z, [x0]
+; VBITS_GE_1024-NEXT: ptrue [[PG2:p[0-9]+]].s
+; VBITS_GE_1024-NEXT: ucvtf [[CVT:z[0-9]+]].h, [[PG2]]/m, [[OP]].s
+; VBITS_GE_1024-NEXT: uzp1 [[RES:z[0-9]+]].h, [[CVT]].h, [[CVT]].h
+; VBITS_GE_1024-NEXT: ptrue [[PG3:p[0-9]+]].h, vl32
+; VBITS_GE_1024-NEXT: st1h { [[RES]].h }, [[PG3]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <32 x i32>, <32 x i32>* %a
+  %res = uitofp <32 x i32> %op1 to <32 x half>
+  store <32 x half> %res, <32 x half>* %b
+  ret void
+}
+
+define void @ucvtf_v64i32_v64f16(<64 x i32>* %a, <64 x half>* %b) #0 {
+; CHECK-LABEL: ucvtf_v64i32_v64f16:
+; VBITS_GE_2048: ptrue [[PG1:p[0-9]+]].s, vl64
+; VBITS_GE_2048-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG1]]/z, [x0]
+; VBITS_GE_2048-NEXT: ptrue [[PG2:p[0-9]+]].s
+; VBITS_GE_2048-NEXT: ucvtf [[RES:z[0-9]+]].h, [[PG2]]/m, [[UPK]].s
+; VBITS_GE_2048-NEXT: uzp1 [[RES:z[0-9]+]].h, [[CVT]].h, [[CVT]].h
+; VBITS_GE_2048-NEXT: ptrue [[PG3:p[0-9]+]].h, vl64
+; VBITS_GE_2048-NEXT: st1h { [[RES]].h }, [[PG3]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <64 x i32>, <64 x i32>* %a
+  %res = uitofp <64 x i32> %op1 to <64 x half>
+  store <64 x half> %res, <64 x half>* %b
+  ret void
+}
+
+;
+; UCVTF S -> S
+;
+
+; Don't use SVE for 64-bit vectors.
+define <2 x float> @ucvtf_v2i32_v2f32(<2 x i32> %op1) #0 {
+; CHECK-LABEL: ucvtf_v2i32_v2f32:
+; CHECK: ucvtf v0.2s, v0.2s
+; CHECK-NEXT: ret
+  %res = uitofp <2 x i32> %op1 to <2 x float>
+  ret <2 x float> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <4 x float> @ucvtf_v4i32_v4f32(<4 x i32> %op1) #0 {
+; CHECK-LABEL: ucvtf_v4i32_v4f32:
+; CHECK: ucvtf v0.4s, v0.4s
+; CHECK-NEXT: ret
+  %res = uitofp <4 x i32> %op1 to <4 x float>
+  ret <4 x float> %res
+}
+
+define void @ucvtf_v8i32_v8f32(<8 x i32>* %a, <8 x float>* %b) #0 {
+; CHECK-LABEL: ucvtf_v8i32_v8f32:
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl8
+; CHECK-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-NEXT: ucvtf [[RES:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; CHECK-NEXT: st1w { [[RES]].s }, [[PG]], [x1]
+; CHECK-NEXT: ret
+  %op1 = load <8 x i32>, <8 x i32>* %a
+  %res = uitofp <8 x i32> %op1 to <8 x float>
+  store <8 x float> %res, <8 x float>* %b
+  ret void
+}
+
+define void @ucvtf_v16i32_v16f32(<16 x i32>* %a, <16 x float>* %b) #0 {
+; CHECK-LABEL: ucvtf_v16i32_v16f32:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_512-NEXT: ucvtf [[RES:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; VBITS_GE_512-NEXT: st1w { [[RES]].s }, [[PG]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <16 x i32>, <16 x i32>* %a
+  %res = uitofp <16 x i32> %op1 to <16 x float>
+  store <16 x float> %res, <16 x float>* %b
+  ret void
+}
+
+define void @ucvtf_v32i32_v32f32(<32 x i32>* %a, <32 x float>* %b) #0 {
+; CHECK-LABEL: ucvtf_v32i32_v32f32:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].s, vl32
+; VBITS_GE_1024-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_1024-NEXT: ucvtf [[RES:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; VBITS_GE_1024-NEXT: st1w { [[RES]].s }, [[PG]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <32 x i32>, <32 x i32>* %a
+  %res = uitofp <32 x i32> %op1 to <32 x float>
+  store <32 x float> %res, <32 x float>* %b
+  ret void
+}
+
+define void @ucvtf_v64i32_v64f32(<64 x i32>* %a, <64 x float>* %b) #0 {
+; CHECK-LABEL: ucvtf_v64i32_v64f32:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].s, vl64
+; VBITS_GE_2048-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_2048-NEXT: ucvtf [[RES:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; VBITS_GE_2048-NEXT: st1w { [[RES]].s }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <64 x i32>, <64 x i32>* %a
+  %res = uitofp <64 x i32> %op1 to <64 x float>
+  store <64 x float> %res, <64 x float>* %b
+  ret void
+}
+
+;
+; UCVTF S -> D
+;
+
+; Don't use SVE for 64-bit vectors.
+define <1 x double> @ucvtf_v1i32_v1f64(<1 x i32> %op1) #0 {
+; CHECK-LABEL: ucvtf_v1i32_v1f64:
+; CHECK: ushll v0.2d, v0.2s, #0
+; CHECK-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-NEXT: ret
+  %res = uitofp <1 x i32> %op1 to <1 x double>
+  ret <1 x double> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <2 x double> @ucvtf_v2i32_v2f64(<2 x i32> %op1) #0 {
+; CHECK-LABEL: ucvtf_v2i32_v2f64:
+; CHECK: ushll v0.2d, v0.2s, #0
+; CHECK-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-NEXT: ret
+  %res = uitofp <2 x i32> %op1 to <2 x double>
+  ret <2 x double> %res
+}
+
+define void @ucvtf_v4i32_v4f64(<4 x i32>* %a, <4 x double>* %b) #0 {
+; CHECK-LABEL: ucvtf_v4i32_v4f64:
+; CHECK: ldr q[[OP:[0-9]+]], [x0]
+; CHECK-NEXT: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: uunpklo [[UPK:z[0-9]+]].d, z[[OP]].s
+; CHECK-NEXT: ucvtf [[RES:z[0-9]+]].d, [[PG]]/m, [[UPK]].d
+; CHECK-NEXT: st1d { [[RES]].d }, [[PG]], [x1]
+; CHECK-NEXT: ret
+  %op1 = load <4 x i32>, <4 x i32>* %a
+  %res = uitofp <4 x i32> %op1 to <4 x double>
+  store <4 x double> %res, <4 x double>* %b
+  ret void
+}
+
+define void @ucvtf_v8i32_v8f64(<8 x i32>* %a, <8 x double>* %b) #0 {
+; CHECK-LABEL: ucvtf_v8i32_v8f64:
+; VBITS_GE_512: ptrue [[PG1:p[0-9]+]].s, vl8
+; VBITS_GE_512-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG1]]/z, [x0]
+; VBITS_GE_512-NEXT: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: uunpklo [[UPK:z[0-9]+]].d, [[OP]].s
+; VBITS_GE_512-NEXT: ucvtf [[RES:z[0-9]+]].d, [[PG1]]/m, [[UPK]].d
+; VBITS_GE_512-NEXT: st1d { [[RES]].d }, [[PG1]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <8 x i32>, <8 x i32>* %a
+  %res = uitofp <8 x i32> %op1 to <8 x double>
+  store <8 x double> %res, <8 x double>* %b
+  ret void
+}
+
+define void @ucvtf_v16i32_v16f64(<16 x i32>* %a, <16 x double>* %b) #0 {
+; CHECK-LABEL: ucvtf_v16i32_v16f64:
+; VBITS_GE_1024: ptrue [[PG1:p[0-9]+]].s, vl16
+; VBITS_GE_1024-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG1]]/z, [x0]
+; VBITS_GE_1024-NEXT: ptrue [[PG2:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: uunpklo [[UPK:z[0-9]+]].d, [[OP]].s
+; VBITS_GE_1024-NEXT: ucvtf [[RES:z[0-9]+]].d, [[PG2]]/m, [[UPK]].d
+; VBITS_GE_1024-NEXT: st1d { [[RES]].d }, [[PG1]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <16 x i32>, <16 x i32>* %a
+  %res = uitofp <16 x i32> %op1 to <16 x double>
+  store <16 x double> %res, <16 x double>* %b
+  ret void
+}
+
+define void @ucvtf_v32i32_v32f64(<32 x i32>* %a, <32 x double>* %b) #0 {
+; CHECK-LABEL: ucvtf_v32i32_v32f64:
+; VBITS_GE_2048: ptrue [[PG1:p[0-9]+]].s, vl32
+; VBITS_GE_2048-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG1]]/z, [x0]
+; VBITS_GE_2048-NEXT: ptrue [[PG2:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: uunpklo [[UPK:z[0-9]+]].d, [[OP]].s
+; VBITS_GE_2048-NEXT: ucvtf [[RES:z[0-9]+]].d, [[PG2]]/m, [[UPK]].d
+; VBITS_GE_2048-NEXT: st1d { [[RES]].d }, [[PG1]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <32 x i32>, <32 x i32>* %a
+  %res = uitofp <32 x i32> %op1 to <32 x double>
+  store <32 x double> %res, <32 x double>* %b
+  ret void
+}
+
+
+;
+; UCVTF D -> H
+;
+
+; Don't use SVE for 64-bit vectors.
+define <1 x half> @ucvtf_v1i64_v1f16(<1 x i64> %op1) #0 {
+; CHECK-LABEL: ucvtf_v1i64_v1f16:
+; CHECK: fmov x8, d0
+; CHECK-NEXT: ucvtf h0, x8
+; CHECK-NEXT: ret
+  %res = uitofp <1 x i64> %op1 to <1 x half>
+  ret <1 x half> %res
+}
+
+; v2f16 is not legal for NEON, so use SVE
+define <2 x half> @ucvtf_v2i64_v2f16(<2 x i64> %op1) #0 {
+; CHECK-LABEL: ucvtf_v2i64_v2f16:
+; CHECK: ptrue [[PG:p[0-9]+]].d
+; CHECK-NEXT: ucvtf [[CVT:z[0-9]+]].h, [[PG]]/m, z0.d
+; CHECK-NEXT: uzp1 [[UZP:z[0-9]+]].s, [[CVT]].s, [[CVT]].s
+; CHECK-NEXT: uzp1 z0.h, [[UZP]].h, [[UZP]].h
+; CHECK-NEXT: ret
+  %res = uitofp <2 x i64> %op1 to <2 x half>
+  ret <2 x half> %res
+}
+
+define <4 x half> @ucvtf_v4i64_v4f16(<4 x i64>* %a) #0 {
+; CHECK-LABEL: ucvtf_v4i64_v4f16:
+; CHECK: ptrue [[PG1:p[0-9]+]].d, vl4
+; CHECK-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG1]]/z, [x0]
+; CHECK-NEXT: ptrue [[PG2:p[0-9]+]].d
+; CHECK-NEXT: ucvtf [[CVT:z[0-9]+]].h, [[PG2]]/m, [[OP]].d
+; CHECK-NEXT: uzp1 [[UZP:z[0-9]+]].s, [[CVT]].s, [[CVT]].s
+; CHECK-NEXT: uzp1 z0.h, [[UZP]].h, [[UZP]].h
+; CHECK-NEXT: ret
+  %op1 = load <4 x i64>, <4 x i64>* %a
+  %res = uitofp <4 x i64> %op1 to <4 x half>
+  ret <4 x half> %res
+}
+
+define <8 x half> @ucvtf_v8i64_v8f16(<8 x i64>* %a) #0 {
+; CHECK-LABEL: ucvtf_v8i64_v8f16:
+; VBITS_GE_512: ptrue [[PG1:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG1]]/z, [x0]
+; VBITS_GE_512-NEXT: ptrue [[PG2:p[0-9]+]].d
+; VBITS_GE_512-NEXT: ucvtf [[CVT:z[0-9]+]].h, [[PG2]]/m, [[OP]].d
+; VBITS_GE_512-NEXT: uzp1 [[UZP:z[0-9]+]].s, [[CVT]].s, [[CVT]].s
+; VBITS_GE_512-NEXT: uzp1 z0.h, [[UZP]].h, [[UZP]].h
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <8 x i64>, <8 x i64>* %a
+  %res = uitofp <8 x i64> %op1 to <8 x half>
+  ret <8 x half> %res
+}
+
+define void @ucvtf_v16i64_v16f16(<16 x i64>* %a, <16 x half>* %b) #0 {
+; CHECK-LABEL: ucvtf_v16i64_v16f16:
+; VBITS_GE_1024: ptrue [[PG1:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG1]]/z, [x0]
+; VBITS_GE_1024-NEXT: ptrue [[PG2:p[0-9]+]].d
+; VBITS_GE_1024-NEXT: ucvtf [[CVT:z[0-9]+]].h, [[PG2]]/m, [[OP]].d
+; VBITS_GE_1024-NEXT: uzp1 [[UZP:z[0-9]+]].s, [[CVT]].s, [[CVT]].s
+; VBITS_GE_1024-NEXT: uzp1 [[RES:z[0-9]+]].h, [[UZP]].h, [[UZP]].h
+; VBITS_GE_1024-NEXT: ptrue [[PG3:p[0-9]+]].h, vl16
+; VBITS_GE_1024-NEXT: st1h { [[RES]].h }, [[PG3]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <16 x i64>, <16 x i64>* %a
+  %res = uitofp <16 x i64> %op1 to <16 x half>
+  store <16 x half> %res, <16 x half>* %b
+  ret void
+}
+
+define void @ucvtf_v32i64_v32f16(<32 x i64>* %a, <32 x half>* %b) #0 {
+; CHECK-LABEL: ucvtf_v32i64_v32f16:
+; VBITS_GE_2048: ptrue [[PG1:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG1]]/z, [x0]
+; VBITS_GE_2048-NEXT: ptrue [[PG2:p[0-9]+]].d
+; VBITS_GE_2048-NEXT: ucvtf [[CVT:z[0-9]+]].h, [[PG2]]/m, [[OP]].d
+; VBITS_GE_2048-NEXT: uzp1 [[UZP:z[0-9]+]].s, [[CVT]].s, [[CVT]].s
+; VBITS_GE_2048-NEXT: uzp1 [[RES:z[0-9]+]].h, [[UZP]].h, [[UZP]].h
+; VBITS_GE_2048-NEXT: ptrue [[PG3:p[0-9]+]].h, vl32
+; VBITS_GE_2048-NEXT: st1h { [[RES]].h }, [[PG3]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <32 x i64>, <32 x i64>* %a
+  %res = uitofp <32 x i64> %op1 to <32 x half>
+  store <32 x half> %res, <32 x half>* %b
+  ret void
+}
+
+;
+; UCVTF D -> S
+;
+
+; Don't use SVE for 64-bit vectors.
+define <1 x float> @ucvtf_v1i64_v1f32(<1 x i64> %op1) #0 {
+; CHECK-LABEL: ucvtf_v1i64_v1f32:
+; CHECK: ucvtf v0.2d, v0.2d
+; CHECK-NEXT: fcvtn v0.2s, v0.2d
+; CHECK-NEXT: ret
+  %res = uitofp <1 x i64> %op1 to <1 x float>
+  ret <1 x float> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <2 x float> @ucvtf_v2i64_v2f32(<2 x i64> %op1) #0 {
+; CHECK-LABEL: ucvtf_v2i64_v2f32:
+; CHECK: ucvtf v0.2d, v0.2d
+; CHECK-NEXT: fcvtn v0.2s, v0.2d
+; CHECK-NEXT: ret
+  %res = uitofp <2 x i64> %op1 to <2 x float>
+  ret <2 x float> %res
+}
+
+define <4 x float> @ucvtf_v4i64_v4f32(<4 x i64>* %a) #0 {
+; CHECK-LABEL: ucvtf_v4i64_v4f32:
+; CHECK: ptrue [[PG1:p[0-9]+]].d, vl4
+; CHECK-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG1]]/z, [x0]
+; CHECK-NEXT: ptrue [[PG2:p[0-9]+]].d
+; CHECK-NEXT: ucvtf [[CVT:z[0-9]+]].s, [[PG2]]/m, [[OP]].d
+; CHECK-NEXT: uzp1 z0.s, [[CVT]].s, [[CVT]].s
+; CHECK-NEXT: ret
+  %op1 = load <4 x i64>, <4 x i64>* %a
+  %res = uitofp <4 x i64> %op1 to <4 x float>
+  ret <4 x float> %res
+}
+
+define void @ucvtf_v8i64_v8f32(<8 x i64>* %a, <8 x float>* %b) #0 {
+; CHECK-LABEL: ucvtf_v8i64_v8f32:
+; VBITS_GE_512: ptrue [[PG1:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG1]]/z, [x0]
+; VBITS_GE_512-NEXT: ptrue [[PG2:p[0-9]+]].d
+; VBITS_GE_512-NEXT: ucvtf [[CVT:z[0-9]+]].s, [[PG2]]/m, [[OP]].d
+; VBITS_GE_512-NEXT: uzp1 [[RES:z[0-9]+]].s, [[CVT]].s, [[CVT]].s
+; VBITS_GE_512-NEXT: ptrue [[PG3:p[0-9]+]].s, vl8
+; VBITS_GE_512-NEXT: st1w { [[RES]].s }, [[PG3]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <8 x i64>, <8 x i64>* %a
+  %res = uitofp <8 x i64> %op1 to <8 x float>
+  store <8 x float> %res, <8 x float>* %b
+  ret void
+}
+
+define void @ucvtf_v16i64_v16f32(<16 x i64>* %a, <16 x float>* %b) #0 {
+; CHECK-LABEL: ucvtf_v16i64_v16f32:
+; VBITS_GE_1024: ptrue [[PG1:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG1]]/z, [x0]
+; VBITS_GE_1024-NEXT: ptrue [[PG2:p[0-9]+]].d
+; VBITS_GE_1024-NEXT: ucvtf [[CVT:z[0-9]+]].s, [[PG2]]/m, [[OP]].d
+; VBITS_GE_1024-NEXT: uzp1 [[RES:z[0-9]+]].s, [[CVT]].s, [[CVT]].s
+; VBITS_GE_1024-NEXT: ptrue [[PG3:p[0-9]+]].s, vl16
+; VBITS_GE_1024-NEXT: st1w { [[RES]].s }, [[PG3]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <16 x i64>, <16 x i64>* %a
+  %res = uitofp <16 x i64> %op1 to <16 x float>
+  store <16 x float> %res, <16 x float>* %b
+  ret void
+}
+
+define void @ucvtf_v32i64_v32f32(<32 x i64>* %a, <32 x float>* %b) #0 {
+; CHECK-LABEL: ucvtf_v32i64_v32f32:
+; VBITS_GE_2048: ptrue [[PG1:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG1]]/z, [x0]
+; VBITS_GE_2048-NEXT: ptrue [[PG2:p[0-9]+]].d
+; VBITS_GE_2048-NEXT: ucvtf [[CVT:z[0-9]+]].s, [[PG2]]/m, [[OP]].d
+; VBITS_GE_2048-NEXT: uzp1 [[RES:z[0-9]+]].s, [[CVT]].s, [[CVT]].s
+; VBITS_GE_2048-NEXT: ptrue [[PG3:p[0-9]+]].s, vl32
+; VBITS_GE_2048-NEXT: st1w { [[RES]].s }, [[PG3]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <32 x i64>, <32 x i64>* %a
+  %res = uitofp <32 x i64> %op1 to <32 x float>
+  store <32 x float> %res, <32 x float>* %b
+  ret void
+}
+
+;
+; UCVTF D -> D
+;
+
+; Don't use SVE for 64-bit vectors.
+define <1 x double> @ucvtf_v1i64_v1f64(<1 x i64> %op1) #0 {
+; CHECK-LABEL: ucvtf_v1i64_v1f64:
+; CHECK: fmov x8, d0
+; CHECK-NEXT: ucvtf d0, x8
+; CHECK-NEXT: ret
+  %res = uitofp <1 x i64> %op1 to <1 x double>
+  ret <1 x double> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <2 x double> @ucvtf_v2i64_v2f64(<2 x i64> %op1) #0 {
+; CHECK-LABEL: ucvtf_v2i64_v2f64:
+; CHECK: ucvtf v0.2d, v0.2d
+; CHECK-NEXT: ret
+  %res = uitofp <2 x i64> %op1 to <2 x double>
+  ret <2 x double> %res
+}
+
+define void @ucvtf_v4i64_v4f64(<4 x i64>* %a, <4 x double>* %b) #0 {
+; CHECK-LABEL: ucvtf_v4i64_v4f64:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-NEXT: ucvtf [[RES:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; CHECK-NEXT: st1d { [[RES]].d }, [[PG]], [x1]
+; CHECK-NEXT: ret
+  %op1 = load <4 x i64>, <4 x i64>* %a
+  %res = uitofp <4 x i64> %op1 to <4 x double>
+  store <4 x double> %res, <4 x double>* %b
+  ret void
+}
+
+define void @ucvtf_v8i64_v8f64(<8 x i64>* %a, <8 x double>* %b) #0 {
+; CHECK-LABEL: ucvtf_v8i64_v8f64:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_512-NEXT: ucvtf [[RES:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; VBITS_GE_512-NEXT: st1d { [[RES]].d }, [[PG]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <8 x i64>, <8 x i64>* %a
+  %res = uitofp <8 x i64> %op1 to <8 x double>
+  store <8 x double> %res, <8 x double>* %b
+  ret void
+}
+
+define void @ucvtf_v16i64_v16f64(<16 x i64>* %a, <16 x double>* %b) #0 {
+; CHECK-LABEL: ucvtf_v16i64_v16f64:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_1024-NEXT: ucvtf [[RES:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; VBITS_GE_1024-NEXT: st1d { [[RES]].d }, [[PG]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <16 x i64>, <16 x i64>* %a
+  %res = uitofp <16 x i64> %op1 to <16 x double>
+  store <16 x double> %res, <16 x double>* %b
+  ret void
+}
+
+define void @ucvtf_v32i64_v32f64(<32 x i64>* %a, <32 x double>* %b) #0 {
+; CHECK-LABEL: ucvtf_v32i64_v32f64:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_2048-NEXT: ucvtf [[RES:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; VBITS_GE_2048-NEXT: st1d { [[RES]].d }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <32 x i64>, <32 x i64>* %a
+  %res = uitofp <32 x i64> %op1 to <32 x double>
+  store <32 x double> %res, <32 x double>* %b
+  ret void
+}
+
+;
+; SCVTF H -> H
+;
+
+; Don't use SVE for 64-bit vectors.
+define <4 x half> @scvtf_v4i16_v4f16(<4 x i16> %op1) #0 {
+; CHECK-LABEL: scvtf_v4i16_v4f16:
+; CHECK: scvtf v0.4h, v0.4h
+; CHECK-NEXT: ret
+  %res = sitofp <4 x i16> %op1 to <4 x half>
+  ret <4 x half> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define void @scvtf_v8i16_v8f16(<8 x i16>* %a, <8 x half>* %b) #0 {
+; CHECK-LABEL: scvtf_v8i16_v8f16:
+; CHECK: ldr q0, [x0]
+; CHECK-NEXT: scvtf v0.8h, v0.8h
+; CHECK-NEXT: str q0, [x1]
+; CHECK-NEXT: ret
+  %op1 = load <8 x i16>, <8 x i16>* %a
+  %res = sitofp <8 x i16> %op1 to <8 x half>
+  store <8 x half> %res, <8 x half>* %b
+  ret void
+}
+
+define void @scvtf_v16i16_v16f16(<16 x i16>* %a, <16 x half>* %b) #0 {
+; CHECK-LABEL: scvtf_v16i16_v16f16:
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl16
+; CHECK-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK-NEXT: scvtf [[RES:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; CHECK-NEXT: st1h { [[RES]].h }, [[PG]], [x1]
+; CHECK-NEXT: ret
+  %op1 = load <16 x i16>, <16 x i16>* %a
+  %res = sitofp <16 x i16> %op1 to <16 x half>
+  store <16 x half> %res, <16 x half>* %b
+  ret void
+}
+
+define void @scvtf_v32i16_v32f16(<32 x i16>* %a, <32 x half>* %b) #0 {
+; CHECK-LABEL: scvtf_v32i16_v32f16:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].h, vl32
+; VBITS_GE_512-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; VBITS_GE_512-NEXT: scvtf [[RES:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; VBITS_GE_512-NEXT: st1h { [[RES]].h }, [[PG]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <32 x i16>, <32 x i16>* %a
+  %res = sitofp <32 x i16> %op1 to <32 x half>
+  store <32 x half> %res, <32 x half>* %b
+  ret void
+}
+
+define void @scvtf_v64i16_v64f16(<64 x i16>* %a, <64 x half>* %b) #0 {
+; CHECK-LABEL: scvtf_v64i16_v64f16:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].h, vl64
+; VBITS_GE_1024-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; VBITS_GE_1024-NEXT: scvtf [[RES:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; VBITS_GE_1024-NEXT: st1h { [[RES]].h }, [[PG]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <64 x i16>, <64 x i16>* %a
+  %res = sitofp <64 x i16> %op1 to <64 x half>
+  store <64 x half> %res, <64 x half>* %b
+  ret void
+}
+
+define void @scvtf_v128i16_v128f16(<128 x i16>* %a, <128 x half>* %b) #0 {
+; CHECK-LABEL: scvtf_v128i16_v128f16:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].h, vl128
+; VBITS_GE_2048-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; VBITS_GE_2048-NEXT: scvtf [[RES:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; VBITS_GE_2048-NEXT: st1h { [[RES]].h }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <128 x i16>, <128 x i16>* %a
+  %res = sitofp <128 x i16> %op1 to <128 x half>
+  store <128 x half> %res, <128 x half>* %b
+  ret void
+}
+
+;
+; SCVTF H -> S
+;
+
+; Don't use SVE for 64-bit vectors.
+define <2 x float> @scvtf_v2i16_v2f32(<2 x i16> %op1) #0 {
+; CHECK-LABEL: scvtf_v2i16_v2f32:
+; CHECK: shl v0.2s, v0.2s, #16
+; CHECK-NEXT: sshr v0.2s, v0.2s, #16
+; CHECK-NEXT: scvtf v0.2s, v0.2s
+; CHECK-NEXT: ret
+  %res = sitofp <2 x i16> %op1 to <2 x float>
+  ret <2 x float> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <4 x float> @scvtf_v4i16_v4f32(<4 x i16> %op1) #0 {
+; CHECK-LABEL: scvtf_v4i16_v4f32:
+; CHECK: scvtf v0.4s, v0.4s
+; CHECK-NEXT: ret
+  %res = sitofp <4 x i16> %op1 to <4 x float>
+  ret <4 x float> %res
+}
+
+define void @scvtf_v8i16_v8f32(<8 x i16>* %a, <8 x float>* %b) #0 {
+; CHECK-LABEL: scvtf_v8i16_v8f32:
+; CHECK: ldr q[[OP:[0-9]+]], [x0]
+; CHECK-NEXT: ptrue [[PG:p[0-9]+]].s, vl8
+; CHECK-NEXT: sunpklo [[UPK:z[0-9]+]].s, z[[OP]].h
+; CHECK-NEXT: scvtf [[RES:z[0-9]+]].s, [[PG]]/m, [[UPK]].s
+; CHECK-NEXT: st1w { [[RES]].s }, [[PG]], [x1]
+; CHECK-NEXT: ret
+  %op1 = load <8 x i16>, <8 x i16>* %a
+  %res = sitofp <8 x i16> %op1 to <8 x float>
+  store <8 x float> %res, <8 x float>* %b
+  ret void
+}
+
+define void @scvtf_v16i16_v16f32(<16 x i16>* %a, <16 x float>* %b) #0 {
+; CHECK-LABEL: scvtf_v16i16_v16f32:
+; VBITS_GE_512: ptrue [[PG1:p[0-9]+]].h, vl16
+; VBITS_GE_512-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG1]]/z, [x0]
+; VBITS_GE_512-NEXT: ptrue [[PG2:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: sunpklo [[UPK:z[0-9]+]].s, [[OP]].h
+; VBITS_GE_512-NEXT: scvtf [[RES:z[0-9]+]].s, [[PG2]]/m, [[UPK]].s
+; VBITS_GE_512-NEXT: st1w { [[RES]].s }, [[PG1]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <16 x i16>, <16 x i16>* %a
+  %res = sitofp <16 x i16> %op1 to <16 x float>
+  store <16 x float> %res, <16 x float>* %b
+  ret void
+}
+
+define void @scvtf_v32i16_v32f32(<32 x i16>* %a, <32 x float>* %b) #0 {
+; CHECK-LABEL: scvtf_v32i16_v32f32:
+; VBITS_GE_1024: ptrue [[PG1:p[0-9]+]].h, vl32
+; VBITS_GE_1024-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG1]]/z, [x0]
+; VBITS_GE_1024-NEXT: ptrue [[PG2:p[0-9]+]].s, vl32
+; VBITS_GE_1024-NEXT: sunpklo [[UPK:z[0-9]+]].s, [[OP]].h
+; VBITS_GE_1024-NEXT: scvtf [[RES:z[0-9]+]].s, [[PG2]]/m, [[UPK]].s
+; VBITS_GE_1024-NEXT: st1w { [[RES]].s }, [[PG1]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <32 x i16>, <32 x i16>* %a
+  %res = sitofp <32 x i16> %op1 to <32 x float>
+  store <32 x float> %res, <32 x float>* %b
+  ret void
+}
+
+define void @scvtf_v64i16_v64f32(<64 x i16>* %a, <64 x float>* %b) #0 {
+; CHECK-LABEL: scvtf_v64i16_v64f32:
+; VBITS_GE_2048: ptrue [[PG1:p[0-9]+]].h, vl64
+; VBITS_GE_2048-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG1]]/z, [x0]
+; VBITS_GE_2048-NEXT: ptrue [[PG2:p[0-9]+]].s, vl64
+; VBITS_GE_2048-NEXT: sunpklo [[UPK:z[0-9]+]].s, [[OP]].h
+; VBITS_GE_2048-NEXT: scvtf [[RES:z[0-9]+]].s, [[PG2]]/m, [[UPK]].s
+; VBITS_GE_2048-NEXT: st1w { [[RES]].s }, [[PG1]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <64 x i16>, <64 x i16>* %a
+  %res = sitofp <64 x i16> %op1 to <64 x float>
+  store <64 x float> %res, <64 x float>* %b
+  ret void
+}
+
+;
+; SCVTF H -> D
+;
+
+; v1i16 is perfered to be widened to v4i16, which pushes the output into SVE types, so use SVE
+define <1 x double> @scvtf_v1i16_v1f64(<1 x i16> %op1) #0 {
+; CHECK-LABEL: scvtf_v1i16_v1f64:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: sunpklo [[UPK1:z[0-9]+]].s, z0.h
+; CHECK-NEXT: sunpklo [[UPK2:z[0-9]+]].d, [[UPK1]].s
+; CHECK-NEXT: scvtf z0.d, [[PG]]/m, [[UPK2]].d
+; CHECK-NEXT: ret
+  %res = sitofp <1 x i16> %op1 to <1 x double>
+  ret <1 x double> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <2 x double> @scvtf_v2i16_v2f64(<2 x i16> %op1) #0 {
+; CHECK-LABEL: scvtf_v2i16_v2f64:
+; CHECK: shl v0.2s, v0.2s, #16
+; CHECK-NEXT: sshr v0.2s, v0.2s, #16
+; CHECK-NEXT: sshll v0.2d, v0.2s, #0
+; CHECK-NEXT: scvtf v0.2d, v0.2d
+; CHECK-NEXT: ret
+  %res = sitofp <2 x i16> %op1 to <2 x double>
+  ret <2 x double> %res
+}
+
+define void @scvtf_v4i16_v4f64(<4 x i16>* %a, <4 x double>* %b) #0 {
+; CHECK-LABEL: scvtf_v4i16_v4f64:
+; CHECK: ldr d[[OP:[0-9]+]], [x0]
+; CHECK-NEXT: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: sunpklo [[UPK1:z[0-9]+]].s, z[[OP]].h
+; CHECK-NEXT: sunpklo [[UPK2:z[0-9]+]].d, [[UPK1]].s
+; CHECK-NEXT: scvtf [[RES:z[0-9]+]].d, [[PG]]/m, [[UPK2]].d
+; CHECK-NEXT: st1d { [[RES]].d }, [[PG]], [x1]
+; CHECK-NEXT: ret
+  %op1 = load <4 x i16>, <4 x i16>* %a
+  %res = sitofp <4 x i16> %op1 to <4 x double>
+  store <4 x double> %res, <4 x double>* %b
+  ret void
+}
+
+define void @scvtf_v8i16_v8f64(<8 x i16>* %a, <8 x double>* %b) #0 {
+; CHECK-LABEL: scvtf_v8i16_v8f64:
+; VBITS_GE_512: ldr q[[OP:[0-9]+]], [x0]
+; VBITS_GE_512-NEXT: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: sunpklo [[UPK1:z[0-9]+]].s, z[[OP]].h
+; VBITS_GE_512-NEXT: sunpklo [[UPK2:z[0-9]+]].d, [[UPK1]].s
+; VBITS_GE_512-NEXT: scvtf [[RES:z[0-9]+]].d, [[PG]]/m, [[UPK2]].d
+; VBITS_GE_512-NEXT: st1d { [[RES]].d }, [[PG]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <8 x i16>, <8 x i16>* %a
+  %res = sitofp <8 x i16> %op1 to <8 x double>
+  store <8 x double> %res, <8 x double>* %b
+  ret void
+}
+
+define void @scvtf_v16i16_v16f64(<16 x i16>* %a, <16 x double>* %b) #0 {
+; CHECK-LABEL: scvtf_v16i16_v16f64:
+; VBITS_GE_1024: ptrue [[PG1:p[0-9]+]].h, vl16
+; VBITS_GE_1024-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG1]]/z, [x0]
+; VBITS_GE_1024-NEXT: ptrue [[PG2:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: sunpklo [[UPK1:z[0-9]+]].s, [[OP]].h
+; VBITS_GE_1024-NEXT: sunpklo [[UPK2:z[0-9]+]].d, [[UPK1]].s
+; VBITS_GE_1024-NEXT: scvtf [[RES:z[0-9]+]].d, [[PG2]]/m, [[UPK2]].d
+; VBITS_GE_1024-NEXT: st1d { [[RES]].d }, [[PG1]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <16 x i16>, <16 x i16>* %a
+  %res = sitofp <16 x i16> %op1 to <16 x double>
+  store <16 x double> %res, <16 x double>* %b
+  ret void
+}
+
+define void @scvtf_v32i16_v32f64(<32 x i16>* %a, <32 x double>* %b) #0 {
+; CHECK-LABEL: scvtf_v32i16_v32f64:
+; VBITS_GE_2048: ptrue [[PG1:p[0-9]+]].h, vl32
+; VBITS_GE_2048-NEXT: ld1h { [[OP:z[0-9]+]].h }, [[PG1]]/z, [x0]
+; VBITS_GE_2048-NEXT: ptrue [[PG2:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: sunpklo [[UPK1:z[0-9]+]].s, [[OP]].h
+; VBITS_GE_2048-NEXT: sunpklo [[UPK2:z[0-9]+]].d, [[UPK]].s
+; VBITS_GE_2048-NEXT: scvtf [[RES:z[0-9]+]].d, [[PG2]]/m, [[UPK2]].d
+; VBITS_GE_2048-NEXT: st1d { [[RES]].d }, [[PG1]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <32 x i16>, <32 x i16>* %a
+  %res = sitofp <32 x i16> %op1 to <32 x double>
+  store <32 x double> %res, <32 x double>* %b
+  ret void
+}
+
+;
+; SCVTF S -> H
+;
+
+; Don't use SVE for 64-bit vectors.
+define <2 x half> @scvtf_v2i32_v2f16(<2 x i32> %op1) #0 {
+; CHECK-LABEL: scvtf_v2i32_v2f16:
+; CHECK: scvtf v0.4s, v0.4s
+; CHECK-NEXT: fcvtn v0.4h, v0.4s
+; CHECK-NEXT: ret
+  %res = sitofp <2 x i32> %op1 to <2 x half>
+  ret <2 x half> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <4 x half> @scvtf_v4i32_v4f16(<4 x i32> %op1) #0 {
+; CHECK-LABEL: scvtf_v4i32_v4f16:
+; CHECK: scvtf v0.4s, v0.4s
+; CHECK-NEXT: fcvtn v0.4h, v0.4s
+; CHECK-NEXT: ret
+  %res = sitofp <4 x i32> %op1 to <4 x half>
+  ret <4 x half> %res
+}
+
+define <8 x half> @scvtf_v8i32_v8f16(<8 x i32>* %a) #0 {
+; CHECK-LABEL: scvtf_v8i32_v8f16:
+; CHECK: ptrue [[PG1:p[0-9]+]].s, vl8
+; CHECK-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG1]]/z, [x0]
+; CHECK-NEXT: ptrue [[PG2:p[0-9]+]].s
+; CHECK-NEXT: scvtf [[CVT:z[0-9]+]].h, [[PG2]]/m, [[OP]].s
+; CHECK-NEXT: uzp1 z0.h, [[CVT]].h, [[CVT]].h
+; CHECK-NEXT: ret
+  %op1 = load <8 x i32>, <8 x i32>* %a
+  %res = sitofp <8 x i32> %op1 to <8 x half>
+  ret <8 x half> %res
+}
+
+define void @scvtf_v16i32_v16f16(<16 x i32>* %a, <16 x half>* %b) #0 {
+; CHECK-LABEL: scvtf_v16i32_v16f16:
+; VBITS_GE_512: ptrue [[PG1:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG1]]/z, [x0]
+; VBITS_GE_512-NEXT: ptrue [[PG2:p[0-9]+]].s
+; VBITS_GE_512-NEXT: scvtf [[CVT:z[0-9]+]].h, [[PG2]]/m, [[OP]].s
+; VBITS_GE_512-NEXT: uzp1 [[RES:z[0-9]+]].h, [[CVT]].h, [[CVT]].h
+; VBITS_GE_512-NEXT: ptrue [[PG3:p[0-9]+]].h, vl16
+; VBITS_GE_512-NEXT: st1h { [[RES]].h }, [[PG3]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <16 x i32>, <16 x i32>* %a
+  %res = sitofp <16 x i32> %op1 to <16 x half>
+  store <16 x half> %res, <16 x half>* %b
+  ret void
+}
+
+define void @scvtf_v32i32_v32f16(<32 x i32>* %a, <32 x half>* %b) #0 {
+; CHECK-LABEL: scvtf_v32i32_v32f16:
+; VBITS_GE_1024: ptrue [[PG1:p[0-9]+]].s, vl32
+; VBITS_GE_1024-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG1]]/z, [x0]
+; VBITS_GE_1024-NEXT: ptrue [[PG2:p[0-9]+]].s
+; VBITS_GE_1024-NEXT: scvtf [[CVT:z[0-9]+]].h, [[PG2]]/m, [[OP]].s
+; VBITS_GE_1024-NEXT: uzp1 [[RES:z[0-9]+]].h, [[CVT]].h, [[CVT]].h
+; VBITS_GE_1024-NEXT: ptrue [[PG3:p[0-9]+]].h, vl32
+; VBITS_GE_1024-NEXT: st1h { [[RES]].h }, [[PG3]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <32 x i32>, <32 x i32>* %a
+  %res = sitofp <32 x i32> %op1 to <32 x half>
+  store <32 x half> %res, <32 x half>* %b
+  ret void
+}
+
+define void @scvtf_v64i32_v64f16(<64 x i32>* %a, <64 x half>* %b) #0 {
+; CHECK-LABEL: scvtf_v64i32_v64f16:
+; VBITS_GE_2048: ptrue [[PG1:p[0-9]+]].s, vl64
+; VBITS_GE_2048-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG1]]/z, [x0]
+; VBITS_GE_2048-NEXT: ptrue [[PG2:p[0-9]+]].s
+; VBITS_GE_2048-NEXT: scvtf [[RES:z[0-9]+]].h, [[PG2]]/m, [[UPK]].s
+; VBITS_GE_2048-NEXT: uzp1 [[RES:z[0-9]+]].h, [[CVT]].h, [[CVT]].h
+; VBITS_GE_2048-NEXT: ptrue [[PG3:p[0-9]+]].h, vl64
+; VBITS_GE_2048-NEXT: st1h { [[RES]].h }, [[PG3]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <64 x i32>, <64 x i32>* %a
+  %res = sitofp <64 x i32> %op1 to <64 x half>
+  store <64 x half> %res, <64 x half>* %b
+  ret void
+}
+
+;
+; SCVTF S -> S
+;
+
+; Don't use SVE for 64-bit vectors.
+define <2 x float> @scvtf_v2i32_v2f32(<2 x i32> %op1) #0 {
+; CHECK-LABEL: scvtf_v2i32_v2f32:
+; CHECK: scvtf v0.2s, v0.2s
+; CHECK-NEXT: ret
+  %res = sitofp <2 x i32> %op1 to <2 x float>
+  ret <2 x float> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <4 x float> @scvtf_v4i32_v4f32(<4 x i32> %op1) #0 {
+; CHECK-LABEL: scvtf_v4i32_v4f32:
+; CHECK: scvtf v0.4s, v0.4s
+; CHECK-NEXT: ret
+  %res = sitofp <4 x i32> %op1 to <4 x float>
+  ret <4 x float> %res
+}
+
+define void @scvtf_v8i32_v8f32(<8 x i32>* %a, <8 x float>* %b) #0 {
+; CHECK-LABEL: scvtf_v8i32_v8f32:
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl8
+; CHECK-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-NEXT: scvtf [[RES:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; CHECK-NEXT: st1w { [[RES]].s }, [[PG]], [x1]
+; CHECK-NEXT: ret
+  %op1 = load <8 x i32>, <8 x i32>* %a
+  %res = sitofp <8 x i32> %op1 to <8 x float>
+  store <8 x float> %res, <8 x float>* %b
+  ret void
+}
+
+define void @scvtf_v16i32_v16f32(<16 x i32>* %a, <16 x float>* %b) #0 {
+; CHECK-LABEL: scvtf_v16i32_v16f32:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_512-NEXT: scvtf [[RES:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; VBITS_GE_512-NEXT: st1w { [[RES]].s }, [[PG]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <16 x i32>, <16 x i32>* %a
+  %res = sitofp <16 x i32> %op1 to <16 x float>
+  store <16 x float> %res, <16 x float>* %b
+  ret void
+}
+
+define void @scvtf_v32i32_v32f32(<32 x i32>* %a, <32 x float>* %b) #0 {
+; CHECK-LABEL: scvtf_v32i32_v32f32:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].s, vl32
+; VBITS_GE_1024-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_1024-NEXT: scvtf [[RES:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; VBITS_GE_1024-NEXT: st1w { [[RES]].s }, [[PG]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <32 x i32>, <32 x i32>* %a
+  %res = sitofp <32 x i32> %op1 to <32 x float>
+  store <32 x float> %res, <32 x float>* %b
+  ret void
+}
+
+define void @scvtf_v64i32_v64f32(<64 x i32>* %a, <64 x float>* %b) #0 {
+; CHECK-LABEL: scvtf_v64i32_v64f32:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].s, vl64
+; VBITS_GE_2048-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_2048-NEXT: scvtf [[RES:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; VBITS_GE_2048-NEXT: st1w { [[RES]].s }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <64 x i32>, <64 x i32>* %a
+  %res = sitofp <64 x i32> %op1 to <64 x float>
+  store <64 x float> %res, <64 x float>* %b
+  ret void
+}
+
+;
+; SCVTF S -> D
+;
+
+; Don't use SVE for 64-bit vectors.
+define <1 x double> @scvtf_v1i32_v1f64(<1 x i32> %op1) #0 {
+; CHECK-LABEL: scvtf_v1i32_v1f64:
+; CHECK: sshll v0.2d, v0.2s, #0
+; CHECK-NEXT: scvtf v0.2d, v0.2d
+; CHECK-NEXT: ret
+  %res = sitofp <1 x i32> %op1 to <1 x double>
+  ret <1 x double> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <2 x double> @scvtf_v2i32_v2f64(<2 x i32> %op1) #0 {
+; CHECK-LABEL: scvtf_v2i32_v2f64:
+; CHECK: sshll v0.2d, v0.2s, #0
+; CHECK-NEXT: scvtf v0.2d, v0.2d
+; CHECK-NEXT: ret
+  %res = sitofp <2 x i32> %op1 to <2 x double>
+  ret <2 x double> %res
+}
+
+define void @scvtf_v4i32_v4f64(<4 x i32>* %a, <4 x double>* %b) #0 {
+; CHECK-LABEL: scvtf_v4i32_v4f64:
+; CHECK: ldr q[[OP:[0-9]+]], [x0]
+; CHECK-NEXT: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: sunpklo [[UPK:z[0-9]+]].d, z[[OP]].s
+; CHECK-NEXT: scvtf [[RES:z[0-9]+]].d, [[PG]]/m, [[UPK]].d
+; CHECK-NEXT: st1d { [[RES]].d }, [[PG]], [x1]
+; CHECK-NEXT: ret
+  %op1 = load <4 x i32>, <4 x i32>* %a
+  %res = sitofp <4 x i32> %op1 to <4 x double>
+  store <4 x double> %res, <4 x double>* %b
+  ret void
+}
+
+define void @scvtf_v8i32_v8f64(<8 x i32>* %a, <8 x double>* %b) #0 {
+; CHECK-LABEL: scvtf_v8i32_v8f64:
+; VBITS_GE_512: ptrue [[PG1:p[0-9]+]].s, vl8
+; VBITS_GE_512-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG1]]/z, [x0]
+; VBITS_GE_512-NEXT: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: sunpklo [[UPK:z[0-9]+]].d, [[OP]].s
+; VBITS_GE_512-NEXT: scvtf [[RES:z[0-9]+]].d, [[PG1]]/m, [[UPK]].d
+; VBITS_GE_512-NEXT: st1d { [[RES]].d }, [[PG1]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <8 x i32>, <8 x i32>* %a
+  %res = sitofp <8 x i32> %op1 to <8 x double>
+  store <8 x double> %res, <8 x double>* %b
+  ret void
+}
+
+define void @scvtf_v16i32_v16f64(<16 x i32>* %a, <16 x double>* %b) #0 {
+; CHECK-LABEL: scvtf_v16i32_v16f64:
+; VBITS_GE_1024: ptrue [[PG1:p[0-9]+]].s, vl16
+; VBITS_GE_1024-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG1]]/z, [x0]
+; VBITS_GE_1024-NEXT: ptrue [[PG2:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: sunpklo [[UPK:z[0-9]+]].d, [[OP]].s
+; VBITS_GE_1024-NEXT: scvtf [[RES:z[0-9]+]].d, [[PG2]]/m, [[UPK]].d
+; VBITS_GE_1024-NEXT: st1d { [[RES]].d }, [[PG1]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <16 x i32>, <16 x i32>* %a
+  %res = sitofp <16 x i32> %op1 to <16 x double>
+  store <16 x double> %res, <16 x double>* %b
+  ret void
+}
+
+define void @scvtf_v32i32_v32f64(<32 x i32>* %a, <32 x double>* %b) #0 {
+; CHECK-LABEL: scvtf_v32i32_v32f64:
+; VBITS_GE_2048: ptrue [[PG1:p[0-9]+]].s, vl32
+; VBITS_GE_2048-NEXT: ld1w { [[OP:z[0-9]+]].s }, [[PG1]]/z, [x0]
+; VBITS_GE_2048-NEXT: ptrue [[PG2:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: sunpklo [[UPK:z[0-9]+]].d, [[OP]].s
+; VBITS_GE_2048-NEXT: scvtf [[RES:z[0-9]+]].d, [[PG2]]/m, [[UPK]].d
+; VBITS_GE_2048-NEXT: st1d { [[RES]].d }, [[PG1]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <32 x i32>, <32 x i32>* %a
+  %res = sitofp <32 x i32> %op1 to <32 x double>
+  store <32 x double> %res, <32 x double>* %b
+  ret void
+}
+
+
+;
+; SCVTF D -> H
+;
+
+; Don't use SVE for 64-bit vectors.
+define <1 x half> @scvtf_v1i64_v1f16(<1 x i64> %op1) #0 {
+; CHECK-LABEL: scvtf_v1i64_v1f16:
+; CHECK: fmov x8, d0
+; CHECK-NEXT: scvtf h0, x8
+; CHECK-NEXT: ret
+  %res = sitofp <1 x i64> %op1 to <1 x half>
+  ret <1 x half> %res
+}
+
+; v2f16 is not legal for NEON, so use SVE
+define <2 x half> @scvtf_v2i64_v2f16(<2 x i64> %op1) #0 {
+; CHECK-LABEL: scvtf_v2i64_v2f16:
+; CHECK: ptrue [[PG:p[0-9]+]].d
+; CHECK-NEXT: scvtf [[CVT:z[0-9]+]].h, [[PG]]/m, z0.d
+; CHECK-NEXT: uzp1 [[UZP:z[0-9]+]].s, [[CVT]].s, [[CVT]].s
+; CHECK-NEXT: uzp1 z0.h, [[UZP]].h, [[UZP]].h
+; CHECK-NEXT: ret
+  %res = sitofp <2 x i64> %op1 to <2 x half>
+  ret <2 x half> %res
+}
+
+define <4 x half> @scvtf_v4i64_v4f16(<4 x i64>* %a) #0 {
+; CHECK-LABEL: scvtf_v4i64_v4f16:
+; CHECK: ptrue [[PG1:p[0-9]+]].d, vl4
+; CHECK-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG1]]/z, [x0]
+; CHECK-NEXT: ptrue [[PG2:p[0-9]+]].d
+; CHECK-NEXT: scvtf [[CVT:z[0-9]+]].h, [[PG2]]/m, [[OP]].d
+; CHECK-NEXT: uzp1 [[UZP:z[0-9]+]].s, [[CVT]].s, [[CVT]].s
+; CHECK-NEXT: uzp1 z0.h, [[UZP]].h, [[UZP]].h
+; CHECK-NEXT: ret
+  %op1 = load <4 x i64>, <4 x i64>* %a
+  %res = sitofp <4 x i64> %op1 to <4 x half>
+  ret <4 x half> %res
+}
+
+define <8 x half> @scvtf_v8i64_v8f16(<8 x i64>* %a) #0 {
+; CHECK-LABEL: scvtf_v8i64_v8f16:
+; VBITS_GE_512: ptrue [[PG1:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG1]]/z, [x0]
+; VBITS_GE_512-NEXT: ptrue [[PG2:p[0-9]+]].d
+; VBITS_GE_512-NEXT: scvtf [[CVT:z[0-9]+]].h, [[PG2]]/m, [[OP]].d
+; VBITS_GE_512-NEXT: uzp1 [[UZP:z[0-9]+]].s, [[CVT]].s, [[CVT]].s
+; VBITS_GE_512-NEXT: uzp1 z0.h, [[UZP]].h, [[UZP]].h
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <8 x i64>, <8 x i64>* %a
+  %res = sitofp <8 x i64> %op1 to <8 x half>
+  ret <8 x half> %res
+}
+
+define void @scvtf_v16i64_v16f16(<16 x i64>* %a, <16 x half>* %b) #0 {
+; CHECK-LABEL: scvtf_v16i64_v16f16:
+; VBITS_GE_1024: ptrue [[PG1:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG1]]/z, [x0]
+; VBITS_GE_1024-NEXT: ptrue [[PG2:p[0-9]+]].d
+; VBITS_GE_1024-NEXT: scvtf [[CVT:z[0-9]+]].h, [[PG2]]/m, [[OP]].d
+; VBITS_GE_1024-NEXT: uzp1 [[UZP:z[0-9]+]].s, [[CVT]].s, [[CVT]].s
+; VBITS_GE_1024-NEXT: uzp1 [[RES:z[0-9]+]].h, [[UZP]].h, [[UZP]].h
+; VBITS_GE_1024-NEXT: ptrue [[PG3:p[0-9]+]].h, vl16
+; VBITS_GE_1024-NEXT: st1h { [[RES]].h }, [[PG3]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <16 x i64>, <16 x i64>* %a
+  %res = sitofp <16 x i64> %op1 to <16 x half>
+  store <16 x half> %res, <16 x half>* %b
+  ret void
+}
+
+define void @scvtf_v32i64_v32f16(<32 x i64>* %a, <32 x half>* %b) #0 {
+; CHECK-LABEL: scvtf_v32i64_v32f16:
+; VBITS_GE_2048: ptrue [[PG1:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG1]]/z, [x0]
+; VBITS_GE_2048-NEXT: ptrue [[PG2:p[0-9]+]].d
+; VBITS_GE_2048-NEXT: scvtf [[CVT:z[0-9]+]].h, [[PG2]]/m, [[OP]].d
+; VBITS_GE_2048-NEXT: uzp1 [[UZP:z[0-9]+]].s, [[CVT]].s, [[CVT]].s
+; VBITS_GE_2048-NEXT: uzp1 [[RES:z[0-9]+]].h, [[UZP]].h, [[UZP]].h
+; VBITS_GE_2048-NEXT: ptrue [[PG3:p[0-9]+]].h, vl32
+; VBITS_GE_2048-NEXT: st1h { [[RES]].h }, [[PG3]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <32 x i64>, <32 x i64>* %a
+  %res = sitofp <32 x i64> %op1 to <32 x half>
+  store <32 x half> %res, <32 x half>* %b
+  ret void
+}
+
+;
+; SCVTF D -> S
+;
+
+; Don't use SVE for 64-bit vectors.
+define <1 x float> @scvtf_v1i64_v1f32(<1 x i64> %op1) #0 {
+; CHECK-LABEL: scvtf_v1i64_v1f32:
+; CHECK: scvtf v0.2d, v0.2d
+; CHECK-NEXT: fcvtn v0.2s, v0.2d
+; CHECK-NEXT: ret
+  %res = sitofp <1 x i64> %op1 to <1 x float>
+  ret <1 x float> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <2 x float> @scvtf_v2i64_v2f32(<2 x i64> %op1) #0 {
+; CHECK-LABEL: scvtf_v2i64_v2f32:
+; CHECK: scvtf v0.2d, v0.2d
+; CHECK-NEXT: fcvtn v0.2s, v0.2d
+; CHECK-NEXT: ret
+  %res = sitofp <2 x i64> %op1 to <2 x float>
+  ret <2 x float> %res
+}
+
+define <4 x float> @scvtf_v4i64_v4f32(<4 x i64>* %a) #0 {
+; CHECK-LABEL: scvtf_v4i64_v4f32:
+; CHECK: ptrue [[PG1:p[0-9]+]].d, vl4
+; CHECK-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG1]]/z, [x0]
+; CHECK-NEXT: ptrue [[PG2:p[0-9]+]].d
+; CHECK-NEXT: scvtf [[CVT:z[0-9]+]].s, [[PG2]]/m, [[OP]].d
+; CHECK-NEXT: uzp1 z0.s, [[CVT]].s, [[CVT]].s
+; CHECK-NEXT: ret
+  %op1 = load <4 x i64>, <4 x i64>* %a
+  %res = sitofp <4 x i64> %op1 to <4 x float>
+  ret <4 x float> %res
+}
+
+define void @scvtf_v8i64_v8f32(<8 x i64>* %a, <8 x float>* %b) #0 {
+; CHECK-LABEL: scvtf_v8i64_v8f32:
+; VBITS_GE_512: ptrue [[PG1:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG1]]/z, [x0]
+; VBITS_GE_512-NEXT: ptrue [[PG2:p[0-9]+]].d
+; VBITS_GE_512-NEXT: scvtf [[CVT:z[0-9]+]].s, [[PG2]]/m, [[OP]].d
+; VBITS_GE_512-NEXT: uzp1 [[RES:z[0-9]+]].s, [[CVT]].s, [[CVT]].s
+; VBITS_GE_512-NEXT: ptrue [[PG3:p[0-9]+]].s, vl8
+; VBITS_GE_512-NEXT: st1w { [[RES]].s }, [[PG3]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <8 x i64>, <8 x i64>* %a
+  %res = sitofp <8 x i64> %op1 to <8 x float>
+  store <8 x float> %res, <8 x float>* %b
+  ret void
+}
+
+define void @scvtf_v16i64_v16f32(<16 x i64>* %a, <16 x float>* %b) #0 {
+; CHECK-LABEL: scvtf_v16i64_v16f32:
+; VBITS_GE_1024: ptrue [[PG1:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG1]]/z, [x0]
+; VBITS_GE_1024-NEXT: ptrue [[PG2:p[0-9]+]].d
+; VBITS_GE_1024-NEXT: scvtf [[CVT:z[0-9]+]].s, [[PG2]]/m, [[OP]].d
+; VBITS_GE_1024-NEXT: uzp1 [[RES:z[0-9]+]].s, [[CVT]].s, [[CVT]].s
+; VBITS_GE_1024-NEXT: ptrue [[PG3:p[0-9]+]].s, vl16
+; VBITS_GE_1024-NEXT: st1w { [[RES]].s }, [[PG3]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <16 x i64>, <16 x i64>* %a
+  %res = sitofp <16 x i64> %op1 to <16 x float>
+  store <16 x float> %res, <16 x float>* %b
+  ret void
+}
+
+define void @scvtf_v32i64_v32f32(<32 x i64>* %a, <32 x float>* %b) #0 {
+; CHECK-LABEL: scvtf_v32i64_v32f32:
+; VBITS_GE_2048: ptrue [[PG1:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG1]]/z, [x0]
+; VBITS_GE_2048-NEXT: ptrue [[PG2:p[0-9]+]].d
+; VBITS_GE_2048-NEXT: scvtf [[CVT:z[0-9]+]].s, [[PG2]]/m, [[OP]].d
+; VBITS_GE_2048-NEXT: uzp1 [[RES:z[0-9]+]].s, [[CVT]].s, [[CVT]].s
+; VBITS_GE_2048-NEXT: ptrue [[PG3:p[0-9]+]].s, vl32
+; VBITS_GE_2048-NEXT: st1w { [[RES]].s }, [[PG3]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <32 x i64>, <32 x i64>* %a
+  %res = sitofp <32 x i64> %op1 to <32 x float>
+  store <32 x float> %res, <32 x float>* %b
+  ret void
+}
+
+;
+; SCVTF D -> D
+;
+
+; Don't use SVE for 64-bit vectors.
+define <1 x double> @scvtf_v1i64_v1f64(<1 x i64> %op1) #0 {
+; CHECK-LABEL: scvtf_v1i64_v1f64:
+; CHECK: fmov x8, d0
+; CHECK-NEXT: scvtf d0, x8
+; CHECK-NEXT: ret
+  %res = sitofp <1 x i64> %op1 to <1 x double>
+  ret <1 x double> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <2 x double> @scvtf_v2i64_v2f64(<2 x i64> %op1) #0 {
+; CHECK-LABEL: scvtf_v2i64_v2f64:
+; CHECK: scvtf v0.2d, v0.2d
+; CHECK-NEXT: ret
+  %res = sitofp <2 x i64> %op1 to <2 x double>
+  ret <2 x double> %res
+}
+
+define void @scvtf_v4i64_v4f64(<4 x i64>* %a, <4 x double>* %b) #0 {
+; CHECK-LABEL: scvtf_v4i64_v4f64:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-NEXT: scvtf [[RES:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; CHECK-NEXT: st1d { [[RES]].d }, [[PG]], [x1]
+; CHECK-NEXT: ret
+  %op1 = load <4 x i64>, <4 x i64>* %a
+  %res = sitofp <4 x i64> %op1 to <4 x double>
+  store <4 x double> %res, <4 x double>* %b
+  ret void
+}
+
+define void @scvtf_v8i64_v8f64(<8 x i64>* %a, <8 x double>* %b) #0 {
+; CHECK-LABEL: scvtf_v8i64_v8f64:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_512-NEXT: scvtf [[RES:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; VBITS_GE_512-NEXT: st1d { [[RES]].d }, [[PG]], [x1]
+; VBITS_GE_512-NEXT: ret
+  %op1 = load <8 x i64>, <8 x i64>* %a
+  %res = sitofp <8 x i64> %op1 to <8 x double>
+  store <8 x double> %res, <8 x double>* %b
+  ret void
+}
+
+define void @scvtf_v16i64_v16f64(<16 x i64>* %a, <16 x double>* %b) #0 {
+; CHECK-LABEL: scvtf_v16i64_v16f64:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_1024-NEXT: scvtf [[RES:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; VBITS_GE_1024-NEXT: st1d { [[RES]].d }, [[PG]], [x1]
+; VBITS_GE_1024-NEXT: ret
+  %op1 = load <16 x i64>, <16 x i64>* %a
+  %res = sitofp <16 x i64> %op1 to <16 x double>
+  store <16 x double> %res, <16 x double>* %b
+  ret void
+}
+
+define void @scvtf_v32i64_v32f64(<32 x i64>* %a, <32 x double>* %b) #0 {
+; CHECK-LABEL: scvtf_v32i64_v32f64:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048-NEXT: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_2048-NEXT: scvtf [[RES:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; VBITS_GE_2048-NEXT: st1d { [[RES]].d }, [[PG]], [x1]
+; VBITS_GE_2048-NEXT: ret
+  %op1 = load <32 x i64>, <32 x i64>* %a
+  %res = sitofp <32 x i64> %op1 to <32 x double>
+  store <32 x double> %res, <32 x double>* %b
+  ret void
+}
+
+attributes #0 = { "target-features"="+sve" }