[SVE] Code generation for fixed length vector truncates.

[paulwalker-arm]

Summary

Lower fixed length vector truncates to a sequence of SVE UZP1 instructions.

[efriedma]

Probably worth adding a testcase for truncating from <4 x i64> to <4 x i8>.

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

1072

This specifically applies to the result type. You might want to note that you're implicitly depending on the fact that we do custom legalization for NEON TRUNCATE operations for other reasons.

[paulwalker-arm]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

1072

I wondered about that. Do you think it would be better if I was just explicit and add the necessary setOperation calls even though they're duplicates?

[cameron.mcinally]

llvm/test/CodeGen/AArch64/sve-fixed-length-trunc.ll

214

Just passing by and had to comment that this is an expensive truncate. It's around 5x slower than the equivalent on x86, with 1/2 the throughput.

Might be a good candidate for a dedicated hardware instruction on future SVE revisions...

[efriedma]

llvm/test/CodeGen/AArch64/sve-fixed-length-trunc.ll

214

We could experiment with using tbl or compact, if this comes up in practice.

[paulwalker-arm]

In D83395#2139892, @efriedma wrote:

Probably worth adding a testcase for truncating from <4 x i64> to <4 x i8>.

I'm happy to add this but just wanted to query what it gives. <4 x i8> is not a legal type so the test just exercises the same truncate path as <4 x i64> to <4 x i16>, or is this what you want protected (i.e. ensure the bytes remain where they're expected to be).

[paulwalker-arm]

Made custom lowering for all truncates explicit. Added test for trunc_v4i64_v4i8 and tighten up the register based tests.

[efriedma]

LGTM

I'm happy to add this but just wanted to query what it gives. <4 x i8> is not a legal type so the test just exercises the same truncate path as <4 x i64> to <4 x i16>, or is this what you want protected (i.e. ensure the bytes remain where they're expected to be).

That's fine; I just want to test that it doesn't get caught in the custom lowering and crash somehow.

llvm/lib/Target/AArch64/AArch64ISelLowering.h

private:
   SDValue LowerWindowsDYNAMIC_STACKALLOC(SDValue Op, SDValue Chain,
                                          SDValue &Size,
                                          SelectionDAG &DAG) const;
   SDValue LowerSVEStructLoad(unsigned Intrinsic, ArrayRef<SDValue> LoadOps,
                              EVT VT, SelectionDAG &DAG, const SDLoc &DL) const;
 
   SDValue LowerFixedLengthVectorLoadToSVE(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFixedLengthVectorStoreToSVE(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerFixedLengthVectorTruncateToSVE(SDValue Op,
+                                              SelectionDAG &DAG) const;
 
   SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
                         SmallVectorImpl<SDNode *> &Created) const override;
   SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
                           int &ExtraSteps, bool &UseOneConst,
                           bool Reciprocal) const override;
   SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
                            int &ExtraSteps) const override;

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

if (Subtarget->hasSVE()) {
     // than the preferred option of combining it with the addRegisterClass call.
     if (useSVEForFixedLengthVectors()) {
       for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
         if (useSVEForFixedLengthVectorVT(VT))
           addTypeForFixedLengthSVE(VT);
       for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())
         if (useSVEForFixedLengthVectorVT(VT))
           addTypeForFixedLengthSVE(VT);
+
+      // 64bit results can mean a bigger than NEON input.
+      for (auto VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32})
+        setOperationAction(ISD::TRUNCATE, VT, Custom);
+
+      // 128bit results imply a bigger than NEON input.
+      for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
+        setOperationAction(ISD::TRUNCATE, VT, Custom);
     }
   }
 
   PredictableSelectIsExpensive = Subtarget->predictableSelectIsExpensive();
 }
 
 void AArch64TargetLowering::addTypeForNEON(MVT VT, MVT PromotedBitwiseVT) {
   assert(VT.isVector() && "VT should be a vector type");
void AArch64TargetLowering::addTypeForFixedLengthSVE(MVT VT) {
   // We use EXTRACT_SUBVECTOR to "cast" a scalable vector to a fixed length one.
   setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
 
   // Lower fixed length vector operations to scalable equivalents.
   setOperationAction(ISD::ADD, VT, Custom);
   setOperationAction(ISD::FADD, VT, Custom);
   setOperationAction(ISD::LOAD, VT, Custom);
   setOperationAction(ISD::STORE, VT, Custom);
+  setOperationAction(ISD::TRUNCATE, VT, Custom);
 }
 
 void AArch64TargetLowering::addDRTypeForNEON(MVT VT) {
   addRegisterClass(VT, &AArch64::FPR64RegClass);
   addTypeForNEON(VT, MVT::v2i32);
 }
 
 void AArch64TargetLowering::addQRTypeForNEON(MVT VT) {

 // extension instead of zero extension and form a srhadd(OpA, OpB) from it.
 SDValue AArch64TargetLowering::LowerTRUNCATE(SDValue Op,
                                              SelectionDAG &DAG) const {
   EVT VT = Op.getValueType();
 
   if (!VT.isVector() || VT.isScalableVector())
     return Op;
 
+  if (useSVEForFixedLengthVectorVT(Op.getOperand(0).getValueType()))
+    return LowerFixedLengthVectorTruncateToSVE(Op, DAG);
+
   // Since we are looking for a right shift by a constant value of 1 and we are
   // operating on types at least 16 bits in length (sign/zero extended OpA and
   // OpB, which are at least 8 bits), it follows that the truncate will always
   // discard the shifted-in bit and therefore the right shift will be logical
   // regardless of the signedness of OpA and OpB.
   SDValue Shift = Op.getOperand(0);
   if (Shift.getOpcode() != AArch64ISD::VLSHR)
     return Op;
SDValue AArch64TargetLowering::LowerFixedLengthVectorStoreToSVE(
   auto NewValue = convertToScalableVector(DAG, ContainerVT, Store->getValue());
   return DAG.getMaskedStore(
       Store->getChain(), DL, NewValue, Store->getBasePtr(), Store->getOffset(),
       getPredicateForFixedLengthVector(DAG, DL, VT), Store->getMemoryVT(),
       Store->getMemOperand(), Store->getAddressingMode(),
       Store->isTruncatingStore());
 }
 
+SDValue AArch64TargetLowering::LowerFixedLengthVectorTruncateToSVE(
+    SDValue Op, SelectionDAG &DAG) const {
+  EVT VT = Op.getValueType();
+  assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
+
+  SDLoc DL(Op);
+  SDValue Val = Op.getOperand(0);
+  EVT ContainerVT = getContainerForFixedLengthVector(DAG, Val.getValueType());
+  Val = convertToScalableVector(DAG, ContainerVT, Val);
+
+  // Repeatedly truncate Val until the result is of the desired element type.
+  switch (ContainerVT.getSimpleVT().SimpleTy) {
+  default:
+    llvm_unreachable("unimplemented container type");
+  case MVT::nxv2i64:
+    Val = DAG.getNode(ISD::BITCAST, DL, MVT::nxv4i32, Val);
+    Val = DAG.getNode(AArch64ISD::UZP1, DL, MVT::nxv4i32, Val, Val);
+    if (VT.getVectorElementType() == MVT::i32)
+      break;
+    LLVM_FALLTHROUGH;
+  case MVT::nxv4i32:
+    Val = DAG.getNode(ISD::BITCAST, DL, MVT::nxv8i16, Val);
+    Val = DAG.getNode(AArch64ISD::UZP1, DL, MVT::nxv8i16, Val, Val);
+    if (VT.getVectorElementType() == MVT::i16)
+      break;
+    LLVM_FALLTHROUGH;
+  case MVT::nxv8i16:
+    Val = DAG.getNode(ISD::BITCAST, DL, MVT::nxv16i8, Val);
+    Val = DAG.getNode(AArch64ISD::UZP1, DL, MVT::nxv16i8, Val, Val);
+    assert(VT.getVectorElementType() == MVT::i8 && "Unexpected element type!");
+    break;
+  }
+
+  return convertFromScalableVector(DAG, VT, Val);
+}
+
 SDValue AArch64TargetLowering::LowerToPredicatedOp(SDValue Op,
                                                    SelectionDAG &DAG,
                                                    unsigned NewOp) const {
   EVT VT = Op.getValueType();
   SDLoc DL(Op);
   auto Pg = getPredicateForVector(DAG, DL, VT);
 
   if (useSVEForFixedLengthVectorVT(VT)) {

llvm/test/CodeGen/AArch64/sve-fixed-length-trunc.ll

+; RUN: llc -aarch64-sve-vector-bits-min=128  -asm-verbose=0 < %s | FileCheck %s -check-prefix=NO_SVE
+; RUN: llc -aarch64-sve-vector-bits-min=256  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK
+; RUN: llc -aarch64-sve-vector-bits-min=384  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK
+; RUN: llc -aarch64-sve-vector-bits-min=512  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=640  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=768  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=896  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=1024 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1152 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1280 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1408 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1536 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1664 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1792 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1920 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=2048 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024,VBITS_GE_2048
+
+target triple = "aarch64-unknown-linux-gnu"
+
+; Don't use SVE when its registers are no bigger than NEON.
+; NO_SVE-NOT: z{0-9}
+
+;
+; truncate i16 -> i8
+;
+
+define <16 x i8> @trunc_v16i16_v16i8(<16 x i16>* %in) #0 {
+; CHECK-LABEL: trunc_v16i16_v16i8:
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl16
+; CHECK-NEXT: ld1h { [[A_HALFS:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK-NEXT: uzp1 z0.b, [[A_HALFS]].b, [[A_HALFS]].b
+; CHECK-NEXT: ret
+  %a = load <16 x i16>, <16 x i16>* %in
+  %b = trunc <16 x i16> %a to <16 x i8>
+  ret <16 x i8> %b
+}
+
+; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
+define void @trunc_v32i16_v32i8(<32 x i16>* %in, <32 x i8>* %out) #0 {
+; CHECK-LABEL: trunc_v32i16_v32i8:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].h, vl32
+; VBITS_GE_512: ld1h { [[A_HALFS:z[0-9]+]].h }, [[PG]]/z, [x0]
+; VBITS_GE_512: uzp1 [[A_BYTES:z[0-9]+]].b, [[A_HALFS]].b, [[A_HALFS]].b
+; VBITS_GE_512: add [[A_BYTES]].b, [[PG]]/m, [[A_BYTES]].b, [[A_BYTES]].b
+  %a = load <32 x i16>, <32 x i16>* %in
+  %b = trunc <32 x i16> %a to <32 x i8>
+  %c = add <32 x i8> %b, %b
+  store <32 x i8> %c, <32 x i8>* %out
+  ret void
+}
+
+; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
+define void @trunc_v64i16_v64i8(<64 x i16>* %in, <64 x i8>* %out) #0 {
+; CHECK-LABEL: trunc_v64i16_v64i8:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].h, vl64
+; VBITS_GE_1024: ld1h { [[A_HALFS:z[0-9]+]].h }, [[PG]]/z, [x0]
+; VBITS_GE_1024: uzp1 [[A_BYTES:z[0-9]+]].b, [[A_HALFS]].b, [[A_HALFS]].b
+; VBITS_GE_1024: add [[A_BYTES]].b, [[PG]]/m, [[A_BYTES]].b, [[A_BYTES]].b
+  %a = load <64 x i16>, <64 x i16>* %in
+  %b = trunc <64 x i16> %a to <64 x i8>
+  %c = add <64 x i8> %b, %b
+  store <64 x i8> %c, <64 x i8>* %out
+  ret void
+}
+
+; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
+define void @trunc_v128i16_v128i8(<128 x i16>* %in, <128 x i8>* %out) #0 {
+; CHECK-LABEL: trunc_v128i16_v128i8:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].h, vl128
+; VBITS_GE_2048: ld1h { [[A_HALFS:z[0-9]+]].h }, [[PG]]/z, [x0]
+; VBITS_GE_2048: uzp1 [[A_BYTES:z[0-9]+]].b, [[A_HALFS]].b, [[A_HALFS]].b
+; VBITS_GE_2048: add [[A_BYTES]].b, [[PG]]/m, [[A_BYTES]].b, [[A_BYTES]].b
+  %a = load <128 x i16>, <128 x i16>* %in
+  %b = trunc <128 x i16> %a to <128 x i8>
+  %c = add <128 x i8> %b, %b
+  store <128 x i8> %c, <128 x i8>* %out
+  ret void
+}
+
+;
+; truncate i32 -> i8
+;
+
+define <8 x i8> @trunc_v8i32_v8i8(<8 x i32>* %in) #0 {
+; CHECK-LABEL: trunc_v8i32_v8i8:
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl8
+; CHECK-NEXT: ld1w { [[A_WORDS:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-NEXT: uzp1 [[A_HALFS:z[0-9]+]].h, [[A_WORDS]].h, [[A_WORDS]].h
+; CHECK-NEXT: uzp1 z0.b, [[A_HALFS]].b, [[A_HALFS]].b
+; CHECK-NEXT: ret
+  %a = load <8 x i32>, <8 x i32>* %in
+  %b = trunc <8 x i32> %a to <8 x i8>
+  ret <8 x i8> %b
+}
+
+define <16 x i8> @trunc_v16i32_v16i8(<16 x i32>* %in) #0 {
+; CHECK-LABEL: trunc_v16i32_v16i8:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].s, vl16
+; VBITS_GE_512-NEXT: ld1w { [[A_WORDS:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_512-NEXT: uzp1 [[A_HALFS:z[0-9]+]].h, [[A_WORDS]].h, [[A_WORDS]].h
+; VBITS_GE_512-NEXT: uzp1 z0.b, [[A_HALFS]].b, [[A_HALFS]].b
+; VBITS_GE_512-NEXT: ret
+  %a = load <16 x i32>, <16 x i32>* %in
+  %b = trunc <16 x i32> %a to <16 x i8>
+  ret <16 x i8> %b
+}
+
+; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
+define void @trunc_v32i32_v32i8(<32 x i32>* %in, <32 x i8>* %out) #0 {
+; CHECK-LABEL: trunc_v32i32_v32i8:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].s, vl32
+; VBITS_GE_1024: ld1w { [[A_WORDS:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_1024: uzp1 [[A_HALFS:z[0-9]+]].h, [[A_WORDS]].h, [[A_WORDS]].h
+; VBITS_GE_1024: uzp1 [[A_BYTES:z[0-9]+]].b, [[A_HALFS]].b, [[A_HALFS]].b
+; VBITS_GE_1024: add [[A_BYTES]].b, [[PG]]/m, [[A_BYTES]].b, [[A_BYTES]].b
+  %a = load <32 x i32>, <32 x i32>* %in
+  %b = trunc <32 x i32> %a to <32 x i8>
+  %c = add <32 x i8> %b, %b
+  store <32 x i8> %c, <32 x i8>* %out
+  ret void
+}
+
+; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
+define void @trunc_v64i32_v64i8(<64 x i32>* %in, <64 x i8>* %out) #0 {
+; CHECK-LABEL: trunc_v64i32_v64i8:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].s, vl64
+; VBITS_GE_2048: ld1w { [[A_WORDS:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_2048: uzp1 [[A_HALFS:z[0-9]+]].h, [[A_WORDS]].h, [[A_WORDS]].h
+; VBITS_GE_2048: uzp1 [[A_BYTES:z[0-9]+]].b, [[A_HALFS]].b, [[A_HALFS]].b
+; VBITS_GE_2048: add [[A_BYTES]].b, [[PG]]/m, [[A_BYTES]].b, [[A_BYTES]].b
+  %a = load <64 x i32>, <64 x i32>* %in
+  %b = trunc <64 x i32> %a to <64 x i8>
+  %c = add <64 x i8> %b, %b
+  store <64 x i8> %c, <64 x i8>* %out
+  ret void
+}
+
+;
+; truncate i32 -> i16
+;
+
+define <8 x i16> @trunc_v8i32_v8i16(<8 x i32>* %in) #0 {
+; CHECK-LABEL: trunc_v8i32_v8i16:
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl8
+; CHECK-NEXT: ld1w { [[A_WORDS:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-NEXT: uzp1 z0.h, [[A_WORDS]].h, [[A_WORDS]].h
+; CHECK-NEXT: ret
+  %a = load <8 x i32>, <8 x i32>* %in
+  %b = trunc <8 x i32> %a to <8 x i16>
+  ret <8 x i16> %b
+}
+
+; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
+define void @trunc_v16i32_v16i16(<16 x i32>* %in, <16 x i16>* %out) #0 {
+; CHECK-LABEL: trunc_v16i32_v16i16:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].s, vl16
+; VBITS_GE_512: ld1w { [[A_WORDS:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_512: uzp1 [[A_HALFS:z[0-9]+]].h, [[A_WORDS]].h, [[A_WORDS]].h
+; VBITS_GE_512: add [[A_HALFS]].h, [[PG]]/m, [[A_HALFS]].h, [[A_HALFS]].h
+  %a = load <16 x i32>, <16 x i32>* %in
+  %b = trunc <16 x i32> %a to <16 x i16>
+  %c = add <16 x i16> %b, %b
+  store <16 x i16> %c, <16 x i16>* %out
+  ret void
+}
+
+; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
+define void @trunc_v32i32_v32i16(<32 x i32>* %in, <32 x i16>* %out) #0 {
+; CHECK-LABEL: trunc_v32i32_v32i16:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].s, vl32
+; VBITS_GE_1024: ld1w { [[A_WORDS:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_1024: uzp1 [[A_HALFS:z[0-9]+]].h, [[A_WORDS]].h, [[A_WORDS]].h
+; VBITS_GE_1024: add [[A_HALFS]].h, [[PG]]/m, [[A_HALFS]].h, [[A_HALFS]].h
+  %a = load <32 x i32>, <32 x i32>* %in
+  %b = trunc <32 x i32> %a to <32 x i16>
+  %c = add <32 x i16> %b, %b
+  store <32 x i16> %c, <32 x i16>* %out
+  ret void
+}
+
+; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
+define void @trunc_v64i32_v64i16(<64 x i32>* %in, <64 x i16>* %out) #0 {
+; CHECK-LABEL: trunc_v64i32_v64i16:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].s, vl64
+; VBITS_GE_2048: ld1w { [[A_WORDS:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_2048: uzp1 [[A_HALFS:z[0-9]+]].h, [[A_WORDS]].h, [[A_WORDS]].h
+; VBITS_GE_2048: add [[A_HALFS]].h, [[PG]]/m, [[A_HALFS]].h, [[A_HALFS]].h
+  %a = load <64 x i32>, <64 x i32>* %in
+  %b = trunc <64 x i32> %a to <64 x i16>
+  %c = add <64 x i16> %b, %b
+  store <64 x i16> %c, <64 x i16>* %out
+  ret void
+}
+
+;
+; truncate i64 -> i8
+;
+
+; NOTE: v4i8 is not legal so result i8 elements are held within i16 containers.
+define <4 x i8> @trunc_v4i64_v4i8(<4 x i64>* %in) #0 {
+; CHECK-LABEL: trunc_v4i64_v4i8:
+; VBITS_GE_256: ptrue [[PG:p[0-9]+]].d, vl4
+; VBITS_GE_256-NEXT: ld1d { [[A_DWORDS:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_256-NEXT: uzp1 [[A_WORDS:z[0-9]+]].s, [[A_DWORDS]].s, [[A_DWORDS]].s
+; VBITS_GE_256-NEXT: uzp1 z0.h, [[A_WORDS]].h, [[A_WORDS]].h
+; VBITS_GE_256-NEXT: ret
+  %a = load <4 x i64>, <4 x i64>* %in
+  %b = trunc <4 x i64> %a to <4 x i8>
+  ret <4 x i8> %b
+}
+
+define <8 x i8> @trunc_v8i64_v8i8(<8 x i64>* %in) #0 {
+; CHECK-LABEL: trunc_v8i64_v8i8:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: ld1d { [[A_DWORDS:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_512-NEXT: uzp1 [[A_WORDS:z[0-9]+]].s, [[A_DWORDS]].s, [[A_DWORDS]].s
+; VBITS_GE_512-NEXT: uzp1 [[A_HALFS:z[0-9]+]].h, [[A_WORDS]].h, [[A_WORDS]].h
+; VBITS_GE_512-NEXT: uzp1 z0.b, [[A_HALFS]].b, [[A_HALFS]].b
+; VBITS_GE_512-NEXT: ret
+  %a = load <8 x i64>, <8 x i64>* %in
+  %b = trunc <8 x i64> %a to <8 x i8>
+  ret <8 x i8> %b
+}
+
+define <16 x i8> @trunc_v16i64_v16i8(<16 x i64>* %in) #0 {
+; CHECK-LABEL: trunc_v16i64_v16i8:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024-NEXT: ld1d { [[A_DWORDS:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_1024-NEXT: uzp1 [[A_WORDS:z[0-9]+]].s, [[A_DWORDS]].s, [[A_DWORDS]].s
+; VBITS_GE_1024-NEXT: uzp1 [[A_HALFS:z[0-9]+]].h, [[A_WORDS]].h, [[A_WORDS]].h
+; VBITS_GE_1024-NEXT: uzp1 z0.b, [[A_HALFS]].b, [[A_HALFS]].b
+; VBITS_GE_1024-NEXT: ret
+  %a = load <16 x i64>, <16 x i64>* %in
+  %b = trunc <16 x i64> %a to <16 x i8>
+  ret <16 x i8> %b
+}
+
+; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
+define void @trunc_v32i64_v32i8(<32 x i64>* %in, <32 x i8>* %out) #0 {
+; CHECK-LABEL: trunc_v32i64_v32i8:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048: ld1d { [[A_DWORDS:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_2048: uzp1 [[A_WORDS:z[0-9]+]].s, [[A_DWORDS]].s, [[A_DWORDS]].s
+; VBITS_GE_2048: uzp1 [[A_HALFS:z[0-9]+]].h, [[A_WORDS]].h, [[A_WORDS]].h
+; VBITS_GE_2048: uzp1 [[A_BYTES:z[0-9]+]].b, [[A_HALFS]].b, [[A_HALFS]].b
+; VBITS_GE_2048: add [[A_BYTES]].b, [[PG]]/m, [[A_BYTES]].b, [[A_BYTES]].b
+  %a = load <32 x i64>, <32 x i64>* %in
+  %b = trunc <32 x i64> %a to <32 x i8>
+  %c = add <32 x i8> %b, %b
+  store <32 x i8> %c, <32 x i8>* %out
+  ret void
+}
+
+;
+; truncate i64 -> i16
+;
+
+define <4 x i16> @trunc_v4i64_v4i16(<4 x i64>* %in) #0 {
+; CHECK-LABEL: trunc_v4i64_v4i16:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: ld1d { [[A_DWORDS:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-NEXT: uzp1 [[A_WORDS:z[0-9]+]].s, [[A_DWORDS]].s, [[A_DWORDS]].s
+; CHECK-NEXT: uzp1 z0.h, [[A_WORDS]].h, [[A_WORDS]].h
+; CHECK-NEXT: ret
+  %a = load <4 x i64>, <4 x i64>* %in
+  %b = trunc <4 x i64> %a to <4 x i16>
+  ret <4 x i16> %b
+}
+
+define <8 x i16> @trunc_v8i64_v8i16(<8 x i64>* %in) #0 {
+; CHECK-LABEL: trunc_v8i64_v8i16:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512-NEXT: ld1d { [[A_DWORDS:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_512-NEXT: uzp1 [[A_WORDS:z[0-9]+]].s, [[A_DWORDS]].s, [[A_DWORDS]].s
+; VBITS_GE_512-NEXT: uzp1 z0.h, [[A_WORDS]].h, [[A_WORDS]].h
+; VBITS_GE_512-NEXT: ret
+  %a = load <8 x i64>, <8 x i64>* %in
+  %b = trunc <8 x i64> %a to <8 x i16>
+  ret <8 x i16> %b
+}
+
+; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
+define void @trunc_v16i64_v16i16(<16 x i64>* %in, <16 x i16>* %out) #0 {
+; CHECK-LABEL: trunc_v16i64_v16i16:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024: ld1d { [[A_DWORDS:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_1024: uzp1 [[A_WORDS:z[0-9]+]].s, [[A_DWORDS]].s, [[A_DWORDS]].s
+; VBITS_GE_1024: uzp1 [[A_HALFS:z[0-9]+]].h, [[A_WORDS]].h, [[A_WORDS]].h
+; VBITS_GE_1024: add [[A_HALFS]].h, [[PG]]/m, [[A_HALFS]].h, [[A_HALFS]].h
+  %a = load <16 x i64>, <16 x i64>* %in
+  %b = trunc <16 x i64> %a to <16 x i16>
+  %c = add <16 x i16> %b, %b
+  store <16 x i16> %c, <16 x i16>* %out
+  ret void
+}
+
+; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
+define void @trunc_v32i64_v32i16(<32 x i64>* %in, <32 x i16>* %out) #0 {
+; CHECK-LABEL: trunc_v32i64_v32i16:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048: ld1d { [[A_DWORDS:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_2048: uzp1 [[A_WORDS:z[0-9]+]].s, [[A_DWORDS]].s, [[A_DWORDS]].s
+; VBITS_GE_2048: uzp1 [[A_HALFS:z[0-9]+]].h, [[A_WORDS]].h, [[A_WORDS]].h
+; VBITS_GE_2048: add [[A_HALFS]].h, [[PG]]/m, [[A_HALFS]].h, [[A_HALFS]].h
+  %a = load <32 x i64>, <32 x i64>* %in
+  %b = trunc <32 x i64> %a to <32 x i16>
+  %c = add <32 x i16> %b, %b
+  store <32 x i16> %c, <32 x i16>* %out
+  ret void
+}
+
+;
+; truncate i64 -> i32
+;
+
+define <4 x i32> @trunc_v4i64_v4i32(<4 x i64>* %in) #0 {
+; CHECK-LABEL: trunc_v4i64_v4i32:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-NEXT: ld1d { [[A_DWORDS:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-NEXT: uzp1 z0.s, [[A_DWORDS]].s, [[A_DWORDS]].s
+; CHECK-NEXT: ret
+  %a = load <4 x i64>, <4 x i64>* %in
+  %b = trunc <4 x i64> %a to <4 x i32>
+  ret <4 x i32> %b
+}
+
+; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
+define void @trunc_v8i64_v8i32(<8 x i64>* %in, <8 x i32>* %out) #0 {
+; CHECK-LABEL: trunc_v8i64_v8i32:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512: ld1d { [[A_DWORDS:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_512: uzp1 [[A_WORDS:z[0-9]+]].s, [[A_DWORDS]].s, [[A_DWORDS]].s
+; VBITS_GE_512: add [[A_WORDS]].s, [[PG]]/m, [[A_WORDS]].s, [[A_WORDS]].s
+  %a = load <8 x i64>, <8 x i64>* %in
+  %b = trunc <8 x i64> %a to <8 x i32>
+  %c = add <8 x i32> %b, %b
+  store <8 x i32> %c, <8 x i32>* %out
+  ret void
+}
+
+; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
+define void @trunc_v16i64_v16i32(<16 x i64>* %in, <16 x i32>* %out) #0 {
+; CHECK-LABEL: trunc_v16i64_v16i32:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024: ld1d { [[A_DWORDS:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_1024: uzp1 [[A_WORDS:z[0-9]+]].s, [[A_DWORDS]].s, [[A_DWORDS]].s
+; VBITS_GE_1024: add [[A_WORDS]].s, [[PG]]/m, [[A_WORDS]].s, [[A_WORDS]].s
+  %a = load <16 x i64>, <16 x i64>* %in
+  %b = trunc <16 x i64> %a to <16 x i32>
+  %c = add <16 x i32> %b, %b
+  store <16 x i32> %c, <16 x i32>* %out
+  ret void
+}
+
+; NOTE: Extra 'add' is to prevent the truncate being combined with the store.
+define void @trunc_v32i64_v32i32(<32 x i64>* %in, <32 x i32>* %out) #0 {
+; CHECK-LABEL: trunc_v32i64_v32i32:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048: ld1d { [[A_DWORDS:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_2048: uzp1 [[A_WORDS:z[0-9]+]].s, [[A_DWORDS]].s, [[A_DWORDS]].s
+; VBITS_GE_2048: add [[A_WORDS]].s, [[PG]]/m, [[A_WORDS]].s, [[A_WORDS]].s
+  %a = load <32 x i64>, <32 x i64>* %in
+  %b = trunc <32 x i64> %a to <32 x i32>
+  %c = add <32 x i32> %b, %b
+  store <32 x i32> %c, <32 x i32>* %out
+  ret void
+}
+
+attributes #0 = { nounwind "target-features"="+sve" }