diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -1389,6 +1389,26 @@ for (auto VT : {MVT::v4f16, MVT::v8f16, MVT::v2f32, MVT::v4f32, MVT::v2f64}) setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom); + for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) { + if (useSVEForFixedLengthVectorVT(VT, + Subtarget->forceSVEInStreamingMode())) { + setOperationAction(ISD::ANY_EXTEND, VT, Custom); + setOperationAction(ISD::ZERO_EXTEND, VT, Custom); + setOperationAction(ISD::SIGN_EXTEND, VT, Custom); + setOperationAction(ISD::LOAD, VT, Custom); + } + } + + for (MVT VT : MVT::fp_fixedlen_vector_valuetypes()) { + if (useSVEForFixedLengthVectorVT(VT, + Subtarget->forceSVEInStreamingMode())) { + setOperationAction(ISD::ANY_EXTEND, VT, Custom); + setOperationAction(ISD::ZERO_EXTEND, VT, Custom); + setOperationAction(ISD::SIGN_EXTEND, VT, Custom); + setOperationAction(ISD::LOAD, VT, Custom); + } + } + // NOTE: Currently this has to happen after computeRegisterProperties rather // than the preferred option of combining it with the addRegisterClass call. if (Subtarget->useSVEForFixedLengthVectors()) { @@ -5725,7 +5745,8 @@ case ISD::MLOAD: return LowerMLOAD(Op, DAG); case ISD::LOAD: - if (useSVEForFixedLengthVectorVT(Op.getValueType())) + if (useSVEForFixedLengthVectorVT(Op.getValueType(), + Subtarget->forceSVEInStreamingMode())) return LowerFixedLengthVectorLoadToSVE(Op, DAG); return LowerLOAD(Op, DAG); case ISD::ADD: @@ -11536,7 +11557,8 @@ SelectionDAG &DAG) const { EVT VT = Op.getValueType(); - if (useSVEForFixedLengthVectorVT(VT)) { + // override NEON if possible. + if (useSVEForFixedLengthVectorVT(VT, Subtarget->forceSVEInStreamingMode())) { if (auto SeqInfo = cast(Op)->isConstantSequence()) { SDLoc DL(Op); EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT); @@ -11972,7 +11994,8 @@ return DAG.getAnyExtOrTrunc(Extract, DL, Op.getValueType()); } - if (useSVEForFixedLengthVectorVT(VT)) + // try overriding NEON if possible. + if (useSVEForFixedLengthVectorVT(VT, Subtarget->forceSVEInStreamingMode())) return LowerFixedLengthExtractVectorElt(Op, DAG); // Check for non-constant or out of range lane. @@ -12031,13 +12054,16 @@ if (Idx == 0 && InVT.getSizeInBits() <= 128) return Op; - // If this is extracting the upper 64-bits of a 128-bit vector, we match - // that directly. - if (Size == 64 && Idx * InVT.getScalarSizeInBits() == 64 && - InVT.getSizeInBits() == 128) - return Op; + if (!Subtarget->forceSVEInStreamingMode()) { + // If this is extracting the upper 64-bits of a 128-bit vector, we match + // that directly. + if (Size == 64 && Idx * InVT.getScalarSizeInBits() == 64 && + InVT.getSizeInBits() == 128) + return Op; + } - if (useSVEForFixedLengthVectorVT(InVT)) { + if (useSVEForFixedLengthVectorVT(InVT, + Subtarget->forceSVEInStreamingMode())) { SDLoc DL(Op); EVT ContainerVT = getContainerForFixedLengthVector(DAG, InVT); @@ -12335,7 +12361,9 @@ switch (Op.getOpcode()) { case ISD::SHL: - if (VT.isScalableVector() || useSVEForFixedLengthVectorVT(VT)) + // override NEON if possible. + if (VT.isScalableVector() || + useSVEForFixedLengthVectorVT(VT, Subtarget->forceSVEInStreamingMode())) return LowerToPredicatedOp(Op, DAG, AArch64ISD::SHL_PRED); if (isVShiftLImm(Op.getOperand(1), VT, false, Cnt) && Cnt < EltSize) @@ -12347,7 +12375,10 @@ Op.getOperand(0), Op.getOperand(1)); case ISD::SRA: case ISD::SRL: - if (VT.isScalableVector() || useSVEForFixedLengthVectorVT(VT)) { + // override NEON if possible. + if (VT.isScalableVector() || + useSVEForFixedLengthVectorVT(VT, + Subtarget->forceSVEInStreamingMode())) { unsigned Opc = Op.getOpcode() == ISD::SRA ? AArch64ISD::SRA_PRED : AArch64ISD::SRL_PRED; return LowerToPredicatedOp(Op, DAG, Opc); diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -218,6 +218,8 @@ def UseScalarIncVL : Predicate<"Subtarget->useScalarIncVL()">; +def IsForcingSVEDisabled : Predicate<"!Subtarget->forceSVEInStreamingMode()">; + def AArch64LocalRecover : SDNode<"ISD::LOCAL_RECOVER", SDTypeProfile<1, 1, [SDTCisSameAs<0, 1>, SDTCisInt<1>]>>; diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -3032,30 +3032,31 @@ (EXTRACT_SUBREG (DUP_ZZI_D ZPR:$vec, sve_elm_idx_extdup_d:$index), dsub)>; // Extract element from vector with immediate index that's within the bottom 128-bits. - let AddedComplexity = 1 in { - def : Pat<(i32 (vector_extract (nxv16i8 ZPR:$vec), VectorIndexB:$index)), - (i32 (UMOVvi8 (v16i8 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexB:$index))>; - def : Pat<(i32 (vector_extract (nxv8i16 ZPR:$vec), VectorIndexH:$index)), - (i32 (UMOVvi16 (v8i16 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexH:$index))>; - def : Pat<(i32 (vector_extract (nxv4i32 ZPR:$vec), VectorIndexS:$index)), - (i32 (UMOVvi32 (v4i32 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexS:$index))>; - def : Pat<(i64 (vector_extract (nxv2i64 ZPR:$vec), VectorIndexD:$index)), - (i64 (UMOVvi64 (v2i64 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexD:$index))>; - } - - def : Pat<(sext_inreg (vector_extract (nxv16i8 ZPR:$vec), VectorIndexB:$index), i8), - (i32 (SMOVvi8to32 (v16i8 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexB:$index))>; - def : Pat<(sext_inreg (anyext (vector_extract (nxv16i8 ZPR:$vec), VectorIndexB:$index)), i8), - (i64 (SMOVvi8to64 (v16i8 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexB:$index))>; + let Predicates = [IsForcingSVEDisabled] in { + let AddedComplexity = 1 in { + def : Pat<(i32 (vector_extract (nxv16i8 ZPR:$vec), VectorIndexB:$index)), + (i32 (UMOVvi8 (v16i8 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexB:$index))>; + def : Pat<(i32 (vector_extract (nxv8i16 ZPR:$vec), VectorIndexH:$index)), + (i32 (UMOVvi16 (v8i16 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexH:$index))>; + def : Pat<(i32 (vector_extract (nxv4i32 ZPR:$vec), VectorIndexS:$index)), + (i32 (UMOVvi32 (v4i32 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexS:$index))>; + def : Pat<(i64 (vector_extract (nxv2i64 ZPR:$vec), VectorIndexD:$index)), + (i64 (UMOVvi64 (v2i64 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexD:$index))>; + } - def : Pat<(sext_inreg (vector_extract (nxv8i16 ZPR:$vec), VectorIndexH:$index), i16), - (i32 (SMOVvi16to32 (v8i16 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexH:$index))>; - def : Pat<(sext_inreg (anyext (vector_extract (nxv8i16 ZPR:$vec), VectorIndexH:$index)), i16), - (i64 (SMOVvi16to64 (v8i16 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexH:$index))>; + def : Pat<(sext_inreg (vector_extract (nxv16i8 ZPR:$vec), VectorIndexB:$index), i8), + (i32 (SMOVvi8to32 (v16i8 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexB:$index))>; + def : Pat<(sext_inreg (anyext (vector_extract (nxv16i8 ZPR:$vec), VectorIndexB:$index)), i8), + (i64 (SMOVvi8to64 (v16i8 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexB:$index))>; - def : Pat<(sext (vector_extract (nxv4i32 ZPR:$vec), VectorIndexS:$index)), - (i64 (SMOVvi32to64 (v4i32 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexS:$index))>; + def : Pat<(sext_inreg (vector_extract (nxv8i16 ZPR:$vec), VectorIndexH:$index), i16), + (i32 (SMOVvi16to32 (v8i16 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexH:$index))>; + def : Pat<(sext_inreg (anyext (vector_extract (nxv8i16 ZPR:$vec), VectorIndexH:$index)), i16), + (i64 (SMOVvi16to64 (v8i16 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexH:$index))>; + def : Pat<(sext (vector_extract (nxv4i32 ZPR:$vec), VectorIndexS:$index)), + (i64 (SMOVvi32to64 (v4i32 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexS:$index))>; + } // Extract first element from vector. let AddedComplexity = 2 in { def : Pat<(vector_extract (nxv16i8 ZPR:$Zs), (i64 0)), diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.h b/llvm/lib/Target/AArch64/AArch64Subtarget.h --- a/llvm/lib/Target/AArch64/AArch64Subtarget.h +++ b/llvm/lib/Target/AArch64/AArch64Subtarget.h @@ -359,10 +359,9 @@ return MinSVEVectorSizeInBits; } - bool useSVEForFixedLengthVectors() const { - // Prefer NEON unless larger SVE registers are available. - return hasSVE() && getMinSVEVectorSizeInBits() >= 256; - } + bool useSVEForFixedLengthVectors() const; + + bool forceSVEInStreamingMode() const; unsigned getVScaleForTuning() const { return VScaleForTuning; } diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp --- a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp +++ b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp @@ -65,6 +65,10 @@ "Should only be used for testing register allocator."), cl::CommaSeparated, cl::Hidden); +static cl::opt + ForceSVEWhenStreamingCompatible("force-sve-when-streaming-compatible", + cl::init(false), cl::Hidden); + unsigned AArch64Subtarget::getVectorInsertExtractBaseCost() const { if (OverrideVectorInsertExtractBaseCost.getNumOccurrences() > 0) return OverrideVectorInsertExtractBaseCost; @@ -431,3 +435,17 @@ } bool AArch64Subtarget::useAA() const { return UseAA; } + +bool AArch64Subtarget::useSVEForFixedLengthVectors() const { + if (ForceSVEWhenStreamingCompatible) + return hasSVE(); + + // Prefer NEON unless larger SVE registers are available. + return hasSVE() && getMinSVEVectorSizeInBits() >= 256; +} + +bool AArch64Subtarget::forceSVEInStreamingMode() const { + if (ForceSVEWhenStreamingCompatible) + return hasSVE(); + return false; +} diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-stores.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-stores.ll --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-stores.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-stores.ll @@ -417,3 +417,4 @@ declare void @llvm.masked.store.v32i8(<32 x i8>, <32 x i8>*, i32, <32 x i1>) attributes #0 = { "target-features"="+sve" } + diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-mload.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-mload.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-mload.ll @@ -0,0 +1,157 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -aarch64-sve-vector-bits-min=128 -force-sve-when-streaming-compatible < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_128_STREAMING +; RUN: llc -aarch64-sve-vector-bits-min=256 -force-sve-when-streaming-compatible < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_256_STREAMING +; RUN: llc -aarch64-sve-vector-bits-min=512 -force-sve-when-streaming-compatible < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512_STREAMING + +target triple = "aarch64-unknown-linux-gnu" + +; +; Masked Load +; + +define <16 x i8> @masked_load_v16i8(<16 x i8>* %src, <16 x i1> %mask) #0 { +; CHECK-LABEL: masked_load_v16i8: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x8, .LCPI0_0 +; CHECK-NEXT: add x8, x8, :lo12:.LCPI0_0 +; CHECK-NEXT: ptrue p0.b, vl16 +; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: ld1b { z1.b }, p0/z, [x8] +; CHECK-NEXT: lsl z0.b, p0/m, z0.b, z1.b +; CHECK-NEXT: asr z0.b, p0/m, z0.b, z1.b +; CHECK-NEXT: cmpne p0.b, p0/z, z0.b, #0 +; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0] +; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 +; CHECK-NEXT: ret + %load = call <16 x i8> @llvm.masked.load.v16i8(<16 x i8>* %src, i32 8, <16 x i1> %mask, <16 x i8> zeroinitializer) + ret <16 x i8> %load +} + +define <2 x half> @masked_load_v2f16(<2 x half>* %src, <2 x i1> %mask) #0 { +; CHECK-LABEL: masked_load_v2f16: +; CHECK: // %bb.0: +; CHECK-NEXT: sub sp, sp, #16 +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: str wzr, [sp, #12] +; CHECK-NEXT: mov z0.s, z0.s[1] +; CHECK-NEXT: ptrue p0.h, vl4 +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: strh w8, [sp, #8] +; CHECK-NEXT: adrp x8, .LCPI1_0 +; CHECK-NEXT: add x8, x8, :lo12:.LCPI1_0 +; CHECK-NEXT: strh w9, [sp, #10] +; CHECK-NEXT: ld1h { z0.h }, p0/z, [x8] +; CHECK-NEXT: add x8, sp, #8 +; CHECK-NEXT: ld1h { z1.h }, p0/z, [x8] +; CHECK-NEXT: lsl z1.h, p0/m, z1.h, z0.h +; CHECK-NEXT: asrr z0.h, p0/m, z0.h, z1.h +; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0 +; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0] +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: add sp, sp, #16 +; CHECK-NEXT: ret + %load = call <2 x half> @llvm.masked.load.v2f16(<2 x half>* %src, i32 8, <2 x i1> %mask, <2 x half> zeroinitializer) + ret <2 x half> %load +} + +define <4 x float> @masked_load_v4f32(<4 x float>* %src, <4 x i1> %mask) vscale_range(1,16) #0 { +; CHECK-LABEL: masked_load_v4f32: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x8, .LCPI2_0 +; CHECK-NEXT: add x8, x8, :lo12:.LCPI2_0 +; CHECK-NEXT: ptrue p0.s, vl4 +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: uunpklo z0.s, z0.h +; CHECK-NEXT: ld1w { z1.s }, p0/z, [x8] +; CHECK-NEXT: lsl z0.s, p0/m, z0.s, z1.s +; CHECK-NEXT: asr z0.s, p0/m, z0.s, z1.s +; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0 +; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0] +; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 +; CHECK-NEXT: ret + %load = call <4 x float> @llvm.masked.load.v4f32(<4 x float>* %src, i32 8, <4 x i1> %mask, <4 x float> zeroinitializer) + ret <4 x float> %load +} + +define <2 x double> @masked_load_v2f64(<2 x double>* %src, <2 x i1> %mask) #0 { +; CHECK-LABEL: masked_load_v2f64: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x8, .LCPI3_0 +; CHECK-NEXT: add x8, x8, :lo12:.LCPI3_0 +; CHECK-NEXT: ptrue p0.d, vl2 +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: uunpklo z0.d, z0.s +; CHECK-NEXT: ld1d { z1.d }, p0/z, [x8] +; CHECK-NEXT: lsl z0.d, p0/m, z0.d, z1.d +; CHECK-NEXT: asr z0.d, p0/m, z0.d, z1.d +; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] +; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 +; CHECK-NEXT: ret + %load = call <2 x double> @llvm.masked.load.v2f64(<2 x double>* %src, i32 8, <2 x i1> %mask, <2 x double> zeroinitializer) + ret <2 x double> %load +} + +define <4 x double> @masked_load_v4f64(<4 x double>* %src, <4 x i1> %mask) #0 { +; VBITS_GE_128_STREAMING-LABEL: masked_load_v4f64: +; VBITS_GE_128_STREAMING: // %bb.0: +; VBITS_GE_128_STREAMING-NEXT: adrp x8, .LCPI4_0 +; VBITS_GE_128_STREAMING-NEXT: add x8, x8, :lo12:.LCPI4_0 +; VBITS_GE_128_STREAMING-NEXT: ptrue p0.d, vl2 +; VBITS_GE_128_STREAMING-NEXT: // kill: def $d0 killed $d0 def $z0 +; VBITS_GE_128_STREAMING-NEXT: uunpklo z0.s, z0.h +; VBITS_GE_128_STREAMING-NEXT: ld1d { z1.d }, p0/z, [x8] +; VBITS_GE_128_STREAMING-NEXT: uunpklo z2.d, z0.s +; VBITS_GE_128_STREAMING-NEXT: ext z0.b, z0.b, z0.b, #8 +; VBITS_GE_128_STREAMING-NEXT: mov x8, #2 +; VBITS_GE_128_STREAMING-NEXT: uunpklo z0.d, z0.s +; VBITS_GE_128_STREAMING-NEXT: lsl z2.d, p0/m, z2.d, z1.d +; VBITS_GE_128_STREAMING-NEXT: lsl z0.d, p0/m, z0.d, z1.d +; VBITS_GE_128_STREAMING-NEXT: asr z2.d, p0/m, z2.d, z1.d +; VBITS_GE_128_STREAMING-NEXT: asr z0.d, p0/m, z0.d, z1.d +; VBITS_GE_128_STREAMING-NEXT: cmpne p1.d, p0/z, z2.d, #0 +; VBITS_GE_128_STREAMING-NEXT: cmpne p0.d, p0/z, z0.d, #0 +; VBITS_GE_128_STREAMING-NEXT: ld1d { z0.d }, p1/z, [x0] +; VBITS_GE_128_STREAMING-NEXT: ld1d { z1.d }, p0/z, [x0, x8, lsl #3] +; VBITS_GE_128_STREAMING-NEXT: // kill: def $q0 killed $q0 killed $z0 +; VBITS_GE_128_STREAMING-NEXT: // kill: def $q1 killed $q1 killed $z1 +; VBITS_GE_128_STREAMING-NEXT: ret +; +; VBITS_GE_256_STREAMING-LABEL: masked_load_v4f64: +; VBITS_GE_256_STREAMING: // %bb.0: +; VBITS_GE_256_STREAMING-NEXT: // kill: def $d0 killed $d0 def $z0 +; VBITS_GE_256_STREAMING-NEXT: ptrue p0.d, vl4 +; VBITS_GE_256_STREAMING-NEXT: uunpklo z0.s, z0.h +; VBITS_GE_256_STREAMING-NEXT: uunpklo z0.d, z0.s +; VBITS_GE_256_STREAMING-NEXT: lsl z0.d, p0/m, z0.d, #63 +; VBITS_GE_256_STREAMING-NEXT: asr z0.d, p0/m, z0.d, #63 +; VBITS_GE_256_STREAMING-NEXT: cmpne p1.d, p0/z, z0.d, #0 +; VBITS_GE_256_STREAMING-NEXT: ld1d { z0.d }, p1/z, [x0] +; VBITS_GE_256_STREAMING-NEXT: st1d { z0.d }, p0, [x8] +; VBITS_GE_256_STREAMING-NEXT: ret +; +; VBITS_GE_512_STREAMING-LABEL: masked_load_v4f64: +; VBITS_GE_512_STREAMING: // %bb.0: +; VBITS_GE_512_STREAMING-NEXT: // kill: def $d0 killed $d0 def $z0 +; VBITS_GE_512_STREAMING-NEXT: ptrue p0.d, vl4 +; VBITS_GE_512_STREAMING-NEXT: uunpklo z0.s, z0.h +; VBITS_GE_512_STREAMING-NEXT: uunpklo z0.d, z0.s +; VBITS_GE_512_STREAMING-NEXT: lsl z0.d, p0/m, z0.d, #63 +; VBITS_GE_512_STREAMING-NEXT: asr z0.d, p0/m, z0.d, #63 +; VBITS_GE_512_STREAMING-NEXT: cmpne p1.d, p0/z, z0.d, #0 +; VBITS_GE_512_STREAMING-NEXT: ld1d { z0.d }, p1/z, [x0] +; VBITS_GE_512_STREAMING-NEXT: st1d { z0.d }, p0, [x8] +; VBITS_GE_512_STREAMING-NEXT: ret + %load = call <4 x double> @llvm.masked.load.v4f64(<4 x double>* %src, i32 8, <4 x i1> %mask, <4 x double> zeroinitializer) + ret <4 x double> %load +} + +declare <16 x i8> @llvm.masked.load.v16i8(<16 x i8>*, i32, <16 x i1>, <16 x i8>) +declare <2 x half> @llvm.masked.load.v2f16(<2 x half>*, i32, <2 x i1>, <2 x half>) +declare <4 x float> @llvm.masked.load.v4f32(<4 x float>*, i32, <4 x i1>, <4 x float>) +declare <2 x double> @llvm.masked.load.v2f64(<2 x double>*, i32, <2 x i1>, <2 x double>) +declare <4 x double> @llvm.masked.load.v4f64(<4 x double>*, i32, <4 x i1>, <4 x double>) + +attributes #0 = { "target-features"="+sve" } diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-mstore.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-mstore.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-mstore.ll @@ -0,0 +1,114 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -aarch64-sve-vector-bits-min=128 -force-sve-when-streaming-compatible < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_128_STREAMING +; RUN: llc -aarch64-sve-vector-bits-min=256 -force-sve-when-streaming-compatible < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_256_STREAMING +; RUN: llc -aarch64-sve-vector-bits-min=512 -force-sve-when-streaming-compatible < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512_STREAMING + +target triple = "aarch64-unknown-linux-gnu" + +; +; Masked Store +; + +define void @masked_store_v16i8(<16 x i8>* %dst, <16 x i1> %mask) #0 { +; CHECK-LABEL: masked_store_v16i8: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x8, .LCPI0_0 +; CHECK-NEXT: add x8, x8, :lo12:.LCPI0_0 +; CHECK-NEXT: ptrue p0.b, vl16 +; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: ld1b { z1.b }, p0/z, [x8] +; CHECK-NEXT: adrp x8, .LCPI0_1 +; CHECK-NEXT: add x8, x8, :lo12:.LCPI0_1 +; CHECK-NEXT: ld1b { z2.b }, p0/z, [x8] +; CHECK-NEXT: lsl z0.b, p0/m, z0.b, z1.b +; CHECK-NEXT: asr z0.b, p0/m, z0.b, z1.b +; CHECK-NEXT: cmpne p0.b, p0/z, z0.b, #0 +; CHECK-NEXT: st1b { z2.b }, p0, [x0] +; CHECK-NEXT: ret + call void @llvm.masked.store.v16i8(<16 x i8> zeroinitializer, <16 x i8>* %dst, i32 8, <16 x i1> %mask) + ret void +} + +define void @masked_store_v2f16(<2 x half>* %dst, <2 x i1> %mask) #0 { +; CHECK-LABEL: masked_store_v2f16: +; CHECK: // %bb.0: +; CHECK-NEXT: sub sp, sp, #16 +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: str wzr, [sp, #12] +; CHECK-NEXT: mov z0.s, z0.s[1] +; CHECK-NEXT: ptrue p0.h, vl4 +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: strh w8, [sp, #8] +; CHECK-NEXT: adrp x8, .LCPI1_0 +; CHECK-NEXT: add x8, x8, :lo12:.LCPI1_0 +; CHECK-NEXT: strh w9, [sp, #10] +; CHECK-NEXT: ld1h { z0.h }, p0/z, [x8] +; CHECK-NEXT: add x8, sp, #8 +; CHECK-NEXT: ld1h { z1.h }, p0/z, [x8] +; CHECK-NEXT: adrp x8, .LCPI1_1 +; CHECK-NEXT: add x8, x8, :lo12:.LCPI1_1 +; CHECK-NEXT: ld1h { z2.h }, p0/z, [x8] +; CHECK-NEXT: lsl z1.h, p0/m, z1.h, z0.h +; CHECK-NEXT: asrr z0.h, p0/m, z0.h, z1.h +; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0 +; CHECK-NEXT: st1h { z2.h }, p0, [x0] +; CHECK-NEXT: add sp, sp, #16 +; CHECK-NEXT: ret + call void @llvm.masked.store.v2f16(<2 x half> zeroinitializer, <2 x half>* %dst, i32 8, <2 x i1> %mask) + ret void +} + +define void @masked_store_v4f32(<4 x float>* %dst, <4 x i1> %mask) #0 { +; CHECK-LABEL: masked_store_v4f32: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x8, .LCPI2_0 +; CHECK-NEXT: add x8, x8, :lo12:.LCPI2_0 +; CHECK-NEXT: ptrue p0.s, vl4 +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: uunpklo z0.s, z0.h +; CHECK-NEXT: ld1w { z1.s }, p0/z, [x8] +; CHECK-NEXT: adrp x8, .LCPI2_1 +; CHECK-NEXT: add x8, x8, :lo12:.LCPI2_1 +; CHECK-NEXT: ld1w { z2.s }, p0/z, [x8] +; CHECK-NEXT: lsl z0.s, p0/m, z0.s, z1.s +; CHECK-NEXT: asr z0.s, p0/m, z0.s, z1.s +; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0 +; CHECK-NEXT: st1w { z2.s }, p0, [x0] +; CHECK-NEXT: ret + call void @llvm.masked.store.v4f32(<4 x float> zeroinitializer, <4 x float>* %dst, i32 8, <4 x i1> %mask) + ret void +} + +define void @masked_store_v2f64(<2 x double>* %dst, <2 x i1> %mask) #0 { +; CHECK-LABEL: masked_store_v2f64: +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x8, .LCPI3_0 +; CHECK-NEXT: add x8, x8, :lo12:.LCPI3_0 +; CHECK-NEXT: ptrue p0.d, vl2 +; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: uunpklo z0.d, z0.s +; CHECK-NEXT: ld1d { z1.d }, p0/z, [x8] +; CHECK-NEXT: adrp x8, .LCPI3_1 +; CHECK-NEXT: add x8, x8, :lo12:.LCPI3_1 +; CHECK-NEXT: ld1d { z2.d }, p0/z, [x8] +; CHECK-NEXT: lsl z0.d, p0/m, z0.d, z1.d +; CHECK-NEXT: asr z0.d, p0/m, z0.d, z1.d +; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 +; CHECK-NEXT: st1d { z2.d }, p0, [x0] +; CHECK-NEXT: ret + call void @llvm.masked.store.v2f64(<2 x double> zeroinitializer, <2 x double>* %dst, i32 8, <2 x i1> %mask) + ret void +} + +declare void @llvm.masked.store.v16i8(<16 x i8>, <16 x i8>*, i32, <16 x i1>) +declare void @llvm.masked.store.v2f16(<2 x half>, <2 x half>*, i32, <2 x i1>) +declare void @llvm.masked.store.v4f32(<4 x float>, <4 x float>*, i32, <4 x i1>) +declare void @llvm.masked.store.v2f64(<2 x double>, <2 x double>*, i32, <2 x i1>) + +attributes #0 = { "target-features"="+sve" } +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; VBITS_GE_128_STREAMING: {{.*}} +; VBITS_GE_256_STREAMING: {{.*}} +; VBITS_GE_512_STREAMING: {{.*}}