Index: llvm/include/llvm/CodeGen/SelectionDAG.h =================================================================== --- llvm/include/llvm/CodeGen/SelectionDAG.h +++ llvm/include/llvm/CodeGen/SelectionDAG.h @@ -1361,7 +1361,8 @@ ISD::MemIndexType IndexType); SDValue getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef Ops, MachineMemOperand *MMO, - ISD::MemIndexType IndexType); + ISD::MemIndexType IndexType, + bool IsTruncating = false); /// Construct a node to track a Value* through the backend. SDValue getSrcValue(const Value *v); Index: llvm/include/llvm/CodeGen/SelectionDAGNodes.h =================================================================== --- llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -523,6 +523,7 @@ class StoreSDNodeBitfields { friend class StoreSDNode; friend class MaskedStoreSDNode; + friend class MaskedScatterSDNode; uint16_t : NumLSBaseSDNodeBits; @@ -2413,10 +2414,17 @@ friend class SelectionDAG; MaskedScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, - EVT MemVT, MachineMemOperand *MMO, + bool IsTrunc, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexType IndexType) : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO, - IndexType) {} + IndexType) { + StoreSDNodeBits.IsTruncating = IsTrunc; + } + + /// Return true if the op does a truncation before store. + /// For integers this is the same as doing a TRUNCATE and storing the result. + /// For floats, it is the same as doing an FP_ROUND and storing the result. + bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; } const SDValue &getValue() const { return getOperand(1); } Index: llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -9310,16 +9310,73 @@ TopHalf->isNullValue() ? RHS->getOperand(1) : LHS->getOperand(1)); } +bool refineUniformBase(SDValue &BasePtr, SDValue &Index) { + if (!isNullConstant(BasePtr) || Index.getOpcode() != ISD::ADD) + return false; + + // For now we check only the LHS of the add. + SDValue LHS = Index.getOperand(0); + if (LHS.getOpcode() != ISD::SPLAT_VECTOR) + return false; + + SDValue SplatValue = LHS.getOperand(0); + if (SplatValue.getSimpleValueType() != MVT::i64) + return false; + + BasePtr = SplatValue; + Index = Index.getOperand(1); + return true; +} + +// Fold sext/zext of index into index type. +bool refineIndexType(SDValue &Index, bool Scaled, + ISD::MemIndexType &IndexType) { + if (Index.getOpcode() == ISD::ZERO_EXTEND) { + Index = Index.getOperand(0); + IndexType = Scaled ? ISD::UNSIGNED_SCALED : ISD::UNSIGNED_UNSCALED; + return true; + } + + if (Index.getOpcode() == ISD::SIGN_EXTEND) { + Index = Index.getOperand(0); + IndexType = Scaled ? ISD::SIGNED_SCALED : ISD::SIGNED_UNSCALED; + return true; + } + + return false; +} + SDValue DAGCombiner::visitMSCATTER(SDNode *N) { MaskedScatterSDNode *MSC = cast(N); SDValue Mask = MSC->getMask(); SDValue Chain = MSC->getChain(); + SDValue Index = MSC->getIndex(); + SDValue Scale = MSC->getScale(); + SDValue StoreVal = MSC->getValue(); + SDValue BasePtr = MSC->getBasePtr(); + ISD::MemIndexType IndexType = MSC->getIndexType(); SDLoc DL(N); // Zap scatters with a zero mask. if (ISD::isBuildVectorAllZeros(Mask.getNode())) return Chain; + if (refineUniformBase(BasePtr, Index)) { + SDValue Ops[] = {Chain, StoreVal, Mask, BasePtr, Index, Scale}; + return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), + StoreVal.getValueType(), DL, Ops, + MSC->getMemOperand(), + IndexType, MSC->isTruncatingStore()); + } + + if (refineIndexType(Index, MSC->isIndexScaled(), IndexType)) { + SDValue Ops[] = {Chain, StoreVal, Mask, BasePtr, Index, Scale}; + return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), + StoreVal.getValueType(), DL, Ops, + MSC->getMemOperand(), + IndexType, MSC->isTruncatingStore()); + } + return SDValue(); } Index: llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -1851,6 +1851,7 @@ SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N, unsigned OpNo) { + bool TruncateStore = N->isTruncatingStore(); SmallVector NewOps(N->op_begin(), N->op_end()); if (OpNo == 2) { // The Mask @@ -1863,9 +1864,14 @@ NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo)); else NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo)); - } else + } else { NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo)); - return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); + TruncateStore = true; + } + + return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), N->getMemoryVT(), + SDLoc(N), NewOps, N->getMemOperand(), + N->getIndexType(), TruncateStore); } SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) { Index: llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp +++ llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp @@ -2513,14 +2513,16 @@ SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Scale}; Lo = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataLo.getValueType(), - DL, OpsLo, MMO, N->getIndexType()); + DL, OpsLo, MMO, N->getIndexType(), + N->isTruncatingStore()); // The order of the Scatter operation after split is well defined. The "Hi" // part comes after the "Lo". So these two operations should be chained one // after another. SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Scale}; return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataHi.getValueType(), - DL, OpsHi, MMO, N->getIndexType()); + DL, OpsHi, MMO, N->getIndexType(), + N->isTruncatingStore()); } SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) { @@ -4694,7 +4696,8 @@ Scale}; return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), MSC->getMemoryVT(), SDLoc(N), Ops, - MSC->getMemOperand(), MSC->getIndexType()); + MSC->getMemOperand(), MSC->getIndexType(), + MSC->isTruncatingStore()); } SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) { Index: llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -7357,14 +7357,15 @@ SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef Ops, MachineMemOperand *MMO, - ISD::MemIndexType IndexType) { + ISD::MemIndexType IndexType, + bool IsTrunc) { assert(Ops.size() == 6 && "Incompatible number of operands"); FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops); ID.AddInteger(VT.getRawBits()); ID.AddInteger(getSyntheticNodeSubclassData( - dl.getIROrder(), VTs, VT, MMO, IndexType)); + dl.getIROrder(), VTs, IsTrunc, VT, MMO, IndexType)); ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { @@ -7372,14 +7373,15 @@ return SDValue(E, 0); } auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), - VTs, VT, MMO, IndexType); + VTs, IsTrunc, VT, MMO, IndexType); createOperands(N, Ops); - assert(N->getMask().getValueType().getVectorNumElements() == - N->getValue().getValueType().getVectorNumElements() && + assert(N->getMask().getValueType().getVectorElementCount() == + N->getValue().getValueType().getVectorElementCount() && "Vector width mismatch between mask and data"); - assert(N->getIndex().getValueType().getVectorNumElements() >= - N->getValue().getValueType().getVectorNumElements() && + assert(ElementCount::isKnownGE( + N->getIndex().getValueType().getVectorElementCount(), + N->getValue().getValueType().getVectorElementCount()) && "Vector width mismatch between index and data"); assert(isa(N->getScale()) && cast(N->getScale())->getAPIntValue().isPowerOf2() && Index: llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -4297,12 +4297,12 @@ if (!UniformBase) { Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout())); Index = getValue(Ptr); - IndexType = ISD::SIGNED_SCALED; + IndexType = ISD::SIGNED_UNSCALED; Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout())); } SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale }; SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl, - Ops, MMO, IndexType); + Ops, MMO, IndexType, false); DAG.setRoot(Scatter); setValue(&I, Scatter); } Index: llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp +++ llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp @@ -735,7 +735,19 @@ OS << ", compressing"; OS << ">"; - } else if (const MemSDNode* M = dyn_cast(this)) { + } else if (const auto *MScatter = dyn_cast(this)) { + OS << "<"; + printMemOperand(OS, *MScatter->getMemOperand(), G); + + if (MScatter->isTruncatingStore()) + OS << ", trunc to " << MScatter->getMemoryVT().getEVTString(); + + auto Signed = MScatter->isIndexSigned() ? "signed" : "unsigned"; + auto Scaled = MScatter->isIndexScaled() ? "scaled" : "unscaled"; + OS << ", " << Signed << " " << Scaled << " offset"; + + OS << ">"; + } else if (const MemSDNode *M = dyn_cast(this)) { OS << "<"; printMemOperand(OS, *M->getMemOperand(), G); OS << ">"; Index: llvm/lib/Target/AArch64/AArch64ISelLowering.h =================================================================== --- llvm/lib/Target/AArch64/AArch64ISelLowering.h +++ llvm/lib/Target/AArch64/AArch64ISelLowering.h @@ -810,6 +810,8 @@ SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerMSCATTER(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const; bool isEligibleForTailCallOptimization( Index: llvm/lib/Target/AArch64/AArch64ISelLowering.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -1001,6 +1001,7 @@ setOperationAction(ISD::SINT_TO_FP, VT, Custom); setOperationAction(ISD::FP_TO_UINT, VT, Custom); setOperationAction(ISD::FP_TO_SINT, VT, Custom); + setOperationAction(ISD::MSCATTER, VT, Custom); setOperationAction(ISD::MUL, VT, Custom); setOperationAction(ISD::SPLAT_VECTOR, VT, Custom); setOperationAction(ISD::SELECT, VT, Custom); @@ -1041,6 +1042,7 @@ MVT::nxv4f32, MVT::nxv2f64}) { setOperationAction(ISD::CONCAT_VECTORS, VT, Custom); setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom); + setOperationAction(ISD::MSCATTER, VT, Custom); setOperationAction(ISD::SPLAT_VECTOR, VT, Custom); setOperationAction(ISD::SELECT, VT, Custom); setOperationAction(ISD::FADD, VT, Custom); @@ -1064,6 +1066,9 @@ setOperationAction(ISD::SPLAT_VECTOR, MVT::nxv8bf16, Custom); + for (auto VT : {MVT::nxv2bf16, MVT::nxv4bf16, MVT::nxv8bf16}) + setOperationAction(ISD::MSCATTER, VT, Custom); + setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i8, Custom); setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i16, Custom); @@ -3715,6 +3720,96 @@ ST->getBasePtr(), ST->getMemOperand()); } +unsigned getScatterVecOpcode(bool IsScaled, bool IsSigned, bool NeedsExtend) { + std::map, unsigned> AddrModes = { + {std::make_tuple(/*Scaled*/ false, /*Signed*/ false, /*Extend*/ false), + AArch64ISD::SST1_PRED}, + {std::make_tuple(/*Scaled*/ false, /*Signed*/ false, /*Extend*/ true), + AArch64ISD::SST1_UXTW_PRED}, + {std::make_tuple(/*Scaled*/ false, /*Signed*/ true, /*Extend*/ false), + AArch64ISD::SST1_PRED}, + {std::make_tuple(/*Scaled*/ false, /*Signed*/ true, /*Extend*/ true), + AArch64ISD::SST1_SXTW_PRED}, + {std::make_tuple(/*Scaled*/ true, /*Signed*/ false, /*Extend*/ false), + AArch64ISD::SST1_SCALED_PRED}, + {std::make_tuple(/*Scaled*/ true, /*Signed*/ false, /*Extend*/ true), + AArch64ISD::SST1_UXTW_SCALED_PRED}, + {std::make_tuple(/*Scaled*/ true, /*Signed*/ true, /*Extend*/ false), + AArch64ISD::SST1_SCALED_PRED}, + {std::make_tuple(/*Scaled*/ true, /*Signed*/ true, /*Extend*/ true), + AArch64ISD::SST1_SXTW_SCALED_PRED}, + }; + auto Key = std::make_tuple(IsScaled, IsSigned, NeedsExtend); + return AddrModes.find(Key)->second; +} + +bool getScatterIndexIsExtended(SDValue Index) { + unsigned Opcode = Index.getOpcode(); + if (Opcode == ISD::SIGN_EXTEND_INREG) + return true; + + if (Opcode == ISD::AND) { + SDValue Splat = Index.getOperand(1); + if (Splat.getOpcode() != ISD::SPLAT_VECTOR) + return false; + ConstantSDNode *Mask = dyn_cast(Splat.getOperand(0)); + if (!Mask || Mask->getZExtValue() != 0xFFFFFFFF) + return false; + return true; + } + + return false; +} + +SDValue AArch64TargetLowering::LowerMSCATTER(SDValue Op, + SelectionDAG &DAG) const { + SDLoc DL(Op); + MaskedScatterSDNode *MSC = cast(Op); + assert(MSC && "Can only custom lower scatter store nodes"); + + SDValue Index = MSC->getIndex(); + SDValue Chain = MSC->getChain(); + SDValue StoreVal = MSC->getValue(); + SDValue Mask = MSC->getMask(); + SDValue BasePtr = MSC->getBasePtr(); + + ISD::MemIndexType IndexType = MSC->getIndexType(); + bool IsScaled = + IndexType == ISD::SIGNED_SCALED || IndexType == ISD::UNSIGNED_SCALED; + bool IsSigned = + IndexType == ISD::SIGNED_SCALED || IndexType == ISD::SIGNED_UNSCALED; + bool NeedsExtend = getScatterIndexIsExtended(Index) || + Index.getSimpleValueType().getVectorElementType() == MVT::i32; + + EVT VT = StoreVal.getSimpleValueType(); + SDVTList VTs = DAG.getVTList(MVT::Other); + EVT MemVT = MSC->getMemoryVT(); + SDValue InputVT = DAG.getValueType(MemVT); + + if (VT.getVectorElementType() == MVT::bf16 && + !static_cast(DAG.getSubtarget()).hasBF16()) + return SDValue(); + + // Handle FP data + if (VT.isFloatingPoint()) { + VT = VT.changeVectorElementTypeToInteger(); + ElementCount EC = VT.getVectorElementCount(); + auto ScalarIntVT = + MVT::getIntegerVT(AArch64::SVEBitsPerBlock / EC.getKnownMinValue()); + StoreVal = DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, + MVT::getVectorVT(ScalarIntVT, EC), StoreVal); + + InputVT = DAG.getValueType(MemVT.changeVectorElementTypeToInteger()); + } + + if (getScatterIndexIsExtended(Index)) + Index = Index.getOperand(0); + + SDValue Ops[] = {Chain, StoreVal, Mask, BasePtr, Index, InputVT}; + return DAG.getNode(getScatterVecOpcode(IsScaled, IsSigned, NeedsExtend), DL, + VTs, Ops); +} + // Custom lowering for any store, vector or scalar and/or default or with // a truncate operations. Currently only custom lower truncate operation // from vector v4i16 to v4i8 or volatile stores of i128. @@ -3960,6 +4055,8 @@ return LowerINTRINSIC_WO_CHAIN(Op, DAG); case ISD::STORE: return LowerSTORE(Op, DAG); + case ISD::MSCATTER: + return LowerMSCATTER(Op, DAG); case ISD::VECREDUCE_ADD: case ISD::VECREDUCE_AND: case ISD::VECREDUCE_OR: Index: llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td =================================================================== --- llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -1769,6 +1769,16 @@ def : Pat<(nxv2i1 (reinterpret_cast (nxv8i1 PPR:$src))), (COPY_TO_REGCLASS PPR:$src, PPR)>; def : Pat<(nxv2i1 (reinterpret_cast (nxv4i1 PPR:$src))), (COPY_TO_REGCLASS PPR:$src, PPR)>; + def : Pat<(nxv2i64 (reinterpret_cast (nxv2f64 ZPR:$src))), (COPY_TO_REGCLASS ZPR:$src, ZPR)>; + def : Pat<(nxv2i64 (reinterpret_cast (nxv2f32 ZPR:$src))), (COPY_TO_REGCLASS ZPR:$src, ZPR)>; + def : Pat<(nxv2i64 (reinterpret_cast (nxv2f16 ZPR:$src))), (COPY_TO_REGCLASS ZPR:$src, ZPR)>; + def : Pat<(nxv4i32 (reinterpret_cast (nxv4f32 ZPR:$src))), (COPY_TO_REGCLASS ZPR:$src, ZPR)>; + def : Pat<(nxv4i32 (reinterpret_cast (nxv4f16 ZPR:$src))), (COPY_TO_REGCLASS ZPR:$src, ZPR)>; + let Predicates = [HasSVE, HasBF16] in { + def : Pat<(nxv2i64 (reinterpret_cast (nxv2bf16 ZPR:$src))), (COPY_TO_REGCLASS ZPR:$src, ZPR)>; + def : Pat<(nxv4i32 (reinterpret_cast (nxv4bf16 ZPR:$src))), (COPY_TO_REGCLASS ZPR:$src, ZPR)>; + } + def : Pat<(nxv16i1 (and PPR:$Ps1, PPR:$Ps2)), (AND_PPzPP (PTRUE_B 31), PPR:$Ps1, PPR:$Ps2)>; def : Pat<(nxv8i1 (and PPR:$Ps1, PPR:$Ps2)), Index: llvm/lib/Target/X86/X86ISelLowering.cpp =================================================================== --- llvm/lib/Target/X86/X86ISelLowering.cpp +++ llvm/lib/Target/X86/X86ISelLowering.cpp @@ -47559,7 +47559,8 @@ return DAG.getMaskedScatter(Scatter->getVTList(), Scatter->getMemoryVT(), DL, Ops, Scatter->getMemOperand(), - Scatter->getIndexType()); + Scatter->getIndexType(), + Scatter->isTruncatingStore()); } static SDValue combineGatherScatter(SDNode *N, SelectionDAG &DAG, Index: llvm/test/CodeGen/AArch64/sve-masked-scatter-32b-scaled.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/AArch64/sve-masked-scatter-32b-scaled.ll @@ -0,0 +1,265 @@ +; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s -asm-verbose=0 | FileCheck %s + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; scaled unpacked 32-bit offsets +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +; CHECK-LABEL: masked_scatter_nxv2i16_sext: +; CHECK-NEXT: st1h { z0.d }, p0, [x0, z1.d, sxtw #1] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2i16_sext( %data, i16* %base, %indexes, %masks) nounwind { + %ext = sext %indexes to + %ptrs = getelementptr i16, i16* %base, %ext + call void @llvm.masked.scatter.nxv2i16( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2i32_sext: +; CHECK-NEXT: st1w { z0.d }, p0, [x0, z1.d, sxtw #2] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2i32_sext( %data, i32* %base, %indexes, %masks) nounwind { + %ext = sext %indexes to + %ptrs = getelementptr i32, i32* %base, %ext + call void @llvm.masked.scatter.nxv2i32( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2i64_sext: +; CHECK-NEXT: st1d { z0.d }, p0, [x0, z1.d, sxtw #3] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2i64_sext( %data, i64* %base, %indexes, %masks) nounwind { + %ext = sext %indexes to + %ptrs = getelementptr i64, i64* %base, %ext + call void @llvm.masked.scatter.nxv2i64( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2f16_sext: +; CHECK-NEXT: st1h { z0.d }, p0, [x0, z1.d, sxtw #1] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2f16_sext( %data, half* %base, %indexes, %masks) nounwind { + %ext = sext %indexes to + %ptrs = getelementptr half, half* %base, %ext + call void @llvm.masked.scatter.nxv2f16( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2bf16_sext: +; CHECK-NEXT: st1h { z0.d }, p0, [x0, z1.d, sxtw #1] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2bf16_sext( %data, bfloat* %base, %indexes, %masks) nounwind #0 { + %ext = sext %indexes to + %ptrs = getelementptr bfloat, bfloat* %base, %ext + call void @llvm.masked.scatter.nxv2bf16( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2f32_sext: +; CHECK-NEXT: st1w { z0.d }, p0, [x0, z1.d, sxtw #2] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2f32_sext( %data, float* %base, %indexes, %masks) nounwind { + %ext = sext %indexes to + %ptrs = getelementptr float, float* %base, %ext + call void @llvm.masked.scatter.nxv2f32( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2f64_sext: +; CHECK-NEXT: st1d { z0.d }, p0, [x0, z1.d, sxtw #3] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2f64_sext( %data, double* %base, %indexes, %masks) nounwind { + %ext = sext %indexes to + %ptrs = getelementptr double, double* %base, %ext + call void @llvm.masked.scatter.nxv2f64( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2i16_zext: +; CHECK-NEXT: st1h { z0.d }, p0, [x0, z1.d, uxtw #1] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2i16_zext( %data, i16* %base, %indexes, %masks) nounwind { + %ext = zext %indexes to + %ptrs = getelementptr i16, i16* %base, %ext + call void @llvm.masked.scatter.nxv2i16( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2i32_zext: +; CHECK-NEXT: st1w { z0.d }, p0, [x0, z1.d, uxtw #2] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2i32_zext( %data, i32* %base, %indexes, %masks) nounwind { + %ext = zext %indexes to + %ptrs = getelementptr i32, i32* %base, %ext + call void @llvm.masked.scatter.nxv2i32( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2i64_zext: +; CHECK-NEXT: st1d { z0.d }, p0, [x0, z1.d, uxtw #3] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2i64_zext( %data, i64* %base, %indexes, %masks) nounwind { + %ext = zext %indexes to + %ptrs = getelementptr i64, i64* %base, %ext + call void @llvm.masked.scatter.nxv2i64( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2f16_zext: +; CHECK-NEXT: st1h { z0.d }, p0, [x0, z1.d, uxtw #1] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2f16_zext( %data, half* %base, %indexes, %masks) nounwind { + %ext = zext %indexes to + %ptrs = getelementptr half, half* %base, %ext + call void @llvm.masked.scatter.nxv2f16( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2bf16_zext: +; CHECK-NEXT: st1h { z0.d }, p0, [x0, z1.d, uxtw #1] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2bf16_zext( %data, bfloat* %base, %indexes, %masks) nounwind #0 { + %ext = zext %indexes to + %ptrs = getelementptr bfloat, bfloat* %base, %ext + call void @llvm.masked.scatter.nxv2bf16( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2f32_zext: +; CHECK-NEXT: st1w { z0.d }, p0, [x0, z1.d, uxtw #2] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2f32_zext( %data, float* %base, %indexes, %masks) nounwind { + %ext = zext %indexes to + %ptrs = getelementptr float, float* %base, %ext + call void @llvm.masked.scatter.nxv2f32( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2f64_zext: +; CHECK-NEXT: st1d { z0.d }, p0, [x0, z1.d, uxtw #3] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2f64_zext( %data, double* %base, %indexes, %masks) nounwind { + %ext = zext %indexes to + %ptrs = getelementptr double, double* %base, %ext + call void @llvm.masked.scatter.nxv2f64( %data, %ptrs, i32 0, %masks) + ret void +} + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; scaled packed 32-bit offset +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +; CHECK-LABEL: masked_scatter_nxv4i16_sext: +; CHECK-NEXT: st1h { z0.s }, p0, [x0, z1.s, sxtw #1] +; CHECK-NEXT: ret +define void @masked_scatter_nxv4i16_sext( %data, i16* %base, %indexes, %masks) nounwind { + %ext = sext %indexes to + %ptrs = getelementptr i16, i16* %base, %ext + call void @llvm.masked.scatter.nxv4i16( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv4i32_sext: +; CHECK-NEXT: st1w { z0.s }, p0, [x0, z1.s, sxtw #2] +; CHECK-NEXT: ret +define void @masked_scatter_nxv4i32_sext( %data, i32* %base, %indexes, %masks) nounwind { + %ext = sext %indexes to + %ptrs = getelementptr i32, i32* %base, %ext + call void @llvm.masked.scatter.nxv4i32( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv4f16_sext: +; CHECK-NEXT: st1h { z0.s }, p0, [x0, z1.s, sxtw #1] +; CHECK-NEXT: ret +define void @masked_scatter_nxv4f16_sext( %data, half* %base, %indexes, %masks) nounwind { + %ext = sext %indexes to + %ptrs = getelementptr half, half* %base, %ext + call void @llvm.masked.scatter.nxv4f16( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv4bf16_sext: +; CHECK-NEXT: st1h { z0.s }, p0, [x0, z1.s, sxtw #1] +; CHECK-NEXT: ret +define void @masked_scatter_nxv4bf16_sext( %data, bfloat* %base, %indexes, %masks) nounwind #0 { + %ext = sext %indexes to + %ptrs = getelementptr bfloat, bfloat* %base, %ext + call void @llvm.masked.scatter.nxv4bf16( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv4f32_sext: +; CHECK-NEXT: st1w { z0.s }, p0, [x0, z1.s, sxtw #2] +; CHECK-NEXT: ret +define void @masked_scatter_nxv4f32_sext( %data, float* %base, %indexes, %masks) nounwind #0 { + %ext = sext %indexes to + %ptrs = getelementptr float, float* %base, %ext + call void @llvm.masked.scatter.nxv4f32( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv4i16_zext: +; CHECK-NEXT: st1h { z0.s }, p0, [x0, z1.s, uxtw #1] +; CHECK-NEXT: ret +define void @masked_scatter_nxv4i16_zext( %data, i16* %base, %indexes, %masks) nounwind { + %ext = zext %indexes to + %ptrs = getelementptr i16, i16* %base, %ext + call void @llvm.masked.scatter.nxv4i16( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv4i32_zext: +; CHECK-NEXT: st1w { z0.s }, p0, [x0, z1.s, uxtw #2] +; CHECK-NEXT: ret +define void @masked_scatter_nxv4i32_zext( %data, i32* %base, %indexes, %masks) nounwind { + %ext = zext %indexes to + %ptrs = getelementptr i32, i32* %base, %ext + call void @llvm.masked.scatter.nxv4i32( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv4f16_zext: +; CHECK-NEXT: st1h { z0.s }, p0, [x0, z1.s, uxtw #1] +; CHECK-NEXT: ret +define void @masked_scatter_nxv4f16_zext( %data, half* %base, %indexes, %masks) nounwind { + %ext = zext %indexes to + %ptrs = getelementptr half, half* %base, %ext + call void @llvm.masked.scatter.nxv4f16( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv4bf16_zext: +; CHECK-NEXT: st1h { z0.s }, p0, [x0, z1.s, uxtw #1] +; CHECK-NEXT: ret +define void @masked_scatter_nxv4bf16_zext( %data, bfloat* %base, %indexes, %masks) nounwind #0 { + %ext = zext %indexes to + %ptrs = getelementptr bfloat, bfloat* %base, %ext + call void @llvm.masked.scatter.nxv4bf16( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv4f32_zext: +; CHECK-NEXT: st1w { z0.s }, p0, [x0, z1.s, uxtw #2] +; CHECK-NEXT: ret +define void @masked_scatter_nxv4f32_zext( %data, float* %base, %indexes, %masks) nounwind #0 { + %ext = zext %indexes to + %ptrs = getelementptr float, float* %base, %ext + call void @llvm.masked.scatter.nxv4f32( %data, %ptrs, i32 0, %masks) + ret void +} + +declare void @llvm.masked.scatter.nxv2f16(, , i32, ) +declare void @llvm.masked.scatter.nxv4f16(, , i32, ) +declare void @llvm.masked.scatter.nxv4bf16(, , i32, ) +declare void @llvm.masked.scatter.nxv4f32(, , i32, ) +declare void @llvm.masked.scatter.nxv2bf16(, , i32, ) +declare void @llvm.masked.scatter.nxv2f32(, , i32, ) +declare void @llvm.masked.scatter.nxv2f64(, , i32, ) +declare void @llvm.masked.scatter.nxv2i16(, , i32, ) +declare void @llvm.masked.scatter.nxv2i32(, , i32, ) +declare void @llvm.masked.scatter.nxv2i64(, , i32, ) +declare void @llvm.masked.scatter.nxv2i8(, , i32, ) +declare void @llvm.masked.scatter.nxv4i16(, , i32, ) +declare void @llvm.masked.scatter.nxv4i32(, , i32, ) +declare void @llvm.masked.scatter.nxv4i8(, , i32, ) +attributes #0 = { "target-features"="+sve,+bf16" } Index: llvm/test/CodeGen/AArch64/sve-masked-scatter-32b-unscaled.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/AArch64/sve-masked-scatter-32b-unscaled.ll @@ -0,0 +1,332 @@ +; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t -asm-verbose=0 | FileCheck %s + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; unscaled unpacked 32-bit offsets +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +; CHECK-LABEL: masked_scatter_nxv2i8_sext_offsets: +; CHECK-NEXT: st1b { z0.d }, p0, [x0, z1.d, sxtw] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2i8_sext_offsets( %data, i8* %base, %i32offsets, %masks) nounwind { + %offsets = sext %i32offsets to + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv2i8( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2i16_sext_offsets: +; CHECK-NEXT: st1h { z0.d }, p0, [x0, z1.d, sxtw] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2i16_sext_offsets( %data, i8* %base, %i32offsets, %masks) nounwind { + %offsets = sext %i32offsets to + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv2i16( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2i32_sext_offsets: +; CHECK-NEXT: st1w { z0.d }, p0, [x0, z1.d, sxtw] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2i32_sext_offsets( %data, i8* %base, %i32offsets, %masks) nounwind { + %offsets = sext %i32offsets to + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv2i32( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2i64_sext_offsets: +; CHECK-NEXT: st1d { z0.d }, p0, [x0, z1.d, sxtw] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2i64_sext_offsets( %data, i8* %base, %i32offsets, %masks) nounwind { + %offsets = sext %i32offsets to + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv2i64( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2f16_sext_offsets: +; CHECK-NEXT: st1h { z0.d }, p0, [x0, z1.d, sxtw] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2f16_sext_offsets( %data, i8* %base, %i32offsets, %masks) nounwind { + %offsets = sext %i32offsets to + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv2f16( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2bf16_sext_offsets: +; CHECK-NEXT: st1h { z0.d }, p0, [x0, z1.d, sxtw] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2bf16_sext_offsets( %data, i8* %base, %i32offsets, %masks) nounwind #0 { + %offsets = sext %i32offsets to + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv2bf16( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2f32_sext_offsets: +; CHECK-NEXT: st1w { z0.d }, p0, [x0, z1.d, sxtw] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2f32_sext_offsets( %data, i8* %base, %i32offsets, %masks) nounwind { + %offsets = sext %i32offsets to + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv2f32( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2f64_sext_offsets: +; CHECK-NEXT: st1d { z0.d }, p0, [x0, z1.d, sxtw] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2f64_sext_offsets( %data, i8* %base, %i32offsets, %masks) nounwind { + %offsets = sext %i32offsets to + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv2f64( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2i8_zext_offsets: +; CHECK-NEXT: st1b { z0.d }, p0, [x0, z1.d, uxtw] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2i8_zext_offsets( %data, i8* %base, %i32offsets, %masks) nounwind { + %offsets = zext %i32offsets to + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv2i8( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2i16_zext_offsets: +; CHECK-NEXT: st1h { z0.d }, p0, [x0, z1.d, uxtw] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2i16_zext_offsets( %data, i8* %base, %i32offsets, %masks) nounwind { + %offsets = zext %i32offsets to + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv2i16( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2i32_zext_offsets: +; CHECK-NEXT: st1w { z0.d }, p0, [x0, z1.d, uxtw] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2i32_zext_offsets( %data, i8* %base, %i32offsets, %masks) nounwind { + %offsets = zext %i32offsets to + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv2i32( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2i64_zext_offsets: +; CHECK-NEXT: st1d { z0.d }, p0, [x0, z1.d, uxtw] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2i64_zext_offsets( %data, i8* %base, %i32offsets, %masks) nounwind { + %offsets = zext %i32offsets to + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv2i64( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2f16_zext_offsets: +; CHECK-NEXT: st1h { z0.d }, p0, [x0, z1.d, uxtw] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2f16_zext_offsets( %data, i8* %base, %i32offsets, %masks) nounwind { + %offsets = zext %i32offsets to + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv2f16( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2bf16_zext_offsets: +; CHECK-NEXT: st1h { z0.d }, p0, [x0, z1.d, uxtw] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2bf16_zext_offsets( %data, i8* %base, %i32offsets, %masks) nounwind #0 { + %offsets = zext %i32offsets to + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv2bf16( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2f32_zext_offsets: +; CHECK-NEXT: st1w { z0.d }, p0, [x0, z1.d, uxtw] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2f32_zext_offsets( %data, i8* %base, %i32offsets, %masks) nounwind { + %offsets = zext %i32offsets to + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv2f32( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2f64_zext_offsets: +; CHECK-NEXT: st1d { z0.d }, p0, [x0, z1.d, uxtw] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2f64_zext_offsets( %data, i8* %base, %i32offsets, %masks) nounwind { + %offsets = zext %i32offsets to + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv2f64( %data, %ptrs, i32 0, %masks) + ret void +} + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; unscaled packed 32-bit offsets +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; CHECK-LABEL: masked_scatter_nxv4i8_sext_offsets: +; CHECK-NEXT: st1b { z0.s }, p0, [x0, z1.s, sxtw] +; CHECK-NEXT: ret +define void @masked_scatter_nxv4i8_sext_offsets( %data, i8* %base, %i32offsets, %masks) nounwind { + %offsets = sext %i32offsets to + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv4i8( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv4i16_sext_offsets: +; CHECK-NEXT: st1h { z0.s }, p0, [x0, z1.s, sxtw] +; CHECK-NEXT: ret +define void @masked_scatter_nxv4i16_sext_offsets( %data, i8* %base, %i32offsets, %masks) nounwind { + %offsets = sext %i32offsets to + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv4i16( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv4i32_sext_offsets: +; CHECK-NEXT: st1w { z0.s }, p0, [x0, z1.s, sxtw] +; CHECK-NEXT: ret +define void @masked_scatter_nxv4i32_sext_offsets( %data, i8* %base, %i32offsets, %masks) nounwind { + %offsets = sext %i32offsets to + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv4i32( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv4f16_sext_offsets: +; CHECK-NEXT: st1h { z0.s }, p0, [x0, z1.s, sxtw] +; CHECK-NEXT: ret +define void @masked_scatter_nxv4f16_sext_offsets( %data, i8* %base, %i32offsets, %masks) nounwind { + %offsets = sext %i32offsets to + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv4f16( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv4bf16_sext_offsets: +; CHECK-NEXT: st1h { z0.s }, p0, [x0, z1.s, sxtw] +; CHECK-NEXT: ret +define void @masked_scatter_nxv4bf16_sext_offsets( %data, i8* %base, %i32offsets, %masks) nounwind #0 { + %offsets = sext %i32offsets to + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv4bf16( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv4f32_sext_offsets: +; CHECK-NEXT: st1w { z0.s }, p0, [x0, z1.s, sxtw] +; CHECK-NEXT: ret +define void @masked_scatter_nxv4f32_sext_offsets( %data, i8* %base, %i32offsets, %masks) nounwind #0 { + %offsets = sext %i32offsets to + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv4f32( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv4i8_zext_offsets: +; CHECK-NEXT: st1b { z0.s }, p0, [x0, z1.s, uxtw] +; CHECK-NEXT: ret +define void @masked_scatter_nxv4i8_zext_offsets( %data, i8* %base, %i32offsets, %masks) nounwind { + %offsets = zext %i32offsets to + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv4i8( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv4i16_zext_offsets: +; CHECK-NEXT: st1h { z0.s }, p0, [x0, z1.s, uxtw] +; CHECK-NEXT: ret +define void @masked_scatter_nxv4i16_zext_offsets( %data, i8* %base, %i32offsets, %masks) nounwind { + %offsets = zext %i32offsets to + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv4i16( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv4i32_zext_offsets: +; CHECK-NEXT: st1w { z0.s }, p0, [x0, z1.s, uxtw] +; CHECK-NEXT: ret +define void @masked_scatter_nxv4i32_zext_offsets( %data, i8* %base, %i32offsets, %masks) nounwind { + %offsets = zext %i32offsets to + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv4i32( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv4f16_zext_offsets: +; CHECK-NEXT: st1h { z0.s }, p0, [x0, z1.s, uxtw] +; CHECK-NEXT: ret +define void @masked_scatter_nxv4f16_zext_offsets( %data, i8* %base, %i32offsets, %masks) nounwind { + %offsets = zext %i32offsets to + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv4f16( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv4bf16_zext_offsets: +; CHECK-NEXT: st1h { z0.s }, p0, [x0, z1.s, uxtw] +; CHECK-NEXT: ret +define void @masked_scatter_nxv4bf16_zext_offsets( %data, i8* %base, %i32offsets, %masks) nounwind #0 { + %offsets = zext %i32offsets to + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv4bf16( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv4f32_zext_offsets: +; CHECK-NEXT: st1w { z0.s }, p0, [x0, z1.s, uxtw] +; CHECK-NEXT: ret +define void @masked_scatter_nxv4f32_zext_offsets( %data, i8* %base, %i32offsets, %masks) nounwind #0 { + %offsets = zext %i32offsets to + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv4f32( %data, %ptrs, i32 0, %masks) + ret void +} + +declare void @llvm.masked.scatter.nxv2f16(, , i32, ) +declare void @llvm.masked.scatter.nxv4f16(, , i32, ) +declare void @llvm.masked.scatter.nxv4bf16(, , i32, ) +declare void @llvm.masked.scatter.nxv4f32(, , i32, ) +declare void @llvm.masked.scatter.nxv2bf16(, , i32, ) +declare void @llvm.masked.scatter.nxv2f32(, , i32, ) +declare void @llvm.masked.scatter.nxv2f64(, , i32, ) +declare void @llvm.masked.scatter.nxv2i16(, , i32, ) +declare void @llvm.masked.scatter.nxv2i32(, , i32, ) +declare void @llvm.masked.scatter.nxv2i64(, , i32, ) +declare void @llvm.masked.scatter.nxv2i8(, , i32, ) +declare void @llvm.masked.scatter.nxv4i16(, , i32, ) +declare void @llvm.masked.scatter.nxv4i32(, , i32, ) +declare void @llvm.masked.scatter.nxv4i8(, , i32, ) +attributes #0 = { "target-features"="+sve,+bf16" } Index: llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-scaled.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-scaled.ll @@ -0,0 +1,66 @@ +; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve < %s | FileCheck %s + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; scaled 64-bit offsets +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +define void @masked_scatter_nxv2i16( %data, i16* %base, %offsets, %mask) { +; CHECK-LABEL: masked_scatter_nxv2i16: +; CHECK: st1h { z0.d }, p0, [x0, z1.d, lsl #1] +; CHECK-NEXT: ret + %ptrs = getelementptr i16, i16* %base, %offsets + call void @llvm.masked.scatter.nxv2i16( %data, %ptrs, i32 2, %mask) + ret void +} + +define void @masked_scatter_nxv2i32( %data, i32* %base, %offsets, %mask) { +; CHECK-LABEL: masked_scatter_nxv2i32: +; CHECK: st1w { z0.d }, p0, [x0, z1.d, lsl #2] +; CHECK-NEXT: ret + %ptrs = getelementptr i32, i32* %base, %offsets + call void @llvm.masked.scatter.nxv2i32( %data, %ptrs, i32 4, %mask) + ret void +} + +define void @masked_scatter_nxv2i64( %data, i64* %base, %offsets, %mask) { +; CHECK-LABEL: masked_scatter_nxv2i64: +; CHECK: st1d { z0.d }, p0, [x0, z1.d, lsl #3] +; CHECK-NEXT: ret + %ptrs = getelementptr i64, i64* %base, %offsets + call void @llvm.masked.scatter.nxv2i64( %data, %ptrs, i32 8, %mask) + ret void +} + +define void @masked_scatter_nxv2f16( %data, half* %base, %offsets, %mask) { +; CHECK-LABEL: masked_scatter_nxv2f16: +; CHECK: st1h { z0.d }, p0, [x0, z1.d, lsl #1] +; CHECK-NEXT: ret + %ptrs = getelementptr half, half* %base, %offsets + call void @llvm.masked.scatter.nxv2f16( %data, %ptrs, i32 2, %mask) + ret void +} + +define void @masked_scatter_nxv2f32( %data, float* %base, %offsets, %mask) { +; CHECK-LABEL: masked_scatter_nxv2f32: +; CHECK: st1w { z0.d }, p0, [x0, z1.d, lsl #2] +; CHECK-NEXT: ret + %ptrs = getelementptr float, float* %base, %offsets + call void @llvm.masked.scatter.nxv2f32( %data, %ptrs, i32 4, %mask) + ret void +} + +define void @masked_scatter_nxv2f64( %data, double* %base, %offsets, %mask) { +; CHECK-LABEL: masked_scatter_nxv2f64: +; CHECK: st1d { z0.d }, p0, [x0, z1.d, lsl #3] +; CHECK-NEXT: ret + %ptrs = getelementptr double, double* %base, %offsets + call void @llvm.masked.scatter.nxv2f64( %data, %ptrs, i32 8, %mask) + ret void +} + +declare void @llvm.masked.scatter.nxv2i16(, , i32, ) +declare void @llvm.masked.scatter.nxv2i32(, , i32, ) +declare void @llvm.masked.scatter.nxv2i64(, , i32, ) +declare void @llvm.masked.scatter.nxv2f16(, , i32, ) +declare void @llvm.masked.scatter.nxv2f32(, , i32, ) +declare void @llvm.masked.scatter.nxv2f64(, , i32, ) Index: llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-unscaled.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-unscaled.ll @@ -0,0 +1,99 @@ +; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t -asm-verbose=0 | FileCheck %s + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; unscaled 64-bit offsets +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +; CHECK-LABEL: masked_scatter_nxv2i8_unscaled_64bit_offsets: +; CHECK-NEXT: st1b { z0.d }, p0, [x0, z1.d] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2i8_unscaled_64bit_offsets( %data, i8* %base, %offsets, %masks) nounwind { + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv2i8( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2i16_unscaled_64bit_offsets: +; CHECK-NEXT: st1h { z0.d }, p0, [x0, z1.d] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2i16_unscaled_64bit_offsets( %data, i8* %base, %offsets, %masks) nounwind { + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv2i16( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2i32_unscaled_64bit_offsets: +; CHECK-NEXT: st1w { z0.d }, p0, [x0, z1.d] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2i32_unscaled_64bit_offsets( %data, i8* %base, %offsets, %masks) nounwind { + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv2i32( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2i64_unscaled_64bit_offsets: +; CHECK-NEXT: st1d { z0.d }, p0, [x0, z1.d] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2i64_unscaled_64bit_offsets( %data, i8* %base, %offsets, %masks) nounwind { + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv2i64( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2f16_unscaled_64bit_offsets: +; CHECK-NEXT: st1h { z0.d }, p0, [x0, z1.d] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2f16_unscaled_64bit_offsets( %data, i8* %base, %offsets, %masks) nounwind { + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv2f16( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2bf16_unscaled_64bit_offsets: +; CHECK-NEXT: st1h { z0.d }, p0, [x0, z1.d] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2bf16_unscaled_64bit_offsets( %data, i8* %base, %offsets, %masks) nounwind #0 { + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv2bf16( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2f32_unscaled_64bit_offsets: +; CHECK-NEXT: st1w { z0.d }, p0, [x0, z1.d] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2f32_unscaled_64bit_offsets( %data, i8* %base, %offsets, %masks) nounwind #0 { + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv2f32( %data, %ptrs, i32 0, %masks) + ret void +} + +; CHECK-LABEL: masked_scatter_nxv2f64_unscaled_64bit_offsets: +; CHECK-NEXT: st1d { z0.d }, p0, [x0, z1.d] +; CHECK-NEXT: ret +define void @masked_scatter_nxv2f64_unscaled_64bit_offsets( %data, i8* %base, %offsets, %masks) nounwind #0 { + %byte_ptrs = getelementptr i8, i8* %base, %offsets + %ptrs = bitcast %byte_ptrs to + call void @llvm.masked.scatter.nxv2f64( %data, %ptrs, i32 0, %masks) + ret void +} + +declare void @llvm.masked.scatter.nxv2f16(, , i32, ) +declare void @llvm.masked.scatter.nxv4f16(, , i32, ) +declare void @llvm.masked.scatter.nxv2bf16(, , i32, ) +declare void @llvm.masked.scatter.nxv2f32(, , i32, ) +declare void @llvm.masked.scatter.nxv2f64(, , i32, ) +declare void @llvm.masked.scatter.nxv2i16(, , i32, ) +declare void @llvm.masked.scatter.nxv2i32(, , i32, ) +declare void @llvm.masked.scatter.nxv2i64(, , i32, ) +declare void @llvm.masked.scatter.nxv2i8(, , i32, ) +declare void @llvm.masked.scatter.nxv4i16(, , i32, ) +declare void @llvm.masked.scatter.nxv4i32(, , i32, ) +declare void @llvm.masked.scatter.nxv4i8(, , i32, ) +attributes #0 = { "target-features"="+sve,+bf16" }