diff --git a/llvm/include/llvm/IR/IntrinsicsRISCV.td b/llvm/include/llvm/IR/IntrinsicsRISCV.td --- a/llvm/include/llvm/IR/IntrinsicsRISCV.td +++ b/llvm/include/llvm/IR/IntrinsicsRISCV.td @@ -543,6 +543,25 @@ LLVMMatchType<1>]), [NoCapture>, IntrReadMem]>, RISCVVIntrinsic; + // For indexed segment load + // Input: (pointer, index, vl) + class RISCVISegLoad + : Intrinsic, + !add(nf, -1))), + [LLVMPointerToElt<0>, llvm_anyvector_ty, llvm_anyint_ty], + [NoCapture>, IntrReadMem]>, RISCVVIntrinsic; + // For indexed segment load with mask + // Input: (maskedoff, pointer, index, mask, vl) + class RISCVISegLoadMask + : Intrinsic, + !add(nf, -1))), + !listconcat(!listsplat(LLVMMatchType<0>, nf), + [LLVMPointerToElt<0>, + llvm_anyvector_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_anyint_ty]), + [NoCapture>, IntrReadMem]>, RISCVVIntrinsic; + // For unit stride segment store // Input: (value, pointer, vl) class RISCVUSSegStore @@ -696,6 +715,10 @@ def "int_riscv_" # NAME : RISCVSSegLoad; def "int_riscv_" # NAME # "_mask" : RISCVSSegLoadMask; } + multiclass RISCVISegLoad { + def "int_riscv_" # NAME : RISCVISegLoad; + def "int_riscv_" # NAME # "_mask" : RISCVISegLoadMask; + } multiclass RISCVUSSegStore { def "int_riscv_" # NAME : RISCVUSSegStore; def "int_riscv_" # NAME # "_mask" : RISCVUSSegStoreMask; @@ -1002,6 +1025,8 @@ foreach nf = [2, 3, 4, 5, 6, 7, 8] in { defm vlseg # nf : RISCVUSSegLoad; defm vlsseg # nf : RISCVSSegLoad; + defm vloxseg # nf : RISCVISegLoad; + defm vluxseg # nf : RISCVISegLoad; defm vsseg # nf : RISCVUSSegStore; defm vssseg # nf : RISCVSSegStore; } diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h --- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h +++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h @@ -57,6 +57,8 @@ void selectVLSEG(SDNode *Node, unsigned IntNo, bool IsStrided); void selectVLSEGMask(SDNode *Node, unsigned IntNo, bool IsStrided); + void selectVLXSEG(SDNode *Node, unsigned IntNo); + void selectVLXSEGMask(SDNode *Node, unsigned IntNo); void selectVSSEG(SDNode *Node, unsigned IntNo, bool IsStrided); void selectVSSEGMask(SDNode *Node, unsigned IntNo, bool IsStrided); diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp --- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp @@ -169,7 +169,8 @@ Operands.push_back(SEW); Operands.push_back(Node->getOperand(0)); // Chain. const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo( - IntNo, ScalarSize, static_cast(LMUL)); + IntNo, ScalarSize, static_cast(LMUL), + static_cast(RISCVVLMUL::LMUL_1)); SDNode *Load = CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands); SDValue SuperReg = SDValue(Load, 0); @@ -207,7 +208,79 @@ Operands.push_back(SEW); Operands.push_back(Node->getOperand(0)); /// Chain. const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo( - IntNo, ScalarSize, static_cast(LMUL)); + IntNo, ScalarSize, static_cast(LMUL), + static_cast(RISCVVLMUL::LMUL_1)); + SDNode *Load = + CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands); + SDValue SuperReg = SDValue(Load, 0); + for (unsigned I = 0; I < NF; ++I) + ReplaceUses(SDValue(Node, I), + CurDAG->getTargetExtractSubreg(getSubregIndexByEVT(VT, I), DL, + VT, SuperReg)); + + ReplaceUses(SDValue(Node, NF), SDValue(Load, 1)); + CurDAG->RemoveDeadNode(Node); +} + +void RISCVDAGToDAGISel::selectVLXSEG(SDNode *Node, unsigned IntNo) { + SDLoc DL(Node); + unsigned NF = Node->getNumValues() - 1; + EVT VT = Node->getValueType(0); + unsigned ScalarSize = VT.getScalarSizeInBits(); + MVT XLenVT = Subtarget->getXLenVT(); + RISCVVLMUL LMUL = getLMUL(VT); + SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT); + SDValue Operands[] = { + Node->getOperand(2), // Base pointer. + Node->getOperand(3), // Index. + Node->getOperand(4), // VL. + SEW, Node->getOperand(0) // Chain. + }; + + EVT IndexVT = Node->getOperand(3)->getValueType(0); + RISCVVLMUL IndexLMUL = getLMUL(IndexVT); + unsigned IndexScalarSize = IndexVT.getScalarSizeInBits(); + const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo( + IntNo, IndexScalarSize, static_cast(LMUL), + static_cast(IndexLMUL)); + SDNode *Load = + CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands); + SDValue SuperReg = SDValue(Load, 0); + for (unsigned I = 0; I < NF; ++I) + ReplaceUses(SDValue(Node, I), + CurDAG->getTargetExtractSubreg(getSubregIndexByEVT(VT, I), DL, + VT, SuperReg)); + + ReplaceUses(SDValue(Node, NF), SDValue(Load, 1)); + CurDAG->RemoveDeadNode(Node); +} + +void RISCVDAGToDAGISel::selectVLXSEGMask(SDNode *Node, unsigned IntNo) { + SDLoc DL(Node); + unsigned NF = Node->getNumValues() - 1; + EVT VT = Node->getValueType(0); + unsigned ScalarSize = VT.getScalarSizeInBits(); + MVT XLenVT = Subtarget->getXLenVT(); + RISCVVLMUL LMUL = getLMUL(VT); + SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT); + SmallVector Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF); + SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL); + SDValue Operands[] = { + MaskedOff, + Node->getOperand(NF + 2), // Base pointer. + Node->getOperand(NF + 3), // Index. + Node->getOperand(NF + 4), // Mask. + Node->getOperand(NF + 5), // VL. + SEW, + Node->getOperand(0) // Chain. + }; + + EVT IndexVT = Node->getOperand(NF + 3)->getValueType(0); + RISCVVLMUL IndexLMUL = getLMUL(IndexVT); + unsigned IndexScalarSize = IndexVT.getScalarSizeInBits(); + const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo( + IntNo, IndexScalarSize, static_cast(LMUL), + static_cast(IndexLMUL)); SDNode *Load = CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands); SDValue SuperReg = SDValue(Load, 0); @@ -245,7 +318,8 @@ Operands.push_back(SEW); Operands.push_back(Node->getOperand(0)); // Chain. const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo( - IntNo, ScalarSize, static_cast(LMUL)); + IntNo, ScalarSize, static_cast(LMUL), + static_cast(RISCVVLMUL::LMUL_1)); SDNode *Store = CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands); ReplaceNode(Node, Store); @@ -278,7 +352,8 @@ Operands.push_back(SEW); Operands.push_back(Node->getOperand(0)); // Chain. const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo( - IntNo, ScalarSize, static_cast(LMUL)); + IntNo, ScalarSize, static_cast(LMUL), + static_cast(RISCVVLMUL::LMUL_1)); SDNode *Store = CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands); ReplaceNode(Node, Store); @@ -446,6 +521,40 @@ selectVLSEGMask(Node, IntNo, /*IsStrided=*/true); return; } + case Intrinsic::riscv_vloxseg2: + case Intrinsic::riscv_vloxseg3: + case Intrinsic::riscv_vloxseg4: + case Intrinsic::riscv_vloxseg5: + case Intrinsic::riscv_vloxseg6: + case Intrinsic::riscv_vloxseg7: + case Intrinsic::riscv_vloxseg8: + case Intrinsic::riscv_vluxseg2: + case Intrinsic::riscv_vluxseg3: + case Intrinsic::riscv_vluxseg4: + case Intrinsic::riscv_vluxseg5: + case Intrinsic::riscv_vluxseg6: + case Intrinsic::riscv_vluxseg7: + case Intrinsic::riscv_vluxseg8: { + selectVLXSEG(Node, IntNo); + return; + } + case Intrinsic::riscv_vloxseg2_mask: + case Intrinsic::riscv_vloxseg3_mask: + case Intrinsic::riscv_vloxseg4_mask: + case Intrinsic::riscv_vloxseg5_mask: + case Intrinsic::riscv_vloxseg6_mask: + case Intrinsic::riscv_vloxseg7_mask: + case Intrinsic::riscv_vloxseg8_mask: + case Intrinsic::riscv_vluxseg2_mask: + case Intrinsic::riscv_vluxseg3_mask: + case Intrinsic::riscv_vluxseg4_mask: + case Intrinsic::riscv_vluxseg5_mask: + case Intrinsic::riscv_vluxseg6_mask: + case Intrinsic::riscv_vluxseg7_mask: + case Intrinsic::riscv_vluxseg8_mask: { + selectVLXSEGMask(Node, IntNo); + return; + } } break; } diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h --- a/llvm/lib/Target/RISCV/RISCVISelLowering.h +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h @@ -330,6 +330,7 @@ unsigned int IntrinsicID; unsigned int SEW; unsigned int LMUL; + unsigned int IndexLMUL; unsigned int Pseudo; }; diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td @@ -417,17 +417,18 @@ let PrimaryKeyName = "getRISCVVIntrinsicInfo"; } -class RISCVZvlsseg S, bits<3> L> { +class RISCVZvlsseg S, bits<3> L, bits<3> IL = V_M1.value> { Intrinsic IntrinsicID = !cast(IntrName); bits<11> SEW = S; bits<3> LMUL = L; + bits<3> IndexLMUL = IL; Pseudo Pseudo = !cast(NAME); } def RISCVZvlssegTable : GenericTable { let FilterClass = "RISCVZvlsseg"; - let Fields = ["IntrinsicID", "SEW", "LMUL", "Pseudo"]; - let PrimaryKey = ["IntrinsicID", "SEW", "LMUL"]; + let Fields = ["IntrinsicID", "SEW", "LMUL", "IndexLMUL", "Pseudo"]; + let PrimaryKey = ["IntrinsicID", "SEW", "LMUL", "IndexLMUL"]; let PrimaryKeyName = "getPseudo"; } @@ -458,7 +459,9 @@ string L = !subst("VLSEG", "vlseg", !subst("VLSSEG", "vlsseg", !subst("VSSEG", "vsseg", - !subst("VSSSEG", "vssseg", Upper)))); + !subst("VSSSEG", "vssseg", + !subst("VLOXSEG", "vloxseg", + !subst("VLUXSEG", "vluxseg", Upper)))))); } // Example: PseudoVLSEG2E32_V_M2 -> int_riscv_vlseg2 @@ -470,7 +473,11 @@ !subst("E16", "", !subst("E32", "", !subst("E64", "", - !subst("_V", "", PseudoToVInst.VInst)))))>.L, + !subst("EI8", "", + !subst("EI16", "", + !subst("EI32", "", + !subst("EI64", "", + !subst("_V", "", PseudoToVInst.VInst)))))))))>.L, !if(IsMasked, "_mask", "")); } @@ -1062,6 +1069,45 @@ let BaseInstr = !cast(PseudoToVInst.VInst); } +class VPseudoISegLoadNoMask EEW, bits<3> LMUL>: + Pseudo<(outs RetClass:$rd), + (ins GPR:$rs1, IdxClass:$offset, GPR:$vl, ixlenimm:$sew),[]>, + RISCVVPseudo, + RISCVZvlsseg.Intrinsic, EEW, VLMul, LMUL> { + let mayLoad = 1; + let mayStore = 0; + let hasSideEffects = 0; + let usesCustomInserter = 1; + // For vector indexed segment loads, the destination vector register groups + // cannot overlap the source vector register group + let Constraints = "@earlyclobber $rd"; + let Uses = [VL, VTYPE]; + let HasVLOp = 1; + let HasSEWOp = 1; + let HasDummyMask = 1; + let BaseInstr = !cast(PseudoToVInst.VInst); +} + +class VPseudoISegLoadMask EEW, bits<3> LMUL>: + Pseudo<(outs GetVRegNoV0.R:$rd), + (ins GetVRegNoV0.R:$merge, GPR:$rs1, + IdxClass:$offset, VMaskOp:$vm, GPR:$vl, ixlenimm:$sew),[]>, + RISCVVPseudo, + RISCVZvlsseg.Intrinsic, EEW, VLMul, LMUL> { + let mayLoad = 1; + let mayStore = 0; + let hasSideEffects = 0; + let usesCustomInserter = 1; + // For vector indexed segment loads, the destination vector register groups + // cannot overlap the source vector register group + let Constraints = "@earlyclobber $rd, $rd = $merge"; + let Uses = [VL, VTYPE]; + let HasVLOp = 1; + let HasSEWOp = 1; + let HasMergeOp = 1; + let BaseInstr = !cast(PseudoToVInst.VInst); +} + class VPseudoUSSegStoreNoMask EEW>: Pseudo<(outs), (ins ValClass:$rd, GPR:$rs1, GPR:$vl, ixlenimm:$sew),[]>, @@ -1693,6 +1739,27 @@ } } +multiclass VPseudoISegLoad { + foreach idx_eew = EEWList in { // EEW for index argument. + foreach idx_lmul = MxSet.m in { // LMUL for index argument. + foreach val_lmul = MxList.m in { // LMUL for the value. + defvar IdxLInfo = idx_lmul.MX; + defvar IdxVreg = idx_lmul.vrclass; + defvar ValLInfo = val_lmul.MX; + let VLMul = val_lmul.value in { + foreach nf = NFSet.L in { + defvar ValVreg = SegRegClass.RC; + def nf # "EI" # idx_eew # "_V_" # IdxLInfo # "_" # ValLInfo : + VPseudoISegLoadNoMask; + def nf # "EI" # idx_eew # "_V_" # IdxLInfo # "_" # ValLInfo # "_MASK" : + VPseudoISegLoadMask; + } + } + } + } + } +} + multiclass VPseudoUSSegStore { foreach eew = EEWList in { foreach lmul = MxSet.m in { @@ -2965,6 +3032,8 @@ //===----------------------------------------------------------------------===// defm PseudoVLSEG : VPseudoUSSegLoad; defm PseudoVLSSEG : VPseudoSSegLoad; +defm PseudoVLOXSEG : VPseudoISegLoad; +defm PseudoVLUXSEG : VPseudoISegLoad; defm PseudoVSSEG : VPseudoUSSegStore; defm PseudoVSSSEG : VPseudoSSegStore;