diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td --- a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td @@ -878,33 +878,41 @@ multiclass VSLD_IV_X_I funct6, Operand optype = simm5, string vw = "v"> { def X : VALUVX, - Sched<[WriteVISlideX, ReadVISlideV, ReadVISlideX, ReadVMask]>; + Sched<[WriteVISlideX_UpperBound, ReadVISlideV_UpperBound, + ReadVISlideX_UpperBound, ReadVMask]>; def I : VALUVI, - Sched<[WriteVISlideI, ReadVISlideV, ReadVMask]>; + Sched<[WriteVISlideI_UpperBound, ReadVISlideV_UpperBound, + ReadVMask]>; } multiclass VSLD1_MV_X funct6, string vw = "v"> { def X : VALUVX, - Sched<[WriteVISlide1X, ReadVISlideV, ReadVISlideX, ReadVMask]>; + Sched<[WriteVISlide1X_UpperBound, ReadVISlideV_UpperBound, + ReadVISlideX_UpperBound, ReadVMask]>; } multiclass VSLD1_FV_F funct6, string vw = "v"> { def F : VALUVF, - Sched<[WriteVFSlide1F, ReadVFSlideV, ReadVFSlideF, ReadVMask]>; + Sched<[WriteVFSlide1F_UpperBound, ReadVFSlideV_UpperBound, + ReadVFSlideF_UpperBound, ReadVMask]>; } multiclass VGTR_IV_V_X_I funct6, Operand optype = simm5, string vw = "v"> { def V : VALUVV, - Sched<[WriteVGatherV, ReadVGatherV, ReadVGatherV, ReadVMask]>; + Sched<[WriteVGatherV_UpperBound, ReadVGatherV_UpperBound, + ReadVGatherV_UpperBound, ReadVMask]>; def X : VALUVX, - Sched<[WriteVGatherX, ReadVGatherV, ReadVGatherX, ReadVMask]>; + Sched<[WriteVGatherX_UpperBound, ReadVGatherV_UpperBound, + ReadVGatherX_UpperBound, ReadVMask]>; def I : VALUVI, - Sched<[WriteVGatherI, ReadVGatherV, ReadVMask]>; + Sched<[WriteVGatherI_UpperBound, ReadVGatherV_UpperBound, + ReadVMask]>; } multiclass VCPR_MV_Mask funct6, string vm = "v"> { def M : VALUVVNoVm, - Sched<[WriteVCompressV, ReadVCompressV, ReadVCompressV]>; + Sched<[WriteVCompressV_UpperBound, ReadVCompressV_UpperBound, + ReadVCompressV_UpperBound]>; } multiclass VWholeLoadN nf, string opcodestr, RegisterClass VRC> { @@ -1559,11 +1567,12 @@ let vm = 1, RVVConstraint = NoConstraint in { def VMV_X_S : RVInstV<0b010000, 0b00000, OPMVV, (outs GPR:$vd), (ins VR:$vs2), "vmv.x.s", "$vd, $vs2">, - Sched<[WriteVIMovVX, ReadVIMovVX]>; + Sched<[WriteVIMovVX_UpperBound, ReadVIMovVX_UpperBound]>; let Constraints = "$vd = $vd_wb" in def VMV_S_X : RVInstV2<0b010000, 0b00000, OPMVX, (outs VR:$vd_wb), (ins VR:$vd, GPR:$rs1), "vmv.s.x", "$vd, $rs1">, - Sched<[WriteVIMovXV, ReadVIMovXV, ReadVIMovXX]>; + Sched<[WriteVIMovXV_UpperBound, ReadVIMovXV_UpperBound, + ReadVIMovXX_UpperBound]>; } } // hasSideEffects = 0, mayLoad = 0, mayStore = 0 @@ -1577,11 +1586,12 @@ // Floating-Point Scalar Move Instructions def VFMV_F_S : RVInstV<0b010000, 0b00000, OPFVV, (outs FPR32:$vd), (ins VR:$vs2), "vfmv.f.s", "$vd, $vs2">, - Sched<[WriteVFMovVF, ReadVFMovVF]>; + Sched<[WriteVFMovVF_UpperBound, ReadVFMovVF_UpperBound]>; let Constraints = "$vd = $vd_wb" in def VFMV_S_F : RVInstV2<0b010000, 0b00000, OPFVF, (outs VR:$vd_wb), (ins VR:$vd, FPR32:$rs1), "vfmv.s.f", "$vd, $rs1">, - Sched<[WriteVFMovFV, ReadVFMovFV, ReadVFMovFX]>; + Sched<[WriteVFMovFV_UpperBound, ReadVFMovFV_UpperBound, + ReadVFMovFX_UpperBound]>; } // hasSideEffects = 0, mayLoad = 0, mayStore = 0, vm = 1 @@ -1609,7 +1619,8 @@ let Constraints = "@earlyclobber $vd", RVVConstraint = Vrgather in { defm VRGATHER_V : VGTR_IV_V_X_I<"vrgather", 0b001100, uimm5>; def VRGATHEREI16_VV : VALUVV<0b001110, OPIVV, "vrgatherei16.vv">, - Sched<[WriteVGatherV, ReadVGatherV, ReadVGatherV]>; + Sched<[WriteVGatherV_UpperBound, ReadVGatherV_UpperBound, + ReadVGatherV_UpperBound]>; } // Constraints = "@earlyclobber $vd", RVVConstraint = Vrgather // Vector Compress Instruction diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td @@ -1889,9 +1889,13 @@ multiclass VPseudoVCPR_V { foreach m = MxList in { + defvar mx = m.MX; + defvar WriteVCompressV_MX = !cast("WriteVCompressV_" # mx); + defvar ReadVCompressV_MX = !cast("ReadVCompressV_" # mx); + let VLMul = m.value in def _VM # "_" # m.MX : VPseudoUnaryAnyMask, - Sched<[WriteVCompressV, ReadVCompressV, ReadVCompressV]>; + Sched<[WriteVCompressV_MX, ReadVCompressV_MX, ReadVCompressV_MX]>; } } @@ -1971,6 +1975,10 @@ multiclass VPseudoVGTR_VV_EEW { foreach m = MxList in { + defvar mx = m.MX; + defvar WriteVGatherV_MX = !cast("WriteVGatherV_" # mx); + defvar ReadVGatherV_MX = !cast("ReadVGatherV_" # mx); + foreach sew = EEWList in { defvar octuple_lmul = m.octuple; // emul = lmul * eew / sew @@ -1979,7 +1987,7 @@ defvar emulMX = octuple_to_str.ret; defvar emul = !cast("V_" # emulMX); defm _VV : VPseudoBinaryEmul, - Sched<[WriteVGatherV, ReadVGatherV, ReadVGatherV]>; + Sched<[WriteVGatherV_MX, ReadVGatherV_MX, ReadVGatherV_MX]>; } } } @@ -1995,9 +2003,15 @@ } multiclass VPseudoVSLD1_VX { - foreach m = MxList in + foreach m = MxList in { + defvar mx = m.MX; + defvar WriteVISlide1X_MX = !cast("WriteVISlide1X_" # mx); + defvar ReadVISlideV_MX = !cast("ReadVISlideV_" # mx); + defvar ReadVISlideX_MX = !cast("ReadVISlideX_" # mx); + defm "_VX" : VPseudoBinary, - Sched<[WriteVISlide1X, ReadVISlideV, ReadVISlideX, ReadVMask]>; + Sched<[WriteVISlide1X_MX, ReadVISlideV_MX, ReadVISlideX_MX, ReadVMask]>; + } } multiclass VPseudoBinaryV_VF { @@ -2006,11 +2020,18 @@ } multiclass VPseudoVSLD1_VF { - foreach f = FPList in - foreach m = f.MxList in + foreach f = FPList in { + foreach m = f.MxList in { + defvar mx = m.MX; + defvar WriteVFSlide1F_MX = !cast("WriteVFSlide1F_" # mx); + defvar ReadVFSlideV_MX = !cast("ReadVFSlideV_" # mx); + defvar ReadVFSlideF_MX = !cast("ReadVFSlideF_" # mx); + defm "_V" # f.FX : VPseudoBinary, - Sched<[WriteVFSlide1F, ReadVFSlideV, ReadVFSlideF, ReadVMask]>; + Sched<[WriteVFSlide1F_MX, ReadVFSlideV_MX, ReadVFSlideF_MX, ReadVMask]>; + } + } } multiclass VPseudoBinaryV_VI { @@ -2409,12 +2430,21 @@ } multiclass VPseudoVGTR_VV_VX_VI { - defm "" : VPseudoBinaryV_VV, - Sched<[WriteVGatherV, ReadVGatherV, ReadVGatherV, ReadVMask]>; - defm "" : VPseudoBinaryV_VX, - Sched<[WriteVGatherX, ReadVGatherV, ReadVGatherX, ReadVMask]>; - defm "" : VPseudoBinaryV_VI, - Sched<[WriteVGatherI, ReadVGatherV, ReadVMask]>; + foreach m = MxList in { + defvar mx = m.MX; + defvar WriteVGatherV_MX = !cast("WriteVGatherV_" # mx); + defvar WriteVGatherX_MX = !cast("WriteVGatherX_" # mx); + defvar WriteVGatherI_MX = !cast("WriteVGatherI_" # mx); + defvar ReadVGatherV_MX = !cast("ReadVGatherV_" # mx); + defvar ReadVGatherX_MX = !cast("ReadVGatherX_" # mx); + + defm "" : VPseudoBinaryV_VV_LMUL, + Sched<[WriteVGatherV_MX, ReadVGatherV_MX, ReadVGatherV_MX, ReadVMask]>; + defm "" : VPseudoBinaryV_VX_LMUL, + Sched<[WriteVGatherX_MX, ReadVGatherV_MX, ReadVGatherX_MX, ReadVMask]>; + defm "" : VPseudoBinaryV_VI_LMUL, + Sched<[WriteVGatherI_MX, ReadVGatherV_MX, ReadVMask]>; + } } multiclass VPseudoVSALU_VV_VX_VI { @@ -3086,9 +3116,8 @@ Constraint, /*Commutable*/1>; } -multiclass VPseudoVSLDV_VX { - foreach m = MxList in - defm _VX : VPseudoTernaryWithPolicy; +multiclass VPseudoVSLDV_VX { + defm _VX : VPseudoTernaryWithPolicy; } multiclass VPseudoTernaryV_VX_AAXA { @@ -3127,9 +3156,8 @@ m.vrclass, m, constraint>; } -multiclass VPseudoVSLDV_VI { - foreach m = MxList in - defm _VI : VPseudoTernaryWithPolicy; +multiclass VPseudoVSLDV_VI { + defm _VI : VPseudoTernaryWithPolicy; } multiclass VPseudoVMAC_VV_VX_AAXA { @@ -3173,10 +3201,19 @@ } multiclass VPseudoVSLD_VX_VI { - defm "" : VPseudoVSLDV_VX, - Sched<[WriteVISlideX, ReadVISlideV, ReadVISlideV, ReadVISlideX, ReadVMask]>; - defm "" : VPseudoVSLDV_VI, - Sched<[WriteVISlideI, ReadVISlideV, ReadVISlideV, ReadVMask]>; + foreach m = MxList in { + defvar mx = m.MX; + defvar WriteVISlideX_MX = !cast("WriteVISlideX_" # mx); + defvar WriteVISlideI_MX = !cast("WriteVISlideI_" # mx); + defvar ReadVISlideV_MX = !cast("ReadVISlideV_" # mx); + defvar ReadVISlideX_MX = !cast("ReadVISlideX_" # mx); + + defm "" : VPseudoVSLDV_VX, + Sched<[WriteVISlideX_MX, ReadVISlideV_MX, ReadVISlideV_MX, + ReadVISlideX_MX, ReadVMask]>; + defm "" : VPseudoVSLDV_VI, + Sched<[WriteVISlideI_MX, ReadVISlideV_MX, ReadVISlideV_MX, ReadVMask]>; + } } multiclass VPseudoVWMAC_VV_VX { @@ -5689,19 +5726,25 @@ let Predicates = [HasVInstructions] in { let mayLoad = 0, mayStore = 0, hasSideEffects = 0 in { foreach m = MxList in { + defvar mx = m.MX; + defvar WriteVIMovVX_MX = !cast("WriteVIMovVX_" # mx); + defvar WriteVIMovXV_MX = !cast("WriteVIMovXV_" # mx); + defvar ReadVIMovVX_MX = !cast("ReadVIMovVX_" # mx); + defvar ReadVIMovXV_MX = !cast("ReadVIMovXV_" # mx); + defvar ReadVIMovXX_MX = !cast("ReadVIMovXX_" # mx); let VLMul = m.value in { let HasSEWOp = 1, BaseInstr = VMV_X_S in - def PseudoVMV_X_S # "_" # m.MX: + def PseudoVMV_X_S # "_" # mx: Pseudo<(outs GPR:$rd), (ins m.vrclass:$rs2, ixlenimm:$sew), []>, - Sched<[WriteVIMovVX, ReadVIMovVX]>, + Sched<[WriteVIMovVX_MX, ReadVIMovVX_MX]>, RISCVVPseudo; let HasVLOp = 1, HasSEWOp = 1, BaseInstr = VMV_S_X, Constraints = "$rd = $rs1" in - def PseudoVMV_S_X # "_" # m.MX: Pseudo<(outs m.vrclass:$rd), + def PseudoVMV_S_X # "_" # mx: Pseudo<(outs m.vrclass:$rd), (ins m.vrclass:$rs1, GPR:$rs2, AVL:$vl, ixlenimm:$sew), []>, - Sched<[WriteVIMovXV, ReadVIMovXV, ReadVIMovXX]>, + Sched<[WriteVIMovXV_MX, ReadVIMovXV_MX, ReadVIMovXX_MX]>, RISCVVPseudo; } } @@ -5716,21 +5759,27 @@ let mayLoad = 0, mayStore = 0, hasSideEffects = 0 in { foreach f = FPList in { foreach m = f.MxList in { + defvar mx = m.MX; + defvar WriteVFMovVF_MX = !cast("WriteVFMovVF_" # mx); + defvar WriteVFMovFV_MX = !cast("WriteVFMovFV_" # mx); + defvar ReadVFMovVF_MX = !cast("ReadVFMovVF_" # mx); + defvar ReadVFMovFV_MX = !cast("ReadVFMovFV_" # mx); + defvar ReadVFMovFX_MX = !cast("ReadVFMovFX_" # mx); let VLMul = m.value in { let HasSEWOp = 1, BaseInstr = VFMV_F_S in - def "PseudoVFMV_" # f.FX # "_S_" # m.MX : + def "PseudoVFMV_" # f.FX # "_S_" # mx : Pseudo<(outs f.fprclass:$rd), (ins m.vrclass:$rs2, ixlenimm:$sew), []>, - Sched<[WriteVFMovVF, ReadVFMovVF]>, + Sched<[WriteVFMovVF_MX, ReadVFMovVF_MX]>, RISCVVPseudo; let HasVLOp = 1, HasSEWOp = 1, BaseInstr = VFMV_S_F, Constraints = "$rd = $rs1" in - def "PseudoVFMV_S_" # f.FX # "_" # m.MX : + def "PseudoVFMV_S_" # f.FX # "_" # mx : Pseudo<(outs m.vrclass:$rd), (ins m.vrclass:$rs1, f.fprclass:$rs2, AVL:$vl, ixlenimm:$sew), []>, - Sched<[WriteVFMovFV, ReadVFMovFV, ReadVFMovFX]>, + Sched<[WriteVFMovFV_MX, ReadVFMovFV_MX, ReadVFMovFX_MX]>, RISCVVPseudo; } } diff --git a/llvm/lib/Target/RISCV/RISCVScheduleV.td b/llvm/lib/Target/RISCV/RISCVScheduleV.td --- a/llvm/lib/Target/RISCV/RISCVScheduleV.td +++ b/llvm/lib/Target/RISCV/RISCVScheduleV.td @@ -325,23 +325,24 @@ // 16. Vector Permutation Instructions // 16.1. Integer Scalar Move Instructions -def WriteVIMovVX : SchedWrite; -def WriteVIMovXV : SchedWrite; +defm "" : LMULSchedWrites<"WriteVIMovVX">; +defm "" : LMULSchedWrites<"WriteVIMovXV">; // 16.2. Floating-Point Scalar Move Instructions -def WriteVFMovVF : SchedWrite; -def WriteVFMovFV : SchedWrite; +defm "" : LMULSchedWrites<"WriteVFMovVF">; +defm "" : LMULSchedWrites<"WriteVFMovFV">; // 16.3. Vector Slide Instructions -def WriteVISlideX : SchedWrite; -def WriteVISlideI : SchedWrite; -def WriteVISlide1X : SchedWrite; -def WriteVFSlide1F : SchedWrite; +defm "" : LMULSchedWrites<"WriteVISlideX">; +defm "" : LMULSchedWrites<"WriteVISlideI">; +defm "" : LMULSchedWrites<"WriteVISlide1X">; +defm "" : LMULSchedWrites<"WriteVFSlide1F">; // 16.4. Vector Register Gather Instructions -def WriteVGatherV : SchedWrite; -def WriteVGatherX : SchedWrite; -def WriteVGatherI : SchedWrite; +defm "" : LMULSchedWrites<"WriteVGatherV">; +defm "" : LMULSchedWrites<"WriteVGatherX">; +defm "" : LMULSchedWrites<"WriteVGatherI">; // 16.5. Vector Compress Instruction -def WriteVCompressV : SchedWrite; +defm "" : LMULSchedWrites<"WriteVCompressV">; // 16.6. Whole Vector Register Move +// These are already LMUL aware def WriteVMov1V : SchedWrite; def WriteVMov2V : SchedWrite; def WriteVMov4V : SchedWrite; @@ -543,24 +544,25 @@ // 16. Vector Permutation Instructions // 16.1. Integer Scalar Move Instructions -def ReadVIMovVX : SchedRead; -def ReadVIMovXV : SchedRead; -def ReadVIMovXX : SchedRead; +defm "" : LMULSchedReads<"ReadVIMovVX">; +defm "" : LMULSchedReads<"ReadVIMovXV">; +defm "" : LMULSchedReads<"ReadVIMovXX">; // 16.2. Floating-Point Scalar Move Instructions -def ReadVFMovVF : SchedRead; -def ReadVFMovFV : SchedRead; -def ReadVFMovFX : SchedRead; +defm "" : LMULSchedReads<"ReadVFMovVF">; +defm "" : LMULSchedReads<"ReadVFMovFV">; +defm "" : LMULSchedReads<"ReadVFMovFX">; // 16.3. Vector Slide Instructions -def ReadVISlideV : SchedRead; -def ReadVISlideX : SchedRead; -def ReadVFSlideV : SchedRead; -def ReadVFSlideF : SchedRead; +defm "" : LMULSchedReads<"ReadVISlideV">; +defm "" : LMULSchedReads<"ReadVISlideX">; +defm "" : LMULSchedReads<"ReadVFSlideV">; +defm "" : LMULSchedReads<"ReadVFSlideF">; // 16.4. Vector Register Gather Instructions -def ReadVGatherV : SchedRead; -def ReadVGatherX : SchedRead; +defm "" : LMULSchedReads<"ReadVGatherV">; +defm "" : LMULSchedReads<"ReadVGatherX">; // 16.5. Vector Compress Instruction -def ReadVCompressV : SchedRead; +defm "" : LMULSchedReads<"ReadVCompressV">; // 16.6. Whole Vector Register Move +// These are already LMUL aware def ReadVMov1V : SchedRead; def ReadVMov2V : SchedRead; def ReadVMov4V : SchedRead; @@ -738,18 +740,19 @@ defm "" : LMULWriteRes<"WriteVMIdxV", []>; // 16. Vector Permutation Instructions -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; +defm "" : LMULWriteRes<"WriteVIMovVX", []>; +defm "" : LMULWriteRes<"WriteVIMovXV", []>; +defm "" : LMULWriteRes<"WriteVFMovVF", []>; +defm "" : LMULWriteRes<"WriteVFMovFV", []>; +defm "" : LMULWriteRes<"WriteVISlideX", []>; +defm "" : LMULWriteRes<"WriteVISlideI", []>; +defm "" : LMULWriteRes<"WriteVISlide1X", []>; +defm "" : LMULWriteRes<"WriteVFSlide1F", []>; +defm "" : LMULWriteRes<"WriteVGatherV", []>; +defm "" : LMULWriteRes<"WriteVGatherX", []>; +defm "" : LMULWriteRes<"WriteVGatherI", []>; +defm "" : LMULWriteRes<"WriteVCompressV", []>; +// These are already LMUL aware def : WriteRes; def : WriteRes; def : WriteRes; @@ -892,19 +895,20 @@ defm "" : LMULReadAdvance<"ReadVMIotV", 0>; // 16. Vector Permutation Instructions -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; +defm "" : LMULReadAdvance<"ReadVIMovVX", 0>; +defm "" : LMULReadAdvance<"ReadVIMovXV", 0>; +defm "" : LMULReadAdvance<"ReadVIMovXX", 0>; +defm "" : LMULReadAdvance<"ReadVFMovVF", 0>; +defm "" : LMULReadAdvance<"ReadVFMovFV", 0>; +defm "" : LMULReadAdvance<"ReadVFMovFX", 0>; +defm "" : LMULReadAdvance<"ReadVISlideV", 0>; +defm "" : LMULReadAdvance<"ReadVISlideX", 0>; +defm "" : LMULReadAdvance<"ReadVFSlideV", 0>; +defm "" : LMULReadAdvance<"ReadVFSlideF", 0>; +defm "" : LMULReadAdvance<"ReadVGatherV", 0>; +defm "" : LMULReadAdvance<"ReadVGatherX", 0>; +defm "" : LMULReadAdvance<"ReadVCompressV", 0>; +// These are already LMUL aware def : ReadAdvance; def : ReadAdvance; def : ReadAdvance;