diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td @@ -115,8 +115,14 @@ // Used for widening and narrowing instructions as it doesn't contain M8. defvar MxListW = [V_MF8, V_MF4, V_MF2, V_M1, V_M2, V_M4]; +// Used for widening reductions. It can contain M8 because wider operands are +// scalar operands. +defvar MxListWRed = MxList; // For floating point which don't need MF8. defvar MxListFW = [V_MF4, V_MF2, V_M1, V_M2, V_M4]; +// For widening floating-point Reduction as it doesn't contain MF8. It can +// contain M8 because wider operands are scalar operands. +defvar MxListFWRed = [V_MF4, V_MF2, V_M1, V_M2, V_M4, V_M8]; // Use for zext/sext.vf2 defvar MxListVF2 = [V_MF4, V_MF2, V_M1, V_M2, V_M4, V_M8]; @@ -3180,16 +3186,14 @@ RegisterClass Op1Class, DAGOperand Op2Class, LMULInfo MInfo, + int sew, string Constraint = "", bit Commutable = 0> { let VLMul = MInfo.value in { defvar mx = MInfo.MX; - defvar sews = SchedSEWSet.val; - foreach e = sews in { let isCommutable = Commutable in - def "_" # mx # "_E" # e : VPseudoTernaryNoMaskWithPolicy; - def "_" # mx # "_E" # e # "_MASK" : VPseudoBinaryTailPolicy; - } + def "_" # mx # "_E" # sew : VPseudoTernaryNoMaskWithPolicy; + def "_" # mx # "_E" # sew # "_MASK" : VPseudoBinaryTailPolicy; } } @@ -3448,50 +3452,60 @@ multiclass VPseudoVRED_VS { foreach m = MxList in { defvar mx = m.MX; - defvar WriteVIRedV_From_MX = !cast("WriteVIRedV_From_" # mx); - defm _VS : VPseudoTernaryWithTailPolicy_E, - Sched<[WriteVIRedV_From_MX, ReadVIRedV, ReadVIRedV, ReadVIRedV, - ReadVMask]>; + foreach e = SchedSEWSet.val in { + defvar WriteVIRedV_From_MX_E = !cast("WriteVIRedV_From_" # mx # "_E" # e); + defm _VS : VPseudoTernaryWithTailPolicy_E, + Sched<[WriteVIRedV_From_MX_E, ReadVIRedV, ReadVIRedV, ReadVIRedV, + ReadVMask]>; + } } } multiclass VPseudoVWRED_VS { - foreach m = MxList in { + foreach m = MxListWRed in { defvar mx = m.MX; - defvar WriteVIWRedV_From_MX = !cast("WriteVIWRedV_From_" # mx); - defm _VS : VPseudoTernaryWithTailPolicy_E, - Sched<[WriteVIWRedV_From_MX, ReadVIWRedV, ReadVIWRedV, - ReadVIWRedV, ReadVMask]>; + foreach e = SchedSEWSet.val in { + defvar WriteVIWRedV_From_MX_E = !cast("WriteVIWRedV_From_" # mx # "_E" # e); + defm _VS : VPseudoTernaryWithTailPolicy_E, + Sched<[WriteVIWRedV_From_MX_E, ReadVIWRedV, ReadVIWRedV, + ReadVIWRedV, ReadVMask]>; + } } } multiclass VPseudoVFRED_VS { foreach m = MxListF in { defvar mx = m.MX; - defvar WriteVFRedV_From_MX = !cast("WriteVFRedV_From_" # mx); - defm _VS : VPseudoTernaryWithTailPolicy_E, - Sched<[WriteVFRedV_From_MX, ReadVFRedV, ReadVFRedV, ReadVFRedV, - ReadVMask]>; + foreach e = SchedSEWSetF.val in { + defvar WriteVFRedV_From_MX_E = !cast("WriteVFRedV_From_" # mx # "_E" # e); + defm _VS : VPseudoTernaryWithTailPolicy_E, + Sched<[WriteVFRedV_From_MX_E, ReadVFRedV, ReadVFRedV, ReadVFRedV, + ReadVMask]>; + } } } multiclass VPseudoVFREDO_VS { foreach m = MxListF in { defvar mx = m.MX; - defvar WriteVFRedOV_From_MX = !cast("WriteVFRedOV_From_" # mx); - defm _VS : VPseudoTernaryWithTailPolicy_E, - Sched<[WriteVFRedOV_From_MX, ReadVFRedOV, ReadVFRedOV, - ReadVFRedOV, ReadVMask]>; + foreach e = SchedSEWSetF.val in { + defvar WriteVFRedOV_From_MX_E = !cast("WriteVFRedOV_From_" # mx # "_E" # e); + defm _VS : VPseudoTernaryWithTailPolicy_E, + Sched<[WriteVFRedOV_From_MX_E, ReadVFRedOV, ReadVFRedOV, + ReadVFRedOV, ReadVMask]>; + } } } multiclass VPseudoVFWRED_VS { - foreach m = MxListF in { + foreach m = MxListFWRed in { defvar mx = m.MX; - defvar WriteVFWRedV_From_MX = !cast("WriteVFWRedV_From_" # mx); - defm _VS : VPseudoTernaryWithTailPolicy_E, - Sched<[WriteVFWRedV_From_MX, ReadVFWRedV, ReadVFWRedV, - ReadVFWRedV, ReadVMask]>; + foreach e = SchedSEWSetF.val in { + defvar WriteVFWRedV_From_MX_E = !cast("WriteVFWRedV_From_" # mx # "_E" # e); + defm _VS : VPseudoTernaryWithTailPolicy_E, + Sched<[WriteVFWRedV_From_MX_E, ReadVFWRedV, ReadVFWRedV, + ReadVFWRedV, ReadVMask]>; + } } } diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td --- a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td +++ b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td @@ -620,12 +620,12 @@ // 14. Vector Reduction Operations let Latency = 32 in { -defm "" : LMULWriteRes<"WriteVIRedV_From", [SiFive7VA]>; -defm "" : LMULWriteRes<"WriteVIWRedV_From", [SiFive7VA]>; -defm "" : LMULWriteRes<"WriteVFRedV_From", [SiFive7VA]>; -defm "" : LMULWriteRes<"WriteVFRedOV_From", [SiFive7VA]>; -defm "" : LMULWriteResFWRed<"WriteVFWRedV_From", [SiFive7VA]>; -defm "" : LMULWriteResFWRed<"WriteVFWRedOV_From", [SiFive7VA]>; +defm "" : LMULSEWWriteRes<"WriteVIRedV_From", [SiFive7VA]>; +defm "" : LMULSEWWriteRes<"WriteVIWRedV_From", [SiFive7VA]>; +defm "" : LMULSEWWriteRes<"WriteVFRedV_From", [SiFive7VA]>; +defm "" : LMULSEWWriteRes<"WriteVFRedOV_From", [SiFive7VA]>; +defm "" : LMULSEWWriteResFWRed<"WriteVFWRedV_From", [SiFive7VA]>; +defm "" : LMULSEWWriteResFWRed<"WriteVFWRedOV_From", [SiFive7VA]>; } // 15. Vector Mask Instructions diff --git a/llvm/lib/Target/RISCV/RISCVScheduleV.td b/llvm/lib/Target/RISCV/RISCVScheduleV.td --- a/llvm/lib/Target/RISCV/RISCVScheduleV.td +++ b/llvm/lib/Target/RISCV/RISCVScheduleV.td @@ -12,6 +12,8 @@ defvar SchedMxList = ["MF8", "MF4", "MF2", "M1", "M2", "M4", "M8"]; // Used for widening and narrowing instructions as it doesn't contain M8. defvar SchedMxListW = !listremove(SchedMxList, ["M8"]); +// Used for widening reductions, which does contain M8. +defvar SchedMxListWRed = SchedMxList; defvar SchedMxListFW = !listremove(SchedMxList, ["M8", "MF8"]); // Used for floating-point as it doesn't contain MF8. defvar SchedMxListF = !listremove(SchedMxList, ["MF8"]); @@ -117,19 +119,23 @@ } } multiclass LMULSEWWriteResImpl resources, - bit isF = 0> { - def : WriteRes(name # "_WorstCase"), resources>; - foreach mx = !if(isF, SchedMxListF, SchedMxList) in { + list MxList, bit isF = 0> { + if !exists(name # "_WorstCase") then + def : WriteRes(name # "_WorstCase"), resources>; + foreach mx = MxList in { foreach sew = !if(isF, SchedSEWSetF.val, SchedSEWSet.val) in - def : WriteRes(name # "_" # mx # "_E" # sew), resources>; + if !exists(name # "_" # mx # "_E" # sew) then + def : WriteRes(name # "_" # mx # "_E" # sew), resources>; } } multiclass LMULSEWReadAdvanceImpl writes = [], - bit isF = 0> { - def : ReadAdvance(name # "_WorstCase"), val, writes>; - foreach mx = !if(isF, SchedMxListF, SchedMxList) in { + list MxList, bit isF = 0> { + if !exists(name # "_WorstCase") then + def : ReadAdvance(name # "_WorstCase"), val, writes>; + foreach mx = MxList in { foreach sew = !if(isF, SchedSEWSetF.val, SchedSEWSet.val) in - def : ReadAdvance(name # "_" # mx # "_E" # sew), val, writes>; + if !exists(name # "_" # mx # "_E" # sew) then + def : ReadAdvance(name # "_" # mx # "_E" # sew), val, writes>; } } // Define classes to define list containing all SchedWrites for each (name, LMUL) @@ -159,16 +165,26 @@ multiclass LMULSEWSchedWrites : LMULSEWSchedWritesImpl; multiclass LMULSEWSchedReads : LMULSEWSchedReadsImpl; multiclass LMULSEWWriteRes resources> - : LMULSEWWriteResImpl; + : LMULSEWWriteResImpl; multiclass LMULSEWReadAdvance writes = []> - : LMULSEWReadAdvanceImpl; + : LMULSEWReadAdvanceImpl; + +multiclass LMULSEWSchedWritesWRed + : LMULSEWSchedWritesImpl; +multiclass LMULSEWWriteResWRed resources> + : LMULSEWWriteResImpl; + +multiclass LMULSEWSchedWritesFWRed + : LMULSEWSchedWritesImpl; +multiclass LMULSEWWriteResFWRed resources> + : LMULSEWWriteResImpl; multiclass LMULSEWSchedWritesF : LMULSEWSchedWritesImpl; multiclass LMULSEWSchedReadsF : LMULSEWSchedReadsImpl; multiclass LMULSEWWriteResF resources> - : LMULSEWWriteResImpl; + : LMULSEWWriteResImpl; multiclass LMULSEWReadAdvanceF writes = []> - : LMULSEWReadAdvanceImpl; + : LMULSEWReadAdvanceImpl; multiclass LMULSchedWritesW : LMULSchedWritesImpl; multiclass LMULSchedReadsW : LMULSchedReadsImpl; @@ -186,12 +202,6 @@ : LMULReadAdvanceImpl; class LMULSchedWriteListFW names> : LMULSchedWriteListImpl; -multiclass LMULSchedWritesFWRed : LMULSchedWritesImpl; -multiclass LMULWriteResFWRed resources> - : LMULWriteResImpl; -class LMULSchedWriteListFWRed names> : LMULSchedWriteListImpl; - - // 3.6 Vector Byte Length vlenb def WriteRdVLENB : SchedWrite; @@ -389,15 +399,15 @@ // MF8 and M8. Use the _From suffix to indicate the number of the // LMUL from VS2. // 14.1. Vector Single-Width Integer Reduction Instructions -defm "" : LMULSchedWrites<"WriteVIRedV_From">; +defm "" : LMULSEWSchedWrites<"WriteVIRedV_From">; // 14.2. Vector Widening Integer Reduction Instructions -defm "" : LMULSchedWrites<"WriteVIWRedV_From">; +defm "" : LMULSEWSchedWritesWRed<"WriteVIWRedV_From">; // 14.3. Vector Single-Width Floating-Point Reduction Instructions -defm "" : LMULSchedWrites<"WriteVFRedV_From">; -defm "" : LMULSchedWrites<"WriteVFRedOV_From">; +defm "" : LMULSEWSchedWritesF<"WriteVFRedV_From">; +defm "" : LMULSEWSchedWritesF<"WriteVFRedOV_From">; // 14.4. Vector Widening Floating-Point Reduction Instructions -defm "" : LMULSchedWritesFWRed<"WriteVFWRedV_From">; -defm "" : LMULSchedWritesFWRed<"WriteVFWRedOV_From">; +defm "" : LMULSEWSchedWritesFWRed<"WriteVFWRedV_From">; +defm "" : LMULSEWSchedWritesFWRed<"WriteVFWRedOV_From">; // 15. Vector Mask Instructions // 15.1. Vector Mask-Register Logical Instructions @@ -821,12 +831,12 @@ defm "" : LMULWriteResFW<"WriteVFNCvtFToFV", []>; // 14. Vector Reduction Operations -defm "" : LMULWriteRes<"WriteVIRedV_From", []>; -defm "" : LMULWriteRes<"WriteVIWRedV_From", []>; -defm "" : LMULWriteRes<"WriteVFRedV_From", []>; -defm "" : LMULWriteRes<"WriteVFRedOV_From", []>; -defm "" : LMULWriteResFWRed<"WriteVFWRedV_From", []>; -defm "" : LMULWriteResFWRed<"WriteVFWRedOV_From", []>; +defm "" : LMULSEWWriteRes<"WriteVIRedV_From", []>; +defm "" : LMULSEWWriteResWRed<"WriteVIWRedV_From", []>; +defm "" : LMULSEWWriteResF<"WriteVFRedV_From", []>; +defm "" : LMULSEWWriteResF<"WriteVFRedOV_From", []>; +defm "" : LMULSEWWriteResFWRed<"WriteVFWRedV_From", []>; +defm "" : LMULSEWWriteResFWRed<"WriteVFWRedOV_From", []>; // 15. Vector Mask Instructions defm "" : LMULWriteRes<"WriteVMALUV", []>;