diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td @@ -117,6 +117,8 @@ defvar MxListW = [V_MF8, V_MF4, V_MF2, V_M1, V_M2, V_M4]; // For floating point which don't need MF8. defvar MxListFW = [V_MF4, V_MF2, V_M1, V_M2, V_M4]; +// For widening floating-point Reduction as it doesn't contain MF8. +defvar MxListFWRed = [V_MF4, V_MF2, V_M1, V_M2, V_M4]; // Use for zext/sext.vf2 defvar MxListVF2 = [V_MF4, V_MF2, V_M1, V_M2, V_M4, V_M8]; @@ -3180,16 +3182,14 @@ RegisterClass Op1Class, DAGOperand Op2Class, LMULInfo MInfo, + int sew, string Constraint = "", bit Commutable = 0> { let VLMul = MInfo.value in { defvar mx = MInfo.MX; - defvar sews = SchedSEWSet.val; - foreach e = sews in { let isCommutable = Commutable in - def "_" # mx # "_E" # e : VPseudoTernaryNoMaskWithPolicy; - def "_" # mx # "_E" # e # "_MASK" : VPseudoBinaryTailPolicy; - } + def "_" # mx # "_E" # sew : VPseudoTernaryNoMaskWithPolicy; + def "_" # mx # "_E" # sew # "_MASK" : VPseudoBinaryTailPolicy; } } @@ -3448,50 +3448,60 @@ multiclass VPseudoVRED_VS { foreach m = MxList in { defvar mx = m.MX; - defvar WriteVIRedV_From_MX = !cast("WriteVIRedV_From_" # mx); - defm _VS : VPseudoTernaryWithTailPolicy_E, - Sched<[WriteVIRedV_From_MX, ReadVIRedV, ReadVIRedV, ReadVIRedV, + foreach e = SchedSEWSet.val in { + defvar WriteVIRedV_From_MX_E = !cast("WriteVIRedV_From_" # mx # "_E" # e); + defm _VS : VPseudoTernaryWithTailPolicy_E, + Sched<[WriteVIRedV_From_MX_E, ReadVIRedV, ReadVIRedV, ReadVIRedV, ReadVMask]>; + } } } multiclass VPseudoVWRED_VS { - foreach m = MxList in { + foreach m = MxListW in { defvar mx = m.MX; - defvar WriteVIWRedV_From_MX = !cast("WriteVIWRedV_From_" # mx); - defm _VS : VPseudoTernaryWithTailPolicy_E, - Sched<[WriteVIWRedV_From_MX, ReadVIWRedV, ReadVIWRedV, + foreach e = SchedSEWSet.val in { + defvar WriteVIWRedV_From_MX_E = !cast("WriteVIWRedV_From_" # mx # "_E" # e); + defm _VS : VPseudoTernaryWithTailPolicy_E, + Sched<[WriteVIWRedV_From_MX_E, ReadVIWRedV, ReadVIWRedV, ReadVIWRedV, ReadVMask]>; + } } } multiclass VPseudoVFRED_VS { foreach m = MxListF in { defvar mx = m.MX; - defvar WriteVFRedV_From_MX = !cast("WriteVFRedV_From_" # mx); - defm _VS : VPseudoTernaryWithTailPolicy_E, - Sched<[WriteVFRedV_From_MX, ReadVFRedV, ReadVFRedV, ReadVFRedV, + foreach e = SchedSEWSetF.val in { + defvar WriteVFRedV_From_MX_E = !cast("WriteVFRedV_From_" # mx # "_E" # e); + defm _VS : VPseudoTernaryWithTailPolicy_E, + Sched<[WriteVFRedV_From_MX_E, ReadVFRedV, ReadVFRedV, ReadVFRedV, ReadVMask]>; + } } } multiclass VPseudoVFREDO_VS { foreach m = MxListF in { defvar mx = m.MX; - defvar WriteVFRedOV_From_MX = !cast("WriteVFRedOV_From_" # mx); - defm _VS : VPseudoTernaryWithTailPolicy_E, - Sched<[WriteVFRedOV_From_MX, ReadVFRedOV, ReadVFRedOV, + foreach e = SchedSEWSetF.val in { + defvar WriteVFRedOV_From_MX_E = !cast("WriteVFRedOV_From_" # mx # "_E" # e); + defm _VS : VPseudoTernaryWithTailPolicy_E, + Sched<[WriteVFRedOV_From_MX_E, ReadVFRedOV, ReadVFRedOV, ReadVFRedOV, ReadVMask]>; + } } } multiclass VPseudoVFWRED_VS { - foreach m = MxListF in { + foreach m = MxListFWRed in { defvar mx = m.MX; - defvar WriteVFWRedV_From_MX = !cast("WriteVFWRedV_From_" # mx); - defm _VS : VPseudoTernaryWithTailPolicy_E, - Sched<[WriteVFWRedV_From_MX, ReadVFWRedV, ReadVFWRedV, - ReadVFWRedV, ReadVMask]>; + foreach e = SchedSEWSetF.val in { + defvar WriteVFWRedV_From_MX_E = !cast("WriteVFWRedV_From_" # mx # "_E" # e); + defm _VS : VPseudoTernaryWithTailPolicy_E, + Sched<[WriteVFWRedV_From_MX_E, ReadVFWRedV, ReadVFWRedV, + ReadVFWRedV, ReadVMask]>; + } } } @@ -5344,7 +5354,9 @@ foreach vti = !if(IsFloat, AllFloatVectors, AllIntegerVectors) in { defvar wtiSEW = !mul(vti.SEW, 2); - if !le(wtiSEW, 64) then { + // The second part of the and statement exists because widening reductions + // do not operate on LMUL M8. + if !and(!le(wtiSEW, 64), !ne(vti.LMul.MX, "M8")) then { defvar wtiM1 = !cast(!if(IsFloat, "VF", "VI") # wtiSEW # "M1"); let Predicates = GetVTypePredicates.Predicates in defm : VPatTernaryTA_E; -defm "" : LMULWriteRes<"WriteVIWRedV_From", [SiFive7VA]>; -defm "" : LMULWriteRes<"WriteVFRedV_From", [SiFive7VA]>; -defm "" : LMULWriteRes<"WriteVFRedOV_From", [SiFive7VA]>; -defm "" : LMULWriteResFWRed<"WriteVFWRedV_From", [SiFive7VA]>; -defm "" : LMULWriteResFWRed<"WriteVFWRedOV_From", [SiFive7VA]>; +defm "" : LMULSEWWriteRes<"WriteVIRedV_From", [SiFive7VA]>; +defm "" : LMULSEWWriteRes<"WriteVIWRedV_From", [SiFive7VA]>; +defm "" : LMULSEWWriteRes<"WriteVFRedV_From", [SiFive7VA]>; +defm "" : LMULSEWWriteRes<"WriteVFRedOV_From", [SiFive7VA]>; +defm "" : LMULSEWWriteResFWRed<"WriteVFWRedV_From", [SiFive7VA]>; +defm "" : LMULSEWWriteResFWRed<"WriteVFWRedOV_From", [SiFive7VA]>; } // 15. Vector Mask Instructions diff --git a/llvm/lib/Target/RISCV/RISCVScheduleV.td b/llvm/lib/Target/RISCV/RISCVScheduleV.td --- a/llvm/lib/Target/RISCV/RISCVScheduleV.td +++ b/llvm/lib/Target/RISCV/RISCVScheduleV.td @@ -117,19 +117,23 @@ } } multiclass LMULSEWWriteResImpl resources, - bit isF = 0> { - def : WriteRes(name # "_WorstCase"), resources>; - foreach mx = !if(isF, SchedMxListF, SchedMxList) in { + list MxList, bit isF = 0> { + if !exists(name # "_WorstCase") then + def : WriteRes(name # "_WorstCase"), resources>; + foreach mx = MxList in { foreach sew = !if(isF, SchedSEWSetF.val, SchedSEWSet.val) in - def : WriteRes(name # "_" # mx # "_E" # sew), resources>; + if !exists(name # "_" # mx # "_E" # sew) then + def : WriteRes(name # "_" # mx # "_E" # sew), resources>; } } multiclass LMULSEWReadAdvanceImpl writes = [], - bit isF = 0> { - def : ReadAdvance(name # "_WorstCase"), val, writes>; - foreach mx = !if(isF, SchedMxListF, SchedMxList) in { + list MxList, bit isF = 0> { + if !exists(name # "_WorstCase") then + def : ReadAdvance(name # "_WorstCase"), val, writes>; + foreach mx = MxList in { foreach sew = !if(isF, SchedSEWSetF.val, SchedSEWSet.val) in - def : ReadAdvance(name # "_" # mx # "_E" # sew), val, writes>; + if !exists(name # "_" # mx # "_E" # sew) then + def : ReadAdvance(name # "_" # mx # "_E" # sew), val, writes>; } } // Define classes to define list containing all SchedWrites for each (name, LMUL) @@ -159,16 +163,26 @@ multiclass LMULSEWSchedWrites : LMULSEWSchedWritesImpl; multiclass LMULSEWSchedReads : LMULSEWSchedReadsImpl; multiclass LMULSEWWriteRes resources> - : LMULSEWWriteResImpl; + : LMULSEWWriteResImpl; multiclass LMULSEWReadAdvance writes = []> - : LMULSEWReadAdvanceImpl; + : LMULSEWReadAdvanceImpl; + +multiclass LMULSEWSchedWritesW + : LMULSEWSchedWritesImpl; +multiclass LMULSEWWriteResW resources> + : LMULSEWWriteResImpl; + +multiclass LMULSEWSchedWritesFWRed + : LMULSEWSchedWritesImpl; +multiclass LMULSEWWriteResFWRed resources> + : LMULSEWWriteResImpl; multiclass LMULSEWSchedWritesF : LMULSEWSchedWritesImpl; multiclass LMULSEWSchedReadsF : LMULSEWSchedReadsImpl; multiclass LMULSEWWriteResF resources> - : LMULSEWWriteResImpl; + : LMULSEWWriteResImpl; multiclass LMULSEWReadAdvanceF writes = []> - : LMULSEWReadAdvanceImpl; + : LMULSEWReadAdvanceImpl; multiclass LMULSchedWritesW : LMULSchedWritesImpl; multiclass LMULSchedReadsW : LMULSchedReadsImpl; @@ -186,12 +200,6 @@ : LMULReadAdvanceImpl; class LMULSchedWriteListFW names> : LMULSchedWriteListImpl; -multiclass LMULSchedWritesFWRed : LMULSchedWritesImpl; -multiclass LMULWriteResFWRed resources> - : LMULWriteResImpl; -class LMULSchedWriteListFWRed names> : LMULSchedWriteListImpl; - - // 3.6 Vector Byte Length vlenb def WriteRdVLENB : SchedWrite; @@ -389,15 +397,15 @@ // MF8 and M8. Use the _From suffix to indicate the number of the // LMUL from VS2. // 14.1. Vector Single-Width Integer Reduction Instructions -defm "" : LMULSchedWrites<"WriteVIRedV_From">; +defm "" : LMULSEWSchedWrites<"WriteVIRedV_From">; // 14.2. Vector Widening Integer Reduction Instructions -defm "" : LMULSchedWrites<"WriteVIWRedV_From">; +defm "" : LMULSEWSchedWritesW<"WriteVIWRedV_From">; // 14.3. Vector Single-Width Floating-Point Reduction Instructions -defm "" : LMULSchedWrites<"WriteVFRedV_From">; -defm "" : LMULSchedWrites<"WriteVFRedOV_From">; +defm "" : LMULSEWSchedWritesF<"WriteVFRedV_From">; +defm "" : LMULSEWSchedWritesF<"WriteVFRedOV_From">; // 14.4. Vector Widening Floating-Point Reduction Instructions -defm "" : LMULSchedWritesFWRed<"WriteVFWRedV_From">; -defm "" : LMULSchedWritesFWRed<"WriteVFWRedOV_From">; +defm "" : LMULSEWSchedWritesFWRed<"WriteVFWRedV_From">; +defm "" : LMULSEWSchedWritesFWRed<"WriteVFWRedOV_From">; // 15. Vector Mask Instructions // 15.1. Vector Mask-Register Logical Instructions @@ -821,12 +829,12 @@ defm "" : LMULWriteResFW<"WriteVFNCvtFToFV", []>; // 14. Vector Reduction Operations -defm "" : LMULWriteRes<"WriteVIRedV_From", []>; -defm "" : LMULWriteRes<"WriteVIWRedV_From", []>; -defm "" : LMULWriteRes<"WriteVFRedV_From", []>; -defm "" : LMULWriteRes<"WriteVFRedOV_From", []>; -defm "" : LMULWriteResFWRed<"WriteVFWRedV_From", []>; -defm "" : LMULWriteResFWRed<"WriteVFWRedOV_From", []>; +defm "" : LMULSEWWriteRes<"WriteVIRedV_From", []>; +defm "" : LMULSEWWriteResW<"WriteVIWRedV_From", []>; +defm "" : LMULSEWWriteResF<"WriteVFRedV_From", []>; +defm "" : LMULSEWWriteResF<"WriteVFRedOV_From", []>; +defm "" : LMULSEWWriteResFWRed<"WriteVFWRedV_From", []>; +defm "" : LMULSEWWriteResFWRed<"WriteVFWRedOV_From", []>; // 15. Vector Mask Instructions defm "" : LMULWriteRes<"WriteVMALUV", []>;