diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td --- a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td @@ -99,18 +99,25 @@ //===----------------------------------------------------------------------===// // Common class of scheduling definitions. -// `ReadVMask` will be appended if instruction is masked. +// `ReadVMergeOp` will be prepended to reads if instruction is masked. +// `ReadVMask` will be appended to reads if instruction is masked. // Operands: // `writes` SchedWrites that are listed for each explicit def operand // in order. // `reads` SchedReads that are listed for each explicit use operand. // `forceMasked` Forced to be masked (e.g. Add-with-Carry Instructions). class SchedCommon writes, list reads, - bit forceMasked = 0>: Sched<[]> { + string mx = "WorstCase", int sew = 0, bit forceMasked = 0> + : Sched<[]> { defvar isMasked = !ne(!find(NAME, "_MASK"), -1); - defvar needReadVMask = !or(forceMasked, isMasked); - let SchedRW = !listconcat(writes, reads, - !if(needReadVMask, [ReadVMask], [])); + defvar isMaskedOrForceMasked = !or(forceMasked, isMasked); + defvar ReadVMergeOp = !if(!or(!eq(mx, "WorstCase"), !eq(sew, 0)), + !cast("ReadVMergeOp_" # mx), + !cast("ReadVMergeOp_" # mx # "_E" #sew)); + defvar allReads = !if(isMaskedOrForceMasked, + !listconcat([ReadVMergeOp], reads, [ReadVMask]), + reads); + let SchedRW = !listconcat(writes, allReads); } // Common class of scheduling definitions for n-ary instructions. @@ -124,7 +131,7 @@ !cast( !if(sew, read # "_" # mx # "_E" # sew, read # "_" # mx))), - forceMasked>; + mx, sew, forceMasked>; // Classes with postfix "MC" are only used in MC layer. // For these classes, we assume that they are with the worst case costs and @@ -159,7 +166,7 @@ // For reduction instructions. class SchedReduction: SchedCommon<[!cast(write # "_" # mx # "_E" # sew)], - !listsplat(!cast(read), 3)>; + !listsplat(!cast(read), 3), mx, sew>; class SchedReductionMC: SchedCommon<[!cast(write # "_WorstCase")], [!cast(readV), !cast(readV0)], @@ -167,82 +174,83 @@ // Whole Vector Register Move class VMVRSched : SchedCommon< - [!cast("WriteVMov" #n #"V")], - [!cast("ReadVMov" #n #"V")] + [!cast("WriteVMov" # n # "V")], + [!cast("ReadVMov" # n # "V")] >; // Vector Unit-Stride Loads and Stores class VLESched : SchedCommon< - [!cast("WriteVLDE_" #lmul)], - [ReadVLDX], forceMasked + [!cast("WriteVLDE_" # lmul)], + [ReadVLDX], mx=lmul, forceMasked=forceMasked >; class VLESchedMC : VLESched<"WorstCase", forceMasked=1>; class VSESched : SchedCommon< - [!cast("WriteVSTE_" #lmul)], - [!cast("ReadVSTEV_" #lmul), ReadVSTX], forceMasked + [!cast("WriteVSTE_" # lmul)], + [!cast("ReadVSTEV_" # lmul), ReadVSTX], mx=lmul, + forceMasked=forceMasked >; class VSESchedMC : VSESched<"WorstCase", forceMasked=1>; // Vector Strided Loads and Stores class VLSSched : SchedCommon< - [!cast("WriteVLDS" #eew #"_" #emul)], - [ReadVLDX, ReadVLDSX], forceMasked + [!cast("WriteVLDS" # eew # "_" # emul)], + [ReadVLDX, ReadVLDSX], emul, eew, forceMasked >; class VLSSchedMC : VLSSched; class VSSSched : SchedCommon< - [!cast("WriteVSTS" #eew #"_" #emul)], - [!cast("ReadVSTS" #eew #"V_" #emul), ReadVSTX, ReadVSTSX], - forceMasked + [!cast("WriteVSTS" # eew # "_" # emul)], + [!cast("ReadVSTS" # eew # "V_" # emul), ReadVSTX, ReadVSTSX], + emul, eew, forceMasked >; class VSSSchedMC : VSSSched; // Vector Indexed Loads and Stores class VLXSched : SchedCommon< - [!cast("WriteVLD" #isOrdered #"X" #dataEEW #"_" #dataEMUL)], - [ReadVLDX, !cast("ReadVLD" #isOrdered #"XV_" #idxEMUL)], - forceMasked + [!cast("WriteVLD" # isOrdered # "X" # dataEEW # "_" # dataEMUL)], + [ReadVLDX, !cast("ReadVLD" # isOrdered # "XV_" # idxEMUL)], + dataEMUL, dataEEW, forceMasked >; class VLXSchedMC: VLXSched; class VSXSched : SchedCommon< - [!cast("WriteVST" #isOrdered #"X" #dataEEW #"_" #dataEMUL)], - [!cast("ReadVST" #isOrdered #"X" #dataEEW #"_" #dataEMUL), - ReadVSTX, !cast("ReadVST" #isOrdered #"XV_" #idxEMUL)], - forceMasked + [!cast("WriteVST" # isOrdered # "X" # dataEEW # "_" # dataEMUL)], + [!cast("ReadVST" # isOrdered #"X" # dataEEW # "_" # dataEMUL), + ReadVSTX, !cast("ReadVST" # isOrdered # "XV_" # idxEMUL)], + dataEMUL, dataEEW, forceMasked >; class VSXSchedMC: VSXSched; // Unit-stride Fault-Only-First Loads class VLFSched : SchedCommon< - [!cast("WriteVLDFF_" #lmul)], - [ReadVLDX], forceMasked + [!cast("WriteVLDFF_" # lmul)], + [ReadVLDX], mx=lmul, forceMasked=forceMasked >; class VLFSchedMC: VLFSched<"WorstCase", forceMasked=1>; // Unit-Stride Segment Loads and Stores class VLSEGSched : SchedCommon< [!cast("WriteVLSEG" #nf #"e" #eew #"_" #emul)], - [ReadVLDX], forceMasked + [ReadVLDX], emul, eew, forceMasked >; class VLSEGSchedMC : VLSEGSched; class VSSEGSched : SchedCommon< - [!cast("WriteVSSEG" #nf #"e" #eew #"_" #emul)], - [!cast("ReadVSTEV_" #emul), ReadVSTX], forceMasked + [!cast("WriteVSSEG" # nf # "e" # eew # "_" # emul)], + [!cast("ReadVSTEV_" #emul), ReadVSTX], emul, eew, forceMasked >; class VSSEGSchedMC : VSSEGSched; class VLSEGFFSched : SchedCommon< - [!cast("WriteVLSEGFF" #nf #"e" #eew #"_" #emul)], - [ReadVLDX], forceMasked + [!cast("WriteVLSEGFF" # nf # "e" # eew # "_" # emul)], + [ReadVLDX], emul, eew, forceMasked >; class VLSEGFFSchedMC : VLSEGFFSched; @@ -250,7 +258,7 @@ // Strided Segment Loads and Stores class VLSSEGSched : SchedCommon< [!cast("WriteVLSSEG" #nf #"e" #eew #"_" #emul)], - [ReadVLDX, ReadVLDSX], forceMasked + [ReadVLDX, ReadVLDSX], emul, eew, forceMasked >; class VLSSEGSchedMC : VLSSEGSched; @@ -258,7 +266,7 @@ class VSSSEGSched : SchedCommon< [!cast("WriteVSSSEG" #nf #"e" #eew #"_" #emul)], [!cast("ReadVSTS" #eew #"V_" #emul), - ReadVSTX, ReadVSTSX], forceMasked + ReadVSTX, ReadVSTSX], emul, eew, forceMasked >; class VSSSEGSchedMC : VSSSEGSched; @@ -268,17 +276,18 @@ bit forceMasked = 0> : SchedCommon< [!cast("WriteVL" #isOrdered #"XSEG" #nf #"e" #eew #"_" #emul)], [ReadVLDX, !cast("ReadVLD" #isOrdered #"XV_" #emul)], - forceMasked + emul, eew, forceMasked >; class VLXSEGSchedMC: VLXSEGSched; +// Passes sew=0 instead of eew=0 since this pseudo does not follow MX_E form. class VSXSEGSched : SchedCommon< [!cast("WriteVS" #isOrdered #"XSEG" #nf #"e" #eew #"_" #emul)], [!cast("ReadVST" #isOrdered #"X" #eew #"_" #emul), ReadVSTX, !cast("ReadVST" #isOrdered #"XV_" #emul)], - forceMasked + emul, sew=0, forceMasked=forceMasked >; class VSXSEGSchedMC: VSXSEGSched; diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td @@ -2501,7 +2501,6 @@ } } - multiclass VPseudoVSHT_VV_VX_VI { foreach m = MxList in { defvar mx = m.MX; diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td --- a/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td @@ -474,6 +474,7 @@ //===----------------------------------------------------------------------===// multiclass VPseudoVMAQA_VV_VX { foreach m = MxListTHVdot in { + // TODO: Add Sched defm "" : VPseudoTernaryW_VV; defm "" : VPseudoTernaryW_VX; } @@ -481,6 +482,7 @@ multiclass VPseudoVMAQA_VX { foreach m = MxListTHVdot in { + // TODO: Add Sched defm "" : VPseudoTernaryW_VX; } } diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td --- a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td +++ b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td @@ -1154,6 +1154,12 @@ // Others def : ReadAdvance; +def : ReadAdvance("ReadVMergeOp_WorstCase"), 0>; +foreach mx = SchedMxList in { + def : ReadAdvance("ReadVMergeOp_" # mx), 0>; + foreach sew = SchedSEWSet.val in + def : ReadAdvance("ReadVMergeOp_" # mx # "_E" # sew), 0>; +} //===----------------------------------------------------------------------===// // Unsupported extensions diff --git a/llvm/lib/Target/RISCV/RISCVScheduleV.td b/llvm/lib/Target/RISCV/RISCVScheduleV.td --- a/llvm/lib/Target/RISCV/RISCVScheduleV.td +++ b/llvm/lib/Target/RISCV/RISCVScheduleV.td @@ -687,6 +687,12 @@ // Others def ReadVMask : SchedRead; +def ReadVMergeOp_WorstCase : SchedRead; +foreach mx = SchedMxList in { + def ReadVMergeOp_ # mx : SchedRead; + foreach sew = SchedSEWSet.val in + def ReadVMergeOp_ # mx # "_E" # sew : SchedRead; +} //===----------------------------------------------------------------------===// /// Define default scheduler resources for V. @@ -1050,6 +1056,12 @@ // Others def : ReadAdvance; +def : ReadAdvance("ReadVMergeOp_WorstCase"), 0>; +foreach mx = SchedMxList in { + def : ReadAdvance("ReadVMergeOp_" # mx), 0>; + foreach sew = SchedSEWSet.val in + def : ReadAdvance("ReadVMergeOp_" # mx # "_E" # sew), 0>; +} } // Unsupported } // UnsupportedSchedV