diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td @@ -49,6 +49,10 @@ class PseudoToVInst { defvar AffixSubsts = [["Pseudo", ""], + ["_E64", ""], + ["_E32", ""], + ["_E16", ""], + ["_E8", ""], ["_F64", "_F"], ["_F32", "_F"], ["_F16", "_F"], @@ -1906,12 +1910,16 @@ multiclass VPseudoVCPR_V { foreach m = MxList in { defvar mx = m.MX; - defvar WriteVCompressV_MX = !cast("WriteVCompressV_" # mx); - defvar ReadVCompressV_MX = !cast("ReadVCompressV_" # mx); - + defvar sews = SchedSEWSet.val; let VLMul = m.value in - def _VM # "_" # m.MX : VPseudoUnaryAnyMask, - Sched<[WriteVCompressV_MX, ReadVCompressV_MX, ReadVCompressV_MX]>; + foreach e = sews in { + defvar WriteVCompressV_MX_E = !cast("WriteVCompressV_" # mx # "_E" # e); + defvar ReadVCompressV_MX_E = !cast("ReadVCompressV_" # mx # "_E" # e); + + defvar suffix = "_" # m.MX # "_E" # e; + def _VM # suffix : VPseudoUnaryAnyMask, + Sched<[WriteVCompressV_MX_E, ReadVCompressV_MX_E, ReadVCompressV_MX_E]>; + } } } @@ -1931,6 +1939,24 @@ } } +multiclass VPseudoBinary_E { + let VLMul = MInfo.value in { + defvar suffix = "_" # MInfo.MX # "_E" # sew; + def suffix : VPseudoBinaryNoMask; + def suffix # "_TU" : VPseudoBinaryNoMaskTU; + def suffix # "_MASK" : VPseudoBinaryMaskPolicy, + RISCVMaskedPseudo; + } +} + multiclass VPseudoBinaryM { + let VLMul = lmul.value in { + defvar suffix = "_" # lmul.MX # "_E" # sew # "_" # emul.MX; + def suffix : VPseudoBinaryNoMask; + def suffix # "_TU" : VPseudoBinaryNoMaskTU; + def suffix # "_MASK" : VPseudoBinaryMaskPolicy, + RISCVMaskedPseudo; + } +} + multiclass VPseudoTiedBinary; } +multiclass VPseudoBinaryV_VV_E { + defm _VV : VPseudoBinary_E; +} + // Similar to VPseudoBinaryV_VV, but uses MxListF. multiclass VPseudoBinaryFV_VV { defm _VV : VPseudoBinary; } +multiclass VPseudoBinaryFV_VV_E { + defm _VV : VPseudoBinary_E; +} + multiclass VPseudoVGTR_VV_EEW { foreach m = MxList in { defvar mx = m.MX; - defvar WriteVGatherV_MX = !cast("WriteVGatherV_" # mx); - defvar ReadVGatherV_MX = !cast("ReadVGatherV_" # mx); foreach sew = EEWList in { defvar octuple_lmul = m.octuple; @@ -1997,8 +2048,13 @@ if !and(!ge(octuple_emul, 1), !le(octuple_emul, 64)) then { defvar emulMX = octuple_to_str.ret; defvar emul = !cast("V_" # emulMX); - defm _VV : VPseudoBinaryEmul, - Sched<[WriteVGatherV_MX, ReadVGatherV_MX, ReadVGatherV_MX]>; + defvar sews = SchedSEWSet.val; + foreach e = sews in { + defvar WriteVGatherV_MX_E = !cast("WriteVGatherV_" # mx # "_E" # e); + defvar ReadVGatherV_MX_E = !cast("ReadVGatherV_" # mx # "_E" # e); + defm _VV : VPseudoBinaryEmul_E, + Sched<[WriteVGatherV_MX_E, ReadVGatherV_MX_E, ReadVGatherV_MX_E]>; + } } } } @@ -2008,6 +2064,10 @@ defm "_VX" : VPseudoBinary; } +multiclass VPseudoBinaryV_VX_E { + defm "_VX" : VPseudoBinary_E; +} + multiclass VPseudoVSLD1_VX { foreach m = MxList in { defvar mx = m.MX; @@ -2025,6 +2085,12 @@ f.fprclass, m, Constraint>; } +multiclass VPseudoBinaryV_VF_E { + defm "_V" # f.FX : VPseudoBinary_E; +} + multiclass VPseudoVSLD1_VF { foreach f = FPList in { foreach m = f.MxList in { @@ -2262,21 +2328,26 @@ multiclass VPseudoVSQR_V { foreach m = MxListF in { defvar mx = m.MX; - defvar WriteVFSqrtV_MX = !cast("WriteVFSqrtV_" # mx); - defvar ReadVFSqrtV_MX = !cast("ReadVFSqrtV_" # mx); + defvar sews = SchedSEWSet.val; - let VLMul = m.value in { - def "_V_" # mx : VPseudoUnaryNoMask, - Sched<[WriteVFSqrtV_MX, ReadVFSqrtV_MX, ReadVMask]>; - def "_V_" # mx # "_TU": VPseudoUnaryNoMaskTU, + let VLMul = m.value in + foreach e = sews in { + defvar WriteVFSqrtV_MX = !cast("WriteVFSqrtV_" # mx # "_E" # e); + defvar ReadVFSqrtV_MX = !cast("ReadVFSqrtV_" # mx # "_E" # e); + defvar suffix = mx # "_E" # e; + + def "_V_" # suffix : VPseudoUnaryNoMask, Sched<[WriteVFSqrtV_MX, ReadVFSqrtV_MX, ReadVMask]>; - def "_V_" # mx # "_MASK" : VPseudoUnaryMaskTA, - RISCVMaskedPseudo, - Sched<[WriteVFSqrtV_MX, ReadVFSqrtV_MX, ReadVMask]>; - } + def "_V_" # suffix # "_TU": VPseudoUnaryNoMaskTU, + Sched<[WriteVFSqrtV_MX, ReadVFSqrtV_MX, ReadVMask]>; + def "_V_" # suffix # "_MASK" : VPseudoUnaryMaskTA, + RISCVMaskedPseudo, + Sched<[WriteVFSqrtV_MX, ReadVFSqrtV_MX, ReadVMask]>; + } } } +// TODO: Should I make this SEW aware? multiclass VPseudoVRCP_V { foreach m = MxListF in { defvar mx = m.MX; @@ -2394,18 +2465,24 @@ multiclass VPseudoVGTR_VV_VX_VI { foreach m = MxList in { defvar mx = m.MX; - defvar WriteVGatherV_MX = !cast("WriteVGatherV_" # mx); + defvar sews = SchedSEWSet.val; defvar WriteVGatherX_MX = !cast("WriteVGatherX_" # mx); defvar WriteVGatherI_MX = !cast("WriteVGatherI_" # mx); - defvar ReadVGatherV_MX = !cast("ReadVGatherV_" # mx); defvar ReadVGatherX_MX = !cast("ReadVGatherX_" # mx); + defvar ReadVGatherI_MX = !cast("ReadVGatherI_" # mx); - defm "" : VPseudoBinaryV_VV, - Sched<[WriteVGatherV_MX, ReadVGatherV_MX, ReadVGatherV_MX, ReadVMask]>; + // TODO: Is this change okay? V->X defm "" : VPseudoBinaryV_VX, - Sched<[WriteVGatherX_MX, ReadVGatherV_MX, ReadVGatherX_MX, ReadVMask]>; + Sched<[WriteVGatherX_MX, ReadVGatherX_MX, ReadVGatherX_MX, ReadVMask]>; defm "" : VPseudoBinaryV_VI, - Sched<[WriteVGatherI_MX, ReadVGatherV_MX, ReadVMask]>; + Sched<[WriteVGatherI_MX, ReadVGatherI_MX, ReadVMask]>; + + foreach e = sews in { + defvar WriteVGatherV_MX_E = !cast("WriteVGatherV_" # mx # "_E" # e); + defvar ReadVGatherV_MX_E = !cast("ReadVGatherV_" # mx # "_E" # e); + defm "" : VPseudoBinaryV_VV_E, + Sched<[WriteVGatherV_MX_E, ReadVGatherV_MX_E, ReadVGatherV_MX_E, ReadVMask]>; + } } } @@ -2560,15 +2637,18 @@ multiclass VPseudoVDIV_VV_VX { foreach m = MxList in { defvar mx = m.MX; - defvar WriteVIDivV_MX = !cast("WriteVIDivV_" # mx); - defvar WriteVIDivX_MX = !cast("WriteVIDivX_" # mx); - defvar ReadVIDivV_MX = !cast("ReadVIDivV_" # mx); - defvar ReadVIDivX_MX = !cast("ReadVIDivX_" # mx); - - defm "" : VPseudoBinaryV_VV, - Sched<[WriteVIDivV_MX, ReadVIDivV_MX, ReadVIDivV_MX, ReadVMask]>; - defm "" : VPseudoBinaryV_VX, - Sched<[WriteVIDivX_MX, ReadVIDivV_MX, ReadVIDivX_MX, ReadVMask]>; + defvar sews = SchedSEWSet.val; + foreach e = sews in { + defvar WriteVIDivV_MX_E = !cast("WriteVIDivV_" # mx # "_E" # e); + defvar WriteVIDivX_MX_E = !cast("WriteVIDivX_" # mx # "_E" # e); + defvar ReadVIDivV_MX_E = !cast("ReadVIDivV_" # mx # "_E" # e); + defvar ReadVIDivX_MX_E = !cast("ReadVIDivX_" # mx # "_E" # e); + + defm "" : VPseudoBinaryV_VV_E, + Sched<[WriteVIDivV_MX_E, ReadVIDivV_MX_E, ReadVIDivV_MX_E, ReadVMask]>; + defm "" : VPseudoBinaryV_VX_E, + Sched<[WriteVIDivX_MX_E, ReadVIDivV_MX_E, ReadVIDivX_MX_E, ReadVMask]>; + } } } @@ -2598,22 +2678,28 @@ multiclass VPseudoVFDIV_VV_VF { foreach m = MxListF in { defvar mx = m.MX; - defvar WriteVFDivV_MX = !cast("WriteVFDivV_" # mx); - defvar ReadVFDivV_MX = !cast("ReadVFDivV_" # mx); + defvar sews = SchedSEWSet.val; + foreach e = sews in { + defvar WriteVFDivV_MX_E = !cast("WriteVFDivV_" # mx # "_E" # e); + defvar ReadVFDivV_MX_E = !cast("ReadVFDivV_" # mx # "_E" # e); - defm "" : VPseudoBinaryFV_VV, - Sched<[WriteVFDivV_MX, ReadVFDivV_MX, ReadVFDivV_MX, ReadVMask]>; + defm "" : VPseudoBinaryFV_VV_E, + Sched<[WriteVFDivV_MX_E, ReadVFDivV_MX_E, ReadVFDivV_MX_E, ReadVMask]>; + } } foreach f = FPList in { foreach m = f.MxList in { defvar mx = m.MX; - defvar WriteVFDivF_MX = !cast("WriteVFDivF_" # mx); - defvar ReadVFDivV_MX = !cast("ReadVFDivV_" # mx); - defvar ReadVFDivF_MX = !cast("ReadVFDivF_" # mx); - - defm "" : VPseudoBinaryV_VF, - Sched<[WriteVFDivF_MX, ReadVFDivV_MX, ReadVFDivF_MX, ReadVMask]>; + defvar sews = SchedSEWSet.val; + foreach e = sews in { + defvar WriteVFDivF_MX_E = !cast("WriteVFDivF_" # mx # "_E" # e); + defvar ReadVFDivV_MX_E = !cast("ReadVFDivV_" # mx # "_E" # e); + defvar ReadVFDivF_MX_E = !cast("ReadVFDivF_" # mx # "_E" # e); + + defm "" : VPseudoBinaryV_VF_E, + Sched<[WriteVFDivF_MX_E, ReadVFDivV_MX_E, ReadVFDivF_MX_E, ReadVMask]>; + } } } } @@ -2622,11 +2708,15 @@ foreach f = FPList in { foreach m = f.MxList in { defvar mx = m.MX; - defvar WriteVFDivF_MX = !cast("WriteVFDivF_" # mx); - defvar ReadVFDivV_MX = !cast("ReadVFDivV_" # mx); - defvar ReadVFDivF_MX = !cast("ReadVFDivF_" # mx); - defm "" : VPseudoBinaryV_VF, - Sched<[WriteVFDivF_MX, ReadVFDivV_MX, ReadVFDivF_MX, ReadVMask]>; + defvar sews = SchedSEWSet.val; + foreach e = sews in { + defvar WriteVFDivF_MX_E = !cast("WriteVFDivF_" # mx # "_E" # e); + defvar ReadVFDivV_MX_E = !cast("ReadVFDivV_" # mx # "_E" # e); + defvar ReadVFDivF_MX_E = !cast("ReadVFDivF_" # mx # "_E" # e); + + defm "" : VPseudoBinaryV_VF_E, + Sched<[WriteVFDivF_MX_E, ReadVFDivV_MX_E, ReadVFDivF_MX_E, ReadVMask]>; + } } } } @@ -3747,6 +3837,23 @@ (op2_type op2_reg_class:$rs2), GPR:$vl, sew)>; +class VPatUnaryNoMask_E : + Pat<(result_type (!cast(intrinsic_name) + (result_type undef), + (op2_type op2_reg_class:$rs2), + VLOpFrag)), + (!cast(inst#"_"#kind#"_"#vlmul.MX#"_E"#sew) + (op2_type op2_reg_class:$rs2), + GPR:$vl, log2sew)>; + class VPatUnaryNoMaskTU; +class VPatUnaryNoMaskTU_E : + Pat<(result_type (!cast(intrinsic_name) + (result_type result_reg_class:$merge), + (op2_type op2_reg_class:$rs2), + VLOpFrag)), + (!cast(inst#"_"#kind#"_"#vlmul.MX#"_E"#sew#"_TU") + (result_type result_reg_class:$merge), + (op2_type op2_reg_class:$rs2), + GPR:$vl, log2sew)>; + class VPatUnaryMask; +class VPatUnaryMaskTA_E : + Pat<(result_type (!cast(intrinsic_name#"_mask") + (result_type result_reg_class:$merge), + (op2_type op2_reg_class:$rs2), + (mask_type V0), + VLOpFrag, (XLenVT timm:$policy))), + (!cast(inst#"_"#kind#"_"#vlmul.MX#"_E"#sew#"_MASK") + (result_type result_reg_class:$merge), + (op2_type op2_reg_class:$rs2), + (mask_type V0), GPR:$vl, log2sew, (XLenVT timm:$policy))>; + class VPatMaskUnaryNoMask : @@ -3849,6 +3996,28 @@ (mask_type VR:$rs2), GPR:$vl, sew)>; +class VPatUnaryAnyMask_E : + Pat<(result_type (!cast(intrinsic) + (result_type result_reg_class:$merge), + (op1_type op1_reg_class:$rs1), + (mask_type VR:$rs2), + VLOpFrag)), + (!cast(inst#"_"#kind#"_"#vlmul.MX#"_E"#sew) + (result_type result_reg_class:$merge), + (op1_type op1_reg_class:$rs1), + (mask_type VR:$rs2), + GPR:$vl, log2sew)>; + class VPatBinaryM vtilist> { + foreach vti = vtilist in { + def : VPatUnaryAnyMask_E; + } +} + multiclass VPatUnaryM_M { @@ -4216,6 +4395,23 @@ } } +multiclass VPatUnaryV_V_E vtilist> { + foreach vti = vtilist in { + def : VPatUnaryNoMask_E; + def : VPatUnaryNoMaskTU_E; + def : VPatUnaryMaskTA_E; + } +} + multiclass VPatNullaryV { foreach vti = AllIntegerVectors in { @@ -4410,18 +4606,29 @@ vti.RegClass, vti.RegClass>; } -multiclass VPatBinaryV_VV_INT vtilist> { + foreach vti = vtilist in + defm : VPatBinaryTA; +} + +multiclass VPatBinaryV_VV_INT_E vtilist> { foreach vti = vtilist in { defvar ivti = GetIntVTypeInfo.Vti; - defm : VPatBinaryTA; } } -multiclass VPatBinaryV_VV_INT_EEW vtilist> { foreach vti = vtilist in { // emul = lmul * eew / sew @@ -4431,7 +4638,7 @@ if !and(!ge(octuple_emul, 1), !le(octuple_emul, 64)) then { defvar emul_str = octuple_to_str.ret; defvar ivti = !cast("VI" # eew # emul_str); - defvar inst = instruction # "_VV_" # vti.LMul.MX # "_" # emul_str; + defvar inst = instruction # "_VV_" # vti.LMul.MX # "_E" # vti.SEW # "_" # emul_str; defm : VPatBinaryTA vtilist> { + foreach vti = vtilist in { + defvar kind = "V"#vti.ScalarSuffix; + defm : VPatBinaryTA; + } +} + multiclass VPatBinaryV_VX_INT vtilist> { foreach vti = vtilist in @@ -4716,6 +4935,11 @@ : VPatBinaryV_VV, VPatBinaryV_VX; +multiclass VPatBinaryV_VV_VX_E vtilist> + : VPatBinaryV_VV_E, + VPatBinaryV_VX_E; + multiclass VPatBinaryV_VX_VI vtilist> : VPatBinaryV_VX, @@ -4920,7 +5144,7 @@ multiclass VPatBinaryV_VV_VX_VI_INT vtilist, Operand ImmType = simm5> - : VPatBinaryV_VV_INT, + : VPatBinaryV_VV_INT_E, VPatBinaryV_VX_INT, VPatBinaryV_VI; @@ -5997,10 +6221,10 @@ //===----------------------------------------------------------------------===// // 11.11. Vector Integer Divide Instructions //===----------------------------------------------------------------------===// -defm : VPatBinaryV_VV_VX<"int_riscv_vdivu", "PseudoVDIVU", AllIntegerVectors>; -defm : VPatBinaryV_VV_VX<"int_riscv_vdiv", "PseudoVDIV", AllIntegerVectors>; -defm : VPatBinaryV_VV_VX<"int_riscv_vremu", "PseudoVREMU", AllIntegerVectors>; -defm : VPatBinaryV_VV_VX<"int_riscv_vrem", "PseudoVREM", AllIntegerVectors>; +defm : VPatBinaryV_VV_VX_E<"int_riscv_vdivu", "PseudoVDIVU", AllIntegerVectors>; +defm : VPatBinaryV_VV_VX_E<"int_riscv_vdiv", "PseudoVDIV", AllIntegerVectors>; +defm : VPatBinaryV_VV_VX_E<"int_riscv_vremu", "PseudoVREMU", AllIntegerVectors>; +defm : VPatBinaryV_VV_VX_E<"int_riscv_vrem", "PseudoVREM", AllIntegerVectors>; //===----------------------------------------------------------------------===// // 11.12. Vector Widening Integer Multiply Instructions @@ -6113,8 +6337,8 @@ // 13.4. Vector Single-Width Floating-Point Multiply/Divide Instructions //===----------------------------------------------------------------------===// defm : VPatBinaryV_VV_VX<"int_riscv_vfmul", "PseudoVFMUL", AllFloatVectors>; -defm : VPatBinaryV_VV_VX<"int_riscv_vfdiv", "PseudoVFDIV", AllFloatVectors>; -defm : VPatBinaryV_VX<"int_riscv_vfrdiv", "PseudoVFRDIV", AllFloatVectors>; +defm : VPatBinaryV_VV_VX_E<"int_riscv_vfdiv", "PseudoVFDIV", AllFloatVectors>; +defm : VPatBinaryV_VX_E<"int_riscv_vfrdiv", "PseudoVFRDIV", AllFloatVectors>; //===----------------------------------------------------------------------===// // 13.5. Vector Widening Floating-Point Multiply @@ -6144,7 +6368,7 @@ //===----------------------------------------------------------------------===// // 13.8. Vector Floating-Point Square-Root Instruction //===----------------------------------------------------------------------===// -defm : VPatUnaryV_V<"int_riscv_vfsqrt", "PseudoVFSQRT", AllFloatVectors>; +defm : VPatUnaryV_V_E<"int_riscv_vfsqrt", "PseudoVFSQRT", AllFloatVectors>; //===----------------------------------------------------------------------===// // 13.9. Vector Floating-Point Reciprocal Square-Root Estimate Instruction @@ -6408,14 +6632,14 @@ let Predicates = [HasVInstructions] in { defm : VPatBinaryV_VV_VX_VI_INT<"int_riscv_vrgather", "PseudoVRGATHER", AllIntegerVectors, uimm5>; - defm : VPatBinaryV_VV_INT_EEW<"int_riscv_vrgatherei16_vv", "PseudoVRGATHEREI16", + defm : VPatBinaryV_VV_INT_E_EEW<"int_riscv_vrgatherei16_vv", "PseudoVRGATHEREI16", /* eew */ 16, AllIntegerVectors>; } // Predicates = [HasVInstructions] let Predicates = [HasVInstructionsAnyF] in { defm : VPatBinaryV_VV_VX_VI_INT<"int_riscv_vrgather", "PseudoVRGATHER", AllFloatVectors, uimm5>; - defm : VPatBinaryV_VV_INT_EEW<"int_riscv_vrgatherei16_vv", "PseudoVRGATHEREI16", + defm : VPatBinaryV_VV_INT_E_EEW<"int_riscv_vrgatherei16_vv", "PseudoVRGATHEREI16", /* eew */ 16, AllFloatVectors>; } // Predicates = [HasVInstructionsAnyF] @@ -6423,11 +6647,11 @@ // 16.5. Vector Compress Instruction //===----------------------------------------------------------------------===// let Predicates = [HasVInstructions] in { - defm : VPatUnaryV_V_AnyMask<"int_riscv_vcompress", "PseudoVCOMPRESS", AllIntegerVectors>; + defm : VPatUnaryV_V_AnyMask_E<"int_riscv_vcompress", "PseudoVCOMPRESS", AllIntegerVectors>; } // Predicates = [HasVInstructions] let Predicates = [HasVInstructionsAnyF] in { - defm : VPatUnaryV_V_AnyMask<"int_riscv_vcompress", "PseudoVCOMPRESS", AllFloatVectors>; + defm : VPatUnaryV_V_AnyMask_E<"int_riscv_vcompress", "PseudoVCOMPRESS", AllFloatVectors>; } // Predicates = [HasVInstructionsAnyF] // Include the non-intrinsic ISel patterns diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td @@ -88,6 +88,23 @@ op_reg_class:$rs2, avl, sew)>; +class VPatBinarySDNode_VV_E : + Pat<(result_type (vop + (op_type op_reg_class:$rs1), + (op_type op_reg_class:$rs2))), + (!cast(instruction_name#"_VV_"# vlmul.MX#"_E"#sew) + op_reg_class:$rs1, + op_reg_class:$rs2, + avl, log2sew)>; + class VPatBinarySDNode_XI; +class VPatBinarySDNode_XI_E : + Pat<(result_type (vop + (vop_type vop_reg_class:$rs1), + (vop_type (SplatPatKind xop_kind:$rs2)))), + (!cast(instruction_name#_#suffix#_# vlmul.MX#"_E"#sew) + vop_reg_class:$rs1, + xop_kind:$rs2, + avl, log2sew)>; + multiclass VPatBinarySDNode_VV_VX { foreach vti = AllIntegerVectors in { def : VPatBinarySDNode_VV { + foreach vti = AllIntegerVectors in { + def : VPatBinarySDNode_VV_E; + def : VPatBinarySDNode_XI_E; + } +} + multiclass VPatBinarySDNode_VV_VX_VI : VPatBinarySDNode_VV_VX { @@ -148,6 +197,24 @@ (xop_type xop_kind:$rs2), avl, sew)>; +class VPatBinarySDNode_VF_E : + Pat<(result_type (vop (vop_type vop_reg_class:$rs1), + (vop_type (SplatFPOp xop_kind:$rs2)))), + (!cast(instruction_name#"_"#vlmul.MX#"_E"#sew) + vop_reg_class:$rs1, + (xop_type xop_kind:$rs2), + avl, log2sew)>; + multiclass VPatBinaryFPSDNode_VV_VF { foreach vti = AllFloatVectors in { def : VPatBinarySDNode_VV { + foreach vti = AllFloatVectors in { + def : VPatBinarySDNode_VV_E; + def : VPatBinarySDNode_VF_E; + } +} + multiclass VPatBinaryFPSDNode_R_VF { foreach fvti = AllFloatVectors in def : Pat<(fvti.Vector (vop (fvti.Vector (SplatFPOp fvti.Scalar:$rs2)), @@ -170,6 +249,16 @@ fvti.AVL, fvti.Log2SEW)>; } +multiclass VPatBinaryFPSDNode_R_VF_E { + foreach fvti = AllFloatVectors in + def : Pat<(fvti.Vector (vop (fvti.Vector (SplatFPOp fvti.Scalar:$rs2)), + (fvti.Vector fvti.RegClass:$rs1))), + (!cast(instruction_name#"_V"#fvti.ScalarSuffix#"_"#fvti.LMul.MX#"_E"#fvti.SEW) + fvti.RegClass:$rs1, + (fvti.Scalar fvti.ScalarRegClass:$rs2), + fvti.AVL, fvti.Log2SEW)>; +} + multiclass VPatIntegerSetCCSDNode_VV { foreach vti = AllIntegerVectors in { @@ -723,10 +812,10 @@ defm : VPatBinarySDNode_VV_VX; // 11.11. Vector Integer Divide Instructions -defm : VPatBinarySDNode_VV_VX; -defm : VPatBinarySDNode_VV_VX; -defm : VPatBinarySDNode_VV_VX; -defm : VPatBinarySDNode_VV_VX; +defm : VPatBinarySDNode_VV_VX_E; +defm : VPatBinarySDNode_VV_VX_E; +defm : VPatBinarySDNode_VV_VX_E; +defm : VPatBinarySDNode_VV_VX_E; // 11.12. Vector Widening Integer Multiply Instructions defm : VPatWidenBinarySDNode_VV_VX; -defm : VPatBinaryFPSDNode_VV_VF; -defm : VPatBinaryFPSDNode_R_VF; +defm : VPatBinaryFPSDNode_VV_VF_E; +defm : VPatBinaryFPSDNode_R_VF_E; // 13.5. Vector Widening Floating-Point Multiply Instructions defm : VPatWidenBinaryFPSDNode_VV_VF; @@ -914,7 +1003,7 @@ foreach vti = AllFloatVectors in { // 13.8. Vector Floating-Point Square-Root Instruction def : Pat<(fsqrt (vti.Vector vti.RegClass:$rs2)), - (!cast("PseudoVFSQRT_V_"# vti.LMul.MX) + (!cast("PseudoVFSQRT_V_"# vti.LMul.MX#"_E"#vti.SEW) vti.RegClass:$rs2, vti.AVL, vti.Log2SEW)>; // 13.12. Vector Floating-Point Sign-Injection Instructions diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td @@ -441,6 +441,32 @@ (mask_type V0), GPR:$vl, sew, TAIL_AGNOSTIC)>; } +multiclass VPatBinaryVL_V_E { + def : Pat<(result_type (vop + (op1_type op1_reg_class:$rs1), + (op2_type op2_reg_class:$rs2), + (result_type result_reg_class:$merge), + (mask_type V0), + VLOpFrag)), + (!cast(instruction_name#"_"#suffix#"_"# vlmul.MX#"_E"# sew#"_MASK") + result_reg_class:$merge, + op1_reg_class:$rs1, + op2_reg_class:$rs2, + (mask_type V0), GPR:$vl, log2sew, TAIL_AGNOSTIC)>; +} + multiclass VPatTiedBinaryNoMaskVL_V; } +multiclass VPatBinaryVL_XI_E { + def : Pat<(result_type (vop + (vop1_type vop_reg_class:$rs1), + (vop2_type (SplatPatKind (XLenVT xop_kind:$rs2))), + (result_type result_reg_class:$merge), + (mask_type V0), + VLOpFrag)), + (!cast(instruction_name#_#suffix#_# vlmul.MX#"_E"# sew#"_MASK") + result_reg_class:$merge, + vop_reg_class:$rs1, + xop_kind:$rs2, + (mask_type V0), GPR:$vl, log2sew, TAIL_AGNOSTIC)>; +} + multiclass VPatBinaryVL_VV_VX { foreach vti = AllIntegerVectors in { defm : VPatBinaryVL_V { + foreach vti = AllIntegerVectors in { + defm : VPatBinaryVL_V_E; + defm : VPatBinaryVL_XI_E; + } +} + multiclass VPatBinaryVL_VV_VX_VI : VPatBinaryVL_VV_VX { @@ -602,6 +668,29 @@ (mask_type V0), GPR:$vl, sew, TAIL_AGNOSTIC)>; } +multiclass VPatBinaryVL_VF_E { + def : Pat<(result_type (vop (vop_type vop_reg_class:$rs1), + (vop_type (SplatFPOp scalar_reg_class:$rs2)), + (result_type result_reg_class:$merge), + (mask_type V0), + VLOpFrag)), + (!cast(instruction_name#"_"#vlmul.MX#"_E"#sew#"_MASK") + result_reg_class:$merge, + vop_reg_class:$rs1, + scalar_reg_class:$rs2, + (mask_type V0), GPR:$vl, log2sew, TAIL_AGNOSTIC)>; +} + multiclass VPatBinaryFPVL_VV_VF { foreach vti = AllFloatVectors in { defm : VPatBinaryVL_V { + foreach vti = AllFloatVectors in { + defm : VPatBinaryVL_V_E; + defm : VPatBinaryVL_VF_E; + } +} + multiclass VPatBinaryFPVL_R_VF { foreach fvti = AllFloatVectors in { def : Pat<(fvti.Vector (vop (SplatFPOp fvti.ScalarRegClass:$rs2), @@ -629,6 +731,20 @@ } } +multiclass VPatBinaryFPVL_R_VF_E { + foreach fvti = AllFloatVectors in { + def : Pat<(fvti.Vector (vop (SplatFPOp fvti.ScalarRegClass:$rs2), + fvti.RegClass:$rs1, + (fvti.Vector fvti.RegClass:$merge), + (fvti.Mask V0), + VLOpFrag)), + (!cast(instruction_name#"_V"#fvti.ScalarSuffix#"_"#fvti.LMul.MX#"_E"#fvti.SEW#"_MASK") + fvti.RegClass:$merge, + fvti.RegClass:$rs1, fvti.ScalarRegClass:$rs2, + (fvti.Mask V0), GPR:$vl, fvti.Log2SEW, TAIL_AGNOSTIC)>; + } +} + multiclass VPatIntegerSetCCVL_VV { def : Pat<(vti.Mask (riscv_setcc_vl (vti.Vector vti.RegClass:$rs1), @@ -1488,10 +1604,10 @@ defm : VPatBinaryVL_VV_VX; // 11.11. Vector Integer Divide Instructions -defm : VPatBinaryVL_VV_VX; -defm : VPatBinaryVL_VV_VX; -defm : VPatBinaryVL_VV_VX; -defm : VPatBinaryVL_VV_VX; +defm : VPatBinaryVL_VV_VX_E; +defm : VPatBinaryVL_VV_VX_E; +defm : VPatBinaryVL_VV_VX_E; +defm : VPatBinaryVL_VV_VX_E; // 11.12. Vector Widening Integer Multiply Instructions defm : VPatBinaryWVL_VV_VX; @@ -1617,8 +1733,8 @@ // 13.4. Vector Single-Width Floating-Point Multiply/Divide Instructions defm : VPatBinaryFPVL_VV_VF; -defm : VPatBinaryFPVL_VV_VF; -defm : VPatBinaryFPVL_R_VF; +defm : VPatBinaryFPVL_VV_VF_E; +defm : VPatBinaryFPVL_R_VF_E; // 13.5. Vector Widening Floating-Point Multiply Instructions defm : VPatWidenBinaryFPVL_VV_VF; @@ -1660,7 +1776,7 @@ // 13.8. Vector Floating-Point Square-Root Instruction def : Pat<(riscv_fsqrt_vl (vti.Vector vti.RegClass:$rs2), (vti.Mask V0), VLOpFrag), - (!cast("PseudoVFSQRT_V_"# vti.LMul.MX #"_MASK") + (!cast("PseudoVFSQRT_V_"# vti.LMul.MX # "_E" # vti.SEW # "_MASK") (vti.Vector (IMPLICIT_DEF)), vti.RegClass:$rs2, (vti.Mask V0), GPR:$vl, vti.Log2SEW, TA_MA)>; @@ -2023,7 +2139,7 @@ vti.RegClass:$merge, (vti.Mask V0), VLOpFrag)), - (!cast("PseudoVRGATHER_VV_"# vti.LMul.MX#"_MASK") + (!cast("PseudoVRGATHER_VV_"# vti.LMul.MX#"_E"# vti.SEW#"_MASK") vti.RegClass:$merge, vti.RegClass:$rs2, vti.RegClass:$rs1, (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(vti.Vector (riscv_vrgather_vx_vl vti.RegClass:$rs2, GPR:$rs1, @@ -2049,7 +2165,7 @@ if !and(!ge(octuple_emul, 1), !le(octuple_emul, 64)) then { defvar emul_str = octuple_to_str.ret; defvar ivti = !cast("VI16" # emul_str); - defvar inst = "PseudoVRGATHEREI16_VV_" # vti.LMul.MX # "_" # emul_str; + defvar inst = "PseudoVRGATHEREI16_VV_" # vti.LMul.MX # "_E" # vti.SEW # "_" # emul_str; def : Pat<(vti.Vector (riscv_vrgatherei16_vv_vl vti.RegClass:$rs2, @@ -2093,7 +2209,7 @@ vti.RegClass:$merge, (vti.Mask V0), VLOpFrag)), - (!cast("PseudoVRGATHER_VV_"# vti.LMul.MX#"_MASK") + (!cast("PseudoVRGATHER_VV_"# vti.LMul.MX#"_E"# vti.SEW#"_MASK") vti.RegClass:$merge, vti.RegClass:$rs2, vti.RegClass:$rs1, (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(vti.Vector (riscv_vrgather_vx_vl vti.RegClass:$rs2, GPR:$rs1, @@ -2119,7 +2235,7 @@ if !and(!ge(octuple_emul, 1), !le(octuple_emul, 64)) then { defvar emul_str = octuple_to_str.ret; defvar ivti = !cast("VI16" # emul_str); - defvar inst = "PseudoVRGATHEREI16_VV_" # vti.LMul.MX # "_" # emul_str; + defvar inst = "PseudoVRGATHEREI16_VV_" # vti.LMul.MX # "_E" # vti.SEW # "_" # emul_str; def : Pat<(vti.Vector (riscv_vrgatherei16_vv_vl vti.RegClass:$rs2, diff --git a/llvm/lib/Target/RISCV/RISCVScheduleV.td b/llvm/lib/Target/RISCV/RISCVScheduleV.td --- a/llvm/lib/Target/RISCV/RISCVScheduleV.td +++ b/llvm/lib/Target/RISCV/RISCVScheduleV.td @@ -17,6 +17,16 @@ // Used for widening floating-point Reduction as it doesn't contain MF8. defvar SchedMxListFWRed = ["UpperBound", "MF4", "MF2", "M1", "M2", "M4", "M8"]; +class SchedSEWSet { + list val = !cond(!eq(mx, "M1"): [8, 16, 32, 64], + !eq(mx, "M2"): [8, 16, 32, 64], + !eq(mx, "M4"): [8, 16, 32, 64], + !eq(mx, "M8"): [8, 16, 32, 64], + !eq(mx, "MF2"): [8, 16, 32], + !eq(mx, "MF4"): [8, 16], + !eq(mx, "MF8"): [8]); +} + // Creates SchedWrite for each (name, LMUL) pair for LMUL in SchedMxList multiclass LMULSchedWrites { foreach mx = SchedMxList in { @@ -24,6 +34,17 @@ } } +// Creates SchedWrite for each (name, LMUL, SEW) tuple +multiclass LMULSEWSchedWrites { + foreach mx = SchedMxList in { + if !eq(mx, "UpperBound") then + def name # "_" # mx : SchedWrite; + else + foreach sew = SchedSEWSet.val in + def name # "_" # mx # "_E" # sew : SchedWrite; + } +} + // Creates SchedWrite for each (name, LMUL) pair for LMUL in SchedMxListW multiclass LMULSchedWritesW { foreach mx = SchedMxListW in { @@ -52,6 +73,17 @@ } } +// Creates SchedRead for each (name, LMUL) pair for LMUL in SchedMxList +multiclass LMULSEWSchedReads { + foreach mx = SchedMxList in { + if !eq(mx, "UpperBound") then + def name # "_" # mx : SchedRead; + else + foreach sew = SchedSEWSet.val in + def name # "_" # mx # "_E" # sew : SchedRead; + } +} + // Creates SchedRead for each (name, LMUL) pair for LMUL in SchedMxListW multiclass LMULSchedReadsW { foreach mx = SchedMxListW in { @@ -74,6 +106,18 @@ } } +// Creates WriteRes for each (name, LMUL, resources) tuple for LMUL +// in SchedMxList +multiclass LMULSEWWriteRes resources> { + foreach mx = SchedMxList in { + if !eq(mx, "UpperBound") then + def : WriteRes(name # "_" # mx), resources>; + else + foreach sew = SchedSEWSet.val in + def : WriteRes(name # "_" # mx # "_E" # sew), resources>; + } +} + // Creates WriteRes for each (name, LMUL, resources) tuple for LMUL // in SchedMxListW multiclass LMULWriteResW resources> { @@ -106,6 +150,18 @@ } } +// Creates ReadAdvance for each (name, LMUL, val) tuple for LMUL +// in SchedMxList +multiclass LMULSEWReadAdvance writes = []> { + foreach mx = SchedMxList in { + if !eq(mx, "UpperBound") then + def : ReadAdvance(name # "_" # mx), val, writes>; + else + foreach sew = SchedSEWSet.val in + def : ReadAdvance(name # "_" # mx # "_E" # sew), val, writes>; + } +} + // Creates ReadAdvance for each (name, LMUL, val) tuple for LMUL // in SchedMxListW multiclass LMULReadAdvanceW writes = []> { @@ -222,8 +278,8 @@ defm "" : LMULSchedWrites<"WriteVIMulV">; defm "" : LMULSchedWrites<"WriteVIMulX">; // 11.11. Vector Integer Divide Instructions -defm "" : LMULSchedWrites<"WriteVIDivV">; -defm "" : LMULSchedWrites<"WriteVIDivX">; +defm "" : LMULSEWSchedWrites<"WriteVIDivV">; +defm "" : LMULSEWSchedWrites<"WriteVIDivX">; // 11.12. Vector Widening Integer Multiply Instructions defm "" : LMULSchedWritesW<"WriteVIWMulV">; defm "" : LMULSchedWritesW<"WriteVIWMulX">; @@ -272,8 +328,8 @@ // 13.4. Vector Single-Width Floating-Point Multiply/Divide Instructions defm "" : LMULSchedWrites<"WriteVFMulV">; defm "" : LMULSchedWrites<"WriteVFMulF">; -defm "" : LMULSchedWrites<"WriteVFDivV">; -defm "" : LMULSchedWrites<"WriteVFDivF">; +defm "" : LMULSEWSchedWrites<"WriteVFDivV">; +defm "" : LMULSEWSchedWrites<"WriteVFDivF">; // 13.5. Vector Widening Floating-Point Multiply defm "" : LMULSchedWritesFW<"WriteVFWMulV">; defm "" : LMULSchedWritesFW<"WriteVFWMulF">; @@ -284,7 +340,7 @@ defm "" : LMULSchedWritesFW<"WriteVFWMulAddV">; defm "" : LMULSchedWritesFW<"WriteVFWMulAddF">; // 13.8. Vector Floating-Point Square-Root Instruction -defm "" : LMULSchedWrites<"WriteVFSqrtV">; +defm "" : LMULSEWSchedWrites<"WriteVFSqrtV">; // 13.9. Vector Floating-Point Reciprocal Square-Root Estimate Instruction // 13.10. Vector Floating-Point Reciprocal Estimate Instruction defm "" : LMULSchedWrites<"WriteVFRecpV">; @@ -358,11 +414,11 @@ defm "" : LMULSchedWrites<"WriteVISlide1X">; defm "" : LMULSchedWrites<"WriteVFSlide1F">; // 16.4. Vector Register Gather Instructions -defm "" : LMULSchedWrites<"WriteVGatherV">; +defm "" : LMULSEWSchedWrites<"WriteVGatherV">; defm "" : LMULSchedWrites<"WriteVGatherX">; defm "" : LMULSchedWrites<"WriteVGatherI">; // 16.5. Vector Compress Instruction -defm "" : LMULSchedWrites<"WriteVCompressV">; +defm "" : LMULSEWSchedWrites<"WriteVCompressV">; // 16.6. Whole Vector Register Move // These are already LMUL aware def WriteVMov1V : SchedWrite; @@ -446,8 +502,8 @@ defm "" : LMULSchedReads<"ReadVIMulV">; defm "" : LMULSchedReads<"ReadVIMulX">; // 11.11. Vector Integer Divide Instructions -defm "" : LMULSchedReads<"ReadVIDivV">; -defm "" : LMULSchedReads<"ReadVIDivX">; +defm "" : LMULSEWSchedReads<"ReadVIDivV">; +defm "" : LMULSEWSchedReads<"ReadVIDivX">; // 11.12. Vector Widening Integer Multiply Instructions defm "" : LMULSchedReadsW<"ReadVIWMulV">; defm "" : LMULSchedReadsW<"ReadVIWMulX">; @@ -491,8 +547,8 @@ // 13.4. Vector Single-Width Floating-Point Multiply/Divide Instructions defm "" : LMULSchedReads<"ReadVFMulV">; defm "" : LMULSchedReads<"ReadVFMulF">; -defm "" : LMULSchedReads<"ReadVFDivV">; -defm "" : LMULSchedReads<"ReadVFDivF">; +defm "" : LMULSEWSchedReads<"ReadVFDivV">; +defm "" : LMULSEWSchedReads<"ReadVFDivF">; // 13.5. Vector Widening Floating-Point Multiply defm "" : LMULSchedReadsFW<"ReadVFWMulV">; defm "" : LMULSchedReadsFW<"ReadVFWMulF">; @@ -503,7 +559,7 @@ defm "" : LMULSchedReadsFW<"ReadVFWMulAddV">; defm "" : LMULSchedReadsFW<"ReadVFWMulAddF">; // 13.8. Vector Floating-Point Square-Root Instruction -defm "" : LMULSchedReads<"ReadVFSqrtV">; +defm "" : LMULSEWSchedReads<"ReadVFSqrtV">; // 13.9. Vector Floating-Point Reciprocal Square-Root Estimate Instruction // 13.10. Vector Floating-Point Reciprocal Estimate Instruction defm "" : LMULSchedReads<"ReadVFRecpV">; @@ -580,10 +636,11 @@ defm "" : LMULSchedReads<"ReadVFSlideV">; defm "" : LMULSchedReads<"ReadVFSlideF">; // 16.4. Vector Register Gather Instructions -defm "" : LMULSchedReads<"ReadVGatherV">; +defm "" : LMULSEWSchedReads<"ReadVGatherV">; defm "" : LMULSchedReads<"ReadVGatherX">; +defm "" : LMULSchedReads<"ReadVGatherI">; // 16.5. Vector Compress Instruction -defm "" : LMULSchedReads<"ReadVCompressV">; +defm "" : LMULSEWSchedReads<"ReadVCompressV">; // 16.6. Whole Vector Register Move // These are already LMUL aware def ReadVMov1V : SchedRead; @@ -684,8 +741,8 @@ defm "" : LMULWriteRes<"WriteVICmpI", []>; defm "" : LMULWriteRes<"WriteVIMulV", []>; defm "" : LMULWriteRes<"WriteVIMulX", []>; -defm "" : LMULWriteRes<"WriteVIDivV", []>; -defm "" : LMULWriteRes<"WriteVIDivX", []>; +defm "" : LMULSEWWriteRes<"WriteVIDivV", []>; +defm "" : LMULSEWWriteRes<"WriteVIDivX", []>; defm "" : LMULWriteResW<"WriteVIWMulV", []>; defm "" : LMULWriteResW<"WriteVIWMulX", []>; defm "" : LMULWriteRes<"WriteVIMulAddV", []>; @@ -721,15 +778,15 @@ defm "" : LMULWriteResFW<"WriteVFWALUF", []>; defm "" : LMULWriteRes<"WriteVFMulV", []>; defm "" : LMULWriteRes<"WriteVFMulF", []>; -defm "" : LMULWriteRes<"WriteVFDivV", []>; -defm "" : LMULWriteRes<"WriteVFDivF", []>; +defm "" : LMULSEWWriteRes<"WriteVFDivV", []>; +defm "" : LMULSEWWriteRes<"WriteVFDivF", []>; defm "" : LMULWriteResFW<"WriteVFWMulV", []>; defm "" : LMULWriteResFW<"WriteVFWMulF", []>; defm "" : LMULWriteRes<"WriteVFMulAddV", []>; defm "" : LMULWriteRes<"WriteVFMulAddF", []>; defm "" : LMULWriteResFW<"WriteVFWMulAddV", []>; defm "" : LMULWriteResFW<"WriteVFWMulAddF", []>; -defm "" : LMULWriteRes<"WriteVFSqrtV", []>; +defm "" : LMULSEWWriteRes<"WriteVFSqrtV", []>; defm "" : LMULWriteRes<"WriteVFRecpV", []>; defm "" : LMULWriteRes<"WriteVFCmpV", []>; defm "" : LMULWriteRes<"WriteVFCmpF", []>; @@ -772,10 +829,10 @@ defm "" : LMULWriteRes<"WriteVISlideI", []>; defm "" : LMULWriteRes<"WriteVISlide1X", []>; defm "" : LMULWriteRes<"WriteVFSlide1F", []>; -defm "" : LMULWriteRes<"WriteVGatherV", []>; +defm "" : LMULSEWWriteRes<"WriteVGatherV", []>; defm "" : LMULWriteRes<"WriteVGatherX", []>; defm "" : LMULWriteRes<"WriteVGatherI", []>; -defm "" : LMULWriteRes<"WriteVCompressV", []>; +defm "" : LMULSEWWriteRes<"WriteVCompressV", []>; // These are already LMUL aware def : WriteRes; def : WriteRes; @@ -839,8 +896,8 @@ defm "" : LMULReadAdvance<"ReadVICmpX", 0>; defm "" : LMULReadAdvance<"ReadVIMulV", 0>; defm "" : LMULReadAdvance<"ReadVIMulX", 0>; -defm "" : LMULReadAdvance<"ReadVIDivV", 0>; -defm "" : LMULReadAdvance<"ReadVIDivX", 0>; +defm "" : LMULSEWReadAdvance<"ReadVIDivV", 0>; +defm "" : LMULSEWReadAdvance<"ReadVIDivX", 0>; defm "" : LMULReadAdvanceW<"ReadVIWMulV", 0>; defm "" : LMULReadAdvanceW<"ReadVIWMulX", 0>; defm "" : LMULReadAdvance<"ReadVIMulAddV", 0>; @@ -871,15 +928,15 @@ defm "" : LMULReadAdvanceFW<"ReadVFWALUF", 0>; defm "" : LMULReadAdvance<"ReadVFMulV", 0>; defm "" : LMULReadAdvance<"ReadVFMulF", 0>; -defm "" : LMULReadAdvance<"ReadVFDivV", 0>; -defm "" : LMULReadAdvance<"ReadVFDivF", 0>; +defm "" : LMULSEWReadAdvance<"ReadVFDivV", 0>; +defm "" : LMULSEWReadAdvance<"ReadVFDivF", 0>; defm "" : LMULReadAdvanceFW<"ReadVFWMulV", 0>; defm "" : LMULReadAdvanceFW<"ReadVFWMulF", 0>; defm "" : LMULReadAdvance<"ReadVFMulAddV", 0>; defm "" : LMULReadAdvance<"ReadVFMulAddF", 0>; defm "" : LMULReadAdvanceFW<"ReadVFWMulAddV", 0>; defm "" : LMULReadAdvanceFW<"ReadVFWMulAddF", 0>; -defm "" : LMULReadAdvance<"ReadVFSqrtV", 0>; +defm "" : LMULSEWReadAdvance<"ReadVFSqrtV", 0>; defm "" : LMULReadAdvance<"ReadVFRecpV", 0>; defm "" : LMULReadAdvance<"ReadVFCmpV", 0>; defm "" : LMULReadAdvance<"ReadVFCmpF", 0>; @@ -930,9 +987,10 @@ defm "" : LMULReadAdvance<"ReadVISlideX", 0>; defm "" : LMULReadAdvance<"ReadVFSlideV", 0>; defm "" : LMULReadAdvance<"ReadVFSlideF", 0>; -defm "" : LMULReadAdvance<"ReadVGatherV", 0>; +defm "" : LMULSEWReadAdvance<"ReadVGatherV", 0>; defm "" : LMULReadAdvance<"ReadVGatherX", 0>; -defm "" : LMULReadAdvance<"ReadVCompressV", 0>; +defm "" : LMULReadAdvance<"ReadVGatherI", 0>; +defm "" : LMULSEWReadAdvance<"ReadVCompressV", 0>; // These are already LMUL aware def : ReadAdvance; def : ReadAdvance;