diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td @@ -49,6 +49,10 @@ class PseudoToVInst { defvar AffixSubsts = [["Pseudo", ""], + ["_E64", ""], + ["_E32", ""], + ["_E16", ""], + ["_E8", ""], ["_F64", "_F"], ["_F32", "_F"], ["_F16", "_F"], @@ -1906,12 +1910,16 @@ multiclass VPseudoVCPR_V { foreach m = MxList in { defvar mx = m.MX; - defvar WriteVCompressV_MX = !cast("WriteVCompressV_" # mx); - defvar ReadVCompressV_MX = !cast("ReadVCompressV_" # mx); - + defvar sews = SchedSEWSet.val; let VLMul = m.value in - def _VM # "_" # m.MX : VPseudoUnaryAnyMask, - Sched<[WriteVCompressV_MX, ReadVCompressV_MX, ReadVCompressV_MX]>; + foreach e = sews in { + defvar suffix = "_" # m.MX # "_E" # e; + defvar WriteVCompressV_MX_E = !cast("WriteVCompressV" # suffix); + defvar ReadVCompressV_MX_E = !cast("ReadVCompressV" # suffix); + + def _VM # suffix : VPseudoUnaryAnyMask, + Sched<[WriteVCompressV_MX_E, ReadVCompressV_MX_E, ReadVCompressV_MX_E]>; + } } } @@ -1931,6 +1939,24 @@ } } +multiclass VPseudoBinary_E { + let VLMul = MInfo.value in { + defvar suffix = "_" # MInfo.MX # "_E" # sew; + def suffix : VPseudoBinaryNoMask; + def suffix # "_TU" : VPseudoBinaryNoMaskTU; + def suffix # "_MASK" : VPseudoBinaryMaskPolicy, + RISCVMaskedPseudo; + } +} + multiclass VPseudoBinaryM { + let VLMul = lmul.value in { + defvar suffix = "_" # lmul.MX # "_E" # sew # "_" # emul.MX; + def suffix : VPseudoBinaryNoMask; + def suffix # "_TU" : VPseudoBinaryNoMaskTU; + def suffix # "_MASK" : VPseudoBinaryMaskPolicy, + RISCVMaskedPseudo; + } +} + multiclass VPseudoTiedBinary; } +multiclass VPseudoBinaryV_VV_E { + defm _VV : VPseudoBinary_E; +} + // Similar to VPseudoBinaryV_VV, but uses MxListF. multiclass VPseudoBinaryFV_VV { defm _VV : VPseudoBinary; } +multiclass VPseudoBinaryFV_VV_E { + defm _VV : VPseudoBinary_E; +} + multiclass VPseudoVGTR_VV_EEW { foreach m = MxList in { defvar mx = m.MX; - defvar WriteVRGatherVV_MX = !cast("WriteVRGatherVV_" # mx); - defvar ReadVRGatherVV_data_MX = !cast("ReadVRGatherVV_data_" # mx); - defvar ReadVRGatherVV_index_MX = !cast("ReadVRGatherVV_index_" # mx); - foreach sew = EEWList in { defvar octuple_lmul = m.octuple; // emul = lmul * eew / sew @@ -1998,9 +2047,14 @@ if !and(!ge(octuple_emul, 1), !le(octuple_emul, 64)) then { defvar emulMX = octuple_to_str.ret; defvar emul = !cast("V_" # emulMX); - defm _VV : VPseudoBinaryEmul, - Sched<[WriteVRGatherVV_MX, ReadVRGatherVV_data_MX, - ReadVRGatherVV_index_MX]>; + defvar sews = SchedSEWSet.val; + foreach e = sews in { + defvar WriteVRGatherVV_MX_E = !cast("WriteVRGatherVV_" # mx # "_E" # e); + defvar ReadVRGatherVV_data_MX_E = !cast("ReadVRGatherVV_data_" # mx # "_E" # e); + defvar ReadVRGatherVV_index_MX_E = !cast("ReadVRGatherVV_index_" # mx # "_E" # e); + defm _VV : VPseudoBinaryEmul_E, + Sched<[WriteVRGatherVV_MX_E, ReadVRGatherVV_data_MX_E, ReadVRGatherVV_index_MX_E]>; + } } } } @@ -2010,6 +2064,10 @@ defm "_VX" : VPseudoBinary; } +multiclass VPseudoBinaryV_VX_E { + defm "_VX" : VPseudoBinary_E; +} + multiclass VPseudoVSLD1_VX { foreach m = MxList in { defvar mx = m.MX; @@ -2027,6 +2085,12 @@ f.fprclass, m, Constraint>; } +multiclass VPseudoBinaryV_VF_E { + defm "_V" # f.FX : VPseudoBinary_E; +} + multiclass VPseudoVSLD1_VF { foreach f = FPList in { foreach m = f.MxList in { @@ -2264,18 +2328,25 @@ multiclass VPseudoVSQR_V { foreach m = MxListF in { defvar mx = m.MX; - defvar WriteVFSqrtV_MX = !cast("WriteVFSqrtV_" # mx); - defvar ReadVFSqrtV_MX = !cast("ReadVFSqrtV_" # mx); + defvar sews = SchedSEWSet.val; - let VLMul = m.value in { - def "_V_" # mx : VPseudoUnaryNoMask, - Sched<[WriteVFSqrtV_MX, ReadVFSqrtV_MX, ReadVMask]>; - def "_V_" # mx # "_TU": VPseudoUnaryNoMaskTU, - Sched<[WriteVFSqrtV_MX, ReadVFSqrtV_MX, ReadVMask]>; - def "_V_" # mx # "_MASK" : VPseudoUnaryMaskTA, - RISCVMaskedPseudo, - Sched<[WriteVFSqrtV_MX, ReadVFSqrtV_MX, ReadVMask]>; - } + let VLMul = m.value in + foreach e = sews in { + defvar suffix = "_" # mx # "_E" # e; + defvar WriteVFSqrtV_MX_E = !cast("WriteVFSqrtV" # suffix); + defvar ReadVFSqrtV_MX_E = !cast("ReadVFSqrtV" # suffix); + + def "_V" # suffix : VPseudoUnaryNoMask, + Sched<[WriteVFSqrtV_MX_E, ReadVFSqrtV_MX_E, + ReadVMask]>; + def "_V" # suffix # "_TU": VPseudoUnaryNoMaskTU, + Sched<[WriteVFSqrtV_MX_E, ReadVFSqrtV_MX_E, + ReadVMask]>; + def "_V" # suffix # "_MASK" : VPseudoUnaryMaskTA, + RISCVMaskedPseudo, + Sched<[WriteVFSqrtV_MX_E, ReadVFSqrtV_MX_E, + ReadVMask]>; + } } } @@ -2396,23 +2467,27 @@ multiclass VPseudoVGTR_VV_VX_VI { foreach m = MxList in { defvar mx = m.MX; - defvar WriteVRGatherVV_MX = !cast("WriteVRGatherVV_" # mx); defvar WriteVRGatherVX_MX = !cast("WriteVRGatherVX_" # mx); defvar WriteVRGatherVI_MX = !cast("WriteVRGatherVI_" # mx); - defvar ReadVRGatherVV_data_MX = !cast("ReadVRGatherVV_data_" # mx); - defvar ReadVRGatherVV_index_MX = !cast("ReadVRGatherVV_index_" # mx); defvar ReadVRGatherVX_data_MX = !cast("ReadVRGatherVX_data_" # mx); defvar ReadVRGatherVX_index_MX = !cast("ReadVRGatherVX_index_" # mx); defvar ReadVRGatherVI_data_MX = !cast("ReadVRGatherVI_data_" # mx); - defm "" : VPseudoBinaryV_VV, - Sched<[WriteVRGatherVV_MX, ReadVRGatherVV_data_MX, - ReadVRGatherVV_index_MX, ReadVMask]>; defm "" : VPseudoBinaryV_VX, Sched<[WriteVRGatherVX_MX, ReadVRGatherVX_data_MX, ReadVRGatherVX_index_MX, ReadVMask]>; defm "" : VPseudoBinaryV_VI, Sched<[WriteVRGatherVI_MX, ReadVRGatherVI_data_MX, ReadVMask]>; + + defvar sews = SchedSEWSet.val; + foreach e = sews in { + defvar WriteVRGatherVV_MX_E = !cast("WriteVRGatherVV_" # mx # "_E" # e); + defvar ReadVRGatherVV_data_MX_E = !cast("ReadVRGatherVV_data_" # mx # "_E" # e); + defvar ReadVRGatherVV_index_MX_E = !cast("ReadVRGatherVV_index_" # mx # "_E" # e); + defm "" : VPseudoBinaryV_VV_E, + Sched<[WriteVRGatherVV_MX_E, ReadVRGatherVV_data_MX_E, + ReadVRGatherVV_index_MX_E, ReadVMask]>; + } } } @@ -2567,15 +2642,18 @@ multiclass VPseudoVDIV_VV_VX { foreach m = MxList in { defvar mx = m.MX; - defvar WriteVIDivV_MX = !cast("WriteVIDivV_" # mx); - defvar WriteVIDivX_MX = !cast("WriteVIDivX_" # mx); - defvar ReadVIDivV_MX = !cast("ReadVIDivV_" # mx); - defvar ReadVIDivX_MX = !cast("ReadVIDivX_" # mx); - - defm "" : VPseudoBinaryV_VV, - Sched<[WriteVIDivV_MX, ReadVIDivV_MX, ReadVIDivV_MX, ReadVMask]>; - defm "" : VPseudoBinaryV_VX, - Sched<[WriteVIDivX_MX, ReadVIDivV_MX, ReadVIDivX_MX, ReadVMask]>; + defvar sews = SchedSEWSet.val; + foreach e = sews in { + defvar WriteVIDivV_MX_E = !cast("WriteVIDivV_" # mx # "_E" # e); + defvar WriteVIDivX_MX_E = !cast("WriteVIDivX_" # mx # "_E" # e); + defvar ReadVIDivV_MX_E = !cast("ReadVIDivV_" # mx # "_E" # e); + defvar ReadVIDivX_MX_E = !cast("ReadVIDivX_" # mx # "_E" # e); + + defm "" : VPseudoBinaryV_VV_E, + Sched<[WriteVIDivV_MX_E, ReadVIDivV_MX_E, ReadVIDivV_MX_E, ReadVMask]>; + defm "" : VPseudoBinaryV_VX_E, + Sched<[WriteVIDivX_MX_E, ReadVIDivV_MX_E, ReadVIDivX_MX_E, ReadVMask]>; + } } } @@ -2605,22 +2683,28 @@ multiclass VPseudoVFDIV_VV_VF { foreach m = MxListF in { defvar mx = m.MX; - defvar WriteVFDivV_MX = !cast("WriteVFDivV_" # mx); - defvar ReadVFDivV_MX = !cast("ReadVFDivV_" # mx); + defvar sews = SchedSEWSet.val; + foreach e = sews in { + defvar WriteVFDivV_MX_E = !cast("WriteVFDivV_" # mx # "_E" # e); + defvar ReadVFDivV_MX_E = !cast("ReadVFDivV_" # mx # "_E" # e); - defm "" : VPseudoBinaryFV_VV, - Sched<[WriteVFDivV_MX, ReadVFDivV_MX, ReadVFDivV_MX, ReadVMask]>; + defm "" : VPseudoBinaryFV_VV_E, + Sched<[WriteVFDivV_MX_E, ReadVFDivV_MX_E, ReadVFDivV_MX_E, ReadVMask]>; + } } foreach f = FPList in { foreach m = f.MxList in { defvar mx = m.MX; - defvar WriteVFDivF_MX = !cast("WriteVFDivF_" # mx); - defvar ReadVFDivV_MX = !cast("ReadVFDivV_" # mx); - defvar ReadVFDivF_MX = !cast("ReadVFDivF_" # mx); - - defm "" : VPseudoBinaryV_VF, - Sched<[WriteVFDivF_MX, ReadVFDivV_MX, ReadVFDivF_MX, ReadVMask]>; + defvar sews = SchedSEWSet.val; + foreach e = sews in { + defvar WriteVFDivF_MX_E = !cast("WriteVFDivF_" # mx # "_E" # e); + defvar ReadVFDivV_MX_E = !cast("ReadVFDivV_" # mx # "_E" # e); + defvar ReadVFDivF_MX_E = !cast("ReadVFDivF_" # mx # "_E" # e); + + defm "" : VPseudoBinaryV_VF_E, + Sched<[WriteVFDivF_MX_E, ReadVFDivV_MX_E, ReadVFDivF_MX_E, ReadVMask]>; + } } } } @@ -2629,11 +2713,15 @@ foreach f = FPList in { foreach m = f.MxList in { defvar mx = m.MX; - defvar WriteVFDivF_MX = !cast("WriteVFDivF_" # mx); - defvar ReadVFDivV_MX = !cast("ReadVFDivV_" # mx); - defvar ReadVFDivF_MX = !cast("ReadVFDivF_" # mx); - defm "" : VPseudoBinaryV_VF, - Sched<[WriteVFDivF_MX, ReadVFDivV_MX, ReadVFDivF_MX, ReadVMask]>; + defvar sews = SchedSEWSet.val; + foreach e = sews in { + defvar WriteVFDivF_MX_E = !cast("WriteVFDivF_" # mx # "_E" # e); + defvar ReadVFDivV_MX_E = !cast("ReadVFDivV_" # mx # "_E" # e); + defvar ReadVFDivF_MX_E = !cast("ReadVFDivF_" # mx # "_E" # e); + + defm "" : VPseudoBinaryV_VF_E, + Sched<[WriteVFDivF_MX_E, ReadVFDivV_MX_E, ReadVFDivF_MX_E, ReadVMask]>; + } } } } @@ -3754,6 +3842,23 @@ (op2_type op2_reg_class:$rs2), GPR:$vl, sew)>; +class VPatUnaryNoMask_E : + Pat<(result_type (!cast(intrinsic_name) + (result_type undef), + (op2_type op2_reg_class:$rs2), + VLOpFrag)), + (!cast(inst#"_"#kind#"_"#vlmul.MX#"_E"#sew) + (op2_type op2_reg_class:$rs2), + GPR:$vl, log2sew)>; + class VPatUnaryNoMaskTU; +class VPatUnaryNoMaskTU_E : + Pat<(result_type (!cast(intrinsic_name) + (result_type result_reg_class:$merge), + (op2_type op2_reg_class:$rs2), + VLOpFrag)), + (!cast(inst#"_"#kind#"_"#vlmul.MX#"_E"#sew#"_TU") + (result_type result_reg_class:$merge), + (op2_type op2_reg_class:$rs2), + GPR:$vl, log2sew)>; + class VPatUnaryMask; +class VPatUnaryMaskTA_E : + Pat<(result_type (!cast(intrinsic_name#"_mask") + (result_type result_reg_class:$merge), + (op2_type op2_reg_class:$rs2), + (mask_type V0), + VLOpFrag, (XLenVT timm:$policy))), + (!cast(inst#"_"#kind#"_"#vlmul.MX#"_E"#sew#"_MASK") + (result_type result_reg_class:$merge), + (op2_type op2_reg_class:$rs2), + (mask_type V0), GPR:$vl, log2sew, (XLenVT timm:$policy))>; + class VPatMaskUnaryNoMask : @@ -3856,6 +4001,28 @@ (mask_type VR:$rs2), GPR:$vl, sew)>; +class VPatUnaryAnyMask_E : + Pat<(result_type (!cast(intrinsic) + (result_type result_reg_class:$merge), + (op1_type op1_reg_class:$rs1), + (mask_type VR:$rs2), + VLOpFrag)), + (!cast(inst#"_"#kind#"_"#vlmul.MX#"_E"#sew) + (result_type result_reg_class:$merge), + (op1_type op1_reg_class:$rs1), + (mask_type VR:$rs2), + GPR:$vl, log2sew)>; + class VPatBinaryM vtilist> { + foreach vti = vtilist in { + def : VPatUnaryAnyMask_E; + } +} + multiclass VPatUnaryM_M { @@ -4223,6 +4400,23 @@ } } +multiclass VPatUnaryV_V_E vtilist> { + foreach vti = vtilist in { + def : VPatUnaryNoMask_E; + def : VPatUnaryNoMaskTU_E; + def : VPatUnaryMaskTA_E; + } +} + multiclass VPatNullaryV { foreach vti = AllIntegerVectors in { @@ -4417,18 +4611,29 @@ vti.RegClass, vti.RegClass>; } -multiclass VPatBinaryV_VV_INT vtilist> { + foreach vti = vtilist in + defm : VPatBinaryTA; +} + +multiclass VPatBinaryV_VV_INT_E vtilist> { foreach vti = vtilist in { defvar ivti = GetIntVTypeInfo.Vti; - defm : VPatBinaryTA; } } -multiclass VPatBinaryV_VV_INT_EEW vtilist> { foreach vti = vtilist in { // emul = lmul * eew / sew @@ -4438,7 +4643,7 @@ if !and(!ge(octuple_emul, 1), !le(octuple_emul, 64)) then { defvar emul_str = octuple_to_str.ret; defvar ivti = !cast("VI" # eew # emul_str); - defvar inst = instruction # "_VV_" # vti.LMul.MX # "_" # emul_str; + defvar inst = instruction # "_VV_" # vti.LMul.MX # "_E" # vti.SEW # "_" # emul_str; defm : VPatBinaryTA vtilist> { + foreach vti = vtilist in { + defvar kind = "V"#vti.ScalarSuffix; + defm : VPatBinaryTA; + } +} + multiclass VPatBinaryV_VX_INT vtilist> { foreach vti = vtilist in @@ -4723,6 +4940,11 @@ : VPatBinaryV_VV, VPatBinaryV_VX; +multiclass VPatBinaryV_VV_VX_E vtilist> + : VPatBinaryV_VV_E, + VPatBinaryV_VX_E; + multiclass VPatBinaryV_VX_VI vtilist> : VPatBinaryV_VX, @@ -4927,7 +5149,7 @@ multiclass VPatBinaryV_VV_VX_VI_INT vtilist, Operand ImmType = simm5> - : VPatBinaryV_VV_INT, + : VPatBinaryV_VV_INT_E, VPatBinaryV_VX_INT, VPatBinaryV_VI; @@ -6004,10 +6226,10 @@ //===----------------------------------------------------------------------===// // 11.11. Vector Integer Divide Instructions //===----------------------------------------------------------------------===// -defm : VPatBinaryV_VV_VX<"int_riscv_vdivu", "PseudoVDIVU", AllIntegerVectors>; -defm : VPatBinaryV_VV_VX<"int_riscv_vdiv", "PseudoVDIV", AllIntegerVectors>; -defm : VPatBinaryV_VV_VX<"int_riscv_vremu", "PseudoVREMU", AllIntegerVectors>; -defm : VPatBinaryV_VV_VX<"int_riscv_vrem", "PseudoVREM", AllIntegerVectors>; +defm : VPatBinaryV_VV_VX_E<"int_riscv_vdivu", "PseudoVDIVU", AllIntegerVectors>; +defm : VPatBinaryV_VV_VX_E<"int_riscv_vdiv", "PseudoVDIV", AllIntegerVectors>; +defm : VPatBinaryV_VV_VX_E<"int_riscv_vremu", "PseudoVREMU", AllIntegerVectors>; +defm : VPatBinaryV_VV_VX_E<"int_riscv_vrem", "PseudoVREM", AllIntegerVectors>; //===----------------------------------------------------------------------===// // 11.12. Vector Widening Integer Multiply Instructions @@ -6120,8 +6342,8 @@ // 13.4. Vector Single-Width Floating-Point Multiply/Divide Instructions //===----------------------------------------------------------------------===// defm : VPatBinaryV_VV_VX<"int_riscv_vfmul", "PseudoVFMUL", AllFloatVectors>; -defm : VPatBinaryV_VV_VX<"int_riscv_vfdiv", "PseudoVFDIV", AllFloatVectors>; -defm : VPatBinaryV_VX<"int_riscv_vfrdiv", "PseudoVFRDIV", AllFloatVectors>; +defm : VPatBinaryV_VV_VX_E<"int_riscv_vfdiv", "PseudoVFDIV", AllFloatVectors>; +defm : VPatBinaryV_VX_E<"int_riscv_vfrdiv", "PseudoVFRDIV", AllFloatVectors>; //===----------------------------------------------------------------------===// // 13.5. Vector Widening Floating-Point Multiply @@ -6151,7 +6373,7 @@ //===----------------------------------------------------------------------===// // 13.8. Vector Floating-Point Square-Root Instruction //===----------------------------------------------------------------------===// -defm : VPatUnaryV_V<"int_riscv_vfsqrt", "PseudoVFSQRT", AllFloatVectors>; +defm : VPatUnaryV_V_E<"int_riscv_vfsqrt", "PseudoVFSQRT", AllFloatVectors>; //===----------------------------------------------------------------------===// // 13.9. Vector Floating-Point Reciprocal Square-Root Estimate Instruction @@ -6415,14 +6637,14 @@ let Predicates = [HasVInstructions] in { defm : VPatBinaryV_VV_VX_VI_INT<"int_riscv_vrgather", "PseudoVRGATHER", AllIntegerVectors, uimm5>; - defm : VPatBinaryV_VV_INT_EEW<"int_riscv_vrgatherei16_vv", "PseudoVRGATHEREI16", + defm : VPatBinaryV_VV_INT_E_EEW<"int_riscv_vrgatherei16_vv", "PseudoVRGATHEREI16", /* eew */ 16, AllIntegerVectors>; } // Predicates = [HasVInstructions] let Predicates = [HasVInstructionsAnyF] in { defm : VPatBinaryV_VV_VX_VI_INT<"int_riscv_vrgather", "PseudoVRGATHER", AllFloatVectors, uimm5>; - defm : VPatBinaryV_VV_INT_EEW<"int_riscv_vrgatherei16_vv", "PseudoVRGATHEREI16", + defm : VPatBinaryV_VV_INT_E_EEW<"int_riscv_vrgatherei16_vv", "PseudoVRGATHEREI16", /* eew */ 16, AllFloatVectors>; } // Predicates = [HasVInstructionsAnyF] @@ -6430,11 +6652,11 @@ // 16.5. Vector Compress Instruction //===----------------------------------------------------------------------===// let Predicates = [HasVInstructions] in { - defm : VPatUnaryV_V_AnyMask<"int_riscv_vcompress", "PseudoVCOMPRESS", AllIntegerVectors>; + defm : VPatUnaryV_V_AnyMask_E<"int_riscv_vcompress", "PseudoVCOMPRESS", AllIntegerVectors>; } // Predicates = [HasVInstructions] let Predicates = [HasVInstructionsAnyF] in { - defm : VPatUnaryV_V_AnyMask<"int_riscv_vcompress", "PseudoVCOMPRESS", AllFloatVectors>; + defm : VPatUnaryV_V_AnyMask_E<"int_riscv_vcompress", "PseudoVCOMPRESS", AllFloatVectors>; } // Predicates = [HasVInstructionsAnyF] // Include the non-intrinsic ISel patterns diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td @@ -88,6 +88,23 @@ op_reg_class:$rs2, avl, sew)>; +class VPatBinarySDNode_VV_E : + Pat<(result_type (vop + (op_type op_reg_class:$rs1), + (op_type op_reg_class:$rs2))), + (!cast(instruction_name#"_VV_"# vlmul.MX#"_E"#sew) + op_reg_class:$rs1, + op_reg_class:$rs2, + avl, log2sew)>; + class VPatBinarySDNode_XI; +class VPatBinarySDNode_XI_E : + Pat<(result_type (vop + (vop_type vop_reg_class:$rs1), + (vop_type (SplatPatKind xop_kind:$rs2)))), + (!cast(instruction_name#_#suffix#_# vlmul.MX#"_E"#sew) + vop_reg_class:$rs1, + xop_kind:$rs2, + avl, log2sew)>; + multiclass VPatBinarySDNode_VV_VX { foreach vti = AllIntegerVectors in { def : VPatBinarySDNode_VV { + foreach vti = AllIntegerVectors in { + def : VPatBinarySDNode_VV_E; + def : VPatBinarySDNode_XI_E; + } +} + multiclass VPatBinarySDNode_VV_VX_VI : VPatBinarySDNode_VV_VX { @@ -148,6 +197,24 @@ (xop_type xop_kind:$rs2), avl, sew)>; +class VPatBinarySDNode_VF_E : + Pat<(result_type (vop (vop_type vop_reg_class:$rs1), + (vop_type (SplatFPOp xop_kind:$rs2)))), + (!cast(instruction_name#"_"#vlmul.MX#"_E"#sew) + vop_reg_class:$rs1, + (xop_type xop_kind:$rs2), + avl, log2sew)>; + multiclass VPatBinaryFPSDNode_VV_VF { foreach vti = AllFloatVectors in { def : VPatBinarySDNode_VV { + foreach vti = AllFloatVectors in { + def : VPatBinarySDNode_VV_E; + def : VPatBinarySDNode_VF_E; + } +} + multiclass VPatBinaryFPSDNode_R_VF { foreach fvti = AllFloatVectors in def : Pat<(fvti.Vector (vop (fvti.Vector (SplatFPOp fvti.Scalar:$rs2)), @@ -170,6 +249,16 @@ fvti.AVL, fvti.Log2SEW)>; } +multiclass VPatBinaryFPSDNode_R_VF_E { + foreach fvti = AllFloatVectors in + def : Pat<(fvti.Vector (vop (fvti.Vector (SplatFPOp fvti.Scalar:$rs2)), + (fvti.Vector fvti.RegClass:$rs1))), + (!cast(instruction_name#"_V"#fvti.ScalarSuffix#"_"#fvti.LMul.MX#"_E"#fvti.SEW) + fvti.RegClass:$rs1, + (fvti.Scalar fvti.ScalarRegClass:$rs2), + fvti.AVL, fvti.Log2SEW)>; +} + multiclass VPatIntegerSetCCSDNode_VV { foreach vti = AllIntegerVectors in { @@ -723,10 +812,10 @@ defm : VPatBinarySDNode_VV_VX; // 11.11. Vector Integer Divide Instructions -defm : VPatBinarySDNode_VV_VX; -defm : VPatBinarySDNode_VV_VX; -defm : VPatBinarySDNode_VV_VX; -defm : VPatBinarySDNode_VV_VX; +defm : VPatBinarySDNode_VV_VX_E; +defm : VPatBinarySDNode_VV_VX_E; +defm : VPatBinarySDNode_VV_VX_E; +defm : VPatBinarySDNode_VV_VX_E; // 11.12. Vector Widening Integer Multiply Instructions defm : VPatWidenBinarySDNode_VV_VX; -defm : VPatBinaryFPSDNode_VV_VF; -defm : VPatBinaryFPSDNode_R_VF; +defm : VPatBinaryFPSDNode_VV_VF_E; +defm : VPatBinaryFPSDNode_R_VF_E; // 13.5. Vector Widening Floating-Point Multiply Instructions defm : VPatWidenBinaryFPSDNode_VV_VF; @@ -914,7 +1003,7 @@ foreach vti = AllFloatVectors in { // 13.8. Vector Floating-Point Square-Root Instruction def : Pat<(fsqrt (vti.Vector vti.RegClass:$rs2)), - (!cast("PseudoVFSQRT_V_"# vti.LMul.MX) + (!cast("PseudoVFSQRT_V_"# vti.LMul.MX#"_E"#vti.SEW) vti.RegClass:$rs2, vti.AVL, vti.Log2SEW)>; // 13.12. Vector Floating-Point Sign-Injection Instructions diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td @@ -446,6 +446,32 @@ (mask_type V0), GPR:$vl, sew, TAIL_AGNOSTIC)>; } +multiclass VPatBinaryVL_V_E { + def : Pat<(result_type (vop + (op1_type op1_reg_class:$rs1), + (op2_type op2_reg_class:$rs2), + (result_type result_reg_class:$merge), + (mask_type V0), + VLOpFrag)), + (!cast(instruction_name#"_"#suffix#"_"# vlmul.MX#"_E"# sew#"_MASK") + result_reg_class:$merge, + op1_reg_class:$rs1, + op2_reg_class:$rs2, + (mask_type V0), GPR:$vl, log2sew, TAIL_AGNOSTIC)>; +} + multiclass VPatTiedBinaryNoMaskVL_V; } +multiclass VPatBinaryVL_XI_E { + def : Pat<(result_type (vop + (vop1_type vop_reg_class:$rs1), + (vop2_type (SplatPatKind (XLenVT xop_kind:$rs2))), + (result_type result_reg_class:$merge), + (mask_type V0), + VLOpFrag)), + (!cast(instruction_name#_#suffix#_# vlmul.MX#"_E"# sew#"_MASK") + result_reg_class:$merge, + vop_reg_class:$rs1, + xop_kind:$rs2, + (mask_type V0), GPR:$vl, log2sew, TAIL_AGNOSTIC)>; +} + multiclass VPatBinaryVL_VV_VX { foreach vti = AllIntegerVectors in { defm : VPatBinaryVL_V { + foreach vti = AllIntegerVectors in { + defm : VPatBinaryVL_V_E; + defm : VPatBinaryVL_XI_E; + } +} + multiclass VPatBinaryVL_VV_VX_VI : VPatBinaryVL_VV_VX { @@ -607,6 +673,29 @@ (mask_type V0), GPR:$vl, sew, TAIL_AGNOSTIC)>; } +multiclass VPatBinaryVL_VF_E { + def : Pat<(result_type (vop (vop_type vop_reg_class:$rs1), + (vop_type (SplatFPOp scalar_reg_class:$rs2)), + (result_type result_reg_class:$merge), + (mask_type V0), + VLOpFrag)), + (!cast(instruction_name#"_"#vlmul.MX#"_E"#sew#"_MASK") + result_reg_class:$merge, + vop_reg_class:$rs1, + scalar_reg_class:$rs2, + (mask_type V0), GPR:$vl, log2sew, TAIL_AGNOSTIC)>; +} + multiclass VPatBinaryFPVL_VV_VF { foreach vti = AllFloatVectors in { defm : VPatBinaryVL_V { + foreach vti = AllFloatVectors in { + defm : VPatBinaryVL_V_E; + defm : VPatBinaryVL_VF_E; + } +} + multiclass VPatBinaryFPVL_R_VF { foreach fvti = AllFloatVectors in { def : Pat<(fvti.Vector (vop (SplatFPOp fvti.ScalarRegClass:$rs2), @@ -634,6 +736,20 @@ } } +multiclass VPatBinaryFPVL_R_VF_E { + foreach fvti = AllFloatVectors in { + def : Pat<(fvti.Vector (vop (SplatFPOp fvti.ScalarRegClass:$rs2), + fvti.RegClass:$rs1, + (fvti.Vector fvti.RegClass:$merge), + (fvti.Mask V0), + VLOpFrag)), + (!cast(instruction_name#"_V"#fvti.ScalarSuffix#"_"#fvti.LMul.MX#"_E"#fvti.SEW#"_MASK") + fvti.RegClass:$merge, + fvti.RegClass:$rs1, fvti.ScalarRegClass:$rs2, + (fvti.Mask V0), GPR:$vl, fvti.Log2SEW, TAIL_AGNOSTIC)>; + } +} + multiclass VPatIntegerSetCCVL_VV { def : Pat<(vti.Mask (riscv_setcc_vl (vti.Vector vti.RegClass:$rs1), @@ -1493,10 +1609,10 @@ defm : VPatBinaryVL_VV_VX; // 11.11. Vector Integer Divide Instructions -defm : VPatBinaryVL_VV_VX; -defm : VPatBinaryVL_VV_VX; -defm : VPatBinaryVL_VV_VX; -defm : VPatBinaryVL_VV_VX; +defm : VPatBinaryVL_VV_VX_E; +defm : VPatBinaryVL_VV_VX_E; +defm : VPatBinaryVL_VV_VX_E; +defm : VPatBinaryVL_VV_VX_E; // 11.12. Vector Widening Integer Multiply Instructions defm : VPatBinaryWVL_VV_VX; @@ -1622,8 +1738,8 @@ // 13.4. Vector Single-Width Floating-Point Multiply/Divide Instructions defm : VPatBinaryFPVL_VV_VF; -defm : VPatBinaryFPVL_VV_VF; -defm : VPatBinaryFPVL_R_VF; +defm : VPatBinaryFPVL_VV_VF_E; +defm : VPatBinaryFPVL_R_VF_E; // 13.5. Vector Widening Floating-Point Multiply Instructions defm : VPatWidenBinaryFPVL_VV_VF; @@ -1665,7 +1781,7 @@ // 13.8. Vector Floating-Point Square-Root Instruction def : Pat<(riscv_fsqrt_vl (vti.Vector vti.RegClass:$rs2), (vti.Mask V0), VLOpFrag), - (!cast("PseudoVFSQRT_V_"# vti.LMul.MX #"_MASK") + (!cast("PseudoVFSQRT_V_"# vti.LMul.MX # "_E" # vti.SEW # "_MASK") (vti.Vector (IMPLICIT_DEF)), vti.RegClass:$rs2, (vti.Mask V0), GPR:$vl, vti.Log2SEW, TA_MA)>; @@ -2029,7 +2145,7 @@ vti.RegClass:$merge, (vti.Mask V0), VLOpFrag)), - (!cast("PseudoVRGATHER_VV_"# vti.LMul.MX#"_MASK") + (!cast("PseudoVRGATHER_VV_"# vti.LMul.MX#"_E"# vti.SEW#"_MASK") vti.RegClass:$merge, vti.RegClass:$rs2, vti.RegClass:$rs1, (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(vti.Vector (riscv_vrgather_vx_vl vti.RegClass:$rs2, GPR:$rs1, @@ -2055,7 +2171,7 @@ if !and(!ge(octuple_emul, 1), !le(octuple_emul, 64)) then { defvar emul_str = octuple_to_str.ret; defvar ivti = !cast("VI16" # emul_str); - defvar inst = "PseudoVRGATHEREI16_VV_" # vti.LMul.MX # "_" # emul_str; + defvar inst = "PseudoVRGATHEREI16_VV_" # vti.LMul.MX # "_E" # vti.SEW # "_" # emul_str; def : Pat<(vti.Vector (riscv_vrgatherei16_vv_vl vti.RegClass:$rs2, @@ -2099,7 +2215,7 @@ vti.RegClass:$merge, (vti.Mask V0), VLOpFrag)), - (!cast("PseudoVRGATHER_VV_"# vti.LMul.MX#"_MASK") + (!cast("PseudoVRGATHER_VV_"# vti.LMul.MX#"_E"# vti.SEW#"_MASK") vti.RegClass:$merge, vti.RegClass:$rs2, vti.RegClass:$rs1, (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(vti.Vector (riscv_vrgather_vx_vl vti.RegClass:$rs2, GPR:$rs1, @@ -2125,7 +2241,7 @@ if !and(!ge(octuple_emul, 1), !le(octuple_emul, 64)) then { defvar emul_str = octuple_to_str.ret; defvar ivti = !cast("VI16" # emul_str); - defvar inst = "PseudoVRGATHEREI16_VV_" # vti.LMul.MX # "_" # emul_str; + defvar inst = "PseudoVRGATHEREI16_VV_" # vti.LMul.MX # "_E" # vti.SEW # "_" # emul_str; def : Pat<(vti.Vector (riscv_vrgatherei16_vv_vl vti.RegClass:$rs2, diff --git a/llvm/lib/Target/RISCV/RISCVScheduleV.td b/llvm/lib/Target/RISCV/RISCVScheduleV.td --- a/llvm/lib/Target/RISCV/RISCVScheduleV.td +++ b/llvm/lib/Target/RISCV/RISCVScheduleV.td @@ -17,10 +17,19 @@ // Used for widening floating-point Reduction as it doesn't contain MF8. defvar SchedMxListFWRed = !listremove(SchedMxList, ["MF8"]); +class SchedSEWSet { + list val = !cond(!eq(mx, "M1"): [8, 16, 32, 64], + !eq(mx, "M2"): [8, 16, 32, 64], + !eq(mx, "M4"): [8, 16, 32, 64], + !eq(mx, "M8"): [8, 16, 32, 64], + !eq(mx, "MF2"): [8, 16, 32], + !eq(mx, "MF4"): [8, 16], + !eq(mx, "MF8"): [8]); +} + // Define multiclasses to define SchedWrite, SchedRead, WriteRes, and // ReadAdvance for each (name, LMUL) pair for each LMUL in each of the // SchedMxList variants above. - multiclass LMULSchedWritesImpl MxList> { foreach mx = MxList in { def name # "_" # mx : SchedWrite; @@ -45,6 +54,46 @@ } } +// Define multiclasses to define SchedWrite, SchedRead, WriteRes, and +// ReadAdvance for each (name, LMUL, SEW) tuple for each LMUL in each of the +// SchedMxList variants above. +multiclass LMULSEWSchedWritesImpl MxList> { + foreach mx = MxList in { + if !eq(mx, "UpperBound") then + def name # "_" # mx : SchedWrite; + else + foreach sew = SchedSEWSet.val in + def name # "_" # mx # "_E" # sew : SchedWrite; + } +} +multiclass LMULSEWSchedReadsImpl MxList> { + foreach mx = MxList in { + if !eq(mx, "UpperBound") then + def name # "_" # mx : SchedRead; + else + foreach sew = SchedSEWSet.val in + def name # "_" # mx # "_E" # sew : SchedRead; + } +} +multiclass LMULSEWWriteResImpl resources> { + foreach mx = SchedMxList in { + if !eq(mx, "UpperBound") then + def : WriteRes(name # "_" # mx), resources>; + else + foreach sew = SchedSEWSet.val in + def : WriteRes(name # "_" # mx # "_E" # sew), resources>; + } +} +multiclass LMULSEWReadAdvanceImpl writes = []> { + foreach mx = SchedMxList in { + if !eq(mx, "UpperBound") then + def : ReadAdvance(name # "_" # mx), val, writes>; + else + foreach sew = SchedSEWSet.val in + def : ReadAdvance(name # "_" # mx # "_E" # sew), val, writes>; + } +} // Define classes to define list containing all SchedWrites for each (name, LMUL) // pair for each LMUL in each of the SchedMxList variants above and name in // argument `names`. These classes can be used to construct a list of existing @@ -69,6 +118,13 @@ : LMULReadAdvanceImpl; class LMULSchedWriteList names> : LMULSchedWriteListImpl; +multiclass LMULSEWSchedWrites : LMULSEWSchedWritesImpl; +multiclass LMULSEWSchedReads : LMULSEWSchedReadsImpl; +multiclass LMULSEWWriteRes resources> + : LMULSEWWriteResImpl; +multiclass LMULSEWReadAdvance writes = []> + : LMULSEWReadAdvanceImpl; + multiclass LMULSchedWritesW : LMULSchedWritesImpl; multiclass LMULSchedReadsW : LMULSchedReadsImpl; multiclass LMULWriteResW resources> @@ -191,8 +247,8 @@ defm "" : LMULSchedWrites<"WriteVIMulV">; defm "" : LMULSchedWrites<"WriteVIMulX">; // 11.11. Vector Integer Divide Instructions -defm "" : LMULSchedWrites<"WriteVIDivV">; -defm "" : LMULSchedWrites<"WriteVIDivX">; +defm "" : LMULSEWSchedWrites<"WriteVIDivV">; +defm "" : LMULSEWSchedWrites<"WriteVIDivX">; // 11.12. Vector Widening Integer Multiply Instructions defm "" : LMULSchedWritesW<"WriteVIWMulV">; defm "" : LMULSchedWritesW<"WriteVIWMulX">; @@ -241,8 +297,8 @@ // 13.4. Vector Single-Width Floating-Point Multiply/Divide Instructions defm "" : LMULSchedWrites<"WriteVFMulV">; defm "" : LMULSchedWrites<"WriteVFMulF">; -defm "" : LMULSchedWrites<"WriteVFDivV">; -defm "" : LMULSchedWrites<"WriteVFDivF">; +defm "" : LMULSEWSchedWrites<"WriteVFDivV">; +defm "" : LMULSEWSchedWrites<"WriteVFDivF">; // 13.5. Vector Widening Floating-Point Multiply defm "" : LMULSchedWritesFW<"WriteVFWMulV">; defm "" : LMULSchedWritesFW<"WriteVFWMulF">; @@ -253,7 +309,7 @@ defm "" : LMULSchedWritesFW<"WriteVFWMulAddV">; defm "" : LMULSchedWritesFW<"WriteVFWMulAddF">; // 13.8. Vector Floating-Point Square-Root Instruction -defm "" : LMULSchedWrites<"WriteVFSqrtV">; +defm "" : LMULSEWSchedWrites<"WriteVFSqrtV">; // 13.9. Vector Floating-Point Reciprocal Square-Root Estimate Instruction // 13.10. Vector Floating-Point Reciprocal Estimate Instruction defm "" : LMULSchedWrites<"WriteVFRecpV">; @@ -327,11 +383,11 @@ defm "" : LMULSchedWrites<"WriteVISlide1X">; defm "" : LMULSchedWrites<"WriteVFSlide1F">; // 16.4. Vector Register Gather Instructions -defm "" : LMULSchedWrites<"WriteVRGatherVV">; +defm "" : LMULSEWSchedWrites<"WriteVRGatherVV">; defm "" : LMULSchedWrites<"WriteVRGatherVX">; defm "" : LMULSchedWrites<"WriteVRGatherVI">; // 16.5. Vector Compress Instruction -defm "" : LMULSchedWrites<"WriteVCompressV">; +defm "" : LMULSEWSchedWrites<"WriteVCompressV">; // 16.6. Whole Vector Register Move // These are already LMUL aware def WriteVMov1V : SchedWrite; @@ -415,8 +471,8 @@ defm "" : LMULSchedReads<"ReadVIMulV">; defm "" : LMULSchedReads<"ReadVIMulX">; // 11.11. Vector Integer Divide Instructions -defm "" : LMULSchedReads<"ReadVIDivV">; -defm "" : LMULSchedReads<"ReadVIDivX">; +defm "" : LMULSEWSchedReads<"ReadVIDivV">; +defm "" : LMULSEWSchedReads<"ReadVIDivX">; // 11.12. Vector Widening Integer Multiply Instructions defm "" : LMULSchedReadsW<"ReadVIWMulV">; defm "" : LMULSchedReadsW<"ReadVIWMulX">; @@ -460,8 +516,8 @@ // 13.4. Vector Single-Width Floating-Point Multiply/Divide Instructions defm "" : LMULSchedReads<"ReadVFMulV">; defm "" : LMULSchedReads<"ReadVFMulF">; -defm "" : LMULSchedReads<"ReadVFDivV">; -defm "" : LMULSchedReads<"ReadVFDivF">; +defm "" : LMULSEWSchedReads<"ReadVFDivV">; +defm "" : LMULSEWSchedReads<"ReadVFDivF">; // 13.5. Vector Widening Floating-Point Multiply defm "" : LMULSchedReadsFW<"ReadVFWMulV">; defm "" : LMULSchedReadsFW<"ReadVFWMulF">; @@ -472,7 +528,7 @@ defm "" : LMULSchedReadsFW<"ReadVFWMulAddV">; defm "" : LMULSchedReadsFW<"ReadVFWMulAddF">; // 13.8. Vector Floating-Point Square-Root Instruction -defm "" : LMULSchedReads<"ReadVFSqrtV">; +defm "" : LMULSEWSchedReads<"ReadVFSqrtV">; // 13.9. Vector Floating-Point Reciprocal Square-Root Estimate Instruction // 13.10. Vector Floating-Point Reciprocal Estimate Instruction defm "" : LMULSchedReads<"ReadVFRecpV">; @@ -549,13 +605,13 @@ defm "" : LMULSchedReads<"ReadVFSlideV">; defm "" : LMULSchedReads<"ReadVFSlideF">; // 16.4. Vector Register Gather Instructions -defm "" : LMULSchedReads<"ReadVRGatherVV_data">; -defm "" : LMULSchedReads<"ReadVRGatherVV_index">; +defm "" : LMULSEWSchedReads<"ReadVRGatherVV_data">; +defm "" : LMULSEWSchedReads<"ReadVRGatherVV_index">; defm "" : LMULSchedReads<"ReadVRGatherVX_data">; defm "" : LMULSchedReads<"ReadVRGatherVX_index">; defm "" : LMULSchedReads<"ReadVRGatherVI_data">; // 16.5. Vector Compress Instruction -defm "" : LMULSchedReads<"ReadVCompressV">; +defm "" : LMULSEWSchedReads<"ReadVCompressV">; // 16.6. Whole Vector Register Move // These are already LMUL aware def ReadVMov1V : SchedRead; @@ -656,8 +712,8 @@ defm "" : LMULWriteRes<"WriteVICmpI", []>; defm "" : LMULWriteRes<"WriteVIMulV", []>; defm "" : LMULWriteRes<"WriteVIMulX", []>; -defm "" : LMULWriteRes<"WriteVIDivV", []>; -defm "" : LMULWriteRes<"WriteVIDivX", []>; +defm "" : LMULSEWWriteRes<"WriteVIDivV", []>; +defm "" : LMULSEWWriteRes<"WriteVIDivX", []>; defm "" : LMULWriteResW<"WriteVIWMulV", []>; defm "" : LMULWriteResW<"WriteVIWMulX", []>; defm "" : LMULWriteRes<"WriteVIMulAddV", []>; @@ -693,15 +749,15 @@ defm "" : LMULWriteResFW<"WriteVFWALUF", []>; defm "" : LMULWriteRes<"WriteVFMulV", []>; defm "" : LMULWriteRes<"WriteVFMulF", []>; -defm "" : LMULWriteRes<"WriteVFDivV", []>; -defm "" : LMULWriteRes<"WriteVFDivF", []>; +defm "" : LMULSEWWriteRes<"WriteVFDivV", []>; +defm "" : LMULSEWWriteRes<"WriteVFDivF", []>; defm "" : LMULWriteResFW<"WriteVFWMulV", []>; defm "" : LMULWriteResFW<"WriteVFWMulF", []>; defm "" : LMULWriteRes<"WriteVFMulAddV", []>; defm "" : LMULWriteRes<"WriteVFMulAddF", []>; defm "" : LMULWriteResFW<"WriteVFWMulAddV", []>; defm "" : LMULWriteResFW<"WriteVFWMulAddF", []>; -defm "" : LMULWriteRes<"WriteVFSqrtV", []>; +defm "" : LMULSEWWriteRes<"WriteVFSqrtV", []>; defm "" : LMULWriteRes<"WriteVFRecpV", []>; defm "" : LMULWriteRes<"WriteVFCmpV", []>; defm "" : LMULWriteRes<"WriteVFCmpF", []>; @@ -744,10 +800,10 @@ defm "" : LMULWriteRes<"WriteVISlideI", []>; defm "" : LMULWriteRes<"WriteVISlide1X", []>; defm "" : LMULWriteRes<"WriteVFSlide1F", []>; -defm "" : LMULWriteRes<"WriteVRGatherVV", []>; +defm "" : LMULSEWWriteRes<"WriteVRGatherVV", []>; defm "" : LMULWriteRes<"WriteVRGatherVX", []>; defm "" : LMULWriteRes<"WriteVRGatherVI", []>; -defm "" : LMULWriteRes<"WriteVCompressV", []>; +defm "" : LMULSEWWriteRes<"WriteVCompressV", []>; // These are already LMUL aware def : WriteRes; def : WriteRes; @@ -811,8 +867,8 @@ defm "" : LMULReadAdvance<"ReadVICmpX", 0>; defm "" : LMULReadAdvance<"ReadVIMulV", 0>; defm "" : LMULReadAdvance<"ReadVIMulX", 0>; -defm "" : LMULReadAdvance<"ReadVIDivV", 0>; -defm "" : LMULReadAdvance<"ReadVIDivX", 0>; +defm "" : LMULSEWReadAdvance<"ReadVIDivV", 0>; +defm "" : LMULSEWReadAdvance<"ReadVIDivX", 0>; defm "" : LMULReadAdvanceW<"ReadVIWMulV", 0>; defm "" : LMULReadAdvanceW<"ReadVIWMulX", 0>; defm "" : LMULReadAdvance<"ReadVIMulAddV", 0>; @@ -843,15 +899,15 @@ defm "" : LMULReadAdvanceFW<"ReadVFWALUF", 0>; defm "" : LMULReadAdvance<"ReadVFMulV", 0>; defm "" : LMULReadAdvance<"ReadVFMulF", 0>; -defm "" : LMULReadAdvance<"ReadVFDivV", 0>; -defm "" : LMULReadAdvance<"ReadVFDivF", 0>; +defm "" : LMULSEWReadAdvance<"ReadVFDivV", 0>; +defm "" : LMULSEWReadAdvance<"ReadVFDivF", 0>; defm "" : LMULReadAdvanceFW<"ReadVFWMulV", 0>; defm "" : LMULReadAdvanceFW<"ReadVFWMulF", 0>; defm "" : LMULReadAdvance<"ReadVFMulAddV", 0>; defm "" : LMULReadAdvance<"ReadVFMulAddF", 0>; defm "" : LMULReadAdvanceFW<"ReadVFWMulAddV", 0>; defm "" : LMULReadAdvanceFW<"ReadVFWMulAddF", 0>; -defm "" : LMULReadAdvance<"ReadVFSqrtV", 0>; +defm "" : LMULSEWReadAdvance<"ReadVFSqrtV", 0>; defm "" : LMULReadAdvance<"ReadVFRecpV", 0>; defm "" : LMULReadAdvance<"ReadVFCmpV", 0>; defm "" : LMULReadAdvance<"ReadVFCmpF", 0>; @@ -902,12 +958,13 @@ defm "" : LMULReadAdvance<"ReadVISlideX", 0>; defm "" : LMULReadAdvance<"ReadVFSlideV", 0>; defm "" : LMULReadAdvance<"ReadVFSlideF", 0>; -defm "" : LMULReadAdvance<"ReadVRGatherVV_data", 0>; -defm "" : LMULReadAdvance<"ReadVRGatherVV_index", 0>; +defm "" : LMULSEWReadAdvance<"ReadVRGatherVV_data", 0>; +defm "" : LMULSEWReadAdvance<"ReadVRGatherVV_index", 0>; defm "" : LMULReadAdvance<"ReadVRGatherVX_data", 0>; defm "" : LMULReadAdvance<"ReadVRGatherVX_index", 0>; defm "" : LMULReadAdvance<"ReadVRGatherVI_data", 0>; -defm "" : LMULReadAdvance<"ReadVCompressV", 0>; +defm "" : LMULReadAdvance<"ReadVGatherV", 0>; +defm "" : LMULSEWReadAdvance<"ReadVCompressV", 0>; // These are already LMUL aware def : ReadAdvance; def : ReadAdvance;