diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h --- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h +++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h @@ -68,14 +68,20 @@ HasMergeOpMask = 1 << HasMergeOpShift, // Does this instruction have a SEW operand. It will be the last explicit - // operand. Used by RVV Pseudos. + // operand unless there is a vector policy operand. Used by RVV Pseudos. HasSEWOpShift = HasMergeOpShift + 1, HasSEWOpMask = 1 << HasSEWOpShift, // Does this instruction have a VL operand. It will be the second to last - // explicit operand. Used by RVV Pseudos. + // explicit operand unless there is a vector policy operand. Used by RVV + // Pseudos. HasVLOpShift = HasSEWOpShift + 1, HasVLOpMask = 1 << HasVLOpShift, + + // Does this instruction have a vector policy operand. It will be the last + // explicit operand. Used by RVV Pseudos. + HasVecPolicyOpShift = HasVLOpShift + 1, + HasVecPolicyOpMask = 1 << HasVecPolicyOpShift, }; // Match with the definitions in RISCVInstrFormatsV.td @@ -131,6 +137,10 @@ static inline bool hasVLOp(uint64_t TSFlags) { return TSFlags & HasVLOpMask; } +/// \returns true if there is a vector policy operand for this instruction. +static inline bool hasVecPolicyOp(uint64_t TSFlags) { + return TSFlags & HasVecPolicyOpMask; +} // RISC-V Specific Machine Operand Flags enum { diff --git a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp --- a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp +++ b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp @@ -362,14 +362,7 @@ const MachineRegisterInfo *MRI) { VSETVLIInfo InstrInfo; unsigned NumOperands = MI.getNumExplicitOperands(); - - RISCVII::VLMUL VLMul = RISCVII::getLMul(TSFlags); - - unsigned Log2SEW = MI.getOperand(NumOperands - 1).getImm(); - // A Log2SEW of 0 is an operation on mask registers only. - bool MaskRegOp = Log2SEW == 0; - unsigned SEW = Log2SEW ? 1 << Log2SEW : 8; - assert(RISCVVType::isValidSEW(SEW) && "Unexpected SEW"); + bool HasPolicy = RISCVII::hasVecPolicyOp(TSFlags); // Default to tail agnostic unless the destination is tied to a source. // Unless the source is undef. In that case the user would have some control @@ -377,8 +370,15 @@ // despite having a tied def. bool ForceTailAgnostic = RISCVII::doesForceTailAgnostic(TSFlags); bool TailAgnostic = true; + // If the instruction has policy argument, use the argument. + if (HasPolicy) { + const MachineOperand &Op = MI.getOperand(MI.getNumExplicitOperands() - 1); + TailAgnostic = Op.getImm() & 0x1; + } + unsigned UseOpIdx; - if (!ForceTailAgnostic && MI.isRegTiedToUseOperand(0, &UseOpIdx)) { + if (!(ForceTailAgnostic || (HasPolicy && TailAgnostic)) && + MI.isRegTiedToUseOperand(0, &UseOpIdx)) { TailAgnostic = false; // If the tied operand is an IMPLICIT_DEF we can keep TailAgnostic. const MachineOperand &UseMO = MI.getOperand(UseOpIdx); @@ -390,8 +390,20 @@ } } + // Remove the tail policy so we can find the SEW and VL. + if (HasPolicy) + --NumOperands; + + RISCVII::VLMUL VLMul = RISCVII::getLMul(TSFlags); + + unsigned Log2SEW = MI.getOperand(NumOperands - 1).getImm(); + // A Log2SEW of 0 is an operation on mask registers only. + bool MaskRegOp = Log2SEW == 0; + unsigned SEW = Log2SEW ? 1 << Log2SEW : 8; + assert(RISCVVType::isValidSEW(SEW) && "Unexpected SEW"); + if (RISCVII::hasVLOp(TSFlags)) { - const MachineOperand &VLOp = MI.getOperand(MI.getNumExplicitOperands() - 2); + const MachineOperand &VLOp = MI.getOperand(NumOperands - 2); if (VLOp.isImm()) InstrInfo.setAVLImm(VLOp.getImm()); else diff --git a/llvm/lib/Target/RISCV/RISCVInstrFormats.td b/llvm/lib/Target/RISCV/RISCVInstrFormats.td --- a/llvm/lib/Target/RISCV/RISCVInstrFormats.td +++ b/llvm/lib/Target/RISCV/RISCVInstrFormats.td @@ -178,6 +178,9 @@ bit HasVLOp = 0; let TSFlags{15} = HasVLOp; + + bit HasVecPolicyOp = 0; + let TSFlags{16} = HasVecPolicyOp; } // Pseudo instructions diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp @@ -1139,7 +1139,7 @@ // clang-format off #define CASE_VFMA_OPCODE_COMMON(OP, TYPE, LMUL) \ - RISCV::PseudoV##OP##_##TYPE##_##LMUL##_COMMUTABLE + RISCV::PseudoV##OP##_##TYPE##_##LMUL #define CASE_VFMA_OPCODE_LMULS(OP, TYPE) \ CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF8): \ @@ -1182,6 +1182,11 @@ case CASE_VFMA_OPCODE_LMULS(NMSAC, VX): case CASE_VFMA_OPCODE_LMULS(MACC, VV): case CASE_VFMA_OPCODE_LMULS(NMSAC, VV): { + // If the tail policy is undisturbed we can't commute. + assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags)); + if ((MI.getOperand(MI.getNumExplicitOperands() - 1).getImm() & 1) == 0) + return false; + // For these instructions we can only swap operand 1 and operand 3 by // changing the opcode. unsigned CommutableOpIdx1 = 1; @@ -1197,6 +1202,11 @@ case CASE_VFMA_OPCODE_LMULS(FNMSUB, VV): case CASE_VFMA_OPCODE_LMULS(MADD, VV): case CASE_VFMA_OPCODE_LMULS(NMSUB, VV): { + // If the tail policy is undisturbed we can't commute. + assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags)); + if ((MI.getOperand(MI.getNumExplicitOperands() - 1).getImm() & 1) == 0) + return false; + // For these instructions we have more freedom. We can commute with the // other multiplicand or with the addend/subtrahend/minuend. @@ -1261,8 +1271,8 @@ } #define CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL) \ - case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL##_COMMUTABLE: \ - Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL##_COMMUTABLE; \ + case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL: \ + Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL; \ break; #define CASE_VFMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE) \ diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td @@ -40,6 +40,9 @@ N->getValueType(0)); }]>; +defvar TAIL_UNDISTURBED = 0; +defvar TAIL_AGNOSTIC = 1; + //===----------------------------------------------------------------------===// // Utilities. //===----------------------------------------------------------------------===// @@ -577,13 +580,11 @@ !subst("_B32", "", !subst("_B64", "", !subst("_MASK", "", - !subst("_COMMUTABLE", "", - !subst("_TA", "", !subst("_TIED", "", !subst("F16", "F", !subst("F32", "F", !subst("F64", "F", - !subst("Pseudo", "", PseudoInst)))))))))))))))))))))); + !subst("Pseudo", "", PseudoInst)))))))))))))))))))); } // The destination vector register group for a masked vector instruction cannot @@ -1060,6 +1061,27 @@ let BaseInstr = !cast(PseudoToVInst.VInst); } +class VPseudoTernaryNoMaskWithPolicy : + Pseudo<(outs RetClass:$rd), + (ins RetClass:$rs3, Op1Class:$rs1, Op2Class:$rs2, + AVL:$vl, ixlenimm:$sew, ixlenimm:$policy), + []>, + RISCVVPseudo { + let mayLoad = 0; + let mayStore = 0; + let hasSideEffects = 0; + let Constraints = Join<[Constraint, "$rd = $rs3"], ",">.ret; + let HasVecPolicyOp = 1; + let HasVLOp = 1; + let HasSEWOp = 1; + let HasMergeOp = 1; + let HasDummyMask = 1; + let BaseInstr = !cast(PseudoToVInst.VInst); +} + class VPseudoAMOWDNoMask : Pseudo<(outs GetVRegNoV0.R:$vd_wd), @@ -1887,16 +1909,23 @@ } } -multiclass VPseudoTernaryV_VV { +multiclass VPseudoTernaryWithPolicy { + let VLMul = MInfo.value in { + let isCommutable = Commutable in + def "_" # MInfo.MX : VPseudoTernaryNoMaskWithPolicy; + def "_" # MInfo.MX # "_MASK" : VPseudoBinaryMask; + } +} + +multiclass VPseudoTernaryV_VV_AAXA { foreach m = MxList.m in { - defm _VV : VPseudoTernary; - - // Add a commutable version for use by IR mul+add. - let isCommutable = 1, ForceTailAgnostic = true, VLMul = m.value in - def "_VV_" # m.MX # "_COMMUTABLE" : VPseudoTernaryNoMask; + defm _VV : VPseudoTernaryWithPolicy; } } @@ -1906,68 +1935,39 @@ } multiclass VPseudoTernaryV_VX_AAXA { - foreach m = MxList.m in { - defm "_VX" : VPseudoTernary; - - // Add a commutable version for use by IR mul+add. - let isCommutable = 1, ForceTailAgnostic = true, VLMul = m.value in - def "_VX_" # m.MX # "_COMMUTABLE" : - VPseudoTernaryNoMask; - } + foreach m = MxList.m in + defm "_VX" : VPseudoTernaryWithPolicy; } multiclass VPseudoTernaryV_VF_AAXA { - foreach m = MxList.m in { - foreach f = FPList.fpinfo in { - defm "_V" # f.FX : VPseudoTernary; - - // Add a commutable version for use by IR mul+add. - let isCommutable = 1, ForceTailAgnostic = true, VLMul = m.value in - def "_V" # f.FX # "_" # m.MX # "_COMMUTABLE" : - VPseudoTernaryNoMask; - } - } + foreach m = MxList.m in + foreach f = FPList.fpinfo in + defm "_V" # f.FX : VPseudoTernaryWithPolicy; } multiclass VPseudoTernaryW_VV { defvar constraint = "@earlyclobber $rd"; - foreach m = MxListW.m in { - defm _VV : VPseudoTernary; - - // Add a tail agnostic version for us by IR mul+add. - let ForceTailAgnostic = true, VLMul = m.value in - def "_VV_" # m.MX # "_TA" : VPseudoTernaryNoMask; - } + foreach m = MxListW.m in + defm _VV : VPseudoTernaryWithPolicy; } multiclass VPseudoTernaryW_VX { defvar constraint = "@earlyclobber $rd"; - foreach m = MxListW.m in { - defm "_VX" : VPseudoTernary; - - // Add a tail agnostic version for use by IR mul+add. - let ForceTailAgnostic = true, VLMul = m.value in - def "_VX_" # m.MX # "_TA" : - VPseudoTernaryNoMask; - } + foreach m = MxListW.m in + defm "_VX" : VPseudoTernaryWithPolicy; } multiclass VPseudoTernaryW_VF { defvar constraint = "@earlyclobber $rd"; foreach m = MxListW.m in - foreach f = FPListW.fpinfo in { - defm "_V" # f.FX : VPseudoTernary; - - // Add a tail agnostic version for use by IR mul+add. - let ForceTailAgnostic = true, VLMul = m.value in - def "_V" # f.FX # "_" # m.MX # "_TA" : - VPseudoTernaryNoMask; - } + foreach f = FPListW.fpinfo in + defm "_V" # f.FX : VPseudoTernaryWithPolicy; } multiclass VPseudoTernaryV_VI { @@ -1976,12 +1976,12 @@ } multiclass VPseudoTernaryV_VV_VX_AAXA { - defm "" : VPseudoTernaryV_VV; + defm "" : VPseudoTernaryV_VV_AAXA; defm "" : VPseudoTernaryV_VX_AAXA; } multiclass VPseudoTernaryV_VV_VF_AAXA { - defm "" : VPseudoTernaryV_VV; + defm "" : VPseudoTernaryV_VV_AAXA; defm "" : VPseudoTernaryV_VF_AAXA; } @@ -2399,6 +2399,29 @@ op2_kind:$rs2, GPR:$vl, sew)>; +class VPatTernaryNoMaskWithPolicy : + Pat<(result_type (!cast(intrinsic) + (result_type result_reg_class:$rs3), + (op1_type op1_reg_class:$rs1), + (op2_type op2_kind:$rs2), + VLOpFrag)), + (!cast(inst#"_"#kind#"_"#vlmul.MX) + result_reg_class:$rs3, + (op1_type op1_reg_class:$rs1), + op2_kind:$rs2, + GPR:$vl, sew, TAIL_UNDISTURBED)>; + class VPatTernaryMask; } -multiclass VPatTernaryV_VV vtilist> { +multiclass VPatTernaryWithPolicy { + def : VPatTernaryNoMaskWithPolicy; + def : VPatTernaryMask; +} + +multiclass VPatTernaryV_VV_AAXA vtilist> { foreach vti = vtilist in - defm : VPatTernary; + defm : VPatTernaryWithPolicy; } multiclass VPatTernaryV_VX vtilist> { foreach vti = vtilist in - defm : VPatTernary; + defm : VPatTernaryWithPolicy; } multiclass VPatTernaryV_VI; + defm : VPatTernaryWithPolicy; } } @@ -3050,17 +3093,17 @@ foreach vtiToWti = vtilist in { defvar vti = vtiToWti.Vti; defvar wti = vtiToWti.Wti; - defm : VPatTernary; + defm : VPatTernaryWithPolicy; } } multiclass VPatTernaryV_VV_VX_AAXA vtilist> - : VPatTernaryV_VV, + : VPatTernaryV_VV_AAXA, VPatTernaryV_VX_AAXA; multiclass VPatTernaryV_VX_VI("PseudoVMADD_VV_"# suffix) vti.RegClass:$rd, vti.RegClass:$rs1, vti.RegClass:$rs2, - vti.AVL, vti.Log2SEW)>; + vti.AVL, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(vti.Vector (sub vti.RegClass:$rs2, (mul_oneuse vti.RegClass:$rs1, vti.RegClass:$rd))), (!cast("PseudoVNMSUB_VV_"# suffix) vti.RegClass:$rd, vti.RegClass:$rs1, vti.RegClass:$rs2, - vti.AVL, vti.Log2SEW)>; + vti.AVL, vti.Log2SEW, TAIL_AGNOSTIC)>; // The choice of VMADD here is arbitrary, vmadd.vx and vmacc.vx are equally // commutable. @@ -510,13 +510,13 @@ vti.RegClass:$rd))), (!cast("PseudoVMADD_VX_" # suffix) vti.RegClass:$rd, vti.ScalarRegClass:$rs1, vti.RegClass:$rs2, - vti.AVL, vti.Log2SEW)>; + vti.AVL, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(vti.Vector (sub vti.RegClass:$rs2, (mul_oneuse (SplatPat XLenVT:$rs1), vti.RegClass:$rd))), (!cast("PseudoVNMSUB_VX_" # suffix) vti.RegClass:$rd, vti.ScalarRegClass:$rs1, vti.RegClass:$rs2, - vti.AVL, vti.Log2SEW)>; + vti.AVL, vti.Log2SEW, TAIL_AGNOSTIC)>; } // 12.15. Vector Integer Merge Instructions @@ -597,27 +597,27 @@ foreach fvti = AllFloatVectors in { // NOTE: We choose VFMADD because it has the most commuting freedom. So it // works best with how TwoAddressInstructionPass tries commuting. - defvar suffix = fvti.LMul.MX # "_COMMUTABLE"; + defvar suffix = fvti.LMul.MX; def : Pat<(fvti.Vector (fma fvti.RegClass:$rs1, fvti.RegClass:$rd, fvti.RegClass:$rs2)), (!cast("PseudoVFMADD_VV_"# suffix) fvti.RegClass:$rd, fvti.RegClass:$rs1, fvti.RegClass:$rs2, - fvti.AVL, fvti.Log2SEW)>; + fvti.AVL, fvti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(fvti.Vector (fma fvti.RegClass:$rs1, fvti.RegClass:$rd, (fneg fvti.RegClass:$rs2))), (!cast("PseudoVFMSUB_VV_"# suffix) fvti.RegClass:$rd, fvti.RegClass:$rs1, fvti.RegClass:$rs2, - fvti.AVL, fvti.Log2SEW)>; + fvti.AVL, fvti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(fvti.Vector (fma (fneg fvti.RegClass:$rs1), fvti.RegClass:$rd, (fneg fvti.RegClass:$rs2))), (!cast("PseudoVFNMADD_VV_"# suffix) fvti.RegClass:$rd, fvti.RegClass:$rs1, fvti.RegClass:$rs2, - fvti.AVL, fvti.Log2SEW)>; + fvti.AVL, fvti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(fvti.Vector (fma (fneg fvti.RegClass:$rs1), fvti.RegClass:$rd, fvti.RegClass:$rs2)), (!cast("PseudoVFNMSUB_VV_"# suffix) fvti.RegClass:$rd, fvti.RegClass:$rs1, fvti.RegClass:$rs2, - fvti.AVL, fvti.Log2SEW)>; + fvti.AVL, fvti.Log2SEW, TAIL_AGNOSTIC)>; // The choice of VFMADD here is arbitrary, vfmadd.vf and vfmacc.vf are equally // commutable. @@ -625,35 +625,35 @@ fvti.RegClass:$rd, fvti.RegClass:$rs2)), (!cast("PseudoVFMADD_V" # fvti.ScalarSuffix # "_" # suffix) fvti.RegClass:$rd, fvti.ScalarRegClass:$rs1, fvti.RegClass:$rs2, - fvti.AVL, fvti.Log2SEW)>; + fvti.AVL, fvti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(fvti.Vector (fma (splat_vector fvti.ScalarRegClass:$rs1), fvti.RegClass:$rd, (fneg fvti.RegClass:$rs2))), (!cast("PseudoVFMSUB_V" # fvti.ScalarSuffix # "_" # suffix) fvti.RegClass:$rd, fvti.ScalarRegClass:$rs1, fvti.RegClass:$rs2, - fvti.AVL, fvti.Log2SEW)>; + fvti.AVL, fvti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(fvti.Vector (fma (splat_vector fvti.ScalarRegClass:$rs1), (fneg fvti.RegClass:$rd), (fneg fvti.RegClass:$rs2))), (!cast("PseudoVFNMADD_V" # fvti.ScalarSuffix # "_" # suffix) fvti.RegClass:$rd, fvti.ScalarRegClass:$rs1, fvti.RegClass:$rs2, - fvti.AVL, fvti.Log2SEW)>; + fvti.AVL, fvti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(fvti.Vector (fma (splat_vector fvti.ScalarRegClass:$rs1), (fneg fvti.RegClass:$rd), fvti.RegClass:$rs2)), (!cast("PseudoVFNMSUB_V" # fvti.ScalarSuffix # "_" # suffix) fvti.RegClass:$rd, fvti.ScalarRegClass:$rs1, fvti.RegClass:$rs2, - fvti.AVL, fvti.Log2SEW)>; + fvti.AVL, fvti.Log2SEW, TAIL_AGNOSTIC)>; // The splat might be negated. def : Pat<(fvti.Vector (fma (fneg (splat_vector fvti.ScalarRegClass:$rs1)), fvti.RegClass:$rd, (fneg fvti.RegClass:$rs2))), (!cast("PseudoVFNMADD_V" # fvti.ScalarSuffix # "_" # suffix) fvti.RegClass:$rd, fvti.ScalarRegClass:$rs1, fvti.RegClass:$rs2, - fvti.AVL, fvti.Log2SEW)>; + fvti.AVL, fvti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(fvti.Vector (fma (fneg (splat_vector fvti.ScalarRegClass:$rs1)), fvti.RegClass:$rd, fvti.RegClass:$rs2)), (!cast("PseudoVFNMSUB_V" # fvti.ScalarSuffix # "_" # suffix) fvti.RegClass:$rd, fvti.ScalarRegClass:$rs1, fvti.RegClass:$rs2, - fvti.AVL, fvti.Log2SEW)>; + fvti.AVL, fvti.Log2SEW, TAIL_AGNOSTIC)>; } foreach vti = AllFloatVectors in { diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td @@ -794,7 +794,7 @@ foreach vti = AllIntegerVectors in { // NOTE: We choose VMADD because it has the most commuting freedom. So it // works best with how TwoAddressInstructionPass tries commuting. - defvar suffix = vti.LMul.MX # "_COMMUTABLE"; + defvar suffix = vti.LMul.MX; def : Pat<(vti.Vector (riscv_add_vl vti.RegClass:$rs2, (riscv_mul_vl_oneuse vti.RegClass:$rs1, @@ -803,7 +803,7 @@ (vti.Mask true_mask), VLOpFrag)), (!cast("PseudoVMADD_VV_"# suffix) vti.RegClass:$rd, vti.RegClass:$rs1, vti.RegClass:$rs2, - GPR:$vl, vti.Log2SEW)>; + GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(vti.Vector (riscv_sub_vl vti.RegClass:$rs2, (riscv_mul_vl_oneuse vti.RegClass:$rs1, @@ -812,7 +812,7 @@ (vti.Mask true_mask), VLOpFrag)), (!cast("PseudoVNMSUB_VV_"# suffix) vti.RegClass:$rd, vti.RegClass:$rs1, vti.RegClass:$rs2, - GPR:$vl, vti.Log2SEW)>; + GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; // The choice of VMADD here is arbitrary, vmadd.vx and vmacc.vx are equally // commutable. @@ -824,7 +824,7 @@ (vti.Mask true_mask), VLOpFrag)), (!cast("PseudoVMADD_VX_" # suffix) vti.RegClass:$rd, vti.ScalarRegClass:$rs1, vti.RegClass:$rs2, - GPR:$vl, vti.Log2SEW)>; + GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(vti.Vector (riscv_sub_vl vti.RegClass:$rs2, (riscv_mul_vl_oneuse (SplatPat XLenVT:$rs1), @@ -834,7 +834,7 @@ (vti.Mask true_mask), VLOpFrag)), (!cast("PseudoVNMSUB_VX_" # suffix) vti.RegClass:$rd, vti.ScalarRegClass:$rs1, vti.RegClass:$rs2, - GPR:$vl, vti.Log2SEW)>; + GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; } // 12.14. Vector Widening Integer Multiply-Add Instructions @@ -847,18 +847,18 @@ (vti.Vector vti.RegClass:$rs2), (vti.Mask true_mask), VLOpFrag), (vti.Mask true_mask), VLOpFrag)), - (!cast("PseudoVWMACC_VV_" # vti.LMul.MX # "_TA") + (!cast("PseudoVWMACC_VV_" # vti.LMul.MX) wti.RegClass:$rd, vti.RegClass:$rs1, vti.RegClass:$rs2, - GPR:$vl, vti.Log2SEW)>; + GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(wti.Vector (riscv_add_vl wti.RegClass:$rd, (riscv_vwmulu_vl_oneuse vti.RegClass:$rs1, (vti.Vector vti.RegClass:$rs2), (vti.Mask true_mask), VLOpFrag), (vti.Mask true_mask), VLOpFrag)), - (!cast("PseudoVWMACCU_VV_" # vti.LMul.MX # "_TA") + (!cast("PseudoVWMACCU_VV_" # vti.LMul.MX) wti.RegClass:$rd, vti.RegClass:$rs1, vti.RegClass:$rs2, - GPR:$vl, vti.Log2SEW)>; + GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(wti.Vector (riscv_add_vl wti.RegClass:$rd, @@ -866,18 +866,18 @@ (vti.Vector vti.RegClass:$rs2), (vti.Mask true_mask), VLOpFrag), (vti.Mask true_mask), VLOpFrag)), - (!cast("PseudoVWMACC_VX_" # vti.LMul.MX # "_TA") + (!cast("PseudoVWMACC_VX_" # vti.LMul.MX) wti.RegClass:$rd, vti.ScalarRegClass:$rs1, vti.RegClass:$rs2, - GPR:$vl, vti.Log2SEW)>; + GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(wti.Vector (riscv_add_vl wti.RegClass:$rd, (riscv_vwmulu_vl_oneuse (SplatPat XLenVT:$rs1), (vti.Vector vti.RegClass:$rs2), (vti.Mask true_mask), VLOpFrag), (vti.Mask true_mask), VLOpFrag)), - (!cast("PseudoVWMACCU_VX_" # vti.LMul.MX # "_TA") + (!cast("PseudoVWMACCU_VX_" # vti.LMul.MX) wti.RegClass:$rd, vti.ScalarRegClass:$rs1, vti.RegClass:$rs2, - GPR:$vl, vti.Log2SEW)>; + GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; } // 12.15. Vector Integer Merge Instructions @@ -961,13 +961,13 @@ foreach vti = AllFloatVectors in { // NOTE: We choose VFMADD because it has the most commuting freedom. So it // works best with how TwoAddressInstructionPass tries commuting. - defvar suffix = vti.LMul.MX # "_COMMUTABLE"; + defvar suffix = vti.LMul.MX; def : Pat<(vti.Vector (riscv_fma_vl vti.RegClass:$rs1, vti.RegClass:$rd, vti.RegClass:$rs2, (vti.Mask true_mask), VLOpFrag)), (!cast("PseudoVFMADD_VV_"# suffix) vti.RegClass:$rd, vti.RegClass:$rs1, vti.RegClass:$rs2, - GPR:$vl, vti.Log2SEW)>; + GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(vti.Vector (riscv_fma_vl vti.RegClass:$rs1, vti.RegClass:$rd, (riscv_fneg_vl vti.RegClass:$rs2, (vti.Mask true_mask), @@ -976,7 +976,7 @@ VLOpFrag)), (!cast("PseudoVFMSUB_VV_"# suffix) vti.RegClass:$rd, vti.RegClass:$rs1, vti.RegClass:$rs2, - GPR:$vl, vti.Log2SEW)>; + GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(vti.Vector (riscv_fma_vl (riscv_fneg_vl vti.RegClass:$rs1, (vti.Mask true_mask), VLOpFrag), @@ -988,7 +988,7 @@ VLOpFrag)), (!cast("PseudoVFNMADD_VV_"# suffix) vti.RegClass:$rd, vti.RegClass:$rs1, vti.RegClass:$rs2, - GPR:$vl, vti.Log2SEW)>; + GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(vti.Vector (riscv_fma_vl (riscv_fneg_vl vti.RegClass:$rs1, (vti.Mask true_mask), VLOpFrag), @@ -997,7 +997,7 @@ VLOpFrag)), (!cast("PseudoVFNMSUB_VV_"# suffix) vti.RegClass:$rd, vti.RegClass:$rs1, vti.RegClass:$rs2, - GPR:$vl, vti.Log2SEW)>; + GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; // The choice of VFMADD here is arbitrary, vfmadd.vf and vfmacc.vf are equally // commutable. @@ -1007,7 +1007,7 @@ VLOpFrag)), (!cast("PseudoVFMADD_V" # vti.ScalarSuffix # "_" # suffix) vti.RegClass:$rd, vti.ScalarRegClass:$rs1, vti.RegClass:$rs2, - GPR:$vl, vti.Log2SEW)>; + GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(vti.Vector (riscv_fma_vl (SplatFPOp vti.ScalarRegClass:$rs1), vti.RegClass:$rd, (riscv_fneg_vl vti.RegClass:$rs2, @@ -1017,7 +1017,7 @@ VLOpFrag)), (!cast("PseudoVFMSUB_V" # vti.ScalarSuffix # "_" # suffix) vti.RegClass:$rd, vti.ScalarRegClass:$rs1, vti.RegClass:$rs2, - GPR:$vl, vti.Log2SEW)>; + GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(vti.Vector (riscv_fma_vl (SplatFPOp vti.ScalarRegClass:$rs1), (riscv_fneg_vl vti.RegClass:$rd, (vti.Mask true_mask), @@ -1029,7 +1029,7 @@ VLOpFrag)), (!cast("PseudoVFNMADD_V" # vti.ScalarSuffix # "_" # suffix) vti.RegClass:$rd, vti.ScalarRegClass:$rs1, vti.RegClass:$rs2, - GPR:$vl, vti.Log2SEW)>; + GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(vti.Vector (riscv_fma_vl (SplatFPOp vti.ScalarRegClass:$rs1), (riscv_fneg_vl vti.RegClass:$rd, (vti.Mask true_mask), @@ -1039,7 +1039,7 @@ VLOpFrag)), (!cast("PseudoVFNMSUB_V" # vti.ScalarSuffix # "_" # suffix) vti.RegClass:$rd, vti.ScalarRegClass:$rs1, vti.RegClass:$rs2, - GPR:$vl, vti.Log2SEW)>; + GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; // The splat might be negated. def : Pat<(vti.Vector (riscv_fma_vl (riscv_fneg_vl (SplatFPOp vti.ScalarRegClass:$rs1), @@ -1053,7 +1053,7 @@ VLOpFrag)), (!cast("PseudoVFNMADD_V" # vti.ScalarSuffix # "_" # suffix) vti.RegClass:$rd, vti.ScalarRegClass:$rs1, vti.RegClass:$rs2, - GPR:$vl, vti.Log2SEW)>; + GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(vti.Vector (riscv_fma_vl (riscv_fneg_vl (SplatFPOp vti.ScalarRegClass:$rs1), (vti.Mask true_mask), VLOpFrag), @@ -1062,7 +1062,7 @@ VLOpFrag)), (!cast("PseudoVFNMSUB_V" # vti.ScalarSuffix # "_" # suffix) vti.RegClass:$rd, vti.ScalarRegClass:$rs1, vti.RegClass:$rs2, - GPR:$vl, vti.Log2SEW)>; + GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; } // 14.11. Vector Floating-Point MIN/MAX Instructions diff --git a/llvm/lib/Target/RISCV/RISCVMCInstLower.cpp b/llvm/lib/Target/RISCV/RISCVMCInstLower.cpp --- a/llvm/lib/Target/RISCV/RISCVMCInstLower.cpp +++ b/llvm/lib/Target/RISCV/RISCVMCInstLower.cpp @@ -148,17 +148,18 @@ assert(TRI && "TargetRegisterInfo expected"); uint64_t TSFlags = MI->getDesc().TSFlags; - int NumOps = MI->getNumExplicitOperands(); - - for (const MachineOperand &MO : MI->explicit_operands()) { - int OpNo = (int)MI->getOperandNo(&MO); - assert(OpNo >= 0 && "Operand number doesn't fit in an 'int' type"); - - // Skip VL and SEW operands which are the last two operands if present. - if (RISCVII::hasVLOp(TSFlags) && OpNo == (NumOps - 2)) - continue; - if (RISCVII::hasSEWOp(TSFlags) && OpNo == (NumOps - 1)) - continue; + unsigned NumOps = MI->getNumExplicitOperands(); + + // Skip policy, VL and SEW operands which are the last operands if present. + if (RISCVII::hasVecPolicyOp(TSFlags)) + --NumOps; + if (RISCVII::hasVLOp(TSFlags)) + --NumOps; + if (RISCVII::hasSEWOp(TSFlags)) + --NumOps; + + for (unsigned OpNo = 0; OpNo != NumOps; ++OpNo) { + const MachineOperand &MO = MI->getOperand(OpNo); // Skip merge op. It should be the first operand after the result. if (RISCVII::hasMergeOp(TSFlags) && OpNo == 1) {