diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp --- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp +++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp @@ -3375,16 +3375,21 @@ } unsigned DestReg = Inst.getOperand(0).getReg(); + unsigned Offset = 0; + int TiedOp = MCID.getOperandConstraint(1, MCOI::TIED_TO); + if (TiedOp == 0) + Offset = 1; + // Operands[1] will be the first operand, DestReg. SMLoc Loc = Operands[1]->getStartLoc(); if (MCID.TSFlags & RISCVII::VS2Constraint) { - unsigned CheckReg = Inst.getOperand(1).getReg(); + unsigned CheckReg = Inst.getOperand(Offset + 1).getReg(); if (DestReg == CheckReg) return Error(Loc, "The destination vector register group cannot overlap" " the source vector register group."); } - if ((MCID.TSFlags & RISCVII::VS1Constraint) && (Inst.getOperand(2).isReg())) { - unsigned CheckReg = Inst.getOperand(2).getReg(); + if ((MCID.TSFlags & RISCVII::VS1Constraint) && Inst.getOperand(Offset + 2).isReg()) { + unsigned CheckReg = Inst.getOperand(Offset + 2).getReg(); if (DestReg == CheckReg) return Error(Loc, "The destination vector register group cannot overlap" " the source vector register group."); diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp --- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp @@ -3187,7 +3187,8 @@ // instruction is tail agnostic, the unmasked instruction should not have a // merge op. uint64_t TSFlags = TII.get(Opc).TSFlags; - assert((UseTUPseudo == RISCVII::hasMergeOp(TSFlags)) && + assert(((UseTUPseudo == RISCVII::hasMergeOp(TSFlags)) || + I->UnmaskedTUPseudo == I->UnmaskedPseudo) && RISCVII::hasDummyMaskOp(TSFlags) && "Unexpected pseudo to transform to"); (void)TSFlags; @@ -3260,7 +3261,9 @@ if (!Info) return false; - if (HasMergeOp) { + bool TUisTA = Info->UnmaskedTUPseudo == Info->UnmaskedPseudo; + + if (HasMergeOp || TUisTA) { // The vmerge instruction must be TU. // FIXME: This could be relaxed, but we need to handle the policy for the // resulting op correctly. @@ -3335,7 +3338,7 @@ unsigned MaskedOpc = Info->MaskedPseudo; assert(RISCVII::hasVecPolicyOp(TII->get(MaskedOpc).TSFlags) && "Expected instructions with mask have policy operand."); - assert(RISCVII::hasMergeOp(TII->get(MaskedOpc).TSFlags) && + assert((RISCVII::hasMergeOp(TII->get(MaskedOpc).TSFlags) || TUisTA) && "Expected instructions with mask have merge operand."); SmallVector Ops; @@ -3346,7 +3349,7 @@ CurDAG->getTargetConstant(Policy, DL, Subtarget->getXLenVT())); Ops.append(True->op_begin() + TrueVLIndex + 3, True->op_end()); } else { - if (!HasMergeOp) + if (!HasMergeOp && !TUisTA) Ops.push_back(False); Ops.append(True->op_begin(), True->op_begin() + TrueVLIndex); Ops.append({Mask, VL, /* SEW */ True.getOperand(TrueVLIndex + 1)}); diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td --- a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td @@ -328,9 +328,11 @@ // op vd, vs1, vs2, vm (reverse the order of vs1 and vs2) class VALUrVV funct6, RISCVVFormat opv, string opcodestr> - : RVInstVV; + : RVInstVV { + let Constraints = "$vd = $vd_wb"; +} // op vd, vs2, vs1 class VALUVVNoVm funct6, RISCVVFormat opv, string opcodestr> @@ -356,9 +358,11 @@ // op vd, rs1, vs2, vm (reverse the order of rs1 and vs2) class VALUrVX funct6, RISCVVFormat opv, string opcodestr> - : RVInstVX; + : RVInstVX { + let Constraints = "$vd = $vd_wb"; +} // op vd, vs1, vs2 class VALUVXNoVm funct6, RISCVVFormat opv, string opcodestr> @@ -398,9 +402,11 @@ // op vd, rs1, vs2, vm (Float) (with mask, reverse the order of rs1 and vs2) class VALUrVF funct6, RISCVVFormat opv, string opcodestr> - : RVInstVX; + : RVInstVX { + let Constraints = "$vd = $vd_wb"; +} // op vd, vs2, vm (use vs1 as instruction encoding) class VALUVs2 funct6, bits<5> vs1, RISCVVFormat opv, string opcodestr> @@ -484,25 +490,25 @@ multiclass VMAC_MV_V_X funct6, string vw = "v"> { def V : VALUrVV, Sched<[WriteVIMulAddV_WorstCase, ReadVIMulAddV_WorstCase, - ReadVIMulAddV_WorstCase, ReadVMask]>; + ReadVIMulAddV_WorstCase, ReadVIMulAddV_WorstCase, ReadVMask]>; def X : VALUrVX, Sched<[WriteVIMulAddX_WorstCase, ReadVIMulAddV_WorstCase, - ReadVIMulAddX_WorstCase, ReadVMask]>; + ReadVIMulAddX_WorstCase, ReadVIMulAddV_WorstCase, ReadVMask]>; } multiclass VWMAC_MV_V_X funct6, string vw = "v"> { def V : VALUrVV, Sched<[WriteVIWMulAddV_WorstCase, ReadVIWMulAddV_WorstCase, - ReadVIWMulAddV_WorstCase, ReadVMask]>; + ReadVIWMulAddV_WorstCase, ReadVIWMulAddV_WorstCase, ReadVMask]>; def X : VALUrVX, Sched<[WriteVIWMulAddX_WorstCase, ReadVIWMulAddV_WorstCase, - ReadVIWMulAddX_WorstCase, ReadVMask]>; + ReadVIWMulAddX_WorstCase, ReadVIWMulAddV_WorstCase, ReadVMask]>; } multiclass VWMAC_MV_X funct6, string vw = "v"> { def X : VALUrVX, Sched<[WriteVIWMulAddX_WorstCase, ReadVIWMulAddV_WorstCase, - ReadVIWMulAddX_WorstCase, ReadVMask]>; + ReadVIWMulAddX_WorstCase, ReadVIWMulAddV_WorstCase, ReadVMask]>; } multiclass VALU_MV_VS2 funct6, bits<5> vs1> { @@ -623,19 +629,19 @@ multiclass VMAC_FV_V_F funct6, string vw = "v"> { def V : VALUrVV, Sched<[WriteVFMulAddV_WorstCase, ReadVFMulAddV_WorstCase, - ReadVFMulAddV_WorstCase, ReadVMask]>; + ReadVFMulAddV_WorstCase, ReadVFMulAddV_WorstCase, ReadVMask]>; def F : VALUrVF, Sched<[WriteVFMulAddF_WorstCase, ReadVFMulAddV_WorstCase, - ReadVFMulAddF_WorstCase, ReadVMask]>; + ReadVFMulAddF_WorstCase, ReadVFMulAddV_WorstCase, ReadVMask]>; } multiclass VWMAC_FV_V_F funct6, string vw = "v"> { def V : VALUrVV, Sched<[WriteVFWMulAddV_WorstCase, ReadVFWMulAddV_WorstCase, - ReadVFWMulAddV_WorstCase, ReadVMask]>; + ReadVFWMulAddV_WorstCase, ReadVFWMulAddV_WorstCase, ReadVMask]>; def F : VALUrVF, Sched<[WriteVFWMulAddF_WorstCase, ReadVFWMulAddV_WorstCase, - ReadVFWMulAddF_WorstCase, ReadVMask]>; + ReadVFWMulAddF_WorstCase, ReadVFWMulAddV_WorstCase, ReadVMask]>; } multiclass VSQR_FV_VS2 funct6, bits<5> vs1> { @@ -1259,7 +1265,8 @@ defm VNMSUB_V : VMAC_MV_V_X<"vnmsub", 0b101011>; // Vector Widening Integer Multiply-Add Instructions -let Constraints = "@earlyclobber $vd", RVVConstraint = WidenV in { +let Constraints = "@earlyclobber $vd_wb, $vd = $vd_wb", + RVVConstraint = WidenV in { defm VWMACCU_V : VWMAC_MV_V_X<"vwmaccu", 0b111100>; defm VWMACC_V : VWMAC_MV_V_X<"vwmacc", 0b111101>; defm VWMACCSU_V : VWMAC_MV_V_X<"vwmaccsu", 0b111111>; @@ -1364,7 +1371,7 @@ } // Vector Widening Floating-Point Fused Multiply-Add Instructions -let Constraints = "@earlyclobber $vd", RVVConstraint = WidenV, +let Constraints = "@earlyclobber $vd_wb, $vd = $vd_wb", RVVConstraint = WidenV, Uses = [FRM], mayRaiseFPException = true in { defm VFWMACC_V : VWMAC_FV_V_F<"vfwmacc", 0b111100>; defm VFWNMACC_V : VWMAC_FV_V_F<"vfwnmacc", 0b111101>; diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td @@ -1237,7 +1237,8 @@ class VPseudoBinaryMaskPolicy : + string Constraint, + bit MergeOp = 1> : Pseudo<(outs GetVRegNoV0.R:$rd), (ins GetVRegNoV0.R:$merge, Op1Class:$rs2, Op2Class:$rs1, @@ -1249,7 +1250,7 @@ let Constraints = Join<[Constraint, "$rd = $merge"], ",">.ret; let HasVLOp = 1; let HasSEWOp = 1; - let HasMergeOp = 1; + let HasMergeOp = MergeOp; let HasVecPolicyOp = 1; let UsesMaskPolicy = 1; } @@ -1382,7 +1383,8 @@ class VPseudoTernaryNoMaskWithPolicy : + string Constraint, + bit MergeOp = 1> : Pseudo<(outs RetClass:$rd), (ins RetClass:$rs3, Op1Class:$rs1, Op2Class:$rs2, AVL:$vl, ixlenimm:$sew, ixlenimm:$policy), @@ -1395,7 +1397,7 @@ let HasVecPolicyOp = 1; let HasVLOp = 1; let HasSEWOp = 1; - let HasMergeOp = 1; + let HasMergeOp = MergeOp; let HasDummyMask = 1; } @@ -3205,8 +3207,8 @@ bit Commutable = 0> { let VLMul = MInfo.value in { let isCommutable = Commutable in - def "_" # MInfo.MX : VPseudoTernaryNoMaskWithPolicy; - def "_" # MInfo.MX # "_MASK" : VPseudoBinaryMaskPolicy, + def "_" # MInfo.MX : VPseudoTernaryNoMaskWithPolicy; + def "_" # MInfo.MX # "_MASK" : VPseudoBinaryMaskPolicy, RISCVMaskedPseudo; } } @@ -3230,19 +3232,20 @@ multiclass VPseudoTernaryW_VV { defvar constraint = "@earlyclobber $rd"; defm _VV : VPseudoTernaryWithPolicy; + constraint, /*Commutable*/0>; } multiclass VPseudoTernaryW_VX { defvar constraint = "@earlyclobber $rd"; defm "_VX" : VPseudoTernaryWithPolicy; + constraint, /*Commutable*/0>; } multiclass VPseudoTernaryW_VF { defvar constraint = "@earlyclobber $rd"; defm "_V" # f.FX : VPseudoTernaryWithPolicy; + m.vrclass, m, constraint, + /*Commutable*/0>; } multiclass VPseudoVSLDVWithPolicy; defm "" : VPseudoTernaryV_VX_AAXA, - Sched<[WriteVIMulAddX_MX, ReadVIMulAddV_MX, ReadVIMulAddV_MX, - ReadVIMulAddX_MX, ReadVMask]>; + Sched<[WriteVIMulAddX_MX, ReadVIMulAddV_MX, ReadVIMulAddX_MX, + ReadVIMulAddV_MX, ReadVMask]>; } } @@ -3299,7 +3302,7 @@ defvar ReadVFMulAddF_MX = !cast("ReadVFMulAddF_" # mx); defm "" : VPseudoTernaryV_VF_AAXA, - Sched<[WriteVFMulAddF_MX, ReadVFMulAddV_MX, ReadVFMulAddV_MX, ReadVFMulAddF_MX, ReadVMask]>; + Sched<[WriteVFMulAddF_MX, ReadVFMulAddV_MX, ReadVFMulAddF_MX, ReadVFMulAddV_MX, ReadVMask]>; } } } @@ -3332,8 +3335,8 @@ Sched<[WriteVIWMulAddV_MX, ReadVIWMulAddV_MX, ReadVIWMulAddV_MX, ReadVIWMulAddV_MX, ReadVMask]>; defm "" : VPseudoTernaryW_VX, - Sched<[WriteVIWMulAddX_MX, ReadVIWMulAddV_MX, ReadVIWMulAddV_MX, - ReadVIWMulAddX_MX, ReadVMask]>; + Sched<[WriteVIWMulAddX_MX, ReadVIWMulAddV_MX, ReadVIWMulAddX_MX, + ReadVIWMulAddV_MX, ReadVMask]>; } } @@ -3345,8 +3348,8 @@ defvar ReadVIWMulAddX_MX = !cast("ReadVIWMulAddX_" # mx); defm "" : VPseudoTernaryW_VX, - Sched<[WriteVIWMulAddX_MX, ReadVIWMulAddV_MX, ReadVIWMulAddV_MX, - ReadVIWMulAddX_MX, ReadVMask]>; + Sched<[WriteVIWMulAddX_MX, ReadVIWMulAddV_MX, ReadVIWMulAddX_MX, + ReadVIWMulAddV_MX, ReadVMask]>; } } @@ -3370,7 +3373,7 @@ defm "" : VPseudoTernaryW_VF, Sched<[WriteVFWMulAddF_MX, ReadVFWMulAddV_MX, - ReadVFWMulAddV_MX, ReadVFWMulAddF_MX, ReadVMask]>; + ReadVFWMulAddF_MX, ReadVFWMulAddV_MX, ReadVMask]>; } } } diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td --- a/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td @@ -51,16 +51,22 @@ let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in { // op vd, vs1, vs2, vm (reverse the order of vs1 and vs2) -class THVdotALUrVV funct6, RISCVVFormat opv, string opcodestr> - : THInstVdotVV; +class THVdotALUrVV funct6, RISCVVFormat opv, string opcodestr, + bit EarlyClobber> + : THInstVdotVV { + let Constraints = !if(EarlyClobber, "@earlyclobber $vd_wb, $vd = $vd_wb", "$vd = $vd_wb"); +} // op vd, rs1, vs2, vm (reverse the order of rs1 and vs2) -class THVdotALUrVX funct6, RISCVVFormat opv, string opcodestr> - : THInstVdotVX; +class THVdotALUrVX funct6, RISCVVFormat opv, string opcodestr, + bit EarlyClobber> + : THInstVdotVX { + let Constraints = !if(EarlyClobber, "@earlyclobber $vd_wb, $vd = $vd_wb", "$vd = $vd_wb"); +} } // hasSideEffects = 0, mayLoad = 0, mayStore = 0 let Predicates = [HasVendorXTHeadBa], DecoderNamespace = "THeadBa", @@ -220,13 +226,14 @@ // Combination of instruction classes. // Use these multiclasses to define instructions more easily. //===----------------------------------------------------------------------===// -multiclass THVdotVMAQA_VX funct6> { - def _VX : THVdotALUrVX; +multiclass THVdotVMAQA_VX funct6, + bit EarlyClobber = 0> { + def _VX : THVdotALUrVX; } -multiclass THVdotVMAQA funct6> { - def _VV : THVdotALUrVV; - defm "" : THVdotVMAQA_VX; +multiclass THVdotVMAQA funct6, bit EarlyClobber = 0> { + def _VV : THVdotALUrVV; + defm "" : THVdotVMAQA_VX; } //===----------------------------------------------------------------------===// @@ -444,12 +451,11 @@ } let Predicates = [HasVendorXTHeadVdot], - Constraints = "@earlyclobber $vd", RVVConstraint = WidenV in { -defm THVdotVMAQA : THVdotVMAQA<"th.vmaqa", 0b100000>; -defm THVdotVMAQAU : THVdotVMAQA<"th.vmaqau", 0b100010>; -defm THVdotVMAQASU : THVdotVMAQA<"th.vmaqasu", 0b100100>; -defm THVdotVMAQAUS : THVdotVMAQA_VX<"th.vmaqaus",0b100110>; +defm THVdotVMAQA : THVdotVMAQA<"th.vmaqa", 0b100000, /*EarlyClobber*/1>; +defm THVdotVMAQAU : THVdotVMAQA<"th.vmaqau", 0b100010, /*EarlyClobber*/1>; +defm THVdotVMAQASU : THVdotVMAQA<"th.vmaqasu", 0b100100, /*EarlyClobber*/1>; +defm THVdotVMAQAUS : THVdotVMAQA_VX<"th.vmaqaus",0b100110, /*EarlyClobber*/1>; } // Associate LMUL with tablegen records of register classes. diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZvfbf.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZvfbf.td --- a/llvm/lib/Target/RISCV/RISCVInstrInfoZvfbf.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZvfbf.td @@ -25,7 +25,8 @@ defm VFNCVTBF16_F_F_W : VNCVTF_FV_VS2<"vfncvtbf16.f.f.w", 0b010010, 0b11101>; } -let Predicates = [HasStdExtZvfbfwma], Constraints = "@earlyclobber $vd", +let Predicates = [HasStdExtZvfbfwma], + Constraints = "@earlyclobber $vd_wb, $vd = $vd_wb", RVVConstraint = WidenV, Uses = [FRM], mayRaiseFPException = true in { defm VFWMACCBF16_V : VWMAC_FV_V_F<"vfwmaccbf16", 0b100011>; }