diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp --- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp +++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp @@ -3375,16 +3375,21 @@ } unsigned DestReg = Inst.getOperand(0).getReg(); + unsigned Offset = 0; + int TiedOp = MCID.getOperandConstraint(1, MCOI::TIED_TO); + if (TiedOp == 0) + Offset = 1; + // Operands[1] will be the first operand, DestReg. SMLoc Loc = Operands[1]->getStartLoc(); if (MCID.TSFlags & RISCVII::VS2Constraint) { - unsigned CheckReg = Inst.getOperand(1).getReg(); + unsigned CheckReg = Inst.getOperand(Offset + 1).getReg(); if (DestReg == CheckReg) return Error(Loc, "The destination vector register group cannot overlap" " the source vector register group."); } - if ((MCID.TSFlags & RISCVII::VS1Constraint) && (Inst.getOperand(2).isReg())) { - unsigned CheckReg = Inst.getOperand(2).getReg(); + if ((MCID.TSFlags & RISCVII::VS1Constraint) && Inst.getOperand(Offset + 2).isReg()) { + unsigned CheckReg = Inst.getOperand(Offset + 2).getReg(); if (DestReg == CheckReg) return Error(Loc, "The destination vector register group cannot overlap" " the source vector register group."); diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp --- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp @@ -3187,7 +3187,8 @@ // instruction is tail agnostic, the unmasked instruction should not have a // merge op. uint64_t TSFlags = TII.get(Opc).TSFlags; - assert((UseTUPseudo == RISCVII::hasMergeOp(TSFlags)) && + assert(((UseTUPseudo == RISCVII::hasMergeOp(TSFlags)) || + I->UnmaskedTUPseudo == I->UnmaskedPseudo) && RISCVII::hasDummyMaskOp(TSFlags) && "Unexpected pseudo to transform to"); (void)TSFlags; @@ -3260,7 +3261,9 @@ if (!Info) return false; - if (HasMergeOp) { + bool TUisTA = Info->UnmaskedTUPseudo == Info->UnmaskedPseudo; + + if (HasMergeOp || TUisTA) { // The vmerge instruction must be TU. // FIXME: This could be relaxed, but we need to handle the policy for the // resulting op correctly. @@ -3335,7 +3338,7 @@ unsigned MaskedOpc = Info->MaskedPseudo; assert(RISCVII::hasVecPolicyOp(TII->get(MaskedOpc).TSFlags) && "Expected instructions with mask have policy operand."); - assert(RISCVII::hasMergeOp(TII->get(MaskedOpc).TSFlags) && + assert((RISCVII::hasMergeOp(TII->get(MaskedOpc).TSFlags) || TUisTA) && "Expected instructions with mask have merge operand."); SmallVector Ops; @@ -3346,7 +3349,7 @@ CurDAG->getTargetConstant(Policy, DL, Subtarget->getXLenVT())); Ops.append(True->op_begin() + TrueVLIndex + 3, True->op_end()); } else { - if (!HasMergeOp) + if (!HasMergeOp && !TUisTA) Ops.push_back(False); Ops.append(True->op_begin(), True->op_begin() + TrueVLIndex); Ops.append({Mask, VL, /* SEW */ True.getOperand(TrueVLIndex + 1)}); diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td --- a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td @@ -328,9 +328,11 @@ // op vd, vs1, vs2, vm (reverse the order of vs1 and vs2) class VALUrVV funct6, RISCVVFormat opv, string opcodestr> - : RVInstVV; + : RVInstVV { + let Constraints = "$vd = $vd_wb"; +} // op vd, vs2, vs1 class VALUVVNoVm funct6, RISCVVFormat opv, string opcodestr> @@ -356,9 +358,11 @@ // op vd, rs1, vs2, vm (reverse the order of rs1 and vs2) class VALUrVX funct6, RISCVVFormat opv, string opcodestr> - : RVInstVX; + : RVInstVX { + let Constraints = "$vd = $vd_wb"; +} // op vd, vs1, vs2 class VALUVXNoVm funct6, RISCVVFormat opv, string opcodestr> @@ -398,9 +402,11 @@ // op vd, rs1, vs2, vm (Float) (with mask, reverse the order of rs1 and vs2) class VALUrVF funct6, RISCVVFormat opv, string opcodestr> - : RVInstVX; + : RVInstVX { + let Constraints = "$vd = $vd_wb"; +} // op vd, vs2, vm (use vs1 as instruction encoding) class VALUVs2 funct6, bits<5> vs1, RISCVVFormat opv, string opcodestr> @@ -484,25 +490,25 @@ multiclass VMAC_MV_V_X funct6, string vw = "v"> { def V : VALUrVV, Sched<[WriteVIMulAddV_WorstCase, ReadVIMulAddV_WorstCase, - ReadVIMulAddV_WorstCase, ReadVMask]>; + ReadVIMulAddV_WorstCase, ReadVIMulAddV_WorstCase, ReadVMask]>; def X : VALUrVX, Sched<[WriteVIMulAddX_WorstCase, ReadVIMulAddV_WorstCase, - ReadVIMulAddX_WorstCase, ReadVMask]>; + ReadVIMulAddX_WorstCase, ReadVIMulAddV_WorstCase, ReadVMask]>; } multiclass VWMAC_MV_V_X funct6, string vw = "v"> { def V : VALUrVV, Sched<[WriteVIWMulAddV_WorstCase, ReadVIWMulAddV_WorstCase, - ReadVIWMulAddV_WorstCase, ReadVMask]>; + ReadVIWMulAddV_WorstCase, ReadVIWMulAddV_WorstCase, ReadVMask]>; def X : VALUrVX, Sched<[WriteVIWMulAddX_WorstCase, ReadVIWMulAddV_WorstCase, - ReadVIWMulAddX_WorstCase, ReadVMask]>; + ReadVIWMulAddX_WorstCase, ReadVIWMulAddV_WorstCase, ReadVMask]>; } multiclass VWMAC_MV_X funct6, string vw = "v"> { def X : VALUrVX, Sched<[WriteVIWMulAddX_WorstCase, ReadVIWMulAddV_WorstCase, - ReadVIWMulAddX_WorstCase, ReadVMask]>; + ReadVIWMulAddX_WorstCase, ReadVIWMulAddV_WorstCase, ReadVMask]>; } multiclass VALU_MV_VS2 funct6, bits<5> vs1> { @@ -623,19 +629,19 @@ multiclass VMAC_FV_V_F funct6, string vw = "v"> { def V : VALUrVV, Sched<[WriteVFMulAddV_WorstCase, ReadVFMulAddV_WorstCase, - ReadVFMulAddV_WorstCase, ReadVMask]>; + ReadVFMulAddV_WorstCase, ReadVFMulAddV_WorstCase, ReadVMask]>; def F : VALUrVF, Sched<[WriteVFMulAddF_WorstCase, ReadVFMulAddV_WorstCase, - ReadVFMulAddF_WorstCase, ReadVMask]>; + ReadVFMulAddF_WorstCase, ReadVFMulAddV_WorstCase, ReadVMask]>; } multiclass VWMAC_FV_V_F funct6, string vw = "v"> { def V : VALUrVV, Sched<[WriteVFWMulAddV_WorstCase, ReadVFWMulAddV_WorstCase, - ReadVFWMulAddV_WorstCase, ReadVMask]>; + ReadVFWMulAddV_WorstCase, ReadVFWMulAddV_WorstCase, ReadVMask]>; def F : VALUrVF, Sched<[WriteVFWMulAddF_WorstCase, ReadVFWMulAddV_WorstCase, - ReadVFWMulAddF_WorstCase, ReadVMask]>; + ReadVFWMulAddF_WorstCase, ReadVFWMulAddV_WorstCase, ReadVMask]>; } multiclass VSQR_FV_VS2 funct6, bits<5> vs1> { @@ -1259,7 +1265,8 @@ defm VNMSUB_V : VMAC_MV_V_X<"vnmsub", 0b101011>; // Vector Widening Integer Multiply-Add Instructions -let Constraints = "@earlyclobber $vd", RVVConstraint = WidenV in { +let Constraints = "@earlyclobber $vd_wb, $vd = $vd_wb", + RVVConstraint = WidenV in { defm VWMACCU_V : VWMAC_MV_V_X<"vwmaccu", 0b111100>; defm VWMACC_V : VWMAC_MV_V_X<"vwmacc", 0b111101>; defm VWMACCSU_V : VWMAC_MV_V_X<"vwmaccsu", 0b111111>; @@ -1364,7 +1371,7 @@ } // Vector Widening Floating-Point Fused Multiply-Add Instructions -let Constraints = "@earlyclobber $vd", RVVConstraint = WidenV, +let Constraints = "@earlyclobber $vd_wb, $vd = $vd_wb", RVVConstraint = WidenV, Uses = [FRM], mayRaiseFPException = true in { defm VFWMACC_V : VWMAC_FV_V_F<"vfwmacc", 0b111100>; defm VFWNMACC_V : VWMAC_FV_V_F<"vfwnmacc", 0b111101>; diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td @@ -3261,12 +3261,14 @@ defvar ReadVIMulAddV_MX = !cast("ReadVIMulAddV_" # mx); defvar ReadVIMulAddX_MX = !cast("ReadVIMulAddX_" # mx); + let HasMergeOp = 0 in defm "" : VPseudoTernaryV_VV_AAXA, Sched<[WriteVIMulAddV_MX, ReadVIMulAddV_MX, ReadVIMulAddV_MX, ReadVIMulAddV_MX, ReadVMask]>; + let HasMergeOp = 0 in defm "" : VPseudoTernaryV_VX_AAXA, - Sched<[WriteVIMulAddX_MX, ReadVIMulAddV_MX, ReadVIMulAddV_MX, - ReadVIMulAddX_MX, ReadVMask]>; + Sched<[WriteVIMulAddX_MX, ReadVIMulAddV_MX, ReadVIMulAddX_MX, + ReadVIMulAddV_MX, ReadVMask]>; } } @@ -3276,6 +3278,7 @@ defvar WriteVFMulAddV_MX = !cast("WriteVFMulAddV_" # mx); defvar ReadVFMulAddV_MX = !cast("ReadVFMulAddV_" # mx); + let HasMergeOp = 0 in defm "" : VPseudoTernaryV_VV_AAXA, Sched<[WriteVFMulAddV_MX, ReadVFMulAddV_MX, ReadVFMulAddV_MX, ReadVFMulAddV_MX, ReadVMask]>; } @@ -3287,8 +3290,9 @@ defvar ReadVFMulAddV_MX = !cast("ReadVFMulAddV_" # mx); defvar ReadVFMulAddF_MX = !cast("ReadVFMulAddF_" # mx); + let HasMergeOp = 0 in defm "" : VPseudoTernaryV_VF_AAXA, - Sched<[WriteVFMulAddF_MX, ReadVFMulAddV_MX, ReadVFMulAddV_MX, ReadVFMulAddF_MX, ReadVMask]>; + Sched<[WriteVFMulAddF_MX, ReadVFMulAddV_MX, ReadVFMulAddF_MX, ReadVFMulAddV_MX, ReadVMask]>; } } } @@ -3317,12 +3321,14 @@ defvar ReadVIWMulAddV_MX = !cast("ReadVIWMulAddV_" # mx); defvar ReadVIWMulAddX_MX = !cast("ReadVIWMulAddX_" # mx); + let HasMergeOp = 0 in defm "" : VPseudoTernaryW_VV, Sched<[WriteVIWMulAddV_MX, ReadVIWMulAddV_MX, ReadVIWMulAddV_MX, ReadVIWMulAddV_MX, ReadVMask]>; + let HasMergeOp = 0 in defm "" : VPseudoTernaryW_VX, - Sched<[WriteVIWMulAddX_MX, ReadVIWMulAddV_MX, ReadVIWMulAddV_MX, - ReadVIWMulAddX_MX, ReadVMask]>; + Sched<[WriteVIWMulAddX_MX, ReadVIWMulAddV_MX, ReadVIWMulAddX_MX, + ReadVIWMulAddV_MX, ReadVMask]>; } } @@ -3333,9 +3339,10 @@ defvar ReadVIWMulAddV_MX= !cast("ReadVIWMulAddV_" # mx); defvar ReadVIWMulAddX_MX = !cast("ReadVIWMulAddX_" # mx); + let HasMergeOp = 0 in defm "" : VPseudoTernaryW_VX, - Sched<[WriteVIWMulAddX_MX, ReadVIWMulAddV_MX, ReadVIWMulAddV_MX, - ReadVIWMulAddX_MX, ReadVMask]>; + Sched<[WriteVIWMulAddX_MX, ReadVIWMulAddV_MX, ReadVIWMulAddX_MX, + ReadVIWMulAddV_MX, ReadVMask]>; } } @@ -3345,6 +3352,7 @@ defvar WriteVFWMulAddV_MX = !cast("WriteVFWMulAddV_" # mx); defvar ReadVFWMulAddV_MX = !cast("ReadVFWMulAddV_" # mx); + let HasMergeOp = 0 in defm "" : VPseudoTernaryW_VV, Sched<[WriteVFWMulAddV_MX, ReadVFWMulAddV_MX, ReadVFWMulAddV_MX, ReadVFWMulAddV_MX, ReadVMask]>; @@ -3357,9 +3365,10 @@ defvar ReadVFWMulAddV_MX = !cast("ReadVFWMulAddV_" # mx); defvar ReadVFWMulAddF_MX = !cast("ReadVFWMulAddF_" # mx); + let HasMergeOp = 0 in defm "" : VPseudoTernaryW_VF, Sched<[WriteVFWMulAddF_MX, ReadVFWMulAddV_MX, - ReadVFWMulAddV_MX, ReadVFWMulAddF_MX, ReadVMask]>; + ReadVFWMulAddF_MX, ReadVFWMulAddV_MX, ReadVMask]>; } } } diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZvfbf.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZvfbf.td --- a/llvm/lib/Target/RISCV/RISCVInstrInfoZvfbf.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZvfbf.td @@ -25,7 +25,8 @@ defm VFNCVTBF16_F_F_W : VNCVTF_FV_VS2<"vfncvtbf16.f.f.w", 0b010010, 0b11101>; } -let Predicates = [HasStdExtZvfbfwma], Constraints = "@earlyclobber $vd", +let Predicates = [HasStdExtZvfbfwma], + Constraints = "@earlyclobber $vd_wb, $vd = $vd_wb", RVVConstraint = WidenV, Uses = [FRM], mayRaiseFPException = true in { defm VFWMACCBF16_V : VWMAC_FV_V_F<"vfwmaccbf16", 0b100011>; }