Index: llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp =================================================================== --- llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp +++ llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp @@ -3242,6 +3242,11 @@ return true; } +static bool isImplicitDef(SDValue V) { + return V.isMachineOpcode() && + V.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF; +} + // Try to fold away VMERGE_VVM instructions. We handle these cases: // -Masked TU VMERGE_VVM combined with an unmasked TA instruction instruction // folds to a masked TU instruction. VMERGE_VVM must have have merge operand @@ -3279,23 +3284,26 @@ const RISCV::RISCVMaskedPseudoInfo *Info = RISCV::lookupMaskedIntrinsicByUnmaskedTA(TrueOpc); if (!Info && HasTiedDest) { - Info = RISCV::getMaskedPseudoInfo(TrueOpc); - IsMasked = true; + Info = RISCV::lookupMaskedIntrinsicByUnmaskedTU(TrueOpc); + if (Info && !isImplicitDef(True->getOperand(0))) + // We only support the TA form of the _TU pseudos + return false; + // FIXME: Expect undef operand here? + if (!Info) { + Info = RISCV::getMaskedPseudoInfo(TrueOpc); + IsMasked = true; + } } if (!Info) return false; if (HasTiedDest) { - // The vmerge instruction must be TU. - // FIXME: This could be relaxed, but we need to handle the policy for the - // resulting op correctly. - if (IsTA) - return false; + // The vmerge instruction must be TU or both merge and op need to be TA. + // Either both the vmerge instruction and the True instruction must have the + // same merge operand or the true instructions merge must be undef. SDValue MergeOpTrue = True->getOperand(0); - // Both the vmerge instruction and the True instruction must have the same - // merge operand. - if (False != MergeOpTrue) + if (!isImplicitDef(MergeOpTrue) && (IsTA || False != MergeOpTrue)) return false; } @@ -3370,15 +3378,15 @@ SmallVector Ops; if (IsMasked) { - Ops.append(True->op_begin(), True->op_begin() + TrueVLIndex); + Ops.push_back(False); + Ops.append(True->op_begin()+1, True->op_begin() + TrueVLIndex); Ops.append({VL, /* SEW */ True.getOperand(TrueVLIndex + 1)}); Ops.push_back( CurDAG->getTargetConstant(Policy, DL, Subtarget->getXLenVT())); Ops.append(True->op_begin() + TrueVLIndex + 3, True->op_end()); } else { - if (!HasTiedDest) - Ops.push_back(False); - Ops.append(True->op_begin(), True->op_begin() + TrueVLIndex); + Ops.push_back(False); + Ops.append(True->op_begin() + HasTiedDest, True->op_begin() + TrueVLIndex); Ops.append({Mask, VL, /* SEW */ True.getOperand(TrueVLIndex + 1)}); Ops.push_back( CurDAG->getTargetConstant(Policy, DL, Subtarget->getXLenVT())); Index: llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td =================================================================== --- llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td +++ llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td @@ -506,6 +506,11 @@ let Key = ["UnmaskedPseudo"]; } +def lookupMaskedIntrinsicByUnmaskedTU : SearchIndex { + let Table = RISCVMaskedPseudosTable; + let Key = ["UnmaskedTUPseudo"]; +} + def RISCVVLETable : GenericTable { let FilterClass = "RISCVVLE"; let CppTypeName = "VLEPseudo"; @@ -4395,8 +4400,8 @@ def : Pat<(vti.Vector (!cast(intrinsic) (vti.Vector undef), VLOpFrag)), - (!cast(instruction#"_V_" # vti.LMul.MX) - GPR:$vl, vti.Log2SEW)>; + (!cast(instruction#"_V_" # vti.LMul.MX # "_TU") + (vti.Vector (IMPLICIT_DEF)), GPR:$vl, vti.Log2SEW)>; def : Pat<(vti.Vector (!cast(intrinsic) (vti.Vector vti.RegClass:$merge), VLOpFrag)), Index: llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td =================================================================== --- llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td +++ llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td @@ -2382,7 +2382,6 @@ (!cast("PseudoVID_V_"#vti.LMul.MX#"_MASK") (vti.Vector (IMPLICIT_DEF)), (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; - def : Pat<(vti.Vector (riscv_slide1up_vl (vti.Vector undef), (vti.Vector vti.RegClass:$rs1), GPR:$rs2, (vti.Mask true_mask),