diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h --- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h +++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h @@ -254,7 +254,6 @@ struct RISCVMaskedPseudoInfo { uint16_t MaskedPseudo; uint16_t UnmaskedPseudo; - uint16_t UnmaskedTUPseudo; uint8_t MaskOpIdx; }; diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp --- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp @@ -41,22 +41,6 @@ #include "RISCVGenSearchableTables.inc" } // namespace llvm::RISCV -static unsigned getLastNonGlueOrChainOpIdx(const SDNode *Node) { - assert(Node->getNumOperands() > 0 && "Node with no operands"); - unsigned LastOpIdx = Node->getNumOperands() - 1; - if (Node->getOperand(LastOpIdx).getValueType() == MVT::Glue) - --LastOpIdx; - if (Node->getOperand(LastOpIdx).getValueType() == MVT::Other) - --LastOpIdx; - return LastOpIdx; -} - -static unsigned getVecPolicyOpIdx(const SDNode *Node, const MCInstrDesc &MCID) { - assert(RISCVII::hasVecPolicyOp(MCID.TSFlags)); - (void)MCID; - return getLastNonGlueOrChainOpIdx(Node); -} - void RISCVDAGToDAGISel::PreprocessISelDAG() { SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end(); @@ -3167,46 +3151,27 @@ if (!usesAllOnesMask(N, MaskOpIdx)) return false; - // Retrieve the tail policy operand index, if any. - std::optional TailPolicyOpIdx; - const MCInstrDesc &MaskedMCID = TII->get(N->getMachineOpcode()); - - bool UseTUPseudo = false; - if (RISCVII::hasVecPolicyOp(MaskedMCID.TSFlags)) { - TailPolicyOpIdx = getVecPolicyOpIdx(N, MaskedMCID); - // Some operations are their own TU. - if (I->UnmaskedTUPseudo == I->UnmaskedPseudo) { - UseTUPseudo = true; - } else { - if (!isImplicitDef(N->getOperand(0))) { - // Keep the true-masked instruction when there is no unmasked TU - // instruction - if (I->UnmaskedTUPseudo == I->MaskedPseudo) - return false; - UseTUPseudo = true; - } - } - } - unsigned Opc = UseTUPseudo ? I->UnmaskedTUPseudo : I->UnmaskedPseudo; + // There are two classes of pseudos in the table - compares and + // everything else. See the comment on RISCVMaskedPseudo for details. + const unsigned Opc = I->UnmaskedPseudo; const MCInstrDesc &MCID = TII->get(Opc); - - // If this instruction is tail agnostic, the unmasked instruction should not - // have a tied destination. + const bool UseTUPseudo = RISCVII::hasVecPolicyOp(MCID.TSFlags); #ifndef NDEBUG - bool HasTiedDest = RISCVII::isFirstDefTiedToFirstUse(MCID); - assert((UseTUPseudo == HasTiedDest) && "Unexpected pseudo to transform to"); + const MCInstrDesc &MaskedMCID = TII->get(N->getMachineOpcode()); + assert(RISCVII::hasVecPolicyOp(MaskedMCID.TSFlags) == + RISCVII::hasVecPolicyOp(MCID.TSFlags) && + "Masked and unmasked pseudos are inconsistent"); + const bool HasTiedDest = RISCVII::isFirstDefTiedToFirstUse(MCID); + assert(UseTUPseudo == HasTiedDest && "Unexpected pseudo structure"); #endif SmallVector Ops; // Skip the merge operand at index 0 if !UseTUPseudo. for (unsigned I = !UseTUPseudo, E = N->getNumOperands(); I != E; I++) { - // Skip the mask, the policy (if the unmasked doesn't have a policy op), and - // the Glue. + // Skip the mask, and the Glue. SDValue Op = N->getOperand(I); - if (I == MaskOpIdx || - (I == TailPolicyOpIdx && !RISCVII::hasVecPolicyOp(MCID.TSFlags)) || - Op.getValueType() == MVT::Glue) + if (I == MaskOpIdx || Op.getValueType() == MVT::Glue) continue; Ops.push_back(Op); } @@ -3269,17 +3234,10 @@ bool IsMasked = false; const RISCV::RISCVMaskedPseudoInfo *Info = - RISCV::lookupMaskedIntrinsicByUnmaskedTA(TrueOpc); + RISCV::lookupMaskedIntrinsicByUnmasked(TrueOpc); if (!Info && HasTiedDest) { - Info = RISCV::lookupMaskedIntrinsicByUnmaskedTU(TrueOpc); - if (Info && !isImplicitDef(True->getOperand(0))) - // We only support the TA form of the _TU pseudos - return false; - // FIXME: Expect undef operand here? - if (!Info) { - Info = RISCV::getMaskedPseudoInfo(TrueOpc); - IsMasked = true; - } + Info = RISCV::getMaskedPseudoInfo(TrueOpc); + IsMasked = true; } if (!Info) diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td @@ -528,30 +528,20 @@ } // Describes the relation of a masked pseudo to the unmasked variants. -// (HasTU = true, IsCombined = false) -// Has both a TA (unsuffixed) and _TU variant defined. TU variant -// may (or may not) have a vector policy operand. -// (HasTU = false, IsCombined = false) -// No TA (unsuffixed) or _TU variants. Masked version is only pseudo -// (HasTU = false, IsCombined = true) -// The unsuffixed version has a merge operand; no explicit _TU variant -// exists. The unsuffixed version has a policy operand, and can thus -// represent all policy states. -// (HasTU = true, IsCombined = true) -// Invalid and unused state. -class RISCVMaskedPseudo MaskIdx, bit HasTU = true, bit IsCombined = false> { +// Note that all masked variants (in this table) have exactly one +// unmasked variant. For all but compares, both the masked and +// unmasked variant have a passthru and policy operand. For compares, +// neither has a policy op, and only the masked version has a passthru. +class RISCVMaskedPseudo MaskIdx> { Pseudo MaskedPseudo = !cast(NAME); Pseudo UnmaskedPseudo = !cast(!subst("_MASK", "", NAME)); - Pseudo UnmaskedTUPseudo = !cond(HasTU : !cast(!subst("_MASK", "", NAME # "_TU")), - IsCombined : UnmaskedPseudo, - true : MaskedPseudo); bits<4> MaskOpIdx = MaskIdx; } def RISCVMaskedPseudosTable : GenericTable { let FilterClass = "RISCVMaskedPseudo"; let CppTypeName = "RISCVMaskedPseudoInfo"; - let Fields = ["MaskedPseudo", "UnmaskedPseudo", "UnmaskedTUPseudo", "MaskOpIdx"]; + let Fields = ["MaskedPseudo", "UnmaskedPseudo", "MaskOpIdx"]; let PrimaryKey = ["MaskedPseudo"]; let PrimaryKeyName = "getMaskedPseudoInfo"; } @@ -565,16 +555,11 @@ Pseudo Pseudo = !cast(NAME); } -def lookupMaskedIntrinsicByUnmaskedTA : SearchIndex { +def lookupMaskedIntrinsicByUnmasked : SearchIndex { let Table = RISCVMaskedPseudosTable; let Key = ["UnmaskedPseudo"]; } -def lookupMaskedIntrinsicByUnmaskedTU : SearchIndex { - let Table = RISCVMaskedPseudosTable; - let Key = ["UnmaskedTUPseudo"]; -} - def RISCVVLETable : GenericTable { let FilterClass = "RISCVVLE"; let CppTypeName = "VLEPseudo"; @@ -1600,8 +1585,7 @@ VLESched; def "E" # eew # "_V_" # LInfo # "_MASK" : VPseudoUSLoadMask, - RISCVMaskedPseudo, + RISCVMaskedPseudo, VLESched; } } @@ -1619,8 +1603,7 @@ VLFSched; def "E" # eew # "FF_V_" # LInfo # "_MASK": VPseudoUSLoadFFMask, - RISCVMaskedPseudo, + RISCVMaskedPseudo, VLFSched; } } @@ -1648,8 +1631,7 @@ VLSSched; def "E" # eew # "_V_" # LInfo # "_MASK" : VPseudoSLoadMask, - RISCVMaskedPseudo, + RISCVMaskedPseudo, VLSSched; } } @@ -1678,8 +1660,7 @@ VLXSched; def "EI" # idxEEW # "_V_" # IdxLInfo # "_" # DataLInfo # "_MASK" : VPseudoILoadMask, - RISCVMaskedPseudo, + RISCVMaskedPseudo, VLXSched; } } @@ -1811,9 +1792,7 @@ def "_V_" # m.MX : VPseudoNullaryNoMask, Sched<[WriteVMIdxV_MX, ReadVMask]>; def "_V_" # m.MX # "_MASK" : VPseudoNullaryMask, - RISCVMaskedPseudo, + RISCVMaskedPseudo, Sched<[WriteVMIdxV_MX, ReadVMask]>; } } @@ -1842,9 +1821,7 @@ def "_" # m.MX : VPseudoUnaryNoMask, Sched<[WriteVMIotV_MX, ReadVMIotV_MX, ReadVMask]>; def "_" # m.MX # "_MASK" : VPseudoUnaryMask, - RISCVMaskedPseudo, + RISCVMaskedPseudo, Sched<[WriteVMIotV_MX, ReadVMIotV_MX, ReadVMask]>; } } @@ -1878,8 +1855,7 @@ Constraint>; def suffix # "_MASK" : VPseudoBinaryMaskPolicy, - RISCVMaskedPseudo; + RISCVMaskedPseudo; } } @@ -1897,9 +1873,7 @@ Op1Class, Op2Class, Constraint>, - RISCVMaskedPseudo; + RISCVMaskedPseudo; } } @@ -1914,7 +1888,7 @@ let ForceTailAgnostic = true in def "_" # MInfo.MX # "_MASK" : VPseudoBinaryMOutMask, - RISCVMaskedPseudo; + RISCVMaskedPseudo; } } @@ -1931,9 +1905,7 @@ Constraint>; def suffix # "_" # emul.MX # "_MASK" : VPseudoBinaryMaskPolicy, - RISCVMaskedPseudo; + RISCVMaskedPseudo; } } @@ -2240,9 +2212,7 @@ def "_V_" # mx : VPseudoUnaryNoMask, Sched<[WriteVFClassV_MX, ReadVFClassV_MX, ReadVMask]>; def "_V_" # mx # "_MASK" : VPseudoUnaryMask, - RISCVMaskedPseudo, + RISCVMaskedPseudo, Sched<[WriteVFClassV_MX, ReadVFClassV_MX, ReadVMask]>; } } @@ -2263,9 +2233,7 @@ Sched<[WriteVFSqrtV_MX_E, ReadVFSqrtV_MX_E, ReadVMask]>; def "_V" # suffix # "_MASK" : VPseudoUnaryMask, - RISCVMaskedPseudo, + RISCVMaskedPseudo, Sched<[WriteVFSqrtV_MX_E, ReadVFSqrtV_MX_E, ReadVMask]>; } @@ -2282,9 +2250,7 @@ def "_V_" # mx : VPseudoUnaryNoMask, Sched<[WriteVFRecpV_MX, ReadVFRecpV_MX, ReadVMask]>; def "_V_" # mx # "_MASK" : VPseudoUnaryMask, - RISCVMaskedPseudo, + RISCVMaskedPseudo, Sched<[WriteVFRecpV_MX, ReadVFRecpV_MX, ReadVMask]>; } } @@ -2302,7 +2268,7 @@ Sched<[WriteVExtV_MX, ReadVExtV_MX, ReadVMask]>; def "_" # mx # "_MASK" : VPseudoUnaryMask, - RISCVMaskedPseudo, + RISCVMaskedPseudo, Sched<[WriteVExtV_MX, ReadVExtV_MX, ReadVMask]>; } } @@ -2320,7 +2286,7 @@ Sched<[WriteVExtV_MX, ReadVExtV_MX, ReadVMask]>; def "_" # mx # "_MASK" : VPseudoUnaryMask, - RISCVMaskedPseudo, + RISCVMaskedPseudo, Sched<[WriteVExtV_MX, ReadVExtV_MX, ReadVMask]>; } } @@ -2338,7 +2304,7 @@ Sched<[WriteVExtV_MX, ReadVExtV_MX, ReadVMask]>; def "_" # mx # "_MASK" : VPseudoUnaryMask, - RISCVMaskedPseudo, + RISCVMaskedPseudo, Sched<[WriteVExtV_MX, ReadVExtV_MX, ReadVMask]>; } } @@ -3090,7 +3056,7 @@ let isCommutable = Commutable in def "_" # MInfo.MX : VPseudoTernaryNoMaskWithPolicy; def "_" # MInfo.MX # "_MASK" : VPseudoBinaryMaskPolicy, - RISCVMaskedPseudo; + RISCVMaskedPseudo; } } @@ -3136,7 +3102,7 @@ let VLMul = MInfo.value in { def "_" # MInfo.MX : VPseudoTernaryNoMaskWithPolicy; def "_" # MInfo.MX # "_MASK" : VPseudoBinaryMaskPolicy, - RISCVMaskedPseudo; + RISCVMaskedPseudo; } } @@ -3424,9 +3390,7 @@ def "_" # MInfo.MX : VPseudoUnaryNoMask; def "_" # MInfo.MX # "_MASK" : VPseudoUnaryMask, - RISCVMaskedPseudo; + RISCVMaskedPseudo; } } @@ -3439,9 +3403,7 @@ Constraint>; def "_" # MInfo.MX # "_MASK" : VPseudoUnaryMask_FRM, - RISCVMaskedPseudo; + RISCVMaskedPseudo; } }