diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp --- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp @@ -3209,6 +3209,10 @@ SDValue True = N->getOperand(2); SDValue Mask = N->getOperand(3); SDValue VL = N->getOperand(4); + // We always have a glue node for the mask at v0 + assert(cast(Mask)->getReg() == RISCV::V0); + SDValue Glue = N->getOperand(N->getNumOperands() - 1); + assert(Glue.getValueType() == MVT::Glue); // We require that either merge and false are the same, or that merge // is undefined. @@ -3254,32 +3258,34 @@ return false; } - if (IsMasked) { - assert(HasTiedDest && "Expected tied dest"); - // The vmerge instruction must be TU. - if (isImplicitDef(Merge)) - return false; - // The vmerge instruction must have an all 1s mask since we're going to keep - // the mask from the True instruction. - // FIXME: Support mask agnostic True instruction which would have an - // undef merge operand. - if (!usesAllOnesMask(N, /* MaskOpIdx */ 3)) - return false; - } - // Skip if True has side effect. // TODO: Support vleff and vlsegff. if (TII->get(TrueOpc).hasUnmodeledSideEffects()) return false; - // The last operand of a masked instruction may be glued. + // Get the operands from True. They take the form: + // ([merge] ops* [mask] [rm] vl sew [policy] [chain] [glue]) + // ^ + // TrueVOpStart bool HasGlueOp = True->getGluedNode() != nullptr; - // The chain operand may exist either before the glued operands or in the last // position. - unsigned TrueChainOpIdx = True.getNumOperands() - HasGlueOp - 1; - bool HasChainOp = - True.getOperand(TrueChainOpIdx).getValueType() == MVT::Other; + unsigned TrueChainIdx = True.getNumOperands() - HasGlueOp - 1; + bool HasChainOp = True.getOperand(TrueChainIdx).getValueType() == MVT::Other; + bool HasVLOp = RISCVII::hasVLOp(TrueTSFlags); + bool HasSEWOp = RISCVII::hasSEWOp(TrueTSFlags); + bool HasVecPolicyOp = RISCVII::hasVecPolicyOp(TrueTSFlags); + bool HasRoundModeOp = RISCVII::hasRoundModeOp(TrueTSFlags); + assert(HasVLOp && HasSEWOp); + + unsigned TrueVOpStart = True.getNumOperands() - HasChainOp - HasGlueOp - + HasVecPolicyOp - HasVLOp - HasSEWOp - HasRoundModeOp - + IsMasked; + + unsigned TrueRoundModeIdx = TrueVOpStart + IsMasked; + unsigned TrueVLIdx = TrueVOpStart + IsMasked + HasRoundModeOp; + SDValue TrueVL = True.getOperand(TrueVLIdx); + SDValue TrueSEW = True.getOperand(TrueVLIdx + 1); if (HasChainOp) { // Avoid creating cycles in the DAG. We must ensure that none of the other @@ -3289,17 +3295,27 @@ LoopWorklist.push_back(False.getNode()); LoopWorklist.push_back(Mask.getNode()); LoopWorklist.push_back(VL.getNode()); - if (SDNode *Glued = N->getGluedNode()) - LoopWorklist.push_back(Glued); + LoopWorklist.push_back(Glue.getNode()); if (SDNode::hasPredecessorHelper(True.getNode(), Visited, LoopWorklist)) return false; } - // The vector policy operand may be present for masked intrinsics - bool HasVecPolicyOp = RISCVII::hasVecPolicyOp(TrueTSFlags); - unsigned TrueVLIndex = - True.getNumOperands() - HasVecPolicyOp - HasChainOp - HasGlueOp - 2; - SDValue TrueVL = True.getOperand(TrueVLIndex); + if (IsMasked) { + assert(HasTiedDest && "Expected tied dest"); + // The vmerge instruction must be TU. + if (isImplicitDef(Merge)) + return false; + // The vmerge instruction must have an all 1s mask since we're going to keep + // the mask from the True instruction. + // FIXME: Support mask agnostic True instruction which would have an + // undef merge operand. + if (!usesAllOnesMask(N, /* MaskOpIdx */ 3)) + return false; + + Mask = True->getOperand(TrueVOpStart); + assert(True->getGluedNode()); + Glue = True->getOperand(True->getNumOperands() - 1); + } // We need the VLs to be the same. But if True has a VL of VLMAX then we can // go ahead and use N's VL because we know it will be smaller, so any tail @@ -3331,39 +3347,26 @@ SDValue PolicyOp = CurDAG->getTargetConstant(Policy, DL, Subtarget->getXLenVT()); + // The resulting masked pseudo operands will be in the form: + // (merge, ops*, mask, [rm], vl, policy, [chain], glue) SmallVector Ops; - if (IsMasked) { - Ops.push_back(False); - Ops.append(True->op_begin() + 1, True->op_begin() + TrueVLIndex); - Ops.append({VL, /* SEW */ True.getOperand(TrueVLIndex + 1)}); - Ops.push_back(PolicyOp); - Ops.append(True->op_begin() + TrueVLIndex + 3, True->op_end()); - } else { - Ops.push_back(False); - if (RISCVII::hasRoundModeOp(TrueTSFlags)) { - // For unmasked "VOp" with rounding mode operand, that is interfaces like - // (..., rm, vl) or (..., rm, vl, policy). - // Its masked version is (..., vm, rm, vl, policy). - // Check the rounding mode pseudo nodes under RISCVInstrInfoVPseudos.td - SDValue RoundMode = True->getOperand(TrueVLIndex - 1); - Ops.append(True->op_begin() + HasTiedDest, - True->op_begin() + TrueVLIndex - 1); - Ops.append( - {Mask, RoundMode, VL, /* SEW */ True.getOperand(TrueVLIndex + 1)}); - } else { - Ops.append(True->op_begin() + HasTiedDest, - True->op_begin() + TrueVLIndex); - Ops.append({Mask, VL, /* SEW */ True.getOperand(TrueVLIndex + 1)}); - } - Ops.push_back(PolicyOp); + Ops.push_back(False); - // Result node should have chain operand of True. - if (HasChainOp) - Ops.push_back(True.getOperand(TrueChainOpIdx)); + Ops.append(True->op_begin() + HasTiedDest, True->op_begin() + TrueVOpStart); - if (N->getGluedNode()) - Ops.push_back(N->getOperand(N->getNumOperands() - 1)); - } + Ops.push_back(Mask); + + if (HasRoundModeOp) + Ops.push_back(True.getOperand(TrueRoundModeIdx)); + + Ops.append({VL, TrueSEW, PolicyOp}); + + // True might have a chain, e.g. if it's a load + if (HasChainOp) + Ops.push_back(True.getOperand(TrueChainIdx)); + + // Add the glue for the CopyToReg of mask->v0. + Ops.push_back(Glue); SDNode *Result = CurDAG->getMachineNode(MaskedOpc, DL, True->getVTList(), Ops);