[RISCV] Transform VMERGE_VVM_<LMUL>_TU with all ones mask to VADD_VI_<LMUL>_TU.

[fakepaper56]

Summary

The transformation is benefit because vmerge.vvm always needs mask operand but
vadd.vi may not.

[fakepaper56]

llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp

2511

Do we have an instruction that mask operand is not v0? I am confused by the condition.

[craig.topper]

llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp

2509

use -> uses?

2710

decrase -> decrease

2713

Should we move this check and useAllOnesMask up to the caller? They are common to performVMergeToVAdd and performCombineVMergeAndVOps.

[fakepaper56]

Address Craig's comment and -verify-machineinstrs into test case.

[fakepaper56]

llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp

2509

Done.

2710

Done.

2713

Good point. Done.

[kito-cheng]

llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp

2511

We *might* allow mask hold other than v0 register in future vector extension, so having this checking should be harmless, and could catch unexpected case in future.

[fakepaper56]

llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp

2511

Thank your explanation.

[fakepaper56]

I still moved useAllOnesMask into performVMergeToVAdd, since only
performVMergeToVAdd needs to call useAllOnesMask.

[craig.topper]

In D133255#3769555, @fakepaper56 wrote:

I still moved useAllOnesMask into performVMergeToVAdd, since only
performVMergeToVAdd needs to call useAllOnesMask.

Doesn't doPeepholeMaskedRVV call usesAllOnesMask on line 2555?

[fakepaper56]

In D133255#3769573, @craig.topper wrote:

In D133255#3769555, @fakepaper56 wrote:

I still moved useAllOnesMask into performVMergeToVAdd, since only
performVMergeToVAdd needs to call useAllOnesMask.

Doesn't doPeepholeMaskedRVV call usesAllOnesMask on line 2555?

But the new function performCombineVMergeAndVOps also serves vmerge.vvm without all ones mask.

[fakepaper56]

Rebase.

[craig.topper]

LGTM

llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h

 
 // Include the pieces autogenerated from the target description.
 #include "RISCVGenDAGISel.inc"
 
 private:
   bool doPeepholeSExtW(SDNode *Node);
   bool doPeepholeMaskedRVV(SDNode *Node);
   bool doPeepholeMergeVVMFold();
+  bool performVMergeToVAdd(SDNode *N);
+  bool performCombineVMergeAndVOps(SDNode *N);
 };
 
 namespace RISCV {
 struct VLSEGPseudo {
   uint16_t NF : 4;
   uint16_t Masked : 1;
   uint16_t IsTU : 1;
   uint16_t Strided : 1;

llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp

case RISCV::GORCIW:
     // NOTE: We only handle the nodes that are selected with hasAllWUsers.
     ReplaceUses(N, N0.getNode());
     return true;
   }
 
   return false;
 }
 
-// Optimize masked RVV pseudo instructions with a known all-ones mask to their
-// corresponding "unmasked" pseudo versions. The mask we're interested in will
-// take the form of a V0 physical register operand, with a glued
-// register-setting instruction.
-bool RISCVDAGToDAGISel::doPeepholeMaskedRVV(SDNode *N) {
-  const RISCV::RISCVMaskedPseudoInfo *I =
-      RISCV::getMaskedPseudoInfo(N->getMachineOpcode());
-  if (!I)
-    return false;
-
-  unsigned MaskOpIdx = I->MaskOpIdx;
-
+// Return true if we can make sure mask of N is all-ones mask.
+static bool usesAllOnesMask(SDNode *N, unsigned MaskOpIdx) {
   // Check that we're using V0 as a mask register.
   if (!isa<RegisterSDNode>(N->getOperand(MaskOpIdx)) ||
       cast<RegisterSDNode>(N->getOperand(MaskOpIdx))->getReg() != RISCV::V0)
     return false;
 
   // The glued user defines V0.
   const auto *Glued = N->getGluedNode();
 
   if (!Glued || Glued->getOpcode() != ISD::CopyToReg)
     return false;
return Opc == RISCV::PseudoVMSET_M_B1 || Opc == RISCV::PseudoVMSET_M_B16 ||
            Opc == RISCV::PseudoVMSET_M_B2 || Opc == RISCV::PseudoVMSET_M_B32 ||
            Opc == RISCV::PseudoVMSET_M_B4 || Opc == RISCV::PseudoVMSET_M_B64 ||
            Opc == RISCV::PseudoVMSET_M_B8;
   };
 
   // TODO: Check that the VMSET is the expected bitwidth? The pseudo has
   // undefined behaviour if it's the wrong bitwidth, so we could choose to
   // assume that it's all-ones? Same applies to its VL.
-  if (!MaskSetter->isMachineOpcode() || !IsVMSet(MaskSetter.getMachineOpcode()))
+  return MaskSetter->isMachineOpcode() &&
+         IsVMSet(MaskSetter.getMachineOpcode());
+}
+
+// Optimize masked RVV pseudo instructions with a known all-ones mask to their
+// corresponding "unmasked" pseudo versions. The mask we're interested in will
+// take the form of a V0 physical register operand, with a glued
+// register-setting instruction.
+bool RISCVDAGToDAGISel::doPeepholeMaskedRVV(SDNode *N) {
+  const RISCV::RISCVMaskedPseudoInfo *I =
+      RISCV::getMaskedPseudoInfo(N->getMachineOpcode());
+  if (!I)
+    return false;
+
+  unsigned MaskOpIdx = I->MaskOpIdx;
+
+  if (!usesAllOnesMask(N, MaskOpIdx))
     return false;
 
   // Retrieve the tail policy operand index, if any.
   Optional<unsigned> TailPolicyOpIdx;
   const RISCVInstrInfo &TII = *Subtarget->getInstrInfo();
   const MCInstrDesc &MaskedMCID = TII.get(N->getMachineOpcode());
 
   bool IsTA = true;
for (unsigned I = IsTA, E = N->getNumOperands(); I != E; I++) {
     SDValue Op = N->getOperand(I);
     if (I == MaskOpIdx || I == TailPolicyOpIdx ||
         Op.getValueType() == MVT::Glue)
       continue;
     Ops.push_back(Op);
   }
 
   // Transitively apply any node glued to our new node.
+  const auto *Glued = N->getGluedNode();
   if (auto *TGlued = Glued->getGluedNode())
     Ops.push_back(SDValue(TGlued, TGlued->getNumValues() - 1));
 
   SDNode *Result = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
   Result->setFlags(N->getFlags());
   ReplaceUses(N, Result);
 
   return true;
 }
 
 // Try to fold VMERGE_VVM with unmasked intrinsic to masked intrinsic. The
 // peephole only deals with VMERGE_VVM which is TU and has false operand same as
 // its true operand now. E.g. (VMERGE_VVM_M1_TU False, False, (VADD_M1 ...),
 // ...) -> (VADD_VV_M1_MASK)
-bool RISCVDAGToDAGISel::doPeepholeMergeVVMFold() {
-  bool MadeChange = false;
-  SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
-
-  while (Position != CurDAG->allnodes_begin()) {
-    SDNode *N = &*--Position;
-    if (N->use_empty() || !N->isMachineOpcode())
-      continue;
-
-    auto IsVMergeTU = [](unsigned Opcode) {
-      return Opcode == RISCV::PseudoVMERGE_VVM_MF8_TU ||
-             Opcode == RISCV::PseudoVMERGE_VVM_MF4_TU ||
-             Opcode == RISCV::PseudoVMERGE_VVM_MF2_TU ||
-             Opcode == RISCV::PseudoVMERGE_VVM_M1_TU ||
-             Opcode == RISCV::PseudoVMERGE_VVM_M2_TU ||
-             Opcode == RISCV::PseudoVMERGE_VVM_M4_TU ||
-             Opcode == RISCV::PseudoVMERGE_VVM_M8_TU;
-    };
-
-    unsigned Opc = N->getMachineOpcode();
-    // TODO: Also deal with TA VMerge nodes.
-    if (!IsVMergeTU(Opc))
-      continue;
-
-    SDValue Merge = N->getOperand(0);
-    SDValue False = N->getOperand(1);
-    SDValue True = N->getOperand(2);
-    SDValue Mask = N->getOperand(3);
-    SDValue VL = N->getOperand(4);
+bool RISCVDAGToDAGISel::performCombineVMergeAndVOps(SDNode *N) {
 
-    if (Merge != False)
-      continue;
-
-    assert(True.getResNo() == 0 &&
-           "Expect True is the first output of an instruction.");
 
-    // Need N is the exactly one using True.
-    if (!True.hasOneUse())
-      continue;
+    return false;
 
-    if (!True.isMachineOpcode())
-      continue;
+    return false;
 
-    unsigned TrueOpc = True.getMachineOpcode();
 
-    // Skip if True has merge operand.
-    // TODO: Deal with True having same merge operand with N.
-    if (RISCVII::hasMergeOp(TII->get(TrueOpc).TSFlags))
-      continue;
+    return false;
 
-    // Skip if True has side effect.
-    // TODO: Support velff and vlsegff.
-    if (TII->get(TrueOpc).hasUnmodeledSideEffects())
-      continue;
+    return false;
 
-    // Only deal with True when True is unmasked intrinsic now.
-    const RISCV::RISCVMaskedPseudoInfo *Info =
-        RISCV::lookupMaskedIntrinsicByUnmaskedTA(TrueOpc);
 
-    if (!Info)
-      continue;
+    return false;
 
-    // The last operand of unmasked intrinsic should be sew or chain.
-    bool HasChainOp =
-        True.getOperand(True.getNumOperands() - 1).getValueType() == MVT::Other;
 
-    // Need True has same VL with N.
-    unsigned TrueVLIndex = True.getNumOperands() - HasChainOp - 2;
-    SDValue TrueVL = True.getOperand(TrueVLIndex);
 
-    auto IsNoFPExcept = [this](SDValue N) {
-      return !this->mayRaiseFPException(N.getNode()) ||
-             N->getFlags().hasNoFPExcept();
-    };
 
-    // Allow the peephole for non-exception True with VLMAX vector length, since
-    // all the values after VL of N are dependent on Merge. VLMAX should be
-    // lowered to (XLenVT -1).
-    if (TrueVL != VL && !(IsNoFPExcept(True) && isAllOnesConstant(TrueVL)))
-      continue;
+    return false;
 
-    SDLoc DL(N);
-    unsigned MaskedOpc = Info->MaskedPseudo;
-    assert(RISCVII::hasVecPolicyOp(TII->get(MaskedOpc).TSFlags) &&
-           "Expected instructions with mask have policy operand.");
 
-    SmallVector<SDValue, 8> Ops;
-    Ops.push_back(Merge);
-    Ops.append(True->op_begin(), True->op_begin() + TrueVLIndex);
-    Ops.append({Mask, VL, /* SEW */ True.getOperand(TrueVLIndex + 1)});
-    Ops.push_back(
-        CurDAG->getTargetConstant(/* TUMU */ 0, DL, Subtarget->getXLenVT()));
 
-    // Result node should have chain operand of True.
-    if (HasChainOp)
-      Ops.push_back(True.getOperand(True.getNumOperands() - 1));
 
-    // Result node should take over glued node of N.
-    if (N->getGluedNode())
-      Ops.push_back(N->getOperand(N->getNumOperands() - 1));
 
-    SDNode *Result =
-        CurDAG->getMachineNode(MaskedOpc, DL, True->getVTList(), Ops);
-    Result->setFlags(True->getFlags());
 
-    // Replace vmerge.vvm node by Result.
-    ReplaceUses(SDValue(N, 0), SDValue(Result, 0));
 
-    // Replace another value of True. E.g. chain and VL.
-    for (unsigned Idx = 1; Idx < True->getNumValues(); ++Idx)
-      ReplaceUses(True.getValue(Idx), SDValue(Result, Idx));
 
-    // Try to transform Result to unmasked intrinsic.
-    doPeepholeMaskedRVV(Result);
-    MadeChange = true;
+  return true;
+}
+
+// Transform (VMERGE_VVM_<LMUL>_TU false, false, true, allones, vl, sew) to
+// (VADD_VI_<LMUL>_TU false, true, 0, vl, sew). It may decrease uses of VMSET.
+bool RISCVDAGToDAGISel::performVMergeToVAdd(SDNode *N) {
+  unsigned NewOpc;
+  switch (N->getMachineOpcode()) {
+  default:
+    llvm_unreachable("Expected VMERGE_VVM_<LMUL>_TU instruction.");
+  case RISCV::PseudoVMERGE_VVM_MF8_TU:
+    NewOpc = RISCV::PseudoVADD_VI_MF8_TU;
+    break;
+  case RISCV::PseudoVMERGE_VVM_MF4_TU:
+    NewOpc = RISCV::PseudoVADD_VI_MF4_TU;
+    break;
+  case RISCV::PseudoVMERGE_VVM_MF2_TU:
+    NewOpc = RISCV::PseudoVADD_VI_MF2_TU;
+    break;
+  case RISCV::PseudoVMERGE_VVM_M1_TU:
+    NewOpc = RISCV::PseudoVADD_VI_M1_TU;
+    break;
+  case RISCV::PseudoVMERGE_VVM_M2_TU:
+    NewOpc = RISCV::PseudoVADD_VI_M2_TU;
+    break;
+  case RISCV::PseudoVMERGE_VVM_M4_TU:
+    NewOpc = RISCV::PseudoVADD_VI_M4_TU;
+    break;
+  case RISCV::PseudoVMERGE_VVM_M8_TU:
+    NewOpc = RISCV::PseudoVADD_VI_M8_TU;
+    break;
+  }
+
+  if (!usesAllOnesMask(N, /* MaskOpIdx */ 3))
+    return false;
+
+  SDLoc DL(N);
+  EVT VT = N->getValueType(0);
+  SDValue Ops[] = {N->getOperand(1), N->getOperand(2),
+                   CurDAG->getTargetConstant(0, DL, Subtarget->getXLenVT()),
+                   N->getOperand(4), N->getOperand(5)};
+  SDNode *Result = CurDAG->getMachineNode(NewOpc, DL, VT, Ops);
+  ReplaceUses(N, Result);
+  return true;
+}
+
+bool RISCVDAGToDAGISel::doPeepholeMergeVVMFold() {
+  bool MadeChange = false;
+  SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
+
+  while (Position != CurDAG->allnodes_begin()) {
+    SDNode *N = &*--Position;
+    if (N->use_empty() || !N->isMachineOpcode())
+      continue;
+
+    auto IsVMergeTU = [](unsigned Opcode) {
+      return Opcode == RISCV::PseudoVMERGE_VVM_MF8_TU ||
+             Opcode == RISCV::PseudoVMERGE_VVM_MF4_TU ||
+             Opcode == RISCV::PseudoVMERGE_VVM_MF2_TU ||
+             Opcode == RISCV::PseudoVMERGE_VVM_M1_TU ||
+             Opcode == RISCV::PseudoVMERGE_VVM_M2_TU ||
+             Opcode == RISCV::PseudoVMERGE_VVM_M4_TU ||
+             Opcode == RISCV::PseudoVMERGE_VVM_M8_TU;
+    };
+
+    unsigned Opc = N->getMachineOpcode();
+    // The following optimizations require that the merge operand of N is same
+    // as the false operand of N.
+    // TODO: Also deal with TA VMerge nodes.
+    if (!IsVMergeTU(Opc) || N->getOperand(0) != N->getOperand(1))
+      continue;
+
+    MadeChange |= performCombineVMergeAndVOps(N);
+    MadeChange |= performVMergeToVAdd(N);
   }
   return MadeChange;
 }
 
 // This pass converts a legalized DAG into a RISCV-specific DAG, ready
 // for instruction scheduling.
 FunctionPass *llvm::createRISCVISelDag(RISCVTargetMachine &TM,
                                        CodeGenOpt::Level OptLevel) {
   return new RISCVDAGToDAGISel(TM, OptLevel);
 }

llvm/test/CodeGen/RISCV/rvv/rvv-vmerge-to-vadd.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v -verify-machineinstrs | FileCheck %s
+define <vscale x 1 x i8> @vpmerge_mf8(<vscale x 1 x i8> %x, <vscale x 1 x i8> %y, i32 zeroext %vl) {
+; CHECK-LABEL: vpmerge_mf8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, mu
+; CHECK-NEXT:    vadd.vi v8, v9, 0
+; CHECK-NEXT:    ret
+  %splat = insertelement <vscale x 1 x i1> poison, i1 -1, i8 0
+  %allones = shufflevector <vscale x 1 x i1> %splat, <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
+  %1 = call <vscale x 1 x i8> @llvm.vp.merge.nxv1i8(<vscale x 1 x i1> %allones, <vscale x 1 x i8> %y, <vscale x 1 x i8> %x, i32 %vl)
+  ret <vscale x 1 x i8> %1
+}
+
+define <vscale x 2 x i8> @vpmerge_mf4(<vscale x 2 x i8> %x, <vscale x 2 x i8> %y, i32 zeroext %vl) {
+; CHECK-LABEL: vpmerge_mf4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, tu, mu
+; CHECK-NEXT:    vadd.vi v8, v9, 0
+; CHECK-NEXT:    ret
+  %splat = insertelement <vscale x 2 x i1> poison, i1 -1, i8 0
+  %allones = shufflevector <vscale x 2 x i1> %splat, <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
+  %1 = call <vscale x 2 x i8> @llvm.vp.merge.nxv2i8(<vscale x 2 x i1> %allones, <vscale x 2 x i8> %y, <vscale x 2 x i8> %x, i32 %vl)
+  ret <vscale x 2 x i8> %1
+}
+
+define <vscale x 4 x i8> @vpmerge_mf2(<vscale x 4 x i8> %x, <vscale x 4 x i8> %y, i32 zeroext %vl) {
+; CHECK-LABEL: vpmerge_mf2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, tu, mu
+; CHECK-NEXT:    vadd.vi v8, v9, 0
+; CHECK-NEXT:    ret
+  %splat = insertelement <vscale x 4 x i1> poison, i1 -1, i8 0
+  %allones = shufflevector <vscale x 4 x i1> %splat, <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
+  %1 = call <vscale x 4 x i8> @llvm.vp.merge.nxv4i8(<vscale x 4 x i1> %allones, <vscale x 4 x i8> %y, <vscale x 4 x i8> %x, i32 %vl)
+  ret <vscale x 4 x i8> %1
+}
+
+define <vscale x 8 x i8> @vpmerge_m1(<vscale x 8 x i8> %x, <vscale x 8 x i8> %y, i32 zeroext %vl) {
+; CHECK-LABEL: vpmerge_m1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, mu
+; CHECK-NEXT:    vadd.vi v8, v9, 0
+; CHECK-NEXT:    ret
+  %splat = insertelement <vscale x 8 x i1> poison, i1 -1, i8 0
+  %allones = shufflevector <vscale x 8 x i1> %splat, <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer
+  %1 = call <vscale x 8 x i8> @llvm.vp.merge.nxv8i8(<vscale x 8 x i1> %allones, <vscale x 8 x i8> %y, <vscale x 8 x i8> %x, i32 %vl)
+  ret <vscale x 8 x i8> %1
+}
+
+define <vscale x 8 x i16> @vpmerge_m2(<vscale x 8 x i16> %x, <vscale x 8 x i16> %y, i32 zeroext %vl) {
+; CHECK-LABEL: vpmerge_m2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, mu
+; CHECK-NEXT:    vadd.vi v8, v10, 0
+; CHECK-NEXT:    ret
+  %splat = insertelement <vscale x 8 x i1> poison, i1 -1, i16 0
+  %allones = shufflevector <vscale x 8 x i1> %splat, <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer
+  %1 = call <vscale x 8 x i16> @llvm.vp.merge.nxv8i16(<vscale x 8 x i1> %allones, <vscale x 8 x i16> %y, <vscale x 8 x i16> %x, i32 %vl)
+  ret <vscale x 8 x i16> %1
+}
+
+define <vscale x 8 x i32> @vpmerge_m4(<vscale x 8 x i32> %x, <vscale x 8 x i32> %y, i32 zeroext %vl) {
+; CHECK-LABEL: vpmerge_m4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, mu
+; CHECK-NEXT:    vadd.vi v8, v12, 0
+; CHECK-NEXT:    ret
+  %splat = insertelement <vscale x 8 x i1> poison, i1 -1, i32 0
+  %allones = shufflevector <vscale x 8 x i1> %splat, <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer
+  %1 = call <vscale x 8 x i32> @llvm.vp.merge.nxv8i32(<vscale x 8 x i1> %allones, <vscale x 8 x i32> %y, <vscale x 8 x i32> %x, i32 %vl)
+  ret <vscale x 8 x i32> %1
+}
+
+define <vscale x 8 x i64> @vpmerge_m8(<vscale x 8 x i64> %x, <vscale x 8 x i64> %y, i32 zeroext %vl) {
+; CHECK-LABEL: vpmerge_m8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e64, m8, tu, mu
+; CHECK-NEXT:    vadd.vi v8, v16, 0
+; CHECK-NEXT:    ret
+  %splat = insertelement <vscale x 8 x i1> poison, i1 -1, i64 0
+  %allones = shufflevector <vscale x 8 x i1> %splat, <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer
+  %1 = call <vscale x 8 x i64> @llvm.vp.merge.nxv8i64(<vscale x 8 x i1> %allones, <vscale x 8 x i64> %y, <vscale x 8 x i64> %x, i32 %vl)
+  ret <vscale x 8 x i64> %1
+}
+
+declare <vscale x 1 x i8> @llvm.vp.merge.nxv1i8(<vscale x 1 x i1>, <vscale x 1 x i8>, <vscale x 1 x i8>, i32)
+declare <vscale x 2 x i8> @llvm.vp.merge.nxv2i8(<vscale x 2 x i1>, <vscale x 2 x i8>, <vscale x 2 x i8>, i32)
+declare <vscale x 4 x i8> @llvm.vp.merge.nxv4i8(<vscale x 4 x i1>, <vscale x 4 x i8>, <vscale x 4 x i8>, i32)
+declare <vscale x 8 x i8> @llvm.vp.merge.nxv8i8(<vscale x 8 x i1>, <vscale x 8 x i8>, <vscale x 8 x i8>, i32)
+declare <vscale x 8 x i16> @llvm.vp.merge.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>, i32)
+declare <vscale x 8 x i32> @llvm.vp.merge.nxv8i32(<vscale x 8 x i1>, <vscale x 8 x i32>, <vscale x 8 x i32>, i32)
+declare <vscale x 8 x i64> @llvm.vp.merge.nxv8i64(<vscale x 8 x i1>, <vscale x 8 x i64>, <vscale x 8 x i64>, i32)