[RISCV] Use maskedoff to decide mask policy for masked compare and vmsbf/vmsif/vmsof.

[khchen]

Summary

masked compare and vmsbf/vmsif/vmsof are always tail agnostic, we could
check maskedoff value to decide mask policy rather than have a addtional
policy operand.

[arcbbb]

llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp

546

This change implies that ForceTailAgnostic overrides all other rules.
Maybe we still want to keep the previous logic:
If there is a policy, we follow it.
Otherwise we look at the tied operand if ForceTailAgnostic is unset.

[craig.topper]

llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp

1111

policyt -> policy

llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp

546

ForceTailAgnostic is an instruction property. It shouldn't be set on any instructions that have a policy operand.

[khchen]

Fix typo. Thanks!
It's good to update this chagned soon if it does make sense,
because I change the IR interface again...
I believe changing the IR interface frequently is annoying.

llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp

546

I think this change still keep the original logic because I force tail policy as agnostic if ForceTailAgnostic is true , but has more ability to decide the mask policy according to merge operand is undef or not.
You could see the ForceTailAgnostic is only used in this else if statement to keep the TailAgnostic would not be changed.

[craig.topper]

LGTM with the typo fixed

llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp

1120

assgin -> assign

llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp

#undef CASE_VMXOR_VMANDN_VMOR_OPCODES
 
       // Mask needs to be copied to V0.
       SDValue Chain = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL,
                                            RISCV::V0, Mask, SDValue());
       SDValue Glue = Chain.getValue(1);
       SDValue V0 = CurDAG->getRegister(RISCV::V0, VT);
 
       // Otherwise use
-      // vmslt{u}.vx vd, va, x, v0.t; vmxor.mm vd, vd, v0
+      // vmslt{u}.vx vd, va, x, v0.t; if mask policy is agnostic.
       SDValue Cmp = SDValue(
           CurDAG->getMachineNode(VMSLTMaskOpcode, DL, VT,
                                  {MaskedOff, Src1, Src2, V0, VL, SEW, Glue}),
           0);
+      if (MaskedOff.isUndef()) {
+        ReplaceNode(Node, Cmp.getNode());
+        return;
+      }
+      // Need vmxor.mm vd, vd, v0 to assign inactive value.
       ReplaceNode(Node, CurDAG->getMachineNode(VMXOROpcode, DL, VT,
                                                {Cmp, Mask, VL, MaskSEW}));
       return;
     }
     case Intrinsic::riscv_vsetvli_opt:
     case Intrinsic::riscv_vsetvlimax_opt:
       return selectVSETVLI(Node);
     }

llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp

static VSETVLIInfo computeInfoForInstr(const MachineInstr &MI, uint64_t TSFlags,
                                        const MachineRegisterInfo *MRI) {
   VSETVLIInfo InstrInfo;
   unsigned NumOperands = MI.getNumExplicitOperands();
   bool HasPolicy = RISCVII::hasVecPolicyOp(TSFlags);
 
   // If the instruction has policy argument, use the argument.
   // If there is no policy argument, default to tail agnostic unless the
   // destination is tied to a source. Unless the source is undef. In that case
-  // the user would have some control over the policy values. Some pseudo
-  // instructions force a tail agnostic policy despite having a tied def.
-  bool ForceTailAgnostic = RISCVII::doesForceTailAgnostic(TSFlags);
+  // the user would have some control over the policy values.
   bool TailAgnostic = true;
   bool UsesMaskPolicy = RISCVII::UsesMaskPolicy(TSFlags);
   // FIXME: Could we look at the above or below instructions to choose the
   // matched mask policy to reduce vsetvli instructions? Default mask policy is
   // agnostic if instructions use mask policy, otherwise is undisturbed. Because
   // most mask operations are mask undisturbed, so we could possibly reduce the
   // vsetvli between mask and nomasked instruction sequence.
   bool MaskAgnostic = UsesMaskPolicy;
   unsigned UseOpIdx;
   if (HasPolicy) {
     const MachineOperand &Op = MI.getOperand(MI.getNumExplicitOperands() - 1);
     uint64_t Policy = Op.getImm();
     assert(Policy <= (RISCVII::TAIL_AGNOSTIC | RISCVII::MASK_AGNOSTIC) &&
            "Invalid Policy Value");
     // Although in some cases, mismatched passthru/maskedoff with policy value
     // does not make sense (ex. tied operand is IMPLICIT_DEF with non-TAMA
     // policy, or tied operand is not IMPLICIT_DEF with TAMA policy), but users
     // have set the policy value explicitly, so compiler would not fix it.
     TailAgnostic = Policy & RISCVII::TAIL_AGNOSTIC;
     MaskAgnostic = Policy & RISCVII::MASK_AGNOSTIC;
-  } else if (!ForceTailAgnostic && MI.isRegTiedToUseOperand(0, &UseOpIdx)) {
+  } else if (MI.isRegTiedToUseOperand(0, &UseOpIdx)) {
     TailAgnostic = false;
     if (UsesMaskPolicy)
       MaskAgnostic = false;
     // If the tied operand is an IMPLICIT_DEF we can keep TailAgnostic.
     const MachineOperand &UseMO = MI.getOperand(UseOpIdx);
     MachineInstr *UseMI = MRI->getVRegDef(UseMO.getReg());
     if (UseMI) {
       UseMI = elideCopies(UseMI, MRI);
       if (UseMI && UseMI->isImplicitDef()) {
         TailAgnostic = true;
         if (UsesMaskPolicy)
           MaskAgnostic = true;
       }
     }
+    // Some pseudo instructions force a tail agnostic policy despite having a
+    // tied def.
+    if (RISCVII::doesForceTailAgnostic(TSFlags))
+      TailAgnostic = true;
   }
 
   // Remove the tail policy so we can find the SEW and VL.
   if (HasPolicy)
     --NumOperands;
 
   RISCVII::VLMUL VLMul = RISCVII::getLMul(TSFlags);
 

llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td

class VPseudoBinaryMOutMask<VReg RetClass,
         RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
   let hasSideEffects = 0;
   let Constraints = Join<[Constraint, "$rd = $merge"], ",">.ret;
   let HasVLOp = 1;
   let HasSEWOp = 1;
   let HasMergeOp = 1;
-  // FIXME: In current design, we would not change the mask policy, so
-  // UsesMaskPolicy is false. We could fix after add the policy operand.
-  let UsesMaskPolicy = 0;
+  let UsesMaskPolicy = 1;
   let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst);
 }
 
 // Special version of VPseudoBinaryMask where we pretend the first source is
 // tied to the destination so we can workaround the earlyclobber constraint.
 // This allows maskedoff and rs2 to be the same register.
 class VPseudoTiedBinaryMask<VReg RetClass,
                             DAGOperand Op2Class,

llvm/test/CodeGen/RISCV/rvv/masked-tama.ll

entry:
   %a = call <vscale x 1 x i8> @llvm.riscv.viota.mask.nxv1i8(
     <vscale x 1 x i8> undef,
     <vscale x 1 x i1> %0,
     <vscale x 1 x i1> %1,
     iXLen %2, iXLen 3)
 
   ret <vscale x 1 x i8> %a
 }
+
+declare <vscale x 1 x i1> @llvm.riscv.vmsbf.mask.nxv1i1(
+  <vscale x 1 x i1>,
+  <vscale x 1 x i1>,
+  <vscale x 1 x i1>,
+  iXLen);
+
+define <vscale x 1 x i1> @intrinsic_vmsbf_mask_m_nxv1i1_nxv1i1(<vscale x 1 x i1> %0, <vscale x 1 x i1> %1, iXLen %2) nounwind {
+; CHECK-LABEL: intrinsic_vmsbf_mask_m_nxv1i1_nxv1i1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v9, v0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsbf.m v8, v9, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 1 x i1> @llvm.riscv.vmsbf.mask.nxv1i1(
+    <vscale x 1 x i1> undef,
+    <vscale x 1 x i1> %0,
+    <vscale x 1 x i1> %1,
+    iXLen %2)
+  ret <vscale x 1 x i1> %a
+}
+
+declare <vscale x 1 x i1> @llvm.riscv.vmfeq.mask.nxv1f16(
+  <vscale x 1 x i1>,
+  <vscale x 1 x half>,
+  <vscale x 1 x half>,
+  <vscale x 1 x i1>,
+  iXLen);
+
+declare <vscale x 1 x i1> @llvm.riscv.vmfeq.nxv1f16(
+  <vscale x 1 x half>,
+  <vscale x 1 x half>,
+  iXLen);
+
+define <vscale x 1 x i1> @intrinsic_vmfeq_mask_vv_nxv1f16_nxv1f16(<vscale x 1 x half> %0, <vscale x 1 x half> %1, <vscale x 1 x half> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vmfeq_mask_vv_nxv1f16_nxv1f16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
+; CHECK-NEXT:    vmfeq.vv v0, v9, v10
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vmfeq.vv v0, v9, v10, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %mask = call <vscale x 1 x i1> @llvm.riscv.vmfeq.nxv1f16(
+    <vscale x 1 x half> %1,
+    <vscale x 1 x half> %2,
+    iXLen %3)
+  %a = call <vscale x 1 x i1> @llvm.riscv.vmfeq.mask.nxv1f16(
+    <vscale x 1 x i1> undef,
+    <vscale x 1 x half> %1,
+    <vscale x 1 x half> %2,
+    <vscale x 1 x i1> %mask,
+    iXLen %3)
+
+  ret <vscale x 1 x i1> %a
+}
+
+declare <vscale x 1 x i1> @llvm.riscv.vmseq.mask.nxv1i64.i64(
+  <vscale x 1 x i1>,
+  <vscale x 1 x i64>,
+  i64,
+  <vscale x 1 x i1>,
+  iXLen);
+
+define <vscale x 1 x i1> @intrinsic_vmseq_mask_vx_nxv1i64_i64(<vscale x 1 x i64> %0, i64 %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
+; RV32-LABEL: intrinsic_vmseq_mask_vx_nxv1i64_i64:
+; RV32:       # %bb.0: # %entry
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    sw a1, 12(sp)
+; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
+; RV32-NEXT:    addi a0, sp, 8
+; RV32-NEXT:    vlse64.v v9, (a0), zero
+; RV32-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
+; RV32-NEXT:    vmseq.vv v0, v8, v9, v0.t
+; RV32-NEXT:    addi sp, sp, 16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: intrinsic_vmseq_mask_vx_nxv1i64_i64:
+; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
+; RV64-NEXT:    vmseq.vx v0, v8, a0, v0.t
+; RV64-NEXT:    ret
+entry:
+  %a = call <vscale x 1 x i1> @llvm.riscv.vmseq.mask.nxv1i64.i64(
+    <vscale x 1 x i1> undef,
+    <vscale x 1 x i64> %0,
+    i64 %1,
+    <vscale x 1 x i1> %2,
+    iXLen %3)
+
+  ret <vscale x 1 x i1> %a
+}
+
+declare <vscale x 1 x i1> @llvm.riscv.vmsge.mask.nxv1i64.i64(
+  <vscale x 1 x i1>,
+  <vscale x 1 x i64>,
+  i64,
+  <vscale x 1 x i1>,
+  iXLen);
+
+define <vscale x 1 x i1> @intrinsic_vmsge_mask_vx_nxv1i64_i64(<vscale x 1 x i64> %0, i64 %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
+; RV32-LABEL: intrinsic_vmsge_mask_vx_nxv1i64_i64:
+; RV32:       # %bb.0: # %entry
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    sw a1, 12(sp)
+; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
+; RV32-NEXT:    addi a0, sp, 8
+; RV32-NEXT:    vlse64.v v9, (a0), zero
+; RV32-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
+; RV32-NEXT:    vmsle.vv v0, v9, v8, v0.t
+; RV32-NEXT:    addi sp, sp, 16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: intrinsic_vmsge_mask_vx_nxv1i64_i64:
+; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
+; RV64-NEXT:    vmslt.vx v0, v8, a0, v0.t
+; RV64-NEXT:    ret
+entry:
+  %a = call <vscale x 1 x i1> @llvm.riscv.vmsge.mask.nxv1i64.i64(
+    <vscale x 1 x i1> undef,
+    <vscale x 1 x i64> %0,
+    i64 %1,
+    <vscale x 1 x i1> %2,
+    iXLen %3)
+
+  ret <vscale x 1 x i1> %a
+}
+
+declare <vscale x 64 x i1> @llvm.riscv.vmsbf.mask.nxv64i1(
+  <vscale x 64 x i1>,
+  <vscale x 64 x i1>,
+  <vscale x 64 x i1>,
+  iXLen);
+
+define <vscale x 64 x i1> @intrinsic_vmsbf_mask_m_nxv64i1_nxv64i1(<vscale x 64 x i1> %0, <vscale x 64 x i1> %1, iXLen %2) nounwind {
+; CHECK-LABEL: intrinsic_vmsbf_mask_m_nxv64i1_nxv64i1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v9, v0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmsbf.m v8, v9, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 64 x i1> @llvm.riscv.vmsbf.mask.nxv64i1(
+    <vscale x 64 x i1> undef,
+    <vscale x 64 x i1> %0,
+    <vscale x 64 x i1> %1,
+    iXLen %2)
+
+  ret <vscale x 64 x i1> %a
+}