[AMDGPU] Asm/disasm v_cndmask_b32_e64 with abs/neg source modifiers

[tpr]

Summary

This commit allows v_cndmask_b32_e64 with abs, neg source
modifiers on src0, src1 to be assembled and disassembled.

This does appear to be allowed, even though they are floating point
modifiers and the operand type is b32.

To do this, I added src0_modifiers and src1_modifiers to the
MachineInstr, which involved fixing up several places in codegen and mir
tests.

Change-Id: I69bf4a8c73ebc65744f6110bb8fc4e937d79fbea

[arsenm]

Are you sure these actually do anything? I thought I tried this a long time ago, and found these are ignored by the instruction (along with a bunch of others like mov)

[tpr]

I'll double check.

[arsenm]

If these do work, we do have a lot of code assuming they don't to try moving the modifiers around so they can fold in the inputs/users

[tpr]

The answer I have got from the HW team is that you can use abs and neg modifiers on f32 data in v_cndmask_b32, v_mov_b32 and v_movrel*_b32.

[vpykhtin]

LGTM, Thanks!

llvm/trunk/lib/Target/AMDGPU/SIFoldOperands.cpp

static bool tryFoldInst(const SIInstrInfo *TII,
                         MachineInstr *MI) {
   unsigned Opc = MI->getOpcode();
 
   if (Opc == AMDGPU::V_CNDMASK_B32_e32    ||
       Opc == AMDGPU::V_CNDMASK_B32_e64    ||
       Opc == AMDGPU::V_CNDMASK_B64_PSEUDO) {
     const MachineOperand *Src0 = TII->getNamedOperand(*MI, AMDGPU::OpName::src0);
     const MachineOperand *Src1 = TII->getNamedOperand(*MI, AMDGPU::OpName::src1);
-    if (Src1->isIdenticalTo(*Src0)) {
+    int Src1ModIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1_modifiers);
+    int Src0ModIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers);
+    if (Src1->isIdenticalTo(*Src0) &&
+        (Src1ModIdx == -1 || !MI->getOperand(Src1ModIdx).getImm()) &&
+        (Src0ModIdx == -1 || !MI->getOperand(Src0ModIdx).getImm())) {
       LLVM_DEBUG(dbgs() << "Folded " << *MI << " into ");
+      auto &NewDesc =
+          TII->get(Src0->isReg() ? (unsigned)AMDGPU::COPY : getMovOpc(false));
       int Src2Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
       if (Src2Idx != -1)
         MI->RemoveOperand(Src2Idx);
       MI->RemoveOperand(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1));
-      mutateCopyOp(*MI, TII->get(Src0->isReg() ? (unsigned)AMDGPU::COPY
-                                               : getMovOpc(false)));
+      if (Src1ModIdx != -1)
+        MI->RemoveOperand(Src1ModIdx);
+      if (Src0ModIdx != -1)
+        MI->RemoveOperand(Src0ModIdx);
+      mutateCopyOp(*MI, NewDesc);
       LLVM_DEBUG(dbgs() << *MI << '\n');
       return true;
     }
   }
 
   return false;
 }
 

llvm/trunk/lib/Target/AMDGPU/SIISelLowering.cpp

case AMDGPU::V_CNDMASK_B64_PSEUDO: {
 
     unsigned DstLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
     unsigned DstHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
     unsigned SrcCondCopy = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
 
     BuildMI(*BB, MI, DL, TII->get(AMDGPU::COPY), SrcCondCopy)
       .addReg(SrcCond);
     BuildMI(*BB, MI, DL, TII->get(AMDGPU::V_CNDMASK_B32_e64), DstLo)
+      .addImm(0)
       .addReg(Src0, 0, AMDGPU::sub0)
+      .addImm(0)
       .addReg(Src1, 0, AMDGPU::sub0)
       .addReg(SrcCondCopy);
     BuildMI(*BB, MI, DL, TII->get(AMDGPU::V_CNDMASK_B32_e64), DstHi)
+      .addImm(0)
       .addReg(Src0, 0, AMDGPU::sub1)
+      .addImm(0)
       .addReg(Src1, 0, AMDGPU::sub1)
       .addReg(SrcCondCopy);
 
     BuildMI(*BB, MI, DL, TII->get(AMDGPU::REG_SEQUENCE), Dst)
       .addReg(DstLo)
       .addImm(AMDGPU::sub0)
       .addReg(DstHi)
       .addImm(AMDGPU::sub1);

llvm/trunk/lib/Target/AMDGPU/SIInstrInfo.cpp

void SIInstrInfo::insertVectorSelect(MachineBasicBlock &MBB,
   assert(MRI.getRegClass(DstReg) == &AMDGPU::VGPR_32RegClass &&
          "Not a VGPR32 reg");
 
   if (Cond.size() == 1) {
     unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
     BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
       .add(Cond[0]);
     BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
+      .addImm(0)
       .addReg(FalseReg)
+      .addImm(0)
       .addReg(TrueReg)
       .addReg(SReg);
   } else if (Cond.size() == 2) {
     assert(Cond[0].isImm() && "Cond[0] is not an immediate");
     switch (Cond[0].getImm()) {
     case SIInstrInfo::SCC_TRUE: {
       unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
       BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), SReg)
         .addImm(-1)
         .addImm(0);
       BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
+        .addImm(0)
         .addReg(FalseReg)
+        .addImm(0)
         .addReg(TrueReg)
         .addReg(SReg);
       break;
     }
     case SIInstrInfo::SCC_FALSE: {
       unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
       BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), SReg)
         .addImm(0)
         .addImm(-1);
       BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
+        .addImm(0)
         .addReg(FalseReg)
+        .addImm(0)
         .addReg(TrueReg)
         .addReg(SReg);
       break;
     }
     case SIInstrInfo::VCCNZ: {
       MachineOperand RegOp = Cond[1];
       RegOp.setImplicit(false);
       unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
       BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
         .add(RegOp);
       BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
+          .addImm(0)
           .addReg(FalseReg)
+          .addImm(0)
           .addReg(TrueReg)
           .addReg(SReg);
       break;
     }
     case SIInstrInfo::VCCZ: {
       MachineOperand RegOp = Cond[1];
       RegOp.setImplicit(false);
       unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
       BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
         .add(RegOp);
       BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
+          .addImm(0)
           .addReg(TrueReg)
+          .addImm(0)
           .addReg(FalseReg)
           .addReg(SReg);
       break;
     }
     case SIInstrInfo::EXECNZ: {
       unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
       unsigned SReg2 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
       BuildMI(MBB, I, DL, get(AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
         .addImm(0);
       BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), SReg)
         .addImm(-1)
         .addImm(0);
       BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
+        .addImm(0)
         .addReg(FalseReg)
+        .addImm(0)
         .addReg(TrueReg)
         .addReg(SReg);
       break;
     }
     case SIInstrInfo::EXECZ: {
       unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
       unsigned SReg2 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
       BuildMI(MBB, I, DL, get(AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
         .addImm(0);
       BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), SReg)
         .addImm(0)
         .addImm(-1);
       BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
+        .addImm(0)
         .addReg(FalseReg)
+        .addImm(0)
         .addReg(TrueReg)
         .addReg(SReg);
       llvm_unreachable("Unhandled branch predicate EXECZ");
       break;
     }
     default:
       llvm_unreachable("invalid branch predicate");
     }

 }
 
 bool SIInstrInfo::canShrink(const MachineInstr &MI,
                             const MachineRegisterInfo &MRI) const {
   const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
   // Can't shrink instruction with three operands.
   // FIXME: v_cndmask_b32 has 3 operands and is shrinkable, but we need to add
   // a special case for it.  It can only be shrunk if the third operand
-  // is vcc.  We should handle this the same way we handle vopc, by addding
+  // is vcc, and src0_modifiers and src1_modifiers are not set.
+  // We should handle this the same way we handle vopc, by addding
   // a register allocation hint pre-regalloc and then do the shrinking
   // post-regalloc.
   if (Src2) {
     switch (MI.getOpcode()) {
       default: return false;
 
       case AMDGPU::V_ADDC_U32_e64:
       case AMDGPU::V_SUBB_U32_e64:

llvm/trunk/lib/Target/AMDGPU/SIInstrInfo.td

bit ret =
     !if(!eq(SrcVT.Value, f32.Value), 1,
     !if(!eq(SrcVT.Value, f64.Value), 1,
     !if(!eq(SrcVT.Value, v2f16.Value), 1,
     !if(!eq(SrcVT.Value, v2i16.Value), 1,
     0)))));
 }
 
 // Return type of input modifiers operand for specified input operand
-class getSrcMod <ValueType VT> {
+class getSrcMod <ValueType VT, bit EnableF32SrcMods> {
   bit isFP = !if(!eq(VT.Value, f16.Value), 1,
                !if(!eq(VT.Value, f32.Value), 1,
                !if(!eq(VT.Value, f64.Value), 1,
                0)));
   bit isPacked = isPackedType<VT>.ret;
   Operand ret =  !if(!eq(VT.Size, 64),
                      !if(isFP, FP64InputMods, Int64InputMods),
                        !if(isFP,
                          !if(!eq(VT.Value, f16.Value),
                             FP16InputMods,
                             FP32InputMods
                           ),
-                         Int32InputMods)
+                         !if(EnableF32SrcMods, FP32InputMods, Int32InputMods))
                      );
 }
 
 class getOpSelMod <ValueType VT> {
   Operand ret = !if(!eq(VT.Value, f16.Value), FP16InputMods, IntOpSelMods);
 }
 
 // Return type of input modifiers operand specified input operand for DPP
class getIns32 <RegisterOperand Src0RC, RegisterClass Src1RC, int NumSrcArgs> {
   dag ret = !if(!eq(NumSrcArgs, 1), (ins Src0RC:$src0),               // VOP1
             !if(!eq(NumSrcArgs, 2), (ins Src0RC:$src0, Src1RC:$src1), // VOP2
                                     (ins)));
 }
 
 // Returns the input arguments for VOP3 instructions for the given SrcVT.
 class getIns64 <RegisterOperand Src0RC, RegisterOperand Src1RC,
                 RegisterOperand Src2RC, int NumSrcArgs,
-                bit HasIntClamp, bit HasModifiers, bit HasOMod,
+                bit HasIntClamp, bit HasModifiers, bit HasSrc2Mods, bit HasOMod,
                 Operand Src0Mod, Operand Src1Mod, Operand Src2Mod> {
 
   dag ret =
     !if (!eq(NumSrcArgs, 0),
       // VOP1 without input operands (V_NOP, V_CLREXCP)
       (ins),
       /* else */
     !if (!eq(NumSrcArgs, 1),
!if (!eq(NumSrcArgs, 2),
         // VOP2 without modifiers
         !if (!eq(HasIntClamp, 1),
           (ins Src0RC:$src0, Src1RC:$src1, clampmod:$clamp),
           (ins Src0RC:$src0, Src1RC:$src1))
 
       /* endif */ )
     /* NumSrcArgs == 3 */,
       !if (!eq(HasModifiers, 1),
+        !if (!eq(HasSrc2Mods, 1),
           // VOP3 with modifiers
           !if (!eq(HasOMod, 1),
             (ins Src0Mod:$src0_modifiers, Src0RC:$src0,
                  Src1Mod:$src1_modifiers, Src1RC:$src1,
                  Src2Mod:$src2_modifiers, Src2RC:$src2,
                  clampmod:$clamp, omod:$omod),
+            !if (!eq(HasIntClamp, 1),
               (ins Src0Mod:$src0_modifiers, Src0RC:$src0,
                    Src1Mod:$src1_modifiers, Src1RC:$src1,
                    Src2Mod:$src2_modifiers, Src2RC:$src2,
-               clampmod:$clamp))
+                   clampmod:$clamp),
+              (ins Src0Mod:$src0_modifiers, Src0RC:$src0,
+                   Src1Mod:$src1_modifiers, Src1RC:$src1,
+                   Src2Mod:$src2_modifiers, Src2RC:$src2))),
+          // VOP3 with modifiers except src2
+          !if (!eq(HasOMod, 1),
+            (ins Src0Mod:$src0_modifiers, Src0RC:$src0,
+                 Src1Mod:$src1_modifiers, Src1RC:$src1,
+                 Src2RC:$src2, clampmod:$clamp, omod:$omod),
+            !if (!eq(HasIntClamp, 1),
+              (ins Src0Mod:$src0_modifiers, Src0RC:$src0,
+                   Src1Mod:$src1_modifiers, Src1RC:$src1,
+                   Src2RC:$src2, clampmod:$clamp),
+              (ins Src0Mod:$src0_modifiers, Src0RC:$src0,
+                   Src1Mod:$src1_modifiers, Src1RC:$src1,
+                   Src2RC:$src2))))
       /* else */,
         // VOP3 without modifiers
         !if (!eq(HasIntClamp, 1),
           (ins Src0RC:$src0, Src1RC:$src1, Src2RC:$src2, clampmod:$clamp),
           (ins Src0RC:$src0, Src1RC:$src1, Src2RC:$src2))
       /* endif */ ))));
 }
 
class BitAnd<bit a, bit b> {
   bit ret = !if(a, !if(b, 1, 0), 0);
 }
 
 def PatGenMode {
   int NoPattern = 0;
   int Pattern   = 1;
 }
 
-class VOPProfile <list<ValueType> _ArgVT> {
+class VOPProfile <list<ValueType> _ArgVT, bit _EnableF32SrcMods = 0> {
 
   field list<ValueType> ArgVT = _ArgVT;
+  field bit EnableF32SrcMods = _EnableF32SrcMods;
 
   field ValueType DstVT = ArgVT[0];
   field ValueType Src0VT = ArgVT[1];
   field ValueType Src1VT = ArgVT[2];
   field ValueType Src2VT = ArgVT[3];
   field RegisterOperand DstRC = getVALUDstForVT<DstVT>.ret;
   field RegisterOperand DstRCDPP = getVALUDstForVT<DstVT>.ret;
   field RegisterOperand DstRCSDWA = getSDWADstForVT<DstVT>.ret;
   field RegisterOperand Src0RC32 = getVOPSrc0ForVT<Src0VT>.ret;
   field RegisterClass Src1RC32 = getVregSrcForVT<Src1VT>.ret;
   field RegisterOperand Src0RC64 = getVOP3SrcForVT<Src0VT>.ret;
   field RegisterOperand Src1RC64 = getVOP3SrcForVT<Src1VT>.ret;
   field RegisterOperand Src2RC64 = getVOP3SrcForVT<Src2VT>.ret;
   field RegisterClass Src0DPP = getVregSrcForVT<Src0VT>.ret;
   field RegisterClass Src1DPP = getVregSrcForVT<Src1VT>.ret;
   field RegisterOperand Src0SDWA = getSDWASrcForVT<Src0VT>.ret;
   field RegisterOperand Src1SDWA = getSDWASrcForVT<Src0VT>.ret;
-  field Operand Src0Mod = getSrcMod<Src0VT>.ret;
-  field Operand Src1Mod = getSrcMod<Src1VT>.ret;
-  field Operand Src2Mod = getSrcMod<Src2VT>.ret;
+  field Operand Src0Mod = getSrcMod<Src0VT, EnableF32SrcMods>.ret;
+  field Operand Src1Mod = getSrcMod<Src1VT, EnableF32SrcMods>.ret;
+  field Operand Src2Mod = getSrcMod<Src2VT, EnableF32SrcMods>.ret;
   field Operand Src0ModDPP = getSrcModExt<Src0VT>.ret;
   field Operand Src1ModDPP = getSrcModExt<Src1VT>.ret;
   field Operand Src0ModSDWA = getSrcModSDWA<Src0VT>.ret;
   field Operand Src1ModSDWA = getSrcModSDWA<Src1VT>.ret;
 
 
   field bit HasDst = !if(!eq(DstVT.Value, untyped.Value), 0, 1);
   field bit HasDst32 = HasDst;
   field bit EmitDst = HasDst; // force dst encoding, see v_movreld_b32 special case
   field int NumSrcArgs = getNumSrcArgs<Src0VT, Src1VT, Src2VT>.ret;
   field bit HasSrc0 = !if(!eq(Src0VT.Value, untyped.Value), 0, 1);
   field bit HasSrc1 = !if(!eq(Src1VT.Value, untyped.Value), 0, 1);
   field bit HasSrc2 = !if(!eq(Src2VT.Value, untyped.Value), 0, 1);
 
   // TODO: Modifiers logic is somewhat adhoc here, to be refined later
-  field bit HasModifiers = isModifierType<Src0VT>.ret;
+  // HasModifiers affects the normal and DPP encodings. We take note of EnableF32SrcMods, which
+  // enables modifiers for i32 type.
+  field bit HasModifiers = BitOr<isModifierType<Src0VT>.ret, EnableF32SrcMods>.ret;
 
+  // HasSrc*FloatMods affects the SDWA encoding. We ignore EnableF32SrcMods.
   field bit HasSrc0FloatMods = isFloatType<Src0VT>.ret;
   field bit HasSrc1FloatMods = isFloatType<Src1VT>.ret;
   field bit HasSrc2FloatMods = isFloatType<Src2VT>.ret;
 
+  // HasSrc*IntMods affects the SDWA encoding. We ignore EnableF32SrcMods.
   field bit HasSrc0IntMods = isIntType<Src0VT>.ret;
   field bit HasSrc1IntMods = isIntType<Src1VT>.ret;
   field bit HasSrc2IntMods = isIntType<Src2VT>.ret;
 
   field bit HasSrc0Mods = HasModifiers;
   field bit HasSrc1Mods = !if(HasModifiers, BitOr<HasSrc1FloatMods, HasSrc1IntMods>.ret, 0);
   field bit HasSrc2Mods = !if(HasModifiers, BitOr<HasSrc2FloatMods, HasSrc2IntMods>.ret, 0);
 
-  field bit HasClamp = HasModifiers;
+  field bit HasClamp = isModifierType<Src0VT>.ret;
   field bit HasSDWAClamp = EmitDst;
   field bit HasFPClamp = BitAnd<isFloatType<DstVT>.ret, HasClamp>.ret;
   field bit HasIntClamp = !if(isFloatType<DstVT>.ret, 0, HasClamp);
   field bit HasClampLo = HasClamp;
   field bit HasClampHi = BitAnd<isPackedType<DstVT>.ret, HasClamp>.ret;
   field bit HasHigh = 0;
 
   field bit IsPacked = isPackedType<Src0VT>.ret;
class VOPProfile <list<ValueType> _ArgVT, bit _EnableF32SrcMods = 0> {
   // output. This is manually overridden for them.
   field dag Outs32 = Outs;
   field dag Outs64 = Outs;
   field dag OutsDPP = getOutsExt<HasDst, DstVT, DstRCDPP>.ret;
   field dag OutsSDWA = getOutsSDWA<HasDst, DstVT, DstRCSDWA>.ret;
 
   field dag Ins32 = getIns32<Src0RC32, Src1RC32, NumSrcArgs>.ret;
   field dag Ins64 = getIns64<Src0RC64, Src1RC64, Src2RC64, NumSrcArgs,
-                             HasIntClamp, HasModifiers, HasOMod, Src0Mod, Src1Mod,
-                             Src2Mod>.ret;
+                             HasIntClamp, HasModifiers, HasSrc2Mods,
+                             HasOMod, Src0Mod, Src1Mod, Src2Mod>.ret;
   field dag InsVOP3P = getInsVOP3P<Src0RC64, Src1RC64, Src2RC64,
                                    NumSrcArgs, HasClamp,
                                    Src0PackedMod, Src1PackedMod, Src2PackedMod>.ret;
   field dag InsVOP3OpSel = getInsVOP3OpSel<Src0RC64, Src1RC64, Src2RC64,
                                            NumSrcArgs,
                                            HasClamp,
                                            getOpSelMod<Src0VT>.ret,
                                            getOpSelMod<Src1VT>.ret,

llvm/trunk/lib/Target/AMDGPU/SIInstructions.td

 def : FMADModsPat<V_MAD_F32, AMDGPUfmad_ftz, f32>;
 def : FMADModsPat<V_MAD_F16, AMDGPUfmad_ftz, f16> {
   let SubtargetPredicate = Has16BitInsts;
 }
 
 multiclass SelectPat <ValueType vt, Instruction inst> {
   def : GCNPat <
     (vt (select i1:$src0, vt:$src1, vt:$src2)),
-    (inst $src2, $src1, $src0)
+    (inst (i32 0), $src2, (i32 0), $src1, $src0)
   >;
 }
 
 defm : SelectPat <i16, V_CNDMASK_B32_e64>;
 defm : SelectPat <i32, V_CNDMASK_B32_e64>;
 defm : SelectPat <f16, V_CNDMASK_B32_e64>;
 defm : SelectPat <f32, V_CNDMASK_B32_e64>;
 

 /********** ================== **********/
 /********** Intrinsic Patterns **********/
 /********** ================== **********/
 
 def : POW_Common <V_LOG_F32_e32, V_EXP_F32_e32, V_MUL_LEGACY_F32_e32>;
 
 def : GCNPat <
   (i32 (sext i1:$src0)),
-  (V_CNDMASK_B32_e64 (i32 0), (i32 -1), $src0)
+  (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),
+                     /*src1mod*/(i32 0), /*src1*/(i32 -1), $src0)
 >;
 
 class Ext32Pat <SDNode ext> : GCNPat <
   (i32 (ext i1:$src0)),
-  (V_CNDMASK_B32_e64 (i32 0), (i32 1), $src0)
+  (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),
+                     /*src1mod*/(i32 0), /*src1*/(i32 1), $src0)
 >;
 
 def : Ext32Pat <zext>;
 def : Ext32Pat <anyext>;
 
 // The multiplication scales from [0,1] to the unsigned integer range
 def : GCNPat <
   (AMDGPUurecip i32:$src0),

 def : GCNPat <
   (i64 (anyext i32:$src)),
   (REG_SEQUENCE SReg_64, $src, sub0, (i32 (IMPLICIT_DEF)), sub1)
 >;
 
 class ZExt_i64_i1_Pat <SDNode ext> : GCNPat <
   (i64 (ext i1:$src)),
     (REG_SEQUENCE VReg_64,
-      (V_CNDMASK_B32_e64 (i32 0), (i32 1), $src), sub0,
-      (S_MOV_B32 (i32 0)), sub1)
+      (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),
+                         /*src1mod*/(i32 0), /*src1*/(i32 1), $src),
+      sub0, (S_MOV_B32 (i32 0)), sub1)
 >;
 
 
 def : ZExt_i64_i1_Pat<zext>;
 def : ZExt_i64_i1_Pat<anyext>;
 
 // FIXME: We need to use COPY_TO_REGCLASS to work-around the fact that
 // REG_SEQUENCE patterns don't support instructions with multiple outputs.
 def : GCNPat <
   (i64 (sext i32:$src)),
     (REG_SEQUENCE SReg_64, $src, sub0,
     (i32 (COPY_TO_REGCLASS (S_ASHR_I32 $src, (i32 31)), SReg_32_XM0)), sub1)
 >;
 
 def : GCNPat <
   (i64 (sext i1:$src)),
   (REG_SEQUENCE VReg_64,
-    (V_CNDMASK_B32_e64 (i32 0), (i32 -1), $src), sub0,
-    (V_CNDMASK_B32_e64 (i32 0), (i32 -1), $src), sub1)
+    (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),
+                       /*src1mod*/(i32 0), /*src1*/(i32 -1), $src), sub0,
+    (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),
+                       /*src1mod*/(i32 0), /*src1*/(i32 -1), $src), sub1)
 >;
 
 class FPToI1Pat<Instruction Inst, int KOne, ValueType kone_type, ValueType vt, SDPatternOperator fp_to_int> : GCNPat <
   (i1 (fp_to_int (vt (VOP3Mods vt:$src0, i32:$src0_modifiers)))),
   (i1 (Inst 0, (kone_type KOne), $src0_modifiers, $src0, DSTCLAMP.NONE))
 >;
 
 def : FPToI1Pat<V_CMP_EQ_F32_e64, CONST.FP32_ONE, i32, f32, fp_to_uint>;

 def : GCNPat <
   (i1 (sub i1:$src0, (i1 -1))),
   (S_NOT_B64 $src0)
 >;
 }
 
 def : GCNPat <
   (f16 (sint_to_fp i1:$src)),
-  (V_CVT_F16_F32_e32 (V_CNDMASK_B32_e64 (i32 0), (i32 CONST.FP32_NEG_ONE), $src))
+  (V_CVT_F16_F32_e32 (
+      V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),
+                        /*src1mod*/(i32 0), /*src1*/(i32 CONST.FP32_NEG_ONE),
+                        $src))
 >;
 
 def : GCNPat <
   (f16 (uint_to_fp i1:$src)),
-  (V_CVT_F16_F32_e32 (V_CNDMASK_B32_e64 (i32 0), (i32 CONST.FP32_ONE), $src))
+  (V_CVT_F16_F32_e32 (
+      V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),
+                        /*src1mod*/(i32 0), /*src1*/(i32 CONST.FP32_ONE),
+                        $src))
 >;
 
 def : GCNPat <
   (f32 (sint_to_fp i1:$src)),
-  (V_CNDMASK_B32_e64 (i32 0), (i32 CONST.FP32_NEG_ONE), $src)
+  (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),
+                        /*src1mod*/(i32 0), /*src1*/(i32 CONST.FP32_NEG_ONE),
+                        $src)
 >;
 
 def : GCNPat <
   (f32 (uint_to_fp i1:$src)),
-  (V_CNDMASK_B32_e64 (i32 0), (i32 CONST.FP32_ONE), $src)
+  (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),
+                        /*src1mod*/(i32 0), /*src1*/(i32 CONST.FP32_ONE),
+                        $src)
 >;
 
 def : GCNPat <
   (f64 (sint_to_fp i1:$src)),
-  (V_CVT_F64_I32_e32 (V_CNDMASK_B32_e64 (i32 0), (i32 -1), $src))
+  (V_CVT_F64_I32_e32 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),
+                                        /*src1mod*/(i32 0), /*src1*/(i32 -1),
+                                        $src))
 >;
 
 def : GCNPat <
   (f64 (uint_to_fp i1:$src)),
-  (V_CVT_F64_U32_e32 (V_CNDMASK_B32_e64 (i32 0), (i32 1), $src))
+  (V_CVT_F64_U32_e32 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),
+                                        /*src1mod*/(i32 0), /*src1*/(i32 1),
+                                        $src))
 >;
 
 //===----------------------------------------------------------------------===//
 // Miscellaneous Patterns
 //===----------------------------------------------------------------------===//
 def : GCNPat <
   (i32 (AMDGPUfp16_zext f16:$src)),
   (COPY $src)

llvm/trunk/lib/Target/AMDGPU/SILowerI1Copies.cpp

for (MachineInstr &MI : MBB) {
       DebugLoc DL = MI.getDebugLoc();
 
       assert(TII->getRegisterInfo().getRegSizeInBits(DstReg, *MRI) == 32);
       assert(!MI.getOperand(0).getSubReg());
 
       ConstrainRegs.insert(SrcReg);
       BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
           .addImm(0)
+          .addImm(0)
+          .addImm(0)
           .addImm(-1)
           .addReg(SrcReg);
       DeadCopies.push_back(&MI);
     }
 
     for (MachineInstr *MI : DeadCopies)
       MI->eraseFromParent();
     DeadCopies.clear();

llvm/trunk/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp

static unsigned optimizeVcndVcmpPair(MachineBasicBlock &MBB,
   if (!Op1->isReg() || !Op2->isImm() || Op2->getImm() != 1)
     return AMDGPU::NoRegister;
 
   unsigned SelReg = Op1->getReg();
   auto *Sel = TRI->findReachingDef(SelReg, Op1->getSubReg(), *Cmp, MRI, LIS);
   if (!Sel || Sel->getOpcode() != AMDGPU::V_CNDMASK_B32_e64)
     return AMDGPU::NoRegister;
 
+  if (TII->hasModifiersSet(*Sel, AMDGPU::OpName::src0_modifiers) ||
+      TII->hasModifiersSet(*Sel, AMDGPU::OpName::src0_modifiers))
+    return AMDGPU::NoRegister;
+
   Op1 = TII->getNamedOperand(*Sel, AMDGPU::OpName::src0);
   Op2 = TII->getNamedOperand(*Sel, AMDGPU::OpName::src1);
   MachineOperand *CC = TII->getNamedOperand(*Sel, AMDGPU::OpName::src2);
   if (!Op1->isImm() || !Op2->isImm() || !CC->isReg() ||
       Op1->getImm() != 0 || Op2->getImm() != 1)
     return AMDGPU::NoRegister;
 
   LLVM_DEBUG(dbgs() << "Folding sequence:\n\t" << *Sel << '\t'

llvm/trunk/lib/Target/AMDGPU/VOP2Instructions.td

 def VOP_MADMK_F16 : VOP_MADMK <f16>;
 def VOP_MADMK_F32 : VOP_MADMK <f32>;
 
 // FIXME: Remove src2_modifiers. It isn't used, so is wasting memory
 // and processing time but it makes it easier to convert to mad.
 class VOP_MAC <ValueType vt> : VOPProfile <[vt, vt, vt, vt]> {
   let Ins32 = (ins Src0RC32:$src0, Src1RC32:$src1, VGPR_32:$src2);
   let Ins64 = getIns64<Src0RC64, Src1RC64, RegisterOperand<VGPR_32>, 3,
-                       0, HasModifiers, HasOMod, Src0Mod, Src1Mod, Src2Mod>.ret;
+                       0, HasModifiers, HasModifiers, HasOMod,
+                       Src0Mod, Src1Mod, Src2Mod>.ret;
   let InsDPP = (ins Src0ModDPP:$src0_modifiers, Src0DPP:$src0,
                     Src1ModDPP:$src1_modifiers, Src1DPP:$src1,
                     VGPR_32:$src2, // stub argument
                     dpp_ctrl:$dpp_ctrl, row_mask:$row_mask,
                     bank_mask:$bank_mask, bound_ctrl:$bound_ctrl);
 
   let InsSDWA = (ins Src0ModSDWA:$src0_modifiers, Src0SDWA:$src0,
                      Src1ModSDWA:$src1_modifiers, Src1SDWA:$src1,
let InsDPP = (ins DstRCDPP:$old,
                     dpp_ctrl:$dpp_ctrl, row_mask:$row_mask,
                     bank_mask:$bank_mask, bound_ctrl:$bound_ctrl);
   let HasExt = 1;
   let HasExtDPP = 1;
   let HasExtSDWA = 1;
   let HasExtSDWA9 = 1;
 }
 
-// Read in from vcc or arbitrary SGPR
-def VOP2e_I32_I32_I32_I1 : VOPProfile<[i32, i32, i32, i1]> {
+// Read in from vcc or arbitrary SGPR.
+// Enable f32 source modifiers on i32 input type.
+def VOP2e_I32_I32_I32_I1 : VOPProfile<[i32, i32, i32, i1], /*EnableF32SrcMods=*/1> {
   let Src0RC32 = VCSrc_b32; // See comment in def VOP2b_I32_I1_I32_I32_I1 above.
   let Asm32 = "$vdst, $src0, $src1, vcc";
-  let Asm64 = "$vdst, $src0, $src1, $src2";
+  let Asm64 = "$vdst, $src0_modifiers, $src1_modifiers, $src2";
   let AsmSDWA = "$vdst, $src0_modifiers, $src1_modifiers, vcc $clamp $dst_sel $dst_unused $src0_sel $src1_sel";
   let AsmSDWA9 = "$vdst, $src0_modifiers, $src1_modifiers, vcc $clamp $dst_sel $dst_unused $src0_sel $src1_sel";
   let AsmDPP = "$vdst, $src0, $src1, vcc $dpp_ctrl$row_mask$bank_mask$bound_ctrl";
 
   let Outs32 = (outs DstRC:$vdst);
   let Outs64 = (outs DstRC:$vdst);
 
   // Suppress src2 implied by type since the 32-bit encoding uses an
   // implicit VCC use.
   let Ins32 = (ins Src0RC32:$src0, Src1RC32:$src1);
 
   let InsSDWA = (ins Src0ModSDWA:$src0_modifiers, Src0SDWA:$src0,
                      Src1ModSDWA:$src1_modifiers, Src1SDWA:$src1,
                      clampmod:$clamp,
                      dst_sel:$dst_sel, dst_unused:$dst_unused,
                      src0_sel:$src0_sel, src1_sel:$src1_sel);
 
   let InsDPP = (ins DstRCDPP:$old,
-                    Src0DPP:$src0,
-                    Src1DPP:$src1,
+                    Src0ModDPP:$src0_modifiers, Src0DPP:$src0,
+                    Src1ModDPP:$src1_modifiers, Src1DPP:$src1,
                     dpp_ctrl:$dpp_ctrl, row_mask:$row_mask,
                     bank_mask:$bank_mask, bound_ctrl:$bound_ctrl);
   let HasExt = 1;
   let HasExtDPP = 1;
   let HasExtSDWA = 1;
   let HasExtSDWA9 = 1;
 }
 
def : GCNPat<
    (REG_SEQUENCE VReg_64,
      (inst $src1, $src0), sub0,
      (V_MOV_B32_e32 (i32 0)), sub1)
 >;
 }
 
 class ZExt_i16_i1_Pat <SDNode ext> : GCNPat <
   (i16 (ext i1:$src)),
-  (V_CNDMASK_B32_e64 (i32 0), (i32 1), $src)
+  (V_CNDMASK_B32_e64 (i32 0/*src0mod*/), (i32 0/*src0*/),
+                     (i32 0/*src1mod*/), (i32 1/*src1*/),
+                     $src)
 >;
 
 let Predicates = [Has16BitInsts] in {
 
 defm : Arithmetic_i16_Pats<add, V_ADD_U16_e64>;
 defm : Arithmetic_i16_Pats<mul, V_MUL_LO_U16_e64>;
 defm : Arithmetic_i16_Pats<sub, V_SUB_U16_e64>;
 defm : Arithmetic_i16_Pats<smin, V_MIN_I16_e64>;

 defm : Bits_OpsRev_i16_Pats<srl, V_LSHRREV_B16_e64>;
 defm : Bits_OpsRev_i16_Pats<sra, V_ASHRREV_I16_e64>;
 
 def : ZExt_i16_i1_Pat<zext>;
 def : ZExt_i16_i1_Pat<anyext>;
 
 def : GCNPat <
   (i16 (sext i1:$src)),
-  (V_CNDMASK_B32_e64 (i32 0), (i32 -1), $src)
+  (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),
+                     /*src1mod*/(i32 0), /*src1*/(i32 -1), $src)
 >;
 
 // Undo sub x, c -> add x, -c canonicalization since c is more likely
 // an inline immediate than -c.
 // TODO: Also do for 64-bit.
 def : GCNPat<
   (add i16:$src0, (i16 NegSubInlineConst16:$src1)),
   (V_SUB_U16_e64 $src0, NegSubInlineConst16:$src1)

llvm/trunk/lib/Target/AMDGPU/VOP3Instructions.td

" " # !if(Features.HasOpSel,
                            HasModifiers, HasOMod, DstVT>.ret,
                   P.Asm64));
   let NeedPatGen = P.NeedPatGen;
 }
 
 class VOP3b_Profile<ValueType vt> : VOPProfile<[vt, vt, vt, vt]> {
   // v_div_scale_{f32|f64} do not support input modifiers.
   let HasModifiers = 0;
+  let HasClamp = 0;
   let HasOMod = 0;
   let Outs64 = (outs DstRC:$vdst, SReg_64:$sdst);
   let Asm64 = " $vdst, $sdst, $src0, $src1, $src2";
 }
 
 def VOP3b_F32_I1_F32_F32_F32 : VOP3b_Profile<f32> {
   // FIXME: Hack to stop printing _e64
   let DstRC = RegisterOperand<VGPR_32>;

llvm/trunk/test/CodeGen/AMDGPU/coalescer-extend-pruned-subrange.mir

bb.0:
     %10:vreg_128 = COPY %6
     S_BRANCH %bb.2
 
   bb.1:
     %11:vgpr_32 = V_OR_B32_e32 %12.sub0, %12.sub1, implicit $exec
     %13:vgpr_32 = V_OR_B32_e32 %11, %12.sub2, implicit $exec
     %14:vgpr_32 = V_AND_B32_e32 1, %13, implicit $exec
     %15:sreg_64_xexec = V_CMP_EQ_U32_e64 0, %14, implicit $exec
-    %16:vgpr_32 = V_CNDMASK_B32_e64 0, 1, %15, implicit $exec
+    %16:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, %15, implicit $exec
     BUFFER_STORE_DWORD_OFFEN_exact %16, undef %17:vgpr_32, undef %18:sreg_128, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into constant-pool, align 1, addrspace 4)
     S_ENDPGM 0
 
   bb.2:
     successors: %bb.3, %bb.4
     %19:sreg_64 = V_CMP_EQ_U32_e64 1, %7, implicit $exec
     %20:sreg_64 = COPY $exec, implicit-def $exec
     %21:sreg_64 = S_AND_B64 %20, %19, implicit-def dead $scc
bb.7:
     $exec = S_MOV_B64_term %30
     SI_MASK_BRANCH %bb.10, implicit $exec
     S_BRANCH %bb.8
 
   bb.8:
     successors: %bb.10
     %31:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN undef %32:vgpr_32, undef %33:sreg_128, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4 from constant-pool, align 1, addrspace 4)
     %34:sreg_64_xexec = V_CMP_NE_U32_e64 0, %31, implicit $exec
-    %35:vgpr_32 = V_CNDMASK_B32_e64 0, -1, %34, implicit $exec
+    %35:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, -1, %34, implicit $exec
     %28:vreg_1 = COPY %35
     S_BRANCH %bb.10
 
   bb.9:
     successors: %bb.11
     S_BRANCH %bb.11
 
   bb.10:
     successors: %bb.9
     $exec = S_OR_B64 $exec, %29, implicit-def $scc
     %36:vreg_1 = COPY %28
     %37:sreg_64_xexec = V_CMP_NE_U32_e64 0, %36, implicit $exec
-    %38:vgpr_32 = V_CNDMASK_B32_e64 0, 1, %37, implicit $exec
+    %38:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, %37, implicit $exec
     %39:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
     undef %40.sub0:vreg_128 = COPY %39
     %40.sub1:vreg_128 = COPY %39
     %40.sub2:vreg_128 = COPY %38
     %41:vreg_128 = COPY %40
     %12:vreg_128 = COPY %41
     S_BRANCH %bb.9
 

llvm/trunk/test/CodeGen/AMDGPU/coalescer-subranges-another-prune-error.mir

bb.4:
     %24:sreg_64 = COPY killed %16
     %25:vgpr_32 = V_OR_B32_e32 %22, %11, implicit $exec
     %26:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN killed %25, undef %27:sreg_128, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from constant-pool, align 1, addrspace 4)
     %28:vgpr_32 = V_LSHRREV_B32_e32 30, killed %26.sub0, implicit $exec
     %29:vreg_128 = COPY killed %21
     %29.sub0:vreg_128 = COPY %1
     %30:sreg_64 = V_CMP_NE_U32_e64 0, %28, implicit $exec
     %31:sreg_64_xexec = V_CMP_EQ_U32_e64 0, %28, implicit $exec
-    dead %32:vgpr_32 = V_CNDMASK_B32_e64 0, -1, killed %31, implicit $exec
+    dead %32:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, -1, killed %31, implicit $exec
     %33:vreg_128 = COPY %29
     %33.sub1:vreg_128 = COPY undef %32
     %34:vgpr_32 = V_MOV_B32_e32 -1, implicit $exec
     %35:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
     %36:sreg_64 = COPY %24
     %37:vreg_128 = COPY %23
     %38:vreg_128 = IMPLICIT_DEF
     %39:vreg_128 = IMPLICIT_DEF

llvm/trunk/test/CodeGen/AMDGPU/coalescer-with-subregs-bad-identical.mir

bb.3:
     S_BRANCH %bb.6
 
   bb.4:
     successors: %bb.5(0x40000000), %bb.7(0x40000000)
     %17:vgpr_32 = V_MAD_F32 0, killed %9, 0, 0, 0, 0, 0, 0, implicit $exec
     %18:vgpr_32 = V_MIN_F32_e32 1065353216, killed %17, implicit $exec
     %19:sreg_64_xexec = V_CMP_NEQ_F32_e64 0, 1065353216, 0, killed %18, 0, implicit $exec
     %20:vgpr_32 = V_MOV_B32_e32 2143289344, implicit $exec
-    %21:vgpr_32 = V_CNDMASK_B32_e64 0, killed %20, killed %19, implicit $exec
+    %21:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, killed %20, killed %19, implicit $exec
     %22:sreg_64 = V_CMP_LT_F32_e64 0, 0, 0, killed %21, 0, implicit $exec
     %23:sreg_64 = COPY $exec, implicit-def $exec
     %24:sreg_64 = S_AND_B64 %23, %22, implicit-def dead $scc
     $exec = S_MOV_B64_term killed %24
     SI_MASK_BRANCH %bb.7, implicit $exec
     S_BRANCH %bb.5
 
   bb.5:

llvm/trunk/test/CodeGen/AMDGPU/fold-cndmask.mir

registers:
   - { id: 3, class: vgpr_32 }
   - { id: 4, class: vgpr_32 }
   - { id: 5, class: vgpr_32 }
   - { id: 6, class: vgpr_32 }
   - { id: 7, class: vgpr_32 }
 body:             |
   bb.0.entry:
     %0 = IMPLICIT_DEF
-    %1 = V_CNDMASK_B32_e64 0, 0, %0, implicit $exec
-    %2 = V_CNDMASK_B32_e64 %1, %1, %0, implicit $exec
+    %1 = V_CNDMASK_B32_e64 0, 0, 0, 0, %0, implicit $exec
+    %2 = V_CNDMASK_B32_e64 0, %1, 0, %1, %0, implicit $exec
     %3 = IMPLICIT_DEF
-    %4 = V_CNDMASK_B32_e64 %3, %3, %0, implicit $exec
+    %4 = V_CNDMASK_B32_e64 0, %3, 0, %3, %0, implicit $exec
     %5 = COPY %1
-    %6 = V_CNDMASK_B32_e64 %5, 0, %0, implicit $exec
+    %6 = V_CNDMASK_B32_e64 0, %5, 0, 0, %0, implicit $exec
     $vcc = IMPLICIT_DEF
     %7 = V_CNDMASK_B32_e32 %3, %3, implicit $exec, implicit $vcc
 
 ...

llvm/trunk/test/CodeGen/AMDGPU/macro-fusion-cluster-vcc-uses.mir

bb.0:
     %7 = V_MOV_B32_e32 0, implicit $exec
     S_NOP 0, implicit def $vcc
     %4, %5 = V_SUBB_U32_e64 %6, %7, %3, implicit $exec
 ...
 
 # GCN-LABEL: name: cluster_cmp_cndmask
 # GCN: S_NOP 0, implicit-def $vcc
 # GCN-NEXT: %3:sreg_64_xexec = V_CMP_EQ_I32_e64 %0, %1, implicit $exec
-# GCN-NEXT: dead %4:vgpr_32 = V_CNDMASK_B32_e64 %0, %1, %3, implicit $exec
+# GCN-NEXT: dead %4:vgpr_32 = V_CNDMASK_B32_e64 0, %0, 0, %1, %3, implicit $exec
 name: cluster_cmp_cndmask
 registers:
   - { id: 0, class: vgpr_32 }
   - { id: 1, class: vgpr_32 }
   - { id: 2, class: vgpr_32 }
   - { id: 3, class: sreg_64_xexec }
   - { id: 4, class: vgpr_32 }
   - { id: 5, class: sreg_64_xexec }
   - { id: 6, class: vgpr_32 }
   - { id: 7, class: vgpr_32 }
 
 body: |
   bb.0:
     %0 = V_MOV_B32_e32 0, implicit $exec
     %1 = V_MOV_B32_e32 0, implicit $exec
     %3 = V_CMP_EQ_I32_e64 %0, %1, implicit $exec
     S_NOP 0, implicit def $vcc
-    %4 = V_CNDMASK_B32_e64 %0, %1, %3, implicit $exec
+    %4 = V_CNDMASK_B32_e64 0, %0, 0, %1, %3, implicit $exec
 ...
 
 # GCN-LABEL: name: cluster_multi_use_cmp_cndmask
 # GCN: %4:sreg_64_xexec = V_CMP_EQ_I32_e64 %0, %1, implicit $exec
-# GCN-NEXT: dead %5:vgpr_32 = V_CNDMASK_B32_e64 %2, %1, %4, implicit $exec
-# GCN-NEXT: dead %6:vgpr_32 = V_CNDMASK_B32_e64 %1, %3, %4, implicit $exec
+# GCN-NEXT: dead %5:vgpr_32 = V_CNDMASK_B32_e64 0, %2, 0, %1, %4, implicit $exec
+# GCN-NEXT: dead %6:vgpr_32 = V_CNDMASK_B32_e64 0, %1, 0, %3, %4, implicit $exec
 name: cluster_multi_use_cmp_cndmask
 registers:
   - { id: 0, class: vgpr_32 }
   - { id: 1, class: vgpr_32 }
   - { id: 2, class: vgpr_32 }
   - { id: 3, class: vgpr_32 }
   - { id: 4, class: sreg_64_xexec }
   - { id: 5, class: vgpr_32 }
   - { id: 6, class: vgpr_32 }
   - { id: 7, class: vgpr_32 }
 
 body: |
   bb.0:
     %0 = V_MOV_B32_e32 0, implicit $exec
     %1 = V_MOV_B32_e32 0, implicit $exec
     %2 = V_MOV_B32_e32 0, implicit $exec
     %3 = V_MOV_B32_e32 0, implicit $exec
 
     %4 = V_CMP_EQ_I32_e64 %0, %1, implicit $exec
     S_NOP 0, implicit def $vcc
-    %5 = V_CNDMASK_B32_e64 %2, %1, %4, implicit $exec
-    %6 = V_CNDMASK_B32_e64 %1, %3, %4, implicit $exec
+    %5 = V_CNDMASK_B32_e64 0, %2, 0, %1, %4, implicit $exec
+    %6 = V_CNDMASK_B32_e64 0, %1, 0, %3, %4, implicit $exec
 ...
 
 # GCN-LABEL: name: cluster_multi_use_cmp_cndmask2
 # GCN: %4:sreg_64_xexec = V_CMP_EQ_I32_e64 %0, %1, implicit $exec
-# GCN-NEXT: dead %5:vgpr_32 = V_CNDMASK_B32_e64 %2, %1, %4, implicit $exec
+# GCN-NEXT: dead %5:vgpr_32 = V_CNDMASK_B32_e64 0, %2, 0, %1, %4, implicit $exec
 # GCN-NEXT: %3:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
-# GCN-NEXT: dead %6:vgpr_32 = V_CNDMASK_B32_e64 %1, %3, %4, implicit $exec
+# GCN-NEXT: dead %6:vgpr_32 = V_CNDMASK_B32_e64 0, %1, 0, %3, %4, implicit $exec
 name: cluster_multi_use_cmp_cndmask2
 registers:
   - { id: 0, class: vgpr_32 }
   - { id: 1, class: vgpr_32 }
   - { id: 2, class: vgpr_32 }
   - { id: 3, class: vgpr_32 }
   - { id: 4, class: sreg_64_xexec }
   - { id: 5, class: vgpr_32 }
   - { id: 6, class: vgpr_32 }
   - { id: 7, class: vgpr_32 }
 
 body: |
   bb.0:
     %0 = V_MOV_B32_e32 0, implicit $exec
     %1 = V_MOV_B32_e32 0, implicit $exec
     %4 = V_CMP_EQ_I32_e64 %0, %1, implicit $exec
     %2 = V_MOV_B32_e32 0, implicit $exec
-    %5 = V_CNDMASK_B32_e64 %2, %1, %4, implicit $exec
+    %5 = V_CNDMASK_B32_e64 0, %2, 0, %1, %4, implicit $exec
     %3 = V_MOV_B32_e32 0, implicit $exec
-    %6 = V_CNDMASK_B32_e64 %1, %3, %4, implicit $exec
+    %6 = V_CNDMASK_B32_e64 0, %1, 0, %3, %4, implicit $exec
 ...

llvm/trunk/test/CodeGen/AMDGPU/merge-load-store-vreg.mir

bb.0:
     %0:vgpr_32 = IMPLICIT_DEF
     S_BRANCH %bb.2
 
   bb.1:
     S_ENDPGM 0
 
   bb.2:
     %1:sreg_64_xexec = V_CMP_NE_U32_e64 %0, 0, implicit $exec
-    %2:vgpr_32 = V_CNDMASK_B32_e64 0, 1, %1, implicit $exec
+    %2:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, %1, implicit $exec
     V_CMP_NE_U32_e32 1, %2, implicit-def $vcc, implicit $exec
     DS_WRITE_B32 %0, %0, 1024, 0, implicit $m0, implicit $exec :: (store 4 into %ir.tmp)
     %3:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
     DS_WRITE_B32 %0, %3, 1056, 0, implicit $m0, implicit $exec :: (store 4 into %ir.tmp1)
     %4:vgpr_32 = DS_READ_B32 %3, 1088, 0, implicit $m0, implicit $exec :: (load 4 from %ir.tmp2)
     %5:vgpr_32 = DS_READ_B32 %3, 1120, 0, implicit $m0, implicit $exec :: (load 4 from %ir.tmp3)
     $vcc = S_AND_B64 $exec, $vcc, implicit-def $scc
     S_CBRANCH_VCCNZ %bb.1, implicit $vcc
bb.0:
     %0:vreg_64 = IMPLICIT_DEF
     S_BRANCH %bb.2
 
   bb.1:
     S_ENDPGM 0
 
   bb.2:
     %1:sreg_64_xexec = V_CMP_NE_U32_e64 %0.sub0, 0, implicit $exec
-    %2:vgpr_32 = V_CNDMASK_B32_e64 0, 1, %1, implicit $exec
+    %2:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, %1, implicit $exec
     V_CMP_NE_U32_e32 1, %2, implicit-def $vcc, implicit $exec
     DS_WRITE_B32 %0.sub0, %0.sub0, 1024, 0, implicit $m0, implicit $exec :: (store 4 into %ir.tmp)
     undef %3.sub0:vreg_64 = V_MOV_B32_e32 0, implicit $exec
     DS_WRITE_B32 %0.sub0, %3.sub0, 1056, 0, implicit $m0, implicit $exec :: (store 4 into %ir.tmp1)
     %4:vgpr_32 = DS_READ_B32 %3.sub0, 1088, 0, implicit $m0, implicit $exec :: (load 4 from %ir.tmp2)
     %5:vgpr_32 = DS_READ_B32 %3.sub0, 1120, 0, implicit $m0, implicit $exec :: (load 4 from %ir.tmp3)
     $vcc = S_AND_B64 $exec, $vcc, implicit-def $scc
     S_CBRANCH_VCCNZ %bb.1, implicit $vcc
bb.0:
     %0:vreg_64 = IMPLICIT_DEF
     S_BRANCH %bb.2
 
   bb.1:
     S_ENDPGM 0
 
   bb.2:
     %1:sreg_64_xexec = V_CMP_NE_U32_e64 %0.sub0, 0, implicit $exec
-    %2:vgpr_32 = V_CNDMASK_B32_e64 0, 1, %1, implicit $exec
+    %2:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, %1, implicit $exec
     V_CMP_NE_U32_e32 1, %2, implicit-def $vcc, implicit $exec
     DS_WRITE_B32 %0.sub0, %0.sub0, 0, 0, implicit $m0, implicit $exec :: (store 4 into %ir.tmp)
     undef %3.sub0:vreg_64 = V_MOV_B32_e32 0, implicit $exec
     DS_WRITE_B32 %0.sub0, %3.sub0, 32, 0, implicit $m0, implicit $exec :: (store 4 into %ir.tmp1)
     %4:vgpr_32 = DS_READ_B32 %3.sub0, 0, 0, implicit $m0, implicit $exec :: (load 4 from %ir.tmp2)
     %5:vgpr_32 = DS_READ_B32 %3.sub0, 32, 0, implicit $m0, implicit $exec :: (load 4 from %ir.tmp3)
     $vcc = S_AND_B64 $exec, $vcc, implicit-def $scc
     S_CBRANCH_VCCNZ %bb.1, implicit $vcc
     S_BRANCH %bb.1
 ...

llvm/trunk/test/CodeGen/AMDGPU/optimize-negated-cond-exec-masking.mir

 # RUN: llc -march=amdgcn -verify-machineinstrs -run-pass=si-optimize-exec-masking-pre-ra -o - %s | FileCheck -check-prefix=GCN %s
 
 # GCN: name: negated_cond_vop2
 # GCN:      %0:sreg_64_xexec = IMPLICIT_DEF
 # GCN-NEXT: $vcc = S_ANDN2_B64 $exec, %0, implicit-def $scc
 # GCN-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
 ---
 name:            negated_cond_vop2
 body:             |
   bb.0:
     %0:sreg_64_xexec = IMPLICIT_DEF
-    %1:vgpr_32 = V_CNDMASK_B32_e64 0, 1, %0, implicit $exec
+    %1:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, %0, implicit $exec
     V_CMP_NE_U32_e32 1, %1, implicit-def $vcc, implicit $exec
     $vcc = S_AND_B64 $exec, killed $vcc, implicit-def dead $scc
     S_CBRANCH_VCCNZ %bb.2, implicit killed $vcc
     S_BRANCH %bb.1
 
   bb.1:
     S_BRANCH %bb.0
 
   bb.2:
     S_ENDPGM 0
 ...
 
 # GCN: name: negated_cond_vop3
 # GCN:      %0:sreg_64_xexec = IMPLICIT_DEF
 # GCN-NEXT: $vcc = S_ANDN2_B64 $exec, %0, implicit-def $scc
 # GCN-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
 ---
 name:            negated_cond_vop3
 body:             |
   bb.0:
     %0:sreg_64_xexec = IMPLICIT_DEF
-    %1:vgpr_32 = V_CNDMASK_B32_e64 0, 1, %0, implicit $exec
+    %1:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, %0, implicit $exec
     %2:sreg_64_xexec = V_CMP_NE_U32_e64 %1, 1, implicit $exec
     $vcc = S_AND_B64 killed %2, $exec, implicit-def dead $scc
     S_CBRANCH_VCCNZ %bb.2, implicit killed $vcc
     S_BRANCH %bb.1
 
   bb.1:
     S_BRANCH %bb.0
 
   bb.2:
     S_ENDPGM 0
 ...
 
 # GCN: name: negated_cond_vop2_redef_vcc1
 # GCN:      %0:sreg_64_xexec = IMPLICIT_DEF
-# GCN-NEXT: %1:vgpr_32 = V_CNDMASK_B32_e64 0, 1, %0, implicit $exec
+# GCN-NEXT: %1:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, %0, implicit $exec
 # GCN-NEXT: V_CMP_NE_U32_e32 1, %1, implicit-def $vcc, implicit $exec
 # GCN-NEXT: $vcc_lo = COPY $sgpr0
 # GCN-NEXT: $vcc = S_AND_B64 $exec, $vcc, implicit-def dead $scc
 # GCN-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
 ---
 name:            negated_cond_vop2_redef_vcc1
 body:             |
   bb.0:
     %0:sreg_64_xexec = IMPLICIT_DEF
-    %1:vgpr_32 = V_CNDMASK_B32_e64 0, 1, %0, implicit $exec
+    %1:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, %0, implicit $exec
     V_CMP_NE_U32_e32 1, %1, implicit-def $vcc, implicit $exec
     $vcc_lo = COPY $sgpr0
     $vcc = S_AND_B64 $exec, killed $vcc, implicit-def dead $scc
     S_CBRANCH_VCCNZ %bb.2, implicit killed $vcc
     S_BRANCH %bb.1
 
   bb.1:
     S_BRANCH %bb.0
 
   bb.2:
     S_ENDPGM 0
 ...
 
 # GCN: name: negated_cond_vop2_redef_vcc2
 # GCN:      %0:sreg_64_xexec = IMPLICIT_DEF
-# GCN-NEXT: %1:vgpr_32 = V_CNDMASK_B32_e64 0, 1, %0, implicit $exec
+# GCN-NEXT: %1:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, %0, implicit $exec
 # GCN-NEXT: V_CMP_NE_U32_e32 1, %1, implicit-def $vcc, implicit $exec
 # GCN-NEXT: $vcc_hi = COPY $sgpr0
 # GCN-NEXT: $vcc = S_AND_B64 $exec, $vcc, implicit-def dead $scc
 # GCN-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
 ---
 name:            negated_cond_vop2_redef_vcc2
 body:             |
   bb.0:
     %0:sreg_64_xexec = IMPLICIT_DEF
-    %1:vgpr_32 = V_CNDMASK_B32_e64 0, 1, %0, implicit $exec
+    %1:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, %0, implicit $exec
     V_CMP_NE_U32_e32 1, %1, implicit-def $vcc, implicit $exec
     $vcc_hi = COPY $sgpr0
     $vcc = S_AND_B64 $exec, killed $vcc, implicit-def dead $scc
     S_CBRANCH_VCCNZ %bb.2, implicit killed $vcc
     S_BRANCH %bb.1
 
   bb.1:
     S_BRANCH %bb.0
 
   bb.2:
     S_ENDPGM 0
 ...
 
 # GCN: name: negated_cond_vop3_redef_cmp
 # GCN:      %0:sreg_64_xexec = IMPLICIT_DEF
-# GCN-NEXT: %1:vgpr_32 = V_CNDMASK_B32_e64 0, 1, %0, implicit $exec
+# GCN-NEXT: %1:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, %0, implicit $exec
 # GCN-NEXT: %2:sreg_64_xexec = V_CMP_NE_U32_e64 %1, 1, implicit $exec
 # GCN-NEXT: %2.sub1:sreg_64_xexec = COPY $sgpr0
 # GCN-NEXT: $vcc = S_AND_B64 %2, $exec, implicit-def dead $scc
 # GCN-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
 ---
 name:            negated_cond_vop3_redef_cmp
 body:             |
   bb.0:
     %0:sreg_64_xexec = IMPLICIT_DEF
-    %1:vgpr_32 = V_CNDMASK_B32_e64 0, 1, %0, implicit $exec
+    %1:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, %0, implicit $exec
     %2:sreg_64_xexec = V_CMP_NE_U32_e64 %1, 1, implicit $exec
     %2.sub1 = COPY $sgpr0
     $vcc = S_AND_B64 killed %2, $exec, implicit-def dead $scc
     S_CBRANCH_VCCNZ %bb.2, implicit killed $vcc
     S_BRANCH %bb.1
 
   bb.1:
     S_BRANCH %bb.0

 # GCN:      $vcc = IMPLICIT_DEF
 # GCN-NEXT: $vcc = S_ANDN2_B64 $exec, $vcc, implicit-def $scc
 # GCN-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
 ---
 name:            negated_cond_vop3_imp_vcc
 body:             |
   bb.0:
     $vcc = IMPLICIT_DEF
-    %1:vgpr_32 = V_CNDMASK_B32_e64 0, 1, $vcc, implicit $exec
+    %1:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, $vcc, implicit $exec
     %2:sreg_64_xexec = V_CMP_NE_U32_e64 %1, 1, implicit $exec
     $vcc = S_AND_B64 killed %2, $exec, implicit-def dead $scc
     S_CBRANCH_VCCNZ %bb.2, implicit killed $vcc
     S_BRANCH %bb.1
 
   bb.1:
     S_BRANCH %bb.0
 
   bb.2:
     S_ENDPGM 0
 ...
 
 # GCN: name: negated_cond_vop2_imp_vcc
 # GCN:      $vcc = IMPLICIT_DEF
 # GCN-NEXT: $vcc = S_ANDN2_B64 $exec, $vcc, implicit-def $scc
 # GCN-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
 ---
 name:            negated_cond_vop2_imp_vcc
 body:             |
   bb.0:
     $vcc = IMPLICIT_DEF
-    %1:vgpr_32 = V_CNDMASK_B32_e64 0, 1, $vcc, implicit $exec
+    %1:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, $vcc, implicit $exec
     V_CMP_NE_U32_e32 1, %1, implicit-def $vcc, implicit $exec
     $vcc = S_AND_B64 killed $vcc, $exec, implicit-def dead $scc
     S_CBRANCH_VCCNZ %bb.2, implicit killed $vcc
     S_BRANCH %bb.1
 
   bb.1:
     S_BRANCH %bb.0
 
   bb.2:
     S_ENDPGM 0
 ...
 
 # GCN: name: negated_cond_vop3_redef_sel
 # GCN:      %0:sreg_64_xexec = IMPLICIT_DEF
-# GCN-NEXT: %1:vgpr_32 = V_CNDMASK_B32_e64 0, 1, %0, implicit $exec
+# GCN-NEXT: %1:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, %0, implicit $exec
 # GCN-NEXT: %1:vgpr_32 = COPY $vgpr0
 # GCN-NEXT: %2:sreg_64_xexec = V_CMP_NE_U32_e64 %1, 1, implicit $exec
 # GCN-NEXT: $vcc = S_AND_B64 %2, $exec, implicit-def dead $scc
 # GCN-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
 ---
 name:            negated_cond_vop3_redef_sel
 body:             |
   bb.0:
     %0:sreg_64_xexec = IMPLICIT_DEF
-    %1:vgpr_32 = V_CNDMASK_B32_e64 0, 1, %0, implicit $exec
+    %1:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, %0, implicit $exec
     %1:vgpr_32 = COPY $vgpr0
     %2:sreg_64_xexec = V_CMP_NE_U32_e64 %1, 1, implicit $exec
     $vcc = S_AND_B64 killed %2, $exec, implicit-def dead $scc
     S_CBRANCH_VCCNZ %bb.2, implicit killed $vcc
     S_BRANCH %bb.1
 
   bb.1:
     S_BRANCH %bb.0
 
   bb.2:
     S_ENDPGM 0
 ...
 
 # GCN: name: negated_cond_vop2_used_sel
 # GCN:      %0:sreg_64_xexec = IMPLICIT_DEF
-# GCN-NEXT: %1:vgpr_32 = V_CNDMASK_B32_e64 0, 1, %0, implicit $exec
+# GCN-NEXT: %1:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, %0, implicit $exec
 # GCN-NEXT: $vcc = S_ANDN2_B64 $exec, %0, implicit-def $scc
 # GCN-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
 ---
 name:            negated_cond_vop2_used_sel
 body:             |
   bb.0:
     %0:sreg_64_xexec = IMPLICIT_DEF
-    %1:vgpr_32 = V_CNDMASK_B32_e64 0, 1, %0, implicit $exec
+    %1:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, %0, implicit $exec
     V_CMP_NE_U32_e32 1, %1, implicit-def $vcc, implicit $exec
     $vcc = S_AND_B64 $exec, killed $vcc, implicit-def dead $scc
     S_CBRANCH_VCCNZ %bb.2, implicit killed $vcc
     S_BRANCH %bb.1
 
   bb.1:
     S_BRANCH %bb.0
 
   bb.2:
     $vgpr0 = COPY %1
     S_ENDPGM 0
 ...
 
 # GCN: name: negated_cond_vop2_used_vcc
 # GCN:      %0:sreg_64_xexec = IMPLICIT_DEF
-# GCN-NEXT: %1:vgpr_32 = V_CNDMASK_B32_e64 0, 1, %0, implicit $exec
+# GCN-NEXT: %1:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, %0, implicit $exec
 # GCN-NEXT: V_CMP_NE_U32_e32 1, %1, implicit-def $vcc, implicit $exec
 # GCN-NEXT: $sgpr0_sgpr1 = COPY $vcc
 # GCN-NEXT: $vcc = S_ANDN2_B64 $exec, %0, implicit-def $scc
 # GCN-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
 ---
 name:            negated_cond_vop2_used_vcc
 body:             |
   bb.0:
     %0:sreg_64_xexec = IMPLICIT_DEF
-    %1:vgpr_32 = V_CNDMASK_B32_e64 0, 1, %0, implicit $exec
+    %1:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, %0, implicit $exec
     V_CMP_NE_U32_e32 1, %1, implicit-def $vcc, implicit $exec
     $sgpr0_sgpr1 = COPY $vcc
     $vcc = S_AND_B64 $exec, killed $vcc, implicit-def dead $scc
     S_CBRANCH_VCCNZ %bb.2, implicit killed $vcc
     S_BRANCH %bb.1
 
   bb.1:
     S_BRANCH %bb.0
 
   bb.2:
     S_ENDPGM 0
 ...
 
 # GCN: name: negated_cond_vop3_sel_wrong_subreg1
 # GCN:      %0:sreg_64_xexec = IMPLICIT_DEF
 # GCN-NEXT: %1.sub1:vreg_64 = IMPLICIT_DEF
-# GCN-NEXT: %1.sub0:vreg_64 = V_CNDMASK_B32_e64 0, 1, %0, implicit $exec
+# GCN-NEXT: %1.sub0:vreg_64 = V_CNDMASK_B32_e64 0, 0, 0, 1, %0, implicit $exec
 # GCN-NEXT: %2:sreg_64_xexec = V_CMP_NE_U32_e64 %1.sub1, 1, implicit $exec
 # GCN-NEXT: $vcc = S_AND_B64 %2, $exec, implicit-def dead $scc
 # GCN-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
 ---
 name:            negated_cond_vop3_sel_wrong_subreg1
 body:             |
   bb.0:
     %0:sreg_64_xexec = IMPLICIT_DEF
     %1.sub1 = IMPLICIT_DEF
-    %1.sub0:vreg_64 = V_CNDMASK_B32_e64 0, 1, %0, implicit $exec
+    %1.sub0:vreg_64 = V_CNDMASK_B32_e64 0, 0, 0, 1, %0, implicit $exec
     %2:sreg_64_xexec = V_CMP_NE_U32_e64 %1.sub1, 1, implicit $exec
     $vcc = S_AND_B64 killed %2, $exec, implicit-def dead $scc
     S_CBRANCH_VCCNZ %bb.2, implicit killed $vcc
     S_BRANCH %bb.1
 
   bb.1:
     S_BRANCH %bb.0
 
   bb.2:
     S_ENDPGM 0
 ...
 
 # GCN: name: negated_cond_vop3_sel_wrong_subreg2
 # GCN:      %0:sreg_64_xexec = IMPLICIT_DEF
-# GCN-NEXT: %1.sub0:vreg_64 = V_CNDMASK_B32_e64 0, 1, %0, implicit $exec
+# GCN-NEXT: %1.sub0:vreg_64 = V_CNDMASK_B32_e64 0, 0, 0, 1, %0, implicit $exec
 # GCN-NEXT: %1.sub1:vreg_64 = IMPLICIT_DEF
 # GCN-NEXT: %2:sreg_64_xexec = V_CMP_NE_U32_e64 %1.sub1, 1, implicit $exec
 # GCN-NEXT: $vcc = S_AND_B64 %2, $exec, implicit-def dead $scc
 # GCN-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
 ---
 name:            negated_cond_vop3_sel_wrong_subreg2
 body:             |
   bb.0:
     %0:sreg_64_xexec = IMPLICIT_DEF
-    %1.sub0:vreg_64 = V_CNDMASK_B32_e64 0, 1, %0, implicit $exec
+    %1.sub0:vreg_64 = V_CNDMASK_B32_e64 0, 0, 0, 1, %0, implicit $exec
     %1.sub1 = IMPLICIT_DEF
     %2:sreg_64_xexec = V_CMP_NE_U32_e64 %1.sub1, 1, implicit $exec
     $vcc = S_AND_B64 killed %2, $exec, implicit-def dead $scc
     S_CBRANCH_VCCNZ %bb.2, implicit killed $vcc
     S_BRANCH %bb.1
 
   bb.1:
     S_BRANCH %bb.0
 
   bb.2:
     S_ENDPGM 0
 ...
 
 # GCN: name: negated_cond_vop3_sel_right_subreg1
 # GCN:      %0:sreg_64_xexec = IMPLICIT_DEF
 # GCN-NEXT: %1.sub1:vreg_64 = IMPLICIT_DEF
 # GCN-NEXT: $vcc = S_ANDN2_B64 $exec, %0, implicit-def $scc
 # GCN-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
 ---
 name:            negated_cond_vop3_sel_right_subreg1
 body:             |
   bb.0:
     %0:sreg_64_xexec = IMPLICIT_DEF
     %1.sub1 = IMPLICIT_DEF
-    %1.sub0:vreg_64 = V_CNDMASK_B32_e64 0, 1, %0, implicit $exec
+    %1.sub0:vreg_64 = V_CNDMASK_B32_e64 0, 0, 0, 1, %0, implicit $exec
     %2:sreg_64_xexec = V_CMP_NE_U32_e64 %1.sub0, 1, implicit $exec
     $vcc = S_AND_B64 killed %2, $exec, implicit-def dead $scc
     S_CBRANCH_VCCNZ %bb.2, implicit killed $vcc
     S_BRANCH %bb.1
 
   bb.1:
     S_BRANCH %bb.0
 
   bb.2:
     S_ENDPGM 0
 ...
 
 # GCN: name: negated_cond_vop3_sel_right_subreg2
 # GCN:      %0:sreg_64_xexec = IMPLICIT_DEF
 # GCN-NEXT: %1.sub1:vreg_64 = IMPLICIT_DEF
 # GCN-NEXT: $vcc = S_ANDN2_B64 $exec, %0, implicit-def $scc
 # GCN-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
 ---
 name:            negated_cond_vop3_sel_right_subreg2
 body:             |
   bb.0:
     %0:sreg_64_xexec = IMPLICIT_DEF
-    %1.sub0:vreg_64 = V_CNDMASK_B32_e64 0, 1, %0, implicit $exec
+    %1.sub0:vreg_64 = V_CNDMASK_B32_e64 0, 0, 0, 1, %0, implicit $exec
     %1.sub1 = IMPLICIT_DEF
     %2:sreg_64_xexec = V_CMP_NE_U32_e64 %1.sub0, 1, implicit $exec
     $vcc = S_AND_B64 killed %2, $exec, implicit-def dead $scc
     S_CBRANCH_VCCNZ %bb.2, implicit killed $vcc
     S_BRANCH %bb.1
 
   bb.1:
     S_BRANCH %bb.0
 
   bb.2:
     S_ENDPGM 0
 ...
 
 # GCN: name: negated_cond_vop3_sel_subreg_overlap
 # GCN:      %0:sreg_64_xexec = IMPLICIT_DEF
-# GCN-NEXT: %1.sub2:vreg_128 = V_CNDMASK_B32_e64 0, 1, %0, implicit $exec
+# GCN-NEXT: %1.sub2:vreg_128 = V_CNDMASK_B32_e64 0, 0, 0, 1, %0, implicit $exec
 # GCN-NEXT: %1.sub2_sub3:vreg_128 = IMPLICIT_DEF
 # GCN-NEXT: %2:sreg_64_xexec = V_CMP_NE_U32_e64 %1.sub2, 1, implicit $exec
 # GCN-NEXT: $vcc = S_AND_B64 %2, $exec, implicit-def dead $scc
 # GCN-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
 ---
 name:            negated_cond_vop3_sel_subreg_overlap
 body:             |
   bb.0:
     %0:sreg_64_xexec = IMPLICIT_DEF
-    %1.sub2:vreg_128 = V_CNDMASK_B32_e64 0, 1, %0, implicit $exec
+    %1.sub2:vreg_128 = V_CNDMASK_B32_e64 0, 0, 0, 1, %0, implicit $exec
     %1.sub2_sub3 = IMPLICIT_DEF
     %2:sreg_64_xexec = V_CMP_NE_U32_e64 %1.sub2, 1, implicit $exec
     $vcc = S_AND_B64 killed %2, $exec, implicit-def dead $scc
     S_CBRANCH_VCCNZ %bb.2, implicit killed $vcc
     S_BRANCH %bb.1
 
   bb.1:
     S_BRANCH %bb.0
 
   bb.2:
     S_ENDPGM 0
 ...
 
 # GCN: name: negated_cond_vop2_dominated_blocks
 # GCN:      %0:sreg_64_xexec = IMPLICIT_DEF
 # GCN:      $vcc = S_ANDN2_B64 $exec, %0, implicit-def $scc
 # GCN-NEXT: S_CBRANCH_VCCNZ %bb.3, implicit $vcc
 ---
 name:            negated_cond_vop2_dominated_blocks
 body:             |
   bb.0:
     %0:sreg_64_xexec = IMPLICIT_DEF
-    %1:vgpr_32 = V_CNDMASK_B32_e64 0, 1, %0, implicit $exec
+    %1:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, %0, implicit $exec
 
   bb.1:
     V_CMP_NE_U32_e32 1, %1, implicit-def $vcc, implicit $exec
     $vcc = S_AND_B64 $exec, killed $vcc, implicit-def dead $scc
     S_CBRANCH_VCCNZ %bb.3, implicit killed $vcc
     S_BRANCH %bb.2
 
   bb.2:
     S_BRANCH %bb.1
 
   bb.3:
     S_ENDPGM 0
 ...
 
 # GCN: name: negated_cond_vop2_different_blocks_cmp_and
-# GCN:      %1:vgpr_32 = V_CNDMASK_B32_e64 0, 1, %0, implicit $exec
+# GCN:      %1:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, %0, implicit $exec
 # GCN:      $vcc = S_AND_B64 $exec, %2, implicit-def dead $scc
 # GCN-NEXT: S_CBRANCH_VCCNZ %bb.3, implicit $vcc
 ---
 name:            negated_cond_vop2_different_blocks_cmp_and
 body:             |
   bb.0:
     %0:sreg_64_xexec = IMPLICIT_DEF
-    %1:vgpr_32 = V_CNDMASK_B32_e64 0, 1, %0, implicit $exec
+    %1:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, %0, implicit $exec
     %2:sreg_64_xexec = V_CMP_NE_U32_e64 %1, 1, implicit $exec
 
   bb.1:
     $vcc = S_AND_B64 $exec, killed %2, implicit-def dead $scc
     S_CBRANCH_VCCNZ %bb.3, implicit killed $vcc
     S_BRANCH %bb.2
 
   bb.2:
     S_BRANCH %bb.1
 
   bb.3:
     S_ENDPGM 0
 ...
 
 # GCN: name: negated_cond_vop2_not_dominated_blocks
-# GCN:      V_CNDMASK_B32_e64 0, 1,
+# GCN:      V_CNDMASK_B32_e64 0, 0, 0, 1,
 # GCN:      $vcc = S_AND_B64 $exec, $vcc, implicit-def dead $scc
 # GCN-NEXT: S_CBRANCH_VCCNZ %bb.4, implicit $vcc
 ---
 name:            negated_cond_vop2_not_dominated_blocks
 body:             |
   bb.0:
     $vcc = IMPLICIT_DEF
     %1 = IMPLICIT_DEF
     S_CBRANCH_VCCNZ %bb.2, implicit killed $vcc
     S_BRANCH %bb.1
 
   bb.1:
     %0:sreg_64_xexec = IMPLICIT_DEF
-    %1:vgpr_32 = V_CNDMASK_B32_e64 0, 1, %0, implicit $exec
+    %1:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, %0, implicit $exec
 
   bb.2:
     V_CMP_NE_U32_e32 1, %1, implicit-def $vcc, implicit $exec
     $vcc = S_AND_B64 $exec, killed $vcc, implicit-def dead $scc
     S_CBRANCH_VCCNZ %bb.4, implicit killed $vcc
     S_BRANCH %bb.3
 
   bb.3:
     S_BRANCH %bb.2
 
   bb.4:
     S_ENDPGM 0
 ...

llvm/trunk/test/CodeGen/AMDGPU/regcoal-subrange-join-seg.mir

body:             |
 
   bb.28:
     %9 = S_FF1_I32_B32 undef %10
     %13 = V_MAD_U32_U24 killed %9, 48, 32, 0, implicit $exec
     %45 = BUFFER_LOAD_DWORD_OFFEN killed %13, undef %15, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 4)
     %46 = V_AND_B32_e32 1, killed %45, implicit $exec
     %21 = S_BUFFER_LOAD_DWORD_SGPR undef %22, undef %23, 0 :: (dereferenceable invariant load 4)
     %25 = V_CMP_GE_F32_e64 0, 0, 0, killed %21, 0, implicit $exec
-    %26 = V_CNDMASK_B32_e64 0, -1, killed %25, implicit $exec
+    %26 = V_CNDMASK_B32_e64 0, 0, 0, -1, killed %25, implicit $exec
     %62 = IMPLICIT_DEF
 
   bb.29:
     successors: %bb.30(0x30000000), %bb.36(0x50000000)
 
     %53 = COPY killed %62
     %47 = V_ADD_I32_e32 -1, %46, implicit-def dead $vcc, implicit $exec
     %48 = V_OR_B32_e32 killed %47, %26, implicit $exec

llvm/trunk/test/CodeGen/AMDGPU/shrink-vop3-carry-out.mir

 # RUN: llc -verify-machineinstrs -march=amdgcn -run-pass si-shrink-instructions -o - %s | FileCheck -check-prefix=GCN %s
 # Check that add with carry out isn't incorrectly reduced to e32 when
 # the carry out is a virtual register.
 
 # TODO: We should run this test until the end of codegen to make sure
 # that the post-RA run does manage to shrink it, but right now the
 # resume crashes
 
 ...
 # GCN-LABEL: name: shrink_add_vop3{{$}}
 # GCN: %29:vgpr_32, %9:sreg_64_xexec = V_ADD_I32_e64 %19, %17, implicit $exec
-# GCN: %24:vgpr_32 = V_CNDMASK_B32_e64 0, 1, killed %9, implicit $exec
+# GCN: %24:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed %9, implicit $exec
 name:            shrink_add_vop3
 alignment:       0
 exposesReturnsTwice: false
 legalized:       false
 regBankSelected: false
 selected:        false
 tracksRegLiveness: true
 registers:
bb.0:
     %11 = S_MOV_B32 0
     %12 = REG_SEQUENCE killed %11, 1, killed %10, 2
     %13 = REG_SEQUENCE killed %5, 17, %12, 18
     %28 = V_LSHL_B64 killed %27, 2, implicit $exec
     %16 = REG_SEQUENCE killed %4, 17, %12, 18
     %17 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 0, 0, 0, 0, implicit $exec
     %19 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 4, 0, 0, 0, implicit $exec
     %29, %9 = V_ADD_I32_e64 %19, %17, implicit $exec
-    %24 = V_CNDMASK_B32_e64 0, 1, killed %9, implicit $exec
+    %24 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed %9, implicit $exec
     BUFFER_STORE_DWORD_ADDR64 %24, %28, killed %16, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
 ---
 # GCN-LABEL: name: shrink_sub_vop3{{$}}
 # GCN: %29:vgpr_32, %9:sreg_64_xexec = V_SUB_I32_e64 %19, %17, implicit $exec
-# GCN: %24:vgpr_32 = V_CNDMASK_B32_e64 0, 1, killed %9, implicit $exec
+# GCN: %24:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed %9, implicit $exec
 
 name:            shrink_sub_vop3
 alignment:       0
 exposesReturnsTwice: false
 legalized:       false
 regBankSelected: false
 selected:        false
 tracksRegLiveness: true
bb.0:
     %11 = S_MOV_B32 0
     %12 = REG_SEQUENCE killed %11, 1, killed %10, 2
     %13 = REG_SEQUENCE killed %5, 17, %12, 18
     %28 = V_LSHL_B64 killed %27, 2, implicit $exec
     %16 = REG_SEQUENCE killed %4, 17, %12, 18
     %17 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 0, 0, 0, 0, implicit $exec
     %19 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 4, 0, 0, 0, implicit $exec
     %29, %9 = V_SUB_I32_e64 %19, %17, implicit $exec
-    %24 = V_CNDMASK_B32_e64 0, 1, killed %9, implicit $exec
+    %24 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed %9, implicit $exec
     BUFFER_STORE_DWORD_ADDR64 %24, %28, killed %16, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
 ---
 # GCN-LABEL: name: shrink_subrev_vop3{{$}}
 # GCN: %29:vgpr_32, %9:sreg_64_xexec = V_SUBREV_I32_e64 %19, %17, implicit $exec
-# GCN: %24:vgpr_32 = V_CNDMASK_B32_e64 0, 1, killed %9, implicit $exec
+# GCN: %24:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed %9, implicit $exec
 
 name:            shrink_subrev_vop3
 alignment:       0
 exposesReturnsTwice: false
 legalized:       false
 regBankSelected: false
 selected:        false
 tracksRegLiveness: true
bb.0:
     %11 = S_MOV_B32 0
     %12 = REG_SEQUENCE killed %11, 1, killed %10, 2
     %13 = REG_SEQUENCE killed %5, 17, %12, 18
     %28 = V_LSHL_B64 killed %27, 2, implicit $exec
     %16 = REG_SEQUENCE killed %4, 17, %12, 18
     %17 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 0, 0, 0, 0, implicit $exec
     %19 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 4, 0, 0, 0, implicit $exec
     %29, %9 = V_SUBREV_I32_e64 %19, %17, implicit $exec
-    %24 = V_CNDMASK_B32_e64 0, 1, killed %9, implicit $exec
+    %24 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed %9, implicit $exec
     BUFFER_STORE_DWORD_ADDR64 %29, %28, killed %16, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
 ---
 # GCN-LABEL: name: check_addc_src2_vop3{{$}}
 # GCN: %29:vgpr_32, $vcc = V_ADDC_U32_e64 %19, %17, %9, implicit $exec
-# GCN: %24:vgpr_32 = V_CNDMASK_B32_e64 0, 1, killed $vcc, implicit $exec
+# GCN: %24:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed $vcc, implicit $exec
 name: check_addc_src2_vop3
 alignment:       0
 exposesReturnsTwice: false
 legalized:       false
 regBankSelected: false
 selected:        false
 tracksRegLiveness: true
 registers:
bb.0:
     %12 = REG_SEQUENCE killed %11, 1, killed %10, 2
     %13 = REG_SEQUENCE killed %5, 17, %12, 18
     %28 = V_LSHL_B64 killed %27, 2, implicit $exec
     %16 = REG_SEQUENCE killed %4, 17, %12, 18
     %17 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 0, 0, 0, 0, implicit $exec
     %19 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 4, 0, 0, 0, implicit $exec
     %9 = S_MOV_B64 0
     %29, $vcc = V_ADDC_U32_e64 %19, %17, %9, implicit $exec
-    %24 = V_CNDMASK_B32_e64 0, 1, killed $vcc, implicit $exec
+    %24 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed $vcc, implicit $exec
     BUFFER_STORE_DWORD_ADDR64 %24, %28, killed %16, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
 ---
 # GCN-LABEL: name: shrink_addc_vop3{{$}}
 # GCN: %29:vgpr_32 = V_ADDC_U32_e32 %19, %17, implicit-def $vcc, implicit $vcc, implicit $exec
-# GCN %24 = V_CNDMASK_B32_e64 0, 1, killed $vcc, implicit $exec
+# GCN %24 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed $vcc, implicit $exec
 
 name:            shrink_addc_vop3
 alignment:       0
 exposesReturnsTwice: false
 legalized:       false
 regBankSelected: false
 selected:        false
 tracksRegLiveness: true
bb.0:
     %12 = REG_SEQUENCE killed %11, 1, killed %10, 2
     %13 = REG_SEQUENCE killed %5, 17, %12, 18
     %28 = V_LSHL_B64 killed %27, 2, implicit $exec
     %16 = REG_SEQUENCE killed %4, 17, %12, 18
     %17 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 0, 0, 0, 0, implicit $exec
     %19 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 4, 0, 0, 0, implicit $exec
     $vcc = S_MOV_B64 0
     %29, $vcc = V_ADDC_U32_e64 %19, %17, $vcc, implicit $exec
-    %24 = V_CNDMASK_B32_e64 0, 1, killed $vcc, implicit $exec
+    %24 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed $vcc, implicit $exec
     BUFFER_STORE_DWORD_ADDR64 %24, %28, killed %16, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
 
 ---
 # GCN-LABEL: name: shrink_addc_undef_vcc{{$}}
 # GCN: %29:vgpr_32 = V_ADDC_U32_e32 %19, %17, implicit-def $vcc, implicit undef $vcc, implicit $exec
-# GCN: %24:vgpr_32 = V_CNDMASK_B32_e64 0, 1, killed $vcc, implicit $exec
+# GCN: %24:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed $vcc, implicit $exec
 name:            shrink_addc_undef_vcc
 alignment:       0
 exposesReturnsTwice: false
 legalized:       false
 regBankSelected: false
 selected:        false
 tracksRegLiveness: true
 registers:
bb.0:
     %11 = S_MOV_B32 0
     %12 = REG_SEQUENCE killed %11, 1, killed %10, 2
     %13 = REG_SEQUENCE killed %5, 17, %12, 18
     %28 = V_LSHL_B64 killed %27, 2, implicit $exec
     %16 = REG_SEQUENCE killed %4, 17, %12, 18
     %17 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 0, 0, 0, 0, implicit $exec
     %19 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 4, 0, 0, 0, implicit $exec
     %29, $vcc = V_ADDC_U32_e64 %19, %17, undef $vcc, implicit $exec
-    %24 = V_CNDMASK_B32_e64 0, 1, killed $vcc, implicit $exec
+    %24 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed $vcc, implicit $exec
     BUFFER_STORE_DWORD_ADDR64 %24, %28, killed %16, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...

llvm/trunk/test/CodeGen/AMDGPU/subreg-split-live-in-error.mir

bb.13:
     %32:vgpr_32 = V_ADD_F32_e32 %25, %31, implicit $exec
     %33:vgpr_32 = V_MUL_F32_e32 %22, %30, implicit $exec
     %34:vgpr_32 = V_MUL_F32_e32 %23, %30, implicit $exec
     %35:vgpr_32 = V_MUL_F32_e32 %32, %30, implicit $exec
     %36:vgpr_32 = V_MUL_F32_e32 0, %34, implicit $exec
     %36:vgpr_32 = V_MAC_F32_e32 0, %33, %36, implicit $exec
     %37:vgpr_32 = V_MAD_F32 0, %35, 0, 0, 0, 0, 0, 0, implicit $exec
     %38:sreg_64_xexec = V_CMP_NE_U32_e64 0, %5, implicit $exec
-    %39:vgpr_32 = V_CNDMASK_B32_e64 0, 1, %38, implicit $exec
+    %39:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, %38, implicit $exec
     V_CMP_NE_U32_e32 1, %39, implicit-def $vcc, implicit $exec
     $vcc = S_AND_B64 $exec, $vcc, implicit-def dead $scc
     %40:vgpr_32 = V_ADD_F32_e32 %36, %37, implicit $exec
     S_CBRANCH_VCCZ %bb.15, implicit $vcc
 
   bb.14:
     successors: %bb.17(0x80000000)
     S_BRANCH %bb.17

llvm/trunk/test/CodeGen/AMDGPU/undefined-physreg-sgpr-spill.mir

body:             |
   bb.0:
     successors: %bb.1, %bb.2
     liveins: $vgpr0, $sgpr4_sgpr5
 
     $vgpr1_vgpr2 = COPY killed $sgpr4_sgpr5, implicit $exec
     $vgpr1 = GLOBAL_LOAD_UBYTE killed $vgpr1_vgpr2, 0, 0, 0, implicit $exec :: (non-temporal dereferenceable invariant load 1 from `i1 addrspace(4)* undef`)
     $vcc = V_CMP_NE_U32_e64 0, $vgpr0, implicit $exec
     $sgpr0_sgpr1 = V_CMP_EQ_U32_e64 1, killed $vgpr1, implicit $exec
-    $vgpr1 = V_CNDMASK_B32_e64 0, -1, killed $sgpr0_sgpr1, implicit $exec
+    $vgpr1 = V_CNDMASK_B32_e64 0, 0, 0, -1, killed $sgpr0_sgpr1, implicit $exec
     $sgpr0_sgpr1 = COPY $exec, implicit-def $exec
     SI_SPILL_S64_SAVE $sgpr0_sgpr1, %stack.0, implicit $exec, implicit $sgpr8_sgpr9_sgpr10_sgpr11, implicit $sgpr13, implicit-def dead $m0 :: (store 8 into %stack.0, align 4, addrspace 5)
     $sgpr2_sgpr3 = S_AND_B64 killed $sgpr0_sgpr1, killed $vcc, implicit-def dead $scc
     $exec = S_MOV_B64_term killed $sgpr2_sgpr3
     SI_MASK_BRANCH %bb.2, implicit $exec
     S_BRANCH %bb.1
 
   bb.1:
body:             |
   bb.0:
     successors: %bb.1, %bb.2
     liveins: $vgpr0, $sgpr4_sgpr5
 
     $vgpr1_vgpr2 = COPY killed $sgpr4_sgpr5, implicit $exec
     $vgpr1 = GLOBAL_LOAD_UBYTE killed $vgpr1_vgpr2, 0, 0, 0, implicit $exec :: (non-temporal dereferenceable invariant load 1 from `i1 addrspace(4)* undef`)
     $vcc = V_CMP_NE_U32_e64 0, $vgpr0, implicit $exec
     $sgpr0_sgpr1 = V_CMP_EQ_U32_e64 1, killed $vgpr1, implicit $exec
-    $vgpr1 = V_CNDMASK_B32_e64 0, -1, killed $sgpr0_sgpr1, implicit $exec
+    $vgpr1 = V_CNDMASK_B32_e64 0, 0, 0, -1, killed $sgpr0_sgpr1, implicit $exec
     $sgpr0_sgpr1 = COPY $exec, implicit-def $exec
     $sgpr2_sgpr3 = S_AND_B64 $sgpr0_sgpr1, killed $vcc, implicit-def dead $scc
     SI_SPILL_S64_SAVE killed $sgpr0_sgpr1, %stack.0, implicit $exec, implicit $sgpr8_sgpr9_sgpr10_sgpr11, implicit $sgpr13, implicit-def dead $m0 :: (store 8 into %stack.0, align 4, addrspace 5)
     $exec = S_MOV_B64_term killed $sgpr2_sgpr3
     SI_MASK_BRANCH %bb.2, implicit $exec
     S_BRANCH %bb.1
 
   bb.1:

llvm/trunk/test/CodeGen/AMDGPU/waitcnt-back-edge-loop.mir

body:             |
   bb.0:
     successors: %bb.1
 
     $vgpr1 = V_MOV_B32_e32 0, implicit $exec, implicit-def $vgpr1_vgpr2
     $vgpr2 = V_MOV_B32_e32 0, implicit $exec, implicit-def $vgpr1_vgpr2
     $vgpr4 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, implicit $exec, implicit $flat_scr :: (volatile load 4 from `float addrspace(1)* null`, addrspace 1)
     $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, implicit $exec, implicit $flat_scr :: (volatile load 4 from `float addrspace(1)* null`, addrspace 1)
     $sgpr0_sgpr1 = V_CMP_EQ_U32_e64 3, killed $sgpr4, implicit $exec
-    $vgpr3 = V_CNDMASK_B32_e64 -1082130432, 1065353216, killed $sgpr0_sgpr1, implicit $exec
+    $vgpr3 = V_CNDMASK_B32_e64 0, -1082130432, 0, 1065353216, killed $sgpr0_sgpr1, implicit $exec
     $vgpr5 = V_MOV_B32_e32 $vgpr0, implicit $exec, implicit $exec
     S_BRANCH %bb.1
 
   bb.3:
     successors: %bb.1
 
     $vgpr5 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, implicit $exec, implicit $flat_scr :: (volatile load 4 from `float addrspace(1)* null`, addrspace 1)
 

llvm/trunk/test/MC/AMDGPU/vop3.s

 
 // Test forced e64 encoding with e32 operands
 
 v_add_f32_e64 v1, v3, v5
 // SICI: v_add_f32_e64 v1, v3, v5 ; encoding: [0x01,0x00,0x06,0xd2,0x03,0x0b,0x02,0x00]
 // VI:   v_add_f32_e64 v1, v3, v5 ; encoding: [0x01,0x00,0x01,0xd1,0x03,0x0b,0x02,0x00]
 
 
-// TODO: Modifier tests
+// TODO: Modifier tests (v_cndmask done)
 
 v_cndmask_b32 v1, v3, v5, s[4:5]
 // SICI: v_cndmask_b32_e64 v1, v3, v5, s[4:5] ; encoding: [0x01,0x00,0x00,0xd2,0x03,0x0b,0x12,0x00]
 // VI:   v_cndmask_b32_e64 v1, v3, v5, s[4:5] ; encoding: [0x01,0x00,0x00,0xd1,0x03,0x0b,0x12,0x00]
 
 v_cndmask_b32_e64 v1, v3, v5, s[4:5]
 // SICI: v_cndmask_b32_e64 v1, v3, v5, s[4:5] ; encoding: [0x01,0x00,0x00,0xd2,0x03,0x0b,0x12,0x00]
 // VI:   v_cndmask_b32_e64 v1, v3, v5, s[4:5] ; encoding: [0x01,0x00,0x00,0xd1,0x03,0x0b,0x12,0x00]
 
 v_cndmask_b32_e64 v1, v3, v5, vcc
 // SICI: v_cndmask_b32_e64 v1, v3, v5, vcc ; encoding: [0x01,0x00,0x00,0xd2,0x03,0x0b,0xaa,0x01]
 // VI:   v_cndmask_b32_e64 v1, v3, v5, vcc ; encoding: [0x01,0x00,0x00,0xd1,0x03,0x0b,0xaa,0x01]
 
+v_cndmask_b32 v1, -v3, v5, s[4:5]
+// SICI: v_cndmask_b32_e64 v1, -v3, v5, s[4:5] ; encoding: [0x01,0x00,0x00,0xd2,0x03,0x0b,0x12,0x20]
+// VI:   v_cndmask_b32_e64 v1, -v3, v5, s[4:5] ; encoding: [0x01,0x00,0x00,0xd1,0x03,0x0b,0x12,0x20]
+
+v_cndmask_b32_e64 v1, v3, |v5|, s[4:5]
+// SICI: v_cndmask_b32_e64 v1, v3, |v5|, s[4:5] ; encoding: [0x01,0x02,0x00,0xd2,0x03,0x0b,0x12,0x00]
+// VI:   v_cndmask_b32_e64 v1, v3, |v5|, s[4:5] ; encoding: [0x01,0x02,0x00,0xd1,0x03,0x0b,0x12,0x00]
+
+v_cndmask_b32_e64 v1, -abs(v3), v5, vcc
+// SICI: v_cndmask_b32_e64 v1, -|v3|, v5, vcc ; encoding: [0x01,0x01,0x00,0xd2,0x03,0x0b,0xaa,0x21]
+// VI:   v_cndmask_b32_e64 v1, -|v3|, v5, vcc ; encoding: [0x01,0x01,0x00,0xd1,0x03,0x0b,0xaa,0x21]
+
 //TODO: readlane, writelane
 
 v_add_f32 v1, v3, s5
 // SICI: v_add_f32_e64 v1, v3, s5 ; encoding: [0x01,0x00,0x06,0xd2,0x03,0x0b,0x00,0x00]
 // VI:   v_add_f32_e64 v1, v3, s5 ; encoding: [0x01,0x00,0x01,0xd1,0x03,0x0b,0x00,0x00]
 
 v_sub_f32 v1, v3, s5
 // SICI: v_sub_f32_e64 v1, v3, s5 ; encoding: [0x01,0x00,0x08,0xd2,0x03,0x0b,0x00,0x00]

llvm/trunk/test/MC/Disassembler/AMDGPU/vop3_vi.txt

 0x01 0x00 0x01 0xd1 0x03 0x0b 0x02 0x00
 
 # VI:   v_cndmask_b32_e64 v1, v3, v5, s[4:5] ; encoding: [0x01,0x00,0x00,0xd1,0x03,0x0b,0x12,0x00]
 0x01 0x00 0x00 0xd1 0x03 0x0b 0x12 0x00
 
 # VI:   v_cndmask_b32_e64 v1, v3, v5, vcc ; encoding: [0x01,0x00,0x00,0xd1,0x03,0x0b,0xaa,0x01]
 0x01 0x00 0x00 0xd1 0x03 0x0b 0xaa 0x01
 
+# VI:   v_cndmask_b32_e64 v1, -v3, v5, s[4:5] ; encoding: [0x01,0x00,0x00,0xd1,0x03,0x0b,0x12,0x20]
+0x01,0x00,0x00,0xd1,0x03,0x0b,0x12,0x20
+
+# VI:   v_cndmask_b32_e64 v1, v3, |v5|, s[4:5] ; encoding: [0x01,0x02,0x00,0xd1,0x03,0x0b,0x12,0x00]
+0x01,0x02,0x00,0xd1,0x03,0x0b,0x12,0x00
+
+# VI:   v_cndmask_b32_e64 v1, -|v3|, v5, vcc ; encoding: [0x01,0x01,0x00,0xd1,0x03,0x0b,0xaa,0x21]
+0x01,0x01,0x00,0xd1,0x03,0x0b,0xaa,0x21
+
 # VI:   v_add_f32_e64 v1, v3, s5 ; encoding: [0x01,0x00,0x01,0xd1,0x03,0x0b,0x00,0x00]
 0x01 0x00 0x01 0xd1 0x03 0x0b 0x00 0x00
 
 # VI:   v_sub_f32_e64 v1, v3, s5 ; encoding: [0x01,0x00,0x02,0xd1,0x03,0x0b,0x00,0x00]
 0x01 0x00 0x02 0xd1 0x03 0x0b 0x00 0x00
 
 # VI:   v_subrev_f32_e64 v1, v3, s5 ; encoding: [0x01,0x00,0x03,0xd1,0x03,0x0b,0x00,0x00]
 0x01 0x00 0x03 0xd1 0x03 0x0b 0x00 0x00