AMDGPU: Preserve vcc undef flags when inverting branch

[arsenm]

Summary

If the branch was on a read-undef of vcc, passes that used
analyzeBranch to invert the branch condition wouldn't preserve
the undef flag resulting in a verifier error.

Fixes verifier failures in a future commit.

Also fix verifier error when inserting copy for vccz corruption bug.

[arsenm]

Fix formatting

[• tstellarAMD]

Would it be possible to write a MIR test for this?

[arsenm]

r286133 with MIR tests

lib/Target/AMDGPU/SIInsertWaits.cpp

 char &llvm::SIInsertWaitsID = SIInsertWaits::ID;
 
 FunctionPass *llvm::createSIInsertWaitsPass() {
   return new SIInsertWaits();
 }
 
 const Counters SIInsertWaits::ZeroCounts = { { 0, 0, 0 } };
 
-static bool readsVCCZ(unsigned Opcode) {
-  return Opcode == AMDGPU::S_CBRANCH_VCCNZ || Opcode == AMDGPU::S_CBRANCH_VCCZ;
+static bool readsVCCZ(const MachineInstr &MI) {
+  unsigned Opc = MI.getOpcode();
+  return (Opc == AMDGPU::S_CBRANCH_VCCNZ || Opc == AMDGPU::S_CBRANCH_VCCZ) &&
+         !MI.getOperand(1).isUndef();
 }
 
 bool SIInsertWaits::hasOutstandingLGKM() const {
   return WaitedOn.Named.LGKM != LastIssued.Named.LGKM;
 }
 
 Counters SIInsertWaits::getHwCounts(MachineInstr &MI) {
   uint64_t TSFlags = MI.getDesc().TSFlags;
for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end();
         } else if (!hasOutstandingLGKM() && I->modifiesRegister(AMDGPU::VCC, TRI)) {
           // FIXME: We only care about SMRD instructions here, not LDS or GDS.
           // Whenever we store a value in vcc, the correct value of vccz is
           // restored.
           VCCZCorrupt = false;
         }
 
         // Check if we need to apply the bug work-around
-        if (readsVCCZ(I->getOpcode()) && VCCZCorrupt) {
+        if (VCCZCorrupt && readsVCCZ(*I)) {
           DEBUG(dbgs() << "Inserting vccz bug work-around before: " << *I << '\n');
 
           // Wait on everything, not just LGKM.  vccz reads usually come from
           // terminators, and we always wait on everything at the end of the
           // block, so if we only wait on LGKM here, we might end up with
           // another s_waitcnt inserted right after this if there are non-LGKM
           // instructions still outstanding.
           insertWait(MBB, I, LastIssued);
 
           // Restore the vccz bit.  Any time a value is written to vcc, the vcc
           // bit is updated, so we can restore the bit by reading the value of
           // vcc and then writing it back to the register.
           BuildMI(MBB, I, I->getDebugLoc(), TII->get(AMDGPU::S_MOV_B64),
                   AMDGPU::VCC)
-                  .addReg(AMDGPU::VCC);
         }
       }
 
       // Record pre-existing, explicitly requested waits
       if (I->getOpcode() == AMDGPU::S_WAITCNT) {
         handleExistingWait(*I);
         RemoveMI.push_back(&*I);
         continue;

lib/Target/AMDGPU/SIInstrInfo.cpp

bool SIInstrInfo::analyzeBranchImpl(MachineBasicBlock &MBB,
   }
 
   BranchPredicate Pred = getBranchPredicate(I->getOpcode());
   if (Pred == INVALID_BR)
     return true;
 
   MachineBasicBlock *CondBB = I->getOperand(0).getMBB();
   Cond.push_back(MachineOperand::CreateImm(Pred));
+  Cond.push_back(I->getOperand(1)); // Save the branch register.
 
   ++I;
 
   if (I == MBB.end()) {
     // Conditional branch followed by fall-through.
     TBB = CondBB;
     return false;
   }
unsigned SIInstrInfo::insertBranch(MachineBasicBlock &MBB,
   }
 
   assert(TBB && Cond[0].isImm());
 
   unsigned Opcode
     = getBranchOpcode(static_cast<BranchPredicate>(Cond[0].getImm()));
 
   if (!FBB) {
+    Cond[1].isUndef();
+    MachineInstr *CondBr =
       BuildMI(&MBB, DL, get(Opcode))
       .addMBB(TBB);
 
+    // Copy the flags onto the implicit condition register operand.
+    MachineOperand &CondReg = CondBr->getOperand(1);
+    CondReg.setIsUndef(Cond[1].isUndef());
+    CondReg.setIsKill(Cond[1].isKill());
+
     if (BytesAdded)
       *BytesAdded = 4;
     return 1;
   }
 
   assert(TBB && FBB);
 
+  MachineInstr *CondBr =
     BuildMI(&MBB, DL, get(Opcode))
     .addMBB(TBB);
   BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
     .addMBB(FBB);
 
+  MachineOperand &CondReg = CondBr->getOperand(1);
+  CondReg.setIsUndef(Cond[1].isUndef());
+  CondReg.setIsKill(Cond[1].isKill());
+
   if (BytesAdded)
       *BytesAdded = 8;
 
   return 2;
 }
 
 bool SIInstrInfo::reverseBranchCondition(
   SmallVectorImpl<MachineOperand> &Cond) const {
-  assert(Cond.size() == 1);
+  assert(Cond.size() == 2);
   Cond[0].setImm(-Cond[0].getImm());
   return false;
 }
 
 static void removeModOperands(MachineInstr &MI) {
   unsigned Opc = MI.getOpcode();
   int Src0ModIdx = AMDGPU::getNamedOperandIdx(Opc,
                                               AMDGPU::OpName::src0_modifiers);