diff --git a/llvm/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp b/llvm/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp --- a/llvm/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp +++ b/llvm/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp @@ -31,6 +31,14 @@ const SIRegisterInfo *TRI; const SIInstrInfo *TII; MachineRegisterInfo *MRI; + LiveIntervals *LIS; + + unsigned AndOpc; + unsigned Andn2Opc; + Register CondReg; + Register ExecReg; + + unsigned optimizeVcndVcmpPair(MachineBasicBlock &MBB); public: static char ID; @@ -68,10 +76,8 @@ return new SIOptimizeExecMaskingPreRA(); } -static bool isFullExecCopy(const MachineInstr& MI, const GCNSubtarget& ST) { - unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; - - if (MI.isFullCopy() && MI.getOperand(1).getReg() == Exec) +static bool isFullExecCopy(const MachineInstr &MI, Register ExecReg) { + if (MI.isFullCopy() && MI.getOperand(1).getReg() == ExecReg) return true; return false; @@ -119,18 +125,8 @@ // lanes. // // Returns %cc register on success. -static unsigned optimizeVcndVcmpPair(MachineBasicBlock &MBB, - const GCNSubtarget &ST, - MachineRegisterInfo &MRI, - LiveIntervals *LIS) { - const SIRegisterInfo *TRI = ST.getRegisterInfo(); - const SIInstrInfo *TII = ST.getInstrInfo(); - bool Wave32 = ST.isWave32(); - const unsigned AndOpc = Wave32 ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64; - const unsigned Andn2Opc = Wave32 ? AMDGPU::S_ANDN2_B32 : AMDGPU::S_ANDN2_B64; - const unsigned CondReg = Wave32 ? AMDGPU::VCC_LO : AMDGPU::VCC; - const unsigned ExecReg = Wave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC; - +unsigned +SIOptimizeExecMaskingPreRA::optimizeVcndVcmpPair(MachineBasicBlock &MBB) { auto I = llvm::find_if(MBB.terminators(), [](const MachineInstr &MI) { unsigned Opc = MI.getOpcode(); return Opc == AMDGPU::S_CBRANCH_VCCZ || @@ -138,8 +134,8 @@ if (I == MBB.terminators().end()) return AMDGPU::NoRegister; - auto *And = TRI->findReachingDef(CondReg, AMDGPU::NoSubRegister, - *I, MRI, LIS); + auto *And = + TRI->findReachingDef(CondReg, AMDGPU::NoSubRegister, *I, *MRI, LIS); if (!And || And->getOpcode() != AndOpc || !And->getOperand(1).isReg() || !And->getOperand(2).isReg()) return AMDGPU::NoRegister; @@ -155,7 +151,7 @@ return AMDGPU::NoRegister; } - auto *Cmp = TRI->findReachingDef(CmpReg, CmpSubReg, *And, MRI, LIS); + auto *Cmp = TRI->findReachingDef(CmpReg, CmpSubReg, *And, *MRI, LIS); if (!Cmp || !(Cmp->getOpcode() == AMDGPU::V_CMP_NE_U32_e32 || Cmp->getOpcode() == AMDGPU::V_CMP_NE_U32_e64) || Cmp->getParent() != And->getParent()) @@ -169,7 +165,7 @@ return AMDGPU::NoRegister; Register SelReg = Op1->getReg(); - auto *Sel = TRI->findReachingDef(SelReg, Op1->getSubReg(), *Cmp, MRI, LIS); + auto *Sel = TRI->findReachingDef(SelReg, Op1->getSubReg(), *Cmp, *MRI, LIS); if (!Sel || Sel->getOpcode() != AMDGPU::V_CNDMASK_B32_e64) return AMDGPU::NoRegister; @@ -212,7 +208,7 @@ // Try to remove compare. Cmp value should not used in between of cmp // and s_and_b64 if VCC or just unused if any other register. - if ((CmpReg.isVirtual() && MRI.use_nodbg_empty(CmpReg)) || + if ((CmpReg.isVirtual() && MRI->use_nodbg_empty(CmpReg)) || (CmpReg == CondReg && std::none_of(std::next(Cmp->getIterator()), Andn2->getIterator(), [&](const MachineInstr &MI) { @@ -224,7 +220,7 @@ Cmp->eraseFromParent(); // Try to remove v_cndmask_b32. - if (SelReg.isVirtual() && MRI.use_nodbg_empty(SelReg)) { + if (SelReg.isVirtual() && MRI->use_nodbg_empty(SelReg)) { LLVM_DEBUG(dbgs() << "Erasing: " << *Sel << '\n'); LIS->RemoveMachineInstrFromMaps(*Sel); @@ -243,16 +239,20 @@ TRI = ST.getRegisterInfo(); TII = ST.getInstrInfo(); MRI = &MF.getRegInfo(); + LIS = &getAnalysis(); + + const bool Wave32 = ST.isWave32(); + AndOpc = Wave32 ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64; + Andn2Opc = Wave32 ? AMDGPU::S_ANDN2_B32 : AMDGPU::S_ANDN2_B64; + CondReg = Wave32 ? AMDGPU::VCC_LO : AMDGPU::VCC; + ExecReg = Wave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC; - MachineRegisterInfo &MRI = MF.getRegInfo(); - LiveIntervals *LIS = &getAnalysis(); DenseSet RecalcRegs({AMDGPU::EXEC_LO, AMDGPU::EXEC_HI}); - unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; bool Changed = false; for (MachineBasicBlock &MBB : MF) { - if (unsigned Reg = optimizeVcndVcmpPair(MBB, ST, MRI, LIS)) { + if (unsigned Reg = optimizeVcndVcmpPair(MBB)) { RecalcRegs.insert(Reg); RecalcRegs.insert(AMDGPU::VCC_LO); RecalcRegs.insert(AMDGPU::VCC_HI); @@ -333,16 +333,17 @@ unsigned ScanThreshold = 10; for (auto I = MBB.rbegin(), E = MBB.rend(); I != E && ScanThreshold--; ++I) { - if (!isFullExecCopy(*I, ST)) + if (!isFullExecCopy(*I, ExecReg)) continue; Register SavedExec = I->getOperand(0).getReg(); - if (SavedExec.isVirtual() && MRI.hasOneNonDBGUse(SavedExec) && - MRI.use_instr_nodbg_begin(SavedExec)->getParent() == I->getParent()) { + if (SavedExec.isVirtual() && MRI->hasOneNonDBGUse(SavedExec) && + MRI->use_instr_nodbg_begin(SavedExec)->getParent() == + I->getParent()) { LLVM_DEBUG(dbgs() << "Redundant EXEC COPY: " << *I << '\n'); LIS->RemoveMachineInstrFromMaps(*I); I->eraseFromParent(); - MRI.replaceRegWith(SavedExec, Exec); + MRI->replaceRegWith(SavedExec, ExecReg); LIS->removeInterval(SavedExec); Changed = true; } @@ -354,7 +355,7 @@ for (auto Reg : RecalcRegs) { if (Reg.isVirtual()) { LIS->removeInterval(Reg); - if (!MRI.reg_empty(Reg)) + if (!MRI->reg_empty(Reg)) LIS->createAndComputeVirtRegInterval(Reg); } else { LIS->removeAllRegUnitsForPhysReg(Reg);