Index: llvm/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp =================================================================== --- llvm/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp +++ llvm/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp @@ -7,15 +7,8 @@ //===----------------------------------------------------------------------===// // /// \file -/// This pass removes redundant S_OR_B64 instructions enabling lanes in -/// the exec. If two SI_END_CF (lowered as S_OR_B64) come together without any -/// vector instructions between them we can only keep outer SI_END_CF, given -/// that CFG is structured and exec bits of the outer end statement are always -/// not less than exec bit of the inner one. -/// -/// This needs to be done before the RA to eliminate saved exec bits registers -/// but after register coalescer to have no vector registers copies in between -/// of different end cf statements. +/// This pass performs exec mask handling peephole optimizations which needs +/// to be done before register allocation to reduce register pressure. /// //===----------------------------------------------------------------------===// @@ -31,10 +24,6 @@ #define DEBUG_TYPE "si-optimize-exec-masking-pre-ra" -static cl::opt -RemoveRedundantEndcf("amdgpu-remove-redundant-endcf", - cl::init(false), cl::ReallyHidden); - namespace { class SIOptimizeExecMaskingPreRA : public MachineFunctionPass { @@ -43,14 +32,6 @@ const SIInstrInfo *TII; MachineRegisterInfo *MRI; -public: - MachineBasicBlock::iterator skipIgnoreExecInsts( - MachineBasicBlock::iterator I, MachineBasicBlock::iterator E) const; - - MachineBasicBlock::iterator skipIgnoreExecInstsTrivialSucc( - MachineBasicBlock *&MBB, - MachineBasicBlock::iterator It) const; - public: static char ID; @@ -87,17 +68,6 @@ return new SIOptimizeExecMaskingPreRA(); } -static bool isEndCF(const MachineInstr &MI, const SIRegisterInfo *TRI, - const GCNSubtarget &ST) { - if (ST.isWave32()) { - return MI.getOpcode() == AMDGPU::S_OR_B32 && - MI.modifiesRegister(AMDGPU::EXEC_LO, TRI); - } - - return MI.getOpcode() == AMDGPU::S_OR_B64 && - MI.modifiesRegister(AMDGPU::EXEC, TRI); -} - static bool isFullExecCopy(const MachineInstr& MI, const GCNSubtarget& ST) { unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; @@ -109,71 +79,6 @@ return false; } -static unsigned getOrNonExecReg(const MachineInstr &MI, - const SIInstrInfo &TII, - const GCNSubtarget& ST) { - unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; - auto Op = TII.getNamedOperand(MI, AMDGPU::OpName::src1); - if (Op->isReg() && Op->getReg() != Exec) - return Op->getReg(); - Op = TII.getNamedOperand(MI, AMDGPU::OpName::src0); - if (Op->isReg() && Op->getReg() != Exec) - return Op->getReg(); - return AMDGPU::NoRegister; -} - -static MachineInstr* getOrExecSource(const MachineInstr &MI, - const SIInstrInfo &TII, - const MachineRegisterInfo &MRI, - const GCNSubtarget& ST) { - auto SavedExec = getOrNonExecReg(MI, TII, ST); - if (SavedExec == AMDGPU::NoRegister) - return nullptr; - auto SaveExecInst = MRI.getUniqueVRegDef(SavedExec); - if (!SaveExecInst || !isFullExecCopy(*SaveExecInst, ST)) - return nullptr; - return SaveExecInst; -} - -/// Skip over instructions that don't care about the exec mask. -MachineBasicBlock::iterator SIOptimizeExecMaskingPreRA::skipIgnoreExecInsts( - MachineBasicBlock::iterator I, MachineBasicBlock::iterator E) const { - for ( ; I != E; ++I) { - if (TII->mayReadEXEC(*MRI, *I)) - break; - } - - return I; -} - -// Skip to the next instruction, ignoring debug instructions, and trivial block -// boundaries (blocks that have one (typically fallthrough) successor, and the -// successor has one predecessor. -MachineBasicBlock::iterator -SIOptimizeExecMaskingPreRA::skipIgnoreExecInstsTrivialSucc( - MachineBasicBlock *&MBB, - MachineBasicBlock::iterator It) const { - - do { - It = skipIgnoreExecInsts(It, MBB->end()); - if (It != MBB->end() || MBB->succ_size() != 1) - break; - - // If there is one trivial successor, advance to the next block. - MachineBasicBlock *Succ = *MBB->succ_begin(); - - // TODO: Is this really necessary? - if (!MBB->isLayoutSuccessor(Succ)) - break; - - It = Succ->begin(); - MBB = Succ; - } while (true); - - return It; -} - - // Optimize sequence // %sel = V_CNDMASK_B32_e64 0, 1, %cc // %cmp = V_CMP_NE_U32 1, %1 @@ -383,63 +288,30 @@ continue; } - if (!RemoveRedundantEndcf) - continue; - - // Try to collapse adjacent endifs. - // The assumption is that conditional regions are perfectly nested and - // a mask restored at the exit from the inner block will be completely - // covered by a mask restored in the outer. - auto E = MBB.end(); - auto Lead = skipDebugInstructionsForward(MBB.begin(), E); - if (MBB.succ_size() != 1 || Lead == E || !isEndCF(*Lead, TRI, ST)) - continue; - - MachineBasicBlock *TmpMBB = &MBB; - auto NextLead = skipIgnoreExecInstsTrivialSucc(TmpMBB, std::next(Lead)); - if (NextLead == TmpMBB->end() || !isEndCF(*NextLead, TRI, ST) || - !getOrExecSource(*NextLead, *TII, MRI, ST)) - continue; - - LLVM_DEBUG(dbgs() << "Redundant EXEC = S_OR_B64 found: " << *Lead << '\n'); - - auto SaveExec = getOrExecSource(*Lead, *TII, MRI, ST); - unsigned SaveExecReg = getOrNonExecReg(*Lead, *TII, ST); - for (auto &Op : Lead->operands()) { - if (Op.isReg()) - RecalcRegs.insert(Op.getReg()); - } - - LIS->RemoveMachineInstrFromMaps(*Lead); - Lead->eraseFromParent(); - if (SaveExecReg) { - LIS->removeInterval(SaveExecReg); - LIS->createAndComputeVirtRegInterval(SaveExecReg); - } - - Changed = true; - - // If the only use of saved exec in the removed instruction is S_AND_B64 - // fold the copy now. - if (!SaveExec || !SaveExec->isFullCopy()) - continue; + // If the only user of a logical operation is move to exec, fold it now + // to prevent forming of saveexec. I.e: + // + // %0:sreg_64 = COPY $exec + // %1:sreg_64 = S_AND_B64 %0:sreg_64, %2:sreg_64 + // => + // %1 = S_AND_B64 $exec, %2:sreg_64 + unsigned ScanThreshold = 10; + for (auto I = MBB.rbegin(), E = MBB.rend(); I != E + && ScanThreshold--; ++I) { + if (!isFullExecCopy(*I, ST)) + continue; - Register SavedExec = SaveExec->getOperand(0).getReg(); - bool SafeToReplace = true; - for (auto& U : MRI.use_nodbg_instructions(SavedExec)) { - if (U.getParent() != SaveExec->getParent()) { - SafeToReplace = false; - break; + Register SavedExec = I->getOperand(0).getReg(); + if (SavedExec.isVirtual() && MRI.hasOneNonDBGUse(SavedExec) && + MRI.use_instr_nodbg_begin(SavedExec)->getParent() == I->getParent()) { + LLVM_DEBUG(dbgs() << "Redundant EXEC COPY: " << *I << '\n'); + LIS->RemoveMachineInstrFromMaps(*I); + I->eraseFromParent(); + MRI.replaceRegWith(SavedExec, Exec); + LIS->removeInterval(SavedExec); + Changed = true; } - - LLVM_DEBUG(dbgs() << "Redundant EXEC COPY: " << *SaveExec << '\n'); - } - - if (SafeToReplace) { - LIS->RemoveMachineInstrFromMaps(*SaveExec); - SaveExec->eraseFromParent(); - MRI.replaceRegWith(SavedExec, Exec); - LIS->removeInterval(SavedExec); + break; } } Index: llvm/test/CodeGen/AMDGPU/collapse-endcf.ll =================================================================== --- llvm/test/CodeGen/AMDGPU/collapse-endcf.ll +++ llvm/test/CodeGen/AMDGPU/collapse-endcf.ll @@ -3,10 +3,7 @@ ; GCN-LABEL: {{^}}simple_nested_if: ; GCN: s_and_saveexec_b64 [[SAVEEXEC:s\[[0-9:]+\]]] ; GCN-NEXT: s_cbranch_execz [[ENDIF:BB[0-9_]+]] - -; TODO: this does not need to save exec, just perform the and. -; GCN: s_and_saveexec_b64 s[{{[0-9:]+}}], vcc - +; GCN: s_and_b64 exec, exec, vcc ; GCN-NEXT: s_cbranch_execz [[ENDIF]] ; GCN-NEXT: ; %bb.{{[0-9]+}}: ; GCN: store_dword @@ -145,7 +142,7 @@ ; GCN-NEXT: s_cbranch_execz [[ENDIF_OUTER:BB[0-9_]+]] ; GCN-NEXT: ; %bb.{{[0-9]+}}: ; GCN: store_dword -; GCN-NEXT: s_and_saveexec_b64 [[SAVEEXEC_INNER_IF_OUTER_THEN:s\[[0-9:]+\]]] +; GCN-NEXT: s_and_b64 exec, exec, ; GCN-NEXT: s_cbranch_execz [[FLOW1:BB[0-9_]+]] ; GCN-NEXT: ; %bb.{{[0-9]+}}: ; GCN: store_dword @@ -217,7 +214,10 @@ ; GCN-LABEL: {{^}}scc_liveness: -; GCN: %bb10 +; GCN: [[BB1_OUTER_LOOP:BB[0-9]+_[0-9]+]]: +; GCN: s_or_b64 exec, exec, [[SAVEEXEC_OUTER:s\[[0-9:]+\]]] +; +; GCN: [[BB1_INNER_LOOP:BB[0-9]+_[0-9]+]]: ; GCN: s_or_b64 exec, exec, s{{\[[0-9]+:[0-9]+\]}} ; GCN: s_andn2_b64 ; GCN-NEXT: s_cbranch_execz @@ -228,8 +228,8 @@ ; GCN: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen -; TODO: this does not need to save exec, just perform the and. -; GCN: s_and_saveexec_b64 s[{{[0-9:]+}}], {{vcc|s\[[0-9:]+\]}} +; GCN: s_and_saveexec_b64 [[SAVEEXEC_OUTER]], {{vcc|s\[[0-9:]+\]}} +; GCN-NEXT: s_cbranch_execz [[BB1_OUTER_LOOP]] ; GCN-NOT: s_or_b64 exec, exec