diff --git a/llvm/include/llvm/CodeGen/LiveVariables.h b/llvm/include/llvm/CodeGen/LiveVariables.h --- a/llvm/include/llvm/CodeGen/LiveVariables.h +++ b/llvm/include/llvm/CodeGen/LiveVariables.h @@ -188,6 +188,12 @@ //===--------------------------------------------------------------------===// // API to update live variable information + /// Recompute liveness from scratch for a virtual register \p Reg that is + /// known to have a single def that dominates all uses. This can be useful + /// after removing some uses of \p Reg. It is not necessary for the whole + /// machine function to be in SSA form. + void recomputeForSingleDefVirtReg(Register Reg); + /// replaceKillInstruction - Update register kill info by replacing a kill /// instruction with a new one. void replaceKillInstruction(Register Reg, MachineInstr &OldMI, diff --git a/llvm/lib/CodeGen/LiveVariables.cpp b/llvm/lib/CodeGen/LiveVariables.cpp --- a/llvm/lib/CodeGen/LiveVariables.cpp +++ b/llvm/lib/CodeGen/LiveVariables.cpp @@ -669,6 +669,84 @@ return false; } +void LiveVariables::recomputeForSingleDefVirtReg(Register Reg) { + assert(Reg.isVirtual()); + + VarInfo &VI = getVarInfo(Reg); + VI.AliveBlocks.clear(); + VI.Kills.clear(); + + MachineInstr &DefMI = *MRI->getUniqueVRegDef(Reg); + MachineBasicBlock &DefBB = *DefMI.getParent(); + + // Handle the case where all uses have been removed. + if (MRI->use_nodbg_empty(Reg)) { + VI.Kills.push_back(&DefMI); + DefMI.addRegisterDead(Reg, nullptr); + return; + } + DefMI.clearRegisterDeads(Reg); + + // Initialize a worklist of BBs that Reg is live-to-end of. (Here + // "live-to-end" means Reg is live at the end of a block even if it is only + // live because of phi uses in a successor. This is different from isLiveOut() + // which does not consider phi uses.) + SmallVector LiveToEndBlocks; + SparseBitVector<> UseBlocks; + for (auto &UseMO : MRI->use_nodbg_operands(Reg)) { + UseMO.setIsKill(false); + MachineInstr &UseMI = *UseMO.getParent(); + MachineBasicBlock &UseBB = *UseMI.getParent(); + UseBlocks.set(UseBB.getNumber()); + if (UseMI.isPHI()) { + // If Reg is used in a phi then it is live-to-end of the corresponding + // predecessor. + unsigned Idx = UseMI.getOperandNo(&UseMO); + LiveToEndBlocks.push_back(UseMI.getOperand(Idx + 1).getMBB()); + } else if (&UseBB == &DefBB) { + // A non-phi use in the same BB as the single def must come after the def. + } else { + // Otherwise Reg must be live-to-end of all predecessors. + LiveToEndBlocks.append(UseBB.pred_begin(), UseBB.pred_end()); + } + } + + // Iterate over the worklist adding blocks to AliveBlocks. + while (!LiveToEndBlocks.empty()) { + MachineBasicBlock &BB = *LiveToEndBlocks.pop_back_val(); + if (&BB == &DefBB) + continue; + if (VI.AliveBlocks.test(BB.getNumber())) + continue; + VI.AliveBlocks.set(BB.getNumber()); + LiveToEndBlocks.append(BB.pred_begin(), BB.pred_end()); + } + + // Recompute kill flags. For each block in which Reg is used but is not + // live-through, find the last instruction that uses Reg. Ignore phi nodes + // because they should not be included in Kills. + for (unsigned UseBBNum : UseBlocks) { + if (VI.AliveBlocks.test(UseBBNum)) + continue; + MachineBasicBlock &UseBB = *MF->getBlockNumbered(UseBBNum); + // Only add kills in DefBB if all uses are in DefBB. + if (&UseBB == &DefBB && UseBlocks.count() != 1) + continue; + for (auto &MI : reverse(UseBB)) { + if (MI.isDebugOrPseudoInstr()) + continue; + if (MI.isPHI()) + break; + if (MI.readsRegister(Reg)) { + assert(!MI.killsRegister(Reg)); + MI.addRegisterKilled(Reg, nullptr); + VI.Kills.push_back(&MI); + break; + } + } + } +} + /// replaceKillInstruction - Update register kill info by replacing a kill /// instruction with a new one. void LiveVariables::replaceKillInstruction(Register Reg, MachineInstr &OldMI, diff --git a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp --- a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp +++ b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp @@ -52,6 +52,7 @@ #include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "llvm/ADT/SmallSet.h" #include "llvm/CodeGen/LiveIntervals.h" +#include "llvm/CodeGen/LiveVariables.h" #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFunctionPass.h" @@ -70,6 +71,7 @@ const SIRegisterInfo *TRI = nullptr; const SIInstrInfo *TII = nullptr; LiveIntervals *LIS = nullptr; + LiveVariables *LV = nullptr; MachineDominatorTree *MDT = nullptr; MachineRegisterInfo *MRI = nullptr; SetVector LoweredEndCf; @@ -237,6 +239,8 @@ BuildMI(MBB, I, DL, TII->get(AndOpc), Tmp) .addReg(CopyReg) .add(Cond); + if (LV) + LV->replaceKillInstruction(Cond.getReg(), MI, *And); setImpSCCDefDead(*And, true); @@ -254,6 +258,8 @@ MachineInstr *SetExec = BuildMI(MBB, I, DL, TII->get(MovTermOpc), Exec) .addReg(Tmp, RegState::Kill); + if (LV) + LV->getVarInfo(Tmp).Kills.push_back(SetExec); // Skip ahead to the unconditional branch in case there are other terminators // present. @@ -307,6 +313,8 @@ MachineInstr *OrSaveExec = BuildMI(MBB, Start, DL, TII->get(OrSaveExecOpc), SaveReg) .add(MI.getOperand(1)); // Saved EXEC + if (LV) + LV->replaceKillInstruction(MI.getOperand(1).getReg(), MI, *OrSaveExec); MachineBasicBlock *DestBB = MI.getOperand(2).getMBB(); @@ -380,15 +388,22 @@ And = BuildMI(MBB, &MI, DL, TII->get(AndOpc), AndReg) .addReg(Exec) .add(MI.getOperand(1)); + if (LV) + LV->replaceKillInstruction(MI.getOperand(1).getReg(), MI, *And); Or = BuildMI(MBB, &MI, DL, TII->get(OrOpc), Dst) .addReg(AndReg) .add(MI.getOperand(2)); if (LIS) LIS->createAndComputeVirtRegInterval(AndReg); - } else + } else { Or = BuildMI(MBB, &MI, DL, TII->get(OrOpc), Dst) .add(MI.getOperand(1)) .add(MI.getOperand(2)); + if (LV) + LV->replaceKillInstruction(MI.getOperand(1).getReg(), MI, *Or); + } + if (LV) + LV->replaceKillInstruction(MI.getOperand(2).getReg(), MI, *Or); if (LIS) { if (And) @@ -490,6 +505,8 @@ BuildMI(MBB, InsPt, DL, TII->get(Opcode), Exec) .addReg(Exec) .add(MI.getOperand(0)); + if (LV) + LV->replaceKillInstruction(MI.getOperand(0).getReg(), MI, *NewMI); LoweredEndCf.insert(NewMI); @@ -581,7 +598,12 @@ LLVM_DEBUG(dbgs() << "Skip redundant "; MI->dump()); if (LIS) LIS->RemoveMachineInstrFromMaps(*MI); + Register Reg; + if (LV) + Reg = TII->getNamedOperand(*MI, AMDGPU::OpName::src1)->getReg(); MI->eraseFromParent(); + if (LV) + LV->recomputeForSingleDefVirtReg(Reg); removeMBBifRedundant(MBB); } } @@ -697,6 +719,8 @@ auto BfeMI = BuildMI(*MBB, FirstMI, DL, TII->get(AMDGPU::S_BFE_U32), CountReg) .addReg(InputReg) .addImm((MI.getOperand(1).getImm() & Mask) | 0x70000); + if (LV) + LV->recomputeForSingleDefVirtReg(InputReg); auto BfmMI = BuildMI(*MBB, FirstMI, DL, TII->get(IsWave32 ? AMDGPU::S_BFM_B32 : AMDGPU::S_BFM_B64), Exec) @@ -705,6 +729,8 @@ auto CmpMI = BuildMI(*MBB, FirstMI, DL, TII->get(AMDGPU::S_CMP_EQ_U32)) .addReg(CountReg, RegState::Kill) .addImm(WavefrontSize); + if (LV) + LV->getVarInfo(CountReg).Kills.push_back(CmpMI); auto CmovMI = BuildMI(*MBB, FirstMI, DL, TII->get(IsWave32 ? AMDGPU::S_CMOV_B32 : AMDGPU::S_CMOV_B64), @@ -777,17 +803,14 @@ } bool SILowerControlFlow::runOnMachineFunction(MachineFunction &MF) { - // FIXME: This pass causes verification failures. - // See: https://bugs.llvm.org/show_bug.cgi?id=52204 - MF.getProperties().set( - MachineFunctionProperties::Property::FailsVerification); - const GCNSubtarget &ST = MF.getSubtarget(); TII = ST.getInstrInfo(); TRI = &TII->getRegisterInfo(); // This doesn't actually need LiveIntervals, but we can preserve them. LIS = getAnalysisIfAvailable(); + // This doesn't actually need LiveVariables, but we can preserve them. + LV = getAnalysisIfAvailable(); MDT = getAnalysisIfAvailable(); MRI = &MF.getRegInfo(); BoolRC = TRI->getBoolRC(); diff --git a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp --- a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp +++ b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp @@ -1498,11 +1498,6 @@ } bool SIWholeQuadMode::runOnMachineFunction(MachineFunction &MF) { - // This pass is a convenient place to re-enable machine verification after the - // problems caused by SILowerControlFlow have been fixed. - MF.getProperties().reset( - MachineFunctionProperties::Property::FailsVerification); - LLVM_DEBUG(dbgs() << "SI Whole Quad Mode on " << MF.getName() << " ------------- \n"); LLVM_DEBUG(MF.dump(););