AMDGPU: WQM cleanups

[arsenm]

Summary

• Add new TTI instruction checks
  • Don't use const for blocks that are mutated.
• Checking isBranch and isTerminator should be redundant

[nhaehnle]

LGTM

[arsenm]

r275252

lib/Target/AMDGPU/SIInstrInfo.h

public:
   static bool isDPP(const MachineInstr &MI) {
     return MI.getDesc().TSFlags & SIInstrFlags::DPP;
   }
 
   bool isDPP(uint16_t Opcode) const {
     return get(Opcode).TSFlags & SIInstrFlags::DPP;
   }
 
+  static bool isScalarUnit(const MachineInstr &MI) {
+    return MI.getDesc().TSFlags & (SIInstrFlags::SALU | SIInstrFlags::SMRD);
+  }
+
+  static bool usesVM_CNT(const MachineInstr &MI) {
+    return MI.getDesc().TSFlags & SIInstrFlags::VM_CNT;
+  }
+
   bool isVGPRCopy(const MachineInstr &MI) const {
     assert(MI.isCopy());
     unsigned Dest = MI.getOperand(0).getReg();
     const MachineFunction &MF = *MI.getParent()->getParent();
     const MachineRegisterInfo &MRI = MF.getRegInfo();
     return !RI.isSGPRReg(MRI, Dest);
   }
 

lib/Target/AMDGPU/SIWholeQuadMode.cpp

 ///      block of the region.
 ///
 //===----------------------------------------------------------------------===//
 
 #include "AMDGPU.h"
 #include "AMDGPUSubtarget.h"
 #include "SIInstrInfo.h"
 #include "SIMachineFunctionInfo.h"
-#include "llvm/CodeGen/MachineDominanceFrontier.h"
-#include "llvm/CodeGen/MachineDominators.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/IR/Constants.h"
 
 using namespace llvm;
 
 #define DEBUG_TYPE "si-wqm"
 
 namespace {
 
 enum {
   StateWQM = 0x1,
   StateExact = 0x2,
 };
 
 struct InstrInfo {
   char Needs = 0;
   char OutNeeds = 0;
 };
 
 struct BlockInfo {
   char Needs = 0;
   char InNeeds = 0;
   char OutNeeds = 0;
 };
 
 struct WorkItem {
-  const MachineBasicBlock *MBB = nullptr;
-  const MachineInstr *MI = nullptr;
+  MachineBasicBlock *MBB = nullptr;
+  MachineInstr *MI = nullptr;
 
   WorkItem() {}
-  WorkItem(const MachineBasicBlock *MBB) : MBB(MBB) {}
-  WorkItem(const MachineInstr *MI) : MI(MI) {}
+  WorkItem(MachineBasicBlock *MBB) : MBB(MBB) {}
+  WorkItem(MachineInstr *MI) : MI(MI) {}
 };
 
 class SIWholeQuadMode : public MachineFunctionPass {
 private:
   const SIInstrInfo *TII;
   const SIRegisterInfo *TRI;
   MachineRegisterInfo *MRI;
 
   DenseMap<const MachineInstr *, InstrInfo> Instructions;
-  DenseMap<const MachineBasicBlock *, BlockInfo> Blocks;
+  DenseMap<MachineBasicBlock *, BlockInfo> Blocks;
   SmallVector<const MachineInstr *, 2> ExecExports;
   SmallVector<MachineInstr *, 1> LiveMaskQueries;
 
-  char scanInstructions(MachineFunction &MF, std::vector<WorkItem>& Worklist);
-  void propagateInstruction(const MachineInstr &MI, std::vector<WorkItem>& Worklist);
-  void propagateBlock(const MachineBasicBlock &MBB, std::vector<WorkItem>& Worklist);
+  char scanInstructions(MachineFunction &MF, std::vector<WorkItem> &Worklist);
+  void propagateInstruction(MachineInstr &MI, std::vector<WorkItem> &Worklist);
+  void propagateBlock(MachineBasicBlock &MBB, std::vector<WorkItem> &Worklist);
   char analyzeFunction(MachineFunction &MF);
 
   void toExact(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before,
                unsigned SaveWQM, unsigned LiveMaskReg);
   void toWQM(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before,
              unsigned SavedWQM);
   void processBlock(MachineBasicBlock &MBB, unsigned LiveMaskReg, bool isEntry);
 
void getAnalysisUsage(AnalysisUsage &AU) const override {
     MachineFunctionPass::getAnalysisUsage(AU);
   }
 };
 
 } // End anonymous namespace
 
 char SIWholeQuadMode::ID = 0;
 
-INITIALIZE_PASS_BEGIN(SIWholeQuadMode, DEBUG_TYPE,
-                      "SI Whole Quad Mode", false, false)
-INITIALIZE_PASS_END(SIWholeQuadMode, DEBUG_TYPE,
-                      "SI Whole Quad Mode", false, false)
+INITIALIZE_PASS(SIWholeQuadMode, DEBUG_TYPE,
 
 char &llvm::SIWholeQuadModeID = SIWholeQuadMode::ID;
 
 FunctionPass *llvm::createSIWholeQuadModePass() {
   return new SIWholeQuadMode;
 }
 
 // Scan instructions to determine which ones require an Exact execmask and
 // which ones seed WQM requirements.
 char SIWholeQuadMode::scanInstructions(MachineFunction &MF,
                                        std::vector<WorkItem> &Worklist) {
   char GlobalFlags = 0;
   bool WQMOutputs = MF.getFunction()->hasFnAttribute("amdgpu-ps-wqm-outputs");
 
   for (auto BI = MF.begin(), BE = MF.end(); BI != BE; ++BI) {
     MachineBasicBlock &MBB = *BI;
 
     for (auto II = MBB.begin(), IE = MBB.end(); II != IE; ++II) {
       MachineInstr &MI = *II;
       unsigned Opcode = MI.getOpcode();
       char Flags = 0;
 
       if (TII->isWQM(Opcode) || TII->isDS(Opcode)) {
         Flags = StateWQM;
-      } else if (TII->get(Opcode).mayStore() &&
-                 (MI.getDesc().TSFlags & SIInstrFlags::VM_CNT)) {
+      } else if (MI.mayStore() && TII->usesVM_CNT(MI)) {
         Flags = StateExact;
       } else {
         // Handle export instructions with the exec mask valid flag set
         if (Opcode == AMDGPU::EXP) {
           if (MI.getOperand(4).getImm() != 0)
             ExecExports.push_back(&MI);
         } else if (Opcode == AMDGPU::SI_PS_LIVE) {
           LiveMaskQueries.push_back(&MI);
if (WQMOutputs && MBB.succ_empty()) {
       Worklist.push_back(&MBB);
       GlobalFlags |= StateExact;
     }
   }
 
   return GlobalFlags;
 }
 
-void SIWholeQuadMode::propagateInstruction(const MachineInstr &MI,
+void SIWholeQuadMode::propagateInstruction(MachineInstr &MI,
                                            std::vector<WorkItem>& Worklist) {
-  const MachineBasicBlock &MBB = *MI.getParent();
+  MachineBasicBlock *MBB = MI.getParent();
   InstrInfo II = Instructions[&MI]; // take a copy to prevent dangling references
-  BlockInfo &BI = Blocks[&MBB];
+  BlockInfo &BI = Blocks[MBB];
 
   // Control flow-type instructions that are followed by WQM computations
   // must themselves be in WQM.
-  if ((II.OutNeeds & StateWQM) && !(II.Needs & StateWQM) &&
-      (MI.isBranch() || MI.isTerminator())) {
+  if ((II.OutNeeds & StateWQM) && !(II.Needs & StateWQM) && MI.isTerminator()) {
     Instructions[&MI].Needs = StateWQM;
     II.Needs = StateWQM;
   }
 
   // Propagate to block level
   BI.Needs |= II.Needs;
   if ((BI.InNeeds | II.Needs) != BI.InNeeds) {
     BI.InNeeds |= II.Needs;
-    Worklist.push_back(&MBB);
+    Worklist.push_back(MBB);
   }
 
   // Propagate backwards within block
-  if (const MachineInstr *PrevMI = MI.getPrevNode()) {
+  if (MachineInstr *PrevMI = MI.getPrevNode()) {
     char InNeeds = II.Needs | II.OutNeeds;
     if (!PrevMI->isPHI()) {
       InstrInfo &PrevII = Instructions[PrevMI];
       if ((PrevII.OutNeeds | InNeeds) != PrevII.OutNeeds) {
         PrevII.OutNeeds |= InNeeds;
         Worklist.push_back(PrevMI);
       }
     }
for (const MachineOperand &Use : MI.uses()) {
     // to a NeedsWQM instruction).
     //
     // Note: VCC appears e.g. in 64-bit addition with carry - theoretically we
     // have to trace this, in practice it happens for 64-bit computations like
     // pointers where both dwords are followed already anyway.
     if (!TargetRegisterInfo::isVirtualRegister(Use.getReg()))
       continue;
 
-    for (const MachineOperand &Def : MRI->def_operands(Use.getReg())) {
-      const MachineInstr *DefMI = Def.getParent();
-      InstrInfo &DefII = Instructions[DefMI];
+    for (MachineInstr &DefMI : MRI->def_instructions(Use.getReg())) {
+      InstrInfo &DefII = Instructions[&DefMI];
 
       // Obviously skip if DefMI is already flagged as NeedWQM.
       //
       // The instruction might also be flagged as NeedExact. This happens when
       // the result of an atomic is used in a WQM computation. In this case,
       // the atomic must not run for helper pixels and the WQM result is
       // undefined.
       if (DefII.Needs != 0)
         continue;
 
       DefII.Needs = StateWQM;
-      Worklist.push_back(DefMI);
+      Worklist.push_back(&DefMI);
     }
   }
 }
 
-void SIWholeQuadMode::propagateBlock(const MachineBasicBlock &MBB,
+void SIWholeQuadMode::propagateBlock(MachineBasicBlock &MBB,
                                      std::vector<WorkItem>& Worklist) {
-  BlockInfo BI = Blocks[&MBB]; // take a copy to prevent dangling references
+  BlockInfo BI = Blocks[&MBB]; // Make a copy to prevent dangling references.
 
   // Propagate through instructions
   if (!MBB.empty()) {
-    const MachineInstr *LastMI = &*MBB.rbegin();
+    MachineInstr *LastMI = &*MBB.rbegin();
     InstrInfo &LastII = Instructions[LastMI];
     if ((LastII.OutNeeds | BI.OutNeeds) != LastII.OutNeeds) {
       LastII.OutNeeds |= BI.OutNeeds;
       Worklist.push_back(LastMI);
     }
   }
 
   // Predecessor blocks must provide for our WQM/Exact needs.
-  for (const MachineBasicBlock *Pred : MBB.predecessors()) {
+  for (MachineBasicBlock *Pred : MBB.predecessors()) {
     BlockInfo &PredBI = Blocks[Pred];
     if ((PredBI.OutNeeds | BI.InNeeds) == PredBI.OutNeeds)
       continue;
 
     PredBI.OutNeeds |= BI.InNeeds;
     PredBI.InNeeds |= BI.InNeeds;
     Worklist.push_back(Pred);
   }
 
-  // All successors must be prepared to accept the same set of WQM/Exact
-  // data.
-  for (const MachineBasicBlock *Succ : MBB.successors()) {
+  // All successors must be prepared to accept the same set of WQM/Exact data.
+  for (MachineBasicBlock *Succ : MBB.successors()) {
     BlockInfo &SuccBI = Blocks[Succ];
     if ((SuccBI.InNeeds | BI.OutNeeds) == SuccBI.InNeeds)
       continue;
 
     SuccBI.InNeeds |= BI.OutNeeds;
     Worklist.push_back(Succ);
   }
 }
void SIWholeQuadMode::processBlock(MachineBasicBlock &MBB, unsigned LiveMaskReg,
   char State = isEntry ? StateExact : StateWQM;
 
   auto II = MBB.getFirstNonPHI(), IE = MBB.end();
   while (II != IE) {
     MachineInstr &MI = *II;
     ++II;
 
     // Skip instructions that are not affected by EXEC
-    if (MI.getDesc().TSFlags & (SIInstrFlags::SALU | SIInstrFlags::SMRD) &&
-        !MI.isBranch() && !MI.isTerminator())
+    if (TII->isScalarUnit(MI) && !MI.isTerminator())
       continue;
 
     // Generic instructions such as COPY will either disappear by register
     // coalescing or be lowered to SALU or VALU instructions.
     if (TargetInstrInfo::isGenericOpcode(MI.getOpcode())) {
       if (MI.getNumExplicitOperands() >= 1) {
         const MachineOperand &Op = MI.getOperand(0);
         if (Op.isReg()) {
while (II != IE) {
     char OutNeeds = 0;
     auto InstrInfoIt = Instructions.find(&MI);
     if (InstrInfoIt != Instructions.end()) {
       Needs = InstrInfoIt->second.Needs;
       OutNeeds = InstrInfoIt->second.OutNeeds;
 
       // Make sure to switch to Exact mode before the end of the block when
       // Exact and only Exact is needed further downstream.
-      if (OutNeeds == StateExact && (MI.isBranch() || MI.isTerminator())) {
+      if (OutNeeds == StateExact && MI.isTerminator()) {
         assert(Needs == 0);
         Needs = StateExact;
       }
     }
 
     // State switching
     if (Needs && State != Needs) {
       if (Needs == StateExact) {
if ((BI.OutNeeds & StateWQM) && State != StateWQM) {
     toWQM(MBB, MBB.end(), SavedWQMReg);
   } else if (BI.OutNeeds == StateExact && State != StateExact) {
     toExact(MBB, MBB.end(), 0, LiveMaskReg);
   }
 }
 
 void SIWholeQuadMode::lowerLiveMaskQueries(unsigned LiveMaskReg) {
   for (MachineInstr *MI : LiveMaskQueries) {
-    DebugLoc DL = MI->getDebugLoc();
+    const DebugLoc &DL = MI->getDebugLoc();
     unsigned Dest = MI->getOperand(0).getReg();
     BuildMI(*MI->getParent(), MI, DL, TII->get(AMDGPU::COPY), Dest)
         .addReg(LiveMaskReg);
     MI->eraseFromParent();
   }
 }
 
 bool SIWholeQuadMode::runOnMachineFunction(MachineFunction &MF) {
if (GlobalFlags == StateWQM) {
     // EntryMI may become invalid here
     return true;
   }
 
   lowerLiveMaskQueries(LiveMaskReg);
   EntryMI = nullptr;
 
   // Handle the general case
-  for (const auto &BII : Blocks)
-    processBlock(const_cast<MachineBasicBlock &>(*BII.first), LiveMaskReg,
-                 BII.first == &*MF.begin());
+  for (auto BII : Blocks)
+    processBlock(*BII.first, LiveMaskReg, BII.first == &*MF.begin());
 
   return true;
 }