AMDGPU: Fix liveness when expanding m0 loop

[• tstellarAMD]

LGTM.

[arsenm]

r273514

lib/Target/AMDGPU/SIInstructions.td

def SI_RETURN : InstSI <
   let isReturn = 1;
   let isPseudo = 1;
   let isCodeGenOnly = 1;
   let hasSideEffects = 1;
   let SALU = 1;
   let hasNoSchedulingInfo = 1;
 }
 
-let Uses = [EXEC], Defs = [EXEC, VCC, M0] in {
+let Uses = [EXEC], Defs = [EXEC, VCC, M0],
+  UseNamedOperandTable = 1 in {
 
 class SI_INDIRECT_SRC<RegisterClass rc> : InstSI <
-  (outs VGPR_32:$dst, SReg_64:$temp),
-  (ins rc:$src, VSrc_32:$idx, i32imm:$off)
+  (outs VGPR_32:$vdst, SReg_64:$sdst),
+  (ins rc:$src, VSrc_32:$idx, i32imm:$offset)
 >;
 
 class SI_INDIRECT_DST<RegisterClass rc> : InstSI <
-  (outs rc:$dst, SReg_64:$temp),
-  (ins unknown:$src, VSrc_32:$idx, i32imm:$off, VGPR_32:$val)> {
-  let Constraints = "$src = $dst";
+  (outs rc:$vdst, SReg_64:$sdst),
+  (ins unknown:$src, VSrc_32:$idx, i32imm:$offset, VGPR_32:$val)> {
+  let Constraints = "$src = $vdst";
 }
 
 // TODO: We can support indirect SGPR access.
 def SI_INDIRECT_SRC_V1 : SI_INDIRECT_SRC<VGPR_32>;
 def SI_INDIRECT_SRC_V2 : SI_INDIRECT_SRC<VReg_64>;
 def SI_INDIRECT_SRC_V4 : SI_INDIRECT_SRC<VReg_128>;
 def SI_INDIRECT_SRC_V8 : SI_INDIRECT_SRC<VReg_256>;
 def SI_INDIRECT_SRC_V16 : SI_INDIRECT_SRC<VReg_512>;

lib/Target/AMDGPU/SILowerControlFlow.cpp

 /// label1:
 /// %EXEC = S_OR_B64 %EXEC, %SGPR0     // Re-enable saved exec mask bits
 //===----------------------------------------------------------------------===//
 
 #include "AMDGPU.h"
 #include "AMDGPUSubtarget.h"
 #include "SIInstrInfo.h"
 #include "SIMachineFunctionInfo.h"
+#include "llvm/CodeGen/LivePhysRegs.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/IR/Constants.h"
 
 using namespace llvm;
private:
   void IfBreak(MachineInstr &MI);
   void ElseBreak(MachineInstr &MI);
   void Loop(MachineInstr &MI);
   void EndCf(MachineInstr &MI);
 
   void Kill(MachineInstr &MI);
   void Branch(MachineInstr &MI);
 
+  void splitBlockLiveIns(const MachineBasicBlock &MBB,
+                         const MachineInstr &MI,
+                         MachineBasicBlock &LoopBB,
+                         MachineBasicBlock &RemainderBB,
+                         unsigned SaveReg,
+                         unsigned IdxReg);
+
   void emitLoadM0FromVGPRLoop(MachineBasicBlock &LoopBB, DebugLoc DL,
-                              MachineInstr *MovRel,
-                              unsigned SaveReg, unsigned IdxReg, int Offset);
+                              MachineInstr *MovRel, unsigned IdxReg, int Offset);
 
   void LoadM0(MachineInstr &MI, MachineInstr *MovRel, int Offset = 0);
   void computeIndirectRegAndOffset(unsigned VecReg, unsigned &Reg, int &Offset);
   void IndirectSrc(MachineInstr &MI);
   void IndirectDst(MachineInstr &MI);
 
 public:
   static char ID;
if ((Op.isImm())) {
     BuildMI(MBB, &MI, DL, TII->get(AMDGPU::V_CMPX_LE_F32_e32))
            .addImm(0)
            .addOperand(Op);
   }
 
   MI.eraseFromParent();
 }
 
+// All currently live registers must remain so in the remainder block.
+void SILowerControlFlow::splitBlockLiveIns(const MachineBasicBlock &MBB,
+                                           const MachineInstr &MI,
+                                           MachineBasicBlock &LoopBB,
+                                           MachineBasicBlock &RemainderBB,
+                                           unsigned SaveReg,
+                                           unsigned IdxReg) {
+  LivePhysRegs RemainderLiveRegs(TRI);
+
+  RemainderLiveRegs.addLiveOuts(MBB);
+  for (MachineBasicBlock::const_reverse_iterator I = MBB.rbegin(), E(&MI);
+       I != E; ++I) {
+    RemainderLiveRegs.stepBackward(*I);
+  }
+
+  // Add reg defined in loop body.
+  RemainderLiveRegs.addReg(SaveReg);
+
+  if (const MachineOperand *Val = TII->getNamedOperand(MI, AMDGPU::OpName::val)) {
+    RemainderLiveRegs.addReg(Val->getReg());
+    LoopBB.addLiveIn(Val->getReg());
+  }
+
+  for (unsigned Reg : RemainderLiveRegs)
+    RemainderBB.addLiveIn(Reg);
+
+  unsigned SrcReg = TII->getNamedOperand(MI, AMDGPU::OpName::src)->getReg();
+  LoopBB.addLiveIn(SrcReg);
+  LoopBB.addLiveIn(IdxReg);
+  LoopBB.sortUniqueLiveIns();
+}
+
 void SILowerControlFlow::emitLoadM0FromVGPRLoop(MachineBasicBlock &LoopBB,
                                                 DebugLoc DL,
                                                 MachineInstr *MovRel,
-                                                unsigned SaveReg,
                                                 unsigned IdxReg,
                                                 int Offset) {
   MachineBasicBlock::iterator I = LoopBB.begin();
 
   // Read the next variant into VCC (lower 32 bits) <- also loop target
   BuildMI(LoopBB, I, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), AMDGPU::VCC_LO)
     .addReg(IdxReg);
 
BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_XOR_B64), AMDGPU::EXEC)
     .addReg(AMDGPU::VCC);
 
   // Loop back to V_READFIRSTLANE_B32 if there are still variants to cover
   BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_CBRANCH_EXECNZ))
     .addMBB(&LoopBB);
 }
 
 void SILowerControlFlow::LoadM0(MachineInstr &MI, MachineInstr *MovRel, int Offset) {
-
   MachineBasicBlock &MBB = *MI.getParent();
   DebugLoc DL = MI.getDebugLoc();
-  MachineBasicBlock::iterator I = MI;
+  MachineBasicBlock::iterator I(&MI);
 
-  unsigned Idx = MI.getOperand(3).getReg();
+  unsigned Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx)->getReg();
 
   if (AMDGPU::SReg_32RegClass.contains(Idx)) {
     if (Offset) {
       BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADD_I32), AMDGPU::M0)
         .addReg(Idx)
         .addImm(Offset);
     } else {
       BuildMI(MBB, I, DL, TII->get(AMDGPU::S_MOV_B32), AMDGPU::M0)
         .addReg(Idx);
     }
 
     MBB.insert(I, MovRel);
     MI.eraseFromParent();
     return;
   }
 
   MachineFunction &MF = *MBB.getParent();
-  unsigned Save = MI.getOperand(1).getReg();
+  MachineOperand *SaveOp = TII->getNamedOperand(MI, AMDGPU::OpName::sdst);
+  SaveOp->setIsDead(false);
+  unsigned Save = SaveOp->getReg();
 
   // Reading from a VGPR requires looping over all workitems in the wavefront.
   assert(AMDGPU::SReg_64RegClass.contains(Save) &&
          AMDGPU::VGPR_32RegClass.contains(Idx));
 
   // Save the EXEC mask
-  BuildMI(MBB, &MI, DL, TII->get(AMDGPU::S_MOV_B64), Save)
+  BuildMI(MBB, I, DL, TII->get(AMDGPU::S_MOV_B64), Save)
     .addReg(AMDGPU::EXEC);
 
   // To insert the loop we need to split the block. Move everything after this
   // point to a new block, and insert a new empty block between the two.
   MachineBasicBlock *LoopBB = MF.CreateMachineBasicBlock();
   MachineBasicBlock *RemainderBB = MF.CreateMachineBasicBlock();
   MachineFunction::iterator MBBI(MBB);
   MF.insert(MBBI, LoopBB);
   MF.insert(MBBI, RemainderBB);
 
   LoopBB->moveAfter(&MBB);
   RemainderBB->moveAfter(LoopBB);
   LoopBB->addSuccessor(LoopBB);
   LoopBB->addSuccessor(RemainderBB);
 
+  if (TRI->trackLivenessAfterRegAlloc(MF))
+    splitBlockLiveIns(MBB, MI, *LoopBB, *RemainderBB, Save, Idx);
+
   // Move the rest of the block into a new block.
   RemainderBB->transferSuccessors(&MBB);
   RemainderBB->splice(RemainderBB->begin(), &MBB, I, MBB.end());
 
-  emitLoadM0FromVGPRLoop(*LoopBB, DL, MovRel, Save, Idx, Offset);
+  emitLoadM0FromVGPRLoop(*LoopBB, DL, MovRel, Idx, Offset);
 
   MachineBasicBlock::iterator First = RemainderBB->begin();
   BuildMI(*RemainderBB, First, DL, TII->get(AMDGPU::S_MOV_B64), AMDGPU::EXEC)
     .addReg(Save);
 
   MI.eraseFromParent();
 }
 

 }
 
 void SILowerControlFlow::IndirectSrc(MachineInstr &MI) {
 
   MachineBasicBlock &MBB = *MI.getParent();
   DebugLoc DL = MI.getDebugLoc();
 
   unsigned Dst = MI.getOperand(0).getReg();
-  unsigned Vec = MI.getOperand(2).getReg();
-  int Off = MI.getOperand(4).getImm();
+  unsigned Vec = TII->getNamedOperand(MI, AMDGPU::OpName::src)->getReg();
+  int Off = TII->getNamedOperand(MI, AMDGPU::OpName::offset)->getImm();
   unsigned Reg;
 
   computeIndirectRegAndOffset(Vec, Reg, Off);
 
   MachineInstr *MovRel =
     BuildMI(*MBB.getParent(), DL, TII->get(AMDGPU::V_MOVRELS_B32_e32), Dst)
-            .addReg(Reg)
-            .addReg(Vec, RegState::Implicit);
 
   LoadM0(MI, MovRel, Off);
 }
 
 void SILowerControlFlow::IndirectDst(MachineInstr &MI) {
 
   MachineBasicBlock &MBB = *MI.getParent();
   DebugLoc DL = MI.getDebugLoc();
 
   unsigned Dst = MI.getOperand(0).getReg();
-  int Off = MI.getOperand(4).getImm();
-  unsigned Val = MI.getOperand(5).getReg();
+  int Off = TII->getNamedOperand(MI, AMDGPU::OpName::offset)->getImm();
+  unsigned Val = TII->getNamedOperand(MI, AMDGPU::OpName::val)->getReg();
   unsigned Reg;
 
   computeIndirectRegAndOffset(Dst, Reg, Off);
 
   MachineInstr *MovRel =
     BuildMI(*MBB.getParent(), DL, TII->get(AMDGPU::V_MOVRELD_B32_e32))
-            .addReg(Reg, RegState::Define)
-            .addReg(Val)
-            .addReg(Dst, RegState::Implicit);
 
   LoadM0(MI, MovRel, Off);
 }
 
 bool SILowerControlFlow::runOnMachineFunction(MachineFunction &MF) {
   TII = static_cast<const SIInstrInfo *>(MF.getSubtarget().getInstrInfo());
   TRI =
       static_cast<const SIRegisterInfo *>(MF.getSubtarget().getRegisterInfo());

test/CodeGen/AMDGPU/indirect-addressing-si.ll

 define void @extract_vgpr_offset_multiple_in_block(i32 addrspace(1)* %out0, i32 addrspace(1)* %out1, i32 addrspace(1)* %in) #0 {
 entry:
   %id = call i32 @llvm.amdgcn.workitem.id.x() #1
   %id.ext = zext i32 %id to i64
   %gep = getelementptr inbounds i32, i32 addrspace(1)* %in, i64 %id.ext
   %idx0 = load volatile i32, i32 addrspace(1)* %gep
   %idx1 = add i32 %idx0, 1
   %val0 = extractelement <4 x i32> <i32 7, i32 9, i32 11, i32 13>, i32 %idx0
+  %live.out.reg = call i32 asm sideeffect "s_mov_b32 $0, 17", "={SGPR4}" ()
   %val1 = extractelement <4 x i32> <i32 7, i32 9, i32 11, i32 13>, i32 %idx1
   store volatile i32 %val0, i32 addrspace(1)* %out0
   store volatile i32 %val1, i32 addrspace(1)* %out0
+  %cmp = icmp eq i32 %id, 0
+  br i1 %cmp, label %bb1, label %bb2
+
+bb1:
+  store volatile i32 %live.out.reg, i32 addrspace(1)* undef
+  br label %bb2
+
+bb2:
   ret void
 }
 
 ; CHECK-LABEL: {{^}}insert_vgpr_offset_multiple_in_block:
 ; CHECK-DAG: s_load_dwordx4 s{{\[}}[[S_ELT0:[0-9]+]]:[[S_ELT3:[0-9]+]]{{\]}}
 ; CHECK-DAG: {{buffer|flat}}_load_dword [[IDX0:v[0-9]+]]
 ; CHECK-DAG: v_mov_b32_e32 [[VEC_ELT0:v[0-9]+]], s[[S_ELT0]]
-; CHECK-DAG: v_mov_b32_e32 [[INS0:v[0-9]+]], 62
+; CHECK-DAG: v_mov_b32 [[INS0:v[0-9]+]], 62
 ; CHECK-DAG: s_waitcnt vmcnt(0)
 
 ; CHECK: s_mov_b64 [[MASK:s\[[0-9]+:[0-9]+\]]], exec
 
 ; CHECK: [[LOOP0:BB[0-9]+_[0-9]+]]:
 ; CHECK: v_readfirstlane_b32 vcc_lo, [[IDX0]]
 ; CHECK: s_mov_b32 m0, vcc_lo
 ; CHECK: v_cmp_eq_u32_e32 vcc, m0, [[IDX0]]

 ; CHECK: s_mov_b32 m0, vcc_lo
 ; CHECK: v_cmp_eq_u32_e32 vcc, m0, [[IDX0]]
 ; CHECK: s_and_saveexec_b64 vcc, vcc
 ; CHECK-NEXT: v_movreld_b32_e32 v[[MOVREL1:[0-9]+]], [[INS1]]
 ; CHECK-NEXT: s_xor_b64 exec, exec, vcc
 ; CHECK: s_cbranch_execnz [[LOOP1]]
 
 ; CHECK: buffer_store_dwordx4 v{{\[}}[[MOVREL0]]:
+
+; CHECK: buffer_store_dword [[INS0]]
 define void @insert_vgpr_offset_multiple_in_block(<4 x i32> addrspace(1)* %out0, <4 x i32> addrspace(1)* %out1, i32 addrspace(1)* %in, <4 x i32> %vec0) #0 {
 entry:
   %id = call i32 @llvm.amdgcn.workitem.id.x() #1
   %id.ext = zext i32 %id to i64
   %gep = getelementptr inbounds i32, i32 addrspace(1)* %in, i64 %id.ext
   %idx0 = load volatile i32, i32 addrspace(1)* %gep
   %idx1 = add i32 %idx0, 1
-  %vec1 = insertelement <4 x i32> %vec0, i32 62, i32 %idx0
+  %live.out.val = call i32 asm sideeffect "v_mov_b32 $0, 62", "=v"()
+  %vec1 = insertelement <4 x i32> %vec0, i32 %live.out.val, i32 %idx0
   %vec2 = insertelement <4 x i32> %vec1, i32 63, i32 %idx1
   store volatile <4 x i32> %vec2, <4 x i32> addrspace(1)* %out0
+  %cmp = icmp eq i32 %id, 0
+  br i1 %cmp, label %bb1, label %bb2
+
+bb1:
+  store volatile i32 %live.out.val, i32 addrspace(1)* undef
+  br label %bb2
+
+bb2:
   ret void
 }
 
 declare i32 @llvm.amdgcn.workitem.id.x() #1
 
 attributes #0 = { nounwind }
 attributes #1 = { nounwind readnone }