AMDGPU/SI: Handle wait states required for DPP instructions

[• tstellarAMD]

Fix case where dpp instruction was first in block.

[arsenm]

lib/Target/AMDGPU/SIInsertWaits.cpp

507

Typo strt

509–510

You should be able to use range loop with predecessors()

516

There could be a break here since there will only ever be one fall through block, although it doesn't really matter because with structured control flow there is at most one predecessor

[• tstellarAMD]

Fix typo. Use range for loop. Simplify loop.

[• tstellarAMD]

Ping.

[arsenm]

LGTM

lib/Target/AMDGPU/SIDefines.h

enum {
   MUBUF = 1 << 14,
   MTBUF = 1 << 15,
   SMRD = 1 << 16,
   DS = 1 << 17,
   MIMG = 1 << 18,
   FLAT = 1 << 19,
   WQM = 1 << 20,
   VGPRSpill = 1 << 21,
-  VOPAsmPrefer32Bit = 1 << 22
+  VOPAsmPrefer32Bit = 1 << 22,
+  DPP = 1 << 23
 };
 }
 
 namespace llvm {
 namespace AMDGPU {
   enum OperandType {
     /// Operand with register or 32-bit immediate
     OPERAND_REG_IMM32 = llvm::MCOI::OPERAND_FIRST_TARGET,

lib/Target/AMDGPU/SIInsertWaits.cpp

private:
   bool unorderedDefines(MachineInstr &MI);
 
   /// \brief Resolve all operand dependencies to counter requirements
   Counters handleOperands(MachineInstr &MI);
 
   /// \brief Insert S_NOP between an instruction writing M0 and S_SENDMSG.
   void handleSendMsg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I);
 
+  /// \param DPP The DPP instruction
+  /// \param SearchI The iterator to start look for hazards.
+  /// \param SearchMBB The basic block we are operating on.
+  /// \param WaitStates Then number of wait states that need to be inserted
+  ///                    When a hazard is detected.
+  void insertDPPWaitStates(MachineBasicBlock::iterator DPP,
+                           MachineBasicBlock::reverse_iterator SearchI,
+                           MachineBasicBlock *SearchMBB,
+                           unsigned WaitStates);
+
+  void insertDPPWaitStates(MachineBasicBlock::iterator DPP);
+
   /// Return true if there are LGKM instrucitons that haven't been waited on
   /// yet.
   bool hasOutstandingLGKM() const;
 
 public:
   static char ID;
 
   SIInsertWaits() :
void SIInsertWaits::handleSendMsg(MachineBasicBlock &MBB,
   for (unsigned i = 0; i < NumOperands; i++) {
     const MachineOperand &Op = I->getOperand(i);
 
     if (Op.isReg() && Op.isDef() && Op.getReg() == AMDGPU::M0)
       LastInstWritesM0 = true;
   }
 }
 
+void SIInsertWaits::insertDPPWaitStates(MachineBasicBlock::iterator DPP,
+                                        MachineBasicBlock::reverse_iterator SearchI,
+                                        MachineBasicBlock *SearchMBB,
+                                        unsigned WaitStates) {
+
+  MachineBasicBlock::reverse_iterator E = SearchMBB->rend();
+
+  for (; WaitStates > 0; --WaitStates, ++SearchI) {
+
+    // If we have reached the start of the block, we need to check predecessors.
+    if (SearchI == E) {
+      for (MachineBasicBlock *Pred : SearchMBB->predecessors()) {
+        // We only need to check fall-through blocks.  Branch instructions
+        // give us enough wait states.
+        if (Pred->getFirstTerminator() == Pred->end()) {
+          insertDPPWaitStates(DPP, Pred->rbegin(), Pred, WaitStates);
+          break;
+        }
+      }
+      return;
+    }
+
+    for (MachineOperand &Op : SearchI->operands()) {
+      if (!Op.isReg() || !Op.isDef())
+        continue;
+
+      if (DPP->readsRegister(Op.getReg(), TRI)) {
+        TII->insertWaitStates(DPP, WaitStates);
+        return;
+      }
+    }
+  }
+}
+
+void SIInsertWaits::insertDPPWaitStates(MachineBasicBlock::iterator DPP) {
+  MachineBasicBlock::reverse_iterator I(DPP);
+  insertDPPWaitStates(DPP, I, DPP->getParent(), 2);
+}
+
 // FIXME: Insert waits listed in Table 4.2 "Required User-Inserted Wait States"
 // around other non-memory instructions.
 bool SIInsertWaits::runOnMachineFunction(MachineFunction &MF) {
   bool Changes = false;
 
   TII = static_cast<const SIInstrInfo *>(MF.getSubtarget().getInstrInfo());
   TRI =
       static_cast<const SIRegisterInfo *>(MF.getSubtarget().getRegisterInfo());
for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end();
           // bit is updated, so we can restore the bit by reading the value of
           // vcc and then writing it back to the register.
           BuildMI(MBB, I, I->getDebugLoc(), TII->get(AMDGPU::S_MOV_B64),
                   AMDGPU::VCC)
                   .addReg(AMDGPU::VCC);
         }
       }
 
+      if (TII->isDPP(*I)) {
+        insertDPPWaitStates(I);
+      }
+
       // Wait for everything before a barrier.
       if (I->getOpcode() == AMDGPU::S_BARRIER)
         Changes |= insertWait(MBB, I, LastIssued);
       else
         Changes |= insertWait(MBB, I, handleOperands(*I));
 
       pushInstruction(MBB, I);
       handleSendMsg(MBB, I);

lib/Target/AMDGPU/SIInstrFormats.td

class InstSI <dag outs, dag ins, string asm, list<dag> pattern> :
   field bits<1> MUBUF = 0;
   field bits<1> MTBUF = 0;
   field bits<1> SMRD = 0;
   field bits<1> DS = 0;
   field bits<1> MIMG = 0;
   field bits<1> FLAT = 0;
   field bits<1> WQM = 0;
   field bits<1> VGPRSpill = 0;
+  field bits<1> DPP = 0;
 
   // This bit tells the assembler to use the 32-bit encoding in case it
   // is unable to infer the encoding from the operands.
   field bits<1> VOPAsmPrefer32Bit = 0;
 
   // These need to be kept in sync with the enum in SIInstrFlags.
   let TSFlags{0} = VM_CNT;
   let TSFlags{1} = EXP_CNT;
class InstSI <dag outs, dag ins, string asm, list<dag> pattern> :
   let TSFlags{15} = MTBUF;
   let TSFlags{16} = SMRD;
   let TSFlags{17} = DS;
   let TSFlags{18} = MIMG;
   let TSFlags{19} = FLAT;
   let TSFlags{20} = WQM;
   let TSFlags{21} = VGPRSpill;
   let TSFlags{22} = VOPAsmPrefer32Bit;
+  let TSFlags{23} = DPP;
 
   let SchedRW = [Write32Bit];
 
   field bits<1> DisableSIDecoder = 0;
   field bits<1> DisableVIDecoder = 0;
   field bits<1> DisableDecoder = 0;
 
   let isAsmParserOnly = !if(!eq(DisableDecoder{0}, {0}), 0, 1);

lib/Target/AMDGPU/SIInstrInfo.h

public:
   static bool isVGPRSpill(const MachineInstr &MI) {
     return MI.getDesc().TSFlags & SIInstrFlags::VGPRSpill;
   }
 
   bool isVGPRSpill(uint16_t Opcode) const {
     return get(Opcode).TSFlags & SIInstrFlags::VGPRSpill;
   }
 
+  static bool isDPP(const MachineInstr &MI) {
+    return MI.getDesc().TSFlags & SIInstrFlags::DPP;
+  }
+
+  bool isDPP(uint16_t Opcode) const {
+    return get(Opcode).TSFlags & SIInstrFlags::DPP;
+  }
+
   bool isInlineConstant(const APInt &Imm) const;
   bool isInlineConstant(const MachineOperand &MO, unsigned OpSize) const;
   bool isLiteralConstant(const MachineOperand &MO, unsigned OpSize) const;
 
   bool isImmOperandLegal(const MachineInstr *MI, unsigned OpNo,
                          const MachineOperand &MO) const;
 
   /// \brief Return true if this 64-bit VALU instruction has a 32-bit encoding.

lib/Target/AMDGPU/VIInstrFormats.td

class VOP3be_vi <bits<10> op> : Enc64 {
   let Inst{61} = src0_modifiers{0};
   let Inst{62} = src1_modifiers{0};
   let Inst{63} = src2_modifiers{0};
 }
 
 class VOP_DPP <dag outs, dag ins, string asm, list<dag> pattern> :
     VOPAnyCommon <outs, ins, asm, pattern> {
   let Size = 8;
+  let DPP = 1;
 }
 
 class VOP_DPPe : Enc64 {
   bits<2> src0_modifiers;
   bits<8> src0;
   bits<2> src1_modifiers;
   bits<9> dpp_ctrl;
   bits<1> bound_ctrl;

test/CodeGen/AMDGPU/llvm.amdgcn.mov.dpp.ll

 ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs -show-mc-encoding < %s | FileCheck -check-prefix=VI %s
 
+; FIXME: The register allocator / scheduler should be able to avoid these hazards.
+
 ; VI-LABEL: {{^}}dpp_test:
+; VI: v_mov_b32_e32 v0, s{{[0-9]+}}
+; VI: s_nop 1
 ; VI: v_mov_b32 v0, v0, 1, -1, 1, 1 ; encoding: [0xfa,0x02,0x00,0x7e,0x00,0x01,0x08,0x11]
 define void @dpp_test(i32 addrspace(1)* %out, i32 %in) {
   %tmp0 = call i32 @llvm.amdgcn.mov.dpp.i32(i32 %in, i32 1, i1 1, i32 1, i32 1) #0
   store i32 %tmp0, i32 addrspace(1)* %out
   ret void
 }
 
+; VI-LABEL: {{^}}dpp_wait_states:
+; VI: v_mov_b32_e32 [[VGPR0:v[0-9]+]], s{{[0-9]+}}
+; VI: s_nop 1
+; VI: v_mov_b32 [[VGPR1:v[0-9]+]], [[VGPR0]], 1, -1, 1, 1
+; VI: s_nop 1
+; VI: v_mov_b32 v{{[0-9]+}}, [[VGPR1]], 1, -1, 1, 1
+define void @dpp_wait_states(i32 addrspace(1)* %out, i32 %in) {
+  %tmp0 = call i32 @llvm.amdgcn.mov.dpp.i32(i32 %in, i32 1, i1 1, i32 1, i32 1) #0
+  %tmp1 = call i32 @llvm.amdgcn.mov.dpp.i32(i32 %tmp0, i32 1, i1 1, i32 1, i32 1) #0
+  store i32 %tmp1, i32 addrspace(1)* %out
+  ret void
+}
+
+; VI-LABEL: {{^}}dpp_first_in_bb:
+; VI: s_nop 1
+; VI: v_mov_b32 [[VGPR0:v[0-9]+]], v{{[0-9]+}}, 1, -1, 1, 1
+; VI: s_nop 1
+; VI: v_mov_b32 [[VGPR1:v[0-9]+]], [[VGPR0]], 1, -1, 1, 1
+; VI: s_nop 1
+; VI: v_mov_b32 v{{[0-9]+}}, [[VGPR1]], 1, -1, 1, 1
+define void @dpp_first_in_bb(float addrspace(1)* %out, float addrspace(1)* %in, float %cond, float %a, float %b) {
+  %cmp = fcmp oeq float %cond, 0.0
+  br i1 %cmp, label %if, label %else
+
+if:
+  %out_val = load float, float addrspace(1)* %out
+  %if_val = fadd float %a, %out_val
+  br label %endif
+
+else:
+  %in_val = load float, float addrspace(1)* %in
+  %else_val = fadd float %b, %in_val
+  br label %endif
+
+endif:
+  %val = phi float [%if_val, %if], [%else_val, %else]
+  %val_i32 = bitcast float %val to i32
+  %tmp0 = call i32 @llvm.amdgcn.mov.dpp.i32(i32 %val_i32, i32 1, i1 1, i32 1, i32 1) #0
+  %tmp1 = call i32 @llvm.amdgcn.mov.dpp.i32(i32 %tmp0, i32 1, i1 1, i32 1, i32 1) #0
+  %tmp2 = call i32 @llvm.amdgcn.mov.dpp.i32(i32 %tmp1, i32 1, i1 1, i32 1, i32 1) #0
+  %tmp_float = bitcast i32 %tmp2 to float
+  store float %tmp_float, float addrspace(1)* %out
+  ret void
+}
+
 declare i32 @llvm.amdgcn.mov.dpp.i32(i32, i32, i1, i32, i32) #0
 
 attributes #0 = { nounwind readnone convergent }