[AMDGPU] gfx11 waitcnt support for VINTERP and LDSDIR instructions

llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp

 struct RegisterEncoding {
   unsigned VGPR0;
   unsigned VGPRL;
   unsigned SGPR0;
   unsigned SGPRL;
 };
 
 enum WaitEventType {
-  VMEM_ACCESS,      // vector-memory read & write
-  VMEM_READ_ACCESS, // vector-memory read
-  VMEM_WRITE_ACCESS,// vector-memory write
-  LDS_ACCESS,       // lds read & write
-  GDS_ACCESS,       // gds read & write
-  SQ_MESSAGE,       // send message
-  SMEM_ACCESS,      // scalar-memory read & write
-  EXP_GPR_LOCK,     // export holding on its data src
-  GDS_GPR_LOCK,     // GDS holding on its data and addr src
-  EXP_POS_ACCESS,   // write to export position
-  EXP_PARAM_ACCESS, // write to export parameter
-  VMW_GPR_LOCK,     // vector-memory write holding on its data src
+  VMEM_ACCESS,       // vector-memory read & write
+  VMEM_READ_ACCESS,  // vector-memory read
+  VMEM_WRITE_ACCESS, // vector-memory write
+  LDS_ACCESS,        // lds read & write
+  GDS_ACCESS,        // gds read & write
+  SQ_MESSAGE,        // send message
+  SMEM_ACCESS,       // scalar-memory read & write
+  EXP_GPR_LOCK,      // export holding on its data src
+  GDS_GPR_LOCK,      // GDS holding on its data and addr src
+  EXP_POS_ACCESS,    // write to export position
+  EXP_PARAM_ACCESS,  // write to export parameter
+  VMW_GPR_LOCK,      // vector-memory write holding on its data src
+  EXP_LDS_ACCESS,    // read by ldsdir counting as export
   NUM_WAIT_EVENTS,
 };
 
 static const unsigned WaitEventMaskForInst[NUM_INST_CNTS] = {
     (1 << VMEM_ACCESS) | (1 << VMEM_READ_ACCESS),
     (1 << SMEM_ACCESS) | (1 << LDS_ACCESS) | (1 << GDS_ACCESS) |
         (1 << SQ_MESSAGE),
     (1 << EXP_GPR_LOCK) | (1 << GDS_GPR_LOCK) | (1 << VMW_GPR_LOCK) |
-      (1 << EXP_PARAM_ACCESS) | (1 << EXP_POS_ACCESS),
-  (1 << VMEM_WRITE_ACCESS)
-};
+        (1 << EXP_PARAM_ACCESS) | (1 << EXP_POS_ACCESS) | (1 << EXP_LDS_ACCESS),
+    (1 << VMEM_WRITE_ACCESS)};
 
 // The mapping is:
 //  0                .. SQ_MAX_PGM_VGPRS-1               real VGPRs
 //  SQ_MAX_PGM_VGPRS .. NUM_ALL_VGPRS-1                  extra VGPR-like slots
 //  NUM_ALL_VGPRS    .. NUM_ALL_VGPRS+SQ_MAX_PGM_SGPRS-1 real SGPRs
 // We reserve a fixed number of VGPR slots in the scoring tables for
 // special tokens like SCMEM_LDS (needed for buffer load to LDS).
 enum RegisterMapping {
if (TII->isDS(Inst) && (Inst.mayStore() || Inst.mayLoad())) {
       if (Inst.mayStore()) {
         setExpScore(&Inst, TII, TRI, MRI, 0, CurrScore);
       } else if (SIInstrInfo::isAtomicRet(Inst)) {
         setExpScore(
             &Inst, TII, TRI, MRI,
             AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data),
             CurrScore);
       }
+    } else if (TII->isLDSDIR(Inst)) {
+      // LDSDIR instructions attach the score to the destination.
+      setExpScore(
+          &Inst, TII, TRI, MRI,
+          AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::vdst),
+          CurrScore);
     } else {
       if (TII->isEXP(Inst)) {
         // For export the destination registers are really temps that
         // can be used as the actual source after export patching, so
         // we need to treat them like sources and set the EXP_CNT
         // score.
         for (unsigned I = 0, E = Inst.getNumOperands(); I != E; ++I) {
           MachineOperand &DefMO = Inst.getOperand(I);
if (MI.isCall() && callWaitsOnFunctionEntry(MI)) {
             // results in order anyway.
             if (Op.isUse() || !SIInstrInfo::isVMEM(MI) ||
                 ScoreBrackets.hasOtherPendingVmemTypes(RegNo,
                                                        getVmemType(MI))) {
               ScoreBrackets.determineWait(
                   VM_CNT, ScoreBrackets.getRegScore(RegNo, VM_CNT), Wait);
               ScoreBrackets.clearVgprVmemTypes(RegNo);
             }
-            if (Op.isDef()) {
+            if (Op.isDef() || ScoreBrackets.hasPendingEvent(EXP_LDS_ACCESS)) {
               ScoreBrackets.determineWait(
                   EXP_CNT, ScoreBrackets.getRegScore(RegNo, EXP_CNT), Wait);
             }
           }
           ScoreBrackets.determineWait(
               LGKM_CNT, ScoreBrackets.getRegScore(RegNo, LGKM_CNT), Wait);
         }
       }
if (OldWaitcntInstr) {
     // Also erase redundant waitcnt.
     Modified =
         applyPreexistingWaitcnt(ScoreBrackets, *OldWaitcntInstr, Wait, &MI);
   } else {
     // Update waitcnt brackets after determining the required wait.
     ScoreBrackets.applyWaitcnt(Wait);
   }
 
+  // ExpCnt can be merged into VINTERP.
+  if (Wait.ExpCnt != ~0u && SIInstrInfo::isVINTERP(MI)) {
+    MachineOperand *WaitExp = TII->getNamedOperand(MI, AMDGPU::OpName::waitexp);
+    if (Wait.ExpCnt < WaitExp->getImm()) {
+      WaitExp->setImm(Wait.ExpCnt);
+      Modified = true;
+    }
+    Wait.ExpCnt = ~0u;
+
+    LLVM_DEBUG(dbgs() << "generateWaitcntInstBefore\n"
+                      << "Update Instr: " << MI);
+  }
+
   // Build new waitcnt instructions unless no wait is needed or the old waitcnt
   // instruction was modified to handle the required wait.
   if (Wait.hasWaitExceptVsCnt()) {
     unsigned Enc = AMDGPU::encodeWaitcnt(IV, Wait);
     auto SWaitInst = BuildMI(*MI.getParent(), MI.getIterator(),
                              MI.getDebugLoc(), TII->get(AMDGPU::S_WAITCNT))
                          .addImm(Enc);
     TrackedWaitcntSet.insert(SWaitInst);
void SIInsertWaitcnts::updateEventWaitcntAfter(MachineInstr &Inst,
   } else if (Inst.isCall()) {
     if (callWaitsOnFunctionReturn(Inst)) {
       // Act as a wait on everything
       ScoreBrackets->applyWaitcnt(AMDGPU::Waitcnt::allZero(ST->hasVscnt()));
     } else {
       // May need to way wait for anything.
       ScoreBrackets->applyWaitcnt(AMDGPU::Waitcnt());
     }
+  } else if (SIInstrInfo::isLDSDIR(Inst)) {
+    ScoreBrackets->updateByEvent(TII, TRI, MRI, EXP_LDS_ACCESS, Inst);
+  } else if (TII->isVINTERP(Inst)) {
+    int64_t Imm = TII->getNamedOperand(Inst, AMDGPU::OpName::waitexp)->getImm();
+    ScoreBrackets->applyWaitcnt(EXP_CNT, Imm);
   } else if (SIInstrInfo::isEXP(Inst)) {
     unsigned Imm = TII->getNamedOperand(Inst, AMDGPU::OpName::tgt)->getImm();
     if (Imm >= AMDGPU::Exp::ET_PARAM0 && Imm <= AMDGPU::Exp::ET_PARAM31)
       ScoreBrackets->updateByEvent(TII, TRI, MRI, EXP_PARAM_ACCESS, Inst);
     else if (Imm >= AMDGPU::Exp::ET_POS0 && Imm <= AMDGPU::Exp::ET_POS_LAST)
       ScoreBrackets->updateByEvent(TII, TRI, MRI, EXP_POS_ACCESS, Inst);
     else
       ScoreBrackets->updateByEvent(TII, TRI, MRI, EXP_GPR_LOCK, Inst);

llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.interp.inreg.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
 define amdgpu_ps void @v_interp_f32(float inreg %i, float inreg %j, i32 inreg %m0) #0 {
 ; GCN-LABEL: v_interp_f32:
 ; GCN:       ; %bb.0: ; %main_body
 ; GCN-NEXT:    s_mov_b32 m0, s2
 ; GCN-NEXT:    v_mov_b32_e32 v2, s0
 ; GCN-NEXT:    lds_param_load v0, attr0.y
 ; GCN-NEXT:    lds_param_load v1, attr1.x
 ; GCN-NEXT:    v_mov_b32_e32 v4, s1
-; GCN-NEXT:    v_interp_p10_f32 v3, v0, v2, v0 wait_exp:7
-; GCN-NEXT:    v_interp_p10_f32 v2, v1, v2, v1 wait_exp:7
+; GCN-NEXT:    v_interp_p10_f32 v3, v0, v2, v0 wait_exp:1
+; GCN-NEXT:    v_interp_p10_f32 v2, v1, v2, v1
 ; GCN-NEXT:    v_interp_p2_f32 v0, v0, v4, v3 wait_exp:7
 ; GCN-NEXT:    v_interp_p2_f32 v1, v1, v4, v0 wait_exp:7
 ; GCN-NEXT:    exp mrt0 v3, v2, v0, v1 done
 ; GCN-NEXT:    s_endpgm
 main_body:
   %p0 = call float @llvm.amdgcn.lds.param.load(i32 1, i32 0, i32 %m0)
   %p1 = call float @llvm.amdgcn.lds.param.load(i32 0, i32 1, i32 %m0)
   %p0_0 = call float @llvm.amdgcn.interp.inreg.p10(float %p0, float %i, float %p0)

 ; GCN:       ; %bb.0: ; %main_body
 ; GCN-NEXT:    s_mov_b32 m0, s2
 ; GCN-NEXT:    v_mov_b32_e32 v4, s0
 ; GCN-NEXT:    lds_param_load v0, attr0.x
 ; GCN-NEXT:    lds_param_load v1, attr1.x
 ; GCN-NEXT:    lds_param_load v2, attr2.x
 ; GCN-NEXT:    lds_param_load v3, attr3.x
 ; GCN-NEXT:    v_mov_b32_e32 v5, s1
-; GCN-NEXT:    v_interp_p10_f32 v6, v0, v4, v0 wait_exp:7
-; GCN-NEXT:    v_interp_p10_f32 v7, v1, v4, v1 wait_exp:7
-; GCN-NEXT:    v_interp_p10_f32 v8, v2, v4, v2 wait_exp:7
-; GCN-NEXT:    v_interp_p10_f32 v4, v3, v4, v3 wait_exp:7
+; GCN-NEXT:    v_interp_p10_f32 v6, v0, v4, v0 wait_exp:3
+; GCN-NEXT:    v_interp_p10_f32 v7, v1, v4, v1 wait_exp:2
+; GCN-NEXT:    v_interp_p10_f32 v8, v2, v4, v2 wait_exp:1
+; GCN-NEXT:    v_interp_p10_f32 v4, v3, v4, v3
 ; GCN-NEXT:    v_interp_p2_f32 v0, v0, v5, v6 wait_exp:7
 ; GCN-NEXT:    v_interp_p2_f32 v1, v1, v5, v7 wait_exp:7
 ; GCN-NEXT:    v_interp_p2_f32 v2, v2, v5, v8 wait_exp:7
 ; GCN-NEXT:    v_interp_p2_f32 v3, v3, v5, v4 wait_exp:7
 ; GCN-NEXT:    exp mrt0 v0, v1, v2, v3 done
 ; GCN-NEXT:    s_endpgm
 main_body:
   %p0 = call float @llvm.amdgcn.lds.param.load(i32 0, i32 0, i32 %m0)

 ; GCN:       ; %bb.0: ; %main_body
 ; GCN-NEXT:    global_load_b64 v[0:1], v[0:1], off offset:4
 ; GCN-NEXT:    s_mov_b32 m0, s0
 ; GCN-NEXT:    lds_param_load v2, attr0.x
 ; GCN-NEXT:    lds_param_load v3, attr1.x
 ; GCN-NEXT:    lds_param_load v4, attr2.x
 ; GCN-NEXT:    lds_param_load v5, attr3.x
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NEXT:    v_interp_p10_f32 v6, v2, v0, v2 wait_exp:7
-; GCN-NEXT:    v_interp_p10_f32 v7, v3, v0, v3 wait_exp:7
-; GCN-NEXT:    v_interp_p10_f32 v8, v4, v0, v4 wait_exp:7
-; GCN-NEXT:    v_interp_p10_f32 v0, v5, v0, v5 wait_exp:7
+; GCN-NEXT:    v_interp_p10_f32 v6, v2, v0, v2 wait_exp:3
+; GCN-NEXT:    v_interp_p10_f32 v7, v3, v0, v3 wait_exp:2
+; GCN-NEXT:    v_interp_p10_f32 v8, v4, v0, v4 wait_exp:1
+; GCN-NEXT:    v_interp_p10_f32 v0, v5, v0, v5
 ; GCN-NEXT:    v_interp_p2_f32 v2, v2, v1, v6 wait_exp:7
 ; GCN-NEXT:    v_interp_p2_f32 v3, v3, v1, v7 wait_exp:7
 ; GCN-NEXT:    v_interp_p2_f32 v4, v4, v1, v8 wait_exp:7
 ; GCN-NEXT:    v_interp_p2_f32 v0, v5, v1, v0 wait_exp:7
 ; GCN-NEXT:    exp mrt0 v2, v3, v4, v0 done
 ; GCN-NEXT:    s_endpgm
 main_body:
   %i.ptr = getelementptr float, float addrspace(1)* %ptr, i32 1

 
 define amdgpu_ps half @v_interp_f16(float inreg %i, float inreg %j, i32 inreg %m0) #0 {
 ; GCN-LABEL: v_interp_f16:
 ; GCN:       ; %bb.0: ; %main_body
 ; GCN-NEXT:    s_mov_b32 m0, s2
 ; GCN-NEXT:    v_mov_b32_e32 v1, s0
 ; GCN-NEXT:    lds_param_load v0, attr0.x
 ; GCN-NEXT:    v_mov_b32_e32 v2, s1
-; GCN-NEXT:    v_interp_p10_f16_f32 v3, v0, v1, v0 wait_exp:7
+; GCN-NEXT:    v_interp_p10_f16_f32 v3, v0, v1, v0
 ; GCN-NEXT:    v_interp_p10_f16_f32 v1, v0, v1, v0 op_sel:[1,0,1,0] wait_exp:7
 ; GCN-NEXT:    v_interp_p2_f16_f32 v3, v0, v2, v3 wait_exp:7
 ; GCN-NEXT:    v_interp_p2_f16_f32 v0, v0, v2, v1 op_sel:[1,0,0,0] wait_exp:7
 ; GCN-NEXT:    v_add_f16_e32 v0, v3, v0
 ; GCN-NEXT:    ; return to shader part epilog
 main_body:
   %p0 = call float @llvm.amdgcn.lds.param.load(i32 0, i32 0, i32 %m0)
   %l_p0 = call float @llvm.amdgcn.interp.inreg.p10.f16(float %p0, float %i, float %p0, i1 0)

llvm/test/CodeGen/AMDGPU/llvm.amdgcn.interp.inreg.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -march=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
 define amdgpu_ps void @v_interp_f32(float inreg %i, float inreg %j, i32 inreg %m0) #0 {
 ; GCN-LABEL: v_interp_f32:
 ; GCN:       ; %bb.0: ; %main_body
 ; GCN-NEXT:    s_mov_b32 m0, s2
 ; GCN-NEXT:    v_mov_b32_e32 v2, s0
 ; GCN-NEXT:    lds_param_load v0, attr0.y
 ; GCN-NEXT:    lds_param_load v1, attr1.x
 ; GCN-NEXT:    v_mov_b32_e32 v4, s1
-; GCN-NEXT:    v_interp_p10_f32 v3, v0, v2, v0 wait_exp:7
-; GCN-NEXT:    v_interp_p10_f32 v2, v1, v2, v1 wait_exp:7
+; GCN-NEXT:    v_interp_p10_f32 v3, v0, v2, v0 wait_exp:1
+; GCN-NEXT:    v_interp_p10_f32 v2, v1, v2, v1
 ; GCN-NEXT:    v_interp_p2_f32 v0, v0, v4, v3 wait_exp:7
 ; GCN-NEXT:    v_interp_p2_f32 v1, v1, v4, v0 wait_exp:7
 ; GCN-NEXT:    exp mrt0 v3, v2, v0, v1 done
 ; GCN-NEXT:    s_endpgm
 main_body:
   %p0 = call float @llvm.amdgcn.lds.param.load(i32 1, i32 0, i32 %m0)
   %p1 = call float @llvm.amdgcn.lds.param.load(i32 0, i32 1, i32 %m0)
   %p0_0 = call float @llvm.amdgcn.interp.inreg.p10(float %p0, float %i, float %p0)

 ; GCN:       ; %bb.0: ; %main_body
 ; GCN-NEXT:    s_mov_b32 m0, s2
 ; GCN-NEXT:    v_mov_b32_e32 v4, s0
 ; GCN-NEXT:    lds_param_load v0, attr0.x
 ; GCN-NEXT:    lds_param_load v1, attr1.x
 ; GCN-NEXT:    lds_param_load v2, attr2.x
 ; GCN-NEXT:    lds_param_load v3, attr3.x
 ; GCN-NEXT:    v_mov_b32_e32 v5, s1
-; GCN-NEXT:    v_interp_p10_f32 v6, v0, v4, v0 wait_exp:7
-; GCN-NEXT:    v_interp_p10_f32 v7, v1, v4, v1 wait_exp:7
-; GCN-NEXT:    v_interp_p10_f32 v8, v2, v4, v2 wait_exp:7
-; GCN-NEXT:    v_interp_p10_f32 v4, v3, v4, v3 wait_exp:7
+; GCN-NEXT:    v_interp_p10_f32 v6, v0, v4, v0 wait_exp:3
+; GCN-NEXT:    v_interp_p10_f32 v7, v1, v4, v1 wait_exp:2
+; GCN-NEXT:    v_interp_p10_f32 v8, v2, v4, v2 wait_exp:1
+; GCN-NEXT:    v_interp_p10_f32 v4, v3, v4, v3
 ; GCN-NEXT:    v_interp_p2_f32 v0, v0, v5, v6 wait_exp:7
 ; GCN-NEXT:    v_interp_p2_f32 v1, v1, v5, v7 wait_exp:7
 ; GCN-NEXT:    v_interp_p2_f32 v2, v2, v5, v8 wait_exp:7
 ; GCN-NEXT:    v_interp_p2_f32 v3, v3, v5, v4 wait_exp:7
 ; GCN-NEXT:    exp mrt0 v0, v1, v2, v3 done
 ; GCN-NEXT:    s_endpgm
 main_body:
   %p0 = call float @llvm.amdgcn.lds.param.load(i32 0, i32 0, i32 %m0)

 ; GCN:       ; %bb.0: ; %main_body
 ; GCN-NEXT:    global_load_b64 v[0:1], v[0:1], off offset:4
 ; GCN-NEXT:    s_mov_b32 m0, s0
 ; GCN-NEXT:    lds_param_load v2, attr0.x
 ; GCN-NEXT:    lds_param_load v3, attr1.x
 ; GCN-NEXT:    lds_param_load v4, attr2.x
 ; GCN-NEXT:    lds_param_load v5, attr3.x
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NEXT:    v_interp_p10_f32 v6, v2, v0, v2 wait_exp:7
-; GCN-NEXT:    v_interp_p10_f32 v7, v3, v0, v3 wait_exp:7
-; GCN-NEXT:    v_interp_p10_f32 v8, v4, v0, v4 wait_exp:7
-; GCN-NEXT:    v_interp_p10_f32 v0, v5, v0, v5 wait_exp:7
+; GCN-NEXT:    v_interp_p10_f32 v6, v2, v0, v2 wait_exp:3
+; GCN-NEXT:    v_interp_p10_f32 v7, v3, v0, v3 wait_exp:2
+; GCN-NEXT:    v_interp_p10_f32 v8, v4, v0, v4 wait_exp:1
+; GCN-NEXT:    v_interp_p10_f32 v0, v5, v0, v5
 ; GCN-NEXT:    v_interp_p2_f32 v2, v2, v1, v6 wait_exp:7
 ; GCN-NEXT:    v_interp_p2_f32 v3, v3, v1, v7 wait_exp:7
 ; GCN-NEXT:    v_interp_p2_f32 v4, v4, v1, v8 wait_exp:7
 ; GCN-NEXT:    v_interp_p2_f32 v0, v5, v1, v0 wait_exp:7
 ; GCN-NEXT:    exp mrt0 v2, v3, v4, v0 done
 ; GCN-NEXT:    s_endpgm
 main_body:
   %i.ptr = getelementptr float, float addrspace(1)* %ptr, i32 1

 
 define amdgpu_ps half @v_interp_f16(float inreg %i, float inreg %j, i32 inreg %m0) #0 {
 ; GCN-LABEL: v_interp_f16:
 ; GCN:       ; %bb.0: ; %main_body
 ; GCN-NEXT:    s_mov_b32 m0, s2
 ; GCN-NEXT:    v_mov_b32_e32 v1, s0
 ; GCN-NEXT:    lds_param_load v0, attr0.x
 ; GCN-NEXT:    v_mov_b32_e32 v2, s1
-; GCN-NEXT:    v_interp_p10_f16_f32 v3, v0, v1, v0 wait_exp:7
+; GCN-NEXT:    v_interp_p10_f16_f32 v3, v0, v1, v0
 ; GCN-NEXT:    v_interp_p10_f16_f32 v1, v0, v1, v0 op_sel:[1,0,1,0] wait_exp:7
 ; GCN-NEXT:    v_interp_p2_f16_f32 v3, v0, v2, v3 wait_exp:7
 ; GCN-NEXT:    v_interp_p2_f16_f32 v0, v0, v2, v1 op_sel:[1,0,0,0] wait_exp:7
 ; GCN-NEXT:    v_add_f16_e32 v0, v3, v0
 ; GCN-NEXT:    ; return to shader part epilog
 main_body:
   %p0 = call float @llvm.amdgcn.lds.param.load(i32 0, i32 0, i32 %m0)
   %l_p0 = call float @llvm.amdgcn.interp.inreg.p10.f16(float %p0, float %i, float %p0, i1 0)