AMDGPU: Insert mem_viol check loop around GWS pre-GFX9

[arsenm]

Summary

It is necessary to emit this loop around GWS operations in case the
wave is preempted pre-GFX9.

[rampitec]

Two questions:

1. Should that only be enabled if xnack enabled? Was it even ever on before gfx9, I think no?
2. Should we support it on ASICs older than gfx9? This may be more a question to Brian.

[arsenm]

In D63464#1547397, @rampitec wrote:

Two questions:

1. Should that only be enabled if xnack enabled? Was it even ever on before gfx9, I think no?

I don't think this is related to xnack, but there's a lot of guesswork since the documentation is mostly non-existent

2. Should we support it on ASICs older than gfx9? This may be more a question to Brian.

This is the compiler support for it, I don't know if the driver has more work to support it but I would doubt it

[rampitec]

Tentative LGTM for the code itself. Please check with Brian for feasibility.

[arsenm]

In D63464#1547400, @rampitec wrote:

Tentative LGTM for the code itself. Please check with Brian for feasibility.

The backend can still support it even if the official driver does not advertise it

[arsenm]

r363979

lib/Target/AMDGPU/AMDGPUSubtarget.h

public:
   }
 
   /// Return if most LDS instructions have an m0 use that require m0 to be
   /// iniitalized.
   bool ldsRequiresM0Init() const {
     return getGeneration() < GFX9;
   }
 
+  // True if the hardware rewinds and replays GWS operations if a wave is
+  // preempted.
+  //
+  // If this is false, a GWS operation requires testing if a nack set the
+  // MEM_VIOL bit, and repeating if so.
+  bool hasGWSAutoReplay() const {
+    return getGeneration() >= GFX9;
+  }
+
   bool hasAddNoCarry() const {
     return AddNoCarryInsts;
   }
 
   bool hasUnpackedD16VMem() const {
     return HasUnpackedD16VMem;
   }
 

lib/Target/AMDGPU/DSInstructions.td

 defm DS_CMPST_RTN_F64  : DS_1A2D_RET_mc<"ds_cmpst_rtn_f64", VReg_64, "ds_cmpst_f64">;
 defm DS_MIN_RTN_F64    : DS_1A1D_RET_mc<"ds_min_rtn_f64", VReg_64, "ds_min_f64">;
 defm DS_MAX_RTN_F64    : DS_1A1D_RET_mc<"ds_max_rtn_f64", VReg_64, "ds_max_f64">;
 
 defm DS_WRXCHG_RTN_B64 : DS_1A1D_RET_mc<"ds_wrxchg_rtn_b64", VReg_64>;
 defm DS_WRXCHG2_RTN_B64 : DS_1A2D_Off8_RET_mc<"ds_wrxchg2_rtn_b64", VReg_128, VReg_64>;
 defm DS_WRXCHG2ST64_RTN_B64 : DS_1A2D_Off8_RET_mc<"ds_wrxchg2st64_rtn_b64", VReg_128, VReg_64>;
 
-let isConvergent = 1 in {
+let isConvergent = 1, usesCustomInserter = 1 in {
 def DS_GWS_INIT       : DS_GWS_1D<"ds_gws_init"> {
   let mayLoad = 0;
 }
 def DS_GWS_SEMA_V     : DS_GWS_0D<"ds_gws_sema_v">;
 def DS_GWS_SEMA_BR    : DS_GWS_1D<"ds_gws_sema_br">;
 def DS_GWS_SEMA_P     : DS_GWS_0D<"ds_gws_sema_p">;
 def DS_GWS_BARRIER    : DS_GWS_1D<"ds_gws_barrier">;
 }

lib/Target/AMDGPU/SIDefines.h

 };
 
 enum Offset : unsigned { // Offset, (5) [10:6]
   OFFSET_DEFAULT_ = 0,
   OFFSET_SHIFT_ = 6,
   OFFSET_WIDTH_ = 5,
   OFFSET_MASK_ = (((1 << OFFSET_WIDTH_) - 1) << OFFSET_SHIFT_),
 
+  OFFSET_MEM_VIOL = 8,
+
   OFFSET_SRC_SHARED_BASE = 16,
   OFFSET_SRC_PRIVATE_BASE = 0
 };
 
 enum WidthMinusOne : unsigned { // WidthMinusOne, (5) [15:11]
   WIDTH_M1_DEFAULT_ = 31,
   WIDTH_M1_SHIFT_ = 11,
   WIDTH_M1_WIDTH_ = 5,

lib/Target/AMDGPU/SIISelLowering.h

SDValue LowerCall(CallLoweringInfo &CLI,
                     SmallVectorImpl<SDValue> &InVals) const override;
 
   unsigned getRegisterByName(const char* RegName, EVT VT,
                              SelectionDAG &DAG) const override;
 
   MachineBasicBlock *splitKillBlock(MachineInstr &MI,
                                     MachineBasicBlock *BB) const;
 
+  MachineBasicBlock *emitGWSMemViolTestLoop(MachineInstr &MI,
+                                            MachineBasicBlock *BB) const;
+
   MachineBasicBlock *
   EmitInstrWithCustomInserter(MachineInstr &MI,
                               MachineBasicBlock *BB) const override;
 
   bool hasBitPreservingFPLogic(EVT VT) const override;
   bool enableAggressiveFMAFusion(EVT VT) const override;
   EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
                          EVT VT) const override;

lib/Target/AMDGPU/SIISelLowering.cpp

MachineBasicBlock *SITargetLowering::splitKillBlock(MachineInstr &MI,
 
   SplitBB->transferSuccessorsAndUpdatePHIs(BB);
   BB->addSuccessor(SplitBB);
 
   MI.setDesc(TII->getKillTerminatorFromPseudo(MI.getOpcode()));
   return SplitBB;
 }
 
+// Split block \p MBB at \p MI, as to insert a loop. If \p InstInLoop is true,
+// \p MI will be the only instruction in the loop body block. Otherwise, it will
+// be the first instruction in the remainder block.
+//
+/// \returns { LoopBody, Remainder }
+static std::pair<MachineBasicBlock *, MachineBasicBlock *>
+splitBlockForLoop(MachineInstr &MI, MachineBasicBlock &MBB, bool InstInLoop) {
+  MachineFunction *MF = MBB.getParent();
+  MachineBasicBlock::iterator I(&MI);
+
+  // To insert the loop we need to split the block. Move everything after this
+  // point to a new block, and insert a new empty block between the two.
+  MachineBasicBlock *LoopBB = MF->CreateMachineBasicBlock();
+  MachineBasicBlock *RemainderBB = MF->CreateMachineBasicBlock();
+  MachineFunction::iterator MBBI(MBB);
+  ++MBBI;
+
+  MF->insert(MBBI, LoopBB);
+  MF->insert(MBBI, RemainderBB);
+
+  LoopBB->addSuccessor(LoopBB);
+  LoopBB->addSuccessor(RemainderBB);
+
+  // Move the rest of the block into a new block.
+  RemainderBB->transferSuccessorsAndUpdatePHIs(&MBB);
+
+  if (InstInLoop) {
+    auto Next = std::next(I);
+
+    // Move instruction to loop body.
+    LoopBB->splice(LoopBB->begin(), &MBB, I, Next);
+
+    // Move the rest of the block.
+    RemainderBB->splice(RemainderBB->begin(), &MBB, Next, MBB.end());
+  } else {
+    RemainderBB->splice(RemainderBB->begin(), &MBB, I, MBB.end());
+  }
+
+  MBB.addSuccessor(LoopBB);
+
+  return std::make_pair(LoopBB, RemainderBB);
+}
+
+MachineBasicBlock *
+SITargetLowering::emitGWSMemViolTestLoop(MachineInstr &MI,
+                                         MachineBasicBlock *BB) const {
+  const DebugLoc &DL = MI.getDebugLoc();
+
+  MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
+
+  MachineBasicBlock *LoopBB;
+  MachineBasicBlock *RemainderBB;
+  const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
+
+  MachineBasicBlock::iterator Prev = std::prev(MI.getIterator());
+
+  std::tie(LoopBB, RemainderBB) = splitBlockForLoop(MI, *BB, true);
+
+  MachineBasicBlock::iterator I = LoopBB->end();
+
+  MachineOperand *Src = TII->getNamedOperand(MI, AMDGPU::OpName::data0);
+  assert(Src && "missing operand from GWS instruction");
+
+  const unsigned EncodedReg = AMDGPU::Hwreg::encodeHwreg(
+    AMDGPU::Hwreg::ID_TRAPSTS, AMDGPU::Hwreg::OFFSET_MEM_VIOL, 1);
+
+  // Clear TRAP_STS.MEM_VIOL
+  BuildMI(*LoopBB, LoopBB->begin(), DL, TII->get(AMDGPU::S_SETREG_IMM32_B32))
+    .addImm(0)
+    .addImm(EncodedReg);
+
+  // This is a pain, but we're not allowed to have physical register live-ins
+  // yet. Insert a pair of copies if the VGPR0 hack is necessary.
+  if (TargetRegisterInfo::isPhysicalRegister(Src->getReg())) {
+    unsigned Data0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+    BuildMI(*BB, std::next(Prev), DL, TII->get(AMDGPU::COPY), Data0)
+      .add(*Src);
+
+    BuildMI(*LoopBB, LoopBB->begin(), DL, TII->get(AMDGPU::COPY), Src->getReg())
+      .addReg(Data0);
+
+    MRI.setSimpleHint(Data0, Src->getReg());
+  }
+
+  BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::S_WAITCNT))
+    .addImm(0);
+
+  unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
+
+  // Load and check TRAP_STS.MEM_VIOL
+  BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::S_GETREG_B32), Reg)
+    .addImm(EncodedReg);
+
+  // FIXME: Do we need to use an isel pseudo that may clobber scc?
+  BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::S_CMP_LG_U32))
+    .addReg(Reg, RegState::Kill)
+    .addImm(0);
+  BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::S_CBRANCH_SCC1))
+    .addMBB(LoopBB);
+
+  return RemainderBB;
+}
+
 // Do a v_movrels_b32 or v_movreld_b32 for each unique value of \p IdxReg in the
 // wavefront. If the value is uniform and just happens to be in a VGPR, this
 // will only do one iteration. In the worst case, this will loop 64 times.
 //
 // TODO: Just use v_readlane_b32 if we know the VGPR has a uniform value.
 static MachineBasicBlock::iterator emitLoadM0FromVGPRLoop(
   const SIInstrInfo *TII,
   MachineRegisterInfo &MRI,
static MachineBasicBlock::iterator loadM0FromVGPR(const SIInstrInfo *TII,
   unsigned MovExecOpc = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
 
   BuildMI(MBB, I, DL, TII->get(TargetOpcode::IMPLICIT_DEF), TmpExec);
 
   // Save the EXEC mask
   BuildMI(MBB, I, DL, TII->get(MovExecOpc), SaveExec)
     .addReg(Exec);
 
-  // To insert the loop we need to split the block. Move everything after this
-  // point to a new block, and insert a new empty block between the two.
-  MachineBasicBlock *LoopBB = MF->CreateMachineBasicBlock();
-  MachineBasicBlock *RemainderBB = MF->CreateMachineBasicBlock();
-  MachineFunction::iterator MBBI(MBB);
-  ++MBBI;
-
-  MF->insert(MBBI, LoopBB);
-  MF->insert(MBBI, RemainderBB);
-
-  LoopBB->addSuccessor(LoopBB);
-  LoopBB->addSuccessor(RemainderBB);
-
-  // Move the rest of the block into a new block.
-  RemainderBB->transferSuccessorsAndUpdatePHIs(&MBB);
-  RemainderBB->splice(RemainderBB->begin(), &MBB, I, MBB.end());
-
-  MBB.addSuccessor(LoopBB);
+  MachineBasicBlock *LoopBB;
+  MachineBasicBlock *RemainderBB;
+  std::tie(LoopBB, RemainderBB) = splitBlockForLoop(MI, MBB, false);
 
   const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx);
 
   auto InsPt = emitLoadM0FromVGPRLoop(TII, MRI, MBB, *LoopBB, DL, *Idx,
                                       InitResultReg, DstReg, PhiReg, TmpExec,
                                       Offset, UseGPRIdxMode, IsIndirectSrc);
 
   MachineBasicBlock::iterator First = RemainderBB->begin();
case AMDGPU::V_SUBREV_I32_e32: {
     if (NeedClampOperand)
       I.addImm(0); // clamp bit for e64 encoding
 
     TII->legalizeOperands(*I);
 
     MI.eraseFromParent();
     return BB;
   }
+  case AMDGPU::DS_GWS_INIT:
+  case AMDGPU::DS_GWS_SEMA_V:
+  case AMDGPU::DS_GWS_SEMA_BR:
+  case AMDGPU::DS_GWS_SEMA_P:
+  case AMDGPU::DS_GWS_BARRIER:
+    if (getSubtarget()->hasGWSAutoReplay())
+      return BB;
+    return emitGWSMemViolTestLoop(MI, BB);
   default:
     return AMDGPUTargetLowering::EmitInstrWithCustomInserter(MI, BB);
   }
 }
 
 bool SITargetLowering::hasBitPreservingFPLogic(EVT VT) const {
   return isTypeLegal(VT.getScalarType());
 }

test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.barrier.ll

-; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SI %s
-; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=hawaii -mattr=+flat-for-global -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CIPLUS %s
-; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CIPLUS %s
-; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CIPLUS %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,LOOP %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=hawaii -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,LOOP %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,LOOP %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,NOLOOP %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,NOLOOP,GFX10 %s
 
 ; Minimum offset
 ; GCN-LABEL: {{^}}gws_barrier_offset0:
-; GCN-DAG: s_load_dword [[BAR_NUM:s[0-9]+]]
-; GCN-DAG: s_mov_b32 m0, -1{{$}}
-; GCN: v_mov_b32_e32 v0, [[BAR_NUM]]
-; GCN: ds_gws_barrier v0 offset:1 gds{{$}}
+; NOLOOP-DAG: s_load_dword [[BAR_NUM:s[0-9]+]]
+; NOLOOP-DAG: s_mov_b32 m0, -1{{$}}
+; NOLOOP: v_mov_b32_e32 v0, [[BAR_NUM]]
+; NOLOOP: ds_gws_barrier v0 offset:1 gds{{$}}
+
+; LOOP: [[LOOP:BB[0-9]+_[0-9]+]]:
+; LOOP-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_TRAPSTS, 8, 1), 0
+; LOOP-NEXT: ds_gws_barrier v0 offset:1 gds
+; LOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; LOOP-NEXT: s_getreg_b32 [[GETREG:s[0-9]+]], hwreg(HW_REG_TRAPSTS, 8, 1)
+; LOOP-NEXT: s_cmp_lg_u32 [[GETREG]], 0
+; LOOP-NEXT: s_cbranch_scc1 [[LOOP]]
 define amdgpu_kernel void @gws_barrier_offset0(i32 %val) #0 {
   call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 0)
   ret void
 }
 
 ; Maximum offset
 ; GCN-LABEL: {{^}}gws_barrier_offset63:
-; GCN-DAG: s_load_dword [[BAR_NUM:s[0-9]+]]
-; GCN-DAG: s_mov_b32 m0, -1{{$}}
-; GCN-DAG: v_mov_b32_e32 v0, [[BAR_NUM]]
-; GCN: ds_gws_barrier v0 offset:64 gds{{$}}
+; NOLOOP-DAG: s_load_dword [[BAR_NUM:s[0-9]+]]
+; NOLOOP-DAG: s_mov_b32 m0, -1{{$}}
+; NOLOOP-DAG: v_mov_b32_e32 v0, [[BAR_NUM]]
+; NOLOOP: ds_gws_barrier v0 offset:64 gds{{$}}
 define amdgpu_kernel void @gws_barrier_offset63(i32 %val) #0 {
   call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 63)
   ret void
 }
 
 ; FIXME: Should be able to shift directly into m0
 ; GCN-LABEL: {{^}}gws_barrier_sgpr_offset:
-; GCN-DAG: s_load_dwordx2 s{{\[}}[[BAR_NUM:[0-9]+]]:[[OFFSET:[0-9]+]]{{\]}}
-; GCN-DAG: s_lshl_b32 [[SHL:s[0-9]+]], s[[OFFSET]], 16
-; GCN-DAG: s_mov_b32 m0, [[SHL]]{{$}}
-; GCN-DAG: v_mov_b32_e32 v0, s[[BAR_NUM]]
-; GCN: ds_gws_barrier v0 gds{{$}}
+; NOLOOP-DAG: s_load_dwordx2 s{{\[}}[[BAR_NUM:[0-9]+]]:[[OFFSET:[0-9]+]]{{\]}}
+; NOLOOP-DAG: s_lshl_b32 [[SHL:s[0-9]+]], s[[OFFSET]], 16
+; NOLOOP-DAG: s_mov_b32 m0, [[SHL]]{{$}}
+; NOLOOP-DAG: v_mov_b32_e32 v0, s[[BAR_NUM]]
+; NOLOOP: ds_gws_barrier v0 gds{{$}}
 define amdgpu_kernel void @gws_barrier_sgpr_offset(i32 %val, i32 %offset) #0 {
   call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 %offset)
   ret void
 }
 
 ; Variable offset in SGPR with constant add
 ; GCN-LABEL: {{^}}gws_barrier_sgpr_offset_add1:
-; GCN-DAG: s_load_dwordx2 s{{\[}}[[BAR_NUM:[0-9]+]]:[[OFFSET:[0-9]+]]{{\]}}
-; GCN-DAG: s_lshl_b32 [[SHL:s[0-9]+]], s[[OFFSET]], 16
-; GCN-DAG: s_mov_b32 m0, [[SHL]]{{$}}
-; GCN-DAG: v_mov_b32_e32 v0, s[[BAR_NUM]]
-; GCN: ds_gws_barrier v0 offset:1 gds{{$}}
+; NOLOOP-DAG: s_load_dwordx2 s{{\[}}[[BAR_NUM:[0-9]+]]:[[OFFSET:[0-9]+]]{{\]}}
+; NOLOOP-DAG: s_lshl_b32 [[SHL:s[0-9]+]], s[[OFFSET]], 16
+; NOLOOP-DAG: s_mov_b32 m0, [[SHL]]{{$}}
+; NOLOOP-DAG: v_mov_b32_e32 v0, s[[BAR_NUM]]
+; NOLOOP: ds_gws_barrier v0 offset:1 gds{{$}}
 define amdgpu_kernel void @gws_barrier_sgpr_offset_add1(i32 %val, i32 %offset.base) #0 {
   %offset = add i32 %offset.base, 1
   call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 %offset)
   ret void
 }
 
 ; GCN-LABEL: {{^}}gws_barrier_vgpr_offset:
-; GCN-DAG: s_load_dword [[BAR_NUM:s[0-9]+]]
-; GCN-DAG: v_readfirstlane_b32 [[READLANE:s[0-9]+]], v0
-; GCN-DAG: s_lshl_b32 [[SHL:s[0-9]+]], [[READLANE]], 16
-; GCN-DAG: s_mov_b32 m0, [[SHL]]{{$}}
-; GCN-DAG: v_mov_b32_e32 v0, [[BAR_NUM]]
-; GCN: ds_gws_barrier v0 gds{{$}}
+; NOLOOP-DAG: s_load_dword [[BAR_NUM:s[0-9]+]]
+; NOLOOP-DAG: v_readfirstlane_b32 [[READLANE:s[0-9]+]], v0
+; NOLOOP-DAG: s_lshl_b32 [[SHL:s[0-9]+]], [[READLANE]], 16
+; NOLOOP-DAG: s_mov_b32 m0, [[SHL]]{{$}}
+; NOLOOP-DAG: v_mov_b32_e32 v0, [[BAR_NUM]]
+; NOLOOP: ds_gws_barrier v0 gds{{$}}
 define amdgpu_kernel void @gws_barrier_vgpr_offset(i32 %val) #0 {
   %vgpr.offset = call i32 @llvm.amdgcn.workitem.id.x()
   call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 %vgpr.offset)
   ret void
 }
 
 ; Variable offset in VGPR with constant add
 ; GCN-LABEL: {{^}}gws_barrier_vgpr_offset_add:
-; GCN-DAG: s_load_dword [[BAR_NUM:s[0-9]+]]
-; GCN-DAG: v_readfirstlane_b32 [[READLANE:s[0-9]+]], v0
-; GCN-DAG: s_lshl_b32 [[SHL:s[0-9]+]], [[READLANE]], 16
-; GCN-DAG: s_mov_b32 m0, [[SHL]]{{$}}
-; GCN-DAG: v_mov_b32_e32 v0, [[BAR_NUM]]
-; GCN: ds_gws_barrier v0 offset:3 gds{{$}}
+; NOLOOP-DAG: s_load_dword [[BAR_NUM:s[0-9]+]]
+; NOLOOP-DAG: v_readfirstlane_b32 [[READLANE:s[0-9]+]], v0
+; NOLOOP-DAG: s_lshl_b32 [[SHL:s[0-9]+]], [[READLANE]], 16
+; NOLOOP-DAG: s_mov_b32 m0, [[SHL]]{{$}}
+; NOLOOP-DAG: v_mov_b32_e32 v0, [[BAR_NUM]]
+; NOLOOP: ds_gws_barrier v0 offset:3 gds{{$}}
 define amdgpu_kernel void @gws_barrier_vgpr_offset_add(i32 %val) #0 {
   %vgpr.offset.base = call i32 @llvm.amdgcn.workitem.id.x()
   %vgpr.offset = add i32 %vgpr.offset.base, 3
   call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 %vgpr.offset)
   ret void
 }
 
 @lds = internal unnamed_addr addrspace(3) global i32 undef
 
 ; Check if m0 initialization is shared
 ; GCN-LABEL: {{^}}gws_barrier_save_m0_barrier_constant_offset:
-; GCN: s_mov_b32 m0, -1
-; GCN-NOT: s_mov_b32 m0
+; NOLOOP: s_mov_b32 m0, -1
+; NOLOOP-NOT: s_mov_b32 m0
 define amdgpu_kernel void @gws_barrier_save_m0_barrier_constant_offset(i32 %val) #0 {
   store i32 1, i32 addrspace(3)* @lds
   call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 10)
   store i32 2, i32 addrspace(3)* @lds
   ret void
 }
 
 ; Make sure this increments lgkmcnt
 ; GCN-LABEL: {{^}}gws_barrier_lgkmcnt:
-; GCN: ds_gws_barrier v0 offset:1 gds{{$}}
-; GCN-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
-; GCN-NEXT: s_setpc_b64
+; NOLOOP: ds_gws_barrier v0 offset:1 gds{{$}}
+; NOLOOP-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; NOLOOP-NEXT: s_setpc_b64
 define void @gws_barrier_lgkmcnt(i32 %val) {
   call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 0)
   ret void
 }
 
 ; Does not imply memory fence on its own
 ; GCN-LABEL: {{^}}gws_barrier_wait_before:
-; GCN: store_dword
-; CIPLUS-NOT: s_waitcnt
-; GCN: ds_gws_barrier v0 offset:8 gds
+; NOLOOP: s_waitcnt
+; NOLOOP-NOT: s_waitcnt
 define amdgpu_kernel void @gws_barrier_wait_before(i32 %val, i32 addrspace(1)* %ptr) #0 {
   store i32 0, i32 addrspace(1)* %ptr
   call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 7)
   ret void
 }
 
 ; GCN-LABEL: {{^}}gws_barrier_wait_after:
-; GCN: ds_gws_barrier v0 offset:8 gds
-; GCN-NEXT: s_waitcnt expcnt(0){{$}}
-; GCN-NEXT: load_dword
+; NOLOOP: ds_gws_barrier v0 offset:8 gds
+; NOLOOP-NEXT: s_waitcnt expcnt(0){{$}}
+; NOLOOP-NEXT: load_dword
 define amdgpu_kernel void @gws_barrier_wait_after(i32 %val, i32 addrspace(1)* %ptr) #0 {
   call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 7)
   %load = load volatile i32, i32 addrspace(1)* %ptr
   ret void
 }
 
 ; Does not imply memory fence on its own
 ; GCN-LABEL: {{^}}gws_barrier_fence_before:
-; GCN: store_dword
-; GCN: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN: ds_gws_barrier v0 offset:8 gds
+; NOLOOP: store_dword
+; NOLOOP: s_waitcnt vmcnt(0) lgkmcnt(0)
+; NOLOOP: ds_gws_barrier v0 offset:8 gds
 define amdgpu_kernel void @gws_barrier_fence_before(i32 %val, i32 addrspace(1)* %ptr) #0 {
   store i32 0, i32 addrspace(1)* %ptr
   fence release
   call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 7)
   ret void
 }
 
 ; GCN-LABEL: {{^}}gws_barrier_fence_after:
-; GCN: ds_gws_barrier v0 offset:8 gds
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: load_dword
+; NOLOOP: ds_gws_barrier v0 offset:8 gds
+; NOLOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; NOLOOP-NEXT: load_dword
 define amdgpu_kernel void @gws_barrier_fence_after(i32 %val, i32 addrspace(1)* %ptr) #0 {
   call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 7)
   fence release
   %load = load volatile i32, i32 addrspace(1)* %ptr
   ret void
 }
 
 ; FIXME: Should a wait be inserted here, or is an explicit fence needed?
 ; GCN-LABEL: {{^}}gws_init_barrier:
-; GCN: s_mov_b32 m0, -1
-; GCN: ds_gws_init v0 offset:8 gds
-; GCN-NEXT: ds_gws_barrier v0 offset:8 gds
+; NOLOOP: s_mov_b32 m0, -1
+; NOLOOP: ds_gws_init v0 offset:8 gds
+; NOLOOP-NEXT: ds_gws_barrier v0 offset:8 gds
 define amdgpu_kernel void @gws_init_barrier(i32 %val) #0 {
   call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 7)
   call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 7)
   ret void
 }
 
 ; FIXME: Why vmcnt, not expcnt?
 ; GCN-LABEL: {{^}}gws_init_fence_barrier:
-; GCN: s_mov_b32 m0, -1
-; GCN: ds_gws_init v0 offset:8 gds
-; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN-NEXT: ds_gws_barrier v0 offset:8 gds
+; NOLOOP: s_mov_b32 m0, -1
+; NOLOOP: ds_gws_init v0 offset:8 gds
+; NOLOOP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; NOLOOP-NEXT: ds_gws_barrier v0 offset:8 gds
 define amdgpu_kernel void @gws_init_fence_barrier(i32 %val) #0 {
   call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 7)
   fence release
   call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 7)
   ret void
 }
 
 declare void @llvm.amdgcn.ds.gws.barrier(i32, i32) #1
 declare void @llvm.amdgcn.ds.gws.init(i32, i32) #2
 declare i32 @llvm.amdgcn.workitem.id.x() #3
 
 attributes #0 = { nounwind }
 attributes #1 = { convergent inaccessiblememonly nounwind }
 attributes #2 = { convergent inaccessiblememonly nounwind writeonly }
 attributes #3 = { nounwind readnone speculatable }

test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.init.ll

-; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN %s
-; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=hawaii -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN %s
-; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN %s
-; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,LOOP %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=hawaii -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,LOOP %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,LOOP %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,NOLOOP %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,NOLOOP %s
 
 ; Minimum offset
 ; GCN-LABEL: {{^}}gws_init_offset0:
 ; GCN-DAG: s_load_dword [[BAR_NUM:s[0-9]+]]
 ; GCN-DAG: s_mov_b32 m0, -1{{$}}
 ; GCN: v_mov_b32_e32 v0, [[BAR_NUM]]
-; GCN: ds_gws_init v0 offset:1 gds{{$}}
+; NOLOOP: ds_gws_init v0 offset:1 gds{{$}}
+
+; LOOP: [[LOOP:BB[0-9]+_[0-9]+]]:
+; LOOP-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_TRAPSTS, 8, 1), 0
+; LOOP-NEXT: ds_gws_init v0 offset:1 gds
+; LOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; LOOP-NEXT: s_getreg_b32 [[GETREG:s[0-9]+]], hwreg(HW_REG_TRAPSTS, 8, 1)
+; LOOP-NEXT: s_cmp_lg_u32 [[GETREG]], 0
+; LOOP-NEXT: s_cbranch_scc1 [[LOOP]]
 define amdgpu_kernel void @gws_init_offset0(i32 %val) #0 {
   call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 0)
   ret void
 }
 
 ; Maximum offset
 ; GCN-LABEL: {{^}}gws_init_offset63:
-; GCN-DAG: s_load_dword [[BAR_NUM:s[0-9]+]]
-; GCN-DAG: s_mov_b32 m0, -1{{$}}
-; GCN-DAG: v_mov_b32_e32 v0, [[BAR_NUM]]
-; GCN: ds_gws_init v0 offset:64 gds{{$}}
+; NOLOOP-DAG: s_load_dword [[BAR_NUM:s[0-9]+]]
+; NOLOOP-DAG: s_mov_b32 m0, -1{{$}}
+; NOLOOP-DAG: v_mov_b32_e32 v0, [[BAR_NUM]]
+; NOLOOP: ds_gws_init v0 offset:64 gds{{$}}
+
+
+; LOOP: [[LOOP:BB[0-9]+_[0-9]+]]:
+; LOOP-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_TRAPSTS, 8, 1), 0
+; LOOP-NEXT: ds_gws_init v0 offset:64 gds
+; LOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; LOOP-NEXT: s_getreg_b32 [[GETREG:s[0-9]+]], hwreg(HW_REG_TRAPSTS, 8, 1)
+; LOOP-NEXT: s_cmp_lg_u32 [[GETREG]], 0
+; LOOP-NEXT: s_cbranch_scc1 [[LOOP]]
 define amdgpu_kernel void @gws_init_offset63(i32 %val) #0 {
   call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 63)
   ret void
 }
 
 ; FIXME: Should be able to shift directly into m0
 ; GCN-LABEL: {{^}}gws_init_sgpr_offset:
-; GCN-DAG: s_load_dwordx2 s{{\[}}[[BAR_NUM:[0-9]+]]:[[OFFSET:[0-9]+]]{{\]}}
-; GCN-DAG: s_lshl_b32 [[SHL:s[0-9]+]], s[[OFFSET]], 16
-; GCN-DAG: s_mov_b32 m0, [[SHL]]{{$}}
-; GCN-DAG: v_mov_b32_e32 v0, s[[BAR_NUM]]
-; GCN: ds_gws_init v0 gds{{$}}
+; NOLOOP-DAG: s_load_dwordx2 s{{\[}}[[BAR_NUM:[0-9]+]]:[[OFFSET:[0-9]+]]{{\]}}
+; NOLOOP-DAG: s_lshl_b32 [[SHL:s[0-9]+]], s[[OFFSET]], 16
+; NOLOOP-DAG: s_mov_b32 m0, [[SHL]]{{$}}
+; NOLOOP-DAG: v_mov_b32_e32 v0, s[[BAR_NUM]]
+; NOLOOP: ds_gws_init v0 gds{{$}}
 define amdgpu_kernel void @gws_init_sgpr_offset(i32 %val, i32 %offset) #0 {
   call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 %offset)
   ret void
 }
 
 ; Variable offset in SGPR with constant add
 ; GCN-LABEL: {{^}}gws_init_sgpr_offset_add1:
-; GCN-DAG: s_load_dwordx2 s{{\[}}[[BAR_NUM:[0-9]+]]:[[OFFSET:[0-9]+]]{{\]}}
-; GCN-DAG: s_lshl_b32 [[SHL:s[0-9]+]], s[[OFFSET]], 16
-; GCN-DAG: s_mov_b32 m0, [[SHL]]{{$}}
-; GCN-DAG: v_mov_b32_e32 v0, s[[BAR_NUM]]
-; GCN: ds_gws_init v0 offset:1 gds{{$}}
+; NOLOOP-DAG: s_load_dwordx2 s{{\[}}[[BAR_NUM:[0-9]+]]:[[OFFSET:[0-9]+]]{{\]}}
+; NOLOOP-DAG: s_lshl_b32 [[SHL:s[0-9]+]], s[[OFFSET]], 16
+; NOLOOP-DAG: s_mov_b32 m0, [[SHL]]{{$}}
+; NOLOOP-DAG: v_mov_b32_e32 v0, s[[BAR_NUM]]
+; NOLOOP: ds_gws_init v0 offset:1 gds{{$}}
 define amdgpu_kernel void @gws_init_sgpr_offset_add1(i32 %val, i32 %offset.base) #0 {
   %offset = add i32 %offset.base, 1
   call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 %offset)
   ret void
 }
 
 ; GCN-LABEL: {{^}}gws_init_vgpr_offset:
-; GCN-DAG: s_load_dword [[BAR_NUM:s[0-9]+]]
-; GCN-DAG: v_readfirstlane_b32 [[READLANE:s[0-9]+]], v0
-; GCN-DAG: s_lshl_b32 [[SHL:s[0-9]+]], [[READLANE]], 16
-; GCN-DAG: s_mov_b32 m0, [[SHL]]{{$}}
-; GCN-DAG: v_mov_b32_e32 v0, [[BAR_NUM]]
-; GCN: ds_gws_init v0 gds{{$}}
+; NOLOOP-DAG: s_load_dword [[BAR_NUM:s[0-9]+]]
+; NOLOOP-DAG: v_readfirstlane_b32 [[READLANE:s[0-9]+]], v0
+; NOLOOP-DAG: s_lshl_b32 [[SHL:s[0-9]+]], [[READLANE]], 16
+; NOLOOP-DAG: s_mov_b32 m0, [[SHL]]{{$}}
+; NOLOOP-DAG: v_mov_b32_e32 v0, [[BAR_NUM]]
+; NOLOOP: ds_gws_init v0 gds{{$}}
 define amdgpu_kernel void @gws_init_vgpr_offset(i32 %val) #0 {
   %vgpr.offset = call i32 @llvm.amdgcn.workitem.id.x()
   call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 %vgpr.offset)
   ret void
 }
 
 ; Variable offset in VGPR with constant add
 ; GCN-LABEL: {{^}}gws_init_vgpr_offset_add:
-; GCN-DAG: s_load_dword [[BAR_NUM:s[0-9]+]]
-; GCN-DAG: v_readfirstlane_b32 [[READLANE:s[0-9]+]], v0
-; GCN-DAG: s_lshl_b32 [[SHL:s[0-9]+]], [[READLANE]], 16
-; GCN-DAG: s_mov_b32 m0, [[SHL]]{{$}}
-; GCN-DAG: v_mov_b32_e32 v0, [[BAR_NUM]]
-; GCN: ds_gws_init v0 offset:3 gds{{$}}
+; NOLOOP-DAG: s_load_dword [[BAR_NUM:s[0-9]+]]
+; NOLOOP-DAG: v_readfirstlane_b32 [[READLANE:s[0-9]+]], v0
+; NOLOOP-DAG: s_lshl_b32 [[SHL:s[0-9]+]], [[READLANE]], 16
+; NOLOOP-DAG: s_mov_b32 m0, [[SHL]]{{$}}
+; NOLOOP-DAG: v_mov_b32_e32 v0, [[BAR_NUM]]
+; NOLOOP: ds_gws_init v0 offset:3 gds{{$}}
 define amdgpu_kernel void @gws_init_vgpr_offset_add(i32 %val) #0 {
   %vgpr.offset.base = call i32 @llvm.amdgcn.workitem.id.x()
   %vgpr.offset = add i32 %vgpr.offset.base, 3
   call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 %vgpr.offset)
   ret void
 }
 
 @lds = internal unnamed_addr addrspace(3) global i32 undef
 
 ; Check if m0 initialization is shared.
 ; GCN-LABEL: {{^}}gws_init_save_m0_init_constant_offset:
-; GCN: s_mov_b32 m0, -1
-; GCN-NOT: s_mov_b32 m0
+; NOLOOP: s_mov_b32 m0, -1
+; NOLOOP-NOT: s_mov_b32 m0
 define amdgpu_kernel void @gws_init_save_m0_init_constant_offset(i32 %val) #0 {
   store i32 1, i32 addrspace(3)* @lds
   call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 10)
   store i32 2, i32 addrspace(3)* @lds
   ret void
 }
 
 ; GCN-LABEL: {{^}}gws_init_lgkmcnt:
-; GCN: ds_gws_init v0 offset:1 gds{{$}}
-; GCN-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
-; GCN-NEXT: s_setpc_b64
+; NOLOOP: ds_gws_init v0 offset:1 gds{{$}}
+; NOLOOP-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; NOLOOP-NEXT: s_setpc_b64
 define void @gws_init_lgkmcnt(i32 %val) {
   call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 0)
   ret void
 }
 
 ; Does not imply memory fence on its own
 ; GCN-LABEL: {{^}}gws_init_wait_before:
-; GCN: store_dword
-; CIPLUS-NOT: s_waitcnt
-; GCN: ds_gws_init v0 offset:8 gds
+; NOLOOP: s_waitcnt
+; NOLOOP-NOT: s_waitcnt
 define amdgpu_kernel void @gws_init_wait_before(i32 %val, i32 addrspace(1)* %ptr) #0 {
   store i32 0, i32 addrspace(1)* %ptr
   call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 7)
   ret void
 }
 
 declare void @llvm.amdgcn.ds.gws.init(i32, i32) #1
 declare i32 @llvm.amdgcn.workitem.id.x() #2
 
 attributes #0 = { nounwind }
 attributes #1 = { convergent inaccessiblememonly nounwind writeonly }
 attributes #2 = { nounwind readnone speculatable }