AMDGPU: Remove v0 workaround for DS_GWS_* instructions

[arsenm]

Summary

Any register should work for the src field since r366067, since the
used value is not pulled from the expected encoding field.

[rampitec]

LGTM

[arsenm]

r367598

lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

if (ConstantSDNode *ConstOffset = dyn_cast<ConstantSDNode>(BaseOffset)) {
     // Shift to offset in m0
     SDNode *M0Base
       = CurDAG->getMachineNode(AMDGPU::S_LSHL_B32, SL, MVT::i32,
                                SDValue(SGPROffset, 0),
                                CurDAG->getTargetConstant(16, SL, MVT::i32));
     glueCopyToM0(N, SDValue(M0Base, 0));
   }
 
-  SDValue V0;
   SDValue Chain = N->getOperand(0);
-  SDValue Glue;
-  if (HasVSrc) {
-    SDValue VSrc0 = N->getOperand(2);
-
-    // The manual doesn't mention this, but it seems only v0 works.
-    V0 = CurDAG->getRegister(AMDGPU::VGPR0, MVT::i32);
-
-    SDValue CopyToV0 = CurDAG->getCopyToReg(
-      N->getOperand(0), SL, V0, VSrc0,
-      N->getOperand(N->getNumOperands() - 1));
-    Chain = CopyToV0;
-    Glue = CopyToV0.getValue(1);
-  }
-
   SDValue OffsetField = CurDAG->getTargetConstant(ImmOffset, SL, MVT::i32);
 
   // TODO: Can this just be removed from the instruction?
   SDValue GDS = CurDAG->getTargetConstant(1, SL, MVT::i1);
 
   const unsigned Opc = gwsIntrinToOpcode(IntrID);
   SmallVector<SDValue, 5> Ops;
   if (HasVSrc)
-    Ops.push_back(V0);
+    Ops.push_back(N->getOperand(2));
   Ops.push_back(OffsetField);
   Ops.push_back(GDS);
   Ops.push_back(Chain);
 
-  if (HasVSrc)
-    Ops.push_back(Glue);
-
   SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
   CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO});
 }
 
 void AMDGPUDAGToDAGISel::SelectINTRINSIC_W_CHAIN(SDNode *N) {
   unsigned IntrID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
   switch (IntrID) {
   case Intrinsic::amdgcn_ds_append:

lib/Target/AMDGPU/SIISelLowering.cpp

SITargetLowering::emitGWSMemViolTestLoop(MachineInstr &MI,
   const DebugLoc &DL = MI.getDebugLoc();
 
   MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
 
   MachineBasicBlock *LoopBB;
   MachineBasicBlock *RemainderBB;
   const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
 
-  MachineBasicBlock::iterator Prev = std::prev(MI.getIterator());
+  // Apparently kill flags are only valid if the def is in the same block?
+  if (MachineOperand *Src = TII->getNamedOperand(MI, AMDGPU::OpName::data0))
+    Src->setIsKill(false);
 
   std::tie(LoopBB, RemainderBB) = splitBlockForLoop(MI, *BB, true);
 
   MachineBasicBlock::iterator I = LoopBB->end();
-  MachineOperand *Src = TII->getNamedOperand(MI, AMDGPU::OpName::data0);
 
   const unsigned EncodedReg = AMDGPU::Hwreg::encodeHwreg(
     AMDGPU::Hwreg::ID_TRAPSTS, AMDGPU::Hwreg::OFFSET_MEM_VIOL, 1);
 
   // Clear TRAP_STS.MEM_VIOL
   BuildMI(*LoopBB, LoopBB->begin(), DL, TII->get(AMDGPU::S_SETREG_IMM32_B32))
     .addImm(0)
     .addImm(EncodedReg);
 
-  // This is a pain, but we're not allowed to have physical register live-ins
-  // yet. Insert a pair of copies if the VGPR0 hack is necessary.
-  if (Src && TargetRegisterInfo::isPhysicalRegister(Src->getReg())) {
-    unsigned Data0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
-    BuildMI(*BB, std::next(Prev), DL, TII->get(AMDGPU::COPY), Data0)
-      .add(*Src);
-
-    BuildMI(*LoopBB, LoopBB->begin(), DL, TII->get(AMDGPU::COPY), Src->getReg())
-      .addReg(Data0);
-
-    MRI.setSimpleHint(Data0, Src->getReg());
-  }
-
   bundleInstWithWaitcnt(MI);
 
   unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
 
   // Load and check TRAP_STS.MEM_VIOL
   BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::S_GETREG_B32), Reg)
     .addImm(EncodedReg);
 

test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.barrier.ll

 ; LOOP-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_TRAPSTS, 8, 1), 0
 ; LOOP-NEXT: ds_gws_barrier v0 gds
 ; LOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; LOOP-NEXT: s_getreg_b32 [[GETREG:s[0-9]+]], hwreg(HW_REG_TRAPSTS, 8, 1)
 ; LOOP-NEXT: s_cmp_lg_u32 [[GETREG]], 0
 ; LOOP-NEXT: s_cbranch_scc1 [[LOOP]]
 
 ; MIR-LABEL: name: gws_barrier_offset0{{$}}
-; MIR: BUNDLE implicit{{( killed)?}} $vgpr0, implicit $m0, implicit $exec {
-; MIR-NEXT: DS_GWS_BARRIER $vgpr0, 0, -1, implicit $m0, implicit $exec :: (load 4 from custom GWSResource)
+; MIR: BUNDLE implicit{{( killed)?( renamable)?}} $vgpr0, implicit $m0, implicit $exec {
+; MIR-NEXT: DS_GWS_BARRIER renamable $vgpr0, 0, -1, implicit $m0, implicit $exec :: (load 4 from custom GWSResource)
 ; MIR-NEXT: S_WAITCNT 0
 ; MIR-NEXT: }
 define amdgpu_kernel void @gws_barrier_offset0(i32 %val) #0 {
   call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 0)
   ret void
 }
 
 ; MIR-LABEL: name: gws_barrier_offset63{{$}}
define amdgpu_kernel void @gws_barrier_offset63(i32 %val) #0 {
   ret void
 }
 
 ; FIXME: Should be able to shift directly into m0
 ; GCN-LABEL: {{^}}gws_barrier_sgpr_offset:
 ; NOLOOP-DAG: s_load_dwordx2 s{{\[}}[[BAR_NUM:[0-9]+]]:[[OFFSET:[0-9]+]]{{\]}}
 ; NOLOOP-DAG: s_lshl_b32 [[SHL:s[0-9]+]], s[[OFFSET]], 16
 ; NOLOOP-DAG: s_mov_b32 m0, [[SHL]]{{$}}
-; NOLOOP-DAG: v_mov_b32_e32 v0, s[[BAR_NUM]]
-; NOLOOP: ds_gws_barrier v0 gds{{$}}
+; NOLOOP-DAG: v_mov_b32_e32 [[GWS_VAL:v[0-9]+]], s[[BAR_NUM]]
+; NOLOOP: ds_gws_barrier [[GWS_VAL]] gds{{$}}
 define amdgpu_kernel void @gws_barrier_sgpr_offset(i32 %val, i32 %offset) #0 {
   call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 %offset)
   ret void
 }
 
 ; Variable offset in SGPR with constant add
 ; GCN-LABEL: {{^}}gws_barrier_sgpr_offset_add1:
 ; NOLOOP-DAG: s_load_dwordx2 s{{\[}}[[BAR_NUM:[0-9]+]]:[[OFFSET:[0-9]+]]{{\]}}
 ; NOLOOP-DAG: s_lshl_b32 [[SHL:s[0-9]+]], s[[OFFSET]], 16
 ; NOLOOP-DAG: s_mov_b32 m0, [[SHL]]{{$}}
-; NOLOOP-DAG: v_mov_b32_e32 v0, s[[BAR_NUM]]
-; NOLOOP: ds_gws_barrier v0 offset:1 gds{{$}}
+; NOLOOP-DAG: v_mov_b32_e32 [[GWS_VAL:v[0-9]+]], s[[BAR_NUM]]
+; NOLOOP: ds_gws_barrier [[GWS_VAL]] offset:1 gds{{$}}
 define amdgpu_kernel void @gws_barrier_sgpr_offset_add1(i32 %val, i32 %offset.base) #0 {
   %offset = add i32 %offset.base, 1
   call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 %offset)
   ret void
 }
 
 ; GCN-LABEL: {{^}}gws_barrier_vgpr_offset:
 ; NOLOOP-DAG: s_load_dword [[BAR_NUM:s[0-9]+]]
 ; NOLOOP-DAG: v_readfirstlane_b32 [[READLANE:s[0-9]+]], v0
 ; NOLOOP-DAG: s_lshl_b32 [[SHL:s[0-9]+]], [[READLANE]], 16
 ; NOLOOP-DAG: s_mov_b32 m0, [[SHL]]{{$}}
-; NOLOOP-DAG: v_mov_b32_e32 v0, [[BAR_NUM]]
-; NOLOOP: ds_gws_barrier v0 gds{{$}}
+; NOLOOP-DAG: v_mov_b32_e32 [[GWS_VAL:v[0-9]+]], [[BAR_NUM]]
+; NOLOOP: ds_gws_barrier [[GWS_VAL]] gds{{$}}
 define amdgpu_kernel void @gws_barrier_vgpr_offset(i32 %val) #0 {
   %vgpr.offset = call i32 @llvm.amdgcn.workitem.id.x()
   call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 %vgpr.offset)
   ret void
 }
 
 ; Variable offset in VGPR with constant add
 ; GCN-LABEL: {{^}}gws_barrier_vgpr_offset_add:
 ; NOLOOP-DAG: s_load_dword [[BAR_NUM:s[0-9]+]]
 ; NOLOOP-DAG: v_readfirstlane_b32 [[READLANE:s[0-9]+]], v0
 ; NOLOOP-DAG: s_lshl_b32 [[SHL:s[0-9]+]], [[READLANE]], 16
 ; NOLOOP-DAG: s_mov_b32 m0, [[SHL]]{{$}}
-; NOLOOP-DAG: v_mov_b32_e32 v0, [[BAR_NUM]]
-; NOLOOP: ds_gws_barrier v0 offset:3 gds{{$}}
+; NOLOOP-DAG: v_mov_b32_e32 [[GWS_VAL:v[0-9]+]], [[BAR_NUM]]
+; NOLOOP: ds_gws_barrier [[GWS_VAL]] offset:3 gds{{$}}
 define amdgpu_kernel void @gws_barrier_vgpr_offset_add(i32 %val) #0 {
   %vgpr.offset.base = call i32 @llvm.amdgcn.workitem.id.x()
   %vgpr.offset = add i32 %vgpr.offset.base, 3
   call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 %vgpr.offset)
   ret void
 }
 
 @lds = internal unnamed_addr addrspace(3) global i32 undef
 
 ; Check if m0 initialization is shared
 ; GCN-LABEL: {{^}}gws_barrier_save_m0_barrier_constant_offset:
 ; NOLOOP: s_mov_b32 m0, 0
-; NOLOOP: ds_gws_barrier v0 offset:10 gds
+; NOLOOP: ds_gws_barrier v{{[0-9]+}} offset:10 gds
 
 ; LOOP: s_mov_b32 m0, -1
 ; LOOP: ds_write_b32
 ; LOOP: s_mov_b32 m0, 0
 ; LOOP: s_setreg_imm32_b32
-; LOOP: ds_gws_barrier v0 offset:10 gds
+; LOOP: ds_gws_barrier v{{[0-9]+}} offset:10 gds
 ; LOOP: s_cbranch_scc1
 
 ; LOOP: s_mov_b32 m0, -1
 ; LOOP: ds_write_b32
 define amdgpu_kernel void @gws_barrier_save_m0_barrier_constant_offset(i32 %val) #0 {
   store i32 1, i32 addrspace(3)* @lds
   call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 10)
   store i32 2, i32 addrspace(3)* @lds

 define amdgpu_kernel void @gws_barrier_wait_before(i32 %val, i32 addrspace(1)* %ptr) #0 {
   store i32 0, i32 addrspace(1)* %ptr
   call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 7)
   ret void
 }
 
 ; GCN-LABEL: {{^}}gws_barrier_wait_after:
 ; NOLOOP: s_mov_b32 m0, 0{{$}}
-; NOLOOP: ds_gws_barrier v0 offset:7 gds
+; NOLOOP: ds_gws_barrier v{{[0-9]+}} offset:7 gds
 ; NOLOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; NOLOOP-NEXT: load_dword
 define amdgpu_kernel void @gws_barrier_wait_after(i32 %val, i32 addrspace(1)* %ptr) #0 {
   call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 7)
   %load = load volatile i32, i32 addrspace(1)* %ptr
   ret void
 }
 
 ; Does not imply memory fence on its own
 ; GCN-LABEL: {{^}}gws_barrier_fence_before:
 ; NOLOOP: s_mov_b32 m0, 0{{$}}
 ; NOLOOP: store_dword
 ; NOLOOP: s_waitcnt vmcnt(0) lgkmcnt(0)
-; NOLOOP: ds_gws_barrier v0 offset:7 gds
+; NOLOOP: ds_gws_barrier v{{[0-9]+}} offset:7 gds
 ; NOLOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 define amdgpu_kernel void @gws_barrier_fence_before(i32 %val, i32 addrspace(1)* %ptr) #0 {
   store i32 0, i32 addrspace(1)* %ptr
   fence release
   call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 7)
   ret void
 }
 
 ; FIXME: Extra waitcnt
 ; GCN-LABEL: {{^}}gws_barrier_fence_after:
 ; NOLOOP: s_mov_b32 m0, 0{{$}}
-; NOLOOP: ds_gws_barrier v0 offset:7 gds
+; NOLOOP: ds_gws_barrier v{{[0-9]+}} offset:7 gds
 ; NOLOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; NOLOOP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
 ; NOLOOP-NEXT: load_dword
 define amdgpu_kernel void @gws_barrier_fence_after(i32 %val, i32 addrspace(1)* %ptr) #0 {
   call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 7)
   fence release
   %load = load volatile i32, i32 addrspace(1)* %ptr

test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.init.ll

define amdgpu_kernel void @gws_init_offset63(i32 %val) #0 {
   ret void
 }
 
 ; FIXME: Should be able to shift directly into m0
 ; GCN-LABEL: {{^}}gws_init_sgpr_offset:
 ; NOLOOP-DAG: s_load_dwordx2 s{{\[}}[[BAR_NUM:[0-9]+]]:[[OFFSET:[0-9]+]]{{\]}}
 ; NOLOOP-DAG: s_lshl_b32 [[SHL:s[0-9]+]], s[[OFFSET]], 16
 ; NOLOOP-DAG: s_mov_b32 m0, [[SHL]]{{$}}
-; NOLOOP-DAG: v_mov_b32_e32 v0, s[[BAR_NUM]]
-; NOLOOP: ds_gws_init v0 gds{{$}}
+; NOLOOP-DAG: v_mov_b32_e32 [[GWS_VAL:v[0-9]+]], s[[BAR_NUM]]
+; NOLOOP: ds_gws_init [[GWS_VAL]] gds{{$}}
 define amdgpu_kernel void @gws_init_sgpr_offset(i32 %val, i32 %offset) #0 {
   call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 %offset)
   ret void
 }
 
 ; Variable offset in SGPR with constant add
 ; GCN-LABEL: {{^}}gws_init_sgpr_offset_add1:
 ; NOLOOP-DAG: s_load_dwordx2 s{{\[}}[[BAR_NUM:[0-9]+]]:[[OFFSET:[0-9]+]]{{\]}}
 ; NOLOOP-DAG: s_lshl_b32 [[SHL:s[0-9]+]], s[[OFFSET]], 16
 ; NOLOOP-DAG: s_mov_b32 m0, [[SHL]]{{$}}
-; NOLOOP-DAG: v_mov_b32_e32 v0, s[[BAR_NUM]]
-; NOLOOP: ds_gws_init v0 offset:1 gds{{$}}
+; NOLOOP-DAG: v_mov_b32_e32 [[GWS_VAL:v[0-9]+]], s[[BAR_NUM]]
+; NOLOOP: ds_gws_init [[GWS_VAL]] offset:1 gds{{$}}
 define amdgpu_kernel void @gws_init_sgpr_offset_add1(i32 %val, i32 %offset.base) #0 {
   %offset = add i32 %offset.base, 1
   call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 %offset)
   ret void
 }
 
 ; GCN-LABEL: {{^}}gws_init_vgpr_offset:
 ; NOLOOP-DAG: s_load_dword [[BAR_NUM:s[0-9]+]]
define amdgpu_kernel void @gws_init_vgpr_offset_add(i32 %val) #0 {
   ret void
 }
 
 @lds = internal unnamed_addr addrspace(3) global i32 undef
 
 ; Check if m0 initialization is shared.
 ; GCN-LABEL: {{^}}gws_init_save_m0_init_constant_offset:
 ; NOLOOP: s_mov_b32 m0, 0
-; NOLOOP: ds_gws_init v0 offset:10 gds
+; NOLOOP: ds_gws_init v{{[0-9]+}} offset:10 gds
 
 ; LOOP: s_mov_b32 m0, -1
 ; LOOP: ds_write_b32
 ; LOOP: s_mov_b32 m0, 0
 ; LOOP: s_setreg_imm32_b32
-; LOOP: ds_gws_init v0 offset:10 gds
+; LOOP: ds_gws_init v{{[0-9]+}} offset:10 gds
 ; LOOP: s_cbranch_scc1
 
 ; LOOP: s_mov_b32 m0, -1
 ; LOOP: ds_write_b32
 define amdgpu_kernel void @gws_init_save_m0_init_constant_offset(i32 %val) #0 {
   store i32 1, i32 addrspace(3)* @lds
   call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 10)
   store i32 2, i32 addrspace(3)* @lds