AMDGPU/MemoryModel: Fix monotonic atomic loads

[kzhuravl]

Summary

Those should have glc bit set for system and agent synchronization scopes

[b-sumner]

Do we need tests of 64 bit atomic loads? What about cases where we get buffer instructions rather than flat instructions?

lib/Target/AMDGPU/SIMemoryLegalizer.cpp

bool SIMemoryLegalizer::expandLoad(const SIMemOpInfo &MOI,
                                    MachineBasicBlock::iterator &MI) {
   assert(MI->mayLoad() && !MI->mayStore());
 
   bool Changed = false;
 
   if (MOI.isAtomic()) {
     if (MOI.getSSID() == SyncScope::System ||
         MOI.getSSID() == MMI->getAgentSSID()) {
-      if (MOI.getOrdering() == AtomicOrdering::Acquire ||
+      if (MOI.getOrdering() == AtomicOrdering::Monotonic ||
+          MOI.getOrdering() == AtomicOrdering::Acquire ||
           MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent)
         Changed |= enableGLCBit(MI);
 
       if (MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent)
         Changed |= insertWaitcntVmcnt0(MI);
 
       if (MOI.getOrdering() == AtomicOrdering::Acquire ||
           MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) {

test/CodeGen/AMDGPU/memory-legalizer-load.ll

 entry:
   %val = load atomic i32, i32 addrspace(4)* %in unordered, align 4
   store i32 %val, i32 addrspace(4)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}system_monotonic
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
 ; GCN-NOT:   buffer_wbinvl1_vol
 ; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
 define amdgpu_kernel void @system_monotonic(
     i32 addrspace(4)* %in, i32 addrspace(4)* %out) {
 entry:
   %val = load atomic i32, i32 addrspace(4)* %in monotonic, align 4
   store i32 %val, i32 addrspace(4)* %out

 entry:
   %val = load atomic i32, i32 addrspace(4)* %in syncscope("agent") unordered, align 4
   store i32 %val, i32 addrspace(4)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}agent_monotonic
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
 ; GCN-NOT:   buffer_wbinvl1_vol
 ; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
 define amdgpu_kernel void @agent_monotonic(
     i32 addrspace(4)* %in, i32 addrspace(4)* %out) {
 entry:
   %val = load atomic i32, i32 addrspace(4)* %in syncscope("agent") monotonic, align 4
   store i32 %val, i32 addrspace(4)* %out