AMDGPU/MemoryModel: Fix monotonic atomic loads

[kzhuravl]

Summary

Those should have glc bit set for system and agent synchronization scopes

[arsenm]

lib/Target/AMDGPU/SIMemoryLegalizer.cpp

401–403

Can this be expressed in terms of one of the isStrongerThan* helpers?

[b-sumner]

Do we need tests for 64-bit atomic loads? Do we need tests for cases where we get buffer instead of flat instructions?

[kzhuravl]

rL324314. I will address review feedback in the follow up change.

[t-tye]

LGTM other than @arsenm suggestion which may apply to some other places too. AtomicOrdering.h does not seem to other useful queries such as isAcquire/isRelease that could be used.

lib/Target/AMDGPU/SIMemoryLegalizer.cpp

401–403

Could probably be:

if (isStrongerThanMonotonic(MOI.getOrdering())) {

A load cannot be a release or acq_rel so returning true for those will not happen.

[kzhuravl]

Done.

lib/Target/AMDGPU/SIMemoryLegalizer.cpp

bool SIMemoryLegalizer::expandLoad(const SIMemOpInfo &MOI,
                                    MachineBasicBlock::iterator &MI) {
   assert(MI->mayLoad() && !MI->mayStore());
 
   bool Changed = false;
 
   if (MOI.isAtomic()) {
     if (MOI.getSSID() == SyncScope::System ||
         MOI.getSSID() == MMI->getAgentSSID()) {
-      if (MOI.getOrdering() == AtomicOrdering::Acquire ||
+      if (MOI.getOrdering() == AtomicOrdering::Monotonic ||
+          MOI.getOrdering() == AtomicOrdering::Acquire ||
           MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent)
         Changed |= enableGLCBit(MI);
 
       if (MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent)
         Changed |= insertWaitcntVmcnt0(MI);
 
       if (MOI.getOrdering() == AtomicOrdering::Acquire ||
           MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) {
         Changed |= insertWaitcntVmcnt0(MI, false);

test/CodeGen/AMDGPU/memory-legalizer-load.ll

 entry:
   %val = load atomic i32, i32 addrspace(4)* %in unordered, align 4
   store i32 %val, i32 addrspace(4)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}system_monotonic
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
 ; GCN-NOT:   buffer_wbinvl1_vol
 ; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
 define amdgpu_kernel void @system_monotonic(
     i32 addrspace(4)* %in, i32 addrspace(4)* %out) {
 entry:
   %val = load atomic i32, i32 addrspace(4)* %in monotonic, align 4
   store i32 %val, i32 addrspace(4)* %out

 entry:
   %val = load atomic i32, i32 addrspace(4)* %in syncscope("agent") unordered, align 4
   store i32 %val, i32 addrspace(4)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}agent_monotonic
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
 ; GCN-NOT:   buffer_wbinvl1_vol
 ; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
 define amdgpu_kernel void @agent_monotonic(
     i32 addrspace(4)* %in, i32 addrspace(4)* %out) {
 entry:
   %val = load atomic i32, i32 addrspace(4)* %in syncscope("agent") monotonic, align 4
   store i32 %val, i32 addrspace(4)* %out