[AMDGPU] Skip generating cache invalidating instructions on AMDPAL

[piotr]

Summary

Frontend guarantees that coherent accesses have
corresponding cache policy bits set (glc, dlc).
Therefore there is no need for extra instructions
that invalidate cache.

[nhaehnle]

@t-tye for awareness.

LGTM -- this is a reasonable step to take for us given that the Vulkan memory model has an explicit distinction between Private and NonPrivate memory accesses which we cannot yet model explicitly in LLVM.

[t-tye]

@nhaehnle my thinking was that the Vulkan memory model could be semantically represented in the LLVM IR by adding new memory orderings to make the distinction of the effective two sets of memory that the Vulkam memory model introduces. There needs to be memory orderings that only relates to the subset that Vulkan is requiring to be coherent. However, I have not followed the Vulkan memory model specification recently so not sure if it is still the same as when I was looking at it. What are your thoughts?

llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp

 class SICacheControl {
 protected:
 
   /// Instruction info.
   const SIInstrInfo *TII = nullptr;
 
   IsaVersion IV;
 
+  /// Whether to insert cache invalidation instructions.
+  bool InsertCacheInv;
+
   SICacheControl(const GCNSubtarget &ST);
 
 public:
 
   /// Create a cache control for the subtarget \p ST.
   static std::unique_ptr<SICacheControl> create(const GCNSubtarget &ST);
 
   /// Update \p MI memory load instruction to bypass any caches up to
if (MI->getNumMemOperands() == 0)
     return SIMemOpInfo();
 
   return constructFromMIWithMMO(MI);
 }
 
 SICacheControl::SICacheControl(const GCNSubtarget &ST) {
   TII = ST.getInstrInfo();
   IV = getIsaVersion(ST.getCPU());
+  InsertCacheInv = !ST.isAmdPalOS();
 }
 
 /* static */
 std::unique_ptr<SICacheControl> SICacheControl::create(const GCNSubtarget &ST) {
   GCNSubtarget::Generation Generation = ST.getGeneration();
   if (Generation <= AMDGPUSubtarget::SOUTHERN_ISLANDS)
     return std::make_unique<SIGfx6CacheControl>(ST);
   if (Generation < AMDGPUSubtarget::GFX10)
bool SIGfx6CacheControl::enableNonTemporal(
 
   return Changed;
 }
 
 bool SIGfx6CacheControl::insertCacheInvalidate(MachineBasicBlock::iterator &MI,
                                                SIAtomicScope Scope,
                                                SIAtomicAddrSpace AddrSpace,
                                                Position Pos) const {
+  if (!InsertCacheInv)
+    return false;
+
   bool Changed = false;
 
   MachineBasicBlock &MBB = *MI->getParent();
   DebugLoc DL = MI->getDebugLoc();
 
   if (Pos == Position::AFTER)
     ++MI;
 
bool SIGfx6CacheControl::insertWait(MachineBasicBlock::iterator &MI,
 
   return Changed;
 }
 
 bool SIGfx7CacheControl::insertCacheInvalidate(MachineBasicBlock::iterator &MI,
                                                SIAtomicScope Scope,
                                                SIAtomicAddrSpace AddrSpace,
                                                Position Pos) const {
+  if (!InsertCacheInv)
+    return false;
+
   bool Changed = false;
 
   MachineBasicBlock &MBB = *MI->getParent();
   DebugLoc DL = MI->getDebugLoc();
 
   const GCNSubtarget &STM = MBB.getParent()->getSubtarget<GCNSubtarget>();
 
   const unsigned Flush = STM.isAmdPalOS() || STM.isMesa3DOS()
bool SIGfx10CacheControl::enableNonTemporal(
 
   return Changed;
 }
 
 bool SIGfx10CacheControl::insertCacheInvalidate(MachineBasicBlock::iterator &MI,
                                                 SIAtomicScope Scope,
                                                 SIAtomicAddrSpace AddrSpace,
                                                 Position Pos) const {
+  if (!InsertCacheInv)
+    return false;
+
   bool Changed = false;
 
   MachineBasicBlock &MBB = *MI->getParent();
   DebugLoc DL = MI->getDebugLoc();
 
   if (Pos == Position::AFTER)
     ++MI;
 

llvm/test/CodeGen/AMDGPU/memory-legalizer-amdpal.ll

-; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN %s
-; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN %s
-; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx800 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN %s
-; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN9 %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN9 %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN9 %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN9 %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN10 %s
 
 ; FUNC-LABEL: {{^}}system_acquire:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NEXT:   buffer_wbinvl1{{$}}
+; GCN10:      s_waitcnt_vscnt null, 0x0
+; GCN-NOT:    buffer_wbinvl1{{$}}
 ; GCN:        s_endpgm
 define amdgpu_kernel void @system_acquire() {
 entry:
   fence acquire
   ret void
 }
 
 ; FUNC-LABEL: {{^}}system_release:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN10:      s_waitcnt_vscnt null, 0x0
 ; GCN:        s_endpgm
 define amdgpu_kernel void @system_release() {
 entry:
   fence release
   ret void
 }
 
 ; FUNC-LABEL: {{^}}system_acq_rel:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:        buffer_wbinvl1{{$}}
+; GCN10:      s_waitcnt_vscnt null, 0x0
+; GCN-NOT:    buffer_wbinvl1{{$}}
 ; GCN:        s_endpgm
 define amdgpu_kernel void @system_acq_rel() {
 entry:
   fence acq_rel
   ret void
 }
 
 ; FUNC-LABEL: {{^}}system_seq_cst:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:        buffer_wbinvl1{{$}}
+; GCN10:      s_waitcnt_vscnt null, 0x0
+; GCN-NOT:    buffer_wbinvl1{{$}}
 ; GCN:        s_endpgm
 define amdgpu_kernel void @system_seq_cst() {
 entry:
   fence seq_cst
   ret void
 }
 
 ; FUNC-LABEL: {{^}}system_one_as_acquire:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0){{$}}
-; GCN-NEXT:   buffer_wbinvl1{{$}}
+; GCN10:      s_waitcnt_vscnt null, 0x0
+; GCN-NOT:    buffer_wbinvl1{{$}}
 ; GCN:        s_endpgm
 define amdgpu_kernel void @system_one_as_acquire() {
 entry:
   fence syncscope("one-as") acquire
   ret void
 }
 
 ; FUNC-LABEL: {{^}}system_one_as_release:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0){{$}}
+; GCN10:      s_waitcnt_vscnt null, 0x0
 ; GCN:        s_endpgm
 define amdgpu_kernel void @system_one_as_release() {
 entry:
   fence syncscope("one-as") release
   ret void
 }
 
 ; FUNC-LABEL: {{^}}system_one_as_acq_rel:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0){{$}}
-; GCN:        buffer_wbinvl1{{$}}
+; GCN10:      s_waitcnt_vscnt null, 0x0
+; GCN-NOT:    buffer_wbinvl1{{$}}
 ; GCN:        s_endpgm
 define amdgpu_kernel void @system_one_as_acq_rel() {
 entry:
   fence syncscope("one-as") acq_rel
   ret void
 }
 
 ; FUNC-LABEL: {{^}}system_one_as_seq_cst:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0){{$}}
-; GCN:        buffer_wbinvl1{{$}}
+; GCN10:      s_waitcnt_vscnt null, 0x0
+; GCN-NOT:    buffer_wbinvl1{{$}}
 ; GCN:        s_endpgm
 define amdgpu_kernel void @system_one_as_seq_cst() {
 entry:
   fence syncscope("one-as") seq_cst
   ret void
 }
 
 ; FUNC-LABEL: {{^}}singlethread_acquire:
entry:
   fence syncscope("singlethread-one-as") seq_cst
   ret void
 }
 
 ; FUNC-LABEL: {{^}}agent_acquire:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NEXT:   buffer_wbinvl1{{$}}
+; GCN10:      s_waitcnt_vscnt null, 0x0
+; GCN-NOT:    buffer_wbinvl1{{$}}
 ; GCN:        s_endpgm
 define amdgpu_kernel void @agent_acquire() {
 entry:
   fence syncscope("agent") acquire
   ret void
 }
 
 ; FUNC-LABEL: {{^}}agent_release:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN10:      s_waitcnt_vscnt null, 0x0
 ; GCN:        s_endpgm
 define amdgpu_kernel void @agent_release() {
 entry:
   fence syncscope("agent") release
   ret void
 }
 
 ; FUNC-LABEL: {{^}}agent_acq_rel:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:        buffer_wbinvl1{{$}}
+; GCN10:      s_waitcnt_vscnt null, 0x0
+; GCN-NOT:    buffer_wbinvl1{{$}}
 ; GCN:        s_endpgm
 define amdgpu_kernel void @agent_acq_rel() {
 entry:
   fence syncscope("agent") acq_rel
   ret void
 }
 
 ; FUNC-LABEL: {{^}}agent_seq_cst:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:        buffer_wbinvl1{{$}}
+; GCN10:      s_waitcnt_vscnt null, 0x0
+; GCN-NOT:    buffer_wbinvl1{{$}}
 ; GCN:        s_endpgm
 define amdgpu_kernel void @agent_seq_cst() {
 entry:
   fence syncscope("agent") seq_cst
   ret void
 }
 
 ; FUNC-LABEL: {{^}}agent_one_as_acquire:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0){{$}}
-; GCN-NEXT:   buffer_wbinvl1{{$}}
+; GCN10:      s_waitcnt_vscnt null, 0x0
+; GCN-NOT:    buffer_wbinvl1{{$}}
 ; GCN:        s_endpgm
 define amdgpu_kernel void @agent_one_as_acquire() {
 entry:
   fence syncscope("agent-one-as") acquire
   ret void
 }
 
 ; FUNC-LABEL: {{^}}agent_one_as_release:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0){{$}}
+; GCN10:      s_waitcnt_vscnt null, 0x0
 ; GCN:        s_endpgm
 define amdgpu_kernel void @agent_one_as_release() {
 entry:
   fence syncscope("agent-one-as") release
   ret void
 }
 
 ; FUNC-LABEL: {{^}}agent_one_as_acq_rel:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0){{$}}
-; GCN:        buffer_wbinvl1{{$}}
+; GCN10:      s_waitcnt_vscnt null, 0x0
+; GCN-NOT:    buffer_wbinvl1{{$}}
 ; GCN:        s_endpgm
 define amdgpu_kernel void @agent_one_as_acq_rel() {
 entry:
   fence syncscope("agent-one-as") acq_rel
   ret void
 }
 
 ; FUNC-LABEL: {{^}}agent_one_as_seq_cst:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0){{$}}
-; GCN:        buffer_wbinvl1{{$}}
+; GCN10:      s_waitcnt_vscnt null, 0x0
+; GCN-NOT:    buffer_wbinvl1{{$}}
 ; GCN:        s_endpgm
 define amdgpu_kernel void @agent_one_as_seq_cst() {
 entry:
   fence syncscope("agent-one-as") seq_cst
   ret void
 }
 
 ; FUNC-LABEL: {{^}}workgroup_acquire:
 ; GCN:        %bb.0
-; GCN-NOT:    s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN9-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN10:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN10:      s_waitcnt_vscnt null, 0x0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_endpgm
 define amdgpu_kernel void @workgroup_acquire() {
 entry:
   fence syncscope("workgroup") acquire
   ret void
 }
 
 ; FUNC-LABEL: {{^}}workgroup_release:
 ; GCN:        %bb.0
-; GCN-NOT:    s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN9-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN10:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN10:      s_waitcnt_vscnt null, 0x0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_endpgm
 define amdgpu_kernel void @workgroup_release() {
 entry:
   fence syncscope("workgroup") release
   ret void
 }
 
 ; FUNC-LABEL: {{^}}workgroup_acq_rel:
 ; GCN:        %bb.0
-; GCN-NOT:    s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN9-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN10:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN10:      s_waitcnt_vscnt null, 0x0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_endpgm
 define amdgpu_kernel void @workgroup_acq_rel() {
 entry:
   fence syncscope("workgroup") acq_rel
   ret void
 }
 
 ; FUNC-LABEL: {{^}}workgroup_seq_cst:
 ; GCN:        %bb.0
 ; GCN-NOT:    s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN9-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN10:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN10:      s_waitcnt_vscnt null, 0x0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_endpgm
 define amdgpu_kernel void @workgroup_seq_cst() {
 entry:
   fence syncscope("workgroup") seq_cst
   ret void
 }
 
 ; FUNC-LABEL: {{^}}workgroup_one_as_acquire:
 ; GCN:        %bb.0
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
+; GCN9-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN10:      s_waitcnt vmcnt(0)
+; GCN10:      s_waitcnt_vscnt null, 0x0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_endpgm
 define amdgpu_kernel void @workgroup_one_as_acquire() {
 entry:
   fence syncscope("workgroup-one-as") acquire
   ret void
 }
 
 ; FUNC-LABEL: {{^}}workgroup_one_as_release:
 ; GCN:        %bb.0
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
+; GCN9-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN10:      s_waitcnt vmcnt(0)
+; GCN10:      s_waitcnt_vscnt null, 0x0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_endpgm
 define amdgpu_kernel void @workgroup_one_as_release() {
 entry:
   fence syncscope("workgroup-one-as") release
   ret void
 }
 
 ; FUNC-LABEL: {{^}}workgroup_one_as_acq_rel:
 ; GCN:        %bb.0
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
+; GCN9-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN10:      s_waitcnt vmcnt(0)
+; GCN10:      s_waitcnt_vscnt null, 0x0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_endpgm
 define amdgpu_kernel void @workgroup_one_as_acq_rel() {
 entry:
   fence syncscope("workgroup-one-as") acq_rel
   ret void
 }
 
 ; FUNC-LABEL: {{^}}workgroup_one_as_seq_cst:
 ; GCN:        %bb.0
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
+; GCN9-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN10:      s_waitcnt vmcnt(0)
+; GCN10:      s_waitcnt_vscnt null, 0x0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_endpgm
 define amdgpu_kernel void @workgroup_one_as_seq_cst() {
 entry:
   fence syncscope("workgroup-one-as") seq_cst
   ret void
 }