[AMDGPU] Make generating cache invalidating instructions optional

[piotr]

Summary

D78800 skipped generating cache invalidating instrucions altogether
on AMDPAL. However, this is sometimes too restrictive - we want a
more flexible option to be able to toggle this behaviour on and off
while we work towards developing a correct implementation of the
alternative memory model.

[nhaehnle]

llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp

50–52

amdgcn-skip-cache-invalidations?

[nhaehnle]

LGTM with the option name fixed, and please adjust the variable name also (amdgcn-skip-cache-invalidations --> AmdgcnSkipCacheInvalidations).

[piotr]

Renamed the option and variable.

llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp

 #include <list>
 
 using namespace llvm;
 using namespace llvm::AMDGPU;
 
 #define DEBUG_TYPE "si-memory-legalizer"
 #define PASS_NAME "SI Memory Legalizer"
 
+static cl::opt<bool> AmdgcnSkipCacheInvalidations(
+    "amdgcn-skip-cache-invalidations", cl::init(false), cl::Hidden,
+    cl::desc("Use this to skip inserting cache invalidating instructions."));
+
 namespace {
 
 LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
 
 /// Memory operation flags. Can be ORed together.
 enum class SIMemOp {
   NONE = 0u,
   LOAD = 1u << 0,

 class SICacheControl {
 protected:
 
   /// Instruction info.
   const SIInstrInfo *TII = nullptr;
 
   IsaVersion IV;
 
-  /// Whether to insert cache invalidation instructions.
+  /// Whether to insert cache invalidating instructions.
   bool InsertCacheInv;
 
   SICacheControl(const GCNSubtarget &ST);
 
 public:
 
   /// Create a cache control for the subtarget \p ST.
   static std::unique_ptr<SICacheControl> create(const GCNSubtarget &ST);
if (MI->getNumMemOperands() == 0)
     return SIMemOpInfo();
 
   return constructFromMIWithMMO(MI);
 }
 
 SICacheControl::SICacheControl(const GCNSubtarget &ST) {
   TII = ST.getInstrInfo();
   IV = getIsaVersion(ST.getCPU());
-  InsertCacheInv = !ST.isAmdPalOS();
+  InsertCacheInv = !AmdgcnSkipCacheInvalidations;
 }
 
 /* static */
 std::unique_ptr<SICacheControl> SICacheControl::create(const GCNSubtarget &ST) {
   GCNSubtarget::Generation Generation = ST.getGeneration();
   if (Generation <= AMDGPUSubtarget::SOUTHERN_ISLANDS)
     return std::make_unique<SIGfx6CacheControl>(ST);
   if (Generation < AMDGPUSubtarget::GFX10)

llvm/test/CodeGen/AMDGPU/memory-legalizer-amdpal.ll

-; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN9 %s
-; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN9 %s
-; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN9 %s
-; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN9 %s
-; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN10 %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN9,CACHE_INV %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN9,CACHE_INV %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN9,CACHE_INV %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN9,CACHE_INV %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN10,CACHE_INV10 %s
+
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx900 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN9,SKIP_CACHE_INV %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx1010 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN10,SKIP_CACHE_INV %s
+
 
 ; FUNC-LABEL: {{^}}system_acquire:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
 ; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN-NOT:    buffer_wbinvl1{{$}}
+; CACHE_INV:  buffer_wbinvl1{{$}}
+; CACHE_INV10: buffer_gl0_inv
+; CACHE_INV10: buffer_gl1_inv
+; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
+; SKIP_CACHE_INV-NOT: buffer_gl
 ; GCN:        s_endpgm
 define amdgpu_kernel void @system_acquire() {
 entry:
   fence acquire
   ret void
 }
 
 ; FUNC-LABEL: {{^}}system_release:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
 ; GCN10:      s_waitcnt_vscnt null, 0x0
 ; GCN:        s_endpgm
 define amdgpu_kernel void @system_release() {
 entry:
   fence release
   ret void
 }
 
 ; FUNC-LABEL: {{^}}system_acq_rel:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
 ; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN-NOT:    buffer_wbinvl1{{$}}
+; CACHE_INV:  buffer_wbinvl1{{$}}
+; CACHE_INV10: buffer_gl0_inv
+; CACHE_INV10: buffer_gl1_inv
+; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
+; SKIP_CACHE_INV-NOT: buffer_gl
 ; GCN:        s_endpgm
 define amdgpu_kernel void @system_acq_rel() {
 entry:
   fence acq_rel
   ret void
 }
 
 ; FUNC-LABEL: {{^}}system_seq_cst:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
 ; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN-NOT:    buffer_wbinvl1{{$}}
+; CACHE_INV:  buffer_wbinvl1{{$}}
+; CACHE_INV10: buffer_gl0_inv
+; CACHE_INV10: buffer_gl1_inv
+; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
+; SKIP_CACHE_INV-NOT: buffer_gl
 ; GCN:        s_endpgm
 define amdgpu_kernel void @system_seq_cst() {
 entry:
   fence seq_cst
   ret void
 }
 
 ; FUNC-LABEL: {{^}}system_one_as_acquire:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0){{$}}
 ; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN-NOT:    buffer_wbinvl1{{$}}
+; CACHE_INV:  buffer_wbinvl1{{$}}
+; CACHE_INV10: buffer_gl0_inv
+; CACHE_INV10: buffer_gl1_inv
+; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
+; SKIP_CACHE_INV-NOT: buffer_gl
 ; GCN:        s_endpgm
 define amdgpu_kernel void @system_one_as_acquire() {
 entry:
   fence syncscope("one-as") acquire
   ret void
 }
 
 ; FUNC-LABEL: {{^}}system_one_as_release:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0){{$}}
 ; GCN10:      s_waitcnt_vscnt null, 0x0
 ; GCN:        s_endpgm
 define amdgpu_kernel void @system_one_as_release() {
 entry:
   fence syncscope("one-as") release
   ret void
 }
 
 ; FUNC-LABEL: {{^}}system_one_as_acq_rel:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0){{$}}
 ; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN-NOT:    buffer_wbinvl1{{$}}
+; CACHE_INV:  buffer_wbinvl1{{$}}
+; CACHE_INV10: buffer_gl0_inv
+; CACHE_INV10: buffer_gl1_inv
+; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
+; SKIP_CACHE_INV-NOT: buffer_gl
 ; GCN:        s_endpgm
 define amdgpu_kernel void @system_one_as_acq_rel() {
 entry:
   fence syncscope("one-as") acq_rel
   ret void
 }
 
 ; FUNC-LABEL: {{^}}system_one_as_seq_cst:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0){{$}}
 ; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN-NOT:    buffer_wbinvl1{{$}}
+; CACHE_INV:    buffer_wbinvl1{{$}}
+; CACHE_INV10: buffer_gl0_inv
+; CACHE_INV10: buffer_gl1_inv
+; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
+; SKIP_CACHE_INV-NOT: buffer_gl
 ; GCN:        s_endpgm
 define amdgpu_kernel void @system_one_as_seq_cst() {
 entry:
   fence syncscope("one-as") seq_cst
   ret void
 }
 
 ; FUNC-LABEL: {{^}}singlethread_acquire:
entry:
   ret void
 }
 
 ; FUNC-LABEL: {{^}}agent_acquire:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
 ; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN-NOT:    buffer_wbinvl1{{$}}
+; CACHE_INV:  buffer_wbinvl1{{$}}
+; CACHE_INV10: buffer_gl0_inv
+; CACHE_INV10: buffer_gl1_inv
+; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
+; SKIP_CACHE_INV-NOT: buffer_gl
 ; GCN:        s_endpgm
 define amdgpu_kernel void @agent_acquire() {
 entry:
   fence syncscope("agent") acquire
   ret void
 }
 
 ; FUNC-LABEL: {{^}}agent_release:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
 ; GCN10:      s_waitcnt_vscnt null, 0x0
 ; GCN:        s_endpgm
 define amdgpu_kernel void @agent_release() {
 entry:
   fence syncscope("agent") release
   ret void
 }
 
 ; FUNC-LABEL: {{^}}agent_acq_rel:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
 ; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN-NOT:    buffer_wbinvl1{{$}}
+; CACHE_INV:  buffer_wbinvl1{{$}}
+; CACHE_INV10: buffer_gl0_inv
+; CACHE_INV10: buffer_gl1_inv
+; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
+; SKIP_CACHE_INV-NOT: buffer_gl
 ; GCN:        s_endpgm
 define amdgpu_kernel void @agent_acq_rel() {
 entry:
   fence syncscope("agent") acq_rel
   ret void
 }
 
 ; FUNC-LABEL: {{^}}agent_seq_cst:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
 ; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN-NOT:    buffer_wbinvl1{{$}}
+; CACHE_INV:  buffer_wbinvl1{{$}}
+; CACHE_INV10: buffer_gl0_inv
+; CACHE_INV10: buffer_gl1_inv
+; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
+; SKIP_CACHE_INV-NOT: buffer_gl
 ; GCN:        s_endpgm
 define amdgpu_kernel void @agent_seq_cst() {
 entry:
   fence syncscope("agent") seq_cst
   ret void
 }
 
 ; FUNC-LABEL: {{^}}agent_one_as_acquire:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0){{$}}
 ; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN-NOT:    buffer_wbinvl1{{$}}
+; CACHE_INV:  buffer_wbinvl1{{$}}
+; CACHE_INV10: buffer_gl0_inv
+; CACHE_INV10: buffer_gl1_inv
+; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
+; SKIP_CACHE_INV-NOT: buffer_gl
 ; GCN:        s_endpgm
 define amdgpu_kernel void @agent_one_as_acquire() {
 entry:
   fence syncscope("agent-one-as") acquire
   ret void
 }
 
 ; FUNC-LABEL: {{^}}agent_one_as_release:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0){{$}}
 ; GCN10:      s_waitcnt_vscnt null, 0x0
 ; GCN:        s_endpgm
 define amdgpu_kernel void @agent_one_as_release() {
 entry:
   fence syncscope("agent-one-as") release
   ret void
 }
 
 ; FUNC-LABEL: {{^}}agent_one_as_acq_rel:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0){{$}}
 ; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN-NOT:    buffer_wbinvl1{{$}}
+; CACHE_INV:  buffer_wbinvl1{{$}}
+; CACHE_INV10: buffer_gl0_inv
+; CACHE_INV10: buffer_gl1_inv
+; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
+; SKIP_CACHE_INV-NOT: buffer_gl
 ; GCN:        s_endpgm
 define amdgpu_kernel void @agent_one_as_acq_rel() {
 entry:
   fence syncscope("agent-one-as") acq_rel
   ret void
 }
 
 ; FUNC-LABEL: {{^}}agent_one_as_seq_cst:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0){{$}}
 ; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN-NOT:    buffer_wbinvl1{{$}}
+; CACHE_INV:  buffer_wbinvl1{{$}}
+; CACHE_INV10: buffer_gl0_inv
+; CACHE_INV10: buffer_gl1_inv
+; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
+; SKIP_CACHE_INV-NOT: buffer_gl
 ; GCN:        s_endpgm
 define amdgpu_kernel void @agent_one_as_seq_cst() {
 entry:
   fence syncscope("agent-one-as") seq_cst
   ret void
 }
 
 ; FUNC-LABEL: {{^}}workgroup_acquire: