[AMDGPU] Cluster stores as well as loads for GFX11

[Joe_Nash]

LGTM. Perhaps we want to add the store clustering to postRA scheduling as well?

[foad]

Post RA scheduler.

[foad]

Perhaps we want to add the store clustering to postRA scheduling as well?

Done for consistency, it just doesn't seem to have any effect in practice.

llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp

 }
 
 static ScheduleDAGInstrs *createSIMachineScheduler(MachineSchedContext *C) {
   return new SIScheduleDAGMI(C);
 }
 
 static ScheduleDAGInstrs *
 createGCNMaxOccupancyMachineScheduler(MachineSchedContext *C) {
+  const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
   ScheduleDAGMILive *DAG =
     new GCNScheduleDAGMILive(C, std::make_unique<GCNMaxOccupancySchedStrategy>(C));
   DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
+  if (ST.shouldClusterStores())
+    DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
   DAG->addMutation(createIGroupLPDAGMutation());
   DAG->addMutation(createSchedBarrierDAGMutation());
   DAG->addMutation(createAMDGPUMacroFusionDAGMutation());
   DAG->addMutation(createAMDGPUExportClusteringDAGMutation());
   return DAG;
 }
 
 static ScheduleDAGInstrs *
 createIterativeGCNMaxOccupancyMachineScheduler(MachineSchedContext *C) {
+  const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
   auto DAG = new GCNIterativeScheduler(C,
     GCNIterativeScheduler::SCHEDULE_LEGACYMAXOCCUPANCY);
   DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
+  if (ST.shouldClusterStores())
+    DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
   return DAG;
 }
 
 static ScheduleDAGInstrs *createMinRegScheduler(MachineSchedContext *C) {
   return new GCNIterativeScheduler(C,
     GCNIterativeScheduler::SCHEDULE_MINREGFORCED);
 }
 
 static ScheduleDAGInstrs *
 createIterativeILPMachineScheduler(MachineSchedContext *C) {
+  const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
   auto DAG = new GCNIterativeScheduler(C,
     GCNIterativeScheduler::SCHEDULE_ILP);
   DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
+  if (ST.shouldClusterStores())
+    DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
   DAG->addMutation(createAMDGPUMacroFusionDAGMutation());
   return DAG;
 }
 
 static MachineSchedRegistry
 SISchedRegistry("si", "Run SI's custom scheduler",
                 createSIMachineScheduler);
 
public:
   ScheduleDAGInstrs *
   createMachineScheduler(MachineSchedContext *C) const override;
 
   ScheduleDAGInstrs *
   createPostMachineScheduler(MachineSchedContext *C) const override {
     ScheduleDAGMI *DAG = createGenericSchedPostRA(C);
     const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
     DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
+    if (ST.shouldClusterStores())
+      DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
     DAG->addMutation(ST.createFillMFMAShadowMutation(DAG->TII));
     DAG->addMutation(createIGroupLPDAGMutation());
     DAG->addMutation(createSchedBarrierDAGMutation());
     return DAG;
   }
 
   bool addPreISel() override;
   void addMachineSSAOptimization() override;

 
 bool AMDGPUPassConfig::addGCPasses() {
   // Do nothing. GC is not supported.
   return false;
 }
 
 llvm::ScheduleDAGInstrs *
 AMDGPUPassConfig::createMachineScheduler(MachineSchedContext *C) const {
+  const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
   ScheduleDAGMILive *DAG = createGenericSchedLive(C);
   DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
+  if (ST.shouldClusterStores())
+    DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
   return DAG;
 }
 
 //===----------------------------------------------------------------------===//
 // GCN Pass Setup
 //===----------------------------------------------------------------------===//
 
 ScheduleDAGInstrs *GCNPassConfig::createMachineScheduler(

llvm/test/CodeGen/AMDGPU/cluster_stores.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-xnack -verify-machineinstrs -debug-only=machine-scheduler < %s 2> %t | FileCheck --enable-var-scope --check-prefix=GFX9 %s
 ; RUN: FileCheck --enable-var-scope --check-prefix=DBG %s < %t
 ; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -debug-only=machine-scheduler < %s 2> %t | FileCheck --enable-var-scope --check-prefix=GFX10 %s
 ; RUN: FileCheck --enable-var-scope --check-prefix=DBG %s < %t
+; RUN: llc -march=amdgcn -mcpu=gfx1100 -verify-machineinstrs -debug-only=machine-scheduler < %s 2> %t | FileCheck --enable-var-scope --check-prefix=GFX11 %s
+; RUN: FileCheck --enable-var-scope --check-prefixes=DBG,DBG11 %s < %t
 ; REQUIRES: asserts
 
 ; FIXME: Verifier error with xnack enabled.
 
 ; DBG-LABEL: cluster_load_cluster_store:
 
 ; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 8
 ; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 8
 ; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
 ; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
 ; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
 ; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
 
 ; DBG: Cluster ld/st SU(1) - SU(2)
 
 ; DBG: Cluster ld/st SU([[L1:[0-9]+]]) - SU([[L2:[0-9]+]])
 ; DBG: Cluster ld/st SU([[L2]]) - SU([[L3:[0-9]+]])
 ; DBG: Cluster ld/st SU([[L3]]) - SU([[L4:[0-9]+]])
 
+; DBG11: Cluster ld/st SU([[S1:[0-9]+]]) - SU([[S2:[0-9]+]])
+; DBG11: Cluster ld/st SU([[S2]]) - SU([[S3:[0-9]+]])
+; DBG11: Cluster ld/st SU([[S3]]) - SU([[S4:[0-9]+]])
+
 ; DBG-NOT: Cluster ld/st
 
 define amdgpu_kernel void @cluster_load_cluster_store(i32* noalias %lb, i32* noalias %sb) {
 ; GFX9-LABEL: cluster_load_cluster_store:
 ; GFX9:       ; %bb.0: ; %bb
 ; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)

 ; GFX10-NEXT:    flat_store_dword v[0:1], v8
 ; GFX10-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(3)
 ; GFX10-NEXT:    flat_store_dword v[2:3], v9
 ; GFX10-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(3)
 ; GFX10-NEXT:    flat_store_dword v[4:5], v10
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(3)
 ; GFX10-NEXT:    flat_store_dword v[6:7], v11
 ; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: cluster_load_cluster_store:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[2:3], s[0:1], 0x24
+; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_mov_b32_e32 v0, s2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s3
+; GFX11-NEXT:    s_clause 0x3
+; GFX11-NEXT:    flat_load_b32 v2, v[0:1]
+; GFX11-NEXT:    flat_load_b32 v3, v[0:1] offset:8
+; GFX11-NEXT:    flat_load_b32 v4, v[0:1] offset:16
+; GFX11-NEXT:    flat_load_b32 v5, v[0:1] offset:24
+; GFX11-NEXT:    v_mov_b32_e32 v0, s0
+; GFX11-NEXT:    v_mov_b32_e32 v1, s1
+; GFX11-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(3)
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2
+; GFX11-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(3)
+; GFX11-NEXT:    flat_store_b32 v[0:1], v3 offset:8
+; GFX11-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(3)
+; GFX11-NEXT:    flat_store_b32 v[0:1], v4 offset:16
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(3)
+; GFX11-NEXT:    flat_store_b32 v[0:1], v5 offset:24
+; GFX11-NEXT:    s_endpgm
 bb:
   %la0 = getelementptr inbounds i32, i32* %lb, i32 0
   %ld0 = load i32, i32* %la0
   %la1 = getelementptr inbounds i32, i32* %lb, i32 2
   %ld1 = load i32, i32* %la1
   %la2 = getelementptr inbounds i32, i32* %lb, i32 4
   %ld2 = load i32, i32* %la2
   %la3 = getelementptr inbounds i32, i32* %lb, i32 6

 ; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
 
 ; DBG: Cluster ld/st SU(1) - SU(2)
 
 ; DBG: Cluster ld/st SU([[L1:[0-9]+]]) - SU([[L2:[0-9]+]])
 ; DBG: Cluster ld/st SU([[L2]]) - SU([[L3:[0-9]+]])
 ; DBG: Cluster ld/st SU([[L3]]) - SU([[L4:[0-9]+]])
 
+; DBG11: Cluster ld/st SU([[S1:[0-9]+]]) - SU([[S2:[0-9]+]])
+; DBG11: Cluster ld/st SU([[S2]]) - SU([[S3:[0-9]+]])
+; DBG11: Cluster ld/st SU([[S3]]) - SU([[S4:[0-9]+]])
+
 ; DBG-NOT: Cluster ld/st
 
 define amdgpu_kernel void @cluster_load_valu_cluster_store(i32* noalias %lb, i32* noalias %sb) {
 ; GFX9-LABEL: cluster_load_valu_cluster_store:
 ; GFX9:       ; %bb.0: ; %bb
 ; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)

 ; GFX10-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
 ; GFX10-NEXT:    flat_store_dword v[0:1], v8
 ; GFX10-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(2)
 ; GFX10-NEXT:    flat_store_dword v[4:5], v9
 ; GFX10-NEXT:    flat_store_dword v[2:3], v11
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(3)
 ; GFX10-NEXT:    flat_store_dword v[6:7], v10
 ; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: cluster_load_valu_cluster_store:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b64 s[2:3], s[0:1], 0x24
+; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x2c
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_mov_b32_e32 v0, s2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s3
+; GFX11-NEXT:    s_clause 0x3
+; GFX11-NEXT:    flat_load_b32 v2, v[0:1] offset:8
+; GFX11-NEXT:    flat_load_b32 v3, v[0:1]
+; GFX11-NEXT:    flat_load_b32 v4, v[0:1] offset:16
+; GFX11-NEXT:    flat_load_b32 v5, v[0:1] offset:24
+; GFX11-NEXT:    v_mov_b32_e32 v0, s0
+; GFX11-NEXT:    v_mov_b32_e32 v1, s1
+; GFX11-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(3)
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 1, v2
+; GFX11-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    flat_store_b32 v[0:1], v3
+; GFX11-NEXT:    flat_store_b32 v[0:1], v2 offset:8
+; GFX11-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(3)
+; GFX11-NEXT:    flat_store_b32 v[0:1], v4 offset:16
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(3)
+; GFX11-NEXT:    flat_store_b32 v[0:1], v5 offset:24
+; GFX11-NEXT:    s_endpgm
 bb:
   %la0 = getelementptr inbounds i32, i32* %lb, i32 0
   %ld0 = load i32, i32* %la0
   %la1 = getelementptr inbounds i32, i32* %lb, i32 2
   %ld1 = load i32, i32* %la1
   %la2 = getelementptr inbounds i32, i32* %lb, i32 4
   %ld2 = load i32, i32* %la2
   %la3 = getelementptr inbounds i32, i32* %lb, i32 6

 ; GFX10-NEXT:    image_load v[6:9], v[12:13], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_add_f32_e32 v5, v5, v9
 ; GFX10-NEXT:    v_add_f32_e32 v4, v4, v8
 ; GFX10-NEXT:    v_add_f32_e32 v3, v3, v7
 ; GFX10-NEXT:    v_add_f32_e32 v2, v2, v6
 ; GFX10-NEXT:    image_store v[2:5], v[0:1], s[8:15] dmask:0xf dim:SQ_RSRC_IMG_2D unorm
 ; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: cluster_image_load:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 1, v0
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 1, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 2, v0
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 2, v1
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    image_load v[2:5], v[2:3], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm
+; GFX11-NEXT:    image_load v[6:9], v[6:7], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_add_f32_e32 v5, v5, v9
+; GFX11-NEXT:    v_add_f32_e32 v4, v4, v8
+; GFX11-NEXT:    v_add_f32_e32 v3, v3, v7
+; GFX11-NEXT:    v_add_f32_e32 v2, v2, v6
+; GFX11-NEXT:    image_store v[2:5], v[0:1], s[8:15] dmask:0xf dim:SQ_RSRC_IMG_2D unorm
+; GFX11-NEXT:    s_endpgm
 entry:
   %x1 = add i32 %x, 1
   %y1 = add i32 %y, 1
   %val1 = call <4 x float> @llvm.amdgcn.image.load.2d.v4f32.i32(i32 15, i32 %x1, i32 %y1, <8 x i32> %src, i32 0, i32 0)
   %x2 = add i32 %x, 2
   %y2 = add i32 %y, 2
   %val2 = call <4 x float> @llvm.amdgcn.image.load.2d.v4f32.i32(i32 15, i32 %x2, i32 %y2, <8 x i32> %src, i32 0, i32 0)
   %val = fadd fast <4 x float> %val1, %val2

 ; GFX10-NEXT:    image_load_mip v[6:9], [v0, v1, v10], s[8:15] dmask:0xf dim:SQ_RSRC_IMG_2D unorm
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_add_f32_e32 v5, v5, v9
 ; GFX10-NEXT:    v_add_f32_e32 v4, v4, v8
 ; GFX10-NEXT:    v_add_f32_e32 v3, v3, v7
 ; GFX10-NEXT:    v_add_f32_e32 v2, v2, v6
 ; GFX10-NEXT:    image_store v[2:5], v[0:1], s[16:23] dmask:0xf dim:SQ_RSRC_IMG_2D unorm
 ; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: no_cluster_image_load:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    v_mov_b32_e32 v6, 0
+; GFX11-NEXT:    image_load_mip v[2:5], [v0, v1, v6], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm
+; GFX11-NEXT:    image_load_mip v[6:9], [v0, v1, v6], s[8:15] dmask:0xf dim:SQ_RSRC_IMG_2D unorm
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_add_f32_e32 v5, v5, v9
+; GFX11-NEXT:    v_add_f32_e32 v4, v4, v8
+; GFX11-NEXT:    v_add_f32_e32 v3, v3, v7
+; GFX11-NEXT:    v_add_f32_e32 v2, v2, v6
+; GFX11-NEXT:    image_store v[2:5], v[0:1], s[16:23] dmask:0xf dim:SQ_RSRC_IMG_2D unorm
+; GFX11-NEXT:    s_endpgm
 entry:
   %val1 = call <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32(i32 15, i32 %x, i32 %y, i32 0, <8 x i32> %src1, i32 0, i32 0)
   %val2 = call <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32(i32 15, i32 %x, i32 %y, i32 0, <8 x i32> %src2, i32 0, i32 0)
   %val = fadd fast <4 x float> %val1, %val2
   call void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float> %val, i32 15, i32 %x, i32 %y, <8 x i32> %dst, i32 0, i32 0)
   ret void
 }
 

 ; GFX10-NEXT:    image_sample_d v[18:21], v[8:13], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_add_f32_e32 v5, v17, v21
 ; GFX10-NEXT:    v_add_f32_e32 v4, v16, v20
 ; GFX10-NEXT:    v_add_f32_e32 v3, v15, v19
 ; GFX10-NEXT:    v_add_f32_e32 v2, v14, v18
 ; GFX10-NEXT:    image_store v[2:5], v[0:1], s[12:19] dmask:0xf dim:SQ_RSRC_IMG_2D unorm
 ; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: cluster_image_sample:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    v_cvt_f32_i32_e32 v8, v0
+; GFX11-NEXT:    v_cvt_f32_i32_e32 v9, v1
+; GFX11-NEXT:    v_mov_b32_e32 v4, 0
+; GFX11-NEXT:    v_mov_b32_e32 v10, 1.0
+; GFX11-NEXT:    v_add_f32_e32 v2, 1.0, v8
+; GFX11-NEXT:    v_add_f32_e32 v3, 1.0, v9
+; GFX11-NEXT:    v_mov_b32_e32 v5, v4
+; GFX11-NEXT:    v_mov_b32_e32 v6, v4
+; GFX11-NEXT:    v_mov_b32_e32 v7, v4
+; GFX11-NEXT:    v_add_f32_e32 v8, 2.0, v8
+; GFX11-NEXT:    v_add_f32_e32 v9, 2.0, v9
+; GFX11-NEXT:    v_mov_b32_e32 v11, v10
+; GFX11-NEXT:    v_mov_b32_e32 v12, v10
+; GFX11-NEXT:    v_mov_b32_e32 v13, v10
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    image_sample_d v[2:5], v[2:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX11-NEXT:    image_sample_d v[6:9], v[8:13], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_add_f32_e32 v5, v5, v9
+; GFX11-NEXT:    v_add_f32_e32 v4, v4, v8
+; GFX11-NEXT:    v_add_f32_e32 v3, v3, v7
+; GFX11-NEXT:    v_add_f32_e32 v2, v2, v6
+; GFX11-NEXT:    image_store v[2:5], v[0:1], s[12:19] dmask:0xf dim:SQ_RSRC_IMG_2D unorm
+; GFX11-NEXT:    s_endpgm
 entry:
   %s = sitofp i32 %x to float
   %t = sitofp i32 %y to float
   %s1 = fadd float %s, 1.0
   %t1 = fadd float %t, 1.0
   %val1 = call <4 x float> @llvm.amdgcn.image.sample.d.2d.v4f32.f32(i32 15, float %s1, float %t1, float 0.0, float 0.0, float 0.0, float 0.0, <8 x i32> %src, <4 x i32> %smp, i1 false, i32 0, i32 0)
   %s2 = fadd float %s, 2.0
   %t2 = fadd float %t, 2.0