[AMDGPU/MemOpsCluster] Compute `width` for `MIMG` instruction class.

[hsmhsm]

Summary

width computation is missing for newly added MIMG
instruction class. Add it.

[rampitec]

Test?

[hsmhsm]

Added test.

[foad]

llvm/lib/Target/AMDGPU/SIInstrInfo.cpp

400–401

Do you need this bit? Does it work if you just use "vdata" for all MIMG instructions?

[foad]

llvm/test/CodeGen/AMDGPU/cluster_stores.ll

8–10

I don't think this really adds any value to the test, unless you're going to check for specific numbers like "Width: 4" or similar?

[hsmhsm]

llvm/lib/Target/AMDGPU/SIInstrInfo.cpp

400–401

I do not know much about MIMG instructions, I added it based on my experience of dealing with other instructions. If you confirm only "vdata" is needed, then, I will update it.

llvm/test/CodeGen/AMDGPU/cluster_stores.ll

8–10

I can do that by extracting width by dbg dump, and updating the same in the test.

[hsmhsm]

llvm/test/CodeGen/AMDGPU/cluster_stores.ll

8–10

Morever, to clarify, my intension here was to test that Width is greather than 0 (should not be 0)

[hsmhsm]

Taken care of review comments by Jay.

[hsmhsm]

llvm/lib/Target/AMDGPU/SIInstrInfo.cpp

400–401

Done

llvm/test/CodeGen/AMDGPU/cluster_stores.ll

8–10

Done

llvm/lib/CodeGen/MachineScheduler.cpp

void BaseMemOpClusterMutation::clusterNeighboringMemOps(
   SmallVector<MemOpInfo, 32> MemOpRecords;
   for (SUnit *SU : MemOps) {
     const MachineInstr &MI = *SU->getInstr();
     SmallVector<const MachineOperand *, 4> BaseOps;
     int64_t Offset;
     bool OffsetIsScalable;
     unsigned Width;
     if (TII->getMemOperandsWithOffsetWidth(MI, BaseOps, Offset,
-                                           OffsetIsScalable, Width, TRI))
+                                           OffsetIsScalable, Width, TRI)) {
       MemOpRecords.push_back(MemOpInfo(SU, BaseOps, Offset, Width));
+
+      LLVM_DEBUG(dbgs() << "Num BaseOps: " << BaseOps.size() << ", Offset: "
+                        << Offset << ", OffsetIsScalable: " << OffsetIsScalable
+                        << ", Width: " << Width << "\n");
+    }
 #ifndef NDEBUG
     for (auto *Op : BaseOps)
       assert(Op);
 #endif
   }
   if (MemOpRecords.size() < 2)
     return;
 
for (unsigned Idx = 0, End = MemOpRecords.size(); Idx < (End - 1); ++Idx) {
     // nearby loads should have effectively the same inputs.
     for (const SDep &Succ : SUa->Succs) {
       if (Succ.getSUnit() == SUb)
         continue;
       LLVM_DEBUG(dbgs() << "  Copy Succ SU(" << Succ.getSUnit()->NodeNum
                         << ")\n");
       DAG->addEdge(Succ.getSUnit(), SDep(SUb, SDep::Artificial));
     }
+
+    LLVM_DEBUG(dbgs() << "  Curr cluster length: " << ClusterLength
+                      << ", Curr cluster bytes: " << CurrentClusterBytes
+                      << "\n");
   }
 }
 
 /// Callback from DAG postProcessing to create cluster edges for loads.
 void BaseMemOpClusterMutation::apply(ScheduleDAGInstrs *DAG) {
   // Map DAG NodeNum to a set of dependent MemOps in store chain.
   DenseMap<unsigned, SmallVector<SUnit *, 4>> StoreChains;
   for (SUnit &SU : DAG->SUnits) {

llvm/lib/Target/AMDGPU/SIInstrInfo.cpp

if (isMIMG(LdSt)) {
     if (VAddr0Idx >= 0) {
       // GFX10 possible NSA encoding.
       for (int I = VAddr0Idx; I < SRsrcIdx; ++I)
         BaseOps.push_back(&LdSt.getOperand(I));
     } else {
       BaseOps.push_back(getNamedOperand(LdSt, AMDGPU::OpName::vaddr));
     }
     Offset = 0;
+    // Get appropriate operand, and compute width accordingly.
+    DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);
+    Width = getOpSize(LdSt, DataOpIdx);
     return true;
   }
 
   if (isSMRD(LdSt)) {
     BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::sbase);
     if (!BaseOp) // e.g. S_MEMTIME
       return false;
     BaseOps.push_back(BaseOp);

llvm/test/CodeGen/AMDGPU/cluster_stores.ll

 ; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -debug-only=machine-scheduler < %s 2> %t | FileCheck --enable-var-scope --check-prefixes=CHECK,GCN %s
 ; RUN: FileCheck --enable-var-scope --check-prefixes=CHECK,DBG %s < %t
 ; REQUIRES: asserts
 
 ; CHECK-LABEL: {{^}}cluster_load_cluster_store:
 define amdgpu_kernel void @cluster_load_cluster_store(i32* noalias %lb, i32* noalias %sb) {
 bb:
+
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 8
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 8
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
+
 ; DBG: Cluster ld/st SU(1) - SU(2)
 
 ; DBG: Cluster ld/st SU([[L1:[0-9]+]]) - SU([[L2:[0-9]+]])
 ; DBG: Cluster ld/st SU([[L2]]) - SU([[L3:[0-9]+]])
 ; DBG: Cluster ld/st SU([[L3]]) - SU([[L4:[0-9]+]])
 ; GCN:      flat_load_dword [[LD1:v[0-9]+]], v[{{[0-9:]+}}]
 ; GCN-NEXT: flat_load_dword [[LD2:v[0-9]+]], v[{{[0-9:]+}}] offset:8
 ; GCN-NEXT: flat_load_dword [[LD3:v[0-9]+]], v[{{[0-9:]+}}] offset:16
; GCN-NEXT: flat_store_dword v[{{[0-9:]+}}], [[LD4]] offset:24
   store i32 %ld3, i32* %sa3
 
   ret void
 }
 
 ; CHECK-LABEL: {{^}}cluster_load_valu_cluster_store:
 define amdgpu_kernel void @cluster_load_valu_cluster_store(i32* noalias %lb, i32* noalias %sb) {
 bb:
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 8
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 8
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
+
 ; DBG: Cluster ld/st SU(1) - SU(2)
 
 ; DBG: Cluster ld/st SU([[L1:[0-9]+]]) - SU([[L2:[0-9]+]])
 ; DBG: Cluster ld/st SU([[L2]]) - SU([[L3:[0-9]+]])
 ; DBG: Cluster ld/st SU([[L3]]) - SU([[L4:[0-9]+]])
 ; GCN:      flat_load_dword [[LD1:v[0-9]+]], v[{{[0-9:]+}}]
 ; GCN-NEXT: flat_load_dword [[LD2:v[0-9]+]], v[{{[0-9:]+}}] offset:8
 ; GCN-NEXT: flat_load_dword [[LD3:v[0-9]+]], v[{{[0-9:]+}}] offset:16
; GCN-NEXT: flat_store_dword v[{{[0-9:]+}}], [[LD4]] offset:24
   %sa3 = getelementptr inbounds i32, i32* %sb, i32 6
   store i32 %ld3, i32* %sa3
 
   ret void
 }
 
 ; Cluster loads from the same texture with different coordinates
 ; CHECK-LABEL: {{^}}cluster_image_load:
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16
 ; DBG: {{^}}Cluster ld/st [[SU1:SU\([0-9]+\)]] - [[SU2:SU\([0-9]+\)]]
 ; DBG: {{^}}[[SU1]]: {{.*}} IMAGE_LOAD
 ; DBG: {{^}}[[SU2]]: {{.*}} IMAGE_LOAD
 ; GCN:      image_load v
 ; GCN-NEXT: image_load v
 define amdgpu_ps void @cluster_image_load(<8 x i32> inreg %src, <8 x i32> inreg %dst, i32 %x, i32 %y) {
 entry:
   %x1 = add i32 %x, 1
   %y1 = add i32 %y, 1
   %val1 = call <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32(i32 15, i32 %x1, i32 %y1, i32 0, <8 x i32> %src, i32 0, i32 0)
   %x2 = add i32 %x, 2
   %y2 = add i32 %y, 2
   %val2 = call <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32(i32 15, i32 %x2, i32 %y2, i32 0, <8 x i32> %src, i32 0, i32 0)
   %val = fadd fast <4 x float> %val1, %val2
   call void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float> %val, i32 15, i32 %x, i32 %y, <8 x i32> %dst, i32 0, i32 0)
   ret void
 }
 
 ; Don't cluster loads from different textures
 ; CHECK-LABEL: {{^}}no_cluster_image_load:
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16
 ; DBG-NOT: {{^}}Cluster ld/st
 define amdgpu_ps void @no_cluster_image_load(<8 x i32> inreg %src1, <8 x i32> inreg %src2, <8 x i32> inreg %dst, i32 %x, i32 %y) {
 entry:
   %val1 = call <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32(i32 15, i32 %x, i32 %y, i32 0, <8 x i32> %src1, i32 0, i32 0)
   %val2 = call <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32(i32 15, i32 %x, i32 %y, i32 0, <8 x i32> %src2, i32 0, i32 0)
   %val = fadd fast <4 x float> %val1, %val2
   call void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float> %val, i32 15, i32 %x, i32 %y, <8 x i32> %dst, i32 0, i32 0)
   ret void
 }
 
 ; Cluster loads from the same texture and sampler with different coordinates
 ; CHECK-LABEL: {{^}}cluster_image_sample:
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16
+; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16
 ; DBG: {{^}}Cluster ld/st [[SU1:SU\([0-9]+\)]] - [[SU2:SU\([0-9]+\)]]
 ; DBG: {{^}}[[SU1]]: {{.*}} IMAGE_SAMPLE
 ; DBG: {{^}}[[SU2]]: {{.*}} IMAGE_SAMPLE
 ; GCN:      image_sample_d
 ; GCN-NEXT: image_sample_d
 define amdgpu_ps void @cluster_image_sample(<8 x i32> inreg %src, <4 x i32> inreg %smp, <8 x i32> inreg %dst, i32 %x, i32 %y) {
 entry:
   %s = sitofp i32 %x to float