diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp @@ -354,6 +354,21 @@ return true; } + if (isMIMG(LdSt)) { + int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc); + BaseOps.push_back(&LdSt.getOperand(SRsrcIdx)); + int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0); + if (VAddr0Idx >= 0) { + // GFX10 possible NSA encoding. + for (int I = VAddr0Idx; I < SRsrcIdx; ++I) + BaseOps.push_back(&LdSt.getOperand(I)); + } else { + BaseOps.push_back(getNamedOperand(LdSt, AMDGPU::OpName::vaddr)); + } + Offset = 0; + return true; + } + if (isSMRD(LdSt)) { BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::sbase); if (!BaseOp) // e.g. S_MEMTIME @@ -379,22 +394,14 @@ return false; } -static bool -memOpsHaveSameBaseOperands(ArrayRef BaseOps1, - ArrayRef BaseOps2) { - if (BaseOps1.size() != BaseOps2.size()) - return false; - for (size_t I = 0, E = BaseOps1.size(); I < E; ++I) - if (!BaseOps1[I]->isIdenticalTo(*BaseOps2[I])) - return false; - return true; -} - static bool memOpsHaveSameBasePtr(const MachineInstr &MI1, ArrayRef BaseOps1, const MachineInstr &MI2, ArrayRef BaseOps2) { - if (memOpsHaveSameBaseOperands(BaseOps1, BaseOps2)) + // Only examine the first "base" operand of each instruction, on the + // assumption that it represents the real base address of the memory access. + // Other operands are typically offsets or indices from this base address. + if (BaseOps1.front()->isIdenticalTo(*BaseOps2.front())) return true; if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand()) @@ -435,6 +442,7 @@ if ((isMUBUF(FirstLdSt) && isMUBUF(SecondLdSt)) || (isMTBUF(FirstLdSt) && isMTBUF(SecondLdSt)) || + (isMIMG(FirstLdSt) && isMIMG(SecondLdSt)) || (isFLAT(FirstLdSt) && isFLAT(SecondLdSt))) { const unsigned MaxGlobalLoadCluster = 7; if (NumLoads > MaxGlobalLoadCluster) @@ -2598,6 +2606,17 @@ return false; } +static bool +memOpsHaveSameBaseOperands(ArrayRef BaseOps1, + ArrayRef BaseOps2) { + if (BaseOps1.size() != BaseOps2.size()) + return false; + for (size_t I = 0, E = BaseOps1.size(); I < E; ++I) + if (!BaseOps1[I]->isIdenticalTo(*BaseOps2[I])) + return false; + return true; +} + static bool offsetsDoNotOverlap(int WidthA, int OffsetA, int WidthB, int OffsetB) { int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB; diff --git a/llvm/test/CodeGen/AMDGPU/cluster_stores.ll b/llvm/test/CodeGen/AMDGPU/cluster_stores.ll --- a/llvm/test/CodeGen/AMDGPU/cluster_stores.ll +++ b/llvm/test/CodeGen/AMDGPU/cluster_stores.ll @@ -83,3 +83,36 @@ ret void } + +; Cluster loads from the same texture with different coordinates +; CHECK-LABEL: {{^}}cluster_image_load: +; DBG: {{^}}Cluster ld/st [[SU1:SU\([0-9]+\)]] - [[SU2:SU\([0-9]+\)]] +; DBG: {{^}}[[SU1]]: {{.*}} IMAGE_LOAD +; DBG: {{^}}[[SU2]]: {{.*}} IMAGE_LOAD +define amdgpu_ps void @cluster_image_load(<8 x i32> inreg %src, <8 x i32> inreg %dst, i32 %x, i32 %y) { +entry: + %x1 = add i32 %x, 1 + %y1 = add i32 %y, 1 + %val1 = call <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32(i32 15, i32 %x1, i32 %y1, i32 0, <8 x i32> %src, i32 0, i32 0) + %x2 = add i32 %x, 2 + %y2 = add i32 %y, 2 + %val2 = call <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32(i32 15, i32 %x2, i32 %y2, i32 0, <8 x i32> %src, i32 0, i32 0) + %val = fadd fast <4 x float> %val1, %val2 + call void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float> %val, i32 15, i32 %x, i32 %y, <8 x i32> %dst, i32 0, i32 0) + ret void +} + +; Don't cluster loads from different textures +; CHECK-LABEL: {{^}}no_cluster_image_load: +; DBG-NOT: {{^}}Cluster ld/st +define amdgpu_ps void @no_cluster_image_load(<8 x i32> inreg %src1, <8 x i32> inreg %src2, <8 x i32> inreg %dst, i32 %x, i32 %y) { +entry: + %val1 = call <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32(i32 15, i32 %x, i32 %y, i32 0, <8 x i32> %src1, i32 0, i32 0) + %val2 = call <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32(i32 15, i32 %x, i32 %y, i32 0, <8 x i32> %src2, i32 0, i32 0) + %val = fadd fast <4 x float> %val1, %val2 + call void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float> %val, i32 15, i32 %x, i32 %y, <8 x i32> %dst, i32 0, i32 0) + ret void +} + +declare <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32(i32 immarg, i32, i32, i32, <8 x i32>, i32 immarg, i32 immarg) +declare void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float>, i32 immarg, i32, i32, <8 x i32>, i32 immarg, i32 immarg)