diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp --- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp @@ -81,7 +81,9 @@ }; typedef DenseMap> SUnitsToCandidateSGsMap; - +typedef function_ref, + const SIInstrInfo *)> + InstructionRuleType; // Classify instructions into groups to enable fine tuned control over the // scheduler. These groups may be more specific than current SchedModel // instruction classes. @@ -95,6 +97,9 @@ // Maximum number of SUnits that can be added to this group. std::optional MaxSize; + // The different rules each instruction in this SchedGroup must conform to + std::optional> Rules; + // SchedGroups will only synchronize with other SchedGroups that have the same // SyncID. int SyncID = 0; @@ -145,6 +150,18 @@ // Returns true if no more instructions may be added to this group. bool isFull() const { return MaxSize && Collection.size() >= *MaxSize; } + // Returns true if the SU matches all rules + bool allowedByRules(const SUnit *SU) const { + if (!Rules.has_value()) + return true; + for (auto &Rule : *Rules) { + if (!Rule(SU, Collection, TII)) { + return false; + } + } + return true; + } + // Add SU to the SchedGroup. void add(SUnit &SU) { LLVM_DEBUG(dbgs() << "For SchedGroup with mask " @@ -176,14 +193,17 @@ SchedGroupMask getMask() { return SGMask; } SchedGroup(SchedGroupMask SGMask, std::optional MaxSize, + std::optional> Rules, ScheduleDAGInstrs *DAG, const SIInstrInfo *TII) - : SGMask(SGMask), MaxSize(MaxSize), DAG(DAG), TII(TII) { + : SGMask(SGMask), MaxSize(MaxSize), Rules(Rules), DAG(DAG), TII(TII) { SGID = NumSchedGroups++; } - SchedGroup(SchedGroupMask SGMask, std::optional MaxSize, int SyncID, - ScheduleDAGInstrs *DAG, const SIInstrInfo *TII) - : SGMask(SGMask), MaxSize(MaxSize), SyncID(SyncID), DAG(DAG), TII(TII) { + SchedGroup(SchedGroupMask SGMask, std::optional MaxSize, + std::optional> Rules, + int SyncID, ScheduleDAGInstrs *DAG, const SIInstrInfo *TII) + : SGMask(SGMask), MaxSize(MaxSize), Rules(Rules), SyncID(SyncID), + DAG(DAG), TII(TII) { SGID = NumSchedGroups++; } }; @@ -569,6 +589,9 @@ if (Match->isFull()) continue; + if (!Match->allowedByRules(CurrSU.first)) + continue; + LLVM_DEBUG(dbgs() << "Assigning to SchedGroup with Mask " << (int)Match->getMask() << "and ID " << CandSGID << "\n"); @@ -656,6 +679,11 @@ LLVM_DEBUG(dbgs() << "SGID # " << CandSGID << " is full\n"); continue; } + if (!Match->allowedByRules(CurrSU.first)) { + LLVM_DEBUG(dbgs() << "SGID # " << CandSGID + << " has conflicting rule\n"); + continue; + } TempCost = addEdges(SyncPipeline, CurrSU.first, CandSGID, AddedEdges); LLVM_DEBUG(dbgs() << "Cost of Group " << TempCost << "\n"); if (TempCost < BestNodeCost || BestNodeCost == -1) { @@ -723,7 +751,7 @@ makePipeline(); } -enum IGLPStrategyID : int { MFMASmallGemmOptID = 0 }; +enum IGLPStrategyID : int { MFMASmallGemmOptID = 0, DemoID = 1 }; // Implement a IGLP scheduling strategy. class IGLPStrategy { @@ -772,11 +800,82 @@ SchedGroup *SG = nullptr; for (unsigned I = 0; I < MFMACount * 3; ++I) { SG = &SyncedSchedGroups[PipelineSyncID].emplace_back( - SchedGroupMask::DS, 2, PipelineSyncID, DAG, TII); + SchedGroupMask::DS, 2, std::nullopt, PipelineSyncID, DAG, TII); SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]); SG = &SyncedSchedGroups[PipelineSyncID].emplace_back( - SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII); + SchedGroupMask::MFMA, 1, std::nullopt, PipelineSyncID, DAG, TII); + SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]); + } +} + +class DemoOpt final : public IGLPStrategy { +public: + void applyIGLPStrategy( + DenseMap &SyncedInstrs, + DenseMap> &SyncedSchedGroups) override; + + bool shouldApplyStrategy(ScheduleDAGInstrs *DAG) override { return true; } + + DemoOpt(ScheduleDAGInstrs *DAG, const SIInstrInfo *TII) + : IGLPStrategy(DAG, TII) {} +}; + +void DemoOpt::applyIGLPStrategy( + DenseMap &SyncedInstrs, + DenseMap> &SyncedSchedGroups) { + // Count the number of MFMA instructions. + const unsigned PipelineSyncID = 0; + SchedGroup *SG = nullptr; + + // The SchedGroup has 1 MFMA and 1 DS_W, where the DS_W is a successor of the + // MFMA + InstructionRuleType Rule1 = [](const SUnit *SU, ArrayRef Collection, + const SIInstrInfo *TII) { + auto MI = SU->getInstr(); + if (MI->getOpcode() == TargetOpcode::BUNDLE) + return false; + if (TII->isDS(*MI) && MI->mayStore()) { + if (Collection.size() > 1) + return false; + if (Collection.size() == 0) + return true; + + auto OtherElt = Collection[0]; + if (TII->isMFMAorWMMA(*OtherElt->getInstr())) { + for (auto &S : OtherElt->Succs) { + if (S.getSUnit() == SU) + return true; + } + } + return false; + } + if (TII->isMFMAorWMMA(*MI)) { + if (Collection.size() > 1) + return false; + if (Collection.size() == 0) + return true; + + auto OtherElt = Collection[0]; + if (TII->isDS(*OtherElt->getInstr()) && MI->mayStore()) { + for (auto &S : OtherElt->Preds) { + if (S.getSUnit() == SU) + return true; + } + } + return false; + } + return false; + }; + + SmallVector DemoRules; + DemoRules.push_back(Rule1); + + auto Mask = SchedGroupMask::MFMA | SchedGroupMask::DS_WRITE; + + for (unsigned I = 0; I < 5; ++I) { + SG = &SyncedSchedGroups[PipelineSyncID].emplace_back( + Mask, 2, DemoRules, PipelineSyncID, DAG, TII); SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]); } } @@ -787,6 +886,8 @@ switch (ID) { case MFMASmallGemmOptID: return std::make_unique(DAG, TII); + case DemoID: + return std::make_unique(DAG, TII); } llvm_unreachable("Unknown IGLPStrategyID"); @@ -844,6 +945,7 @@ bool SchedGroup::canAddMI(const MachineInstr &MI) const { bool Result = false; + if (MI.isMetaInstruction()) Result = false; @@ -950,6 +1052,17 @@ bool SchedGroup::canAddSU(SUnit &SU) const { MachineInstr &MI = *SU.getInstr(); + + // At SchedGroup init time, collections will be empty. Thus, any rule + // inspecting the stored contents of collections will not be relevant during + // SchedGroup initialization + if (Rules.has_value()) { + for (auto &Rule : *Rules) { + if (!Rule(&SU, Collection, TII)) + return false; + } + } + if (MI.getOpcode() != TargetOpcode::BUNDLE) return canAddMI(MI); @@ -1049,7 +1162,7 @@ resetEdges(SchedBarrier, DAG); auto InvertedMask = invertSchedBarrierMask((SchedGroupMask)MI.getOperand(0).getImm()); - SchedGroup SG(InvertedMask, std::nullopt, DAG, TII); + SchedGroup SG(InvertedMask, std::nullopt, std::nullopt, DAG, TII); SG.initSchedGroup(); // Preserve original instruction ordering relative to the SCHED_BARRIER. SG.link( @@ -1104,8 +1217,8 @@ int32_t Size = SGB.getOperand(1).getImm(); int32_t SyncID = SGB.getOperand(2).getImm(); - auto &SG = SyncedSchedGroups[SyncID].emplace_back((SchedGroupMask)SGMask, - Size, SyncID, DAG, TII); + auto &SG = SyncedSchedGroups[SyncID].emplace_back( + (SchedGroupMask)SGMask, Size, std::nullopt, SyncID, DAG, TII); SG.initSchedGroup(RIter, SyncedInstrs[SG.getSyncID()]); } diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll @@ -147,9 +147,99 @@ ret void } + +define amdgpu_kernel void @test_iglp_opt_demo(ptr addrspace(3) noalias %in, ptr addrspace(3) %in2, ptr addrspace(3) noalias %out) #0 { +; GCN-LABEL: test_iglp_opt_demo: +; GCN: ; %bb.0: ; %entry +; GCN-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; GCN-NEXT: v_lshlrev_b32_e32 v4, 6, v0 +; GCN-NEXT: s_load_dword s0, s[0:1], 0x2c +; GCN-NEXT: ; iglp_opt mask(0x00000001) +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v0, s3 +; GCN-NEXT: ds_read2_b64 v[0:3], v0 offset1:1 +; GCN-NEXT: v_add_u32_e32 v5, s2, v4 +; GCN-NEXT: ds_read_b128 a[16:19], v5 +; GCN-NEXT: ds_read_b128 a[20:23], v5 offset:16 +; GCN-NEXT: ds_read_b128 a[24:27], v5 offset:32 +; GCN-NEXT: ds_read_b128 a[28:31], v5 offset:48 +; GCN-NEXT: v_add_u32_e32 v4, s0, v4 +; GCN-NEXT: ds_read_b128 a[12:15], v5 offset:4144 +; GCN-NEXT: ds_read_b128 a[8:11], v5 offset:4128 +; GCN-NEXT: s_waitcnt lgkmcnt(2) +; GCN-NEXT: v_mfma_f32_32x32x8f16 a[16:31], v[0:1], v[2:3], a[16:31] cbsz:1 abid:2 blgp:3 +; GCN-NEXT: ds_read_b128 a[4:7], v5 offset:4112 +; GCN-NEXT: ds_read_b128 a[0:3], v5 offset:4096 +; GCN-NEXT: ds_read_b128 a[40:43], v5 offset:12336 +; GCN-NEXT: ds_read_b128 a[36:39], v5 offset:12320 +; GCN-NEXT: ds_read_b128 a[32:35], v5 offset:12304 +; GCN-NEXT: v_mov_b32_e32 v6, s0 +; GCN-NEXT: s_nop 7 +; GCN-NEXT: s_nop 4 +; GCN-NEXT: ds_write_b128 v4, a[28:31] offset:48 +; GCN-NEXT: ds_read_b128 a[28:31], v5 offset:12288 +; GCN-NEXT: s_waitcnt lgkmcnt(5) +; GCN-NEXT: v_mfma_f32_32x32x8f16 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GCN-NEXT: ds_write_b128 v4, a[24:27] offset:32 +; GCN-NEXT: ds_write_b128 v4, a[20:23] offset:16 +; GCN-NEXT: ds_write_b128 v4, a[16:19] +; GCN-NEXT: s_waitcnt lgkmcnt(3) +; GCN-NEXT: v_mfma_f32_32x32x8f16 a[28:43], v[0:1], v[2:3], a[28:43] cbsz:1 abid:2 blgp:3 +; GCN-NEXT: s_nop 7 +; GCN-NEXT: s_nop 5 +; GCN-NEXT: ds_write_b128 v6, a[8:11] offset:4128 +; GCN-NEXT: ds_write_b128 v6, a[12:15] offset:4144 +; GCN-NEXT: ds_write_b128 v6, a[0:3] offset:4096 +; GCN-NEXT: ds_write_b128 v6, a[4:7] offset:4112 +; GCN-NEXT: s_nop 0 +; GCN-NEXT: ds_write_b128 v6, a[36:39] offset:8224 +; GCN-NEXT: ds_write_b128 v6, a[40:43] offset:8240 +; GCN-NEXT: ds_write_b128 v6, a[28:31] offset:8192 +; GCN-NEXT: ds_write_b128 v6, a[32:35] offset:8208 +; GCN-NEXT: s_endpgm +entry: + call void @llvm.amdgcn.iglp.opt(i32 1) + %idx = call i32 @llvm.amdgcn.workitem.id.x() + %load.0.addr = getelementptr <16 x float>, ptr addrspace(3) %in, i32 %idx + %load.0 = load <16 x float>, ptr addrspace(3) %load.0.addr + %load.1.addr = getelementptr <16 x float>, ptr addrspace(3) %load.0.addr, i32 64 + %load.1 = load <16 x float>, ptr addrspace(3) %load.1.addr + %load.2.addr = getelementptr <16 x float>, ptr addrspace(3) %load.1.addr, i32 128 + %load.2 = load <16 x float>, ptr addrspace(3) %load.2.addr + %load.3.addr = getelementptr <16 x float>, ptr addrspace(3) %load.1.addr, i32 128 + %load.3 = load <16 x float>, ptr addrspace(3) %load.2.addr + %load.4.addr = getelementptr <16 x float>, ptr addrspace(3) %load.1.addr, i32 128 + %load.4 = load <16 x float>, ptr addrspace(3) %load.2.addr + %load.5.addr = getelementptr <16 x float>, ptr addrspace(3) %load.1.addr, i32 128 + %load.5 = load <16 x float>, ptr addrspace(3) %load.2.addr + %in2.0 = load <4 x half>, ptr addrspace(3) %in2 + %inp.1 = getelementptr <4 x half>, ptr addrspace(3) %in2, i64 1 + %in2.1 = load <4 x half>, ptr addrspace(3) %inp.1 + %mai.0 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x8f16(<4 x half> %in2.0, <4 x half> %in2.1, <16 x float> %load.0, i32 1, i32 2, i32 3) + %mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x8f16(<4 x half> %in2.0, <4 x half> %in2.1, <16 x float> %load.1, i32 1, i32 2, i32 3) + %mai.2 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x8f16(<4 x half> %in2.0, <4 x half> %in2.1, <16 x float> %load.2, i32 1, i32 2, i32 3) + %mai.3 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x8f16(<4 x half> %in2.0, <4 x half> %in2.1, <16 x float> %load.3, i32 1, i32 2, i32 3) + %mai.4 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x8f16(<4 x half> %in2.0, <4 x half> %in2.1, <16 x float> %load.4, i32 1, i32 2, i32 3) + %mai.5 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x8f16(<4 x half> %in2.0, <4 x half> %in2.1, <16 x float> %load.5, i32 1, i32 2, i32 3) + %store.0.addr = getelementptr <16 x float>, ptr addrspace(3) %out, i32 %idx + store <16 x float> %mai.0, ptr addrspace(3) %store.0.addr + %store.1.addr = getelementptr <16 x float>, ptr addrspace(3) %out, i32 64 + store <16 x float> %mai.1, ptr addrspace(3) %store.1.addr + %store.2.addr = getelementptr <16 x float>, ptr addrspace(3) %out, i32 128 + store <16 x float> %mai.2, ptr addrspace(3) %store.2.addr + %store.3.addr = getelementptr <16 x float>, ptr addrspace(3) %out, i32 128 + store <16 x float> %mai.2, ptr addrspace(3) %store.3.addr + %store.4.addr = getelementptr <16 x float>, ptr addrspace(3) %out, i32 128 + store <16 x float> %mai.2, ptr addrspace(3) %store.4.addr + %store.5.addr = getelementptr <16 x float>, ptr addrspace(3) %out, i32 128 + store <16 x float> %mai.2, ptr addrspace(3) %store.5.addr + ret void +} + declare void @llvm.amdgcn.iglp.opt(i32) #1 declare i32 @llvm.amdgcn.workitem.id.x() #1 declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float, float, <32 x float>, i32, i32, i32) #1 +declare <16 x float> @llvm.amdgcn.mfma.f32.32x32x8f16(<4 x half>, <4 x half>, <16 x float>, i32, i32, i32) #1 attributes #0 = { nounwind "amdgpu-flat-work-group-size"="1,256" } attributes #1 = { convergent nounwind } diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.s b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.s new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.s @@ -0,0 +1,297 @@ + .text + .section .AMDGPU.config + .long 47176 + .long 11468800 + .long 47180 + .long 128 + .long 47200 + .long 0 + .long 4 + .long 0 + .long 8 + .long 0 + .text + .globl test_iglp_opt ; -- Begin function test_iglp_opt + .p2align 8 + .type test_iglp_opt,@function +test_iglp_opt: ; @test_iglp_opt +; %bb.0: ; %entry + ; iglp_opt mask(0x00000000) + s_endpgm +.Lfunc_end0: + .size test_iglp_opt, .Lfunc_end0-test_iglp_opt + ; -- End function + .section .AMDGPU.csdata +; Kernel info: +; codeLenInByte = 4 +; NumSgprs: 4 +; NumVgprs: 0 +; NumAgprs: 0 +; TotalNumVgprs: 0 +; ScratchSize: 0 +; MemoryBound: 0 +; FloatMode: 240 +; IeeeMode: 1 +; LDSByteSize: 0 bytes/workgroup (compile time only) +; SGPRBlocks: 0 +; VGPRBlocks: 0 +; NumSGPRsForWavesPerEU: 4 +; NumVGPRsForWavesPerEU: 1 +; AccumOffset: 4 +; Occupancy: 8 +; WaveLimiterHint : 0 +; COMPUTE_PGM_RSRC2:SCRATCH_EN: 0 +; COMPUTE_PGM_RSRC2:USER_SGPR: 0 +; COMPUTE_PGM_RSRC2:TRAP_HANDLER: 0 +; COMPUTE_PGM_RSRC2:TGID_X_EN: 1 +; COMPUTE_PGM_RSRC2:TGID_Y_EN: 0 +; COMPUTE_PGM_RSRC2:TGID_Z_EN: 0 +; COMPUTE_PGM_RSRC2:TIDIG_COMP_CNT: 0 +; COMPUTE_PGM_RSRC3_GFX90A:ACCUM_OFFSET: 0 +; COMPUTE_PGM_RSRC3_GFX90A:TG_SPLIT: 0 + .section .AMDGPU.config + .long 47176 + .long 11468820 + .long 47180 + .long 132 + .long 47200 + .long 0 + .long 4 + .long 0 + .long 8 + .long 0 + .text + .globl test_iglp_opt_mfma_gemm ; -- Begin function test_iglp_opt_mfma_gemm + .p2align 8 + .type test_iglp_opt_mfma_gemm,@function +test_iglp_opt_mfma_gemm: ; @test_iglp_opt_mfma_gemm +; %bb.0: ; %entry + s_load_dwordx2 s[0:1], s[0:1], 0x24 + v_lshlrev_b32_e32 v0, 7, v0 + v_mov_b32_e32 v3, 2.0 + ; iglp_opt mask(0x00000000) + s_waitcnt lgkmcnt(0) + v_add_u32_e32 v1, s0, v0 + v_add_u32_e32 v2, 0x6000, v1 + ds_read_b128 a[28:31], v2 offset:57456 + ds_read_b128 a[24:27], v2 offset:57440 + ds_read_b128 a[20:23], v2 offset:57424 + ds_read_b128 a[16:19], v2 offset:57408 + ds_read_b128 a[0:3], v2 offset:57344 + ds_read_b128 a[4:7], v2 offset:57360 + ds_read_b128 a[8:11], v2 offset:57376 + ds_read_b128 a[12:15], v2 offset:57392 + v_mov_b32_e32 v2, 1.0 + ds_read_b128 a[60:63], v1 offset:49264 + ds_read_b128 a[56:59], v1 offset:49248 + ds_read_b128 a[52:55], v1 offset:49232 + ds_read_b128 a[48:51], v1 offset:49216 + ds_read_b128 a[44:47], v1 offset:49200 + ds_read_b128 a[40:43], v1 offset:49184 + ds_read_b128 a[36:39], v1 offset:49168 + ds_read_b128 a[32:35], v1 offset:49152 + s_waitcnt lgkmcnt(8) + v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31] + ds_read_b128 a[156:159], v1 offset:112 + ds_read_b128 a[152:155], v1 offset:96 + ds_read_b128 a[68:71], v1 offset:24592 + ds_read_b128 a[64:67], v1 offset:24576 + v_add_u32_e32 v0, s1, v0 + s_waitcnt lgkmcnt(4) + v_mfma_f32_32x32x1f32 a[32:63], v2, v3, a[32:63] + ds_read_b128 a[148:151], v1 offset:80 + ds_read_b128 a[144:147], v1 offset:64 + ds_read_b128 a[128:131], v1 + ds_read_b128 a[132:135], v1 offset:16 + ds_read_b128 a[136:139], v1 offset:32 + ds_read_b128 a[140:143], v1 offset:48 + s_waitcnt lgkmcnt(0) + v_mfma_f32_32x32x1f32 a[128:159], v2, v3, a[128:159] + ds_read_b128 a[124:127], v1 offset:8304 + ds_read_b128 a[120:123], v1 offset:8288 + ds_read_b128 a[116:119], v1 offset:8272 + ds_read_b128 a[112:115], v1 offset:8256 + ds_read_b128 a[108:111], v1 offset:8240 + ds_read_b128 a[104:107], v1 offset:8224 + ds_read_b128 a[100:103], v1 offset:8208 + ds_read_b128 a[96:99], v1 offset:8192 + s_waitcnt lgkmcnt(0) + v_mfma_f32_32x32x1f32 a[96:127], v2, v3, a[96:127] + ds_read_b128 a[92:95], v1 offset:24688 + ds_read_b128 a[88:91], v1 offset:24672 + ds_read_b128 a[84:87], v1 offset:24656 + ds_read_b128 a[80:83], v1 offset:24640 + ds_read_b128 a[76:79], v1 offset:24624 + ds_read_b128 a[72:75], v1 offset:24608 + s_nop 2 + ds_write_b128 v0, a[156:159] offset:112 + ds_write_b128 v0, a[152:155] offset:96 + ds_write_b128 v0, a[148:151] offset:80 + ds_write_b128 v0, a[144:147] offset:64 + ds_write_b128 v0, a[140:143] offset:48 + ds_write_b128 v0, a[136:139] offset:32 + ds_write_b128 v0, a[132:135] offset:16 + ds_write_b128 v0, a[128:131] + v_mov_b32_e32 v0, s1 + s_waitcnt lgkmcnt(8) + v_mfma_f32_32x32x1f32 a[64:95], v2, v3, a[64:95] + ds_write_b128 v0, a[56:59] offset:24672 + ds_write_b128 v0, a[60:63] offset:24688 + ds_write_b128 v0, a[48:51] offset:24640 + ds_write_b128 v0, a[120:123] offset:8288 + ds_write_b128 v0, a[124:127] offset:8304 + ds_write_b128 v0, a[112:115] offset:8256 + ds_write_b128 v0, a[116:119] offset:8272 + ds_write_b128 v0, a[104:107] offset:8224 + ds_write_b128 v0, a[108:111] offset:8240 + ds_write_b128 v0, a[96:99] offset:8192 + ds_write_b128 v0, a[100:103] offset:8208 + ds_write_b128 v0, a[52:55] offset:24656 + ds_write_b128 v0, a[40:43] offset:24608 + ds_write_b128 v0, a[44:47] offset:24624 + ds_write_b128 v0, a[32:35] offset:24576 + ds_write_b128 v0, a[36:39] offset:24592 + ds_write_b128 v0, a[24:27] offset:32864 + ds_write_b128 v0, a[28:31] offset:32880 + ds_write_b128 v0, a[16:19] offset:32832 + ds_write_b128 v0, a[88:91] offset:16480 + ds_write_b128 v0, a[92:95] offset:16496 + ds_write_b128 v0, a[80:83] offset:16448 + ds_write_b128 v0, a[84:87] offset:16464 + ds_write_b128 v0, a[72:75] offset:16416 + ds_write_b128 v0, a[76:79] offset:16432 + ds_write_b128 v0, a[64:67] offset:16384 + ds_write_b128 v0, a[68:71] offset:16400 + ds_write_b128 v0, a[20:23] offset:32848 + ds_write_b128 v0, a[8:11] offset:32800 + ds_write_b128 v0, a[12:15] offset:32816 + ds_write_b128 v0, a[0:3] offset:32768 + ds_write_b128 v0, a[4:7] offset:32784 + s_endpgm +.Lfunc_end1: + .size test_iglp_opt_mfma_gemm, .Lfunc_end1-test_iglp_opt_mfma_gemm + ; -- End function + .section .AMDGPU.csdata +; Kernel info: +; codeLenInByte = 772 +; NumSgprs: 6 +; NumVgprs: 4 +; NumAgprs: 160 +; TotalNumVgprs: 164 +; ScratchSize: 0 +; MemoryBound: 1 +; FloatMode: 240 +; IeeeMode: 1 +; LDSByteSize: 0 bytes/workgroup (compile time only) +; SGPRBlocks: 0 +; VGPRBlocks: 20 +; NumSGPRsForWavesPerEU: 6 +; NumVGPRsForWavesPerEU: 164 +; AccumOffset: 4 +; Occupancy: 3 +; WaveLimiterHint : 1 +; COMPUTE_PGM_RSRC2:SCRATCH_EN: 0 +; COMPUTE_PGM_RSRC2:USER_SGPR: 2 +; COMPUTE_PGM_RSRC2:TRAP_HANDLER: 0 +; COMPUTE_PGM_RSRC2:TGID_X_EN: 1 +; COMPUTE_PGM_RSRC2:TGID_Y_EN: 0 +; COMPUTE_PGM_RSRC2:TGID_Z_EN: 0 +; COMPUTE_PGM_RSRC2:TIDIG_COMP_CNT: 0 +; COMPUTE_PGM_RSRC3_GFX90A:ACCUM_OFFSET: 0 +; COMPUTE_PGM_RSRC3_GFX90A:TG_SPLIT: 0 + .section .AMDGPU.config + .long 47176 + .long 11468806 + .long 47180 + .long 132 + .long 47200 + .long 0 + .long 4 + .long 0 + .long 8 + .long 0 + .text + .globl test_iglp_opt_demo ; -- Begin function test_iglp_opt_demo + .p2align 8 + .type test_iglp_opt_demo,@function +test_iglp_opt_demo: ; @test_iglp_opt_demo +; %bb.0: ; %entry + s_load_dwordx2 s[2:3], s[0:1], 0x24 + v_lshlrev_b32_e32 v4, 6, v0 + s_load_dword s0, s[0:1], 0x2c + ; iglp_opt mask(0x00000001) + s_waitcnt lgkmcnt(0) + v_mov_b32_e32 v0, s3 + ds_read2_b64 v[0:3], v0 offset1:1 + v_add_u32_e32 v5, s2, v4 + ds_read_b128 a[16:19], v5 + ds_read_b128 a[20:23], v5 offset:16 + ds_read_b128 a[24:27], v5 offset:32 + ds_read_b128 a[28:31], v5 offset:48 + v_add_u32_e32 v4, s0, v4 + ds_read_b128 a[12:15], v5 offset:4144 + ds_read_b128 a[8:11], v5 offset:4128 + s_waitcnt lgkmcnt(2) + v_mfma_f32_32x32x8f16 a[16:31], v[0:1], v[2:3], a[16:31] cbsz:1 abid:2 blgp:3 + ds_read_b128 a[4:7], v5 offset:4112 + ds_read_b128 a[0:3], v5 offset:4096 + ds_read_b128 a[40:43], v5 offset:12336 + ds_read_b128 a[36:39], v5 offset:12320 + ds_read_b128 a[32:35], v5 offset:12304 + v_mov_b32_e32 v6, s0 + s_nop 7 + s_nop 4 + ds_write_b128 v4, a[28:31] offset:48 + ds_read_b128 a[28:31], v5 offset:12288 + s_waitcnt lgkmcnt(5) + v_mfma_f32_32x32x8f16 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 + ds_write_b128 v4, a[24:27] offset:32 + ds_write_b128 v4, a[20:23] offset:16 + ds_write_b128 v4, a[16:19] + s_waitcnt lgkmcnt(3) + v_mfma_f32_32x32x8f16 a[28:43], v[0:1], v[2:3], a[28:43] cbsz:1 abid:2 blgp:3 + s_nop 7 + s_nop 5 + ds_write_b128 v6, a[8:11] offset:4128 + ds_write_b128 v6, a[12:15] offset:4144 + ds_write_b128 v6, a[0:3] offset:4096 + ds_write_b128 v6, a[4:7] offset:4112 + s_nop 0 + ds_write_b128 v6, a[36:39] offset:8224 + ds_write_b128 v6, a[40:43] offset:8240 + ds_write_b128 v6, a[28:31] offset:8192 + ds_write_b128 v6, a[32:35] offset:8208 + s_endpgm +.Lfunc_end2: + .size test_iglp_opt_demo, .Lfunc_end2-test_iglp_opt_demo + ; -- End function + .section .AMDGPU.csdata +; Kernel info: +; codeLenInByte = 312 +; NumSgprs: 8 +; NumVgprs: 7 +; NumAgprs: 44 +; TotalNumVgprs: 52 +; ScratchSize: 0 +; MemoryBound: 1 +; FloatMode: 240 +; IeeeMode: 1 +; LDSByteSize: 0 bytes/workgroup (compile time only) +; SGPRBlocks: 0 +; VGPRBlocks: 6 +; NumSGPRsForWavesPerEU: 8 +; NumVGPRsForWavesPerEU: 52 +; AccumOffset: 8 +; Occupancy: 8 +; WaveLimiterHint : 1 +; COMPUTE_PGM_RSRC2:SCRATCH_EN: 0 +; COMPUTE_PGM_RSRC2:USER_SGPR: 2 +; COMPUTE_PGM_RSRC2:TRAP_HANDLER: 0 +; COMPUTE_PGM_RSRC2:TGID_X_EN: 1 +; COMPUTE_PGM_RSRC2:TGID_Y_EN: 0 +; COMPUTE_PGM_RSRC2:TGID_Z_EN: 0 +; COMPUTE_PGM_RSRC2:TIDIG_COMP_CNT: 0 +; COMPUTE_PGM_RSRC3_GFX90A:ACCUM_OFFSET: 1 +; COMPUTE_PGM_RSRC3_GFX90A:TG_SPLIT: 0 + .section ".note.GNU-stack" + .amd_amdgpu_isa "amdgcn-unknown-linux-gnu-gfx90a" diff --git a/llvm/test/CodeGen/AMDGPU/myOuty b/llvm/test/CodeGen/AMDGPU/myOuty new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/myOuty @@ -0,0 +1,20 @@ +Found match: DS_WRITE_B128_gfx9 %93:vgpr_32, %286.sub12_sub13_sub14_sub15:areg_512_align2, 48, 0, implicit $exec :: (store (s128) into %ir.store.0.addr + 48, addrspace 3) +nd DS_WRITE_B128_gfx9 %93:vgpr_32, %286.sub12_sub13_sub14_sub15:areg_512_align2, 48, 0, implicit $exec :: (store (s128) into %ir.store.0.addr + 48, addrspace 3) + +Hit +Found match: DS_WRITE_B128_gfx9 %123:vgpr_32, %270.sub8_sub9_sub10_sub11:areg_512_align2, 4128, 0, implicit $exec :: (store (s128) into %ir.store.1.addr + 32, align 32, addrspace 3) +nd DS_WRITE_B128_gfx9 %123:vgpr_32, %270.sub8_sub9_sub10_sub11:areg_512_align2, 4128, 0, implicit $exec :: (store (s128) into %ir.store.1.addr + 32, align 32, addrspace 3) + +Hit +Found match: DS_WRITE_B128_gfx9 %123:vgpr_32, %330.sub8_sub9_sub10_sub11:areg_512_align2, 8224, 0, implicit $exec :: (store (s128) into %ir.store.2.addr + 32, align 32, addrspace 3) +nd DS_WRITE_B128_gfx9 %123:vgpr_32, %330.sub8_sub9_sub10_sub11:areg_512_align2, 8224, 0, implicit $exec :: (store (s128) into %ir.store.2.addr + 32, align 32, addrspace 3) + +Hit +Found match: DS_WRITE_B128_gfx9 renamable $vgpr4, renamable $agpr28_agpr29_agpr30_agpr31, 48, 0, implicit $exec :: (store (s128) into %ir.store.0.addr + 48, addrspace 3) +nd DS_WRITE_B128_gfx9 renamable $vgpr4, renamable $agpr28_agpr29_agpr30_agpr31, 48, 0, implicit $exec :: (store (s128) into %ir.store.0.addr + 48, addrspace 3) + +Hit +Found match: DS_WRITE_B128_gfx9 renamable $vgpr6, renamable $agpr8_agpr9_agpr10_agpr11, 4128, 0, implicit $exec :: (store (s128) into %ir.store.1.addr + 32, align 32, addrspace 3) +nd DS_WRITE_B128_gfx9 renamable $vgpr6, renamable $agpr8_agpr9_agpr10_agpr11, 4128, 0, implicit $exec :: (store (s128) into %ir.store.1.addr + 32, align 32, addrspace 3) + +Hit