diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h @@ -554,6 +554,14 @@ return MI.getDesc().TSFlags & SIInstrFlags::EXP; } + static bool isDualSourceBlendEXP(const MachineInstr &MI) { + if (!isEXP(MI)) + return false; + unsigned Target = MI.getOperand(0).getImm(); + return Target == AMDGPU::Exp::ET_DUAL_SRC_BLEND0 || + Target == AMDGPU::Exp::ET_DUAL_SRC_BLEND1; + } + bool isEXP(uint16_t Opcode) const { return get(Opcode).TSFlags & SIInstrFlags::EXP; } diff --git a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp --- a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp +++ b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp @@ -534,13 +534,28 @@ GlobalFlags |= StateStrictWWM; LowerToMovInstrs.push_back(&MI); continue; - } else if (Opcode == AMDGPU::STRICT_WQM) { + } else if (Opcode == AMDGPU::STRICT_WQM || + TII->isDualSourceBlendEXP(MI)) { // STRICT_WQM is similar to STRICTWWM, but instead of enabling all // threads of the wave like STRICTWWM, STRICT_WQM enables all threads in // quads that have at least one active thread. markInstructionUses(MI, StateStrictWQM, Worklist); GlobalFlags |= StateStrictWQM; - LowerToMovInstrs.push_back(&MI); + + if (Opcode == AMDGPU::STRICT_WQM) { + LowerToMovInstrs.push_back(&MI); + } else { + // Dual source blend export acts as implicit strict-wqm, its sources + // need to be shuffled in strict wqm, but the export itself needs to + // run in exact mode. + BBI.Needs |= StateExact; + if (!(BBI.InNeeds & StateExact)) { + BBI.InNeeds |= StateExact; + Worklist.push_back(MBB); + } + GlobalFlags |= StateExact; + III.Disabled = StateWQM | StateStrict; + } continue; } else if (Opcode == AMDGPU::LDS_PARAM_LOAD || Opcode == AMDGPU::LDS_DIRECT_LOAD) { diff --git a/llvm/test/CodeGen/AMDGPU/dual-source-blend-export.ll b/llvm/test/CodeGen/AMDGPU/dual-source-blend-export.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/dual-source-blend-export.ll @@ -0,0 +1,99 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -march=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck %s -check-prefix=GCN + +; This is a slightly modified IR from real case to make it concise. +define amdgpu_ps void @_amdgpu_ps_main(i32 inreg %PrimMask, <2 x float> %InterpCenter) #0 { +; GCN-LABEL: _amdgpu_ps_main: +; GCN: ; %bb.0: ; %.entry +; GCN-NEXT: s_mov_b32 s1, exec_lo +; GCN-NEXT: s_wqm_b32 exec_lo, exec_lo +; GCN-NEXT: s_mov_b32 m0, s0 +; GCN-NEXT: v_mov_b32_e32 v2, v0 +; GCN-NEXT: lds_param_load v3, attr1.x wait_vdst:15 +; GCN-NEXT: lds_param_load v4, attr1.y wait_vdst:15 +; GCN-NEXT: lds_param_load v5, attr1.z wait_vdst:15 +; GCN-NEXT: lds_param_load v6, attr1.w wait_vdst:15 +; GCN-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0 +; GCN-NEXT: v_mbcnt_hi_u32_b32 v7, -1, v7 +; GCN-NEXT: v_and_b32_e32 v7, 1, v7 +; GCN-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7 +; GCN-NEXT: v_interp_p10_f32 v8, v4, v2, v4 wait_exp:2 +; GCN-NEXT: v_interp_p10_f32 v10, v5, v2, v5 wait_exp:1 +; GCN-NEXT: v_interp_p10_f32 v9, v6, v2, v6 +; GCN-NEXT: v_interp_p10_f32 v2, v3, v2, v3 wait_exp:7 +; GCN-NEXT: v_interp_p2_f32 v4, v4, v1, v8 wait_exp:7 +; GCN-NEXT: v_interp_p2_f32 v5, v5, v1, v10 wait_exp:7 +; GCN-NEXT: v_interp_p2_f32 v6, v6, v1, v9 wait_exp:7 +; GCN-NEXT: v_interp_p2_f32 v2, v3, v1, v2 wait_exp:7 +; GCN-NEXT: v_mov_b32_dpp v4, v4 dpp8:[1,0,3,2,5,4,7,6] +; GCN-NEXT: v_mov_b32_dpp v6, v6 dpp8:[1,0,3,2,5,4,7,6] +; GCN-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc_lo +; GCN-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc_lo +; GCN-NEXT: v_cndmask_b32_e32 v5, v2, v6, vcc_lo +; GCN-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc_lo +; GCN-NEXT: v_mov_b32_dpp v4, v4 dpp8:[1,0,3,2,5,4,7,6] +; GCN-NEXT: v_mov_b32_dpp v5, v5 dpp8:[1,0,3,2,5,4,7,6] +; GCN-NEXT: s_mov_b32 exec_lo, s1 +; GCN-NEXT: exp dual_src_blend0 v3, v2, off, off +; GCN-NEXT: exp dual_src_blend1 v4, v5, off, off done +; GCN-NEXT: s_endpgm +.entry: + %InterpCenter.i0 = extractelement <2 x float> %InterpCenter, i64 0 + %InterpCenter.i1 = extractelement <2 x float> %InterpCenter, i64 1 + %i6 = call float @llvm.amdgcn.lds.param.load(i32 immarg 0, i32 immarg 1, i32 %PrimMask) + %i7 = call float @llvm.amdgcn.lds.param.load(i32 immarg 1, i32 immarg 1, i32 %PrimMask) + %i8 = call float @llvm.amdgcn.lds.param.load(i32 immarg 2, i32 immarg 1, i32 %PrimMask) + %i9 = call float @llvm.amdgcn.lds.param.load(i32 immarg 3, i32 immarg 1, i32 %PrimMask) + + %i14 = call float @llvm.amdgcn.interp.inreg.p10(float %i8, float %InterpCenter.i0, float %i8) + %i15 = call float @llvm.amdgcn.interp.inreg.p2(float %i8, float %InterpCenter.i1, float %i14) + + %i16 = call float @llvm.amdgcn.interp.inreg.p10(float %i7, float %InterpCenter.i0, float %i7) + %i17 = call float @llvm.amdgcn.interp.inreg.p2(float %i7, float %InterpCenter.i1, float %i16) + + %i18 = call float @llvm.amdgcn.interp.inreg.p10(float %i6, float %InterpCenter.i0, float %i6) + %i19 = call float @llvm.amdgcn.interp.inreg.p2(float %i6, float %InterpCenter.i1, float %i18) + + %i20 = call float @llvm.amdgcn.interp.inreg.p10(float %i9, float %InterpCenter.i0, float %i9) + %i21 = call float @llvm.amdgcn.interp.inreg.p2(float %i9, float %InterpCenter.i1, float %i20) + + %i34 = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) + %i35 = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %i34) + %i36 = and i32 %i35, 1 + %.not = icmp eq i32 %i36, 0 + + %i37 = bitcast float %i15 to i32 + %i38 = bitcast float %i17 to i32 + %i39 = call i32 @llvm.amdgcn.mov.dpp8.i32(i32 %i38, i32 14570689) + %i40 = select i1 %.not, i32 %i37, i32 %i39 + %i41 = bitcast i32 %i40 to float + %i42 = select i1 %.not, i32 %i39, i32 %i37 + %i43 = call i32 @llvm.amdgcn.mov.dpp8.i32(i32 %i42, i32 14570689) + %i44 = bitcast i32 %i43 to float + + %i45 = bitcast float %i19 to i32 + %i46 = bitcast float %i21 to i32 + %i47 = call i32 @llvm.amdgcn.mov.dpp8.i32(i32 %i46, i32 14570689) + %i48 = select i1 %.not, i32 %i45, i32 %i47 + %i49 = bitcast i32 %i48 to float + %i50 = select i1 %.not, i32 %i47, i32 %i45 + %i51 = call i32 @llvm.amdgcn.mov.dpp8.i32(i32 %i50, i32 14570689) + %i52 = bitcast i32 %i51 to float + call void @llvm.amdgcn.exp.f32(i32 immarg 21, i32 immarg 3, float %i41, float %i49, float undef, float undef, i1 immarg false, i1 immarg true) + call void @llvm.amdgcn.exp.f32(i32 immarg 22, i32 immarg 3, float %i44, float %i52, float undef, float undef, i1 immarg true, i1 immarg true) + ret void +} + +declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32) #2 +declare i32 @llvm.amdgcn.mbcnt.hi(i32, i32) #2 +declare i32 @llvm.amdgcn.mov.dpp8.i32(i32, i32 immarg) #3 +declare void @llvm.amdgcn.exp.f32(i32 immarg, i32 immarg, float, float, float, float, i1 immarg, i1 immarg) #4 +declare float @llvm.amdgcn.interp.inreg.p10(float, float, float) #1 +declare float @llvm.amdgcn.interp.inreg.p2(float, float, float) #1 +declare float @llvm.amdgcn.lds.param.load(i32 immarg, i32 immarg, i32) #1 + +attributes #0 = { nounwind } +attributes #1 = { nounwind readnone speculatable willreturn } +attributes #2 = { nounwind readnone willreturn } +attributes #3 = { convergent nounwind readnone willreturn } +attributes #4 = { inaccessiblememonly nounwind willreturn writeonly }