diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp --- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp @@ -1005,7 +1005,12 @@ void AMDGPUDAGToDAGISel::SelectMAD_64_32(SDNode *N) { SDLoc SL(N); bool Signed = N->getOpcode() == AMDGPUISD::MAD_I64_I32; - unsigned Opc = Signed ? AMDGPU::V_MAD_I64_I32_e64 : AMDGPU::V_MAD_U64_U32_e64; + unsigned Opc; + if (Subtarget->getGeneration() == AMDGPUSubtarget::GFX11) + Opc = Signed ? AMDGPU::V_MAD_I64_I32_gfx11_e64 + : AMDGPU::V_MAD_U64_U32_gfx11_e64; + else + Opc = Signed ? AMDGPU::V_MAD_I64_I32_e64 : AMDGPU::V_MAD_U64_U32_e64; SDValue Clamp = CurDAG->getTargetConstant(0, SL, MVT::i1); SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), @@ -1018,7 +1023,12 @@ void AMDGPUDAGToDAGISel::SelectMUL_LOHI(SDNode *N) { SDLoc SL(N); bool Signed = N->getOpcode() == ISD::SMUL_LOHI; - unsigned Opc = Signed ? AMDGPU::V_MAD_I64_I32_e64 : AMDGPU::V_MAD_U64_U32_e64; + unsigned Opc; + if (Subtarget->getGeneration() == AMDGPUSubtarget::GFX11) + Opc = Signed ? AMDGPU::V_MAD_I64_I32_gfx11_e64 + : AMDGPU::V_MAD_U64_U32_gfx11_e64; + else + Opc = Signed ? AMDGPU::V_MAD_I64_I32_e64 : AMDGPU::V_MAD_U64_U32_e64; SDValue Zero = CurDAG->getTargetConstant(0, SL, MVT::i64); SDValue Clamp = CurDAG->getTargetConstant(0, SL, MVT::i1); diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp --- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp @@ -464,8 +464,13 @@ MachineFunction *MF = BB->getParent(); const bool IsUnsigned = I.getOpcode() == AMDGPU::G_AMDGPU_MAD_U64_U32; - I.setDesc(TII.get(IsUnsigned ? AMDGPU::V_MAD_U64_U32_e64 - : AMDGPU::V_MAD_I64_I32_e64)); + unsigned Opc; + if (Subtarget->getGeneration() == AMDGPUSubtarget::GFX11) + Opc = IsUnsigned ? AMDGPU::V_MAD_U64_U32_gfx11_e64 + : AMDGPU::V_MAD_I64_I32_gfx11_e64; + else + Opc = IsUnsigned ? AMDGPU::V_MAD_U64_U32_e64 : AMDGPU::V_MAD_I64_I32_e64; + I.setDesc(TII.get(Opc)); I.addOperand(*MF, MachineOperand::CreateImm(0)); I.addImplicitDefUseOperands(*MF); return constrainSelectedInstRegOperands(I, TII, TRI, RBI); diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-mad_64_32.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-mad_64_32.mir --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-mad_64_32.mir +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-mad_64_32.mir @@ -1,5 +1,6 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -march=amdgcn -mcpu=gfx1030 -run-pass=instruction-select -global-isel-abort=2 -pass-remarks-missed='gisel*' -verify-machineinstrs %s -o - 2>%t | FileCheck -check-prefix=GCN %s +# RUN: llc -march=amdgcn -mcpu=gfx1030 -run-pass=instruction-select -global-isel-abort=2 -pass-remarks-missed='gisel*' -verify-machineinstrs %s -o - 2>%t | FileCheck -check-prefix=GFX10 %s +# RUN: llc -march=amdgcn -mcpu=gfx1100 -run-pass=instruction-select -global-isel-abort=2 -pass-remarks-missed='gisel*' -verify-machineinstrs %s -o - 2>%t | FileCheck -check-prefix=GFX11 %s --- name: mad_u64_u32_vvv @@ -9,12 +10,22 @@ body: | bb.0: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3 - ; GCN-LABEL: name: mad_u64_u32_vvv - ; GCN: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 - ; GCN-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY $vgpr3 - ; GCN-NEXT: [[V_MAD_U64_U32_e64_:%[0-9]+]]:vreg_64, [[V_MAD_U64_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_MAD_U64_U32_e64 [[COPY]], [[COPY1]], [[COPY2]], 0, implicit $exec - ; GCN-NEXT: S_ENDPGM 0, implicit [[V_MAD_U64_U32_e64_]], implicit [[V_MAD_U64_U32_e64_1]] + ; GFX10-LABEL: name: mad_u64_u32_vvv + ; GFX10: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3 + ; GFX10-NEXT: {{ $}} + ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 + ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY $vgpr3 + ; GFX10-NEXT: [[V_MAD_U64_U32_e64_:%[0-9]+]]:vreg_64, [[V_MAD_U64_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_MAD_U64_U32_e64 [[COPY]], [[COPY1]], [[COPY2]], 0, implicit $exec + ; GFX10-NEXT: S_ENDPGM 0, implicit [[V_MAD_U64_U32_e64_]], implicit [[V_MAD_U64_U32_e64_1]] + ; GFX11-LABEL: name: mad_u64_u32_vvv + ; GFX11: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3 + ; GFX11-NEXT: {{ $}} + ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 + ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY $vgpr3 + ; GFX11-NEXT: [[V_MAD_U64_U32_gfx11_e64_:%[0-9]+]]:vreg_64, [[V_MAD_U64_U32_gfx11_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_MAD_U64_U32_gfx11_e64 [[COPY]], [[COPY1]], [[COPY2]], 0, implicit $exec + ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_MAD_U64_U32_gfx11_e64_]], implicit [[V_MAD_U64_U32_gfx11_e64_1]] %0:vgpr(s32) = COPY $vgpr0 %1:vgpr(s32) = COPY $vgpr1 %2:vgpr(s32) = COPY $vgpr2 @@ -32,12 +43,22 @@ body: | bb.0: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3 - ; GCN-LABEL: name: mad_i64_i32_vvv - ; GCN: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 - ; GCN-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY $vgpr3 - ; GCN-NEXT: [[V_MAD_I64_I32_e64_:%[0-9]+]]:vreg_64, [[V_MAD_I64_I32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_MAD_I64_I32_e64 [[COPY]], [[COPY1]], [[COPY2]], 0, implicit $exec - ; GCN-NEXT: S_ENDPGM 0, implicit [[V_MAD_I64_I32_e64_]], implicit [[V_MAD_I64_I32_e64_1]] + ; GFX10-LABEL: name: mad_i64_i32_vvv + ; GFX10: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3 + ; GFX10-NEXT: {{ $}} + ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 + ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY $vgpr3 + ; GFX10-NEXT: [[V_MAD_I64_I32_e64_:%[0-9]+]]:vreg_64, [[V_MAD_I64_I32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_MAD_I64_I32_e64 [[COPY]], [[COPY1]], [[COPY2]], 0, implicit $exec + ; GFX10-NEXT: S_ENDPGM 0, implicit [[V_MAD_I64_I32_e64_]], implicit [[V_MAD_I64_I32_e64_1]] + ; GFX11-LABEL: name: mad_i64_i32_vvv + ; GFX11: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3 + ; GFX11-NEXT: {{ $}} + ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 + ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY $vgpr3 + ; GFX11-NEXT: [[V_MAD_I64_I32_gfx11_e64_:%[0-9]+]]:vreg_64, [[V_MAD_I64_I32_gfx11_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_MAD_I64_I32_gfx11_e64 [[COPY]], [[COPY1]], [[COPY2]], 0, implicit $exec + ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_MAD_I64_I32_gfx11_e64_]], implicit [[V_MAD_I64_I32_gfx11_e64_1]] %0:vgpr(s32) = COPY $vgpr0 %1:vgpr(s32) = COPY $vgpr1 %2:vgpr(s32) = COPY $vgpr2 diff --git a/llvm/test/CodeGen/AMDGPU/llvm.mulo.ll b/llvm/test/CodeGen/AMDGPU/llvm.mulo.ll --- a/llvm/test/CodeGen/AMDGPU/llvm.mulo.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.mulo.ll @@ -2,6 +2,7 @@ ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck --check-prefix=SI %s ; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck --check-prefix=GFX9 %s ; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefix=GFX10 %s +; RUN: llc -march=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck --check-prefix=GFX11 %s define { i64, i1 } @umulo_i64_v_v(i64 %x, i64 %y) { ; SI-LABEL: umulo_i64_v_v: @@ -75,6 +76,31 @@ ; GFX10-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3] ; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: umulo_i64_v_v: +; GFX11: ; %bb.0: ; %bb +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: v_mov_b32_e32 v4, v0 +; GFX11-NEXT: v_mov_b32_e32 v5, v1 +; GFX11-NEXT: v_mad_u64_u32 v[0:1], s0, v4, v2, 0 +; GFX11-NEXT: v_mad_u64_u32 v[6:7], s0, v4, v3, 0 +; GFX11-NEXT: v_mad_u64_u32 v[9:10], s0, v5, v2, 0 +; GFX11-NEXT: v_mad_u64_u32 v[11:12], s0, v5, v3, 0 +; GFX11-NEXT: v_mov_b32_e32 v8, v1 +; GFX11-NEXT: v_mul_lo_u32 v5, v5, v2 +; GFX11-NEXT: v_mul_lo_u32 v4, v4, v3 +; GFX11-NEXT: v_add_co_u32 v6, vcc_lo, v8, v6 +; GFX11-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, 0, v7, vcc_lo +; GFX11-NEXT: v_add3_u32 v1, v1, v4, v5 +; GFX11-NEXT: v_add_co_u32 v2, vcc_lo, v6, v9 +; GFX11-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v7, v10, vcc_lo +; GFX11-NEXT: v_add_co_ci_u32_e32 v6, vcc_lo, 0, v12, vcc_lo +; GFX11-NEXT: v_add_co_u32 v2, vcc_lo, v2, v11 +; GFX11-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v6, vcc_lo +; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3] +; GFX11-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo +; GFX11-NEXT: s_setpc_b64 s[30:31] bb: %umulo = tail call { i64, i1 } @llvm.umul.with.overflow.i64(i64 %x, i64 %y) ret { i64, i1 } %umulo @@ -190,6 +216,43 @@ ; GFX10-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[2:3] ; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: smulo_i64_v_v: +; GFX11: ; %bb.0: ; %bb +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: v_mov_b32_e32 v4, v0 +; GFX11-NEXT: v_mov_b32_e32 v5, v1 +; GFX11-NEXT: v_mad_u64_u32 v[0:1], s0, v4, v2, 0 +; GFX11-NEXT: v_mad_u64_u32 v[6:7], s0, v4, v3, 0 +; GFX11-NEXT: v_mad_u64_u32 v[9:10], s0, v5, v2, 0 +; GFX11-NEXT: v_mad_i64_i32 v[11:12], s0, v5, v3, 0 +; GFX11-NEXT: v_mov_b32_e32 v8, v1 +; GFX11-NEXT: v_add_co_u32 v6, vcc_lo, v8, v6 +; GFX11-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, 0, v7, vcc_lo +; GFX11-NEXT: v_mul_lo_u32 v8, v5, v2 +; GFX11-NEXT: v_add_co_u32 v6, vcc_lo, v6, v9 +; GFX11-NEXT: v_add_co_ci_u32_e32 v6, vcc_lo, v7, v10, vcc_lo +; GFX11-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, 0, v12, vcc_lo +; GFX11-NEXT: v_mul_lo_u32 v9, v4, v3 +; GFX11-NEXT: v_add_co_u32 v6, vcc_lo, v6, v11 +; GFX11-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, 0, v7, vcc_lo +; GFX11-NEXT: v_sub_co_u32 v2, vcc_lo, v6, v2 +; GFX11-NEXT: v_subrev_co_ci_u32_e32 v10, vcc_lo, 0, v7, vcc_lo +; GFX11-NEXT: v_cmp_gt_i32_e32 vcc_lo, 0, v5 +; GFX11-NEXT: v_add3_u32 v1, v1, v9, v8 +; GFX11-NEXT: v_cndmask_b32_e32 v6, v6, v2, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v10, vcc_lo +; GFX11-NEXT: v_ashrrev_i32_e32 v2, 31, v1 +; GFX11-NEXT: v_sub_co_u32 v4, vcc_lo, v6, v4 +; GFX11-NEXT: v_subrev_co_ci_u32_e32 v7, vcc_lo, 0, v5, vcc_lo +; GFX11-NEXT: v_cmp_gt_i32_e32 vcc_lo, 0, v3 +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc_lo +; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[2:3] +; GFX11-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo +; GFX11-NEXT: s_setpc_b64 s[30:31] bb: %smulo = tail call { i64, i1 } @llvm.smul.with.overflow.i64(i64 %x, i64 %y) ret { i64, i1 } %smulo @@ -285,6 +348,34 @@ ; GFX10-NEXT: v_cndmask_b32_e64 v0, s0, 0, s2 ; GFX10-NEXT: global_store_dwordx2 v[0:1], v[0:1], off ; GFX10-NEXT: s_endpgm +; +; GFX11-LABEL: umulo_i64_s: +; GFX11: ; %bb.0: ; %bb +; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x24 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: s_mul_i32 s7, s0, s3 +; GFX11-NEXT: s_mul_hi_u32 s8, s0, s2 +; GFX11-NEXT: s_mul_hi_u32 s5, s0, s3 +; GFX11-NEXT: s_mul_hi_u32 s4, s1, s2 +; GFX11-NEXT: s_mul_i32 s6, s1, s2 +; GFX11-NEXT: s_mul_hi_u32 s9, s1, s3 +; GFX11-NEXT: s_mul_i32 s1, s1, s3 +; GFX11-NEXT: s_add_u32 s3, s8, s7 +; GFX11-NEXT: s_addc_u32 s5, 0, s5 +; GFX11-NEXT: s_add_u32 s3, s3, s6 +; GFX11-NEXT: s_addc_u32 s3, s5, s4 +; GFX11-NEXT: s_addc_u32 s5, s9, 0 +; GFX11-NEXT: s_add_u32 s4, s3, s1 +; GFX11-NEXT: s_addc_u32 s5, 0, s5 +; GFX11-NEXT: s_add_i32 s1, s8, s7 +; GFX11-NEXT: s_mul_i32 s0, s0, s2 +; GFX11-NEXT: s_add_i32 s1, s1, s6 +; GFX11-NEXT: s_cmp_lg_u64 s[4:5], 0 +; GFX11-NEXT: s_cselect_b32 s2, -1, 0 +; GFX11-NEXT: v_cndmask_b32_e64 v1, s1, 0, s2 +; GFX11-NEXT: v_cndmask_b32_e64 v0, s0, 0, s2 +; GFX11-NEXT: global_store_b64 v[0:1], v[0:1], off +; GFX11-NEXT: s_endpgm bb: %umulo = tail call { i64, i1 } @llvm.umul.with.overflow.i64(i64 %x, i64 %y) %mul = extractvalue { i64, i1 } %umulo, 0 @@ -430,6 +521,49 @@ ; GFX10-NEXT: v_cndmask_b32_e64 v0, s0, 0, vcc_lo ; GFX10-NEXT: global_store_dwordx2 v[0:1], v[0:1], off ; GFX10-NEXT: s_endpgm +; +; GFX11-LABEL: smulo_i64_s: +; GFX11: ; %bb.0: ; %bb +; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x24 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: s_mul_i32 s7, s0, s3 +; GFX11-NEXT: s_mul_hi_u32 s8, s0, s2 +; GFX11-NEXT: s_mul_hi_u32 s6, s0, s3 +; GFX11-NEXT: s_mul_i32 s5, s1, s2 +; GFX11-NEXT: s_add_u32 s11, s8, s7 +; GFX11-NEXT: s_mul_hi_u32 s4, s1, s2 +; GFX11-NEXT: s_addc_u32 s6, 0, s6 +; GFX11-NEXT: s_mul_hi_i32 s9, s1, s3 +; GFX11-NEXT: s_add_u32 s11, s11, s5 +; GFX11-NEXT: s_mul_i32 s10, s1, s3 +; GFX11-NEXT: s_addc_u32 s4, s6, s4 +; GFX11-NEXT: s_addc_u32 s6, s9, 0 +; GFX11-NEXT: s_add_u32 s4, s4, s10 +; GFX11-NEXT: s_addc_u32 s6, 0, s6 +; GFX11-NEXT: s_sub_u32 s9, s4, s2 +; GFX11-NEXT: s_subb_u32 s10, s6, 0 +; GFX11-NEXT: v_mov_b32_e32 v1, s9 +; GFX11-NEXT: s_cmp_lt_i32 s1, 0 +; GFX11-NEXT: v_mov_b32_e32 v0, s10 +; GFX11-NEXT: s_cselect_b32 vcc_lo, -1, 0 +; GFX11-NEXT: s_cmp_lt_i32 s3, 0 +; GFX11-NEXT: v_cndmask_b32_e32 v2, s4, v1, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v0, s6, v0, vcc_lo +; GFX11-NEXT: v_sub_co_u32 v3, vcc_lo, v2, s0 +; GFX11-NEXT: v_subrev_co_ci_u32_e32 v1, vcc_lo, 0, v0, vcc_lo +; GFX11-NEXT: s_cselect_b32 vcc_lo, -1, 0 +; GFX11-NEXT: s_add_i32 s1, s8, s7 +; GFX11-NEXT: s_mul_i32 s0, s0, s2 +; GFX11-NEXT: s_add_i32 s1, s1, s5 +; GFX11-NEXT: v_cndmask_b32_e32 v1, v0, v1, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo +; GFX11-NEXT: s_ashr_i32 s4, s1, 31 +; GFX11-NEXT: s_mov_b32 s5, s4 +; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, s[4:5], v[0:1] +; GFX11-NEXT: v_cndmask_b32_e64 v1, s1, 0, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v0, s0, 0, vcc_lo +; GFX11-NEXT: global_store_b64 v[0:1], v[0:1], off +; GFX11-NEXT: s_endpgm bb: %umulo = tail call { i64, i1 } @llvm.smul.with.overflow.i64(i64 %x, i64 %y) %mul = extractvalue { i64, i1 } %umulo, 0 @@ -476,6 +610,19 @@ ; GFX10-NEXT: v_mov_b32_e32 v1, v3 ; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: smulo_i64_v_4: +; GFX11: ; %bb.0: ; %bb +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: v_lshlrev_b64 v[4:5], 2, v[0:1] +; GFX11-NEXT: v_alignbit_b32 v3, v1, v0, 30 +; GFX11-NEXT: v_ashrrev_i64 v[5:6], 2, v[4:5] +; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[5:6], v[0:1] +; GFX11-NEXT: v_mov_b32_e32 v0, v4 +; GFX11-NEXT: v_mov_b32_e32 v1, v3 +; GFX11-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo +; GFX11-NEXT: s_setpc_b64 s[30:31] bb: %umulo = tail call { i64, i1 } @llvm.smul.with.overflow.i64(i64 %i, i64 4) ret { i64, i1 } %umulo @@ -521,6 +668,20 @@ ; GFX10-NEXT: v_mov_b32_e32 v1, v3 ; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: umulo_i64_v_4: +; GFX11: ; %bb.0: ; %bb +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: v_and_b32_e32 v7, 0x3fffffff, v1 +; GFX11-NEXT: v_mov_b32_e32 v6, v0 +; GFX11-NEXT: v_lshlrev_b64 v[4:5], 2, v[0:1] +; GFX11-NEXT: v_alignbit_b32 v3, v1, v0, 30 +; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[6:7], v[0:1] +; GFX11-NEXT: v_mov_b32_e32 v0, v4 +; GFX11-NEXT: v_mov_b32_e32 v1, v3 +; GFX11-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo +; GFX11-NEXT: s_setpc_b64 s[30:31] bb: %umulo = tail call { i64, i1 } @llvm.umul.with.overflow.i64(i64 %i, i64 4) ret { i64, i1 } %umulo diff --git a/llvm/test/CodeGen/AMDGPU/mad_64_32.ll b/llvm/test/CodeGen/AMDGPU/mad_64_32.ll --- a/llvm/test/CodeGen/AMDGPU/mad_64_32.ll +++ b/llvm/test/CodeGen/AMDGPU/mad_64_32.ll @@ -2,6 +2,9 @@ ; RUN: llc -march=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -check-prefixes=CI %s ; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefixes=SI %s ; RUN: llc -march=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9 %s +; RUN: llc -march=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11 %s + +; On GFX11, ensure vdst and src2 do not partially overlap. Full overlap is ok. define i64 @mad_i64_i32_sextops(i32 %arg0, i32 %arg1, i64 %arg2) #0 { ; CI-LABEL: mad_i64_i32_sextops: @@ -24,6 +27,15 @@ ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3] ; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: mad_i64_i32_sextops: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: v_mov_b32_e32 v4, v1 +; GFX11-NEXT: v_mov_b32_e32 v5, v0 +; GFX11-NEXT: v_mad_i64_i32 v[0:1], s0, v5, v4, v[2:3] +; GFX11-NEXT: s_setpc_b64 s[30:31] %sext0 = sext i32 %arg0 to i64 %sext1 = sext i32 %arg1 to i64 %mul = mul i64 %sext0, %sext1 @@ -52,6 +64,15 @@ ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3] ; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: mad_i64_i32_sextops_commute: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: v_mov_b32_e32 v4, v1 +; GFX11-NEXT: v_mov_b32_e32 v5, v0 +; GFX11-NEXT: v_mad_i64_i32 v[0:1], s0, v5, v4, v[2:3] +; GFX11-NEXT: s_setpc_b64 s[30:31] %sext0 = sext i32 %arg0 to i64 %sext1 = sext i32 %arg1 to i64 %mul = mul i64 %sext0, %sext1 @@ -80,6 +101,15 @@ ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[2:3] ; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: mad_u64_u32_zextops: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: v_mov_b32_e32 v4, v1 +; GFX11-NEXT: v_mov_b32_e32 v5, v0 +; GFX11-NEXT: v_mad_u64_u32 v[0:1], s0, v5, v4, v[2:3] +; GFX11-NEXT: s_setpc_b64 s[30:31] %sext0 = zext i32 %arg0 to i64 %sext1 = zext i32 %arg1 to i64 %mul = mul i64 %sext0, %sext1 @@ -108,6 +138,15 @@ ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[2:3] ; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: mad_u64_u32_zextops_commute: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: v_mov_b32_e32 v4, v1 +; GFX11-NEXT: v_mov_b32_e32 v5, v0 +; GFX11-NEXT: v_mad_u64_u32 v[0:1], s0, v5, v4, v[2:3] +; GFX11-NEXT: s_setpc_b64 s[30:31] %sext0 = zext i32 %arg0 to i64 %sext1 = zext i32 %arg1 to i64 %mul = mul i64 %sext0, %sext1 @@ -200,6 +239,33 @@ ; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, v7, v4, vcc ; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v8, v5, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: mad_i64_i32_sextops_i32_i128: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: v_mad_u64_u32 v[6:7], s0, v0, v1, 0 +; GFX11-NEXT: v_mov_b32_e32 v8, 0 +; GFX11-NEXT: v_ashrrev_i32_e32 v14, 31, v0 +; GFX11-NEXT: v_ashrrev_i32_e32 v15, 31, v1 +; GFX11-NEXT: v_mad_u64_u32 v[9:10], s0, v14, v1, v[7:8] +; GFX11-NEXT: v_mov_b32_e32 v7, v10 +; GFX11-NEXT: v_mov_b32_e32 v10, v8 +; GFX11-NEXT: v_mad_u64_u32 v[11:12], s0, v0, v15, v[9:10] +; GFX11-NEXT: v_mad_i64_i32 v[9:10], s0, v1, v14, 0 +; GFX11-NEXT: v_mov_b32_e32 v8, v12 +; GFX11-NEXT: v_add_co_u32 v7, s0, v7, v8 +; GFX11-NEXT: v_add_co_ci_u32_e64 v8, s0, 0, 0, s0 +; GFX11-NEXT: v_mad_i64_i32 v[12:13], s0, v15, v0, v[9:10] +; GFX11-NEXT: v_mad_u64_u32 v[0:1], s0, v14, v15, v[7:8] +; GFX11-NEXT: v_mov_b32_e32 v7, v11 +; GFX11-NEXT: v_add_co_u32 v8, vcc_lo, v0, v12 +; GFX11-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, v1, v13, vcc_lo +; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v6, v2 +; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v7, v3, vcc_lo +; GFX11-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v8, v4, vcc_lo +; GFX11-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v9, v5, vcc_lo +; GFX11-NEXT: s_setpc_b64 s[30:31] %sext0 = sext i32 %arg0 to i128 %sext1 = sext i32 %arg1 to i128 %mul = mul i128 %sext0, %sext1 @@ -228,6 +294,15 @@ ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3] ; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: mad_i64_i32_sextops_i32_i63: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: v_mov_b32_e32 v4, v1 +; GFX11-NEXT: v_mov_b32_e32 v5, v0 +; GFX11-NEXT: v_mad_i64_i32 v[0:1], s0, v5, v4, v[2:3] +; GFX11-NEXT: s_setpc_b64 s[30:31] %sext0 = sext i32 %arg0 to i63 %sext1 = sext i32 %arg1 to i63 %mul = mul i63 %sext0, %sext1 @@ -264,6 +339,15 @@ ; GFX9-NEXT: v_bfe_i32 v0, v0, 0, 31 ; GFX9-NEXT: v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3] ; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: mad_i64_i32_sextops_i31_i63: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: v_bfe_i32 v4, v1, 0, 31 +; GFX11-NEXT: v_bfe_i32 v5, v0, 0, 31 +; GFX11-NEXT: v_mad_i64_i32 v[0:1], s0, v5, v4, v[2:3] +; GFX11-NEXT: s_setpc_b64 s[30:31] %sext0 = sext i31 %arg0 to i63 %sext1 = sext i31 %arg1 to i63 %mul = mul i63 %sext0, %sext1 @@ -303,6 +387,18 @@ ; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v5, v4, v[2:3] ; GFX9-NEXT: v_mov_b32_e32 v1, v2 ; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: mad_i64_i32_extops_i32_i64: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: v_mov_b32_e32 v4, v1 +; GFX11-NEXT: v_mov_b32_e32 v5, v0 +; GFX11-NEXT: v_mad_u64_u32 v[0:1], s0, v5, v4, v[2:3] +; GFX11-NEXT: v_ashrrev_i32_e32 v5, 31, v5 +; GFX11-NEXT: v_mov_b32_e32 v3, v1 +; GFX11-NEXT: v_mad_u64_u32 v[1:2], s0, v5, v4, v[3:4] +; GFX11-NEXT: s_setpc_b64 s[30:31] %ext0 = sext i32 %arg0 to i64 %ext1 = zext i32 %arg1 to i64 %mul = mul i64 %ext0, %ext1 @@ -331,6 +427,14 @@ ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v2, v[4:5] ; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: mad_u64_u32_bitops: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: v_mov_b32_e32 v3, v0 +; GFX11-NEXT: v_mad_u64_u32 v[0:1], s0, v3, v2, v[4:5] +; GFX11-NEXT: s_setpc_b64 s[30:31] %trunc.lhs = and i64 %arg0, 4294967295 %trunc.rhs = and i64 %arg1, 4294967295 %mul = mul i64 %trunc.lhs, %trunc.rhs @@ -369,6 +473,19 @@ ; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v3, v2, v[4:5] ; GFX9-NEXT: v_mov_b32_e32 v1, v2 ; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: mad_u64_u32_bitops_lhs_mask_small: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v2, v0 +; GFX11-NEXT: v_mov_b32_e32 v6, v1 +; GFX11-NEXT: v_mad_u64_u32 v[0:1], s0, v2, v3, v[4:5] +; GFX11-NEXT: v_and_b32_e32 v5, 1, v6 +; GFX11-NEXT: v_mov_b32_e32 v4, v1 +; GFX11-NEXT: v_mad_u64_u32 v[1:2], s0, v5, v3, v[4:5] +; GFX11-NEXT: s_setpc_b64 s[30:31] %trunc.lhs = and i64 %arg0, 8589934591 %trunc.rhs = and i64 %arg1, 4294967295 %mul = mul i64 %trunc.lhs, %trunc.rhs @@ -409,6 +526,17 @@ ; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, v3, v[2:3] ; GFX9-NEXT: v_mov_b32_e32 v1, v2 ; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: mad_u64_u32_bitops_rhs_mask_small: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: v_mov_b32_e32 v6, v0 +; GFX11-NEXT: v_mad_u64_u32 v[0:1], s0, v6, v2, v[4:5] +; GFX11-NEXT: v_and_b32_e32 v4, 1, v3 +; GFX11-NEXT: v_mov_b32_e32 v3, v1 +; GFX11-NEXT: v_mad_u64_u32 v[1:2], s0, v6, v4, v[3:4] +; GFX11-NEXT: s_setpc_b64 s[30:31] %trunc.lhs = and i64 %arg0, 4294967295 %trunc.rhs = and i64 %arg1, 8589934591 %mul = mul i64 %trunc.lhs, %trunc.rhs @@ -437,6 +565,14 @@ ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: v_mad_i64_i32 v[0:1], s[4:5], v0, v2, v[4:5] ; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: mad_i64_i32_bitops: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: v_mov_b32_e32 v3, v0 +; GFX11-NEXT: v_mad_i64_i32 v[0:1], s0, v3, v2, v[4:5] +; GFX11-NEXT: s_setpc_b64 s[30:31] %shl.lhs = shl i64 %arg0, 32 %trunc.lhs = ashr i64 %shl.lhs, 32 %shl.rhs = shl i64 %arg1, 32 @@ -468,6 +604,15 @@ ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v1, v0, v[0:1] ; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: mad_i64_i32_unpack_i64ops: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: v_mad_u64_u32 v[2:3], s0, v1, v0, v[0:1] +; GFX11-NEXT: v_mov_b32_e32 v0, v2 +; GFX11-NEXT: v_mov_b32_e32 v1, v3 +; GFX11-NEXT: s_setpc_b64 s[30:31] %tmp4 = lshr i64 %arg0, 32 %tmp5 = and i64 %arg0, 4294967295 %mul = mul nuw i64 %tmp4, %tmp5 @@ -523,6 +668,23 @@ ; GFX9-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] ; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7] ; GFX9-NEXT: s_endpgm +; +; GFX11-LABEL: mad_i64_i32_uniform: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_clause 0x2 +; GFX11-NEXT: s_load_b64 s[2:3], s[0:1], 0x2c +; GFX11-NEXT: s_load_b64 s[4:5], s[0:1], 0x34 +; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: s_mul_i32 s6, s2, s3 +; GFX11-NEXT: s_mul_hi_u32 s3, s2, s3 +; GFX11-NEXT: s_add_u32 s2, s6, s4 +; GFX11-NEXT: s_addc_u32 s3, s3, s5 +; GFX11-NEXT: v_mov_b32_e32 v0, s2 +; GFX11-NEXT: v_mov_b32_e32 v1, s3 +; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1] +; GFX11-NEXT: s_endpgm %ext0 = zext i32 %arg0 to i64 %ext1 = zext i32 %arg1 to i64 %mul = mul i64 %ext0, %ext1 @@ -562,6 +724,16 @@ ; GFX9-NEXT: v_xor_b32_e32 v1, v3, v1 ; GFX9-NEXT: v_xor_b32_e32 v0, v2, v0 ; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: mad_i64_i32_twice: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: v_mad_i64_i32 v[6:7], s0, v0, v1, v[2:3] +; GFX11-NEXT: v_mad_i64_i32 v[2:3], s0, v0, v1, v[4:5] +; GFX11-NEXT: v_xor_b32_e32 v0, v6, v2 +; GFX11-NEXT: v_xor_b32_e32 v1, v7, v3 +; GFX11-NEXT: s_setpc_b64 s[30:31] %sext0 = sext i32 %arg0 to i64 %sext1 = sext i32 %arg1 to i64 %mul = mul i64 %sext0, %sext1 @@ -616,6 +788,23 @@ ; GFX9-NEXT: v_xor_b32_e32 v1, v3, v1 ; GFX9-NEXT: v_xor_b32_e32 v0, v2, v0 ; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: mad_i64_i32_thrice: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: v_mad_i64_i32 v[8:9], s0, v0, v1, 0 +; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v8, v2 +; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v9, v3, vcc_lo +; GFX11-NEXT: v_add_co_u32 v2, vcc_lo, v8, v4 +; GFX11-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v9, v5, vcc_lo +; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, v8, v6 +; GFX11-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, v9, v7, vcc_lo +; GFX11-NEXT: v_xor_b32_e32 v0, v0, v2 +; GFX11-NEXT: v_xor_b32_e32 v1, v1, v3 +; GFX11-NEXT: v_xor_b32_e32 v0, v0, v4 +; GFX11-NEXT: v_xor_b32_e32 v1, v1, v5 +; GFX11-NEXT: s_setpc_b64 s[30:31] %sext0 = sext i32 %arg0 to i64 %sext1 = sext i32 %arg1 to i64 %mul = mul i64 %sext0, %sext1 @@ -657,6 +846,17 @@ ; GFX9-NEXT: v_xor_b32_e32 v1, v1, v5 ; GFX9-NEXT: v_xor_b32_e32 v0, v0, v4 ; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: mad_i64_i32_secondary_use: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: v_mad_i64_i32 v[4:5], s0, v0, v1, 0 +; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v4, v2 +; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v5, v3, vcc_lo +; GFX11-NEXT: v_xor_b32_e32 v0, v0, v4 +; GFX11-NEXT: v_xor_b32_e32 v1, v1, v5 +; GFX11-NEXT: s_setpc_b64 s[30:31] %sext0 = sext i32 %arg0 to i64 %sext1 = sext i32 %arg1 to i64 %mul = mul i64 %sext0, %sext1 @@ -701,6 +901,18 @@ ; GFX9-NEXT: v_mul_lo_u32 v2, v6, v2 ; GFX9-NEXT: v_add3_u32 v1, v2, v1, v3 ; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: mad_i48_i48: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: v_mov_b32_e32 v6, v1 +; GFX11-NEXT: v_mov_b32_e32 v7, v0 +; GFX11-NEXT: v_mad_u64_u32 v[0:1], s0, v7, v2, v[4:5] +; GFX11-NEXT: v_mul_lo_u32 v3, v7, v3 +; GFX11-NEXT: v_mul_lo_u32 v2, v6, v2 +; GFX11-NEXT: v_add3_u32 v1, v2, v1, v3 +; GFX11-NEXT: s_setpc_b64 s[30:31] %m = mul i48 %arg0, %arg1 %a = add i48 %m, %arg2 ret i48 %a