Index: lib/Target/AMDGPU/AMDGPUISelLowering.h =================================================================== --- lib/Target/AMDGPU/AMDGPUISelLowering.h +++ lib/Target/AMDGPU/AMDGPUISelLowering.h @@ -70,6 +70,9 @@ SDValue performSraCombine(SDNode *N, DAGCombinerInfo &DCI) const; SDValue performSrlCombine(SDNode *N, DAGCombinerInfo &DCI) const; SDValue performMulCombine(SDNode *N, DAGCombinerInfo &DCI) const; + SDValue performMulhsCombine(SDNode *N, DAGCombinerInfo &DCI) const; + SDValue performMulhuCombine(SDNode *N, DAGCombinerInfo &DCI) const; + SDValue performMulLoHi24Combine(SDNode *N, DAGCombinerInfo &DCI) const; SDValue performCtlzCombine(const SDLoc &SL, SDValue Cond, SDValue LHS, SDValue RHS, DAGCombinerInfo &DCI) const; SDValue performSelectCombine(SDNode *N, DAGCombinerInfo &DCI) const; @@ -226,9 +229,9 @@ DWORDADDR, FRACT, CLAMP, - // This is SETCC with the full mask result which is used for a compare with a + // This is SETCC with the full mask result which is used for a compare with a // result bit per item in the wavefront. - SETCC, + SETCC, // SIN_HW, COS_HW - f32 for SI, 1 ULP max error, valid from -100 pi to 100 pi. // Denormals handled on some parts. @@ -272,8 +275,12 @@ FFBH_I32, MUL_U24, MUL_I24, + MULHI_U24, + MULHI_I24, MAD_U24, MAD_I24, + MUL_LOHI_I24, + MUL_LOHI_U24, TEXTURE_FETCH, EXPORT, CONST_ADDRESS, Index: lib/Target/AMDGPU/AMDGPUISelLowering.cpp =================================================================== --- lib/Target/AMDGPU/AMDGPUISelLowering.cpp +++ lib/Target/AMDGPU/AMDGPUISelLowering.cpp @@ -467,6 +467,8 @@ setTargetDAGCombine(ISD::SRA); setTargetDAGCombine(ISD::SRL); setTargetDAGCombine(ISD::MUL); + setTargetDAGCombine(ISD::MULHU); + setTargetDAGCombine(ISD::MULHS); setTargetDAGCombine(ISD::SELECT); setTargetDAGCombine(ISD::SELECT_CC); setTargetDAGCombine(ISD::STORE); @@ -2285,11 +2287,36 @@ return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildPair); } +// We need to specifically handle i64 mul here to avoid unnecessary conversion +// instructions. If we only match on the legalized i64 mul expansion, +// SimplifyDemandedBits will be unable to remove them because there will be +// multiple uses due to the separate mul + mulh[su]. +static SDValue getMul24(SelectionDAG &DAG, const SDLoc &SL, + SDValue N0, SDValue N1, unsigned Size, bool Signed) { + if (Size <= 32) { + unsigned MulOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24; + return DAG.getNode(MulOpc, SL, MVT::i32, N0, N1); + } + + // Because we want to eliminate extension instructions before the + // operation, we need to create a single user here (i.e. not the separate + // mul_lo + mul_hi) so that SimplifyDemandedBits will deal with it. + + unsigned MulOpc = Signed ? AMDGPUISD::MUL_LOHI_I24 : AMDGPUISD::MUL_LOHI_U24; + + SDValue Mul = DAG.getNode(MulOpc, SL, + DAG.getVTList(MVT::i32, MVT::i32), N0, N1); + + return DAG.getNode(ISD::BUILD_PAIR, SL, MVT::i64, + Mul.getValue(0), Mul.getValue(1)); +} + SDValue AMDGPUTargetLowering::performMulCombine(SDNode *N, DAGCombinerInfo &DCI) const { EVT VT = N->getValueType(0); - if (VT.isVector() || VT.getSizeInBits() > 32) + unsigned Size = VT.getSizeInBits(); + if (VT.isVector() || Size > 64) return SDValue(); SelectionDAG &DAG = DCI.DAG; @@ -2302,11 +2329,11 @@ if (Subtarget->hasMulU24() && isU24(N0, DAG) && isU24(N1, DAG)) { N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32); N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32); - Mul = DAG.getNode(AMDGPUISD::MUL_U24, DL, MVT::i32, N0, N1); + Mul = getMul24(DAG, DL, N0, N1, Size, false); } else if (Subtarget->hasMulI24() && isI24(N0, DAG) && isI24(N1, DAG)) { N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32); N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32); - Mul = DAG.getNode(AMDGPUISD::MUL_I24, DL, MVT::i32, N0, N1); + Mul = getMul24(DAG, DL, N0, N1, Size, true); } else { return SDValue(); } @@ -2316,6 +2343,77 @@ return DAG.getSExtOrTrunc(Mul, DL, VT); } +SDValue AMDGPUTargetLowering::performMulhsCombine(SDNode *N, + DAGCombinerInfo &DCI) const { + EVT VT = N->getValueType(0); + + if (!Subtarget->hasMulI24() || VT.isVector()) + return SDValue(); + + SelectionDAG &DAG = DCI.DAG; + SDLoc DL(N); + + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); + + if (!isI24(N0, DAG) || !isI24(N1, DAG)) + return SDValue(); + + N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32); + N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32); + + SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_I24, DL, MVT::i32, N0, N1); + DCI.AddToWorklist(Mulhi.getNode()); + return DAG.getSExtOrTrunc(Mulhi, DL, VT); +} + +SDValue AMDGPUTargetLowering::performMulhuCombine(SDNode *N, + DAGCombinerInfo &DCI) const { + EVT VT = N->getValueType(0); + + if (!Subtarget->hasMulU24() || VT.isVector() || VT.getSizeInBits() > 32) + return SDValue(); + + SelectionDAG &DAG = DCI.DAG; + SDLoc DL(N); + + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); + + if (!isU24(N0, DAG) || !isU24(N1, DAG)) + return SDValue(); + + N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32); + N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32); + + SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_U24, DL, MVT::i32, N0, N1); + DCI.AddToWorklist(Mulhi.getNode()); + return DAG.getZExtOrTrunc(Mulhi, DL, VT); +} + +SDValue AMDGPUTargetLowering::performMulLoHi24Combine( + SDNode *N, DAGCombinerInfo &DCI) const { + SelectionDAG &DAG = DCI.DAG; + + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); + + // Simplify demanded bits before splitting into multiple users. + if (simplifyI24(N0, DCI) || simplifyI24(N1, DCI)) + return SDValue(); + + bool Signed = (N->getOpcode() == AMDGPUISD::MUL_LOHI_I24); + + unsigned MulLoOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24; + unsigned MulHiOpc = Signed ? AMDGPUISD::MULHI_I24 : AMDGPUISD::MULHI_U24; + + SDLoc SL(N); + + SDValue MulLo = DAG.getNode(MulLoOpc, SL, MVT::i32, N0, N1); + SDValue MulHi = DAG.getNode(MulHiOpc, SL, MVT::i32, N0, N1); + return DAG.getMergeValues({ MulLo, MulHi }, SL); +} + static bool isNegativeOne(SDValue Val) { if (ConstantSDNode *C = dyn_cast(Val)) return C->isAllOnesValue(); @@ -2476,14 +2574,23 @@ } case ISD::MUL: return performMulCombine(N, DCI); + case ISD::MULHS: + return performMulhsCombine(N, DCI); + case ISD::MULHU: + return performMulhuCombine(N, DCI); case AMDGPUISD::MUL_I24: - case AMDGPUISD::MUL_U24: { + case AMDGPUISD::MUL_U24: + case AMDGPUISD::MULHI_I24: + case AMDGPUISD::MULHI_U24: { SDValue N0 = N->getOperand(0); SDValue N1 = N->getOperand(1); simplifyI24(N0, DCI); simplifyI24(N1, DCI); return SDValue(); } + case AMDGPUISD::MUL_LOHI_I24: + case AMDGPUISD::MUL_LOHI_U24: + return performMulLoHi24Combine(N, DCI); case ISD::SELECT: return performSelectCombine(N, DCI); case AMDGPUISD::BFE_I32: @@ -2695,6 +2802,10 @@ NODE_NAME_CASE(FFBH_I32) NODE_NAME_CASE(MUL_U24) NODE_NAME_CASE(MUL_I24) + NODE_NAME_CASE(MULHI_U24) + NODE_NAME_CASE(MULHI_I24) + NODE_NAME_CASE(MUL_LOHI_U24) + NODE_NAME_CASE(MUL_LOHI_I24) NODE_NAME_CASE(MAD_U24) NODE_NAME_CASE(MAD_I24) NODE_NAME_CASE(TEXTURE_FETCH) Index: lib/Target/AMDGPU/AMDGPUInstrInfo.td =================================================================== --- lib/Target/AMDGPU/AMDGPUInstrInfo.td +++ lib/Target/AMDGPU/AMDGPUInstrInfo.td @@ -216,13 +216,20 @@ def AMDGPUffbh_u32 : SDNode<"AMDGPUISD::FFBH_U32", SDTIntUnaryOp>; def AMDGPUffbh_i32 : SDNode<"AMDGPUISD::FFBH_I32", SDTIntUnaryOp>; -// Signed and unsigned 24-bit mulitply. The highest 8-bits are ignore when -// performing the mulitply. The result is a 32-bit value. +// Signed and unsigned 24-bit multiply. The highest 8-bits are ignore +// when performing the mulitply. The result is a 32-bit value. def AMDGPUmul_u24 : SDNode<"AMDGPUISD::MUL_U24", SDTIntBinOp, - [SDNPCommutative] + [SDNPCommutative, SDNPAssociative] >; def AMDGPUmul_i24 : SDNode<"AMDGPUISD::MUL_I24", SDTIntBinOp, - [SDNPCommutative] + [SDNPCommutative, SDNPAssociative] +>; + +def AMDGPUmulhi_u24 : SDNode<"AMDGPUISD::MULHI_U24", SDTIntBinOp, + [SDNPCommutative, SDNPAssociative] +>; +def AMDGPUmulhi_i24 : SDNode<"AMDGPUISD::MULHI_I24", SDTIntBinOp, + [SDNPCommutative, SDNPAssociative] >; def AMDGPUmad_u24 : SDNode<"AMDGPUISD::MAD_U24", AMDGPUDTIntTernaryOp, Index: lib/Target/AMDGPU/CaymanInstructions.td =================================================================== --- lib/Target/AMDGPU/CaymanInstructions.td +++ lib/Target/AMDGPU/CaymanInstructions.td @@ -37,6 +37,9 @@ def MULHI_INT_cm : MULHI_INT_Common<0x90>; def MULLO_UINT_cm : MULLO_UINT_Common<0x91>; def MULHI_UINT_cm : MULHI_UINT_Common<0x92>; +def MULHI_INT_cm24 : MULHI_INT24_Common<0x5c>; +def MULHI_UINT_cm24 : MULHI_UINT24_Common<0xb2>; + def RECIPSQRT_CLAMPED_cm : RECIPSQRT_CLAMPED_Common<0x87>; def EXP_IEEE_cm : EXP_IEEE_Common<0x81>; def LOG_IEEE_cm : LOG_IEEE_Common<0x83>; Index: lib/Target/AMDGPU/EvergreenInstructions.td =================================================================== --- lib/Target/AMDGPU/EvergreenInstructions.td +++ lib/Target/AMDGPU/EvergreenInstructions.td @@ -72,6 +72,8 @@ def MULHI_INT_eg : MULHI_INT_Common<0x90>; def MULLO_UINT_eg : MULLO_UINT_Common<0x91>; def MULHI_UINT_eg : MULHI_UINT_Common<0x92>; +def MULHI_UINT24_eg : MULHI_UINT24_Common<0xb2>; + def RECIP_UINT_eg : RECIP_UINT_Common<0x94>; def RECIPSQRT_CLAMPED_eg : RECIPSQRT_CLAMPED_Common<0x87>; def EXP_IEEE_eg : EXP_IEEE_Common<0x81>; Index: lib/Target/AMDGPU/R600Instructions.td =================================================================== --- lib/Target/AMDGPU/R600Instructions.td +++ lib/Target/AMDGPU/R600Instructions.td @@ -1073,18 +1073,27 @@ class LSHR_Common inst> : R600_2OP_Helper ; class ASHR_Common inst> : R600_2OP_Helper ; class MULHI_INT_Common inst> : R600_2OP_Helper < - inst, "MULHI_INT", mulhs -> { + inst, "MULHI_INT", mulhs> { let Itinerary = TransALU; } + +class MULHI_INT24_Common inst> : R600_2OP_Helper < + inst, "MULHI_INT24", AMDGPUmulhi_i24> { + let Itinerary = VecALU; +} + class MULHI_UINT_Common inst> : R600_2OP_Helper < - inst, "MULHI", mulhu -> { + inst, "MULHI", mulhu> { let Itinerary = TransALU; } + +class MULHI_UINT24_Common inst> : R600_2OP_Helper < + inst, "MULHI_UINT24", AMDGPUmulhi_u24> { + let Itinerary = VecALU; +} + class MULLO_INT_Common inst> : R600_2OP_Helper < - inst, "MULLO_INT", mul -> { + inst, "MULLO_INT", mul> { let Itinerary = TransALU; } class MULLO_UINT_Common inst> : R600_2OP { Index: lib/Target/AMDGPU/SIInstructions.td =================================================================== --- lib/Target/AMDGPU/SIInstructions.td +++ lib/Target/AMDGPU/SIInstructions.td @@ -1362,7 +1362,7 @@ >; defm V_MUL_HI_I32_I24 : VOP2Inst , "v_mul_hi_i32_i24", - VOP_I32_I32_I32 + VOP_I32_I32_I32, AMDGPUmulhi_i24 >; defm V_MUL_U32_U24 : VOP2Inst , "v_mul_u32_u24", @@ -1370,7 +1370,7 @@ >; defm V_MUL_HI_U32_U24 : VOP2Inst , "v_mul_hi_u32_u24", - VOP_I32_I32_I32 + VOP_I32_I32_I32, AMDGPUmulhi_u24 >; defm V_MIN_F32 : VOP2Inst , "v_min_f32", VOP_F32_F32_F32, Index: test/CodeGen/AMDGPU/mul_int24.ll =================================================================== --- test/CodeGen/AMDGPU/mul_int24.ll +++ test/CodeGen/AMDGPU/mul_int24.ll @@ -1,23 +1,151 @@ -; RUN: llc < %s -march=amdgcn -verify-machineinstrs | FileCheck %s --check-prefix=SI --check-prefix=FUNC -; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s --check-prefix=SI --check-prefix=FUNC -; RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck %s --check-prefix=EG --check-prefix=FUNC -; RUN: llc < %s -march=r600 -mcpu=cayman | FileCheck %s --check-prefix=CM --check-prefix=FUNC +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=FUNC %s +; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s +; RUN: llc -march=r600 -mcpu=cayman < %s | FileCheck -check-prefix=CM -check-prefix=FUNC %s + +; FUNC-LABEL: {{^}}test_smul24_i32: +; GCN-NOT: bfe +; GCN: v_mul_i32_i24 -; FUNC-LABEL: {{^}}i32_mul24: ; Signed 24-bit multiply is not supported on pre-Cayman GPUs. ; EG: MULLO_INT + ; Make sure we are not masking the inputs ; CM-NOT: AND ; CM: MUL_INT24 -; SI-NOT: and -; SI: v_mul_i32_i24 -define void @i32_mul24(i32 addrspace(1)* %out, i32 %a, i32 %b) { +define void @test_smul24_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) #0 { entry: - %0 = shl i32 %a, 8 - %a_24 = ashr i32 %0, 8 - %1 = shl i32 %b, 8 - %b_24 = ashr i32 %1, 8 - %2 = mul i32 %a_24, %b_24 - store i32 %2, i32 addrspace(1)* %out + %a.shl = shl i32 %a, 8 + %a.24 = ashr i32 %a.shl, 8 + %b.shl = shl i32 %b, 8 + %b.24 = ashr i32 %b.shl, 8 + %mul24 = mul i32 %a.24, %b.24 + store i32 %mul24, i32 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}test_smulhi24_i64: +; GCN-NOT: bfe +; GCN-NOT: ashr +; GCN: v_mul_hi_i32_i24_e32 [[RESULT:v[0-9]+]], +; GCN-NEXT: buffer_store_dword [[RESULT]] + +; EG: ASHR +; EG: ASHR +; EG: MULHI_INT + +; CM-NOT: ASHR +; CM: MULHI_INT24 +; CM: MULHI_INT24 +; CM: MULHI_INT24 +; CM: MULHI_INT24 +define void @test_smulhi24_i64(i32 addrspace(1)* %out, i32 %a, i32 %b) #0 { +entry: + %a.shl = shl i32 %a, 8 + %a.24 = ashr i32 %a.shl, 8 + %b.shl = shl i32 %b, 8 + %b.24 = ashr i32 %b.shl, 8 + %a.24.i64 = sext i32 %a.24 to i64 + %b.24.i64 = sext i32 %b.24 to i64 + %mul48 = mul i64 %a.24.i64, %b.24.i64 + %mul48.hi = lshr i64 %mul48, 32 + %mul24hi = trunc i64 %mul48.hi to i32 + store i32 %mul24hi, i32 addrspace(1)* %out + ret void +} + +; This requires handling of the original 64-bit mul node to eliminate +; unnecessary extension instructions because after legalization they +; will not be removed by SimplifyDemandedBits because there are +; multiple uses by the separate mul and mulhi. + +; FUNC-LABEL: {{^}}test_smul24_i64: +; GCN: s_load_dword s +; GCN: s_load_dword s + +; GCN-NOT: bfe +; GCN-NOT: ashr + +; GCN-DAG: v_mul_hi_i32_i24_e32 +; GCN-DAG: v_mul_i32_i24_e32 + +; GCN: buffer_store_dwordx2 +define void @test_smul24_i64(i64 addrspace(1)* %out, i32 %a, i32 %b) #0 { + %shl.i = shl i32 %a, 8 + %shr.i = ashr i32 %shl.i, 8 + %conv.i = sext i32 %shr.i to i64 + %shl1.i = shl i32 %b, 8 + %shr2.i = ashr i32 %shl1.i, 8 + %conv3.i = sext i32 %shr2.i to i64 + %mul.i = mul i64 %conv3.i, %conv.i + store i64 %mul.i, i64 addrspace(1)* %out + ret void +} + +; FIXME: Should be able to eliminate bfe +; FUNC-LABEL: {{^}}test_smul24_i64_square: +; GCN: s_load_dword [[A:s[0-9]+]] +; GCN: s_bfe_i32 [[SEXT:s[0-9]+]], [[A]], 0x180000{{$}} +; GCN-DAG: v_mul_hi_i32_i24_e64 v{{[0-9]+}}, [[SEXT]], [[SEXT]] +; GCN-DAG: v_mul_i32_i24_e64 v{{[0-9]+}}, [[SEXT]], [[SEXT]] +; GCN: buffer_store_dwordx2 +define void @test_smul24_i64_square(i64 addrspace(1)* %out, i32 %a, i32 %b) #0 { + %shl.i = shl i32 %a, 8 + %shr.i = ashr i32 %shl.i, 8 + %conv.i = sext i32 %shr.i to i64 + %mul.i = mul i64 %conv.i, %conv.i + store i64 %mul.i, i64 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}test_smul24_i33: +; GCN: s_load_dword s +; GCN: s_load_dword s + +; GCN-NOT: and +; GCN-NOT: lshr + +; GCN-DAG: v_mul_i32_i24_e32 +; GCN-DAG: v_mul_hi_i32_i24_e32 +; SI: v_lshl_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, 31 +; SI: v_ashr_i64 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, 31 + +; VI: v_lshlrev_b64 v{{\[[0-9]+:[0-9]+\]}}, 31, v{{\[[0-9]+:[0-9]+\]}} +; VI: v_ashrrev_i64 v{{\[[0-9]+:[0-9]+\]}}, 31, v{{\[[0-9]+:[0-9]+\]}} + +; GCN: buffer_store_dwordx2 +define void @test_smul24_i33(i64 addrspace(1)* %out, i33 %a, i33 %b) #0 { +entry: + %a.shl = shl i33 %a, 9 + %a.24 = ashr i33 %a.shl, 9 + %b.shl = shl i33 %b, 9 + %b.24 = ashr i33 %b.shl, 9 + %mul24 = mul i33 %a.24, %b.24 + %ext = sext i33 %mul24 to i64 + store i64 %ext, i64 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}test_smulhi24_i33: +; SI: s_load_dword s +; SI: s_load_dword s + +; SI-NOT: bfe + +; SI: v_mul_hi_i32_i24_e32 v[[MUL_HI:[0-9]+]], +; SI-NEXT: v_and_b32_e32 v[[HI:[0-9]+]], 1, v[[MUL_HI]] +; SI-NEXT: buffer_store_dword v[[HI]] +define void @test_smulhi24_i33(i32 addrspace(1)* %out, i33 %a, i33 %b) { +entry: + %tmp0 = shl i33 %a, 9 + %a_24 = ashr i33 %tmp0, 9 + %tmp1 = shl i33 %b, 9 + %b_24 = ashr i33 %tmp1, 9 + %tmp2 = mul i33 %a_24, %b_24 + %hi = lshr i33 %tmp2, 32 + %trunc = trunc i33 %hi to i32 + + store i32 %trunc, i32 addrspace(1)* %out ret void } +attributes #0 = { nounwind } Index: test/CodeGen/AMDGPU/mul_uint24.ll =================================================================== --- test/CodeGen/AMDGPU/mul_uint24.ll +++ test/CodeGen/AMDGPU/mul_uint24.ll @@ -1,13 +1,12 @@ -; RUN: llc < %s -march=amdgcn -verify-machineinstrs | FileCheck %s --check-prefix=SI --check-prefix=FUNC -; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s --check-prefix=SI --check-prefix=FUNC -; RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck %s --check-prefix=EG --check-prefix=FUNC -; RUN: llc < %s -march=r600 -mcpu=cayman | FileCheck %s --check-prefix=EG --check-prefix=FUNC +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s +; RUN: llc -march=r600 -mcpu=cayman < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s -; FUNC-LABEL: {{^}}u32_mul24: +; FUNC-LABEL: {{^}}test_umul24_i32: ; EG: MUL_UINT24 {{[* ]*}}T{{[0-9]\.[XYZW]}}, KC0[2].Z, KC0[2].W ; SI: v_mul_u32_u24 - -define void @u32_mul24(i32 addrspace(1)* %out, i32 %a, i32 %b) { +define void @test_umul24_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) { entry: %0 = shl i32 %a, 8 %a_24 = lshr i32 %0, 8 @@ -18,46 +17,98 @@ ret void } -; FUNC-LABEL: {{^}}i16_mul24: +; FUNC-LABEL: {{^}}test_umul24_i16_sext: ; EG: MUL_UINT24 {{[* ]*}}T{{[0-9]}}.[[MUL_CHAN:[XYZW]]] ; The result must be sign-extended ; EG: BFE_INT {{[* ]*}}T{{[0-9]}}.{{[XYZW]}}, PV.[[MUL_CHAN]], 0.0, literal.x ; EG: 16 + ; SI: v_mul_u32_u24_e{{(32|64)}} [[MUL:v[0-9]]], {{[sv][0-9], [sv][0-9]}} ; SI: v_bfe_i32 v{{[0-9]}}, [[MUL]], 0, 16 -define void @i16_mul24(i32 addrspace(1)* %out, i16 %a, i16 %b) { +define void @test_umul24_i16_sext(i32 addrspace(1)* %out, i16 %a, i16 %b) { +entry: + %mul = mul i16 %a, %b + %ext = sext i16 %mul to i32 + store i32 %ext, i32 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}test_umul24_i16: +; SI: s_and_b32 +; SI: v_mul_u32_u24_e32 +; SI: v_and_b32_e32 +define void @test_umul24_i16(i32 addrspace(1)* %out, i16 %a, i16 %b) { entry: - %0 = mul i16 %a, %b - %1 = sext i16 %0 to i32 - store i32 %1, i32 addrspace(1)* %out + %mul = mul i16 %a, %b + %ext = zext i16 %mul to i32 + store i32 %ext, i32 addrspace(1)* %out ret void } -; FUNC-LABEL: {{^}}i8_mul24: +; FUNC-LABEL: {{^}}test_umul24_i8: ; EG: MUL_UINT24 {{[* ]*}}T{{[0-9]}}.[[MUL_CHAN:[XYZW]]] ; The result must be sign-extended ; EG: BFE_INT {{[* ]*}}T{{[0-9]}}.{{[XYZW]}}, PV.[[MUL_CHAN]], 0.0, literal.x ; SI: v_mul_u32_u24_e{{(32|64)}} [[MUL:v[0-9]]], {{[sv][0-9], [sv][0-9]}} ; SI: v_bfe_i32 v{{[0-9]}}, [[MUL]], 0, 8 -define void @i8_mul24(i32 addrspace(1)* %out, i8 %a, i8 %b) { +define void @test_umul24_i8(i32 addrspace(1)* %out, i8 %a, i8 %b) { +entry: + %mul = mul i8 %a, %b + %ext = sext i8 %mul to i32 + store i32 %ext, i32 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}test_umulhi24_i32_i64: +; SI-NOT: and +; SI: v_mul_hi_u32_u24_e32 [[RESULT:v[0-9]+]], +; SI-NEXT: buffer_store_dword [[RESULT]] + +; EG: MULHI_UINT24 {{[* ]*}}T{{[0-9]\.[XYZW]}}, KC0[2].Z, KC0[2].W +define void @test_umulhi24_i32_i64(i32 addrspace(1)* %out, i32 %a, i32 %b) { +entry: + %a.24 = and i32 %a, 16777215 + %b.24 = and i32 %b, 16777215 + %a.24.i64 = zext i32 %a.24 to i64 + %b.24.i64 = zext i32 %b.24 to i64 + %mul48 = mul i64 %a.24.i64, %b.24.i64 + %mul48.hi = lshr i64 %mul48, 32 + %mul24hi = trunc i64 %mul48.hi to i32 + store i32 %mul24hi, i32 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}test_umulhi24: +; SI-NOT: and +; SI: v_mul_hi_u32_u24_e32 [[RESULT:v[0-9]+]], +; SI-NEXT: buffer_store_dword [[RESULT]] + +; EG: MULHI_UINT24 {{[* ]*}}T{{[0-9]\.[XYZW]}}, KC0[2].W, KC0[3].Y +define void @test_umulhi24(i32 addrspace(1)* %out, i64 %a, i64 %b) { entry: - %0 = mul i8 %a, %b - %1 = sext i8 %0 to i32 - store i32 %1, i32 addrspace(1)* %out + %a.24 = and i64 %a, 16777215 + %b.24 = and i64 %b, 16777215 + %mul48 = mul i64 %a.24, %b.24 + %mul48.hi = lshr i64 %mul48, 32 + %mul24.hi = trunc i64 %mul48.hi to i32 + store i32 %mul24.hi, i32 addrspace(1)* %out ret void } ; Multiply with 24-bit inputs and 64-bit output -; FUNC_LABEL: {{^}}mul24_i64: +; FUNC-LABEL: {{^}}test_umul24_i64: ; EG; MUL_UINT24 ; EG: MULHI -; FIXME: SI support 24-bit mulhi -; SI-DAG: v_mul_u32_u24 -; SI-DAG: v_mul_hi_u32 -; SI: s_endpgm -define void @mul24_i64(i64 addrspace(1)* %out, i64 %a, i64 %b, i64 %c) { +; SI-NOT: and +; SI-NOT: lshr + +; SI-DAG: v_mul_u32_u24_e32 +; SI-DAG: v_mul_hi_u32_u24_e32 + +; SI: buffer_store_dwordx2 +define void @test_umul24_i64(i64 addrspace(1)* %out, i64 %a, i64 %b) { entry: %tmp0 = shl i64 %a, 40 %a_24 = lshr i64 %tmp0, 40 @@ -67,3 +118,80 @@ store i64 %tmp2, i64 addrspace(1)* %out ret void } + +; FIXME: Should be able to eliminate the and +; FUNC-LABEL: {{^}}test_umul24_i64_square: +; SI: s_load_dword [[A:s[0-9]+]] +; SI: s_and_b32 [[TRUNC:s[0-9]+]], [[A]], 0xffffff{{$}} +; SI-DAG: v_mul_hi_u32_u24_e64 v{{[0-9]+}}, [[TRUNC]], [[TRUNC]] +; SI-DAG: v_mul_u32_u24_e64 v{{[0-9]+}}, [[TRUNC]], [[TRUNC]] +define void @test_umul24_i64_square(i64 addrspace(1)* %out, i64 %a) { +entry: + %tmp0 = shl i64 %a, 40 + %a.24 = lshr i64 %tmp0, 40 + %tmp2 = mul i64 %a.24, %a.24 + store i64 %tmp2, i64 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}test_umulhi16_i32: +; SI: s_and_b32 +; SI: s_and_b32 +; SI: v_mul_u32_u24_e32 [[MUL24:v[0-9]+]] +; SI: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, [[MUL24]] +define void @test_umulhi16_i32(i16 addrspace(1)* %out, i32 %a, i32 %b) { +entry: + %a.16 = and i32 %a, 65535 + %b.16 = and i32 %b, 65535 + %mul = mul i32 %a.16, %b.16 + %hi = lshr i32 %mul, 16 + %mulhi = trunc i32 %hi to i16 + store i16 %mulhi, i16 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}test_umul24_i33: +; SI: s_load_dword s +; SI: s_load_dword s + +; SI-NOT: and +; SI-NOT: lshr + +; SI-DAG: v_mul_u32_u24_e32 v[[MUL_LO:[0-9]+]], +; SI-DAG: v_mul_hi_u32_u24_e32 v[[MUL_HI:[0-9]+]], +; SI-DAG: v_and_b32_e32 v[[HI:[0-9]+]], 1, v[[MUL_HI]] +; SI: buffer_store_dwordx2 v{{\[}}[[MUL_LO]]:[[HI]]{{\]}} +define void @test_umul24_i33(i64 addrspace(1)* %out, i33 %a, i33 %b) { +entry: + %tmp0 = shl i33 %a, 9 + %a_24 = lshr i33 %tmp0, 9 + %tmp1 = shl i33 %b, 9 + %b_24 = lshr i33 %tmp1, 9 + %tmp2 = mul i33 %a_24, %b_24 + %ext = zext i33 %tmp2 to i64 + store i64 %ext, i64 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}test_umulhi24_i33: +; SI: s_load_dword s +; SI: s_load_dword s + +; SI-NOT: and +; SI-NOT: lshr + +; SI: v_mul_hi_u32_u24_e32 v[[MUL_HI:[0-9]+]], +; SI-NEXT: v_and_b32_e32 v[[HI:[0-9]+]], 1, v[[MUL_HI]] +; SI-NEXT: buffer_store_dword v[[HI]] +define void @test_umulhi24_i33(i32 addrspace(1)* %out, i33 %a, i33 %b) { +entry: + %tmp0 = shl i33 %a, 9 + %a_24 = lshr i33 %tmp0, 9 + %tmp1 = shl i33 %b, 9 + %b_24 = lshr i33 %tmp1, 9 + %tmp2 = mul i33 %a_24, %b_24 + %hi = lshr i33 %tmp2, 32 + %trunc = trunc i33 %hi to i32 + store i32 %trunc, i32 addrspace(1)* %out + ret void +}