AMDGPU: Select mulhi 24-bit instructions

[• tstellarAMD]

LGTM.

[arsenm]

r279902

lib/Target/AMDGPU/AMDGPUISelLowering.h

protected:
   bool shouldCombineMemoryType(EVT VT) const;
   SDValue performLoadCombine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performStoreCombine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performAndCombine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performShlCombine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performSraCombine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performSrlCombine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performMulCombine(SDNode *N, DAGCombinerInfo &DCI) const;
+  SDValue performMulhsCombine(SDNode *N, DAGCombinerInfo &DCI) const;
+  SDValue performMulhuCombine(SDNode *N, DAGCombinerInfo &DCI) const;
+  SDValue performMulLoHi24Combine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performCtlzCombine(const SDLoc &SL, SDValue Cond, SDValue LHS,
                              SDValue RHS, DAGCombinerInfo &DCI) const;
   SDValue performSelectCombine(SDNode *N, DAGCombinerInfo &DCI) const;
 
   static EVT getEquivalentMemType(LLVMContext &Context, EVT VT);
 
   virtual SDValue LowerGlobalAddress(AMDGPUMachineFunction *MFI, SDValue Op,
                                      SelectionDAG &DAG) const;
enum NodeType : unsigned {
   UMUL,        // 32bit unsigned multiplication
   BRANCH_COND,
   // End AMDIL ISD Opcodes
   ENDPGM,
   RETURN,
   DWORDADDR,
   FRACT,
   CLAMP,
-  // This is SETCC with the full mask result which is used for a compare with a 
+  // This is SETCC with the full mask result which is used for a compare with a
   // result bit per item in the wavefront.
-  SETCC,    
+  SETCC,
 
   // SIN_HW, COS_HW - f32 for SI, 1 ULP max error, valid from -100 pi to 100 pi.
   // Denormals handled on some parts.
   COS_HW,
   SIN_HW,
   FMAX_LEGACY,
   FMIN_LEGACY,
   FMAX3,
enum NodeType : unsigned {
   BFE_U32, // Extract range of bits with zero extension to 32-bits.
   BFE_I32, // Extract range of bits with sign extension to 32-bits.
   BFI, // (src0 & src1) | (~src0 & src2)
   BFM, // Insert a range of bits into a 32-bit word.
   FFBH_U32, // ctlz with -1 if input is zero.
   FFBH_I32,
   MUL_U24,
   MUL_I24,
+  MULHI_U24,
+  MULHI_I24,
   MAD_U24,
   MAD_I24,
+  MUL_LOHI_I24,
+  MUL_LOHI_U24,
   TEXTURE_FETCH,
   EXPORT,
   CONST_ADDRESS,
   REGISTER_LOAD,
   REGISTER_STORE,
   LOAD_INPUT,
   SAMPLE,
   SAMPLEB,

lib/Target/AMDGPU/AMDGPUISelLowering.cpp

AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
   MaxStoresPerMemset  = 4096;
 
   setTargetDAGCombine(ISD::BITCAST);
   setTargetDAGCombine(ISD::AND);
   setTargetDAGCombine(ISD::SHL);
   setTargetDAGCombine(ISD::SRA);
   setTargetDAGCombine(ISD::SRL);
   setTargetDAGCombine(ISD::MUL);
+  setTargetDAGCombine(ISD::MULHU);
+  setTargetDAGCombine(ISD::MULHS);
   setTargetDAGCombine(ISD::SELECT);
   setTargetDAGCombine(ISD::SELECT_CC);
   setTargetDAGCombine(ISD::STORE);
   setTargetDAGCombine(ISD::FADD);
   setTargetDAGCombine(ISD::FSUB);
 }
 
 //===----------------------------------------------------------------------===//
SDValue AMDGPUTargetLowering::performSrlCombine(SDNode *N,
   SDValue NewConst = DAG.getConstant(ShiftAmt - 32, SL, MVT::i32);
   SDValue NewShift = DAG.getNode(ISD::SRL, SL, MVT::i32, Hi, NewConst);
 
   SDValue BuildPair = DAG.getBuildVector(MVT::v2i32, SL, {NewShift, Zero});
 
   return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildPair);
 }
 
+// We need to specifically handle i64 mul here to avoid unnecessary conversion
+// instructions. If we only match on the legalized i64 mul expansion,
+// SimplifyDemandedBits will be unable to remove them because there will be
+// multiple uses due to the separate mul + mulh[su].
+static SDValue getMul24(SelectionDAG &DAG, const SDLoc &SL,
+                        SDValue N0, SDValue N1, unsigned Size, bool Signed) {
+  if (Size <= 32) {
+    unsigned MulOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24;
+    return DAG.getNode(MulOpc, SL, MVT::i32, N0, N1);
+  }
+
+  // Because we want to eliminate extension instructions before the
+  // operation, we need to create a single user here (i.e. not the separate
+  // mul_lo + mul_hi) so that SimplifyDemandedBits will deal with it.
+
+  unsigned MulOpc = Signed ? AMDGPUISD::MUL_LOHI_I24 : AMDGPUISD::MUL_LOHI_U24;
+
+  SDValue Mul = DAG.getNode(MulOpc, SL,
+                            DAG.getVTList(MVT::i32, MVT::i32), N0, N1);
+
+  return DAG.getNode(ISD::BUILD_PAIR, SL, MVT::i64,
+                     Mul.getValue(0), Mul.getValue(1));
+}
+
 SDValue AMDGPUTargetLowering::performMulCombine(SDNode *N,
                                                 DAGCombinerInfo &DCI) const {
   EVT VT = N->getValueType(0);
 
-  if (VT.isVector() || VT.getSizeInBits() > 32)
+  unsigned Size = VT.getSizeInBits();
+  if (VT.isVector() || Size > 64)
     return SDValue();
 
   SelectionDAG &DAG = DCI.DAG;
   SDLoc DL(N);
 
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
   SDValue Mul;
 
   if (Subtarget->hasMulU24() && isU24(N0, DAG) && isU24(N1, DAG)) {
     N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32);
     N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32);
-    Mul = DAG.getNode(AMDGPUISD::MUL_U24, DL, MVT::i32, N0, N1);
+    Mul = getMul24(DAG, DL, N0, N1, Size, false);
   } else if (Subtarget->hasMulI24() && isI24(N0, DAG) && isI24(N1, DAG)) {
     N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32);
     N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32);
-    Mul = DAG.getNode(AMDGPUISD::MUL_I24, DL, MVT::i32, N0, N1);
+    Mul = getMul24(DAG, DL, N0, N1, Size, true);
   } else {
     return SDValue();
   }
 
   // We need to use sext even for MUL_U24, because MUL_U24 is used
   // for signed multiply of 8 and 16-bit types.
   return DAG.getSExtOrTrunc(Mul, DL, VT);
 }
 
+SDValue AMDGPUTargetLowering::performMulhsCombine(SDNode *N,
+                                                  DAGCombinerInfo &DCI) const {
+  EVT VT = N->getValueType(0);
+
+  if (!Subtarget->hasMulI24() || VT.isVector())
+    return SDValue();
+
+  SelectionDAG &DAG = DCI.DAG;
+  SDLoc DL(N);
+
+  SDValue N0 = N->getOperand(0);
+  SDValue N1 = N->getOperand(1);
+
+  if (!isI24(N0, DAG) || !isI24(N1, DAG))
+    return SDValue();
+
+  N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32);
+  N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32);
+
+  SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_I24, DL, MVT::i32, N0, N1);
+  DCI.AddToWorklist(Mulhi.getNode());
+  return DAG.getSExtOrTrunc(Mulhi, DL, VT);
+}
+
+SDValue AMDGPUTargetLowering::performMulhuCombine(SDNode *N,
+                                                  DAGCombinerInfo &DCI) const {
+  EVT VT = N->getValueType(0);
+
+  if (!Subtarget->hasMulU24() || VT.isVector() || VT.getSizeInBits() > 32)
+    return SDValue();
+
+  SelectionDAG &DAG = DCI.DAG;
+  SDLoc DL(N);
+
+  SDValue N0 = N->getOperand(0);
+  SDValue N1 = N->getOperand(1);
+
+  if (!isU24(N0, DAG) || !isU24(N1, DAG))
+    return SDValue();
+
+  N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32);
+  N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32);
+
+  SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_U24, DL, MVT::i32, N0, N1);
+  DCI.AddToWorklist(Mulhi.getNode());
+  return DAG.getZExtOrTrunc(Mulhi, DL, VT);
+}
+
+SDValue AMDGPUTargetLowering::performMulLoHi24Combine(
+  SDNode *N, DAGCombinerInfo &DCI) const {
+  SelectionDAG &DAG = DCI.DAG;
+
+  SDValue N0 = N->getOperand(0);
+  SDValue N1 = N->getOperand(1);
+
+  // Simplify demanded bits before splitting into multiple users.
+  if (simplifyI24(N0, DCI) || simplifyI24(N1, DCI))
+    return SDValue();
+
+  bool Signed = (N->getOpcode() == AMDGPUISD::MUL_LOHI_I24);
+
+  unsigned MulLoOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24;
+  unsigned MulHiOpc = Signed ? AMDGPUISD::MULHI_I24 : AMDGPUISD::MULHI_U24;
+
+  SDLoc SL(N);
+
+  SDValue MulLo = DAG.getNode(MulLoOpc, SL, MVT::i32, N0, N1);
+  SDValue MulHi = DAG.getNode(MulHiOpc, SL, MVT::i32, N0, N1);
+  return DAG.getMergeValues({ MulLo, MulHi }, SL);
+}
+
 static bool isNegativeOne(SDValue Val) {
   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val))
     return C->isAllOnesValue();
   return false;
 }
 
 static bool isCtlzOpc(unsigned Opc) {
   return Opc == ISD::CTLZ || Opc == ISD::CTLZ_ZERO_UNDEF;
SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N,
   case ISD::AND: {
     if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
       break;
 
     return performAndCombine(N, DCI);
   }
   case ISD::MUL:
     return performMulCombine(N, DCI);
+  case ISD::MULHS:
+    return performMulhsCombine(N, DCI);
+  case ISD::MULHU:
+    return performMulhuCombine(N, DCI);
   case AMDGPUISD::MUL_I24:
-  case AMDGPUISD::MUL_U24: {
+  case AMDGPUISD::MUL_U24:
+  case AMDGPUISD::MULHI_I24:
+  case AMDGPUISD::MULHI_U24: {
     SDValue N0 = N->getOperand(0);
     SDValue N1 = N->getOperand(1);
     simplifyI24(N0, DCI);
     simplifyI24(N1, DCI);
     return SDValue();
   }
+  case AMDGPUISD::MUL_LOHI_I24:
+  case AMDGPUISD::MUL_LOHI_U24:
+    return performMulLoHi24Combine(N, DCI);
   case ISD::SELECT:
     return performSelectCombine(N, DCI);
   case AMDGPUISD::BFE_I32:
   case AMDGPUISD::BFE_U32: {
     assert(!N->getValueType(0).isVector() &&
            "Vector handling of BFE not implemented");
     ConstantSDNode *Width = dyn_cast<ConstantSDNode>(N->getOperand(2));
     if (!Width)
const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const {
   NODE_NAME_CASE(BFE_U32)
   NODE_NAME_CASE(BFE_I32)
   NODE_NAME_CASE(BFI)
   NODE_NAME_CASE(BFM)
   NODE_NAME_CASE(FFBH_U32)
   NODE_NAME_CASE(FFBH_I32)
   NODE_NAME_CASE(MUL_U24)
   NODE_NAME_CASE(MUL_I24)
+  NODE_NAME_CASE(MULHI_U24)
+  NODE_NAME_CASE(MULHI_I24)
+  NODE_NAME_CASE(MUL_LOHI_U24)
+  NODE_NAME_CASE(MUL_LOHI_I24)
   NODE_NAME_CASE(MAD_U24)
   NODE_NAME_CASE(MAD_I24)
   NODE_NAME_CASE(TEXTURE_FETCH)
   NODE_NAME_CASE(EXPORT)
   NODE_NAME_CASE(CONST_ADDRESS)
   NODE_NAME_CASE(REGISTER_LOAD)
   NODE_NAME_CASE(REGISTER_STORE)
   NODE_NAME_CASE(LOAD_INPUT)

lib/Target/AMDGPU/AMDGPUInstrInfo.td

 def AMDGPUbfe_u32 : SDNode<"AMDGPUISD::BFE_U32", AMDGPUDTIntTernaryOp>;
 def AMDGPUbfe_i32 : SDNode<"AMDGPUISD::BFE_I32", AMDGPUDTIntTernaryOp>;
 def AMDGPUbfi : SDNode<"AMDGPUISD::BFI", AMDGPUDTIntTernaryOp>;
 def AMDGPUbfm : SDNode<"AMDGPUISD::BFM", SDTIntBinOp>;
 
 def AMDGPUffbh_u32 : SDNode<"AMDGPUISD::FFBH_U32", SDTIntUnaryOp>;
 def AMDGPUffbh_i32 : SDNode<"AMDGPUISD::FFBH_I32", SDTIntUnaryOp>;
 
-// Signed and unsigned 24-bit mulitply.  The highest 8-bits are ignore when
-// performing the mulitply.  The result is a 32-bit value.
+// Signed and unsigned 24-bit multiply. The highest 8-bits are ignore
+// when performing the mulitply. The result is a 32-bit value.
 def AMDGPUmul_u24 : SDNode<"AMDGPUISD::MUL_U24", SDTIntBinOp,
-  [SDNPCommutative]
+  [SDNPCommutative, SDNPAssociative]
 >;
 def AMDGPUmul_i24 : SDNode<"AMDGPUISD::MUL_I24", SDTIntBinOp,
-  [SDNPCommutative]
+  [SDNPCommutative, SDNPAssociative]
+>;
+
+def AMDGPUmulhi_u24 : SDNode<"AMDGPUISD::MULHI_U24", SDTIntBinOp,
+  [SDNPCommutative, SDNPAssociative]
+>;
+def AMDGPUmulhi_i24 : SDNode<"AMDGPUISD::MULHI_I24", SDTIntBinOp,
+  [SDNPCommutative, SDNPAssociative]
 >;
 
 def AMDGPUmad_u24 : SDNode<"AMDGPUISD::MAD_U24", AMDGPUDTIntTernaryOp,
   []
 >;
 def AMDGPUmad_i24 : SDNode<"AMDGPUISD::MAD_I24", AMDGPUDTIntTernaryOp,
   []
 >;

lib/Target/AMDGPU/CaymanInstructions.td

 let isVector = 1 in {
 
 def RECIP_IEEE_cm : RECIP_IEEE_Common<0x86>;
 
 def MULLO_INT_cm : MULLO_INT_Common<0x8F>;
 def MULHI_INT_cm : MULHI_INT_Common<0x90>;
 def MULLO_UINT_cm : MULLO_UINT_Common<0x91>;
 def MULHI_UINT_cm : MULHI_UINT_Common<0x92>;
+def MULHI_INT_cm24 : MULHI_INT24_Common<0x5c>;
+def MULHI_UINT_cm24 : MULHI_UINT24_Common<0xb2>;
+
 def RECIPSQRT_CLAMPED_cm : RECIPSQRT_CLAMPED_Common<0x87>;
 def EXP_IEEE_cm : EXP_IEEE_Common<0x81>;
 def LOG_IEEE_cm : LOG_IEEE_Common<0x83>;
 def RECIP_CLAMPED_cm : RECIP_CLAMPED_Common<0x84>;
 def RECIPSQRT_IEEE_cm : RECIPSQRT_IEEE_Common<0x89>;
 def SIN_cm : SIN_Common<0x8D>;
 def COS_cm : COS_Common<0x8E>;
 } // End isVector = 1

lib/Target/AMDGPU/EvergreenInstructions.td

 
 def RECIP_IEEE_eg : RECIP_IEEE_Common<0x86>;
 defm DIV_eg : DIV_Common<RECIP_IEEE_eg>;
 
 def MULLO_INT_eg : MULLO_INT_Common<0x8F>;
 def MULHI_INT_eg : MULHI_INT_Common<0x90>;
 def MULLO_UINT_eg : MULLO_UINT_Common<0x91>;
 def MULHI_UINT_eg : MULHI_UINT_Common<0x92>;
+def MULHI_UINT24_eg : MULHI_UINT24_Common<0xb2>;
+
 def RECIP_UINT_eg : RECIP_UINT_Common<0x94>;
 def RECIPSQRT_CLAMPED_eg : RECIPSQRT_CLAMPED_Common<0x87>;
 def EXP_IEEE_eg : EXP_IEEE_Common<0x81>;
 def LOG_IEEE_eg : LOG_IEEE_Common<0x83>;
 def RECIP_CLAMPED_eg : RECIP_CLAMPED_Common<0x84>;
 def RECIPSQRT_IEEE_eg : RECIPSQRT_IEEE_Common<0x89>;
 def : RsqPat<RECIPSQRT_IEEE_eg, f32>;
 def SIN_eg : SIN_Common<0x8D>;

lib/Target/AMDGPU/R600Instructions.td

 > {
   let Itinerary = TransALU;
 }
 
 class LSHL_Common <bits<11> inst> : R600_2OP_Helper <inst, "LSHL", shl>;
 class LSHR_Common <bits<11> inst> : R600_2OP_Helper <inst, "LSHR", srl>;
 class ASHR_Common <bits<11> inst> : R600_2OP_Helper <inst, "ASHR", sra>;
 class MULHI_INT_Common <bits<11> inst> : R600_2OP_Helper <
-  inst, "MULHI_INT", mulhs
-> {
+  inst, "MULHI_INT", mulhs> {
   let Itinerary = TransALU;
 }
+
+class MULHI_INT24_Common <bits<11> inst> : R600_2OP_Helper <
+  inst, "MULHI_INT24", AMDGPUmulhi_i24> {
+  let Itinerary = VecALU;
+}
+
 class MULHI_UINT_Common <bits<11> inst> : R600_2OP_Helper <
-  inst, "MULHI", mulhu
-> {
+  inst, "MULHI", mulhu> {
   let Itinerary = TransALU;
 }
+
+class MULHI_UINT24_Common <bits<11> inst> : R600_2OP_Helper <
+  inst, "MULHI_UINT24", AMDGPUmulhi_u24> {
+  let Itinerary = VecALU;
+}
+
 class MULLO_INT_Common <bits<11> inst> : R600_2OP_Helper <
-  inst, "MULLO_INT", mul
-> {
+  inst, "MULLO_INT", mul> {
   let Itinerary = TransALU;
 }
 class MULLO_UINT_Common <bits<11> inst> : R600_2OP <inst, "MULLO_UINT", []> {
   let Itinerary = TransALU;
 }
 
 class RECIP_CLAMPED_Common <bits<11> inst> : R600_1OP <
   inst, "RECIP_CLAMPED", []

lib/Target/AMDGPU/SIInstructions.td

defm V_MUL_F32 : VOP2Inst <vop2<0x8, 0x5>, "v_mul_f32",
   VOP_F32_F32_F32, fmul
 >;
 
 defm V_MUL_I32_I24 : VOP2Inst <vop2<0x9, 0x6>, "v_mul_i32_i24",
   VOP_I32_I32_I32, AMDGPUmul_i24
 >;
 
 defm V_MUL_HI_I32_I24 : VOP2Inst <vop2<0xa,0x7>, "v_mul_hi_i32_i24",
-  VOP_I32_I32_I32
+  VOP_I32_I32_I32, AMDGPUmulhi_i24
 >;
 
 defm V_MUL_U32_U24 : VOP2Inst <vop2<0xb, 0x8>, "v_mul_u32_u24",
   VOP_I32_I32_I32, AMDGPUmul_u24
 >;
 
 defm V_MUL_HI_U32_U24 : VOP2Inst <vop2<0xc,0x9>, "v_mul_hi_u32_u24",
- VOP_I32_I32_I32
+ VOP_I32_I32_I32, AMDGPUmulhi_u24
 >;
 
 defm V_MIN_F32 : VOP2Inst <vop2<0xf, 0xa>, "v_min_f32", VOP_F32_F32_F32,
   fminnum>;
 defm V_MAX_F32 : VOP2Inst <vop2<0x10, 0xb>, "v_max_f32", VOP_F32_F32_F32,
   fmaxnum>;
 defm V_MIN_I32 : VOP2Inst <vop2<0x11, 0xc>, "v_min_i32", VOP_I32_I32_I32>;
 defm V_MAX_I32 : VOP2Inst <vop2<0x12, 0xd>, "v_max_i32", VOP_I32_I32_I32>;

test/CodeGen/AMDGPU/mul_int24.ll

-; RUN: llc < %s -march=amdgcn -verify-machineinstrs | FileCheck %s --check-prefix=SI --check-prefix=FUNC
-; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s --check-prefix=SI --check-prefix=FUNC
-; RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck %s --check-prefix=EG --check-prefix=FUNC
-; RUN: llc < %s -march=r600 -mcpu=cayman | FileCheck %s --check-prefix=CM --check-prefix=FUNC
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=FUNC %s
+; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
+; RUN: llc -march=r600 -mcpu=cayman < %s | FileCheck -check-prefix=CM -check-prefix=FUNC %s
+
+; FUNC-LABEL: {{^}}test_smul24_i32:
+; GCN-NOT: bfe
+; GCN: v_mul_i32_i24
 
-; FUNC-LABEL: {{^}}i32_mul24:
 ; Signed 24-bit multiply is not supported on pre-Cayman GPUs.
 ; EG: MULLO_INT
+
 ; Make sure we are not masking the inputs
 ; CM-NOT: AND
 ; CM: MUL_INT24
-; SI-NOT: and
-; SI: v_mul_i32_i24
-define void @i32_mul24(i32 addrspace(1)* %out, i32 %a, i32 %b) {
+define void @test_smul24_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) #0 {
 entry:
-  %0 = shl i32 %a, 8
-  %a_24 = ashr i32 %0, 8
-  %1 = shl i32 %b, 8
-  %b_24 = ashr i32 %1, 8
-  %2 = mul i32 %a_24, %b_24
-  store i32 %2, i32 addrspace(1)* %out
+  %a.shl = shl i32 %a, 8
+  %a.24 = ashr i32 %a.shl, 8
+  %b.shl = shl i32 %b, 8
+  %b.24 = ashr i32 %b.shl, 8
+  %mul24 = mul i32 %a.24, %b.24
+  store i32 %mul24, i32 addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}test_smulhi24_i64:
+; GCN-NOT: bfe
+; GCN-NOT: ashr
+; GCN: v_mul_hi_i32_i24_e32 [[RESULT:v[0-9]+]],
+; GCN-NEXT: buffer_store_dword [[RESULT]]
+
+; EG: ASHR
+; EG: ASHR
+; EG: MULHI_INT
+
+; CM-NOT: ASHR
+; CM: MULHI_INT24
+; CM: MULHI_INT24
+; CM: MULHI_INT24
+; CM: MULHI_INT24
+define void @test_smulhi24_i64(i32 addrspace(1)* %out, i32 %a, i32 %b) #0 {
+entry:
+  %a.shl = shl i32 %a, 8
+  %a.24 = ashr i32 %a.shl, 8
+  %b.shl = shl i32 %b, 8
+  %b.24 = ashr i32 %b.shl, 8
+  %a.24.i64 = sext i32 %a.24 to i64
+  %b.24.i64 = sext i32 %b.24 to i64
+  %mul48 = mul i64 %a.24.i64, %b.24.i64
+  %mul48.hi = lshr i64 %mul48, 32
+  %mul24hi = trunc i64 %mul48.hi to i32
+  store i32 %mul24hi, i32 addrspace(1)* %out
+  ret void
+}
+
+; This requires handling of the original 64-bit mul node to eliminate
+; unnecessary extension instructions because after legalization they
+; will not be removed by SimplifyDemandedBits because there are
+; multiple uses by the separate mul and mulhi.
+
+; FUNC-LABEL: {{^}}test_smul24_i64:
+; GCN: s_load_dword s
+; GCN: s_load_dword s
+
+; GCN-NOT: bfe
+; GCN-NOT: ashr
+
+; GCN-DAG: v_mul_hi_i32_i24_e32
+; GCN-DAG: v_mul_i32_i24_e32
+
+; GCN: buffer_store_dwordx2
+define void @test_smul24_i64(i64 addrspace(1)* %out, i32 %a, i32 %b) #0 {
+  %shl.i = shl i32 %a, 8
+  %shr.i = ashr i32 %shl.i, 8
+  %conv.i = sext i32 %shr.i to i64
+  %shl1.i = shl i32 %b, 8
+  %shr2.i = ashr i32 %shl1.i, 8
+  %conv3.i = sext i32 %shr2.i to i64
+  %mul.i = mul i64 %conv3.i, %conv.i
+  store i64 %mul.i, i64 addrspace(1)* %out
+  ret void
+}
+
+; FIXME: Should be able to eliminate bfe
+; FUNC-LABEL: {{^}}test_smul24_i64_square:
+; GCN: s_load_dword [[A:s[0-9]+]]
+; GCN: s_bfe_i32 [[SEXT:s[0-9]+]], [[A]], 0x180000{{$}}
+; GCN-DAG: v_mul_hi_i32_i24_e64 v{{[0-9]+}}, [[SEXT]], [[SEXT]]
+; GCN-DAG: v_mul_i32_i24_e64 v{{[0-9]+}}, [[SEXT]], [[SEXT]]
+; GCN: buffer_store_dwordx2
+define void @test_smul24_i64_square(i64 addrspace(1)* %out, i32 %a, i32 %b) #0 {
+  %shl.i = shl i32 %a, 8
+  %shr.i = ashr i32 %shl.i, 8
+  %conv.i = sext i32 %shr.i to i64
+  %mul.i = mul i64 %conv.i, %conv.i
+  store i64 %mul.i, i64 addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}test_smul24_i33:
+; GCN: s_load_dword s
+; GCN: s_load_dword s
+
+; GCN-NOT: and
+; GCN-NOT: lshr
+
+; GCN-DAG: v_mul_i32_i24_e32
+; GCN-DAG: v_mul_hi_i32_i24_e32
+; SI: v_lshl_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, 31
+; SI: v_ashr_i64 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, 31
+
+; VI: v_lshlrev_b64 v{{\[[0-9]+:[0-9]+\]}}, 31, v{{\[[0-9]+:[0-9]+\]}}
+; VI: v_ashrrev_i64 v{{\[[0-9]+:[0-9]+\]}}, 31, v{{\[[0-9]+:[0-9]+\]}}
+
+; GCN: buffer_store_dwordx2
+define void @test_smul24_i33(i64 addrspace(1)* %out, i33 %a, i33 %b) #0 {
+entry:
+  %a.shl = shl i33 %a, 9
+  %a.24 = ashr i33 %a.shl, 9
+  %b.shl = shl i33 %b, 9
+  %b.24 = ashr i33 %b.shl, 9
+  %mul24 = mul i33 %a.24, %b.24
+  %ext = sext i33 %mul24 to i64
+  store i64 %ext, i64 addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}test_smulhi24_i33:
+; SI: s_load_dword s
+; SI: s_load_dword s
+
+; SI-NOT: bfe
+
+; SI: v_mul_hi_i32_i24_e32 v[[MUL_HI:[0-9]+]],
+; SI-NEXT: v_and_b32_e32 v[[HI:[0-9]+]], 1, v[[MUL_HI]]
+; SI-NEXT: buffer_store_dword v[[HI]]
+define void @test_smulhi24_i33(i32 addrspace(1)* %out, i33 %a, i33 %b) {
+entry:
+  %tmp0 = shl i33 %a, 9
+  %a_24 = ashr i33 %tmp0, 9
+  %tmp1 = shl i33 %b, 9
+  %b_24 = ashr i33 %tmp1, 9
+  %tmp2 = mul i33 %a_24, %b_24
+  %hi = lshr i33 %tmp2, 32
+  %trunc = trunc i33 %hi to i32
+
+  store i32 %trunc, i32 addrspace(1)* %out
   ret void
 }
+attributes #0 = { nounwind }

test/CodeGen/AMDGPU/mul_uint24.ll

-; RUN: llc < %s -march=amdgcn -verify-machineinstrs | FileCheck %s --check-prefix=SI --check-prefix=FUNC
-; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s --check-prefix=SI --check-prefix=FUNC
-; RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck %s --check-prefix=EG --check-prefix=FUNC
-; RUN: llc < %s -march=r600 -mcpu=cayman | FileCheck %s --check-prefix=EG --check-prefix=FUNC
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
+; RUN: llc -march=r600 -mcpu=cayman < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
 
-; FUNC-LABEL: {{^}}u32_mul24:
+; FUNC-LABEL: {{^}}test_umul24_i32:
 ; EG: MUL_UINT24 {{[* ]*}}T{{[0-9]\.[XYZW]}}, KC0[2].Z, KC0[2].W
 ; SI: v_mul_u32_u24
-
-define void @u32_mul24(i32 addrspace(1)* %out, i32 %a, i32 %b) {
+define void @test_umul24_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) {
 entry:
   %0 = shl i32 %a, 8
   %a_24 = lshr i32 %0, 8
   %1 = shl i32 %b, 8
   %b_24 = lshr i32 %1, 8
   %2 = mul i32 %a_24, %b_24
   store i32 %2, i32 addrspace(1)* %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}i16_mul24:
+; FUNC-LABEL: {{^}}test_umul24_i16_sext:
 ; EG: MUL_UINT24 {{[* ]*}}T{{[0-9]}}.[[MUL_CHAN:[XYZW]]]
 ; The result must be sign-extended
 ; EG: BFE_INT {{[* ]*}}T{{[0-9]}}.{{[XYZW]}}, PV.[[MUL_CHAN]], 0.0, literal.x
 ; EG: 16
+
 ; SI: v_mul_u32_u24_e{{(32|64)}} [[MUL:v[0-9]]], {{[sv][0-9], [sv][0-9]}}
 ; SI: v_bfe_i32 v{{[0-9]}}, [[MUL]], 0, 16
-define void @i16_mul24(i32 addrspace(1)* %out, i16 %a, i16 %b) {
+define void @test_umul24_i16_sext(i32 addrspace(1)* %out, i16 %a, i16 %b) {
 entry:
-  %0 = mul i16 %a, %b
-  %1 = sext i16 %0 to i32
-  store i32 %1, i32 addrspace(1)* %out
+  %mul = mul i16 %a, %b
+  %ext = sext i16 %mul to i32
+  store i32 %ext, i32 addrspace(1)* %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}i8_mul24:
+; FUNC-LABEL: {{^}}test_umul24_i16:
+; SI: s_and_b32
+; SI: v_mul_u32_u24_e32
+; SI: v_and_b32_e32
+define void @test_umul24_i16(i32 addrspace(1)* %out, i16 %a, i16 %b) {
+entry:
+  %mul = mul i16 %a, %b
+  %ext = zext i16 %mul to i32
+  store i32 %ext, i32 addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}test_umul24_i8:
 ; EG: MUL_UINT24 {{[* ]*}}T{{[0-9]}}.[[MUL_CHAN:[XYZW]]]
 ; The result must be sign-extended
 ; EG: BFE_INT {{[* ]*}}T{{[0-9]}}.{{[XYZW]}}, PV.[[MUL_CHAN]], 0.0, literal.x
 ; SI: v_mul_u32_u24_e{{(32|64)}} [[MUL:v[0-9]]], {{[sv][0-9], [sv][0-9]}}
 ; SI: v_bfe_i32 v{{[0-9]}}, [[MUL]], 0, 8
 
-define void @i8_mul24(i32 addrspace(1)* %out, i8 %a, i8 %b) {
+define void @test_umul24_i8(i32 addrspace(1)* %out, i8 %a, i8 %b) {
+entry:
+  %mul = mul i8 %a, %b
+  %ext = sext i8 %mul to i32
+  store i32 %ext, i32 addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}test_umulhi24_i32_i64:
+; SI-NOT: and
+; SI: v_mul_hi_u32_u24_e32 [[RESULT:v[0-9]+]],
+; SI-NEXT: buffer_store_dword [[RESULT]]
+
+; EG: MULHI_UINT24 {{[* ]*}}T{{[0-9]\.[XYZW]}}, KC0[2].Z, KC0[2].W
+define void @test_umulhi24_i32_i64(i32 addrspace(1)* %out, i32 %a, i32 %b) {
 entry:
-  %0 = mul i8 %a, %b
-  %1 = sext i8 %0 to i32
-  store i32 %1, i32 addrspace(1)* %out
+  %a.24 = and i32 %a, 16777215
+  %b.24 = and i32 %b, 16777215
+  %a.24.i64 = zext i32 %a.24 to i64
+  %b.24.i64 = zext i32 %b.24 to i64
+  %mul48 = mul i64 %a.24.i64, %b.24.i64
+  %mul48.hi = lshr i64 %mul48, 32
+  %mul24hi = trunc i64 %mul48.hi to i32
+  store i32 %mul24hi, i32 addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}test_umulhi24:
+; SI-NOT: and
+; SI: v_mul_hi_u32_u24_e32 [[RESULT:v[0-9]+]],
+; SI-NEXT: buffer_store_dword [[RESULT]]
+
+; EG: MULHI_UINT24 {{[* ]*}}T{{[0-9]\.[XYZW]}}, KC0[2].W, KC0[3].Y
+define void @test_umulhi24(i32 addrspace(1)* %out, i64 %a, i64 %b) {
+entry:
+  %a.24 = and i64 %a, 16777215
+  %b.24 = and i64 %b, 16777215
+  %mul48 = mul i64 %a.24, %b.24
+  %mul48.hi = lshr i64 %mul48, 32
+  %mul24.hi = trunc i64 %mul48.hi to i32
+  store i32 %mul24.hi, i32 addrspace(1)* %out
   ret void
 }
 
 ; Multiply with 24-bit inputs and 64-bit output
-; FUNC_LABEL: {{^}}mul24_i64:
+; FUNC-LABEL: {{^}}test_umul24_i64:
 ; EG; MUL_UINT24
 ; EG: MULHI
-; FIXME: SI support 24-bit mulhi
 
-; SI-DAG: v_mul_u32_u24
-; SI-DAG: v_mul_hi_u32
-; SI: s_endpgm
-define void @mul24_i64(i64 addrspace(1)* %out, i64 %a, i64 %b, i64 %c) {
+; SI-NOT: and
+; SI-NOT: lshr
+
+; SI-DAG: v_mul_u32_u24_e32
+; SI-DAG: v_mul_hi_u32_u24_e32
+
+; SI: buffer_store_dwordx2
+define void @test_umul24_i64(i64 addrspace(1)* %out, i64 %a, i64 %b) {
 entry:
   %tmp0 = shl i64 %a, 40
   %a_24 = lshr i64 %tmp0, 40
   %tmp1 = shl i64 %b, 40
   %b_24 = lshr i64 %tmp1, 40
   %tmp2 = mul i64 %a_24, %b_24
   store i64 %tmp2, i64 addrspace(1)* %out
   ret void
 }
+
+; FIXME: Should be able to eliminate the and
+; FUNC-LABEL: {{^}}test_umul24_i64_square:
+; SI: s_load_dword [[A:s[0-9]+]]
+; SI: s_and_b32 [[TRUNC:s[0-9]+]], [[A]], 0xffffff{{$}}
+; SI-DAG: v_mul_hi_u32_u24_e64 v{{[0-9]+}}, [[TRUNC]], [[TRUNC]]
+; SI-DAG: v_mul_u32_u24_e64 v{{[0-9]+}}, [[TRUNC]], [[TRUNC]]
+define void @test_umul24_i64_square(i64 addrspace(1)* %out, i64 %a) {
+entry:
+  %tmp0 = shl i64 %a, 40
+  %a.24 = lshr i64 %tmp0, 40
+  %tmp2 = mul i64 %a.24, %a.24
+  store i64 %tmp2, i64 addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}test_umulhi16_i32:
+; SI: s_and_b32
+; SI: s_and_b32
+; SI: v_mul_u32_u24_e32 [[MUL24:v[0-9]+]]
+; SI: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, [[MUL24]]
+define void @test_umulhi16_i32(i16 addrspace(1)* %out, i32 %a, i32 %b) {
+entry:
+  %a.16 = and i32 %a, 65535
+  %b.16 = and i32 %b, 65535
+  %mul = mul i32 %a.16, %b.16
+  %hi = lshr i32 %mul, 16
+  %mulhi = trunc i32 %hi to i16
+  store i16 %mulhi, i16 addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}test_umul24_i33:
+; SI: s_load_dword s
+; SI: s_load_dword s
+
+; SI-NOT: and
+; SI-NOT: lshr
+
+; SI-DAG: v_mul_u32_u24_e32 v[[MUL_LO:[0-9]+]],
+; SI-DAG: v_mul_hi_u32_u24_e32 v[[MUL_HI:[0-9]+]],
+; SI-DAG: v_and_b32_e32 v[[HI:[0-9]+]], 1, v[[MUL_HI]]
+; SI: buffer_store_dwordx2 v{{\[}}[[MUL_LO]]:[[HI]]{{\]}}
+define void @test_umul24_i33(i64 addrspace(1)* %out, i33 %a, i33 %b) {
+entry:
+  %tmp0 = shl i33 %a, 9
+  %a_24 = lshr i33 %tmp0, 9
+  %tmp1 = shl i33 %b, 9
+  %b_24 = lshr i33 %tmp1, 9
+  %tmp2 = mul i33 %a_24, %b_24
+  %ext = zext i33 %tmp2 to i64
+  store i64 %ext, i64 addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}test_umulhi24_i33:
+; SI: s_load_dword s
+; SI: s_load_dword s
+
+; SI-NOT: and
+; SI-NOT: lshr
+
+; SI: v_mul_hi_u32_u24_e32 v[[MUL_HI:[0-9]+]],
+; SI-NEXT: v_and_b32_e32 v[[HI:[0-9]+]], 1, v[[MUL_HI]]
+; SI-NEXT: buffer_store_dword v[[HI]]
+define void @test_umulhi24_i33(i32 addrspace(1)* %out, i33 %a, i33 %b) {
+entry:
+  %tmp0 = shl i33 %a, 9
+  %a_24 = lshr i33 %tmp0, 9
+  %tmp1 = shl i33 %b, 9
+  %b_24 = lshr i33 %tmp1, 9
+  %tmp2 = mul i33 %a_24, %b_24
+  %hi = lshr i33 %tmp2, 32
+  %trunc = trunc i33 %hi to i32
+  store i32 %trunc, i32 addrspace(1)* %out
+  ret void
+}