[AMDGPU] Enable 48-bit mul in AMDGPUCodeGenPrepare.

[abinavpp]

Summary

We were bailing out of creating 24-bit muls for results wider than 32
bits in AMDGPUCodeGenPrepare. With the 24-bit mulhi intrinsic, this
change teaches AMDGPUCodeGenPrepare to generate the 48-bit mul
correctly.

[abinavpp]

Use the original destination size in getMul24().

[abinavpp]

llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp

532

I think we were incorrect in doing the LHSBits + RHSBits > ... check in D111523. We did not consider the case when an operand has > 24 known bits, but the sum of known bits being in limits. mul24 instruction works only on the low-order 24 bits of its operands.

[abinavpp]

llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp

532

Ignore this comment. I missed the operand width check in the if above.

[abinavpp]

llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp

532

Just realized that this if and the similar one for signed below is not required.

[abinavpp]

Removed the unnecessary ifs.

[foad]

Seems reasonable, just some nits inline.

llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp

487

This would be neater with IsSigned ? ... : ...

494

This would be neater with IsSigned ? ... : ...

502

Can you use CreateIntrinsic to create all the calls?

529

Do all subtargets that have mul_u24 also have mulhi_u24, and the same for i24?

549–550

I prefer not to have a negated condition for an "if" if there's an "else" as well, otherwise the condition for the "else" is a double negative which is harder to understand.

560–561

Same.

[abinavpp]

Addressed review comments.

[abinavpp]

llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp

529

I guess so. We're doing the same thing in AMDGPUISelLowering.cpp. For amdgcn, I can see a corresponding 24-bit mulhi instruction for GFX6 and above. For r600, I can see that in HD6900 and Evergreen. I haven't looked at all the subtargets. The subtarget initialization should take care of this.

[foad]

LGTM.

[arsenm]

Needs some end to end codegen tests that show this actually produces the ISA we want

[abinavpp]

In D112395#3087182, @arsenm wrote:

Needs some end to end codegen tests that show this actually produces the ISA we want

Added some in D112554.

llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp

static Value *insertValues(IRBuilder<> &Builder,
 
   Value *NewVal = UndefValue::get(Ty);
   for (int I = 0, E = Values.size(); I != E; ++I)
     NewVal = Builder.CreateInsertElement(NewVal, Values[I], I);
 
   return NewVal;
 }
 
+// Returns 24-bit or 48-bit (as per `NumBits` and `Size`) mul of `LHS` and
+// `RHS`. `NumBits` is the number of KnownBits of the result and `Size` is the
+// width of the original destination.
+static Value *getMul24(IRBuilder<> &Builder, Value *LHS, Value *RHS,
+                       unsigned Size, unsigned NumBits, bool IsSigned) {
+  if (Size <= 32 || (IsSigned ? NumBits <= 30 : NumBits <= 32)) {
+    Intrinsic::ID ID =
+        IsSigned ? Intrinsic::amdgcn_mul_i24 : Intrinsic::amdgcn_mul_u24;
+    return Builder.CreateIntrinsic(ID, {}, {LHS, RHS});
+  }
+
+  assert(IsSigned ? NumBits <= 46 : NumBits <= 48);
+
+  Intrinsic::ID LoID =
+      IsSigned ? Intrinsic::amdgcn_mul_i24 : Intrinsic::amdgcn_mul_u24;
+  Intrinsic::ID HiID =
+      IsSigned ? Intrinsic::amdgcn_mulhi_i24 : Intrinsic::amdgcn_mulhi_u24;
+
+  Value *Lo = Builder.CreateIntrinsic(LoID, {}, {LHS, RHS});
+  Value *Hi = Builder.CreateIntrinsic(HiID, {}, {LHS, RHS});
+
+  IntegerType *I64Ty = Builder.getInt64Ty();
+  Lo = Builder.CreateZExtOrTrunc(Lo, I64Ty);
+  Hi = Builder.CreateZExtOrTrunc(Hi, I64Ty);
+
+  return Builder.CreateOr(Lo, Builder.CreateShl(Hi, 32));
+}
+
 bool AMDGPUCodeGenPrepare::replaceMulWithMul24(BinaryOperator &I) const {
   if (I.getOpcode() != Instruction::Mul)
     return false;
 
   Type *Ty = I.getType();
   unsigned Size = Ty->getScalarSizeInBits();
   if (Size <= 16 && ST->has16BitInsts())
     return false;
 
   // Prefer scalar if this could be s_mul_i32
   if (DA->isUniform(&I))
     return false;
 
   Value *LHS = I.getOperand(0);
   Value *RHS = I.getOperand(1);
   IRBuilder<> Builder(&I);
   Builder.SetCurrentDebugLocation(I.getDebugLoc());
 
-  Intrinsic::ID IntrID = Intrinsic::not_intrinsic;
-
   unsigned LHSBits = 0, RHSBits = 0;
+  bool IsSigned = false;
 
   if (ST->hasMulU24() && (LHSBits = numBitsUnsigned(LHS, Size)) <= 24 &&
       (RHSBits = numBitsUnsigned(RHS, Size)) <= 24) {
-    // The mul24 instruction yields the low-order 32 bits. If the original
-    // result and the destination is wider than 32 bits, the mul24 would
-    // truncate the result.
-    if (Size > 32 && LHSBits + RHSBits > 32)
-      return false;
+    IsSigned = false;
 
-    IntrID = Intrinsic::amdgcn_mul_u24;
   } else if (ST->hasMulI24() &&
              (LHSBits = numBitsSigned(LHS, Size)) < 24 &&
              (RHSBits = numBitsSigned(RHS, Size)) < 24) {
-    // The original result is positive if its destination is wider than 32 bits
-    // and its highest set bit is at bit 31. Generating mul24 and sign-extending
-    // it would yield a negative value.
-    if (Size > 32 && LHSBits + RHSBits > 30)
-      return false;
+    IsSigned = true;
 
-    IntrID = Intrinsic::amdgcn_mul_i24;
   } else
     return false;
 
   SmallVector<Value *, 4> LHSVals;
   SmallVector<Value *, 4> RHSVals;
   SmallVector<Value *, 4> ResultVals;
   extractValues(Builder, LHSVals, LHS);
   extractValues(Builder, RHSVals, RHS);
 
-
   IntegerType *I32Ty = Builder.getInt32Ty();
-  FunctionCallee Intrin = Intrinsic::getDeclaration(Mod, IntrID);
   for (int I = 0, E = LHSVals.size(); I != E; ++I) {
     Value *LHS, *RHS;
-    if (IntrID == Intrinsic::amdgcn_mul_u24) {
-      LHS = Builder.CreateZExtOrTrunc(LHSVals[I], I32Ty);
-      RHS = Builder.CreateZExtOrTrunc(RHSVals[I], I32Ty);
-    } else {
+    if (IsSigned) {
       LHS = Builder.CreateSExtOrTrunc(LHSVals[I], I32Ty);
       RHS = Builder.CreateSExtOrTrunc(RHSVals[I], I32Ty);
+    } else {
+      LHS = Builder.CreateZExtOrTrunc(LHSVals[I], I32Ty);
+      RHS = Builder.CreateZExtOrTrunc(RHSVals[I], I32Ty);
     }
 
-    Value *Result = Builder.CreateCall(Intrin, {LHS, RHS});
+    Value *Result =
+        getMul24(Builder, LHS, RHS, Size, LHSBits + RHSBits, IsSigned);
 
-    if (IntrID == Intrinsic::amdgcn_mul_u24) {
-      ResultVals.push_back(Builder.CreateZExtOrTrunc(Result,
-                                                     LHSVals[I]->getType()));
+    if (IsSigned) {
+      ResultVals.push_back(
+          Builder.CreateSExtOrTrunc(Result, LHSVals[I]->getType()));
     } else {
-      ResultVals.push_back(Builder.CreateSExtOrTrunc(Result,
-                                                     LHSVals[I]->getType()));
+      ResultVals.push_back(
+          Builder.CreateZExtOrTrunc(Result, LHSVals[I]->getType()));
     }
   }
 
   Value *NewVal = insertValues(Builder, Ty, ResultVals);
   NewVal->takeName(&I);
   I.replaceAllUsesWith(NewVal);
   I.eraseFromParent();
 

llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-mul24.ll

 }
 
 define i64 @smul24_i64(i64 %lhs, i64 %rhs) {
 ; SI-LABEL: @smul24_i64(
 ; SI-NEXT:    [[SHL_LHS:%.*]] = shl i64 [[LHS:%.*]], 40
 ; SI-NEXT:    [[LHS24:%.*]] = ashr i64 [[SHL_LHS]], 40
 ; SI-NEXT:    [[LSHR_RHS:%.*]] = shl i64 [[RHS:%.*]], 40
 ; SI-NEXT:    [[RHS24:%.*]] = ashr i64 [[LHS]], 40
-; SI-NEXT:    [[MUL:%.*]] = mul i64 [[LHS24]], [[RHS24]]
+; SI-NEXT:    [[TMP1:%.*]] = trunc i64 [[LHS24]] to i32
+; SI-NEXT:    [[TMP2:%.*]] = trunc i64 [[RHS24]] to i32
+; SI-NEXT:    [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP1]], i32 [[TMP2]])
+; SI-NEXT:    [[TMP4:%.*]] = call i32 @llvm.amdgcn.mulhi.i24(i32 [[TMP1]], i32 [[TMP2]])
+; SI-NEXT:    [[TMP5:%.*]] = zext i32 [[TMP3]] to i64
+; SI-NEXT:    [[TMP6:%.*]] = zext i32 [[TMP4]] to i64
+; SI-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP6]], 32
+; SI-NEXT:    [[MUL:%.*]] = or i64 [[TMP5]], [[TMP7]]
 ; SI-NEXT:    ret i64 [[MUL]]
 ;
 ; VI-LABEL: @smul24_i64(
 ; VI-NEXT:    [[SHL_LHS:%.*]] = shl i64 [[LHS:%.*]], 40
 ; VI-NEXT:    [[LHS24:%.*]] = ashr i64 [[SHL_LHS]], 40
 ; VI-NEXT:    [[LSHR_RHS:%.*]] = shl i64 [[RHS:%.*]], 40
 ; VI-NEXT:    [[RHS24:%.*]] = ashr i64 [[LHS]], 40
-; VI-NEXT:    [[MUL:%.*]] = mul i64 [[LHS24]], [[RHS24]]
+; VI-NEXT:    [[TMP1:%.*]] = trunc i64 [[LHS24]] to i32
+; VI-NEXT:    [[TMP2:%.*]] = trunc i64 [[RHS24]] to i32
+; VI-NEXT:    [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP1]], i32 [[TMP2]])
+; VI-NEXT:    [[TMP4:%.*]] = call i32 @llvm.amdgcn.mulhi.i24(i32 [[TMP1]], i32 [[TMP2]])
+; VI-NEXT:    [[TMP5:%.*]] = zext i32 [[TMP3]] to i64
+; VI-NEXT:    [[TMP6:%.*]] = zext i32 [[TMP4]] to i64
+; VI-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP6]], 32
+; VI-NEXT:    [[MUL:%.*]] = or i64 [[TMP5]], [[TMP7]]
 ; VI-NEXT:    ret i64 [[MUL]]
 ;
 ; DISABLED-LABEL: @smul24_i64(
 ; DISABLED-NEXT:    [[SHL_LHS:%.*]] = shl i64 [[LHS:%.*]], 40
 ; DISABLED-NEXT:    [[LHS24:%.*]] = ashr i64 [[SHL_LHS]], 40
 ; DISABLED-NEXT:    [[LSHR_RHS:%.*]] = shl i64 [[RHS:%.*]], 40
 ; DISABLED-NEXT:    [[RHS24:%.*]] = ashr i64 [[LHS]], 40
 ; DISABLED-NEXT:    [[MUL:%.*]] = mul i64 [[LHS24]], [[RHS24]]

 }
 
 define i64 @smul24_i64_3(i64 %lhs, i64 %rhs) {
 ; SI-LABEL: @smul24_i64_3(
 ; SI-NEXT:    [[LHS_TRUNC:%.*]] = trunc i64 [[LHS:%.*]] to i16
 ; SI-NEXT:    [[LHS24:%.*]] = sext i16 [[LHS_TRUNC]] to i64
 ; SI-NEXT:    [[RHS_TRUNC:%.*]] = trunc i64 [[RHS:%.*]] to i17
 ; SI-NEXT:    [[RHS24:%.*]] = sext i17 [[RHS_TRUNC]] to i64
-; SI-NEXT:    [[MUL:%.*]] = mul i64 [[LHS24]], [[RHS24]]
+; SI-NEXT:    [[TMP1:%.*]] = trunc i64 [[LHS24]] to i32
+; SI-NEXT:    [[TMP2:%.*]] = trunc i64 [[RHS24]] to i32
+; SI-NEXT:    [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP1]], i32 [[TMP2]])
+; SI-NEXT:    [[TMP4:%.*]] = call i32 @llvm.amdgcn.mulhi.i24(i32 [[TMP1]], i32 [[TMP2]])
+; SI-NEXT:    [[TMP5:%.*]] = zext i32 [[TMP3]] to i64
+; SI-NEXT:    [[TMP6:%.*]] = zext i32 [[TMP4]] to i64
+; SI-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP6]], 32
+; SI-NEXT:    [[MUL:%.*]] = or i64 [[TMP5]], [[TMP7]]
 ; SI-NEXT:    ret i64 [[MUL]]
 ;
 ; VI-LABEL: @smul24_i64_3(
 ; VI-NEXT:    [[LHS_TRUNC:%.*]] = trunc i64 [[LHS:%.*]] to i16
 ; VI-NEXT:    [[LHS24:%.*]] = sext i16 [[LHS_TRUNC]] to i64
 ; VI-NEXT:    [[RHS_TRUNC:%.*]] = trunc i64 [[RHS:%.*]] to i17
 ; VI-NEXT:    [[RHS24:%.*]] = sext i17 [[RHS_TRUNC]] to i64
-; VI-NEXT:    [[MUL:%.*]] = mul i64 [[LHS24]], [[RHS24]]
+; VI-NEXT:    [[TMP1:%.*]] = trunc i64 [[LHS24]] to i32
+; VI-NEXT:    [[TMP2:%.*]] = trunc i64 [[RHS24]] to i32
+; VI-NEXT:    [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP1]], i32 [[TMP2]])
+; VI-NEXT:    [[TMP4:%.*]] = call i32 @llvm.amdgcn.mulhi.i24(i32 [[TMP1]], i32 [[TMP2]])
+; VI-NEXT:    [[TMP5:%.*]] = zext i32 [[TMP3]] to i64
+; VI-NEXT:    [[TMP6:%.*]] = zext i32 [[TMP4]] to i64
+; VI-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP6]], 32
+; VI-NEXT:    [[MUL:%.*]] = or i64 [[TMP5]], [[TMP7]]
 ; VI-NEXT:    ret i64 [[MUL]]
 ;
 ; DISABLED-LABEL: @smul24_i64_3(
 ; DISABLED-NEXT:    [[LHS_TRUNC:%.*]] = trunc i64 [[LHS:%.*]] to i16
 ; DISABLED-NEXT:    [[LHS24:%.*]] = sext i16 [[LHS_TRUNC]] to i64
 ; DISABLED-NEXT:    [[RHS_TRUNC:%.*]] = trunc i64 [[RHS:%.*]] to i17
 ; DISABLED-NEXT:    [[RHS24:%.*]] = sext i17 [[RHS_TRUNC]] to i64
 ; DISABLED-NEXT:    [[MUL:%.*]] = mul i64 [[LHS24]], [[RHS24]]
 ; DISABLED-NEXT:    ret i64 [[MUL]]
 ;
   %lhs.trunc = trunc i64 %lhs to i16
   %lhs24 = sext i16 %lhs.trunc to i64
   %rhs.trunc = trunc i64 %rhs to i17
   %rhs24 = sext i17 %rhs.trunc to i64
   %mul = mul i64 %lhs24, %rhs24
   ret i64 %mul
 }
 
+define i64 @smul24_i64_4(i64 %lhs, i64 %rhs) {
+; SI-LABEL: @smul24_i64_4(
+; SI-NEXT:    [[LHS_TRUNC:%.*]] = trunc i64 [[LHS:%.*]] to i24
+; SI-NEXT:    [[LHS24:%.*]] = sext i24 [[LHS_TRUNC]] to i64
+; SI-NEXT:    [[RHS_TRUNC:%.*]] = trunc i64 [[RHS:%.*]] to i25
+; SI-NEXT:    [[RHS24:%.*]] = sext i25 [[RHS_TRUNC]] to i64
+; SI-NEXT:    [[MUL:%.*]] = mul i64 [[LHS24]], [[RHS24]]
+; SI-NEXT:    ret i64 [[MUL]]
+;
+; VI-LABEL: @smul24_i64_4(
+; VI-NEXT:    [[LHS_TRUNC:%.*]] = trunc i64 [[LHS:%.*]] to i24
+; VI-NEXT:    [[LHS24:%.*]] = sext i24 [[LHS_TRUNC]] to i64
+; VI-NEXT:    [[RHS_TRUNC:%.*]] = trunc i64 [[RHS:%.*]] to i25
+; VI-NEXT:    [[RHS24:%.*]] = sext i25 [[RHS_TRUNC]] to i64
+; VI-NEXT:    [[MUL:%.*]] = mul i64 [[LHS24]], [[RHS24]]
+; VI-NEXT:    ret i64 [[MUL]]
+;
+; DISABLED-LABEL: @smul24_i64_4(
+; DISABLED-NEXT:    [[LHS_TRUNC:%.*]] = trunc i64 [[LHS:%.*]] to i24
+; DISABLED-NEXT:    [[LHS24:%.*]] = sext i24 [[LHS_TRUNC]] to i64
+; DISABLED-NEXT:    [[RHS_TRUNC:%.*]] = trunc i64 [[RHS:%.*]] to i25
+; DISABLED-NEXT:    [[RHS24:%.*]] = sext i25 [[RHS_TRUNC]] to i64
+; DISABLED-NEXT:    [[MUL:%.*]] = mul i64 [[LHS24]], [[RHS24]]
+; DISABLED-NEXT:    ret i64 [[MUL]]
+;
+  %lhs.trunc = trunc i64 %lhs to i24
+  %lhs24 = sext i24 %lhs.trunc to i64
+  %rhs.trunc = trunc i64 %rhs to i25
+  %rhs24 = sext i25 %rhs.trunc to i64
+  %mul = mul i64 %lhs24, %rhs24
+  ret i64 %mul
+}
+
 define i64 @umul24_i64(i64 %lhs, i64 %rhs) {
 ; SI-LABEL: @umul24_i64(
 ; SI-NEXT:    [[LHS24:%.*]] = and i64 [[LHS:%.*]], 16777215
 ; SI-NEXT:    [[RHS24:%.*]] = and i64 [[RHS:%.*]], 16777215
-; SI-NEXT:    [[MUL:%.*]] = mul i64 [[LHS24]], [[RHS24]]
+; SI-NEXT:    [[TMP1:%.*]] = trunc i64 [[LHS24]] to i32
+; SI-NEXT:    [[TMP2:%.*]] = trunc i64 [[RHS24]] to i32
+; SI-NEXT:    [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[TMP1]], i32 [[TMP2]])
+; SI-NEXT:    [[TMP4:%.*]] = call i32 @llvm.amdgcn.mulhi.u24(i32 [[TMP1]], i32 [[TMP2]])
+; SI-NEXT:    [[TMP5:%.*]] = zext i32 [[TMP3]] to i64
+; SI-NEXT:    [[TMP6:%.*]] = zext i32 [[TMP4]] to i64
+; SI-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP6]], 32
+; SI-NEXT:    [[MUL:%.*]] = or i64 [[TMP5]], [[TMP7]]
 ; SI-NEXT:    ret i64 [[MUL]]
 ;
 ; VI-LABEL: @umul24_i64(
 ; VI-NEXT:    [[LHS24:%.*]] = and i64 [[LHS:%.*]], 16777215
 ; VI-NEXT:    [[RHS24:%.*]] = and i64 [[RHS:%.*]], 16777215
-; VI-NEXT:    [[MUL:%.*]] = mul i64 [[LHS24]], [[RHS24]]
+; VI-NEXT:    [[TMP1:%.*]] = trunc i64 [[LHS24]] to i32
+; VI-NEXT:    [[TMP2:%.*]] = trunc i64 [[RHS24]] to i32
+; VI-NEXT:    [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[TMP1]], i32 [[TMP2]])
+; VI-NEXT:    [[TMP4:%.*]] = call i32 @llvm.amdgcn.mulhi.u24(i32 [[TMP1]], i32 [[TMP2]])
+; VI-NEXT:    [[TMP5:%.*]] = zext i32 [[TMP3]] to i64
+; VI-NEXT:    [[TMP6:%.*]] = zext i32 [[TMP4]] to i64
+; VI-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP6]], 32
+; VI-NEXT:    [[MUL:%.*]] = or i64 [[TMP5]], [[TMP7]]
 ; VI-NEXT:    ret i64 [[MUL]]
 ;
 ; DISABLED-LABEL: @umul24_i64(
 ; DISABLED-NEXT:    [[LHS24:%.*]] = and i64 [[LHS:%.*]], 16777215
 ; DISABLED-NEXT:    [[RHS24:%.*]] = and i64 [[RHS:%.*]], 16777215
 ; DISABLED-NEXT:    [[MUL:%.*]] = mul i64 [[LHS24]], [[RHS24]]
 ; DISABLED-NEXT:    ret i64 [[MUL]]
 ;

 }
 
 define i33 @smul24_i33(i33 %lhs, i33 %rhs) {
 ; SI-LABEL: @smul24_i33(
 ; SI-NEXT:    [[SHL_LHS:%.*]] = shl i33 [[LHS:%.*]], 9
 ; SI-NEXT:    [[LHS24:%.*]] = ashr i33 [[SHL_LHS]], 9
 ; SI-NEXT:    [[LSHR_RHS:%.*]] = shl i33 [[RHS:%.*]], 9
 ; SI-NEXT:    [[RHS24:%.*]] = ashr i33 [[LHS]], 9
-; SI-NEXT:    [[MUL:%.*]] = mul i33 [[LHS24]], [[RHS24]]
+; SI-NEXT:    [[TMP1:%.*]] = trunc i33 [[LHS24]] to i32
+; SI-NEXT:    [[TMP2:%.*]] = trunc i33 [[RHS24]] to i32
+; SI-NEXT:    [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP1]], i32 [[TMP2]])
+; SI-NEXT:    [[TMP4:%.*]] = call i32 @llvm.amdgcn.mulhi.i24(i32 [[TMP1]], i32 [[TMP2]])
+; SI-NEXT:    [[TMP5:%.*]] = zext i32 [[TMP3]] to i64
+; SI-NEXT:    [[TMP6:%.*]] = zext i32 [[TMP4]] to i64
+; SI-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP6]], 32
+; SI-NEXT:    [[TMP8:%.*]] = or i64 [[TMP5]], [[TMP7]]
+; SI-NEXT:    [[MUL:%.*]] = trunc i64 [[TMP8]] to i33
 ; SI-NEXT:    ret i33 [[MUL]]
 ;
 ; VI-LABEL: @smul24_i33(
 ; VI-NEXT:    [[SHL_LHS:%.*]] = shl i33 [[LHS:%.*]], 9
 ; VI-NEXT:    [[LHS24:%.*]] = ashr i33 [[SHL_LHS]], 9
 ; VI-NEXT:    [[LSHR_RHS:%.*]] = shl i33 [[RHS:%.*]], 9
 ; VI-NEXT:    [[RHS24:%.*]] = ashr i33 [[LHS]], 9
-; VI-NEXT:    [[MUL:%.*]] = mul i33 [[LHS24]], [[RHS24]]
+; VI-NEXT:    [[TMP1:%.*]] = trunc i33 [[LHS24]] to i32
+; VI-NEXT:    [[TMP2:%.*]] = trunc i33 [[RHS24]] to i32
+; VI-NEXT:    [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP1]], i32 [[TMP2]])
+; VI-NEXT:    [[TMP4:%.*]] = call i32 @llvm.amdgcn.mulhi.i24(i32 [[TMP1]], i32 [[TMP2]])
+; VI-NEXT:    [[TMP5:%.*]] = zext i32 [[TMP3]] to i64
+; VI-NEXT:    [[TMP6:%.*]] = zext i32 [[TMP4]] to i64
+; VI-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP6]], 32
+; VI-NEXT:    [[TMP8:%.*]] = or i64 [[TMP5]], [[TMP7]]
+; VI-NEXT:    [[MUL:%.*]] = trunc i64 [[TMP8]] to i33
 ; VI-NEXT:    ret i33 [[MUL]]
 ;
 ; DISABLED-LABEL: @smul24_i33(
 ; DISABLED-NEXT:    [[SHL_LHS:%.*]] = shl i33 [[LHS:%.*]], 9
 ; DISABLED-NEXT:    [[LHS24:%.*]] = ashr i33 [[SHL_LHS]], 9
 ; DISABLED-NEXT:    [[LSHR_RHS:%.*]] = shl i33 [[RHS:%.*]], 9
 ; DISABLED-NEXT:    [[RHS24:%.*]] = ashr i33 [[LHS]], 9
 ; DISABLED-NEXT:    [[MUL:%.*]] = mul i33 [[LHS24]], [[RHS24]]
 ; DISABLED-NEXT:    ret i33 [[MUL]]
 ;
   %shl.lhs = shl i33 %lhs, 9
   %lhs24 = ashr i33 %shl.lhs, 9
   %lshr.rhs = shl i33 %rhs, 9
   %rhs24 = ashr i33 %lhs, 9
   %mul = mul i33 %lhs24, %rhs24
   ret i33 %mul
 }
 
 define i33 @umul24_i33(i33 %lhs, i33 %rhs) {
 ; SI-LABEL: @umul24_i33(
 ; SI-NEXT:    [[LHS24:%.*]] = and i33 [[LHS:%.*]], 16777215
 ; SI-NEXT:    [[RHS24:%.*]] = and i33 [[RHS:%.*]], 16777215
-; SI-NEXT:    [[MUL:%.*]] = mul i33 [[LHS24]], [[RHS24]]
+; SI-NEXT:    [[TMP1:%.*]] = trunc i33 [[LHS24]] to i32
+; SI-NEXT:    [[TMP2:%.*]] = trunc i33 [[RHS24]] to i32
+; SI-NEXT:    [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[TMP1]], i32 [[TMP2]])
+; SI-NEXT:    [[TMP4:%.*]] = call i32 @llvm.amdgcn.mulhi.u24(i32 [[TMP1]], i32 [[TMP2]])
+; SI-NEXT:    [[TMP5:%.*]] = zext i32 [[TMP3]] to i64
+; SI-NEXT:    [[TMP6:%.*]] = zext i32 [[TMP4]] to i64
+; SI-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP6]], 32
+; SI-NEXT:    [[TMP8:%.*]] = or i64 [[TMP5]], [[TMP7]]
+; SI-NEXT:    [[MUL:%.*]] = trunc i64 [[TMP8]] to i33
 ; SI-NEXT:    ret i33 [[MUL]]
 ;
 ; VI-LABEL: @umul24_i33(
 ; VI-NEXT:    [[LHS24:%.*]] = and i33 [[LHS:%.*]], 16777215
 ; VI-NEXT:    [[RHS24:%.*]] = and i33 [[RHS:%.*]], 16777215
-; VI-NEXT:    [[MUL:%.*]] = mul i33 [[LHS24]], [[RHS24]]
+; VI-NEXT:    [[TMP1:%.*]] = trunc i33 [[LHS24]] to i32
+; VI-NEXT:    [[TMP2:%.*]] = trunc i33 [[RHS24]] to i32
+; VI-NEXT:    [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[TMP1]], i32 [[TMP2]])
+; VI-NEXT:    [[TMP4:%.*]] = call i32 @llvm.amdgcn.mulhi.u24(i32 [[TMP1]], i32 [[TMP2]])
+; VI-NEXT:    [[TMP5:%.*]] = zext i32 [[TMP3]] to i64
+; VI-NEXT:    [[TMP6:%.*]] = zext i32 [[TMP4]] to i64
+; VI-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP6]], 32
+; VI-NEXT:    [[TMP8:%.*]] = or i64 [[TMP5]], [[TMP7]]
+; VI-NEXT:    [[MUL:%.*]] = trunc i64 [[TMP8]] to i33
 ; VI-NEXT:    ret i33 [[MUL]]
 ;
 ; DISABLED-LABEL: @umul24_i33(
 ; DISABLED-NEXT:    [[LHS24:%.*]] = and i33 [[LHS:%.*]], 16777215
 ; DISABLED-NEXT:    [[RHS24:%.*]] = and i33 [[RHS:%.*]], 16777215
 ; DISABLED-NEXT:    [[MUL:%.*]] = mul i33 [[LHS24]], [[RHS24]]
 ; DISABLED-NEXT:    ret i33 [[MUL]]
 ;

 }
 
 define <2 x i33> @smul24_v2i33(<2 x i33> %lhs, <2 x i33> %rhs) {
 ; SI-LABEL: @smul24_v2i33(
 ; SI-NEXT:    [[SHL_LHS:%.*]] = shl <2 x i33> [[LHS:%.*]], <i33 9, i33 9>
 ; SI-NEXT:    [[LHS24:%.*]] = ashr <2 x i33> [[SHL_LHS]], <i33 9, i33 9>
 ; SI-NEXT:    [[LSHR_RHS:%.*]] = shl <2 x i33> [[RHS:%.*]], <i33 9, i33 9>
 ; SI-NEXT:    [[RHS24:%.*]] = ashr <2 x i33> [[LHS]], <i33 9, i33 9>
-; SI-NEXT:    [[MUL:%.*]] = mul <2 x i33> [[LHS24]], [[RHS24]]
+; SI-NEXT:    [[TMP1:%.*]] = extractelement <2 x i33> [[LHS24]], i64 0
+; SI-NEXT:    [[TMP2:%.*]] = extractelement <2 x i33> [[LHS24]], i64 1
+; SI-NEXT:    [[TMP3:%.*]] = extractelement <2 x i33> [[RHS24]], i64 0
+; SI-NEXT:    [[TMP4:%.*]] = extractelement <2 x i33> [[RHS24]], i64 1
+; SI-NEXT:    [[TMP5:%.*]] = trunc i33 [[TMP1]] to i32
+; SI-NEXT:    [[TMP6:%.*]] = trunc i33 [[TMP3]] to i32
+; SI-NEXT:    [[TMP7:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP5]], i32 [[TMP6]])
+; SI-NEXT:    [[TMP8:%.*]] = call i32 @llvm.amdgcn.mulhi.i24(i32 [[TMP5]], i32 [[TMP6]])
+; SI-NEXT:    [[TMP9:%.*]] = zext i32 [[TMP7]] to i64
+; SI-NEXT:    [[TMP10:%.*]] = zext i32 [[TMP8]] to i64
+; SI-NEXT:    [[TMP11:%.*]] = shl i64 [[TMP10]], 32
+; SI-NEXT:    [[TMP12:%.*]] = or i64 [[TMP9]], [[TMP11]]
+; SI-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i33
+; SI-NEXT:    [[TMP14:%.*]] = trunc i33 [[TMP2]] to i32
+; SI-NEXT:    [[TMP15:%.*]] = trunc i33 [[TMP4]] to i32
+; SI-NEXT:    [[TMP16:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP14]], i32 [[TMP15]])
+; SI-NEXT:    [[TMP17:%.*]] = call i32 @llvm.amdgcn.mulhi.i24(i32 [[TMP14]], i32 [[TMP15]])
+; SI-NEXT:    [[TMP18:%.*]] = zext i32 [[TMP16]] to i64
+; SI-NEXT:    [[TMP19:%.*]] = zext i32 [[TMP17]] to i64
+; SI-NEXT:    [[TMP20:%.*]] = shl i64 [[TMP19]], 32
+; SI-NEXT:    [[TMP21:%.*]] = or i64 [[TMP18]], [[TMP20]]
+; SI-NEXT:    [[TMP22:%.*]] = trunc i64 [[TMP21]] to i33
+; SI-NEXT:    [[TMP23:%.*]] = insertelement <2 x i33> undef, i33 [[TMP13]], i64 0
+; SI-NEXT:    [[MUL:%.*]] = insertelement <2 x i33> [[TMP23]], i33 [[TMP22]], i64 1
 ; SI-NEXT:    ret <2 x i33> [[MUL]]
 ;
 ; VI-LABEL: @smul24_v2i33(
 ; VI-NEXT:    [[SHL_LHS:%.*]] = shl <2 x i33> [[LHS:%.*]], <i33 9, i33 9>
 ; VI-NEXT:    [[LHS24:%.*]] = ashr <2 x i33> [[SHL_LHS]], <i33 9, i33 9>
 ; VI-NEXT:    [[LSHR_RHS:%.*]] = shl <2 x i33> [[RHS:%.*]], <i33 9, i33 9>
 ; VI-NEXT:    [[RHS24:%.*]] = ashr <2 x i33> [[LHS]], <i33 9, i33 9>
-; VI-NEXT:    [[MUL:%.*]] = mul <2 x i33> [[LHS24]], [[RHS24]]
+; VI-NEXT:    [[TMP1:%.*]] = extractelement <2 x i33> [[LHS24]], i64 0
+; VI-NEXT:    [[TMP2:%.*]] = extractelement <2 x i33> [[LHS24]], i64 1
+; VI-NEXT:    [[TMP3:%.*]] = extractelement <2 x i33> [[RHS24]], i64 0
+; VI-NEXT:    [[TMP4:%.*]] = extractelement <2 x i33> [[RHS24]], i64 1
+; VI-NEXT:    [[TMP5:%.*]] = trunc i33 [[TMP1]] to i32
+; VI-NEXT:    [[TMP6:%.*]] = trunc i33 [[TMP3]] to i32
+; VI-NEXT:    [[TMP7:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP5]], i32 [[TMP6]])
+; VI-NEXT:    [[TMP8:%.*]] = call i32 @llvm.amdgcn.mulhi.i24(i32 [[TMP5]], i32 [[TMP6]])
+; VI-NEXT:    [[TMP9:%.*]] = zext i32 [[TMP7]] to i64
+; VI-NEXT:    [[TMP10:%.*]] = zext i32 [[TMP8]] to i64
+; VI-NEXT:    [[TMP11:%.*]] = shl i64 [[TMP10]], 32
+; VI-NEXT:    [[TMP12:%.*]] = or i64 [[TMP9]], [[TMP11]]
+; VI-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i33
+; VI-NEXT:    [[TMP14:%.*]] = trunc i33 [[TMP2]] to i32
+; VI-NEXT:    [[TMP15:%.*]] = trunc i33 [[TMP4]] to i32
+; VI-NEXT:    [[TMP16:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP14]], i32 [[TMP15]])
+; VI-NEXT:    [[TMP17:%.*]] = call i32 @llvm.amdgcn.mulhi.i24(i32 [[TMP14]], i32 [[TMP15]])
+; VI-NEXT:    [[TMP18:%.*]] = zext i32 [[TMP16]] to i64
+; VI-NEXT:    [[TMP19:%.*]] = zext i32 [[TMP17]] to i64
+; VI-NEXT:    [[TMP20:%.*]] = shl i64 [[TMP19]], 32
+; VI-NEXT:    [[TMP21:%.*]] = or i64 [[TMP18]], [[TMP20]]
+; VI-NEXT:    [[TMP22:%.*]] = trunc i64 [[TMP21]] to i33
+; VI-NEXT:    [[TMP23:%.*]] = insertelement <2 x i33> undef, i33 [[TMP13]], i64 0
+; VI-NEXT:    [[MUL:%.*]] = insertelement <2 x i33> [[TMP23]], i33 [[TMP22]], i64 1
 ; VI-NEXT:    ret <2 x i33> [[MUL]]
 ;
 ; DISABLED-LABEL: @smul24_v2i33(
 ; DISABLED-NEXT:    [[SHL_LHS:%.*]] = shl <2 x i33> [[LHS:%.*]], <i33 9, i33 9>
 ; DISABLED-NEXT:    [[LHS24:%.*]] = ashr <2 x i33> [[SHL_LHS]], <i33 9, i33 9>
 ; DISABLED-NEXT:    [[LSHR_RHS:%.*]] = shl <2 x i33> [[RHS:%.*]], <i33 9, i33 9>
 ; DISABLED-NEXT:    [[RHS24:%.*]] = ashr <2 x i33> [[LHS]], <i33 9, i33 9>
 ; DISABLED-NEXT:    [[MUL:%.*]] = mul <2 x i33> [[LHS24]], [[RHS24]]