Index: lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp =================================================================== --- lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp +++ lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp @@ -55,6 +55,12 @@ cl::ReallyHidden, cl::init(true)); +static cl::opt UseMul24Intrin( + "amdgpu-codegenprepare-mul24", + cl::desc("mul24 intrinsic"), + cl::ReallyHidden, + cl::init(true)); + class AMDGPUCodeGenPrepare : public FunctionPass, public InstVisitor { const GCNSubtarget *ST = nullptr; @@ -881,7 +887,7 @@ DA->isUniform(&I) && promoteUniformOpToI32(I)) return true; - if (replaceMulWithMul24(I)) + if (UseMul24Intrin && replaceMulWithMul24(I)) return true; bool Changed = false; Index: test/CodeGen/AMDGPU/amdgpu-codegenprepare-mul24.ll =================================================================== --- test/CodeGen/AMDGPU/amdgpu-codegenprepare-mul24.ll +++ test/CodeGen/AMDGPU/amdgpu-codegenprepare-mul24.ll @@ -1,18 +1,23 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt -S -mtriple=amdgcn-- -mcpu=tahiti -amdgpu-codegenprepare %s | FileCheck -check-prefix=SI %s ; RUN: opt -S -mtriple=amdgcn-- -mcpu=fiji -amdgpu-codegenprepare %s | FileCheck -check-prefix=VI %s +; RUN: opt -S -mtriple=amdgcn-- -mcpu=fiji -amdgpu-codegenprepare-mul24=0 -amdgpu-codegenprepare %s | FileCheck -check-prefix=DISABLED %s define i16 @mul_i16(i16 %lhs, i16 %rhs) { ; SI-LABEL: @mul_i16( ; SI-NEXT: [[TMP1:%.*]] = zext i16 [[LHS:%.*]] to i32 ; SI-NEXT: [[TMP2:%.*]] = zext i16 [[RHS:%.*]] to i32 ; SI-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[TMP1]], i32 [[TMP2]]) -; SI-NEXT: [[TMP4:%.*]] = trunc i32 [[TMP3]] to i16 -; SI-NEXT: ret i16 [[TMP4]] +; SI-NEXT: [[MUL:%.*]] = trunc i32 [[TMP3]] to i16 +; SI-NEXT: ret i16 [[MUL]] ; ; VI-LABEL: @mul_i16( ; VI-NEXT: [[MUL:%.*]] = mul i16 [[LHS:%.*]], [[RHS:%.*]] ; VI-NEXT: ret i16 [[MUL]] +; +; DISABLED-LABEL: @mul_i16( +; DISABLED-NEXT: [[MUL:%.*]] = mul i16 [[LHS:%.*]], [[RHS:%.*]] +; DISABLED-NEXT: ret i16 [[MUL]] ; %mul = mul i16 %lhs, %rhs ret i16 %mul @@ -24,16 +29,24 @@ ; SI-NEXT: [[LHS24:%.*]] = ashr i32 [[SHL_LHS]], 8 ; SI-NEXT: [[LSHR_RHS:%.*]] = shl i32 [[RHS:%.*]], 8 ; SI-NEXT: [[RHS24:%.*]] = ashr i32 [[LHS]], 8 -; SI-NEXT: [[TMP1:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[LHS24]], i32 [[RHS24]]) -; SI-NEXT: ret i32 [[TMP1]] +; SI-NEXT: [[MUL:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[LHS24]], i32 [[RHS24]]) +; SI-NEXT: ret i32 [[MUL]] ; ; VI-LABEL: @smul24_i32( ; VI-NEXT: [[SHL_LHS:%.*]] = shl i32 [[LHS:%.*]], 8 ; VI-NEXT: [[LHS24:%.*]] = ashr i32 [[SHL_LHS]], 8 ; VI-NEXT: [[LSHR_RHS:%.*]] = shl i32 [[RHS:%.*]], 8 ; VI-NEXT: [[RHS24:%.*]] = ashr i32 [[LHS]], 8 -; VI-NEXT: [[TMP1:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[LHS24]], i32 [[RHS24]]) -; VI-NEXT: ret i32 [[TMP1]] +; VI-NEXT: [[MUL:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[LHS24]], i32 [[RHS24]]) +; VI-NEXT: ret i32 [[MUL]] +; +; DISABLED-LABEL: @smul24_i32( +; DISABLED-NEXT: [[SHL_LHS:%.*]] = shl i32 [[LHS:%.*]], 8 +; DISABLED-NEXT: [[LHS24:%.*]] = ashr i32 [[SHL_LHS]], 8 +; DISABLED-NEXT: [[LSHR_RHS:%.*]] = shl i32 [[RHS:%.*]], 8 +; DISABLED-NEXT: [[RHS24:%.*]] = ashr i32 [[LHS]], 8 +; DISABLED-NEXT: [[MUL:%.*]] = mul i32 [[LHS24]], [[RHS24]] +; DISABLED-NEXT: ret i32 [[MUL]] ; %shl.lhs = shl i32 %lhs, 8 %lhs24 = ashr i32 %shl.lhs, 8 @@ -56,8 +69,8 @@ ; SI-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP1]], i32 [[TMP3]]) ; SI-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP2]], i32 [[TMP4]]) ; SI-NEXT: [[TMP7:%.*]] = insertelement <2 x i32> undef, i32 [[TMP5]], i64 0 -; SI-NEXT: [[TMP8:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP6]], i64 1 -; SI-NEXT: ret <2 x i32> [[TMP8]] +; SI-NEXT: [[MUL:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP6]], i64 1 +; SI-NEXT: ret <2 x i32> [[MUL]] ; ; VI-LABEL: @smul24_v2i32( ; VI-NEXT: [[SHL_LHS:%.*]] = shl <2 x i32> [[LHS:%.*]], @@ -71,8 +84,16 @@ ; VI-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP1]], i32 [[TMP3]]) ; VI-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP2]], i32 [[TMP4]]) ; VI-NEXT: [[TMP7:%.*]] = insertelement <2 x i32> undef, i32 [[TMP5]], i64 0 -; VI-NEXT: [[TMP8:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP6]], i64 1 -; VI-NEXT: ret <2 x i32> [[TMP8]] +; VI-NEXT: [[MUL:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP6]], i64 1 +; VI-NEXT: ret <2 x i32> [[MUL]] +; +; DISABLED-LABEL: @smul24_v2i32( +; DISABLED-NEXT: [[SHL_LHS:%.*]] = shl <2 x i32> [[LHS:%.*]], +; DISABLED-NEXT: [[LHS24:%.*]] = ashr <2 x i32> [[SHL_LHS]], +; DISABLED-NEXT: [[LSHR_RHS:%.*]] = shl <2 x i32> [[RHS:%.*]], +; DISABLED-NEXT: [[RHS24:%.*]] = ashr <2 x i32> [[LHS]], +; DISABLED-NEXT: [[MUL:%.*]] = mul <2 x i32> [[LHS24]], [[RHS24]] +; DISABLED-NEXT: ret <2 x i32> [[MUL]] ; %shl.lhs = shl <2 x i32> %lhs, %lhs24 = ashr <2 x i32> %shl.lhs, @@ -86,14 +107,20 @@ ; SI-LABEL: @umul24_i32( ; SI-NEXT: [[LHS24:%.*]] = and i32 [[LHS:%.*]], 16777215 ; SI-NEXT: [[RHS24:%.*]] = and i32 [[RHS:%.*]], 16777215 -; SI-NEXT: [[TMP1:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[LHS24]], i32 [[RHS24]]) -; SI-NEXT: ret i32 [[TMP1]] +; SI-NEXT: [[MUL:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[LHS24]], i32 [[RHS24]]) +; SI-NEXT: ret i32 [[MUL]] ; ; VI-LABEL: @umul24_i32( ; VI-NEXT: [[LHS24:%.*]] = and i32 [[LHS:%.*]], 16777215 ; VI-NEXT: [[RHS24:%.*]] = and i32 [[RHS:%.*]], 16777215 -; VI-NEXT: [[TMP1:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[LHS24]], i32 [[RHS24]]) -; VI-NEXT: ret i32 [[TMP1]] +; VI-NEXT: [[MUL:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[LHS24]], i32 [[RHS24]]) +; VI-NEXT: ret i32 [[MUL]] +; +; DISABLED-LABEL: @umul24_i32( +; DISABLED-NEXT: [[LHS24:%.*]] = and i32 [[LHS:%.*]], 16777215 +; DISABLED-NEXT: [[RHS24:%.*]] = and i32 [[RHS:%.*]], 16777215 +; DISABLED-NEXT: [[MUL:%.*]] = mul i32 [[LHS24]], [[RHS24]] +; DISABLED-NEXT: ret i32 [[MUL]] ; %lhs24 = and i32 %lhs, 16777215 %rhs24 = and i32 %rhs, 16777215 @@ -112,8 +139,8 @@ ; SI-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[TMP1]], i32 [[TMP3]]) ; SI-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[TMP2]], i32 [[TMP4]]) ; SI-NEXT: [[TMP7:%.*]] = insertelement <2 x i32> undef, i32 [[TMP5]], i64 0 -; SI-NEXT: [[TMP8:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP6]], i64 1 -; SI-NEXT: ret <2 x i32> [[TMP8]] +; SI-NEXT: [[MUL:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP6]], i64 1 +; SI-NEXT: ret <2 x i32> [[MUL]] ; ; VI-LABEL: @umul24_v2i32( ; VI-NEXT: [[LHS24:%.*]] = and <2 x i32> [[LHS:%.*]], @@ -125,8 +152,14 @@ ; VI-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[TMP1]], i32 [[TMP3]]) ; VI-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[TMP2]], i32 [[TMP4]]) ; VI-NEXT: [[TMP7:%.*]] = insertelement <2 x i32> undef, i32 [[TMP5]], i64 0 -; VI-NEXT: [[TMP8:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP6]], i64 1 -; VI-NEXT: ret <2 x i32> [[TMP8]] +; VI-NEXT: [[MUL:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP6]], i64 1 +; VI-NEXT: ret <2 x i32> [[MUL]] +; +; DISABLED-LABEL: @umul24_v2i32( +; DISABLED-NEXT: [[LHS24:%.*]] = and <2 x i32> [[LHS:%.*]], +; DISABLED-NEXT: [[RHS24:%.*]] = and <2 x i32> [[RHS:%.*]], +; DISABLED-NEXT: [[MUL:%.*]] = mul <2 x i32> [[LHS24]], [[RHS24]] +; DISABLED-NEXT: ret <2 x i32> [[MUL]] ; %lhs24 = and <2 x i32> %lhs, %rhs24 = and <2 x i32> %rhs, @@ -143,8 +176,8 @@ ; SI-NEXT: [[TMP1:%.*]] = trunc i64 [[LHS24]] to i32 ; SI-NEXT: [[TMP2:%.*]] = trunc i64 [[RHS24]] to i32 ; SI-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP1]], i32 [[TMP2]]) -; SI-NEXT: [[TMP4:%.*]] = sext i32 [[TMP3]] to i64 -; SI-NEXT: ret i64 [[TMP4]] +; SI-NEXT: [[MUL:%.*]] = sext i32 [[TMP3]] to i64 +; SI-NEXT: ret i64 [[MUL]] ; ; VI-LABEL: @smul24_i64( ; VI-NEXT: [[SHL_LHS:%.*]] = shl i64 [[LHS:%.*]], 40 @@ -154,8 +187,16 @@ ; VI-NEXT: [[TMP1:%.*]] = trunc i64 [[LHS24]] to i32 ; VI-NEXT: [[TMP2:%.*]] = trunc i64 [[RHS24]] to i32 ; VI-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP1]], i32 [[TMP2]]) -; VI-NEXT: [[TMP4:%.*]] = sext i32 [[TMP3]] to i64 -; VI-NEXT: ret i64 [[TMP4]] +; VI-NEXT: [[MUL:%.*]] = sext i32 [[TMP3]] to i64 +; VI-NEXT: ret i64 [[MUL]] +; +; DISABLED-LABEL: @smul24_i64( +; DISABLED-NEXT: [[SHL_LHS:%.*]] = shl i64 [[LHS:%.*]], 40 +; DISABLED-NEXT: [[LHS24:%.*]] = ashr i64 [[SHL_LHS]], 40 +; DISABLED-NEXT: [[LSHR_RHS:%.*]] = shl i64 [[RHS:%.*]], 40 +; DISABLED-NEXT: [[RHS24:%.*]] = ashr i64 [[LHS]], 40 +; DISABLED-NEXT: [[MUL:%.*]] = mul i64 [[LHS24]], [[RHS24]] +; DISABLED-NEXT: ret i64 [[MUL]] ; %shl.lhs = shl i64 %lhs, 40 %lhs24 = ashr i64 %shl.lhs, 40 @@ -172,8 +213,8 @@ ; SI-NEXT: [[TMP1:%.*]] = trunc i64 [[LHS24]] to i32 ; SI-NEXT: [[TMP2:%.*]] = trunc i64 [[RHS24]] to i32 ; SI-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[TMP1]], i32 [[TMP2]]) -; SI-NEXT: [[TMP4:%.*]] = zext i32 [[TMP3]] to i64 -; SI-NEXT: ret i64 [[TMP4]] +; SI-NEXT: [[MUL:%.*]] = zext i32 [[TMP3]] to i64 +; SI-NEXT: ret i64 [[MUL]] ; ; VI-LABEL: @umul24_i64( ; VI-NEXT: [[LHS24:%.*]] = and i64 [[LHS:%.*]], 16777215 @@ -181,8 +222,14 @@ ; VI-NEXT: [[TMP1:%.*]] = trunc i64 [[LHS24]] to i32 ; VI-NEXT: [[TMP2:%.*]] = trunc i64 [[RHS24]] to i32 ; VI-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[TMP1]], i32 [[TMP2]]) -; VI-NEXT: [[TMP4:%.*]] = zext i32 [[TMP3]] to i64 -; VI-NEXT: ret i64 [[TMP4]] +; VI-NEXT: [[MUL:%.*]] = zext i32 [[TMP3]] to i64 +; VI-NEXT: ret i64 [[MUL]] +; +; DISABLED-LABEL: @umul24_i64( +; DISABLED-NEXT: [[LHS24:%.*]] = and i64 [[LHS:%.*]], 16777215 +; DISABLED-NEXT: [[RHS24:%.*]] = and i64 [[RHS:%.*]], 16777215 +; DISABLED-NEXT: [[MUL:%.*]] = mul i64 [[LHS24]], [[RHS24]] +; DISABLED-NEXT: ret i64 [[MUL]] ; %lhs24 = and i64 %lhs, 16777215 %rhs24 = and i64 %rhs, 16777215 @@ -199,8 +246,8 @@ ; SI-NEXT: [[TMP1:%.*]] = sext i31 [[LHS24]] to i32 ; SI-NEXT: [[TMP2:%.*]] = sext i31 [[RHS24]] to i32 ; SI-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP1]], i32 [[TMP2]]) -; SI-NEXT: [[TMP4:%.*]] = trunc i32 [[TMP3]] to i31 -; SI-NEXT: ret i31 [[TMP4]] +; SI-NEXT: [[MUL:%.*]] = trunc i32 [[TMP3]] to i31 +; SI-NEXT: ret i31 [[MUL]] ; ; VI-LABEL: @smul24_i31( ; VI-NEXT: [[SHL_LHS:%.*]] = shl i31 [[LHS:%.*]], 7 @@ -210,8 +257,16 @@ ; VI-NEXT: [[TMP1:%.*]] = sext i31 [[LHS24]] to i32 ; VI-NEXT: [[TMP2:%.*]] = sext i31 [[RHS24]] to i32 ; VI-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP1]], i32 [[TMP2]]) -; VI-NEXT: [[TMP4:%.*]] = trunc i32 [[TMP3]] to i31 -; VI-NEXT: ret i31 [[TMP4]] +; VI-NEXT: [[MUL:%.*]] = trunc i32 [[TMP3]] to i31 +; VI-NEXT: ret i31 [[MUL]] +; +; DISABLED-LABEL: @smul24_i31( +; DISABLED-NEXT: [[SHL_LHS:%.*]] = shl i31 [[LHS:%.*]], 7 +; DISABLED-NEXT: [[LHS24:%.*]] = ashr i31 [[SHL_LHS]], 7 +; DISABLED-NEXT: [[LSHR_RHS:%.*]] = shl i31 [[RHS:%.*]], 7 +; DISABLED-NEXT: [[RHS24:%.*]] = ashr i31 [[LHS]], 7 +; DISABLED-NEXT: [[MUL:%.*]] = mul i31 [[LHS24]], [[RHS24]] +; DISABLED-NEXT: ret i31 [[MUL]] ; %shl.lhs = shl i31 %lhs, 7 %lhs24 = ashr i31 %shl.lhs, 7 @@ -228,8 +283,8 @@ ; SI-NEXT: [[TMP1:%.*]] = zext i31 [[LHS24]] to i32 ; SI-NEXT: [[TMP2:%.*]] = zext i31 [[RHS24]] to i32 ; SI-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[TMP1]], i32 [[TMP2]]) -; SI-NEXT: [[TMP4:%.*]] = trunc i32 [[TMP3]] to i31 -; SI-NEXT: ret i31 [[TMP4]] +; SI-NEXT: [[MUL:%.*]] = trunc i32 [[TMP3]] to i31 +; SI-NEXT: ret i31 [[MUL]] ; ; VI-LABEL: @umul24_i31( ; VI-NEXT: [[LHS24:%.*]] = and i31 [[LHS:%.*]], 16777215 @@ -237,8 +292,14 @@ ; VI-NEXT: [[TMP1:%.*]] = zext i31 [[LHS24]] to i32 ; VI-NEXT: [[TMP2:%.*]] = zext i31 [[RHS24]] to i32 ; VI-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[TMP1]], i32 [[TMP2]]) -; VI-NEXT: [[TMP4:%.*]] = trunc i32 [[TMP3]] to i31 -; VI-NEXT: ret i31 [[TMP4]] +; VI-NEXT: [[MUL:%.*]] = trunc i32 [[TMP3]] to i31 +; VI-NEXT: ret i31 [[MUL]] +; +; DISABLED-LABEL: @umul24_i31( +; DISABLED-NEXT: [[LHS24:%.*]] = and i31 [[LHS:%.*]], 16777215 +; DISABLED-NEXT: [[RHS24:%.*]] = and i31 [[RHS:%.*]], 16777215 +; DISABLED-NEXT: [[MUL:%.*]] = mul i31 [[LHS24]], [[RHS24]] +; DISABLED-NEXT: ret i31 [[MUL]] ; %lhs24 = and i31 %lhs, 16777215 %rhs24 = and i31 %rhs, 16777215 @@ -263,8 +324,8 @@ ; SI-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[TMP9]], i32 [[TMP10]]) ; SI-NEXT: [[TMP12:%.*]] = trunc i32 [[TMP11]] to i31 ; SI-NEXT: [[TMP13:%.*]] = insertelement <2 x i31> undef, i31 [[TMP8]], i64 0 -; SI-NEXT: [[TMP14:%.*]] = insertelement <2 x i31> [[TMP13]], i31 [[TMP12]], i64 1 -; SI-NEXT: ret <2 x i31> [[TMP14]] +; SI-NEXT: [[MUL:%.*]] = insertelement <2 x i31> [[TMP13]], i31 [[TMP12]], i64 1 +; SI-NEXT: ret <2 x i31> [[MUL]] ; ; VI-LABEL: @umul24_v2i31( ; VI-NEXT: [[LHS24:%.*]] = and <2 x i31> [[LHS:%.*]], @@ -282,8 +343,14 @@ ; VI-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[TMP9]], i32 [[TMP10]]) ; VI-NEXT: [[TMP12:%.*]] = trunc i32 [[TMP11]] to i31 ; VI-NEXT: [[TMP13:%.*]] = insertelement <2 x i31> undef, i31 [[TMP8]], i64 0 -; VI-NEXT: [[TMP14:%.*]] = insertelement <2 x i31> [[TMP13]], i31 [[TMP12]], i64 1 -; VI-NEXT: ret <2 x i31> [[TMP14]] +; VI-NEXT: [[MUL:%.*]] = insertelement <2 x i31> [[TMP13]], i31 [[TMP12]], i64 1 +; VI-NEXT: ret <2 x i31> [[MUL]] +; +; DISABLED-LABEL: @umul24_v2i31( +; DISABLED-NEXT: [[LHS24:%.*]] = and <2 x i31> [[LHS:%.*]], +; DISABLED-NEXT: [[RHS24:%.*]] = and <2 x i31> [[RHS:%.*]], +; DISABLED-NEXT: [[MUL:%.*]] = mul <2 x i31> [[LHS24]], [[RHS24]] +; DISABLED-NEXT: ret <2 x i31> [[MUL]] ; %lhs24 = and <2 x i31> %lhs, %rhs24 = and <2 x i31> %rhs, @@ -310,8 +377,8 @@ ; SI-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP9]], i32 [[TMP10]]) ; SI-NEXT: [[TMP12:%.*]] = trunc i32 [[TMP11]] to i31 ; SI-NEXT: [[TMP13:%.*]] = insertelement <2 x i31> undef, i31 [[TMP8]], i64 0 -; SI-NEXT: [[TMP14:%.*]] = insertelement <2 x i31> [[TMP13]], i31 [[TMP12]], i64 1 -; SI-NEXT: ret <2 x i31> [[TMP14]] +; SI-NEXT: [[MUL:%.*]] = insertelement <2 x i31> [[TMP13]], i31 [[TMP12]], i64 1 +; SI-NEXT: ret <2 x i31> [[MUL]] ; ; VI-LABEL: @smul24_v2i31( ; VI-NEXT: [[SHL_LHS:%.*]] = shl <2 x i31> [[LHS:%.*]], @@ -331,8 +398,16 @@ ; VI-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP9]], i32 [[TMP10]]) ; VI-NEXT: [[TMP12:%.*]] = trunc i32 [[TMP11]] to i31 ; VI-NEXT: [[TMP13:%.*]] = insertelement <2 x i31> undef, i31 [[TMP8]], i64 0 -; VI-NEXT: [[TMP14:%.*]] = insertelement <2 x i31> [[TMP13]], i31 [[TMP12]], i64 1 -; VI-NEXT: ret <2 x i31> [[TMP14]] +; VI-NEXT: [[MUL:%.*]] = insertelement <2 x i31> [[TMP13]], i31 [[TMP12]], i64 1 +; VI-NEXT: ret <2 x i31> [[MUL]] +; +; DISABLED-LABEL: @smul24_v2i31( +; DISABLED-NEXT: [[SHL_LHS:%.*]] = shl <2 x i31> [[LHS:%.*]], +; DISABLED-NEXT: [[LHS24:%.*]] = ashr <2 x i31> [[SHL_LHS]], +; DISABLED-NEXT: [[LSHR_RHS:%.*]] = shl <2 x i31> [[RHS:%.*]], +; DISABLED-NEXT: [[RHS24:%.*]] = ashr <2 x i31> [[LHS]], +; DISABLED-NEXT: [[MUL:%.*]] = mul <2 x i31> [[LHS24]], [[RHS24]] +; DISABLED-NEXT: ret <2 x i31> [[MUL]] ; %shl.lhs = shl <2 x i31> %lhs, %lhs24 = ashr <2 x i31> %shl.lhs, @@ -351,8 +426,8 @@ ; SI-NEXT: [[TMP1:%.*]] = trunc i33 [[LHS24]] to i32 ; SI-NEXT: [[TMP2:%.*]] = trunc i33 [[RHS24]] to i32 ; SI-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP1]], i32 [[TMP2]]) -; SI-NEXT: [[TMP4:%.*]] = sext i32 [[TMP3]] to i33 -; SI-NEXT: ret i33 [[TMP4]] +; SI-NEXT: [[MUL:%.*]] = sext i32 [[TMP3]] to i33 +; SI-NEXT: ret i33 [[MUL]] ; ; VI-LABEL: @smul24_i33( ; VI-NEXT: [[SHL_LHS:%.*]] = shl i33 [[LHS:%.*]], 9 @@ -362,8 +437,16 @@ ; VI-NEXT: [[TMP1:%.*]] = trunc i33 [[LHS24]] to i32 ; VI-NEXT: [[TMP2:%.*]] = trunc i33 [[RHS24]] to i32 ; VI-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP1]], i32 [[TMP2]]) -; VI-NEXT: [[TMP4:%.*]] = sext i32 [[TMP3]] to i33 -; VI-NEXT: ret i33 [[TMP4]] +; VI-NEXT: [[MUL:%.*]] = sext i32 [[TMP3]] to i33 +; VI-NEXT: ret i33 [[MUL]] +; +; DISABLED-LABEL: @smul24_i33( +; DISABLED-NEXT: [[SHL_LHS:%.*]] = shl i33 [[LHS:%.*]], 9 +; DISABLED-NEXT: [[LHS24:%.*]] = ashr i33 [[SHL_LHS]], 9 +; DISABLED-NEXT: [[LSHR_RHS:%.*]] = shl i33 [[RHS:%.*]], 9 +; DISABLED-NEXT: [[RHS24:%.*]] = ashr i33 [[LHS]], 9 +; DISABLED-NEXT: [[MUL:%.*]] = mul i33 [[LHS24]], [[RHS24]] +; DISABLED-NEXT: ret i33 [[MUL]] ; %shl.lhs = shl i33 %lhs, 9 %lhs24 = ashr i33 %shl.lhs, 9 @@ -380,8 +463,8 @@ ; SI-NEXT: [[TMP1:%.*]] = trunc i33 [[LHS24]] to i32 ; SI-NEXT: [[TMP2:%.*]] = trunc i33 [[RHS24]] to i32 ; SI-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[TMP1]], i32 [[TMP2]]) -; SI-NEXT: [[TMP4:%.*]] = zext i32 [[TMP3]] to i33 -; SI-NEXT: ret i33 [[TMP4]] +; SI-NEXT: [[MUL:%.*]] = zext i32 [[TMP3]] to i33 +; SI-NEXT: ret i33 [[MUL]] ; ; VI-LABEL: @umul24_i33( ; VI-NEXT: [[LHS24:%.*]] = and i33 [[LHS:%.*]], 16777215 @@ -389,8 +472,14 @@ ; VI-NEXT: [[TMP1:%.*]] = trunc i33 [[LHS24]] to i32 ; VI-NEXT: [[TMP2:%.*]] = trunc i33 [[RHS24]] to i32 ; VI-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[TMP1]], i32 [[TMP2]]) -; VI-NEXT: [[TMP4:%.*]] = zext i32 [[TMP3]] to i33 -; VI-NEXT: ret i33 [[TMP4]] +; VI-NEXT: [[MUL:%.*]] = zext i32 [[TMP3]] to i33 +; VI-NEXT: ret i33 [[MUL]] +; +; DISABLED-LABEL: @umul24_i33( +; DISABLED-NEXT: [[LHS24:%.*]] = and i33 [[LHS:%.*]], 16777215 +; DISABLED-NEXT: [[RHS24:%.*]] = and i33 [[RHS:%.*]], 16777215 +; DISABLED-NEXT: [[MUL:%.*]] = mul i33 [[LHS24]], [[RHS24]] +; DISABLED-NEXT: ret i33 [[MUL]] ; %lhs24 = and i33 %lhs, 16777215 %rhs24 = and i33 %rhs, 16777215 @@ -414,6 +503,14 @@ ; VI-NEXT: [[RHS24:%.*]] = ashr i32 [[LHS]], 7 ; VI-NEXT: [[MUL:%.*]] = mul i32 [[LHS24]], [[RHS24]] ; VI-NEXT: ret i32 [[MUL]] +; +; DISABLED-LABEL: @smul25_i32( +; DISABLED-NEXT: [[SHL_LHS:%.*]] = shl i32 [[LHS:%.*]], 7 +; DISABLED-NEXT: [[LHS24:%.*]] = ashr i32 [[SHL_LHS]], 7 +; DISABLED-NEXT: [[LSHR_RHS:%.*]] = shl i32 [[RHS:%.*]], 7 +; DISABLED-NEXT: [[RHS24:%.*]] = ashr i32 [[LHS]], 7 +; DISABLED-NEXT: [[MUL:%.*]] = mul i32 [[LHS24]], [[RHS24]] +; DISABLED-NEXT: ret i32 [[MUL]] ; %shl.lhs = shl i32 %lhs, 7 %lhs24 = ashr i32 %shl.lhs, 7 @@ -435,6 +532,12 @@ ; VI-NEXT: [[RHS24:%.*]] = and i32 [[RHS:%.*]], 33554431 ; VI-NEXT: [[MUL:%.*]] = mul i32 [[LHS24]], [[RHS24]] ; VI-NEXT: ret i32 [[MUL]] +; +; DISABLED-LABEL: @umul25_i32( +; DISABLED-NEXT: [[LHS24:%.*]] = and i32 [[LHS:%.*]], 33554431 +; DISABLED-NEXT: [[RHS24:%.*]] = and i32 [[RHS:%.*]], 33554431 +; DISABLED-NEXT: [[MUL:%.*]] = mul i32 [[LHS24]], [[RHS24]] +; DISABLED-NEXT: ret i32 [[MUL]] ; %lhs24 = and i32 %lhs, 33554431 %rhs24 = and i32 %rhs, 33554431 @@ -461,8 +564,8 @@ ; SI-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP9]], i32 [[TMP10]]) ; SI-NEXT: [[TMP12:%.*]] = sext i32 [[TMP11]] to i33 ; SI-NEXT: [[TMP13:%.*]] = insertelement <2 x i33> undef, i33 [[TMP8]], i64 0 -; SI-NEXT: [[TMP14:%.*]] = insertelement <2 x i33> [[TMP13]], i33 [[TMP12]], i64 1 -; SI-NEXT: ret <2 x i33> [[TMP14]] +; SI-NEXT: [[MUL:%.*]] = insertelement <2 x i33> [[TMP13]], i33 [[TMP12]], i64 1 +; SI-NEXT: ret <2 x i33> [[MUL]] ; ; VI-LABEL: @smul24_v2i33( ; VI-NEXT: [[SHL_LHS:%.*]] = shl <2 x i33> [[LHS:%.*]], @@ -482,8 +585,16 @@ ; VI-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.mul.i24(i32 [[TMP9]], i32 [[TMP10]]) ; VI-NEXT: [[TMP12:%.*]] = sext i32 [[TMP11]] to i33 ; VI-NEXT: [[TMP13:%.*]] = insertelement <2 x i33> undef, i33 [[TMP8]], i64 0 -; VI-NEXT: [[TMP14:%.*]] = insertelement <2 x i33> [[TMP13]], i33 [[TMP12]], i64 1 -; VI-NEXT: ret <2 x i33> [[TMP14]] +; VI-NEXT: [[MUL:%.*]] = insertelement <2 x i33> [[TMP13]], i33 [[TMP12]], i64 1 +; VI-NEXT: ret <2 x i33> [[MUL]] +; +; DISABLED-LABEL: @smul24_v2i33( +; DISABLED-NEXT: [[SHL_LHS:%.*]] = shl <2 x i33> [[LHS:%.*]], +; DISABLED-NEXT: [[LHS24:%.*]] = ashr <2 x i33> [[SHL_LHS]], +; DISABLED-NEXT: [[LSHR_RHS:%.*]] = shl <2 x i33> [[RHS:%.*]], +; DISABLED-NEXT: [[RHS24:%.*]] = ashr <2 x i33> [[LHS]], +; DISABLED-NEXT: [[MUL:%.*]] = mul <2 x i33> [[LHS24]], [[RHS24]] +; DISABLED-NEXT: ret <2 x i33> [[MUL]] ; %shl.lhs = shl <2 x i33> %lhs, %lhs24 = ashr <2 x i33> %shl.lhs,