diff --git a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp --- a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp @@ -188,6 +188,15 @@ static Value *takeLog2(IRBuilderBase &Builder, Value *Op, unsigned Depth, bool DoFold); +static std::optional shouldFoldOpIntoSelect(BinaryOperator &I, Value *Op, + Value *OpOther) { + if (match(Op, m_Select(m_Value(), m_Value(), m_Value()))) + return match(OpOther, m_ImmConstant()) && + match(Op, m_Select(m_Value(), m_ImmConstant(), m_ImmConstant())); + + return std::nullopt; +} + Instruction *InstCombinerImpl::visitMul(BinaryOperator &I) { Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); if (Value *V = @@ -488,6 +497,14 @@ return replaceInstUsesWith(I, Builder.CreateShl(Op0, Res, I.getName())); } + if (auto MultiUse = shouldFoldOpIntoSelect(I, Op0, Op1)) + if (Instruction *R = FoldOpIntoSelect(I, cast(Op0), *MultiUse)) + return R; + + if (auto MultiUse = shouldFoldOpIntoSelect(I, Op1, Op0)) + if (Instruction *R = FoldOpIntoSelect(I, cast(Op1), *MultiUse)) + return R; + bool Changed = false; if (!HasNSW && willNotOverflowSignedMul(Op0, Op1, I)) { Changed = true; @@ -972,12 +989,13 @@ // If the divisor is a select-of-constants, try to constant fold all div ops: // C / (select Cond, TrueC, FalseC) --> select Cond, (C / TrueC), (C / FalseC) // TODO: Adapt simplifyDivRemOfSelectWithZeroOp to allow this and other folds. - if (match(Op0, m_ImmConstant()) && - match(Op1, m_Select(m_Value(), m_ImmConstant(), m_ImmConstant()))) { - if (Instruction *R = FoldOpIntoSelect(I, cast(Op1), - /*FoldWithMultiUse*/ true)) + if (auto MultiUse = shouldFoldOpIntoSelect(I, Op0, Op1)) + if (Instruction *R = FoldOpIntoSelect(I, cast(Op0), *MultiUse)) + return R; + + if (auto MultiUse = shouldFoldOpIntoSelect(I, Op1, Op0)) + if (Instruction *R = FoldOpIntoSelect(I, cast(Op1), *MultiUse)) return R; - } const APInt *C2; if (match(Op1, m_APInt(C2))) { @@ -1793,12 +1811,13 @@ // If the divisor is a select-of-constants, try to constant fold all rem ops: // C % (select Cond, TrueC, FalseC) --> select Cond, (C % TrueC), (C % FalseC) // TODO: Adapt simplifyDivRemOfSelectWithZeroOp to allow this and other folds. - if (match(Op0, m_ImmConstant()) && - match(Op1, m_Select(m_Value(), m_ImmConstant(), m_ImmConstant()))) { - if (Instruction *R = FoldOpIntoSelect(I, cast(Op1), - /*FoldWithMultiUse*/ true)) + if (auto MultiUse = shouldFoldOpIntoSelect(I, Op0, Op1)) + if (Instruction *R = FoldOpIntoSelect(I, cast(Op0), *MultiUse)) + return R; + + if (auto MultiUse = shouldFoldOpIntoSelect(I, Op1, Op0)) + if (Instruction *R = FoldOpIntoSelect(I, cast(Op1), *MultiUse)) return R; - } if (isa(Op1)) { if (Instruction *Op0I = dyn_cast(Op0)) { diff --git a/llvm/test/Transforms/InstCombine/binop-select.ll b/llvm/test/Transforms/InstCombine/binop-select.ll --- a/llvm/test/Transforms/InstCombine/binop-select.ll +++ b/llvm/test/Transforms/InstCombine/binop-select.ll @@ -96,8 +96,8 @@ define i32 @test8(i1 %c, i32 %x, i32 %y) { ; CHECK-LABEL: @test8( -; CHECK-NEXT: [[COND:%.*]] = select i1 [[C:%.*]], i32 7, i32 [[Y:%.*]] -; CHECK-NEXT: [[DIV:%.*]] = sdiv i32 42, [[COND]] +; CHECK-NEXT: [[Y_OP:%.*]] = sdiv i32 42, [[Y:%.*]] +; CHECK-NEXT: [[DIV:%.*]] = select i1 [[C:%.*]], i32 6, i32 [[Y_OP]] ; CHECK-NEXT: ret i32 [[DIV]] ; %cond = select i1 %c, i32 7, i32 %y @@ -118,8 +118,8 @@ define i32 @test10(i1 %c, i32 %x, i32 %y) { ; CHECK-LABEL: @test10( -; CHECK-NEXT: [[COND:%.*]] = select i1 [[C:%.*]], i32 7, i32 [[Y:%.*]] -; CHECK-NEXT: [[DIV:%.*]] = udiv i32 42, [[COND]] +; CHECK-NEXT: [[Y_OP:%.*]] = udiv i32 42, [[Y:%.*]] +; CHECK-NEXT: [[DIV:%.*]] = select i1 [[C:%.*]], i32 6, i32 [[Y_OP]] ; CHECK-NEXT: ret i32 [[DIV]] ; %cond = select i1 %c, i32 7, i32 %y @@ -129,8 +129,8 @@ define i32 @test11(i1 %c, i32 %x, i32 %y) { ; CHECK-LABEL: @test11( -; CHECK-NEXT: [[COND:%.*]] = select i1 [[C:%.*]], i32 7, i32 [[Y:%.*]] -; CHECK-NEXT: [[DIV:%.*]] = srem i32 42, [[COND]] +; CHECK-NEXT: [[Y_OP:%.*]] = srem i32 42, [[Y:%.*]] +; CHECK-NEXT: [[DIV:%.*]] = select i1 [[C:%.*]], i32 0, i32 [[Y_OP]] ; CHECK-NEXT: ret i32 [[DIV]] ; %cond = select i1 %c, i32 7, i32 %y @@ -140,8 +140,8 @@ define i32 @test12(i1 %c, i32 %x, i32 %y) { ; CHECK-LABEL: @test12( -; CHECK-NEXT: [[COND:%.*]] = select i1 [[C:%.*]], i32 7, i32 [[Y:%.*]] -; CHECK-NEXT: [[DIV:%.*]] = urem i32 42, [[COND]] +; CHECK-NEXT: [[Y_OP:%.*]] = urem i32 42, [[Y:%.*]] +; CHECK-NEXT: [[DIV:%.*]] = select i1 [[C:%.*]], i32 0, i32 [[Y_OP]] ; CHECK-NEXT: ret i32 [[DIV]] ; %cond = select i1 %c, i32 7, i32 %y @@ -324,9 +324,8 @@ define <2 x i32> @test_udiv_to_const_shr(i1 %c, <2 x i32> %x, <2 x i32> %yy) { ; CHECK-LABEL: @test_udiv_to_const_shr( -; CHECK-NEXT: [[Y:%.*]] = shl nuw <2 x i32> , [[YY:%.*]] -; CHECK-NEXT: [[COND:%.*]] = select i1 [[C:%.*]], <2 x i32> , <2 x i32> [[Y]] -; CHECK-NEXT: [[DIV:%.*]] = udiv <2 x i32> , [[COND]] +; CHECK-NEXT: [[Y_OP1:%.*]] = lshr <2 x i32> , [[YY:%.*]] +; CHECK-NEXT: [[DIV:%.*]] = select i1 [[C:%.*]], <2 x i32> , <2 x i32> [[Y_OP1]] ; CHECK-NEXT: ret <2 x i32> [[DIV]] ; %y = shl <2 x i32> , %yy @@ -353,8 +352,8 @@ define i32 @test_udiv_to_const_Cudiv(i32 %x) { ; CHECK-LABEL: @test_udiv_to_const_Cudiv( ; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[X:%.*]], 90 -; CHECK-NEXT: [[COND:%.*]] = select i1 [[C]], i32 7, i32 19 -; CHECK-NEXT: [[DIV:%.*]] = udiv i32 [[X]], [[COND]] +; CHECK-NEXT: [[DOTOP:%.*]] = udiv i32 [[X]], 19 +; CHECK-NEXT: [[DIV:%.*]] = select i1 [[C]], i32 12, i32 [[DOTOP]] ; CHECK-NEXT: ret i32 [[DIV]] ; %c = icmp eq i32 %x, 90 @@ -379,8 +378,8 @@ define <2 x i32> @test_sdiv_to_const_Csdiv(<2 x i32> %x) { ; CHECK-LABEL: @test_sdiv_to_const_Csdiv( ; CHECK-NEXT: [[C_NOT:%.*]] = icmp eq <2 x i32> [[X:%.*]], -; CHECK-NEXT: [[COND:%.*]] = select <2 x i1> [[C_NOT]], <2 x i32> , <2 x i32> -; CHECK-NEXT: [[DIV:%.*]] = sdiv <2 x i32> [[X]], [[COND]] +; CHECK-NEXT: [[DOTOP:%.*]] = sdiv <2 x i32> [[X]], +; CHECK-NEXT: [[DIV:%.*]] = select <2 x i1> [[C_NOT]], <2 x i32> , <2 x i32> [[DOTOP]] ; CHECK-NEXT: ret <2 x i32> [[DIV]] ; %c = icmp ne <2 x i32> %x, @@ -392,8 +391,8 @@ define i32 @test_srem_to_const_Csrem(i32 %x) { ; CHECK-LABEL: @test_srem_to_const_Csrem( ; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[X:%.*]], 24 -; CHECK-NEXT: [[COND:%.*]] = select i1 [[C]], i32 7, i32 16 -; CHECK-NEXT: [[DIV:%.*]] = srem i32 [[X]], [[COND]] +; CHECK-NEXT: [[DOTOP:%.*]] = srem i32 [[X]], 16 +; CHECK-NEXT: [[DIV:%.*]] = select i1 [[C]], i32 3, i32 [[DOTOP]] ; CHECK-NEXT: ret i32 [[DIV]] ; %c = icmp eq i32 %x, 24 @@ -434,9 +433,10 @@ define i32 @test_urem_to_const_and(i1 %c, i32 %yy) { ; CHECK-LABEL: @test_urem_to_const_and( -; CHECK-NEXT: [[Y:%.*]] = shl nuw i32 1, [[YY:%.*]] -; CHECK-NEXT: [[COND:%.*]] = select i1 [[C:%.*]], i32 [[Y]], i32 19 -; CHECK-NEXT: [[DIV:%.*]] = urem i32 44, [[COND]] +; CHECK-NEXT: [[NOTMASK:%.*]] = shl nsw i32 -1, [[YY:%.*]] +; CHECK-NEXT: [[TMP1:%.*]] = and i32 [[NOTMASK]], 44 +; CHECK-NEXT: [[Y_OP:%.*]] = xor i32 [[TMP1]], 44 +; CHECK-NEXT: [[DIV:%.*]] = select i1 [[C:%.*]], i32 [[Y_OP]], i32 6 ; CHECK-NEXT: ret i32 [[DIV]] ; %y = shl i32 1, %yy @@ -448,8 +448,8 @@ define i32 @test_mul_to_const_Cmul(i32 %x) { ; CHECK-LABEL: @test_mul_to_const_Cmul( ; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[X:%.*]], 61 -; CHECK-NEXT: [[COND:%.*]] = select i1 [[C]], i32 9, i32 14 -; CHECK-NEXT: [[DIV:%.*]] = mul i32 [[COND]], [[X]] +; CHECK-NEXT: [[DOTOP:%.*]] = mul i32 [[X]], 14 +; CHECK-NEXT: [[DIV:%.*]] = select i1 [[C]], i32 549, i32 [[DOTOP]] ; CHECK-NEXT: ret i32 [[DIV]] ; %c = icmp eq i32 %x, 61 diff --git a/llvm/test/Transforms/InstCombine/div.ll b/llvm/test/Transforms/InstCombine/div.ll --- a/llvm/test/Transforms/InstCombine/div.ll +++ b/llvm/test/Transforms/InstCombine/div.ll @@ -1128,8 +1128,8 @@ define i32 @sdiv_constant_dividend_select_divisor1(i1 %b, i32 %x) { ; CHECK-LABEL: @sdiv_constant_dividend_select_divisor1( -; CHECK-NEXT: [[S:%.*]] = select i1 [[B:%.*]], i32 [[X:%.*]], i32 -3 -; CHECK-NEXT: [[R:%.*]] = sdiv i32 42, [[S]] +; CHECK-NEXT: [[X_OP:%.*]] = sdiv i32 42, [[X:%.*]] +; CHECK-NEXT: [[R:%.*]] = select i1 [[B:%.*]], i32 [[X_OP]], i32 -14 ; CHECK-NEXT: ret i32 [[R]] ; %s = select i1 %b, i32 %x, i32 -3 @@ -1141,8 +1141,8 @@ define i32 @sdiv_constant_dividend_select_divisor2(i1 %b, i32 %x) { ; CHECK-LABEL: @sdiv_constant_dividend_select_divisor2( -; CHECK-NEXT: [[S:%.*]] = select i1 [[B:%.*]], i32 12, i32 [[X:%.*]] -; CHECK-NEXT: [[R:%.*]] = sdiv i32 42, [[S]] +; CHECK-NEXT: [[X_OP:%.*]] = sdiv i32 42, [[X:%.*]] +; CHECK-NEXT: [[R:%.*]] = select i1 [[B:%.*]], i32 3, i32 [[X_OP]] ; CHECK-NEXT: ret i32 [[R]] ; %s = select i1 %b, i32 12, i32 %x @@ -1234,8 +1234,8 @@ define i32 @udiv_constant_dividend_select_divisor1(i1 %b, i32 %x) { ; CHECK-LABEL: @udiv_constant_dividend_select_divisor1( -; CHECK-NEXT: [[S:%.*]] = select i1 [[B:%.*]], i32 [[X:%.*]], i32 -3 -; CHECK-NEXT: [[R:%.*]] = udiv i32 42, [[S]] +; CHECK-NEXT: [[X_OP:%.*]] = udiv i32 42, [[X:%.*]] +; CHECK-NEXT: [[R:%.*]] = select i1 [[B:%.*]], i32 [[X_OP]], i32 0 ; CHECK-NEXT: ret i32 [[R]] ; %s = select i1 %b, i32 %x, i32 -3 @@ -1247,8 +1247,8 @@ define i32 @udiv_constant_dividend_select_divisor2(i1 %b, i32 %x) { ; CHECK-LABEL: @udiv_constant_dividend_select_divisor2( -; CHECK-NEXT: [[S:%.*]] = select i1 [[B:%.*]], i32 12, i32 [[X:%.*]] -; CHECK-NEXT: [[R:%.*]] = udiv i32 42, [[S]] +; CHECK-NEXT: [[X_OP:%.*]] = udiv i32 42, [[X:%.*]] +; CHECK-NEXT: [[R:%.*]] = select i1 [[B:%.*]], i32 3, i32 [[X_OP]] ; CHECK-NEXT: ret i32 [[R]] ; %s = select i1 %b, i32 12, i32 %x diff --git a/llvm/test/Transforms/InstCombine/rem.ll b/llvm/test/Transforms/InstCombine/rem.ll --- a/llvm/test/Transforms/InstCombine/rem.ll +++ b/llvm/test/Transforms/InstCombine/rem.ll @@ -847,8 +847,8 @@ define i32 @srem_constant_dividend_select_divisor1(i1 %b, i32 %x) { ; CHECK-LABEL: @srem_constant_dividend_select_divisor1( -; CHECK-NEXT: [[S:%.*]] = select i1 [[B:%.*]], i32 [[X:%.*]], i32 -3 -; CHECK-NEXT: [[R:%.*]] = srem i32 42, [[S]] +; CHECK-NEXT: [[X_OP:%.*]] = srem i32 42, [[X:%.*]] +; CHECK-NEXT: [[R:%.*]] = select i1 [[B:%.*]], i32 [[X_OP]], i32 0 ; CHECK-NEXT: ret i32 [[R]] ; %s = select i1 %b, i32 %x, i32 -3 @@ -860,8 +860,8 @@ define i32 @srem_constant_dividend_select_divisor2(i1 %b, i32 %x) { ; CHECK-LABEL: @srem_constant_dividend_select_divisor2( -; CHECK-NEXT: [[S:%.*]] = select i1 [[B:%.*]], i32 12, i32 [[X:%.*]] -; CHECK-NEXT: [[R:%.*]] = srem i32 42, [[S]] +; CHECK-NEXT: [[X_OP:%.*]] = srem i32 42, [[X:%.*]] +; CHECK-NEXT: [[R:%.*]] = select i1 [[B:%.*]], i32 6, i32 [[X_OP]] ; CHECK-NEXT: ret i32 [[R]] ; %s = select i1 %b, i32 12, i32 %x @@ -953,8 +953,8 @@ define i32 @urem_constant_dividend_select_divisor1(i1 %b, i32 %x) { ; CHECK-LABEL: @urem_constant_dividend_select_divisor1( -; CHECK-NEXT: [[S:%.*]] = select i1 [[B:%.*]], i32 [[X:%.*]], i32 -3 -; CHECK-NEXT: [[R:%.*]] = urem i32 42, [[S]] +; CHECK-NEXT: [[X_OP:%.*]] = urem i32 42, [[X:%.*]] +; CHECK-NEXT: [[R:%.*]] = select i1 [[B:%.*]], i32 [[X_OP]], i32 42 ; CHECK-NEXT: ret i32 [[R]] ; %s = select i1 %b, i32 %x, i32 -3 @@ -966,8 +966,8 @@ define i32 @urem_constant_dividend_select_divisor2(i1 %b, i32 %x) { ; CHECK-LABEL: @urem_constant_dividend_select_divisor2( -; CHECK-NEXT: [[S:%.*]] = select i1 [[B:%.*]], i32 12, i32 [[X:%.*]] -; CHECK-NEXT: [[R:%.*]] = urem i32 42, [[S]] +; CHECK-NEXT: [[X_OP:%.*]] = urem i32 42, [[X:%.*]] +; CHECK-NEXT: [[R:%.*]] = select i1 [[B:%.*]], i32 6, i32 [[X_OP]] ; CHECK-NEXT: ret i32 [[R]] ; %s = select i1 %b, i32 12, i32 %x