Index: lib/Transforms/InstCombine/InstCombineSelect.cpp =================================================================== --- lib/Transforms/InstCombine/InstCombineSelect.cpp +++ lib/Transforms/InstCombine/InstCombineSelect.cpp @@ -532,6 +532,44 @@ } /// We want to turn: +/// (select (icmp sgt x, C), lshr (X, Y), ashr (X, Y)); iff C s>= -1 +/// (select (icmp slt x, C), ashr (X, Y), lshr (X, Y)); iff C s>= 0 +/// into: +/// ashr (X, Y) +static Value *foldSelectICmpLshrAshr(const ICmpInst *IC, Value *TrueVal, + Value *FalseVal, + InstCombiner::BuilderTy &Builder) { + ICmpInst::Predicate Pred = IC->getPredicate(); + Value *CmpLHS = IC->getOperand(0); + Value *CmpRHS = IC->getOperand(1); + + Value *X, *Y; + unsigned Bitwidth = CmpRHS->getType()->getScalarSizeInBits(); + if ((Pred != ICmpInst::ICMP_SGT || + !match(CmpRHS, + m_SpecificInt_ICMP(ICmpInst::ICMP_SGE, APInt(Bitwidth, -1)))) && + (Pred != ICmpInst::ICMP_SLT || + !match(CmpRHS, + m_SpecificInt_ICMP(ICmpInst::ICMP_SGE, APInt(Bitwidth, 0))))) + return nullptr; + + // Canonicalize so that ashr is in FalseVal. + if (Pred == ICmpInst::ICMP_SLT) + std::swap(TrueVal, FalseVal); + + if (match(TrueVal, m_LShr(m_Value(X), m_Value(Y))) && + match(FalseVal, m_AShr(m_Specific(X), m_Specific(Y))) && + match(CmpLHS, m_Specific(X))) { + const auto *Ashr = cast(FalseVal); + // if lshr is not exact and ashr is, this new ashr must not be exact. + bool IsExact = Ashr->isExact() && cast(TrueVal)->isExact(); + return Builder.CreateAShr(X, Y, IC->getName(), IsExact); + } + + return nullptr; +} + +/// We want to turn: /// (select (icmp eq (and X, C1), 0), Y, (or Y, C2)) /// into: /// (or (shl (and X, C1), C3), Y) @@ -1112,6 +1150,9 @@ if (Value *V = foldSelectICmpAndOr(ICI, TrueVal, FalseVal, Builder)) return replaceInstUsesWith(SI, V); + if (Value *V = foldSelectICmpLshrAshr(ICI, TrueVal, FalseVal, Builder)) + return replaceInstUsesWith(SI, V); + if (Value *V = foldSelectCttzCtlz(ICI, TrueVal, FalseVal, Builder)) return replaceInstUsesWith(SI, V); Index: test/Transforms/InstCombine/ashr-lshr.ll =================================================================== --- test/Transforms/InstCombine/ashr-lshr.ll +++ test/Transforms/InstCombine/ashr-lshr.ll @@ -3,11 +3,8 @@ define i32 @ashr_lshr_exact_ashr_only(i32 %x, i32 %y) { ; CHECK-LABEL: @ashr_lshr_exact_ashr_only( -; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[X:%.*]], -1 -; CHECK-NEXT: [[L:%.*]] = lshr i32 [[X]], [[Y:%.*]] -; CHECK-NEXT: [[R:%.*]] = ashr exact i32 [[X]], [[Y]] -; CHECK-NEXT: [[RET:%.*]] = select i1 [[CMP]], i32 [[L]], i32 [[R]] -; CHECK-NEXT: ret i32 [[RET]] +; CHECK-NEXT: [[CMP1:%.*]] = ashr i32 [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: ret i32 [[CMP1]] ; %cmp = icmp sgt i32 %x, -1 %l = lshr i32 %x, %y @@ -18,11 +15,8 @@ define i32 @ashr_lshr_no_exact(i32 %x, i32 %y) { ; CHECK-LABEL: @ashr_lshr_no_exact( -; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[X:%.*]], -1 -; CHECK-NEXT: [[L:%.*]] = lshr i32 [[X]], [[Y:%.*]] -; CHECK-NEXT: [[R:%.*]] = ashr i32 [[X]], [[Y]] -; CHECK-NEXT: [[RET:%.*]] = select i1 [[CMP]], i32 [[L]], i32 [[R]] -; CHECK-NEXT: ret i32 [[RET]] +; CHECK-NEXT: [[CMP1:%.*]] = ashr i32 [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: ret i32 [[CMP1]] ; %cmp = icmp sgt i32 %x, -1 %l = lshr i32 %x, %y @@ -33,11 +27,8 @@ define i32 @ashr_lshr_exact_both(i32 %x, i32 %y) { ; CHECK-LABEL: @ashr_lshr_exact_both( -; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[X:%.*]], -1 -; CHECK-NEXT: [[L:%.*]] = lshr exact i32 [[X]], [[Y:%.*]] -; CHECK-NEXT: [[R:%.*]] = ashr exact i32 [[X]], [[Y]] -; CHECK-NEXT: [[RET:%.*]] = select i1 [[CMP]], i32 [[L]], i32 [[R]] -; CHECK-NEXT: ret i32 [[RET]] +; CHECK-NEXT: [[CMP1:%.*]] = ashr exact i32 [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: ret i32 [[CMP1]] ; %cmp = icmp sgt i32 %x, -1 %l = lshr exact i32 %x, %y @@ -48,11 +39,8 @@ define i32 @ashr_lshr_exact_lshr_only(i32 %x, i32 %y) { ; CHECK-LABEL: @ashr_lshr_exact_lshr_only( -; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[X:%.*]], -1 -; CHECK-NEXT: [[L:%.*]] = lshr exact i32 [[X]], [[Y:%.*]] -; CHECK-NEXT: [[R:%.*]] = ashr i32 [[X]], [[Y]] -; CHECK-NEXT: [[RET:%.*]] = select i1 [[CMP]], i32 [[L]], i32 [[R]] -; CHECK-NEXT: ret i32 [[RET]] +; CHECK-NEXT: [[CMP1:%.*]] = ashr i32 [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: ret i32 [[CMP1]] ; %cmp = icmp sgt i32 %x, -1 %l = lshr exact i32 %x, %y @@ -63,11 +51,8 @@ define i32 @ashr_lshr2(i32 %x, i32 %y) { ; CHECK-LABEL: @ashr_lshr2( -; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[X:%.*]], 5 -; CHECK-NEXT: [[L:%.*]] = lshr i32 [[X]], [[Y:%.*]] -; CHECK-NEXT: [[R:%.*]] = ashr exact i32 [[X]], [[Y]] -; CHECK-NEXT: [[RET:%.*]] = select i1 [[CMP]], i32 [[L]], i32 [[R]] -; CHECK-NEXT: ret i32 [[RET]] +; CHECK-NEXT: [[CMP1:%.*]] = ashr i32 [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: ret i32 [[CMP1]] ; %cmp = icmp sgt i32 %x, 5 %l = lshr i32 %x, %y @@ -78,11 +63,8 @@ define <2 x i32> @ashr_lshr_splat_vec(<2 x i32> %x, <2 x i32> %y) { ; CHECK-LABEL: @ashr_lshr_splat_vec( -; CHECK-NEXT: [[CMP:%.*]] = icmp sgt <2 x i32> [[X:%.*]], -; CHECK-NEXT: [[L:%.*]] = lshr <2 x i32> [[X]], [[Y:%.*]] -; CHECK-NEXT: [[R:%.*]] = ashr <2 x i32> [[X]], [[Y]] -; CHECK-NEXT: [[RET:%.*]] = select <2 x i1> [[CMP]], <2 x i32> [[L]], <2 x i32> [[R]] -; CHECK-NEXT: ret <2 x i32> [[RET]] +; CHECK-NEXT: [[CMP1:%.*]] = ashr <2 x i32> [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: ret <2 x i32> [[CMP1]] ; %cmp = icmp sgt <2 x i32> %x, %l = lshr <2 x i32> %x, %y @@ -93,11 +75,8 @@ define <2 x i32> @ashr_lshr_splat_vec2(<2 x i32> %x, <2 x i32> %y) { ; CHECK-LABEL: @ashr_lshr_splat_vec2( -; CHECK-NEXT: [[CMP:%.*]] = icmp sgt <2 x i32> [[X:%.*]], -; CHECK-NEXT: [[L:%.*]] = lshr exact <2 x i32> [[X]], [[Y:%.*]] -; CHECK-NEXT: [[R:%.*]] = ashr exact <2 x i32> [[X]], [[Y]] -; CHECK-NEXT: [[RET:%.*]] = select <2 x i1> [[CMP]], <2 x i32> [[L]], <2 x i32> [[R]] -; CHECK-NEXT: ret <2 x i32> [[RET]] +; CHECK-NEXT: [[CMP1:%.*]] = ashr exact <2 x i32> [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: ret <2 x i32> [[CMP1]] ; %cmp = icmp sgt <2 x i32> %x, %l = lshr exact <2 x i32> %x, %y @@ -108,11 +87,8 @@ define <2 x i32> @ashr_lshr_splat_vec3(<2 x i32> %x, <2 x i32> %y) { ; CHECK-LABEL: @ashr_lshr_splat_vec3( -; CHECK-NEXT: [[CMP:%.*]] = icmp sgt <2 x i32> [[X:%.*]], -; CHECK-NEXT: [[L:%.*]] = lshr exact <2 x i32> [[X]], [[Y:%.*]] -; CHECK-NEXT: [[R:%.*]] = ashr <2 x i32> [[X]], [[Y]] -; CHECK-NEXT: [[RET:%.*]] = select <2 x i1> [[CMP]], <2 x i32> [[L]], <2 x i32> [[R]] -; CHECK-NEXT: ret <2 x i32> [[RET]] +; CHECK-NEXT: [[CMP1:%.*]] = ashr <2 x i32> [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: ret <2 x i32> [[CMP1]] ; %cmp = icmp sgt <2 x i32> %x, %l = lshr exact <2 x i32> %x, %y @@ -123,11 +99,8 @@ define <2 x i32> @ashr_lshr_splat_vec4(<2 x i32> %x, <2 x i32> %y) { ; CHECK-LABEL: @ashr_lshr_splat_vec4( -; CHECK-NEXT: [[CMP:%.*]] = icmp sgt <2 x i32> [[X:%.*]], -; CHECK-NEXT: [[L:%.*]] = lshr <2 x i32> [[X]], [[Y:%.*]] -; CHECK-NEXT: [[R:%.*]] = ashr exact <2 x i32> [[X]], [[Y]] -; CHECK-NEXT: [[RET:%.*]] = select <2 x i1> [[CMP]], <2 x i32> [[L]], <2 x i32> [[R]] -; CHECK-NEXT: ret <2 x i32> [[RET]] +; CHECK-NEXT: [[CMP1:%.*]] = ashr <2 x i32> [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: ret <2 x i32> [[CMP1]] ; %cmp = icmp sgt <2 x i32> %x, %l = lshr <2 x i32> %x, %y @@ -138,11 +111,8 @@ define <2 x i32> @ashr_lshr_nonsplat_vec(<2 x i32> %x, <2 x i32> %y) { ; CHECK-LABEL: @ashr_lshr_nonsplat_vec( -; CHECK-NEXT: [[CMP:%.*]] = icmp sgt <2 x i32> [[X:%.*]], -; CHECK-NEXT: [[L:%.*]] = lshr <2 x i32> [[X]], [[Y:%.*]] -; CHECK-NEXT: [[R:%.*]] = ashr <2 x i32> [[X]], [[Y]] -; CHECK-NEXT: [[RET:%.*]] = select <2 x i1> [[CMP]], <2 x i32> [[L]], <2 x i32> [[R]] -; CHECK-NEXT: ret <2 x i32> [[RET]] +; CHECK-NEXT: [[CMP1:%.*]] = ashr <2 x i32> [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: ret <2 x i32> [[CMP1]] ; %cmp = icmp sgt <2 x i32> %x, %l = lshr <2 x i32> %x, %y @@ -153,11 +123,8 @@ define <2 x i32> @ashr_lshr_nonsplat_vec2(<2 x i32> %x, <2 x i32> %y) { ; CHECK-LABEL: @ashr_lshr_nonsplat_vec2( -; CHECK-NEXT: [[CMP:%.*]] = icmp sgt <2 x i32> [[X:%.*]], -; CHECK-NEXT: [[L:%.*]] = lshr exact <2 x i32> [[X]], [[Y:%.*]] -; CHECK-NEXT: [[R:%.*]] = ashr exact <2 x i32> [[X]], [[Y]] -; CHECK-NEXT: [[RET:%.*]] = select <2 x i1> [[CMP]], <2 x i32> [[L]], <2 x i32> [[R]] -; CHECK-NEXT: ret <2 x i32> [[RET]] +; CHECK-NEXT: [[CMP1:%.*]] = ashr exact <2 x i32> [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: ret <2 x i32> [[CMP1]] ; %cmp = icmp sgt <2 x i32> %x, %l = lshr exact <2 x i32> %x, %y @@ -168,11 +135,8 @@ define <2 x i32> @ashr_lshr_nonsplat_vec3(<2 x i32> %x, <2 x i32> %y) { ; CHECK-LABEL: @ashr_lshr_nonsplat_vec3( -; CHECK-NEXT: [[CMP:%.*]] = icmp sgt <2 x i32> [[X:%.*]], -; CHECK-NEXT: [[L:%.*]] = lshr exact <2 x i32> [[X]], [[Y:%.*]] -; CHECK-NEXT: [[R:%.*]] = ashr <2 x i32> [[X]], [[Y]] -; CHECK-NEXT: [[RET:%.*]] = select <2 x i1> [[CMP]], <2 x i32> [[L]], <2 x i32> [[R]] -; CHECK-NEXT: ret <2 x i32> [[RET]] +; CHECK-NEXT: [[CMP1:%.*]] = ashr <2 x i32> [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: ret <2 x i32> [[CMP1]] ; %cmp = icmp sgt <2 x i32> %x, %l = lshr exact <2 x i32> %x, %y @@ -183,11 +147,8 @@ define <2 x i32> @ashr_lshr_nonsplat_vec4(<2 x i32> %x, <2 x i32> %y) { ; CHECK-LABEL: @ashr_lshr_nonsplat_vec4( -; CHECK-NEXT: [[CMP:%.*]] = icmp sgt <2 x i32> [[X:%.*]], -; CHECK-NEXT: [[L:%.*]] = lshr <2 x i32> [[X]], [[Y:%.*]] -; CHECK-NEXT: [[R:%.*]] = ashr exact <2 x i32> [[X]], [[Y]] -; CHECK-NEXT: [[RET:%.*]] = select <2 x i1> [[CMP]], <2 x i32> [[L]], <2 x i32> [[R]] -; CHECK-NEXT: ret <2 x i32> [[RET]] +; CHECK-NEXT: [[CMP1:%.*]] = ashr <2 x i32> [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: ret <2 x i32> [[CMP1]] ; %cmp = icmp sgt <2 x i32> %x, %l = lshr <2 x i32> %x, %y @@ -198,11 +159,8 @@ define i32 @ashr_lshr_cst(i32 %x, i32 %y) { ; CHECK-LABEL: @ashr_lshr_cst( -; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 1 -; CHECK-NEXT: [[L:%.*]] = lshr i32 [[X]], 8 -; CHECK-NEXT: [[R:%.*]] = ashr exact i32 [[X]], 8 -; CHECK-NEXT: [[RET:%.*]] = select i1 [[CMP]], i32 [[R]], i32 [[L]] -; CHECK-NEXT: ret i32 [[RET]] +; CHECK-NEXT: [[CMP1:%.*]] = ashr i32 [[X:%.*]], 8 +; CHECK-NEXT: ret i32 [[CMP1]] ; %cmp = icmp slt i32 %x, 1 %l = lshr i32 %x, 8 @@ -213,11 +171,8 @@ define i32 @ashr_lshr_cst2(i32 %x, i32 %y) { ; CHECK-LABEL: @ashr_lshr_cst2( -; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[X:%.*]], -1 -; CHECK-NEXT: [[L:%.*]] = lshr i32 [[X]], 8 -; CHECK-NEXT: [[R:%.*]] = ashr exact i32 [[X]], 8 -; CHECK-NEXT: [[RET:%.*]] = select i1 [[CMP]], i32 [[L]], i32 [[R]] -; CHECK-NEXT: ret i32 [[RET]] +; CHECK-NEXT: [[CMP1:%.*]] = ashr i32 [[X:%.*]], 8 +; CHECK-NEXT: ret i32 [[CMP1]] ; %cmp = icmp sgt i32 %x, -1 %l = lshr i32 %x, 8 @@ -228,11 +183,8 @@ define i32 @ashr_lshr_inv(i32 %x, i32 %y) { ; CHECK-LABEL: @ashr_lshr_inv( -; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 1 -; CHECK-NEXT: [[L:%.*]] = lshr i32 [[X]], [[Y:%.*]] -; CHECK-NEXT: [[R:%.*]] = ashr exact i32 [[X]], [[Y]] -; CHECK-NEXT: [[RET:%.*]] = select i1 [[CMP]], i32 [[R]], i32 [[L]] -; CHECK-NEXT: ret i32 [[RET]] +; CHECK-NEXT: [[CMP1:%.*]] = ashr i32 [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: ret i32 [[CMP1]] ; %cmp = icmp slt i32 %x, 1 %l = lshr i32 %x, %y @@ -243,11 +195,8 @@ define i32 @ashr_lshr_inv2(i32 %x, i32 %y) { ; CHECK-LABEL: @ashr_lshr_inv2( -; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 7 -; CHECK-NEXT: [[L:%.*]] = lshr i32 [[X]], [[Y:%.*]] -; CHECK-NEXT: [[R:%.*]] = ashr exact i32 [[X]], [[Y]] -; CHECK-NEXT: [[RET:%.*]] = select i1 [[CMP]], i32 [[R]], i32 [[L]] -; CHECK-NEXT: ret i32 [[RET]] +; CHECK-NEXT: [[CMP1:%.*]] = ashr i32 [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: ret i32 [[CMP1]] ; %cmp = icmp slt i32 %x, 7 %l = lshr i32 %x, %y @@ -258,11 +207,8 @@ define <2 x i32> @ashr_lshr_inv_splat_vec(<2 x i32> %x, <2 x i32> %y) { ; CHECK-LABEL: @ashr_lshr_inv_splat_vec( -; CHECK-NEXT: [[CMP:%.*]] = icmp slt <2 x i32> [[X:%.*]], -; CHECK-NEXT: [[L:%.*]] = lshr <2 x i32> [[X]], [[Y:%.*]] -; CHECK-NEXT: [[R:%.*]] = ashr exact <2 x i32> [[X]], [[Y]] -; CHECK-NEXT: [[RET:%.*]] = select <2 x i1> [[CMP]], <2 x i32> [[R]], <2 x i32> [[L]] -; CHECK-NEXT: ret <2 x i32> [[RET]] +; CHECK-NEXT: [[CMP1:%.*]] = ashr <2 x i32> [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: ret <2 x i32> [[CMP1]] ; %cmp = icmp slt <2 x i32> %x, %l = lshr <2 x i32> %x, %y @@ -273,11 +219,8 @@ define <2 x i32> @ashr_lshr_inv_nonsplat_vec(<2 x i32> %x, <2 x i32> %y) { ; CHECK-LABEL: @ashr_lshr_inv_nonsplat_vec( -; CHECK-NEXT: [[CMP:%.*]] = icmp slt <2 x i32> [[X:%.*]], -; CHECK-NEXT: [[L:%.*]] = lshr <2 x i32> [[X]], [[Y:%.*]] -; CHECK-NEXT: [[R:%.*]] = ashr exact <2 x i32> [[X]], [[Y]] -; CHECK-NEXT: [[RET:%.*]] = select <2 x i1> [[CMP]], <2 x i32> [[R]], <2 x i32> [[L]] -; CHECK-NEXT: ret <2 x i32> [[RET]] +; CHECK-NEXT: [[CMP1:%.*]] = ashr <2 x i32> [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: ret <2 x i32> [[CMP1]] ; %cmp = icmp slt <2 x i32> %x, %l = lshr <2 x i32> %x, %y @@ -288,11 +231,8 @@ define <2 x i32> @ashr_lshr_vec_undef(<2 x i32> %x, <2 x i32> %y) { ; CHECK-LABEL: @ashr_lshr_vec_undef( -; CHECK-NEXT: [[CMP:%.*]] = icmp sgt <2 x i32> [[X:%.*]], -; CHECK-NEXT: [[L:%.*]] = lshr <2 x i32> [[X]], [[Y:%.*]] -; CHECK-NEXT: [[R:%.*]] = ashr exact <2 x i32> [[X]], [[Y]] -; CHECK-NEXT: [[RET:%.*]] = select <2 x i1> [[CMP]], <2 x i32> [[L]], <2 x i32> [[R]] -; CHECK-NEXT: ret <2 x i32> [[RET]] +; CHECK-NEXT: [[CMP1:%.*]] = ashr <2 x i32> [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: ret <2 x i32> [[CMP1]] ; %cmp = icmp sgt <2 x i32> %x, %l = lshr <2 x i32> %x, %y @@ -303,11 +243,8 @@ define <2 x i32> @ashr_lshr_vec_undef2(<2 x i32> %x, <2 x i32> %y) { ; CHECK-LABEL: @ashr_lshr_vec_undef2( -; CHECK-NEXT: [[CMP:%.*]] = icmp slt <2 x i32> [[X:%.*]], -; CHECK-NEXT: [[L:%.*]] = lshr exact <2 x i32> [[X]], [[Y:%.*]] -; CHECK-NEXT: [[R:%.*]] = ashr exact <2 x i32> [[X]], [[Y]] -; CHECK-NEXT: [[RET:%.*]] = select <2 x i1> [[CMP]], <2 x i32> [[R]], <2 x i32> [[L]] -; CHECK-NEXT: ret <2 x i32> [[RET]] +; CHECK-NEXT: [[CMP1:%.*]] = ashr exact <2 x i32> [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: ret <2 x i32> [[CMP1]] ; %cmp = icmp slt <2 x i32> %x, %l = lshr exact <2 x i32> %x, %y