Index: lib/Transforms/InstCombine/InstCombineMulDivRem.cpp =================================================================== --- lib/Transforms/InstCombine/InstCombineMulDivRem.cpp +++ lib/Transforms/InstCombine/InstCombineMulDivRem.cpp @@ -918,23 +918,48 @@ } Constant *C; - if ((match(N, m_OneUse(m_ZExt(m_Value(X)))) && match(D, m_Constant(C))) || - (match(D, m_OneUse(m_ZExt(m_Value(X)))) && match(N, m_Constant(C)))) { - // If the constant is the same in the smaller type, use the narrow version. - Constant *TruncC = ConstantExpr::getTrunc(C, X->getType()); - if (ConstantExpr::getZExt(TruncC, Ty) != C) - return nullptr; - - // udiv (zext X), C --> zext (udiv X, C') - // urem (zext X), C --> zext (urem X, C') - // udiv C, (zext X) --> zext (udiv C', X) - // urem C, (zext X) --> zext (urem C', X) - Value *NarrowOp = isa(D) ? Builder.CreateBinOp(Opcode, X, TruncC) - : Builder.CreateBinOp(Opcode, TruncC, X); - return new ZExtInst(NarrowOp, Ty); + bool CanConvert = false; + if ((match(N, m_ZExt(m_Value(X))) && match(D, m_Constant(C)))) { + if (N->hasOneUse()) { + CanConvert = true; + } else if (N->hasNUses(2)) { + // Handle the case with two uses to support this common pattern: + // warp_id = N UDIV 32 + // lane_id = N UREM 32 + Value::user_iterator UI = N->user_begin(); + llvm::Instruction *I1 = + dyn_cast(*UI); + llvm::Instruction *I2 = + dyn_cast(*(++UI)); + I2 = (I1 == &I) ? I2 : I1; + // Instead of allowing a second constant C2 that meets the requirement of + // ConstantExpr::getZExt(Trunc(C2), Ty) == C2, we simply require the + // constant to be the same as the first constant. + if (I2 && (match(I2, m_UDiv(m_Specific(N), m_Specific(C))) || + match(I2, m_URem(m_Specific(N), m_Specific(C))))) { + CanConvert = true; + } + } + } else if (match(D, m_OneUse(m_ZExt(m_Value(X)))) && + match(N, m_Constant(C))) { + CanConvert = true; } - return nullptr; + if (!CanConvert) + return nullptr; + + // If the constant is the same in the smaller type, use the narrow version. + Constant *TruncC = ConstantExpr::getTrunc(C, X->getType()); + if (ConstantExpr::getZExt(TruncC, Ty) != C) + return nullptr; + + // udiv (zext X), C --> zext (udiv X, C') + // urem (zext X), C --> zext (urem X, C') + // udiv C, (zext X) --> zext (udiv C', X) + // urem C, (zext X) --> zext (urem C', X) + Value *NarrowOp = isa(D) ? Builder.CreateBinOp(Opcode, X, TruncC) + : Builder.CreateBinOp(Opcode, TruncC, X); + return new ZExtInst(NarrowOp, Ty); } Instruction *InstCombiner::visitUDiv(BinaryOperator &I) { Index: test/Transforms/InstCombine/udivrem-change-width.ll =================================================================== --- test/Transforms/InstCombine/udivrem-change-width.ll +++ test/Transforms/InstCombine/udivrem-change-width.ll @@ -75,6 +75,36 @@ ret <2 x i32> %udiv } +define i64 @udiv_urem_twouses_xform(i32 %a) { +; CHECK-LABEL: @udiv_urem_twouses_xform( +; CHECK-NEXT: [[UDIV:%.*]] = udiv i32 [[A:%.*]], 33 +; CHECK-NEXT: [[UREM:%.*]] = urem i32 [[A:%.*]], 33 +; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[UDIV]], [[UREM]] +; CHECK-NEXT: [[ADD_ZEXT:%.*]] = zext i32 [[ADD:%.*]] to i64 +; CHECK-NEXT: ret i64 [[ADD_ZEXT]] +; + %za = zext i32 %a to i64 + %udiv = udiv i64 %za, 33 + %urem = urem i64 %za, 33 + %uadd = add i64 %udiv, %urem + ret i64 %uadd +} + +define i64 @udiv_urem_twouses_not_xform(i32 %a) { +; CHECK-LABEL: @udiv_urem_twouses_not_xform( +; CHECK-NEXT: [[A_ZEXT:%.*]] = zext i32 {{%.*}} to i64 +; CHECK-NEXT: [[UDIV:%.*]] = udiv i64 [[A_ZEXT:%.*]], 33 +; CHECK-NEXT: [[UREM:%.*]] = urem i64 [[A_ZEXT:%.*]], 2147483649 +; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i64 [[UDIV]], [[UREM]] +; CHECK-NEXT: ret i64 [[ADD]] +; + %za = zext i32 %a to i64 + %udiv = udiv i64 %za, 33 + %urem = urem i64 %za, 2147483649 + %uadd = add i64 %udiv, %urem + ret i64 %uadd +} + define i32 @udiv_i32_multiuse(i8 %a, i8 %b) { ; CHECK-LABEL: @udiv_i32_multiuse( ; CHECK-NEXT: [[ZA:%.*]] = zext i8 %a to i32