Index: cfe/trunk/lib/CodeGen/CGBuiltin.cpp =================================================================== --- cfe/trunk/lib/CodeGen/CGBuiltin.cpp +++ cfe/trunk/lib/CodeGen/CGBuiltin.cpp @@ -839,6 +839,93 @@ return RValue::get(BufAddr.getPointer()); } +/// Determine if a binop is a checked mixed-sign multiply we can specialize. +static bool isSpecialMixedSignMultiply(unsigned BuiltinID, + WidthAndSignedness Op1Info, + WidthAndSignedness Op2Info, + WidthAndSignedness ResultInfo) { + return BuiltinID == Builtin::BI__builtin_mul_overflow && + Op1Info.Width == Op2Info.Width && Op1Info.Width >= ResultInfo.Width && + Op1Info.Signed != Op2Info.Signed; +} + +/// Emit a checked mixed-sign multiply. This is a cheaper specialization of +/// the generic checked-binop irgen. +static RValue +EmitCheckedMixedSignMultiply(CodeGenFunction &CGF, const clang::Expr *Op1, + WidthAndSignedness Op1Info, const clang::Expr *Op2, + WidthAndSignedness Op2Info, + const clang::Expr *ResultArg, QualType ResultQTy, + WidthAndSignedness ResultInfo) { + assert(isSpecialMixedSignMultiply(Builtin::BI__builtin_mul_overflow, Op1Info, + Op2Info, ResultInfo) && + "Not a mixed-sign multipliction we can specialize"); + + // Emit the signed and unsigned operands. + const clang::Expr *SignedOp = Op1Info.Signed ? Op1 : Op2; + const clang::Expr *UnsignedOp = Op1Info.Signed ? Op2 : Op1; + llvm::Value *Signed = CGF.EmitScalarExpr(SignedOp); + llvm::Value *Unsigned = CGF.EmitScalarExpr(UnsignedOp); + + llvm::Type *OpTy = Signed->getType(); + llvm::Value *Zero = llvm::Constant::getNullValue(OpTy); + Address ResultPtr = CGF.EmitPointerWithAlignment(ResultArg); + llvm::Type *ResTy = ResultPtr.getElementType(); + + // Take the absolute value of the signed operand. + llvm::Value *IsNegative = CGF.Builder.CreateICmpSLT(Signed, Zero); + llvm::Value *AbsOfNegative = CGF.Builder.CreateSub(Zero, Signed); + llvm::Value *AbsSigned = + CGF.Builder.CreateSelect(IsNegative, AbsOfNegative, Signed); + + // Perform a checked unsigned multiplication. + llvm::Value *UnsignedOverflow; + llvm::Value *UnsignedResult = + EmitOverflowIntrinsic(CGF, llvm::Intrinsic::umul_with_overflow, AbsSigned, + Unsigned, UnsignedOverflow); + + llvm::Value *Overflow, *Result; + if (ResultInfo.Signed) { + // Signed overflow occurs if the result is greater than INT_MAX or lesser + // than INT_MIN, i.e when |Result| > (INT_MAX + IsNegative). + auto IntMax = llvm::APInt::getSignedMaxValue(ResultInfo.Width) + .zextOrSelf(Op1Info.Width); + llvm::Value *MaxResult = + CGF.Builder.CreateAdd(llvm::ConstantInt::get(OpTy, IntMax), + CGF.Builder.CreateZExt(IsNegative, OpTy)); + llvm::Value *SignedOverflow = + CGF.Builder.CreateICmpUGT(UnsignedResult, MaxResult); + Overflow = CGF.Builder.CreateOr(UnsignedOverflow, SignedOverflow); + + // Prepare the signed result (possibly by negating it). + llvm::Value *NegativeResult = CGF.Builder.CreateNeg(UnsignedResult); + llvm::Value *SignedResult = + CGF.Builder.CreateSelect(IsNegative, NegativeResult, UnsignedResult); + Result = CGF.Builder.CreateTrunc(SignedResult, ResTy); + } else { + // Unsigned overflow occurs if the result is < 0 or greater than UINT_MAX. + llvm::Value *Underflow = CGF.Builder.CreateAnd( + IsNegative, CGF.Builder.CreateIsNotNull(UnsignedResult)); + Overflow = CGF.Builder.CreateOr(UnsignedOverflow, Underflow); + if (ResultInfo.Width < Op1Info.Width) { + auto IntMax = + llvm::APInt::getMaxValue(ResultInfo.Width).zext(Op1Info.Width); + llvm::Value *TruncOverflow = CGF.Builder.CreateICmpUGT( + UnsignedResult, llvm::ConstantInt::get(OpTy, IntMax)); + Overflow = CGF.Builder.CreateOr(Overflow, TruncOverflow); + } + + Result = CGF.Builder.CreateTrunc(UnsignedResult, ResTy); + } + assert(Overflow && Result && "Missing overflow or result"); + + bool isVolatile = + ResultArg->getType()->getPointeeType().isVolatileQualified(); + CGF.Builder.CreateStore(CGF.EmitToMemory(Result, ResultQTy), ResultPtr, + isVolatile); + return RValue::get(Overflow); +} + RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue) { @@ -2323,6 +2410,14 @@ getIntegerWidthAndSignedness(CGM.getContext(), RightArg->getType()); WidthAndSignedness ResultInfo = getIntegerWidthAndSignedness(CGM.getContext(), ResultQTy); + + // Handle mixed-sign multiplication as a special case, because adding + // runtime or backend support for our generic irgen would be too expensive. + if (isSpecialMixedSignMultiply(BuiltinID, LeftInfo, RightInfo, ResultInfo)) + return EmitCheckedMixedSignMultiply(*this, LeftArg, LeftInfo, RightArg, + RightInfo, ResultArg, ResultQTy, + ResultInfo); + WidthAndSignedness EncompassingInfo = EncompassingIntegerType({LeftInfo, RightInfo, ResultInfo}); Index: cfe/trunk/test/CodeGen/builtins-overflow.c =================================================================== --- cfe/trunk/test/CodeGen/builtins-overflow.c +++ cfe/trunk/test/CodeGen/builtins-overflow.c @@ -338,3 +338,122 @@ return LongLongErrorCode; return result; } + +int test_mixed_sign_mull_overflow(int x, unsigned y) { +// CHECK: @test_mixed_sign_mull_overflow +// CHECK: [[IsNeg:%.*]] = icmp slt i32 [[Op1:%.*]], 0 +// CHECK-NEXT: [[Signed:%.*]] = sub i32 0, [[Op1]] +// CHECK-NEXT: [[AbsSigned:%.*]] = select i1 [[IsNeg]], i32 [[Signed]], i32 [[Op1]] +// CHECK-NEXT: call { i32, i1 } @llvm.umul.with.overflow.i32(i32 [[AbsSigned]], i32 %{{.*}}) +// CHECK-NEXT: [[UnsignedOFlow:%.*]] = extractvalue { i32, i1 } %{{.*}}, 1 +// CHECK-NEXT: [[UnsignedResult:%.*]] = extractvalue { i32, i1 } %{{.*}}, 0 +// CHECK-NEXT: [[IsNegZext:%.*]] = zext i1 [[IsNeg]] to i32 +// CHECK-NEXT: [[MaxResult:%.*]] = add i32 2147483647, [[IsNegZext]] +// CHECK-NEXT: [[SignedOFlow:%.*]] = icmp ugt i32 [[UnsignedResult]], [[MaxResult]] +// CHECK-NEXT: [[OFlow:%.*]] = or i1 [[UnsignedOFlow]], [[SignedOFlow]] +// CHECK-NEXT: [[NegativeResult:%.*]] = sub i32 0, [[UnsignedResult]] +// CHECK-NEXT: [[Result:%.*]] = select i1 [[IsNeg]], i32 [[NegativeResult]], i32 [[UnsignedResult]] +// CHECK-NEXT: store i32 [[Result]], i32* %{{.*}}, align 4 +// CHECK: br i1 [[OFlow]] + + int result; + if (__builtin_mul_overflow(x, y, &result)) + return LongErrorCode; + return result; +} + +int test_mixed_sign_mull_overflow_unsigned(int x, unsigned y) { +// CHECK: @test_mixed_sign_mull_overflow_unsigned +// CHECK: [[IsNeg:%.*]] = icmp slt i32 [[Op1:%.*]], 0 +// CHECK-NEXT: [[Signed:%.*]] = sub i32 0, [[Op1]] +// CHECK-NEXT: [[AbsSigned:%.*]] = select i1 [[IsNeg]], i32 [[Signed]], i32 [[Op1]] +// CHECK-NEXT: call { i32, i1 } @llvm.umul.with.overflow.i32(i32 [[AbsSigned]], i32 %{{.*}}) +// CHECK-NEXT: [[UnsignedOFlow:%.*]] = extractvalue { i32, i1 } %{{.*}}, 1 +// CHECK-NEXT: [[UnsignedResult:%.*]] = extractvalue { i32, i1 } %{{.*}}, 0 +// CHECK-NEXT: [[NotNull:%.*]] = icmp ne i32 [[UnsignedResult]], 0 +// CHECK-NEXT: [[Underflow:%.*]] = and i1 [[IsNeg]], [[NotNull]] +// CHECK-NEXT: [[OFlow:%.*]] = or i1 [[UnsignedOFlow]], [[Underflow]] +// CHECK-NEXT: store i32 [[UnsignedResult]], i32* %{{.*}}, align 4 +// CHECK: br i1 [[OFlow]] + + unsigned result; + if (__builtin_mul_overflow(x, y, &result)) + return LongErrorCode; + return result; +} + +int test_mixed_sign_mull_overflow_swapped(int x, unsigned y) { +// CHECK: @test_mixed_sign_mull_overflow_swapped +// CHECK: call { i32, i1 } @llvm.umul.with.overflow.i32 +// CHECK: add i32 2147483647 + int result; + if (__builtin_mul_overflow(y, x, &result)) + return LongErrorCode; + return result; +} + +long long test_mixed_sign_mulll_overflow(long long x, unsigned long long y) { +// CHECK: @test_mixed_sign_mulll_overflow +// CHECK: call { i64, i1 } @llvm.umul.with.overflow.i64 +// CHECK: add i64 92233720368547 + long long result; + if (__builtin_mul_overflow(x, y, &result)) + return LongLongErrorCode; + return result; +} + +long long test_mixed_sign_mulll_overflow_swapped(long long x, unsigned long long y) { +// CHECK: @test_mixed_sign_mulll_overflow_swapped +// CHECK: call { i64, i1 } @llvm.umul.with.overflow.i64 +// CHECK: add i64 92233720368547 + long long result; + if (__builtin_mul_overflow(y, x, &result)) + return LongLongErrorCode; + return result; +} + +long long test_mixed_sign_mulll_overflow_trunc_signed(long long x, unsigned long long y) { +// CHECK: @test_mixed_sign_mulll_overflow_trunc_signed +// CHECK: call { i64, i1 } @llvm.umul.with.overflow.i64 +// CHECK: add i64 2147483647 +// CHECK: trunc +// CHECK: store + int result; + if (__builtin_mul_overflow(y, x, &result)) + return LongLongErrorCode; + return result; +} + +long long test_mixed_sign_mulll_overflow_trunc_unsigned(long long x, unsigned long long y) { +// CHECK: @test_mixed_sign_mulll_overflow_trunc_unsigned +// CHECK: call { i64, i1 } @llvm.umul.with.overflow.i64 +// CHECK: [[NON_ZERO:%.*]] = icmp ne i64 [[UNSIGNED_RESULT:%.*]], 0 +// CHECK-NEXT: [[UNDERFLOW:%.*]] = and i1 {{.*}}, [[NON_ZERO]] +// CHECK-NEXT: [[OVERFLOW_PRE_TRUNC:%.*]] = or i1 {{.*}}, [[UNDERFLOW]] +// CHECK-NEXT: [[TRUNC_OVERFLOW:%.*]] = icmp ugt i64 [[UNSIGNED_RESULT]], 4294967295 +// CHECK-NEXT: [[OVERFLOW:%.*]] = or i1 [[OVERFLOW_PRE_TRUNC]], [[TRUNC_OVERFLOW]] +// CHECK-NEXT: trunc i64 [[UNSIGNED_RESULT]] to i32 +// CHECK-NEXT: store + unsigned result; + if (__builtin_mul_overflow(y, x, &result)) + return LongLongErrorCode; + return result; +} + +long long test_mixed_sign_mul_overflow_extend_signed(int x, unsigned y) { +// CHECK: @test_mixed_sign_mul_overflow_extend_signed +// CHECK: call { i64, i1 } @llvm.smul.with.overflow.i64 + long long result; + if (__builtin_mul_overflow(y, x, &result)) + return LongLongErrorCode; + return result; +} + +long long test_mixed_sign_mul_overflow_extend_unsigned(int x, unsigned y) { +// CHECK: @test_mixed_sign_mul_overflow_extend_unsigned +// CHECK: call { i65, i1 } @llvm.smul.with.overflow.i65 + unsigned long long result; + if (__builtin_mul_overflow(y, x, &result)) + return LongLongErrorCode; + return result; +}