[CodeGen] Handle mixed-width ops in mixed-sign mul-with-overflow lowering

[vsk]

Summary

The special lowering for __builtin_mul_overflow introduced in r320902
fixed an ICE seen when passing mixed-sign operands to the builtin.

This patch extends the special lowering to cover mixed-width, mixed-sign
operands. In a few common scenarios, calls to muloti4 will no longer be
emitted.

This should address the latest comments in PR34920 and work around the
link failure seen in:

https://bugzilla.redhat.com/show_bug.cgi?id=1657544.

Testing:

• check-clang
• A/B output comparison with: https://gist.github.com/vedantk/3eb9c88f82e5c32f2e590555b4af5081

[efriedma]

This looks reasonable.

clang/lib/CodeGen/CGBuiltin.cpp

1273

This is a weird way to express this... could you instead compute the width of the signed operand and the unsigned operand, and compare those?

[vsk]

• Simplify logic per Eli's review feedback.

[efriedma]

LGTM

clang/lib/CodeGen/CGBuiltin.cpp

 }
 
 /// Determine if a binop is a checked mixed-sign multiply we can specialize.
 static bool isSpecialMixedSignMultiply(unsigned BuiltinID,
                                        WidthAndSignedness Op1Info,
                                        WidthAndSignedness Op2Info,
                                        WidthAndSignedness ResultInfo) {
   return BuiltinID == Builtin::BI__builtin_mul_overflow &&
-         Op1Info.Width == Op2Info.Width && Op1Info.Width >= ResultInfo.Width &&
+         std::max(Op1Info.Width, Op2Info.Width) >= ResultInfo.Width &&
          Op1Info.Signed != Op2Info.Signed;
 }
 
 /// Emit a checked mixed-sign multiply. This is a cheaper specialization of
 /// the generic checked-binop irgen.
 static RValue
 EmitCheckedMixedSignMultiply(CodeGenFunction &CGF, const clang::Expr *Op1,
                              WidthAndSignedness Op1Info, const clang::Expr *Op2,
                              WidthAndSignedness Op2Info,
                              const clang::Expr *ResultArg, QualType ResultQTy,
                              WidthAndSignedness ResultInfo) {
   assert(isSpecialMixedSignMultiply(Builtin::BI__builtin_mul_overflow, Op1Info,
                                     Op2Info, ResultInfo) &&
          "Not a mixed-sign multipliction we can specialize");
 
   // Emit the signed and unsigned operands.
   const clang::Expr *SignedOp = Op1Info.Signed ? Op1 : Op2;
   const clang::Expr *UnsignedOp = Op1Info.Signed ? Op2 : Op1;
   llvm::Value *Signed = CGF.EmitScalarExpr(SignedOp);
   llvm::Value *Unsigned = CGF.EmitScalarExpr(UnsignedOp);
+  unsigned SignedOpWidth = Op1Info.Signed ? Op1Info.Width : Op2Info.Width;
+  unsigned UnsignedOpWidth = Op1Info.Signed ? Op2Info.Width : Op1Info.Width;
+
+  // One of the operands may be smaller than the other. If so, [s|z]ext it.
+  if (SignedOpWidth < UnsignedOpWidth)
+    Signed = CGF.Builder.CreateSExt(Signed, Unsigned->getType(), "op.sext");
+  if (UnsignedOpWidth < SignedOpWidth)
+    Unsigned = CGF.Builder.CreateZExt(Unsigned, Signed->getType(), "op.zext");
 
   llvm::Type *OpTy = Signed->getType();
   llvm::Value *Zero = llvm::Constant::getNullValue(OpTy);
   Address ResultPtr = CGF.EmitPointerWithAlignment(ResultArg);
   llvm::Type *ResTy = ResultPtr.getElementType();
+  unsigned OpWidth = std::max(Op1Info.Width, Op2Info.Width);
 
   // Take the absolute value of the signed operand.
   llvm::Value *IsNegative = CGF.Builder.CreateICmpSLT(Signed, Zero);
   llvm::Value *AbsOfNegative = CGF.Builder.CreateSub(Zero, Signed);
   llvm::Value *AbsSigned =
       CGF.Builder.CreateSelect(IsNegative, AbsOfNegative, Signed);
 
   // Perform a checked unsigned multiplication.
   llvm::Value *UnsignedOverflow;
   llvm::Value *UnsignedResult =
       EmitOverflowIntrinsic(CGF, llvm::Intrinsic::umul_with_overflow, AbsSigned,
                             Unsigned, UnsignedOverflow);
 
   llvm::Value *Overflow, *Result;
   if (ResultInfo.Signed) {
     // Signed overflow occurs if the result is greater than INT_MAX or lesser
     // than INT_MIN, i.e when |Result| > (INT_MAX + IsNegative).
-    auto IntMax = llvm::APInt::getSignedMaxValue(ResultInfo.Width)
-                      .zextOrSelf(Op1Info.Width);
+    auto IntMax =
+        llvm::APInt::getSignedMaxValue(ResultInfo.Width).zextOrSelf(OpWidth);
     llvm::Value *MaxResult =
         CGF.Builder.CreateAdd(llvm::ConstantInt::get(OpTy, IntMax),
                               CGF.Builder.CreateZExt(IsNegative, OpTy));
     llvm::Value *SignedOverflow =
         CGF.Builder.CreateICmpUGT(UnsignedResult, MaxResult);
     Overflow = CGF.Builder.CreateOr(UnsignedOverflow, SignedOverflow);
 
     // Prepare the signed result (possibly by negating it).
     llvm::Value *NegativeResult = CGF.Builder.CreateNeg(UnsignedResult);
     llvm::Value *SignedResult =
         CGF.Builder.CreateSelect(IsNegative, NegativeResult, UnsignedResult);
     Result = CGF.Builder.CreateTrunc(SignedResult, ResTy);
   } else {
     // Unsigned overflow occurs if the result is < 0 or greater than UINT_MAX.
     llvm::Value *Underflow = CGF.Builder.CreateAnd(
         IsNegative, CGF.Builder.CreateIsNotNull(UnsignedResult));
     Overflow = CGF.Builder.CreateOr(UnsignedOverflow, Underflow);
-    if (ResultInfo.Width < Op1Info.Width) {
+    if (ResultInfo.Width < OpWidth) {
       auto IntMax =
-          llvm::APInt::getMaxValue(ResultInfo.Width).zext(Op1Info.Width);
+          llvm::APInt::getMaxValue(ResultInfo.Width).zext(OpWidth);
       llvm::Value *TruncOverflow = CGF.Builder.CreateICmpUGT(
           UnsignedResult, llvm::ConstantInt::get(OpTy, IntMax));
       Overflow = CGF.Builder.CreateOr(Overflow, TruncOverflow);
     }
 
     // Negate the product if it would be negative in infinite precision.
     Result = CGF.Builder.CreateSelect(
         IsNegative, CGF.Builder.CreateNeg(UnsignedResult), UnsignedResult);

clang/test/CodeGen/builtins-overflow.c

 // CHECK: @test_smulll_overflow
 // CHECK: %{{.+}} = call { i64, i1 } @llvm.smul.with.overflow.i64(i64 %{{.+}}, i64 %{{.+}})
   long long result;
   if (__builtin_smulll_overflow(x, y, &result))
     return LongLongErrorCode;
   return result;
 }
 
+int test_mixed_sign_mul_overflow_sext_signed_op(int x, unsigned long long y) {
+// CHECK: @test_mixed_sign_mul_overflow_sext_signed_op
+// CHECK: [[SignedOp:%.*]] = sext i32 %0 to i64
+// CHECK: [[IsNeg:%.*]] = icmp slt i64 [[SignedOp]], 0
+  int result;
+  if (__builtin_mul_overflow(x, y, &result))
+    return LongErrorCode;
+  return result;
+}
+
+int test_mixed_sign_mul_overflow_zext_unsigned_op(long long x, unsigned y) {
+// CHECK: @test_mixed_sign_mul_overflow_zext_unsigned_op
+// CHECK: [[UnsignedOp:%.*]] = zext i32 %1 to i64
+// CHECK: [[IsNeg:%.*]] = icmp slt i64 %0, 0
+// CHECK: @llvm.umul.with.overflow.i64({{.*}}, i64 [[UnsignedOp]])
+  int result;
+  if (__builtin_mul_overflow(x, y, &result))
+    return LongErrorCode;
+  return result;
+}
+
 int test_mixed_sign_mull_overflow(int x, unsigned y) {
 // CHECK: @test_mixed_sign_mull_overflow
 // CHECK:       [[IsNeg:%.*]] = icmp slt i32 [[Op1:%.*]], 0
 // CHECK-NEXT:  [[Signed:%.*]] = sub i32 0, [[Op1]]
 // CHECK-NEXT:  [[AbsSigned:%.*]] = select i1 [[IsNeg]], i32 [[Signed]], i32 [[Op1]]
 // CHECK-NEXT:  call { i32, i1 } @llvm.umul.with.overflow.i32(i32 [[AbsSigned]], i32 %{{.*}})
 // CHECK-NEXT:  [[UnsignedOFlow:%.*]] = extractvalue { i32, i1 } %{{.*}}, 1
 // CHECK-NEXT:  [[UnsignedResult:%.*]] = extractvalue { i32, i1 } %{{.*}}, 0