[X86ISelLowering] avoid emitting libcalls to __mulodi4()

[nickdesaulniers]

Summary

Similar to D108842, D108844, and D108926.

__has_builtin(builtin_mul_overflow) returns true for 32b x86 targets,
but Clang is deferring to compiler RT when encountering long long types.
This breaks ARCH=i386 + CONFIG_BLK_DEV_NBD=y builds of the Linux kernel
that are using builtin_mul_overflow with these types for these targets.

If the semantics of __has_builtin mean "the compiler resolves these,
always" then we shouldn't conditionally emit a libcall.

This will still need to be worked around in the Linux kernel in order to
continue to support these builds of the Linux kernel for this
target with older releases of clang.

Link: https://bugs.llvm.org/show_bug.cgi?id=28629
Link: https://github.com/ClangBuiltLinux/linux/issues/1438

[nickdesaulniers]

looks like I need to update CodeGen/X86/smul_fix_sat.ll and CodeGen/X86/xmulo.ll

[nickdesaulniers]

• fix up tests

[lebedev.ri]

Yay, LGTM, but please wait for @craig.topper / @RKSimon :)

[RKSimon]

llvm/test/CodeGen/X86/overflow-intrinsic-optimizations.ll

2

Please use the update_llc_test_checks.py script to generate checks

[nickdesaulniers]

• run update_llc_test_checks.py

llvm/test/CodeGen/X86/overflow-intrinsic-optimizations.ll

2

Honestly, update_llc_test_checks.py is a big hammer. It's going to add 84 lines to this test, which simply checks we don't call one function __mulodi4. I don't care what the resulting code is, as long as we don't call the function. I've added it as you asked, but I think it significantly hurts the readability of this test.

It's convenient for sure, but it ends up generating significant churn in the tests, and reviewers don't need to think about why a test was important or whether they're changing a test for the worst; they can just run an automated tool until the test passes; who cares what the original intent of the test ever was and whether we just steam-rolled it?

[craig.topper]

llvm/test/CodeGen/X86/overflow-intrinsic-optimizations.ll

2

I think I agree with Nick here. The only value this test has over xmulo.ll is that the test is named no__mulodi4. The -NOT was the most straightforward way to check exactly what the intent of this test is. With the script being run on it we lose that and might as well not have it and rely on xmulo.ll

[nickdesaulniers]

llvm/test/CodeGen/X86/overflow-intrinsic-optimizations.ll

2

That's reasonable; in that case @RKSimon would you prefer I either:

1. delete this newly added test, llvm/test/CodeGen/X86/overflow-intrinsic-optimizations.ll and rely on the other modified tests for coverage?
2. revert llvm/test/CodeGen/X86/overflow-intrinsic-optimizations.ll back to 369497?

(I guess there's also 3. commit code as is...)

[rengolin]

llvm/test/CodeGen/X86/overflow-intrinsic-optimizations.ll

2

Agreed. The original check was pretty clear to me what the test was doing.

[RKSimon]

llvm/test/CodeGen/X86/overflow-intrinsic-optimizations.ll

2

OK, going back to the original, minimal checks is fine as long as you're not intending to add many more tests to this file.

[nickdesaulniers]

• revert test

[nickdesaulniers]

bumping for review, @RKSimon

[RKSimon]

LGTM (sorry for delay)

llvm/lib/Target/X86/X86ISelLowering.cpp

   }
 
   if (!Subtarget.is64Bit()) {
     // These libcalls are not available in 32-bit.
     setLibcallName(RTLIB::SHL_I128, nullptr);
     setLibcallName(RTLIB::SRL_I128, nullptr);
     setLibcallName(RTLIB::SRA_I128, nullptr);
     setLibcallName(RTLIB::MUL_I128, nullptr);
+    setLibcallName(RTLIB::MULO_I64, nullptr);
     setLibcallName(RTLIB::MULO_I128, nullptr);
   }
 
   // Combine sin / cos into _sincos_stret if it is available.
   if (getLibcallName(RTLIB::SINCOS_STRET_F32) != nullptr &&
       getLibcallName(RTLIB::SINCOS_STRET_F64) != nullptr) {
     setOperationAction(ISD::FSINCOS, MVT::f64, Custom);
     setOperationAction(ISD::FSINCOS, MVT::f32, Custom);

llvm/test/CodeGen/X86/overflow-intrinsic-optimizations.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc %s -mtriple=i386 -o - | FileCheck %s
+
+define i1 @no__mulodi4(i32 %a, i64 %b, i32* %c) {
+; CHECK-LABEL: no__mulodi4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushl %ebp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    pushl %ebx
+; CHECK-NEXT:    .cfi_def_cfa_offset 12
+; CHECK-NEXT:    pushl %edi
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    .cfi_def_cfa_offset 20
+; CHECK-NEXT:    subl $12, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset %esi, -20
+; CHECK-NEXT:    .cfi_offset %edi, -16
+; CHECK-NEXT:    .cfi_offset %ebx, -12
+; CHECK-NEXT:    .cfi_offset %ebp, -8
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl %eax, %edi
+; CHECK-NEXT:    sarl $31, %edi
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    mull %ecx
+; CHECK-NEXT:    movl %edx, %esi
+; CHECK-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    mull %ecx
+; CHECK-NEXT:    movl %edx, %ebx
+; CHECK-NEXT:    movl %eax, %ecx
+; CHECK-NEXT:    addl %esi, %ecx
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; CHECK-NEXT:    adcl $0, %ebx
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    mull %ebp
+; CHECK-NEXT:    movl %edx, %esi
+; CHECK-NEXT:    addl %ecx, %eax
+; CHECK-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    adcl %ebx, %esi
+; CHECK-NEXT:    setb %cl
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    mull %ebp
+; CHECK-NEXT:    movl %eax, %ebx
+; CHECK-NEXT:    addl %esi, %ebx
+; CHECK-NEXT:    movzbl %cl, %eax
+; CHECK-NEXT:    adcl %eax, %edx
+; CHECK-NEXT:    movl %edx, (%esp) # 4-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl %eax, %ecx
+; CHECK-NEXT:    imull %edi, %ecx
+; CHECK-NEXT:    mull %edi
+; CHECK-NEXT:    movl %eax, %esi
+; CHECK-NEXT:    addl %ecx, %edx
+; CHECK-NEXT:    imull %ebp, %edi
+; CHECK-NEXT:    addl %edx, %edi
+; CHECK-NEXT:    sarl $31, %ebp
+; CHECK-NEXT:    movl %ebp, %eax
+; CHECK-NEXT:    imull {{[0-9]+}}(%esp)
+; CHECK-NEXT:    addl %esi, %eax
+; CHECK-NEXT:    adcl %edi, %edx
+; CHECK-NEXT:    addl %ebx, %eax
+; CHECK-NEXT:    adcl (%esp), %edx # 4-byte Folded Reload
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; CHECK-NEXT:    movl %esi, (%ecx)
+; CHECK-NEXT:    movl %esi, %ecx
+; CHECK-NEXT:    sarl $31, %ecx
+; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; CHECK-NEXT:    xorl %esi, %ecx
+; CHECK-NEXT:    sarl $31, %esi
+; CHECK-NEXT:    xorl %esi, %edx
+; CHECK-NEXT:    xorl %eax, %esi
+; CHECK-NEXT:    orl %edx, %esi
+; CHECK-NEXT:    orl %esi, %ecx
+; CHECK-NEXT:    setne %al
+; CHECK-NEXT:    addl $12, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 20
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popl %edi
+; CHECK-NEXT:    .cfi_def_cfa_offset 12
+; CHECK-NEXT:    popl %ebx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    popl %ebp
+; CHECK-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-NEXT:    retl
+entry:
+  %0 = sext i32 %a to i64
+  %1 = call { i64, i1 } @llvm.smul.with.overflow.i64(i64 %0, i64 %b)
+  %2 = extractvalue { i64, i1 } %1, 1
+  %3 = extractvalue { i64, i1 } %1, 0
+  %4 = trunc i64 %3 to i32
+  %5 = sext i32 %4 to i64
+  %6 = icmp ne i64 %3, %5
+  %7 = or i1 %2, %6
+  store i32 %4, i32* %c, align 4
+  ret i1 %7
+}
+
+declare { i64, i1 } @llvm.smul.with.overflow.i64(i64, i64)

llvm/test/CodeGen/X86/smul_fix_sat.ll

 ; X86-NEXT:    pushl %ebp
 ; X86-NEXT:    .cfi_def_cfa_offset 8
 ; X86-NEXT:    pushl %ebx
 ; X86-NEXT:    .cfi_def_cfa_offset 12
 ; X86-NEXT:    pushl %edi
 ; X86-NEXT:    .cfi_def_cfa_offset 16
 ; X86-NEXT:    pushl %esi
 ; X86-NEXT:    .cfi_def_cfa_offset 20
-; X86-NEXT:    pushl %eax
-; X86-NEXT:    .cfi_def_cfa_offset 24
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 32
 ; X86-NEXT:    .cfi_offset %esi, -20
 ; X86-NEXT:    .cfi_offset %edi, -16
 ; X86-NEXT:    .cfi_offset %ebx, -12
 ; X86-NEXT:    .cfi_offset %ebp, -8
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %edx, %ecx
+; X86-NEXT:    movl %edx, %ebp
+; X86-NEXT:    sarl $31, %ecx
+; X86-NEXT:    movl %eax, %edi
+; X86-NEXT:    imull %ecx, %edi
+; X86-NEXT:    mull %ecx
+; X86-NEXT:    movl %eax, %esi
+; X86-NEXT:    addl %edi, %edx
+; X86-NEXT:    movl %ebx, %edi
+; X86-NEXT:    imull %ebx, %ecx
+; X86-NEXT:    addl %edx, %ecx
+; X86-NEXT:    sarl $31, %edi
+; X86-NEXT:    movl %edi, %ebx
+; X86-NEXT:    imull %ebp, %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    movl %edi, %eax
+; X86-NEXT:    mull %ebp
+; X86-NEXT:    addl %ebx, %edx
+; X86-NEXT:    imull %ebp, %edi
+; X86-NEXT:    addl %edx, %edi
+; X86-NEXT:    addl %esi, %eax
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    adcl %ecx, %edi
+; X86-NEXT:    movl %ebp, %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    mull %ecx
+; X86-NEXT:    movl %edx, %ebp
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl $0, (%esp)
-; X86-NEXT:    movl %esp, %edi
-; X86-NEXT:    xorl %ebp, %ebp
-; X86-NEXT:    xorl %ebx, %ebx
+; X86-NEXT:    mull %ecx
+; X86-NEXT:    movl %edx, %ebx
 ; X86-NEXT:    movl %eax, %esi
+; X86-NEXT:    addl %ebp, %esi
+; X86-NEXT:    adcl $0, %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    mull %ecx
+; X86-NEXT:    movl %edx, %ecx
+; X86-NEXT:    movl %eax, %ebp
+; X86-NEXT:    addl %esi, %ebp
+; X86-NEXT:    adcl %ebx, %ecx
+; X86-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl %ebx, %eax
+; X86-NEXT:    mull {{[0-9]+}}(%esp)
+; X86-NEXT:    movl %edx, %esi
+; X86-NEXT:    addl %ecx, %eax
+; X86-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X86-NEXT:    adcl %ecx, %esi
+; X86-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT:    adcl %edi, %esi
+; X86-NEXT:    movl %ebp, %ecx
+; X86-NEXT:    sarl $31, %ecx
 ; X86-NEXT:    xorl %ecx, %esi
-; X86-NEXT:    movl $-1, %esi
-; X86-NEXT:    cmovsl %ebp, %esi
-; X86-NEXT:    sets %bl
-; X86-NEXT:    addl $2147483647, %ebx # imm = 0x7FFFFFFF
-; X86-NEXT:    pushl %edi
-; X86-NEXT:    .cfi_adjust_cfa_offset 4
-; X86-NEXT:    pushl %ecx
-; X86-NEXT:    .cfi_adjust_cfa_offset 4
-; X86-NEXT:    pushl %edx
-; X86-NEXT:    .cfi_adjust_cfa_offset 4
-; X86-NEXT:    pushl %eax
-; X86-NEXT:    .cfi_adjust_cfa_offset 4
-; X86-NEXT:    pushl {{[0-9]+}}(%esp)
-; X86-NEXT:    .cfi_adjust_cfa_offset 4
-; X86-NEXT:    calll __mulodi4
-; X86-NEXT:    addl $20, %esp
-; X86-NEXT:    .cfi_adjust_cfa_offset -20
-; X86-NEXT:    cmpl $0, (%esp)
-; X86-NEXT:    cmovnel %esi, %eax
-; X86-NEXT:    cmovnel %ebx, %edx
-; X86-NEXT:    addl $4, %esp
+; X86-NEXT:    xorl %eax, %ecx
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    xorl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl $-1, %edi
+; X86-NEXT:    cmovsl %eax, %edi
+; X86-NEXT:    sets %dl
+; X86-NEXT:    addl $2147483647, %edx # imm = 0x7FFFFFFF
+; X86-NEXT:    orl %esi, %ecx
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    cmovnel %edi, %eax
+; X86-NEXT:    cmovel %ebp, %edx
+; X86-NEXT:    addl $12, %esp
 ; X86-NEXT:    .cfi_def_cfa_offset 20
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    .cfi_def_cfa_offset 16
 ; X86-NEXT:    popl %edi
 ; X86-NEXT:    .cfi_def_cfa_offset 12
 ; X86-NEXT:    popl %ebx
 ; X86-NEXT:    .cfi_def_cfa_offset 8
 ; X86-NEXT:    popl %ebp

llvm/test/CodeGen/X86/xmulo.ll

 ; WIN64:       # %bb.0:
 ; WIN64-NEXT:    imulq %rdx, %rcx
 ; WIN64-NEXT:    seto %al
 ; WIN64-NEXT:    movq %rcx, (%r8)
 ; WIN64-NEXT:    retq
 ;
 ; WIN32-LABEL: smuloi64:
 ; WIN32:       # %bb.0:
+; WIN32-NEXT:    pushl %ebp
 ; WIN32-NEXT:    pushl %ebx
 ; WIN32-NEXT:    pushl %edi
 ; WIN32-NEXT:    pushl %esi
-; WIN32-NEXT:    pushl %eax
-; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; WIN32-NEXT:    subl $8, %esp
 ; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; WIN32-NEXT:    movl $0, (%esp)
-; WIN32-NEXT:    movl %esp, %ebx
-; WIN32-NEXT:    pushl %ebx
-; WIN32-NEXT:    pushl %edi
-; WIN32-NEXT:    pushl %edx
-; WIN32-NEXT:    pushl %ecx
-; WIN32-NEXT:    pushl %eax
-; WIN32-NEXT:    calll ___mulodi4
-; WIN32-NEXT:    addl $20, %esp
-; WIN32-NEXT:    cmpl $0, (%esp)
-; WIN32-NEXT:    setne %cl
-; WIN32-NEXT:    movl %edx, 4(%esi)
-; WIN32-NEXT:    movl %eax, (%esi)
-; WIN32-NEXT:    movl %ecx, %eax
-; WIN32-NEXT:    addl $4, %esp
+; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; WIN32-NEXT:    movl %edx, %ecx
+; WIN32-NEXT:    movl %edx, %ebx
+; WIN32-NEXT:    sarl $31, %ecx
+; WIN32-NEXT:    movl %eax, %esi
+; WIN32-NEXT:    imull %ecx, %esi
+; WIN32-NEXT:    mull %ecx
+; WIN32-NEXT:    movl %eax, %ebp
+; WIN32-NEXT:    addl %esi, %edx
+; WIN32-NEXT:    movl %edi, %esi
+; WIN32-NEXT:    imull %edi, %ecx
+; WIN32-NEXT:    addl %edx, %ecx
+; WIN32-NEXT:    sarl $31, %esi
+; WIN32-NEXT:    movl %esi, %edi
+; WIN32-NEXT:    imull %ebx, %edi
+; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; WIN32-NEXT:    movl %esi, %eax
+; WIN32-NEXT:    mull %ebx
+; WIN32-NEXT:    addl %edi, %edx
+; WIN32-NEXT:    imull %ebx, %esi
+; WIN32-NEXT:    addl %edx, %esi
+; WIN32-NEXT:    addl %ebp, %eax
+; WIN32-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; WIN32-NEXT:    adcl %ecx, %esi
+; WIN32-NEXT:    movl %ebx, %eax
+; WIN32-NEXT:    movl %ebx, %edi
+; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; WIN32-NEXT:    mull %ecx
+; WIN32-NEXT:    movl %edx, %ebx
+; WIN32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; WIN32-NEXT:    mull %ecx
+; WIN32-NEXT:    movl %edx, %ebp
+; WIN32-NEXT:    movl %eax, %ecx
+; WIN32-NEXT:    addl %ebx, %ecx
+; WIN32-NEXT:    adcl $0, %ebp
+; WIN32-NEXT:    movl %edi, %eax
+; WIN32-NEXT:    mull {{[0-9]+}}(%esp)
+; WIN32-NEXT:    movl %edx, %ebx
+; WIN32-NEXT:    movl %eax, %edi
+; WIN32-NEXT:    addl %ecx, %edi
+; WIN32-NEXT:    adcl %ebp, %ebx
+; WIN32-NEXT:    setb %cl
+; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; WIN32-NEXT:    mull {{[0-9]+}}(%esp)
+; WIN32-NEXT:    addl %ebx, %eax
+; WIN32-NEXT:    movzbl %cl, %ecx
+; WIN32-NEXT:    adcl %ecx, %edx
+; WIN32-NEXT:    addl (%esp), %eax # 4-byte Folded Reload
+; WIN32-NEXT:    adcl %esi, %edx
+; WIN32-NEXT:    movl %edi, %ecx
+; WIN32-NEXT:    sarl $31, %ecx
+; WIN32-NEXT:    xorl %ecx, %edx
+; WIN32-NEXT:    xorl %eax, %ecx
+; WIN32-NEXT:    orl %edx, %ecx
+; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; WIN32-NEXT:    movl %edi, 4(%eax)
+; WIN32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; WIN32-NEXT:    movl %ecx, (%eax)
+; WIN32-NEXT:    setne %al
+; WIN32-NEXT:    addl $8, %esp
 ; WIN32-NEXT:    popl %esi
 ; WIN32-NEXT:    popl %edi
 ; WIN32-NEXT:    popl %ebx
+; WIN32-NEXT:    popl %ebp
 ; WIN32-NEXT:    retl
   %t = call {i64, i1} @llvm.smul.with.overflow.i64(i64 %v1, i64 %v2)
   %val = extractvalue {i64, i1} %t, 0
   %obit = extractvalue {i64, i1} %t, 1
   store i64 %val, i64* %res
   ret i1 %obit
 }
 

 ;
 ; WIN32-LABEL: smuloselecti64:
 ; WIN32:       # %bb.0:
 ; WIN32-NEXT:    pushl %ebp
 ; WIN32-NEXT:    pushl %ebx
 ; WIN32-NEXT:    pushl %edi
 ; WIN32-NEXT:    pushl %esi
 ; WIN32-NEXT:    pushl %eax
-; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %ebp
 ; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; WIN32-NEXT:    movl %eax, %ecx
+; WIN32-NEXT:    movl %eax, %esi
+; WIN32-NEXT:    sarl $31, %ecx
+; WIN32-NEXT:    movl %ebp, %edi
+; WIN32-NEXT:    imull %ecx, %edi
+; WIN32-NEXT:    movl %ebp, %eax
+; WIN32-NEXT:    mull %ecx
+; WIN32-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; WIN32-NEXT:    addl %edi, %edx
+; WIN32-NEXT:    imull %ebx, %ecx
+; WIN32-NEXT:    addl %edx, %ecx
+; WIN32-NEXT:    sarl $31, %ebx
+; WIN32-NEXT:    movl %ebx, %edi
+; WIN32-NEXT:    imull %esi, %edi
+; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; WIN32-NEXT:    movl %ebx, %eax
+; WIN32-NEXT:    mull %esi
+; WIN32-NEXT:    addl %edi, %edx
+; WIN32-NEXT:    movl %esi, %edi
+; WIN32-NEXT:    imull %esi, %ebx
+; WIN32-NEXT:    addl %edx, %ebx
+; WIN32-NEXT:    addl (%esp), %eax # 4-byte Folded Reload
+; WIN32-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; WIN32-NEXT:    adcl %ecx, %ebx
+; WIN32-NEXT:    movl %edi, %eax
+; WIN32-NEXT:    mull %ebp
+; WIN32-NEXT:    movl %edx, %esi
+; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; WIN32-NEXT:    mull %ebp
+; WIN32-NEXT:    movl %edx, %ebp
+; WIN32-NEXT:    movl %eax, %ecx
+; WIN32-NEXT:    addl %esi, %ecx
+; WIN32-NEXT:    adcl $0, %ebp
+; WIN32-NEXT:    movl %edi, %eax
+; WIN32-NEXT:    mull {{[0-9]+}}(%esp)
+; WIN32-NEXT:    movl %edx, %esi
+; WIN32-NEXT:    movl %eax, %edi
+; WIN32-NEXT:    addl %ecx, %edi
+; WIN32-NEXT:    adcl %ebp, %esi
 ; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; WIN32-NEXT:    movl $0, (%esp)
-; WIN32-NEXT:    movl %esp, %eax
-; WIN32-NEXT:    pushl %eax
-; WIN32-NEXT:    pushl %ebp
-; WIN32-NEXT:    pushl %ebx
-; WIN32-NEXT:    pushl %edi
-; WIN32-NEXT:    pushl %esi
-; WIN32-NEXT:    calll ___mulodi4
-; WIN32-NEXT:    addl $20, %esp
-; WIN32-NEXT:    cmpl $0, (%esp)
+; WIN32-NEXT:    setb %cl
+; WIN32-NEXT:    movl %ebp, %eax
+; WIN32-NEXT:    mull {{[0-9]+}}(%esp)
+; WIN32-NEXT:    addl %esi, %eax
+; WIN32-NEXT:    movzbl %cl, %ecx
+; WIN32-NEXT:    adcl %ecx, %edx
+; WIN32-NEXT:    addl (%esp), %eax # 4-byte Folded Reload
+; WIN32-NEXT:    adcl %ebx, %edx
+; WIN32-NEXT:    sarl $31, %edi
+; WIN32-NEXT:    xorl %edi, %edx
+; WIN32-NEXT:    xorl %eax, %edi
+; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; WIN32-NEXT:    orl %edx, %edi
 ; WIN32-NEXT:    jne LBB12_2
 ; WIN32-NEXT:  # %bb.1:
-; WIN32-NEXT:    movl %ebx, %esi
-; WIN32-NEXT:    movl %ebp, %edi
+; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %ebp
 ; WIN32-NEXT:  LBB12_2:
-; WIN32-NEXT:    movl %esi, %eax
-; WIN32-NEXT:    movl %edi, %edx
+; WIN32-NEXT:    movl %ebp, %edx
 ; WIN32-NEXT:    addl $4, %esp
 ; WIN32-NEXT:    popl %esi
 ; WIN32-NEXT:    popl %edi
 ; WIN32-NEXT:    popl %ebx
 ; WIN32-NEXT:    popl %ebp
 ; WIN32-NEXT:    retl
   %t = call {i64, i1} @llvm.smul.with.overflow.i64(i64 %v1, i64 %v2)
   %obit = extractvalue {i64, i1} %t, 1

 ; WIN64-NEXT:    movb $1, %al
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:  .LBB18_1: # %overflow
 ; WIN64-NEXT:    xorl %eax, %eax
 ; WIN64-NEXT:    retq
 ;
 ; WIN32-LABEL: smulobri64:
 ; WIN32:       # %bb.0:
+; WIN32-NEXT:    pushl %ebp
+; WIN32-NEXT:    pushl %ebx
 ; WIN32-NEXT:    pushl %edi
 ; WIN32-NEXT:    pushl %esi
 ; WIN32-NEXT:    pushl %eax
 ; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; WIN32-NEXT:    movl $0, (%esp)
-; WIN32-NEXT:    movl %esp, %edi
-; WIN32-NEXT:    pushl %edi
-; WIN32-NEXT:    pushl %esi
-; WIN32-NEXT:    pushl %edx
-; WIN32-NEXT:    pushl %ecx
-; WIN32-NEXT:    pushl %eax
-; WIN32-NEXT:    calll ___mulodi4
-; WIN32-NEXT:    addl $20, %esp
-; WIN32-NEXT:    cmpl $0, (%esp)
+; WIN32-NEXT:    movl %edx, %ecx
+; WIN32-NEXT:    movl %edx, %ebx
+; WIN32-NEXT:    sarl $31, %ecx
+; WIN32-NEXT:    movl %eax, %esi
+; WIN32-NEXT:    imull %ecx, %esi
+; WIN32-NEXT:    mull %ecx
+; WIN32-NEXT:    movl %eax, %ebp
+; WIN32-NEXT:    addl %esi, %edx
+; WIN32-NEXT:    movl %edi, %esi
+; WIN32-NEXT:    imull %edi, %ecx
+; WIN32-NEXT:    addl %edx, %ecx
+; WIN32-NEXT:    sarl $31, %esi
+; WIN32-NEXT:    movl %esi, %edi
+; WIN32-NEXT:    imull %ebx, %edi
+; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; WIN32-NEXT:    movl %esi, %eax
+; WIN32-NEXT:    mull %ebx
+; WIN32-NEXT:    addl %edi, %edx
+; WIN32-NEXT:    imull %ebx, %esi
+; WIN32-NEXT:    addl %edx, %esi
+; WIN32-NEXT:    addl %ebp, %eax
+; WIN32-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; WIN32-NEXT:    adcl %ecx, %esi
+; WIN32-NEXT:    movl %ebx, %eax
+; WIN32-NEXT:    movl %ebx, %edi
+; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; WIN32-NEXT:    mull %ecx
+; WIN32-NEXT:    movl %edx, %ebp
+; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; WIN32-NEXT:    mull %ecx
+; WIN32-NEXT:    movl %edx, %ebx
+; WIN32-NEXT:    movl %eax, %ecx
+; WIN32-NEXT:    addl %ebp, %ecx
+; WIN32-NEXT:    adcl $0, %ebx
+; WIN32-NEXT:    movl %edi, %eax
+; WIN32-NEXT:    mull {{[0-9]+}}(%esp)
+; WIN32-NEXT:    movl %edx, %ebp
+; WIN32-NEXT:    movl %eax, %edi
+; WIN32-NEXT:    addl %ecx, %edi
+; WIN32-NEXT:    adcl %ebx, %ebp
+; WIN32-NEXT:    setb %cl
+; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; WIN32-NEXT:    mull {{[0-9]+}}(%esp)
+; WIN32-NEXT:    addl %ebp, %eax
+; WIN32-NEXT:    movzbl %cl, %ecx
+; WIN32-NEXT:    adcl %ecx, %edx
+; WIN32-NEXT:    addl (%esp), %eax # 4-byte Folded Reload
+; WIN32-NEXT:    adcl %esi, %edx
+; WIN32-NEXT:    sarl $31, %edi
+; WIN32-NEXT:    xorl %edi, %edx
+; WIN32-NEXT:    xorl %eax, %edi
+; WIN32-NEXT:    orl %edx, %edi
 ; WIN32-NEXT:    jne LBB18_1
 ; WIN32-NEXT:  # %bb.3: # %continue
 ; WIN32-NEXT:    movb $1, %al
 ; WIN32-NEXT:  LBB18_2: # %overflow
 ; WIN32-NEXT:    addl $4, %esp
 ; WIN32-NEXT:    popl %esi
 ; WIN32-NEXT:    popl %edi
+; WIN32-NEXT:    popl %ebx
+; WIN32-NEXT:    popl %ebp
 ; WIN32-NEXT:    retl
 ; WIN32-NEXT:  LBB18_1: # %overflow
 ; WIN32-NEXT:    xorl %eax, %eax
 ; WIN32-NEXT:    jmp LBB18_2
   %t = call {i64, i1} @llvm.smul.with.overflow.i64(i64 %v1, i64 %v2)
   %val = extractvalue {i64, i1} %t, 0
   %obit = extractvalue {i64, i1} %t, 1
   br i1 %obit, label %overflow, label %continue, !prof !0

 ; WIN64:       # %bb.0:
 ; WIN64-NEXT:    imulq (%rcx), %rdx
 ; WIN64-NEXT:    seto %al
 ; WIN64-NEXT:    movq %rdx, (%r8)
 ; WIN64-NEXT:    retq
 ;
 ; WIN32-LABEL: smuloi64_load:
 ; WIN32:       # %bb.0:
+; WIN32-NEXT:    pushl %ebp
 ; WIN32-NEXT:    pushl %ebx
 ; WIN32-NEXT:    pushl %edi
 ; WIN32-NEXT:    pushl %esi
-; WIN32-NEXT:    pushl %eax
-; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; WIN32-NEXT:    subl $16, %esp
+; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %ebx
 ; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; WIN32-NEXT:    movl (%eax), %esi
+; WIN32-NEXT:    movl 4(%eax), %ebp
+; WIN32-NEXT:    sarl $31, %ebx
+; WIN32-NEXT:    movl %ebx, %ecx
+; WIN32-NEXT:    imull %ebp, %ecx
+; WIN32-NEXT:    movl %ebx, %eax
+; WIN32-NEXT:    mull %esi
+; WIN32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; WIN32-NEXT:    addl %ecx, %edx
+; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; WIN32-NEXT:    imull %esi, %ebx
+; WIN32-NEXT:    addl %edx, %ebx
+; WIN32-NEXT:    movl %ebp, %ecx
+; WIN32-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; WIN32-NEXT:    sarl $31, %ecx
+; WIN32-NEXT:    movl %eax, %edi
+; WIN32-NEXT:    imull %ecx, %edi
+; WIN32-NEXT:    mull %ecx
+; WIN32-NEXT:    addl %edi, %edx
+; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; WIN32-NEXT:    imull %edi, %ecx
+; WIN32-NEXT:    addl %edx, %ecx
+; WIN32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; WIN32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; WIN32-NEXT:    adcl %ebx, %ecx
+; WIN32-NEXT:    movl %esi, %eax
+; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; WIN32-NEXT:    mull %edi
+; WIN32-NEXT:    movl %edx, %ebx
+; WIN32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; WIN32-NEXT:    movl %ebp, %eax
+; WIN32-NEXT:    mull %edi
+; WIN32-NEXT:    movl %edx, %ebp
+; WIN32-NEXT:    movl %eax, %edi
+; WIN32-NEXT:    addl %ebx, %edi
+; WIN32-NEXT:    adcl $0, %ebp
+; WIN32-NEXT:    movl %esi, %eax
 ; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; WIN32-NEXT:    movl (%edx), %edi
-; WIN32-NEXT:    movl 4(%edx), %edx
-; WIN32-NEXT:    movl $0, (%esp)
-; WIN32-NEXT:    movl %esp, %ebx
-; WIN32-NEXT:    pushl %ebx
-; WIN32-NEXT:    pushl %ecx
-; WIN32-NEXT:    pushl %eax
-; WIN32-NEXT:    pushl %edx
-; WIN32-NEXT:    pushl %edi
-; WIN32-NEXT:    calll ___mulodi4
-; WIN32-NEXT:    addl $20, %esp
-; WIN32-NEXT:    cmpl $0, (%esp)
-; WIN32-NEXT:    setne %cl
-; WIN32-NEXT:    movl %eax, (%esi)
-; WIN32-NEXT:    movl %edx, 4(%esi)
-; WIN32-NEXT:    movl %ecx, %eax
-; WIN32-NEXT:    addl $4, %esp
+; WIN32-NEXT:    mull %edx
+; WIN32-NEXT:    movl %edx, %esi
+; WIN32-NEXT:    movl %eax, %ebx
+; WIN32-NEXT:    addl %edi, %ebx
+; WIN32-NEXT:    adcl %ebp, %esi
+; WIN32-NEXT:    setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; WIN32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; WIN32-NEXT:    mull {{[0-9]+}}(%esp)
+; WIN32-NEXT:    addl %esi, %eax
+; WIN32-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 1-byte Folded Reload
+; WIN32-NEXT:    adcl %esi, %edx
+; WIN32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; WIN32-NEXT:    adcl %ecx, %edx
+; WIN32-NEXT:    movl %ebx, %ecx
+; WIN32-NEXT:    sarl $31, %ecx
+; WIN32-NEXT:    xorl %ecx, %edx
+; WIN32-NEXT:    xorl %eax, %ecx
+; WIN32-NEXT:    orl %edx, %ecx
+; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; WIN32-NEXT:    movl %ebx, 4(%eax)
+; WIN32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; WIN32-NEXT:    movl %ecx, (%eax)
+; WIN32-NEXT:    setne %al
+; WIN32-NEXT:    addl $16, %esp
 ; WIN32-NEXT:    popl %esi
 ; WIN32-NEXT:    popl %edi
 ; WIN32-NEXT:    popl %ebx
+; WIN32-NEXT:    popl %ebp
 ; WIN32-NEXT:    retl
   %v1 = load i64, i64* %ptr1
   %t = call {i64, i1} @llvm.smul.with.overflow.i64(i64 %v1, i64 %v2)
   %val = extractvalue {i64, i1} %t, 0
   %obit = extractvalue {i64, i1} %t, 1
   store i64 %val, i64* %res
   ret i1 %obit
 }

 ; WIN64:       # %bb.0:
 ; WIN64-NEXT:    imulq (%rdx), %rcx
 ; WIN64-NEXT:    seto %al
 ; WIN64-NEXT:    movq %rcx, (%r8)
 ; WIN64-NEXT:    retq
 ;
 ; WIN32-LABEL: smuloi64_load2:
 ; WIN32:       # %bb.0:
+; WIN32-NEXT:    pushl %ebp
 ; WIN32-NEXT:    pushl %ebx
 ; WIN32-NEXT:    pushl %edi
 ; WIN32-NEXT:    pushl %esi
-; WIN32-NEXT:    pushl %eax
-; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; WIN32-NEXT:    subl $12, %esp
 ; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; WIN32-NEXT:    movl (%edx), %edi
-; WIN32-NEXT:    movl 4(%edx), %edx
-; WIN32-NEXT:    movl $0, (%esp)
-; WIN32-NEXT:    movl %esp, %ebx
-; WIN32-NEXT:    pushl %ebx
-; WIN32-NEXT:    pushl %edx
-; WIN32-NEXT:    pushl %edi
-; WIN32-NEXT:    pushl %ecx
-; WIN32-NEXT:    pushl %eax
-; WIN32-NEXT:    calll ___mulodi4
-; WIN32-NEXT:    addl $20, %esp
-; WIN32-NEXT:    cmpl $0, (%esp)
-; WIN32-NEXT:    setne %cl
-; WIN32-NEXT:    movl %eax, (%esi)
-; WIN32-NEXT:    movl %edx, 4(%esi)
-; WIN32-NEXT:    movl %ecx, %eax
-; WIN32-NEXT:    addl $4, %esp
+; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; WIN32-NEXT:    movl (%eax), %ebp
+; WIN32-NEXT:    movl 4(%eax), %edi
+; WIN32-NEXT:    movl %edx, %ecx
+; WIN32-NEXT:    movl %edx, %ebx
+; WIN32-NEXT:    sarl $31, %ecx
+; WIN32-NEXT:    movl %ebp, %esi
+; WIN32-NEXT:    imull %ecx, %esi
+; WIN32-NEXT:    movl %ebp, %eax
+; WIN32-NEXT:    mull %ecx
+; WIN32-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; WIN32-NEXT:    addl %esi, %edx
+; WIN32-NEXT:    movl %edi, %esi
+; WIN32-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; WIN32-NEXT:    imull %edi, %ecx
+; WIN32-NEXT:    addl %edx, %ecx
+; WIN32-NEXT:    sarl $31, %esi
+; WIN32-NEXT:    movl %esi, %edi
+; WIN32-NEXT:    imull %ebx, %edi
+; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; WIN32-NEXT:    movl %esi, %eax
+; WIN32-NEXT:    mull %ebx
+; WIN32-NEXT:    addl %edi, %edx
+; WIN32-NEXT:    imull %ebx, %esi
+; WIN32-NEXT:    addl %edx, %esi
+; WIN32-NEXT:    addl (%esp), %eax # 4-byte Folded Reload
+; WIN32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; WIN32-NEXT:    adcl %ecx, %esi
+; WIN32-NEXT:    movl %ebx, %eax
+; WIN32-NEXT:    movl %ebx, %edi
+; WIN32-NEXT:    mull %ebp
+; WIN32-NEXT:    movl %edx, %ebx
+; WIN32-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; WIN32-NEXT:    mull %ebp
+; WIN32-NEXT:    movl %edx, %ebp
+; WIN32-NEXT:    movl %eax, %ecx
+; WIN32-NEXT:    addl %ebx, %ecx
+; WIN32-NEXT:    adcl $0, %ebp
+; WIN32-NEXT:    movl %edi, %eax
+; WIN32-NEXT:    mull {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; WIN32-NEXT:    movl %edx, %ebx
+; WIN32-NEXT:    movl %eax, %edi
+; WIN32-NEXT:    addl %ecx, %edi
+; WIN32-NEXT:    adcl %ebp, %ebx
+; WIN32-NEXT:    setb %cl
+; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; WIN32-NEXT:    mull {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; WIN32-NEXT:    addl %ebx, %eax
+; WIN32-NEXT:    movzbl %cl, %ecx
+; WIN32-NEXT:    adcl %ecx, %edx
+; WIN32-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; WIN32-NEXT:    adcl %esi, %edx
+; WIN32-NEXT:    movl %edi, %ecx
+; WIN32-NEXT:    sarl $31, %ecx
+; WIN32-NEXT:    xorl %ecx, %edx
+; WIN32-NEXT:    xorl %eax, %ecx
+; WIN32-NEXT:    orl %edx, %ecx
+; WIN32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; WIN32-NEXT:    movl %edi, 4(%eax)
+; WIN32-NEXT:    movl (%esp), %ecx # 4-byte Reload
+; WIN32-NEXT:    movl %ecx, (%eax)
+; WIN32-NEXT:    setne %al
+; WIN32-NEXT:    addl $12, %esp
 ; WIN32-NEXT:    popl %esi
 ; WIN32-NEXT:    popl %edi
 ; WIN32-NEXT:    popl %ebx
+; WIN32-NEXT:    popl %ebp
 ; WIN32-NEXT:    retl
   %v2 = load i64, i64* %ptr2
   %t = call {i64, i1} @llvm.smul.with.overflow.i64(i64 %v1, i64 %v2)
   %val = extractvalue {i64, i1} %t, 0
   %obit = extractvalue {i64, i1} %t, 1
   store i64 %val, i64* %res
   ret i1 %obit
 }