[X86] Fix i128 argument passing under SysV ABI

[nikic]

Summary

The x86_64 SysV ABI specifies that __int128 is passed either in two registers (if available) or in a 16 byte aligned stack slot. GCC implements this behavior. However, if only one free register is available, LLVM will instead pass one half of the i128 in a register, and the other on the stack.

Make sure that either both are passed in registers or both on the stack.

Fixes https://github.com/llvm/llvm-project/issues/41784. The patch is basically what @craig.topper proposed to do there.

[pengfei]

Change looks good to me. Maybe add the ABI fix in clang ReleaseNotes?

[pengfei]

Do we have a way to detect it's from i128 and only do for it? I think we will increase unnecessary load/stores to normal 64-bit argument passing which would be more common in practice.

[craig.topper]

In D158169#4597591, @pengfei wrote:

Do we have a way to detect it's from i128 and only do for it? I think we will increase unnecessary load/stores to normal 64-bit argument passing which would be more common in practice.

I think the CCIfSplit means it was larger than i64 to start.

[tmgross]

See also https://reviews.llvm.org/D86310 which is needed in addition to this to fix i128 ABI for non-Clang LLVM frontends (I think that patch is also ready for a review at this point)

[pengfei]

In D158169#4597602, @craig.topper wrote:

In D158169#4597591, @pengfei wrote:

Do we have a way to detect it's from i128 and only do for it? I think we will increase unnecessary load/stores to normal 64-bit argument passing which would be more common in practice.

I think the CCIfSplit means it was larger than i64 to start.

That's great! Thanks Craig!

[RalfJung]

I think the CCIfSplit means it was larger than i64 to start.

What about the case where R9 would take the *second* half of a split i128 (i.e., the value gets split across R8 and R9)? Does that still work after this patch?

[nikic]

In D158169#4645012, @RalfJung wrote:

I think the CCIfSplit means it was larger than i64 to start.

What about the case where R9 would take the *second* half of a split i128 (i.e., the value gets split across R8 and R9)? Does that still work after this patch?

Yes, this is the case tested in @in_reg. CCIfSplit only matches the first part of a split value, not all parts of it.

clang/docs/ReleaseNotes.rst

 C/C++ Language Potentially Breaking Changes
 -------------------------------------------
 
 C++ Specific Potentially Breaking Changes
 -----------------------------------------
 
 ABI Changes in This Version
 ---------------------------
+- Following the SystemV ABI for x86-64, ``__int128`` arguments will no longer
+  be split between a register and a stack slot.
 
 What's New in Clang |release|?
 ==============================
 Some of the major new features and improvements to Clang are listed
 here. Generic improvements to Clang as a whole or to its underlying
 infrastructure are described first, followed by language-specific
 sections with improvements to Clang's support for those languages.
 

llvm/lib/Target/X86/X86CallingConv.td

def CC_X86_64_C : CallingConv<[
   CCIfCC<"CallingConv::SwiftTail",
     CCIfSRet<CCIfType<[i64], CCAssignToReg<[RAX]>>>>,
 
   // Pointers are always passed in full 64-bit registers.
   CCIfPtr<CCCustom<"CC_X86_64_Pointer">>,
 
   // The first 6 integer arguments are passed in integer registers.
   CCIfType<[i32], CCAssignToReg<[EDI, ESI, EDX, ECX, R8D, R9D]>>,
+
+  // i128 can be either passed in two i64 registers, or on the stack, but
+  // not split across register and stack. As such, do not allow using R9
+  // for a split i64.
+  CCIfType<[i64],
+           CCIfSplit<CCAssignToReg<[RDI, RSI, RDX, RCX, R8]>>>,
+  CCIfType<[i64], CCIfSplit<CCAssignToStackWithShadow<8, 16, [R9]>>>,
+
   CCIfType<[i64], CCAssignToReg<[RDI, RSI, RDX, RCX, R8 , R9 ]>>,
 
   // The first 8 MMX vector arguments are passed in XMM registers on Darwin.
   CCIfType<[x86mmx],
             CCIfSubtarget<"isTargetDarwin()",
             CCIfSubtarget<"hasSSE2()",
             CCPromoteToType<v2i64>>>>,
 

llvm/test/CodeGen/X86/addcarry.ll

entry:
   store i128 %1, ptr %b
   ret void
 }
 
 define i256 @add256(i256 %a, i256 %b) nounwind {
 ; CHECK-LABEL: add256:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movq %rdi, %rax
-; CHECK-NEXT:    addq %r9, %rsi
+; CHECK-NEXT:    addq {{[0-9]+}}(%rsp), %rsi
 ; CHECK-NEXT:    adcq {{[0-9]+}}(%rsp), %rdx
 ; CHECK-NEXT:    adcq {{[0-9]+}}(%rsp), %rcx
 ; CHECK-NEXT:    adcq {{[0-9]+}}(%rsp), %r8
 ; CHECK-NEXT:    movq %rcx, 16(%rdi)
 ; CHECK-NEXT:    movq %rdx, 8(%rdi)
 ; CHECK-NEXT:    movq %rsi, (%rdi)
 ; CHECK-NEXT:    movq %r8, 24(%rdi)
 ; CHECK-NEXT:    retq

llvm/test/CodeGen/X86/i128-abi.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --no_x86_scrub_sp --version 2
 ; RUN: llc -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
 
 define i128 @in_reg(i64 %a0, i64 %a1, i64 %a2, i64 %a3, i128 %a4) {
 ; CHECK-LABEL: in_reg:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movq %r9, %rdx
 ; CHECK-NEXT:    movq %r8, %rax
 ; CHECK-NEXT:    retq
   ret i128 %a4
 }
 
 define i128 @on_stack(i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i128 %a5) {
 ; CHECK-LABEL: on_stack:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    movq %r9, %rax
-; CHECK-NEXT:    movq 8(%rsp), %rdx
+; CHECK-NEXT:    movq 8(%rsp), %rax
+; CHECK-NEXT:    movq 16(%rsp), %rdx
 ; CHECK-NEXT:    retq
   ret i128 %a5
 }
 
 define i64 @trailing_arg_on_stack(i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i128 %a5, i64 %a6) {
 ; CHECK-LABEL: trailing_arg_on_stack:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    movq 16(%rsp), %rax
+; CHECK-NEXT:    movq 24(%rsp), %rax
 ; CHECK-NEXT:    retq
   ret i64 %a6
 }
 
 define void @call_in_reg(i128 %x) nounwind {
 ; CHECK-LABEL: call_in_reg:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    pushq %rax
; CHECK-NEXT:    retq
   call i128 @in_reg(i64 0, i64 1, i64 2, i64 3, i128 %x)
   ret void
 }
 
 define void @call_on_stack(i128 %x) nounwind {
 ; CHECK-LABEL: call_on_stack:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    movq %rsi, %rax
 ; CHECK-NEXT:    movq %rdi, %r9
-; CHECK-NEXT:    movq %rsi, (%rsp)
 ; CHECK-NEXT:    movl $1, %esi
 ; CHECK-NEXT:    movl $2, %edx
 ; CHECK-NEXT:    movl $3, %ecx
 ; CHECK-NEXT:    movl $4, %r8d
 ; CHECK-NEXT:    xorl %edi, %edi
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    pushq %r9
 ; CHECK-NEXT:    callq on_stack@PLT
+; CHECK-NEXT:    addq $16, %rsp
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    retq
   call i128 @on_stack(i64 0, i64 1, i64 2, i64 3, i64 4, i128 %x)
   ret void
 }
 
 define void @call_trailing_arg_on_stack(i128 %x, i64 %y) nounwind {
 ; CHECK-LABEL: call_trailing_arg_on_stack:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    pushq %rax
 ; CHECK-NEXT:    movq %rdx, %rax
-; CHECK-NEXT:    movq %rsi, %r10
-; CHECK-NEXT:    movq %rdi, %r9
+; CHECK-NEXT:    movq %rsi, %r9
+; CHECK-NEXT:    movq %rdi, %r10
+; CHECK-NEXT:    subq $8, %rsp
 ; CHECK-NEXT:    movl $1, %esi
 ; CHECK-NEXT:    movl $2, %edx
 ; CHECK-NEXT:    movl $3, %ecx
 ; CHECK-NEXT:    movl $4, %r8d
 ; CHECK-NEXT:    xorl %edi, %edi
 ; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    pushq %r9
 ; CHECK-NEXT:    pushq %r10
 ; CHECK-NEXT:    callq trailing_arg_on_stack@PLT
-; CHECK-NEXT:    addq $16, %rsp
+; CHECK-NEXT:    addq $32, %rsp
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    retq
   call i128 @trailing_arg_on_stack(i64 0, i64 1, i64 2, i64 3, i64 4, i128 %x, i64 %y)
   ret void
 }

llvm/test/CodeGen/X86/sadd_sat_vec.ll

 
 define <2 x i128> @v2i128(<2 x i128> %x, <2 x i128> %y) nounwind {
 ; SSE-LABEL: v2i128:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movq %rdi, %rax
 ; SSE-NEXT:    addq {{[0-9]+}}(%rsp), %rcx
 ; SSE-NEXT:    adcq {{[0-9]+}}(%rsp), %r8
 ; SSE-NEXT:    seto %dil
-; SSE-NEXT:    movq %r8, %r10
-; SSE-NEXT:    sarq $63, %r10
+; SSE-NEXT:    movq %r8, %r9
+; SSE-NEXT:    sarq $63, %r9
 ; SSE-NEXT:    testb %dil, %dil
-; SSE-NEXT:    cmovneq %r10, %rcx
-; SSE-NEXT:    movabsq $-9223372036854775808, %r11 # imm = 0x8000000000000000
-; SSE-NEXT:    xorq %r11, %r10
+; SSE-NEXT:    cmovneq %r9, %rcx
+; SSE-NEXT:    movabsq $-9223372036854775808, %r10 # imm = 0x8000000000000000
+; SSE-NEXT:    xorq %r10, %r9
 ; SSE-NEXT:    testb %dil, %dil
-; SSE-NEXT:    cmoveq %r8, %r10
-; SSE-NEXT:    addq %r9, %rsi
+; SSE-NEXT:    cmoveq %r8, %r9
+; SSE-NEXT:    addq {{[0-9]+}}(%rsp), %rsi
 ; SSE-NEXT:    adcq {{[0-9]+}}(%rsp), %rdx
 ; SSE-NEXT:    seto %dil
 ; SSE-NEXT:    movq %rdx, %r8
 ; SSE-NEXT:    sarq $63, %r8
 ; SSE-NEXT:    testb %dil, %dil
 ; SSE-NEXT:    cmovneq %r8, %rsi
-; SSE-NEXT:    xorq %r11, %r8
+; SSE-NEXT:    xorq %r10, %r8
 ; SSE-NEXT:    testb %dil, %dil
 ; SSE-NEXT:    cmoveq %rdx, %r8
 ; SSE-NEXT:    movq %rcx, 16(%rax)
 ; SSE-NEXT:    movq %rsi, (%rax)
-; SSE-NEXT:    movq %r10, 24(%rax)
+; SSE-NEXT:    movq %r9, 24(%rax)
 ; SSE-NEXT:    movq %r8, 8(%rax)
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: v2i128:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    movq %rdi, %rax
 ; AVX-NEXT:    addq {{[0-9]+}}(%rsp), %rcx
 ; AVX-NEXT:    adcq {{[0-9]+}}(%rsp), %r8
 ; AVX-NEXT:    seto %dil
-; AVX-NEXT:    movq %r8, %r10
-; AVX-NEXT:    sarq $63, %r10
+; AVX-NEXT:    movq %r8, %r9
+; AVX-NEXT:    sarq $63, %r9
 ; AVX-NEXT:    testb %dil, %dil
-; AVX-NEXT:    cmovneq %r10, %rcx
-; AVX-NEXT:    movabsq $-9223372036854775808, %r11 # imm = 0x8000000000000000
-; AVX-NEXT:    xorq %r11, %r10
+; AVX-NEXT:    cmovneq %r9, %rcx
+; AVX-NEXT:    movabsq $-9223372036854775808, %r10 # imm = 0x8000000000000000
+; AVX-NEXT:    xorq %r10, %r9
 ; AVX-NEXT:    testb %dil, %dil
-; AVX-NEXT:    cmoveq %r8, %r10
-; AVX-NEXT:    addq %r9, %rsi
+; AVX-NEXT:    cmoveq %r8, %r9
+; AVX-NEXT:    addq {{[0-9]+}}(%rsp), %rsi
 ; AVX-NEXT:    adcq {{[0-9]+}}(%rsp), %rdx
 ; AVX-NEXT:    seto %dil
 ; AVX-NEXT:    movq %rdx, %r8
 ; AVX-NEXT:    sarq $63, %r8
 ; AVX-NEXT:    testb %dil, %dil
 ; AVX-NEXT:    cmovneq %r8, %rsi
-; AVX-NEXT:    xorq %r11, %r8
+; AVX-NEXT:    xorq %r10, %r8
 ; AVX-NEXT:    testb %dil, %dil
 ; AVX-NEXT:    cmoveq %rdx, %r8
 ; AVX-NEXT:    movq %rcx, 16(%rax)
 ; AVX-NEXT:    movq %rsi, (%rax)
-; AVX-NEXT:    movq %r10, 24(%rax)
+; AVX-NEXT:    movq %r9, 24(%rax)
 ; AVX-NEXT:    movq %r8, 8(%rax)
 ; AVX-NEXT:    retq
   %z = call <2 x i128> @llvm.sadd.sat.v2i128(<2 x i128> %x, <2 x i128> %y)
   ret <2 x i128> %z
 }

llvm/test/CodeGen/X86/ssub_sat_vec.ll

 
 define <2 x i128> @v2i128(<2 x i128> %x, <2 x i128> %y) nounwind {
 ; SSE-LABEL: v2i128:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movq %rdi, %rax
 ; SSE-NEXT:    subq {{[0-9]+}}(%rsp), %rcx
 ; SSE-NEXT:    sbbq {{[0-9]+}}(%rsp), %r8
 ; SSE-NEXT:    seto %dil
-; SSE-NEXT:    movq %r8, %r10
-; SSE-NEXT:    sarq $63, %r10
+; SSE-NEXT:    movq %r8, %r9
+; SSE-NEXT:    sarq $63, %r9
 ; SSE-NEXT:    testb %dil, %dil
-; SSE-NEXT:    cmovneq %r10, %rcx
-; SSE-NEXT:    movabsq $-9223372036854775808, %r11 # imm = 0x8000000000000000
-; SSE-NEXT:    xorq %r11, %r10
+; SSE-NEXT:    cmovneq %r9, %rcx
+; SSE-NEXT:    movabsq $-9223372036854775808, %r10 # imm = 0x8000000000000000
+; SSE-NEXT:    xorq %r10, %r9
 ; SSE-NEXT:    testb %dil, %dil
-; SSE-NEXT:    cmoveq %r8, %r10
-; SSE-NEXT:    subq %r9, %rsi
+; SSE-NEXT:    cmoveq %r8, %r9
+; SSE-NEXT:    subq {{[0-9]+}}(%rsp), %rsi
 ; SSE-NEXT:    sbbq {{[0-9]+}}(%rsp), %rdx
 ; SSE-NEXT:    seto %dil
 ; SSE-NEXT:    movq %rdx, %r8
 ; SSE-NEXT:    sarq $63, %r8
 ; SSE-NEXT:    testb %dil, %dil
 ; SSE-NEXT:    cmovneq %r8, %rsi
-; SSE-NEXT:    xorq %r11, %r8
+; SSE-NEXT:    xorq %r10, %r8
 ; SSE-NEXT:    testb %dil, %dil
 ; SSE-NEXT:    cmoveq %rdx, %r8
 ; SSE-NEXT:    movq %rcx, 16(%rax)
 ; SSE-NEXT:    movq %rsi, (%rax)
-; SSE-NEXT:    movq %r10, 24(%rax)
+; SSE-NEXT:    movq %r9, 24(%rax)
 ; SSE-NEXT:    movq %r8, 8(%rax)
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: v2i128:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    movq %rdi, %rax
 ; AVX-NEXT:    subq {{[0-9]+}}(%rsp), %rcx
 ; AVX-NEXT:    sbbq {{[0-9]+}}(%rsp), %r8
 ; AVX-NEXT:    seto %dil
-; AVX-NEXT:    movq %r8, %r10
-; AVX-NEXT:    sarq $63, %r10
+; AVX-NEXT:    movq %r8, %r9
+; AVX-NEXT:    sarq $63, %r9
 ; AVX-NEXT:    testb %dil, %dil
-; AVX-NEXT:    cmovneq %r10, %rcx
-; AVX-NEXT:    movabsq $-9223372036854775808, %r11 # imm = 0x8000000000000000
-; AVX-NEXT:    xorq %r11, %r10
+; AVX-NEXT:    cmovneq %r9, %rcx
+; AVX-NEXT:    movabsq $-9223372036854775808, %r10 # imm = 0x8000000000000000
+; AVX-NEXT:    xorq %r10, %r9
 ; AVX-NEXT:    testb %dil, %dil
-; AVX-NEXT:    cmoveq %r8, %r10
-; AVX-NEXT:    subq %r9, %rsi
+; AVX-NEXT:    cmoveq %r8, %r9
+; AVX-NEXT:    subq {{[0-9]+}}(%rsp), %rsi
 ; AVX-NEXT:    sbbq {{[0-9]+}}(%rsp), %rdx
 ; AVX-NEXT:    seto %dil
 ; AVX-NEXT:    movq %rdx, %r8
 ; AVX-NEXT:    sarq $63, %r8
 ; AVX-NEXT:    testb %dil, %dil
 ; AVX-NEXT:    cmovneq %r8, %rsi
-; AVX-NEXT:    xorq %r11, %r8
+; AVX-NEXT:    xorq %r10, %r8
 ; AVX-NEXT:    testb %dil, %dil
 ; AVX-NEXT:    cmoveq %rdx, %r8
 ; AVX-NEXT:    movq %rcx, 16(%rax)
 ; AVX-NEXT:    movq %rsi, (%rax)
-; AVX-NEXT:    movq %r10, 24(%rax)
+; AVX-NEXT:    movq %r9, 24(%rax)
 ; AVX-NEXT:    movq %r8, 8(%rax)
 ; AVX-NEXT:    retq
   %z = call <2 x i128> @llvm.ssub.sat.v2i128(<2 x i128> %x, <2 x i128> %y)
   ret <2 x i128> %z
 }

llvm/test/CodeGen/X86/subcarry.ll

entry:
   %0 = sub i128 %a, %b
   ret i128 %0
 }
 
 define i256 @sub256(i256 %a, i256 %b) nounwind {
 ; CHECK-LABEL: sub256:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movq %rdi, %rax
-; CHECK-NEXT:    subq %r9, %rsi
+; CHECK-NEXT:    subq {{[0-9]+}}(%rsp), %rsi
 ; CHECK-NEXT:    sbbq {{[0-9]+}}(%rsp), %rdx
 ; CHECK-NEXT:    sbbq {{[0-9]+}}(%rsp), %rcx
 ; CHECK-NEXT:    sbbq {{[0-9]+}}(%rsp), %r8
 ; CHECK-NEXT:    movq %rcx, 16(%rdi)
 ; CHECK-NEXT:    movq %rdx, 8(%rdi)
 ; CHECK-NEXT:    movq %rsi, (%rdi)
 ; CHECK-NEXT:    movq %r8, 24(%rdi)
 ; CHECK-NEXT:    retq

llvm/test/CodeGen/X86/uadd_sat_vec.ll

; AVX512-NEXT:    retq
   %z = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> %x, <8 x i64> %y)
   ret <8 x i64> %z
 }
 
 define <2 x i128> @v2i128(<2 x i128> %x, <2 x i128> %y) nounwind {
 ; SSE-LABEL: v2i128:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movq %rdi, %rax
-; SSE-NEXT:    addq %r9, %rsi
+; SSE-NEXT:    addq {{[0-9]+}}(%rsp), %rsi
 ; SSE-NEXT:    adcq {{[0-9]+}}(%rsp), %rdx
 ; SSE-NEXT:    movq $-1, %rdi
-; SSE-NEXT:    cmovbq %rdi, %rsi
 ; SSE-NEXT:    cmovbq %rdi, %rdx
+; SSE-NEXT:    cmovbq %rdi, %rsi
 ; SSE-NEXT:    addq {{[0-9]+}}(%rsp), %rcx
 ; SSE-NEXT:    adcq {{[0-9]+}}(%rsp), %r8
 ; SSE-NEXT:    cmovbq %rdi, %r8
 ; SSE-NEXT:    cmovbq %rdi, %rcx
 ; SSE-NEXT:    movq %r8, 24(%rax)
 ; SSE-NEXT:    movq %rcx, 16(%rax)
 ; SSE-NEXT:    movq %rdx, 8(%rax)
 ; SSE-NEXT:    movq %rsi, (%rax)
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: v2i128:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    movq %rdi, %rax
-; AVX-NEXT:    addq %r9, %rsi
+; AVX-NEXT:    addq {{[0-9]+}}(%rsp), %rsi
 ; AVX-NEXT:    adcq {{[0-9]+}}(%rsp), %rdx
 ; AVX-NEXT:    movq $-1, %rdi
-; AVX-NEXT:    cmovbq %rdi, %rsi
 ; AVX-NEXT:    cmovbq %rdi, %rdx
+; AVX-NEXT:    cmovbq %rdi, %rsi
 ; AVX-NEXT:    addq {{[0-9]+}}(%rsp), %rcx
 ; AVX-NEXT:    adcq {{[0-9]+}}(%rsp), %r8
 ; AVX-NEXT:    cmovbq %rdi, %r8
 ; AVX-NEXT:    cmovbq %rdi, %rcx
 ; AVX-NEXT:    movq %r8, 24(%rax)
 ; AVX-NEXT:    movq %rcx, 16(%rax)
 ; AVX-NEXT:    movq %rdx, 8(%rax)
 ; AVX-NEXT:    movq %rsi, (%rax)
 ; AVX-NEXT:    retq
   %z = call <2 x i128> @llvm.uadd.sat.v2i128(<2 x i128> %x, <2 x i128> %y)
   ret <2 x i128> %z
 }

llvm/test/CodeGen/X86/usub_sat_vec.ll

; AVX512-NEXT:    retq
   ret <8 x i64> %z
 }
 
 define <2 x i128> @v2i128(<2 x i128> %x, <2 x i128> %y) nounwind {
 ; SSE-LABEL: v2i128:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movq %rdi, %rax
 ; SSE-NEXT:    xorl %edi, %edi
-; SSE-NEXT:    subq %r9, %rsi
+; SSE-NEXT:    subq {{[0-9]+}}(%rsp), %rsi
 ; SSE-NEXT:    sbbq {{[0-9]+}}(%rsp), %rdx
-; SSE-NEXT:    cmovbq %rdi, %rsi
 ; SSE-NEXT:    cmovbq %rdi, %rdx
+; SSE-NEXT:    cmovbq %rdi, %rsi
 ; SSE-NEXT:    subq {{[0-9]+}}(%rsp), %rcx
 ; SSE-NEXT:    sbbq {{[0-9]+}}(%rsp), %r8
 ; SSE-NEXT:    cmovbq %rdi, %r8
 ; SSE-NEXT:    cmovbq %rdi, %rcx
 ; SSE-NEXT:    movq %r8, 24(%rax)
 ; SSE-NEXT:    movq %rcx, 16(%rax)
 ; SSE-NEXT:    movq %rdx, 8(%rax)
 ; SSE-NEXT:    movq %rsi, (%rax)
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: v2i128:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    movq %rdi, %rax
 ; AVX-NEXT:    xorl %edi, %edi
-; AVX-NEXT:    subq %r9, %rsi
+; AVX-NEXT:    subq {{[0-9]+}}(%rsp), %rsi
 ; AVX-NEXT:    sbbq {{[0-9]+}}(%rsp), %rdx
-; AVX-NEXT:    cmovbq %rdi, %rsi
 ; AVX-NEXT:    cmovbq %rdi, %rdx
+; AVX-NEXT:    cmovbq %rdi, %rsi
 ; AVX-NEXT:    subq {{[0-9]+}}(%rsp), %rcx
 ; AVX-NEXT:    sbbq {{[0-9]+}}(%rsp), %r8
 ; AVX-NEXT:    cmovbq %rdi, %r8
 ; AVX-NEXT:    cmovbq %rdi, %rcx
 ; AVX-NEXT:    movq %r8, 24(%rax)
 ; AVX-NEXT:    movq %rcx, 16(%rax)
 ; AVX-NEXT:    movq %rdx, 8(%rax)
 ; AVX-NEXT:    movq %rsi, (%rax)