[DAGCombine] narrowInsertExtractVectorBinOp - add CONCAT_VECTORS support

[RKSimon]

Summary

We already split extract_subvector(binop(insert_subvector(v,x),insert_subvector(w,y))) -> binop(x,y).

This patch adds support for extract_subvector(binop(concat_vectors(),concat_vectors())) cases as well.

In particular this means we don't have to wait for X86 lowering to convert concat_vectors to insert_subvector chains, which helps avoid some cases where demandedelts/combine calls occur too late to split large vector ops.

The fast-isel-store.ll regression is annoying but I don't think is that critical?

[RKSimon]

ping?

[spatel]

I don't have a good sense of how we make fast-isel speed vs. perf trade-offs, so if anyone else has thoughts about that case, feel free to comment.

The optimization for regular combining overrides that concern for me, so LGTM.

lib/CodeGen/SelectionDAG/DAGCombiner.cpp

17910

I don't think it's possible for an extract index to be bigger than uint64_t given LLVM type limitations, so could go with the possibly more optimizable:

IndexC->getZExtValue() % VT.getVectorNumElements() == 0

lib/CodeGen/SelectionDAG/DAGCombiner.cpp

static SDValue narrowInsertExtractVectorBinOp(SDNode *Extract,
   unsigned BinOpcode = BinOp.getOpcode();
   if (!TLI.isBinOp(BinOpcode) || BinOp.getNode()->getNumValues() != 1)
     return SDValue();
 
   SDValue Bop0 = BinOp.getOperand(0), Bop1 = BinOp.getOperand(1);
   SDValue Index = Extract->getOperand(1);
   EVT VT = Extract->getValueType(0);
 
+  // Helper that peeks through INSERT_SUBVECTOR/CONCAT_VECTORS to find
+  // if the source subvector is the same type as the one being extracted.
   auto GetSubVector = [VT, Index](SDValue V) {
-    if (V.getOpcode() != ISD::INSERT_SUBVECTOR ||
-        V.getOperand(1).getValueType() != VT || V.getOperand(2) != Index)
-      return SDValue();
+    if (V.getOpcode() == ISD::INSERT_SUBVECTOR &&
+        V.getOperand(1).getValueType() == VT && V.getOperand(2) == Index) {
       return V.getOperand(1);
+    }
+    auto *IndexC = dyn_cast<ConstantSDNode>(Index);
+    if (IndexC && V.getOpcode() == ISD::CONCAT_VECTORS &&
+        V.getOperand(0).getValueType() == VT &&
+        IndexC->getAPIntValue().urem(VT.getVectorNumElements()) == 0) {
+      uint64_t SubIdx = IndexC->getZExtValue() / VT.getVectorNumElements();
+      return V.getOperand(SubIdx);
+    }
+    return SDValue();
   };
   SDValue Sub0 = GetSubVector(Bop0);
   SDValue Sub1 = GetSubVector(Bop1);
 
   // TODO: We could handle the case where only 1 operand is being inserted by
   //       creating an extract of the other operand, but that requires checking
   //       number of uses and/or costs.
   if (!Sub0 || !Sub1 || !TLI.isOperationLegalOrCustom(BinOpcode, VT))

test/CodeGen/AMDGPU/fmax_legacy.f16.ll

 ; GFX9-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
 ; GFX9-SAFE-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX9-SAFE-NEXT:    v_lshl_or_b32 v0, v4, 16, v0
 ; GFX9-SAFE-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-NNAN-LABEL: test_fmax_legacy_ugt_v3f16:
 ; GFX9-NNAN:       ; %bb.0:
 ; GFX9-NNAN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NNAN-NEXT:    v_pk_max_f16 v0, v0, v2
 ; GFX9-NNAN-NEXT:    v_pk_max_f16 v1, v1, v3
+; GFX9-NNAN-NEXT:    v_pk_max_f16 v0, v0, v2
 ; GFX9-NNAN-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-SAFE-LABEL: test_fmax_legacy_ugt_v3f16:
 ; VI-SAFE:       ; %bb.0:
 ; VI-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SAFE-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
 ; VI-SAFE-NEXT:    v_lshrrev_b32_e32 v5, 16, v0
 ; VI-SAFE-NEXT:    v_cmp_nle_f16_e32 vcc, v5, v4

test/CodeGen/AMDGPU/fmin_legacy.f16.ll

 ; GFX9-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
 ; GFX9-SAFE-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX9-SAFE-NEXT:    v_lshl_or_b32 v0, v4, 16, v0
 ; GFX9-SAFE-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-NNAN-LABEL: test_fmin_legacy_ule_v3f16:
 ; GFX9-NNAN:       ; %bb.0:
 ; GFX9-NNAN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NNAN-NEXT:    v_pk_min_f16 v0, v0, v2
 ; GFX9-NNAN-NEXT:    v_pk_min_f16 v1, v1, v3
+; GFX9-NNAN-NEXT:    v_pk_min_f16 v0, v0, v2
 ; GFX9-NNAN-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-SAFE-LABEL: test_fmin_legacy_ule_v3f16:
 ; VI-SAFE:       ; %bb.0:
 ; VI-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-SAFE-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
 ; VI-SAFE-NEXT:    v_lshrrev_b32_e32 v5, 16, v0
 ; VI-SAFE-NEXT:    v_cmp_ngt_f16_e32 vcc, v5, v4

test/CodeGen/X86/fast-isel-store.ll

 ; AVX64-NEXT:    retl
   store <8 x float> %value, <8 x float>* %addr, align 32
   ret <8 x float> %value
 }
 
 define <4 x double> @test_store_4xf64(<4 x double>* nocapture %addr, <4 x double> %value, <4 x double> %value2) {
 ; SSE32-LABEL: test_store_4xf64:
 ; SSE32:       # %bb.0:
-; SSE32-NEXT:    addpd %xmm3, %xmm1
 ; SSE32-NEXT:    addpd %xmm2, %xmm0
 ; SSE32-NEXT:    movupd %xmm0, (%rdi)
+; SSE32-NEXT:    addpd %xmm3, %xmm1
 ; SSE32-NEXT:    movupd %xmm1, 16(%rdi)
 ; SSE32-NEXT:    retq
 ;
 ; SSE64-LABEL: test_store_4xf64:
 ; SSE64:       # %bb.0:
 ; SSE64-NEXT:    subl $12, %esp
 ; SSE64-NEXT:    .cfi_def_cfa_offset 16
+; SSE64-NEXT:    movapd {{[0-9]+}}(%esp), %xmm3
 ; SSE64-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; SSE64-NEXT:    addpd {{[0-9]+}}(%esp), %xmm1
 ; SSE64-NEXT:    addpd %xmm2, %xmm0
 ; SSE64-NEXT:    movupd %xmm0, (%eax)
+; SSE64-NEXT:    addpd %xmm3, %xmm1
 ; SSE64-NEXT:    movupd %xmm1, 16(%eax)
 ; SSE64-NEXT:    addl $12, %esp
 ; SSE64-NEXT:    .cfi_def_cfa_offset 4
 ; SSE64-NEXT:    retl
 ;
 ; AVX32-LABEL: test_store_4xf64:
 ; AVX32:       # %bb.0:
 ; AVX32-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
; AVX64-NEXT:    retl
   %foo = fadd <4 x double> %value, %value2 ; to force dobule type on store
   store <4 x double> %foo, <4 x double>* %addr, align 1
   ret <4 x double> %foo
 }
 
 define <4 x double> @test_store_4xf64_aligned(<4 x double>* nocapture %addr, <4 x double> %value, <4 x double> %value2) {
 ; SSE32-LABEL: test_store_4xf64_aligned:
 ; SSE32:       # %bb.0:
-; SSE32-NEXT:    addpd %xmm3, %xmm1
 ; SSE32-NEXT:    addpd %xmm2, %xmm0
 ; SSE32-NEXT:    movapd %xmm0, (%rdi)
+; SSE32-NEXT:    addpd %xmm3, %xmm1
 ; SSE32-NEXT:    movapd %xmm1, 16(%rdi)
 ; SSE32-NEXT:    retq
 ;
 ; SSE64-LABEL: test_store_4xf64_aligned:
 ; SSE64:       # %bb.0:
 ; SSE64-NEXT:    subl $12, %esp
 ; SSE64-NEXT:    .cfi_def_cfa_offset 16
+; SSE64-NEXT:    movapd {{[0-9]+}}(%esp), %xmm3
 ; SSE64-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; SSE64-NEXT:    addpd {{[0-9]+}}(%esp), %xmm1
 ; SSE64-NEXT:    addpd %xmm2, %xmm0
 ; SSE64-NEXT:    movapd %xmm0, (%eax)
+; SSE64-NEXT:    addpd %xmm3, %xmm1
 ; SSE64-NEXT:    movapd %xmm1, 16(%eax)
 ; SSE64-NEXT:    addl $12, %esp
 ; SSE64-NEXT:    .cfi_def_cfa_offset 4
 ; SSE64-NEXT:    retl
 ;
 ; AVX32-LABEL: test_store_4xf64_aligned:
 ; AVX32:       # %bb.0:
 ; AVX32-NEXT:    vaddpd %ymm1, %ymm0, %ymm0

 ; AVX51264-NEXT:    retl
   store <16 x float> %value, <16 x float>* %addr, align 64
   ret <16 x float> %value
 }
 
 define <8 x double> @test_store_8xf64(<8 x double>* nocapture %addr, <8 x double> %value, <8 x double> %value2) {
 ; SSE32-LABEL: test_store_8xf64:
 ; SSE32:       # %bb.0:
-; SSE32-NEXT:    addpd %xmm7, %xmm3
-; SSE32-NEXT:    addpd %xmm6, %xmm2
-; SSE32-NEXT:    addpd %xmm5, %xmm1
 ; SSE32-NEXT:    addpd %xmm4, %xmm0
 ; SSE32-NEXT:    movupd %xmm0, (%rdi)
+; SSE32-NEXT:    addpd %xmm5, %xmm1
 ; SSE32-NEXT:    movupd %xmm1, 16(%rdi)
+; SSE32-NEXT:    addpd %xmm6, %xmm2
 ; SSE32-NEXT:    movupd %xmm2, 32(%rdi)
+; SSE32-NEXT:    addpd %xmm7, %xmm3
 ; SSE32-NEXT:    movupd %xmm3, 48(%rdi)
 ; SSE32-NEXT:    retq
 ;
 ; SSE64-LABEL: test_store_8xf64:
 ; SSE64:       # %bb.0:
 ; SSE64-NEXT:    subl $12, %esp
 ; SSE64-NEXT:    .cfi_def_cfa_offset 16
+; SSE64-NEXT:    movapd {{[0-9]+}}(%esp), %xmm4
+; SSE64-NEXT:    movapd {{[0-9]+}}(%esp), %xmm5
+; SSE64-NEXT:    movapd {{[0-9]+}}(%esp), %xmm6
 ; SSE64-NEXT:    movapd {{[0-9]+}}(%esp), %xmm3
+; SSE64-NEXT:    addpd %xmm4, %xmm3
 ; SSE64-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; SSE64-NEXT:    addpd {{[0-9]+}}(%esp), %xmm3
-; SSE64-NEXT:    addpd {{[0-9]+}}(%esp), %xmm2
-; SSE64-NEXT:    addpd {{[0-9]+}}(%esp), %xmm1
 ; SSE64-NEXT:    addpd {{[0-9]+}}(%esp), %xmm0
 ; SSE64-NEXT:    movupd %xmm0, (%eax)
+; SSE64-NEXT:    addpd %xmm6, %xmm1
 ; SSE64-NEXT:    movupd %xmm1, 16(%eax)
+; SSE64-NEXT:    addpd %xmm5, %xmm2
 ; SSE64-NEXT:    movupd %xmm2, 32(%eax)
 ; SSE64-NEXT:    movupd %xmm3, 48(%eax)
 ; SSE64-NEXT:    addl $12, %esp
 ; SSE64-NEXT:    .cfi_def_cfa_offset 4
 ; SSE64-NEXT:    retl
 ;
 ; AVXONLY32-LABEL: test_store_8xf64:
 ; AVXONLY32:       # %bb.0:
-; AVXONLY32-NEXT:    vaddpd %ymm3, %ymm1, %ymm1
 ; AVXONLY32-NEXT:    vaddpd %ymm2, %ymm0, %ymm0
 ; AVXONLY32-NEXT:    vmovupd %ymm0, (%rdi)
+; AVXONLY32-NEXT:    vaddpd %ymm3, %ymm1, %ymm1
 ; AVXONLY32-NEXT:    vmovupd %ymm1, 32(%rdi)
 ; AVXONLY32-NEXT:    retq
 ;
 ; AVXONLY64-LABEL: test_store_8xf64:
 ; AVXONLY64:       # %bb.0:
 ; AVXONLY64-NEXT:    pushl %ebp
 ; AVXONLY64-NEXT:    .cfi_def_cfa_offset 8
 ; AVXONLY64-NEXT:    .cfi_offset %ebp, -8
 ; AVXONLY64-NEXT:    movl %esp, %ebp
 ; AVXONLY64-NEXT:    .cfi_def_cfa_register %ebp
 ; AVXONLY64-NEXT:    andl $-32, %esp
 ; AVXONLY64-NEXT:    subl $32, %esp
+; AVXONLY64-NEXT:    vmovapd 40(%ebp), %ymm3
 ; AVXONLY64-NEXT:    movl 8(%ebp), %eax
-; AVXONLY64-NEXT:    vaddpd 40(%ebp), %ymm1, %ymm1
 ; AVXONLY64-NEXT:    vaddpd %ymm2, %ymm0, %ymm0
 ; AVXONLY64-NEXT:    vmovupd %ymm0, (%eax)
+; AVXONLY64-NEXT:    vaddpd %ymm3, %ymm1, %ymm1
 ; AVXONLY64-NEXT:    vmovupd %ymm1, 32(%eax)
 ; AVXONLY64-NEXT:    movl %ebp, %esp
 ; AVXONLY64-NEXT:    popl %ebp
 ; AVXONLY64-NEXT:    .cfi_def_cfa %esp, 4
 ; AVXONLY64-NEXT:    retl
 ;
 ; AVX51232-LABEL: test_store_8xf64:
 ; AVX51232:       # %bb.0:
; AVX51264-NEXT:    retl
   %foo = fadd <8 x double> %value, %value2 ; to force dobule type on store
   store <8 x double> %foo, <8 x double>* %addr, align 1
   ret <8 x double> %foo
 }
 
 define <8 x double> @test_store_8xf64_aligned(<8 x double>* nocapture %addr, <8 x double> %value, <8 x double> %value2) {
 ; SSE32-LABEL: test_store_8xf64_aligned:
 ; SSE32:       # %bb.0:
-; SSE32-NEXT:    addpd %xmm7, %xmm3
-; SSE32-NEXT:    addpd %xmm6, %xmm2
-; SSE32-NEXT:    addpd %xmm5, %xmm1
 ; SSE32-NEXT:    addpd %xmm4, %xmm0
 ; SSE32-NEXT:    movapd %xmm0, (%rdi)
+; SSE32-NEXT:    addpd %xmm5, %xmm1
 ; SSE32-NEXT:    movapd %xmm1, 16(%rdi)
+; SSE32-NEXT:    addpd %xmm6, %xmm2
 ; SSE32-NEXT:    movapd %xmm2, 32(%rdi)
+; SSE32-NEXT:    addpd %xmm7, %xmm3
 ; SSE32-NEXT:    movapd %xmm3, 48(%rdi)
 ; SSE32-NEXT:    retq
 ;
 ; SSE64-LABEL: test_store_8xf64_aligned:
 ; SSE64:       # %bb.0:
 ; SSE64-NEXT:    subl $12, %esp
 ; SSE64-NEXT:    .cfi_def_cfa_offset 16
+; SSE64-NEXT:    movapd {{[0-9]+}}(%esp), %xmm4
+; SSE64-NEXT:    movapd {{[0-9]+}}(%esp), %xmm5
+; SSE64-NEXT:    movapd {{[0-9]+}}(%esp), %xmm6
 ; SSE64-NEXT:    movapd {{[0-9]+}}(%esp), %xmm3
+; SSE64-NEXT:    addpd %xmm4, %xmm3
 ; SSE64-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; SSE64-NEXT:    addpd {{[0-9]+}}(%esp), %xmm3
-; SSE64-NEXT:    addpd {{[0-9]+}}(%esp), %xmm2
-; SSE64-NEXT:    addpd {{[0-9]+}}(%esp), %xmm1
 ; SSE64-NEXT:    addpd {{[0-9]+}}(%esp), %xmm0
 ; SSE64-NEXT:    movapd %xmm0, (%eax)
+; SSE64-NEXT:    addpd %xmm6, %xmm1
 ; SSE64-NEXT:    movapd %xmm1, 16(%eax)
+; SSE64-NEXT:    addpd %xmm5, %xmm2
 ; SSE64-NEXT:    movapd %xmm2, 32(%eax)
 ; SSE64-NEXT:    movapd %xmm3, 48(%eax)
 ; SSE64-NEXT:    addl $12, %esp
 ; SSE64-NEXT:    .cfi_def_cfa_offset 4
 ; SSE64-NEXT:    retl
 ;
 ; AVXONLY32-LABEL: test_store_8xf64_aligned:
 ; AVXONLY32:       # %bb.0:
-; AVXONLY32-NEXT:    vaddpd %ymm3, %ymm1, %ymm1
 ; AVXONLY32-NEXT:    vaddpd %ymm2, %ymm0, %ymm0
 ; AVXONLY32-NEXT:    vmovapd %ymm0, (%rdi)
+; AVXONLY32-NEXT:    vaddpd %ymm3, %ymm1, %ymm1
 ; AVXONLY32-NEXT:    vmovapd %ymm1, 32(%rdi)
 ; AVXONLY32-NEXT:    retq
 ;
 ; AVXONLY64-LABEL: test_store_8xf64_aligned:
 ; AVXONLY64:       # %bb.0:
 ; AVXONLY64-NEXT:    pushl %ebp
 ; AVXONLY64-NEXT:    .cfi_def_cfa_offset 8
 ; AVXONLY64-NEXT:    .cfi_offset %ebp, -8
 ; AVXONLY64-NEXT:    movl %esp, %ebp
 ; AVXONLY64-NEXT:    .cfi_def_cfa_register %ebp
 ; AVXONLY64-NEXT:    andl $-32, %esp
 ; AVXONLY64-NEXT:    subl $32, %esp
+; AVXONLY64-NEXT:    vmovapd 40(%ebp), %ymm3
 ; AVXONLY64-NEXT:    movl 8(%ebp), %eax
-; AVXONLY64-NEXT:    vaddpd 40(%ebp), %ymm1, %ymm1
 ; AVXONLY64-NEXT:    vaddpd %ymm2, %ymm0, %ymm0
 ; AVXONLY64-NEXT:    vmovapd %ymm0, (%eax)
+; AVXONLY64-NEXT:    vaddpd %ymm3, %ymm1, %ymm1
 ; AVXONLY64-NEXT:    vmovapd %ymm1, 32(%eax)
 ; AVXONLY64-NEXT:    movl %ebp, %esp
 ; AVXONLY64-NEXT:    popl %ebp
 ; AVXONLY64-NEXT:    .cfi_def_cfa %esp, 4
 ; AVXONLY64-NEXT:    retl
 ;
 ; AVX51232-LABEL: test_store_8xf64_aligned:
 ; AVX51232:       # %bb.0:

test/CodeGen/X86/machine-combiner-int-vec.ll

; AVX-NEXT:    retq
   ret <4 x i32> %t2
 }
 
 ; Verify that 256-bit vector logical ops are reassociated.
 
 define <8 x i32> @reassociate_and_v8i32(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, <8 x i32> %x3) {
 ; SSE-LABEL: reassociate_and_v8i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    paddd %xmm3, %xmm1
 ; SSE-NEXT:    paddd %xmm2, %xmm0
 ; SSE-NEXT:    pand %xmm6, %xmm4
 ; SSE-NEXT:    pand %xmm4, %xmm0
+; SSE-NEXT:    paddd %xmm3, %xmm1
 ; SSE-NEXT:    pand %xmm7, %xmm5
 ; SSE-NEXT:    pand %xmm5, %xmm1
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: reassociate_and_v8i32:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
 ; AVX-NEXT:    vpand %ymm3, %ymm2, %ymm1
 ; AVX-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX-NEXT:    retq
 
   %t0 = add <8 x i32> %x0, %x1
   %t1 = and <8 x i32> %x2, %t0
   %t2 = and <8 x i32> %x3, %t1
   ret <8 x i32> %t2
 }
 
 define <8 x i32> @reassociate_or_v8i32(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, <8 x i32> %x3) {
 ; SSE-LABEL: reassociate_or_v8i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    paddd %xmm3, %xmm1
 ; SSE-NEXT:    paddd %xmm2, %xmm0
 ; SSE-NEXT:    por %xmm6, %xmm4
 ; SSE-NEXT:    por %xmm4, %xmm0
+; SSE-NEXT:    paddd %xmm3, %xmm1
 ; SSE-NEXT:    por %xmm7, %xmm5
 ; SSE-NEXT:    por %xmm5, %xmm1
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: reassociate_or_v8i32:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
 ; AVX-NEXT:    vpor %ymm3, %ymm2, %ymm1
 ; AVX-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX-NEXT:    retq
 
   %t0 = add <8 x i32> %x0, %x1
   %t1 = or <8 x i32> %x2, %t0
   %t2 = or <8 x i32> %x3, %t1
   ret <8 x i32> %t2
 }
 
 define <8 x i32> @reassociate_xor_v8i32(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, <8 x i32> %x3) {
 ; SSE-LABEL: reassociate_xor_v8i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    paddd %xmm3, %xmm1
 ; SSE-NEXT:    paddd %xmm2, %xmm0
 ; SSE-NEXT:    pxor %xmm6, %xmm4
 ; SSE-NEXT:    pxor %xmm4, %xmm0
+; SSE-NEXT:    paddd %xmm3, %xmm1
 ; SSE-NEXT:    pxor %xmm7, %xmm5
 ; SSE-NEXT:    pxor %xmm5, %xmm1
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: reassociate_xor_v8i32:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
 ; AVX-NEXT:    vpxor %ymm3, %ymm2, %ymm1

 ; SSE-NEXT:    pand {{[0-9]+}}(%rsp), %xmm0
 ; SSE-NEXT:    pand {{[0-9]+}}(%rsp), %xmm1
 ; SSE-NEXT:    pand {{[0-9]+}}(%rsp), %xmm2
 ; SSE-NEXT:    pand {{[0-9]+}}(%rsp), %xmm3
 ; SSE-NEXT:    retq
 ;
 ; AVX2-LABEL: reassociate_and_v16i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpaddd %ymm3, %ymm1, %ymm1
 ; AVX2-NEXT:    vpaddd %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpand %ymm6, %ymm4, %ymm2
 ; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %ymm3, %ymm1, %ymm1
 ; AVX2-NEXT:    vpand %ymm7, %ymm5, %ymm2
 ; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: reassociate_and_v16i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vpandd %zmm3, %zmm2, %zmm1

 ; SSE-NEXT:    por {{[0-9]+}}(%rsp), %xmm0
 ; SSE-NEXT:    por {{[0-9]+}}(%rsp), %xmm1
 ; SSE-NEXT:    por {{[0-9]+}}(%rsp), %xmm2
 ; SSE-NEXT:    por {{[0-9]+}}(%rsp), %xmm3
 ; SSE-NEXT:    retq
 ;
 ; AVX2-LABEL: reassociate_or_v16i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpaddd %ymm3, %ymm1, %ymm1
 ; AVX2-NEXT:    vpaddd %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpor %ymm6, %ymm4, %ymm2
 ; AVX2-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %ymm3, %ymm1, %ymm1
 ; AVX2-NEXT:    vpor %ymm7, %ymm5, %ymm2
 ; AVX2-NEXT:    vpor %ymm2, %ymm1, %ymm1
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: reassociate_or_v16i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vpord %zmm3, %zmm2, %zmm1

 ; SSE-NEXT:    pxor {{[0-9]+}}(%rsp), %xmm0
 ; SSE-NEXT:    pxor {{[0-9]+}}(%rsp), %xmm1
 ; SSE-NEXT:    pxor {{[0-9]+}}(%rsp), %xmm2
 ; SSE-NEXT:    pxor {{[0-9]+}}(%rsp), %xmm3
 ; SSE-NEXT:    retq
 ;
 ; AVX2-LABEL: reassociate_xor_v16i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpaddd %ymm3, %ymm1, %ymm1
 ; AVX2-NEXT:    vpaddd %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpxor %ymm6, %ymm4, %ymm2
 ; AVX2-NEXT:    vpxor %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %ymm3, %ymm1, %ymm1
 ; AVX2-NEXT:    vpxor %ymm7, %ymm5, %ymm2
 ; AVX2-NEXT:    vpxor %ymm2, %ymm1, %ymm1
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: reassociate_xor_v16i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vpxord %zmm3, %zmm2, %zmm1

test/CodeGen/X86/nontemporal-2.ll

; VLX-NEXT:    retq
   ret void
 }
 
 ; 256-bit NT stores require 256-bit alignment.
 ; For AVX, we lower 128-bit alignment as 2x movntps %xmm.
 define void @test_unaligned_v8f32(<8 x float> %a, <8 x float> %b, <8 x float>* %dst) {
 ; SSE-LABEL: test_unaligned_v8f32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    addps %xmm2, %xmm0
 ; SSE-NEXT:    addps %xmm3, %xmm1
 ; SSE-NEXT:    movntps %xmm1, 16(%rdi)
+; SSE-NEXT:    addps %xmm2, %xmm0
 ; SSE-NEXT:    movntps %xmm0, (%rdi)
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test_unaligned_v8f32:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vaddps %ymm1, %ymm0, %ymm0
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX-NEXT:    vmovntps %xmm1, 16(%rdi)

test/CodeGen/X86/vec_saddo.ll

 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm9
 ; AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
 ; AVX1-NEXT:    vpcmpgtd %xmm9, %xmm5, %xmm7
 ; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm6, %xmm6
 ; AVX1-NEXT:    vpxor %xmm6, %xmm7, %xmm8
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm7
 ; AVX1-NEXT:    vpcmpgtd %xmm7, %xmm5, %xmm4
-; AVX1-NEXT:    vpxor %xmm6, %xmm4, %xmm10
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm10, %xmm8
-; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm5, %xmm4
-; AVX1-NEXT:    vpxor %xmm6, %xmm4, %xmm11
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm5, %xmm4
 ; AVX1-NEXT:    vpxor %xmm6, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm11, %xmm4, %xmm11
-; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm11, %ymm11
+; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm4, %xmm8
 ; AVX1-NEXT:    vpaddd %xmm9, %xmm7, %xmm9
 ; AVX1-NEXT:    vpcmpgtd %xmm9, %xmm5, %xmm7
 ; AVX1-NEXT:    vpxor %xmm6, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm10, %xmm7
+; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm4, %xmm4
+; AVX1-NEXT:    vpandn %xmm8, %xmm4, %xmm8
+; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm5, %xmm7
+; AVX1-NEXT:    vpxor %xmm6, %xmm7, %xmm7
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm5, %xmm4
+; AVX1-NEXT:    vpxor %xmm6, %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm4, %xmm7
 ; AVX1-NEXT:    vpaddd %xmm3, %xmm1, %xmm10
 ; AVX1-NEXT:    vpcmpgtd %xmm10, %xmm5, %xmm1
 ; AVX1-NEXT:    vpxor %xmm6, %xmm1, %xmm1
 ; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm4, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm1, %ymm1
-; AVX1-NEXT:    vandnps %ymm11, %ymm1, %ymm1
-; AVX1-NEXT:    vpandn %xmm8, %xmm7, %xmm4
-; AVX1-NEXT:    vpackssdw %xmm4, %xmm1, %xmm8
+; AVX1-NEXT:    vpandn %xmm7, %xmm1, %xmm1
+; AVX1-NEXT:    vpackssdw %xmm8, %xmm1, %xmm8
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
 ; AVX1-NEXT:    vpcmpgtd %xmm4, %xmm5, %xmm7
 ; AVX1-NEXT:    vpxor %xmm6, %xmm7, %xmm7
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm5, %xmm3
 ; AVX1-NEXT:    vpxor %xmm6, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm3, %xmm11
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm5, %xmm7
-; AVX1-NEXT:    vpxor %xmm6, %xmm7, %xmm12
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm5, %xmm7
-; AVX1-NEXT:    vpxor %xmm6, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm12, %xmm7, %xmm12
-; AVX1-NEXT:    vinsertf128 $1, %xmm11, %ymm12, %ymm12
+; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm3, %xmm7
 ; AVX1-NEXT:    vpaddd %xmm4, %xmm1, %xmm4
 ; AVX1-NEXT:    vpcmpgtd %xmm4, %xmm5, %xmm1
 ; AVX1-NEXT:    vpxor %xmm6, %xmm1, %xmm1
 ; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vpandn %xmm7, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm5, %xmm3
+; AVX1-NEXT:    vpxor %xmm6, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm5, %xmm7
+; AVX1-NEXT:    vpxor %xmm6, %xmm7, %xmm7
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm7, %xmm3
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm0, %xmm2
 ; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm5, %xmm0
 ; AVX1-NEXT:    vpxor %xmm6, %xmm0, %xmm0
 ; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm7, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT:    vandnps %ymm12, %ymm0, %ymm0
-; AVX1-NEXT:    vpandn %xmm11, %xmm1, %xmm1
+; AVX1-NEXT:    vpandn %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpacksswb %xmm8, %xmm0, %xmm1
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3

test/CodeGen/X86/vec_ssubo.ll

 ; SSE-NEXT:    movdqa %xmm8, %xmm0
 ; SSE-NEXT:    movdqa %xmm9, %xmm1
 ; SSE-NEXT:    movdqa %xmm4, %xmm2
 ; SSE-NEXT:    movdqa %xmm5, %xmm3
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: ssubo_v16i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm9
-; AVX1-NEXT:    vpxor %xmm6, %xmm6, %xmm6
-; AVX1-NEXT:    vpcmpgtd %xmm9, %xmm6, %xmm7
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
+; AVX1-NEXT:    vpxor %xmm9, %xmm9, %xmm9
+; AVX1-NEXT:    vpcmpgtd %xmm4, %xmm9, %xmm7
 ; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
 ; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm8
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm7
-; AVX1-NEXT:    vpcmpgtd %xmm7, %xmm6, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm10
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm10, %xmm8
-; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm6, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm11
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm6, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm11, %xmm4, %xmm11
-; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm11, %ymm11
-; AVX1-NEXT:    vpsubd %xmm9, %xmm7, %xmm9
-; AVX1-NEXT:    vpcmpgtd %xmm9, %xmm6, %xmm7
+; AVX1-NEXT:    vpcmpgtd %xmm7, %xmm9, %xmm6
+; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
+; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm6, %xmm8
+; AVX1-NEXT:    vpsubd %xmm4, %xmm7, %xmm10
+; AVX1-NEXT:    vpcmpgtd %xmm10, %xmm9, %xmm7
 ; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm10, %xmm7
+; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm6, %xmm6
+; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
+; AVX1-NEXT:    vpandn %xmm6, %xmm8, %xmm6
+; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm9, %xmm7
 ; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
-; AVX1-NEXT:    vpsubd %xmm3, %xmm1, %xmm10
-; AVX1-NEXT:    vpcmpgtd %xmm10, %xmm6, %xmm1
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm9, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm4, %xmm7
+; AVX1-NEXT:    vpsubd %xmm3, %xmm1, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm9, %xmm1
 ; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
 ; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm4, %xmm1
 ; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm1, %ymm1
-; AVX1-NEXT:    vandnps %ymm1, %ymm11, %ymm1
-; AVX1-NEXT:    vpandn %xmm7, %xmm8, %xmm4
-; AVX1-NEXT:    vpackssdw %xmm4, %xmm1, %xmm8
+; AVX1-NEXT:    vpandn %xmm1, %xmm7, %xmm1
+; AVX1-NEXT:    vpackssdw %xmm6, %xmm1, %xmm8
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
-; AVX1-NEXT:    vpcmpgtd %xmm4, %xmm6, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm6, %xmm3
-; AVX1-NEXT:    vpxor %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm3, %xmm11
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm6, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm12
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm6, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm12, %xmm7, %xmm12
-; AVX1-NEXT:    vinsertf128 $1, %xmm11, %ymm12, %ymm12
-; AVX1-NEXT:    vpsubd %xmm4, %xmm1, %xmm4
-; AVX1-NEXT:    vpcmpgtd %xmm4, %xmm6, %xmm1
+; AVX1-NEXT:    vpcmpgtd %xmm4, %xmm9, %xmm6
+; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
+; AVX1-NEXT:    vpcmpgtd %xmm7, %xmm9, %xmm1
 ; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm1, %xmm6
+; AVX1-NEXT:    vpsubd %xmm4, %xmm7, %xmm7
+; AVX1-NEXT:    vpcmpgtd %xmm7, %xmm9, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm1, %xmm1
 ; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vpandn %xmm1, %xmm6, %xmm1
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm9, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm9, %xmm6
+; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm6, %xmm4
 ; AVX1-NEXT:    vpsubd %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm6, %xmm0
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm9, %xmm0
 ; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm7, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm6, %xmm0
 ; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT:    vandnps %ymm0, %ymm12, %ymm0
-; AVX1-NEXT:    vpandn %xmm1, %xmm11, %xmm1
+; AVX1-NEXT:    vpandn %xmm0, %xmm4, %xmm0
 ; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpacksswb %xmm8, %xmm0, %xmm1
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,2,3]
-; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
-; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[1,1,2,3]
+; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,0,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
-; AVX1-NEXT:    vmovdqa %xmm9, 48(%rdi)
-; AVX1-NEXT:    vmovdqa %xmm10, 32(%rdi)
-; AVX1-NEXT:    vmovdqa %xmm4, 16(%rdi)
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm4, %ymm1
+; AVX1-NEXT:    vmovdqa %xmm10, 48(%rdi)
+; AVX1-NEXT:    vmovdqa %xmm3, 32(%rdi)
+; AVX1-NEXT:    vmovdqa %xmm7, 16(%rdi)
 ; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: ssubo_v16i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
 ; AVX2-NEXT:    vpcmpgtd %ymm3, %ymm4, %ymm5
 ; AVX2-NEXT:    vpcmpeqd %ymm6, %ymm6, %ymm6