[DAG] visitINSERT_VECTOR_ELT - attempt to reconstruct BUILD_VECTOR before other fold interfere

[RKSimon]

Summary

Another issue unearthed by D127115

We take a long time to canonicalize an insert_vector_elt chain before being able to convert it into a build_vector - even if they are already in ascending insertion order, we fold the nodes one at a time in to the build_vector 'seed', leaving plenty of time for other folds to alter it (in particular recognising when they come from extract_vector_elt resulting in a shuffle_vector that is much harder to fold with).

D127115 makes this particularly difficult as we're almost guaranteed to have the lost the sequence before all possible insertions have been made.

This patch proposes to begin at the last insertion and attempt to collect all the (oneuse) insertions right away and create the build_vector before its too late.

[RKSimon]

fix typo in comment

[RKSimon]

llvm/test/CodeGen/AArch64/sve-fixed-length-insert-vector-elt.ll

232

Minor aarch64 optimization - insertelement v1f64 folds to a move but buildvector doesn't?

[deadalnix]

The test for which this triggers seems overly restrictive, but this is good.

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

19433

That seems overly restrictive:

• The remaining elements could be undef - this is a relatively common occurrence, as far as i can tell.
• The element could be inserted in pretty much any order (unless there is some canonicalization?)

19444

this assumes proper order, do we make sure this is cannonical in some way?

[RKSimon]

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

19376

@deadalnix We canonicalize to ascending order here

[deadalnix]

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

19376

I see. Great.

[dmgreen]

llvm/test/CodeGen/AArch64/sve-fixed-length-insert-vector-elt.ll

232

Should a single element buildvector be canonicalized to a scalar_to_reg?

[RKSimon]

llvm/test/CodeGen/AArch64/sve-fixed-length-insert-vector-elt.ll

232

I tested with folding to scalar_to_vector and this test had the same problem.

[foad]

Personally I think IR should have a buildvector instruction instead of an insertelement instruction. :)

[deadalnix]

In D127595#3577082, @foad wrote:

Personally I think IR should have a buildvector instruction instead of an insertelement instruction. :)

The proliferation of option to build a vector are probably playing against us, indeed. build_vector, insert_element, vector_shuffle, etc...

We are bound to miss patterns.

[deadalnix]

If you rebase this, you'll notice there is a big win in shuffle-extract-subvector.ll

[RKSimon]

rebase

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

SDValue DAGCombiner::visitINSERT_VECTOR_ELT(SDNode *N) {
 
   // Canonicalize insert_vector_elt dag nodes.
   // Example:
   // (insert_vector_elt (insert_vector_elt A, Idx0), Idx1)
   // -> (insert_vector_elt (insert_vector_elt A, Idx1), Idx0)
   //
   // Do this only if the child insert_vector node has one use; also
   // do this only if indices are both constants and Idx1 < Idx0.
   if (InVec.getOpcode() == ISD::INSERT_VECTOR_ELT && InVec.hasOneUse()
       && isa<ConstantSDNode>(InVec.getOperand(2))) {
     unsigned OtherElt = InVec.getConstantOperandVal(2);
     if (Elt < OtherElt) {
       // Swap nodes.
       SDValue NewOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT,
                                   InVec.getOperand(0), InVal, EltNo);
       AddToWorklist(NewOp.getNode());
       return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(InVec.getNode()),
if (InVec.getOpcode() == ISD::SCALAR_TO_VECTOR && InVec.hasOneUse()) {
       Ops.append(NumElts - 1, DAG.getUNDEF(InVec.getOperand(0).getValueType()));
       return UpdateBuildVector(Ops);
     }
 
     if (InVec.isUndef()) {
       Ops.append(NumElts, DAG.getUNDEF(InVal.getValueType()));
       return UpdateBuildVector(Ops);
     }
+
+    // If we're inserting into the end of a vector as part of an sequence, see
+    // if we can create a BUILD_VECTOR by following the sequence back up the
+    // chain.
+    if (Elt == (NumElts - 1)) {
+      SmallVector<SDValue> ReverseInsertions;
+      ReverseInsertions.push_back(InVal);
+
+      EVT MaxEltVT = InVal.getValueType();
+      SDValue CurVec = InVec;
+      for (unsigned I = 1; I != NumElts; ++I) {
+        if (CurVec.getOpcode() != ISD::INSERT_VECTOR_ELT || !CurVec.hasOneUse())
+          break;
+
+        auto *CurIdx = dyn_cast<ConstantSDNode>(CurVec.getOperand(2));
+        if (!CurIdx || CurIdx->getAPIntValue() != ((NumElts - 1) - I))
+          break;
+        SDValue CurVal = CurVec.getOperand(1);
+        ReverseInsertions.push_back(CurVal);
+        if (VT.isInteger()) {
+          EVT CurValVT = CurVal.getValueType();
+          MaxEltVT = MaxEltVT.bitsGE(CurValVT) ? MaxEltVT : CurValVT;
+        }
+        CurVec = CurVec.getOperand(0);
+      }
+
+      if (ReverseInsertions.size() == NumElts) {
+        for (unsigned I = 0; I != NumElts; ++I) {
+          SDValue Val = ReverseInsertions[(NumElts - 1) - I];
+          Val = VT.isInteger() ? DAG.getAnyExtOrTrunc(Val, DL, MaxEltVT) : Val;
+          Ops.push_back(Val);
+        }
+        return DAG.getBuildVector(VT, DL, Ops);
+      }
+    }
   }
 
   return SDValue();
 }
 
 SDValue DAGCombiner::scalarizeExtractedVectorLoad(SDNode *EVE, EVT InVecVT,
                                                   SDValue EltNo,
                                                   LoadSDNode *OriginalLoad) {

llvm/test/CodeGen/AArch64/arm64-neon-copy.ll

; CHECK-NEXT:    ret
   %tmp3 = extractelement <4 x i32> %tmp1, i32 2
   %tmp4 = insertelement <2 x i32> %tmp2, i32 %tmp3, i32 1
   ret <2 x i32> %tmp4
 }
 
 define <1 x i64> @ins2d1(<2 x i64> %tmp1, <1 x i64> %tmp2) {
 ; CHECK-LABEL: ins2d1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-NEXT:    mov v1.d[0], v0.d[0]
-; CHECK-NEXT:    fmov d0, d1
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
   %tmp3 = extractelement <2 x i64> %tmp1, i32 0
   %tmp4 = insertelement <1 x i64> %tmp2, i64 %tmp3, i32 0
   ret <1 x i64> %tmp4
 }
 
 define <2 x float> @ins4f2(<4 x float> %tmp1, <2 x float> %tmp2) {
 ; CHECK-LABEL: ins4f2:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
 ; CHECK-NEXT:    mov v1.s[1], v0.s[2]
 ; CHECK-NEXT:    fmov d0, d1
 ; CHECK-NEXT:    ret
   %tmp3 = extractelement <4 x float> %tmp1, i32 2
   %tmp4 = insertelement <2 x float> %tmp2, float %tmp3, i32 1
   ret <2 x float> %tmp4
 }
 
 define <1 x double> @ins2f1(<2 x double> %tmp1, <1 x double> %tmp2) {
 ; CHECK-LABEL: ins2f1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    dup v0.2d, v0.d[1]
+; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
   %tmp3 = extractelement <2 x double> %tmp1, i32 1
   %tmp4 = insertelement <1 x double> %tmp2, double %tmp3, i32 0
   ret <1 x double> %tmp4
 }
 
 define <8 x i8> @ins8b8(<8 x i8> %tmp1, <8 x i8> %tmp2) {

llvm/test/CodeGen/AArch64/sve-fixed-length-insert-vector-elt.ll

; VBITS_GE_2048-NEXT:    ret
     %r = insertelement <64 x float> %op1, float 5.0, i64 63
     ret <64 x float> %r
 }
 
 ; Don't use SVE for 64-bit vectors.
 define <1 x double> @insertelement_v1f64(<1 x double> %op1) #0 {
 ; VBITS_GE_256-LABEL: insertelement_v1f64:
 ; VBITS_GE_256:       // %bb.0:
-; VBITS_GE_256-NEXT:    fmov d0, #5.00000000
+; VBITS_GE_256-NEXT:    mov x8, #4617315517961601024
+; VBITS_GE_256-NEXT:    fmov d0, x8
 ; VBITS_GE_256-NEXT:    ret
     %r = insertelement <1 x double> %op1, double 5.0, i64 0
     ret <1 x double> %r
 }
 
 ; Don't use SVE for 128-bit vectors.
 define <2 x double> @insertelement_v2f64(<2 x double> %op1) #0 {
 ; VBITS_GE_256-LABEL: insertelement_v2f64:

llvm/test/CodeGen/AArch64/vecreduce-propagate-sd-flags.ll

 ; REQUIRES: arm-registered-target
 ; REQUIRES: asserts
 ; RUN: llc -o /dev/null %s -debug-only=legalize-types 2>&1 | FileCheck %s
 
 ; This test check that when v4f64 gets broken down to two v2f64 it maintains
 ; the "nnan" flags.
 
 ; CHECK: Legalizing node: [[VFOUR:t.*]]: v4f64 = BUILD_VECTOR
 ; CHECK-NEXT: Analyzing result type: v4f64
 ; CHECK-NEXT: Split node result: [[VFOUR]]: v4f64 = BUILD_VECTOR
 
 ; CHECK: Legalizing node: [[VTWOA:t.*]]: v2f64 = BUILD_VECTOR
 ; CHECK: Legally typed node: [[VTWOA]]: v2f64 = BUILD_VECTOR
 ; CHECK: Legalizing node: [[VTWOB:t.*]]: v2f64 = BUILD_VECTOR
 ; CHECK: Legally typed node: [[VTWOB]]: v2f64 = BUILD_VECTOR
-; CHECK: Legalizing node: t34: v2f64 = fmaxnum nnan reassoc [[VTWOB]], [[VTWOA]]
+; CHECK: Legalizing node: t30: v2f64 = fmaxnum nnan reassoc [[VTWOB]], [[VTWOA]]
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-gnu"
 
 
 ; Function Attrs: norecurse nounwind
 define fastcc double @test(double %a0, double %a1, double %a2, double %a3) unnamed_addr #1 {
 entry:

llvm/test/CodeGen/ARM/neon-copy.ll

; CHECK-NEXT:    bx lr
   %tmp3 = extractelement <4 x i32> %tmp1, i32 2
   %tmp4 = insertelement <2 x i32> %tmp2, i32 %tmp3, i32 1
   ret <2 x i32> %tmp4
 }
 
 define <1 x i64> @ins2d1(<2 x i64> %tmp1, <1 x i64> %tmp2) {
 ; CHECK-LABEL: ins2d1:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vmov r0, r1, d0
-; CHECK-NEXT:    vmov.32 d2[0], r0
-; CHECK-NEXT:    vmov.32 d2[1], r1
-; CHECK-NEXT:    vorr d0, d2, d2
+; CHECK-NEXT:    @ kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    bx lr
   %tmp3 = extractelement <2 x i64> %tmp1, i32 0
   %tmp4 = insertelement <1 x i64> %tmp2, i64 %tmp3, i32 0
   ret <1 x i64> %tmp4
 }
 
 define <2 x float> @ins4f2(<4 x float> %tmp1, <2 x float> %tmp2) {
 ; CHECK-LABEL: ins4f2:

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-insert-i1.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -mattr=+v -riscv-v-vector-bits-min=128 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s --check-prefixes=CHECK,RV32
 ; RUN: llc -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-min=128 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s --check-prefixes=CHECK,RV64
 
 define <1 x i1> @insertelt_v1i1(<1 x i1> %x, i1 %elt) nounwind {
 ; CHECK-LABEL: insertelt_v1i1:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    andi a0, a0, 1
 ; CHECK-NEXT:    vsetivli zero, 1, e8, mf8, ta, mu
-; CHECK-NEXT:    vmv.v.i v8, 0
-; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
-; CHECK-NEXT:    vsetvli zero, zero, e8, mf8, tu, mu
-; CHECK-NEXT:    vmv.s.x v8, a0
-; CHECK-NEXT:    vsetvli zero, zero, e8, mf8, ta, mu
-; CHECK-NEXT:    vand.vi v8, v8, 1
+; CHECK-NEXT:    vmv.v.x v8, a0
 ; CHECK-NEXT:    vmsne.vi v0, v8, 0
 ; CHECK-NEXT:    ret
   %y = insertelement <1 x i1> %x, i1 %elt, i64 0
   ret <1 x i1> %y
 }
 
 define <1 x i1> @insertelt_idx_v1i1(<1 x i1> %x, i1 %elt, i32 zeroext %idx) nounwind {
 ; RV32-LABEL: insertelt_idx_v1i1:

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll

 ; RV64ZVE32F:       # %bb.0:
 ; RV64ZVE32F-NEXT:    vsetvli a1, zero, e8, mf4, ta, mu
 ; RV64ZVE32F-NEXT:    vmv.v.i v9, 0
 ; RV64ZVE32F-NEXT:    vmerge.vim v9, v9, 1, v0
 ; RV64ZVE32F-NEXT:    vmv.x.s a1, v9
 ; RV64ZVE32F-NEXT:    andi a1, a1, 1
 ; RV64ZVE32F-NEXT:    beqz a1, .LBB0_2
 ; RV64ZVE32F-NEXT:  # %bb.1: # %cond.load
-; RV64ZVE32F-NEXT:    lb a0, 0(a0)
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, tu, mu
-; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, mu
+; RV64ZVE32F-NEXT:    vlse8.v v8, (a0), zero
 ; RV64ZVE32F-NEXT:  .LBB0_2: # %else
 ; RV64ZVE32F-NEXT:    ret
   %v = call <1 x i8> @llvm.masked.gather.v1i8.v1p0i8(<1 x i8*> %ptrs, i32 1, <1 x i1> %m, <1 x i8> %passthru)
   ret <1 x i8> %v
 }
 
 declare <2 x i8> @llvm.masked.gather.v2i8.v2p0i8(<2 x i8*>, i32, <2 x i1>, <2 x i8>)
 

 ; RV64ZVE32F:       # %bb.0:
 ; RV64ZVE32F-NEXT:    vsetvli a1, zero, e8, mf4, ta, mu
 ; RV64ZVE32F-NEXT:    vmv.v.i v9, 0
 ; RV64ZVE32F-NEXT:    vmerge.vim v9, v9, 1, v0
 ; RV64ZVE32F-NEXT:    vmv.x.s a1, v9
 ; RV64ZVE32F-NEXT:    andi a1, a1, 1
 ; RV64ZVE32F-NEXT:    beqz a1, .LBB13_2
 ; RV64ZVE32F-NEXT:  # %bb.1: # %cond.load
-; RV64ZVE32F-NEXT:    lh a0, 0(a0)
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, tu, mu
-; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, mu
+; RV64ZVE32F-NEXT:    vlse16.v v8, (a0), zero
 ; RV64ZVE32F-NEXT:  .LBB13_2: # %else
 ; RV64ZVE32F-NEXT:    ret
   %v = call <1 x i16> @llvm.masked.gather.v1i16.v1p0i16(<1 x i16*> %ptrs, i32 2, <1 x i1> %m, <1 x i16> %passthru)
   ret <1 x i16> %v
 }
 
 declare <2 x i16> @llvm.masked.gather.v2i16.v2p0i16(<2 x i16*>, i32, <2 x i1>, <2 x i16>)
 

 ; RV64ZVE32F:       # %bb.0:
 ; RV64ZVE32F-NEXT:    vsetvli a1, zero, e8, mf4, ta, mu
 ; RV64ZVE32F-NEXT:    vmv.v.i v9, 0
 ; RV64ZVE32F-NEXT:    vmerge.vim v9, v9, 1, v0
 ; RV64ZVE32F-NEXT:    vmv.x.s a1, v9
 ; RV64ZVE32F-NEXT:    andi a1, a1, 1
 ; RV64ZVE32F-NEXT:    beqz a1, .LBB27_2
 ; RV64ZVE32F-NEXT:  # %bb.1: # %cond.load
-; RV64ZVE32F-NEXT:    lw a0, 0(a0)
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, tu, mu
-; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, mu
+; RV64ZVE32F-NEXT:    vlse32.v v8, (a0), zero
 ; RV64ZVE32F-NEXT:  .LBB27_2: # %else
 ; RV64ZVE32F-NEXT:    ret
   %v = call <1 x i32> @llvm.masked.gather.v1i32.v1p0i32(<1 x i32*> %ptrs, i32 4, <1 x i1> %m, <1 x i32> %passthru)
   ret <1 x i32> %v
 }
 
 declare <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*>, i32, <2 x i1>, <2 x i32>)
 

 ; RV64ZVE32F:       # %bb.0:
 ; RV64ZVE32F-NEXT:    vsetvli a1, zero, e8, mf4, ta, mu
 ; RV64ZVE32F-NEXT:    vmv.v.i v9, 0
 ; RV64ZVE32F-NEXT:    vmerge.vim v9, v9, 1, v0
 ; RV64ZVE32F-NEXT:    vmv.x.s a1, v9
 ; RV64ZVE32F-NEXT:    andi a1, a1, 1
 ; RV64ZVE32F-NEXT:    beqz a1, .LBB58_2
 ; RV64ZVE32F-NEXT:  # %bb.1: # %cond.load
-; RV64ZVE32F-NEXT:    flh ft0, 0(a0)
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, tu, mu
-; RV64ZVE32F-NEXT:    vfmv.s.f v8, ft0
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, mu
+; RV64ZVE32F-NEXT:    vlse16.v v8, (a0), zero
 ; RV64ZVE32F-NEXT:  .LBB58_2: # %else
 ; RV64ZVE32F-NEXT:    ret
   %v = call <1 x half> @llvm.masked.gather.v1f16.v1p0f16(<1 x half*> %ptrs, i32 2, <1 x i1> %m, <1 x half> %passthru)
   ret <1 x half> %v
 }
 
 declare <2 x half> @llvm.masked.gather.v2f16.v2p0f16(<2 x half*>, i32, <2 x i1>, <2 x half>)
 

 ; RV64ZVE32F:       # %bb.0:
 ; RV64ZVE32F-NEXT:    vsetvli a1, zero, e8, mf4, ta, mu
 ; RV64ZVE32F-NEXT:    vmv.v.i v9, 0
 ; RV64ZVE32F-NEXT:    vmerge.vim v9, v9, 1, v0
 ; RV64ZVE32F-NEXT:    vmv.x.s a1, v9
 ; RV64ZVE32F-NEXT:    andi a1, a1, 1
 ; RV64ZVE32F-NEXT:    beqz a1, .LBB68_2
 ; RV64ZVE32F-NEXT:  # %bb.1: # %cond.load
-; RV64ZVE32F-NEXT:    flw ft0, 0(a0)
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, tu, mu
-; RV64ZVE32F-NEXT:    vfmv.s.f v8, ft0
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, mu
+; RV64ZVE32F-NEXT:    vlse32.v v8, (a0), zero
 ; RV64ZVE32F-NEXT:  .LBB68_2: # %else
 ; RV64ZVE32F-NEXT:    ret
   %v = call <1 x float> @llvm.masked.gather.v1f32.v1p0f32(<1 x float*> %ptrs, i32 4, <1 x i1> %m, <1 x float> %passthru)
   ret <1 x float> %v
 }
 
 declare <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*>, i32, <2 x i1>, <2 x float>)
 

llvm/test/CodeGen/X86/avx512-insert-extract.ll

; SKX-NEXT:    retq
   %r1 = insertelement <4 x i64> %x, i64 %val, i32 1
   %r2 = insertelement <4 x i64> %r1, i64 %y, i32 3
   ret <4 x i64> %r2
 }
 
 define <2 x i64> @insert_v2i64(<2 x i64> %x, i64 %y , i64* %ptr) {
 ; CHECK-LABEL: insert_v2i64:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vpinsrq $0, %rdi, %xmm0, %xmm0
-; CHECK-NEXT:    vpinsrq $1, (%rsi), %xmm0, %xmm0
+; CHECK-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    vmovq %rdi, %xmm1
+; CHECK-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
 ; CHECK-NEXT:    retq
   %val = load i64, i64* %ptr
   %r1 = insertelement <2 x i64> %x, i64 %val, i32 1
   %r2 = insertelement <2 x i64> %r1, i64 %y, i32 0
   ret <2 x i64> %r2
 }
 
 define <16 x i32> @insert_v16i32(<16 x i32> %x, i32 %y, i32* %ptr) {

llvm/test/CodeGen/X86/masked_gather_scatter.ll

; SKX_32-NEXT:    retl
   ret <2 x i64>%res
 }
 
 define <2 x i64> @test26(i64* %base, <2 x i32> %ind, <2 x i64> %src0) {
 ; KNL_64-LABEL: test26:
 ; KNL_64:       # %bb.0:
 ; KNL_64-NEXT:    vpmovsxdq %xmm0, %xmm0
 ; KNL_64-NEXT:    vpsllq $3, %xmm0, %xmm0
-; KNL_64-NEXT:    vmovq %rdi, %xmm2
-; KNL_64-NEXT:    vpbroadcastq %xmm2, %xmm2
-; KNL_64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; KNL_64-NEXT:    vmovq %rdi, %xmm1
+; KNL_64-NEXT:    vpbroadcastq %xmm1, %xmm1
+; KNL_64-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
 ; KNL_64-NEXT:    vmovq %xmm0, %rax
-; KNL_64-NEXT:    vpinsrq $0, (%rax), %xmm1, %xmm1
-; KNL_64-NEXT:    vpextrq $1, %xmm0, %rax
-; KNL_64-NEXT:    vpinsrq $1, (%rax), %xmm1, %xmm0
+; KNL_64-NEXT:    vpextrq $1, %xmm0, %rcx
+; KNL_64-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; KNL_64-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; KNL_64-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
 ; KNL_64-NEXT:    retq
 ;
 ; KNL_32-LABEL: test26:
 ; KNL_32:       # %bb.0:
 ; KNL_32-NEXT:    vpslld $3, %xmm0, %xmm0
-; KNL_32-NEXT:    vpbroadcastd {{[0-9]+}}(%esp), %xmm2
-; KNL_32-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
+; KNL_32-NEXT:    vpbroadcastd {{[0-9]+}}(%esp), %xmm1
+; KNL_32-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
 ; KNL_32-NEXT:    vmovd %xmm0, %eax
-; KNL_32-NEXT:    vpinsrd $0, (%eax), %xmm1, %xmm1
-; KNL_32-NEXT:    vpinsrd $1, 4(%eax), %xmm1, %xmm1
-; KNL_32-NEXT:    vpextrd $1, %xmm0, %eax
-; KNL_32-NEXT:    vpinsrd $2, (%eax), %xmm1, %xmm0
-; KNL_32-NEXT:    vpinsrd $3, 4(%eax), %xmm0, %xmm0
+; KNL_32-NEXT:    vpextrd $1, %xmm0, %ecx
+; KNL_32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; KNL_32-NEXT:    vpinsrd $1, 4(%eax), %xmm0, %xmm0
+; KNL_32-NEXT:    vpinsrd $2, (%ecx), %xmm0, %xmm0
+; KNL_32-NEXT:    vpinsrd $3, 4(%ecx), %xmm0, %xmm0
 ; KNL_32-NEXT:    retl
 ;
 ; SKX-LABEL: test26:
 ; SKX:       # %bb.0:
 ; SKX-NEXT:    vpmovsxdq %xmm0, %xmm0
-; SKX-NEXT:    vpbroadcastq %rdi, %xmm2
+; SKX-NEXT:    vpbroadcastq %rdi, %xmm1
 ; SKX-NEXT:    vpsllq $3, %xmm0, %xmm0
-; SKX-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; SKX-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
 ; SKX-NEXT:    vmovq %xmm0, %rax
-; SKX-NEXT:    vpinsrq $0, (%rax), %xmm1, %xmm1
-; SKX-NEXT:    vpextrq $1, %xmm0, %rax
-; SKX-NEXT:    vpinsrq $1, (%rax), %xmm1, %xmm0
+; SKX-NEXT:    vpextrq $1, %xmm0, %rcx
+; SKX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; SKX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; SKX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
 ; SKX-NEXT:    retq
 ;
 ; SKX_32-LABEL: test26:
 ; SKX_32:       # %bb.0:
 ; SKX_32-NEXT:    vpslld $3, %xmm0, %xmm0
 ; SKX_32-NEXT:    vpaddd {{[0-9]+}}(%esp){1to4}, %xmm0, %xmm0
 ; SKX_32-NEXT:    vmovd %xmm0, %eax
-; SKX_32-NEXT:    vpinsrd $0, (%eax), %xmm1, %xmm1
-; SKX_32-NEXT:    vpinsrd $1, 4(%eax), %xmm1, %xmm1
-; SKX_32-NEXT:    vpextrd $1, %xmm0, %eax
-; SKX_32-NEXT:    vpinsrd $2, (%eax), %xmm1, %xmm0
-; SKX_32-NEXT:    vpinsrd $3, 4(%eax), %xmm0, %xmm0
+; SKX_32-NEXT:    vpextrd $1, %xmm0, %ecx
+; SKX_32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SKX_32-NEXT:    vpinsrd $1, 4(%eax), %xmm0, %xmm0
+; SKX_32-NEXT:    vpinsrd $2, (%ecx), %xmm0, %xmm0
+; SKX_32-NEXT:    vpinsrd $3, 4(%ecx), %xmm0, %xmm0
 ; SKX_32-NEXT:    retl
   %sext_ind = sext <2 x i32> %ind to <2 x i64>
   %gep.random = getelementptr i64, i64* %base, <2 x i64> %sext_ind
   %res = call <2 x i64> @llvm.masked.gather.v2i64.v2p0i64(<2 x i64*> %gep.random, i32 8, <2 x i1> <i1 true, i1 true>, <2 x i64> %src0)
   ret <2 x i64>%res
 }
 
 ; Result type requires widening; all-ones mask

llvm/test/CodeGen/X86/shuffle-extract-subvector.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s
 
 define void @f(<4 x half>* %a, <4 x half>* %b, <8 x half>* %c) {
 ; CHECK-LABEL: f:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    movzwl (%rdi), %eax
-; CHECK-NEXT:    movzwl 2(%rdi), %ecx
-; CHECK-NEXT:    movw %cx, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movzwl 6(%rdi), %r8d
-; CHECK-NEXT:    movzwl 4(%rdi), %r11d
-; CHECK-NEXT:    movq (%rsi), %rsi
-; CHECK-NEXT:    movq %rsi, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movq (%rdi), %rax
+; CHECK-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movdqa -{{[0-9]+}}(%rsp), %xmm0
-; CHECK-NEXT:    pextrw $1, %xmm0, %r9d
-; CHECK-NEXT:    movd %xmm0, %r10d
-; CHECK-NEXT:    movl -{{[0-9]+}}(%rsp), %esi
-; CHECK-NEXT:    pextrw $3, %xmm0, %eax
-; CHECK-NEXT:    pextrw $2, %xmm0, %edi
-; CHECK-NEXT:    movw %r11w, 8(%rdx)
-; CHECK-NEXT:    movw %cx, 4(%rdx)
-; CHECK-NEXT:    movw %r8w, 12(%rdx)
-; CHECK-NEXT:    movw %si, (%rdx)
-; CHECK-NEXT:    movw %di, 10(%rdx)
-; CHECK-NEXT:    movw %ax, 14(%rdx)
-; CHECK-NEXT:    movw %r10w, 2(%rdx)
-; CHECK-NEXT:    movw %r9w, 6(%rdx)
+; CHECK-NEXT:    movq (%rsi), %rax
+; CHECK-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
+; CHECK-NEXT:    movdqa %xmm0, (%rdx)
 ; CHECK-NEXT:    retq
   %tmp4 = load <4 x half>, <4 x half>* %a
   %tmp5 = load <4 x half>, <4 x half>* %b
   %tmp7 = shufflevector <4 x half> %tmp4, <4 x half> %tmp5, <2 x i32> <i32 0, i32 4>
   %tmp8 = shufflevector <4 x half> %tmp4, <4 x half> %tmp5, <2 x i32> <i32 1, i32 5>
   %tmp9 = shufflevector <4 x half> %tmp4, <4 x half> %tmp5, <2 x i32> <i32 2, i32 6>
   %tmp10 = shufflevector <4 x half> %tmp4, <4 x half> %tmp5, <2 x i32> <i32 3, i32 7>
   %tmp11 = extractelement <2 x half> %tmp7, i32 0

llvm/test/CodeGen/X86/sse-insertelt-from-mem.ll

 ; AVX-NEXT:    retq
   %s = load i32, i32* %s.addr
   %i0 = insertelement <4 x i32> %x, i32 %s, i32 0
   %i1 = insertelement <4 x i32> %i0, i32 %s, i32 1
   ret <4 x i32> %i1
 }
 
 define <2 x i64> @insert_i64_two_elts(<2 x i64> %x, i64* %s.addr) {
-; SSE2-LABEL: insert_i64_two_elts:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movq (%rdi), %rax
-; SSE2-NEXT:    movq %rax, %xmm0
-; SSE2-NEXT:    movq %rax, %xmm1
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: insert_i64_two_elts:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    movq (%rdi), %rax
-; SSE41-NEXT:    pinsrq $0, %rax, %xmm0
-; SSE41-NEXT:    pinsrq $1, %rax, %xmm0
-; SSE41-NEXT:    retq
+; SSE-LABEL: insert_i64_two_elts:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: insert_i64_two_elts:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    movq (%rdi), %rax
-; AVX-NEXT:    vpinsrq $0, %rax, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrq $1, %rax, %xmm0, %xmm0
+; AVX-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
 ; AVX-NEXT:    retq
   %s = load i64, i64* %s.addr
   %i0 = insertelement <2 x i64> %x, i64 %s, i32 0
   %i1 = insertelement <2 x i64> %i0, i64 %s, i32 1
   ret <2 x i64> %i1
 }
 
 ; Special tests

llvm/test/CodeGen/X86/sse-insertelt.ll

 ; AVX-NEXT:    vpinsrd $1, %edi, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %i0 = insertelement <4 x i32> %x, i32 %s, i32 0
   %i1 = insertelement <4 x i32> %i0, i32 %s, i32 1
   ret <4 x i32> %i1
 }
 
 define <2 x i64> @insert_i64_two_elts(<2 x i64> %x, i64 %s) {
-; SSE2-LABEL: insert_i64_two_elts:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movq %rdi, %xmm0
-; SSE2-NEXT:    movq %rdi, %xmm1
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: insert_i64_two_elts:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pinsrq $0, %rdi, %xmm0
-; SSE41-NEXT:    pinsrq $1, %rdi, %xmm0
-; SSE41-NEXT:    retq
-;
-; AVX-LABEL: insert_i64_two_elts:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpinsrq $0, %rdi, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrq $1, %rdi, %xmm0, %xmm0
-; AVX-NEXT:    retq
+; SSE-LABEL: insert_i64_two_elts:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movq %rdi, %xmm0
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
+; SSE-NEXT:    retq
   %i0 = insertelement <2 x i64> %x, i64 %s, i32 0
   %i1 = insertelement <2 x i64> %i0, i64 %s, i32 1
   ret <2 x i64> %i1
 }

llvm/test/CodeGen/X86/vec_insert-7.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=i686-apple-darwin9 -mattr=+mmx,+sse4.2 | FileCheck %s --check-prefix=X86
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin9 -mattr=+mmx,+sse4.2 | FileCheck %s --check-prefix=X64
 
 ; MMX insertelement is not available; these are promoted to xmm.
 ; (Without SSE they are split to two ints, and the code is much better.)
 
 define x86_mmx @mmx_movzl(x86_mmx %x) nounwind {
 ; X86-LABEL: mmx_movzl:
 ; X86:       ## %bb.0:
-; X86-NEXT:    movq {{\.?LCPI[0-9]+_[0-9]+}}, %mm0
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    movd %eax, %mm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: mmx_movzl:
 ; X64:       ## %bb.0:
 ; X64-NEXT:    movl $32, %eax
 ; X64-NEXT:    movq %rax, %xmm0
 ; X64-NEXT:    retq
   %tmp = bitcast x86_mmx %x to <2 x i32>
   %tmp3 = insertelement <2 x i32> %tmp, i32 32, i32 0
   %tmp8 = insertelement <2 x i32> %tmp3, i32 0, i32 1
   %tmp9 = bitcast <2 x i32> %tmp8 to x86_mmx
   ret x86_mmx %tmp9
 }