[AVX-512] Teach two address instruction pass to replace masked move instructions with blendm instructions when its beneficial.

[craig.topper]

Summary

Isel now selects masked move instructions for vselect instead of blendm. But sometimes it beneficial to register allocation to remove the tied register constraint by using blendm instructions.

This also picks up cases where the masked move was created due to a masked load intrinsic.

[RKSimon]

LGTM

llvm/trunk/lib/Target/X86/X86InstrAVX512.td

multiclass avx512_load<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
 
   let canFoldAsLoad = 1, isReMaterializable = 1,
       SchedRW = [WriteLoad] in
   def rm : AVX512PI<opc, MRMSrcMem, (outs _.RC:$dst), (ins _.MemOp:$src),
                     !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
                     [(set _.RC:$dst, (_.VT (bitconvert (ld_frag addr:$src))))],
                     _.ExeDomain>, EVEX;
 
-  let Constraints = "$src0 = $dst" in {
+  let Constraints = "$src0 = $dst", isConvertibleToThreeAddress = 1 in {
   def rrk : AVX512PI<opc, MRMSrcReg, (outs _.RC:$dst),
                     (ins _.RC:$src0, _.KRCWM:$mask, _.RC:$src1),
                     !strconcat(OpcodeStr, "\t{$src1, ${dst} {${mask}}|",
                     "${dst} {${mask}}, $src1}"),
                     [(set _.RC:$dst, (_.VT (SelectOprr _.KRCWM:$mask,
                                         (_.VT _.RC:$src1),
                                         (_.VT _.RC:$src0))))], _.ExeDomain>,
                      EVEX, EVEX_K;

llvm/trunk/lib/Target/X86/X86InstrInfo.cpp

case X86::ADD16ri8_DB:
     if (DisableLEA16)
       return is64Bit ? convertToThreeAddressWithLEA(MIOpc, MFI, MI, LV)
                      : nullptr;
     assert(MI.getNumOperands() >= 3 && "Unknown add instruction!");
     NewMI = addOffset(
         BuildMI(MF, MI.getDebugLoc(), get(X86::LEA16r)).add(Dest).add(Src),
         MI.getOperand(2));
     break;
+
+  case X86::VMOVDQU8Z128rmk:
+  case X86::VMOVDQU8Z256rmk:
+  case X86::VMOVDQU8Zrmk:
+  case X86::VMOVDQU16Z128rmk:
+  case X86::VMOVDQU16Z256rmk:
+  case X86::VMOVDQU16Zrmk:
+  case X86::VMOVDQU32Z128rmk: case X86::VMOVDQA32Z128rmk:
+  case X86::VMOVDQU32Z256rmk: case X86::VMOVDQA32Z256rmk:
+  case X86::VMOVDQU32Zrmk:    case X86::VMOVDQA32Zrmk:
+  case X86::VMOVDQU64Z128rmk: case X86::VMOVDQA64Z128rmk:
+  case X86::VMOVDQU64Z256rmk: case X86::VMOVDQA64Z256rmk:
+  case X86::VMOVDQU64Zrmk:    case X86::VMOVDQA64Zrmk:
+  case X86::VMOVUPDZ128rmk:   case X86::VMOVAPDZ128rmk:
+  case X86::VMOVUPDZ256rmk:   case X86::VMOVAPDZ256rmk:
+  case X86::VMOVUPDZrmk:      case X86::VMOVAPDZrmk:
+  case X86::VMOVUPSZ128rmk:   case X86::VMOVAPSZ128rmk:
+  case X86::VMOVUPSZ256rmk:   case X86::VMOVAPSZ256rmk:
+  case X86::VMOVUPSZrmk:      case X86::VMOVAPSZrmk: {
+    unsigned Opc;
+    switch (MIOpc) {
+    default: llvm_unreachable("Unreachable!");
+    case X86::VMOVDQU8Z128rmk:  Opc = X86::VPBLENDMBZ128rmk; break;
+    case X86::VMOVDQU8Z256rmk:  Opc = X86::VPBLENDMBZ256rmk; break;
+    case X86::VMOVDQU8Zrmk:     Opc = X86::VPBLENDMBZrmk;    break;
+    case X86::VMOVDQU16Z128rmk: Opc = X86::VPBLENDMWZ128rmk; break;
+    case X86::VMOVDQU16Z256rmk: Opc = X86::VPBLENDMWZ256rmk; break;
+    case X86::VMOVDQU16Zrmk:    Opc = X86::VPBLENDMWZrmk;    break;
+    case X86::VMOVDQU32Z128rmk: Opc = X86::VPBLENDMDZ128rmk; break;
+    case X86::VMOVDQU32Z256rmk: Opc = X86::VPBLENDMDZ256rmk; break;
+    case X86::VMOVDQU32Zrmk:    Opc = X86::VPBLENDMDZrmk;    break;
+    case X86::VMOVDQU64Z128rmk: Opc = X86::VPBLENDMQZ128rmk; break;
+    case X86::VMOVDQU64Z256rmk: Opc = X86::VPBLENDMQZ256rmk; break;
+    case X86::VMOVDQU64Zrmk:    Opc = X86::VPBLENDMQZrmk;    break;
+    case X86::VMOVUPDZ128rmk:   Opc = X86::VBLENDMPDZ128rmk; break;
+    case X86::VMOVUPDZ256rmk:   Opc = X86::VBLENDMPDZ256rmk; break;
+    case X86::VMOVUPDZrmk:      Opc = X86::VBLENDMPDZrmk;    break;
+    case X86::VMOVUPSZ128rmk:   Opc = X86::VBLENDMPSZ128rmk; break;
+    case X86::VMOVUPSZ256rmk:   Opc = X86::VBLENDMPSZ256rmk; break;
+    case X86::VMOVUPSZrmk:      Opc = X86::VBLENDMPSZrmk;    break;
+    case X86::VMOVDQA32Z128rmk: Opc = X86::VPBLENDMDZ128rmk; break;
+    case X86::VMOVDQA32Z256rmk: Opc = X86::VPBLENDMDZ256rmk; break;
+    case X86::VMOVDQA32Zrmk:    Opc = X86::VPBLENDMDZrmk;    break;
+    case X86::VMOVDQA64Z128rmk: Opc = X86::VPBLENDMQZ128rmk; break;
+    case X86::VMOVDQA64Z256rmk: Opc = X86::VPBLENDMQZ256rmk; break;
+    case X86::VMOVDQA64Zrmk:    Opc = X86::VPBLENDMQZrmk;    break;
+    case X86::VMOVAPDZ128rmk:   Opc = X86::VBLENDMPDZ128rmk; break;
+    case X86::VMOVAPDZ256rmk:   Opc = X86::VBLENDMPDZ256rmk; break;
+    case X86::VMOVAPDZrmk:      Opc = X86::VBLENDMPDZrmk;    break;
+    case X86::VMOVAPSZ128rmk:   Opc = X86::VBLENDMPSZ128rmk; break;
+    case X86::VMOVAPSZ256rmk:   Opc = X86::VBLENDMPSZ256rmk; break;
+    case X86::VMOVAPSZrmk:      Opc = X86::VBLENDMPSZrmk;    break;
+    }
+
+    NewMI = BuildMI(MF, MI.getDebugLoc(), get(Opc))
+              .add(Dest)
+              .add(MI.getOperand(2))
+              .add(Src)
+              .add(MI.getOperand(3))
+              .add(MI.getOperand(4))
+              .add(MI.getOperand(5))
+              .add(MI.getOperand(6))
+              .add(MI.getOperand(7));
+    break;
+  }
+  case X86::VMOVDQU8Z128rrk:
+  case X86::VMOVDQU8Z256rrk:
+  case X86::VMOVDQU8Zrrk:
+  case X86::VMOVDQU16Z128rrk:
+  case X86::VMOVDQU16Z256rrk:
+  case X86::VMOVDQU16Zrrk:
+  case X86::VMOVDQU32Z128rrk: case X86::VMOVDQA32Z128rrk:
+  case X86::VMOVDQU32Z256rrk: case X86::VMOVDQA32Z256rrk:
+  case X86::VMOVDQU32Zrrk:    case X86::VMOVDQA32Zrrk:
+  case X86::VMOVDQU64Z128rrk: case X86::VMOVDQA64Z128rrk:
+  case X86::VMOVDQU64Z256rrk: case X86::VMOVDQA64Z256rrk:
+  case X86::VMOVDQU64Zrrk:    case X86::VMOVDQA64Zrrk:
+  case X86::VMOVUPDZ128rrk:   case X86::VMOVAPDZ128rrk:
+  case X86::VMOVUPDZ256rrk:   case X86::VMOVAPDZ256rrk:
+  case X86::VMOVUPDZrrk:      case X86::VMOVAPDZrrk:
+  case X86::VMOVUPSZ128rrk:   case X86::VMOVAPSZ128rrk:
+  case X86::VMOVUPSZ256rrk:   case X86::VMOVAPSZ256rrk:
+  case X86::VMOVUPSZrrk:      case X86::VMOVAPSZrrk: {
+    unsigned Opc;
+    switch (MIOpc) {
+    default: llvm_unreachable("Unreachable!");
+    case X86::VMOVDQU8Z128rrk:  Opc = X86::VPBLENDMBZ128rrk; break;
+    case X86::VMOVDQU8Z256rrk:  Opc = X86::VPBLENDMBZ256rrk; break;
+    case X86::VMOVDQU8Zrrk:     Opc = X86::VPBLENDMBZrrk;    break;
+    case X86::VMOVDQU16Z128rrk: Opc = X86::VPBLENDMWZ128rrk; break;
+    case X86::VMOVDQU16Z256rrk: Opc = X86::VPBLENDMWZ256rrk; break;
+    case X86::VMOVDQU16Zrrk:    Opc = X86::VPBLENDMWZrrk;    break;
+    case X86::VMOVDQU32Z128rrk: Opc = X86::VPBLENDMDZ128rrk; break;
+    case X86::VMOVDQU32Z256rrk: Opc = X86::VPBLENDMDZ256rrk; break;
+    case X86::VMOVDQU32Zrrk:    Opc = X86::VPBLENDMDZrrk;    break;
+    case X86::VMOVDQU64Z128rrk: Opc = X86::VPBLENDMQZ128rrk; break;
+    case X86::VMOVDQU64Z256rrk: Opc = X86::VPBLENDMQZ256rrk; break;
+    case X86::VMOVDQU64Zrrk:    Opc = X86::VPBLENDMQZrrk;    break;
+    case X86::VMOVUPDZ128rrk:   Opc = X86::VBLENDMPDZ128rrk; break;
+    case X86::VMOVUPDZ256rrk:   Opc = X86::VBLENDMPDZ256rrk; break;
+    case X86::VMOVUPDZrrk:      Opc = X86::VBLENDMPDZrrk;    break;
+    case X86::VMOVUPSZ128rrk:   Opc = X86::VBLENDMPSZ128rrk; break;
+    case X86::VMOVUPSZ256rrk:   Opc = X86::VBLENDMPSZ256rrk; break;
+    case X86::VMOVUPSZrrk:      Opc = X86::VBLENDMPSZrrk;    break;
+    case X86::VMOVDQA32Z128rrk: Opc = X86::VPBLENDMDZ128rrk; break;
+    case X86::VMOVDQA32Z256rrk: Opc = X86::VPBLENDMDZ256rrk; break;
+    case X86::VMOVDQA32Zrrk:    Opc = X86::VPBLENDMDZrrk;    break;
+    case X86::VMOVDQA64Z128rrk: Opc = X86::VPBLENDMQZ128rrk; break;
+    case X86::VMOVDQA64Z256rrk: Opc = X86::VPBLENDMQZ256rrk; break;
+    case X86::VMOVDQA64Zrrk:    Opc = X86::VPBLENDMQZrrk;    break;
+    case X86::VMOVAPDZ128rrk:   Opc = X86::VBLENDMPDZ128rrk; break;
+    case X86::VMOVAPDZ256rrk:   Opc = X86::VBLENDMPDZ256rrk; break;
+    case X86::VMOVAPDZrrk:      Opc = X86::VBLENDMPDZrrk;    break;
+    case X86::VMOVAPSZ128rrk:   Opc = X86::VBLENDMPSZ128rrk; break;
+    case X86::VMOVAPSZ256rrk:   Opc = X86::VBLENDMPSZ256rrk; break;
+    case X86::VMOVAPSZrrk:      Opc = X86::VBLENDMPSZrrk;    break;
+    }
+
+    NewMI = BuildMI(MF, MI.getDebugLoc(), get(Opc))
+              .add(Dest)
+              .add(MI.getOperand(2))
+              .add(Src)
+              .add(MI.getOperand(3));
+    break;
+  }
   }
 
   if (!NewMI) return nullptr;
 
   if (LV) {  // Update live variables
     if (Src.isKill())
       LV->replaceKillInstruction(Src.getReg(), MI, *NewMI);
     if (Dest.isDead())

llvm/trunk/test/CodeGen/X86/avx512-bugfix-26264.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=x86_64-apple-darwin -mattr=avx512f,avx512bw < %s | FileCheck %s --check-prefix=AVX512BW
 
 define <32 x double> @test_load_32f64(<32 x double>* %ptrs, <32 x i1> %mask, <32 x double> %src0)  {
 ; AVX512BW-LABEL: test_load_32f64:
 ; AVX512BW:       ## BB#0:
 ; AVX512BW-NEXT:    vpsllw $7, %ymm0, %ymm0
 ; AVX512BW-NEXT:    vpmovb2m %zmm0, %k1
-; AVX512BW-NEXT:    vmovupd (%rdi), %zmm1 {%k1}
+; AVX512BW-NEXT:    vblendmpd (%rdi), %zmm1, %zmm0 {%k1}
 ; AVX512BW-NEXT:    kshiftrd $16, %k1, %k2
-; AVX512BW-NEXT:    vmovupd 128(%rdi), %zmm3 {%k2}
+; AVX512BW-NEXT:    vblendmpd 128(%rdi), %zmm3, %zmm5 {%k2}
 ; AVX512BW-NEXT:    kshiftrw $8, %k1, %k1
-; AVX512BW-NEXT:    vmovupd 64(%rdi), %zmm2 {%k1}
+; AVX512BW-NEXT:    vblendmpd 64(%rdi), %zmm2, %zmm1 {%k1}
 ; AVX512BW-NEXT:    kshiftrw $8, %k2, %k1
-; AVX512BW-NEXT:    vmovupd 192(%rdi), %zmm4 {%k1}
-; AVX512BW-NEXT:    vmovapd %zmm1, %zmm0
-; AVX512BW-NEXT:    vmovapd %zmm2, %zmm1
-; AVX512BW-NEXT:    vmovapd %zmm3, %zmm2
-; AVX512BW-NEXT:    vmovapd %zmm4, %zmm3
+; AVX512BW-NEXT:    vblendmpd 192(%rdi), %zmm4, %zmm3 {%k1}
+; AVX512BW-NEXT:    vmovapd %zmm5, %zmm2
 ; AVX512BW-NEXT:    retq
   %res = call <32 x double> @llvm.masked.load.v32f64.p0v32f64(<32 x double>* %ptrs, i32 4, <32 x i1> %mask, <32 x double> %src0)
   ret <32 x double> %res
 }
 
 define <32 x i64> @test_load_32i64(<32 x i64>* %ptrs, <32 x i1> %mask, <32 x i64> %src0)  {
 ; AVX512BW-LABEL: test_load_32i64:
 ; AVX512BW:       ## BB#0:
 ; AVX512BW-NEXT:    vpsllw $7, %ymm0, %ymm0
 ; AVX512BW-NEXT:    vpmovb2m %zmm0, %k1
-; AVX512BW-NEXT:    vmovdqu64 (%rdi), %zmm1 {%k1}
+; AVX512BW-NEXT:    vpblendmq (%rdi), %zmm1, %zmm0 {%k1}
 ; AVX512BW-NEXT:    kshiftrd $16, %k1, %k2
-; AVX512BW-NEXT:    vmovdqu64 128(%rdi), %zmm3 {%k2}
+; AVX512BW-NEXT:    vpblendmq 128(%rdi), %zmm3, %zmm5 {%k2}
 ; AVX512BW-NEXT:    kshiftrw $8, %k1, %k1
-; AVX512BW-NEXT:    vmovdqu64 64(%rdi), %zmm2 {%k1}
+; AVX512BW-NEXT:    vpblendmq 64(%rdi), %zmm2, %zmm1 {%k1}
 ; AVX512BW-NEXT:    kshiftrw $8, %k2, %k1
-; AVX512BW-NEXT:    vmovdqu64 192(%rdi), %zmm4 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm3
+; AVX512BW-NEXT:    vpblendmq 192(%rdi), %zmm4, %zmm3 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm2
 ; AVX512BW-NEXT:    retq
   %res = call <32 x i64> @llvm.masked.load.v32i64.p0v32i64(<32 x i64>* %ptrs, i32 4, <32 x i1> %mask, <32 x i64> %src0)
   ret <32 x i64> %res
 }
 
 declare <32 x i64> @llvm.masked.load.v32i64.p0v32i64(<32 x i64>* %ptrs, i32, <32 x i1> %mask, <32 x i64> %src0)
 declare <32 x double> @llvm.masked.load.v32f64.p0v32f64(<32 x double>* %ptrs, i32, <32 x i1> %mask, <32 x double> %src0)

llvm/trunk/test/CodeGen/X86/avx512-masked-memop-64-32.ll

; AVX512-NEXT:    retq
   ret void
 }
 
 define <16 x float> @test4(<16 x i32> %trigger, <16 x float>* %addr, <16 x float> %dst) {
 ; AVX512-LABEL: test4:
 ; AVX512:       ## BB#0:
 ; AVX512-NEXT:    vpxord %zmm2, %zmm2, %zmm2
 ; AVX512-NEXT:    vpcmpeqd %zmm2, %zmm0, %k1
-; AVX512-NEXT:    vmovups (%rdi), %zmm1 {%k1}
-; AVX512-NEXT:    vmovaps %zmm1, %zmm0
+; AVX512-NEXT:    vblendmps (%rdi), %zmm1, %zmm0 {%k1}
 ; AVX512-NEXT:    retq
   %mask = icmp eq <16 x i32> %trigger, zeroinitializer
   %res = call <16 x float> @llvm.masked.load.v16f32.p0v16f32(<16 x float>* %addr, i32 4, <16 x i1>%mask, <16 x float> %dst)
   ret <16 x float> %res
 }
 
 define void @test13(<16 x i32> %trigger, <16 x float>* %addr, <16 x float> %val) {
 ; AVX512-LABEL: test13:

 declare void @llvm.masked.store.v16f64.p0v16f64(<16 x double> %src0, <16 x double>* %ptrs, i32, <16 x i1> %mask)
 
 define <16 x i64> @test_load_16i64(<16 x i64>* %ptrs, <16 x i1> %mask, <16 x i64> %src0)  {
 ; AVX512F-LABEL: test_load_16i64:
 ; AVX512F:       ## BB#0:
 ; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
 ; AVX512F-NEXT:    vpslld $31, %zmm0, %zmm0
 ; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k1
-; AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm1 {%k1}
+; AVX512F-NEXT:    vpblendmq (%rdi), %zmm1, %zmm0 {%k1}
 ; AVX512F-NEXT:    kshiftrw $8, %k1, %k1
-; AVX512F-NEXT:    vmovdqu64 64(%rdi), %zmm2 {%k1}
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm1
+; AVX512F-NEXT:    vpblendmq 64(%rdi), %zmm2, %zmm1 {%k1}
 ; AVX512F-NEXT:    retq
 ;
 ; SKX-LABEL: test_load_16i64:
 ; SKX:       ## BB#0:
 ; SKX-NEXT:    vpsllw $7, %xmm0, %xmm0
 ; SKX-NEXT:    vpmovb2m %xmm0, %k1
-; SKX-NEXT:    vmovdqu64 (%rdi), %zmm1 {%k1}
+; SKX-NEXT:    vpblendmq (%rdi), %zmm1, %zmm0 {%k1}
 ; SKX-NEXT:    kshiftrw $8, %k1, %k1
-; SKX-NEXT:    vmovdqu64 64(%rdi), %zmm2 {%k1}
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0
-; SKX-NEXT:    vmovdqa64 %zmm2, %zmm1
+; SKX-NEXT:    vpblendmq 64(%rdi), %zmm2, %zmm1 {%k1}
 ; SKX-NEXT:    retq
   %res = call <16 x i64> @llvm.masked.load.v16i64.p0v16i64(<16 x i64>* %ptrs, i32 4, <16 x i1> %mask, <16 x i64> %src0)
   ret <16 x i64> %res
 }
 declare <16 x i64> @llvm.masked.load.v16i64.p0v16i64(<16 x i64>* %ptrs, i32, <16 x i1> %mask, <16 x i64> %src0)
 
 define <16 x double> @test_load_16f64(<16 x double>* %ptrs, <16 x i1> %mask, <16 x double> %src0)  {
 ; AVX512F-LABEL: test_load_16f64:
 ; AVX512F:       ## BB#0:
 ; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
 ; AVX512F-NEXT:    vpslld $31, %zmm0, %zmm0
 ; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k1
-; AVX512F-NEXT:    vmovupd (%rdi), %zmm1 {%k1}
+; AVX512F-NEXT:    vblendmpd (%rdi), %zmm1, %zmm0 {%k1}
 ; AVX512F-NEXT:    kshiftrw $8, %k1, %k1
-; AVX512F-NEXT:    vmovupd 64(%rdi), %zmm2 {%k1}
-; AVX512F-NEXT:    vmovapd %zmm1, %zmm0
-; AVX512F-NEXT:    vmovapd %zmm2, %zmm1
+; AVX512F-NEXT:    vblendmpd 64(%rdi), %zmm2, %zmm1 {%k1}
 ; AVX512F-NEXT:    retq
 ;
 ; SKX-LABEL: test_load_16f64:
 ; SKX:       ## BB#0:
 ; SKX-NEXT:    vpsllw $7, %xmm0, %xmm0
 ; SKX-NEXT:    vpmovb2m %xmm0, %k1
-; SKX-NEXT:    vmovupd (%rdi), %zmm1 {%k1}
+; SKX-NEXT:    vblendmpd (%rdi), %zmm1, %zmm0 {%k1}
 ; SKX-NEXT:    kshiftrw $8, %k1, %k1
-; SKX-NEXT:    vmovupd 64(%rdi), %zmm2 {%k1}
-; SKX-NEXT:    vmovapd %zmm1, %zmm0
-; SKX-NEXT:    vmovapd %zmm2, %zmm1
+; SKX-NEXT:    vblendmpd 64(%rdi), %zmm2, %zmm1 {%k1}
 ; SKX-NEXT:    retq
   %res = call <16 x double> @llvm.masked.load.v16f64.p0v16f64(<16 x double>* %ptrs, i32 4, <16 x i1> %mask, <16 x double> %src0)
   ret <16 x double> %res
 }
 declare <16 x double> @llvm.masked.load.v16f64.p0v16f64(<16 x double>* %ptrs, i32, <16 x i1> %mask, <16 x double> %src0)
 
 define <32 x double> @test_load_32f64(<32 x double>* %ptrs, <32 x i1> %mask, <32 x double> %src0)  {
 ; AVX512F-LABEL: test_load_32f64:
 ; AVX512F:       ## BB#0:
 ; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm5
 ; AVX512F-NEXT:    vpmovsxbd %xmm5, %zmm5
 ; AVX512F-NEXT:    vpslld $31, %zmm5, %zmm5
 ; AVX512F-NEXT:    vptestmd %zmm5, %zmm5, %k1
-; AVX512F-NEXT:    vmovupd 128(%rdi), %zmm3 {%k1}
+; AVX512F-NEXT:    vblendmpd 128(%rdi), %zmm3, %zmm5 {%k1}
 ; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
 ; AVX512F-NEXT:    vpslld $31, %zmm0, %zmm0
 ; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k2
-; AVX512F-NEXT:    vmovupd (%rdi), %zmm1 {%k2}
+; AVX512F-NEXT:    vblendmpd (%rdi), %zmm1, %zmm0 {%k2}
 ; AVX512F-NEXT:    kshiftrw $8, %k1, %k1
-; AVX512F-NEXT:    vmovupd 192(%rdi), %zmm4 {%k1}
+; AVX512F-NEXT:    vblendmpd 192(%rdi), %zmm4, %zmm3 {%k1}
 ; AVX512F-NEXT:    kshiftrw $8, %k2, %k1
-; AVX512F-NEXT:    vmovupd 64(%rdi), %zmm2 {%k1}
-; AVX512F-NEXT:    vmovapd %zmm1, %zmm0
-; AVX512F-NEXT:    vmovapd %zmm2, %zmm1
-; AVX512F-NEXT:    vmovapd %zmm3, %zmm2
-; AVX512F-NEXT:    vmovapd %zmm4, %zmm3
+; AVX512F-NEXT:    vblendmpd 64(%rdi), %zmm2, %zmm1 {%k1}
+; AVX512F-NEXT:    vmovapd %zmm5, %zmm2
 ; AVX512F-NEXT:    retq
 ;
 ; SKX-LABEL: test_load_32f64:
 ; SKX:       ## BB#0:
 ; SKX-NEXT:    vpsllw $7, %ymm0, %ymm0
 ; SKX-NEXT:    vpmovb2m %ymm0, %k1
-; SKX-NEXT:    vmovupd (%rdi), %zmm1 {%k1}
+; SKX-NEXT:    vblendmpd (%rdi), %zmm1, %zmm0 {%k1}
 ; SKX-NEXT:    kshiftrd $16, %k1, %k2
-; SKX-NEXT:    vmovupd 128(%rdi), %zmm3 {%k2}
+; SKX-NEXT:    vblendmpd 128(%rdi), %zmm3, %zmm5 {%k2}
 ; SKX-NEXT:    kshiftrw $8, %k1, %k1
-; SKX-NEXT:    vmovupd 64(%rdi), %zmm2 {%k1}
+; SKX-NEXT:    vblendmpd 64(%rdi), %zmm2, %zmm1 {%k1}
 ; SKX-NEXT:    kshiftrw $8, %k2, %k1
-; SKX-NEXT:    vmovupd 192(%rdi), %zmm4 {%k1}
-; SKX-NEXT:    vmovapd %zmm1, %zmm0
-; SKX-NEXT:    vmovapd %zmm2, %zmm1
-; SKX-NEXT:    vmovapd %zmm3, %zmm2
-; SKX-NEXT:    vmovapd %zmm4, %zmm3
+; SKX-NEXT:    vblendmpd 192(%rdi), %zmm4, %zmm3 {%k1}
+; SKX-NEXT:    vmovapd %zmm5, %zmm2
 ; SKX-NEXT:    retq
   %res = call <32 x double> @llvm.masked.load.v32f64.p0v32f64(<32 x double>* %ptrs, i32 4, <32 x i1> %mask, <32 x double> %src0)
   ret <32 x double> %res
 }
 
 declare <32 x double> @llvm.masked.load.v32f64.p0v32f64(<32 x double>* %ptrs, i32, <32 x i1> %mask, <32 x double> %src0)

llvm/trunk/test/CodeGen/X86/avx512-masked_memop-16-8.ll

 }
 declare <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>*, i32, <16 x i1>, <16 x i8>)
 
 define <32 x i8> @test_mask_load_32xi8(<32 x i1> %mask, <32 x i8>* %addr, <32 x i8> %val) {
 ; CHECK-LABEL: test_mask_load_32xi8:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpsllw $7, %ymm0, %ymm0
 ; CHECK-NEXT:    vpmovb2m %ymm0, %k1
-; CHECK-NEXT:    vmovdqu8 (%rdi), %ymm1 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    vpblendmb (%rdi), %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %res = call <32 x i8> @llvm.masked.load.v32i8.p0v32i8(<32 x i8>* %addr, i32 4, <32 x i1>%mask, <32 x i8> %val)
   ret <32 x i8> %res
 }
 declare <32 x i8> @llvm.masked.load.v32i8.p0v32i8(<32 x i8>*, i32, <32 x i1>, <32 x i8>)
 
 define <64 x i8> @test_mask_load_64xi8(<64 x i1> %mask, <64 x i8>* %addr, <64 x i8> %val) {
 ; CHECK-LABEL: test_mask_load_64xi8:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpsllw $7, %zmm0, %zmm0
 ; CHECK-NEXT:    vpmovb2m %zmm0, %k1
-; CHECK-NEXT:    vmovdqu8 (%rdi), %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    vpblendmb (%rdi), %zmm1, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %res = call <64 x i8> @llvm.masked.load.v64i8.p0v64i8(<64 x i8>* %addr, i32 4, <64 x i1>%mask, <64 x i8> %val)
   ret <64 x i8> %res
 }
 declare <64 x i8> @llvm.masked.load.v64i8.p0v64i8(<64 x i8>*, i32, <64 x i1>, <64 x i8>)
 
 define <8 x i16> @test_mask_load_8xi16(<8 x i1> %mask, <8 x i16>* %addr, <8 x i16> %val) {
 ; CHECK-LABEL: test_mask_load_8xi16:

 }
 declare <16 x i16> @llvm.masked.load.v16i16.p0v16i16(<16 x i16>*, i32, <16 x i1>, <16 x i16>)
 
 define <32 x i16> @test_mask_load_32xi16(<32 x i1> %mask, <32 x i16>* %addr, <32 x i16> %val) {
 ; CHECK-LABEL: test_mask_load_32xi16:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpsllw $7, %ymm0, %ymm0
 ; CHECK-NEXT:    vpmovb2m %ymm0, %k1
-; CHECK-NEXT:    vmovdqu16 (%rdi), %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    vpblendmw (%rdi), %zmm1, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %res = call <32 x i16> @llvm.masked.load.v32i16.p0v32i16(<32 x i16>* %addr, i32 4, <32 x i1>%mask, <32 x i16> %val)
   ret <32 x i16> %res
 }
 declare <32 x i16> @llvm.masked.load.v32i16.p0v32i16(<32 x i16>*, i32, <32 x i1>, <32 x i16>)
 
 define void @test_mask_store_16xi8(<16 x i1> %mask, <16 x i8>* %addr, <16 x i8> %val) {
 ; CHECK-LABEL: test_mask_store_16xi8:

llvm/trunk/test/CodeGen/X86/avx512-regcall-NoMask.ll

define x86_regcallcc [4 x i32]* @test_CallargRetPointer([4 x i32]* %a)  {
   %e = call x86_regcallcc [4 x i32]* @test_argRetPointer([4 x i32]* %d)
   %f = ptrtoint [4 x i32]* %e to i32
   %g = add i32 %f, 1
   %h = inttoptr i32 %g to [4 x i32]*
   ret [4 x i32]* %h
 }
 
 ; X32-LABEL:  test_argRet128Vector:
-; X32:        vmovdqa{{.*}}  %xmm0, %xmm1
-; X32:        vmovdqa{{.*}}  %xmm1, %xmm0
+; X32:        vpblend{{.*}}  %xmm0, %xmm1, %xmm0
 ; X32:        ret{{.*}}
 
 ; WIN64-LABEL:  test_argRet128Vector:
-; WIN64:        vmovdqa{{.*}}  %xmm0, %xmm1
-; WIN64:        vmovdqa{{.*}}  %xmm1, %xmm0
+; WIN64:        vpblend{{.*}}  %xmm0, %xmm1, %xmm0
 ; WIN64:        ret{{.*}}
 
 ; Test regcall when receiving/returning 128 bit vector
 define x86_regcallcc <4 x i32> @test_argRet128Vector(<4 x i32> %a, <4 x i32> %b)  {
   %d = select <4 x i1> undef , <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %d
 }
 

 ; Test regcall when passing/retrieving 128 bit vector
 define x86_regcallcc <4 x i32> @test_CallargRet128Vector(<4 x i32> %a)  {
   %b = call x86_regcallcc <4 x i32> @test_argRet128Vector(<4 x i32> %a, <4 x i32> %a)
   %c = select <4 x i1> undef , <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %c
 }
 
 ; X32-LABEL:  test_argRet256Vector:
-; X32:        vmovdqa{{.*}}  %ymm0, %ymm1
-; X32:        vmovdqa{{.*}}  %ymm1, %ymm0
+; X32:        vpblend{{.*}}  %ymm0, %ymm1, %ymm0
 ; X32:        ret{{.*}}
 
 ; WIN64-LABEL:  test_argRet256Vector:
-; WIN64:        vmovdqa{{.*}}  %ymm0, %ymm1
-; WIN64:        vmovdqa{{.*}}  %ymm1, %ymm0
+; WIN64:        vpblend{{.*}}  %ymm0, %ymm1, %ymm0
 ; WIN64:        ret{{.*}}
 
 ; Test regcall when receiving/returning 256 bit vector
 define x86_regcallcc <8 x i32> @test_argRet256Vector(<8 x i32> %a, <8 x i32> %b)  {
   %d = select <8 x i1> undef , <8 x i32> %a, <8 x i32> %b
   ret <8 x i32> %d
 }
 

 ; Test regcall when passing/retrieving 256 bit vector
 define x86_regcallcc <8 x i32> @test_CallargRet256Vector(<8 x i32> %a)  {
   %b = call x86_regcallcc <8 x i32> @test_argRet256Vector(<8 x i32> %a, <8 x i32> %a)
   %c = select <8 x i1> undef , <8 x i32> %a, <8 x i32> %b
   ret <8 x i32> %c
 }
 
 ; X32-LABEL:  test_argRet512Vector:
-; X32:        vmovdqa{{.*}}  %zmm0, %zmm1
-; X32:        vmovdqa{{.*}}  %zmm1, %zmm0
+; X32:        vpblend{{.*}}  %zmm0, %zmm1, %zmm0
 ; X32:        ret{{.*}}
 
 ; WIN64-LABEL:  test_argRet512Vector:
-; WIN64:        vmovdqa{{.*}}  %zmm0, %zmm1
-; WIN64:        vmovdqa{{.*}}  %zmm1, %zmm0
+; WIN64:        vpblend{{.*}}  %zmm0, %zmm1, %zmm0
 ; WIN64:        ret{{.*}}
 
 ; Test regcall when receiving/returning 512 bit vector
 define x86_regcallcc <16 x i32> @test_argRet512Vector(<16 x i32> %a, <16 x i32> %b)  {
   %d = select <16 x i1> undef , <16 x i32> %a, <16 x i32> %b
   ret <16 x i32> %d
 }
 

llvm/trunk/test/CodeGen/X86/avx512-vec-cmp.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl | FileCheck %s --check-prefix=CHECK --check-prefix=KNL
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=skx | FileCheck %s --check-prefix=CHECK --check-prefix=SKX
 
 define <16 x float> @test1(<16 x float> %x, <16 x float> %y) nounwind {
 ; CHECK-LABEL: test1:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vcmpleps %zmm1, %zmm0, %k1
-; CHECK-NEXT:    vmovaps %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovaps %zmm1, %zmm0
+; CHECK-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask = fcmp ole <16 x float> %x, %y
   %max = select <16 x i1> %mask, <16 x float> %x, <16 x float> %y
   ret <16 x float> %max
 }
 
 define <8 x double> @test2(<8 x double> %x, <8 x double> %y) nounwind {
 ; CHECK-LABEL: test2:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vcmplepd %zmm1, %zmm0, %k1
-; CHECK-NEXT:    vmovapd %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovapd %zmm1, %zmm0
+; CHECK-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask = fcmp ole <8 x double> %x, %y
   %max = select <8 x i1> %mask, <8 x double> %x, <8 x double> %y
   ret <8 x double> %max
 }
 
 define <16 x i32> @test3(<16 x i32> %x, <16 x i32> %x1, <16 x i32>* %yp) nounwind {
 ; CHECK-LABEL: test3:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpeqd (%rdi), %zmm0, %k1
-; CHECK-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <16 x i32>, <16 x i32>* %yp, align 4
   %mask = icmp eq <16 x i32> %x, %y
   %max = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> %x1
   ret <16 x i32> %max
 }
 
 define <16 x i32> @test4_unsigned(<16 x i32> %x, <16 x i32> %y, <16 x i32> %x1) nounwind {
 ; CHECK-LABEL: test4_unsigned:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpnltud %zmm1, %zmm0, %k1
-; CHECK-NEXT:    vmovdqa32 %zmm2, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    vpblendmd %zmm2, %zmm1, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask = icmp uge <16 x i32> %x, %y
   %max = select <16 x i1> %mask, <16 x i32> %x1, <16 x i32> %y
   ret <16 x i32> %max
 }
 
 define <8 x i64> @test5(<8 x i64> %x, <8 x i64> %y) nounwind {
 ; CHECK-LABEL: test5:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpeqq %zmm1, %zmm0, %k1
-; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask = icmp eq <8 x i64> %x, %y
   %max = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> %y
   ret <8 x i64> %max
 }
 
 define <8 x i64> @test6_unsigned(<8 x i64> %x, <8 x i64> %y, <8 x i64> %x1) nounwind {
 ; CHECK-LABEL: test6_unsigned:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpnleuq %zmm1, %zmm0, %k1
-; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    vpblendmq %zmm2, %zmm1, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask = icmp ugt <8 x i64> %x, %y
   %max = select <8 x i1> %mask, <8 x i64> %x1, <8 x i64> %y
   ret <8 x i64> %max
 }
 
 define <4 x float> @test7(<4 x float> %a, <4 x float> %b) {
 ; KNL-LABEL: test7:
 ; KNL:       ## BB#0:
 ; KNL-NEXT:    vxorps %xmm2, %xmm2, %xmm2
 ; KNL-NEXT:    vcmpltps %xmm2, %xmm0, %xmm2
 ; KNL-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test7:
 ; SKX:       ## BB#0:
 ; SKX-NEXT:    vxorps %xmm2, %xmm2, %xmm2
 ; SKX-NEXT:    vcmpltps %xmm2, %xmm0, %k1
-; SKX-NEXT:    vmovaps %xmm0, %xmm1 {%k1}
-; SKX-NEXT:    vmovaps %xmm1, %xmm0
+; SKX-NEXT:    vblendmps %xmm0, %xmm1, %xmm0 {%k1}
 ; SKX-NEXT:    retq
 
   %mask = fcmp olt <4 x float> %a, zeroinitializer
   %c = select <4 x i1>%mask, <4 x float>%a, <4 x float>%b
   ret <4 x float>%c
 }
 
 define <2 x double> @test8(<2 x double> %a, <2 x double> %b) {
 ; KNL-LABEL: test8:
 ; KNL:       ## BB#0:
 ; KNL-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
 ; KNL-NEXT:    vcmpltpd %xmm2, %xmm0, %xmm2
 ; KNL-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test8:
 ; SKX:       ## BB#0:
 ; SKX-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
 ; SKX-NEXT:    vcmpltpd %xmm2, %xmm0, %k1
-; SKX-NEXT:    vmovapd %xmm0, %xmm1 {%k1}
-; SKX-NEXT:    vmovapd %xmm1, %xmm0
+; SKX-NEXT:    vblendmpd %xmm0, %xmm1, %xmm0 {%k1}
 ; SKX-NEXT:    retq
   %mask = fcmp olt <2 x double> %a, zeroinitializer
   %c = select <2 x i1>%mask, <2 x double>%a, <2 x double>%b
   ret <2 x double>%c
 }
 
 define <8 x i32> @test9(<8 x i32> %x, <8 x i32> %y) nounwind {
 ; KNL-LABEL: test9:
 ; KNL:       ## BB#0:
 ; KNL-NEXT:    ## kill: %YMM1<def> %YMM1<kill> %ZMM1<def>
 ; KNL-NEXT:    ## kill: %YMM0<def> %YMM0<kill> %ZMM0<def>
 ; KNL-NEXT:    vpcmpeqd %zmm1, %zmm0, %k1
-; KNL-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
-; KNL-NEXT:    vmovdqa %ymm1, %ymm0
+; KNL-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
+; KNL-NEXT:    ## kill: %YMM0<def> %YMM0<kill> %ZMM0<kill>
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test9:
 ; SKX:       ## BB#0:
 ; SKX-NEXT:    vpcmpeqd %ymm1, %ymm0, %k1
-; SKX-NEXT:    vmovdqa32 %ymm0, %ymm1 {%k1}
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0
+; SKX-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
 ; SKX-NEXT:    retq
   %mask = icmp eq <8 x i32> %x, %y
   %max = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %y
   ret <8 x i32> %max
 }
 
 define <8 x float> @test10(<8 x float> %x, <8 x float> %y) nounwind {
 ; KNL-LABEL: test10:
 ; KNL:       ## BB#0:
 ; KNL-NEXT:    ## kill: %YMM1<def> %YMM1<kill> %ZMM1<def>
 ; KNL-NEXT:    ## kill: %YMM0<def> %YMM0<kill> %ZMM0<def>
 ; KNL-NEXT:    vcmpeqps %zmm1, %zmm0, %k1
-; KNL-NEXT:    vmovaps %zmm0, %zmm1 {%k1}
-; KNL-NEXT:    vmovaps %ymm1, %ymm0
+; KNL-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1}
+; KNL-NEXT:    ## kill: %YMM0<def> %YMM0<kill> %ZMM0<kill>
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test10:
 ; SKX:       ## BB#0:
 ; SKX-NEXT:    vcmpeqps %ymm1, %ymm0, %k1
-; SKX-NEXT:    vmovaps %ymm0, %ymm1 {%k1}
-; SKX-NEXT:    vmovaps %ymm1, %ymm0
+; SKX-NEXT:    vblendmps %ymm0, %ymm1, %ymm0 {%k1}
 ; SKX-NEXT:    retq
 
   %mask = fcmp oeq <8 x float> %x, %y
   %max = select <8 x i1> %mask, <8 x float> %x, <8 x float> %y
   ret <8 x float> %max
 }
 
 define <8 x i32> @test11_unsigned(<8 x i32> %x, <8 x i32> %y) nounwind {
; CHECK-NEXT:    retq
   %res = select <8 x i1> %mask, <8 x i64> zeroinitializer, <8 x i64> %sub_r
   ret <8 x i64>%res
 }
 
 define <16 x i32> @test16(<16 x i32> %x, <16 x i32> %y, <16 x i32> %x1) nounwind {
 ; CHECK-LABEL: test16:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpled %zmm0, %zmm1, %k1
-; CHECK-NEXT:    vmovdqa32 %zmm2, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    vpblendmd %zmm2, %zmm1, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask = icmp sge <16 x i32> %x, %y
   %max = select <16 x i1> %mask, <16 x i32> %x1, <16 x i32> %y
   ret <16 x i32> %max
 }
 
 define <16 x i32> @test17(<16 x i32> %x, <16 x i32> %x1, <16 x i32>* %y.ptr) nounwind {
 ; CHECK-LABEL: test17:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpgtd (%rdi), %zmm0, %k1
-; CHECK-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <16 x i32>, <16 x i32>* %y.ptr, align 4
   %mask = icmp sgt <16 x i32> %x, %y
   %max = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> %x1
   ret <16 x i32> %max
 }
 
 define <16 x i32> @test18(<16 x i32> %x, <16 x i32> %x1, <16 x i32>* %y.ptr) nounwind {
 ; CHECK-LABEL: test18:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpled (%rdi), %zmm0, %k1
-; CHECK-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <16 x i32>, <16 x i32>* %y.ptr, align 4
   %mask = icmp sle <16 x i32> %x, %y
   %max = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> %x1
   ret <16 x i32> %max
 }
 
 define <16 x i32> @test19(<16 x i32> %x, <16 x i32> %x1, <16 x i32>* %y.ptr) nounwind {
 ; CHECK-LABEL: test19:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpleud (%rdi), %zmm0, %k1
-; CHECK-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <16 x i32>, <16 x i32>* %y.ptr, align 4
   %mask = icmp ule <16 x i32> %x, %y
   %max = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> %x1
   ret <16 x i32> %max
 }
 
 define <16 x i32> @test20(<16 x i32> %x, <16 x i32> %y, <16 x i32> %x1, <16 x i32> %y1) nounwind {
 ; CHECK-LABEL: test20:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpeqd %zmm1, %zmm0, %k1
 ; CHECK-NEXT:    vpcmpeqd %zmm3, %zmm2, %k1 {%k1}
-; CHECK-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask1 = icmp eq <16 x i32> %x1, %y1
   %mask0 = icmp eq <16 x i32> %x, %y
   %mask = select <16 x i1> %mask0, <16 x i1> %mask1, <16 x i1> zeroinitializer
   %max = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> %y
   ret <16 x i32> %max
 }
 
 define <8 x i64> @test21(<8 x i64> %x, <8 x i64> %y, <8 x i64> %x1, <8 x i64> %y1) nounwind {
 ; CHECK-LABEL: test21:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpleq %zmm1, %zmm0, %k1
 ; CHECK-NEXT:    vpcmpleq %zmm2, %zmm3, %k1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    vpblendmq %zmm0, %zmm2, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask1 = icmp sge <8 x i64> %x1, %y1
   %mask0 = icmp sle <8 x i64> %x, %y
   %mask = select <8 x i1> %mask0, <8 x i1> %mask1, <8 x i1> zeroinitializer
   %max = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> %x1
   ret <8 x i64> %max
 }
 
 define <8 x i64> @test22(<8 x i64> %x, <8 x i64>* %y.ptr, <8 x i64> %x1, <8 x i64> %y1) nounwind {
 ; CHECK-LABEL: test22:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpgtq %zmm2, %zmm1, %k1
 ; CHECK-NEXT:    vpcmpgtq (%rdi), %zmm0, %k1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask1 = icmp sgt <8 x i64> %x1, %y1
   %y = load <8 x i64>, <8 x i64>* %y.ptr, align 4
   %mask0 = icmp sgt <8 x i64> %x, %y
   %mask = select <8 x i1> %mask0, <8 x i1> %mask1, <8 x i1> zeroinitializer
   %max = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> %x1
   ret <8 x i64> %max
 }
 
 define <16 x i32> @test23(<16 x i32> %x, <16 x i32>* %y.ptr, <16 x i32> %x1, <16 x i32> %y1) nounwind {
 ; CHECK-LABEL: test23:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpled %zmm1, %zmm2, %k1
 ; CHECK-NEXT:    vpcmpleud (%rdi), %zmm0, %k1 {%k1}
-; CHECK-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask1 = icmp sge <16 x i32> %x1, %y1
   %y = load <16 x i32>, <16 x i32>* %y.ptr, align 4
   %mask0 = icmp ule <16 x i32> %x, %y
   %mask = select <16 x i1> %mask0, <16 x i1> %mask1, <16 x i1> zeroinitializer
   %max = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> %x1
   ret <16 x i32> %max
 }
 
 define <8 x i64> @test24(<8 x i64> %x, <8 x i64> %x1, i64* %yb.ptr) nounwind {
 ; CHECK-LABEL: test24:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpeqq (%rdi){1to8}, %zmm0, %k1
-; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %yb = load i64, i64* %yb.ptr, align 4
   %y.0 = insertelement <8 x i64> undef, i64 %yb, i32 0
   %y = shufflevector <8 x i64> %y.0, <8 x i64> undef, <8 x i32> zeroinitializer
   %mask = icmp eq <8 x i64> %x, %y
   %max = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> %x1
   ret <8 x i64> %max
 }
 
 define <16 x i32> @test25(<16 x i32> %x, i32* %yb.ptr, <16 x i32> %x1) nounwind {
 ; CHECK-LABEL: test25:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpled (%rdi){1to16}, %zmm0, %k1
-; CHECK-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %yb = load i32, i32* %yb.ptr, align 4
   %y.0 = insertelement <16 x i32> undef, i32 %yb, i32 0
   %y = shufflevector <16 x i32> %y.0, <16 x i32> undef, <16 x i32> zeroinitializer
   %mask = icmp sle <16 x i32> %x, %y
   %max = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> %x1
   ret <16 x i32> %max
 }
 
 define <16 x i32> @test26(<16 x i32> %x, i32* %yb.ptr, <16 x i32> %x1, <16 x i32> %y1) nounwind {
 ; CHECK-LABEL: test26:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpled %zmm1, %zmm2, %k1
 ; CHECK-NEXT:    vpcmpgtd (%rdi){1to16}, %zmm0, %k1 {%k1}
-; CHECK-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask1 = icmp sge <16 x i32> %x1, %y1
   %yb = load i32, i32* %yb.ptr, align 4
   %y.0 = insertelement <16 x i32> undef, i32 %yb, i32 0
   %y = shufflevector <16 x i32> %y.0, <16 x i32> undef, <16 x i32> zeroinitializer
   %mask0 = icmp sgt <16 x i32> %x, %y
   %mask = select <16 x i1> %mask0, <16 x i1> %mask1, <16 x i1> zeroinitializer
   %max = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> %x1
   ret <16 x i32> %max
 }
 
 define <8 x i64> @test27(<8 x i64> %x, i64* %yb.ptr, <8 x i64> %x1, <8 x i64> %y1) nounwind {
 ; CHECK-LABEL: test27:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpleq %zmm1, %zmm2, %k1
 ; CHECK-NEXT:    vpcmpleq (%rdi){1to8}, %zmm0, %k1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask1 = icmp sge <8 x i64> %x1, %y1
   %yb = load i64, i64* %yb.ptr, align 4
   %y.0 = insertelement <8 x i64> undef, i64 %yb, i32 0
   %y = shufflevector <8 x i64> %y.0, <8 x i64> undef, <8 x i32> zeroinitializer
   %mask0 = icmp sle <8 x i64> %x, %y
   %mask = select <8 x i1> %mask0, <8 x i1> %mask1, <8 x i1> zeroinitializer
   %max = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> %x1

 ; KNL:       ## BB#0:
 ; KNL-NEXT:    vcmpeqpd %ymm1, %ymm0, %ymm2
 ; KNL-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test30:
 ; SKX:       ## BB#0:
 ; SKX-NEXT:    vcmpeqpd %ymm1, %ymm0, %k1
-; SKX-NEXT:    vmovapd %ymm0, %ymm1 {%k1}
-; SKX-NEXT:    vmovapd %ymm1, %ymm0
+; SKX-NEXT:    vblendmpd %ymm0, %ymm1, %ymm0 {%k1}
 ; SKX-NEXT:    retq
 
   %mask = fcmp oeq <4 x double> %x, %y
   %max = select <4 x i1> %mask, <4 x double> %x, <4 x double> %y
   ret <4 x double> %max
 }
 
 define <2 x double> @test31(<2 x double> %x, <2 x double> %x1, <2 x double>* %yp) nounwind {
 ; KNL-LABEL: test31:
 ; KNL:       ## BB#0:
 ; KNL-NEXT:    vcmpltpd (%rdi), %xmm0, %xmm2
 ; KNL-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test31:
 ; SKX:       ## BB#0:
 ; SKX-NEXT:    vcmpltpd (%rdi), %xmm0, %k1
-; SKX-NEXT:    vmovapd %xmm0, %xmm1 {%k1}
-; SKX-NEXT:    vmovapd %xmm1, %xmm0
+; SKX-NEXT:    vblendmpd %xmm0, %xmm1, %xmm0 {%k1}
 ; SKX-NEXT:    retq
 
   %y = load <2 x double>, <2 x double>* %yp, align 4
   %mask = fcmp olt <2 x double> %x, %y
   %max = select <2 x i1> %mask, <2 x double> %x, <2 x double> %x1
   ret <2 x double> %max
 }
 
 define <4 x double> @test32(<4 x double> %x, <4 x double> %x1, <4 x double>* %yp) nounwind {
 ; KNL-LABEL: test32:
 ; KNL:       ## BB#0:
 ; KNL-NEXT:    vcmpltpd (%rdi), %ymm0, %ymm2
 ; KNL-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test32:
 ; SKX:       ## BB#0:
 ; SKX-NEXT:    vcmpltpd (%rdi), %ymm0, %k1
-; SKX-NEXT:    vmovapd %ymm0, %ymm1 {%k1}
-; SKX-NEXT:    vmovapd %ymm1, %ymm0
+; SKX-NEXT:    vblendmpd %ymm0, %ymm1, %ymm0 {%k1}
 ; SKX-NEXT:    retq
 
   %y = load <4 x double>, <4 x double>* %yp, align 4
   %mask = fcmp ogt <4 x double> %y, %x
   %max = select <4 x i1> %mask, <4 x double> %x, <4 x double> %x1
   ret <4 x double> %max
 }
 
 define <8 x double> @test33(<8 x double> %x, <8 x double> %x1, <8 x double>* %yp) nounwind {
 ; CHECK-LABEL: test33:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vcmpltpd (%rdi), %zmm0, %k1
-; CHECK-NEXT:    vmovapd %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovapd %zmm1, %zmm0
+; CHECK-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <8 x double>, <8 x double>* %yp, align 4
   %mask = fcmp olt <8 x double> %x, %y
   %max = select <8 x i1> %mask, <8 x double> %x, <8 x double> %x1
   ret <8 x double> %max
 }
 
 define <4 x float> @test34(<4 x float> %x, <4 x float> %x1, <4 x float>* %yp) nounwind {
 ; KNL-LABEL: test34:
 ; KNL:       ## BB#0:
 ; KNL-NEXT:    vcmpltps (%rdi), %xmm0, %xmm2
 ; KNL-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test34:
 ; SKX:       ## BB#0:
 ; SKX-NEXT:    vcmpltps (%rdi), %xmm0, %k1
-; SKX-NEXT:    vmovaps %xmm0, %xmm1 {%k1}
-; SKX-NEXT:    vmovaps %xmm1, %xmm0
+; SKX-NEXT:    vblendmps %xmm0, %xmm1, %xmm0 {%k1}
 ; SKX-NEXT:    retq
   %y = load <4 x float>, <4 x float>* %yp, align 4
   %mask = fcmp olt <4 x float> %x, %y
   %max = select <4 x i1> %mask, <4 x float> %x, <4 x float> %x1
   ret <4 x float> %max
 }
 
 define <8 x float> @test35(<8 x float> %x, <8 x float> %x1, <8 x float>* %yp) nounwind {
 ; KNL-LABEL: test35:
 ; KNL:       ## BB#0:
 ; KNL-NEXT:    ## kill: %YMM1<def> %YMM1<kill> %ZMM1<def>
 ; KNL-NEXT:    ## kill: %YMM0<def> %YMM0<kill> %ZMM0<def>
 ; KNL-NEXT:    vmovups (%rdi), %ymm2
 ; KNL-NEXT:    vcmpltps %zmm2, %zmm0, %k1
-; KNL-NEXT:    vmovaps %zmm0, %zmm1 {%k1}
-; KNL-NEXT:    vmovaps %ymm1, %ymm0
+; KNL-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1}
+; KNL-NEXT:    ## kill: %YMM0<def> %YMM0<kill> %ZMM0<kill>
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test35:
 ; SKX:       ## BB#0:
 ; SKX-NEXT:    vcmpltps (%rdi), %ymm0, %k1
-; SKX-NEXT:    vmovaps %ymm0, %ymm1 {%k1}
-; SKX-NEXT:    vmovaps %ymm1, %ymm0
+; SKX-NEXT:    vblendmps %ymm0, %ymm1, %ymm0 {%k1}
 ; SKX-NEXT:    retq
 
   %y = load <8 x float>, <8 x float>* %yp, align 4
   %mask = fcmp ogt <8 x float> %y, %x
   %max = select <8 x i1> %mask, <8 x float> %x, <8 x float> %x1
   ret <8 x float> %max
 }
 
 define <16 x float> @test36(<16 x float> %x, <16 x float> %x1, <16 x float>* %yp) nounwind {
 ; CHECK-LABEL: test36:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vcmpltps (%rdi), %zmm0, %k1
-; CHECK-NEXT:    vmovaps %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovaps %zmm1, %zmm0
+; CHECK-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <16 x float>, <16 x float>* %yp, align 4
   %mask = fcmp olt <16 x float> %x, %y
   %max = select <16 x i1> %mask, <16 x float> %x, <16 x float> %x1
   ret <16 x float> %max
 }
 
 define <8 x double> @test37(<8 x double> %x, <8 x double> %x1, double* %ptr) nounwind {
 ; CHECK-LABEL: test37:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vcmpltpd (%rdi){1to8}, %zmm0, %k1
-; CHECK-NEXT:    vmovapd %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovapd %zmm1, %zmm0
+; CHECK-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
 
   %a = load double, double* %ptr
   %v = insertelement <8 x double> undef, double %a, i32 0
   %shuffle = shufflevector <8 x double> %v, <8 x double> undef, <8 x i32> zeroinitializer
 
   %mask = fcmp ogt <8 x double> %shuffle, %x
   %max = select <8 x i1> %mask, <8 x double> %x, <8 x double> %x1
   ret <8 x double> %max
 }
 
 define <4 x double> @test38(<4 x double> %x, <4 x double> %x1, double* %ptr) nounwind {
 ; KNL-LABEL: test38:
 ; KNL:       ## BB#0:
 ; KNL-NEXT:    vbroadcastsd (%rdi), %ymm2
 ; KNL-NEXT:    vcmpltpd %ymm2, %ymm0, %ymm2
 ; KNL-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test38:
 ; SKX:       ## BB#0:
 ; SKX-NEXT:    vcmpltpd (%rdi){1to4}, %ymm0, %k1
-; SKX-NEXT:    vmovapd %ymm0, %ymm1 {%k1}
-; SKX-NEXT:    vmovapd %ymm1, %ymm0
+; SKX-NEXT:    vblendmpd %ymm0, %ymm1, %ymm0 {%k1}
 ; SKX-NEXT:    retq
 
   %a = load double, double* %ptr
   %v = insertelement <4 x double> undef, double %a, i32 0
   %shuffle = shufflevector <4 x double> %v, <4 x double> undef, <4 x i32> zeroinitializer
 
   %mask = fcmp ogt <4 x double> %shuffle, %x
   %max = select <4 x i1> %mask, <4 x double> %x, <4 x double> %x1
   ret <4 x double> %max
 }
 
 define <2 x double> @test39(<2 x double> %x, <2 x double> %x1, double* %ptr) nounwind {
 ; KNL-LABEL: test39:
 ; KNL:       ## BB#0:
 ; KNL-NEXT:    vmovddup {{.*#+}} xmm2 = mem[0,0]
 ; KNL-NEXT:    vcmpltpd %xmm2, %xmm0, %xmm2
 ; KNL-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test39:
 ; SKX:       ## BB#0:
 ; SKX-NEXT:    vcmpltpd (%rdi){1to2}, %xmm0, %k1
-; SKX-NEXT:    vmovapd %xmm0, %xmm1 {%k1}
-; SKX-NEXT:    vmovapd %xmm1, %xmm0
+; SKX-NEXT:    vblendmpd %xmm0, %xmm1, %xmm0 {%k1}
 ; SKX-NEXT:    retq
 
   %a = load double, double* %ptr
   %v = insertelement <2 x double> undef, double %a, i32 0
   %shuffle = shufflevector <2 x double> %v, <2 x double> undef, <2 x i32> <i32 0, i32 0>
 
   %mask = fcmp ogt <2 x double> %shuffle, %x
   %max = select <2 x i1> %mask, <2 x double> %x, <2 x double> %x1
   ret <2 x double> %max
 }
 
 
 define <16  x float> @test40(<16  x float> %x, <16  x float> %x1, float* %ptr) nounwind {
 ; CHECK-LABEL: test40:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vcmpltps (%rdi){1to16}, %zmm0, %k1
-; CHECK-NEXT:    vmovaps %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovaps %zmm1, %zmm0
+; CHECK-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
 
   %a = load float, float* %ptr
   %v = insertelement <16  x float> undef, float %a, i32 0
   %shuffle = shufflevector <16  x float> %v, <16  x float> undef, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
 
   %mask = fcmp ogt <16  x float> %shuffle, %x
   %max = select <16 x i1> %mask, <16  x float> %x, <16  x float> %x1
   ret <16  x float> %max
 }
 
 define <8  x float> @test41(<8  x float> %x, <8  x float> %x1, float* %ptr) nounwind {
 ; KNL-LABEL: test41:
 ; KNL:       ## BB#0:
 ; KNL-NEXT:    ## kill: %YMM1<def> %YMM1<kill> %ZMM1<def>
 ; KNL-NEXT:    ## kill: %YMM0<def> %YMM0<kill> %ZMM0<def>
 ; KNL-NEXT:    vbroadcastss (%rdi), %ymm2
 ; KNL-NEXT:    vcmpltps %zmm2, %zmm0, %k1
-; KNL-NEXT:    vmovaps %zmm0, %zmm1 {%k1}
-; KNL-NEXT:    vmovaps %ymm1, %ymm0
+; KNL-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1}
+; KNL-NEXT:    ## kill: %YMM0<def> %YMM0<kill> %ZMM0<kill>
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test41:
 ; SKX:       ## BB#0:
 ; SKX-NEXT:    vcmpltps (%rdi){1to8}, %ymm0, %k1
-; SKX-NEXT:    vmovaps %ymm0, %ymm1 {%k1}
-; SKX-NEXT:    vmovaps %ymm1, %ymm0
+; SKX-NEXT:    vblendmps %ymm0, %ymm1, %ymm0 {%k1}
 ; SKX-NEXT:    retq
 
   %a = load float, float* %ptr
   %v = insertelement <8  x float> undef, float %a, i32 0
   %shuffle = shufflevector <8  x float> %v, <8  x float> undef, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
 
   %mask = fcmp ogt <8  x float> %shuffle, %x
   %max = select <8 x i1> %mask, <8  x float> %x, <8  x float> %x1
   ret <8  x float> %max
 }
 
 define <4  x float> @test42(<4  x float> %x, <4  x float> %x1, float* %ptr) nounwind {
 ; KNL-LABEL: test42:
 ; KNL:       ## BB#0:
 ; KNL-NEXT:    vbroadcastss (%rdi), %xmm2
 ; KNL-NEXT:    vcmpltps %xmm2, %xmm0, %xmm2
 ; KNL-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test42:
 ; SKX:       ## BB#0:
 ; SKX-NEXT:    vcmpltps (%rdi){1to4}, %xmm0, %k1
-; SKX-NEXT:    vmovaps %xmm0, %xmm1 {%k1}
-; SKX-NEXT:    vmovaps %xmm1, %xmm0
+; SKX-NEXT:    vblendmps %xmm0, %xmm1, %xmm0 {%k1}
 ; SKX-NEXT:    retq
 
   %a = load float, float* %ptr
   %v = insertelement <4  x float> undef, float %a, i32 0
   %shuffle = shufflevector <4  x float> %v, <4  x float> undef, <4 x i32> <i32 0, i32 0, i32 0, i32 0>
 
   %mask = fcmp ogt <4  x float> %shuffle, %x
   %max = select <4 x i1> %mask, <4  x float> %x, <4  x float> %x1
   ret <4  x float> %max
 }
 
 define <8 x double> @test43(<8 x double> %x, <8 x double> %x1, double* %ptr,<8 x i1> %mask_in) nounwind {
 ; KNL-LABEL: test43:
 ; KNL:       ## BB#0:
 ; KNL-NEXT:    vpmovsxwq %xmm2, %zmm2
 ; KNL-NEXT:    vpsllq $63, %zmm2, %zmm2
 ; KNL-NEXT:    vptestmq %zmm2, %zmm2, %k1
 ; KNL-NEXT:    vcmpltpd (%rdi){1to8}, %zmm0, %k1 {%k1}
-; KNL-NEXT:    vmovapd %zmm0, %zmm1 {%k1}
-; KNL-NEXT:    vmovapd %zmm1, %zmm0
+; KNL-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1}
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test43:
 ; SKX:       ## BB#0:
 ; SKX-NEXT:    vpsllw $15, %xmm2, %xmm2
 ; SKX-NEXT:    vpmovw2m %xmm2, %k1
 ; SKX-NEXT:    vcmpltpd (%rdi){1to8}, %zmm0, %k1 {%k1}
-; SKX-NEXT:    vmovapd %zmm0, %zmm1 {%k1}
-; SKX-NEXT:    vmovapd %zmm1, %zmm0
+; SKX-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1}
 ; SKX-NEXT:    retq
 
   %a = load double, double* %ptr
   %v = insertelement <8 x double> undef, double %a, i32 0
   %shuffle = shufflevector <8 x double> %v, <8 x double> undef, <8 x i32> zeroinitializer
 
   %mask_cmp = fcmp ogt <8 x double> %shuffle, %x
   %mask = and <8 x i1> %mask_cmp, %mask_in

llvm/trunk/test/CodeGen/X86/avx512bw-vec-cmp.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=skx | FileCheck %s
 
 define <64 x i8> @test1(<64 x i8> %x, <64 x i8> %y) nounwind {
 ; CHECK-LABEL: test1:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpeqb %zmm1, %zmm0, %k1
-; CHECK-NEXT:    vmovdqu8 %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    vpblendmb %zmm0, %zmm1, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask = icmp eq <64 x i8> %x, %y
   %max = select <64 x i1> %mask, <64 x i8> %x, <64 x i8> %y
   ret <64 x i8> %max
 }
 
 define <64 x i8> @test2(<64 x i8> %x, <64 x i8> %y, <64 x i8> %x1) nounwind {
 ; CHECK-LABEL: test2:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpgtb %zmm1, %zmm0, %k1
-; CHECK-NEXT:    vmovdqu8 %zmm2, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    vpblendmb %zmm2, %zmm1, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask = icmp sgt <64 x i8> %x, %y
   %max = select <64 x i1> %mask, <64 x i8> %x1, <64 x i8> %y
   ret <64 x i8> %max
 }
 
 define <32 x i16> @test3(<32 x i16> %x, <32 x i16> %y, <32 x i16> %x1) nounwind {
 ; CHECK-LABEL: test3:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmplew %zmm0, %zmm1, %k1
-; CHECK-NEXT:    vmovdqu16 %zmm2, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    vpblendmw %zmm2, %zmm1, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask = icmp sge <32 x i16> %x, %y
   %max = select <32 x i1> %mask, <32 x i16> %x1, <32 x i16> %y
   ret <32 x i16> %max
 }
 
 define <64 x i8> @test4(<64 x i8> %x, <64 x i8> %y, <64 x i8> %x1) nounwind {
 ; CHECK-LABEL: test4:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpnleub %zmm1, %zmm0, %k1
-; CHECK-NEXT:    vmovdqu8 %zmm2, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    vpblendmb %zmm2, %zmm1, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask = icmp ugt <64 x i8> %x, %y
   %max = select <64 x i1> %mask, <64 x i8> %x1, <64 x i8> %y
   ret <64 x i8> %max
 }
 
 define <32 x i16> @test5(<32 x i16> %x, <32 x i16> %x1, <32 x i16>* %yp) nounwind {
 ; CHECK-LABEL: test5:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpeqw (%rdi), %zmm0, %k1
-; CHECK-NEXT:    vmovdqu16 %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    vpblendmw %zmm0, %zmm1, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <32 x i16>, <32 x i16>* %yp, align 4
   %mask = icmp eq <32 x i16> %x, %y
   %max = select <32 x i1> %mask, <32 x i16> %x, <32 x i16> %x1
   ret <32 x i16> %max
 }
 
 define <32 x i16> @test6(<32 x i16> %x, <32 x i16> %x1, <32 x i16>* %y.ptr) nounwind {
 ; CHECK-LABEL: test6:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpgtw (%rdi), %zmm0, %k1
-; CHECK-NEXT:    vmovdqu16 %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    vpblendmw %zmm0, %zmm1, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <32 x i16>, <32 x i16>* %y.ptr, align 4
   %mask = icmp sgt <32 x i16> %x, %y
   %max = select <32 x i1> %mask, <32 x i16> %x, <32 x i16> %x1
   ret <32 x i16> %max
 }
 
 define <32 x i16> @test7(<32 x i16> %x, <32 x i16> %x1, <32 x i16>* %y.ptr) nounwind {
 ; CHECK-LABEL: test7:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmplew (%rdi), %zmm0, %k1
-; CHECK-NEXT:    vmovdqu16 %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    vpblendmw %zmm0, %zmm1, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <32 x i16>, <32 x i16>* %y.ptr, align 4
   %mask = icmp sle <32 x i16> %x, %y
   %max = select <32 x i1> %mask, <32 x i16> %x, <32 x i16> %x1
   ret <32 x i16> %max
 }
 
 define <32 x i16> @test8(<32 x i16> %x, <32 x i16> %x1, <32 x i16>* %y.ptr) nounwind {
 ; CHECK-LABEL: test8:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpleuw (%rdi), %zmm0, %k1
-; CHECK-NEXT:    vmovdqu16 %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    vpblendmw %zmm0, %zmm1, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <32 x i16>, <32 x i16>* %y.ptr, align 4
   %mask = icmp ule <32 x i16> %x, %y
   %max = select <32 x i1> %mask, <32 x i16> %x, <32 x i16> %x1
   ret <32 x i16> %max
 }
 
 define <32 x i16> @test9(<32 x i16> %x, <32 x i16> %y, <32 x i16> %x1, <32 x i16> %y1) nounwind {
 ; CHECK-LABEL: test9:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpeqw %zmm1, %zmm0, %k1
 ; CHECK-NEXT:    vpcmpeqw %zmm3, %zmm2, %k1 {%k1}
-; CHECK-NEXT:    vmovdqu16 %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    vpblendmw %zmm0, %zmm1, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask1 = icmp eq <32 x i16> %x1, %y1
   %mask0 = icmp eq <32 x i16> %x, %y
   %mask = select <32 x i1> %mask0, <32 x i1> %mask1, <32 x i1> zeroinitializer
   %max = select <32 x i1> %mask, <32 x i16> %x, <32 x i16> %y
   ret <32 x i16> %max
 }
 
 define <64 x i8> @test10(<64 x i8> %x, <64 x i8> %y, <64 x i8> %x1, <64 x i8> %y1) nounwind {
 ; CHECK-LABEL: test10:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpleb %zmm1, %zmm0, %k1
 ; CHECK-NEXT:    vpcmpleb %zmm2, %zmm3, %k1 {%k1}
-; CHECK-NEXT:    vmovdqu8 %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    vpblendmb %zmm0, %zmm2, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask1 = icmp sge <64 x i8> %x1, %y1
   %mask0 = icmp sle <64 x i8> %x, %y
   %mask = select <64 x i1> %mask0, <64 x i1> %mask1, <64 x i1> zeroinitializer
   %max = select <64 x i1> %mask, <64 x i8> %x, <64 x i8> %x1
   ret <64 x i8> %max
 }
 
 define <64 x i8> @test11(<64 x i8> %x, <64 x i8>* %y.ptr, <64 x i8> %x1, <64 x i8> %y1) nounwind {
 ; CHECK-LABEL: test11:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpgtb %zmm2, %zmm1, %k1
 ; CHECK-NEXT:    vpcmpgtb (%rdi), %zmm0, %k1 {%k1}
-; CHECK-NEXT:    vmovdqu8 %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    vpblendmb %zmm0, %zmm1, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask1 = icmp sgt <64 x i8> %x1, %y1
   %y = load <64 x i8>, <64 x i8>* %y.ptr, align 4
   %mask0 = icmp sgt <64 x i8> %x, %y
   %mask = select <64 x i1> %mask0, <64 x i1> %mask1, <64 x i1> zeroinitializer
   %max = select <64 x i1> %mask, <64 x i8> %x, <64 x i8> %x1
   ret <64 x i8> %max
 }
 
 define <32 x i16> @test12(<32 x i16> %x, <32 x i16>* %y.ptr, <32 x i16> %x1, <32 x i16> %y1) nounwind {
 ; CHECK-LABEL: test12:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmplew %zmm1, %zmm2, %k1
 ; CHECK-NEXT:    vpcmpleuw (%rdi), %zmm0, %k1 {%k1}
-; CHECK-NEXT:    vmovdqu16 %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
+; CHECK-NEXT:    vpblendmw %zmm0, %zmm1, %zmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask1 = icmp sge <32 x i16> %x1, %y1
   %y = load <32 x i16>, <32 x i16>* %y.ptr, align 4
   %mask0 = icmp ule <32 x i16> %x, %y
   %mask = select <32 x i1> %mask0, <32 x i1> %mask1, <32 x i1> zeroinitializer
   %max = select <32 x i1> %mask, <32 x i16> %x, <32 x i16> %x1
   ret <32 x i16> %max
 }

llvm/trunk/test/CodeGen/X86/avx512bwvl-vec-cmp.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=skx | FileCheck %s
 
 define <32 x i8> @test256_1(<32 x i8> %x, <32 x i8> %y) nounwind {
 ; CHECK-LABEL: test256_1:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpeqb %ymm1, %ymm0, %k1
-; CHECK-NEXT:    vmovdqu8 %ymm0, %ymm1 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    vpblendmb %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask = icmp eq <32 x i8> %x, %y
   %max = select <32 x i1> %mask, <32 x i8> %x, <32 x i8> %y
   ret <32 x i8> %max
 }
 
 define <32 x i8> @test256_2(<32 x i8> %x, <32 x i8> %y, <32 x i8> %x1) nounwind {
 ; CHECK-LABEL: test256_2:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpgtb %ymm1, %ymm0, %k1
-; CHECK-NEXT:    vmovdqu8 %ymm0, %ymm2 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm2, %ymm0
+; CHECK-NEXT:    vpblendmb %ymm0, %ymm2, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask = icmp sgt <32 x i8> %x, %y
   %max = select <32 x i1> %mask, <32 x i8> %x, <32 x i8> %x1
   ret <32 x i8> %max
 }
 
 define <16 x i16> @test256_3(<16 x i16> %x, <16 x i16> %y, <16 x i16> %x1) nounwind {
 ; CHECK-LABEL: test256_3:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmplew %ymm0, %ymm1, %k1
-; CHECK-NEXT:    vmovdqu16 %ymm2, %ymm1 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    vpblendmw %ymm2, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask = icmp sge <16 x i16> %x, %y
   %max = select <16 x i1> %mask, <16 x i16> %x1, <16 x i16> %y
   ret <16 x i16> %max
 }
 
 define <32 x i8> @test256_4(<32 x i8> %x, <32 x i8> %y, <32 x i8> %x1) nounwind {
 ; CHECK-LABEL: test256_4:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpnleub %ymm1, %ymm0, %k1
-; CHECK-NEXT:    vmovdqu8 %ymm0, %ymm2 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm2, %ymm0
+; CHECK-NEXT:    vpblendmb %ymm0, %ymm2, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask = icmp ugt <32 x i8> %x, %y
   %max = select <32 x i1> %mask, <32 x i8> %x, <32 x i8> %x1
   ret <32 x i8> %max
 }
 
 define <16 x i16> @test256_5(<16 x i16> %x, <16 x i16> %x1, <16 x i16>* %yp) nounwind {
 ; CHECK-LABEL: test256_5:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpeqw (%rdi), %ymm0, %k1
-; CHECK-NEXT:    vmovdqu16 %ymm0, %ymm1 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    vpblendmw %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <16 x i16>, <16 x i16>* %yp, align 4
   %mask = icmp eq <16 x i16> %x, %y
   %max = select <16 x i1> %mask, <16 x i16> %x, <16 x i16> %x1
   ret <16 x i16> %max
 }
 
 define <16 x i16> @test256_6(<16 x i16> %x, <16 x i16> %x1, <16 x i16>* %y.ptr) nounwind {
 ; CHECK-LABEL: test256_6:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpgtw (%rdi), %ymm0, %k1
-; CHECK-NEXT:    vmovdqu16 %ymm0, %ymm1 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    vpblendmw %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <16 x i16>, <16 x i16>* %y.ptr, align 4
   %mask = icmp sgt <16 x i16> %x, %y
   %max = select <16 x i1> %mask, <16 x i16> %x, <16 x i16> %x1
   ret <16 x i16> %max
 }
 
 define <16 x i16> @test256_7(<16 x i16> %x, <16 x i16> %x1, <16 x i16>* %y.ptr) nounwind {
 ; CHECK-LABEL: test256_7:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmplew (%rdi), %ymm0, %k1
-; CHECK-NEXT:    vmovdqu16 %ymm0, %ymm1 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    vpblendmw %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <16 x i16>, <16 x i16>* %y.ptr, align 4
   %mask = icmp sle <16 x i16> %x, %y
   %max = select <16 x i1> %mask, <16 x i16> %x, <16 x i16> %x1
   ret <16 x i16> %max
 }
 
 define <16 x i16> @test256_8(<16 x i16> %x, <16 x i16> %x1, <16 x i16>* %y.ptr) nounwind {
 ; CHECK-LABEL: test256_8:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpleuw (%rdi), %ymm0, %k1
-; CHECK-NEXT:    vmovdqu16 %ymm0, %ymm1 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    vpblendmw %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <16 x i16>, <16 x i16>* %y.ptr, align 4
   %mask = icmp ule <16 x i16> %x, %y
   %max = select <16 x i1> %mask, <16 x i16> %x, <16 x i16> %x1
   ret <16 x i16> %max
 }
 
 define <16 x i16> @test256_9(<16 x i16> %x, <16 x i16> %y, <16 x i16> %x1, <16 x i16> %y1) nounwind {
 ; CHECK-LABEL: test256_9:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpeqw %ymm1, %ymm0, %k1
 ; CHECK-NEXT:    vpcmpeqw %ymm3, %ymm2, %k1 {%k1}
-; CHECK-NEXT:    vmovdqu16 %ymm0, %ymm1 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    vpblendmw %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask1 = icmp eq <16 x i16> %x1, %y1
   %mask0 = icmp eq <16 x i16> %x, %y
   %mask = select <16 x i1> %mask0, <16 x i1> %mask1, <16 x i1> zeroinitializer
   %max = select <16 x i1> %mask, <16 x i16> %x, <16 x i16> %y
   ret <16 x i16> %max
 }
 
 define <32 x i8> @test256_10(<32 x i8> %x, <32 x i8> %y, <32 x i8> %x1, <32 x i8> %y1) nounwind {
 ; CHECK-LABEL: test256_10:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpleb %ymm1, %ymm0, %k1
 ; CHECK-NEXT:    vpcmpleb %ymm2, %ymm3, %k1 {%k1}
-; CHECK-NEXT:    vmovdqu8 %ymm0, %ymm2 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm2, %ymm0
+; CHECK-NEXT:    vpblendmb %ymm0, %ymm2, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask1 = icmp sge <32 x i8> %x1, %y1
   %mask0 = icmp sle <32 x i8> %x, %y
   %mask = select <32 x i1> %mask0, <32 x i1> %mask1, <32 x i1> zeroinitializer
   %max = select <32 x i1> %mask, <32 x i8> %x, <32 x i8> %x1
   ret <32 x i8> %max
 }
 
 define <32 x i8> @test256_11(<32 x i8> %x, <32 x i8>* %y.ptr, <32 x i8> %x1, <32 x i8> %y1) nounwind {
 ; CHECK-LABEL: test256_11:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpgtb %ymm2, %ymm1, %k1
 ; CHECK-NEXT:    vpcmpgtb (%rdi), %ymm0, %k1 {%k1}
-; CHECK-NEXT:    vmovdqu8 %ymm0, %ymm1 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    vpblendmb %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask1 = icmp sgt <32 x i8> %x1, %y1
   %y = load <32 x i8>, <32 x i8>* %y.ptr, align 4
   %mask0 = icmp sgt <32 x i8> %x, %y
   %mask = select <32 x i1> %mask0, <32 x i1> %mask1, <32 x i1> zeroinitializer
   %max = select <32 x i1> %mask, <32 x i8> %x, <32 x i8> %x1
   ret <32 x i8> %max
 }
 
 define <16 x i16> @test256_12(<16 x i16> %x, <16 x i16>* %y.ptr, <16 x i16> %x1, <16 x i16> %y1) nounwind {
 ; CHECK-LABEL: test256_12:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmplew %ymm1, %ymm2, %k1
 ; CHECK-NEXT:    vpcmpleuw (%rdi), %ymm0, %k1 {%k1}
-; CHECK-NEXT:    vmovdqu16 %ymm0, %ymm1 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    vpblendmw %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask1 = icmp sge <16 x i16> %x1, %y1
   %y = load <16 x i16>, <16 x i16>* %y.ptr, align 4
   %mask0 = icmp ule <16 x i16> %x, %y
   %mask = select <16 x i1> %mask0, <16 x i1> %mask1, <16 x i1> zeroinitializer
   %max = select <16 x i1> %mask, <16 x i16> %x, <16 x i16> %x1
   ret <16 x i16> %max
 }
 
 define <16 x i8> @test128_1(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; CHECK-LABEL: test128_1:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpeqb %xmm1, %xmm0, %k1
-; CHECK-NEXT:    vmovdqu8 %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vpblendmb %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask = icmp eq <16 x i8> %x, %y
   %max = select <16 x i1> %mask, <16 x i8> %x, <16 x i8> %y
   ret <16 x i8> %max
 }
 
 define <16 x i8> @test128_2(<16 x i8> %x, <16 x i8> %y, <16 x i8> %x1) nounwind {
 ; CHECK-LABEL: test128_2:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpgtb %xmm1, %xmm0, %k1
-; CHECK-NEXT:    vmovdqu8 %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm2, %xmm0
+; CHECK-NEXT:    vpblendmb %xmm0, %xmm2, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask = icmp sgt <16 x i8> %x, %y
   %max = select <16 x i1> %mask, <16 x i8> %x, <16 x i8> %x1
   ret <16 x i8> %max
 }
 
 define <8 x i16> @test128_3(<8 x i16> %x, <8 x i16> %y, <8 x i16> %x1) nounwind {
 ; CHECK-LABEL: test128_3:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmplew %xmm0, %xmm1, %k1
-; CHECK-NEXT:    vmovdqu16 %xmm2, %xmm1 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vpblendmw %xmm2, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask = icmp sge <8 x i16> %x, %y
   %max = select <8 x i1> %mask, <8 x i16> %x1, <8 x i16> %y
   ret <8 x i16> %max
 }
 
 define <16 x i8> @test128_4(<16 x i8> %x, <16 x i8> %y, <16 x i8> %x1) nounwind {
 ; CHECK-LABEL: test128_4:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpnleub %xmm1, %xmm0, %k1
-; CHECK-NEXT:    vmovdqu8 %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm2, %xmm0
+; CHECK-NEXT:    vpblendmb %xmm0, %xmm2, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask = icmp ugt <16 x i8> %x, %y
   %max = select <16 x i1> %mask, <16 x i8> %x, <16 x i8> %x1
   ret <16 x i8> %max
 }
 
 define <8 x i16> @test128_5(<8 x i16> %x, <8 x i16> %x1, <8 x i16>* %yp) nounwind {
 ; CHECK-LABEL: test128_5:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpeqw (%rdi), %xmm0, %k1
-; CHECK-NEXT:    vmovdqu16 %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vpblendmw %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <8 x i16>, <8 x i16>* %yp, align 4
   %mask = icmp eq <8 x i16> %x, %y
   %max = select <8 x i1> %mask, <8 x i16> %x, <8 x i16> %x1
   ret <8 x i16> %max
 }
 
 define <8 x i16> @test128_6(<8 x i16> %x, <8 x i16> %x1, <8 x i16>* %y.ptr) nounwind {
 ; CHECK-LABEL: test128_6:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpgtw (%rdi), %xmm0, %k1
-; CHECK-NEXT:    vmovdqu16 %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vpblendmw %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <8 x i16>, <8 x i16>* %y.ptr, align 4
   %mask = icmp sgt <8 x i16> %x, %y
   %max = select <8 x i1> %mask, <8 x i16> %x, <8 x i16> %x1
   ret <8 x i16> %max
 }
 
 define <8 x i16> @test128_7(<8 x i16> %x, <8 x i16> %x1, <8 x i16>* %y.ptr) nounwind {
 ; CHECK-LABEL: test128_7:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmplew (%rdi), %xmm0, %k1
-; CHECK-NEXT:    vmovdqu16 %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vpblendmw %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <8 x i16>, <8 x i16>* %y.ptr, align 4
   %mask = icmp sle <8 x i16> %x, %y
   %max = select <8 x i1> %mask, <8 x i16> %x, <8 x i16> %x1
   ret <8 x i16> %max
 }
 
 define <8 x i16> @test128_8(<8 x i16> %x, <8 x i16> %x1, <8 x i16>* %y.ptr) nounwind {
 ; CHECK-LABEL: test128_8:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpleuw (%rdi), %xmm0, %k1
-; CHECK-NEXT:    vmovdqu16 %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vpblendmw %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <8 x i16>, <8 x i16>* %y.ptr, align 4
   %mask = icmp ule <8 x i16> %x, %y
   %max = select <8 x i1> %mask, <8 x i16> %x, <8 x i16> %x1
   ret <8 x i16> %max
 }
 
 define <8 x i16> @test128_9(<8 x i16> %x, <8 x i16> %y, <8 x i16> %x1, <8 x i16> %y1) nounwind {
 ; CHECK-LABEL: test128_9:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpeqw %xmm1, %xmm0, %k1
 ; CHECK-NEXT:    vpcmpeqw %xmm3, %xmm2, %k1 {%k1}
-; CHECK-NEXT:    vmovdqu16 %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vpblendmw %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask1 = icmp eq <8 x i16> %x1, %y1
   %mask0 = icmp eq <8 x i16> %x, %y
   %mask = select <8 x i1> %mask0, <8 x i1> %mask1, <8 x i1> zeroinitializer
   %max = select <8 x i1> %mask, <8 x i16> %x, <8 x i16> %y
   ret <8 x i16> %max
 }
 
 define <16 x i8> @test128_10(<16 x i8> %x, <16 x i8> %y, <16 x i8> %x1, <16 x i8> %y1) nounwind {
 ; CHECK-LABEL: test128_10:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpleb %xmm1, %xmm0, %k1
 ; CHECK-NEXT:    vpcmpleb %xmm2, %xmm3, %k1 {%k1}
-; CHECK-NEXT:    vmovdqu8 %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm2, %xmm0
+; CHECK-NEXT:    vpblendmb %xmm0, %xmm2, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask1 = icmp sge <16 x i8> %x1, %y1
   %mask0 = icmp sle <16 x i8> %x, %y
   %mask = select <16 x i1> %mask0, <16 x i1> %mask1, <16 x i1> zeroinitializer
   %max = select <16 x i1> %mask, <16 x i8> %x, <16 x i8> %x1
   ret <16 x i8> %max
 }
 
 define <16 x i8> @test128_11(<16 x i8> %x, <16 x i8>* %y.ptr, <16 x i8> %x1, <16 x i8> %y1) nounwind {
 ; CHECK-LABEL: test128_11:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpgtb %xmm2, %xmm1, %k1
 ; CHECK-NEXT:    vpcmpgtb (%rdi), %xmm0, %k1 {%k1}
-; CHECK-NEXT:    vmovdqu8 %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vpblendmb %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask1 = icmp sgt <16 x i8> %x1, %y1
   %y = load <16 x i8>, <16 x i8>* %y.ptr, align 4
   %mask0 = icmp sgt <16 x i8> %x, %y
   %mask = select <16 x i1> %mask0, <16 x i1> %mask1, <16 x i1> zeroinitializer
   %max = select <16 x i1> %mask, <16 x i8> %x, <16 x i8> %x1
   ret <16 x i8> %max
 }
 
 define <8 x i16> @test128_12(<8 x i16> %x, <8 x i16>* %y.ptr, <8 x i16> %x1, <8 x i16> %y1) nounwind {
 ; CHECK-LABEL: test128_12:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmplew %xmm1, %xmm2, %k1
 ; CHECK-NEXT:    vpcmpleuw (%rdi), %xmm0, %k1 {%k1}
-; CHECK-NEXT:    vmovdqu16 %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vpblendmw %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask1 = icmp sge <8 x i16> %x1, %y1
   %y = load <8 x i16>, <8 x i16>* %y.ptr, align 4
   %mask0 = icmp ule <8 x i16> %x, %y
   %mask = select <8 x i1> %mask0, <8 x i1> %mask1, <8 x i1> zeroinitializer
   %max = select <8 x i1> %mask, <8 x i16> %x, <8 x i16> %x1
   ret <8 x i16> %max
 }

llvm/trunk/test/CodeGen/X86/avx512vl-vec-cmp.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=skx | FileCheck %s
 
 define <4 x i64> @test256_1(<4 x i64> %x, <4 x i64> %y) nounwind {
 ; CHECK-LABEL: test256_1:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpeqq %ymm1, %ymm0, %k1
-; CHECK-NEXT:    vmovdqa64 %ymm0, %ymm1 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask = icmp eq <4 x i64> %x, %y
   %max = select <4 x i1> %mask, <4 x i64> %x, <4 x i64> %y
   ret <4 x i64> %max
 }
 
 define <4 x i64> @test256_2(<4 x i64> %x, <4 x i64> %y, <4 x i64> %x1) nounwind {
 ; CHECK-LABEL: test256_2:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpgtq %ymm1, %ymm0, %k1
-; CHECK-NEXT:    vmovdqa64 %ymm2, %ymm1 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    vpblendmq %ymm2, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask = icmp sgt <4 x i64> %x, %y
   %max = select <4 x i1> %mask, <4 x i64> %x1, <4 x i64> %y
   ret <4 x i64> %max
 }
 
 define <8 x i32> @test256_3(<8 x i32> %x, <8 x i32> %y, <8 x i32> %x1) nounwind {
 ; CHECK-LABEL: test256_3:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpled %ymm0, %ymm1, %k1
-; CHECK-NEXT:    vmovdqa32 %ymm2, %ymm1 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    vpblendmd %ymm2, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask = icmp sge <8 x i32> %x, %y
   %max = select <8 x i1> %mask, <8 x i32> %x1, <8 x i32> %y
   ret <8 x i32> %max
 }
 
 define <4 x i64> @test256_4(<4 x i64> %x, <4 x i64> %y, <4 x i64> %x1) nounwind {
 ; CHECK-LABEL: test256_4:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpnleuq %ymm1, %ymm0, %k1
-; CHECK-NEXT:    vmovdqa64 %ymm2, %ymm1 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    vpblendmq %ymm2, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask = icmp ugt <4 x i64> %x, %y
   %max = select <4 x i1> %mask, <4 x i64> %x1, <4 x i64> %y
   ret <4 x i64> %max
 }
 
 define <8 x i32> @test256_5(<8 x i32> %x, <8 x i32> %x1, <8 x i32>* %yp) nounwind {
 ; CHECK-LABEL: test256_5:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpeqd (%rdi), %ymm0, %k1
-; CHECK-NEXT:    vmovdqa32 %ymm0, %ymm1 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <8 x i32>, <8 x i32>* %yp, align 4
   %mask = icmp eq <8 x i32> %x, %y
   %max = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %x1
   ret <8 x i32> %max
 }
 
 define <8 x i32> @test256_5b(<8 x i32> %x, <8 x i32> %x1, <8 x i32>* %yp) nounwind {
 ; CHECK-LABEL: test256_5b:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpeqd (%rdi), %ymm0, %k1
-; CHECK-NEXT:    vmovdqa32 %ymm0, %ymm1 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <8 x i32>, <8 x i32>* %yp, align 4
   %mask = icmp eq <8 x i32> %y, %x
   %max = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %x1
   ret <8 x i32> %max
 }
 
 define <8 x i32> @test256_6(<8 x i32> %x, <8 x i32> %x1, <8 x i32>* %y.ptr) nounwind {
 ; CHECK-LABEL: test256_6:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpgtd (%rdi), %ymm0, %k1
-; CHECK-NEXT:    vmovdqa32 %ymm0, %ymm1 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <8 x i32>, <8 x i32>* %y.ptr, align 4
   %mask = icmp sgt <8 x i32> %x, %y
   %max = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %x1
   ret <8 x i32> %max
 }
 
 define <8 x i32> @test256_6b(<8 x i32> %x, <8 x i32> %x1, <8 x i32>* %y.ptr) nounwind {
 ; CHECK-LABEL: test256_6b:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpgtd (%rdi), %ymm0, %k1
-; CHECK-NEXT:    vmovdqa32 %ymm0, %ymm1 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <8 x i32>, <8 x i32>* %y.ptr, align 4
   %mask = icmp slt <8 x i32> %y, %x
   %max = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %x1
   ret <8 x i32> %max
 }
 
 define <8 x i32> @test256_7(<8 x i32> %x, <8 x i32> %x1, <8 x i32>* %y.ptr) nounwind {
 ; CHECK-LABEL: test256_7:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpled (%rdi), %ymm0, %k1
-; CHECK-NEXT:    vmovdqa32 %ymm0, %ymm1 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <8 x i32>, <8 x i32>* %y.ptr, align 4
   %mask = icmp sle <8 x i32> %x, %y
   %max = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %x1
   ret <8 x i32> %max
 }
 
 define <8 x i32> @test256_7b(<8 x i32> %x, <8 x i32> %x1, <8 x i32>* %y.ptr) nounwind {
 ; CHECK-LABEL: test256_7b:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpled (%rdi), %ymm0, %k1
-; CHECK-NEXT:    vmovdqa32 %ymm0, %ymm1 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <8 x i32>, <8 x i32>* %y.ptr, align 4
   %mask = icmp sge <8 x i32> %y, %x
   %max = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %x1
   ret <8 x i32> %max
 }
 
 define <8 x i32> @test256_8(<8 x i32> %x, <8 x i32> %x1, <8 x i32>* %y.ptr) nounwind {
 ; CHECK-LABEL: test256_8:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpleud (%rdi), %ymm0, %k1
-; CHECK-NEXT:    vmovdqa32 %ymm0, %ymm1 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <8 x i32>, <8 x i32>* %y.ptr, align 4
   %mask = icmp ule <8 x i32> %x, %y
   %max = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %x1
   ret <8 x i32> %max
 }
 
 define <8 x i32> @test256_8b(<8 x i32> %x, <8 x i32> %x1, <8 x i32>* %y.ptr) nounwind {
 ; CHECK-LABEL: test256_8b:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpleud (%rdi), %ymm0, %k1
-; CHECK-NEXT:    vmovdqa32 %ymm0, %ymm1 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <8 x i32>, <8 x i32>* %y.ptr, align 4
   %mask = icmp uge <8 x i32> %y, %x
   %max = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %x1
   ret <8 x i32> %max
 }
 
 define <8 x i32> @test256_9(<8 x i32> %x, <8 x i32> %y, <8 x i32> %x1, <8 x i32> %y1) nounwind {
 ; CHECK-LABEL: test256_9:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpeqd %ymm1, %ymm0, %k1
 ; CHECK-NEXT:    vpcmpeqd %ymm3, %ymm2, %k1 {%k1}
-; CHECK-NEXT:    vmovdqa32 %ymm0, %ymm1 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask1 = icmp eq <8 x i32> %x1, %y1
   %mask0 = icmp eq <8 x i32> %x, %y
   %mask = select <8 x i1> %mask0, <8 x i1> %mask1, <8 x i1> zeroinitializer
   %max = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %y
   ret <8 x i32> %max
 }
 
 define <4 x i64> @test256_10(<4 x i64> %x, <4 x i64> %y, <4 x i64> %x1, <4 x i64> %y1) nounwind {
 ; CHECK-LABEL: test256_10:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpleq %ymm1, %ymm0, %k1
 ; CHECK-NEXT:    vpcmpleq %ymm2, %ymm3, %k1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %ymm0, %ymm2 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm2, %ymm0
+; CHECK-NEXT:    vpblendmq %ymm0, %ymm2, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask1 = icmp sge <4 x i64> %x1, %y1
   %mask0 = icmp sle <4 x i64> %x, %y
   %mask = select <4 x i1> %mask0, <4 x i1> %mask1, <4 x i1> zeroinitializer
   %max = select <4 x i1> %mask, <4 x i64> %x, <4 x i64> %x1
   ret <4 x i64> %max
 }
 
 define <4 x i64> @test256_11(<4 x i64> %x, <4 x i64>* %y.ptr, <4 x i64> %x1, <4 x i64> %y1) nounwind {
 ; CHECK-LABEL: test256_11:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpgtq %ymm2, %ymm1, %k1
 ; CHECK-NEXT:    vpcmpgtq (%rdi), %ymm0, %k1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %ymm0, %ymm1 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask1 = icmp sgt <4 x i64> %x1, %y1
   %y = load <4 x i64>, <4 x i64>* %y.ptr, align 4
   %mask0 = icmp sgt <4 x i64> %x, %y
   %mask = select <4 x i1> %mask0, <4 x i1> %mask1, <4 x i1> zeroinitializer
   %max = select <4 x i1> %mask, <4 x i64> %x, <4 x i64> %x1
   ret <4 x i64> %max
 }
 
 define <8 x i32> @test256_12(<8 x i32> %x, <8 x i32>* %y.ptr, <8 x i32> %x1, <8 x i32> %y1) nounwind {
 ; CHECK-LABEL: test256_12:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpled %ymm1, %ymm2, %k1
 ; CHECK-NEXT:    vpcmpleud (%rdi), %ymm0, %k1 {%k1}
-; CHECK-NEXT:    vmovdqa32 %ymm0, %ymm1 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask1 = icmp sge <8 x i32> %x1, %y1
   %y = load <8 x i32>, <8 x i32>* %y.ptr, align 4
   %mask0 = icmp ule <8 x i32> %x, %y
   %mask = select <8 x i1> %mask0, <8 x i1> %mask1, <8 x i1> zeroinitializer
   %max = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %x1
   ret <8 x i32> %max
 }
 
 define <4 x i64> @test256_13(<4 x i64> %x, <4 x i64> %x1, i64* %yb.ptr) nounwind {
 ; CHECK-LABEL: test256_13:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpeqq (%rdi){1to4}, %ymm0, %k1
-; CHECK-NEXT:    vmovdqa64 %ymm0, %ymm1 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %yb = load i64, i64* %yb.ptr, align 4
   %y.0 = insertelement <4 x i64> undef, i64 %yb, i32 0
   %y = shufflevector <4 x i64> %y.0, <4 x i64> undef, <4 x i32> zeroinitializer
   %mask = icmp eq <4 x i64> %x, %y
   %max = select <4 x i1> %mask, <4 x i64> %x, <4 x i64> %x1
   ret <4 x i64> %max
 }
 
 define <8 x i32> @test256_14(<8 x i32> %x, i32* %yb.ptr, <8 x i32> %x1) nounwind {
 ; CHECK-LABEL: test256_14:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpled (%rdi){1to8}, %ymm0, %k1
-; CHECK-NEXT:    vmovdqa32 %ymm0, %ymm1 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %yb = load i32, i32* %yb.ptr, align 4
   %y.0 = insertelement <8 x i32> undef, i32 %yb, i32 0
   %y = shufflevector <8 x i32> %y.0, <8 x i32> undef, <8 x i32> zeroinitializer
   %mask = icmp sle <8 x i32> %x, %y
   %max = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %x1
   ret <8 x i32> %max
 }
 
 define <8 x i32> @test256_15(<8 x i32> %x, i32* %yb.ptr, <8 x i32> %x1, <8 x i32> %y1) nounwind {
 ; CHECK-LABEL: test256_15:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpled %ymm1, %ymm2, %k1
 ; CHECK-NEXT:    vpcmpgtd (%rdi){1to8}, %ymm0, %k1 {%k1}
-; CHECK-NEXT:    vmovdqa32 %ymm0, %ymm1 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask1 = icmp sge <8 x i32> %x1, %y1
   %yb = load i32, i32* %yb.ptr, align 4
   %y.0 = insertelement <8 x i32> undef, i32 %yb, i32 0
   %y = shufflevector <8 x i32> %y.0, <8 x i32> undef, <8 x i32> zeroinitializer
   %mask0 = icmp sgt <8 x i32> %x, %y
   %mask = select <8 x i1> %mask0, <8 x i1> %mask1, <8 x i1> zeroinitializer
   %max = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %x1
   ret <8 x i32> %max
 }
 
 define <4 x i64> @test256_16(<4 x i64> %x, i64* %yb.ptr, <4 x i64> %x1, <4 x i64> %y1) nounwind {
 ; CHECK-LABEL: test256_16:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpleq %ymm1, %ymm2, %k1
 ; CHECK-NEXT:    vpcmpgtq (%rdi){1to4}, %ymm0, %k1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %ymm0, %ymm1 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask1 = icmp sge <4 x i64> %x1, %y1
   %yb = load i64, i64* %yb.ptr, align 4
   %y.0 = insertelement <4 x i64> undef, i64 %yb, i32 0
   %y = shufflevector <4 x i64> %y.0, <4 x i64> undef, <4 x i32> zeroinitializer
   %mask0 = icmp sgt <4 x i64> %x, %y
   %mask = select <4 x i1> %mask0, <4 x i1> %mask1, <4 x i1> zeroinitializer
   %max = select <4 x i1> %mask, <4 x i64> %x, <4 x i64> %x1
   ret <4 x i64> %max
 }
 
 define <8 x i32> @test256_17(<8 x i32> %x, <8 x i32> %x1, <8 x i32>* %yp) nounwind {
 ; CHECK-LABEL: test256_17:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpneqd (%rdi), %ymm0, %k1
-; CHECK-NEXT:    vmovdqa32 %ymm0, %ymm1 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <8 x i32>, <8 x i32>* %yp, align 4
   %mask = icmp ne <8 x i32> %x, %y
   %max = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %x1
   ret <8 x i32> %max
 }
 
 define <8 x i32> @test256_18(<8 x i32> %x, <8 x i32> %x1, <8 x i32>* %yp) nounwind {
 ; CHECK-LABEL: test256_18:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpneqd (%rdi), %ymm0, %k1
-; CHECK-NEXT:    vmovdqa32 %ymm0, %ymm1 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <8 x i32>, <8 x i32>* %yp, align 4
   %mask = icmp ne <8 x i32> %y, %x
   %max = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %x1
   ret <8 x i32> %max
 }
 
 define <8 x i32> @test256_19(<8 x i32> %x, <8 x i32> %x1, <8 x i32>* %yp) nounwind {
 ; CHECK-LABEL: test256_19:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpnltud (%rdi), %ymm0, %k1
-; CHECK-NEXT:    vmovdqa32 %ymm0, %ymm1 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <8 x i32>, <8 x i32>* %yp, align 4
   %mask = icmp uge <8 x i32> %x, %y
   %max = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %x1
   ret <8 x i32> %max
 }
 
 define <8 x i32> @test256_20(<8 x i32> %x, <8 x i32> %x1, <8 x i32>* %yp) nounwind {
 ; CHECK-LABEL: test256_20:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpleud (%rdi), %ymm0, %k1
-; CHECK-NEXT:    vmovdqa32 %ymm0, %ymm1 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <8 x i32>, <8 x i32>* %yp, align 4
   %mask = icmp uge <8 x i32> %y, %x
   %max = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %x1
   ret <8 x i32> %max
 }
 
 define <2 x i64> @test128_1(<2 x i64> %x, <2 x i64> %y) nounwind {
 ; CHECK-LABEL: test128_1:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpeqq %xmm1, %xmm0, %k1
-; CHECK-NEXT:    vmovdqa64 %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask = icmp eq <2 x i64> %x, %y
   %max = select <2 x i1> %mask, <2 x i64> %x, <2 x i64> %y
   ret <2 x i64> %max
 }
 
 define <2 x i64> @test128_2(<2 x i64> %x, <2 x i64> %y, <2 x i64> %x1) nounwind {
 ; CHECK-LABEL: test128_2:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpgtq %xmm1, %xmm0, %k1
-; CHECK-NEXT:    vmovdqa64 %xmm2, %xmm1 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vpblendmq %xmm2, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask = icmp sgt <2 x i64> %x, %y
   %max = select <2 x i1> %mask, <2 x i64> %x1, <2 x i64> %y
   ret <2 x i64> %max
 }
 
 define <4 x i32> @test128_3(<4 x i32> %x, <4 x i32> %y, <4 x i32> %x1) nounwind {
 ; CHECK-LABEL: test128_3:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpled %xmm0, %xmm1, %k1
-; CHECK-NEXT:    vmovdqa32 %xmm2, %xmm1 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vpblendmd %xmm2, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask = icmp sge <4 x i32> %x, %y
   %max = select <4 x i1> %mask, <4 x i32> %x1, <4 x i32> %y
   ret <4 x i32> %max
 }
 
 define <2 x i64> @test128_4(<2 x i64> %x, <2 x i64> %y, <2 x i64> %x1) nounwind {
 ; CHECK-LABEL: test128_4:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpnleuq %xmm1, %xmm0, %k1
-; CHECK-NEXT:    vmovdqa64 %xmm2, %xmm1 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vpblendmq %xmm2, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask = icmp ugt <2 x i64> %x, %y
   %max = select <2 x i1> %mask, <2 x i64> %x1, <2 x i64> %y
   ret <2 x i64> %max
 }
 
 define <4 x i32> @test128_5(<4 x i32> %x, <4 x i32> %x1, <4 x i32>* %yp) nounwind {
 ; CHECK-LABEL: test128_5:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpeqd (%rdi), %xmm0, %k1
-; CHECK-NEXT:    vmovdqa32 %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <4 x i32>, <4 x i32>* %yp, align 4
   %mask = icmp eq <4 x i32> %x, %y
   %max = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> %x1
   ret <4 x i32> %max
 }
 
 define <4 x i32> @test128_5b(<4 x i32> %x, <4 x i32> %x1, <4 x i32>* %yp) nounwind {
 ; CHECK-LABEL: test128_5b:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpeqd (%rdi), %xmm0, %k1
-; CHECK-NEXT:    vmovdqa32 %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <4 x i32>, <4 x i32>* %yp, align 4
   %mask = icmp eq <4 x i32> %y, %x
   %max = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> %x1
   ret <4 x i32> %max
 }
 
 define <4 x i32> @test128_6(<4 x i32> %x, <4 x i32> %x1, <4 x i32>* %y.ptr) nounwind {
 ; CHECK-LABEL: test128_6:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpgtd (%rdi), %xmm0, %k1
-; CHECK-NEXT:    vmovdqa32 %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <4 x i32>, <4 x i32>* %y.ptr, align 4
   %mask = icmp sgt <4 x i32> %x, %y
   %max = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> %x1
   ret <4 x i32> %max
 }
 
 define <4 x i32> @test128_6b(<4 x i32> %x, <4 x i32> %x1, <4 x i32>* %y.ptr) nounwind {
 ; CHECK-LABEL: test128_6b:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpgtd (%rdi), %xmm0, %k1
-; CHECK-NEXT:    vmovdqa32 %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <4 x i32>, <4 x i32>* %y.ptr, align 4
   %mask = icmp slt <4 x i32> %y, %x
   %max = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> %x1
   ret <4 x i32> %max
 }
 
 define <4 x i32> @test128_7(<4 x i32> %x, <4 x i32> %x1, <4 x i32>* %y.ptr) nounwind {
 ; CHECK-LABEL: test128_7:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpled (%rdi), %xmm0, %k1
-; CHECK-NEXT:    vmovdqa32 %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <4 x i32>, <4 x i32>* %y.ptr, align 4
   %mask = icmp sle <4 x i32> %x, %y
   %max = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> %x1
   ret <4 x i32> %max
 }
 
 define <4 x i32> @test128_7b(<4 x i32> %x, <4 x i32> %x1, <4 x i32>* %y.ptr) nounwind {
 ; CHECK-LABEL: test128_7b:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpled (%rdi), %xmm0, %k1
-; CHECK-NEXT:    vmovdqa32 %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <4 x i32>, <4 x i32>* %y.ptr, align 4
   %mask = icmp sge <4 x i32> %y, %x
   %max = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> %x1
   ret <4 x i32> %max
 }
 
 define <4 x i32> @test128_8(<4 x i32> %x, <4 x i32> %x1, <4 x i32>* %y.ptr) nounwind {
 ; CHECK-LABEL: test128_8:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpleud (%rdi), %xmm0, %k1
-; CHECK-NEXT:    vmovdqa32 %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <4 x i32>, <4 x i32>* %y.ptr, align 4
   %mask = icmp ule <4 x i32> %x, %y
   %max = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> %x1
   ret <4 x i32> %max
 }
 
 define <4 x i32> @test128_8b(<4 x i32> %x, <4 x i32> %x1, <4 x i32>* %y.ptr) nounwind {
 ; CHECK-LABEL: test128_8b:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpleud (%rdi), %xmm0, %k1
-; CHECK-NEXT:    vmovdqa32 %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <4 x i32>, <4 x i32>* %y.ptr, align 4
   %mask = icmp uge <4 x i32> %y, %x
   %max = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> %x1
   ret <4 x i32> %max
 }
 
 define <4 x i32> @test128_9(<4 x i32> %x, <4 x i32> %y, <4 x i32> %x1, <4 x i32> %y1) nounwind {
 ; CHECK-LABEL: test128_9:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpeqd %xmm1, %xmm0, %k1
 ; CHECK-NEXT:    vpcmpeqd %xmm3, %xmm2, %k1 {%k1}
-; CHECK-NEXT:    vmovdqa32 %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask1 = icmp eq <4 x i32> %x1, %y1
   %mask0 = icmp eq <4 x i32> %x, %y
   %mask = select <4 x i1> %mask0, <4 x i1> %mask1, <4 x i1> zeroinitializer
   %max = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> %y
   ret <4 x i32> %max
 }
 
 define <2 x i64> @test128_10(<2 x i64> %x, <2 x i64> %y, <2 x i64> %x1, <2 x i64> %y1) nounwind {
 ; CHECK-LABEL: test128_10:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpleq %xmm1, %xmm0, %k1
 ; CHECK-NEXT:    vpcmpleq %xmm2, %xmm3, %k1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm2, %xmm0
+; CHECK-NEXT:    vpblendmq %xmm0, %xmm2, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask1 = icmp sge <2 x i64> %x1, %y1
   %mask0 = icmp sle <2 x i64> %x, %y
   %mask = select <2 x i1> %mask0, <2 x i1> %mask1, <2 x i1> zeroinitializer
   %max = select <2 x i1> %mask, <2 x i64> %x, <2 x i64> %x1
   ret <2 x i64> %max
 }
 
 define <2 x i64> @test128_11(<2 x i64> %x, <2 x i64>* %y.ptr, <2 x i64> %x1, <2 x i64> %y1) nounwind {
 ; CHECK-LABEL: test128_11:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpgtq %xmm2, %xmm1, %k1
 ; CHECK-NEXT:    vpcmpgtq (%rdi), %xmm0, %k1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask1 = icmp sgt <2 x i64> %x1, %y1
   %y = load <2 x i64>, <2 x i64>* %y.ptr, align 4
   %mask0 = icmp sgt <2 x i64> %x, %y
   %mask = select <2 x i1> %mask0, <2 x i1> %mask1, <2 x i1> zeroinitializer
   %max = select <2 x i1> %mask, <2 x i64> %x, <2 x i64> %x1
   ret <2 x i64> %max
 }
 
 define <4 x i32> @test128_12(<4 x i32> %x, <4 x i32>* %y.ptr, <4 x i32> %x1, <4 x i32> %y1) nounwind {
 ; CHECK-LABEL: test128_12:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpled %xmm1, %xmm2, %k1
 ; CHECK-NEXT:    vpcmpleud (%rdi), %xmm0, %k1 {%k1}
-; CHECK-NEXT:    vmovdqa32 %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask1 = icmp sge <4 x i32> %x1, %y1
   %y = load <4 x i32>, <4 x i32>* %y.ptr, align 4
   %mask0 = icmp ule <4 x i32> %x, %y
   %mask = select <4 x i1> %mask0, <4 x i1> %mask1, <4 x i1> zeroinitializer
   %max = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> %x1
   ret <4 x i32> %max
 }
 
 define <2 x i64> @test128_13(<2 x i64> %x, <2 x i64> %x1, i64* %yb.ptr) nounwind {
 ; CHECK-LABEL: test128_13:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpeqq (%rdi){1to2}, %xmm0, %k1
-; CHECK-NEXT:    vmovdqa64 %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %yb = load i64, i64* %yb.ptr, align 4
   %y.0 = insertelement <2 x i64> undef, i64 %yb, i32 0
   %y = insertelement <2 x i64> %y.0, i64 %yb, i32 1
   %mask = icmp eq <2 x i64> %x, %y
   %max = select <2 x i1> %mask, <2 x i64> %x, <2 x i64> %x1
   ret <2 x i64> %max
 }
 
 define <4 x i32> @test128_14(<4 x i32> %x, i32* %yb.ptr, <4 x i32> %x1) nounwind {
 ; CHECK-LABEL: test128_14:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpled (%rdi){1to4}, %xmm0, %k1
-; CHECK-NEXT:    vmovdqa32 %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %yb = load i32, i32* %yb.ptr, align 4
   %y.0 = insertelement <4 x i32> undef, i32 %yb, i32 0
   %y = shufflevector <4 x i32> %y.0, <4 x i32> undef, <4 x i32> zeroinitializer
   %mask = icmp sle <4 x i32> %x, %y
   %max = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> %x1
   ret <4 x i32> %max
 }
 
 define <4 x i32> @test128_15(<4 x i32> %x, i32* %yb.ptr, <4 x i32> %x1, <4 x i32> %y1) nounwind {
 ; CHECK-LABEL: test128_15:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpled %xmm1, %xmm2, %k1
 ; CHECK-NEXT:    vpcmpgtd (%rdi){1to4}, %xmm0, %k1 {%k1}
-; CHECK-NEXT:    vmovdqa32 %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask1 = icmp sge <4 x i32> %x1, %y1
   %yb = load i32, i32* %yb.ptr, align 4
   %y.0 = insertelement <4 x i32> undef, i32 %yb, i32 0
   %y = shufflevector <4 x i32> %y.0, <4 x i32> undef, <4 x i32> zeroinitializer
   %mask0 = icmp sgt <4 x i32> %x, %y
   %mask = select <4 x i1> %mask0, <4 x i1> %mask1, <4 x i1> zeroinitializer
   %max = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> %x1
   ret <4 x i32> %max
 }
 
 define <2 x i64> @test128_16(<2 x i64> %x, i64* %yb.ptr, <2 x i64> %x1, <2 x i64> %y1) nounwind {
 ; CHECK-LABEL: test128_16:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpleq %xmm1, %xmm2, %k1
 ; CHECK-NEXT:    vpcmpgtq (%rdi){1to2}, %xmm0, %k1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %mask1 = icmp sge <2 x i64> %x1, %y1
   %yb = load i64, i64* %yb.ptr, align 4
   %y.0 = insertelement <2 x i64> undef, i64 %yb, i32 0
   %y = insertelement <2 x i64> %y.0, i64 %yb, i32 1
   %mask0 = icmp sgt <2 x i64> %x, %y
   %mask = select <2 x i1> %mask0, <2 x i1> %mask1, <2 x i1> zeroinitializer
   %max = select <2 x i1> %mask, <2 x i64> %x, <2 x i64> %x1
   ret <2 x i64> %max
 }
 
 define <4 x i32> @test128_17(<4 x i32> %x, <4 x i32> %x1, <4 x i32>* %y.ptr) nounwind {
 ; CHECK-LABEL: test128_17:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpneqd (%rdi), %xmm0, %k1
-; CHECK-NEXT:    vmovdqa32 %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <4 x i32>, <4 x i32>* %y.ptr, align 4
   %mask = icmp ne <4 x i32> %x, %y
   %max = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> %x1
   ret <4 x i32> %max
 }
 
 define <4 x i32> @test128_18(<4 x i32> %x, <4 x i32> %x1, <4 x i32>* %y.ptr) nounwind {
 ; CHECK-LABEL: test128_18:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpneqd (%rdi), %xmm0, %k1
-; CHECK-NEXT:    vmovdqa32 %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <4 x i32>, <4 x i32>* %y.ptr, align 4
   %mask = icmp ne <4 x i32> %y, %x
   %max = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> %x1
   ret <4 x i32> %max
 }
 
 define <4 x i32> @test128_19(<4 x i32> %x, <4 x i32> %x1, <4 x i32>* %y.ptr) nounwind {
 ; CHECK-LABEL: test128_19:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpnltud (%rdi), %xmm0, %k1
-; CHECK-NEXT:    vmovdqa32 %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <4 x i32>, <4 x i32>* %y.ptr, align 4
   %mask = icmp uge <4 x i32> %x, %y
   %max = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> %x1
   ret <4 x i32> %max
 }
 
 define <4 x i32> @test128_20(<4 x i32> %x, <4 x i32> %x1, <4 x i32>* %y.ptr) nounwind {
 ; CHECK-LABEL: test128_20:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpcmpleud (%rdi), %xmm0, %k1
-; CHECK-NEXT:    vmovdqa32 %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %y = load <4 x i32>, <4 x i32>* %y.ptr, align 4
   %mask = icmp uge <4 x i32> %y, %x
   %max = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> %x1
   ret <4 x i32> %max
 }

llvm/trunk/test/CodeGen/X86/masked_memop.ll

 ; AVX512F-NEXT:    vmaskmovpd (%rdi), %xmm0, %xmm2
 ; AVX512F-NEXT:    vblendvpd %xmm0, %xmm2, %xmm1, %xmm0
 ; AVX512F-NEXT:    retq
 ;
 ; SKX-LABEL: test6:
 ; SKX:       ## BB#0:
 ; SKX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; SKX-NEXT:    vpcmpeqq %xmm2, %xmm0, %k1
-; SKX-NEXT:    vmovupd (%rdi), %xmm1 {%k1}
-; SKX-NEXT:    vmovapd %xmm1, %xmm0
+; SKX-NEXT:    vblendmpd (%rdi), %xmm1, %xmm0 {%k1}
 ; SKX-NEXT:    retq
   %mask = icmp eq <2 x i64> %trigger, zeroinitializer
   %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %addr, i32 4, <2 x i1>%mask, <2 x double>%dst)
   ret <2 x double> %res
 }
 
 define <4 x float> @test7(<4 x i32> %trigger, <4 x float>* %addr, <4 x float> %dst) {
 ; AVX-LABEL: test7:

 ; AVX512F-NEXT:    vmaskmovps (%rdi), %xmm0, %xmm2
 ; AVX512F-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
 ; AVX512F-NEXT:    retq
 ;
 ; SKX-LABEL: test7:
 ; SKX:       ## BB#0:
 ; SKX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; SKX-NEXT:    vpcmpeqd %xmm2, %xmm0, %k1
-; SKX-NEXT:    vmovups (%rdi), %xmm1 {%k1}
-; SKX-NEXT:    vmovaps %xmm1, %xmm0
+; SKX-NEXT:    vblendmps (%rdi), %xmm1, %xmm0 {%k1}
 ; SKX-NEXT:    retq
   %mask = icmp eq <4 x i32> %trigger, zeroinitializer
   %res = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %addr, i32 4, <4 x i1>%mask, <4 x float>%dst)
   ret <4 x float> %res
 }
 
 define <4 x i32> @test8(<4 x i32> %trigger, <4 x i32>* %addr, <4 x i32> %dst) {
 ; AVX1-LABEL: test8:

 ; AVX512F-NEXT:    vpmaskmovd (%rdi), %xmm0, %xmm2
 ; AVX512F-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
 ; AVX512F-NEXT:    retq
 ;
 ; SKX-LABEL: test8:
 ; SKX:       ## BB#0:
 ; SKX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; SKX-NEXT:    vpcmpeqd %xmm2, %xmm0, %k1
-; SKX-NEXT:    vmovdqu32 (%rdi), %xmm1 {%k1}
-; SKX-NEXT:    vmovdqa %xmm1, %xmm0
+; SKX-NEXT:    vpblendmd (%rdi), %xmm1, %xmm0 {%k1}
 ; SKX-NEXT:    retq
   %mask = icmp eq <4 x i32> %trigger, zeroinitializer
   %res = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %addr, i32 4, <4 x i1>%mask, <4 x i32>%dst)
   ret <4 x i32> %res
 }
 
 define void @test9(<4 x i32> %trigger, <4 x i32>* %addr, <4 x i32> %val) {
 ; AVX1-LABEL: test9:

 ; AVX512F-NEXT:    vmaskmovpd (%rdi), %ymm0, %ymm2
 ; AVX512F-NEXT:    vblendvpd %ymm0, %ymm2, %ymm1, %ymm0
 ; AVX512F-NEXT:    retq
 ;
 ; SKX-LABEL: test10:
 ; SKX:       ## BB#0:
 ; SKX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; SKX-NEXT:    vpcmpeqd %xmm2, %xmm0, %k1
-; SKX-NEXT:    vmovapd (%rdi), %ymm1 {%k1}
-; SKX-NEXT:    vmovapd %ymm1, %ymm0
+; SKX-NEXT:    vblendmpd (%rdi), %ymm1, %ymm0 {%k1}
 ; SKX-NEXT:    retq
   %mask = icmp eq <4 x i32> %trigger, zeroinitializer
   %res = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* %addr, i32 32, <4 x i1>%mask, <4 x double>%dst)
   ret <4 x double> %res
 }
 
 define <4 x double> @test10b(<4 x i32> %trigger, <4 x double>* %addr, <4 x double> %dst) {
 ; AVX1-LABEL: test10b:

 ; AVX512F-LABEL: test11a:
 ; AVX512F:       ## BB#0:
 ; AVX512F-NEXT:    ## kill: %YMM1<def> %YMM1<kill> %ZMM1<def>
 ; AVX512F-NEXT:    ## kill: %YMM0<def> %YMM0<kill> %ZMM0<def>
 ; AVX512F-NEXT:    vpxor %ymm2, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpcmpeqd %zmm2, %zmm0, %k0
 ; AVX512F-NEXT:    kshiftlw $8, %k0, %k0
 ; AVX512F-NEXT:    kshiftrw $8, %k0, %k1
-; AVX512F-NEXT:    vmovups (%rdi), %zmm1 {%k1}
-; AVX512F-NEXT:    vmovaps %ymm1, %ymm0
+; AVX512F-NEXT:    vblendmps (%rdi), %zmm1, %zmm0 {%k1}
+; AVX512F-NEXT:    ## kill: %YMM0<def> %YMM0<kill> %ZMM0<kill>
 ; AVX512F-NEXT:    retq
 ;
 ; SKX-LABEL: test11a:
 ; SKX:       ## BB#0:
 ; SKX-NEXT:    vpxor %ymm2, %ymm2, %ymm2
 ; SKX-NEXT:    vpcmpeqd %ymm2, %ymm0, %k1
-; SKX-NEXT:    vmovaps (%rdi), %ymm1 {%k1}
-; SKX-NEXT:    vmovaps %ymm1, %ymm0
+; SKX-NEXT:    vblendmps (%rdi), %ymm1, %ymm0 {%k1}
 ; SKX-NEXT:    retq
   %mask = icmp eq <8 x i32> %trigger, zeroinitializer
   %res = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* %addr, i32 32, <8 x i1>%mask, <8 x float>%dst)
   ret <8 x float> %res
 }
 
 define <8 x i32> @test11b(<8 x i1> %mask, <8 x i32>* %addr, <8 x i32> %dst) {
 ; AVX1-LABEL: test11b:

 ; AVX512F-LABEL: test11b:
 ; AVX512F:       ## BB#0:
 ; AVX512F-NEXT:    ## kill: %YMM1<def> %YMM1<kill> %ZMM1<def>
 ; AVX512F-NEXT:    vpmovsxwq %xmm0, %zmm0
 ; AVX512F-NEXT:    vpsllq $63, %zmm0, %zmm0
 ; AVX512F-NEXT:    vptestmq %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kshiftlw $8, %k0, %k0
 ; AVX512F-NEXT:    kshiftrw $8, %k0, %k1
-; AVX512F-NEXT:    vmovdqu32 (%rdi), %zmm1 {%k1}
-; AVX512F-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512F-NEXT:    vpblendmd (%rdi), %zmm1, %zmm0 {%k1}
+; AVX512F-NEXT:    ## kill: %YMM0<def> %YMM0<kill> %ZMM0<kill>
 ; AVX512F-NEXT:    retq
 ;
 ; SKX-LABEL: test11b:
 ; SKX:       ## BB#0:
 ; SKX-NEXT:    vpsllw $15, %xmm0, %xmm0
 ; SKX-NEXT:    vpmovw2m %xmm0, %k1
-; SKX-NEXT:    vmovdqu32 (%rdi), %ymm1 {%k1}
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0
+; SKX-NEXT:    vpblendmd (%rdi), %ymm1, %ymm0 {%k1}
 ; SKX-NEXT:    retq
   %res = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* %addr, i32 4, <8 x i1>%mask, <8 x i32>%dst)
   ret <8 x i32> %res
 }
 
 define <8 x float> @test11c(<8 x i1> %mask, <8 x float>* %addr) {
 ; AVX1-LABEL: test11c:
 ; AVX1:       ## BB#0:

 ;
 ; SKX-LABEL: test16:
 ; SKX:       ## BB#0:
 ; SKX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; SKX-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
 ; SKX-NEXT:    vpcmpeqq %xmm2, %xmm0, %k0
 ; SKX-NEXT:    kshiftlw $14, %k0, %k0
 ; SKX-NEXT:    kshiftrw $14, %k0, %k1
-; SKX-NEXT:    vmovups (%rdi), %xmm1 {%k1}
-; SKX-NEXT:    vmovaps %xmm1, %xmm0
+; SKX-NEXT:    vblendmps (%rdi), %xmm1, %xmm0 {%k1}
 ; SKX-NEXT:    retq
   %mask = icmp eq <2 x i32> %trigger, zeroinitializer
   %res = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* %addr, i32 4, <2 x i1>%mask, <2 x float>%dst)
   ret <2 x float> %res
 }
 
 define <2 x i32> @test17(<2 x i32> %trigger, <2 x i32>* %addr, <2 x i32> %dst) {
 ; AVX1-LABEL: test17:

llvm/trunk/test/CodeGen/X86/vector-shuffle-128-v16.ll

 ; AVX1OR2-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
 ; AVX1OR2-NEXT:    vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX1OR2-NEXT:    retq
 ;
 ; AVX512VL-LABEL: shuffle_v16i8_00_17_02_19_04_21_06_23_08_25_10_27_12_29_14_31:
 ; AVX512VL:       # BB#0:
 ; AVX512VL-NEXT:    movw $-21846, %ax # imm = 0xAAAA
 ; AVX512VL-NEXT:    kmovw %eax, %k1
-; AVX512VL-NEXT:    vmovdqu8 %xmm0, %xmm1 {%k1}
-; AVX512VL-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512VL-NEXT:    vpblendmb %xmm0, %xmm1, %xmm0 {%k1}
 ; AVX512VL-NEXT:    retq
   %shuffle = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 0, i32 17, i32 2, i32 19, i32 4, i32 21, i32 6, i32 23, i32 8, i32 25, i32 10, i32 27, i32 12, i32 29, i32 14, i32 31>
   ret <16 x i8> %shuffle
 }
 
 define <16 x i8> @shuffle_v16i8_00_01_02_19_04_05_06_23_08_09_10_27_12_13_14_31(<16 x i8> %a, <16 x i8> %b) {
 ; SSE2-LABEL: shuffle_v16i8_00_01_02_19_04_05_06_23_08_09_10_27_12_13_14_31:
 ; SSE2:       # BB#0:

 ; AVX1OR2-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0]
 ; AVX1OR2-NEXT:    vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX1OR2-NEXT:    retq
 ;
 ; AVX512VL-LABEL: shuffle_v16i8_00_01_02_19_04_05_06_23_08_09_10_27_12_13_14_31:
 ; AVX512VL:       # BB#0:
 ; AVX512VL-NEXT:    movw $-30584, %ax # imm = 0x8888
 ; AVX512VL-NEXT:    kmovw %eax, %k1
-; AVX512VL-NEXT:    vmovdqu8 %xmm0, %xmm1 {%k1}
-; AVX512VL-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512VL-NEXT:    vpblendmb %xmm0, %xmm1, %xmm0 {%k1}
 ; AVX512VL-NEXT:    retq
   %shuffle = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 19, i32 4, i32 5, i32 6, i32 23, i32 8, i32 9, i32 10, i32 27, i32 12, i32 13, i32 14, i32 31>
   ret <16 x i8> %shuffle
 }
 
 define <16 x i8> @shuffle_v16i8_00_01_02_zz_04_05_06_zz_08_09_10_zz_12_13_14_zz(<16 x i8> %a) {
 ; SSE-LABEL: shuffle_v16i8_00_01_02_zz_04_05_06_zz_08_09_10_zz_12_13_14_zz:
 ; SSE:       # BB#0:

 ; AVX1OR2-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,255,255,0,255,255,0,255,255,255,255,0,255,255,0]
 ; AVX1OR2-NEXT:    vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX1OR2-NEXT:    retq
 ;
 ; AVX512VL-LABEL: shuffle_v16i8_00_01_02_03_20_05_06_23_08_09_10_11_28_13_14_31:
 ; AVX512VL:       # BB#0:
 ; AVX512VL-NEXT:    movw $-28528, %ax # imm = 0x9090
 ; AVX512VL-NEXT:    kmovw %eax, %k1
-; AVX512VL-NEXT:    vmovdqu8 %xmm0, %xmm1 {%k1}
-; AVX512VL-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512VL-NEXT:    vpblendmb %xmm0, %xmm1, %xmm0 {%k1}
 ; AVX512VL-NEXT:    retq
   %shuffle = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 20, i32 5, i32 6, i32 23, i32 8, i32 9, i32 10, i32 11, i32 28, i32 13, i32 14, i32 31>
   ret <16 x i8> %shuffle
 }
 
 define <16 x i8> @shuffle_v16i8_16_17_18_19_04_05_06_07_24_25_10_11_28_13_30_15(<16 x i8> %a, <16 x i8> %b) {
 ; SSE2-LABEL: shuffle_v16i8_16_17_18_19_04_05_06_07_24_25_10_11_28_13_30_15:
 ; SSE2:       # BB#0:

llvm/trunk/test/CodeGen/X86/vector-shuffle-256-v16.ll

 ; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0]
 ; AVX2-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512VL-LABEL: shuffle_v16i16_00_01_02_03_04_05_06_07_08_09_10_11_12_13_14_31:
 ; AVX512VL:       # BB#0:
 ; AVX512VL-NEXT:    movw $-32768, %ax # imm = 0x8000
 ; AVX512VL-NEXT:    kmovw %eax, %k1
-; AVX512VL-NEXT:    vmovdqu16 %ymm0, %ymm1 {%k1}
-; AVX512VL-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512VL-NEXT:    vpblendmw %ymm0, %ymm1, %ymm0 {%k1}
 ; AVX512VL-NEXT:    retq
   %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 31>
   ret <16 x i16> %shuffle
 }
 
 define <16 x i16> @shuffle_v16i16_16_01_02_03_04_05_06_07_08_09_10_11_12_13_14_15(<16 x i16> %a, <16 x i16> %b) {
 ; AVX1-LABEL: shuffle_v16i16_16_01_02_03_04_05_06_07_08_09_10_11_12_13_14_15:
 ; AVX1:       # BB#0:
 ; AVX1-NEXT:    vmovaps {{.*#+}} ymm2 = [0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
 ; AVX1-NEXT:    vandnps %ymm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vandps %ymm2, %ymm0, %ymm0
 ; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: shuffle_v16i16_16_01_02_03_04_05_06_07_08_09_10_11_12_13_14_15:
 ; AVX2:       # BB#0:
 ; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX2-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512VL-LABEL: shuffle_v16i16_16_01_02_03_04_05_06_07_08_09_10_11_12_13_14_15:
 ; AVX512VL:       # BB#0:
 ; AVX512VL-NEXT:    movw $1, %ax
 ; AVX512VL-NEXT:    kmovw %eax, %k1
-; AVX512VL-NEXT:    vmovdqu16 %ymm0, %ymm1 {%k1}
-; AVX512VL-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512VL-NEXT:    vpblendmw %ymm0, %ymm1, %ymm0 {%k1}
 ; AVX512VL-NEXT:    retq
   %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 16, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   ret <16 x i16> %shuffle
 }
 
 define <16 x i16> @shuffle_v16i16_00_17_02_19_04_21_06_23_24_09_26_11_28_13_30_15(<16 x i16> %a, <16 x i16> %b) {
 ; AVX1-LABEL: shuffle_v16i16_00_17_02_19_04_21_06_23_24_09_26_11_28_13_30_15:
 ; AVX1:       # BB#0:
 ; AVX1-NEXT:    vmovaps {{.*#+}} ymm2 = [65535,0,65535,0,65535,0,65535,0,0,65535,0,65535,0,65535,0,65535]
 ; AVX1-NEXT:    vandnps %ymm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vandps %ymm2, %ymm0, %ymm0
 ; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: shuffle_v16i16_00_17_02_19_04_21_06_23_24_09_26_11_28_13_30_15:
 ; AVX2:       # BB#0:
 ; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,0,0,255,255,0,0,255,255,0,0,255,255,0,0,0,0,255,255,0,0,255,255,0,0,255,255,0,0,255,255]
 ; AVX2-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512VL-LABEL: shuffle_v16i16_00_17_02_19_04_21_06_23_24_09_26_11_28_13_30_15:
 ; AVX512VL:       # BB#0:
 ; AVX512VL-NEXT:    movw $21930, %ax # imm = 0x55AA
 ; AVX512VL-NEXT:    kmovw %eax, %k1
-; AVX512VL-NEXT:    vmovdqu16 %ymm0, %ymm1 {%k1}
-; AVX512VL-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512VL-NEXT:    vpblendmw %ymm0, %ymm1, %ymm0 {%k1}
 ; AVX512VL-NEXT:    retq
   %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 0, i32 17, i32 2, i32 19, i32 4, i32 21, i32 6, i32 23, i32 24, i32 9, i32 26, i32 11, i32 28, i32 13, i32 30, i32 15>
   ret <16 x i16> %shuffle
 }
 
 define <16 x i16> @shuffle_v16i16_16_01_18_03_20_05_22_07_08_25_10_27_12_29_14_31(<16 x i16> %a, <16 x i16> %b) {
 ; AVX1-LABEL: shuffle_v16i16_16_01_18_03_20_05_22_07_08_25_10_27_12_29_14_31:
 ; AVX1:       # BB#0:
 ; AVX1-NEXT:    vmovaps {{.*#+}} ymm2 = [0,65535,0,65535,0,65535,0,65535,65535,0,65535,0,65535,0,65535,0]
 ; AVX1-NEXT:    vandnps %ymm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vandps %ymm2, %ymm0, %ymm0
 ; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: shuffle_v16i16_16_01_18_03_20_05_22_07_08_25_10_27_12_29_14_31:
 ; AVX2:       # BB#0:
 ; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,0,255,255,0,0,255,255,0,0,255,255,0,0,255,255,255,255,0,0,255,255,0,0,255,255,0,0,255,255,0,0]
 ; AVX2-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512VL-LABEL: shuffle_v16i16_16_01_18_03_20_05_22_07_08_25_10_27_12_29_14_31:
 ; AVX512VL:       # BB#0:
 ; AVX512VL-NEXT:    movw $-21931, %ax # imm = 0xAA55
 ; AVX512VL-NEXT:    kmovw %eax, %k1
-; AVX512VL-NEXT:    vmovdqu16 %ymm0, %ymm1 {%k1}
-; AVX512VL-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512VL-NEXT:    vpblendmw %ymm0, %ymm1, %ymm0 {%k1}
 ; AVX512VL-NEXT:    retq
   %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 16, i32 1, i32 18, i32 3, i32 20, i32 5, i32 22, i32 7, i32 8, i32 25, i32 10, i32 27, i32 12, i32 29, i32 14, i32 31>
   ret <16 x i16> %shuffle
 }
 
 define <16 x i16> @shuffle_v16i16_00_01_18_19_20_21_06_07_08_09_26_27_12_13_30_31(<16 x i16> %a, <16 x i16> %b) {
 ; AVX1-LABEL: shuffle_v16i16_00_01_18_19_20_21_06_07_08_09_26_27_12_13_30_31:
 ; AVX1:       # BB#0:

llvm/trunk/test/CodeGen/X86/vector-shuffle-256-v32.ll

 ; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
 ; AVX2-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512VL-LABEL: shuffle_v32i8_00_33_02_35_04_37_06_39_08_41_10_43_12_45_14_47_16_49_18_51_20_53_22_55_24_57_26_59_28_61_30_63:
 ; AVX512VL:       # BB#0:
 ; AVX512VL-NEXT:    movl $-1431655766, %eax # imm = 0xAAAAAAAA
 ; AVX512VL-NEXT:    kmovd %eax, %k1
-; AVX512VL-NEXT:    vmovdqu8 %ymm0, %ymm1 {%k1}
-; AVX512VL-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512VL-NEXT:    vpblendmb %ymm0, %ymm1, %ymm0 {%k1}
 ; AVX512VL-NEXT:    retq
   %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> <i32 0, i32 33, i32 2, i32 35, i32 4, i32 37, i32 6, i32 39, i32 8, i32 41, i32 10, i32 43, i32 12, i32 45, i32 14, i32 47, i32 16, i32 49, i32 18, i32 51, i32 20, i32 53, i32 22, i32 55, i32 24, i32 57, i32 26, i32 59, i32 28, i32 61, i32 30, i32 63>
   ret <32 x i8> %shuffle
 }
 
 define <32 x i8> @shuffle_v32i8_32_01_34_03_36_05_38_07_40_09_42_11_44_13_46_15_48_17_50_19_52_21_54_23_56_25_58_27_60_29_62_31(<32 x i8> %a, <32 x i8> %b) {
 ; AVX1-LABEL: shuffle_v32i8_32_01_34_03_36_05_38_07_40_09_42_11_44_13_46_15_48_17_50_19_52_21_54_23_56_25_58_27_60_29_62_31:
 ; AVX1:       # BB#0:

llvm/trunk/test/CodeGen/X86/vector-shuffle-512-v16.ll

; ALL-NEXT:    retq
   ret <16 x i32> %d
 }
 
 define <16 x i32> @shuffle_v16i32_0_1_2_19_u_u_u_u_u_u_u_u_u_u_u_u(<16 x i32> %a, <16 x i32> %b)  {
 ; ALL-LABEL: shuffle_v16i32_0_1_2_19_u_u_u_u_u_u_u_u_u_u_u_u:
 ; ALL:       # BB#0:
 ; ALL-NEXT:    movw $8, %ax
 ; ALL-NEXT:    kmovw %eax, %k1
-; ALL-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
-; ALL-NEXT:    vmovdqa64 %zmm1, %zmm0
+; ALL-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
 ; ALL-NEXT:    retq
   %c = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 19, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
   ret <16 x i32> %c
 }
 
 define <8 x float> @shuffle_v16f32_extract_256(float* %RET, float* %a) {
 ; ALL-LABEL: shuffle_v16f32_extract_256:
 ; ALL:       # BB#0:

llvm/trunk/test/CodeGen/X86/vector-shuffle-masked.ll

; CHECK-NEXT:    retq
   ret <8 x i32> %res
 }
 
 define <8 x i32> @mask_shuffle_v8i32_23456701(<8 x i32> %a, <8 x i32> %passthru, i8 %mask) {
 ; CHECK-LABEL: mask_shuffle_v8i32_23456701:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[1,2,3,0]
 ; CHECK-NEXT:    kmovb %edi, %k1
-; CHECK-NEXT:    vmovdqa32 %ymm0, %ymm1 {%k1}
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
+; CHECK-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %shuffle = shufflevector <8 x i32> %a, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1>
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res = select <8 x i1> %mask.cast, <8 x i32> %shuffle, <8 x i32> %passthru
   ret <8 x i32> %res
 }
 
 define <8 x i32> @maskz_shuffle_v8i32_23456701(<8 x i32> %a, i8 %mask) {