[X86][AVX512] Adding missing shuffle lowering to blend mask instructions (VPBLENDMB/VPBLENDMW/VPBLENDMD/VPBLENDMQ) .

[m_zuckerman]

Summary

Some shuffles can be lowered to blend mask instruction (VPBLENDMB/VPBLENDMW/VPBLENDMD/VPBLENDMQ) .
In this patch, I added new pattern match for this case.

This pattern only catches zmm, since we are using a more efficient blend instruction (without a mask) in the other cases.

[RKSimon]

lib/Target/X86/X86ISelLowering.cpp

13143

Unnecessary as AVX512BW is a requirement for v32i16 - see the assert at the top of function.

13187

Unnecessary as AVX512BW is a requirement for v64i8 - see the assert at the top of function.

[RKSimon]

test/CodeGen/X86/vector-shuffle-512-v32.ll

115

Any idea why this isn't using a blend with zero: _mm512_maskz_mov_epi16 ?

[igorb]

Hi,
for v16i8 and v32i8 we should use vblendmb if possible.

define <16 x i8> @test_mask_blend_epi8(<16 x i8> %A, <16 x i8> %W){
entry:

%0 = shufflevector <16 x i8> %A, <16 x i8> %W, <16 x i32>  <i32 16, i32 1, i32 18, i32 3, i32 20, i32 5, i32 22, i32 7, i32 24, i32 9, i32 26, i32 11, i32 28, i32 13, i32 30, i32 15>
ret <16 x i8> %0

}

vmovdqu .LCPI0_0(%rip), %xmm2 # xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
retq

lib/Target/X86/X86ISelLowering.cpp

8483

you can use getVectorMaskingNode to simplify the code, all logic already implemented.

8485

not in use.

test/CodeGen/X86/vector-shuffle-to-blend-avx512.ll.ll

4 ↗

(On Diff #83689)

Could you please add tests for the all cases i8/i16/f64

[m_zuckerman]

test/CodeGen/X86/vector-shuffle-512-v32.ll

115

Patterns were missing, This was changed in commit 291368

[RKSimon]

LGTM - possibly move the new tests into the existing vector-shuffle-avx512.ll file for tidiness but that's very minor.

test/CodeGen/X86/vector-shuffle-to-blend-avx512.ll

4

Move these into vector-shuffle-avx512.ll ?

[dlj]

llvm/trunk/lib/Target/X86/X86ISelLowering.cpp

8334 ↗

(On Diff #84256)

This is problematic with v64i8, since the mask can now be 64 bits wide.

With UndefinedBehaviorSanitizer, I see this failure from test/CodeGen/X86/vector-shuffle-avx512.ll:

• TEST 'LLVM :: CodeGen/X86/vector-shuffle-avx512.ll' FAILED ****

Script:

/build/build-ubsan/./bin/llc < /src/test/CodeGen/X86/vector-shuffle-avx512.ll -mtriple=x86_64-pc-linux-gnu -mcpu=skx | /build/build-ubsan/./bin/FileCheck /src/test/CodeGen/X86/vector-shuffle-avx512.ll --check-prefix=SKX

/build/build-ubsan/./bin/llc < /src/test/CodeGen/X86/vector-shuffle-avx512.ll -mtriple=x86_64-pc-linux-gnu -mcpu=knl | /build/build-ubsan/./bin/FileCheck /src/test/CodeGen/X86/vector-shuffle-avx512.ll --check-prefix=KNL

Exit Code: 1

Command Output (stderr):

/src/lib/Target/X86/X86ISelLowering.cpp:8342:23: runtime error: shift exponent 33 is too large for 32-bit type 'unsigned int'
/src/test/CodeGen/X86/vector-shuffle-avx512.ll:248:14: error: expected string not found in input
; SKX-LABEL: expand12:

^

<stdin>:168:11: note: scanning from here
expand11: # @expand11

^

<stdin>:168:14: note: possible intended match here
expand11: # @expand11

^

[craig.topper]

The test case for 64 x i8 is also showing only a 32-bit immediate being used.

lib/Target/X86/X86ISelLowering.cpp

static SDValue lowerVectorShuffleAsBitBlend(const SDLoc &DL, MVT VT, SDValue V1,
   // We have to cast V2 around.
   MVT MaskVT = MVT::getVectorVT(MVT::i64, VT.getSizeInBits() / 64);
   V2 = DAG.getBitcast(VT, DAG.getNode(X86ISD::ANDNP, DL, MaskVT,
                                       DAG.getBitcast(MaskVT, V1Mask),
                                       DAG.getBitcast(MaskVT, V2)));
   return DAG.getNode(ISD::OR, DL, VT, V1, V2);
 }
 
+static SDValue getVectorMaskingNode(SDValue Op, SDValue Mask,
+                                    SDValue PreservedSrc,
+                                    const X86Subtarget &Subtarget,
+                                    SelectionDAG &DAG);
+
 /// \brief Try to emit a blend instruction for a shuffle.
 ///
 /// This doesn't do any checks for the availability of instructions for blending
 /// these values. It relies on the availability of the X86ISD::BLENDI pattern to
 /// be matched in the backend with the type given. What it does check for is
 /// that the shuffle mask is a blend, or convertible into a blend with zero.
 static SDValue lowerVectorShuffleAsBlend(const SDLoc &DL, MVT VT, SDValue V1,
                                          SDValue V2, ArrayRef<int> Original,
for (int i = 0, Size = Mask.size(); i < Size; ++i)
                                           MVT::i8));
 
     V1 = DAG.getBitcast(BlendVT, V1);
     V2 = DAG.getBitcast(BlendVT, V2);
     return DAG.getBitcast(
         VT, DAG.getNode(ISD::VSELECT, DL, BlendVT,
                         DAG.getBuildVector(BlendVT, DL, VSELECTMask), V1, V2));
   }
-
+  case MVT::v16f32:
+  case MVT::v8f64:
+  case MVT::v8i64:
+  case MVT::v16i32:
+  case MVT::v32i16:
+  case MVT::v64i8: {
+    MVT IntegerType =
+        MVT::getIntegerVT(std::max((int)VT.getVectorNumElements(), 8));
+    SDValue MaskNode = DAG.getConstant(BlendMask, DL, IntegerType);
+    return getVectorMaskingNode(V1, MaskNode, V2, Subtarget, DAG);
+  }
   default:
     llvm_unreachable("Not a supported integer vector type!");
   }
 }
 
 /// \brief Try to lower as a blend of elements from two inputs followed by
 /// a single-input permutation.
 ///
 /// This matches the pattern where we can blend elements from two inputs and
static SDValue lowerV8F64VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
   if (SDValue Op =
       lowerVectorShuffleWithSHUFPD(DL, MVT::v8f64, Mask, V1, V2, DAG))
     return Op;
 
   if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v8f64, Zeroable, Mask, V1,
                                              V2, DAG, Subtarget))
     return V;
 
+  if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v8f64, V1, V2, Mask,
+                                                Zeroable, Subtarget, DAG))
+    return Blend;
+
   return lowerVectorShuffleWithPERMV(DL, MVT::v8f64, Mask, V1, V2, DAG);
 }
 
 /// \brief Handle lowering of 16-lane 32-bit floating point shuffles.
 static SDValue lowerV16F32VectorShuffle(SDLoc DL, ArrayRef<int> Mask,
                                         const SmallBitVector &Zeroable, 
                                         SDValue V1, SDValue V2,
                                         const X86Subtarget &Subtarget,
if (V2.isUndef())
       return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v16f32, V1,
                          getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
 
     // Use dedicated unpack instructions for masks that match their pattern.
     if (SDValue Unpck =
             lowerVectorShuffleWithUNPCK(DL, MVT::v16f32, Mask, V1, V2, DAG))
       return Unpck;
 
+    if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v16f32, V1, V2, Mask,
+                                                  Zeroable, Subtarget, DAG))
+      return Blend;
+
     // Otherwise, fall back to a SHUFPS sequence.
     return lowerVectorShuffleWithSHUFPS(DL, MVT::v16f32, RepeatedMask, V1, V2, DAG);
   }
   // If we have AVX512F support, we can use VEXPAND.
   if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v16f32, Zeroable, Mask,
                                              V1, V2, DAG, Subtarget))
     return V;
 
static SDValue lowerV8I64VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
   if (SDValue Unpck =
           lowerVectorShuffleWithUNPCK(DL, MVT::v8i64, Mask, V1, V2, DAG))
     return Unpck;
   // If we have AVX512F support, we can use VEXPAND.
   if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v8i64, Zeroable, Mask, V1,
                                              V2, DAG, Subtarget))
     return V;
 
+  if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v8i64, V1, V2, Mask,
+                                                Zeroable, Subtarget, DAG))
+    return Blend;
+
   return lowerVectorShuffleWithPERMV(DL, MVT::v8i64, Mask, V1, V2, DAG);
 }
 
 /// \brief Handle lowering of 16-lane 32-bit integer shuffles.
 static SDValue lowerV16I32VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
                                         const SmallBitVector &Zeroable,
                                         SDValue V1, SDValue V2,
                                         const X86Subtarget &Subtarget,
SDValue ShufPS = lowerVectorShuffleWithSHUFPS(DL, MVT::v16f32, RepeatedMask,
                                                   CastV1, CastV2, DAG);
     return DAG.getBitcast(MVT::v16i32, ShufPS);
   }
   // If we have AVX512F support, we can use VEXPAND.
   if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v16i32, Zeroable, Mask,
                                              V1, V2, DAG, Subtarget))
     return V;
 
+  if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v16i32, V1, V2, Mask,
+                                                Zeroable, Subtarget, DAG))
+    return Blend;
   return lowerVectorShuffleWithPERMV(DL, MVT::v16i32, Mask, V1, V2, DAG);
 }
 
 /// \brief Handle lowering of 32-lane 16-bit integer shuffles.
 static SDValue lowerV32I16VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
                                         const SmallBitVector &Zeroable,
                                         SDValue V1, SDValue V2,
                                         const X86Subtarget &Subtarget,
if (is128BitLaneRepeatedShuffleMask(MVT::v32i16, Mask, RepeatedMask)) {
       // As this is a single-input shuffle, the repeated mask should be
       // a strictly valid v8i16 mask that we can pass through to the v8i16
       // lowering to handle even the v32 case.
       return lowerV8I16GeneralSingleInputVectorShuffle(
           DL, MVT::v32i16, V1, RepeatedMask, Subtarget, DAG);
     }
   }
 
+  if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v32i16, V1, V2, Mask,
+                                                Zeroable, Subtarget, DAG))
+    return Blend;
+
   return lowerVectorShuffleWithPERMV(DL, MVT::v32i16, Mask, V1, V2, DAG);
 }
 
 /// \brief Handle lowering of 64-lane 8-bit integer shuffles.
 static SDValue lowerV64I8VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
                                        const SmallBitVector &Zeroable,
                                        SDValue V1, SDValue V2,
                                        const X86Subtarget &Subtarget,
static SDValue lowerV64I8VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
   if (SDValue Rotate = lowerVectorShuffleAsByteRotate(
           DL, MVT::v64i8, V1, V2, Mask, Subtarget, DAG))
     return Rotate;
 
   if (SDValue PSHUFB = lowerVectorShuffleWithPSHUFB(
           DL, MVT::v64i8, Mask, V1, V2, Zeroable, Subtarget, DAG))
     return PSHUFB;
 
   // VBMI can use VPERMV/VPERMV3 byte shuffles.
   if (Subtarget.hasVBMI())
     return lowerVectorShuffleWithPERMV(DL, MVT::v64i8, Mask, V1, V2, DAG);
 
   // Try to create an in-lane repeating shuffle mask and then shuffle the
   // the results into the target lanes.
   if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
           DL, MVT::v64i8, V1, V2, Mask, Subtarget, DAG))
     return V;
 
+  if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v64i8, V1, V2, Mask,
+                                                Zeroable, Subtarget, DAG))
+    return Blend;
+
   // FIXME: Implement direct support for this type!
   return splitAndLowerVectorShuffle(DL, MVT::v64i8, V1, V2, Mask, DAG);
 }
 
 /// \brief High-level routine to lower various 512-bit x86 vector shuffles.
 ///
 /// This routine either breaks down the specific type of a 512-bit x86 vector
 /// shuffle or splits it into two 256-bit shuffles and fuses the results back

test/CodeGen/X86/merge-consecutive-loads-512.ll

; X32-AVX512F-NEXT:    retl
   %res6 = insertelement <8 x double> %res3, double   0.0, i32 6
   %res7 = insertelement <8 x double> %res6, double   0.0, i32 7
   ret <8 x double> %res7
 }
 
 define <8 x double> @merge_8f64_f64_1u3u5zu8(double* %ptr) nounwind uwtable noinline ssp {
 ; ALL-LABEL: merge_8f64_f64_1u3u5zu8:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vmovupd 8(%rdi), %zmm1
-; ALL-NEXT:    vpxord %zmm2, %zmm2, %zmm2
-; ALL-NEXT:    vmovapd {{.*#+}} zmm0 = <0,u,2,u,4,13,u,7>
-; ALL-NEXT:    vpermi2pd %zmm2, %zmm1, %zmm0
+; ALL-NEXT:    movb $32, %al
+; ALL-NEXT:    kmovw  %eax, %k1
+; ALL-NEXT:    vmovupd  8(%rdi), %zmm0 {%k1} {z}
 ; ALL-NEXT:    retq
 ;
 ; X32-AVX512F-LABEL: merge_8f64_f64_1u3u5zu8:
 ; X32-AVX512F:       # BB#0:
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vmovupd 8(%eax), %zmm1
-; X32-AVX512F-NEXT:    vpxord %zmm2, %zmm2, %zmm2
-; X32-AVX512F-NEXT:    vmovapd {{.*#+}} zmm0 = <0,0,u,u,2,0,u,u,4,0,13,0,u,u,7,0>
-; X32-AVX512F-NEXT:    vpermi2pd %zmm2, %zmm1, %zmm0
+; X32-AVX512F-NEXT:    movl 4(%esp), %eax
+; X32-AVX512F-NEXT:    movb $32, %cl
+; X32-AVX512F-NEXT:    kmovw  %ecx, %k1
+; X32-AVX512F-NEXT:    vmovupd  8(%eax), %zmm0 {%k1} {z}
 ; X32-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds double, double* %ptr, i64 1
   %ptr2 = getelementptr inbounds double, double* %ptr, i64 3
   %ptr4 = getelementptr inbounds double, double* %ptr, i64 5
   %ptr7 = getelementptr inbounds double, double* %ptr, i64 8
   %val0 = load double, double* %ptr0
   %val2 = load double, double* %ptr2
   %val4 = load double, double* %ptr4
; X32-AVX512F-NEXT:    retl
   %res6 = insertelement <8 x i64> %res4, i64     0, i32 6
   %res7 = insertelement <8 x i64> %res6, i64     0, i32 7
   ret <8 x i64> %res7
 }
 
 define <8 x i64> @merge_8i64_i64_1u3u5zu8(i64* %ptr) nounwind uwtable noinline ssp {
 ; ALL-LABEL: merge_8i64_i64_1u3u5zu8:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vmovdqu64 8(%rdi), %zmm1
-; ALL-NEXT:    vpxord %zmm2, %zmm2, %zmm2
-; ALL-NEXT:    vmovdqa64 {{.*#+}} zmm0 = <0,u,2,u,4,13,u,7>
-; ALL-NEXT:    vpermi2q %zmm2, %zmm1, %zmm0
+; ALL-NEXT:    movb $32, %al
+; ALL-NEXT:    kmovw  %eax, %k1
+; ALL-NEXT:    vmovdqu64  8(%rdi), %zmm0 {%k1} {z}
 ; ALL-NEXT:    retq
 ;
 ; X32-AVX512F-LABEL: merge_8i64_i64_1u3u5zu8:
 ; X32-AVX512F:       # BB#0:
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vmovdqu64 8(%eax), %zmm1
-; X32-AVX512F-NEXT:    vpxord %zmm2, %zmm2, %zmm2
-; X32-AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm0 = <0,0,u,u,2,0,u,u,4,0,13,0,u,u,7,0>
-; X32-AVX512F-NEXT:    vpermi2q %zmm2, %zmm1, %zmm0
+; X32-AVX512F-NEXT:    movl 4(%esp), %eax
+; X32-AVX512F-NEXT:    movb $32, %cl
+; X32-AVX512F-NEXT:    kmovw  %ecx, %k1
+; X32-AVX512F-NEXT:    vmovdqu64  8(%eax), %zmm0 {%k1} {z}
 ; X32-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds i64, i64* %ptr, i64 1
   %ptr2 = getelementptr inbounds i64, i64* %ptr, i64 3
   %ptr4 = getelementptr inbounds i64, i64* %ptr, i64 5
   %ptr7 = getelementptr inbounds i64, i64* %ptr, i64 8
   %val0 = load i64, i64* %ptr0
   %val2 = load i64, i64* %ptr2
   %val4 = load i64, i64* %ptr4
; X32-AVX512F-NEXT:    retl
   %resE = insertelement <16 x i32> %resC, i32 %valE, i32 14
   %resF = insertelement <16 x i32> %resE, i32 %valF, i32 15
   ret <16 x i32> %resF
 }
 
 define <16 x i32> @merge_16i32_i32_0uu3zzuuuuuzCuEF(i32* %ptr) nounwind uwtable noinline ssp {
 ; ALL-LABEL: merge_16i32_i32_0uu3zzuuuuuzCuEF:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vmovdqu32 (%rdi), %zmm1
-; ALL-NEXT:    vpxord %zmm2, %zmm2, %zmm2
-; ALL-NEXT:    vmovdqa32 {{.*#+}} zmm0 = <0,u,u,3,20,21,u,u,u,u,u,u,12,29,14,15>
-; ALL-NEXT:    vpermi2d %zmm2, %zmm1, %zmm0
+; ALL-NEXT:    movw $8240, %ax              # imm = 0x2030
+; ALL-NEXT:    kmovw  %eax, %k1
+; ALL-NEXT:    vmovdqu32  (%rdi), %zmm0 {%k1} {z}
 ; ALL-NEXT:    retq
 ;
 ; X32-AVX512F-LABEL: merge_16i32_i32_0uu3zzuuuuuzCuEF:
 ; X32-AVX512F:       # BB#0:
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vmovdqu32 (%eax), %zmm1
-; X32-AVX512F-NEXT:    vpxord %zmm2, %zmm2, %zmm2
-; X32-AVX512F-NEXT:    vmovdqa32 {{.*#+}} zmm0 = <0,u,u,3,20,21,u,u,u,u,u,u,12,29,14,15>
-; X32-AVX512F-NEXT:    vpermi2d %zmm2, %zmm1, %zmm0
+; X32-AVX512F-NEXT:    movl 4(%esp), %eax
+; X32-AVX512F-NEXT:    movw $8240, %cx              # imm = 0x2030
+; X32-AVX512F-NEXT:    kmovw  %ecx, %k1
+; X32-AVX512F-NEXT:    vmovdqu32  (%eax), %zmm0 {%k1} {z}
 ; X32-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds i32, i32* %ptr, i64 0
   %ptr3 = getelementptr inbounds i32, i32* %ptr, i64 3
   %ptrC = getelementptr inbounds i32, i32* %ptr, i64 12
   %ptrE = getelementptr inbounds i32, i32* %ptr, i64 14
   %ptrF = getelementptr inbounds i32, i32* %ptr, i64 15
   %val0 = load i32, i32* %ptr0
   %val3 = load i32, i32* %ptr3

test/CodeGen/X86/sse3-avx-addsub.ll

 ; AVX1-LABEL: test5:
 ; AVX1:       # BB#0:
 ; AVX1-NEXT:    vaddsubps %ymm2, %ymm0, %ymm0
 ; AVX1-NEXT:    vaddsubps %ymm3, %ymm1, %ymm1
 ; AVX1-NEXT:    retq
 ;
 ; AVX512-LABEL: test5:
 ; AVX512:       # BB#0:
-; AVX512-NEXT:    vaddps %zmm1, %zmm0, %zmm2
-; AVX512-NEXT:    vsubps %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    vshufps {{.*#+}} zmm0 = zmm0[0,2],zmm2[1,3],zmm0[4,6],zmm2[5,7],zmm0[8,10],zmm2[9,11],zmm0[12,14],zmm2[13,15]
-; AVX512-NEXT:    vpermilps {{.*#+}} zmm0 = zmm0[0,2,1,3,4,6,5,7,8,10,9,11,12,14,13,15]
+; AVX512-NEXT:    vaddps  %zmm1, %zmm0, %zmm2
+; AVX512-NEXT:    movw  $-21846, %ax            # imm = 0xAAAA
+; AVX512-NEXT:    kmovw %eax, %k1
+; AVX512-NEXT:    vsubps  %zmm1, %zmm0, %zmm2 {%k1}
+; AVX512-NEXT:    vmovaps %zmm2, %zmm0
 ; AVX512-NEXT:    retq
   %add = fadd <16 x float> %A, %B
   %sub = fsub <16 x float> %A, %B
   %vecinit2 = shufflevector <16 x float> %sub, <16 x float> %add, <16 x i32> <i32 0, i32 17, i32 2, i32 19, i32 4, i32 21, i32 6, i32 23, i32 8, i32 25, i32 10, i32 27, i32 12, i32 29, i32 14, i32 31>
   ret <16 x float> %vecinit2
 }
 
 define <8 x double> @test6(<8 x double> %A, <8 x double> %B) {

test/CodeGen/X86/vector-shuffle-512-v16.ll

; ALL-NEXT:    retq
   %c = load <16 x i32>, <16 x i32>* %b
   %d = shufflevector <16 x i32> %a, <16 x i32> %c, <16 x i32> <i32 15, i32 31, i32 14, i32 22, i32 13, i32 29, i32 4, i32 28, i32 11, i32 27, i32 10, i32 26, i32 9, i32 25, i32 8, i32 24>
   ret <16 x i32> %d
 }
 
 define <16 x i32> @shuffle_v16i32_0_1_2_19_u_u_u_u_u_u_u_u_u_u_u_u(<16 x i32> %a, <16 x i32> %b)  {
 ; ALL-LABEL: shuffle_v16i32_0_1_2_19_u_u_u_u_u_u_u_u_u_u_u_u:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vmovdqa32 {{.*#+}} zmm2 = <0,1,2,19,u,u,u,u,u,u,u,u,u,u,u,u>
-; ALL-NEXT:    vpermt2d %zmm1, %zmm2, %zmm0
+; ALL-NEXT:    movw $8, %ax
+; ALL-NEXT:    kmovw %eax, %k1
+; ALL-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
+; ALL-NEXT:    vmovdqa64 %zmm1, %zmm0
 ; ALL-NEXT:    retq
   %c = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 19, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
   ret <16 x i32> %c
 }
 
 define <8 x float> @shuffle_v16f32_extract_256(float* %RET, float* %a) {
 ; ALL-LABEL: shuffle_v16f32_extract_256:
 ; ALL:       # BB#0:

test/CodeGen/X86/vector-shuffle-512-v32.ll

 ; ALL-NEXT:    retq
   %c = shufflevector <32 x i16> %a, <32 x i16> zeroinitializer, <32 x i32> <i32 1, i32 1, i32 0, i32 0, i32 5, i32 5, i32 4, i32 4, i32 9, i32 9, i32 8, i32 8, i32 13, i32 13, i32 12, i32 12, i32 17, i32 17, i32 16, i32 16, i32 21, i32 21, i32 20, i32 20, i32 25, i32 25, i32 24, i32 24, i32 29, i32 29, i32 28, i32 28>
   ret <32 x i16> %c
 }
 
 define <32 x i16> @shuffle_v32i16_0zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz(<32 x i16> %a) {
 ; ALL-LABEL: shuffle_v32i16_0zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz:
 ; ALL:       # BB#0:
-; ALL-NEXT:    vmovdqu16 {{.*#+}} zmm2 = [32,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31]
-; ALL-NEXT:    vpxord %zmm1, %zmm1, %zmm1
-; ALL-NEXT:    vpermt2w %zmm0, %zmm2, %zmm1
-; ALL-NEXT:    vmovdqa64 %zmm1, %zmm0
+; ALL-NEXT:    movl $1, %eax
+; ALL-NEXT:    kmovd  %eax, %k0
+; ALL-NEXT:    knotd  %k0, %k1
+; ALL-NEXT:    vmovdqu16  %zmm0, %zmm0 {%k1} {z}
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <32 x i16> %a, <32 x i16> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>
   ret <32 x i16> %shuffle
 }
 
 define <32 x i16> @insert_dup_mem_v32i16_i32(i32* %ptr) {
 ; ALL-LABEL: insert_dup_mem_v32i16_i32:
 ; ALL:       # BB#0:

test/CodeGen/X86/vector-shuffle-512-v8.ll

; AVX512F-32-NEXT:    retl
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1>
   ret <8 x i64> %shuffle
 }
 
 define <8 x i64> @shuffle_v8i64_81a3c5e7(<8 x i64> %a, <8 x i64> %b) {
 ;
 ; AVX512F-LABEL: shuffle_v8i64_81a3c5e7:
 ; AVX512F:       # BB#0:
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,9,2,11,4,13,6,15]
-; AVX512F-NEXT:    vpermi2q %zmm0, %zmm1, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512F-NEXT:    retq
+; AVX512F-NEXT:   movb $-86, %al
+; AVX512F-NEXT:   kmovw %eax, %k1
+; AVX512F-NEXT:   vmovdqa64 %zmm1, %zmm0 {%k1}
+; AVX512F-NEXT:   retq  
 ;
+;
 ; AVX512F-32-LABEL: shuffle_v8i64_81a3c5e7:
 ; AVX512F-32:       # BB#0:
-; AVX512F-32-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,0,9,0,2,0,11,0,4,0,13,0,6,0,15,0]
-; AVX512F-32-NEXT:    vpermi2q %zmm0, %zmm1, %zmm2
-; AVX512F-32-NEXT:    vmovdqa64 %zmm2, %zmm0
+; AVX512F-32-NEXT:    movb $-86, %al
+; AVX512F-32-NEXT:    kmovw %eax, %k1
+; AVX512F-32-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
 ; AVX512F-32-NEXT:    retl
   %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 8, i32 1, i32 10, i32 3, i32 12, i32 5, i32 14, i32 7>
   ret <8 x i64> %shuffle
 }
 
 define <8 x i64> @shuffle_v8i64_08080808(<8 x i64> %a, <8 x i64> %b) {
 ;
 ; AVX512F-LABEL: shuffle_v8i64_08080808:

test/CodeGen/X86/vector-shuffle-to-blend-avx512.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skx | FileCheck %s --check-prefix=SKX
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=knl | FileCheck %s --check-prefix=KNL
+
+define <64 x i8> @test_mm512_mask_blend_epi8(<64 x i8> %A, <64 x i8> %W){
+; SKX-LABEL: test_mm512_mask_blend_epi8:
+; SKX:       # BB#0: # %entry
+; SKX-NEXT:    movl $2863311530, %eax # imm = 0xAAAAAAAA
+; SKX-NEXT:    kmovq %rax, %k1
+; SKX-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
+; SKX-NEXT:    retq
+;
+; KNL-LABEL: test_mm512_mask_blend_epi8:
+; KNL:       # BB#0: # %entry
+; KNL-NEXT:    vpbroadcastw {{.*}}(%rip), %ymm4
+; KNL-NEXT:    vpblendvb %ymm4, %ymm2, %ymm0, %ymm0
+; KNL-NEXT:    vpblendvb %ymm4, %ymm3, %ymm1, %ymm1
+; KNL-NEXT:    retq
+entry:
+  %0 = shufflevector <64 x i8> %A, <64 x i8> %W, <64 x i32>  <i32 64, i32 1, i32 66, i32 3, i32 68, i32 5, i32 70, i32 7, i32 72, i32 9, i32 74, i32 11, i32 76, i32 13, i32 78, i32 15, i32 80, i32 17, i32 82, i32 19, i32 84, i32 21, i32 86, i32 23, i32 88, i32 25, i32 90, i32 27, i32 92, i32 29, i32 94, i32 31, i32 96, i32 33, i32 98, i32 35, i32 100, i32 37, i32 102, i32 39, i32 104, i32 41, i32 106, i32 43, i32 108, i32 45, i32 110, i32 47, i32 112, i32 49, i32 114, i32 51, i32 116, i32 53, i32 118, i32 55, i32 120, i32 57, i32 122, i32 59, i32 124, i32 61, i32 126, i32 63>
+  ret <64 x i8> %0
+}
+
+define <32 x i16> @test_mm512_mask_blend_epi16(<32 x i16> %A, <32 x i16> %W){
+; SKX-LABEL: test_mm512_mask_blend_epi16:
+; SKX:       # BB#0: # %entry
+; SKX-NEXT:    movl $-1431655766, %eax # imm = 0xAAAAAAAA
+; SKX-NEXT:    kmovd %eax, %k1
+; SKX-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k1}
+; SKX-NEXT:    retq
+;
+; KNL-LABEL: test_mm512_mask_blend_epi16:
+; KNL:       # BB#0: # %entry
+; KNL-NEXT:    vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2],ymm0[3],ymm2[4],ymm0[5],ymm2[6],ymm0[7],ymm2[8],ymm0[9],ymm2[10],ymm0[11],ymm2[12],ymm0[13],ymm2[14],ymm0[15]
+; KNL-NEXT:    vpblendw {{.*#+}} ymm1 = ymm3[0],ymm1[1],ymm3[2],ymm1[3],ymm3[4],ymm1[5],ymm3[6],ymm1[7],ymm3[8],ymm1[9],ymm3[10],ymm1[11],ymm3[12],ymm1[13],ymm3[14],ymm1[15]
+; KNL-NEXT:    retq
+entry:
+  %0 = shufflevector <32 x i16> %A, <32 x i16> %W, <32 x i32>  <i32 32, i32 1, i32 34, i32 3, i32 36, i32 5, i32 38, i32 7, i32 40, i32 9, i32 42, i32 11, i32 44, i32 13, i32 46, i32 15, i32 48, i32 17, i32 50, i32 19, i32 52, i32 21, i32 54, i32 23, i32 56, i32 25, i32 58, i32 27, i32 60, i32 29, i32 62, i32 31>
+  ret <32 x i16> %0
+}
+
+define <16 x i32> @test_mm512_mask_blend_epi32(<16 x i32> %A, <16 x i32> %W){
+; SKX-LABEL: test_mm512_mask_blend_epi32:
+; SKX:       # BB#0: # %entry
+; SKX-NEXT:    movw $-21846, %ax # imm = 0xAAAA
+; SKX-NEXT:    kmovw %eax, %k1
+; SKX-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
+; SKX-NEXT:    retq
+;
+; KNL-LABEL: test_mm512_mask_blend_epi32:
+; KNL:       # BB#0: # %entry
+; KNL-NEXT:    movw $-21846, %ax # imm = 0xAAAA
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
+; KNL-NEXT:    retq
+entry:
+  %0 = shufflevector <16 x i32> %A, <16 x i32> %W, <16 x i32>  <i32 16, i32 1, i32 18, i32 3, i32 20, i32 5, i32 22, i32 7, i32 24, i32 9, i32 26, i32 11, i32 28, i32 13, i32 30, i32 15>
+  ret <16 x i32> %0
+}
+
+define <8 x i64> @test_mm512_mask_blend_epi64(<8 x i64> %A, <8 x i64> %W){
+; SKX-LABEL: test_mm512_mask_blend_epi64:
+; SKX:       # BB#0: # %entry
+; SKX-NEXT:    movb $-86, %al
+; SKX-NEXT:    kmovb %eax, %k1
+; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
+; SKX-NEXT:    retq
+;
+; KNL-LABEL: test_mm512_mask_blend_epi64:
+; KNL:       # BB#0: # %entry
+; KNL-NEXT:    movb $-86, %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
+; KNL-NEXT:    retq
+entry:
+  %0 = shufflevector <8 x i64> %A, <8 x i64> %W, <8 x i32>  <i32 8, i32 1, i32 10, i32 3, i32 12, i32 5, i32 14, i32 7>
+  ret <8 x i64> %0
+}
+
+define <16 x float> @test_mm512_mask_blend_ps(<16 x float> %A, <16 x float> %W){
+; SKX-LABEL: test_mm512_mask_blend_ps:
+; SKX:       # BB#0: # %entry
+; SKX-NEXT:    movw $-21846, %ax # imm = 0xAAAA
+; SKX-NEXT:    kmovw %eax, %k1
+; SKX-NEXT:    vmovaps %zmm1, %zmm0 {%k1}
+; SKX-NEXT:    retq
+;
+; KNL-LABEL: test_mm512_mask_blend_ps:
+; KNL:       # BB#0: # %entry
+; KNL-NEXT:    movw $-21846, %ax # imm = 0xAAAA
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    vmovaps %zmm1, %zmm0 {%k1}
+; KNL-NEXT:    retq
+entry:
+  %0 = shufflevector <16 x float> %A, <16 x float> %W, <16 x i32>  <i32 16, i32 1, i32 18, i32 3, i32 20, i32 5, i32 22, i32 7, i32 24, i32 9, i32 26, i32 11, i32 28, i32 13, i32 30, i32 15>
+  ret <16 x float> %0
+}
+
+define <8 x double> @test_mm512_mask_blend_pd(<8 x double> %A, <8 x double> %W){
+; SKX-LABEL: test_mm512_mask_blend_pd:
+; SKX:       # BB#0: # %entry
+; SKX-NEXT:    movb $-88, %al
+; SKX-NEXT:    kmovb %eax, %k1
+; SKX-NEXT:    vmovapd %zmm1, %zmm0 {%k1}
+; SKX-NEXT:    retq
+;
+; KNL-LABEL: test_mm512_mask_blend_pd:
+; KNL:       # BB#0: # %entry
+; KNL-NEXT:    movb $-88, %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    vmovapd %zmm1, %zmm0 {%k1}
+; KNL-NEXT:    retq
+entry:
+  %0 = shufflevector <8 x double> %A, <8 x double> %W, <8 x i32>  <i32 8, i32 9, i32 10, i32 3, i32 12, i32 5, i32 14, i32 7>
+  ret <8 x double> %0
+}