[X86] Add InstFixup for masked `unpck{l|h}pd` -> masked `shufpd`

[goldstein.w.n]

Summary

This is a follow up D147507 which removed the prior transformation to
shufps which was incorrect as the mask was for 64-bit double
elements, not 32-bit float elements. Using shufpd for the
replacement, however, preserves the mask semantics and has the same
benefits as shufps.

[pengfei]

LGTM.

llvm/lib/Target/X86/X86FixupInstTuning.cpp

250–255

No sure if we should use VSHUFPD for them too.

[RKSimon]

LGTM

llvm/lib/Target/X86/X86FixupInstTuning.cpp

250–255

+1

[goldstein.w.n]

shufps -> shufpd

[skatkov]

llvm/lib/Target/X86/X86FixupInstTuning.cpp

157

Hello Noah, we've got downstream mis-compile on fuzzer testing on this patch.
Specifically reverting constants 0x00 -> 0x44 and 0xFF -> 0xee fixes the issue.

While we do not have IR reproducer for this problem at the moment, may be you would be able to detect the problem from source code?
At least I see that this comment states that for r,r case we should use old constants and for r,r,k new ones.
However it looks like the patch updates more cases. Is it correct?
It just is wild guess as I'm not familiar with this code.

[pengfei]

llvm/lib/Target/X86/X86FixupInstTuning.cpp

240–247

Guess problem may come from here.

[skatkov]

If it helps. the bad and good compilations differ only in one line:

vshufps	$0, %xmm0, %xmm1, %xmm0         # xmm0 = xmm1[0,0],xmm0[0,0]

vs

vshufps	$68, %xmm0, %xmm1, %xmm0        # xmm0 = xmm1[0,1],xmm0[0,1]

So probably the problem comes from case which have not been updated but uses a lambda which is updated?

[pengfei]

I put D147775 for example, but think we can modify it together with D147728.

[aleksandr.popov]

@pengfei D147775 works, thanks!

[pengfei]

In D147541#4251040, @aleksandr.popov wrote:

@pengfei D147775 works, thanks!

Thanks for the confirmation! Then I'd like to land it as a quick fix.

llvm/lib/Target/X86/X86FixupInstTuning.cpp

auto ProcessVPERMILPSmi = [&](unsigned NewOpc) -> bool {
     // `vpshufd` saves a byte of code size.
     if (!ST->hasNoDomainDelayShuffle() &&
         !NewOpcPreferable(NewOpc, /*ReplaceInTie*/ false))
       return false;
     MI.setDesc(TII->get(NewOpc));
     return true;
   };
 
   // `vunpcklpd/vmovlhps r, r` -> `vshufps r, r, 0x44`
   // `vunpckhpd/vmovlhps r, r` -> `vshufps r, r, 0xee`
+  // `vunpcklpd r, r, k` -> `vshufpd r, r, 0x00`
+  // `vunpckhpd r, r, k` -> `vshufpd r, r, 0xff`
   // iff `vshufps` is faster than `vunpck{l|h}pd`. Otherwise stick with
   // `vunpck{l|h}pd` as it uses less code size.
   // TODO: Look into using `{VP}UNPCK{L|H}QDQ{...}` instead of `{V}SHUF{...}PS`
   // as the replacement. `{VP}UNPCK{L|H}QDQ{...}` has no codesize cost.
   auto ProcessUNPCKPD = [&](unsigned NewOpc, unsigned MaskImm) -> bool {
     if (!NewOpcPreferable(NewOpc, /*ReplaceInTie*/ false))
       return false;
 
     MI.setDesc(TII->get(NewOpc));
     MI.addOperand(MachineOperand::CreateImm(MaskImm));
     return true;
   };
   auto ProcessUNPCKLPDrr = [&](unsigned NewOpc) -> bool {
     return ProcessUNPCKPD(NewOpc, 0x44);
   };
   auto ProcessUNPCKHPDrr = [&](unsigned NewOpc) -> bool {
     return ProcessUNPCKPD(NewOpc, 0xee);
   };
+  auto ProcessUNPCKLPDrrk = [&](unsigned NewOpc) -> bool {
+    return ProcessUNPCKPD(NewOpc, 0x00);
+  };
+  auto ProcessUNPCKHPDrrk = [&](unsigned NewOpc) -> bool {
+    return ProcessUNPCKPD(NewOpc, 0xff);
+  };
 
   switch (Opc) {
   case X86::VPERMILPSri:
     return ProcessVPERMILPSri(X86::VSHUFPSrri);
   case X86::VPERMILPSYri:
     return ProcessVPERMILPSri(X86::VSHUFPSYrri);
   case X86::VPERMILPSZ128ri:
     return ProcessVPERMILPSri(X86::VSHUFPSZ128rri);
case X86::VPERMILPSZmik:
     return ProcessVPERMILPSmi(X86::VPSHUFDZmik);
 
     // TODO: {V}UNPCK{L|H}PD{...} is probably safe to transform to
     // `{VP}UNPCK{L|H}QDQ{...}` which gets the same perf benefit as
     // `{V}SHUF{...}PS` but 1) without increasing code size and 2) can also
     // handle the `mr` case. ICL doesn't have a domain penalty for replacing
     // float unpck -> int unpck, but at this time, I haven't verified the set of
     // processors where its safe.
   case X86::MOVLHPSrr:
   case X86::UNPCKLPDrr:
     return ProcessUNPCKLPDrr(X86::SHUFPSrri);
   case X86::VMOVLHPSrr:
   case X86::VUNPCKLPDrr:
     return ProcessUNPCKLPDrr(X86::VSHUFPSrri);
   case X86::VUNPCKLPDYrr:
     return ProcessUNPCKLPDrr(X86::VSHUFPSYrri);
     // VMOVLHPS is always 128 bits.
   case X86::VMOVLHPSZrr:
   case X86::VUNPCKLPDZ128rr:
     return ProcessUNPCKLPDrr(X86::VSHUFPSZ128rri);
   case X86::VUNPCKLPDZ256rr:
     return ProcessUNPCKLPDrr(X86::VSHUFPSZ256rri);
   case X86::VUNPCKLPDZrr:
     return ProcessUNPCKLPDrr(X86::VSHUFPSZrri);
+  case X86::VUNPCKLPDZ128rrk:
+    return ProcessUNPCKLPDrrk(X86::VSHUFPDZ128rrik);
+  case X86::VUNPCKLPDZ256rrk:
+    return ProcessUNPCKLPDrrk(X86::VSHUFPDZ256rrik);
+  case X86::VUNPCKLPDZrrk:
+    return ProcessUNPCKLPDrrk(X86::VSHUFPDZrrik);
+  case X86::VUNPCKLPDZ128rrkz:
+    return ProcessUNPCKLPDrrk(X86::VSHUFPDZ128rrikz);
+  case X86::VUNPCKLPDZ256rrkz:
+    return ProcessUNPCKLPDrrk(X86::VSHUFPDZ256rrikz);
+  case X86::VUNPCKLPDZrrkz:
+    return ProcessUNPCKLPDrrk(X86::VSHUFPDZrrikz);
   case X86::UNPCKHPDrr:
     return ProcessUNPCKHPDrr(X86::SHUFPSrri);
   case X86::VUNPCKHPDrr:
     return ProcessUNPCKHPDrr(X86::VSHUFPSrri);
   case X86::VUNPCKHPDYrr:
     return ProcessUNPCKHPDrr(X86::VSHUFPSYrri);
   case X86::VUNPCKHPDZ128rr:
     return ProcessUNPCKHPDrr(X86::VSHUFPSZ128rri);
   case X86::VUNPCKHPDZ256rr:
     return ProcessUNPCKHPDrr(X86::VSHUFPSZ256rri);
   case X86::VUNPCKHPDZrr:
     return ProcessUNPCKHPDrr(X86::VSHUFPSZrri);
+  case X86::VUNPCKHPDZ128rrk:
+    return ProcessUNPCKHPDrrk(X86::VSHUFPDZ128rrik);
+  case X86::VUNPCKHPDZ256rrk:
+    return ProcessUNPCKHPDrrk(X86::VSHUFPDZ256rrik);
+  case X86::VUNPCKHPDZrrk:
+    return ProcessUNPCKHPDrrk(X86::VSHUFPDZrrik);
+  case X86::VUNPCKHPDZ128rrkz:
+    return ProcessUNPCKHPDrrk(X86::VSHUFPDZ128rrikz);
+  case X86::VUNPCKHPDZ256rrkz:
+    return ProcessUNPCKHPDrrk(X86::VSHUFPDZ256rrikz);
+  case X86::VUNPCKHPDZrrkz:
+    return ProcessUNPCKHPDrrk(X86::VSHUFPDZrrikz);
   default:
     return false;
   }
 }
 
 bool X86FixupInstTuningPass::runOnMachineFunction(MachineFunction &MF) {
   LLVM_DEBUG(dbgs() << "Start X86FixupInstTuning\n";);
   bool Changed = false;

llvm/test/CodeGen/X86/tuning-shuffle-unpckpd-avx512.ll

 ; CHECK-NEXT:    vunpckhpd {{.*#+}} zmm0 {%k1} {z} = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7]
 ; CHECK-NEXT:    retq
   %mask = bitcast i8 %mask_int to <8 x i1>
   %shufp = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
   %res = select <8 x i1> %mask, <8 x double> %shufp, <8 x double> zeroinitializer
   ret <8 x double> %res
 }
 
-; Check that masked vunpcklpd will not be transformed into vshufps.
 define <4 x double> @transform_VUNPCKLPDYrrkz(<4 x double> %a, <4 x double> %b, i4 %mask_int) nounwind {
-; CHECK-LABEL: transform_VUNPCKLPDYrrkz:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    kmovd %edi, %k1
-; CHECK-NEXT:    vunpcklpd {{.*#+}} ymm0 {%k1} {z} = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
-; CHECK-NEXT:    retq
+; CHECK-SKX-LABEL: transform_VUNPCKLPDYrrkz:
+; CHECK-SKX:       # %bb.0:
+; CHECK-SKX-NEXT:    kmovd %edi, %k1
+; CHECK-SKX-NEXT:    vunpcklpd {{.*#+}} ymm0 {%k1} {z} = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
+; CHECK-SKX-NEXT:    retq
+;
+; CHECK-ICX-LABEL: transform_VUNPCKLPDYrrkz:
+; CHECK-ICX:       # %bb.0:
+; CHECK-ICX-NEXT:    kmovd %edi, %k1
+; CHECK-ICX-NEXT:    vshufpd {{.*#+}} ymm0 {%k1} {z} = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
+; CHECK-ICX-NEXT:    retq
+;
+; CHECK-V4-LABEL: transform_VUNPCKLPDYrrkz:
+; CHECK-V4:       # %bb.0:
+; CHECK-V4-NEXT:    kmovd %edi, %k1
+; CHECK-V4-NEXT:    vunpcklpd {{.*#+}} ymm0 {%k1} {z} = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
+; CHECK-V4-NEXT:    retq
+;
+; CHECK-AVX512-LABEL: transform_VUNPCKLPDYrrkz:
+; CHECK-AVX512:       # %bb.0:
+; CHECK-AVX512-NEXT:    kmovd %edi, %k1
+; CHECK-AVX512-NEXT:    vunpcklpd {{.*#+}} ymm0 {%k1} {z} = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
+; CHECK-AVX512-NEXT:    retq
+;
+; CHECK-ZNVER4-LABEL: transform_VUNPCKLPDYrrkz:
+; CHECK-ZNVER4:       # %bb.0:
+; CHECK-ZNVER4-NEXT:    kmovd %edi, %k1
+; CHECK-ZNVER4-NEXT:    vunpcklpd {{.*#+}} ymm0 {%k1} {z} = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
+; CHECK-ZNVER4-NEXT:    retq
   %mask = bitcast i4 %mask_int to <4 x i1>
   %shufp = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
   %res = select <4 x i1> %mask, <4 x double> %shufp, <4 x double> zeroinitializer
   ret <4 x double> %res
 }
 
-; Check that masked vunpcklpd will not be transformed into vshufps.
 define <4 x double> @transform_VUNPCKHPDYrrkz(<4 x double> %a, <4 x double> %b, i4 %mask_int) nounwind {
-; CHECK-LABEL: transform_VUNPCKHPDYrrkz:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    kmovd %edi, %k1
-; CHECK-NEXT:    vunpckhpd {{.*#+}} ymm0 {%k1} {z} = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
-; CHECK-NEXT:    retq
+; CHECK-SKX-LABEL: transform_VUNPCKHPDYrrkz:
+; CHECK-SKX:       # %bb.0:
+; CHECK-SKX-NEXT:    kmovd %edi, %k1
+; CHECK-SKX-NEXT:    vunpckhpd {{.*#+}} ymm0 {%k1} {z} = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
+; CHECK-SKX-NEXT:    retq
+;
+; CHECK-ICX-LABEL: transform_VUNPCKHPDYrrkz:
+; CHECK-ICX:       # %bb.0:
+; CHECK-ICX-NEXT:    kmovd %edi, %k1
+; CHECK-ICX-NEXT:    vshufpd {{.*#+}} ymm0 {%k1} {z} = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
+; CHECK-ICX-NEXT:    retq
+;
+; CHECK-V4-LABEL: transform_VUNPCKHPDYrrkz:
+; CHECK-V4:       # %bb.0:
+; CHECK-V4-NEXT:    kmovd %edi, %k1
+; CHECK-V4-NEXT:    vunpckhpd {{.*#+}} ymm0 {%k1} {z} = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
+; CHECK-V4-NEXT:    retq
+;
+; CHECK-AVX512-LABEL: transform_VUNPCKHPDYrrkz:
+; CHECK-AVX512:       # %bb.0:
+; CHECK-AVX512-NEXT:    kmovd %edi, %k1
+; CHECK-AVX512-NEXT:    vunpckhpd {{.*#+}} ymm0 {%k1} {z} = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
+; CHECK-AVX512-NEXT:    retq
+;
+; CHECK-ZNVER4-LABEL: transform_VUNPCKHPDYrrkz:
+; CHECK-ZNVER4:       # %bb.0:
+; CHECK-ZNVER4-NEXT:    kmovd %edi, %k1
+; CHECK-ZNVER4-NEXT:    vunpckhpd {{.*#+}} ymm0 {%k1} {z} = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
+; CHECK-ZNVER4-NEXT:    retq
   %mask = bitcast i4 %mask_int to <4 x i1>
   %shufp = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
   %res = select <4 x i1> %mask, <4 x double> %shufp, <4 x double> zeroinitializer
   ret <4 x double> %res
 }
 
-; Check that masked vunpcklpd will not be transformed into vshufps.
 define <2 x double> @transform_VUNPCKLPDrrkz(<2 x double> %a, <2 x double> %b, i2 %mask_int) nounwind {
-; CHECK-LABEL: transform_VUNPCKLPDrrkz:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    kmovd %edi, %k1
-; CHECK-NEXT:    vunpcklpd {{.*#+}} xmm0 {%k1} {z} = xmm0[0],xmm1[0]
-; CHECK-NEXT:    retq
+; CHECK-SKX-LABEL: transform_VUNPCKLPDrrkz:
+; CHECK-SKX:       # %bb.0:
+; CHECK-SKX-NEXT:    kmovd %edi, %k1
+; CHECK-SKX-NEXT:    vunpcklpd {{.*#+}} xmm0 {%k1} {z} = xmm0[0],xmm1[0]
+; CHECK-SKX-NEXT:    retq
+;
+; CHECK-ICX-LABEL: transform_VUNPCKLPDrrkz:
+; CHECK-ICX:       # %bb.0:
+; CHECK-ICX-NEXT:    kmovd %edi, %k1
+; CHECK-ICX-NEXT:    vshufpd {{.*#+}} xmm0 {%k1} {z} = xmm0[0],xmm1[0]
+; CHECK-ICX-NEXT:    retq
+;
+; CHECK-V4-LABEL: transform_VUNPCKLPDrrkz:
+; CHECK-V4:       # %bb.0:
+; CHECK-V4-NEXT:    kmovd %edi, %k1
+; CHECK-V4-NEXT:    vunpcklpd {{.*#+}} xmm0 {%k1} {z} = xmm0[0],xmm1[0]
+; CHECK-V4-NEXT:    retq
+;
+; CHECK-AVX512-LABEL: transform_VUNPCKLPDrrkz:
+; CHECK-AVX512:       # %bb.0:
+; CHECK-AVX512-NEXT:    kmovd %edi, %k1
+; CHECK-AVX512-NEXT:    vunpcklpd {{.*#+}} xmm0 {%k1} {z} = xmm0[0],xmm1[0]
+; CHECK-AVX512-NEXT:    retq
+;
+; CHECK-ZNVER4-LABEL: transform_VUNPCKLPDrrkz:
+; CHECK-ZNVER4:       # %bb.0:
+; CHECK-ZNVER4-NEXT:    kmovd %edi, %k1
+; CHECK-ZNVER4-NEXT:    vunpcklpd {{.*#+}} xmm0 {%k1} {z} = xmm0[0],xmm1[0]
+; CHECK-ZNVER4-NEXT:    retq
   %mask = bitcast i2 %mask_int to <2 x i1>
   %shufp = shufflevector <2 x double> %a, <2 x double> %b, <2 x i32> <i32 0, i32 2>
   %res = select <2 x i1> %mask, <2 x double> %shufp, <2 x double> zeroinitializer
   ret <2 x double> %res
 }
 
-; Check that masked vunpcklpd will not be transformed into vshufps.
 define <2 x double> @transform_VUNPCKHPDrrkz(<2 x double> %a, <2 x double> %b, i2 %mask_int) nounwind {
-; CHECK-LABEL: transform_VUNPCKHPDrrkz:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    kmovd %edi, %k1
-; CHECK-NEXT:    vunpckhpd {{.*#+}} xmm0 {%k1} {z} = xmm0[1],xmm1[1]
-; CHECK-NEXT:    retq
+; CHECK-SKX-LABEL: transform_VUNPCKHPDrrkz:
+; CHECK-SKX:       # %bb.0:
+; CHECK-SKX-NEXT:    kmovd %edi, %k1
+; CHECK-SKX-NEXT:    vunpckhpd {{.*#+}} xmm0 {%k1} {z} = xmm0[1],xmm1[1]
+; CHECK-SKX-NEXT:    retq
+;
+; CHECK-ICX-LABEL: transform_VUNPCKHPDrrkz:
+; CHECK-ICX:       # %bb.0:
+; CHECK-ICX-NEXT:    kmovd %edi, %k1
+; CHECK-ICX-NEXT:    vshufpd {{.*#+}} xmm0 {%k1} {z} = xmm0[1],xmm1[1]
+; CHECK-ICX-NEXT:    retq
+;
+; CHECK-V4-LABEL: transform_VUNPCKHPDrrkz:
+; CHECK-V4:       # %bb.0:
+; CHECK-V4-NEXT:    kmovd %edi, %k1
+; CHECK-V4-NEXT:    vunpckhpd {{.*#+}} xmm0 {%k1} {z} = xmm0[1],xmm1[1]
+; CHECK-V4-NEXT:    retq
+;
+; CHECK-AVX512-LABEL: transform_VUNPCKHPDrrkz:
+; CHECK-AVX512:       # %bb.0:
+; CHECK-AVX512-NEXT:    kmovd %edi, %k1
+; CHECK-AVX512-NEXT:    vunpckhpd {{.*#+}} xmm0 {%k1} {z} = xmm0[1],xmm1[1]
+; CHECK-AVX512-NEXT:    retq
+;
+; CHECK-ZNVER4-LABEL: transform_VUNPCKHPDrrkz:
+; CHECK-ZNVER4:       # %bb.0:
+; CHECK-ZNVER4-NEXT:    kmovd %edi, %k1
+; CHECK-ZNVER4-NEXT:    vunpckhpd {{.*#+}} xmm0 {%k1} {z} = xmm0[1],xmm1[1]
+; CHECK-ZNVER4-NEXT:    retq
   %mask = bitcast i2 %mask_int to <2 x i1>
   %shufp = shufflevector <2 x double> %a, <2 x double> %b, <2 x i32> <i32 1, i32 3>
   %res = select <2 x i1> %mask, <2 x double> %shufp, <2 x double> zeroinitializer
   ret <2 x double> %res
 }
 
 define <8 x double> @transform_VUNPCKLPDZrrk(<8 x double> %a, <8 x double> %b, <8 x double> %c, i8 %mask_int) nounwind {
 ; CHECK-LABEL: transform_VUNPCKLPDZrrk:

 ; CHECK-NEXT:    vmovapd %zmm2, %zmm0
 ; CHECK-NEXT:    retq
   %mask = bitcast i8 %mask_int to <8 x i1>
   %shufp = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
   %res = select <8 x i1> %mask, <8 x double> %shufp, <8 x double> %c
   ret <8 x double> %res
 }
 
-; Check that masked vunpcklpd will not be transformed into vshufps.
 define <4 x double> @transform_VUNPCKLPDYrrk(<4 x double> %a, <4 x double> %b, <4 x double> %c, i4 %mask_int) nounwind {
-; CHECK-LABEL: transform_VUNPCKLPDYrrk:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    kmovd %edi, %k1
-; CHECK-NEXT:    vunpcklpd {{.*#+}} ymm2 {%k1} = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
-; CHECK-NEXT:    vmovapd %ymm2, %ymm0
-; CHECK-NEXT:    retq
+; CHECK-SKX-LABEL: transform_VUNPCKLPDYrrk:
+; CHECK-SKX:       # %bb.0:
+; CHECK-SKX-NEXT:    kmovd %edi, %k1
+; CHECK-SKX-NEXT:    vunpcklpd {{.*#+}} ymm2 {%k1} = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
+; CHECK-SKX-NEXT:    vmovapd %ymm2, %ymm0
+; CHECK-SKX-NEXT:    retq
+;
+; CHECK-ICX-LABEL: transform_VUNPCKLPDYrrk:
+; CHECK-ICX:       # %bb.0:
+; CHECK-ICX-NEXT:    kmovd %edi, %k1
+; CHECK-ICX-NEXT:    vshufpd {{.*#+}} ymm2 {%k1} = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
+; CHECK-ICX-NEXT:    vmovapd %ymm2, %ymm0
+; CHECK-ICX-NEXT:    retq
+;
+; CHECK-V4-LABEL: transform_VUNPCKLPDYrrk:
+; CHECK-V4:       # %bb.0:
+; CHECK-V4-NEXT:    kmovd %edi, %k1
+; CHECK-V4-NEXT:    vunpcklpd {{.*#+}} ymm2 {%k1} = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
+; CHECK-V4-NEXT:    vmovapd %ymm2, %ymm0
+; CHECK-V4-NEXT:    retq
+;
+; CHECK-AVX512-LABEL: transform_VUNPCKLPDYrrk:
+; CHECK-AVX512:       # %bb.0:
+; CHECK-AVX512-NEXT:    kmovd %edi, %k1
+; CHECK-AVX512-NEXT:    vunpcklpd {{.*#+}} ymm2 {%k1} = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
+; CHECK-AVX512-NEXT:    vmovapd %ymm2, %ymm0
+; CHECK-AVX512-NEXT:    retq
+;
+; CHECK-ZNVER4-LABEL: transform_VUNPCKLPDYrrk:
+; CHECK-ZNVER4:       # %bb.0:
+; CHECK-ZNVER4-NEXT:    kmovd %edi, %k1
+; CHECK-ZNVER4-NEXT:    vunpcklpd {{.*#+}} ymm2 {%k1} = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
+; CHECK-ZNVER4-NEXT:    vmovapd %ymm2, %ymm0
+; CHECK-ZNVER4-NEXT:    retq
   %mask = bitcast i4 %mask_int to <4 x i1>
   %shufp = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
   %res = select <4 x i1> %mask, <4 x double> %shufp, <4 x double> %c
   ret <4 x double> %res
 }
 
-; Check that masked vunpcklpd will not be transformed into vshufps.
 define <4 x double> @transform_VUNPCKHPDYrrk(<4 x double> %a, <4 x double> %b, <4 x double> %c, i4 %mask_int) nounwind {
-; CHECK-LABEL: transform_VUNPCKHPDYrrk:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    kmovd %edi, %k1
-; CHECK-NEXT:    vunpckhpd {{.*#+}} ymm2 {%k1} = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
-; CHECK-NEXT:    vmovapd %ymm2, %ymm0
-; CHECK-NEXT:    retq
+; CHECK-SKX-LABEL: transform_VUNPCKHPDYrrk:
+; CHECK-SKX:       # %bb.0:
+; CHECK-SKX-NEXT:    kmovd %edi, %k1
+; CHECK-SKX-NEXT:    vunpckhpd {{.*#+}} ymm2 {%k1} = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
+; CHECK-SKX-NEXT:    vmovapd %ymm2, %ymm0
+; CHECK-SKX-NEXT:    retq
+;
+; CHECK-ICX-LABEL: transform_VUNPCKHPDYrrk:
+; CHECK-ICX:       # %bb.0:
+; CHECK-ICX-NEXT:    kmovd %edi, %k1
+; CHECK-ICX-NEXT:    vshufpd {{.*#+}} ymm2 {%k1} = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
+; CHECK-ICX-NEXT:    vmovapd %ymm2, %ymm0
+; CHECK-ICX-NEXT:    retq
+;
+; CHECK-V4-LABEL: transform_VUNPCKHPDYrrk:
+; CHECK-V4:       # %bb.0:
+; CHECK-V4-NEXT:    kmovd %edi, %k1
+; CHECK-V4-NEXT:    vunpckhpd {{.*#+}} ymm2 {%k1} = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
+; CHECK-V4-NEXT:    vmovapd %ymm2, %ymm0
+; CHECK-V4-NEXT:    retq
+;
+; CHECK-AVX512-LABEL: transform_VUNPCKHPDYrrk:
+; CHECK-AVX512:       # %bb.0:
+; CHECK-AVX512-NEXT:    kmovd %edi, %k1
+; CHECK-AVX512-NEXT:    vunpckhpd {{.*#+}} ymm2 {%k1} = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
+; CHECK-AVX512-NEXT:    vmovapd %ymm2, %ymm0
+; CHECK-AVX512-NEXT:    retq
+;
+; CHECK-ZNVER4-LABEL: transform_VUNPCKHPDYrrk:
+; CHECK-ZNVER4:       # %bb.0:
+; CHECK-ZNVER4-NEXT:    kmovd %edi, %k1
+; CHECK-ZNVER4-NEXT:    vunpckhpd {{.*#+}} ymm2 {%k1} = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
+; CHECK-ZNVER4-NEXT:    vmovapd %ymm2, %ymm0
+; CHECK-ZNVER4-NEXT:    retq
   %mask = bitcast i4 %mask_int to <4 x i1>
   %shufp = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
   %res = select <4 x i1> %mask, <4 x double> %shufp, <4 x double> %c
   ret <4 x double> %res
 }
 
-; Check that masked vunpcklpd will not be transformed into vshufps.
 define <2 x double> @transform_VUNPCKLPDrrk(<2 x double> %a, <2 x double> %b, <2 x double> %c, i2 %mask_int) nounwind {
-; CHECK-LABEL: transform_VUNPCKLPDrrk:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    kmovd %edi, %k1
-; CHECK-NEXT:    vunpcklpd {{.*#+}} xmm2 {%k1} = xmm0[0],xmm1[0]
-; CHECK-NEXT:    vmovapd %xmm2, %xmm0
-; CHECK-NEXT:    retq
+; CHECK-SKX-LABEL: transform_VUNPCKLPDrrk:
+; CHECK-SKX:       # %bb.0:
+; CHECK-SKX-NEXT:    kmovd %edi, %k1
+; CHECK-SKX-NEXT:    vunpcklpd {{.*#+}} xmm2 {%k1} = xmm0[0],xmm1[0]
+; CHECK-SKX-NEXT:    vmovapd %xmm2, %xmm0
+; CHECK-SKX-NEXT:    retq
+;
+; CHECK-ICX-LABEL: transform_VUNPCKLPDrrk:
+; CHECK-ICX:       # %bb.0:
+; CHECK-ICX-NEXT:    kmovd %edi, %k1
+; CHECK-ICX-NEXT:    vshufpd {{.*#+}} xmm2 {%k1} = xmm0[0],xmm1[0]
+; CHECK-ICX-NEXT:    vmovapd %xmm2, %xmm0
+; CHECK-ICX-NEXT:    retq
+;
+; CHECK-V4-LABEL: transform_VUNPCKLPDrrk:
+; CHECK-V4:       # %bb.0:
+; CHECK-V4-NEXT:    kmovd %edi, %k1
+; CHECK-V4-NEXT:    vunpcklpd {{.*#+}} xmm2 {%k1} = xmm0[0],xmm1[0]
+; CHECK-V4-NEXT:    vmovapd %xmm2, %xmm0
+; CHECK-V4-NEXT:    retq
+;
+; CHECK-AVX512-LABEL: transform_VUNPCKLPDrrk:
+; CHECK-AVX512:       # %bb.0:
+; CHECK-AVX512-NEXT:    kmovd %edi, %k1
+; CHECK-AVX512-NEXT:    vunpcklpd {{.*#+}} xmm2 {%k1} = xmm0[0],xmm1[0]
+; CHECK-AVX512-NEXT:    vmovapd %xmm2, %xmm0
+; CHECK-AVX512-NEXT:    retq
+;
+; CHECK-ZNVER4-LABEL: transform_VUNPCKLPDrrk:
+; CHECK-ZNVER4:       # %bb.0:
+; CHECK-ZNVER4-NEXT:    kmovd %edi, %k1
+; CHECK-ZNVER4-NEXT:    vunpcklpd {{.*#+}} xmm2 {%k1} = xmm0[0],xmm1[0]
+; CHECK-ZNVER4-NEXT:    vmovapd %xmm2, %xmm0
+; CHECK-ZNVER4-NEXT:    retq
   %mask = bitcast i2 %mask_int to <2 x i1>
   %shufp = shufflevector <2 x double> %a, <2 x double> %b, <2 x i32> <i32 0, i32 2>
   %res = select <2 x i1> %mask, <2 x double> %shufp, <2 x double> %c
   ret <2 x double> %res
 }
 
-; Check that masked vunpcklpd will not be transformed into vshufps.
 define <2 x double> @transform_VUNPCKHPDrrk(<2 x double> %a, <2 x double> %b, <2 x double> %c, i2 %mask_int) nounwind {
-; CHECK-LABEL: transform_VUNPCKHPDrrk:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    kmovd %edi, %k1
-; CHECK-NEXT:    vunpckhpd {{.*#+}} xmm2 {%k1} = xmm0[1],xmm1[1]
-; CHECK-NEXT:    vmovapd %xmm2, %xmm0
-; CHECK-NEXT:    retq
+; CHECK-SKX-LABEL: transform_VUNPCKHPDrrk:
+; CHECK-SKX:       # %bb.0:
+; CHECK-SKX-NEXT:    kmovd %edi, %k1
+; CHECK-SKX-NEXT:    vunpckhpd {{.*#+}} xmm2 {%k1} = xmm0[1],xmm1[1]
+; CHECK-SKX-NEXT:    vmovapd %xmm2, %xmm0
+; CHECK-SKX-NEXT:    retq
+;
+; CHECK-ICX-LABEL: transform_VUNPCKHPDrrk:
+; CHECK-ICX:       # %bb.0:
+; CHECK-ICX-NEXT:    kmovd %edi, %k1
+; CHECK-ICX-NEXT:    vshufpd {{.*#+}} xmm2 {%k1} = xmm0[1],xmm1[1]
+; CHECK-ICX-NEXT:    vmovapd %xmm2, %xmm0
+; CHECK-ICX-NEXT:    retq
+;
+; CHECK-V4-LABEL: transform_VUNPCKHPDrrk:
+; CHECK-V4:       # %bb.0:
+; CHECK-V4-NEXT:    kmovd %edi, %k1
+; CHECK-V4-NEXT:    vunpckhpd {{.*#+}} xmm2 {%k1} = xmm0[1],xmm1[1]
+; CHECK-V4-NEXT:    vmovapd %xmm2, %xmm0
+; CHECK-V4-NEXT:    retq
+;
+; CHECK-AVX512-LABEL: transform_VUNPCKHPDrrk:
+; CHECK-AVX512:       # %bb.0:
+; CHECK-AVX512-NEXT:    kmovd %edi, %k1
+; CHECK-AVX512-NEXT:    vunpckhpd {{.*#+}} xmm2 {%k1} = xmm0[1],xmm1[1]
+; CHECK-AVX512-NEXT:    vmovapd %xmm2, %xmm0
+; CHECK-AVX512-NEXT:    retq
+;
+; CHECK-ZNVER4-LABEL: transform_VUNPCKHPDrrk:
+; CHECK-ZNVER4:       # %bb.0:
+; CHECK-ZNVER4-NEXT:    kmovd %edi, %k1
+; CHECK-ZNVER4-NEXT:    vunpckhpd {{.*#+}} xmm2 {%k1} = xmm0[1],xmm1[1]
+; CHECK-ZNVER4-NEXT:    vmovapd %xmm2, %xmm0
+; CHECK-ZNVER4-NEXT:    retq
   %mask = bitcast i2 %mask_int to <2 x i1>
   %shufp = shufflevector <2 x double> %a, <2 x double> %b, <2 x i32> <i32 1, i32 3>
   %res = select <2 x i1> %mask, <2 x double> %shufp, <2 x double> %c
   ret <2 x double> %res
 }
 
 define <16 x float> @transform_VUNPCKLPDZrm(<16 x float> %a, ptr %pb) nounwind {
 ; CHECK-LABEL: transform_VUNPCKLPDZrm: