AVX512 : kunpck encoding implementation

[igorb]

Summary

AVX512: kunpck encoding implementation
Added tests for encoding.

[delena]

lib/Target/X86/X86InstrAVX512.td

2077 ↗

(On Diff #32234)

The intrinsics should be completed too.

[igorb]

lib/Target/X86/X86InstrAVX512.td

2077 ↗

(On Diff #32234)

Intrinsics implementation may require not trivial changes in X86InstrInfo.cpp.
I would like to make it as separate patch.

[delena]

LGTM

llvm/trunk/lib/Target/X86/X86ISelLowering.cpp

if (!Subtarget->useSoftFloat() && Subtarget->hasBWI()) {
     setOperationAction(ISD::SETCC,              MVT::v64i1, Custom);
     setOperationAction(ISD::ADD,                MVT::v32i16, Legal);
     setOperationAction(ISD::ADD,                MVT::v64i8, Legal);
     setOperationAction(ISD::SUB,                MVT::v32i16, Legal);
     setOperationAction(ISD::SUB,                MVT::v64i8, Legal);
     setOperationAction(ISD::MUL,                MVT::v32i16, Legal);
     setOperationAction(ISD::MULHS,              MVT::v32i16, Legal);
     setOperationAction(ISD::MULHU,              MVT::v32i16, Legal);
-    setOperationAction(ISD::CONCAT_VECTORS,     MVT::v32i1, Custom);
-    setOperationAction(ISD::CONCAT_VECTORS,     MVT::v64i1, Custom);
+    setOperationAction(ISD::CONCAT_VECTORS,     MVT::v32i1, Legal);
+    setOperationAction(ISD::CONCAT_VECTORS,     MVT::v64i1, Legal);
     setOperationAction(ISD::INSERT_SUBVECTOR,   MVT::v32i1, Custom);
     setOperationAction(ISD::INSERT_SUBVECTOR,   MVT::v64i1, Custom);
     setOperationAction(ISD::SELECT,             MVT::v32i1, Custom);
     setOperationAction(ISD::SELECT,             MVT::v64i1, Custom);
     setOperationAction(ISD::SIGN_EXTEND,        MVT::v32i8, Custom);
     setOperationAction(ISD::ZERO_EXTEND,        MVT::v32i8, Custom);
     setOperationAction(ISD::SIGN_EXTEND,        MVT::v32i16, Custom);
     setOperationAction(ISD::ZERO_EXTEND,        MVT::v32i16, Custom);

llvm/trunk/lib/Target/X86/X86InstrAVX512.td

def : Pat<(xor (xor VK2:$src1, VK2:$src2), (v2i1 immAllOnesV)),
           (COPY_TO_REGCLASS (KXNORWrr (COPY_TO_REGCLASS VK2:$src1, VK16),
                              (COPY_TO_REGCLASS VK2:$src2, VK16)), VK2)>;
 
 def : Pat<(xor (xor VK1:$src1, VK1:$src2), (i1 1)),
           (COPY_TO_REGCLASS (KXNORWrr (COPY_TO_REGCLASS VK1:$src1, VK16),
                              (COPY_TO_REGCLASS VK1:$src2, VK16)), VK1)>;
 
 // Mask unpacking
-multiclass avx512_mask_unpck<bits<8> opc, string OpcodeStr,
-                           RegisterClass KRC> {
-  let Predicates = [HasAVX512] in
-    def rr : I<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src1, KRC:$src2),
-               !strconcat(OpcodeStr,
-                          "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>;
-}
+multiclass avx512_mask_unpck<string Suffix,RegisterClass KRC, ValueType VT,
+                             RegisterClass KRCSrc, Predicate prd> {
+  let Predicates = [prd] in {
+    def rr : I<0x4b, MRMSrcReg, (outs KRC:$dst),
+               (ins KRC:$src1, KRC:$src2),
+               "kunpck"#Suffix#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,
+               VEX_4V, VEX_L;
 
-multiclass avx512_mask_unpck_bw<bits<8> opc, string OpcodeStr> {
-  defm BW : avx512_mask_unpck<opc, !strconcat(OpcodeStr, "bw"), VK16>,
-                            VEX_4V, VEX_L, PD;
+    def : Pat<(VT (concat_vectors KRCSrc:$src1, KRCSrc:$src2)),
+              (!cast<Instruction>(NAME##rr)
+                        (COPY_TO_REGCLASS KRCSrc:$src2, KRC),
+                        (COPY_TO_REGCLASS KRCSrc:$src1, KRC))>;
+  }
 }
 
-defm KUNPCK : avx512_mask_unpck_bw<0x4b, "kunpck">;
-def : Pat<(v16i1 (concat_vectors (v8i1 VK8:$src1), (v8i1 VK8:$src2))),
-          (KUNPCKBWrr (COPY_TO_REGCLASS VK8:$src2, VK16),
-                  (COPY_TO_REGCLASS VK8:$src1, VK16))>;
-
+defm KUNPCKBW : avx512_mask_unpck<"bw", VK16, v16i1, VK8, HasAVX512>, PD;
+defm KUNPCKWD : avx512_mask_unpck<"wd", VK32, v32i1, VK16, HasBWI>, PS;
+defm KUNPCKDQ : avx512_mask_unpck<"dq", VK64, v64i1, VK32, HasBWI>, PS, VEX_W;
 
 multiclass avx512_mask_unpck_int<string IntName, string InstName> {
   let Predicates = [HasAVX512] in
     def : Pat<(!cast<Intrinsic>("int_x86_avx512_"##IntName##"_bw")
                 (i16 GR16:$src1), (i16 GR16:$src2)),
               (COPY_TO_REGCLASS (!cast<Instruction>(InstName##"BWrr")
               (v16i1 (COPY_TO_REGCLASS GR16:$src1, VK16)),
               (v16i1 (COPY_TO_REGCLASS GR16:$src2, VK16))), GR16)>;

llvm/trunk/test/CodeGen/X86/avx512-vec-cmp.ll

 ; KNL:       ## BB#0:
 ; KNL-NEXT:    vpmaxud %ymm1, %ymm0, %ymm0
 ; KNL-NEXT:    retq
   %mask = icmp ugt <8 x i32> %x, %y
   %max = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %y
   ret <8 x i32> %max
 }
 
-
 define i16 @test12(<16 x i64> %a, <16 x i64> %b) nounwind {
 ; KNL-LABEL: test12:
 ; KNL:       ## BB#0:
 ; KNL-NEXT:    vpcmpeqq %zmm2, %zmm0, %k0
 ; KNL-NEXT:    vpcmpeqq %zmm3, %zmm1, %k1
 ; KNL-NEXT:    kunpckbw %k0, %k1, %k0
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    retq
   %res = icmp eq <16 x i64> %a, %b
   %res1 = bitcast <16 x i1> %res to i16
   ret i16 %res1
 }
 
+define i32 @test12_v32i32(<32 x i32> %a, <32 x i32> %b) nounwind {
+; SKX-LABEL: test12_v32i32:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpcmpeqd %zmm2, %zmm0, %k0
+; SKX-NEXT:    vpcmpeqd %zmm3, %zmm1, %k1
+; SKX-NEXT:    kunpckwd %k0, %k1, %k0
+; SKX-NEXT:    kmovd %k0, %eax
+; SKX-NEXT:    retq
+  %res = icmp eq <32 x i32> %a, %b
+  %res1 = bitcast <32 x i1> %res to i32
+  ret i32 %res1
+}
+
+define i64 @test12_v64i16(<64 x i16> %a, <64 x i16> %b) nounwind {
+; SKX-LABEL: test12_v64i16:
+; SKX:       ## BB#0:
+; SKX-NEXT:    vpcmpeqw %zmm2, %zmm0, %k0
+; SKX-NEXT:    vpcmpeqw %zmm3, %zmm1, %k1
+; SKX-NEXT:    kunpckdq %k0, %k1, %k0
+; SKX-NEXT:    kmovq %k0, %rax
+; SKX-NEXT:    retq
+  %res = icmp eq <64 x i16> %a, %b
+  %res1 = bitcast <64 x i1> %res to i64
+  ret i64 %res1
+}
+
 define <16 x i32> @test13(<16 x float>%a, <16 x float>%b)
 ; KNL-LABEL: test13:
 ; KNL:       ## BB#0:
 ; KNL-NEXT:    vcmpeqps %zmm1, %zmm0, %k1
 ; KNL-NEXT:    vpbroadcastd {{.*}}(%rip), %zmm0 {%k1} {z}
 ; KNL-NEXT:    retq
 {
   %cmpvector_i = fcmp oeq <16 x float> %a, %b

llvm/trunk/test/MC/X86/avx512-encodings.s

 // CHECK: vgetexpsd -1024(%rdx), %xmm7, %xmm2
 // CHECK:  encoding: [0x62,0xf2,0xc5,0x08,0x43,0x52,0x80]
           vgetexpsd -1024(%rdx), %xmm7, %xmm2
 
 // CHECK: vgetexpsd -1032(%rdx), %xmm7, %xmm2
 // CHECK:  encoding: [0x62,0xf2,0xc5,0x08,0x43,0x92,0xf8,0xfb,0xff,0xff]
           vgetexpsd -1032(%rdx), %xmm7, %xmm2
 
+// CHECK: kunpckbw %k6, %k5, %k5
+// CHECK:  encoding: [0xc5,0xd5,0x4b,0xee]
+          kunpckbw %k6, %k5, %k5
+
 // CHECK: vgetmantss $171, %xmm12, %xmm2, %xmm3
 // CHECK:  encoding: [0x62,0xd3,0x6d,0x08,0x27,0xdc,0xab]
           vgetmantss $0xab, %xmm12, %xmm2, %xmm3
 
 // CHECK: vgetmantss $171, %xmm12, %xmm2, %xmm3 {%k7}
 // CHECK:  encoding: [0x62,0xd3,0x6d,0x0f,0x27,0xdc,0xab]
           vgetmantss $0xab, %xmm12, %xmm2, %xmm3 {%k7}
 

llvm/trunk/test/MC/X86/x86-64-avx512bw.s

 // CHECK: vpunpckhwd -8192(%rdx), %zmm26, %zmm30
 // CHECK:  encoding: [0x62,0x61,0x2d,0x40,0x69,0x72,0x80]
           vpunpckhwd -8192(%rdx), %zmm26, %zmm30
 
 // CHECK: vpunpckhwd -8256(%rdx), %zmm26, %zmm30
 // CHECK:  encoding: [0x62,0x61,0x2d,0x40,0x69,0xb2,0xc0,0xdf,0xff,0xff]
           vpunpckhwd -8256(%rdx), %zmm26, %zmm30
 
+// CHECK: kunpckdq %k4, %k6, %k4
+// CHECK:  encoding: [0xc4,0xe1,0xcc,0x4b,0xe4]
+          kunpckdq %k4, %k6, %k4
+
+// CHECK: kunpckwd %k6, %k5, %k5
+// CHECK:  encoding: [0xc5,0xd4,0x4b,0xee]
+          kunpckwd %k6, %k5, %k5
+
 // CHECK: ktestd %k3, %k3
 // CHECK:  encoding: [0xc4,0xe1,0xf9,0x99,0xdb]
           ktestd %k3, %k3
 
 // CHECK: ktestq %k6, %k2
 // CHECK:  encoding: [0xc4,0xe1,0xf8,0x99,0xd6]
           ktestq %k6, %k2