Index: include/llvm/CodeGen/SelectionDAG.h =================================================================== --- include/llvm/CodeGen/SelectionDAG.h +++ include/llvm/CodeGen/SelectionDAG.h @@ -974,7 +974,7 @@ MachineMemOperand *MMO, ISD::LoadExtType); SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, - MachineMemOperand *MMO, bool IsTrunc); + MachineMemOperand *MMO, bool IsTrunc, bool isCompressed = false); SDValue getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef Ops, MachineMemOperand *MMO); SDValue getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl, Index: include/llvm/CodeGen/SelectionDAGNodes.h =================================================================== --- include/llvm/CodeGen/SelectionDAGNodes.h +++ include/llvm/CodeGen/SelectionDAGNodes.h @@ -460,6 +460,7 @@ uint16_t : NumLSBaseSDNodeBits; uint16_t IsTruncating : 1; + uint16_t IsCompressed : 1; }; union { @@ -1958,15 +1959,19 @@ public: friend class SelectionDAG; MaskedStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, - bool isTrunc, EVT MemVT, MachineMemOperand *MMO) + bool isTrunc, bool isCompressed, EVT MemVT, + MachineMemOperand *MMO) : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, MemVT, MMO) { StoreSDNodeBits.IsTruncating = isTrunc; + StoreSDNodeBits.IsCompressed = isCompressed; } /// Return true if the op does a truncation before store. /// For integers this is the same as doing a TRUNCATE and storing the result. /// For floats, it is the same as doing an FP_ROUND and storing the result. bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; } + bool isCompressedStore() const { return StoreSDNodeBits.IsCompressed; } + const SDValue &getValue() const { return getOperand(3); } static bool classof(const SDNode *N) { Index: lib/CodeGen/SelectionDAG/SelectionDAG.cpp =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -5346,7 +5346,7 @@ SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, - bool isTrunc) { + bool isTrunc, bool isCompressed) { assert(Chain.getValueType() == MVT::Other && "Invalid chain type"); EVT VT = Val.getValueType(); @@ -5356,7 +5356,7 @@ AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops); ID.AddInteger(VT.getRawBits()); ID.AddInteger(getSyntheticNodeSubclassData( - dl.getIROrder(), VTs, isTrunc, MemVT, MMO)); + dl.getIROrder(), VTs, isTrunc, isCompressed, MemVT, MMO)); ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { @@ -5364,7 +5364,7 @@ return SDValue(E, 0); } auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), VTs, - isTrunc, MemVT, MMO); + isTrunc, isCompressed, MemVT, MMO); createOperands(N, Ops); CSEMap.InsertNode(N, IP); Index: lib/Target/X86/X86ISelLowering.cpp =================================================================== --- lib/Target/X86/X86ISelLowering.cpp +++ lib/Target/X86/X86ISelLowering.cpp @@ -18859,11 +18859,16 @@ return DAG.getStore(Chain, dl, DataToCompress, Addr, MemIntr->getMemOperand()); - SDValue Compressed = - getVectorMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, DataToCompress), - Mask, DAG.getUNDEF(VT), Subtarget, DAG); - return DAG.getStore(Chain, dl, Compressed, Addr, - MemIntr->getMemOperand()); +// SDValue Compressed = +// getVectorMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, DataToCompress), +// Mask, DAG.getUNDEF(VT), Subtarget, DAG); +// return DAG.getStore(Chain, dl, Compressed, Addr, +// MemIntr->getMemOperand()); + MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements()); + SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl); + + return DAG.getMaskedStore(Chain, dl, DataToCompress, Addr, VMask, VT, + MemIntr->getMemOperand(), false, true); } case TRUNCATE_TO_MEM_VI8: case TRUNCATE_TO_MEM_VI16: Index: lib/Target/X86/X86InstrAVX512.td =================================================================== --- lib/Target/X86/X86InstrAVX512.td +++ lib/Target/X86/X86InstrAVX512.td @@ -7372,8 +7372,8 @@ // AVX-512 - COMPRESS and EXPAND // -multiclass compress_by_vec_width opc, X86VectorVTInfo _, - string OpcodeStr> { +multiclass compress_by_vec_width_common opc, X86VectorVTInfo _, + string OpcodeStr > { defm rr : AVX512_maskable, AVX5128IBase; @@ -7387,30 +7387,40 @@ def mrk : AVX5128I, + []>, EVEX_K, EVEX_CD8<_.EltSize, CD8VT1>; } +multiclass compress_by_vec_width_lowering { + + def : Pat<(X86mCompressedStore addr:$dst, _.KRCWM:$mask, + (_.VT _.RC:$src)), + (!cast(NAME#_.ZSuffix##mrk) + addr:$dst, _.KRCWM:$mask, _.RC:$src)>; +} + + multiclass compress_by_elt_width opc, string OpcodeStr, - AVX512VLVectorVTInfo VTInfo> { - defm Z : compress_by_vec_width, EVEX_V512; + AVX512VLVectorVTInfo VTInfo > { + defm Z : compress_by_vec_width_common, + compress_by_vec_width_lowering, EVEX_V512; let Predicates = [HasVLX] in { - defm Z256 : compress_by_vec_width, EVEX_V256; - defm Z128 : compress_by_vec_width, EVEX_V128; + defm Z256 : compress_by_vec_width_common, + compress_by_vec_width_lowering, EVEX_V256; + defm Z128 : compress_by_vec_width_common, + compress_by_vec_width_lowering, EVEX_V128; } } defm VPCOMPRESSD : compress_by_elt_width <0x8B, "vpcompressd", avx512vl_i32_info>, - EVEX; + EVEX; defm VPCOMPRESSQ : compress_by_elt_width <0x8B, "vpcompressq", avx512vl_i64_info>, - EVEX, VEX_W; + EVEX, VEX_W; defm VCOMPRESSPS : compress_by_elt_width <0x8A, "vcompressps", avx512vl_f32_info>, - EVEX; + EVEX; defm VCOMPRESSPD : compress_by_elt_width <0x8A, "vcompresspd", avx512vl_f64_info>, - EVEX, VEX_W; + EVEX, VEX_W; // expand multiclass expand_by_vec_width opc, X86VectorVTInfo _, Index: lib/Target/X86/X86InstrFragmentsSIMD.td =================================================================== --- lib/Target/X86/X86InstrFragmentsSIMD.td +++ lib/Target/X86/X86InstrFragmentsSIMD.td @@ -973,7 +973,8 @@ // do not support vector types (llvm-tblgen will fail). def X86mstore : PatFrag<(ops node:$src1, node:$src2, node:$src3), (masked_store node:$src1, node:$src2, node:$src3), [{ - return !cast(N)->isTruncatingStore(); + return (!cast(N)->isTruncatingStore()) && + (!cast(N)->isCompressedStore()); }]>; def masked_store_aligned128 : PatFrag<(ops node:$src1, node:$src2, node:$src3), @@ -1002,6 +1003,11 @@ return isa(N); }]>; +def X86mCompressedStore : PatFrag<(ops node:$src1, node:$src2, node:$src3), + (masked_store node:$src1, node:$src2, node:$src3), [{ + return cast(N)->isCompressedStore(); +}]>; + // masked truncstore fragments // X86mtruncstore can't be implemented in core DAG files because some targets // doesn't support vector type ( llvm-tblgen will fail) @@ -1009,6 +1015,7 @@ (masked_store node:$src1, node:$src2, node:$src3), [{ return cast(N)->isTruncatingStore(); }]>; + def masked_truncstorevi8 : PatFrag<(ops node:$src1, node:$src2, node:$src3), (X86mtruncstore node:$src1, node:$src2, node:$src3), [{ Index: test/CodeGen/X86/avx512vl-intrinsics.ll =================================================================== --- test/CodeGen/X86/avx512vl-intrinsics.ll +++ test/CodeGen/X86/avx512vl-intrinsics.ll @@ -893,6 +893,37 @@ ret <4 x i32> %res } +@xmm = common global <4 x i32> zeroinitializer, align 16 +@k8 = common global i8 0, align 1 + +define i32 @compr11() { +; CHECK-LABEL: compr11: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: movq _xmm@{{.*}}(%rip), %rax ## encoding: [0x48,0x8b,0x05,A,A,A,A] +; CHECK-NEXT: ## fixup A - offset: 3, value: _xmm@GOTPCREL-4, kind: reloc_riprel_4byte_movq_load +; CHECK-NEXT: vmovdqa32 (%rax), %xmm0 ## encoding: [0x62,0xf1,0x7d,0x08,0x6f,0x00] +; CHECK-NEXT: movq _k8@{{.*}}(%rip), %rax ## encoding: [0x48,0x8b,0x05,A,A,A,A] +; CHECK-NEXT: ## fixup A - offset: 3, value: _k8@GOTPCREL-4, kind: reloc_riprel_4byte_movq_load +; CHECK-NEXT: movzbl (%rax), %eax ## encoding: [0x0f,0xb6,0x00] +; CHECK-NEXT: kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8] +; CHECK-NEXT: vpcompressd %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0x89,0x8b,0xc0] +; CHECK-NEXT: vpxord %xmm1, %xmm1, %xmm1 ## encoding: [0x62,0xf1,0x75,0x08,0xef,0xc9] +; CHECK-NEXT: vmovdqa32 %xmm0, -{{[0-9]+}}(%rsp) ## encoding: [0x62,0xf1,0x7d,0x08,0x7f,0x84,0x24,0xd8,0xff,0xff,0xff] +; CHECK-NEXT: vmovdqa32 %xmm1, -{{[0-9]+}}(%rsp) ## encoding: [0x62,0xf1,0x7d,0x08,0x7f,0x8c,0x24,0xe8,0xff,0xff,0xff] +; CHECK-NEXT: xorl %eax, %eax ## encoding: [0x31,0xc0] +; CHECK-NEXT: retq ## encoding: [0xc3] +entry: + %.compoundliteral = alloca <2 x i64>, align 16 + %res = alloca <4 x i32>, align 16 + %a0 = load <4 x i32>, <4 x i32>* @xmm, align 16 + %a2 = load i8, i8* @k8, align 1 + %a21 = call <4 x i32> @llvm.x86.avx512.mask.compress.d.128(<4 x i32> %a0, <4 x i32> zeroinitializer, i8 %a2) #2 + store volatile <4 x i32> %a21, <4 x i32>* %res, align 16 + store <2 x i64> zeroinitializer, <2 x i64>* %.compoundliteral, align 16 + ret i32 0 +} + + declare <4 x i32> @llvm.x86.avx512.mask.compress.d.128(<4 x i32> %data, <4 x i32> %src0, i8 %mask) ; Expand @@ -5296,9 +5327,9 @@ ; CHECK: ## BB#0: ; CHECK-NEXT: vmovdqa32 {{.*#+}} ymm0 = [2,9,4294967284,23,4294967270,37,4294967256,51] ; CHECK-NEXT: ## encoding: [0x62,0xf1,0x7d,0x28,0x6f,0x05,A,A,A,A] -; CHECK-NEXT: ## fixup A - offset: 6, value: LCPI312_0-4, kind: reloc_riprel_4byte +; CHECK-NEXT: ## fixup A - offset: 6, value: LCPI313_0-4, kind: reloc_riprel_4byte ; CHECK-NEXT: vpsravd {{.*}}(%rip), %ymm0, %ymm0 ## encoding: [0x62,0xf2,0x7d,0x28,0x46,0x05,A,A,A,A] -; CHECK-NEXT: ## fixup A - offset: 6, value: LCPI312_1-4, kind: reloc_riprel_4byte +; CHECK-NEXT: ## fixup A - offset: 6, value: LCPI313_1-4, kind: reloc_riprel_4byte ; CHECK-NEXT: retq ## encoding: [0xc3] %res = call <8 x i32> @llvm.x86.avx512.mask.psrav8.si(<8 x i32> , <8 x i32> , <8 x i32> zeroinitializer, i8 -1) ret <8 x i32> %res @@ -5329,9 +5360,9 @@ ; CHECK: ## BB#0: ; CHECK-NEXT: vmovdqa64 {{.*#+}} xmm0 = [2,18446744073709551607] ; CHECK-NEXT: ## encoding: [0x62,0xf1,0xfd,0x08,0x6f,0x05,A,A,A,A] -; CHECK-NEXT: ## fixup A - offset: 6, value: LCPI314_0-4, kind: reloc_riprel_4byte +; CHECK-NEXT: ## fixup A - offset: 6, value: LCPI315_0-4, kind: reloc_riprel_4byte ; CHECK-NEXT: vpsravq {{.*}}(%rip), %xmm0, %xmm0 ## encoding: [0x62,0xf2,0xfd,0x08,0x46,0x05,A,A,A,A] -; CHECK-NEXT: ## fixup A - offset: 6, value: LCPI314_1-4, kind: reloc_riprel_4byte +; CHECK-NEXT: ## fixup A - offset: 6, value: LCPI315_1-4, kind: reloc_riprel_4byte ; CHECK-NEXT: retq ## encoding: [0xc3] %res = call <2 x i64> @llvm.x86.avx512.mask.psrav.q.128(<2 x i64> , <2 x i64> , <2 x i64> zeroinitializer, i8 -1) ret <2 x i64> %res