Index: lib/Target/X86/X86ISelLowering.cpp =================================================================== --- lib/Target/X86/X86ISelLowering.cpp +++ lib/Target/X86/X86ISelLowering.cpp @@ -5300,11 +5300,12 @@ static SDValue getExtendInVec(unsigned Opc, const SDLoc &DL, EVT VT, SDValue In, SelectionDAG &DAG) { EVT InVT = In.getValueType(); - assert((X86ISD::VSEXT == Opc || X86ISD::VZEXT == Opc) && "Unexpected opcode"); + assert((ISD::SIGN_EXTEND == Opc || ISD::ZERO_EXTEND == Opc) && + "Unexpected opcode"); if (VT.is128BitVector() && InVT.is128BitVector()) - return X86ISD::VSEXT == Opc ? DAG.getSignExtendVectorInReg(In, DL, VT) - : DAG.getZeroExtendVectorInReg(In, DL, VT); + return ISD::SIGN_EXTEND == Opc ? DAG.getSignExtendVectorInReg(In, DL, VT) + : DAG.getZeroExtendVectorInReg(In, DL, VT); // For 256-bit vectors, we only need the lower (128-bit) input half. // For 512-bit vectors, we only need the lower input half or quarter. @@ -5312,9 +5313,14 @@ int Scale = VT.getScalarSizeInBits() / InVT.getScalarSizeInBits(); In = extractSubVector(In, 0, DAG, DL, std::max(128, (int)VT.getSizeInBits() / Scale)); + InVT = In.getValueType(); } - return DAG.getNode(Opc, DL, VT, In); + if (VT.getVectorNumElements() == InVT.getVectorNumElements()) + return DAG.getNode(Opc, DL, VT, In); + + return DAG.getNode((ISD::SIGN_EXTEND == Opc) ? X86ISD::VSEXT : X86ISD::VZEXT, + DL, VT, In); } /// Returns a vector_shuffle node for an unpackl operation. @@ -6215,6 +6221,7 @@ return true; } case ISD::ZERO_EXTEND_VECTOR_INREG: + case ISD::ZERO_EXTEND: case X86ISD::VZEXT: { // TODO - add support for VPMOVZX with smaller input vector types. SDValue Src = N.getOperand(0); @@ -10098,7 +10105,7 @@ MVT ExtVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits * Scale), NumElements / Scale); InputV = ShuffleOffset(InputV); - InputV = getExtendInVec(X86ISD::VZEXT, DL, ExtVT, InputV, DAG); + InputV = getExtendInVec(ISD::ZERO_EXTEND, DL, ExtVT, InputV, DAG); return DAG.getBitcast(VT, InputV); } @@ -16487,7 +16494,7 @@ "Unexpected element type"); if (Subtarget.hasInt256()) - return DAG.getNode(X86ISD::VZEXT, dl, VT, In); + return DAG.getNode(ISD::ZERO_EXTEND, dl, VT, In); // Optimize vectors in AVX mode: // @@ -16806,7 +16813,7 @@ // Make sure we're allowed to promote 512-bits. if (Subtarget.canExtendTo512DQ()) return DAG.getNode(ISD::TRUNCATE, DL, VT, - getExtendInVec(X86ISD::VSEXT, DL, MVT::v16i32, In, + getExtendInVec(ISD::SIGN_EXTEND, DL, MVT::v16i32, In, DAG)); } else { return DAG.getNode(ISD::TRUNCATE, DL, VT, In); @@ -18880,8 +18887,8 @@ if (Subtarget.hasInt256()) { assert(VT.getSizeInBits() > 128 && "Unexpected 128-bit vector extension"); unsigned ExtOpc = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG ? - X86ISD::VSEXT : X86ISD::VZEXT; - return DAG.getNode(ExtOpc, dl, VT, In); + ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; + return getExtendInVec(ExtOpc, dl, VT, In, DAG); } // We should only get here for sign extend. @@ -18945,7 +18952,7 @@ "Unexpected element type"); if (Subtarget.hasInt256()) - return DAG.getNode(X86ISD::VSEXT, dl, VT, In); + return DAG.getNode(ISD::SIGN_EXTEND, dl, VT, In); // Optimize vectors in AVX mode // Sign extend v8i16 to v8i32 and @@ -19143,7 +19150,7 @@ if (Ext == ISD::SEXTLOAD) { // If we have SSE4.1, we can directly emit a VSEXT node. if (Subtarget.hasSSE41()) { - SDValue Sext = getExtendInVec(X86ISD::VSEXT, dl, RegVT, SlicedVec, DAG); + SDValue Sext = getExtendInVec(ISD::SIGN_EXTEND, dl, RegVT, SlicedVec, DAG); DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), TF); return Sext; } @@ -19160,7 +19167,7 @@ if (Ext == ISD::EXTLOAD && !Subtarget.hasBWI() && RegVT == MVT::v8i64 && MemVT == MVT::v8i8) { - SDValue Sext = getExtendInVec(X86ISD::VZEXT, dl, RegVT, SlicedVec, DAG); + SDValue Sext = getExtendInVec(ISD::ZERO_EXTEND, dl, RegVT, SlicedVec, DAG); DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), TF); return Sext; } @@ -22440,7 +22447,8 @@ assert(VT == MVT::v16i8 && "Pre-AVX2 support only supports v16i8 multiplication"); MVT ExVT = MVT::v8i16; - unsigned ExSSE41 = (ISD::MULHU == Opcode ? X86ISD::VZEXT : X86ISD::VSEXT); + unsigned ExSSE41 = (ISD::MULHU == Opcode ? ISD::ZERO_EXTEND + : ISD::SIGN_EXTEND); // Extract the lo parts and zero/sign extend to i16. SDValue ALo, BLo; @@ -28356,6 +28364,10 @@ DstVT = MVT::getIntegerVT(Scale * MaskEltSize); DstVT = MVT::getVectorVT(DstVT, NumDstElts); + + if (V1.getSimpleValueType().getVectorNumElements() == NumDstElts) + Shuffle = unsigned(ISD::ZERO_EXTEND); + return true; } } @@ -34750,7 +34762,7 @@ Mld->getBasePtr(), NewMask, WideSrc0, Mld->getMemoryVT(), Mld->getMemOperand(), ISD::NON_EXTLOAD); - SDValue NewVec = getExtendInVec(X86ISD::VSEXT, dl, VT, WideLd, DAG); + SDValue NewVec = getExtendInVec(ISD::SIGN_EXTEND, dl, VT, WideLd, DAG); return DCI.CombineTo(N, NewVec, WideLd.getValue(1), true); } @@ -37784,8 +37796,9 @@ if (getTargetConstantBitsFromNode(Op, OpEltSizeInBits, UndefElts, EltBits)) { APInt Undefs(NumElts, 0); SmallVector Vals(NumElts, APInt(EltSizeInBits, 0)); - bool IsZEXT = - (Opcode == X86ISD::VZEXT) || (Opcode == ISD::ZERO_EXTEND_VECTOR_INREG); + bool IsZEXT = Opcode == ISD::ZERO_EXTEND || + Opcode == X86ISD::VZEXT || + Opcode == ISD::ZERO_EXTEND_VECTOR_INREG; for (unsigned i = 0; i != NumElts; ++i) { if (UndefElts[i]) { Undefs.setBit(i); Index: lib/Target/X86/X86InstrAVX512.td =================================================================== --- lib/Target/X86/X86InstrAVX512.td +++ lib/Target/X86/X86InstrAVX512.td @@ -8464,8 +8464,8 @@ } } -multiclass avx512_extend_BW opc, string OpcodeStr, - SDNode OpNode, SDNode InVecNode, string ExtTy, +multiclass avx512_extend_BW opc, string OpcodeStr, SDNode OpNode, + SDNode X86OpNode, SDNode InVecNode, string ExtTy, OpndItins itins, PatFrag LdFrag = !cast(ExtTy#"extloadvi8")> { let Predicates = [HasVLX, HasBWI] in { defm Z128: avx512_extend_common opc, string OpcodeStr, - SDNode OpNode, SDNode InVecNode, string ExtTy, +multiclass avx512_extend_BD opc, string OpcodeStr, SDNode OpNode, + SDNode X86OpNode, SDNode InVecNode, string ExtTy, OpndItins itins, PatFrag LdFrag = !cast(ExtTy#"extloadvi8")> { let Predicates = [HasVLX, HasAVX512] in { defm Z128: avx512_extend_common, T8PD, EVEX_V128, VEX_WIG; defm Z256: avx512_extend_common, + v16i8x_info, i64mem, LdFrag, X86OpNode>, EVEX_CD8<8, CD8VQ>, T8PD, EVEX_V256, VEX_WIG; } let Predicates = [HasAVX512] in { @@ -8502,8 +8502,8 @@ } } -multiclass avx512_extend_BQ opc, string OpcodeStr, - SDNode OpNode, SDNode InVecNode, string ExtTy, +multiclass avx512_extend_BQ opc, string OpcodeStr, SDNode OpNode, + SDNode X86OpNode, SDNode InVecNode, string ExtTy, OpndItins itins, PatFrag LdFrag = !cast(ExtTy#"extloadvi8")> { let Predicates = [HasVLX, HasAVX512] in { defm Z128: avx512_extend_common, T8PD, EVEX_V128, VEX_WIG; defm Z256: avx512_extend_common, + v16i8x_info, i32mem, LdFrag, X86OpNode>, EVEX_CD8<8, CD8VO>, T8PD, EVEX_V256, VEX_WIG; } let Predicates = [HasAVX512] in { defm Z : avx512_extend_common, + v16i8x_info, i64mem, LdFrag, X86OpNode>, EVEX_CD8<8, CD8VO>, T8PD, EVEX_V512, VEX_WIG; } } -multiclass avx512_extend_WD opc, string OpcodeStr, - SDNode OpNode, SDNode InVecNode, string ExtTy, +multiclass avx512_extend_WD opc, string OpcodeStr, SDNode OpNode, + SDNode X86OpNode, SDNode InVecNode, string ExtTy, OpndItins itins, PatFrag LdFrag = !cast(ExtTy#"extloadvi16")> { let Predicates = [HasVLX, HasAVX512] in { defm Z128: avx512_extend_common opc, string OpcodeStr, - SDNode OpNode, SDNode InVecNode, string ExtTy, +multiclass avx512_extend_WQ opc, string OpcodeStr, SDNode OpNode, + SDNode X86OpNode, SDNode InVecNode, string ExtTy, OpndItins itins, PatFrag LdFrag = !cast(ExtTy#"extloadvi16")> { let Predicates = [HasVLX, HasAVX512] in { defm Z128: avx512_extend_common, T8PD, EVEX_V128, VEX_WIG; defm Z256: avx512_extend_common, + v8i16x_info, i64mem, LdFrag, X86OpNode>, EVEX_CD8<16, CD8VQ>, T8PD, EVEX_V256, VEX_WIG; } let Predicates = [HasAVX512] in { @@ -8559,8 +8559,8 @@ } } -multiclass avx512_extend_DQ opc, string OpcodeStr, - SDNode OpNode, SDNode InVecNode, string ExtTy, +multiclass avx512_extend_DQ opc, string OpcodeStr, SDNode OpNode, + SDNode X86OpNode, SDNode InVecNode, string ExtTy, OpndItins itins, PatFrag LdFrag = !cast(ExtTy#"extloadvi32")> { let Predicates = [HasVLX, HasAVX512] in { @@ -8579,22 +8579,22 @@ } } -defm VPMOVZXBW : avx512_extend_BW<0x30, "vpmovzxbw", X86vzext, zext_invec, "z", AVX512_EXTEND>; -defm VPMOVZXBD : avx512_extend_BD<0x31, "vpmovzxbd", X86vzext, zext_invec, "z", AVX512_EXTEND>; -defm VPMOVZXBQ : avx512_extend_BQ<0x32, "vpmovzxbq", X86vzext, zext_invec, "z", AVX512_EXTEND>; -defm VPMOVZXWD : avx512_extend_WD<0x33, "vpmovzxwd", X86vzext, zext_invec, "z", AVX512_EXTEND>; -defm VPMOVZXWQ : avx512_extend_WQ<0x34, "vpmovzxwq", X86vzext, zext_invec, "z", AVX512_EXTEND>; -defm VPMOVZXDQ : avx512_extend_DQ<0x35, "vpmovzxdq", X86vzext, zext_invec, "z", AVX512_EXTEND>; +defm VPMOVZXBW : avx512_extend_BW<0x30, "vpmovzxbw", zext, X86vzext, zext_invec, "z", AVX512_EXTEND>; +defm VPMOVZXBD : avx512_extend_BD<0x31, "vpmovzxbd", zext, X86vzext, zext_invec, "z", AVX512_EXTEND>; +defm VPMOVZXBQ : avx512_extend_BQ<0x32, "vpmovzxbq", zext, X86vzext, zext_invec, "z", AVX512_EXTEND>; +defm VPMOVZXWD : avx512_extend_WD<0x33, "vpmovzxwd", zext, X86vzext, zext_invec, "z", AVX512_EXTEND>; +defm VPMOVZXWQ : avx512_extend_WQ<0x34, "vpmovzxwq", zext, X86vzext, zext_invec, "z", AVX512_EXTEND>; +defm VPMOVZXDQ : avx512_extend_DQ<0x35, "vpmovzxdq", zext, X86vzext, zext_invec, "z", AVX512_EXTEND>; -defm VPMOVSXBW: avx512_extend_BW<0x20, "vpmovsxbw", X86vsext, sext_invec, "s", AVX512_EXTEND>; -defm VPMOVSXBD: avx512_extend_BD<0x21, "vpmovsxbd", X86vsext, sext_invec, "s", AVX512_EXTEND>; -defm VPMOVSXBQ: avx512_extend_BQ<0x22, "vpmovsxbq", X86vsext, sext_invec, "s", AVX512_EXTEND>; -defm VPMOVSXWD: avx512_extend_WD<0x23, "vpmovsxwd", X86vsext, sext_invec, "s", AVX512_EXTEND>; -defm VPMOVSXWQ: avx512_extend_WQ<0x24, "vpmovsxwq", X86vsext, sext_invec, "s", AVX512_EXTEND>; -defm VPMOVSXDQ: avx512_extend_DQ<0x25, "vpmovsxdq", X86vsext, sext_invec, "s", AVX512_EXTEND>; +defm VPMOVSXBW: avx512_extend_BW<0x20, "vpmovsxbw", sext, X86vsext, sext_invec, "s", AVX512_EXTEND>; +defm VPMOVSXBD: avx512_extend_BD<0x21, "vpmovsxbd", sext, X86vsext, sext_invec, "s", AVX512_EXTEND>; +defm VPMOVSXBQ: avx512_extend_BQ<0x22, "vpmovsxbq", sext, X86vsext, sext_invec, "s", AVX512_EXTEND>; +defm VPMOVSXWD: avx512_extend_WD<0x23, "vpmovsxwd", sext, X86vsext, sext_invec, "s", AVX512_EXTEND>; +defm VPMOVSXWQ: avx512_extend_WQ<0x24, "vpmovsxwq", sext, X86vsext, sext_invec, "s", AVX512_EXTEND>; +defm VPMOVSXDQ: avx512_extend_DQ<0x25, "vpmovsxdq", sext, X86vsext, sext_invec, "s", AVX512_EXTEND>; -multiclass AVX512_pmovx_patterns { // 128-bit patterns let Predicates = [HasVLX, HasBWI] in { @@ -8669,22 +8669,22 @@ (!cast(OpcPrefix#BWZ256rm) addr:$src)>; } let Predicates = [HasVLX] in { - def : Pat<(v8i32 (ExtOp (bc_v16i8 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))), + def : Pat<(v8i32 (X86ExtOp (bc_v16i8 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))), (!cast(OpcPrefix#BDZ256rm) addr:$src)>; - def : Pat<(v8i32 (ExtOp (v16i8 (vzmovl_v2i64 addr:$src)))), + def : Pat<(v8i32 (X86ExtOp (v16i8 (vzmovl_v2i64 addr:$src)))), (!cast(OpcPrefix#BDZ256rm) addr:$src)>; - def : Pat<(v8i32 (ExtOp (v16i8 (vzload_v2i64 addr:$src)))), + def : Pat<(v8i32 (X86ExtOp (v16i8 (vzload_v2i64 addr:$src)))), (!cast(OpcPrefix#BDZ256rm) addr:$src)>; - def : Pat<(v8i32 (ExtOp (bc_v16i8 (loadv2i64 addr:$src)))), + def : Pat<(v8i32 (X86ExtOp (bc_v16i8 (loadv2i64 addr:$src)))), (!cast(OpcPrefix#BDZ256rm) addr:$src)>; - def : Pat<(v4i64 (ExtOp (bc_v16i8 (v4i32 (scalar_to_vector (loadi32 addr:$src)))))), + def : Pat<(v4i64 (X86ExtOp (bc_v16i8 (v4i32 (scalar_to_vector (loadi32 addr:$src)))))), (!cast(OpcPrefix#BQZ256rm) addr:$src)>; - def : Pat<(v4i64 (ExtOp (v16i8 (vzmovl_v4i32 addr:$src)))), + def : Pat<(v4i64 (X86ExtOp (v16i8 (vzmovl_v4i32 addr:$src)))), (!cast(OpcPrefix#BQZ256rm) addr:$src)>; - def : Pat<(v4i64 (ExtOp (v16i8 (vzload_v2i64 addr:$src)))), + def : Pat<(v4i64 (X86ExtOp (v16i8 (vzload_v2i64 addr:$src)))), (!cast(OpcPrefix#BQZ256rm) addr:$src)>; - def : Pat<(v4i64 (ExtOp (bc_v16i8 (loadv2i64 addr:$src)))), + def : Pat<(v4i64 (X86ExtOp (bc_v16i8 (loadv2i64 addr:$src)))), (!cast(OpcPrefix#BQZ256rm) addr:$src)>; def : Pat<(v8i32 (ExtOp (bc_v8i16 (loadv2i64 addr:$src)))), @@ -8694,13 +8694,13 @@ def : Pat<(v8i32 (ExtOp (v8i16 (vzload_v2i64 addr:$src)))), (!cast(OpcPrefix#WDZ256rm) addr:$src)>; - def : Pat<(v4i64 (ExtOp (bc_v8i16 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))), + def : Pat<(v4i64 (X86ExtOp (bc_v8i16 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))), (!cast(OpcPrefix#WQZ256rm) addr:$src)>; - def : Pat<(v4i64 (ExtOp (v8i16 (vzmovl_v2i64 addr:$src)))), + def : Pat<(v4i64 (X86ExtOp (v8i16 (vzmovl_v2i64 addr:$src)))), (!cast(OpcPrefix#WQZ256rm) addr:$src)>; - def : Pat<(v4i64 (ExtOp (v8i16 (vzload_v2i64 addr:$src)))), + def : Pat<(v4i64 (X86ExtOp (v8i16 (vzload_v2i64 addr:$src)))), (!cast(OpcPrefix#WQZ256rm) addr:$src)>; - def : Pat<(v4i64 (ExtOp (bc_v8i16 (loadv2i64 addr:$src)))), + def : Pat<(v4i64 (X86ExtOp (bc_v8i16 (loadv2i64 addr:$src)))), (!cast(OpcPrefix#WQZ256rm) addr:$src)>; def : Pat<(v4i64 (ExtOp (bc_v4i32 (loadv2i64 addr:$src)))), @@ -8719,9 +8719,9 @@ def : Pat<(v16i32 (ExtOp (bc_v16i8 (loadv2i64 addr:$src)))), (!cast(OpcPrefix#BDZrm) addr:$src)>; - def : Pat<(v8i64 (ExtOp (bc_v16i8 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))), + def : Pat<(v8i64 (X86ExtOp (bc_v16i8 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))), (!cast(OpcPrefix#BQZrm) addr:$src)>; - def : Pat<(v8i64 (ExtOp (bc_v16i8 (loadv2i64 addr:$src)))), + def : Pat<(v8i64 (X86ExtOp (bc_v16i8 (loadv2i64 addr:$src)))), (!cast(OpcPrefix#BQZrm) addr:$src)>; def : Pat<(v16i32 (ExtOp (bc_v16i16 (loadv4i64 addr:$src)))), @@ -8735,8 +8735,17 @@ } } -defm : AVX512_pmovx_patterns<"VPMOVSX", X86vsext, sext_invec, extloadi32i16>; -defm : AVX512_pmovx_patterns<"VPMOVZX", X86vzext, zext_invec, loadi16_anyext>; +defm : AVX512_pmovx_patterns<"VPMOVSX", sext, X86vsext, sext_invec, extloadi32i16>; +defm : AVX512_pmovx_patterns<"VPMOVZX", zext, X86vzext, zext_invec, loadi16_anyext>; + +let Predicates = [HasAVX512, NoBWI] in { +def: Pat<(v16i8 (trunc (v16i16 VR256X:$src))), + (VPMOVDBZrr (v16i32 (VPMOVSXWDZrr VR256X:$src)))>; +def: Pat<(v16i8 (trunc (bc_v16i16 (loadv4i64 addr:$src)))), + (VPMOVDBZrr (v16i32 (VPMOVSXWDZrm addr:$src)))>; +def: Pat<(store (v16i8 (trunc (v16i16 VR256X:$src))), addr:$dst), + (VPMOVDBZmr addr:$dst, (v16i32 (VPMOVSXWDZrr VR256X:$src)))>; +} //===----------------------------------------------------------------------===// // GATHER - SCATTER Operations Index: lib/Target/X86/X86InstrSSE.td =================================================================== --- lib/Target/X86/X86InstrSSE.td +++ lib/Target/X86/X86InstrSSE.td @@ -5390,21 +5390,22 @@ defm BQ : SS41I_pmovx_rm<0x22, "bq", i16mem, i32mem, NoVLX>; // AVX2 Patterns -multiclass SS41I_pmovx_avx2_patterns { +multiclass SS41I_pmovx_avx2_patterns { // Register-Register patterns let Predicates = [HasAVX, NoVLX_Or_NoBWI] in { def : Pat<(v16i16 (ExtOp (v16i8 VR128:$src))), (!cast(OpcPrefix#BWYrr) VR128:$src)>; } let Predicates = [HasAVX, NoVLX] in { - def : Pat<(v8i32 (ExtOp (v16i8 VR128:$src))), + def : Pat<(v8i32 (X86ExtOp (v16i8 VR128:$src))), (!cast(OpcPrefix#BDYrr) VR128:$src)>; - def : Pat<(v4i64 (ExtOp (v16i8 VR128:$src))), + def : Pat<(v4i64 (X86ExtOp (v16i8 VR128:$src))), (!cast(OpcPrefix#BQYrr) VR128:$src)>; def : Pat<(v8i32 (ExtOp (v8i16 VR128:$src))), (!cast(OpcPrefix#WDYrr) VR128:$src)>; - def : Pat<(v4i64 (ExtOp (v8i16 VR128:$src))), + def : Pat<(v4i64 (X86ExtOp (v8i16 VR128:$src))), (!cast(OpcPrefix#WQYrr) VR128:$src)>; def : Pat<(v4i64 (ExtOp (v4i32 VR128:$src))), @@ -5441,22 +5442,22 @@ (!cast(OpcPrefix#BWYrm) addr:$src)>; } let Predicates = [HasAVX, NoVLX] in { - def : Pat<(v8i32 (ExtOp (bc_v16i8 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))), + def : Pat<(v8i32 (X86ExtOp (bc_v16i8 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))), (!cast(OpcPrefix#BDYrm) addr:$src)>; - def : Pat<(v8i32 (ExtOp (v16i8 (vzmovl_v2i64 addr:$src)))), + def : Pat<(v8i32 (X86ExtOp (v16i8 (vzmovl_v2i64 addr:$src)))), (!cast(OpcPrefix#BDYrm) addr:$src)>; - def : Pat<(v8i32 (ExtOp (v16i8 (vzload_v2i64 addr:$src)))), + def : Pat<(v8i32 (X86ExtOp (v16i8 (vzload_v2i64 addr:$src)))), (!cast(OpcPrefix#BDYrm) addr:$src)>; - def : Pat<(v8i32 (ExtOp (bc_v16i8 (loadv2i64 addr:$src)))), + def : Pat<(v8i32 (X86ExtOp (bc_v16i8 (loadv2i64 addr:$src)))), (!cast(OpcPrefix#BDYrm) addr:$src)>; - def : Pat<(v4i64 (ExtOp (bc_v16i8 (v4i32 (scalar_to_vector (loadi32 addr:$src)))))), + def : Pat<(v4i64 (X86ExtOp (bc_v16i8 (v4i32 (scalar_to_vector (loadi32 addr:$src)))))), (!cast(OpcPrefix#BQYrm) addr:$src)>; - def : Pat<(v4i64 (ExtOp (v16i8 (vzmovl_v4i32 addr:$src)))), + def : Pat<(v4i64 (X86ExtOp (v16i8 (vzmovl_v4i32 addr:$src)))), (!cast(OpcPrefix#BQYrm) addr:$src)>; - def : Pat<(v4i64 (ExtOp (v16i8 (vzload_v2i64 addr:$src)))), + def : Pat<(v4i64 (X86ExtOp (v16i8 (vzload_v2i64 addr:$src)))), (!cast(OpcPrefix#BQYrm) addr:$src)>; - def : Pat<(v4i64 (ExtOp (bc_v16i8 (loadv2i64 addr:$src)))), + def : Pat<(v4i64 (X86ExtOp (bc_v16i8 (loadv2i64 addr:$src)))), (!cast(OpcPrefix#BQYrm) addr:$src)>; def : Pat<(v8i32 (ExtOp (bc_v8i16 (loadv2i64 addr:$src)))), @@ -5466,13 +5467,13 @@ def : Pat<(v8i32 (ExtOp (v8i16 (vzload_v2i64 addr:$src)))), (!cast(OpcPrefix#WDYrm) addr:$src)>; - def : Pat<(v4i64 (ExtOp (bc_v8i16 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))), + def : Pat<(v4i64 (X86ExtOp (bc_v8i16 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))), (!cast(OpcPrefix#WQYrm) addr:$src)>; - def : Pat<(v4i64 (ExtOp (v8i16 (vzmovl_v2i64 addr:$src)))), + def : Pat<(v4i64 (X86ExtOp (v8i16 (vzmovl_v2i64 addr:$src)))), (!cast(OpcPrefix#WQYrm) addr:$src)>; - def : Pat<(v4i64 (ExtOp (v8i16 (vzload_v2i64 addr:$src)))), + def : Pat<(v4i64 (X86ExtOp (v8i16 (vzload_v2i64 addr:$src)))), (!cast(OpcPrefix#WQYrm) addr:$src)>; - def : Pat<(v4i64 (ExtOp (bc_v8i16 (loadv2i64 addr:$src)))), + def : Pat<(v4i64 (X86ExtOp (bc_v8i16 (loadv2i64 addr:$src)))), (!cast(OpcPrefix#WQYrm) addr:$src)>; def : Pat<(v4i64 (ExtOp (bc_v4i32 (loadv2i64 addr:$src)))), @@ -5484,8 +5485,8 @@ } } -defm : SS41I_pmovx_avx2_patterns<"VPMOVSX", "s", X86vsext>; -defm : SS41I_pmovx_avx2_patterns<"VPMOVZX", "z", X86vzext>; +defm : SS41I_pmovx_avx2_patterns<"VPMOVSX", "s", sext, X86vsext>; +defm : SS41I_pmovx_avx2_patterns<"VPMOVZX", "z", zext, X86vzext>; // SSE4.1/AVX patterns. multiclass SS41I_pmovx_patterns