Index: lib/Target/X86/X86ISelLowering.cpp =================================================================== --- lib/Target/X86/X86ISelLowering.cpp +++ lib/Target/X86/X86ISelLowering.cpp @@ -19303,9 +19303,34 @@ } } - // Emit a zeroed vector and insert the desired subvector on its - // first half. + assert(Subtarget->hasAVX() && "Shuffle combining 256-bit vector needs AVX"); + + // Emit a zero vector and blend the desired subvector into its low half. SDValue Zeros = getZeroVector(VT, Subtarget, DAG, dl); + + // The blend instruction, and therefore its mask, depend on the data type. + MVT ScalarType = VT.getScalarType().getSimpleVT(); + if (ScalarType.isFloatingPoint()) { + // Choose either vblendps (float) or vblendpd (double). + unsigned ScalarSize = ScalarType.getSizeInBits(); + assert((ScalarSize == 64 || ScalarSize == 32) && "Unknown float type"); + unsigned MaskVal = (ScalarSize == 64) ? 0x03 : 0x0f; + SDValue Mask = DAG.getConstant(MaskVal, MVT::i8); + return DAG.getNode(X86ISD::BLENDI, dl, VT, Zeros, V1, Mask); + } + + // AVX2 is needed for 256-bit integer blend support. + if (Subtarget->hasAVX2()) { + // Integers must be cast to 32-bit because there is only vpblendd... + // vpblendw can't be used for this because it has a handicapped mask. + SDValue Mask = DAG.getConstant(0x0f, MVT::i8); + Zeros = DAG.getNode(ISD::BITCAST, dl, MVT::v8i32, Zeros); + V1 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i32, V1); + V1 = DAG.getNode(X86ISD::BLENDI, dl, MVT::v8i32, Zeros, V1, Mask); + return DAG.getNode(ISD::BITCAST, dl, VT, V1); + } + + // For an AVX1 target, fall back to using vinsertf128 for integers. SDValue InsV = Insert128BitVector(Zeros, V1.getOperand(0), 0, DAG, dl); return DCI.CombineTo(N, InsV); } Index: test/CodeGen/X86/2012-04-26-sdglue.ll =================================================================== --- test/CodeGen/X86/2012-04-26-sdglue.ll +++ test/CodeGen/X86/2012-04-26-sdglue.ll @@ -5,10 +5,10 @@ ; It's hard to test for the ISEL condition because CodeGen optimizes ; away the bugpointed code. Just ensure the basics are still there. ;CHECK-LABEL: func: -;CHECK: vpxor -;CHECK: vinserti128 +;CHECK: vxorps ;CHECK: vpshufd ;CHECK: vpbroadcastd +;CHECK: vinserti128 ;CHECK: vmulps ;CHECK: vmulps ;CHECK: ret Index: test/CodeGen/X86/avx-cast.ll =================================================================== --- test/CodeGen/X86/avx-cast.ll +++ test/CodeGen/X86/avx-cast.ll @@ -1,51 +1,100 @@ -; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx | FileCheck %s +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx | FileCheck %s --check-prefix=AVX1 +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx2 | FileCheck %s --check-prefix=AVX2 + +; Prefer a blend instruction to a vinsert128 instruction because blends +; are simpler (no lane changes) and therefore will have equal or better +; performance. -; CHECK-LABEL: castA: -; CHECK: vxorps -; CHECK-NEXT: vinsertf128 $0 define <8 x float> @castA(<4 x float> %m) nounwind uwtable readnone ssp { +; AVX1-LABEL: castA: +; AVX1: vxorps %ymm1, %ymm1, %ymm1 +; AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] +; AVX1-NEXT: retq +; +; AVX2-LABEL: castA: +; AVX2: vxorps %ymm1, %ymm1, %ymm1 +; AVX2-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] +; AVX2-NEXT: retq + entry: %shuffle.i = shufflevector <4 x float> %m, <4 x float> zeroinitializer, <8 x i32> ret <8 x float> %shuffle.i } -; CHECK-LABEL: castB: -; CHECK: vxorps -; CHECK-NEXT: vinsertf128 $0 define <4 x double> @castB(<2 x double> %m) nounwind uwtable readnone ssp { +; AVX1-LABEL: castB: +; AVX1: vxorpd %ymm1, %ymm1, %ymm1 +; AVX1-NEXT: vblendpd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3] +; AVX1-NEXT: retq +; +; AVX2-LABEL: castB: +; AVX2: vxorpd %ymm1, %ymm1, %ymm1 +; AVX2-NEXT: vblendpd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3] +; AVX2-NEXT: retq + entry: %shuffle.i = shufflevector <2 x double> %m, <2 x double> zeroinitializer, <4 x i32> ret <4 x double> %shuffle.i } -; CHECK-LABEL: castC: -; CHECK: vxorps -; CHECK-NEXT: vinsertf128 $0 +; AVX2 is needed for integer types. + define <4 x i64> @castC(<2 x i64> %m) nounwind uwtable readnone ssp { +; AVX1-LABEL: castC: +; AVX1: vxorps %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $0, %xmm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: castC: +; AVX2: vpxor %ymm1, %ymm1, %ymm1 +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] +; AVX2-NEXT: retq + entry: %shuffle.i = shufflevector <2 x i64> %m, <2 x i64> zeroinitializer, <4 x i32> ret <4 x i64> %shuffle.i } -; CHECK-LABEL: castD: -; CHECK-NOT: vextractf128 $0 +; The next three tests don't need any shuffling. There may or may not be a +; vzeroupper before the return, so just check for the absence of shuffles. + define <4 x float> @castD(<8 x float> %m) nounwind uwtable readnone ssp { +; AVX1-LABEL: castD: +; AVX1-NOT: extract +; AVX1-NOT: blend +; +; AVX2-LABEL: castD: +; AVX2-NOT: extract +; AVX2-NOT: blend + entry: %shuffle.i = shufflevector <8 x float> %m, <8 x float> %m, <4 x i32> ret <4 x float> %shuffle.i } -; CHECK-LABEL: castE: -; CHECK-NOT: vextractf128 $0 define <2 x i64> @castE(<4 x i64> %m) nounwind uwtable readnone ssp { +; AVX1-LABEL: castE: +; AVX1-NOT: extract +; AVX1-NOT: blend +; +; AVX2-LABEL: castE: +; AVX2-NOT: extract +; AVX2-NOT: blend + entry: %shuffle.i = shufflevector <4 x i64> %m, <4 x i64> %m, <2 x i32> ret <2 x i64> %shuffle.i } -; CHECK-LABEL: castF: -; CHECK-NOT: vextractf128 $0 define <2 x double> @castF(<4 x double> %m) nounwind uwtable readnone ssp { +; AVX1-LABEL: castF: +; AVX1-NOT: extract +; AVX1-NOT: blend +; +; AVX2-LABEL: castF: +; AVX2-NOT: extract +; AVX2-NOT: blend + entry: %shuffle.i = shufflevector <4 x double> %m, <4 x double> %m, <2 x i32> ret <2 x double> %shuffle.i