This is an archive of the discontinued LLVM Phabricator instance.

Differential D19579

[AVX512] AVX512 braodcastsd/ss patterns interaction with spilling.
ClosedPublic

Authored by igorb on Apr 27 2016, 12:36 AM.

Details

Reviewers
delena
greened
cameron.mcinally
Commits
rG131008fbcb6c: Change AVX512 braodcastsd/ss patterns interaction with spilling . New…
rL268190: Change AVX512 braodcastsd/ss patterns interaction with spilling . New…
Summary

Change AVX512 braodcastsd/ss patterns interaction with spilling . New implementation take a scalar register and generate a vector without COPY_TO_REGCLASS (turn it into a VR128 register ) .The issue is that during register allocation we may spill a scalar value using 128-bit loads and stores, wasting cache bandwidth.
example

declare void @func_f32(float)
define <8 x float> @_256_broadcast_ss_spill(float %x) {
  %a  = fadd float %x, %x
  call void @func_f32(float %a)
  %b = insertelement <8 x float> undef, float %a, i32 0
  %c = shufflevector <8 x float> %b, <8 x float> undef, <8 x i32> zeroinitializer
  ret <8 x float> %c
}

new implementation

vaddss  %xmm0, %xmm0, %xmm0
vmovss  %xmm0, 4(%rsp)          # 4-byte Folded Spill
callq   func_f32
vbroadcastss    4(%rsp), %ymm0  # 4-byte Folded Reload
popq    %rax
retq

old implementation

vaddss  %xmm0, %xmm0, %xmm0
vmovaps %xmm0, (%rsp)           # 16-byte Spill
callq   func_f32
vbroadcastss    (%rsp), %ymm0   # 16-byte Folded Reload
addq    $24, %rsp
retq

Diff Detail

Repository
rL LLVM

Event Timeline

igorb updated this revision to Diff 55166.Apr 27 2016, 12:36 AM
igorb retitled this revision from to [AVX512] AVX512 braodcastsd/ss patterns interaction with spilling..
igorb updated this object.
igorb added reviewers: delena, greened, cameron.mcinally.
igorb set the repository for this revision to rL LLVM.
igorb updated this object.
igorb added a subscriber: llvm-commits.
greened edited edge metadata.Apr 28 2016, 1:37 PM

LGTM. Thanks!

Closed by commit rL268190: Change AVX512 braodcastsd/ss patterns interaction with spilling . New… (authored by ibreger). · Explain WhyMay 1 2016, 1:45 AM
This revision was automatically updated to reflect the committed changes.

Revision Contents

PathSize
llvm/
trunk/
lib/
Target/
X86/
90 lines
86 lines
32 lines
test/
CodeGen/
X86/
17 lines
39 lines
175 lines

Diff 55740

llvm/trunk/lib/Target/X86/X86InstrAVX512.td

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 815 Lines • ▼ Show 20 Linesdef VEXTRACTPSzmr : AVX512AIi8<0x17, MRMDestMem, (outs),
(ins f32mem:$dst, VR128X:$src1, u8imm:$src2), (ins f32mem:$dst, VR128X:$src1, u8imm:$src2),
"vextractps\t{$src2, $src1, $dst|$dst, $src1, $src2}", "vextractps\t{$src2, $src1, $dst|$dst, $src1, $src2}",
[(store (extractelt (bc_v4i32 (v4f32 VR128X:$src1)), imm:$src2), [(store (extractelt (bc_v4i32 (v4f32 VR128X:$src1)), imm:$src2),
addr:$dst)]>, EVEX, EVEX_CD8<32, CD8VT1>; addr:$dst)]>, EVEX, EVEX_CD8<32, CD8VT1>;
//===---------------------------------------------------------------------===// //===---------------------------------------------------------------------===//
// AVX-512 BROADCAST // AVX-512 BROADCAST
//--- //---
// broadcast with a scalar argument.
multiclass avx512_broadcast_scalar<bits<8> opc, string OpcodeStr,
X86VectorVTInfo DestInfo, X86VectorVTInfo SrcInfo> {
let isCodeGenOnly = 1 in {
def r_s : I< opc, MRMSrcReg, (outs DestInfo.RC:$dst),
(ins SrcInfo.FRC:$src), OpcodeStr#"\t{$src, $dst|$dst, $src}",
[(set DestInfo.RC:$dst, (DestInfo.VT (X86VBroadcast SrcInfo.FRC:$src)))]>,
Requires<[HasAVX512]>, T8PD, EVEX;
let Constraints = "$src0 = $dst" in
def rk_s : I< opc, MRMSrcReg, (outs DestInfo.RC:$dst),
(ins DestInfo.RC:$src0, DestInfo.KRCWM:$mask, SrcInfo.FRC:$src),
OpcodeStr#"\t{$src, $dst {${mask}} |$dst {${mask}}, $src}",
[(set DestInfo.RC:$dst,
(vselect DestInfo.KRCWM:$mask,
(DestInfo.VT (X86VBroadcast SrcInfo.FRC:$src)),
DestInfo.RC:$src0))]>,
Requires<[HasAVX512]>, T8PD, EVEX, EVEX_K;
def rkz_s : I< opc, MRMSrcReg, (outs DestInfo.RC:$dst),
(ins DestInfo.KRCWM:$mask, SrcInfo.FRC:$src),
OpcodeStr#"\t{$src, $dst {${mask}} {z}|$dst {${mask}} {z}, $src}",
[(set DestInfo.RC:$dst,
(vselect DestInfo.KRCWM:$mask,
(DestInfo.VT (X86VBroadcast SrcInfo.FRC:$src)),
DestInfo.ImmAllZerosV))]>,
Requires<[HasAVX512]>, T8PD, EVEX, EVEX_KZ;
} // let isCodeGenOnly = 1 in
}
multiclass avx512_broadcast_rm<bits<8> opc, string OpcodeStr, multiclass avx512_broadcast_rm<bits<8> opc, string OpcodeStr,
X86VectorVTInfo DestInfo, X86VectorVTInfo SrcInfo> { X86VectorVTInfo DestInfo, X86VectorVTInfo SrcInfo> {
defm r : AVX512_maskable<opc, MRMSrcReg, DestInfo, (outs DestInfo.RC:$dst), defm r : AVX512_maskable<opc, MRMSrcReg, DestInfo, (outs DestInfo.RC:$dst),
(ins SrcInfo.RC:$src), OpcodeStr, "$src", "$src", (ins SrcInfo.RC:$src), OpcodeStr, "$src", "$src",
(DestInfo.VT (X86VBroadcast (SrcInfo.VT SrcInfo.RC:$src)))>, (DestInfo.VT (X86VBroadcast (SrcInfo.VT SrcInfo.RC:$src)))>,
T8PD, EVEX; T8PD, EVEX;
let mayLoad = 1 in let mayLoad = 1 in
defm m : AVX512_maskable<opc, MRMSrcMem, DestInfo, (outs DestInfo.RC:$dst), defm m : AVX512_maskable<opc, MRMSrcMem, DestInfo, (outs DestInfo.RC:$dst),
(ins SrcInfo.ScalarMemOp:$src), OpcodeStr, "$src", "$src", (ins SrcInfo.ScalarMemOp:$src), OpcodeStr, "$src", "$src",
(DestInfo.VT (X86VBroadcast (DestInfo.VT (X86VBroadcast
(SrcInfo.ScalarLdFrag addr:$src)))>, (SrcInfo.ScalarLdFrag addr:$src)))>,
T8PD, EVEX, EVEX_CD8<SrcInfo.EltSize, CD8VT1>; T8PD, EVEX, EVEX_CD8<SrcInfo.EltSize, CD8VT1>;
} }
multiclass avx512_fp_broadcast_vl<bits<8> opc, string OpcodeStr, multiclass avx512_fp_broadcast_vl<bits<8> opc, string OpcodeStr,
AVX512VLVectorVTInfo _> { AVX512VLVectorVTInfo _> {
defm Z : avx512_broadcast_rm<opc, OpcodeStr, _.info512, _.info128>, defm Z : avx512_broadcast_rm<opc, OpcodeStr, _.info512, _.info128>,
avx512_broadcast_scalar<opc, OpcodeStr, _.info512, _.info128>,
EVEX_V512; EVEX_V512;
let Predicates = [HasVLX] in { let Predicates = [HasVLX] in {
defm Z256 : avx512_broadcast_rm<opc, OpcodeStr, _.info256, _.info128>, defm Z256 : avx512_broadcast_rm<opc, OpcodeStr, _.info256, _.info128>,
avx512_broadcast_scalar<opc, OpcodeStr, _.info256, _.info128>,
EVEX_V256; EVEX_V256;
} }
} }
let ExeDomain = SSEPackedSingle in { let ExeDomain = SSEPackedSingle in {
defm VBROADCASTSS : avx512_fp_broadcast_vl<0x18, "vbroadcastss", defm VBROADCASTSS : avx512_fp_broadcast_vl<0x18, "vbroadcastss",
avx512vl_f32_info>; avx512vl_f32_info>;
let Predicates = [HasVLX] in { let Predicates = [HasVLX] in {
defm VBROADCASTSSZ128 : avx512_broadcast_rm<0x18, "vbroadcastss", defm VBROADCASTSSZ128 :
v4f32x_info, v4f32x_info>, EVEX_V128; avx512_broadcast_rm<0x18, "vbroadcastss", v4f32x_info, v4f32x_info>,
avx512_broadcast_scalar<0x18, "vbroadcastss", v4f32x_info, v4f32x_info>,
EVEX_V128;
} }
} }
let ExeDomain = SSEPackedDouble in { let ExeDomain = SSEPackedDouble in {
defm VBROADCASTSD : avx512_fp_broadcast_vl<0x19, "vbroadcastsd", defm VBROADCASTSD : avx512_fp_broadcast_vl<0x19, "vbroadcastsd",
avx512vl_f64_info>, VEX_W; avx512vl_f64_info>, VEX_W;
} }
// avx512_broadcast_pat introduces patterns for broadcast with a scalar argument.
// Later, we can canonize broadcast instructions before ISel phase and
// eliminate additional patterns on ISel.
// SrcRC_v and SrcRC_s are RegisterClasses for vector and scalar
// representations of source
multiclass avx512_broadcast_pat<string InstName, SDNode OpNode,
X86VectorVTInfo _, RegisterClass SrcRC_v,
RegisterClass SrcRC_s> {
def : Pat<(_.VT (OpNode (_.EltVT SrcRC_s:$src))),
(!cast<Instruction>(InstName##"r")
(COPY_TO_REGCLASS SrcRC_s:$src, SrcRC_v))>;
let AddedComplexity = 30 in {
def : Pat<(_.VT (vselect _.KRCWM:$mask,
(OpNode (_.EltVT SrcRC_s:$src)), _.RC:$src0)),
(!cast<Instruction>(InstName##"rk") _.RC:$src0, _.KRCWM:$mask,
(COPY_TO_REGCLASS SrcRC_s:$src, SrcRC_v))>;
def : Pat<(_.VT(vselect _.KRCWM:$mask,
(OpNode (_.EltVT SrcRC_s:$src)), _.ImmAllZerosV)),
(!cast<Instruction>(InstName##"rkz") _.KRCWM:$mask,
(COPY_TO_REGCLASS SrcRC_s:$src, SrcRC_v))>;
}
}
defm : avx512_broadcast_pat<"VBROADCASTSSZ", X86VBroadcast, v16f32_info,
VR128X, FR32X>;
defm : avx512_broadcast_pat<"VBROADCASTSDZ", X86VBroadcast, v8f64_info,
VR128X, FR64X>;
let Predicates = [HasVLX] in {
defm : avx512_broadcast_pat<"VBROADCASTSSZ256", X86VBroadcast,
v8f32x_info, VR128X, FR32X>;
defm : avx512_broadcast_pat<"VBROADCASTSSZ128", X86VBroadcast,
v4f32x_info, VR128X, FR32X>;
defm : avx512_broadcast_pat<"VBROADCASTSDZ256", X86VBroadcast,
v4f64x_info, VR128X, FR64X>;
}
def : Pat<(v16f32 (X86VBroadcast (loadf32 addr:$src))),
(VBROADCASTSSZm addr:$src)>;
def : Pat<(v8f64 (X86VBroadcast (loadf64 addr:$src))),
(VBROADCASTSDZm addr:$src)>;
def : Pat<(int_x86_avx512_vbroadcast_ss_512 addr:$src), def : Pat<(int_x86_avx512_vbroadcast_ss_512 addr:$src),
(VBROADCASTSSZm addr:$src)>; (VBROADCASTSSZm addr:$src)>;
def : Pat<(int_x86_avx512_vbroadcast_sd_512 addr:$src), def : Pat<(int_x86_avx512_vbroadcast_sd_512 addr:$src),
(VBROADCASTSDZm addr:$src)>; (VBROADCASTSDZm addr:$src)>;
multiclass avx512_int_broadcast_reg<bits<8> opc, X86VectorVTInfo _, multiclass avx512_int_broadcast_reg<bits<8> opc, X86VectorVTInfo _,
RegisterClass SrcRC> { RegisterClass SrcRC> {
defm r : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), defm r : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
▲ Show 20 Lines • Show All 169 Lines • ▼ Show 20 Lines
def : Pat<(v16f32 (X86VBroadcast (v8f32 VR256X:$src))), def : Pat<(v16f32 (X86VBroadcast (v8f32 VR256X:$src))),
(VBROADCASTSSZr (EXTRACT_SUBREG (v8f32 VR256X:$src), sub_xmm))>; (VBROADCASTSSZr (EXTRACT_SUBREG (v8f32 VR256X:$src), sub_xmm))>;
def : Pat<(v8f64 (X86VBroadcast (v8f64 VR512:$src))), def : Pat<(v8f64 (X86VBroadcast (v8f64 VR512:$src))),
(VBROADCASTSDZr (EXTRACT_SUBREG (v8f64 VR512:$src), sub_xmm))>; (VBROADCASTSDZr (EXTRACT_SUBREG (v8f64 VR512:$src), sub_xmm))>;
def : Pat<(v8f64 (X86VBroadcast (v4f64 VR256X:$src))), def : Pat<(v8f64 (X86VBroadcast (v4f64 VR256X:$src))),
(VBROADCASTSDZr (EXTRACT_SUBREG (v4f64 VR256X:$src), sub_xmm))>; (VBROADCASTSDZr (EXTRACT_SUBREG (v4f64 VR256X:$src), sub_xmm))>;
// Provide fallback in case the load node that is used in the patterns above
// is used by additional users, which prevents the pattern selection.
def : Pat<(v16f32 (X86VBroadcast FR32X:$src)),
(VBROADCASTSSZr (COPY_TO_REGCLASS FR32X:$src, VR128X))>;
def : Pat<(v8f64 (X86VBroadcast FR64X:$src)),
(VBROADCASTSDZr (COPY_TO_REGCLASS FR64X:$src, VR128X))>;
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
// AVX-512 BROADCAST MASK TO VECTOR REGISTER // AVX-512 BROADCAST MASK TO VECTOR REGISTER
//--- //---
multiclass avx512_mask_broadcastm<bits<8> opc, string OpcodeStr, multiclass avx512_mask_broadcastm<bits<8> opc, string OpcodeStr,
X86VectorVTInfo _, RegisterClass KRC> { X86VectorVTInfo _, RegisterClass KRC> {
def rr : AVX512XS8I<opc, MRMSrcReg, (outs _.RC:$dst), (ins KRC:$src), def rr : AVX512XS8I<opc, MRMSrcReg, (outs _.RC:$dst), (ins KRC:$src),
!strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
[(set _.RC:$dst, (_.VT (X86VBroadcastm KRC:$src)))]>, EVEX; [(set _.RC:$dst, (_.VT (X86VBroadcastm KRC:$src)))]>, EVEX;
▲ Show 20 Lines • Show All 6,577 Lines • Show Last 20 Lines

llvm/trunk/lib/Target/X86/X86InstrInfo.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 799 Lines • ▼ Show 20 Lines static const X86MemoryFoldTableEntry MemoryFoldTable1[] = {
{ X86::T1MSKC64rr, X86::T1MSKC64rm, 0 }, { X86::T1MSKC64rr, X86::T1MSKC64rm, 0 },
{ X86::TZCNT16rr, X86::TZCNT16rm, 0 }, { X86::TZCNT16rr, X86::TZCNT16rm, 0 },
{ X86::TZCNT32rr, X86::TZCNT32rm, 0 }, { X86::TZCNT32rr, X86::TZCNT32rm, 0 },
{ X86::TZCNT64rr, X86::TZCNT64rm, 0 }, { X86::TZCNT64rr, X86::TZCNT64rm, 0 },
{ X86::TZMSK32rr, X86::TZMSK32rm, 0 }, { X86::TZMSK32rr, X86::TZMSK32rm, 0 },
{ X86::TZMSK64rr, X86::TZMSK64rm, 0 }, { X86::TZMSK64rr, X86::TZMSK64rm, 0 },
// AVX-512 foldable instructions // AVX-512 foldable instructions
{ X86::VMOV64toPQIZrr, X86::VMOVQI2PQIZrm, 0 }, { X86::VMOV64toPQIZrr, X86::VMOVQI2PQIZrm, 0 },
{ X86::VMOVDI2SSZrr, X86::VMOVDI2SSZrm, 0 }, { X86::VMOVDI2SSZrr, X86::VMOVDI2SSZrm, 0 },
{ X86::VMOVAPDZrr, X86::VMOVAPDZrm, TB_ALIGN_64 }, { X86::VMOVAPDZrr, X86::VMOVAPDZrm, TB_ALIGN_64 },
{ X86::VMOVAPSZrr, X86::VMOVAPSZrm, TB_ALIGN_64 }, { X86::VMOVAPSZrr, X86::VMOVAPSZrm, TB_ALIGN_64 },
{ X86::VMOVDQA32Zrr, X86::VMOVDQA32Zrm, TB_ALIGN_64 }, { X86::VMOVDQA32Zrr, X86::VMOVDQA32Zrm, TB_ALIGN_64 },
{ X86::VMOVDQA64Zrr, X86::VMOVDQA64Zrm, TB_ALIGN_64 }, { X86::VMOVDQA64Zrr, X86::VMOVDQA64Zrm, TB_ALIGN_64 },
{ X86::VMOVDQU8Zrr, X86::VMOVDQU8Zrm, 0 }, { X86::VMOVDQU8Zrr, X86::VMOVDQU8Zrm, 0 },
{ X86::VMOVDQU16Zrr, X86::VMOVDQU16Zrm, 0 }, { X86::VMOVDQU16Zrr, X86::VMOVDQU16Zrm, 0 },
{ X86::VMOVDQU32Zrr, X86::VMOVDQU32Zrm, 0 }, { X86::VMOVDQU32Zrr, X86::VMOVDQU32Zrm, 0 },
{ X86::VMOVDQU64Zrr, X86::VMOVDQU64Zrm, 0 }, { X86::VMOVDQU64Zrr, X86::VMOVDQU64Zrm, 0 },
{ X86::VMOVUPDZrr, X86::VMOVUPDZrm, 0 }, { X86::VMOVUPDZrr, X86::VMOVUPDZrm, 0 },
{ X86::VMOVUPSZrr, X86::VMOVUPSZrm, 0 }, { X86::VMOVUPSZrr, X86::VMOVUPSZrm, 0 },
{ X86::VPABSDZrr, X86::VPABSDZrm, 0 }, { X86::VPABSDZrr, X86::VPABSDZrm, 0 },
{ X86::VPABSQZrr, X86::VPABSQZrm, 0 }, { X86::VPABSQZrr, X86::VPABSQZrm, 0 },
{ X86::VBROADCASTSSZr, X86::VBROADCASTSSZm, TB_NO_REVERSE }, { X86::VBROADCASTSSZr, X86::VBROADCASTSSZm, TB_NO_REVERSE },
{ X86::VBROADCASTSSZr_s, X86::VBROADCASTSSZm, TB_NO_REVERSE },
{ X86::VBROADCASTSDZr, X86::VBROADCASTSDZm, TB_NO_REVERSE }, { X86::VBROADCASTSDZr, X86::VBROADCASTSDZm, TB_NO_REVERSE },
{ X86::VBROADCASTSDZr_s, X86::VBROADCASTSDZm, TB_NO_REVERSE },
// AVX-512 foldable instructions (256-bit versions) // AVX-512 foldable instructions (256-bit versions)
{ X86::VMOVAPDZ256rr, X86::VMOVAPDZ256rm, TB_ALIGN_32 }, { X86::VMOVAPDZ256rr, X86::VMOVAPDZ256rm, TB_ALIGN_32 },
{ X86::VMOVAPSZ256rr, X86::VMOVAPSZ256rm, TB_ALIGN_32 }, { X86::VMOVAPSZ256rr, X86::VMOVAPSZ256rm, TB_ALIGN_32 },
{ X86::VMOVDQA32Z256rr, X86::VMOVDQA32Z256rm, TB_ALIGN_32 }, { X86::VMOVDQA32Z256rr, X86::VMOVDQA32Z256rm, TB_ALIGN_32 },
{ X86::VMOVDQA64Z256rr, X86::VMOVDQA64Z256rm, TB_ALIGN_32 }, { X86::VMOVDQA64Z256rr, X86::VMOVDQA64Z256rm, TB_ALIGN_32 },
{ X86::VMOVDQU8Z256rr, X86::VMOVDQU8Z256rm, 0 }, { X86::VMOVDQU8Z256rr, X86::VMOVDQU8Z256rm, 0 },
{ X86::VMOVDQU16Z256rr, X86::VMOVDQU16Z256rm, 0 }, { X86::VMOVDQU16Z256rr, X86::VMOVDQU16Z256rm, 0 },
{ X86::VMOVDQU32Z256rr, X86::VMOVDQU32Z256rm, 0 }, { X86::VMOVDQU32Z256rr, X86::VMOVDQU32Z256rm, 0 },
{ X86::VMOVDQU64Z256rr, X86::VMOVDQU64Z256rm, 0 }, { X86::VMOVDQU64Z256rr, X86::VMOVDQU64Z256rm, 0 },
{ X86::VMOVUPDZ256rr, X86::VMOVUPDZ256rm, 0 }, { X86::VMOVUPDZ256rr, X86::VMOVUPDZ256rm, 0 },
{ X86::VMOVUPSZ256rr, X86::VMOVUPSZ256rm, 0 }, { X86::VMOVUPSZ256rr, X86::VMOVUPSZ256rm, 0 },
{ X86::VBROADCASTSSZ256r, X86::VBROADCASTSSZ256m, TB_NO_REVERSE }, { X86::VBROADCASTSSZ256r, X86::VBROADCASTSSZ256m, TB_NO_REVERSE },
{ X86::VBROADCASTSSZ256r_s, X86::VBROADCASTSSZ256m, TB_NO_REVERSE },
{ X86::VBROADCASTSDZ256r, X86::VBROADCASTSDZ256m, TB_NO_REVERSE }, { X86::VBROADCASTSDZ256r, X86::VBROADCASTSDZ256m, TB_NO_REVERSE },
{ X86::VBROADCASTSDZ256r_s, X86::VBROADCASTSDZ256m, TB_NO_REVERSE },
// AVX-512 foldable instructions (256-bit versions) // AVX-512 foldable instructions (128-bit versions)
{ X86::VMOVAPDZ128rr, X86::VMOVAPDZ128rm, TB_ALIGN_16 }, { X86::VMOVAPDZ128rr, X86::VMOVAPDZ128rm, TB_ALIGN_16 },
{ X86::VMOVAPSZ128rr, X86::VMOVAPSZ128rm, TB_ALIGN_16 }, { X86::VMOVAPSZ128rr, X86::VMOVAPSZ128rm, TB_ALIGN_16 },
{ X86::VMOVDQA32Z128rr, X86::VMOVDQA32Z128rm, TB_ALIGN_16 }, { X86::VMOVDQA32Z128rr, X86::VMOVDQA32Z128rm, TB_ALIGN_16 },
{ X86::VMOVDQA64Z128rr, X86::VMOVDQA64Z128rm, TB_ALIGN_16 }, { X86::VMOVDQA64Z128rr, X86::VMOVDQA64Z128rm, TB_ALIGN_16 },
{ X86::VMOVDQU8Z128rr, X86::VMOVDQU8Z128rm, 0 }, { X86::VMOVDQU8Z128rr, X86::VMOVDQU8Z128rm, 0 },
{ X86::VMOVDQU16Z128rr, X86::VMOVDQU16Z128rm, 0 }, { X86::VMOVDQU16Z128rr, X86::VMOVDQU16Z128rm, 0 },
{ X86::VMOVDQU32Z128rr, X86::VMOVDQU32Z128rm, 0 }, { X86::VMOVDQU32Z128rr, X86::VMOVDQU32Z128rm, 0 },
{ X86::VMOVDQU64Z128rr, X86::VMOVDQU64Z128rm, 0 }, { X86::VMOVDQU64Z128rr, X86::VMOVDQU64Z128rm, 0 },
{ X86::VMOVUPDZ128rr, X86::VMOVUPDZ128rm, 0 }, { X86::VMOVUPDZ128rr, X86::VMOVUPDZ128rm, 0 },
{ X86::VMOVUPSZ128rr, X86::VMOVUPSZ128rm, 0 }, { X86::VMOVUPSZ128rr, X86::VMOVUPSZ128rm, 0 },
{ X86::VBROADCASTSSZ128r, X86::VBROADCASTSSZ128m, TB_NO_REVERSE }, { X86::VBROADCASTSSZ128r, X86::VBROADCASTSSZ128m, TB_NO_REVERSE },
{ X86::VBROADCASTSSZ128r_s, X86::VBROADCASTSSZ128m, TB_NO_REVERSE },
// F16C foldable instructions // F16C foldable instructions
{ X86::VCVTPH2PSrr, X86::VCVTPH2PSrm, 0 }, { X86::VCVTPH2PSrr, X86::VCVTPH2PSrm, 0 },
{ X86::VCVTPH2PSYrr, X86::VCVTPH2PSYrm, 0 }, { X86::VCVTPH2PSYrr, X86::VCVTPH2PSYrm, 0 },
// AES foldable instructions // AES foldable instructions
{ X86::AESIMCrr, X86::AESIMCrm, TB_ALIGN_16 }, { X86::AESIMCrr, X86::AESIMCrm, TB_ALIGN_16 },
{ X86::AESKEYGENASSIST128rr, X86::AESKEYGENASSIST128rm, TB_ALIGN_16 }, { X86::AESKEYGENASSIST128rr, X86::AESKEYGENASSIST128rm, TB_ALIGN_16 },
{ X86::VAESIMCrr, X86::VAESIMCrm, 0 }, { X86::VAESIMCrr, X86::VAESIMCrm, 0 },
▲ Show 20 Lines • Show All 6,631 Lines • Show Last 20 Lines

llvm/trunk/lib/Target/X86/X86InstrSSE.td

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 8,390 Lines • ▼ Show 20 Lineslet Predicates = [HasAVX2] in {
// Provide aliases for broadcast from the same register class that // Provide aliases for broadcast from the same register class that
// automatically does the extract. // automatically does the extract.
def : Pat<(v8f32 (X86VBroadcast (v8f32 VR256:$src))), def : Pat<(v8f32 (X86VBroadcast (v8f32 VR256:$src))),
(VBROADCASTSSYrr (v4f32 (EXTRACT_SUBREG (v8f32 VR256:$src), (VBROADCASTSSYrr (v4f32 (EXTRACT_SUBREG (v8f32 VR256:$src),
sub_xmm)))>; sub_xmm)))>;
def : Pat<(v4f64 (X86VBroadcast (v4f64 VR256:$src))), def : Pat<(v4f64 (X86VBroadcast (v4f64 VR256:$src))),
(VBROADCASTSDYrr (v2f64 (EXTRACT_SUBREG (v4f64 VR256:$src), (VBROADCASTSDYrr (v2f64 (EXTRACT_SUBREG (v4f64 VR256:$src),
sub_xmm)))>; sub_xmm)))>;
}
let Predicates = [HasAVX2, NoVLX] in {
// Provide fallback in case the load node that is used in the patterns above // Provide fallback in case the load node that is used in the patterns above
// is used by additional users, which prevents the pattern selection. // is used by additional users, which prevents the pattern selection.
let AddedComplexity = 20 in { let AddedComplexity = 20 in {
def : Pat<(v4f32 (X86VBroadcast FR32:$src)), def : Pat<(v4f32 (X86VBroadcast FR32:$src)),
(VBROADCASTSSrr (COPY_TO_REGCLASS FR32:$src, VR128))>; (VBROADCASTSSrr (COPY_TO_REGCLASS FR32:$src, VR128))>;
def : Pat<(v8f32 (X86VBroadcast FR32:$src)), def : Pat<(v8f32 (X86VBroadcast FR32:$src)),
(VBROADCASTSSYrr (COPY_TO_REGCLASS FR32:$src, VR128))>; (VBROADCASTSSYrr (COPY_TO_REGCLASS FR32:$src, VR128))>;
def : Pat<(v4f64 (X86VBroadcast FR64:$src)), def : Pat<(v4f64 (X86VBroadcast FR64:$src)),
(VBROADCASTSDYrr (COPY_TO_REGCLASS FR64:$src, VR128))>; (VBROADCASTSDYrr (COPY_TO_REGCLASS FR64:$src, VR128))>;
} }
} }
let Predicates = [HasAVX2, NoVLX_Or_NoBWI], AddedComplexity = 20 in { let Predicates = [HasAVX2, NoVLX_Or_NoBWI], AddedComplexity = 20 in {
def : Pat<(v16i8 (X86VBroadcast GR8:$src)), def : Pat<(v16i8 (X86VBroadcast GR8:$src)),
(VPBROADCASTBrr (COPY_TO_REGCLASS (VPBROADCASTBrr (COPY_TO_REGCLASS
(i32 (SUBREG_TO_REG (i32 0), GR8:$src, sub_8bit)), (i32 (SUBREG_TO_REG (i32 0), GR8:$src, sub_8bit)),
VR128))>; VR128))>;
def : Pat<(v32i8 (X86VBroadcast GR8:$src)), def : Pat<(v32i8 (X86VBroadcast GR8:$src)),
Show All 35 Lines
def : Pat<(v4i32 (X86VBroadcast (loadi32 addr:$src))), def : Pat<(v4i32 (X86VBroadcast (loadi32 addr:$src))),
(VBROADCASTSSrm addr:$src)>; (VBROADCASTSSrm addr:$src)>;
} }
// Provide fallback in case the load node that is used in the patterns above // Provide fallback in case the load node that is used in the patterns above
// is used by additional users, which prevents the pattern selection. // is used by additional users, which prevents the pattern selection.
let Predicates = [HasAVX], AddedComplexity = 20 in { let Predicates = [HasAVX], AddedComplexity = 20 in {
// 128bit broadcasts: // 128bit broadcasts:
def : Pat<(v2f64 (X86VBroadcast f64:$src)),
(VMOVDDUPrr (COPY_TO_REGCLASS FR64:$src, VR128))>;
}
let Predicates = [HasAVX, NoVLX], AddedComplexity = 20 in {
def : Pat<(v4f32 (X86VBroadcast FR32:$src)), def : Pat<(v4f32 (X86VBroadcast FR32:$src)),
(VPSHUFDri (COPY_TO_REGCLASS FR32:$src, VR128), 0)>; (VPSHUFDri (COPY_TO_REGCLASS FR32:$src, VR128), 0)>;
def : Pat<(v8f32 (X86VBroadcast FR32:$src)), def : Pat<(v8f32 (X86VBroadcast FR32:$src)),
(VINSERTF128rr (INSERT_SUBREG (v8f32 (IMPLICIT_DEF)), (VINSERTF128rr (INSERT_SUBREG (v8f32 (IMPLICIT_DEF)),
(VPSHUFDri (COPY_TO_REGCLASS FR32:$src, VR128), 0), sub_xmm), (VPSHUFDri (COPY_TO_REGCLASS FR32:$src, VR128), 0), sub_xmm),
(VPSHUFDri (COPY_TO_REGCLASS FR32:$src, VR128), 0), 1)>; (VPSHUFDri (COPY_TO_REGCLASS FR32:$src, VR128), 0), 1)>;
def : Pat<(v4f64 (X86VBroadcast FR64:$src)), def : Pat<(v4f64 (X86VBroadcast FR64:$src)),
(VINSERTF128rr (INSERT_SUBREG (v4f64 (IMPLICIT_DEF)), (VINSERTF128rr (INSERT_SUBREG (v4f64 (IMPLICIT_DEF)),
(VPSHUFDri (COPY_TO_REGCLASS FR64:$src, VR128), 0x44), sub_xmm), (VPSHUFDri (COPY_TO_REGCLASS FR64:$src, VR128), 0x44), sub_xmm),
(VPSHUFDri (COPY_TO_REGCLASS FR64:$src, VR128), 0x44), 1)>; (VPSHUFDri (COPY_TO_REGCLASS FR64:$src, VR128), 0x44), 1)>;
def : Pat<(v2f64 (X86VBroadcast f64:$src)),
(VMOVDDUPrr (COPY_TO_REGCLASS FR64:$src, VR128))>;
}
let Predicates = [HasAVX, NoVLX], AddedComplexity = 20 in {
def : Pat<(v4i32 (X86VBroadcast GR32:$src)), def : Pat<(v4i32 (X86VBroadcast GR32:$src)),
(VPSHUFDri (COPY_TO_REGCLASS GR32:$src, VR128), 0)>; (VPSHUFDri (COPY_TO_REGCLASS GR32:$src, VR128), 0)>;
def : Pat<(v8i32 (X86VBroadcast GR32:$src)), def : Pat<(v8i32 (X86VBroadcast GR32:$src)),
(VINSERTF128rr (INSERT_SUBREG (v8i32 (IMPLICIT_DEF)), (VINSERTF128rr (INSERT_SUBREG (v8i32 (IMPLICIT_DEF)),
(VPSHUFDri (COPY_TO_REGCLASS GR32:$src, VR128), 0), sub_xmm), (VPSHUFDri (COPY_TO_REGCLASS GR32:$src, VR128), 0), sub_xmm),
(VPSHUFDri (COPY_TO_REGCLASS GR32:$src, VR128), 0), 1)>; (VPSHUFDri (COPY_TO_REGCLASS GR32:$src, VR128), 0), 1)>;
def : Pat<(v4i64 (X86VBroadcast GR64:$src)), def : Pat<(v4i64 (X86VBroadcast GR64:$src)),
(VINSERTF128rr (INSERT_SUBREG (v4i64 (IMPLICIT_DEF)), (VINSERTF128rr (INSERT_SUBREG (v4i64 (IMPLICIT_DEF)),
▲ Show 20 Lines • Show All 406 Lines • Show Last 20 Lines

llvm/trunk/test/CodeGen/X86/avx-isa-check.ll

Show First 20 Lines • Show All 657 Lines • ▼ Show 20 Lines
define <32 x i8> @test_cmpgtb(<32 x i8> %A) { define <32 x i8> @test_cmpgtb(<32 x i8> %A) {
; generate the follow code ; generate the follow code
; vpxor %ymm1, %ymm1, %ymm1 ; vpxor %ymm1, %ymm1, %ymm1
; vpcmpgtb %ymm0, %ymm1, %ymm0 ; vpcmpgtb %ymm0, %ymm1, %ymm0
%B = ashr <32 x i8> %A, <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7> %B = ashr <32 x i8> %A, <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>
ret <32 x i8> %B ret <32 x i8> %B
} }
define <4 x float> @_inreg4xfloat(float %a) {
%b = insertelement <4 x float> undef, float %a, i32 0
%c = shufflevector <4 x float> %b, <4 x float> undef, <4 x i32> zeroinitializer
ret <4 x float> %c
}
define <8 x float> @_inreg8xfloat(float %a) {
%b = insertelement <8 x float> undef, float %a, i32 0
%c = shufflevector <8 x float> %b, <8 x float> undef, <8 x i32> zeroinitializer
ret <8 x float> %c
}
define <4 x double> @_inreg4xdouble(double %a) {
%b = insertelement <4 x double> undef, double %a, i32 0
%c = shufflevector <4 x double> %b, <4 x double> undef, <4 x i32> zeroinitializer
ret <4 x double> %c
}

llvm/trunk/test/CodeGen/X86/avx512-vbroadcast.ll

Show First 20 Lines • Show All 392 Lines • ▼ Show 20 Lines
; ALL-LABEL: _invec4xi64: ; ALL-LABEL: _invec4xi64:
; ALL: # BB#0: ; ALL: # BB#0:
; ALL-NEXT: vpbroadcastq %xmm0, %zmm0 ; ALL-NEXT: vpbroadcastq %xmm0, %zmm0
; ALL-NEXT: retq ; ALL-NEXT: retq
%res = shufflevector <4 x i64> %a, <4 x i64> undef, <8 x i32> zeroinitializer %res = shufflevector <4 x i64> %a, <4 x i64> undef, <8 x i32> zeroinitializer
ret <8 x i64>%res ret <8 x i64>%res
} }
declare void @func_f32(float)
define <16 x float> @broadcast_ss_spill(float %x) {
; ALL-LABEL: broadcast_ss_spill:
; ALL: # BB#0:
; ALL-NEXT: pushq %rax
; ALL-NEXT: .Ltmp0:
; ALL-NEXT: .cfi_def_cfa_offset 16
; ALL-NEXT: vaddss %xmm0, %xmm0, %xmm0
; ALL-NEXT: vmovss %xmm0, {{[0-9]+}}(%rsp) # 4-byte Folded Spill
; ALL-NEXT: callq func_f32
; ALL-NEXT: vbroadcastss {{[0-9]+}}(%rsp), %zmm0 # 4-byte Folded Reload
; ALL-NEXT: popq %rax
; ALL-NEXT: retq
%a = fadd float %x, %x
call void @func_f32(float %a)
%b = insertelement <16 x float> undef, float %a, i32 0
%c = shufflevector <16 x float> %b, <16 x float> undef, <16 x i32> zeroinitializer
ret <16 x float> %c
}
declare void @func_f64(double)
define <8 x double> @broadcast_sd_spill(double %x) {
; ALL-LABEL: broadcast_sd_spill:
; ALL: # BB#0:
; ALL-NEXT: pushq %rax
; ALL-NEXT: .Ltmp1:
; ALL-NEXT: .cfi_def_cfa_offset 16
; ALL-NEXT: vaddsd %xmm0, %xmm0, %xmm0
; ALL-NEXT: vmovsd %xmm0, (%rsp) # 8-byte Folded Spill
; ALL-NEXT: callq func_f64
; ALL-NEXT: vbroadcastsd (%rsp), %zmm0 # 8-byte Folded Reload
; ALL-NEXT: popq %rax
; ALL-NEXT: retq
%a = fadd double %x, %x
call void @func_f64(double %a)
%b = insertelement <8 x double> undef, double %a, i32 0
%c = shufflevector <8 x double> %b, <8 x double> undef, <8 x i32> zeroinitializer
ret <8 x double> %c
}

llvm/trunk/test/CodeGen/X86/avx512vl-vbroadcast.ll

; NOTE: Assertions have been autogenerated by update_llc_test_checks.py
; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f -mattr=+avx512vl| FileCheck %s
declare void @func_f32(float)
define <8 x float> @_256_broadcast_ss_spill(float %x) {
; CHECK-LABEL: _256_broadcast_ss_spill:
; CHECK: # BB#0:
; CHECK-NEXT: pushq %rax
; CHECK-NEXT: .Ltmp0:
; CHECK-NEXT: .cfi_def_cfa_offset 16
; CHECK-NEXT: vaddss %xmm0, %xmm0, %xmm0
; CHECK-NEXT: vmovss %xmm0, {{[0-9]+}}(%rsp) # 4-byte Folded Spill
; CHECK-NEXT: callq func_f32
; CHECK-NEXT: vbroadcastss {{[0-9]+}}(%rsp), %ymm0 # 4-byte Folded Reload
; CHECK-NEXT: popq %rax
; CHECK-NEXT: retq
%a = fadd float %x, %x
call void @func_f32(float %a)
%b = insertelement <8 x float> undef, float %a, i32 0
%c = shufflevector <8 x float> %b, <8 x float> undef, <8 x i32> zeroinitializer
ret <8 x float> %c
}
define <4 x float> @_128_broadcast_ss_spill(float %x) {
; CHECK-LABEL: _128_broadcast_ss_spill:
; CHECK: # BB#0:
; CHECK-NEXT: pushq %rax
; CHECK-NEXT: .Ltmp1:
; CHECK-NEXT: .cfi_def_cfa_offset 16
; CHECK-NEXT: vaddss %xmm0, %xmm0, %xmm0
; CHECK-NEXT: vmovss %xmm0, {{[0-9]+}}(%rsp) # 4-byte Folded Spill
; CHECK-NEXT: callq func_f32
; CHECK-NEXT: vbroadcastss {{[0-9]+}}(%rsp), %xmm0 # 4-byte Folded Reload
; CHECK-NEXT: popq %rax
; CHECK-NEXT: retq
%a = fadd float %x, %x
call void @func_f32(float %a)
%b = insertelement <4 x float> undef, float %a, i32 0
%c = shufflevector <4 x float> %b, <4 x float> undef, <4 x i32> zeroinitializer
ret <4 x float> %c
}
declare void @func_f64(double)
define <4 x double> @_256_broadcast_sd_spill(double %x) {
; CHECK-LABEL: _256_broadcast_sd_spill:
; CHECK: # BB#0:
; CHECK-NEXT: pushq %rax
; CHECK-NEXT: .Ltmp2:
; CHECK-NEXT: .cfi_def_cfa_offset 16
; CHECK-NEXT: vaddsd %xmm0, %xmm0, %xmm0
; CHECK-NEXT: vmovsd %xmm0, (%rsp) # 8-byte Folded Spill
; CHECK-NEXT: callq func_f64
; CHECK-NEXT: vbroadcastsd (%rsp), %ymm0 # 8-byte Folded Reload
; CHECK-NEXT: popq %rax
; CHECK-NEXT: retq
%a = fadd double %x, %x
call void @func_f64(double %a)
%b = insertelement <4 x double> undef, double %a, i32 0
%c = shufflevector <4 x double> %b, <4 x double> undef, <4 x i32> zeroinitializer
ret <4 x double> %c
}
define <8 x float> @_inreg8xfloat(float %a) {
; CHECK-LABEL: _inreg8xfloat:
; CHECK: # BB#0:
; CHECK-NEXT: vbroadcastss %xmm0, %ymm0
; CHECK-NEXT: retq
%b = insertelement <8 x float> undef, float %a, i32 0
%c = shufflevector <8 x float> %b, <8 x float> undef, <8 x i32> zeroinitializer
ret <8 x float> %c
}
define <8 x float> @_ss8xfloat_mask(<8 x float> %i, float %a, <8 x i32> %mask1) {
; CHECK-LABEL: _ss8xfloat_mask:
; CHECK: # BB#0:
; CHECK-NEXT: vpxor %ymm3, %ymm3, %ymm3
; CHECK-NEXT: vpcmpneqd %ymm3, %ymm2, %k1
; CHECK-NEXT: vbroadcastss %xmm1, %ymm0 {%k1}
; CHECK-NEXT: retq
%mask = icmp ne <8 x i32> %mask1, zeroinitializer
%b = insertelement <8 x float> undef, float %a, i32 0
%c = shufflevector <8 x float> %b, <8 x float> undef, <8 x i32> zeroinitializer
%r = select <8 x i1> %mask, <8 x float> %c, <8 x float> %i
ret <8 x float> %r
}
define <8 x float> @_ss8xfloat_maskz(float %a, <8 x i32> %mask1) {
; CHECK-LABEL: _ss8xfloat_maskz:
; CHECK: # BB#0:
; CHECK-NEXT: vpxor %ymm2, %ymm2, %ymm2
; CHECK-NEXT: vpcmpneqd %ymm2, %ymm1, %k1
; CHECK-NEXT: vbroadcastss %xmm0, %ymm0 {%k1} {z}
; CHECK-NEXT: retq
%mask = icmp ne <8 x i32> %mask1, zeroinitializer
%b = insertelement <8 x float> undef, float %a, i32 0
%c = shufflevector <8 x float> %b, <8 x float> undef, <8 x i32> zeroinitializer
%r = select <8 x i1> %mask, <8 x float> %c, <8 x float> zeroinitializer
ret <8 x float> %r
}
define <4 x float> @_inreg4xfloat(float %a) {
; CHECK-LABEL: _inreg4xfloat:
; CHECK: # BB#0:
; CHECK-NEXT: vbroadcastss %xmm0, %xmm0
; CHECK-NEXT: retq
%b = insertelement <4 x float> undef, float %a, i32 0
%c = shufflevector <4 x float> %b, <4 x float> undef, <4 x i32> zeroinitializer
ret <4 x float> %c
}
define <4 x float> @_ss4xfloat_mask(<4 x float> %i, float %a, <4 x i32> %mask1) {
; CHECK-LABEL: _ss4xfloat_mask:
; CHECK: # BB#0:
; CHECK-NEXT: vpxor %xmm3, %xmm3, %xmm3
; CHECK-NEXT: vpcmpneqd %xmm3, %xmm2, %k1
; CHECK-NEXT: vbroadcastss %xmm1, %xmm0 {%k1}
; CHECK-NEXT: retq
%mask = icmp ne <4 x i32> %mask1, zeroinitializer
%b = insertelement <4 x float> undef, float %a, i32 0
%c = shufflevector <4 x float> %b, <4 x float> undef, <4 x i32> zeroinitializer
%r = select <4 x i1> %mask, <4 x float> %c, <4 x float> %i
ret <4 x float> %r
}
define <4 x float> @_ss4xfloat_maskz(float %a, <4 x i32> %mask1) {
; CHECK-LABEL: _ss4xfloat_maskz:
; CHECK: # BB#0:
; CHECK-NEXT: vpxor %xmm2, %xmm2, %xmm2
; CHECK-NEXT: vpcmpneqd %xmm2, %xmm1, %k1
; CHECK-NEXT: vbroadcastss %xmm0, %xmm0 {%k1} {z}
; CHECK-NEXT: retq
%mask = icmp ne <4 x i32> %mask1, zeroinitializer
%b = insertelement <4 x float> undef, float %a, i32 0
%c = shufflevector <4 x float> %b, <4 x float> undef, <4 x i32> zeroinitializer
%r = select <4 x i1> %mask, <4 x float> %c, <4 x float> zeroinitializer
ret <4 x float> %r
}
define <4 x double> @_inreg4xdouble(double %a) {
; CHECK-LABEL: _inreg4xdouble:
; CHECK: # BB#0:
; CHECK-NEXT: vbroadcastsd %xmm0, %ymm0
; CHECK-NEXT: retq
%b = insertelement <4 x double> undef, double %a, i32 0
%c = shufflevector <4 x double> %b, <4 x double> undef, <4 x i32> zeroinitializer
ret <4 x double> %c
}
define <4 x double> @_ss4xdouble_mask(<4 x double> %i, double %a, <4 x i32> %mask1) {
; CHECK-LABEL: _ss4xdouble_mask:
; CHECK: # BB#0:
; CHECK-NEXT: vpxor %xmm3, %xmm3, %xmm3
; CHECK-NEXT: vpcmpneqd %xmm3, %xmm2, %k1
; CHECK-NEXT: vbroadcastsd %xmm1, %ymm0 {%k1}
; CHECK-NEXT: retq
%mask = icmp ne <4 x i32> %mask1, zeroinitializer
%b = insertelement <4 x double> undef, double %a, i32 0
%c = shufflevector <4 x double> %b, <4 x double> undef, <4 x i32> zeroinitializer
%r = select <4 x i1> %mask, <4 x double> %c, <4 x double> %i
ret <4 x double> %r
}
define <4 x double> @_ss4xdouble_maskz(double %a, <4 x i32> %mask1) {
; CHECK-LABEL: _ss4xdouble_maskz:
; CHECK: # BB#0:
; CHECK-NEXT: vpxor %xmm2, %xmm2, %xmm2
; CHECK-NEXT: vpcmpneqd %xmm2, %xmm1, %k1
; CHECK-NEXT: vbroadcastsd %xmm0, %ymm0 {%k1} {z}
; CHECK-NEXT: retq
%mask = icmp ne <4 x i32> %mask1, zeroinitializer
%b = insertelement <4 x double> undef, double %a, i32 0
%c = shufflevector <4 x double> %b, <4 x double> undef, <4 x i32> zeroinitializer
%r = select <4 x i1> %mask, <4 x double> %c, <4 x double> zeroinitializer
ret <4 x double> %r
}